perf optimization; code refactor

Author: MarshalX

cycode/cli/files_collector/path_documents.py

@@ -1,6 +1,5 @@
 import os
-from collections import defaultdict
-from typing import TYPE_CHECKING, Iterable, List, Set, Tuple
+from typing import TYPE_CHECKING, Iterable, List, Optional, Tuple
 
 import pathspec
 
@@ -11,6 +10,7 @@
     is_iac,
     is_tfplan_file,
 )
+from cycode.cli.files_collector.walk_ignore import walk_ignore
 from cycode.cli.models import Document
 from cycode.cli.utils.path_utils import get_absolute_path, get_file_content
 from cycode.cyclient import logger
@@ -19,82 +19,18 @@
     from cycode.cli.utils.progress_bar import BaseProgressBar, ProgressBarSection
 
 
-def _walk_to_top(path: str) -> Iterable[str]:
-    while os.path.dirname(path) != path:
-        yield path
-        path = os.path.dirname(path)
+def _get_all_existing_files_in_directory(path: str, *, walk_with_ignore_patterns: bool = True) -> List[str]:
+    files: List[str] = []
 
-    if path:
-        yield path  # Include the top-level directory
-
-
-_SUPPORTED_IGNORE_PATTERN_FILES = {'.gitignore', '.cycodeignore'}
-
-
-def _collect_top_level_ignore_files(path: str) -> List[str]:
-    ignore_files = []
-    for dir_path in _walk_to_top(path):
-        for ignore_file in _SUPPORTED_IGNORE_PATTERN_FILES:
-            ignore_file_path = os.path.join(dir_path, ignore_file)
-            if os.path.exists(ignore_file_path):
-                logger.debug('Found top level ignore file: %s', ignore_file_path)
-                ignore_files.append(ignore_file_path)
-    return ignore_files
-
-
-def _get_global_ignore_patterns(path: str) -> List[str]:
-    ignore_patterns = []
-    for ignore_file in _collect_top_level_ignore_files(path):
-        file_patterns = get_file_content(ignore_file).splitlines()
-        ignore_patterns.extend(file_patterns)
-    return ignore_patterns
-
-
-def _apply_ignore_patterns(ignore_patterns: List[str], files: Set[str]) -> Set[str]:
-    if not ignore_patterns:
-        return files
-
-    path_spec = pathspec.PathSpec.from_lines(pathspec.patterns.GitWildMatchPattern, ignore_patterns)
-    excluded_file_paths = set(path_spec.match_files(files))
-
-    return files - excluded_file_paths
-
-
-def _get_all_existing_files_in_directory(path: str, *, apply_ignore_patterns: bool = True) -> Set[str]:
-    files: Set[str] = set()
-
-    global_ignore_patterns = _get_global_ignore_patterns(path)
-    path_to_ignore_patterns = defaultdict(list)
-
-    for root, _, filenames in os.walk(path):
+    walk_func = walk_ignore if walk_with_ignore_patterns else os.walk
+    for root, _, filenames in walk_func(path):
         for filename in filenames:
-            filepath = os.path.join(root, filename)
-
-            if filepath in _SUPPORTED_IGNORE_PATTERN_FILES:
-                logger.debug('Found ignore file: %s', filepath)
-                # TODO(MarshalX): accumulate ignore pattern from previous levels
-                path_to_ignore_patterns[root].extend(get_file_content(filepath).splitlines())
-
-            if apply_ignore_patterns and root in path_to_ignore_patterns:
-                filtered_paths = _apply_ignore_patterns(
-                    path_to_ignore_patterns[root],
-                    {
-                        filepath,
-                    },
-                )
-                if filtered_paths:
-                    files.update(filtered_paths)
-            else:
-                files.add(os.path.join(root, filename))
-
-    if apply_ignore_patterns:
-        logger.debug('Applying global ignore patterns %s', {'global_ignore_patterns': global_ignore_patterns})
-        return _apply_ignore_patterns(global_ignore_patterns, files)
+            files.append(os.path.join(root, filename))
 
     return files
 
 
-def _get_relevant_files_in_path(path: str, exclude_patterns: Iterable[str]) -> List[str]:
+def _get_relevant_files_in_path(path: str, exclude_patterns: Optional[Iterable[str]] = None) -> List[str]:
     absolute_path = get_absolute_path(path)
 
     if not os.path.isfile(absolute_path) and not os.path.isdir(absolute_path):
@@ -103,24 +39,21 @@ def _get_relevant_files_in_path(path: str, exclude_patterns: Iterable[str]) -> L
     if os.path.isfile(absolute_path):
         return [absolute_path]
 
-    all_file_paths = _get_all_existing_files_in_directory(absolute_path)
-
-    path_spec = pathspec.PathSpec.from_lines(pathspec.patterns.GitWildMatchPattern, exclude_patterns)
-    excluded_file_paths = set(path_spec.match_files(all_file_paths))
+    file_paths = _get_all_existing_files_in_directory(absolute_path)
 
-    relevant_file_paths = all_file_paths - excluded_file_paths
+    if exclude_patterns:
+        path_spec = pathspec.PathSpec.from_lines(pathspec.patterns.GitWildMatchPattern, exclude_patterns)
+        file_paths = path_spec.match_files(file_paths, negate=True)
 
-    return [file_path for file_path in relevant_file_paths if os.path.isfile(file_path)]
+    return [file_path for file_path in file_paths if os.path.isfile(file_path)]
 
 
 def _get_relevant_files(
     progress_bar: 'BaseProgressBar', progress_bar_section: 'ProgressBarSection', scan_type: str, paths: Tuple[str]
 ) -> List[str]:
     all_files_to_scan = []
     for path in paths:
-        all_files_to_scan.extend(
-            _get_relevant_files_in_path(path=path, exclude_patterns=['**/.git/**', '**/.cycode/**'])
-        )
+        all_files_to_scan.extend(_get_relevant_files_in_path(path))
 
     # we are double the progress bar section length because we are going to process the files twice
     # first time to get the file list with respect of excluded patterns (excluding takes seconds to execute)

cycode/cli/files_collector/walk_ignore.py

@@ -0,0 +1,73 @@
+import os
+from collections import defaultdict
+from typing import Iterable, List
+
+import pathspec
+from pathspec.util import StrPath
+
+from cycode.cli.utils.path_utils import get_file_content
+from cycode.cyclient import logger
+
+_SUPPORTED_IGNORE_PATTERN_FILES = {'.gitignore', '.cycodeignore'}
+_DEFAULT_GLOBAL_IGNORE_PATTERNS = [
+    '.git',
+    '.cycode',
+    '**/.git/**',
+    '**/.cycode/**',
+]
+
+
+def _walk_to_top(path: str) -> Iterable[str]:
+    while os.path.dirname(path) != path:
+        yield path
+        path = os.path.dirname(path)
+
+    if path:
+        yield path  # Include the top-level directory
+
+
+def _collect_top_level_ignore_files(path: str) -> List[str]:
+    ignore_files = []
+    for dir_path in _walk_to_top(path):
+        for ignore_file in _SUPPORTED_IGNORE_PATTERN_FILES:
+            ignore_file_path = os.path.join(dir_path, ignore_file)
+            if os.path.exists(ignore_file_path):
+                logger.debug('Apply top level ignore file: %s', ignore_file_path)
+                ignore_files.append(ignore_file_path)
+    return ignore_files
+
+
+def _get_global_ignore_patterns(path: str) -> List[str]:
+    ignore_patterns = _DEFAULT_GLOBAL_IGNORE_PATTERNS.copy()
+    for ignore_file in _collect_top_level_ignore_files(path):
+        file_patterns = get_file_content(ignore_file).splitlines()
+        ignore_patterns.extend(file_patterns)
+    return ignore_patterns
+
+
+def _should_include_path(ignore_patterns: List[str], path: StrPath) -> bool:
+    path_spec = pathspec.PathSpec.from_lines(pathspec.patterns.GitWildMatchPattern, ignore_patterns)
+    return not path_spec.match_file(path)  # works with both files and directories; negative match
+
+
+def walk_ignore(path: str) -> List[str]:
+    global_ignore_patterns = _get_global_ignore_patterns(path)
+    path_to_ignore_patterns = defaultdict(list)
+
+    for dirpath, dirnames, filenames in os.walk(path, topdown=True):
+        # finds and processes ignore files first to get the patterns
+        for filename in filenames:
+            filepath = os.path.join(dirpath, filename)
+            if filename in _SUPPORTED_IGNORE_PATTERN_FILES:
+                logger.debug('Apply ignore file: %s', filepath)
+                # TODO(MarshalX): accumulate ignore pattern from previous levels
+                path_to_ignore_patterns[dirpath].extend(get_file_content(filepath).splitlines())
+
+        ignore_patterns = global_ignore_patterns + path_to_ignore_patterns.get(dirpath, [])
+
+        # decrease recursion depth of os.walk() because of topdown=True by changing the list in-place
+        # slicing ([:]) is mandatory to change dict in-place!
+        dirnames[:] = [d for d in dirnames if _should_include_path(ignore_patterns, d)]
+        filenames[:] = [f for f in filenames if _should_include_path(ignore_patterns, f)]
+
+        yield dirpath, dirnames, filenames

…i/files_collector/test_path_documents.py …/cli/files_collector/test_walk_ignore.pytests/cli/files_collector/test_path_documents.py renamed to tests/cli/files_collector/test_walk_ignore.py tests/cli/files_collector/test_path_documents.py renamed to tests/cli/files_collector/test_walk_ignore.py

@@ -1,7 +1,7 @@
 from os.path import normpath
 from typing import TYPE_CHECKING
 
-from cycode.cli.files_collector.path_documents import (
+from cycode.cli.files_collector.walk_ignore import (
     _collect_top_level_ignore_files,
     _get_global_ignore_patterns,
     _walk_to_top,
@@ -87,7 +87,13 @@ def test_get_global_ignore_patterns(fs: 'FakeFilesystem') -> None:
     _create_mocked_file_structure(fs)
     ignore_patterns = _get_global_ignore_patterns('/home/user/project/subdir')
 
-    assert len(ignore_patterns) == 3
+    assert len(ignore_patterns) == 7
+    # default global:
+    assert '.git' in ignore_patterns
+    assert '.cycode' in ignore_patterns
+    assert '**/.git/**' in ignore_patterns
+    assert '**/.cycode/**' in ignore_patterns
+    # additional:
     assert '*.txt' in ignore_patterns
     assert '*.pyc' in ignore_patterns
     assert '*.log' in ignore_patterns