rename SCA deps restore func to clarify

Author: MarshalX

cycode/cli/apps/report/sbom/path/path_command.py

@@ -8,7 +8,7 @@
 from cycode.cli.apps.report.sbom.common import create_sbom_report, send_report_feedback
 from cycode.cli.exceptions.handle_report_sbom_errors import handle_report_exception
 from cycode.cli.files_collector.path_documents import get_relevant_documents
-from cycode.cli.files_collector.sca.sca_code_scanner import perform_pre_scan_documents_actions
+from cycode.cli.files_collector.sca.sca_file_collector import add_sca_dependencies_tree_documents_if_needed
 from cycode.cli.files_collector.zip_documents import zip_documents
 from cycode.cli.utils.get_api_client import get_report_cycode_client
 from cycode.cli.utils.progress_bar import SbomReportProgressBarSection
@@ -41,7 +41,7 @@ def path_command(
         )
         # TODO(MarshalX): combine perform_pre_scan_documents_actions with get_relevant_document.
         #  unhardcode usage of context in perform_pre_scan_documents_actions
-        perform_pre_scan_documents_actions(ctx, consts.SCA_SCAN_TYPE, documents)
+        add_sca_dependencies_tree_documents_if_needed(ctx, consts.SCA_SCAN_TYPE, documents)
 
         zipped_documents = zip_documents(consts.SCA_SCAN_TYPE, documents)
         report_execution = client.request_sbom_report_execution(report_parameters, zip_file=zipped_documents)

cycode/cli/apps/scan/code_scanner.py

@@ -17,7 +17,7 @@
 from cycode.cli.exceptions import custom_exceptions
 from cycode.cli.exceptions.handle_scan_errors import handle_scan_exception
 from cycode.cli.files_collector.path_documents import get_relevant_documents
-from cycode.cli.files_collector.sca.sca_code_scanner import perform_pre_scan_documents_actions
+from cycode.cli.files_collector.sca.sca_file_collector import add_sca_dependencies_tree_documents_if_needed
 from cycode.cli.files_collector.zip_documents import zip_documents
 from cycode.cli.models import CliError, Document, LocalScanResult
 from cycode.cli.utils.progress_bar import ScanProgressBarSection
@@ -42,7 +42,7 @@ def scan_disk_files(ctx: typer.Context, paths: tuple[str, ...]) -> None:
 
     try:
         documents = get_relevant_documents(progress_bar, ScanProgressBarSection.PREPARE_LOCAL_FILES, scan_type, paths)
-        perform_pre_scan_documents_actions(ctx, scan_type, documents)
+        add_sca_dependencies_tree_documents_if_needed(ctx, scan_type, documents)
         scan_documents(ctx, documents, get_scan_parameters(ctx, paths))
     except Exception as e:
         handle_scan_exception(ctx, e)

cycode/cli/apps/scan/commit_range_scanner.py

@@ -28,7 +28,10 @@
     get_pre_commit_modified_documents,
     parse_commit_range,
 )
-from cycode.cli.files_collector.sca import sca_code_scanner
+from cycode.cli.files_collector.sca.sca_file_collector import (
+    perform_pre_commit_range_scan_actions,
+    perform_pre_hook_range_scan_actions,
+)
 from cycode.cli.files_collector.zip_documents import zip_documents
 from cycode.cli.models import Document
 from cycode.cli.utils.git_proxy import git_proxy
@@ -169,13 +172,15 @@ def _scan_commit_range_documents(
 
 def _scan_sca_commit_range(ctx: typer.Context, path: str, commit_range: str, **_) -> None:
     scan_parameters = get_scan_parameters(ctx, (path,))
+
     from_commit_rev, to_commit_rev = parse_commit_range(commit_range, path)
     from_commit_documents, to_commit_documents, _ = get_commit_range_modified_documents(
         ctx.obj['progress_bar'], ScanProgressBarSection.PREPARE_LOCAL_FILES, path, from_commit_rev, to_commit_rev
     )
     from_commit_documents = excluder.exclude_irrelevant_documents_to_scan(consts.SCA_SCAN_TYPE, from_commit_documents)
     to_commit_documents = excluder.exclude_irrelevant_documents_to_scan(consts.SCA_SCAN_TYPE, to_commit_documents)
-    sca_code_scanner.perform_pre_commit_range_scan_actions(
+
+    perform_pre_commit_range_scan_actions(
         path, from_commit_documents, from_commit_rev, to_commit_documents, to_commit_rev
     )
 
@@ -197,6 +202,7 @@ def _scan_secret_commit_range(
 
 def _scan_sast_commit_range(ctx: typer.Context, path: str, commit_range: str, **_) -> None:
     scan_parameters = get_scan_parameters(ctx, (path,))
+
     from_commit_rev, to_commit_rev = parse_commit_range(commit_range, path)
     _, commit_documents, diff_documents = get_commit_range_modified_documents(
         ctx.obj['progress_bar'], ScanProgressBarSection.PREPARE_LOCAL_FILES, path, from_commit_rev, to_commit_rev
@@ -226,6 +232,7 @@ def scan_commit_range(ctx: typer.Context, path: str, commit_range: str, **kwargs
 
 def _scan_sca_pre_commit(ctx: typer.Context, repo_path: str) -> None:
     scan_parameters = get_scan_parameters(ctx)
+
     git_head_documents, pre_committed_documents = get_pre_commit_modified_documents(
         progress_bar=ctx.obj['progress_bar'],
         progress_bar_section=ScanProgressBarSection.PREPARE_LOCAL_FILES,
@@ -235,7 +242,9 @@ def _scan_sca_pre_commit(ctx: typer.Context, repo_path: str) -> None:
     pre_committed_documents = excluder.exclude_irrelevant_documents_to_scan(
         consts.SCA_SCAN_TYPE, pre_committed_documents
     )
-    sca_code_scanner.perform_pre_hook_range_scan_actions(repo_path, git_head_documents, pre_committed_documents)
+
+    perform_pre_hook_range_scan_actions(repo_path, git_head_documents, pre_committed_documents)
+
     _scan_commit_range_documents(
         ctx,
         git_head_documents,

cycode/cli/apps/scan/repository/repository_command.py

@@ -10,7 +10,7 @@
 from cycode.cli.exceptions.handle_scan_errors import handle_scan_exception
 from cycode.cli.files_collector.file_excluder import excluder
 from cycode.cli.files_collector.repository_documents import get_git_repository_tree_file_entries
-from cycode.cli.files_collector.sca.sca_code_scanner import perform_pre_scan_documents_actions
+from cycode.cli.files_collector.sca.sca_file_collector import add_sca_dependencies_tree_documents_if_needed
 from cycode.cli.logger import logger
 from cycode.cli.models import Document
 from cycode.cli.utils.path_utils import get_path_by_os
@@ -60,7 +60,7 @@ def repository_command(
 
         documents_to_scan = excluder.exclude_irrelevant_documents_to_scan(scan_type, documents_to_scan)
 
-        perform_pre_scan_documents_actions(ctx, scan_type, documents_to_scan)
+        add_sca_dependencies_tree_documents_if_needed(ctx, scan_type, documents_to_scan)
 
         logger.debug('Found all relevant files for scanning %s', {'path': path, 'branch': branch})
         scan_documents(ctx, documents_to_scan, get_scan_parameters(ctx, (str(path),)))

cycode/cli/files_collector/repository_documents.py

@@ -4,13 +4,12 @@
 from typing import TYPE_CHECKING, Optional, Union
 
 from cycode.cli import consts
-from cycode.cli.files_collector.sca.sca_code_scanner import get_file_content_from_commit_diff
 from cycode.cli.models import Document
 from cycode.cli.utils.git_proxy import git_proxy
 from cycode.cli.utils.path_utils import get_file_content, get_path_by_os
 
 if TYPE_CHECKING:
-    from git import Blob, Diff
+    from git import Blob, Diff, Repo
     from git.objects.base import IndexObjUnion
     from git.objects.tree import TraversedTreeTup
 
@@ -103,6 +102,18 @@ def get_pre_commit_modified_documents(
     return git_head_documents, pre_committed_documents
 
 
+def get_file_content_from_commit_path(repo: 'Repo', commit: str, file_path: str) -> Optional[str]:
+    try:
+        return repo.git.show(f'{commit}:{file_path}')
+    except git_proxy.get_git_command_error():
+        return None
+
+
+def get_file_content_from_commit_diff(repo: 'Repo', commit: str, diff: 'Diff') -> Optional[str]:
+    file_path = get_diff_file_path(diff, relative=True)
+    return get_file_content_from_commit_path(repo, commit, file_path)
+
+
 def get_commit_range_modified_documents(
     progress_bar: 'BaseProgressBar',
     progress_bar_section: 'ProgressBarSection',

…/files_collector/sca/sca_code_scanner.py …iles_collector/sca/sca_file_collector.pycycode/cli/files_collector/sca/sca_code_scanner.py renamed to cycode/cli/files_collector/sca/sca_file_collector.py cycode/cli/files_collector/sca/sca_code_scanner.py renamed to cycode/cli/files_collector/sca/sca_file_collector.py

@@ -3,6 +3,7 @@
 import typer
 
 from cycode.cli import consts
+from cycode.cli.files_collector.repository_documents import get_file_content_from_commit_path
 from cycode.cli.files_collector.sca.base_restore_dependencies import BaseRestoreDependencies
 from cycode.cli.files_collector.sca.go.restore_go_dependencies import RestoreGoDependencies
 from cycode.cli.files_collector.sca.maven.restore_gradle_dependencies import RestoreGradleDependencies
@@ -17,12 +18,27 @@
 from cycode.logger import get_logger
 
 if TYPE_CHECKING:
-    from git import Diff, Repo
+    from git import Repo
 
 BUILD_DEP_TREE_TIMEOUT = 180
 
 
-logger = get_logger('SCA Code Scanner')
+logger = get_logger('SCA File Collector')
+
+
+def _add_ecosystem_related_files_if_exists(
+    documents: list[Document], repo: Optional['Repo'] = None, commit_rev: Optional[str] = None
+) -> None:
+    documents_to_add: list[Document] = []
+    for doc in documents:
+        ecosystem = _get_project_file_ecosystem(doc)
+        if ecosystem is None:
+            logger.debug('Failed to resolve project file ecosystem: %s', doc.path)
+            continue
+
+        documents_to_add.extend(_get_doc_ecosystem_related_project_files(doc, documents, ecosystem, commit_rev, repo))
+
+    documents.extend(documents_to_add)
 
 
 def perform_pre_commit_range_scan_actions(
@@ -33,40 +49,25 @@ def perform_pre_commit_range_scan_actions(
     to_commit_rev: str,
 ) -> None:
     repo = git_proxy.get_repo(path)
-    add_ecosystem_related_files_if_exists(from_commit_documents, repo, from_commit_rev)
-    add_ecosystem_related_files_if_exists(to_commit_documents, repo, to_commit_rev)
+    _add_ecosystem_related_files_if_exists(from_commit_documents, repo, from_commit_rev)
+    _add_ecosystem_related_files_if_exists(to_commit_documents, repo, to_commit_rev)
 
 
 def perform_pre_hook_range_scan_actions(
     repo_path: str, git_head_documents: list[Document], pre_committed_documents: list[Document]
 ) -> None:
     repo = git_proxy.get_repo(repo_path)
-    add_ecosystem_related_files_if_exists(git_head_documents, repo, consts.GIT_HEAD_COMMIT_REV)
-    add_ecosystem_related_files_if_exists(pre_committed_documents)
-
-
-def add_ecosystem_related_files_if_exists(
-    documents: list[Document], repo: Optional['Repo'] = None, commit_rev: Optional[str] = None
-) -> None:
-    documents_to_add: list[Document] = []
-    for doc in documents:
-        ecosystem = get_project_file_ecosystem(doc)
-        if ecosystem is None:
-            logger.debug('Failed to resolve project file ecosystem: %s', doc.path)
-            continue
+    _add_ecosystem_related_files_if_exists(git_head_documents, repo, consts.GIT_HEAD_COMMIT_REV)
+    _add_ecosystem_related_files_if_exists(pre_committed_documents)
 
-        documents_to_add.extend(get_doc_ecosystem_related_project_files(doc, documents, ecosystem, commit_rev, repo))
-
-    documents.extend(documents_to_add)
 
-
-def get_doc_ecosystem_related_project_files(
+def _get_doc_ecosystem_related_project_files(
     doc: Document, documents: list[Document], ecosystem: str, commit_rev: Optional[str], repo: Optional['Repo']
 ) -> list[Document]:
     documents_to_add: list[Document] = []
     for ecosystem_project_file in consts.PROJECT_FILES_BY_ECOSYSTEM_MAP.get(ecosystem):
         file_to_search = join_paths(get_file_dir(doc.path), ecosystem_project_file)
-        if not is_project_file_exists_in_documents(documents, file_to_search):
+        if not _is_project_file_exists_in_documents(documents, file_to_search):
             if repo:
                 file_content = get_file_content_from_commit_path(repo, commit_rev, file_to_search)
             else:
@@ -78,19 +79,23 @@ def get_doc_ecosystem_related_project_files(
     return documents_to_add
 
 
-def is_project_file_exists_in_documents(documents: list[Document], file: str) -> bool:
+def _is_project_file_exists_in_documents(documents: list[Document], file: str) -> bool:
     return any(doc for doc in documents if file == doc.path)
 
 
-def get_project_file_ecosystem(document: Document) -> Optional[str]:
+def _get_project_file_ecosystem(document: Document) -> Optional[str]:
     for ecosystem, project_files in consts.PROJECT_FILES_BY_ECOSYSTEM_MAP.items():
         for project_file in project_files:
             if document.path.endswith(project_file):
                 return ecosystem
     return None
 
 
-def try_restore_dependencies(
+def _get_manifest_file_path(document: Document, is_monitor_action: bool, project_path: str) -> str:
+    return join_paths(project_path, document.path) if is_monitor_action else document.path
+
+
+def _try_restore_dependencies(
     ctx: typer.Context,
     restore_dependencies: 'BaseRestoreDependencies',
     document: Document,
@@ -110,34 +115,13 @@ def try_restore_dependencies(
         is_monitor_action = ctx.obj.get('monitor', False)
         project_path = get_path_from_context(ctx)
 
-        manifest_file_path = get_manifest_file_path(document, is_monitor_action, project_path)
+        manifest_file_path = _get_manifest_file_path(document, is_monitor_action, project_path)
         logger.debug('Succeeded to generate dependencies tree on path: %s', manifest_file_path)
 
     return restore_dependencies_document
 
 
-def add_dependencies_tree_document(
-    ctx: typer.Context, documents_to_scan: list[Document], is_git_diff: bool = False
-) -> None:
-    documents_to_add: dict[str, Document] = {document.path: document for document in documents_to_scan}
-    restore_dependencies_list = restore_handlers(ctx, is_git_diff)
-
-    for restore_dependencies in restore_dependencies_list:
-        for document in documents_to_scan:
-            restore_dependencies_document = try_restore_dependencies(ctx, restore_dependencies, document)
-            if restore_dependencies_document is None:
-                continue
-
-            if restore_dependencies_document.path in documents_to_add:
-                logger.debug('Duplicate document on restore for path: %s', restore_dependencies_document.path)
-            else:
-                documents_to_add[restore_dependencies_document.path] = restore_dependencies_document
-
-    # mutate original list using slice assignment
-    documents_to_scan[:] = list(documents_to_add.values())
-
-
-def restore_handlers(ctx: typer.Context, is_git_diff: bool) -> list[BaseRestoreDependencies]:
+def _get_restore_handlers(ctx: typer.Context, is_git_diff: bool) -> list[BaseRestoreDependencies]:
     return [
         RestoreGradleDependencies(ctx, is_git_diff, BUILD_DEP_TREE_TIMEOUT),
         RestoreMavenDependencies(ctx, is_git_diff, BUILD_DEP_TREE_TIMEOUT),
@@ -149,28 +133,38 @@ def restore_handlers(ctx: typer.Context, is_git_diff: bool) -> list[BaseRestoreD
     ]
 
 
-def get_manifest_file_path(document: Document, is_monitor_action: bool, project_path: str) -> str:
-    return join_paths(project_path, document.path) if is_monitor_action else document.path
+def _add_dependencies_tree_documents(
+    ctx: typer.Context, documents_to_scan: list[Document], is_git_diff: bool = False
+) -> None:
+    logger.debug(
+        'Adding dependencies tree documents, %s',
+        {'documents_count': len(documents_to_scan), 'is_git_diff': is_git_diff},
+    )
 
+    documents_to_add: dict[str, Document] = {document.path: document for document in documents_to_scan}
+    restore_dependencies_list = _get_restore_handlers(ctx, is_git_diff)
 
-def get_file_content_from_commit_path(repo: 'Repo', commit: str, file_path: str) -> Optional[str]:
-    try:
-        return repo.git.show(f'{commit}:{file_path}')
-    except git_proxy.get_git_command_error():
-        return None
+    for restore_dependencies in restore_dependencies_list:
+        for document in documents_to_scan:
+            restore_dependencies_document = _try_restore_dependencies(ctx, restore_dependencies, document)
+            if restore_dependencies_document is None:
+                continue
 
+            if restore_dependencies_document.path in documents_to_add:
+                logger.debug('Duplicate document on restore for path: %s', restore_dependencies_document.path)
+            else:
+                logger.debug('Adding dependencies tree document, %s', restore_dependencies_document.path)
+                documents_to_add[restore_dependencies_document.path] = restore_dependencies_document
 
-def get_file_content_from_commit_diff(repo: 'Repo', commit: str, diff: 'Diff') -> Optional[str]:
-    from cycode.cli.files_collector.repository_documents import get_diff_file_path
+    logger.debug('Finished adding dependencies tree documents, %s', {'documents_count': len(documents_to_add)})
 
-    file_path = get_diff_file_path(diff, relative=True)
-    return get_file_content_from_commit_path(repo, commit, file_path)
+    # mutate original list using slice assignment
+    documents_to_scan[:] = list(documents_to_add.values())
 
 
-def perform_pre_scan_documents_actions(
+def add_sca_dependencies_tree_documents_if_needed(
     ctx: typer.Context, scan_type: str, documents_to_scan: list[Document], is_git_diff: bool = False
 ) -> None:
     no_restore = ctx.params.get('no-restore', False)
     if scan_type == consts.SCA_SCAN_TYPE and not no_restore:
-        logger.debug('Perform pre-scan document add_dependencies_tree_document action')
-        add_dependencies_tree_document(ctx, documents_to_scan, is_git_diff)
+        _add_dependencies_tree_documents(ctx, documents_to_scan, is_git_diff)