fix file filtering for SAST commit range; fix pre-receive availability for SCA

Author: MarshalX

cycode/cli/apps/scan/commit_range_scanner.py

@@ -18,19 +18,19 @@
 )
 from cycode.cli.config import configuration_manager
 from cycode.cli.exceptions.handle_scan_errors import handle_scan_exception
-from cycode.cli.files_collector.commit_range_documents import collect_commit_range_diff_documents
-from cycode.cli.files_collector.file_excluder import excluder
-from cycode.cli.files_collector.models.in_memory_zip import InMemoryZip
-from cycode.cli.files_collector.repository_documents import (
+from cycode.cli.files_collector.commit_range_documents import (
+    collect_commit_range_diff_documents,
     get_commit_range_modified_documents,
     get_diff_file_content,
     get_diff_file_path,
     get_pre_commit_modified_documents,
     parse_commit_range,
 )
+from cycode.cli.files_collector.file_excluder import excluder
+from cycode.cli.files_collector.models.in_memory_zip import InMemoryZip
 from cycode.cli.files_collector.sca.sca_file_collector import (
-    perform_pre_commit_range_scan_actions,
-    perform_pre_hook_range_scan_actions,
+    perform_sca_pre_commit_range_scan_actions,
+    perform_sca_pre_hook_range_scan_actions,
 )
 from cycode.cli.files_collector.zip_documents import zip_documents
 from cycode.cli.models import Document
@@ -180,7 +180,7 @@ def _scan_sca_commit_range(ctx: typer.Context, path: str, commit_range: str, **_
     from_commit_documents = excluder.exclude_irrelevant_documents_to_scan(consts.SCA_SCAN_TYPE, from_commit_documents)
     to_commit_documents = excluder.exclude_irrelevant_documents_to_scan(consts.SCA_SCAN_TYPE, to_commit_documents)
 
-    perform_pre_commit_range_scan_actions(
+    perform_sca_pre_commit_range_scan_actions(
         path, from_commit_documents, from_commit_rev, to_commit_documents, to_commit_rev
     )
 
@@ -207,6 +207,8 @@ def _scan_sast_commit_range(ctx: typer.Context, path: str, commit_range: str, **
     _, commit_documents, diff_documents = get_commit_range_modified_documents(
         ctx.obj['progress_bar'], ScanProgressBarSection.PREPARE_LOCAL_FILES, path, from_commit_rev, to_commit_rev
     )
+    commit_documents = excluder.exclude_irrelevant_documents_to_scan(consts.SAST_SCAN_TYPE, commit_documents)
+    diff_documents = excluder.exclude_irrelevant_documents_to_scan(consts.SAST_SCAN_TYPE, diff_documents)
 
     _scan_commit_range_documents(ctx, commit_documents, diff_documents, scan_parameters=scan_parameters)
 
@@ -243,7 +245,7 @@ def _scan_sca_pre_commit(ctx: typer.Context, repo_path: str) -> None:
         consts.SCA_SCAN_TYPE, pre_committed_documents
     )
 
-    perform_pre_hook_range_scan_actions(repo_path, git_head_documents, pre_committed_documents)
+    perform_sca_pre_hook_range_scan_actions(repo_path, git_head_documents, pre_committed_documents)
 
     _scan_commit_range_documents(
         ctx,
@@ -256,16 +258,18 @@ def _scan_sca_pre_commit(ctx: typer.Context, repo_path: str) -> None:
 
 def _scan_secret_pre_commit(ctx: typer.Context, repo_path: str) -> None:
     progress_bar = ctx.obj['progress_bar']
-    diff_files = git_proxy.get_repo(repo_path).index.diff(consts.GIT_HEAD_COMMIT_REV, create_patch=True, R=True)
+    diff_index = git_proxy.get_repo(repo_path).index.diff(consts.GIT_HEAD_COMMIT_REV, create_patch=True, R=True)
 
-    progress_bar.set_section_length(ScanProgressBarSection.PREPARE_LOCAL_FILES, len(diff_files))
+    progress_bar.set_section_length(ScanProgressBarSection.PREPARE_LOCAL_FILES, len(diff_index))
 
     documents_to_scan = []
-    for file in diff_files:
+    for diff in diff_index:
         progress_bar.update(ScanProgressBarSection.PREPARE_LOCAL_FILES)
-        documents_to_scan.append(Document(get_path_by_os(get_diff_file_path(file)), get_diff_file_content(file)))
-
+        documents_to_scan.append(
+            Document(get_path_by_os(get_diff_file_path(diff)), get_diff_file_content(diff), is_git_diff_format=True)
+        )
     documents_to_scan = excluder.exclude_irrelevant_documents_to_scan(consts.SECRET_SCAN_TYPE, documents_to_scan)
+
     scan_documents(ctx, documents_to_scan, get_scan_parameters(ctx), is_git_diff=True)
 
 

cycode/cli/apps/scan/pre_receive/pre_receive_command.py

@@ -2,7 +2,6 @@
 import os
 from typing import Annotated, Optional
 
-import click
 import typer
 
 from cycode.cli import consts
@@ -14,7 +13,7 @@
 from cycode.cli.config import configuration_manager
 from cycode.cli.console import console
 from cycode.cli.exceptions.handle_scan_errors import handle_scan_exception
-from cycode.cli.files_collector.repository_documents import (
+from cycode.cli.files_collector.commit_range_documents import (
     calculate_pre_receive_commit_range,
     parse_pre_receive_input,
 )
@@ -32,10 +31,6 @@ def pre_receive_command(
     try:
         add_breadcrumb('pre_receive')
 
-        scan_type = ctx.obj['scan_type']
-        if scan_type != consts.SECRET_SCAN_TYPE:
-            raise click.ClickException(f'Commit range scanning for {scan_type.upper()} is not supported')
-
         if should_skip_pre_receive_scan():
             logger.info(
                 'A scan has been skipped as per your request. '

cycode/cli/files_collector/commit_range_documents.py

@@ -1,14 +1,24 @@
-from typing import Optional
+import os
+import sys
+from typing import TYPE_CHECKING, Optional
 
 import typer
 
-from cycode.cli.files_collector.repository_documents import get_diff_file_content, get_diff_file_path
+from cycode.cli import consts
+from cycode.cli.files_collector.repository_documents import (
+    get_file_content_from_commit_path,
+)
 from cycode.cli.models import Document
 from cycode.cli.utils.git_proxy import git_proxy
-from cycode.cli.utils.path_utils import get_path_by_os
+from cycode.cli.utils.path_utils import get_file_content, get_path_by_os
 from cycode.cli.utils.progress_bar import ScanProgressBarSection
 from cycode.logger import get_logger
 
+if TYPE_CHECKING:
+    from git import Diff, Repo
+
+    from cycode.cli.utils.progress_bar import BaseProgressBar, ProgressBarSection
+
 logger = get_logger('Commit Range Collector')
 
 
@@ -67,3 +77,161 @@ def collect_commit_range_diff_documents(
     logger.debug('List of commit ids to scan, %s', {'commit_ids': commit_ids_to_scan})
 
     return commit_documents_to_scan
+
+
+def calculate_pre_receive_commit_range(branch_update_details: str) -> Optional[str]:
+    end_commit = _get_end_commit_from_branch_update_details(branch_update_details)
+
+    # branch is deleted, no need to perform scan
+    if end_commit == consts.EMPTY_COMMIT_SHA:
+        return None
+
+    start_commit = _get_oldest_unupdated_commit_for_branch(end_commit)
+
+    # no new commit to update found
+    if not start_commit:
+        return None
+
+    return f'{start_commit}~1...{end_commit}'
+
+
+def _get_end_commit_from_branch_update_details(update_details: str) -> str:
+    # update details pattern: <start_commit> <end_commit> <ref>
+    _, end_commit, _ = update_details.split()
+    return end_commit
+
+
+def _get_oldest_unupdated_commit_for_branch(commit: str) -> Optional[str]:
+    # get a list of commits by chronological order that are not in the remote repository yet
+    # more info about rev-list command: https://git-scm.com/docs/git-rev-list
+    repo = git_proxy.get_repo(os.getcwd())
+    not_updated_commits = repo.git.rev_list(commit, '--topo-order', '--reverse', '--not', '--all')
+
+    commits = not_updated_commits.splitlines()
+    if not commits:
+        return None
+
+    return commits[0]
+
+
+def _get_file_content_from_commit_diff(repo: 'Repo', commit: str, diff: 'Diff') -> Optional[str]:
+    file_path = get_diff_file_path(diff, relative=True)
+    return get_file_content_from_commit_path(repo, commit, file_path)
+
+
+def get_commit_range_modified_documents(
+    progress_bar: 'BaseProgressBar',
+    progress_bar_section: 'ProgressBarSection',
+    path: str,
+    from_commit_rev: str,
+    to_commit_rev: str,
+) -> tuple[list[Document], list[Document], list[Document]]:
+    from_commit_documents = []
+    to_commit_documents = []
+    diff_documents = []
+
+    repo = git_proxy.get_repo(path)
+    diff_index = repo.commit(from_commit_rev).diff(to_commit_rev, create_patch=True, R=True)
+
+    modified_files_diff = [
+        diff for diff in diff_index if diff.change_type != consts.COMMIT_DIFF_DELETED_FILE_CHANGE_TYPE
+    ]
+    progress_bar.set_section_length(progress_bar_section, len(modified_files_diff))
+    for diff in modified_files_diff:
+        progress_bar.update(progress_bar_section)
+
+        file_path = get_path_by_os(get_diff_file_path(diff))
+
+        diff_documents.append(
+            Document(
+                path=file_path,
+                content=get_diff_file_content(diff),
+                is_git_diff_format=True,
+            )
+        )
+
+        file_content = _get_file_content_from_commit_diff(repo, from_commit_rev, diff)
+        if file_content is not None:
+            from_commit_documents.append(Document(file_path, file_content))
+
+        file_content = _get_file_content_from_commit_diff(repo, to_commit_rev, diff)
+        if file_content is not None:
+            to_commit_documents.append(Document(file_path, file_content))
+
+    return from_commit_documents, to_commit_documents, diff_documents
+
+
+def parse_pre_receive_input() -> str:
+    """Parse input to pushed branch update details.
+
+    Example input:
+    old_value new_value refname
+    -----------------------------------------------
+    0000000000000000000000000000000000000000 9cf90954ef26e7c58284f8ebf7dcd0fcf711152a refs/heads/main
+    973a96d3e925b65941f7c47fa16129f1577d499f 0000000000000000000000000000000000000000 refs/heads/feature-branch
+    59564ef68745bca38c42fc57a7822efd519a6bd9 3378e52dcfa47fb11ce3a4a520bea5f85d5d0bf3 refs/heads/develop
+
+    :return: First branch update details (input's first line)
+    """
+    # FIXME(MarshalX): this blocks main thread forever if called outside of pre-receive hook
+    pre_receive_input = sys.stdin.read().strip()
+    if not pre_receive_input:
+        raise ValueError(
+            'Pre receive input was not found. Make sure that you are using this command only in pre-receive hook'
+        )
+
+    # each line represents a branch update request, handle the first one only
+    # TODO(MichalBor): support case of multiple update branch requests
+    return pre_receive_input.splitlines()[0]
+
+
+def get_diff_file_path(diff: 'Diff', relative: bool = False) -> Optional[str]:
+    if relative:
+        # relative to the repository root
+        return diff.b_path if diff.b_path else diff.a_path
+
+    if diff.b_blob:
+        return diff.b_blob.abspath
+    return diff.a_blob.abspath
+
+
+def get_diff_file_content(diff: 'Diff') -> str:
+    return diff.diff.decode('UTF-8', errors='replace')
+
+
+def get_pre_commit_modified_documents(
+    progress_bar: 'BaseProgressBar',
+    progress_bar_section: 'ProgressBarSection',
+    repo_path: str,
+) -> tuple[list[Document], list[Document]]:
+    git_head_documents = []
+    pre_committed_documents = []
+
+    repo = git_proxy.get_repo(repo_path)
+    diff_index = repo.index.diff(consts.GIT_HEAD_COMMIT_REV, create_patch=True, R=True)
+    progress_bar.set_section_length(progress_bar_section, len(diff_index))
+    for diff in diff_index:
+        progress_bar.update(progress_bar_section)
+
+        file_path = get_path_by_os(get_diff_file_path(diff))
+        file_content = _get_file_content_from_commit_diff(repo, consts.GIT_HEAD_COMMIT_REV, diff)
+        if file_content is not None:
+            git_head_documents.append(Document(file_path, file_content))
+
+        if os.path.exists(file_path):
+            file_content = get_file_content(file_path)
+            pre_committed_documents.append(Document(file_path, file_content))
+
+    return git_head_documents, pre_committed_documents
+
+
+def parse_commit_range(commit_range: str, path: str) -> tuple[str, str]:
+    from_commit_rev = None
+    to_commit_rev = None
+
+    for commit in git_proxy.get_repo(path).iter_commits(rev=commit_range):
+        if not to_commit_rev:
+            to_commit_rev = commit.hexsha
+        from_commit_rev = commit.hexsha
+
+    return from_commit_rev, to_commit_rev