From 550518625f678a0f1e573b5fc50c78c993be559e Mon Sep 17 00:00:00 2001 From: naftali-hershler Date: Tue, 5 Nov 2024 19:44:40 +0200 Subject: [PATCH 1/4] CM-41217 Add support for SBT restore --- .../sca/base_restore_dependencies.py | 18 +++++++++-- .../cli/files_collector/sca/sbt/__init__.py | 0 .../sca/sbt/restore_sbt_dependencies.py | 30 +++++++++++++++++++ .../files_collector/sca/sca_code_scanner.py | 14 ++++----- cycode/cli/utils/shell_executor.py | 8 +++-- 5 files changed, 56 insertions(+), 14 deletions(-) create mode 100644 cycode/cli/files_collector/sca/sbt/__init__.py create mode 100644 cycode/cli/files_collector/sca/sbt/restore_sbt_dependencies.py diff --git a/cycode/cli/files_collector/sca/base_restore_dependencies.py b/cycode/cli/files_collector/sca/base_restore_dependencies.py index c64b0720..e0d7558a 100644 --- a/cycode/cli/files_collector/sca/base_restore_dependencies.py +++ b/cycode/cli/files_collector/sca/base_restore_dependencies.py @@ -14,10 +14,14 @@ def build_dep_tree_path(path: str, generated_file_name: str) -> str: def execute_command( - command: List[str], file_name: str, command_timeout: int, dependencies_file_name: Optional[str] = None + command: List[str], + file_name: str, + command_timeout: int, + dependencies_file_name: Optional[str] = None, + working_directory: Optional[str] = None, ) -> Optional[str]: try: - dependencies = shell(command=command, timeout=command_timeout) + dependencies = shell(command=command, timeout=command_timeout, working_directory=working_directory) # Write stdout output to the file if output_file_path is provided if dependencies_file_name: with open(dependencies_file_name, 'w') as output_file: @@ -51,18 +55,26 @@ def get_manifest_file_path(self, document: Document) -> str: def try_restore_dependencies(self, document: Document) -> Optional[Document]: manifest_file_path = self.get_manifest_file_path(document) restore_file_path = build_dep_tree_path(document.path, self.get_lock_file_name()) + working_directory_path = self.get_working_directory(document) if self.verify_restore_file_already_exist(restore_file_path): restore_file_content = get_file_content(restore_file_path) else: output_file_path = restore_file_path if self.create_output_file_manually else None execute_command( - self.get_command(manifest_file_path), manifest_file_path, self.command_timeout, output_file_path + self.get_command(manifest_file_path), + manifest_file_path, + self.command_timeout, + output_file_path, + working_directory_path, ) restore_file_content = get_file_content(restore_file_path) return Document(restore_file_path, restore_file_content, self.is_git_diff) + def get_working_directory(self, document: Document) -> Optional[str]: + return None + @abstractmethod def verify_restore_file_already_exist(self, restore_file_path: str) -> bool: pass diff --git a/cycode/cli/files_collector/sca/sbt/__init__.py b/cycode/cli/files_collector/sca/sbt/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cycode/cli/files_collector/sca/sbt/restore_sbt_dependencies.py b/cycode/cli/files_collector/sca/sbt/restore_sbt_dependencies.py new file mode 100644 index 00000000..456df3a3 --- /dev/null +++ b/cycode/cli/files_collector/sca/sbt/restore_sbt_dependencies.py @@ -0,0 +1,30 @@ +import os +from typing import List, Optional + +import click + +from cycode.cli.files_collector.sca.base_restore_dependencies import BaseRestoreDependencies +from cycode.cli.models import Document + +SBT_PROJECT_FILE_EXTENSIONS = ['sbt'] +SBT_LOCK_FILE_NAME = 'build.sbt.lock' + + +class RestoreSbtDependencies(BaseRestoreDependencies): + def __init__(self, context: click.Context, is_git_diff: bool, command_timeout: int) -> None: + super().__init__(context, is_git_diff, command_timeout) + + def is_project(self, document: Document) -> bool: + return any(document.path.endswith(ext) for ext in SBT_PROJECT_FILE_EXTENSIONS) + + def get_command(self, manifest_file_path: str) -> List[str]: + return ['sbt', 'dependencyLockWrite', '--verbose'] + + def get_lock_file_name(self) -> str: + return SBT_LOCK_FILE_NAME + + def verify_restore_file_already_exist(self, restore_file_path: str) -> bool: + return os.path.isfile(restore_file_path) + + def get_working_directory(self, document: Document) -> Optional[str]: + return os.path.dirname(document.path) diff --git a/cycode/cli/files_collector/sca/sca_code_scanner.py b/cycode/cli/files_collector/sca/sca_code_scanner.py index 1090e7bf..9e5ac5b4 100644 --- a/cycode/cli/files_collector/sca/sca_code_scanner.py +++ b/cycode/cli/files_collector/sca/sca_code_scanner.py @@ -7,8 +7,7 @@ from cycode.cli.files_collector.sca.base_restore_dependencies import BaseRestoreDependencies from cycode.cli.files_collector.sca.maven.restore_gradle_dependencies import RestoreGradleDependencies from cycode.cli.files_collector.sca.maven.restore_maven_dependencies import RestoreMavenDependencies -from cycode.cli.files_collector.sca.npm.restore_npm_dependencies import RestoreNpmDependencies -from cycode.cli.files_collector.sca.nuget.restore_nuget_dependencies import RestoreNugetDependencies +from cycode.cli.files_collector.sca.sbt.restore_sbt_dependencies import RestoreSbtDependencies from cycode.cli.models import Document from cycode.cli.utils.git_proxy import git_proxy from cycode.cli.utils.path_utils import get_file_content, get_file_dir, get_path_from_context, join_paths @@ -17,9 +16,7 @@ if TYPE_CHECKING: from git import Repo -BUILD_GRADLE_DEP_TREE_TIMEOUT = 180 -BUILD_NUGET_DEP_TREE_TIMEOUT = 180 -BUILD_NPM_DEP_TREE_TIMEOUT = 180 +BUILD_DEP_TREE_TIMEOUT = 180 def perform_pre_commit_range_scan_actions( @@ -132,10 +129,9 @@ def add_dependencies_tree_document( def restore_handlers(context: click.Context, is_git_diff: bool) -> List[BaseRestoreDependencies]: return [ - RestoreGradleDependencies(context, is_git_diff, BUILD_GRADLE_DEP_TREE_TIMEOUT), - RestoreMavenDependencies(context, is_git_diff, BUILD_GRADLE_DEP_TREE_TIMEOUT), - RestoreNugetDependencies(context, is_git_diff, BUILD_NUGET_DEP_TREE_TIMEOUT), - RestoreNpmDependencies(context, is_git_diff, BUILD_NPM_DEP_TREE_TIMEOUT), + RestoreGradleDependencies(context, is_git_diff, BUILD_DEP_TREE_TIMEOUT), + RestoreMavenDependencies(context, is_git_diff, BUILD_DEP_TREE_TIMEOUT), + RestoreSbtDependencies(context, is_git_diff, BUILD_DEP_TREE_TIMEOUT), ] diff --git a/cycode/cli/utils/shell_executor.py b/cycode/cli/utils/shell_executor.py index a0883d6d..5ac79518 100644 --- a/cycode/cli/utils/shell_executor.py +++ b/cycode/cli/utils/shell_executor.py @@ -8,12 +8,16 @@ _SUBPROCESS_DEFAULT_TIMEOUT_SEC = 60 -def shell(command: Union[str, List[str]], timeout: int = _SUBPROCESS_DEFAULT_TIMEOUT_SEC) -> Optional[str]: +def shell( + command: Union[str, List[str]], + timeout: int = _SUBPROCESS_DEFAULT_TIMEOUT_SEC, + working_directory: Optional[str] = None, +) -> Optional[str]: logger.debug('Executing shell command: %s', command) try: result = subprocess.run( # noqa: S603 - command, timeout=timeout, check=True, capture_output=True + command, cwd=working_directory, timeout=timeout, check=True, capture_output=True ) return result.stdout.decode('UTF-8').strip() From 60b47ce8bd4dffdeb960fc9c7da785594ad22afc Mon Sep 17 00:00:00 2001 From: naftalicy Date: Wed, 6 Nov 2024 13:54:20 +0200 Subject: [PATCH 2/4] CM-41217 Update Readme Update README.md --- README.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/README.md b/README.md index cb6ae032..24e2b742 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,8 @@ This guide will guide you through both installation and usage. 6. [Commit History Scan](#commit-history-scan) 1. [Commit Range Option](#commit-range-option) 7. [Pre-Commit Scan](#pre-commit-scan) + 8. [Lock Restore Options](#lock-restore-options) + 1. [SBT Scan](#sbt-scan) 2. [Scan Results](#scan-results) 1. [Show/Hide Secrets](#showhide-secrets) 2. [Soft Fail](#soft-fail) @@ -496,6 +498,17 @@ After your install the pre-commit hook and, you may, on occasion, wish to skip s `SKIP=cycode git commit -m ` +### Lock Restore Options + +#### SBT Scan + +We use sbt-dependency-lock plugin to restore the lock file for SBT projects. +To disable lock restore in use `--no-restore` option. + +Prerequisites +* sbt-dependency-lock Plugin: Install the plugin by adding the following line to `project/plugins.sbt`: +`addSbtPlugin("software.purpledragon" % "sbt-dependency-lock" % "1.5.1")` + ## Scan Results Each scan will complete with a message stating if any issues were found or not. From ba6d27494acd2934225e68d32e06bf85a44a03b5 Mon Sep 17 00:00:00 2001 From: naftali-hershler Date: Wed, 6 Nov 2024 17:38:54 +0200 Subject: [PATCH 3/4] CM-41217 Fix PR comments --- .../cli/files_collector/sca/sbt/restore_sbt_dependencies.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/cycode/cli/files_collector/sca/sbt/restore_sbt_dependencies.py b/cycode/cli/files_collector/sca/sbt/restore_sbt_dependencies.py index 456df3a3..8023a6d6 100644 --- a/cycode/cli/files_collector/sca/sbt/restore_sbt_dependencies.py +++ b/cycode/cli/files_collector/sca/sbt/restore_sbt_dependencies.py @@ -1,8 +1,6 @@ import os from typing import List, Optional -import click - from cycode.cli.files_collector.sca.base_restore_dependencies import BaseRestoreDependencies from cycode.cli.models import Document @@ -11,9 +9,6 @@ class RestoreSbtDependencies(BaseRestoreDependencies): - def __init__(self, context: click.Context, is_git_diff: bool, command_timeout: int) -> None: - super().__init__(context, is_git_diff, command_timeout) - def is_project(self, document: Document) -> bool: return any(document.path.endswith(ext) for ext in SBT_PROJECT_FILE_EXTENSIONS) From 0f5c9efe6d53b5ec7d1722a5900ff477402fe6d5 Mon Sep 17 00:00:00 2001 From: naftali-hershler Date: Thu, 7 Nov 2024 11:52:59 +0200 Subject: [PATCH 4/4] CM-41217 Add absolute path to document --- .../commands/scan/repository/repository_command.py | 11 +++++++++-- .../sca/sbt/restore_sbt_dependencies.py | 2 +- cycode/cli/models.py | 8 +++++++- 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/cycode/cli/commands/scan/repository/repository_command.py b/cycode/cli/commands/scan/repository/repository_command.py index 87b8bbcc..9485c31c 100644 --- a/cycode/cli/commands/scan/repository/repository_command.py +++ b/cycode/cli/commands/scan/repository/repository_command.py @@ -48,8 +48,15 @@ def repository_command(context: click.Context, path: str, branch: str) -> None: # FIXME(MarshalX): probably file could be tree or submodule too. we expect blob only progress_bar.update(ScanProgressBarSection.PREPARE_LOCAL_FILES) - file_path = file.path if monitor else get_path_by_os(os.path.join(path, file.path)) - documents_to_scan.append(Document(file_path, file.data_stream.read().decode('UTF-8', errors='replace'))) + absolute_path = get_path_by_os(os.path.join(path, file.path)) + file_path = file.path if monitor else absolute_path + documents_to_scan.append( + Document( + file_path, + file.data_stream.read().decode('UTF-8', errors='replace'), + absolute_path=absolute_path, + ) + ) documents_to_scan = exclude_irrelevant_documents_to_scan(scan_type, documents_to_scan) diff --git a/cycode/cli/files_collector/sca/sbt/restore_sbt_dependencies.py b/cycode/cli/files_collector/sca/sbt/restore_sbt_dependencies.py index 8023a6d6..f5073ef0 100644 --- a/cycode/cli/files_collector/sca/sbt/restore_sbt_dependencies.py +++ b/cycode/cli/files_collector/sca/sbt/restore_sbt_dependencies.py @@ -22,4 +22,4 @@ def verify_restore_file_already_exist(self, restore_file_path: str) -> bool: return os.path.isfile(restore_file_path) def get_working_directory(self, document: Document) -> Optional[str]: - return os.path.dirname(document.path) + return os.path.dirname(document.absolute_path) diff --git a/cycode/cli/models.py b/cycode/cli/models.py index 66846725..4d4d241c 100644 --- a/cycode/cli/models.py +++ b/cycode/cli/models.py @@ -7,12 +7,18 @@ class Document: def __init__( - self, path: str, content: str, is_git_diff_format: bool = False, unique_id: Optional[str] = None + self, + path: str, + content: str, + is_git_diff_format: bool = False, + unique_id: Optional[str] = None, + absolute_path: Optional[str] = None, ) -> None: self.path = path self.content = content self.is_git_diff_format = is_git_diff_format self.unique_id = unique_id + self.absolute_path = absolute_path def __repr__(self) -> str: return 'path:{0}, content:{1}'.format(self.path, self.content)