From a7472bca2d30dcbe1e83bfb6149df6721f63dbd3 Mon Sep 17 00:00:00 2001 From: Mor Samouchian Date: Tue, 18 Feb 2025 16:48:48 +0200 Subject: [PATCH 1/4] CM-44581 gradle - support restore projects and by selecting specific project --- cycode/cli/commands/scan/scan_command.py | 12 ++++++- cycode/cli/consts.py | 2 ++ .../sca/maven/restore_gradle_dependencies.py | 36 +++++++++++++++++-- 3 files changed, 46 insertions(+), 4 deletions(-) diff --git a/cycode/cli/commands/scan/scan_command.py b/cycode/cli/commands/scan/scan_command.py index 5282dfb7..1f15ff0f 100644 --- a/cycode/cli/commands/scan/scan_command.py +++ b/cycode/cli/commands/scan/scan_command.py @@ -12,7 +12,7 @@ from cycode.cli.consts import ( ISSUE_DETECTED_STATUS_CODE, NO_ISSUES_STATUS_CODE, - SCA_SKIP_RESTORE_DEPENDENCIES_FLAG, + SCA_SKIP_RESTORE_DEPENDENCIES_FLAG, SCA_GRADLE_ALL_SUB_PROJECTS_FLAG, ) from cycode.cli.models import Severity from cycode.cli.sentry import add_breadcrumb @@ -109,6 +109,14 @@ type=bool, required=False, ) +@click.option( + f'--{SCA_GRADLE_ALL_SUB_PROJECTS_FLAG}', + is_flag=True, + default=False, + help='When specified, Cycode will run gradle restore command for all sub projects. Should run from root project directory ONLY!', + type=bool, + required=False, +) @click.pass_context def scan_command( context: click.Context, @@ -123,6 +131,7 @@ def scan_command( report: bool, no_restore: bool, sync: bool, + gradle_all_sub_projects: bool ) -> int: """Scans for Secrets, IaC, SCA or SAST violations.""" add_breadcrumb('scan') @@ -144,6 +153,7 @@ def scan_command( context.obj['monitor'] = monitor context.obj['report'] = report context.obj[SCA_SKIP_RESTORE_DEPENDENCIES_FLAG] = no_restore + context.obj[SCA_GRADLE_ALL_SUB_PROJECTS_FLAG] = gradle_all_sub_projects _sca_scan_to_context(context, sca_scan) diff --git a/cycode/cli/consts.py b/cycode/cli/consts.py index b4b09a15..be470e85 100644 --- a/cycode/cli/consts.py +++ b/cycode/cli/consts.py @@ -222,3 +222,5 @@ SCA_SHORTCUT_DEPENDENCY_PATHS = 2 SCA_SKIP_RESTORE_DEPENDENCIES_FLAG = 'no-restore' + +SCA_GRADLE_ALL_SUB_PROJECTS_FLAG = 'gradle-all-sub-projects' diff --git a/cycode/cli/files_collector/sca/maven/restore_gradle_dependencies.py b/cycode/cli/files_collector/sca/maven/restore_gradle_dependencies.py index 04fc6b9c..f885c59b 100644 --- a/cycode/cli/files_collector/sca/maven/restore_gradle_dependencies.py +++ b/cycode/cli/files_collector/sca/maven/restore_gradle_dependencies.py @@ -1,28 +1,58 @@ import os -from typing import List +import re +from typing import List, Set, Optional import click +from cycode.cli.consts import SCA_GRADLE_ALL_SUB_PROJECTS_FLAG from cycode.cli.files_collector.sca.base_restore_dependencies import BaseRestoreDependencies from cycode.cli.models import Document +from cycode.cli.utils.shell_executor import shell BUILD_GRADLE_FILE_NAME = 'build.gradle' BUILD_GRADLE_KTS_FILE_NAME = 'build.gradle.kts' BUILD_GRADLE_DEP_TREE_FILE_NAME = 'gradle-dependencies-generated.txt' +BUILD_GRADLE_ALL_PROJECTS_TIMEOUT = 180 +BUILD_GRADLE_ALL_PROJECTS_COMMAND = ['gradle', 'projects'] +ALL_PROJECTS_REGEX = r"[+-]{3} Project '(.*?)'" class RestoreGradleDependencies(BaseRestoreDependencies): - def __init__(self, context: click.Context, is_git_diff: bool, command_timeout: int) -> None: + def __init__(self, context: click.Context, is_git_diff: bool, command_timeout: int, + projects: Set[str] = set()) -> None: super().__init__(context, is_git_diff, command_timeout, create_output_file_manually=True) + self.projects = projects + self.projects = self.get_all_projects() if self.is_gradle_sub_projects() else set() + + def is_gradle_sub_projects(self): + return self.context.obj.get(SCA_GRADLE_ALL_SUB_PROJECTS_FLAG) def is_project(self, document: Document) -> bool: return document.path.endswith(BUILD_GRADLE_FILE_NAME) or document.path.endswith(BUILD_GRADLE_KTS_FILE_NAME) def get_commands(self, manifest_file_path: str) -> List[List[str]]: - return [['gradle', 'dependencies', '-b', manifest_file_path, '-q', '--console', 'plain']] + return self.get_commands_for_sub_projects(manifest_file_path) if self.is_gradle_sub_projects() else [ + ['gradle', 'dependencies', '-b', manifest_file_path, '-q', '--console', 'plain']] def get_lock_file_name(self) -> str: return BUILD_GRADLE_DEP_TREE_FILE_NAME def verify_restore_file_already_exist(self, restore_file_path: str) -> bool: return os.path.isfile(restore_file_path) + + def get_working_directory(self, document: Document) -> Optional[str]: + return self.context.params.get('paths')[0] if self.is_gradle_sub_projects() else None + + def get_all_projects(self) -> List[str]: + projects_output = shell(command=BUILD_GRADLE_ALL_PROJECTS_COMMAND, timeout=BUILD_GRADLE_ALL_PROJECTS_TIMEOUT, + working_directory=self.context.params.get('paths')[0]) + + projects = re.findall(ALL_PROJECTS_REGEX, projects_output) + + return set(projects) + + def get_commands_for_sub_projects(self, manifest_file_path: str) -> List[List[str]]: + project_name = os.path.basename(os.path.dirname(manifest_file_path)) + project_name = f':{project_name}' + return [['gradle', f'{project_name}:dependencies', '-q', '--console', + 'plain']] if project_name in self.projects else [] From dc0533ac9ac7c4876940a902ce2eb21028afd01c Mon Sep 17 00:00:00 2001 From: Mor Samouchian Date: Mon, 24 Feb 2025 13:27:37 +0200 Subject: [PATCH 2/4] CM-44581 gradle - support restore projects and by selecting specific project --- .../files_collector/sca/maven/restore_gradle_dependencies.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cycode/cli/files_collector/sca/maven/restore_gradle_dependencies.py b/cycode/cli/files_collector/sca/maven/restore_gradle_dependencies.py index f885c59b..765d37b3 100644 --- a/cycode/cli/files_collector/sca/maven/restore_gradle_dependencies.py +++ b/cycode/cli/files_collector/sca/maven/restore_gradle_dependencies.py @@ -7,6 +7,7 @@ from cycode.cli.consts import SCA_GRADLE_ALL_SUB_PROJECTS_FLAG from cycode.cli.files_collector.sca.base_restore_dependencies import BaseRestoreDependencies from cycode.cli.models import Document +from cycode.cli.utils.path_utils import get_path_from_context from cycode.cli.utils.shell_executor import shell BUILD_GRADLE_FILE_NAME = 'build.gradle' @@ -41,11 +42,11 @@ def verify_restore_file_already_exist(self, restore_file_path: str) -> bool: return os.path.isfile(restore_file_path) def get_working_directory(self, document: Document) -> Optional[str]: - return self.context.params.get('paths')[0] if self.is_gradle_sub_projects() else None + return get_path_from_context(self.context) if self.is_gradle_sub_projects() else None def get_all_projects(self) -> List[str]: projects_output = shell(command=BUILD_GRADLE_ALL_PROJECTS_COMMAND, timeout=BUILD_GRADLE_ALL_PROJECTS_TIMEOUT, - working_directory=self.context.params.get('paths')[0]) + working_directory=get_path_from_context(self.context)) projects = re.findall(ALL_PROJECTS_REGEX, projects_output) From e81868ca207de556437dfd3e8680dd0e1dc3171d Mon Sep 17 00:00:00 2001 From: Mor Samouchian Date: Mon, 24 Feb 2025 13:43:29 +0200 Subject: [PATCH 3/4] CM-44581 gradle - support restore projects and by selecting specific project --- cycode/cli/utils/scan_batch.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cycode/cli/utils/scan_batch.py b/cycode/cli/utils/scan_batch.py index 1ecfcf49..7bda640e 100644 --- a/cycode/cli/utils/scan_batch.py +++ b/cycode/cli/utils/scan_batch.py @@ -50,7 +50,11 @@ def run_parallel_batched_scan( progress_bar: 'BaseProgressBar', ) -> Tuple[Dict[str, 'CliError'], List['LocalScanResult']]: max_size = consts.SCAN_BATCH_MAX_SIZE_IN_BYTES.get(scan_type, consts.DEFAULT_SCAN_BATCH_MAX_SIZE_IN_BYTES) - batches = split_documents_into_batches(documents, max_size) + + if scan_type == consts.SCA_SCAN_TYPE: + batches = [documents] + else: + batches = split_documents_into_batches(documents, max_size) progress_bar.set_section_length(ScanProgressBarSection.SCAN, len(batches)) # * 3 # TODO(MarshalX): we should multiply the count of batches in SCAN section because each batch has 3 steps: From d616b9ec0e7312e32e8d9b7b864864c8f81cd9de Mon Sep 17 00:00:00 2001 From: Mor Samouchian Date: Mon, 24 Feb 2025 13:52:35 +0200 Subject: [PATCH 4/4] CM-44581 gradle - support restore projects and by selecting specific project --- cycode/cli/commands/scan/scan_command.py | 8 ++-- .../sca/maven/restore_gradle_dependencies.py | 37 ++++++++++++------- cycode/cli/utils/scan_batch.py | 5 +-- 3 files changed, 30 insertions(+), 20 deletions(-) diff --git a/cycode/cli/commands/scan/scan_command.py b/cycode/cli/commands/scan/scan_command.py index 93afd911..95259f4a 100644 --- a/cycode/cli/commands/scan/scan_command.py +++ b/cycode/cli/commands/scan/scan_command.py @@ -13,7 +13,8 @@ from cycode.cli.consts import ( ISSUE_DETECTED_STATUS_CODE, NO_ISSUES_STATUS_CODE, - SCA_SKIP_RESTORE_DEPENDENCIES_FLAG, SCA_GRADLE_ALL_SUB_PROJECTS_FLAG, + SCA_GRADLE_ALL_SUB_PROJECTS_FLAG, + SCA_SKIP_RESTORE_DEPENDENCIES_FLAG, ) from cycode.cli.models import Severity from cycode.cli.sentry import add_breadcrumb @@ -114,7 +115,8 @@ f'--{SCA_GRADLE_ALL_SUB_PROJECTS_FLAG}', is_flag=True, default=False, - help='When specified, Cycode will run gradle restore command for all sub projects. Should run from root project directory ONLY!', + help='When specified, Cycode will run gradle restore command for all sub projects. ' + 'Should run from root project directory ONLY!', type=bool, required=False, ) @@ -132,7 +134,7 @@ def scan_command( report: bool, no_restore: bool, sync: bool, - gradle_all_sub_projects: bool + gradle_all_sub_projects: bool, ) -> int: """Scans for Secrets, IaC, SCA or SAST violations.""" add_breadcrumb('scan') diff --git a/cycode/cli/files_collector/sca/maven/restore_gradle_dependencies.py b/cycode/cli/files_collector/sca/maven/restore_gradle_dependencies.py index 765d37b3..85dc9e20 100644 --- a/cycode/cli/files_collector/sca/maven/restore_gradle_dependencies.py +++ b/cycode/cli/files_collector/sca/maven/restore_gradle_dependencies.py @@ -1,6 +1,6 @@ import os import re -from typing import List, Set, Optional +from typing import List, Optional, Set import click @@ -19,21 +19,26 @@ class RestoreGradleDependencies(BaseRestoreDependencies): - def __init__(self, context: click.Context, is_git_diff: bool, command_timeout: int, - projects: Set[str] = set()) -> None: + def __init__( + self, context: click.Context, is_git_diff: bool, command_timeout: int, projects: Optional[Set[str]] = None + ) -> None: super().__init__(context, is_git_diff, command_timeout, create_output_file_manually=True) - self.projects = projects - self.projects = self.get_all_projects() if self.is_gradle_sub_projects() else set() + if projects is None: + projects = set() + self.projects = self.get_all_projects() if self.is_gradle_sub_projects() else projects - def is_gradle_sub_projects(self): + def is_gradle_sub_projects(self) -> bool: return self.context.obj.get(SCA_GRADLE_ALL_SUB_PROJECTS_FLAG) def is_project(self, document: Document) -> bool: return document.path.endswith(BUILD_GRADLE_FILE_NAME) or document.path.endswith(BUILD_GRADLE_KTS_FILE_NAME) def get_commands(self, manifest_file_path: str) -> List[List[str]]: - return self.get_commands_for_sub_projects(manifest_file_path) if self.is_gradle_sub_projects() else [ - ['gradle', 'dependencies', '-b', manifest_file_path, '-q', '--console', 'plain']] + return ( + self.get_commands_for_sub_projects(manifest_file_path) + if self.is_gradle_sub_projects() + else [['gradle', 'dependencies', '-b', manifest_file_path, '-q', '--console', 'plain']] + ) def get_lock_file_name(self) -> str: return BUILD_GRADLE_DEP_TREE_FILE_NAME @@ -44,9 +49,12 @@ def verify_restore_file_already_exist(self, restore_file_path: str) -> bool: def get_working_directory(self, document: Document) -> Optional[str]: return get_path_from_context(self.context) if self.is_gradle_sub_projects() else None - def get_all_projects(self) -> List[str]: - projects_output = shell(command=BUILD_GRADLE_ALL_PROJECTS_COMMAND, timeout=BUILD_GRADLE_ALL_PROJECTS_TIMEOUT, - working_directory=get_path_from_context(self.context)) + def get_all_projects(self) -> Set[str]: + projects_output = shell( + command=BUILD_GRADLE_ALL_PROJECTS_COMMAND, + timeout=BUILD_GRADLE_ALL_PROJECTS_TIMEOUT, + working_directory=get_path_from_context(self.context), + ) projects = re.findall(ALL_PROJECTS_REGEX, projects_output) @@ -55,5 +63,8 @@ def get_all_projects(self) -> List[str]: def get_commands_for_sub_projects(self, manifest_file_path: str) -> List[List[str]]: project_name = os.path.basename(os.path.dirname(manifest_file_path)) project_name = f':{project_name}' - return [['gradle', f'{project_name}:dependencies', '-q', '--console', - 'plain']] if project_name in self.projects else [] + return ( + [['gradle', f'{project_name}:dependencies', '-q', '--console', 'plain']] + if project_name in self.projects + else [] + ) diff --git a/cycode/cli/utils/scan_batch.py b/cycode/cli/utils/scan_batch.py index 7bda640e..3d2d83dc 100644 --- a/cycode/cli/utils/scan_batch.py +++ b/cycode/cli/utils/scan_batch.py @@ -51,10 +51,7 @@ def run_parallel_batched_scan( ) -> Tuple[Dict[str, 'CliError'], List['LocalScanResult']]: max_size = consts.SCAN_BATCH_MAX_SIZE_IN_BYTES.get(scan_type, consts.DEFAULT_SCAN_BATCH_MAX_SIZE_IN_BYTES) - if scan_type == consts.SCA_SCAN_TYPE: - batches = [documents] - else: - batches = split_documents_into_batches(documents, max_size) + batches = [documents] if scan_type == consts.SCA_SCAN_TYPE else split_documents_into_batches(documents, max_size) progress_bar.set_section_length(ScanProgressBarSection.SCAN, len(batches)) # * 3 # TODO(MarshalX): we should multiply the count of batches in SCAN section because each batch has 3 steps: