diff --git a/cycode/cli/commands/scan/scan_command.py b/cycode/cli/commands/scan/scan_command.py index 37b0a227..95259f4a 100644 --- a/cycode/cli/commands/scan/scan_command.py +++ b/cycode/cli/commands/scan/scan_command.py @@ -13,6 +13,7 @@ from cycode.cli.consts import ( ISSUE_DETECTED_STATUS_CODE, NO_ISSUES_STATUS_CODE, + SCA_GRADLE_ALL_SUB_PROJECTS_FLAG, SCA_SKIP_RESTORE_DEPENDENCIES_FLAG, ) from cycode.cli.models import Severity @@ -110,6 +111,15 @@ type=bool, required=False, ) +@click.option( + f'--{SCA_GRADLE_ALL_SUB_PROJECTS_FLAG}', + is_flag=True, + default=False, + help='When specified, Cycode will run gradle restore command for all sub projects. ' + 'Should run from root project directory ONLY!', + type=bool, + required=False, +) @click.pass_context def scan_command( context: click.Context, @@ -124,6 +134,7 @@ def scan_command( report: bool, no_restore: bool, sync: bool, + gradle_all_sub_projects: bool, ) -> int: """Scans for Secrets, IaC, SCA or SAST violations.""" add_breadcrumb('scan') @@ -145,6 +156,7 @@ def scan_command( context.obj['monitor'] = monitor context.obj['report'] = report context.obj[SCA_SKIP_RESTORE_DEPENDENCIES_FLAG] = no_restore + context.obj[SCA_GRADLE_ALL_SUB_PROJECTS_FLAG] = gradle_all_sub_projects _sca_scan_to_context(context, sca_scan) diff --git a/cycode/cli/consts.py b/cycode/cli/consts.py index 3640d82a..558f5b7b 100644 --- a/cycode/cli/consts.py +++ b/cycode/cli/consts.py @@ -224,3 +224,5 @@ SCA_SHORTCUT_DEPENDENCY_PATHS = 2 SCA_SKIP_RESTORE_DEPENDENCIES_FLAG = 'no-restore' + +SCA_GRADLE_ALL_SUB_PROJECTS_FLAG = 'gradle-all-sub-projects' diff --git a/cycode/cli/files_collector/sca/maven/restore_gradle_dependencies.py b/cycode/cli/files_collector/sca/maven/restore_gradle_dependencies.py index 04fc6b9c..85dc9e20 100644 --- a/cycode/cli/files_collector/sca/maven/restore_gradle_dependencies.py +++ b/cycode/cli/files_collector/sca/maven/restore_gradle_dependencies.py @@ -1,28 +1,70 @@ import os -from typing import List +import re +from typing import List, Optional, Set import click +from cycode.cli.consts import SCA_GRADLE_ALL_SUB_PROJECTS_FLAG from cycode.cli.files_collector.sca.base_restore_dependencies import BaseRestoreDependencies from cycode.cli.models import Document +from cycode.cli.utils.path_utils import get_path_from_context +from cycode.cli.utils.shell_executor import shell BUILD_GRADLE_FILE_NAME = 'build.gradle' BUILD_GRADLE_KTS_FILE_NAME = 'build.gradle.kts' BUILD_GRADLE_DEP_TREE_FILE_NAME = 'gradle-dependencies-generated.txt' +BUILD_GRADLE_ALL_PROJECTS_TIMEOUT = 180 +BUILD_GRADLE_ALL_PROJECTS_COMMAND = ['gradle', 'projects'] +ALL_PROJECTS_REGEX = r"[+-]{3} Project '(.*?)'" class RestoreGradleDependencies(BaseRestoreDependencies): - def __init__(self, context: click.Context, is_git_diff: bool, command_timeout: int) -> None: + def __init__( + self, context: click.Context, is_git_diff: bool, command_timeout: int, projects: Optional[Set[str]] = None + ) -> None: super().__init__(context, is_git_diff, command_timeout, create_output_file_manually=True) + if projects is None: + projects = set() + self.projects = self.get_all_projects() if self.is_gradle_sub_projects() else projects + + def is_gradle_sub_projects(self) -> bool: + return self.context.obj.get(SCA_GRADLE_ALL_SUB_PROJECTS_FLAG) def is_project(self, document: Document) -> bool: return document.path.endswith(BUILD_GRADLE_FILE_NAME) or document.path.endswith(BUILD_GRADLE_KTS_FILE_NAME) def get_commands(self, manifest_file_path: str) -> List[List[str]]: - return [['gradle', 'dependencies', '-b', manifest_file_path, '-q', '--console', 'plain']] + return ( + self.get_commands_for_sub_projects(manifest_file_path) + if self.is_gradle_sub_projects() + else [['gradle', 'dependencies', '-b', manifest_file_path, '-q', '--console', 'plain']] + ) def get_lock_file_name(self) -> str: return BUILD_GRADLE_DEP_TREE_FILE_NAME def verify_restore_file_already_exist(self, restore_file_path: str) -> bool: return os.path.isfile(restore_file_path) + + def get_working_directory(self, document: Document) -> Optional[str]: + return get_path_from_context(self.context) if self.is_gradle_sub_projects() else None + + def get_all_projects(self) -> Set[str]: + projects_output = shell( + command=BUILD_GRADLE_ALL_PROJECTS_COMMAND, + timeout=BUILD_GRADLE_ALL_PROJECTS_TIMEOUT, + working_directory=get_path_from_context(self.context), + ) + + projects = re.findall(ALL_PROJECTS_REGEX, projects_output) + + return set(projects) + + def get_commands_for_sub_projects(self, manifest_file_path: str) -> List[List[str]]: + project_name = os.path.basename(os.path.dirname(manifest_file_path)) + project_name = f':{project_name}' + return ( + [['gradle', f'{project_name}:dependencies', '-q', '--console', 'plain']] + if project_name in self.projects + else [] + ) diff --git a/cycode/cli/utils/scan_batch.py b/cycode/cli/utils/scan_batch.py index 1ecfcf49..3d2d83dc 100644 --- a/cycode/cli/utils/scan_batch.py +++ b/cycode/cli/utils/scan_batch.py @@ -50,7 +50,8 @@ def run_parallel_batched_scan( progress_bar: 'BaseProgressBar', ) -> Tuple[Dict[str, 'CliError'], List['LocalScanResult']]: max_size = consts.SCAN_BATCH_MAX_SIZE_IN_BYTES.get(scan_type, consts.DEFAULT_SCAN_BATCH_MAX_SIZE_IN_BYTES) - batches = split_documents_into_batches(documents, max_size) + + batches = [documents] if scan_type == consts.SCA_SCAN_TYPE else split_documents_into_batches(documents, max_size) progress_bar.set_section_length(ScanProgressBarSection.SCAN, len(batches)) # * 3 # TODO(MarshalX): we should multiply the count of batches in SCAN section because each batch has 3 steps: