CM-46731 - Make all flows use scan service (#296)

Author: MarshalX

README.md

@@ -303,7 +303,6 @@ The Cycode CLI application offers several types of scans so that you can choose
 | `--monitor`                                                | When specified, the scan results will be recorded in the knowledge graph. Please note that when working in `monitor` mode, the knowledge graph will not be updated as a result of SCM events (Push, Repo creation). (Supported for SCA scan type only). |
 | `--report`                                                 | When specified, a violations report will be generated. A URL link to the report will be printed as an output to the command execution.                                                                                                                  |
 | `--no-restore`                                             | When specified, Cycode will not run restore command. Will scan direct dependencies ONLY!                                                                                                                                                                |
-| `--sync`                                                   | Run scan synchronously (the default is asynchronous).                                                                                                                                                                                                   |
 | `--gradle-all-sub-projects`                                | When specified, Cycode will run gradle restore command for all sub projects. Should run from root project directory ONLY!                                                                                                                               |
 | `--help`                                                   | Show options for given command.                                                                                                                                                                                                                         |
 

cycode/cli/apps/scan/code_scanner.py

@@ -100,24 +100,30 @@ def set_issue_detected_by_scan_results(ctx: typer.Context, scan_results: List[Lo
     set_issue_detected(ctx, any(scan_result.issue_detected for scan_result in scan_results))
 
 
-def _should_use_scan_service(scan_type: str, scan_parameters: dict) -> bool:
-    return scan_type == consts.SECRET_SCAN_TYPE and scan_parameters.get('report') is True
+def _should_use_sync_flow(command_scan_type: str, scan_type: str, sync_option: bool) -> bool:
+    """Decide whether to use sync flow or async flow for the scan.
 
+    Note:
+        Passing `--sync` option does not mean that sync flow will be used in all cases.
 
-def _should_use_sync_flow(
-    command_scan_type: str, scan_type: str, sync_option: bool, scan_parameters: Optional[dict] = None
-) -> bool:
-    if not sync_option:
+    The logic:
+    - for IAC scan, sync flow is always used
+    - for SAST scan, sync flow is not supported
+    - for SCA and Secrets scan, sync flow is supported only for path/repository scan
+    """
+    if not sync_option and scan_type != consts.IAC_SCAN_TYPE:
         return False
 
     if command_scan_type not in {'path', 'repository'}:
-        raise ValueError(f'Sync flow is not available for "{command_scan_type}" command type. Remove --sync option.')
+        return False
 
-    if scan_type is consts.SAST_SCAN_TYPE:
-        raise ValueError('Sync scan is not available for SAST scan type.')
+    if scan_type == consts.IAC_SCAN_TYPE:
+        # sync in the only available flow for IAC scan; we do not use detector directly anymore
+        return True
 
-    if scan_parameters.get('report') is True:
-        raise ValueError('You can not use sync flow with report option. Either remove "report" or "sync" option.')
+    if scan_type is consts.SAST_SCAN_TYPE:  # noqa: SIM103
+        # SAST does not support sync flow
+        return False
 
     return True
 
@@ -169,8 +175,7 @@ def _scan_batch_thread_func(batch: List[Document]) -> Tuple[str, CliError, Local
         scan_id = str(_generate_unique_id())
         scan_completed = False
 
-        should_use_scan_service = _should_use_scan_service(scan_type, scan_parameters)
-        should_use_sync_flow = _should_use_sync_flow(command_scan_type, scan_type, sync_option, scan_parameters)
+        should_use_sync_flow = _should_use_sync_flow(command_scan_type, scan_type, sync_option)
 
         try:
             logger.debug('Preparing local files, %s', {'batch_files_count': len(batch)})
@@ -180,11 +185,9 @@ def _scan_batch_thread_func(batch: List[Document]) -> Tuple[str, CliError, Local
                 cycode_client,
                 zipped_documents,
                 scan_type,
-                scan_id,
                 is_git_diff,
                 is_commit_range,
                 scan_parameters,
-                should_use_scan_service,
                 should_use_sync_flow,
             )
 
@@ -224,7 +227,6 @@ def _scan_batch_thread_func(batch: List[Document]) -> Tuple[str, CliError, Local
             zip_file_size,
             command_scan_type,
             error_message,
-            should_use_scan_service or should_use_sync_flow,  # sync flow implies scan service
         )
 
         return scan_id, error, local_scan_result
@@ -456,24 +458,16 @@ def perform_scan(
     cycode_client: 'ScanClient',
     zipped_documents: 'InMemoryZip',
     scan_type: str,
-    scan_id: str,
     is_git_diff: bool,
     is_commit_range: bool,
     scan_parameters: dict,
-    should_use_scan_service: bool = False,
     should_use_sync_flow: bool = False,
 ) -> ZippedFileScanResult:
     if should_use_sync_flow:
         # it does not support commit range scans; should_use_sync_flow handles it
         return perform_scan_sync(cycode_client, zipped_documents, scan_type, scan_parameters, is_git_diff)
 
-    if scan_type in (consts.SCA_SCAN_TYPE, consts.SAST_SCAN_TYPE) or should_use_scan_service:
-        return perform_scan_async(cycode_client, zipped_documents, scan_type, scan_parameters, is_commit_range)
-
-    if is_commit_range:
-        return cycode_client.commit_range_zipped_file_scan(scan_type, zipped_documents, scan_id)
-
-    return cycode_client.zipped_file_scan(scan_type, zipped_documents, scan_id, scan_parameters, is_git_diff)
+    return perform_scan_async(cycode_client, zipped_documents, scan_type, scan_parameters, is_commit_range)
 
 
 def perform_scan_async(
@@ -823,7 +817,6 @@ def _report_scan_status(
     zip_size: int,
     command_scan_type: str,
     error_message: Optional[str],
-    should_use_scan_service: bool = False,
 ) -> None:
     try:
         end_scan_time = time.time()
@@ -840,12 +833,15 @@ def _report_scan_status(
             'scan_type': scan_type,
         }
 
-        cycode_client.report_scan_status(scan_type, scan_id, scan_status, should_use_scan_service)
+        cycode_client.report_scan_status(scan_type, scan_id, scan_status)
     except Exception as e:
         logger.debug('Failed to report scan status', exc_info=e)
 
 
 def _generate_unique_id() -> UUID:
+    if 'PYTEST_TEST_UNIQUE_ID' in os.environ:
+        return UUID(os.environ['PYTEST_TEST_UNIQUE_ID'])
+
     return uuid4()
 
 
@@ -868,13 +864,13 @@ def _get_scan_result(
     if not scan_details.detections_count:
         return init_default_scan_result(scan_id)
 
-    scan_raw_detections = cycode_client.get_scan_raw_detections(scan_type, scan_id)
+    scan_raw_detections = cycode_client.get_scan_raw_detections(scan_id)
 
     return ZippedFileScanResult(
         did_detect=True,
         detections_per_file=_map_detections_per_file_and_commit_id(scan_type, scan_raw_detections),
         scan_id=scan_id,
-        report_url=_try_get_any_report_url_if_needed(cycode_client, scan_id, scan_type, scan_parameters),
+        report_url=_try_get_aggregation_report_url_if_needed(scan_parameters, cycode_client, scan_type),
     )
 
 
@@ -886,37 +882,6 @@ def init_default_scan_result(scan_id: str) -> ZippedFileScanResult:
     )
 
 
-def _try_get_any_report_url_if_needed(
-    cycode_client: 'ScanClient',
-    scan_id: str,
-    scan_type: str,
-    scan_parameters: dict,
-) -> Optional[str]:
-    """Tries to get aggregation report URL if needed, otherwise tries to get report URL."""
-    aggregation_report_url = None
-    if scan_parameters:
-        _try_get_report_url_if_needed(cycode_client, scan_id, scan_type, scan_parameters)
-        aggregation_report_url = _try_get_aggregation_report_url_if_needed(scan_parameters, cycode_client, scan_type)
-
-    if aggregation_report_url:
-        return aggregation_report_url
-
-    return _try_get_report_url_if_needed(cycode_client, scan_id, scan_type, scan_parameters)
-
-
-def _try_get_report_url_if_needed(
-    cycode_client: 'ScanClient', scan_id: str, scan_type: str, scan_parameters: dict
-) -> Optional[str]:
-    if not scan_parameters.get('report', False):
-        return None
-
-    try:
-        report_url_response = cycode_client.get_scan_report_url(scan_id, scan_type)
-        return report_url_response.report_url
-    except Exception as e:
-        logger.debug('Failed to get report URL', exc_info=e)
-
-
 def _set_aggregation_report_url(ctx: typer.Context, aggregation_report_url: Optional[str] = None) -> None:
     ctx.obj['aggregation_report_url'] = aggregation_report_url
 

cycode/cli/apps/scan/scan_command.py

@@ -54,7 +54,9 @@ def scan_command(
     ] = SeverityOption.INFO,
     sync: Annotated[
         bool,
-        typer.Option('--sync', help='Run scan synchronously.', show_default='asynchronously'),
+        typer.Option(
+            '--sync', help='Run scan synchronously (INTERNAL FOR IDEs).', show_default='asynchronously', hidden=True
+        ),
     ] = False,
     report: Annotated[
         bool,

cycode/cyclient/models.py

@@ -59,19 +59,6 @@ def __init__(self, file_name: str, detections: List[Detection], commit_id: Optio
         self.commit_id = commit_id
 
 
-class DetectionsPerFileSchema(Schema):
-    class Meta:
-        unknown = EXCLUDE
-
-    file_name = fields.String()
-    detections = fields.List(fields.Nested(DetectionSchema))
-    commit_id = fields.String(allow_none=True)
-
-    @post_load
-    def build_dto(self, data: Dict[str, Any], **_) -> 'DetectionsPerFile':
-        return DetectionsPerFile(**data)
-
-
 class ZippedFileScanResult(Schema):
     def __init__(
         self,
@@ -89,21 +76,6 @@ def __init__(
         self.err = err
 
 
-class ZippedFileScanResultSchema(Schema):
-    class Meta:
-        unknown = EXCLUDE
-
-    did_detect = fields.Boolean()
-    scan_id = fields.String()
-    report_url = fields.String(allow_none=True)
-    detections_per_file = fields.List(fields.Nested(DetectionsPerFileSchema))
-    err = fields.String()
-
-    @post_load
-    def build_dto(self, data: Dict[str, Any], **_) -> 'ZippedFileScanResult':
-        return ZippedFileScanResult(**data)
-
-
 class ScanResult(Schema):
     def __init__(
         self,