CM-60184-Commit range scans

Author: mateusz-sterczewski

cycode/cli/apps/scan/code_scanner.py

@@ -29,6 +29,7 @@
     generate_unique_scan_id,
     is_cycodeignore_allowed_by_scan_config,
     set_issue_detected_by_scan_results,
+    should_use_presigned_upload,
 )
 from cycode.cyclient.models import ZippedFileScanResult
 from cycode.logger import get_logger
@@ -106,7 +107,10 @@ def _should_use_sync_flow(command_scan_type: str, scan_type: str, sync_option: b
 
 
 def _get_scan_documents_thread_func(
-    ctx: typer.Context, is_git_diff: bool, is_commit_range: bool, scan_parameters: dict
+    ctx: typer.Context,
+    is_git_diff: bool,
+    is_commit_range: bool,
+    scan_parameters: dict,
 ) -> Callable[[list[Document]], tuple[str, CliError, LocalScanResult]]:
     cycode_client = ctx.obj['client']
     scan_type = ctx.obj['scan_type']
@@ -202,11 +206,38 @@ def scan_documents(
         )
         return
 
-    scan_batch_thread_func = _get_scan_documents_thread_func(ctx, is_git_diff, is_commit_range, scan_parameters)
-    errors, local_scan_results = run_parallel_batched_scan(
-        scan_batch_thread_func, scan_type, documents_to_scan, progress_bar=progress_bar
+    scan_batch_thread_func = _get_scan_documents_thread_func(
+        ctx, is_git_diff, is_commit_range, scan_parameters
     )
 
+    if should_use_presigned_upload(scan_type):
+        try:
+            # Try to zip all documents as a single batch; ZipTooLargeError raised if it exceeds the scan type's limit
+            zip_documents(scan_type, documents_to_scan)
+            # It fits: skip batching and upload everything as one ZIP
+            errors, local_scan_results = run_parallel_batched_scan(
+                scan_batch_thread_func,
+                scan_type,
+                documents_to_scan,
+                progress_bar=progress_bar,
+                skip_batching=True,
+            )
+        except custom_exceptions.ZipTooLargeError:
+            printer.print_warning(
+                'The repository is too large to upload as a single file. '
+                'Falling back to batched scanning. This may result in multiple scan results.'
+            )
+            errors, local_scan_results = run_parallel_batched_scan(
+                scan_batch_thread_func,
+                scan_type,
+                documents_to_scan,
+                progress_bar=progress_bar,
+            )
+    else:
+        errors, local_scan_results = run_parallel_batched_scan(
+            scan_batch_thread_func, scan_type, documents_to_scan, progress_bar=progress_bar
+        )
+
     try_set_aggregation_report_url_if_needed(ctx, scan_parameters, ctx.obj['client'], scan_type)
 
     progress_bar.set_section_length(ScanProgressBarSection.GENERATE_REPORT, 1)

cycode/cli/apps/scan/commit_range_scanner.py

@@ -86,6 +86,38 @@ def _perform_commit_range_scan_async(
     return poll_scan_results(cycode_client, scan_async_result.scan_id, scan_type, scan_parameters, timeout)
 
 
+def _perform_commit_range_scan_v4_async(
+    cycode_client: 'ScanClient',
+    from_commit_zipped_documents: 'InMemoryZip',
+    to_commit_zipped_documents: 'InMemoryZip',
+    scan_type: str,
+    scan_parameters: dict,
+    timeout: Optional[int] = None,
+) -> ZippedFileScanResult:
+    from_upload_link = cycode_client.get_upload_link(scan_type)
+    logger.debug('Got from-commit upload link, %s', {'upload_id': from_upload_link.upload_id})
+
+    cycode_client.upload_to_presigned_post(
+        from_upload_link.url, from_upload_link.presigned_post_fields, from_commit_zipped_documents
+    )
+    logger.debug('Uploaded from-commit zip')
+
+    to_upload_link = cycode_client.get_upload_link(scan_type)
+    logger.debug('Got to-commit upload link, %s', {'upload_id': to_upload_link.upload_id})
+
+    cycode_client.upload_to_presigned_post(
+        to_upload_link.url, to_upload_link.presigned_post_fields, to_commit_zipped_documents
+    )
+    logger.debug('Uploaded to-commit zip')
+
+    scan_async_result = cycode_client.commit_range_scan_from_upload_ids(
+        scan_type, from_upload_link.upload_id, to_upload_link.upload_id, scan_parameters
+    )
+    logger.debug('V4 commit range scan request triggered, %s', {'scan_id': scan_async_result.scan_id})
+
+    return poll_scan_results(cycode_client, scan_async_result.scan_id, scan_type, scan_parameters, timeout)
+
+
 def _scan_commit_range_documents(
     ctx: typer.Context,
     from_documents_to_scan: list[Document],
@@ -118,14 +150,24 @@ def _scan_commit_range_documents(
             # for SAST it is files with diff between from_commit and to_commit
             to_commit_zipped_documents = zip_documents(scan_type, to_documents_to_scan)
 
-            scan_result = _perform_commit_range_scan_async(
-                cycode_client,
-                from_commit_zipped_documents,
-                to_commit_zipped_documents,
-                scan_type,
-                scan_parameters,
-                timeout,
-            )
+            if scan_type == consts.SAST_SCAN_TYPE:
+                scan_result = _perform_commit_range_scan_v4_async(
+                    cycode_client,
+                    from_commit_zipped_documents,
+                    to_commit_zipped_documents,
+                    scan_type,
+                    scan_parameters,
+                    timeout,
+                )
+            else:
+                scan_result = _perform_commit_range_scan_async(
+                    cycode_client,
+                    from_commit_zipped_documents,
+                    to_commit_zipped_documents,
+                    scan_type,
+                    scan_parameters,
+                    timeout,
+                )
             enrich_scan_result_with_data_from_detection_rules(cycode_client, scan_result)
 
         progress_bar.update(ScanProgressBarSection.SCAN)

cycode/cli/consts.py

@@ -166,7 +166,7 @@
     COMMIT_HISTORY_COMMAND_SCAN_TYPE_OLD,
 ]
 
-DEFAULT_CYCODE_DOMAIN = 'cycode.com'
+DEFAULT_CYCODE_DOMAIN = 'cycode.xyz'
 DEFAULT_CYCODE_API_URL = f'https://api.{DEFAULT_CYCODE_DOMAIN}'
 DEFAULT_CYCODE_APP_URL = f'https://app.{DEFAULT_CYCODE_DOMAIN}'
 
@@ -192,15 +192,18 @@
 # 5MB in bytes (in decimal)
 FILE_MAX_SIZE_LIMIT_IN_BYTES = 5000000
 
+PRESIGNED_LINK_UPLOADED_ZIP_MAX_SIZE_LIMIT_IN_BYTES = 5 * 1024 * 1024 * 1024  # 5 GB (S3 presigned POST limit)
+PRESIGNED_UPLOAD_SCAN_TYPES = {SAST_SCAN_TYPE}
+
 DEFAULT_ZIP_MAX_SIZE_LIMIT_IN_BYTES = 20 * 1024 * 1024
 ZIP_MAX_SIZE_LIMIT_IN_BYTES = {
     SCA_SCAN_TYPE: 200 * 1024 * 1024,
-    SAST_SCAN_TYPE: 50 * 1024 * 1024,
+    SAST_SCAN_TYPE: PRESIGNED_LINK_UPLOADED_ZIP_MAX_SIZE_LIMIT_IN_BYTES,
 }
 
 # scan in batches
 DEFAULT_SCAN_BATCH_MAX_SIZE_IN_BYTES = 9 * 1024 * 1024
-SCAN_BATCH_MAX_SIZE_IN_BYTES = {SAST_SCAN_TYPE: 50 * 1024 * 1024}
+SCAN_BATCH_MAX_SIZE_IN_BYTES = {SAST_SCAN_TYPE: PRESIGNED_LINK_UPLOADED_ZIP_MAX_SIZE_LIMIT_IN_BYTES}
 SCAN_BATCH_MAX_SIZE_IN_BYTES_ENV_VAR_NAME = 'SCAN_BATCH_MAX_SIZE_IN_BYTES'
 
 DEFAULT_SCAN_BATCH_MAX_FILES_COUNT = 1000

cycode/cli/files_collector/zip_documents.py

@@ -17,7 +17,11 @@ def _validate_zip_file_size(scan_type: str, zip_file_size: int) -> None:
         raise custom_exceptions.ZipTooLargeError(max_size_limit)
 
 
-def zip_documents(scan_type: str, documents: list[Document], zip_file: Optional[InMemoryZip] = None) -> InMemoryZip:
+def zip_documents(
+    scan_type: str,
+    documents: list[Document],
+    zip_file: Optional[InMemoryZip] = None,
+) -> InMemoryZip:
     if zip_file is None:
         zip_file = InMemoryZip()
 

cycode/cli/utils/scan_batch.py

@@ -111,9 +111,13 @@ def run_parallel_batched_scan(
     scan_type: str,
     documents: list[Document],
     progress_bar: 'BaseProgressBar',
+    skip_batching: bool = False,
 ) -> tuple[dict[str, 'CliError'], list['LocalScanResult']]:
     # batching is disabled for SCA; requested by Mor
-    batches = [documents] if scan_type == consts.SCA_SCAN_TYPE else split_documents_into_batches(scan_type, documents)
+    if scan_type == consts.SCA_SCAN_TYPE or skip_batching:
+        batches = [documents]
+    else:
+        batches = split_documents_into_batches(scan_type, documents)
 
     progress_bar.set_section_length(ScanProgressBarSection.SCAN, len(batches))  # * 3
     # TODO(MarshalX): we should multiply the count of batches in SCAN section because each batch has 3 steps:

cycode/cli/utils/scan_utils.py

@@ -5,6 +5,7 @@
 
 import typer
 
+from cycode.cli import consts
 from cycode.cli.cli_types import SeverityOption
 
 if TYPE_CHECKING:
@@ -31,6 +32,10 @@ def is_cycodeignore_allowed_by_scan_config(ctx: typer.Context) -> bool:
     return scan_config.is_cycode_ignore_allowed if scan_config else True
 
 
+def should_use_presigned_upload(scan_type: str) -> bool:
+    return scan_type in consts.PRESIGNED_UPLOAD_SCAN_TYPES
+
+
 def generate_unique_scan_id() -> UUID:
     if 'PYTEST_TEST_UNIQUE_ID' in os.environ:
         return UUID(os.environ['PYTEST_TEST_UNIQUE_ID'])

cycode/cyclient/scan_client.py

@@ -153,7 +153,7 @@ def scan_repository_from_upload_id(
         url_path = f'{self.get_scan_service_v4_url_path(scan_type)}/{async_scan_type}/repository'
         response = self.scan_cycode_client.post(
             url_path=url_path,
-            data={
+            body={
                 'upload_id': upload_id,
                 'is_git_diff': is_git_diff,
                 'is_commit_range': is_commit_range,
@@ -200,6 +200,27 @@ def commit_range_scan_async(
         )
         return models.ScanInitializationResponseSchema().load(response.json())
 
+    def commit_range_scan_from_upload_ids(
+        self,
+        scan_type: str,
+        from_upload_id: str,
+        to_upload_id: str,
+        scan_parameters: dict,
+        is_git_diff: bool = False,
+    ) -> models.ScanInitializationResponse:
+        async_scan_type = self.scan_config.get_async_scan_type(scan_type)
+        url_path = f'{self.get_scan_service_v4_url_path(scan_type)}/{async_scan_type}/repository/commit-range'
+        response = self.scan_cycode_client.post(
+            url_path=url_path,
+            body={
+                'from_upload_id': from_upload_id,
+                'to_upload_id': to_upload_id,
+                'is_git_diff': is_git_diff,
+                'scan_parameters': json.dumps(scan_parameters),
+            },
+        )
+        return models.ScanInitializationResponseSchema().load(response.json())
+
     def get_scan_details_path(self, scan_type: str, scan_id: str) -> str:
         return f'{self.get_scan_service_url_path(scan_type)}/{scan_id}'