update for cve-report; new rl-deploy

Author: rl-devops

Makefile

@@ -10,7 +10,6 @@ endif
 
 VOLUMES 		:= -v ./output:/output -v ./input:/input
 USER_GROUP		:= $(shell id -u):$(shell id -u )
-
 COMMON_DOCKER	:= -i --rm -u $(USER_GROUP) --env-file=$(HOME)/.envfile_rl-scanner.docker
 
 # IMAGE_NAME		:= rlsecure/scanner:latest
@@ -21,17 +20,45 @@ ARTIFACT_OK		:=	vim
 ARTIFACT_ERR	:=	eicarcom2.zip
 
 LINE_LENGTH = 120
+PL_LINTERS = "eradicate,mccabe,pycodestyle,pyflakes,pylint"
+PL_IGNORE = C0114,C0115,C0116
+SCRIPTS = scripts/
 
 IMAGE ?= reversinglabs/rl-scanner
 TAG   ?= latest
 
 .PHONY: build clean
 
-all: clean prep format build testFail test_ok test_err clean
+all: clean prep build tests
+
+clean:
+	docker image prune -f
+	-docker image rm $(IMAGE_NAME)
+	rm -f eicarcom2.zip
+	rm -rf .mypy_cache */.mypy_cache
 
-prep:
+prep: format pycheck mypy
 	wget 'https://www.eicar.org/download/eicar-com-2-2/?wpdmdl=8848&refresh=65d33af627b351708342006' --output-document 'eicarcom2.zip'
 
+format: $(SCRIPTS)
+	black \
+		--line-length $(LINE_LENGTH) \
+		$(SCRIPTS)/*
+
+pycheck: $(SCRIPTS)
+	pylama \
+		--max-line-length $(LINE_LENGTH) \
+		--linters $(PL_LINTERS) \
+		--ignore $(PL_IGNORE) \
+		$(SCRIPTS)
+
+mypy: $(SCRIPTS)
+	mypy \
+		--strict \
+		--no-incremental \
+		$(SCRIPTS)
+
+
 # build a new docker image from the Dockerfile generated
 build:
 	mkdir -p tmp
@@ -40,6 +67,8 @@ build:
 	docker image inspect $(IMAGE_NAME) --format '{{ .Config.Labels }}'
 	docker image inspect $(IMAGE_NAME) --format '{{ .RepoTags }}'
 
+tests: testFail test_ok test_err
+
 testFail:
 	# we know that specifying no arguments should print usage() and fail
 	-docker run $(COMMON_DOCKER) $(VOLUMES) $(IMAGE_NAME) # will fail but we will ignore that
@@ -50,7 +79,8 @@ test_ok:
 	rm -rf output input
 	mkdir -m 777 -p input output
 	cp /bin/$(ARTIFACT_OK) ./input/$(ARTIFACT_OK)
-	docker run $(COMMON_DOCKER) $(VOLUMES) $(IMAGE_NAME) rl-scan --package-path=/input/$(ARTIFACT_OK) --report-path=/output
+	docker run $(COMMON_DOCKER) $(VOLUMES) $(IMAGE_NAME) \
+		rl-scan --package-path=/input/$(ARTIFACT_OK) --report-path=/output --report-format all
 	ls -laR input output >./tmp/list_in_out_ok.txt
 	cat output/report.rl.json | jq -r . >tmp/test_ok.json
 
@@ -60,16 +90,8 @@ test_err:
 	curl -o $(ARTIFACT_ERR) -sS https://secure.eicar.org/$(ARTIFACT_ERR)
 	cp $(ARTIFACT_ERR) ./input/$(ARTIFACT_ERR)
 	# as we are now scanning a item that makes the scan fail (non zero exit code) we have to ignore the error in the makefile
-	-docker run $(COMMON_DOCKER) $(VOLUMES) $(IMAGE_NAME) rl-scan --package-path=/input/$(ARTIFACT_ERR) --report-path=/output
+	-docker run $(COMMON_DOCKER) $(VOLUMES) $(IMAGE_NAME) \
+		rl-scan --package-path=/input/$(ARTIFACT_ERR) --report-path=/output --report-format all
 	ls -laR input output >./tmp/list_in_out_err.txt
 	cat output/report.rl.json | jq -r . >tmp/test_err.json
 
-clean:
-	docker image prune -f
-	-docker image rm $(IMAGE_NAME)
-	rm -f eicarcom2.zip
-	rm -rf .mypy_cache */.mypy_cache
-
-format:
-	black --line-length $(LINE_LENGTH) scripts/*
-

README.md

@@ -186,7 +186,7 @@ docker run --rm \
 
 To perform a build reproducibility check, we need two build artifacts of a package version.
 
-Assuming there is already an existing package store with a previously scanned package version `project/[email protected]`, we perform a reproducibility check by scanning another build artifact of the same package version with the specially crafted project URL `project/[email protected]?build=repro`. 
+Assuming there is already an existing package store with a previously scanned package version `project/[email protected]`, we perform a reproducibility check by scanning another build artifact of the same package version with the specially crafted project URL `project/[email protected]?build=repro`.
 
 The previously scanned artifact for the package version is used as the reference against which the artifact we're scanning will be compared.
 The `build=repro` qualifier indicates our intention to perform a reproducibility build check.
@@ -220,7 +220,7 @@ The `rl-scan` helper tool supports the following parameters.
 | :--------- | :------ |
 | `--package-path` | Required. Path to the package file you want to scan. The specified package file must exist in the **package source** directory mounted to the container.  |
 | `--report-path` | Required. Path to the location where you want to store analysis reports. The specified path must exist in the **reports destination** directory mounted to the container. |
-| `--report-format` | Required. A comma-separated list of report formats to generate. Supported values: `cyclonedx`, `sarif`, `spdx`, `rl-html`, `rl-json`, `all` |
+| `--report-format` | Required. A comma-separated list of report formats to generate. Supported values: `cyclonedx`, `sarif`, `spdx`, `rl-html`, `rl-json`, `rl-checks`, `rl-cve`, `all` |
 | `--rl-store` | Optional. Path to existing rl-secure package store that is to be used for scan. |
 | `--purl` | Optional. Package URL used for scan (format `[pkg:namespace/]<project></package><@version>`). |
 | `--diff-with` | Optional. Package version that will be used as a base for difference report. |

scripts/cimessages.py

@@ -6,7 +6,7 @@
 )
 
 
-class Messages:
+class Messages:  # pylint: disable=too-few-public-methods
     @classmethod
     def create(cls, name: str) -> Any:
         if name == "teamcity":
@@ -48,8 +48,7 @@ def escape(msg: str) -> str:
         if isinstance(msg, dict):
             msg_content: List[str] = [f"{k}='{escape(v)}'" for k, v in msg.items()]
             return f"##teamcity[{name} {' '.join(msg_content)}]"
-        else:
-            return f"##teamcity[{name} '{escape(msg)}']"
+        return f"##teamcity[{name} '{escape(msg)}']"
 
     def block_start(self, msg: str) -> None:
         print(TeamCityMesages.service_message("progressStart", msg), flush=True)

scripts/rl-scan

@@ -5,10 +5,16 @@ import argparse
 import os
 import sys
 import glob
-from urllib.parse import urlsplit, parse_qs
+from urllib.parse import (
+    urlsplit,
+    parse_qs,
+)
 
 import rlsecure
 from cimessages import Messages
+from typing import (
+    List,
+)
 
 
 def __expand_specs(specs: str) -> str:
@@ -20,7 +26,20 @@ def __expand_specs(specs: str) -> str:
     return files[0]
 
 
+ReportFormatLIst: List[str] = [
+    "cyclonedx",
+    "sarif",
+    "spdx",
+    "rl-html",
+    "rl-json",
+    "rl-checks",
+    "rl-cve",
+    "all",
+]
+
+
 def main() -> int:
+    reportList = ", ".join(ReportFormatLIst)
     parser = argparse.ArgumentParser(
         formatter_class=argparse.RawDescriptionHelpFormatter,
         prog="rl-scan",
@@ -78,7 +97,7 @@ def main() -> int:
     parser.add_argument(
         "--report-format",
         default="all",
-        help="A comma-separated list of report formats to generate. Supported values: cyclonedx, sarif, spdx, rl-html, rl-json, rl-checks, all",
+        help=f"A comma-separated list of report formats to generate. Supported values: {reportList}",
     )
     parser.add_argument(
         "--message-reporter",

scripts/rlsecure.py

@@ -3,9 +3,16 @@
 import glob
 import shutil
 import subprocess
-from distutils.dir_util import copy_tree
+
+# from distutils.dir_util import copy_tree
 from pathlib import Path
-from urllib.parse import urlsplit, parse_qs, urlunsplit, urlencode, SplitResult
+from urllib.parse import (
+    urlsplit,
+    parse_qs,
+    urlunsplit,
+    urlencode,
+    SplitResult,
+)
 from typing import (
     Optional,
     Any,
@@ -23,7 +30,7 @@ def __is_empty_dir(path: str) -> bool:
 
 def __run(*args: Any, **kwargs: Any) -> Any:
     try:
-        return subprocess.run(*args, **kwargs)
+        return subprocess.run(*args, **kwargs)  # pylint: disable=subprocess-run-check
     except subprocess.CalledProcessError as ex:
         raise RuntimeError(f'Command "{" ".join(*args)}" returned non-zero exit code ({ex.returncode})') from ex
     except Exception as ex:
@@ -64,7 +71,7 @@ def install(stream: Optional[str] = None) -> None:
 
 
 def use_store(store_path: str) -> None:
-    global __RLSTORE
+    global __RLSTORE  # pylint: disable=global-statement
     __RLSTORE = store_path
     if not os.path.isdir(__RLSTORE):
         raise RuntimeError(f"'{__RLSTORE}' is not a directory")
@@ -120,7 +127,7 @@ def prune(
     __run(cmd, check=True)
 
 
-class ScanResult:
+class ScanResult:  # pylint: disable=too-few-public-methods
     def __init__(self, passed: bool, msg: str) -> None:
         self.passed = passed
         self.msg = msg
@@ -161,7 +168,11 @@ def generate_report(
 
     # copy report to desired location
     os.makedirs(rpt_path, exist_ok=True)
-    copy_tree(__RLREPORT_LOCATION, rpt_path)
+    shutil.copytree(
+        src=__RLREPORT_LOCATION,
+        dst=rpt_path,
+        dirs_exist_ok=True,
+    )
 
     # collect scan results
     if not is_repro:
@@ -184,12 +195,18 @@ def generate_report(
             return ScanResult(False, msg.group(1) if msg is not None else "rl-secure analysis: failed")
     else:
         # repro scan
-        def make_base_purl(purl):
+        def make_base_purl(purl: str) -> str:
             elements = urlsplit(purl)
             query = parse_qs(elements.query)
             del query["build"]
             return urlunsplit(
-                SplitResult(elements.scheme, elements.netloc, elements.path, urlencode(query), elements.fragment)
+                SplitResult(
+                    elements.scheme,
+                    elements.netloc,
+                    elements.path,
+                    urlencode(query),
+                    elements.fragment,
+                )
             )
 
         base_purl = make_base_purl(purl)