diff --git a/ChangeLog.md b/ChangeLog.md index ea4ccc1..a95c609 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -5,6 +5,10 @@ # CaPyCli - Clearing Automation Python Command Line Tool for SW360 +## NEXT + +* fix namespace handling in `bom componentcheck` and `project componentcheck`. + ## 2.11.0 * Two new commands: `bom componentcheck` and `project componentcheck`. The first one diff --git a/capycli/bom/check_bom.py b/capycli/bom/check_bom.py index 6af5b20..cf6db04 100644 --- a/capycli/bom/check_bom.py +++ b/capycli/bom/check_bom.py @@ -15,12 +15,12 @@ from colorama import Fore, Style from cyclonedx.model.bom import Bom from cyclonedx.model.component import Component -from sw360 import SW360Error import capycli.common.script_base from capycli.common.capycli_bom_support import CaPyCliBom, CycloneDxSupport from capycli.common.print import print_green, print_red, print_text, print_yellow from capycli.main.result_codes import ResultCode +from sw360 import SW360Error LOG = capycli.get_logger(__name__) diff --git a/capycli/bom/check_bom_item_status.py b/capycli/bom/check_bom_item_status.py index beade56..e40008e 100644 --- a/capycli/bom/check_bom_item_status.py +++ b/capycli/bom/check_bom_item_status.py @@ -15,12 +15,12 @@ from colorama import Fore, Style from cyclonedx.model.bom import Bom from cyclonedx.model.component import Component -from sw360 import SW360Error import capycli.common.script_base from capycli.common.capycli_bom_support import CaPyCliBom, CycloneDxSupport from capycli.common.print import print_red, print_text from capycli.main.result_codes import ResultCode +from sw360 import SW360Error LOG = capycli.get_logger(__name__) diff --git a/capycli/bom/component_check.py b/capycli/bom/component_check.py index bff4cde..751a359 100644 --- a/capycli/bom/component_check.py +++ b/capycli/bom/component_check.py @@ -6,6 +6,8 @@ # SPDX-License-Identifier: MIT # ------------------------------------------------------------------------------- +import logging + try: import importlib.resources as pkg_resources except ImportError: @@ -38,6 +40,7 @@ class ComponentCheck(capycli.common.script_base.ScriptBase): def __init__(self) -> None: self.component_check_list: Dict[str, Any] = {} self.files_to_ignore: List[Dict[str, Any]] = [] + self.verbose = False @staticmethod def get_component_check_list(download_url: str) -> None: @@ -54,6 +57,8 @@ def read_component_check_list(self, download_url: str = "", local_check_list_fil try: with open(local_check_list_file, "r", encoding="utf-8") as fin: self.component_check_list = json.load(fin) + if self.verbose: + print_text(f" Reading component checklist from {local_check_list_file}...") except FileNotFoundError as e: print_yellow(f"File not found: {e} \n Reading the default component check list") except Exception as e: @@ -63,6 +68,8 @@ def read_component_check_list(self, download_url: str = "", local_check_list_fil ComponentCheck.get_component_check_list(download_url) with open("component_checks.json", "r", encoding="utf-8") as fin: self.component_check_list = json.load(fin) + if self.verbose: + print_text(f" Reading component checklist from {download_url}...") except FileNotFoundError as e: print_yellow(f"File not found: {e} \n Reading the default component check list") except Exception as e: @@ -92,11 +99,20 @@ def is_dev_dependency(self, comp: Component) -> bool: pd = comp.purl.to_dict() ecosystem = pd.get("type", "").lower() for entry in self.get_dev_dependencies(ecosystem): - if comp.group: - if comp.group.lower() != entry.get("namespace", ""): + name_to_compare = entry.get("name", "") + if comp.purl.namespace and entry.get("namespace", ""): + if comp.purl.namespace.lower() != entry.get("namespace", ""): continue - if comp.name.lower() == entry.get("name", ""): + if comp.name.lower() == name_to_compare.lower(): + return True + + # it can happen that comp.purl.namespace is empty, but the namespace + # ...is included in the name + if entry.get("namespace", ""): + name_to_compare = entry.get("namespace", "") + "/" + name_to_compare + + if comp.name.lower() == name_to_compare.lower(): return True else: # fallback: only check by name @@ -154,6 +170,11 @@ def run(self, args: Any) -> None: if args.debug: global LOG LOG = capycli.get_logger(__name__) + else: + # suppress (debug) log output from requests and urllib + logging.getLogger("requests").setLevel(logging.WARNING) + logging.getLogger("urllib3").setLevel(logging.WARNING) + logging.getLogger("urllib3.connectionpool").setLevel(logging.WARNING) print_text( "\n" + capycli.get_app_signature() + @@ -179,11 +200,13 @@ def run(self, args: Any) -> None: print_red("Input file not found!") sys.exit(ResultCode.RESULT_FILE_NOT_FOUND) - print_text("Reading component check list from component_checks.json...") + self.verbose = args.verbose + + print_text("Reading component checklist...") try: self.read_component_check_list(args.remote_check_list, args.local_checklist_list) except Exception as ex: - print_red("Error reading component check list " + repr(ex)) + print_red("Error reading component checklist " + repr(ex)) sys.exit(ResultCode.RESULT_GENERAL_ERROR) if len(self.component_check_list) > 0: print_text(" Got component checklist.") diff --git a/capycli/bom/create_components.py b/capycli/bom/create_components.py index e6afb74..08b1c8b 100644 --- a/capycli/bom/create_components.py +++ b/capycli/bom/create_components.py @@ -20,7 +20,6 @@ from cyclonedx.model.bom import Bom from cyclonedx.model.component import Component from cyclonedx.model.license import DisjunctiveLicense, LicenseExpression -from sw360 import SW360Error import capycli.common.json_support import capycli.common.script_base @@ -30,6 +29,7 @@ from capycli.common.purl_utils import PurlUtils from capycli.common.script_support import ScriptSupport from capycli.main.result_codes import ResultCode +from sw360 import SW360Error LOG = capycli.get_logger(__name__) diff --git a/capycli/bom/findsources.py b/capycli/bom/findsources.py index 267dc91..e3a2024 100644 --- a/capycli/bom/findsources.py +++ b/capycli/bom/findsources.py @@ -24,7 +24,6 @@ from cyclonedx.model import ExternalReferenceType, XsUri from cyclonedx.model.bom import Bom from cyclonedx.model.component import Component -from sw360 import SW360Error import capycli.common.script_base from capycli import get_logger @@ -32,6 +31,7 @@ from capycli.common.github_support import GitHubSupport from capycli.common.print import print_green, print_red, print_text, print_yellow from capycli.main.result_codes import ResultCode +from sw360 import SW360Error LOG = get_logger(__name__) diff --git a/capycli/bom/map_bom.py b/capycli/bom/map_bom.py index 2a51218..1b66862 100644 --- a/capycli/bom/map_bom.py +++ b/capycli/bom/map_bom.py @@ -21,7 +21,6 @@ from cyclonedx.model.bom import Bom from cyclonedx.model.component import Component from packageurl import PackageURL -from sw360 import SW360 import capycli.common.file_support import capycli.common.script_base @@ -34,6 +33,7 @@ from capycli.common.print import print_green, print_red, print_text, print_yellow from capycli.common.purl_service import PurlService from capycli.main.result_codes import ResultCode +from sw360 import SW360 LOG = get_logger(__name__) diff --git a/capycli/common/component_cache.py b/capycli/common/component_cache.py index 780e7ff..883ad12 100644 --- a/capycli/common/component_cache.py +++ b/capycli/common/component_cache.py @@ -12,7 +12,6 @@ from typing import Any, Dict, List, Optional import sw360 - from capycli import get_logger from capycli.common.print import print_red, print_text, print_yellow from capycli.common.script_support import ScriptSupport diff --git a/capycli/common/purl_service.py b/capycli/common/purl_service.py index b141554..a2b7951 100644 --- a/capycli/common/purl_service.py +++ b/capycli/common/purl_service.py @@ -9,11 +9,11 @@ from typing import Any, Dict, List, Optional import packageurl -from sw360 import SW360 from capycli.common.print import print_green, print_text, print_yellow from capycli.common.purl_store import PurlStore from capycli.common.purl_utils import PurlUtils +from sw360 import SW360 class PurlService: diff --git a/capycli/common/script_base.py b/capycli/common/script_base.py index 0dc35b7..0b07299 100644 --- a/capycli/common/script_base.py +++ b/capycli/common/script_base.py @@ -19,10 +19,10 @@ import jwt import requests from cyclonedx.model.bom import Bom -from sw360 import SW360, SW360Error from capycli.common.print import print_red, print_text, print_yellow from capycli.main.result_codes import ResultCode +from sw360 import SW360, SW360Error class ScriptBase: diff --git a/capycli/main/options.py b/capycli/main/options.py index 76ffcd4..38eda84 100644 --- a/capycli/main/options.py +++ b/capycli/main/options.py @@ -50,6 +50,7 @@ def __init__(self) -> None: Findsources determine the source code for SBOM items Validate validate an SBOM BomPackage create a single archive that contains the SBOM and all source and binary files + ComponentCheck Check the SBOM for special components mapping ToHtml create a HTML page showing the mapping result @@ -72,6 +73,7 @@ def __init__(self) -> None: CreateReadme create a Readme_OSS Vulnerabilities show security vulnerabilities of a project ECC show export control status of a project + ComponentCheck Check the project for special components Note that each command has also its own help display, i.e. if you enter `capycli project vulnerabilities -h` you will get a help that only shows the options diff --git a/capycli/project/create_bom.py b/capycli/project/create_bom.py index 40491e9..2c94cb2 100644 --- a/capycli/project/create_bom.py +++ b/capycli/project/create_bom.py @@ -14,7 +14,6 @@ from cyclonedx.model.bom import Bom from cyclonedx.model.component import Component from packageurl import PackageURL -from sw360 import SW360Error import capycli.common.script_base from capycli import get_logger @@ -22,6 +21,7 @@ from capycli.common.print import print_red, print_text, print_yellow from capycli.common.purl_utils import PurlUtils from capycli.main.result_codes import ResultCode +from sw360 import SW360Error LOG = get_logger(__name__) diff --git a/capycli/project/create_project.py b/capycli/project/create_project.py index 61d45ce..846ecf6 100644 --- a/capycli/project/create_project.py +++ b/capycli/project/create_project.py @@ -13,13 +13,13 @@ import requests from cyclonedx.model.bom import Bom -from sw360 import SW360Error import capycli.common.script_base from capycli import get_logger from capycli.common.capycli_bom_support import CaPyCliBom, CycloneDxSupport from capycli.common.print import print_red, print_text, print_yellow from capycli.main.result_codes import ResultCode +from sw360 import SW360Error LOG = get_logger(__name__) diff --git a/capycli/project/find_project.py b/capycli/project/find_project.py index 23aab8c..4b480cd 100644 --- a/capycli/project/find_project.py +++ b/capycli/project/find_project.py @@ -12,9 +12,9 @@ from typing import Any, Dict, Optional import requests -import sw360 import capycli.common.script_base +import sw360 from capycli.common.print import print_red, print_text, print_yellow from capycli.main.result_codes import ResultCode diff --git a/capycli/project/get_license_info.py b/capycli/project/get_license_info.py index 2b1ccfa..24af8b6 100644 --- a/capycli/project/get_license_info.py +++ b/capycli/project/get_license_info.py @@ -12,13 +12,12 @@ import sys from typing import Any, Dict, List -from sw360 import SW360Error - import capycli.common.script_base from capycli.common.json_support import load_json_file from capycli.common.print import print_red, print_text, print_yellow from capycli.common.script_support import ScriptSupport from capycli.main.result_codes import ResultCode +from sw360 import SW360Error LOG = capycli.get_logger(__name__) diff --git a/capycli/project/handle_project.py b/capycli/project/handle_project.py index 371c19f..fb9b0fa 100644 --- a/capycli/project/handle_project.py +++ b/capycli/project/handle_project.py @@ -47,6 +47,7 @@ def run_project_command(args: Any) -> None: print(" CreateReadme create a Readme_OSS") print(" Vulnerabilities show security vulnerabilities of a project") print(" ECC Show export control status of a project") + print(" ComponentCheck Check the project for special components") return subcommand = args.command[1].lower() diff --git a/capycli/project/project_component_check.py b/capycli/project/project_component_check.py index a127d62..e8ceb14 100644 --- a/capycli/project/project_component_check.py +++ b/capycli/project/project_component_check.py @@ -10,9 +10,8 @@ import sys from typing import Any -import sw360 - import capycli.common.script_base +import sw360 from capycli.bom.component_check import ComponentCheck from capycli.common.print import print_red, print_text, print_yellow from capycli.main.result_codes import ResultCode @@ -26,13 +25,17 @@ class ProjectComponentCheck(capycli.common.script_base.ScriptBase): """ def __init__(self) -> None: self.component_check = ComponentCheck() + self.verbose = False def is_dev_dependency(self, name: str) -> bool: """Check whether the given component matches any known development dependency.""" dd = self.component_check.component_check_list.get("dev_dependencies", []) for ecosystem in dd: for entry in dd.get(ecosystem, []): - if name.lower() == entry.get("name", ""): + to_compare = entry.get("name", "") + if entry.get("namespace", ""): + to_compare = entry.get("namespace", "") + "/" + to_compare + if name.lower() == to_compare.lower(): return True return False @@ -129,11 +132,14 @@ def run(self, args: Any) -> None: print_text(" --forceerror force an error exit code in case of validation errors or warnings") return - print_text("Reading component check list from component_checks.json...") + self.verbose = args.verbose + self.component_check.verbose = args.verbose + + print_text("Reading component checklist...") try: self.component_check.read_component_check_list(args.remote_check_list, args.local_checklist_list) except Exception as ex: - print_red("Error reading component check list " + repr(ex)) + print_red("Error reading component checklist " + repr(ex)) sys.exit(ResultCode.RESULT_GENERAL_ERROR) if len(self.component_check.component_check_list) > 0: print_text(" Got component checklist.") diff --git a/capycli/project/show_ecc.py b/capycli/project/show_ecc.py index a45d996..f9f9e50 100644 --- a/capycli/project/show_ecc.py +++ b/capycli/project/show_ecc.py @@ -10,9 +10,8 @@ import sys from typing import Any, Dict -import sw360 - import capycli.common.script_base +import sw360 from capycli.common.json_support import write_json_to_file from capycli.common.print import print_green, print_red, print_text, print_yellow from capycli.main.result_codes import ResultCode diff --git a/capycli/project/show_project.py b/capycli/project/show_project.py index 5ad0d1d..a2b49c1 100644 --- a/capycli/project/show_project.py +++ b/capycli/project/show_project.py @@ -10,11 +10,11 @@ import sys from typing import Any, Dict, Optional -import sw360 from colorama import Fore import capycli.common.json_support import capycli.common.script_base +import sw360 from capycli.common.print import print_red, print_text, print_yellow from capycli.main.result_codes import ResultCode diff --git a/capycli/project/show_vulnerabilities.py b/capycli/project/show_vulnerabilities.py index 6512573..6c0662f 100644 --- a/capycli/project/show_vulnerabilities.py +++ b/capycli/project/show_vulnerabilities.py @@ -12,12 +12,12 @@ import requests from colorama import Fore, Style -from sw360 import SW360Error import capycli.common.json_support import capycli.common.script_base from capycli.common.print import print_green, print_red, print_text, print_yellow from capycli.main.result_codes import ResultCode +from sw360 import SW360Error LOG = capycli.get_logger(__name__) diff --git a/tests/fixtures/sbom_for_component_check.json b/tests/fixtures/sbom_for_component_check.json index db7882d..f683dea 100644 --- a/tests/fixtures/sbom_for_component_check.json +++ b/tests/fixtures/sbom_for_component_check.json @@ -125,7 +125,21 @@ ], "purl": "pkg:maven/org.eclipse.jdt/junit@1.2.3", "type": "library", - "version": "123" + "version": "1.2.3" + }, + { + "bom-ref": "pkg:maven/org.eclipse.jdt/junit@1.2.4", + "name": "junit", + "group": "org.eclipse.jdt", + "properties": [ + { + "name": "siemens:primaryLanguage", + "value": "Java" + } + ], + "purl": "pkg:maven/org.eclipse.jdt/junit@1.2.4", + "type": "library", + "version": "1.2.4" }, { "bom-ref": "pkg:npm/gulp@1.2.3", diff --git a/tests/test_check_components.py b/tests/test_check_components.py index 643b7df..b99251c 100644 --- a/tests/test_check_components.py +++ b/tests/test_check_components.py @@ -94,10 +94,10 @@ def test_real_bom1(self) -> None: out = self.capture_stdout(sut.run, args) self.assertTrue(self.INPUTFILE1 in out) - self.assertTrue("Reading component check list from component_checks.json..." in out) + self.assertTrue("Reading component checklist..." in out) self.assertTrue("Got component checklist." in out) self.assertTrue("0 components will be ignored." in out) - self.assertTrue("6 components read from SBOM" in out) + self.assertTrue("7 components read from SBOM" in out) self.assertTrue("pandas 5.0 is known as a Python component that has additional binary dependencies" in out) self.assertTrue("pytest 7.4.3 seems to be a development dependency" in out) @@ -116,11 +116,12 @@ def test_real_bom2(self) -> None: out = self.capture_stdout(sut.run, args) self.assertTrue(self.INPUTFILE1 in out) - self.assertTrue("Reading component check list from component_checks.json..." in out) + self.assertTrue("Reading component checklist..." in out) self.assertTrue("Got component checklist." in out) self.assertTrue("1 components will be ignored." in out) self.assertTrue("gulp 123 seems to be a development dependency" in out) - self.assertTrue("junit 123 seems to be a development dependency" in out) + self.assertTrue("junit 1.2.3 seems to be a development dependency" in out) + self.assertTrue("junit 1.2.4 seems to be a development dependency" in out) self.assertFalse("pytest 7.4.3 seems to be a development dependency" in out) @responses.activate @@ -217,4 +218,4 @@ def test_read_component_check_download_error(self) -> None: if __name__ == '__main__': APP = TestComponentCheck() - APP.test_read_granularity_list_local_file_not_found() + APP.test_real_bom2() diff --git a/tests/test_project_component_check.py b/tests/test_project_component_check.py index 10fe258..a2a09e9 100644 --- a/tests/test_project_component_check.py +++ b/tests/test_project_component_check.py @@ -252,7 +252,7 @@ def test_project_show_by_id(self) -> None: out = self.capture_stdout(sut.run, args) # self.dump_textfile(out, "DUMP.TXT") - self.assertTrue("Reading component check list from component_checks.json" in out) + self.assertTrue("Reading component checklist" in out) self.assertTrue("Got component checklist." in out) self.assertTrue("0 components will be ignored." in out) diff --git a/tests/test_script_base.py b/tests/test_script_base.py index 46ea141..0e231c0 100644 --- a/tests/test_script_base.py +++ b/tests/test_script_base.py @@ -10,10 +10,10 @@ import pytest import responses -from sw360.sw360error import SW360Error from capycli.common.script_base import ScriptBase from capycli.main.result_codes import ResultCode +from sw360.sw360error import SW360Error from tests.test_base import TestBase diff --git a/tests/test_update_project.py b/tests/test_update_project.py index 2887731..596112f 100644 --- a/tests/test_update_project.py +++ b/tests/test_update_project.py @@ -4,10 +4,10 @@ from cyclonedx.model.bom import Bom from pytest import fixture, raises -from sw360 import SW360Error from capycli.main.result_codes import ResultCode from capycli.project.create_project import CreateProject +from sw360 import SW360Error IRRELEVANT_STR = "irrelevant" IRRELEVANT_DICT = {"irrelevant": "data"}