Refactor conan mining pipeline for git deployment (#787)

* Refactor conan mining pipeline for git deployment

Signed-off-by: Tushar Goel <tushar.goel.dav@gmail.com>

* Refactor tests

Signed-off-by: Tushar Goel <tushar.goel.dav@gmail.com>

* Fix tests

Signed-off-by: Tushar Goel <tushar.goel.dav@gmail.com>

* Restructure code and tests

Signed-off-by: Tushar Goel <tushar.goel.dav@gmail.com>

* Release minecode for conan pipeline deployment

Signed-off-by: Keshav Priyadarshi <git@keshav.space>

* Fix saneyaml import

Signed-off-by: Keshav Priyadarshi <git@keshav.space>

* Fix missing MINECODE_CONAN_INDEX_REPO

Signed-off-by: Keshav Priyadarshi <git@keshav.space>

* Fix errors

Signed-off-by: Tushar Goel <tushar.goel.dav@gmail.com>

* Fix errors

Signed-off-by: Tushar Goel <tushar.goel.dav@gmail.com>

* Fix errors

Signed-off-by: Tushar Goel <tushar.goel.dav@gmail.com>

* Fix errors

Signed-off-by: Tushar Goel <tushar.goel.dav@gmail.com>

* Fix errors

Signed-off-by: Tushar Goel <tushar.goel.dav@gmail.com>

* Fix errors

Signed-off-by: Tushar Goel <tushar.goel.dav@gmail.com>

---------

Signed-off-by: Tushar Goel <tushar.goel.dav@gmail.com>
Signed-off-by: Keshav Priyadarshi <git@keshav.space>
Co-authored-by: Keshav Priyadarshi <git@keshav.space>

Author: TG1999

minecode_pipelines/miners/conan.py

@@ -20,60 +20,44 @@
 # ScanCode.io is a free software code scanning tool from nexB Inc. and others.
 # Visit https://github.com/aboutcode-org/scancode.io for support and download.
 
-import os
-from scanpipe.pipelines import Pipeline
-from minecode_pipelines import pipes
-from minecode_pipelines.miners import conan
+from pathlib import Path
+from minecode_pipelines.pipes import conan
+from minecode_pipelines.pipelines import MineCodeBasePipeline
 from scanpipe.pipes import federatedcode
 
-MINECODE_CONAN_INDEX_REPO = "https://github.com/conan-io/conan-center-index"
 
-MINECODE_DATA_CONAN_REPO = os.environ.get(
-    "MINECODE_DATA_CONAN_REPO", "https://github.com/aboutcode-data/minecode-data-conan-test"
-)
-
-
-class MineConan(Pipeline):
+class MineConan(MineCodeBasePipeline):
     """Pipeline to mine Conan packages and publish them to FederatedCode repo."""
 
+    MINECODE_CONAN_INDEX_REPO = "https://github.com/conan-io/conan-center-index"
+
     @classmethod
     def steps(cls):
         return (
             cls.check_federatedcode_eligibility,
-            cls.clone_conan_repos,
-            cls.mine_and_publish_conan_package_urls,
+            cls.create_federatedcode_working_dir,
+            cls.clone_conan_index,
+            cls.fetch_federation_config,
+            cls.mine_and_publish_packageurls,
+            cls.delete_working_dir,
         )
 
-    def check_federatedcode_eligibility(self):
-        """
-        Check if the project fulfills the following criteria for
-        pushing the project result to FederatedCode.
-        """
-        federatedcode.check_federatedcode_configured_and_available(logger=self.log)
-
-    def clone_conan_repos(self):
-        """
-        Clone the Conan-related repositories (index, data, and pipelines config)
-        and store their Repo objects in the corresponding instance variables.
-        """
-        self.conan_index_repo = federatedcode.clone_repository(MINECODE_CONAN_INDEX_REPO)
-        self.cloned_data_repo = federatedcode.clone_repository(MINECODE_DATA_CONAN_REPO)
-
-        if self.log:
-            self.log(
-                f"{MINECODE_CONAN_INDEX_REPO} repo cloned at: {self.conan_index_repo.working_dir}"
-            )
-            self.log(
-                f"{MINECODE_DATA_CONAN_REPO} repo cloned at: {self.cloned_data_repo.working_dir}"
-            )
-
-    def mine_and_publish_conan_package_urls(self):
-        conan.mine_and_publish_conan_packageurls(
-            self.conan_index_repo, self.cloned_data_repo, self.log
+    def clone_conan_index(self):
+        """Clone the Cargo index Repo."""
+        self.conan_index_repo = federatedcode.clone_repository(
+            repo_url=self.MINECODE_CONAN_INDEX_REPO,
+            clone_path=self.working_path / "conan-index",
+            logger=self.log,
         )
 
-    def delete_cloned_repos(self):
-        pipes.delete_cloned_repos(
-            repos=[self.conan_index_repo, self.cloned_data_repo],
+    def packages_count(self):
+        base_path = Path(self.conan_index_repo.working_tree_dir)
+        package_dir = [p for p in base_path.iterdir() if p.is_dir() and not p.name.startswith(".")]
+        return sum(1 for dir in package_dir for f in dir.rglob("*") if f.is_file())
+
+    def mine_packageurls(self):
+        """Yield PackageURLs from Cargo index."""
+        return conan.mine_conan_packageurls(
+            conan_index_repo=self.conan_index_repo,
             logger=self.log,
         )

minecode_pipelines/pipelines/mine_conan.py

@@ -20,24 +20,40 @@
 # ScanCode.io is a free software code scanning tool from nexB Inc. and others.
 # Visit https://github.com/aboutcode-org/scancode.io for support and download.
 
-from packageurl import PackageURL
 from pathlib import Path
-from aboutcode import hashid
-from minecode_pipelines.pipes import write_data_to_yaml_file
+from packageurl import PackageURL
 
+import saneyaml
 
-def store_conan_packages(pacakge_name, versions_data, fed_repo):
-    """Collect Conan package versions into purls and write them to the repo."""
 
-    base_purl = PackageURL(type="conan", name=pacakge_name)
+def get_conan_packages(file_path, file_versions_data):
+    # Example: file_path = Path("repo_path/recipes/7zip/config.yml")
+    # - file_path.parts = ("repo_path", "recipes", "7zip", "config.yml")
+    # - file_path.parts[-2] = "7zip"  (the package name)
+    if len(file_path.parts) < 2:
+        return None, []
+    package_name = file_path.parts[-2]
+    base_purl = PackageURL(type="conan", name=package_name)
 
     updated_purls = []
-    versions = list(versions_data["versions"].keys())
+    versions = file_versions_data.get("versions") or []
     for version in versions:
-        purl = PackageURL(type="conan", name=pacakge_name, version=version).to_string()
+        purl = PackageURL(type="conan", name=package_name, version=str(version)).to_string()
         updated_purls.append(purl)
+    return base_purl, updated_purls
+
+
+def mine_conan_packageurls(conan_index_repo, logger):
+    """Mine Conan PackageURLs from package index."""
+
+    base_path = Path(conan_index_repo.working_dir)
+    for file_path in base_path.glob("recipes/**/*"):
+        if not file_path.name == "config.yml":
+            continue
+        with open(file_path, encoding="utf-8") as f:
+            versions = saneyaml.load(f)
+
+        if not versions:
+            continue
 
-    ppath = hashid.get_package_purls_yml_file_path(base_purl)
-    purl_file_full_path = Path(fed_repo.working_dir) / ppath
-    write_data_to_yaml_file(path=purl_file_full_path, data=updated_purls)
-    return purl_file_full_path, base_purl
+        yield get_conan_packages(file_path=file_path, file_versions_data=versions)

minecode_pipelines/pipes/conan.py

@@ -7,25 +7,20 @@
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
 
-import tempfile
 from pathlib import Path
-from unittest import mock
-from unittest.mock import Mock, patch
 import saneyaml
 import yaml
 
 from django.test import TestCase
 
-from minecode_pipelines.pipes import write_data_to_yaml_file
-from minecode_pipelines.pipes.conan import store_conan_packages
+from minecode_pipelines.pipes.conan import get_conan_packages
 
 DATA_DIR = Path(__file__).parent.parent / "test_data" / "conan"
 
 
 class ConanPipelineTests(TestCase):
-    @patch("minecode_pipelines.pipes.conan.write_data_to_yaml_file")
-    def test_collect_packages_from_cargo_calls_write(self, mock_write):
-        packages_file = DATA_DIR / "cairo-config.yml"
+    def test_collect_packages_from_conan_calls_write(self, mock_write):
+        packages_file = DATA_DIR / "cairo" / "cairo-config.yml"
         expected_file = DATA_DIR / "expected-cairo-purls.yml"
 
         with open(packages_file, encoding="utf-8") as f:
@@ -34,51 +29,6 @@ def test_collect_packages_from_cargo_calls_write(self, mock_write):
         with open(expected_file, encoding="utf-8") as f:
             expected = saneyaml.load(f)
 
-        with tempfile.TemporaryDirectory() as tmpdir:
-            repo = Mock()
-            repo.working_dir = tmpdir
-
-            store_conan_packages("cairo", versions_data, repo)
-
-            mock_write.assert_called_once()
-            args, kwargs = mock_write.call_args
-            base_purl, written_packages = kwargs["path"], kwargs["data"]
-
-            expected_base_purl = (
-                Path(tmpdir) / "aboutcode-packages-conan-0" / "conan" / "cairo" / "purls.yml"
-            )
-
-            self.assertEqual(str(base_purl), str(expected_base_purl))
-            self.assertEqual(written_packages, expected)
-
-    def _assert_purls_written(self, purls):
-        with tempfile.TemporaryDirectory() as tmpdir:
-            repo_dir = Path(tmpdir)
-
-            mock_repo = mock.MagicMock()
-            mock_repo.working_dir = str(repo_dir)
-            mock_repo.index.add = mock.MagicMock()
-
-            purls_file = repo_dir / "purls.yaml"
-
-            write_data_to_yaml_file(purls_file, purls)
-
-            self.assertTrue(purls_file.exists())
-
-            with open(purls_file, encoding="utf-8") as f:
-                content = saneyaml.load(f)
-
-            self.assertEqual(content, purls)
-
-    def test_add_purl_result_with_mock_repo(self):
-        purls = [
-            {"purl": "pkg:conan/cairo@1.18.0"},
-            {"purl": "pkg:conan/cairo@1.17.8"},
-            {"purl": "pkg:conan/cairo@1.17.6"},
-            {"purl": "pkg:conan/cairo@1.17.4"},
-        ]
-
-        self._assert_purls_written(purls)
-
-    def test_add_empty_purl_result_with_mock_repo(self):
-        self._assert_purls_written([])
+        base, purls = get_conan_packages(packages_file, versions_data)
+        self.assertEqual(purls, expected)
+        self.assertEqual(str(base), "pkg:conan/cairo")

minecode_pipelines/tests/pipes/test_conan.py

@@ -4,7 +4,7 @@ build-backend = "flot.buildapi"
 
 [project]
 name = "minecode_pipelines"
-version = "0.0.1b32"
+version = "0.0.1b41"
 description = "A library for mining packageURLs and package metadata from ecosystem repositories."
 readme = "minecode_pipelines/README.rst"
 license = { text = "Apache-2.0" }