Add ability to remove packages from exported repos

alws/utils/exporter.py

@@ -39,7 +39,8 @@ async def get_repodata_file_links(base_url: str):
         async with session.get(base_url) as response:
             content = await response.text()
             doc = document_fromstring(content)
-            children_urls = [base_url + a.get('href') for a in doc.xpath('//a')]
+            children_urls = [base_url + a.get('href')
+                             for a in doc.xpath('//a') if '..' not in a.get('href')]
             return children_urls
 
 

scripts/exporters/base_exporter.py

@@ -5,6 +5,7 @@
 import shutil
 import sys
 import urllib.parse
+from fnmatch import fnmatch
 from pathlib import Path
 from typing import List, Literal, Optional, Tuple, Union
 
@@ -91,6 +92,31 @@ def regenerate_repo_metadata(self, repo_path: str):
 
         shutil.copytree(repodata_path, cache_repodata_dir)
 
+    def remove_packages_from_repo(
+        self,
+        repo_path: str,
+        exclude_patterns: List[str],
+    ):
+        packages_dir = Path(repo_path)
+        if not packages_dir.exists():
+            self.logger.warning(
+                'Packages directory does not exist: %s', repo_path,
+            )
+            return
+        globs = [
+            p + '*' if not p.endswith('*') else p
+            for p in exclude_patterns
+        ]
+        removed_count = 0
+        for rpm_file in packages_dir.rglob('*.rpm'):
+            if any(fnmatch(rpm_file.name, g) for g in globs):
+                self.logger.info('Removing package: %s', rpm_file)
+                rpm_file.unlink()
+                removed_count += 1
+        self.logger.info(
+            'Removed %d packages from %s', removed_count, repo_path,
+        )
+
     async def create_filesystem_exporters(
         self,
         repository_ids: List[int],
@@ -252,6 +278,7 @@ async def repomd_signer(self, repodata_path, key_id, token):
         file_path = os.path.join(repodata_path, "repomd.xml")
         result = await self.sign_repomd_xml(file_path, key_id, token)
         self.logger.info('PGP key id: %s', key_id)
+        self.logger.info('sign-file result: %s', result)
         result_data = result.get("asc_content")
         if result_data is None:
             self.logger.error(
@@ -296,6 +323,7 @@ async def make_request(
         async with aiohttp.ClientSession(
             headers=headers,
             raise_for_status=True,
+            timeout=aiohttp.ClientTimeout(total=900),
         ) as session:
             async with session.request(
                 method,

scripts/exporters/packages_exporter.py

@@ -433,7 +433,6 @@ async def export_repos_from_release(
         exported_paths = await self.export_repositories(repo_ids)
         return exported_paths, db_release.platform_id
 
-
 async def sign_repodata(
     exporter: PackagesExporter,
     exported_paths: List[str],
@@ -520,13 +519,16 @@ def export_errata_and_oval(
         if not os.path.exists(errata_export_base_path):
             os.mkdir(errata_export_base_path)
         for platform in platform_names:
+            errata_cache = platform_errata_cache.get(platform, {}).get('cache')
+            if not errata_cache:
+                exporter.logger.debug('No errata cache for %s platform, skipping', platform)
+                continue
             platform_path = os.path.join(errata_export_base_path, platform)
             if not os.path.exists(platform_path):
                 os.mkdir(platform_path)
             html_path = os.path.join(platform_path, "html")
             if not os.path.exists(html_path):
                 os.mkdir(html_path)
-            errata_cache = platform_errata_cache[platform]["cache"]
             exporter.process_osv_data(errata_cache, platform)
             exporter.logger.debug("Generating HTML errata pages")
             for record in errata_cache:

scripts/exporters/products_exporter.py

@@ -82,6 +82,15 @@ def parse_args():
         required=False,
         help="Path to export log",
     )
+    parser.add_argument(
+        "-rp",
+        "--remove-packages",
+        type=str,
+        nargs="+",
+        default=None,
+        required=False,
+        help="List of package names to remove from exported repos",
+    )
     parser.add_argument(
         "-v",
         "--verbose",
@@ -215,6 +224,11 @@ async def main():
         distr_name=args.distribution,
         arches=args.arches,
     )
+
+    if args.remove_packages:
+        for path in exported_paths:
+            exporter.remove_packages_from_repo(path, args.remove_packages)
+
     await asyncio.gather(
         *(repo_post_processing(exporter, path) for path in exported_paths)
     )