From 801b1d3e3c3f1150828c4f4011b6eed6f8941ee5 Mon Sep 17 00:00:00 2001 From: ReenigneArcher <42013603+ReenigneArcher@users.noreply.github.com> Date: Sat, 16 May 2026 20:02:05 -0400 Subject: [PATCH 1/2] fix: Add package-config & asset filtering to include in gh-pages Introduce packages.config.json and package-config.js to control which release assets are mirrored (wildcard patterns, per-repo include lists). Update sync-assets.js to honor the config: only configured assets are downloaded, stored assets excluded by the config are cleaned up, hash sidecar handling improved, and new helpers added (cleanup, hash/file helpers, download limit checks). Enhance generate-packages.js to normalize GitHub metadata, build packages.json from metadata (with stats), and export helper functions. Remove the standalone sync-release-assets workflow and consolidate asset sync and pages publishing in update-pages.yml (adds scheduled runs, artifact preparation, and updated checkout/worktree logic). Update README and the gh-pages client to reference the published packages.json path. Add example packages.config.json. --- .github/scripts/generate-packages.js | 97 +++++++--- .github/scripts/package-config.js | 130 +++++++++++++ .github/scripts/sync-assets.js | 218 +++++++++++++++++----- .github/workflows/sync-release-assets.yml | 91 --------- .github/workflows/update-pages.yml | 90 ++++++++- README.md | 38 +++- gh-pages-template/assets/js/app.js | 49 +++-- packages.config.json | 11 ++ 8 files changed, 537 insertions(+), 187 deletions(-) create mode 100644 .github/scripts/package-config.js delete mode 100644 .github/workflows/sync-release-assets.yml create mode 100644 packages.config.json diff --git a/.github/scripts/generate-packages.js b/.github/scripts/generate-packages.js index 13e34ea4..48a96f0e 100644 --- a/.github/scripts/generate-packages.js +++ b/.github/scripts/generate-packages.js @@ -2,12 +2,76 @@ const fs = require('fs'); const path = require('path'); /** - * Generate packages.json file by scanning the dist directory structure - * @param {string} distPath - Path to the dist directory - * @param {Array} repositoryMetadata - Repository metadata from sync process with archived status + * Build packages.json data with stats from repository entries. + * @param {Array} repositories - Repository data to include. + * @returns {Object} packages.json data. + */ +function buildPackagesData(repositories) { + // Sort repositories by name + repositories.sort((a, b) => a.name.localeCompare(b.name)); + + // Calculate totals + const totalReleases = repositories.reduce((sum, repo) => sum + repo.releases.length, 0); + const totalAssets = repositories.reduce((sum, repo) => + sum + repo.releases.reduce((releaseSum, release) => releaseSum + release.assetCount, 0), 0 + ); + + const packagesData = { + repositories: repositories, + stats: { + totalRepositories: repositories.length, + totalReleases: totalReleases, + totalAssets: totalAssets + } + }; + + console.log(`Generated packages data:`); + console.log(` Repositories: ${repositories.length}`); + console.log(` Total Releases: ${totalReleases}`); + console.log(` Total Assets: ${totalAssets}`); + + return packagesData; +} + +/** + * Normalize GitHub-derived repository metadata for packages.json. + * @param {Array} repositoryMetadata - Repository metadata from the sync process. + * @returns {Array} Normalized repositories with sorted releases. + */ +function normalizeRepositoryMetadata(repositoryMetadata = []) { + return repositoryMetadata + .filter(repo => repo && Array.isArray(repo.releases) && repo.releases.length > 0) + .map(repo => { + const releases = repo.releases + .filter(release => release && release.assetCount > 0) + .map(release => ({ + tag: release.tag, + assetCount: release.assetCount + })); + + releases.sort((a, b) => b.tag.localeCompare(a.tag, undefined, { numeric: true, sensitivity: 'base' })); + + return { + name: repo.name, + archived: Boolean(repo.archived), + releases + }; + }) + .filter(repo => repo.releases.length > 0); +} + +/** + * Generate packages.json data from GitHub metadata, with a dist scan fallback. + * @param {string} distPath - Path to the dist directory. + * @param {Array} repositoryMetadata - GitHub-derived metadata for all release assets. * @returns {Object} Generated packages data */ function generatePackagesJson(distPath = '.', repositoryMetadata = []) { + if (repositoryMetadata.length > 0) { + console.log('Generating packages.json from GitHub release metadata'); + return buildPackagesData(normalizeRepositoryMetadata(repositoryMetadata)); + } + console.log(`Scanning dist directory: ${distPath}`); const repositories = []; @@ -42,30 +106,7 @@ function generatePackagesJson(distPath = '.', repositoryMetadata = []) { } } - // Sort repositories by name - repositories.sort((a, b) => a.name.localeCompare(b.name)); - - // Calculate totals - const totalReleases = repositories.reduce((sum, repo) => sum + repo.releases.length, 0); - const totalAssets = repositories.reduce((sum, repo) => - sum + repo.releases.reduce((releaseSum, release) => releaseSum + release.assetCount, 0), 0 - ); - - const packagesData = { - repositories: repositories, - stats: { - totalRepositories: repositories.length, - totalReleases: totalReleases, - totalAssets: totalAssets - } - }; - - console.log(`Generated packages data:`); - console.log(` Repositories: ${repositories.length}`); - console.log(` Total Releases: ${totalReleases}`); - console.log(` Total Assets: ${totalAssets}`); - - return packagesData; + return buildPackagesData(repositories); } catch (error) { console.error('Error scanning dist directory:', error); @@ -176,7 +217,9 @@ function writePackagesJson(packagesData, outputPath = './packages.json') { } module.exports = { + buildPackagesData, generatePackagesJson, + normalizeRepositoryMetadata, scanRepositoryDirectory, scanReleaseDirectory, writePackagesJson diff --git a/.github/scripts/package-config.js b/.github/scripts/package-config.js new file mode 100644 index 00000000..1e94e12e --- /dev/null +++ b/.github/scripts/package-config.js @@ -0,0 +1,130 @@ +const fs = require('fs'); +const path = require('path'); + +const CONFIG_FILENAME = 'packages.config.json'; + +/** + * Find packages.config.json by walking up from a starting directory. + * @param {string} startDir - Directory to start searching from. + * @returns {string|null} Absolute path to the config file, or null if absent. + */ +function findConfigPath(startDir = process.cwd()) { + let currentDir = path.resolve(startDir); + + while (true) { + const candidate = path.join(currentDir, CONFIG_FILENAME); + if (fs.existsSync(candidate)) { + return candidate; + } + + const parentDir = path.dirname(currentDir); + if (parentDir === currentDir) { + return null; + } + currentDir = parentDir; + } +} + +/** + * Normalize a possibly missing pattern list to non-empty string patterns. + * @param {unknown} value - Config value to normalize. + * @returns {string[]} Valid pattern strings. + */ +function normalizePatterns(value) { + if (!Array.isArray(value)) { + return []; + } + + return value.filter(pattern => typeof pattern === 'string' && pattern.length > 0); +} + +/** + * Load release asset retention config. + * @param {string} startDir - Directory used to locate packages.config.json. + * @returns {{defaultInclude: boolean, repositories: Object}} Normalized package config. + */ +function loadPackageConfig(startDir = process.cwd()) { + const configPath = findConfigPath(startDir); + if (!configPath) { + console.log(`No ${CONFIG_FILENAME} found; including all release assets.`); + return { + defaultInclude: true, + repositories: {} + }; + } + + const config = JSON.parse(fs.readFileSync(configPath, 'utf8')); + const repositories = config.repositories && typeof config.repositories === 'object' + ? config.repositories + : {}; + + console.log(`Loaded package config from ${configPath}`); + + return { + defaultInclude: config.defaultInclude === true, + repositories + }; +} + +/** + * Escape literal text for use inside a regular expression. + * @param {string} value - Raw pattern segment. + * @returns {string} Regex-escaped segment. + */ +function escapeRegex(value) { + return value.replace(/[|\\{}()[\]^$+?.]/g, '\\$&'); +} + +/** + * Convert a simple wildcard pattern to a regular expression. + * @param {string} pattern - Pattern using '*' as a wildcard. + * @returns {RegExp} Anchored regular expression. + */ +function patternToRegex(pattern) { + const source = pattern + .split('*') + .map(escapeRegex) + .join('.*'); + + return new RegExp(`^${source}$`); +} + +/** + * Check whether a value matches any configured wildcard pattern. + * @param {string} value - Value to test. + * @param {string[]} patterns - Wildcard patterns. + * @returns {boolean} True when any pattern matches. + */ +function matchesPattern(value, patterns) { + return patterns.some(pattern => patternToRegex(pattern).test(value)); +} + +/** + * Determine whether a release asset should be mirrored into dist and GitHub Pages. + * @param {{defaultInclude: boolean, repositories: Object}} config - Package config. + * @param {string} repoName - Repository name. + * @param {string} assetName - Release asset filename. + * @returns {boolean} True when the asset should be downloaded and retained. + */ +function shouldIncludeAsset(config, repoName, assetName) { + const repoConfig = config.repositories[repoName]; + if (!repoConfig) { + return config.defaultInclude; + } + + if (repoConfig.includeAll === true) { + return true; + } + + const includeAssets = normalizePatterns(repoConfig.includeAssets || repoConfig.include); + if (includeAssets.length === 0) { + return false; + } + + return matchesPattern(assetName, includeAssets); +} + +module.exports = { + loadPackageConfig, + shouldIncludeAsset +}; diff --git a/.github/scripts/sync-assets.js b/.github/scripts/sync-assets.js index 98357ab8..bb466b78 100644 --- a/.github/scripts/sync-assets.js +++ b/.github/scripts/sync-assets.js @@ -2,15 +2,152 @@ * Asset Synchronization Script * Main script for downloading and organizing release assets */ +const fs = require('fs'); const path = require('path'); -const { ensureDir, fileExists, writeFile } = require('./file-utils'); +const { ensureDir, fileExists } = require('./file-utils'); const { generateHashFiles } = require('./hash-utils'); const { downloadAssetWithRetry } = require('./download-utils'); +const { loadPackageConfig, shouldIncludeAsset } = require('./package-config'); /** - * Process a single repository and download its release assets + * Check whether a filename is one of the generated hash sidecar files. + * @param {string} filename - File name to check. + * @returns {boolean} True when the file is a generated hash file. */ -async function processRepository(github, context, repo, repositoryData, totalAssets, isPullRequest = false, releaseLimit = null, maxNewAssets = 0, newAssetsDownloaded = 0) { +function isHashFile(filename) { + return filename.endsWith('.sha256') || + filename.endsWith('.sha512') || + filename.endsWith('.md5'); +} + +/** + * Remove generated hash sidecars for a stored asset. + * @param {string} assetPath - Path to the primary asset file. + */ +function removeHashFiles(assetPath) { + ['sha256', 'sha512', 'md5'].forEach(hashType => { + const hashFile = `${assetPath}.${hashType}`; + if (fileExists(hashFile)) { + fs.unlinkSync(hashFile); + console.log(`Removed hash file: ${hashFile}`); + } + }); +} + +/** + * Remove an asset and any generated hash sidecars. + * @param {string} assetPath - Path to the primary asset file. + */ +function removeAsset(assetPath) { + if (fileExists(assetPath)) { + fs.unlinkSync(assetPath); + console.log(`Removed asset: ${assetPath}`); + } + + removeHashFiles(assetPath); +} + +/** + * Check whether a release directory still contains retained assets. + * @param {string} directoryPath - Release directory path. + * @returns {boolean} True when at least one non-hash asset remains. + */ +function hasAssetFiles(directoryPath) { + return fs.readdirSync(directoryPath, { withFileTypes: true }) + .some(dirent => dirent.isFile() && !isHashFile(dirent.name)); +} + +/** + * Check whether another new asset may be downloaded during this run. + * @param {number} maxNewAssets - Maximum new assets to download; 0 means unlimited. + * @param {number} currentNewAssets - Number of new assets already downloaded. + * @param {number} releaseNewAssets - Number of new assets downloaded in the current release. + * @returns {boolean} True when another new asset download is allowed. + */ +function canDownloadNewAsset(maxNewAssets, currentNewAssets, releaseNewAssets) { + return maxNewAssets <= 0 || (currentNewAssets + releaseNewAssets) < maxNewAssets; +} + +/** + * Remove stored files from dist that are no longer allowed by packages.config.json. + * This only affects mirrored files; release metadata is still generated from GitHub. + * @param {string} distPath - Path to the dist checkout. + * @param {Object} packageConfig - Loaded package config. + */ +function cleanupStoredAssets(distPath, packageConfig) { + console.log('Cleaning up stored assets excluded by package config...'); + + const distContents = fs.readdirSync(distPath, { withFileTypes: true }); + + for (const repoDir of distContents) { + if (!repoDir.isDirectory() || repoDir.name === '.git' || repoDir.name.startsWith('.')) { + continue; + } + + const repoPath = path.join(distPath, repoDir.name); + const repoContents = fs.readdirSync(repoPath, { withFileTypes: true }); + + for (const releaseDir of repoContents) { + if (!releaseDir.isDirectory() || !releaseDir.name.startsWith('v')) { + continue; + } + + const releasePath = path.join(repoPath, releaseDir.name); + const releaseContents = fs.readdirSync(releasePath, { withFileTypes: true }); + + for (const assetFile of releaseContents) { + if (!assetFile.isFile() || isHashFile(assetFile.name)) { + continue; + } + + if (!shouldIncludeAsset(packageConfig, repoDir.name, assetFile.name)) { + removeAsset(path.join(releasePath, assetFile.name)); + } + } + + for (const hashFile of fs.readdirSync(releasePath, { withFileTypes: true })) { + if (!hashFile.isFile() || !isHashFile(hashFile.name)) { + continue; + } + + const assetName = hashFile.name.replace(/\.(sha256|sha512|md5)$/, ''); + if (!fileExists(path.join(releasePath, assetName))) { + const hashPath = path.join(releasePath, hashFile.name); + fs.unlinkSync(hashPath); + console.log(`Removed orphan hash file: ${hashPath}`); + } + } + + if (!hasAssetFiles(releasePath)) { + fs.rmSync(releasePath, { recursive: true, force: true }); + console.log(`Removed empty release directory: ${releasePath}`); + } + } + + if (fs.readdirSync(repoPath).length === 0) { + fs.rmSync(repoPath, { recursive: true, force: true }); + console.log(`Removed empty repository directory: ${repoPath}`); + } + } + + console.log('Package config cleanup completed successfully'); +} + +/** + * Process a single repository, collecting all release metadata while downloading only configured assets. + * @param {Object} github - GitHub API client. + * @param {Object} context - GitHub Actions context. + * @param {Object} repo - Repository API response object. + * @param {Array} repositoryData - Accumulated package metadata. + * @param {number} totalAssets - Current total release asset count. + * @param {Object} packageConfig - Loaded package config. + * @param {boolean} isPullRequest - Whether this is a pull request run. + * @param {number|null} releaseLimit - Optional release limit for pull request runs. + * @param {number} maxNewAssets - Maximum new assets to download. + * @param {number} newAssetsDownloaded - Number of new assets already downloaded. + * @returns {Promise<{totalAssets: number, processedReleases: number, newAssetsDownloaded: number}>} + */ +async function processRepository(github, context, repo, repositoryData, totalAssets, packageConfig, isPullRequest = false, releaseLimit = null, maxNewAssets = 0, newAssetsDownloaded = 0) { console.log(`Processing repository: ${repo.name}`); let processedReleasesWithAssets = 0; @@ -43,19 +180,13 @@ async function processRepository(github, context, repo, repositoryData, totalAss }; for (const release of publishedReleases) { - // Check if we've reached the asset limit globally - if (maxNewAssets > 0 && (newAssetsDownloaded + repoNewAssets) >= maxNewAssets) { - console.log(`Reached maximum new assets limit (${maxNewAssets}). Stopping processing for ${repo.name}.`); - break; - } - // For pull requests, stop after processing the specified number of releases with assets if (isPullRequest && releaseLimit && processedReleasesWithAssets >= releaseLimit) { console.log(`PR mode: Reached limit of ${releaseLimit} releases with assets for ${repo.name}`); break; } - const result = await processRelease(repo.name, release, maxNewAssets, newAssetsDownloaded + repoNewAssets); + const result = await processRelease(repo.name, release, packageConfig, maxNewAssets, newAssetsDownloaded + repoNewAssets); const assetCount = result.assetCount; const newAssets = result.newAssets; @@ -83,9 +214,15 @@ async function processRepository(github, context, repo, repositoryData, totalAss } /** - * Process a single release and download its assets + * Process a single release, counting all release assets while downloading only configured assets. + * @param {string} repoName - Repository name. + * @param {Object} release - GitHub release API response object. + * @param {Object} packageConfig - Loaded package config. + * @param {number} maxNewAssets - Maximum new assets to download. + * @param {number} currentNewAssets - Number of new assets already downloaded. + * @returns {Promise<{assetCount: number, newAssets: number}>} */ -async function processRelease(repoName, release, maxNewAssets = 0, currentNewAssets = 0) { +async function processRelease(repoName, release, packageConfig, maxNewAssets = 0, currentNewAssets = 0) { console.log(`Processing release: ${release.tag_name}`); if (release.assets.length === 0) { @@ -93,23 +230,29 @@ async function processRelease(repoName, release, maxNewAssets = 0, currentNewAss return { assetCount: 0, newAssets: 0 }; } + const assetCount = release.assets.length; + const includedAssets = release.assets.filter(asset => shouldIncludeAsset(packageConfig, repoName, asset.name)); + + if (includedAssets.length === 0) { + console.log(`No configured assets found for release ${release.tag_name}`); + return { assetCount, newAssets: 0 }; + } + // Create directory structure const releaseDir = path.join(repoName, release.tag_name); ensureDir(releaseDir); - let assetCount = 0; let newAssets = 0; - for (const asset of release.assets) { + for (const asset of includedAssets) { // Check if we've reached the new assets download limit - if (maxNewAssets > 0 && (currentNewAssets + newAssets) >= maxNewAssets) { + if (!canDownloadNewAsset(maxNewAssets, currentNewAssets, newAssets)) { console.log(`Reached maximum new assets limit (${maxNewAssets}) for this run. Stopping asset processing for release ${release.tag_name}.`); break; } const result = await processAsset(releaseDir, asset); if (result.downloaded) { - assetCount++; if (result.isNew) { newAssets++; } @@ -121,6 +264,9 @@ async function processRelease(repoName, release, maxNewAssets = 0, currentNewAss /** * Process a single asset - download and generate hashes if not exists + * @param {string} releaseDir - Directory where the asset should be stored. + * @param {Object} asset - GitHub release asset API response object. + * @returns {Promise<{downloaded: boolean, isNew: boolean}>} */ async function processAsset(releaseDir, asset) { const assetPath = path.join(releaseDir, asset.name); @@ -135,16 +281,7 @@ async function processAsset(releaseDir, asset) { if (fileExists(assetPath)) { console.log(`Removing existing oversized file: ${assetPath}`); try { - const fs = require('fs'); - fs.unlinkSync(assetPath); - // Also remove associated hash files - ['sha256', 'sha512', 'md5'].forEach(hashType => { - const hashFile = `${assetPath}.${hashType}`; - if (fileExists(hashFile)) { - fs.unlinkSync(hashFile); - console.log(`Removed hash file: ${hashFile}`); - } - }); + removeAsset(assetPath); } catch (error) { console.error(`Failed to remove oversized file ${assetPath}: ${error.message}`); } @@ -159,20 +296,11 @@ async function processAsset(releaseDir, asset) { // Check if existing file is over size limit and remove it try { - const fs = require('fs'); const stats = fs.statSync(assetPath); if (stats.size > maxSizeBytes) { const sizeMB = (stats.size / (1024 * 1024)).toFixed(2); console.log(`Removing existing oversized file: ${assetPath} (${sizeMB}MB)`); - fs.unlinkSync(assetPath); - // Also remove associated hash files - ['sha256', 'sha512', 'md5'].forEach(hashType => { - const hashFile = `${assetPath}.${hashType}`; - if (fileExists(hashFile)) { - fs.unlinkSync(hashFile); - console.log(`Removed hash file: ${hashFile}`); - } - }); + removeAsset(assetPath); // Continue to download the asset since we removed the oversized one // But first check again if the new asset would be over the limit if (asset.size > maxSizeBytes) { @@ -213,7 +341,12 @@ async function processAsset(releaseDir, asset) { } /** - * Main function to sync all release assets + * Sync package metadata for all release assets and mirror only configured assets to dist. + * @param {Object} github - GitHub API client. + * @param {Object} context - GitHub Actions context. + * @param {boolean} isPullRequest - Whether this is a pull request run. + * @param {number} maxNewAssets - Maximum new assets to download. + * @returns {Promise} Repository metadata for packages.json. */ async function syncReleaseAssets(github, context, isPullRequest = false, maxNewAssets = 0) { console.log('Getting repositories from organization...'); @@ -237,6 +370,9 @@ async function syncReleaseAssets(github, context, isPullRequest = false, maxNewA console.log(`Found ${repos.length} repositories`); + const packageConfig = loadPackageConfig(process.cwd()); + cleanupStoredAssets('.', packageConfig); + const repositoryData = []; let totalAssets = 0; let totalProcessedReleases = 0; @@ -244,18 +380,13 @@ async function syncReleaseAssets(github, context, isPullRequest = false, maxNewA // Process each repository for (const repo of repos) { - // Check if we've reached the asset limit - if (maxNewAssets > 0 && newAssetsDownloaded >= maxNewAssets) { - console.log(`Reached maximum new assets limit (${maxNewAssets}). Stopping processing.`); - break; - } - const result = await processRepository( github, context, repo, repositoryData, totalAssets, + packageConfig, isPullRequest, isPullRequest ? 2 : null, maxNewAssets, @@ -283,5 +414,6 @@ async function syncReleaseAssets(github, context, isPullRequest = false, maxNewA } module.exports = { + canDownloadNewAsset, syncReleaseAssets }; diff --git a/.github/workflows/sync-release-assets.yml b/.github/workflows/sync-release-assets.yml deleted file mode 100644 index 70769da2..00000000 --- a/.github/workflows/sync-release-assets.yml +++ /dev/null @@ -1,91 +0,0 @@ ---- -name: Sync Release Assets -permissions: {} - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - -env: - MAX_NEW_ASSETS: 50 # Set to 0 for unlimited, or any positive number to limit downloads per run - -on: - workflow_dispatch: - pull_request: - push: - branches: - - master - schedule: - - cron: '0 * * * *' - -jobs: - sync-assets: - permissions: - contents: read - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - token: ${{ secrets.GH_BOT_TOKEN }} - - - name: Create or checkout dist branch into dist directory - run: | - git fetch origin - if git rev-parse --verify origin/dist >/dev/null 2>&1; then - echo "Dist branch exists, checking it out" - git worktree add dist origin/dist - cd dist - git pull origin dist - else - echo "Dist branch doesn't exist, creating new one" - git worktree add --orphan dist - cd dist - git rm -rf . 2>/dev/null || true - fi - - - name: Get organization repositories and download assets - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 - with: - github-token: ${{ secrets.GH_BOT_TOKEN }} - script: | - // Import the sync assets with metadata module - const { syncAssetsWithMetadata } = require('./.github/scripts/sync-assets-with-metadata.js'); - - // Check if this is a pull request event - const isPullRequest = context.eventName === 'pull_request'; - - // Get the max new assets limit from environment - const maxNewAssets = parseInt(process.env.MAX_NEW_ASSETS) || 0; - - // Change working directory to dist for file operations - process.chdir('./dist'); - - // Run the asset synchronization and store metadata - await syncAssetsWithMetadata(github, context, isPullRequest, maxNewAssets); - - - name: Generate packages.json - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 - with: - github-token: ${{ secrets.GH_BOT_TOKEN }} - script: | - // Import the packages generation module - const { generatePackagesWithCleanup } = require('./.github/scripts/generate-packages-with-cleanup.js'); - - console.log('Generating packages.json from dist directory...'); - - // Change to dist directory - process.chdir('./dist'); - - // Generate packages.json with cleanup - generatePackagesWithCleanup('.'); - - - name: Commit and push changes - if: github.event_name != 'pull_request' - uses: actions-js/push@5a7cbd780d82c0c937b5977586e641b2fd94acc5 # v1.5 - with: - author_email: ${{ secrets.GH_BOT_EMAIL }} - author_name: ${{ vars.GH_BOT_NAME }} - branch: dist - directory: dist - github_token: ${{ secrets.GH_BOT_TOKEN }} - message: 'Update release assets - ${{ github.run_id }}' diff --git a/.github/workflows/update-pages.yml b/.github/workflows/update-pages.yml index 97d06342..2f7fb2d1 100644 --- a/.github/workflows/update-pages.yml +++ b/.github/workflows/update-pages.yml @@ -1,5 +1,5 @@ --- -name: Build GH-Pages +name: Update permissions: {} on: @@ -7,14 +7,19 @@ on: push: branches: - master + schedule: + - cron: '0 * * * *' workflow_dispatch: concurrency: group: "${{ github.workflow }}-${{ github.ref }}" cancel-in-progress: true +env: + MAX_NEW_ASSETS: 50 # Set to 0 for unlimited, or any positive number to limit downloads per run + jobs: - prep: + update: permissions: contents: read runs-on: ubuntu-latest @@ -22,17 +27,89 @@ jobs: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - name: Create or checkout dist branch into dist directory + run: | + if git ls-remote --exit-code --heads origin dist >/dev/null; then + echo "Dist branch exists, checking it out at depth 1" + git fetch --no-tags --depth=1 origin dist:refs/remotes/origin/dist + git worktree add dist origin/dist + else + status=$? + if [ "$status" -ne 2 ]; then + exit "$status" + fi + + echo "Dist branch doesn't exist, creating new one" + git worktree add --orphan dist + cd dist + git rm -rf . 2>/dev/null || true + fi + + - name: Get organization repositories and download assets + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + github-token: ${{ secrets.GH_BOT_TOKEN || github.token }} + script: | + // Import the sync assets with metadata module + const { syncAssetsWithMetadata } = require('./.github/scripts/sync-assets-with-metadata.js'); + + // Check if this is a pull request event + const isPullRequest = context.eventName === 'pull_request'; + + // Get the max new assets limit from environment + const maxNewAssets = parseInt(process.env.MAX_NEW_ASSETS) || 0; + + // Change working directory to dist for file operations + process.chdir('./dist'); + + // Run the asset synchronization and store metadata + await syncAssetsWithMetadata(github, context, isPullRequest, maxNewAssets); + + - name: Generate packages.json + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + github-token: ${{ secrets.GH_BOT_TOKEN || github.token }} + script: | + // Import the packages generation module + const { generatePackagesWithCleanup } = require('./.github/scripts/generate-packages-with-cleanup.js'); + + console.log('Generating packages.json from dist directory...'); + + // Change to dist directory + process.chdir('./dist'); + + // Generate packages.json with cleanup + generatePackagesWithCleanup('.'); + + - name: GitHub Commit & Push + if: github.event_name != 'pull_request' + uses: actions-js/push@5a7cbd780d82c0c937b5977586e641b2fd94acc5 # v1.5 + with: + author_email: ${{ secrets.GH_BOT_EMAIL }} + author_name: ${{ vars.GH_BOT_NAME }} + branch: dist + directory: dist + github_token: ${{ secrets.GH_BOT_TOKEN }} + message: 'Update release assets - ${{ github.run_id }}' + + - name: Prepare artifact + run: | + cd gh-pages-template + 7z a ../build.zip . + cd ../dist + 7z a ../build.zip . "-xr!.git" + - name: Upload artifact uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: prep - path: gh-pages-template/ + name: update + path: build.zip if-no-files-found: error include-hidden-files: true retention-days: 1 call-jekyll-build: - needs: prep + needs: update permissions: contents: read uses: LizardByte/LizardByte.github.io/.github/workflows/jekyll-build.yml@master @@ -41,6 +118,7 @@ jobs: GH_BOT_TOKEN: ${{ secrets.GH_BOT_TOKEN }} with: clean_gh_pages: true + extract_archive: 'build.zip' gh_bot_name: ${{ vars.GH_BOT_NAME }} - site_artifact: 'prep' + site_artifact: 'update' target_branch: 'gh-pages' diff --git a/README.md b/README.md index 5eb992ee..5e0f97d5 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # LizardByte Package Repository -This repository serves as a centralized storage for all packages and release assets from repositories within the -LizardByte organization. All release assets are automatically downloaded and organized in the `dist` branch for +This repository serves as a centralized package index and filtered release asset mirror for repositories within the +LizardByte organization. Configured release assets are automatically downloaded and organized in the `dist` branch for easy access and distribution. ## Structure @@ -34,18 +34,42 @@ dist/ ## Workflow -The release asset collection is handled by the `sync-release-assets.yml` workflow which: +The release asset collection and GitHub Pages deployment are handled by the `update-pages.yml` workflow. Asset +retention is controlled by `packages.config.json`. 1. Discovers all repositories in the LizardByte organization 2. Fetches release information for each repository -3. Downloads missing release assets -4. Generates hash files for integrity verification -5. Commits changes to the `dist` branch +3. Downloads configured release assets +4. Removes previously stored assets that are no longer configured +5. Generates hash files for integrity verification +6. Commits changes to the `dist` branch +7. Rebuilds GitHub Pages and publishes the filtered `dist` assets there too + +## Configuration + +`packages.config.json` controls which release assets are retained in `dist` and published to GitHub Pages. +`packages.json` is still generated from GitHub release metadata for all discovered assets. `defaultInclude` is +`false`, so repositories must opt in with asset patterns for files that should be mirrored. + +```json +{ + "defaultInclude": false, + "repositories": { + "Sunshine": { + "includeAssets": [ + "*-installer.exe", + "*-installer.msi" + ] + } + } +} +``` ## Usage -All release assets are available in the `dist` branch of this repository. You can: +Configured release assets are available in the `dist` branch of this repository. You can: - Browse assets directly on GitHub - Clone the `dist` branch for offline access +- Use GitHub Pages direct URLs that mirror the `dist` layout, e.g. `/packages///` - Use the hash files to verify asset integrity diff --git a/gh-pages-template/assets/js/app.js b/gh-pages-template/assets/js/app.js index 46f67ebf..7249e984 100644 --- a/gh-pages-template/assets/js/app.js +++ b/gh-pages-template/assets/js/app.js @@ -6,8 +6,10 @@ class RepositoryDataManager { constructor() { this.repositoryData = []; this.orgName = 'LizardByte'; // Organization name - this.distBranch = 'dist'; - this.rawBase = 'https://raw.githubusercontent.com'; + this.packagesPaths = [ + 'packages.json', + `https://raw.githubusercontent.com/${this.orgName}/packages/dist/packages.json` + ]; } /** @@ -17,8 +19,8 @@ class RepositoryDataManager { try { console.log('Fetching last successful workflow run...'); - // Fetch workflow runs for the sync-release-assets.yml workflow - const response = await fetch(`https://api.github.com/repos/${this.orgName}/packages/actions/workflows/sync-release-assets.yml/runs?event=schedule&status=success&branch=master&per_page=1`); + // Fetch workflow runs for the update-pages.yml workflow + const response = await fetch(`https://api.github.com/repos/${this.orgName}/packages/actions/workflows/update-pages.yml/runs?event=schedule&status=success&branch=master&per_page=1`); if (!response.ok) { console.warn(`Failed to fetch workflow runs: ${response.status}`); @@ -42,20 +44,41 @@ class RepositoryDataManager { } /** - * Load repository data from packages.json in the dist branch + * Fetch packages.json from the first available source. + * + * The generated GitHub Pages site serves packages.json locally. ReadTheDocs + * previews only serve the template files, so they need the dist branch + * fallback to show repository data. */ - async loadRepositoryData() { - try { - console.log('Loading repository data from packages.json...'); + async fetchPackagesJson() { + let lastError = null; - // Fetch the packages.json file from the dist branch - const response = await fetch(`${this.rawBase}/${this.orgName}/packages/${this.distBranch}/packages.json`); + for (const packagesPath of this.packagesPaths) { + try { + const response = await fetch(packagesPath); - if (!response.ok) { - throw new Error(`Failed to fetch packages.json: ${response.status}`); + if (!response.ok) { + throw new Error(`Failed to fetch packages.json from ${packagesPath}: ${response.status}`); + } + + return await response.json(); + } catch (error) { + console.warn(error.message); + lastError = error; } + } - const data = await response.json(); + throw lastError || new Error('Failed to fetch packages.json'); + } + + /** + * Load repository data from the packages.json published with GitHub Pages + */ + async loadRepositoryData() { + try { + console.log('Loading repository data from packages.json...'); + + const data = await this.fetchPackagesJson(); // Validate the data structure if (!data.repositories || !Array.isArray(data.repositories)) { diff --git a/packages.config.json b/packages.config.json new file mode 100644 index 00000000..4ab1bb17 --- /dev/null +++ b/packages.config.json @@ -0,0 +1,11 @@ +{ + "defaultInclude": false, + "repositories": { + "Sunshine": { + "includeAssets": [ + "*-installer.exe", + "*-installer.msi" + ] + } + } +} From a316156bde6deb10f7f0bcca1331cd83c434d088 Mon Sep 17 00:00:00 2001 From: ReenigneArcher <42013603+ReenigneArcher@users.noreply.github.com> Date: Sat, 16 May 2026 21:19:38 -0400 Subject: [PATCH 2/2] Simplify packages.json fetching Replace the packagesPaths array/fallback logic with a single packagesPath and inline fetch. Removed the fetchPackagesJson method and updated loadRepositoryData to fetch this.packagesPath directly, validating response.ok and parsing JSON. This simplifies the code by removing the raw GitHub fallback and centralizing the packages.json request. --- gh-pages-template/assets/js/app.js | 41 ++++++------------------------ 1 file changed, 8 insertions(+), 33 deletions(-) diff --git a/gh-pages-template/assets/js/app.js b/gh-pages-template/assets/js/app.js index 7249e984..b232cdd9 100644 --- a/gh-pages-template/assets/js/app.js +++ b/gh-pages-template/assets/js/app.js @@ -6,10 +6,7 @@ class RepositoryDataManager { constructor() { this.repositoryData = []; this.orgName = 'LizardByte'; // Organization name - this.packagesPaths = [ - 'packages.json', - `https://raw.githubusercontent.com/${this.orgName}/packages/dist/packages.json` - ]; + this.packagesPath = 'packages.json'; } /** @@ -43,34 +40,6 @@ class RepositoryDataManager { } } - /** - * Fetch packages.json from the first available source. - * - * The generated GitHub Pages site serves packages.json locally. ReadTheDocs - * previews only serve the template files, so they need the dist branch - * fallback to show repository data. - */ - async fetchPackagesJson() { - let lastError = null; - - for (const packagesPath of this.packagesPaths) { - try { - const response = await fetch(packagesPath); - - if (!response.ok) { - throw new Error(`Failed to fetch packages.json from ${packagesPath}: ${response.status}`); - } - - return await response.json(); - } catch (error) { - console.warn(error.message); - lastError = error; - } - } - - throw lastError || new Error('Failed to fetch packages.json'); - } - /** * Load repository data from the packages.json published with GitHub Pages */ @@ -78,7 +47,13 @@ class RepositoryDataManager { try { console.log('Loading repository data from packages.json...'); - const data = await this.fetchPackagesJson(); + const response = await fetch(this.packagesPath); + + if (!response.ok) { + throw new Error(`Failed to fetch packages.json: ${response.status}`); + } + + const data = await response.json(); // Validate the data structure if (!data.repositories || !Array.isArray(data.repositories)) {