diff --git a/.github/scripts/generate-packages.js b/.github/scripts/generate-packages.js index 13e34ea4..48a96f0e 100644 --- a/.github/scripts/generate-packages.js +++ b/.github/scripts/generate-packages.js @@ -2,12 +2,76 @@ const fs = require('fs'); const path = require('path'); /** - * Generate packages.json file by scanning the dist directory structure - * @param {string} distPath - Path to the dist directory - * @param {Array} repositoryMetadata - Repository metadata from sync process with archived status + * Build packages.json data with stats from repository entries. + * @param {Array} repositories - Repository data to include. + * @returns {Object} packages.json data. + */ +function buildPackagesData(repositories) { + // Sort repositories by name + repositories.sort((a, b) => a.name.localeCompare(b.name)); + + // Calculate totals + const totalReleases = repositories.reduce((sum, repo) => sum + repo.releases.length, 0); + const totalAssets = repositories.reduce((sum, repo) => + sum + repo.releases.reduce((releaseSum, release) => releaseSum + release.assetCount, 0), 0 + ); + + const packagesData = { + repositories: repositories, + stats: { + totalRepositories: repositories.length, + totalReleases: totalReleases, + totalAssets: totalAssets + } + }; + + console.log(`Generated packages data:`); + console.log(` Repositories: ${repositories.length}`); + console.log(` Total Releases: ${totalReleases}`); + console.log(` Total Assets: ${totalAssets}`); + + return packagesData; +} + +/** + * Normalize GitHub-derived repository metadata for packages.json. + * @param {Array} repositoryMetadata - Repository metadata from the sync process. + * @returns {Array} Normalized repositories with sorted releases. + */ +function normalizeRepositoryMetadata(repositoryMetadata = []) { + return repositoryMetadata + .filter(repo => repo && Array.isArray(repo.releases) && repo.releases.length > 0) + .map(repo => { + const releases = repo.releases + .filter(release => release && release.assetCount > 0) + .map(release => ({ + tag: release.tag, + assetCount: release.assetCount + })); + + releases.sort((a, b) => b.tag.localeCompare(a.tag, undefined, { numeric: true, sensitivity: 'base' })); + + return { + name: repo.name, + archived: Boolean(repo.archived), + releases + }; + }) + .filter(repo => repo.releases.length > 0); +} + +/** + * Generate packages.json data from GitHub metadata, with a dist scan fallback. + * @param {string} distPath - Path to the dist directory. + * @param {Array} repositoryMetadata - GitHub-derived metadata for all release assets. * @returns {Object} Generated packages data */ function generatePackagesJson(distPath = '.', repositoryMetadata = []) { + if (repositoryMetadata.length > 0) { + console.log('Generating packages.json from GitHub release metadata'); + return buildPackagesData(normalizeRepositoryMetadata(repositoryMetadata)); + } + console.log(`Scanning dist directory: ${distPath}`); const repositories = []; @@ -42,30 +106,7 @@ function generatePackagesJson(distPath = '.', repositoryMetadata = []) { } } - // Sort repositories by name - repositories.sort((a, b) => a.name.localeCompare(b.name)); - - // Calculate totals - const totalReleases = repositories.reduce((sum, repo) => sum + repo.releases.length, 0); - const totalAssets = repositories.reduce((sum, repo) => - sum + repo.releases.reduce((releaseSum, release) => releaseSum + release.assetCount, 0), 0 - ); - - const packagesData = { - repositories: repositories, - stats: { - totalRepositories: repositories.length, - totalReleases: totalReleases, - totalAssets: totalAssets - } - }; - - console.log(`Generated packages data:`); - console.log(` Repositories: ${repositories.length}`); - console.log(` Total Releases: ${totalReleases}`); - console.log(` Total Assets: ${totalAssets}`); - - return packagesData; + return buildPackagesData(repositories); } catch (error) { console.error('Error scanning dist directory:', error); @@ -176,7 +217,9 @@ function writePackagesJson(packagesData, outputPath = './packages.json') { } module.exports = { + buildPackagesData, generatePackagesJson, + normalizeRepositoryMetadata, scanRepositoryDirectory, scanReleaseDirectory, writePackagesJson diff --git a/.github/scripts/package-config.js b/.github/scripts/package-config.js new file mode 100644 index 00000000..1e94e12e --- /dev/null +++ b/.github/scripts/package-config.js @@ -0,0 +1,130 @@ +const fs = require('fs'); +const path = require('path'); + +const CONFIG_FILENAME = 'packages.config.json'; + +/** + * Find packages.config.json by walking up from a starting directory. + * @param {string} startDir - Directory to start searching from. + * @returns {string|null} Absolute path to the config file, or null if absent. + */ +function findConfigPath(startDir = process.cwd()) { + let currentDir = path.resolve(startDir); + + while (true) { + const candidate = path.join(currentDir, CONFIG_FILENAME); + if (fs.existsSync(candidate)) { + return candidate; + } + + const parentDir = path.dirname(currentDir); + if (parentDir === currentDir) { + return null; + } + currentDir = parentDir; + } +} + +/** + * Normalize a possibly missing pattern list to non-empty string patterns. + * @param {unknown} value - Config value to normalize. + * @returns {string[]} Valid pattern strings. + */ +function normalizePatterns(value) { + if (!Array.isArray(value)) { + return []; + } + + return value.filter(pattern => typeof pattern === 'string' && pattern.length > 0); +} + +/** + * Load release asset retention config. + * @param {string} startDir - Directory used to locate packages.config.json. + * @returns {{defaultInclude: boolean, repositories: Object}} Normalized package config. + */ +function loadPackageConfig(startDir = process.cwd()) { + const configPath = findConfigPath(startDir); + if (!configPath) { + console.log(`No ${CONFIG_FILENAME} found; including all release assets.`); + return { + defaultInclude: true, + repositories: {} + }; + } + + const config = JSON.parse(fs.readFileSync(configPath, 'utf8')); + const repositories = config.repositories && typeof config.repositories === 'object' + ? config.repositories + : {}; + + console.log(`Loaded package config from ${configPath}`); + + return { + defaultInclude: config.defaultInclude === true, + repositories + }; +} + +/** + * Escape literal text for use inside a regular expression. + * @param {string} value - Raw pattern segment. + * @returns {string} Regex-escaped segment. + */ +function escapeRegex(value) { + return value.replace(/[|\\{}()[\]^$+?.]/g, '\\$&'); +} + +/** + * Convert a simple wildcard pattern to a regular expression. + * @param {string} pattern - Pattern using '*' as a wildcard. + * @returns {RegExp} Anchored regular expression. + */ +function patternToRegex(pattern) { + const source = pattern + .split('*') + .map(escapeRegex) + .join('.*'); + + return new RegExp(`^${source}$`); +} + +/** + * Check whether a value matches any configured wildcard pattern. + * @param {string} value - Value to test. + * @param {string[]} patterns - Wildcard patterns. + * @returns {boolean} True when any pattern matches. + */ +function matchesPattern(value, patterns) { + return patterns.some(pattern => patternToRegex(pattern).test(value)); +} + +/** + * Determine whether a release asset should be mirrored into dist and GitHub Pages. + * @param {{defaultInclude: boolean, repositories: Object}} config - Package config. + * @param {string} repoName - Repository name. + * @param {string} assetName - Release asset filename. + * @returns {boolean} True when the asset should be downloaded and retained. + */ +function shouldIncludeAsset(config, repoName, assetName) { + const repoConfig = config.repositories[repoName]; + if (!repoConfig) { + return config.defaultInclude; + } + + if (repoConfig.includeAll === true) { + return true; + } + + const includeAssets = normalizePatterns(repoConfig.includeAssets || repoConfig.include); + if (includeAssets.length === 0) { + return false; + } + + return matchesPattern(assetName, includeAssets); +} + +module.exports = { + loadPackageConfig, + shouldIncludeAsset +}; diff --git a/.github/scripts/sync-assets.js b/.github/scripts/sync-assets.js index 98357ab8..bb466b78 100644 --- a/.github/scripts/sync-assets.js +++ b/.github/scripts/sync-assets.js @@ -2,15 +2,152 @@ * Asset Synchronization Script * Main script for downloading and organizing release assets */ +const fs = require('fs'); const path = require('path'); -const { ensureDir, fileExists, writeFile } = require('./file-utils'); +const { ensureDir, fileExists } = require('./file-utils'); const { generateHashFiles } = require('./hash-utils'); const { downloadAssetWithRetry } = require('./download-utils'); +const { loadPackageConfig, shouldIncludeAsset } = require('./package-config'); /** - * Process a single repository and download its release assets + * Check whether a filename is one of the generated hash sidecar files. + * @param {string} filename - File name to check. + * @returns {boolean} True when the file is a generated hash file. */ -async function processRepository(github, context, repo, repositoryData, totalAssets, isPullRequest = false, releaseLimit = null, maxNewAssets = 0, newAssetsDownloaded = 0) { +function isHashFile(filename) { + return filename.endsWith('.sha256') || + filename.endsWith('.sha512') || + filename.endsWith('.md5'); +} + +/** + * Remove generated hash sidecars for a stored asset. + * @param {string} assetPath - Path to the primary asset file. + */ +function removeHashFiles(assetPath) { + ['sha256', 'sha512', 'md5'].forEach(hashType => { + const hashFile = `${assetPath}.${hashType}`; + if (fileExists(hashFile)) { + fs.unlinkSync(hashFile); + console.log(`Removed hash file: ${hashFile}`); + } + }); +} + +/** + * Remove an asset and any generated hash sidecars. + * @param {string} assetPath - Path to the primary asset file. + */ +function removeAsset(assetPath) { + if (fileExists(assetPath)) { + fs.unlinkSync(assetPath); + console.log(`Removed asset: ${assetPath}`); + } + + removeHashFiles(assetPath); +} + +/** + * Check whether a release directory still contains retained assets. + * @param {string} directoryPath - Release directory path. + * @returns {boolean} True when at least one non-hash asset remains. + */ +function hasAssetFiles(directoryPath) { + return fs.readdirSync(directoryPath, { withFileTypes: true }) + .some(dirent => dirent.isFile() && !isHashFile(dirent.name)); +} + +/** + * Check whether another new asset may be downloaded during this run. + * @param {number} maxNewAssets - Maximum new assets to download; 0 means unlimited. + * @param {number} currentNewAssets - Number of new assets already downloaded. + * @param {number} releaseNewAssets - Number of new assets downloaded in the current release. + * @returns {boolean} True when another new asset download is allowed. + */ +function canDownloadNewAsset(maxNewAssets, currentNewAssets, releaseNewAssets) { + return maxNewAssets <= 0 || (currentNewAssets + releaseNewAssets) < maxNewAssets; +} + +/** + * Remove stored files from dist that are no longer allowed by packages.config.json. + * This only affects mirrored files; release metadata is still generated from GitHub. + * @param {string} distPath - Path to the dist checkout. + * @param {Object} packageConfig - Loaded package config. + */ +function cleanupStoredAssets(distPath, packageConfig) { + console.log('Cleaning up stored assets excluded by package config...'); + + const distContents = fs.readdirSync(distPath, { withFileTypes: true }); + + for (const repoDir of distContents) { + if (!repoDir.isDirectory() || repoDir.name === '.git' || repoDir.name.startsWith('.')) { + continue; + } + + const repoPath = path.join(distPath, repoDir.name); + const repoContents = fs.readdirSync(repoPath, { withFileTypes: true }); + + for (const releaseDir of repoContents) { + if (!releaseDir.isDirectory() || !releaseDir.name.startsWith('v')) { + continue; + } + + const releasePath = path.join(repoPath, releaseDir.name); + const releaseContents = fs.readdirSync(releasePath, { withFileTypes: true }); + + for (const assetFile of releaseContents) { + if (!assetFile.isFile() || isHashFile(assetFile.name)) { + continue; + } + + if (!shouldIncludeAsset(packageConfig, repoDir.name, assetFile.name)) { + removeAsset(path.join(releasePath, assetFile.name)); + } + } + + for (const hashFile of fs.readdirSync(releasePath, { withFileTypes: true })) { + if (!hashFile.isFile() || !isHashFile(hashFile.name)) { + continue; + } + + const assetName = hashFile.name.replace(/\.(sha256|sha512|md5)$/, ''); + if (!fileExists(path.join(releasePath, assetName))) { + const hashPath = path.join(releasePath, hashFile.name); + fs.unlinkSync(hashPath); + console.log(`Removed orphan hash file: ${hashPath}`); + } + } + + if (!hasAssetFiles(releasePath)) { + fs.rmSync(releasePath, { recursive: true, force: true }); + console.log(`Removed empty release directory: ${releasePath}`); + } + } + + if (fs.readdirSync(repoPath).length === 0) { + fs.rmSync(repoPath, { recursive: true, force: true }); + console.log(`Removed empty repository directory: ${repoPath}`); + } + } + + console.log('Package config cleanup completed successfully'); +} + +/** + * Process a single repository, collecting all release metadata while downloading only configured assets. + * @param {Object} github - GitHub API client. + * @param {Object} context - GitHub Actions context. + * @param {Object} repo - Repository API response object. + * @param {Array} repositoryData - Accumulated package metadata. + * @param {number} totalAssets - Current total release asset count. + * @param {Object} packageConfig - Loaded package config. + * @param {boolean} isPullRequest - Whether this is a pull request run. + * @param {number|null} releaseLimit - Optional release limit for pull request runs. + * @param {number} maxNewAssets - Maximum new assets to download. + * @param {number} newAssetsDownloaded - Number of new assets already downloaded. + * @returns {Promise<{totalAssets: number, processedReleases: number, newAssetsDownloaded: number}>} + */ +async function processRepository(github, context, repo, repositoryData, totalAssets, packageConfig, isPullRequest = false, releaseLimit = null, maxNewAssets = 0, newAssetsDownloaded = 0) { console.log(`Processing repository: ${repo.name}`); let processedReleasesWithAssets = 0; @@ -43,19 +180,13 @@ async function processRepository(github, context, repo, repositoryData, totalAss }; for (const release of publishedReleases) { - // Check if we've reached the asset limit globally - if (maxNewAssets > 0 && (newAssetsDownloaded + repoNewAssets) >= maxNewAssets) { - console.log(`Reached maximum new assets limit (${maxNewAssets}). Stopping processing for ${repo.name}.`); - break; - } - // For pull requests, stop after processing the specified number of releases with assets if (isPullRequest && releaseLimit && processedReleasesWithAssets >= releaseLimit) { console.log(`PR mode: Reached limit of ${releaseLimit} releases with assets for ${repo.name}`); break; } - const result = await processRelease(repo.name, release, maxNewAssets, newAssetsDownloaded + repoNewAssets); + const result = await processRelease(repo.name, release, packageConfig, maxNewAssets, newAssetsDownloaded + repoNewAssets); const assetCount = result.assetCount; const newAssets = result.newAssets; @@ -83,9 +214,15 @@ async function processRepository(github, context, repo, repositoryData, totalAss } /** - * Process a single release and download its assets + * Process a single release, counting all release assets while downloading only configured assets. + * @param {string} repoName - Repository name. + * @param {Object} release - GitHub release API response object. + * @param {Object} packageConfig - Loaded package config. + * @param {number} maxNewAssets - Maximum new assets to download. + * @param {number} currentNewAssets - Number of new assets already downloaded. + * @returns {Promise<{assetCount: number, newAssets: number}>} */ -async function processRelease(repoName, release, maxNewAssets = 0, currentNewAssets = 0) { +async function processRelease(repoName, release, packageConfig, maxNewAssets = 0, currentNewAssets = 0) { console.log(`Processing release: ${release.tag_name}`); if (release.assets.length === 0) { @@ -93,23 +230,29 @@ async function processRelease(repoName, release, maxNewAssets = 0, currentNewAss return { assetCount: 0, newAssets: 0 }; } + const assetCount = release.assets.length; + const includedAssets = release.assets.filter(asset => shouldIncludeAsset(packageConfig, repoName, asset.name)); + + if (includedAssets.length === 0) { + console.log(`No configured assets found for release ${release.tag_name}`); + return { assetCount, newAssets: 0 }; + } + // Create directory structure const releaseDir = path.join(repoName, release.tag_name); ensureDir(releaseDir); - let assetCount = 0; let newAssets = 0; - for (const asset of release.assets) { + for (const asset of includedAssets) { // Check if we've reached the new assets download limit - if (maxNewAssets > 0 && (currentNewAssets + newAssets) >= maxNewAssets) { + if (!canDownloadNewAsset(maxNewAssets, currentNewAssets, newAssets)) { console.log(`Reached maximum new assets limit (${maxNewAssets}) for this run. Stopping asset processing for release ${release.tag_name}.`); break; } const result = await processAsset(releaseDir, asset); if (result.downloaded) { - assetCount++; if (result.isNew) { newAssets++; } @@ -121,6 +264,9 @@ async function processRelease(repoName, release, maxNewAssets = 0, currentNewAss /** * Process a single asset - download and generate hashes if not exists + * @param {string} releaseDir - Directory where the asset should be stored. + * @param {Object} asset - GitHub release asset API response object. + * @returns {Promise<{downloaded: boolean, isNew: boolean}>} */ async function processAsset(releaseDir, asset) { const assetPath = path.join(releaseDir, asset.name); @@ -135,16 +281,7 @@ async function processAsset(releaseDir, asset) { if (fileExists(assetPath)) { console.log(`Removing existing oversized file: ${assetPath}`); try { - const fs = require('fs'); - fs.unlinkSync(assetPath); - // Also remove associated hash files - ['sha256', 'sha512', 'md5'].forEach(hashType => { - const hashFile = `${assetPath}.${hashType}`; - if (fileExists(hashFile)) { - fs.unlinkSync(hashFile); - console.log(`Removed hash file: ${hashFile}`); - } - }); + removeAsset(assetPath); } catch (error) { console.error(`Failed to remove oversized file ${assetPath}: ${error.message}`); } @@ -159,20 +296,11 @@ async function processAsset(releaseDir, asset) { // Check if existing file is over size limit and remove it try { - const fs = require('fs'); const stats = fs.statSync(assetPath); if (stats.size > maxSizeBytes) { const sizeMB = (stats.size / (1024 * 1024)).toFixed(2); console.log(`Removing existing oversized file: ${assetPath} (${sizeMB}MB)`); - fs.unlinkSync(assetPath); - // Also remove associated hash files - ['sha256', 'sha512', 'md5'].forEach(hashType => { - const hashFile = `${assetPath}.${hashType}`; - if (fileExists(hashFile)) { - fs.unlinkSync(hashFile); - console.log(`Removed hash file: ${hashFile}`); - } - }); + removeAsset(assetPath); // Continue to download the asset since we removed the oversized one // But first check again if the new asset would be over the limit if (asset.size > maxSizeBytes) { @@ -213,7 +341,12 @@ async function processAsset(releaseDir, asset) { } /** - * Main function to sync all release assets + * Sync package metadata for all release assets and mirror only configured assets to dist. + * @param {Object} github - GitHub API client. + * @param {Object} context - GitHub Actions context. + * @param {boolean} isPullRequest - Whether this is a pull request run. + * @param {number} maxNewAssets - Maximum new assets to download. + * @returns {Promise} Repository metadata for packages.json. */ async function syncReleaseAssets(github, context, isPullRequest = false, maxNewAssets = 0) { console.log('Getting repositories from organization...'); @@ -237,6 +370,9 @@ async function syncReleaseAssets(github, context, isPullRequest = false, maxNewA console.log(`Found ${repos.length} repositories`); + const packageConfig = loadPackageConfig(process.cwd()); + cleanupStoredAssets('.', packageConfig); + const repositoryData = []; let totalAssets = 0; let totalProcessedReleases = 0; @@ -244,18 +380,13 @@ async function syncReleaseAssets(github, context, isPullRequest = false, maxNewA // Process each repository for (const repo of repos) { - // Check if we've reached the asset limit - if (maxNewAssets > 0 && newAssetsDownloaded >= maxNewAssets) { - console.log(`Reached maximum new assets limit (${maxNewAssets}). Stopping processing.`); - break; - } - const result = await processRepository( github, context, repo, repositoryData, totalAssets, + packageConfig, isPullRequest, isPullRequest ? 2 : null, maxNewAssets, @@ -283,5 +414,6 @@ async function syncReleaseAssets(github, context, isPullRequest = false, maxNewA } module.exports = { + canDownloadNewAsset, syncReleaseAssets }; diff --git a/.github/workflows/sync-release-assets.yml b/.github/workflows/sync-release-assets.yml deleted file mode 100644 index 70769da2..00000000 --- a/.github/workflows/sync-release-assets.yml +++ /dev/null @@ -1,91 +0,0 @@ ---- -name: Sync Release Assets -permissions: {} - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - -env: - MAX_NEW_ASSETS: 50 # Set to 0 for unlimited, or any positive number to limit downloads per run - -on: - workflow_dispatch: - pull_request: - push: - branches: - - master - schedule: - - cron: '0 * * * *' - -jobs: - sync-assets: - permissions: - contents: read - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - token: ${{ secrets.GH_BOT_TOKEN }} - - - name: Create or checkout dist branch into dist directory - run: | - git fetch origin - if git rev-parse --verify origin/dist >/dev/null 2>&1; then - echo "Dist branch exists, checking it out" - git worktree add dist origin/dist - cd dist - git pull origin dist - else - echo "Dist branch doesn't exist, creating new one" - git worktree add --orphan dist - cd dist - git rm -rf . 2>/dev/null || true - fi - - - name: Get organization repositories and download assets - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 - with: - github-token: ${{ secrets.GH_BOT_TOKEN }} - script: | - // Import the sync assets with metadata module - const { syncAssetsWithMetadata } = require('./.github/scripts/sync-assets-with-metadata.js'); - - // Check if this is a pull request event - const isPullRequest = context.eventName === 'pull_request'; - - // Get the max new assets limit from environment - const maxNewAssets = parseInt(process.env.MAX_NEW_ASSETS) || 0; - - // Change working directory to dist for file operations - process.chdir('./dist'); - - // Run the asset synchronization and store metadata - await syncAssetsWithMetadata(github, context, isPullRequest, maxNewAssets); - - - name: Generate packages.json - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 - with: - github-token: ${{ secrets.GH_BOT_TOKEN }} - script: | - // Import the packages generation module - const { generatePackagesWithCleanup } = require('./.github/scripts/generate-packages-with-cleanup.js'); - - console.log('Generating packages.json from dist directory...'); - - // Change to dist directory - process.chdir('./dist'); - - // Generate packages.json with cleanup - generatePackagesWithCleanup('.'); - - - name: Commit and push changes - if: github.event_name != 'pull_request' - uses: actions-js/push@5a7cbd780d82c0c937b5977586e641b2fd94acc5 # v1.5 - with: - author_email: ${{ secrets.GH_BOT_EMAIL }} - author_name: ${{ vars.GH_BOT_NAME }} - branch: dist - directory: dist - github_token: ${{ secrets.GH_BOT_TOKEN }} - message: 'Update release assets - ${{ github.run_id }}' diff --git a/.github/workflows/update-pages.yml b/.github/workflows/update-pages.yml index 97d06342..2f7fb2d1 100644 --- a/.github/workflows/update-pages.yml +++ b/.github/workflows/update-pages.yml @@ -1,5 +1,5 @@ --- -name: Build GH-Pages +name: Update permissions: {} on: @@ -7,14 +7,19 @@ on: push: branches: - master + schedule: + - cron: '0 * * * *' workflow_dispatch: concurrency: group: "${{ github.workflow }}-${{ github.ref }}" cancel-in-progress: true +env: + MAX_NEW_ASSETS: 50 # Set to 0 for unlimited, or any positive number to limit downloads per run + jobs: - prep: + update: permissions: contents: read runs-on: ubuntu-latest @@ -22,17 +27,89 @@ jobs: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - name: Create or checkout dist branch into dist directory + run: | + if git ls-remote --exit-code --heads origin dist >/dev/null; then + echo "Dist branch exists, checking it out at depth 1" + git fetch --no-tags --depth=1 origin dist:refs/remotes/origin/dist + git worktree add dist origin/dist + else + status=$? + if [ "$status" -ne 2 ]; then + exit "$status" + fi + + echo "Dist branch doesn't exist, creating new one" + git worktree add --orphan dist + cd dist + git rm -rf . 2>/dev/null || true + fi + + - name: Get organization repositories and download assets + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + github-token: ${{ secrets.GH_BOT_TOKEN || github.token }} + script: | + // Import the sync assets with metadata module + const { syncAssetsWithMetadata } = require('./.github/scripts/sync-assets-with-metadata.js'); + + // Check if this is a pull request event + const isPullRequest = context.eventName === 'pull_request'; + + // Get the max new assets limit from environment + const maxNewAssets = parseInt(process.env.MAX_NEW_ASSETS) || 0; + + // Change working directory to dist for file operations + process.chdir('./dist'); + + // Run the asset synchronization and store metadata + await syncAssetsWithMetadata(github, context, isPullRequest, maxNewAssets); + + - name: Generate packages.json + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + github-token: ${{ secrets.GH_BOT_TOKEN || github.token }} + script: | + // Import the packages generation module + const { generatePackagesWithCleanup } = require('./.github/scripts/generate-packages-with-cleanup.js'); + + console.log('Generating packages.json from dist directory...'); + + // Change to dist directory + process.chdir('./dist'); + + // Generate packages.json with cleanup + generatePackagesWithCleanup('.'); + + - name: GitHub Commit & Push + if: github.event_name != 'pull_request' + uses: actions-js/push@5a7cbd780d82c0c937b5977586e641b2fd94acc5 # v1.5 + with: + author_email: ${{ secrets.GH_BOT_EMAIL }} + author_name: ${{ vars.GH_BOT_NAME }} + branch: dist + directory: dist + github_token: ${{ secrets.GH_BOT_TOKEN }} + message: 'Update release assets - ${{ github.run_id }}' + + - name: Prepare artifact + run: | + cd gh-pages-template + 7z a ../build.zip . + cd ../dist + 7z a ../build.zip . "-xr!.git" + - name: Upload artifact uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: prep - path: gh-pages-template/ + name: update + path: build.zip if-no-files-found: error include-hidden-files: true retention-days: 1 call-jekyll-build: - needs: prep + needs: update permissions: contents: read uses: LizardByte/LizardByte.github.io/.github/workflows/jekyll-build.yml@master @@ -41,6 +118,7 @@ jobs: GH_BOT_TOKEN: ${{ secrets.GH_BOT_TOKEN }} with: clean_gh_pages: true + extract_archive: 'build.zip' gh_bot_name: ${{ vars.GH_BOT_NAME }} - site_artifact: 'prep' + site_artifact: 'update' target_branch: 'gh-pages' diff --git a/README.md b/README.md index 5eb992ee..5e0f97d5 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # LizardByte Package Repository -This repository serves as a centralized storage for all packages and release assets from repositories within the -LizardByte organization. All release assets are automatically downloaded and organized in the `dist` branch for +This repository serves as a centralized package index and filtered release asset mirror for repositories within the +LizardByte organization. Configured release assets are automatically downloaded and organized in the `dist` branch for easy access and distribution. ## Structure @@ -34,18 +34,42 @@ dist/ ## Workflow -The release asset collection is handled by the `sync-release-assets.yml` workflow which: +The release asset collection and GitHub Pages deployment are handled by the `update-pages.yml` workflow. Asset +retention is controlled by `packages.config.json`. 1. Discovers all repositories in the LizardByte organization 2. Fetches release information for each repository -3. Downloads missing release assets -4. Generates hash files for integrity verification -5. Commits changes to the `dist` branch +3. Downloads configured release assets +4. Removes previously stored assets that are no longer configured +5. Generates hash files for integrity verification +6. Commits changes to the `dist` branch +7. Rebuilds GitHub Pages and publishes the filtered `dist` assets there too + +## Configuration + +`packages.config.json` controls which release assets are retained in `dist` and published to GitHub Pages. +`packages.json` is still generated from GitHub release metadata for all discovered assets. `defaultInclude` is +`false`, so repositories must opt in with asset patterns for files that should be mirrored. + +```json +{ + "defaultInclude": false, + "repositories": { + "Sunshine": { + "includeAssets": [ + "*-installer.exe", + "*-installer.msi" + ] + } + } +} +``` ## Usage -All release assets are available in the `dist` branch of this repository. You can: +Configured release assets are available in the `dist` branch of this repository. You can: - Browse assets directly on GitHub - Clone the `dist` branch for offline access +- Use GitHub Pages direct URLs that mirror the `dist` layout, e.g. `/packages///` - Use the hash files to verify asset integrity diff --git a/gh-pages-template/assets/js/app.js b/gh-pages-template/assets/js/app.js index 46f67ebf..b232cdd9 100644 --- a/gh-pages-template/assets/js/app.js +++ b/gh-pages-template/assets/js/app.js @@ -6,8 +6,7 @@ class RepositoryDataManager { constructor() { this.repositoryData = []; this.orgName = 'LizardByte'; // Organization name - this.distBranch = 'dist'; - this.rawBase = 'https://raw.githubusercontent.com'; + this.packagesPath = 'packages.json'; } /** @@ -17,8 +16,8 @@ class RepositoryDataManager { try { console.log('Fetching last successful workflow run...'); - // Fetch workflow runs for the sync-release-assets.yml workflow - const response = await fetch(`https://api.github.com/repos/${this.orgName}/packages/actions/workflows/sync-release-assets.yml/runs?event=schedule&status=success&branch=master&per_page=1`); + // Fetch workflow runs for the update-pages.yml workflow + const response = await fetch(`https://api.github.com/repos/${this.orgName}/packages/actions/workflows/update-pages.yml/runs?event=schedule&status=success&branch=master&per_page=1`); if (!response.ok) { console.warn(`Failed to fetch workflow runs: ${response.status}`); @@ -42,14 +41,13 @@ class RepositoryDataManager { } /** - * Load repository data from packages.json in the dist branch + * Load repository data from the packages.json published with GitHub Pages */ async loadRepositoryData() { try { console.log('Loading repository data from packages.json...'); - // Fetch the packages.json file from the dist branch - const response = await fetch(`${this.rawBase}/${this.orgName}/packages/${this.distBranch}/packages.json`); + const response = await fetch(this.packagesPath); if (!response.ok) { throw new Error(`Failed to fetch packages.json: ${response.status}`); diff --git a/packages.config.json b/packages.config.json new file mode 100644 index 00000000..4ab1bb17 --- /dev/null +++ b/packages.config.json @@ -0,0 +1,11 @@ +{ + "defaultInclude": false, + "repositories": { + "Sunshine": { + "includeAssets": [ + "*-installer.exe", + "*-installer.msi" + ] + } + } +}