Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 0 additions & 45 deletions .github/scripts/generate-packages-with-cleanup.js

This file was deleted.

63 changes: 56 additions & 7 deletions .github/scripts/generate-packages.js
Original file line number Diff line number Diff line change
@@ -1,6 +1,15 @@
const fs = require('fs');
const path = require('path');

/**
* Encode path segments for a browser URL without changing the dist directory layout.
* @param {...string} segments - URL path segments.
* @returns {string} Relative URL for GitHub Pages.
*/
function encodeDirectAssetUrl(...segments) {
return segments.map(segment => encodeURIComponent(segment)).join('/');
}

/**
* Build packages.json data with stats from repository entries.
* @param {Array} repositories - Repository data to include.
Expand All @@ -13,10 +22,12 @@ function buildPackagesData(repositories) {
// Calculate totals
const totalReleases = repositories.reduce((sum, repo) => sum + repo.releases.length, 0);
const totalAssets = repositories.reduce((sum, repo) =>
sum + repo.releases.reduce((releaseSum, release) => releaseSum + release.assetCount, 0), 0
sum + repo.releases.reduce((releaseSum, release) =>
releaseSum + (release.assetCount || (Array.isArray(release.assets) ? release.assets.length : 0)), 0), 0
);

const packagesData = {
generatedAt: new Date().toISOString(),
repositories: repositories,
stats: {
totalRepositories: repositories.length,
Expand All @@ -43,17 +54,40 @@ function normalizeRepositoryMetadata(repositoryMetadata = []) {
.filter(repo => repo && Array.isArray(repo.releases) && repo.releases.length > 0)
.map(repo => {
const releases = repo.releases
.filter(release => release && release.assetCount > 0)
.map(release => ({
tag: release.tag,
assetCount: release.assetCount
}));
.filter(release => release && (release.assetCount > 0 || (Array.isArray(release.assets) && release.assets.length > 0)))
.map(release => {
const assets = Array.isArray(release.assets)
? release.assets
.filter(asset => asset && asset.name)
.map(asset => ({
name: asset.name,
size: asset.size,
githubUrl: asset.githubUrl || asset.browserDownloadUrl,
directUrl: asset.directUrl
}))
: [];

const releaseData = {
tag: release.tag,
url: release.url,
publishedAt: release.publishedAt,
assetCount: release.assetCount || assets.length
};

if (assets.length > 0) {
assets.sort((a, b) => a.name.localeCompare(b.name, undefined, { numeric: true, sensitivity: 'base' }));
releaseData.assets = assets;
}

return releaseData;
});

releases.sort((a, b) => b.tag.localeCompare(a.tag, undefined, { numeric: true, sensitivity: 'base' }));

return {
name: repo.name,
archived: Boolean(repo.archived),
url: repo.url,
releases
};
})
Expand Down Expand Up @@ -186,9 +220,23 @@ function scanReleaseDirectory(releasePath) {
});

if (assetFiles.length > 0) {
const assets = assetFiles
.map(assetFile => {
const assetPath = path.join(releasePath, assetFile.name);
const repoName = path.basename(path.dirname(releasePath));

return {
name: assetFile.name,
size: fs.statSync(assetPath).size,
directUrl: encodeDirectAssetUrl(repoName, releaseTag, assetFile.name)
};
})
.sort((a, b) => a.name.localeCompare(b.name, undefined, { numeric: true, sensitivity: 'base' }));

return {
tag: releaseTag,
assetCount: assetFiles.length
assetCount: assetFiles.length,
assets
};
}

Expand Down Expand Up @@ -218,6 +266,7 @@ function writePackagesJson(packagesData, outputPath = './packages.json') {

module.exports = {
buildPackagesData,
encodeDirectAssetUrl,
generatePackagesJson,
normalizeRepositoryMetadata,
scanRepositoryDirectory,
Expand Down
21 changes: 15 additions & 6 deletions .github/scripts/sync-assets-with-metadata.js
Original file line number Diff line number Diff line change
@@ -1,8 +1,11 @@
const fs = require('fs');
const { syncReleaseAssets } = require('./sync-assets.js');
const { buildPackagesData, writePackagesJson } = require('./generate-packages.js');

const LEGACY_METADATA_PATH = 'repo-metadata.json';

/**
* Main function to sync assets and store repository metadata
* Main function to sync assets and write the package index.
* @param {Object} github - GitHub API client
* @param {Object} context - GitHub Actions context
* @param {boolean} isPullRequest - Whether this is a pull request event
Expand All @@ -15,14 +18,20 @@ async function syncAssetsWithMetadata(github, context, isPullRequest = false, ma
// Run the asset synchronization
const repositoryData = await syncReleaseAssets(github, context, isPullRequest, maxNewAssets);

// Store repository data for use in next step
const metadataPath = 'repo-metadata.json';
fs.writeFileSync(metadataPath, JSON.stringify(repositoryData, null, 2));
// Store the public package index directly. This replaces the old
// repo-metadata.json handoff so GitHub Pages only needs one JSON file.
const packagesData = buildPackagesData(repositoryData);
writePackagesJson(packagesData, 'packages.json');

if (fs.existsSync(LEGACY_METADATA_PATH)) {
fs.unlinkSync(LEGACY_METADATA_PATH);
console.log(`Removed legacy metadata file: ${LEGACY_METADATA_PATH}`);
}

console.log(`Stored metadata for ${repositoryData.length} repositories in ${metadataPath}`);
console.log(`Stored package index for ${repositoryData.length} repositories in packages.json`);
console.log('Asset synchronization completed successfully');

return repositoryData;
return packagesData;

} catch (error) {
console.error('Error during asset synchronization:', error);
Expand Down
80 changes: 58 additions & 22 deletions .github/scripts/sync-assets.js
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ const { ensureDir, fileExists } = require('./file-utils');
const { generateHashFiles } = require('./hash-utils');
const { downloadAssetWithRetry } = require('./download-utils');
const { loadPackageConfig, shouldIncludeAsset } = require('./package-config');
const { cleanupNonVPrefixedReleases } = require('./cleanup-releases');

/**
* Check whether a filename is one of the generated hash sidecar files.
Expand Down Expand Up @@ -68,6 +69,36 @@ function canDownloadNewAsset(maxNewAssets, currentNewAssets, releaseNewAssets) {
return maxNewAssets <= 0 || (currentNewAssets + releaseNewAssets) < maxNewAssets;
}

/**
* Encode path segments for a browser URL without changing the dist directory layout.
* @param {...string} segments - URL path segments.
* @returns {string} Relative URL for GitHub Pages.
*/
function encodeDirectAssetUrl(...segments) {
return segments.map(segment => encodeURIComponent(segment)).join('/');
}

/**
* Build public metadata for a GitHub release asset.
* @param {string} repoName - Repository name.
* @param {string} releaseTag - Release tag.
* @param {Object} asset - GitHub release asset API response object.
* @returns {Object} Package index asset metadata.
*/
function buildAssetMetadata(repoName, releaseTag, asset) {
const assetData = {
name: asset.name,
size: asset.size,
githubUrl: asset.browser_download_url
};

if (fileExists(path.join(repoName, releaseTag, asset.name))) {
assetData.directUrl = encodeDirectAssetUrl(repoName, releaseTag, asset.name);
}

return assetData;
}

/**
* Remove stored files from dist that are no longer allowed by packages.config.json.
* This only affects mirrored files; release metadata is still generated from GitHub.
Expand Down Expand Up @@ -176,6 +207,7 @@ async function processRepository(github, context, repo, repositoryData, totalAss
const repoData = {
name: repo.name,
archived: repo.archived,
url: repo.html_url,
releases: []
};

Expand All @@ -196,7 +228,10 @@ async function processRepository(github, context, repo, repositoryData, totalAss
if (assetCount > 0) {
repoData.releases.push({
tag: release.tag_name,
assetCount: assetCount
url: release.html_url,
publishedAt: release.published_at,
assetCount: assetCount,
assets: result.assets
});
processedReleasesWithAssets++;
}
Expand All @@ -220,46 +255,46 @@ async function processRepository(github, context, repo, repositoryData, totalAss
* @param {Object} packageConfig - Loaded package config.
* @param {number} maxNewAssets - Maximum new assets to download.
* @param {number} currentNewAssets - Number of new assets already downloaded.
* @returns {Promise<{assetCount: number, newAssets: number}>}
* @returns {Promise<{assetCount: number, newAssets: number, assets: Array}>}
*/
async function processRelease(repoName, release, packageConfig, maxNewAssets = 0, currentNewAssets = 0) {
console.log(`Processing release: ${release.tag_name}`);

if (release.assets.length === 0) {
console.log(`No assets found for release ${release.tag_name}`);
return { assetCount: 0, newAssets: 0 };
return { assetCount: 0, newAssets: 0, assets: [] };
}

const assetCount = release.assets.length;
const includedAssets = release.assets.filter(asset => shouldIncludeAsset(packageConfig, repoName, asset.name));
let newAssets = 0;

if (includedAssets.length === 0) {
console.log(`No configured assets found for release ${release.tag_name}`);
return { assetCount, newAssets: 0 };
}

// Create directory structure
const releaseDir = path.join(repoName, release.tag_name);
ensureDir(releaseDir);

let newAssets = 0;

for (const asset of includedAssets) {
// Check if we've reached the new assets download limit
if (!canDownloadNewAsset(maxNewAssets, currentNewAssets, newAssets)) {
console.log(`Reached maximum new assets limit (${maxNewAssets}) for this run. Stopping asset processing for release ${release.tag_name}.`);
break;
}
} else {
// Create directory structure
const releaseDir = path.join(repoName, release.tag_name);
ensureDir(releaseDir);

for (const asset of includedAssets) {
// Check if we've reached the new assets download limit
if (!canDownloadNewAsset(maxNewAssets, currentNewAssets, newAssets)) {
console.log(`Reached maximum new assets limit (${maxNewAssets}) for this run. Stopping asset processing for release ${release.tag_name}.`);
break;
}

const result = await processAsset(releaseDir, asset);
if (result.downloaded) {
if (result.isNew) {
const result = await processAsset(releaseDir, asset);
if (result.downloaded && result.isNew) {
newAssets++;
}
}
}

return { assetCount, newAssets };
const assets = release.assets
.map(asset => buildAssetMetadata(repoName, release.tag_name, asset))
.sort((a, b) => a.name.localeCompare(b.name, undefined, { numeric: true, sensitivity: 'base' }));

return { assetCount, newAssets, assets };
}

/**
Expand Down Expand Up @@ -371,6 +406,7 @@ async function syncReleaseAssets(github, context, isPullRequest = false, maxNewA
console.log(`Found ${repos.length} repositories`);

const packageConfig = loadPackageConfig(process.cwd());
cleanupNonVPrefixedReleases('.');
cleanupStoredAssets('.', packageConfig);

const repositoryData = [];
Expand Down
24 changes: 5 additions & 19 deletions .github/workflows/update-pages.yml
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ jobs:
git init --initial-branch=dist
git remote add origin "https://github.com/${GITHUB_REPOSITORY}.git"

- name: Get organization repositories and download assets
- name: Get organization repositories, download assets, and update package index
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
with:
github-token: ${{ secrets.GH_BOT_TOKEN || github.token }}
Expand All @@ -75,35 +75,21 @@ jobs:
// Change working directory to dist for file operations
process.chdir('./dist');

// Run the asset synchronization and store metadata
// Run the asset synchronization and write packages.json
await syncAssetsWithMetadata(github, context, isPullRequest, maxNewAssets);

- name: Generate packages.json
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
with:
github-token: ${{ secrets.GH_BOT_TOKEN || github.token }}
script: |
// Import the packages generation module
const { generatePackagesWithCleanup } = require('./.github/scripts/generate-packages-with-cleanup.js');

console.log('Generating packages.json from dist directory...');

// Change to dist directory
process.chdir('./dist');

// Generate packages.json with cleanup
generatePackagesWithCleanup('.');

- name: GitHub Commit & Push
if: github.event_name != 'pull_request'
uses: actions-js/push@5a7cbd780d82c0c937b5977586e641b2fd94acc5 # v1.5
with:
amend: ${{ steps.dist_branch.outputs.exists == 'true' }}
author_email: ${{ secrets.GH_BOT_EMAIL }}
author_name: ${{ vars.GH_BOT_NAME }}
branch: dist
directory: dist
force: ${{ steps.dist_branch.outputs.exists == 'true' }}
github_token: ${{ secrets.GH_BOT_TOKEN }}
message: 'Update release assets - ${{ github.run_id }}'
message: 'Update release assets'

- name: Prepare artifact
run: |
Expand Down
Loading