From 2d48b179f7eed326bfd7fa7e3203944a6df0d5cb Mon Sep 17 00:00:00 2001 From: vltbaudbot <264001112+vltbaudbot@users.noreply.github.com> Date: Thu, 14 May 2026 00:39:40 +0000 Subject: [PATCH 1/3] feat: use vlt registry as default for all package manager benchmarks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Switch all non-registry benchmark variations to use the vlt registry (https://registry.vlt.io/vlt/npm/) as the default package registry instead of the public npm registry. This serves two purposes: 1. The vlt registry should outperform the default npm registry, so total benchmark suite time should decrease. 2. Running the full benchmark suite against the vlt registry provides continuous load/performance testing to keep us honest as usage grows. Registry benchmark variations (registry-clean, registry-lockfile) are NOT affected — they continue to compare registries as before. Changes: - New `scripts/setup-vlt-registry.sh`: writes vlt registry URL + auth token to project-level .npmrc. No-op when VLT_REGISTRY_AUTH_TOKEN is not set (graceful fallback to npm registry for local development). - `scripts/variations/common.sh`: each PM's setup now calls the registry script before install: - npm, yarn classic, pnpm, pacquet, bun, deno, aube, nx, turbo, vp, node: configured via .npmrc - berry, zpm: configured via npmRegistryServer/npmAuthToken in .yarnrc.yml - vlt: configured via --registry CLI flag + VLT_REGISTRY/VLT_TOKEN env vars (vlt doesn't read .npmrc for registry config) - `scripts/clean-helpers.sh`: clean_all() now calls clean_npmrc() to strip registry/auth lines between runs, preventing stale config. --- scripts/clean-helpers.sh | 1 + scripts/setup-vlt-registry.sh | 32 +++++++++++++++ scripts/variations/common.sh | 74 +++++++++++++++++++++++++++++------ 3 files changed, 95 insertions(+), 12 deletions(-) create mode 100755 scripts/setup-vlt-registry.sh diff --git a/scripts/clean-helpers.sh b/scripts/clean-helpers.sh index b67640da60..8971d24a74 100644 --- a/scripts/clean-helpers.sh +++ b/scripts/clean-helpers.sh @@ -226,6 +226,7 @@ clean_all() { clean_all_cache clean_build_files clean_build_output + clean_npmrc echo "Cleanup completed successfully!" } diff --git a/scripts/setup-vlt-registry.sh b/scripts/setup-vlt-registry.sh new file mode 100755 index 0000000000..7d5d3cf5b9 --- /dev/null +++ b/scripts/setup-vlt-registry.sh @@ -0,0 +1,32 @@ +#!/bin/bash +# Write vlt registry config to .npmrc (project-level). +# Preserves existing non-registry lines (e.g. engine-strict=true). +# No-op when VLT_REGISTRY_AUTH_TOKEN is not available. +# +# Usage: bash setup-vlt-registry.sh +# +# Required env vars: +# VLT_REGISTRY_AUTH_TOKEN — auth token for the vlt registry +# +# The script is idempotent: it strips any previous registry/auth lines +# before appending the vlt registry config. + +set -euo pipefail + +REGISTRY_URL="https://registry.vlt.io/vlt/npm/" +REGISTRY_NPMRC_KEY="${REGISTRY_URL#http*://}" + +if [ -z "${VLT_REGISTRY_AUTH_TOKEN:-}" ]; then + exit 0 +fi + +# Strip any previous registry/auth lines, then append ours +if [ -f ".npmrc" ]; then + awk '!/^[[:space:]]*(registry=|\/\/)/ { print }' ".npmrc" > ".npmrc.tmp" + mv ".npmrc.tmp" ".npmrc" +fi + +{ + echo "registry=${REGISTRY_URL}" + echo "//${REGISTRY_NPMRC_KEY}:_authToken=\${VLT_REGISTRY_AUTH_TOKEN}" +} >> ".npmrc" diff --git a/scripts/variations/common.sh b/scripts/variations/common.sh index 86f19092d7..5f47c5a192 100644 --- a/scripts/variations/common.sh +++ b/scripts/variations/common.sh @@ -56,31 +56,73 @@ for pm in npm yarn berry zpm pnpm pacquet vlt bun deno aube nx turbo vp node; do fi done BENCH_OUTPUT_FOLDER="$BENCH_RESULTS/$BENCH_FIXTURE/$BENCH_VARIATION" + +# --------------------------------------------------------------------------- +# Default registry: vlt registry +# All non-registry benchmark variations use the vlt registry instead of the +# public npm registry. This requires VLT_REGISTRY_AUTH_TOKEN to be set. +# When the token is absent (e.g. local development) the default npm registry +# is used as a fallback. +# --------------------------------------------------------------------------- +BENCH_DEFAULT_REGISTRY_URL="https://registry.vlt.io/vlt/npm/" +BENCH_DEFAULT_REGISTRY_NPMRC_KEY="${BENCH_DEFAULT_REGISTRY_URL#http*://}" + +if [ -n "${VLT_REGISTRY_AUTH_TOKEN:-}" ]; then + BENCH_USE_VLT_REGISTRY=1 + echo "Using vlt registry as default: $BENCH_DEFAULT_REGISTRY_URL" +else + BENCH_USE_VLT_REGISTRY="" + echo "Warning: VLT_REGISTRY_AUTH_TOKEN not set — falling back to default npm registry" +fi + +# Helper script that writes vlt registry + auth to .npmrc. +# No-op when VLT_REGISTRY_AUTH_TOKEN is not set. +# This is a standalone script (not a function) so it works in hyperfine's +# sh -c subshells without requiring bash or export -f. +BENCH_SETUP_VLT_REGISTRY="bash $BENCH_SCRIPTS/setup-vlt-registry.sh" + BENCH_COMMAND_YARN_MODERN_CONFIG=$(cat < .yarnrc.yml" BENCH_SETUP_ZPM="echo \"$BENCH_COMMAND_YARN_MODERN_CONFIG\" > .yarnrc.yml; { echo '[zpm prepare]'; echo 'cwd:'; pwd; echo 'package.json:'; ls -la package.json || true; echo 'yarn path:'; command -v yarn || true; echo 'yarn version:'; yarn -v || true; echo 'canary version:'; echo \"$BENCH_ZPM_VERSION\"; echo 'packageManager (before):'; npm pkg get packageManager || true; echo 'set packageManager=yarn@'"$BENCH_ZPM_VERSION"':' ; npm pkg set packageManager=\"yarn@$BENCH_ZPM_VERSION\" || true; echo 'packageManager (after):'; npm pkg get packageManager || true; } >> $BENCH_OUTPUT_FOLDER/zpm-prepare.log 2>&1" -BENCH_SETUP_PNPM="npm pkg delete packageManager >/dev/null 2>&1 || true" +BENCH_SETUP_PNPM="$BENCH_SETUP_VLT_REGISTRY; npm pkg delete packageManager >/dev/null 2>&1 || true" -BENCH_SETUP_PACQUET="npm pkg delete packageManager >/dev/null 2>&1 || true" +BENCH_SETUP_PACQUET="$BENCH_SETUP_VLT_REGISTRY; npm pkg delete packageManager >/dev/null 2>&1 || true" BENCH_SETUP_VLT="node $BENCH_SCRIPTS/add-workspace-protocol.js . >> $BENCH_OUTPUT_FOLDER/vlt-prepare.log 2>&1" -BENCH_SETUP_BUN="" -BENCH_SETUP_DENO="" -BENCH_SETUP_AUBE="npm pkg delete packageManager >/dev/null 2>&1 || true" -BENCH_SETUP_NX="" -BENCH_SETUP_TURBO="" -BENCH_SETUP_VP="" -BENCH_SETUP_NODE="" +BENCH_SETUP_BUN="$BENCH_SETUP_VLT_REGISTRY" +BENCH_SETUP_DENO="$BENCH_SETUP_VLT_REGISTRY" +BENCH_SETUP_AUBE="$BENCH_SETUP_VLT_REGISTRY; npm pkg delete packageManager >/dev/null 2>&1 || true" +BENCH_SETUP_NX="$BENCH_SETUP_VLT_REGISTRY" +BENCH_SETUP_TURBO="$BENCH_SETUP_VLT_REGISTRY" +BENCH_SETUP_VP="$BENCH_SETUP_VLT_REGISTRY" +BENCH_SETUP_NODE="$BENCH_SETUP_VLT_REGISTRY" # Bare install commands (no log redirection) — used by strace process counting. # Install scripts are disabled where the PM runs them by default, so benchmarks @@ -99,7 +141,15 @@ BENCH_INSTALL_BERRY="corepack yarn@latest install" BENCH_INSTALL_ZPM="yarn install --silent" BENCH_INSTALL_PNPM="corepack pnpm@latest install --ignore-scripts --silent" BENCH_INSTALL_PACQUET="pacquet install" -BENCH_INSTALL_VLT="vlt install --view=silent" +# vlt reads its own config (not .npmrc), so pass --registry on the CLI. +# Auth is handled via VLT_REGISTRY + VLT_TOKEN env vars exported below. +if [ -n "$BENCH_USE_VLT_REGISTRY" ]; then + export VLT_REGISTRY="$BENCH_DEFAULT_REGISTRY_URL" + export VLT_TOKEN="$VLT_REGISTRY_AUTH_TOKEN" + BENCH_INSTALL_VLT="vlt install --registry=$BENCH_DEFAULT_REGISTRY_URL --view=silent" +else + BENCH_INSTALL_VLT="vlt install --view=silent" +fi BENCH_INSTALL_BUN="bun install --ignore-scripts --silent" BENCH_INSTALL_DENO="deno install --quiet" BENCH_INSTALL_AUBE="aube install --silent" From c25f1539279ae252168ca7392dfc53e424391b35 Mon Sep 17 00:00:00 2001 From: vltbaudbot <264001112+vltbaudbot@users.noreply.github.com> Date: Thu, 14 May 2026 00:54:48 +0000 Subject: [PATCH 2/3] fix: read registry URL from VLT_REGISTRY_URL env var instead of hardcoding The registry URL must match/be tied to the auth token since all calls to the vlt registry are authenticated. Reading both VLT_REGISTRY_URL and VLT_REGISTRY_AUTH_TOKEN from environment (configured as GitHub secrets) ensures they always stay in sync. - Updated setup-vlt-registry.sh to read VLT_REGISTRY_URL from env - Updated common.sh to use VLT_REGISTRY_URL env var - Added VLT_REGISTRY_URL to workflow env vars (from secrets) - Both URL and token must be set for vlt registry to activate; if either is missing, falls back to default npm registry with a diagnostic warning --- .github/workflows/benchmark.yaml | 1 + scripts/setup-vlt-registry.sh | 13 ++++++++----- scripts/variations/common.sh | 22 ++++++++++++++++------ 3 files changed, 25 insertions(+), 11 deletions(-) diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml index 19cc78748d..96c0fcfde8 100644 --- a/.github/workflows/benchmark.yaml +++ b/.github/workflows/benchmark.yaml @@ -151,6 +151,7 @@ jobs: - name: Run Benchmarks variations env: CODEARTIFACT_AUTH_TOKEN: ${{ steps.aws.outputs.token }} + VLT_REGISTRY_URL: ${{ secrets.VLT_REGISTRY_URL }} VLT_REGISTRY_AUTH_TOKEN: ${{ secrets.VLT_REGISTRY_AUTH_TOKEN }} GH_REGISTRY: ${{ secrets.GH_REGISTRY }} GH_AUTH_TOKEN: ${{ secrets.GH_AUTH_TOKEN }} diff --git a/scripts/setup-vlt-registry.sh b/scripts/setup-vlt-registry.sh index 7d5d3cf5b9..56b54f17bf 100755 --- a/scripts/setup-vlt-registry.sh +++ b/scripts/setup-vlt-registry.sh @@ -1,25 +1,28 @@ #!/bin/bash # Write vlt registry config to .npmrc (project-level). # Preserves existing non-registry lines (e.g. engine-strict=true). -# No-op when VLT_REGISTRY_AUTH_TOKEN is not available. +# No-op when VLT_REGISTRY_URL or VLT_REGISTRY_AUTH_TOKEN is not available. # # Usage: bash setup-vlt-registry.sh # # Required env vars: -# VLT_REGISTRY_AUTH_TOKEN — auth token for the vlt registry +# VLT_REGISTRY_URL — registry URL (e.g. https://registry.vlt.io/vlt/npm/) +# VLT_REGISTRY_AUTH_TOKEN — auth token for the vlt registry # # The script is idempotent: it strips any previous registry/auth lines # before appending the vlt registry config. set -euo pipefail -REGISTRY_URL="https://registry.vlt.io/vlt/npm/" -REGISTRY_NPMRC_KEY="${REGISTRY_URL#http*://}" +REGISTRY_URL="${VLT_REGISTRY_URL:-}" +AUTH_TOKEN="${VLT_REGISTRY_AUTH_TOKEN:-}" -if [ -z "${VLT_REGISTRY_AUTH_TOKEN:-}" ]; then +if [ -z "$REGISTRY_URL" ] || [ -z "$AUTH_TOKEN" ]; then exit 0 fi +REGISTRY_NPMRC_KEY="${REGISTRY_URL#http*://}" + # Strip any previous registry/auth lines, then append ours if [ -f ".npmrc" ]; then awk '!/^[[:space:]]*(registry=|\/\/)/ { print }' ".npmrc" > ".npmrc.tmp" diff --git a/scripts/variations/common.sh b/scripts/variations/common.sh index 5f47c5a192..fc5499f79d 100644 --- a/scripts/variations/common.sh +++ b/scripts/variations/common.sh @@ -60,19 +60,29 @@ BENCH_OUTPUT_FOLDER="$BENCH_RESULTS/$BENCH_FIXTURE/$BENCH_VARIATION" # --------------------------------------------------------------------------- # Default registry: vlt registry # All non-registry benchmark variations use the vlt registry instead of the -# public npm registry. This requires VLT_REGISTRY_AUTH_TOKEN to be set. -# When the token is absent (e.g. local development) the default npm registry +# public npm registry. This requires both VLT_REGISTRY_URL and +# VLT_REGISTRY_AUTH_TOKEN to be set (configured as GitHub secrets). +# The URL and token must match — the token authenticates requests to the +# specific registry URL. +# When either is absent (e.g. local development) the default npm registry # is used as a fallback. # --------------------------------------------------------------------------- -BENCH_DEFAULT_REGISTRY_URL="https://registry.vlt.io/vlt/npm/" -BENCH_DEFAULT_REGISTRY_NPMRC_KEY="${BENCH_DEFAULT_REGISTRY_URL#http*://}" +BENCH_DEFAULT_REGISTRY_URL="${VLT_REGISTRY_URL:-}" -if [ -n "${VLT_REGISTRY_AUTH_TOKEN:-}" ]; then +if [ -n "${VLT_REGISTRY_AUTH_TOKEN:-}" ] && [ -n "$BENCH_DEFAULT_REGISTRY_URL" ]; then BENCH_USE_VLT_REGISTRY=1 + BENCH_DEFAULT_REGISTRY_NPMRC_KEY="${BENCH_DEFAULT_REGISTRY_URL#http*://}" echo "Using vlt registry as default: $BENCH_DEFAULT_REGISTRY_URL" else BENCH_USE_VLT_REGISTRY="" - echo "Warning: VLT_REGISTRY_AUTH_TOKEN not set — falling back to default npm registry" + BENCH_DEFAULT_REGISTRY_NPMRC_KEY="" + if [ -n "${VLT_REGISTRY_AUTH_TOKEN:-}" ] && [ -z "$BENCH_DEFAULT_REGISTRY_URL" ]; then + echo "Warning: VLT_REGISTRY_AUTH_TOKEN is set but VLT_REGISTRY_URL is not — falling back to default npm registry" + elif [ -z "${VLT_REGISTRY_AUTH_TOKEN:-}" ] && [ -n "$BENCH_DEFAULT_REGISTRY_URL" ]; then + echo "Warning: VLT_REGISTRY_URL is set but VLT_REGISTRY_AUTH_TOKEN is not — falling back to default npm registry" + else + echo "Warning: VLT_REGISTRY_URL and VLT_REGISTRY_AUTH_TOKEN not set — falling back to default npm registry" + fi fi # Helper script that writes vlt registry + auth to .npmrc. From 34f36372afe9e820a5904ec824c726744c4e4075 Mon Sep 17 00:00:00 2001 From: vltbaudbot <264001112+vltbaudbot@users.noreply.github.com> Date: Thu, 14 May 2026 01:03:39 +0000 Subject: [PATCH 3/3] fix: make aube installation non-fatal in setup.sh MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit aube@1.13.0 (published 2026-05-13) has a broken arm64 binary — the @endevco/aube-linux-arm64 platform package declares bin/aubr but the file doesn't exist in the tarball, causing npm install to fail with ENOENT during preinstall. Rather than pinning to a specific version, make the aube install non-fatal so a broken aube release doesn't block the entire benchmark suite (all 41+ jobs). If aube fails to install, a warning is logged and the suite continues benchmarking all other package managers. --- scripts/setup.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/scripts/setup.sh b/scripts/setup.sh index a0528cc8ed..9923a18fd9 100644 --- a/scripts/setup.sh +++ b/scripts/setup.sh @@ -50,8 +50,12 @@ npm install -g npm@latest corepack@latest vlt@latest bun@latest deno@latest nx@l # Install Vite+ (vp) via npm (available as the `vite-plus` package) npm install -g vite-plus@latest -# Install aube via npm (available as the `@endevco/aube` package) -npm install -g @endevco/aube@latest +# Install aube via npm (available as the `@endevco/aube` package). +# Non-fatal: aube may not have a working binary for all platforms (e.g. arm64). +# If it fails to install, the benchmark suite continues without aube. +if ! npm install -g @endevco/aube@latest 2>/dev/null; then + echo "Warning: aube installation failed (may not support this platform) — skipping aube benchmarks" +fi # Configure Package Managers echo "Configuring package managers..."