diff --git a/feature-versions/state.json b/feature-versions/state.json index f10e955c..3025cd7e 100644 --- a/feature-versions/state.json +++ b/feature-versions/state.json @@ -88,5 +88,10 @@ "tag": "4", "installed": "sha256:0cb4a78491b6e62ee8a9bf4fbeacbd15b5013d19bc420591b05383a696315e60", "filter": "startupscript\\/butane\\/050-parse-devcontainer\\.sh" + }, + "gcr.io/go-containerregistry/crane": { + "tag": "latest", + "installed": "sha256:d3a706262093746258f20107ab4e95536f9d6d45c8c3f3acf6b02b1801b440d6", + "filter": "startupscript\\/butane\\/prefetch-oci-features\\.sh" } } diff --git a/startupscript/butane/050-parse-devcontainer.sh b/startupscript/butane/050-parse-devcontainer.sh index 012b8f6a..c5053f0f 100755 --- a/startupscript/butane/050-parse-devcontainer.sh +++ b/startupscript/butane/050-parse-devcontainer.sh @@ -78,6 +78,9 @@ if [[ -d "${DEVCONTAINER_FEATURES_PATH}" ]]; then rsync -a --ignore-existing "${DEVCONTAINER_FEATURES_PATH}/" "${DEVCONTAINER_PATH}/.devcontainer/features" fi +/home/core/prefetch-oci-features.sh "${DEVCONTAINER_CONFIG_PATH}" || \ + echo "WARNING: prefetch-oci-features.sh failed, continuing with remote features" >&2 + replace_template_options() { local TEMPLATE_PATH="$1" diff --git a/startupscript/butane/prefetch-oci-features.sh b/startupscript/butane/prefetch-oci-features.sh new file mode 100755 index 00000000..7878fe37 --- /dev/null +++ b/startupscript/butane/prefetch-oci-features.sh @@ -0,0 +1,182 @@ +#!/bin/bash + +# prefetch-oci-features.sh pre-downloads OCI devcontainer features to local +# directories, then rewrites devcontainer.json to use local paths. This avoids +# ghcr.io rate limits during devcontainer up. + +set -o errexit +set -o nounset +set -o pipefail +set -o xtrace + +readonly CRANE_IMAGE="gcr.io/go-containerregistry/crane@sha256:d3a706262093746258f20107ab4e95536f9d6d45c8c3f3acf6b02b1801b440d6" + +crane() { + docker run --rm "${CRANE_IMAGE}" "$@" +} + +readonly MAX_RETRIES=3 +readonly RETRY_DELAY=5 + +retry() { + local attempt + for attempt in $(seq 1 "${MAX_RETRIES}"); do + if "$@"; then + return 0 + fi + if [[ "${attempt}" -lt "${MAX_RETRIES}" ]]; then + echo "Attempt ${attempt}/${MAX_RETRIES} failed, retrying in ${RETRY_DELAY}s..." >&2 + sleep "${RETRY_DELAY}" + fi + done + return 1 +} + +if [[ $# -lt 1 ]]; then + echo "Usage: $0 " >&2 + exit 1 +fi + +readonly CONFIG_PATH="$1" + +# Derive the project root from the config path. +# .devcontainer.json is at the project root; .devcontainer/devcontainer.json is one level down. +CONFIG_DIR="$(cd "$(dirname "${CONFIG_PATH}")" && pwd)" +if [[ "$(basename "${CONFIG_DIR}")" == ".devcontainer" ]]; then + PROJECT_DIR="$(dirname "${CONFIG_DIR}")" +else + PROJECT_DIR="${CONFIG_DIR}" +fi +readonly PROJECT_DIR +readonly FEATURES_DIR="${PROJECT_DIR}/.devcontainer/features" + +# ref_to_dir_name converts an OCI feature reference to a namespaced directory +# name, avoiding collisions with local features (e.g., features/src/java). +# ghcr.io/devcontainers/features/java@sha256:... -> devcontainers-features-java +# ghcr.io/rocker-org/devcontainer-features/r-packages:1 -> rocker-org-devcontainer-features-r-packages +ref_to_dir_name() { + local ref="$1" + ref_to_resource "${ref}" \ + | sed 's|^[^/]*/||' \ + | tr '/' '-' +} + +# ref_to_resource extracts the registry/path portion without the version or digest. +# ghcr.io/devcontainers/features/java@sha256:... -> ghcr.io/devcontainers/features/java +# ghcr.io/devcontainers/features/java:1 -> ghcr.io/devcontainers/features/java +ref_to_resource() { + local ref="$1" + echo "${ref}" | sed 's/@sha256:.*//' | sed 's/:[^/]*$//' +} + +# Extract ghcr.io feature references that are JSON keys (followed by ":"). +FEATURE_REFS=() +while IFS= read -r ref; do + FEATURE_REFS+=("${ref}") +done < <(grep -o '"ghcr\.io/[^"]*"[[:space:]]*:' "${CONFIG_PATH}" | sed 's/[[:space:]]*:$//' | tr -d '"') + +if [[ ${#FEATURE_REFS[@]} -eq 0 ]]; then + echo "No OCI feature references found in ${CONFIG_PATH}" + exit 0 +fi + +mkdir -p "${FEATURES_DIR}" + +# Track which OCI refs were successfully prefetched, for installsAfter rewriting. +declare -A PREFETCHED_REFS # OCI resource -> local path + +for FEATURE_REF in "${FEATURE_REFS[@]}"; do + DIR_NAME="$(ref_to_dir_name "${FEATURE_REF}")" + LOCAL_DIR="${FEATURES_DIR}/${DIR_NAME}" + RESOURCE="$(ref_to_resource "${FEATURE_REF}")" + + if [[ -d "${LOCAL_DIR}" ]] && [[ -f "${LOCAL_DIR}/devcontainer-feature.json" ]]; then + echo "Already exists: ${LOCAL_DIR}, skipping download" + PREFETCHED_REFS["${RESOURCE}"]="./.devcontainer/features/${DIR_NAME}" + continue + fi + + echo "Prefetching ${FEATURE_REF} -> ${LOCAL_DIR}" + + # For tag-based refs (no @sha256:), resolve to a digest first. + RESOLVED_REF="${FEATURE_REF}" + if [[ "${FEATURE_REF}" != *"@sha256:"* ]]; then + DIGEST="$(retry crane digest "${FEATURE_REF}")" || { + echo "WARNING: Failed to resolve digest for ${FEATURE_REF}, skipping" >&2 + continue + } + RESOLVED_REF="${RESOURCE}@${DIGEST}" + fi + + MANIFEST="$(retry crane manifest "${RESOLVED_REF}")" || { + echo "WARNING: Failed to fetch manifest for ${RESOLVED_REF}, skipping" >&2 + continue + } + + LAYER_DIGEST="$(echo "${MANIFEST}" | jq -r '.layers[0].digest')" + if [[ -z "${LAYER_DIGEST}" ]] || [[ "${LAYER_DIGEST}" == "null" ]]; then + echo "WARNING: No layer found in manifest for ${RESOLVED_REF}, skipping" >&2 + continue + fi + + BLOB_TAR="$(mktemp)" + download_blob() { + crane blob "${RESOURCE}@${LAYER_DIGEST}" > "${BLOB_TAR}" + } + retry download_blob || { + echo "WARNING: Failed to download blob for ${RESOLVED_REF}, skipping" >&2 + rm -f "${BLOB_TAR}" + continue + } + + mkdir -p "${LOCAL_DIR}" + tar -xf "${BLOB_TAR}" -C "${LOCAL_DIR}" || { + echo "WARNING: Failed to extract blob for ${RESOLVED_REF}, skipping" >&2 + rm -f "${BLOB_TAR}" + rm -rf "${LOCAL_DIR}" + continue + } + rm -f "${BLOB_TAR}" + + if [[ ! -f "${LOCAL_DIR}/devcontainer-feature.json" ]]; then + echo "WARNING: Downloaded feature missing devcontainer-feature.json, skipping" >&2 + rm -rf "${LOCAL_DIR}" + continue + fi + + PREFETCHED_REFS["${RESOURCE}"]="./.devcontainer/features/${DIR_NAME}" + echo "Prefetched ${FEATURE_REF} -> ${LOCAL_DIR}" +done + +if [[ ${#PREFETCHED_REFS[@]} -eq 0 ]]; then + echo "No features were prefetched" + exit 0 +fi + +# Rewrite devcontainer.json: replace OCI refs with local paths. +for FEATURE_REF in "${FEATURE_REFS[@]}"; do + RESOURCE="$(ref_to_resource "${FEATURE_REF}")" + LOCAL_PATH="${PREFETCHED_REFS[${RESOURCE}]:-}" + if [[ -z "${LOCAL_PATH}" ]]; then + continue + fi + + # shellcheck disable=SC2016 # backslash-ampersand is intentional sed replacement syntax + ESCAPED_REF="$(printf '%s' "${FEATURE_REF}" | sed 's/[[\.*^$()+?{|]/\\&/g')" + sed -i "s|\"${ESCAPED_REF}\"|\"${LOCAL_PATH}\"|g" "${CONFIG_PATH}" +done + +# Rewrite installsAfter in all features, not just prefetched ones. Local features +# like workbench-tools may reference OCI features we've prefetched. +for FEATURE_JSON in "${FEATURES_DIR}"/*/devcontainer-feature.json; do + if [[ ! -f "${FEATURE_JSON}" ]]; then + continue + fi + + for INSTALL_AFTER_RESOURCE in "${!PREFETCHED_REFS[@]}"; do + INSTALL_AFTER_LOCAL="${PREFETCHED_REFS[${INSTALL_AFTER_RESOURCE}]}" + sed -i "s|\"${INSTALL_AFTER_RESOURCE}\"|\"${INSTALL_AFTER_LOCAL}\"|g" "${FEATURE_JSON}" + done +done + +echo "Prefetched ${#PREFETCHED_REFS[@]} of ${#FEATURE_REFS[@]} OCI features" diff --git a/tests/common/build.sh b/tests/common/build.sh index f5c5ef0b..132f78f0 100755 --- a/tests/common/build.sh +++ b/tests/common/build.sh @@ -58,6 +58,16 @@ popd mkdir -p "${SRC_DIR}/.devcontainer/features" rsync -a --ignore-existing "features/src/" "${SRC_DIR}/.devcontainer/features" +############################ +# Prefetch OCI features +############################ +PREFETCH_SCRIPT="./startupscript/butane/prefetch-oci-features.sh" +if [[ -f "${SRC_DIR}/.devcontainer.json" ]]; then + "${PREFETCH_SCRIPT}" "${SRC_DIR}/.devcontainer.json" +elif [[ -f "${SRC_DIR}/.devcontainer/devcontainer.json" ]]; then + "${PREFETCH_SCRIPT}" "${SRC_DIR}/.devcontainer/devcontainer.json" +fi + ############################ # Install Devcontainer CLI ############################