diff --git a/architectures b/architectures deleted file mode 100644 index 1c542bf2e9..0000000000 --- a/architectures +++ /dev/null @@ -1,7 +0,0 @@ -bashbrew-arch variants -amd64 alpine3.22,alpine3.23,bookworm,bookworm-slim,bullseye,bullseye-slim,trixie,trixie-slim -arm32v6 alpine3.22,alpine3.23 -arm32v7 alpine3.22,alpine3.23,bookworm,bookworm-slim,bullseye,bullseye-slim,trixie,trixie-slim -arm64v8 alpine3.22,alpine3.23,bookworm,bookworm-slim,bullseye,bullseye-slim,trixie,trixie-slim -ppc64le bookworm,bookworm-slim,trixie,trixie-slim -s390x alpine3.22,alpine3.23,bookworm,bookworm-slim,trixie,trixie-slim diff --git a/config b/config deleted file mode 100644 index 0c4717b9f9..0000000000 --- a/config +++ /dev/null @@ -1,4 +0,0 @@ -baseuri https://nodejs.org/dist -default_variant bookworm -alpine_version 3.23 -debian_versions bookworm bullseye trixie diff --git a/functions.sh b/functions.sh deleted file mode 100755 index f19c5003bd..0000000000 --- a/functions.sh +++ /dev/null @@ -1,368 +0,0 @@ -#!/usr/bin/env bash -# -# Utility functions -# Don't change this file unless needed -# The GitHub Action for automating new builds rely on this file - -info() { - printf "%s\\n" "$@" -} - -fatal() { - printf "**********\\n" - printf "Fatal Error: %s\\n" "$@" - printf "**********\\n" - exit 1 -} - -# Get system architecture -# -# This is used to get the target architecture for docker image. -# For crossing building, we need a way to specify the target -# architecture manually. -function get_arch() { - local arch - case $(uname -m) in - x86_64) - arch="amd64" - ;; - ppc64le) - arch="ppc64le" - ;; - s390x) - arch="s390x" - ;; - aarch64 | arm64) - arch="arm64" - ;; - armv7l) - arch="arm32v7" - ;; - *) - echo "$0 does not support architecture ${arch:-unknown} ... aborting" - exit 1 - ;; - esac - - echo "${arch}" -} - -# Get corresponding variants based on the architecture. -# All supported variants of each supported architecture are listed in a -# file - 'architectures'. Its format is: -# ,... -# ,... -function get_variants() { - local dir - dir=${1:-.} - shift - - local arch - local availablevariants - local variantsfilter - local variants=() - - arch=$(get_arch) - variantsfilter=("$@") - IFS=' ' read -ra availablevariants <<< "$(grep "^${arch}" "${dir}/architectures" | sed -E 's/'"${arch}"'[[:space:]]*//' | sed -E 's/,/ /g')" - - if [ ${#variantsfilter[@]} -gt 0 ]; then - for variant1 in "${availablevariants[@]}"; do - for variant2 in "${variantsfilter[@]}"; do - if [ "${variant1}" = "${variant2}" ]; then - variants+=("${variant1}") - fi - done - done - - if [ ${#variants[@]} -gt 0 ]; then - echo "${variants[@]}" - fi - else - echo "${availablevariants[@]}" - fi -} - -# Get supported architectures for a specific version and variant -# -# Get default supported architectures from 'architectures'. Then go to the version folder -# to see if there is a local architectures file. The local architectures will override the -# default architectures. This will give us some benefits: -# - a specific version may or may not support some architectures -# - if there is no specialization for a version, just don't provide local architectures -function get_supported_arches() { - local version - local variant - local arches - local lines - local line - version="$1" - shift - variant="$1" - shift - - # Get default supported arches - lines=$(grep "${variant}" "$(dirname "${version}")"/architectures 2> /dev/null | cut -d' ' -f1) - - # Get version specific supported architectures if there is specialized information - if [ -e "${version}"/architectures ]; then - lines=$(grep "${variant}" "${version}"/architectures 2> /dev/null | cut -d' ' -f1) - fi - - while IFS='' read -r line; do - arches+=("${line}") - done <<< "${lines}" - - echo "${arches[@]}" -} - -# Get configuration values from the config file -# -# The configuration entries are simple key/value pairs which are whitespace separated. -function get_config() { - local dir - dir=${1:-.} - shift - - local name - name=${1} - shift - - local value - value=$(grep "^${name}" "${dir}/config" | sed -E 's/'"${name}"'[[:space:]]*//') - echo "${value}" -} - -# Get available versions for a given path -# -# The result is a list of valid versions. -# shellcheck disable=SC2120 -function get_versions() { - shift - - local versions=() - local dirs=("$@") - - local default_variant - default_variant=$(get_config "./" "default_variant") - if [ ${#dirs[@]} -eq 0 ]; then - IFS=' ' read -ra dirs <<< "$(echo "./"*/)" - fi - - for dir in "${dirs[@]}"; do - if [ -e "${dir}/Dockerfile" ] || [ -e "${dir}/${default_variant}/Dockerfile" ]; then - versions+=("${dir#./}") - fi - done - - if [ ${#versions[@]} -gt 0 ]; then - echo "${versions[@]%/}" - fi -} - -function is_alpine() { - local variant - variant=${1} - shift - - if [ "${variant}" = "${variant#alpine}" ]; then - return 1 - fi -} - -function is_debian() { - local variant - variant=$1 - shift - - IFS=' ' read -ra debianVersions <<< "$(get_config "./" "debian_versions")" - for d in "${debianVersions[@]}"; do - if [ "${d}" = "${variant}" ]; then - return 0 - fi - done - return 1 -} - -function is_debian_slim() { - local variant - variant=$1 - shift - - IFS=' ' read -ra debianVersions <<< "$(get_config "./" "debian_versions")" - for d in "${debianVersions[@]}"; do - if [ "${d}-slim" = "${variant}" ]; then - return 0 - fi - done - return 1 -} - -function get_fork_name() { - local version - version=$1 - shift - - IFS='/' read -ra versionparts <<< "${version}" - if [ ${#versionparts[@]} -gt 1 ]; then - echo "${versionparts[0]}" - fi -} - -function get_full_tag() { - local variant - local tag - local full_tag - variant="$1" - shift - tag="$1" - shift - if [ -z "${variant}" ]; then - full_tag="${tag}" - elif [ "${variant}" = "default" ]; then - full_tag="${tag}" - else - full_tag="${tag}-${variant}" - fi - echo "${full_tag}" -} - -function get_full_version() { - local version - version=$1 - shift - - local default_dockerfile - if [ -f "${version}/${default_variant}/Dockerfile" ]; then - default_dockerfile="${version}/${default_variant}/Dockerfile" - else - default_dockerfile="${version}/Dockerfile" - fi - - grep -m1 'ENV NODE_VERSION=' "${default_dockerfile}" | cut -d= -f2 -} - -function get_major_minor_version() { - local version - version=$1 - shift - - local fullversion - fullversion=$(get_full_version "${version}") - - echo "$(echo "${fullversion}" | cut -d'.' -f1).$(echo "${fullversion}" | cut -d'.' -f2)" -} - -function get_path() { - local version - local variant - local path - version="$1" - shift - variant="$1" - shift - - if [ -z "${variant}" ]; then - path="${version}/${variant}" - elif [ "${variant}" = "default" ]; then - path="${version}" - else - path="${version}/${variant}" - fi - echo "${path}" -} - -function get_tag() { - local version - version=$1 - shift - - local versiontype - versiontype=${1:-full} - shift - - local tagversion - if [ "${versiontype}" = full ]; then - tagversion=$(get_full_version "${version}") - elif [ "${versiontype}" = majorminor ]; then - tagversion=$(get_major_minor_version "${version}") - fi - - local tagparts - IFS=' ' read -ra tagparts <<< "$(get_fork_name "${version}") ${tagversion}" - IFS='-' - echo "${tagparts[*]}" - unset IFS -} - -function sort_versions() { - local versions=("$@") - local sorted - local lines - local line - - IFS=$'\n' - lines="${versions[*]}" - unset IFS - - while IFS='' read -r line; do - sorted+=("${line}") - done <<< "$(echo "${lines}" | grep "^[0-9]" | sort -r)" - - while IFS='' read -r line; do - sorted+=("${line}") - done <<< "$(echo "${lines}" | grep -v "^[0-9]" | sort -r)" - - echo "${sorted[@]}" -} - -function commit_range() { - local commit_id_end=${1} - shift - local commit_id_start=${1} - - if [ -z "${commit_id_start}" ]; then - if [ -z "${commit_id_end}" ]; then - echo "HEAD~1..HEAD" - elif [[ "${commit_id_end}" =~ .. ]]; then - echo "${commit_id_end}" - else - echo "${commit_id_end}~1..${commit_id_end}" - fi - else - echo "${commit_id_end}..${commit_id_start}" - fi -} - -function images_updated() { - local commit_range - local versions - local images_changed - - commit_range="$(commit_range "$@")" - - IFS=' ' read -ra versions <<< "$( - IFS=',' - get_versions - )" - images_changed=$(git diff --name-only "${commit_range}" "${versions[@]}") - - if [ -z "${images_changed}" ]; then - return 1 - fi - return 0 -} - -function tests_updated() { - local commit_range - local test_changed - - commit_range="$(commit_range "$@")" - - test_changed=$(git diff --name-only "${commit_range}" test*) - - if [ -z "${test_changed}" ]; then - return 1 - fi - return 0 -} diff --git a/genMatrix.js b/genMatrix.js index dc76724328..88f23afd81 100644 --- a/genMatrix.js +++ b/genMatrix.js @@ -1,31 +1,19 @@ 'use strict'; const path = require('path'); -const fs = require('fs'); +const { getAllDockerfiles, getDockerfileNodeVersion } = require('./utils'); const testFiles = [ 'genMatrix.js', '.github/workflows/build-test.yml', + 'update.js', + 'updateLib.js', + 'utils.js', + 'versions.json', ]; -const nodeDirRegex = /^\d+$/; - const areTestFilesChanged = (changedFiles) => changedFiles .some((file) => testFiles.includes(file)); -// Returns a list of the child directories in the given path -const getChildDirectories = (parent) => fs.readdirSync(parent, { withFileTypes: true }) - .filter((dirent) => dirent.isDirectory()) - .map(({ name }) => path.resolve(parent, name)); - -const getNodeVersionDirs = (base) => getChildDirectories(base) - .filter((childPath) => nodeDirRegex.test(path.basename(childPath))); - -// Returns the paths of Dockerfiles that are at: base/*/Dockerfile -const getDockerfilesInChildDirs = (base) => getChildDirectories(base) - .map((childDir) => path.resolve(childDir, 'Dockerfile')); - -const getAllDockerfiles = (base) => getNodeVersionDirs(base).flatMap(getDockerfilesInChildDirs); - const getAffectedDockerfiles = (filesAdded, filesModified, filesRenamed) => { const files = [ ...filesAdded, @@ -52,13 +40,10 @@ const getAffectedDockerfiles = (filesAdded, filesModified, filesRenamed) => { ]; }; -const getFullNodeVersionFromDockerfile = (file) => fs.readFileSync(file, 'utf8') - .match(/^ENV NODE_VERSION=(\d*\.*\d*\.\d*)/m)[1]; - const getDockerfileMatrixEntry = (file) => { const [variant] = path.dirname(file).split(path.sep).slice(-1); - const version = getFullNodeVersionFromDockerfile(file); + const version = getDockerfileNodeVersion(file); return { version, diff --git a/update.js b/update.js new file mode 100755 index 0000000000..086146fe89 --- /dev/null +++ b/update.js @@ -0,0 +1,62 @@ +#!/usr/bin/env node +'use strict'; +const update = require('./updateLib'); + +const usage = ` + Update the node docker images. + + Usage: + ./update.js [ OPTIONS ] + + OPTIONS: + -h, --help\tthis message + -a, --all\tupdate all images even if no node version update`; + +const printUsage = () => { + console.log(usage); +}; + +const runUpdate = async (updateAll) => { + const updated = await update(updateAll); + + updated.forEach(({ file }) => { + console.log('Updated', file); + }); + + if (!updated.length) { + console.log('Nothing updated'); + } +}; + +const main = async () => { + if (process.argv.length > 3) { + printUsage(); + process.exit(1); + } + + if (process.argv.length === 2) { + await runUpdate(false); + return; + } + + switch (process.argv[2]) { + case '-a': + case '--all': + await runUpdate(true); + return; + + case '-h': + case '--help': + printUsage(); + return; + + default: + printUsage(); + process.exit(1); + } +}; + +main().catch((e) => { + console.error(e); + process.exit(1); +}); diff --git a/update.sh b/update.sh deleted file mode 100755 index 5d2d884ae8..0000000000 --- a/update.sh +++ /dev/null @@ -1,244 +0,0 @@ -#!/usr/bin/env bash - -set -ue - -function usage() { - cat << EOF - - Update the node docker images. - - Usage: - $0 [-s] [MAJOR_VERSION(S)] [VARIANT(S)] - - Examples: - - update.sh # Update all images - - update.sh -s # Update all images, skip updating Alpine if the musl build is unavailable - - update.sh 22,24 # Update all variants of version 22 and 24 - - update.sh -s 24 # Update all variants of version 24, except skip updating Alpine if the musl build is unavailable - - update.sh 24 alpine3.22,alpine3.23 # Update only alpine3.22 & alpine3.23 variants for version 24 - - update.sh . trixie,trixie-slim # Update only trixie & trixie-slim Debian variants for all versions - - OPTIONS: - -s Security update; allows Debian updates even if musl build for Alpine is unavailable - -h Show this message - -EOF -} - -SKIP_ALPINE=false -while getopts "sh" opt; do - case "${opt}" in - s) - SKIP_ALPINE=true - shift - ;; - h) - usage - exit - ;; - \?) - usage - exit - ;; - esac -done - -. functions.sh - -cd "$(cd "${0%/*}" && pwd -P)" - -IFS=',' read -ra versions_arg <<< "${1:-}" -IFS=',' read -ra variant_arg <<< "${2:-}" - -IFS=' ' read -ra versions <<< "$(get_versions .)" -IFS=' ' read -ra update_versions <<< "$(get_versions . "${versions_arg[@]:-}")" -IFS=' ' read -ra update_variants <<< "$(get_variants . "${variant_arg[@]:-}")" -if [ ${#versions[@]} -eq 0 ]; then - fatal "No valid versions found!" -fi - -# Global variables -# Get architecture and use this as target architecture for docker image -# See details in function.sh -# TODO: Should be able to specify target architecture manually -arch=$(get_arch) - -function in_versions_to_update() { - local version=$1 - - if [ "${#update_versions[@]}" -eq 0 ]; then - echo 0 - return - fi - - for version_to_update in "${update_versions[@]}"; do - if [ "${version_to_update}" = "${version}" ]; then - echo 0 - return - fi - done - - echo 1 -} - -function in_variants_to_update() { - local variant=$1 - - if [ "${#update_variants[@]}" -eq 0 ]; then - echo 0 - return - fi - - for variant_to_update in "${update_variants[@]}"; do - if [ "${variant_to_update}" = "${variant}" ]; then - echo 0 - return - fi - done - - echo 1 -} - -function update_node_version() { - - local baseuri=${1} - shift - local version=${1} - shift - local template=${1} - shift - local dockerfile=${1} - shift - local variant="" - if [ $# -eq 1 ]; then - variant=${1} - shift - fi - - fullVersion="$(curl -sSL --compressed "${baseuri}" | grep ' /dev/null; then - echo "${dockerfile} is already up to date!" - else - echo "${dockerfile} updated!" - fi - - # Required for POSIX sed - if [ -f "${dockerfile}-tmp-e" ]; then - rm "${dockerfile}-tmp-e" - fi - - # Guard the move because Alpine sometimes will be missing - if [ -f "${dockerfile}-tmp" ]; then - mv -f "${dockerfile}-tmp" "${dockerfile}" - fi - ) -} - -pids=() - -for version in "${versions[@]}"; do - parentpath=$(dirname "${version}") - versionnum=$(basename "${version}") - baseuri=$(get_config "${parentpath}" "baseuri") - update_version=$(in_versions_to_update "${version}") - - [ "${update_version}" -eq 0 ] && info "Updating version ${version}..." - - # Get supported variants according the target architecture - # See details in function.sh - IFS=' ' read -ra variants <<< "$(get_variants "${parentpath}")" - - if [ -f "${version}/Dockerfile" ]; then - if [ "${update_version}" -eq 0 ]; then - update_node_version "${baseuri}" "${versionnum}" "${parentpath}/Dockerfile.template" "${version}/Dockerfile" & - pids+=($!) - fi - fi - - for variant in "${variants[@]}"; do - # Skip non-docker directories - [ -f "${version}/${variant}/Dockerfile" ] || continue - - update_variant=$(in_variants_to_update "${variant}") - template_file="${parentpath}/Dockerfile-${variant}.template" - - if is_debian "${variant}"; then - template_file="${parentpath}/Dockerfile-debian.template" - elif is_debian_slim "${variant}"; then - template_file="${parentpath}/Dockerfile-slim.template" - elif is_alpine "${variant}"; then - template_file="${parentpath}/Dockerfile-alpine.template" - fi - - cp "${parentpath}/docker-entrypoint.sh" "${version}/${variant}/docker-entrypoint.sh" - if [ "${update_version}" -eq 0 ] && [ "${update_variant}" -eq 0 ]; then - update_node_version "${baseuri}" "${versionnum}" "${template_file}" "${version}/${variant}/Dockerfile" "${variant}" & - pids+=($!) - fi - done -done - -# The reason we explicitly wait on each pid is so the return status of this script is set properly -# if one of the jobs fails. If we just called "wait", the exit status would always be 0 -for pid in "${pids[@]}"; do - wait "$pid" -done - -info "Done!" diff --git a/updateLib.js b/updateLib.js new file mode 100644 index 0000000000..a67623bf7c --- /dev/null +++ b/updateLib.js @@ -0,0 +1,141 @@ +'use strict'; +const path = require('path'); +const { readFileSync, writeFileSync } = require('fs'); +const { getAllDockerfiles, getDockerfileNodeVersion } = require('./utils'); + +const templates = Object.freeze({ + alpine: 1, + debian: 2, + debianSlim: 3, +}); + +const templateFileMap = Object.freeze({ + [templates.alpine]: 'Dockerfile-alpine.template', + [templates.debian]: 'Dockerfile-debian.template', + [templates.debianSlim]: 'Dockerfile-slim.template', +}); + +const templateRepoMap = Object.freeze({ + [templates.alpine]: 'alpine', + [templates.debian]: 'buildpack-deps', + [templates.debianSlim]: 'debian', +}); + +// nodeVersions is sorted +const getLatestNodeVersion = (nodeVersions, majorVersion) => nodeVersions + .find((version) => version.startsWith(`${majorVersion}.`)); + +const getTemplate = (variant) => { + if (variant.startsWith('alpine')) { + return templates.alpine; + } + + if (variant.endsWith('-slim')) { + return templates.debianSlim; + } + + return templates.debian; +}; + +const getDockerfileMetadata = (nodeVersions, file) => { + const [nodeMajorVersion, variant] = path.dirname(file).split(path.sep).slice(-2); + const fileNodeVersion = getDockerfileNodeVersion(file); + + return { + file, + variant, + fileNodeVersion, + nodeMajorVersion, + latestVersion: getLatestNodeVersion(nodeVersions, nodeMajorVersion), + template: getTemplate(variant), + }; +}; + +const isDockerfileOutdated = ({ fileNodeVersion, latestVersion }) => fileNodeVersion + !== latestVersion; + +const fetchLatestNodeVersions = async () => { + const nodeDist = await fetch('https://nodejs.org/dist/index.json'); + const content = await nodeDist.json(); + return content.map(({ version }) => version.substring(1)); +}; + +const findOutdated = async (updateAll) => { + const nodeVersions = await fetchLatestNodeVersions(); + + const dockerfileMetadatas = getAllDockerfiles(__dirname) + .map((file) => getDockerfileMetadata(nodeVersions, file)); + + return updateAll + ? dockerfileMetadatas + : dockerfileMetadatas.filter(isDockerfileOutdated); +}; + +const getKeys = (basename) => readFileSync(path.resolve(__dirname, 'keys', basename)) + .toString().trim().split('\n'); + +const readTemplate = (template) => readFileSync( + path.resolve(__dirname, templateFileMap[template]), +).toString(); + +const getBaseImage = ({ template, variant }) => { + const tag = template === templates.alpine + ? variant.replace(/alpine/, '') + : variant; + + return `${templateRepoMap[template]}:${tag}`; +}; + +const formatKeys = (keys) => keys.map((key) => `$1${key} \\`).join('\n'); + +const formatTemplate = (nodeKeys, muslChecksum, base, metadata) => { + const { latestVersion, template, nodeMajorVersion } = metadata; + const baseImage = getBaseImage(metadata); + let initialFormat = base.replace(/^FROM.+$/m, `FROM ${baseImage}`) + .replace(/^ENV NODE_VERSION=.+$/m, `ENV NODE_VERSION=${latestVersion}`) + .replace(/^(\s*)"\${NODE_KEYS\[@]}".*$/m, formatKeys(nodeKeys)) + + if (parseInt(nodeMajorVersion, 10) >= 26) { + initialFormat = initialFormat.replace(/ENV YARN_VERSION.*\*\n/s, ''); + } + + if (template !== templates.alpine) { + return initialFormat; + } + else { + return initialFormat.replace(/CHECKSUM=CHECKSUM_x64/m, `CHECKSUM="${muslChecksum}"`); + } +}; + +const fetchMuslChecksum = async (nodeVersion) => { + const checksums = await fetch( + `https://unofficial-builds.nodejs.org/download/release/v${nodeVersion}/SHASUMS256.txt`, + ); + const content = await checksums.text(); + return await content.match(/(\S+)\s+\S+-linux-x64-musl.tar.xz/m)[1]; +}; + +const updateDockerfile = async (nodeKeys, metadata) => { + const { file, template, latestVersion } = metadata; + const base = readTemplate(template); + const muslChecksum = await fetchMuslChecksum(latestVersion); + + const formatted = formatTemplate(nodeKeys, muslChecksum, base, metadata); + writeFileSync(file, formatted); +}; + +const updateDockerfiles = async (outdated) => { + const nodeKeys = getKeys('node.keys'); + + await Promise.all( + outdated.map((metadata) => updateDockerfile(nodeKeys, metadata)), + ); +}; + +const update = async (updateAll) => { + const outdated = await findOutdated(updateAll); + await updateDockerfiles(outdated); + return outdated; +}; + +module.exports = update; diff --git a/utils.js b/utils.js new file mode 100644 index 0000000000..1ccaf60aeb --- /dev/null +++ b/utils.js @@ -0,0 +1,27 @@ +'use strict'; +const path = require('path'); +const { readFileSync, readdirSync } = require('fs'); + +const nodeDirRegex = /^\d+$/; + +// Returns a list of the child directories in the given path +const getChildDirectories = (parent) => readdirSync(parent, { withFileTypes: true }) + .filter((dirent) => dirent.isDirectory()) + .map(({ name }) => path.resolve(parent, name)); + +const getNodeVersionDirs = (base) => getChildDirectories(base) + .filter((childPath) => nodeDirRegex.test(path.basename(childPath))); + +// Returns the paths of Dockerfiles that are at: base/*/Dockerfile +const getDockerfilesInChildDirs = (base) => getChildDirectories(base) + .map((childDir) => path.resolve(childDir, 'Dockerfile')); + +const getAllDockerfiles = (base) => getNodeVersionDirs(base).flatMap(getDockerfilesInChildDirs); + +const getDockerfileNodeVersion = (file) => readFileSync(file, 'utf8') + .match(/^ENV NODE_VERSION=(\d*\.*\d*\.\d*)/m)[1]; + +module.exports = { + getAllDockerfiles, + getDockerfileNodeVersion, +};