diff --git a/package-lock.json b/package-lock.json index 2a384090..9b96f908 100644 --- a/package-lock.json +++ b/package-lock.json @@ -196,6 +196,7 @@ "integrity": "sha512-CGOfOJqWjg2qW/Mb6zNsDm+u5vFQ8DxXfbM09z69p5Z6+mE1ikP2jUXw+j42Pf1XTYED2Rni5f95npYeuwMDQA==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@babel/code-frame": "^7.29.0", "@babel/generator": "^7.29.0", @@ -2212,6 +2213,34 @@ "node": ">=v18" } }, + "node_modules/@commitlint/load/node_modules/cosmiconfig": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/cosmiconfig/-/cosmiconfig-9.0.0.tgz", + "integrity": "sha512-itvL5h8RETACmOTFc4UfIyB2RfEHi71Ax6E/PivVxq9NseKbOWpeyHEOIbmAw1rs8Ak0VursQNww7lf7YtUwzg==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "env-paths": "^2.2.1", + "import-fresh": "^3.3.0", + "js-yaml": "^4.1.0", + "parse-json": "^5.2.0" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/d-fischer" + }, + "peerDependencies": { + "typescript": ">=4.9.5" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": true + } + } + }, "node_modules/@commitlint/load/node_modules/cosmiconfig-typescript-loader": { "version": "6.1.0", "resolved": "https://registry.npmjs.org/cosmiconfig-typescript-loader/-/cosmiconfig-typescript-loader-6.1.0.tgz", @@ -2653,6 +2682,7 @@ "integrity": "sha512-Tdfx4eH2uS+gv9V9NCr3Rz+c7RSS6ntXp3Blliud18ibRUlRxO9dTaOjG4iv4x0nAmMeedP1ORkEpeXSkh2QiQ==", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=20" } @@ -2734,7 +2764,8 @@ "resolved": "https://registry.npmjs.org/@cspell/dict-css/-/dict-css-4.0.19.tgz", "integrity": "sha512-VYHtPnZt/Zd/ATbW3rtexWpBnHUohUrQOHff/2JBhsVgxOrksAxJnLAO43Q1ayLJBJUUwNVo+RU0sx0aaysZfg==", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/@cspell/dict-dart": { "version": "2.3.2", @@ -2874,14 +2905,16 @@ "resolved": "https://registry.npmjs.org/@cspell/dict-html/-/dict-html-4.0.14.tgz", "integrity": "sha512-2bf7n+kS92g+cMKV0wr9o/Oq9n8JzU7CcrB96gIh2GHgnF+0xDOqO2W/1KeFAqOfqosoOVE48t+4dnEMkkoJ2Q==", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/@cspell/dict-html-symbol-entities": { "version": "4.0.5", "resolved": "https://registry.npmjs.org/@cspell/dict-html-symbol-entities/-/dict-html-symbol-entities-4.0.5.tgz", "integrity": "sha512-429alTD4cE0FIwpMucvSN35Ld87HCyuM8mF731KU5Rm4Je2SG6hmVx7nkBsLyrmH3sQukTcr1GaiZsiEg8svPA==", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/@cspell/dict-java": { "version": "5.0.12", @@ -3079,7 +3112,8 @@ "resolved": "https://registry.npmjs.org/@cspell/dict-typescript/-/dict-typescript-3.2.3.tgz", "integrity": "sha512-zXh1wYsNljQZfWWdSPYwQhpwiuW0KPW1dSd8idjMRvSD0aSvWWHoWlrMsmZeRl4qM4QCEAjua8+cjflm41cQBg==", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/@cspell/dict-vue": { "version": "3.0.5", @@ -4290,6 +4324,7 @@ "integrity": "sha512-+owLCBBdfpgL3HU+BD5etr1SvbXpSitJK0is1kiYjJxAAJggYMRQz5hSdd5pq1sSggfxPbw2ld71pt4x5wwViA==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@jest/environment": "30.3.0", "@jest/expect": "30.3.0", @@ -4809,6 +4844,7 @@ "integrity": "sha512-DhGl4xMVFGVIyMwswXeyzdL4uXD5OGILGX5N8Y+f6W7LhC1Ze2poSNrkF/fedpVDHEEZ+PHFW0vL14I+mm8K3Q==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@octokit/auth-token": "^6.0.0", "@octokit/graphql": "^9.0.3", @@ -6276,7 +6312,8 @@ "version": "7.0.15", "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz", "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==", - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/@types/json5": { "version": "0.0.29", @@ -6358,6 +6395,7 @@ "resolved": "https://registry.npmjs.org/@types/node/-/node-24.10.1.tgz", "integrity": "sha512-GNWcUTRBgIRJD5zj+Tq0fKOJ5XZajIiBroOF0yvj2bSU1WvNdYS/dn9UxwsujGW4JX06dnHyjV2y9rRaybH0iQ==", "license": "MIT", + "peer": true, "dependencies": { "undici-types": "~7.16.0" } @@ -6524,6 +6562,7 @@ "integrity": "sha512-XZzOmihLIr8AD1b9hL9ccNMzEMWt/dE2u7NyTY9jJG6YNiNthaD5XtUHVF2uCXZ15ng+z2hT3MVuxnUYhq6k1g==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@typescript-eslint/scope-manager": "8.57.0", "@typescript-eslint/types": "8.57.0", @@ -7092,6 +7131,7 @@ "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", "dev": true, "license": "MIT", + "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -8120,6 +8160,7 @@ } ], "license": "MIT", + "peer": true, "dependencies": { "baseline-browser-mapping": "^2.9.0", "caniuse-lite": "^1.0.30001759", @@ -10840,6 +10881,7 @@ "integrity": "sha512-LEyamqS7W5HB3ujJyvi0HQK/dtVINZvd5mAAp9eT5S/ujByGjiZLCzPcHVzuXbpJDJF/cxwHlfceVUDZ2lnSTw==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.8.0", "@eslint-community/regexpp": "^4.12.1", @@ -10900,6 +10942,7 @@ "integrity": "sha512-82GZUjRS0p/jganf6q1rEO25VSoHH0hKPCTrgillPjdI/3bgBhAE1QzHrHTizjpRvy6pGAvKjDJtk2pF9NDq8w==", "dev": true, "license": "MIT", + "peer": true, "bin": { "eslint-config-prettier": "bin/cli.js" }, @@ -11026,6 +11069,7 @@ "integrity": "sha512-whOE1HFo/qJDyX4SnXzP4N6zOWn79WhnCUY/iDR0mPfQZO8wcYE4JClzI2oZrhBnnMUCBCHZhO6VQyoBU95mZA==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@rtsao/scc": "^1.1.0", "array-includes": "^3.1.9", @@ -11560,6 +11604,7 @@ "resolved": "https://registry.npmjs.org/express/-/express-5.2.1.tgz", "integrity": "sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw==", "license": "MIT", + "peer": true, "dependencies": { "accepts": "^2.0.0", "body-parser": "^2.2.1", @@ -11646,6 +11691,7 @@ "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.17.1.tgz", "integrity": "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==", "license": "MIT", + "peer": true, "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", @@ -14322,6 +14368,7 @@ "integrity": "sha512-AkXIIFcaazymvey2i/+F94XRnM6TsVLZDhBMLsd1Sf/W0wzsvvpjeyUrCZD6HGG4SDYPgDJDBKeiJTBb10WzMg==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@jest/core": "30.3.0", "@jest/types": "30.3.0", @@ -15200,6 +15247,7 @@ "resolved": "https://registry.npmjs.org/jsep/-/jsep-1.4.0.tgz", "integrity": "sha512-B7qPcEVE3NVkmSJbaYxvv4cHkVW7DQsZz13pUMrfS8z8Q/BuShN+gcTXrUlPiGqM2/t/EEaI030bpxMqY8gMlw==", "license": "MIT", + "peer": true, "engines": { "node": ">= 10.16.0" } @@ -16560,6 +16608,7 @@ "integrity": "sha512-8dD6FusOQSrpv9Z1rdNMdlSgQOIP880DHqnohobOmYLElGEqAL/JvxvuxZO16r4HtjTlfPRDC1hbvxC9dPN2nA==", "dev": true, "license": "MIT", + "peer": true, "bin": { "marked": "bin/marked.js" }, @@ -19818,6 +19867,7 @@ "dev": true, "inBundle": true, "license": "MIT", + "peer": true, "engines": { "node": ">=12" }, @@ -21075,6 +21125,7 @@ "integrity": "sha512-UOnG6LftzbdaHZcKoPFtOcCKztrQ57WkHDeRD9t/PTQtmT0NHSeWWepj6pS0z/N7+08BHFDQVUrfmfMRcZwbMg==", "dev": true, "license": "MIT", + "peer": true, "bin": { "prettier": "bin/prettier.cjs" }, @@ -22194,6 +22245,7 @@ "integrity": "sha512-WRgl5GcypwramYX4HV+eQGzUbD7UUbljVmS+5G1uMwX/wLgYuJAxGeerXJDMO2xshng4+FXqCgyB5QfClV6WjA==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@semantic-release/commit-analyzer": "^13.0.1", "@semantic-release/error": "^4.0.0", @@ -24339,6 +24391,7 @@ "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=12" }, @@ -24561,6 +24614,7 @@ "integrity": "sha512-f0FFpIdcHgn8zcPSbf1dRevwt047YMnaiJM3u2w2RewrB+fob/zePZcrOyQoLMMO7aBIddLcQIEK5dYjkLnGrQ==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@cspotcode/source-map-support": "^0.8.0", "@tsconfig/node10": "^1.0.7", @@ -24825,6 +24879,7 @@ "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", "dev": true, "license": "Apache-2.0", + "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -25004,6 +25059,7 @@ "dev": true, "hasInstallScript": true, "license": "MIT", + "peer": true, "dependencies": { "napi-postinstall": "^0.3.0" }, @@ -25436,6 +25492,7 @@ "version": "7.5.7", "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.7.tgz", "integrity": "sha512-KMvVuFzpKBuiIXW3E4u3mySRO2/mCHSyZDJQM5NQ9Q9KHWHWh0NHgfbRMLLrceUK5qAL4ytALJbpRMjixFZh8A==", + "peer": true, "engines": { "node": ">=8.3.0" }, diff --git a/src/api/v2/catalogs/refresh.ts b/src/api/v2/catalogs/refresh.ts new file mode 100644 index 00000000..7b30c27f --- /dev/null +++ b/src/api/v2/catalogs/refresh.ts @@ -0,0 +1,22 @@ +import Debug from 'debug' +import { Response } from 'express' +import { BadRequestError } from 'src/error' +import { OpenApiRequestExt } from 'src/otomi-models' + +const debug = Debug('otomi:api:v2:catalogs:refresh') + +/** + * POST /v2/catalogs/refresh + * Refresh BYO catalog cache(s) immediately + */ +export const refreshAplCatalogCache = async (req: OpenApiRequestExt, res: Response): Promise => { + const catalogId = typeof req.query.catalogId === 'string' ? decodeURIComponent(req.query.catalogId) : undefined + debug(`refreshAplCatalogCache(${catalogId || 'all'})`) + try { + await req.otomi.refreshBYOCatalogCache(catalogId) + res.status(200).json({ code: 200, message: 'Successfully refreshed catalog cache' }) + } catch (e) { + debug('refreshAplCatalogCache failed', e) + res.status(400).json(new BadRequestError('Failed to refresh catalog cache')) + } +} diff --git a/src/app.ts b/src/app.ts index 7d98f311..f6ec68da 100644 --- a/src/app.ts +++ b/src/app.ts @@ -24,6 +24,7 @@ import { AplResponseObject, OpenAPIDoc, Schema, SealedSecretManifestResponse } f import { default as OtomiStack } from 'src/otomi-stack' import { extract, getPaths, getValuesSchema } from 'src/utils' import { + CATALOG_CACHE_REFRESH_INTERVAL_MS, CHECK_LATEST_COMMIT_INTERVAL, cleanEnv, EXPRESS_PAYLOAD_LIMIT, @@ -35,6 +36,7 @@ import getLatestRemoteCommitSha from './git/connect' import { getBuildStatus, getSealedSecretStatus, getServiceStatus, getWorkloadStatus } from './k8s_operations' const env = cleanEnv({ + CATALOG_CACHE_REFRESH_INTERVAL_MS, CHECK_LATEST_COMMIT_INTERVAL, EXPRESS_PAYLOAD_LIMIT, GIT_PUSH_RETRIES, @@ -146,6 +148,24 @@ export const getAppList = (): string[] => { return appsSchema.enum as string[] } +let isCatalogRefreshRunning = false +let catalogRefreshInterval: ReturnType | undefined + +export const cloneCatalogRepositories = async (): Promise => { + if (isCatalogRefreshRunning) { + debug('Catalog cache refresh is already running, skipping scheduled run') + return + } + + isCatalogRefreshRunning = true + try { + const otomiStack = await getSessionStack() + await otomiStack.refreshBYOCatalogCache() + } finally { + isCatalogRefreshRunning = false + } +} + export async function initApp(inOtomiStack?: OtomiStack) { // Only create lightship in production (not in tests) const lightship = env.isTest ? null : createLightship() @@ -158,7 +178,6 @@ export async function initApp(inOtomiStack?: OtomiStack) { app.set('trust proxy', env.TRUST_PROXY) } - const apiRoutesPath = path.resolve(__dirname, 'api') await loadSpec() const authz = new Authz(otomiSpec.spec) app.use(logger('dev')) @@ -221,14 +240,33 @@ export async function initApp(inOtomiStack?: OtomiStack) { .listen(PORT, async () => { debug(`Listening on :::${PORT}`) lightship?.signalReady() - // Clone repo after the application is ready to avoid Pod NotReady phenomenon, and thus infinite Pod crash loopback - ;(await getSessionStack()).initGit() + + // Initialize git, warm catalog cache, and then start periodic refresh in one sequenced background task + void (async () => { + const sessionStack = await getSessionStack() + await sessionStack.initGit() + void cloneCatalogRepositories() + + if (!catalogRefreshInterval) { + catalogRefreshInterval = setInterval(() => { + void cloneCatalogRepositories().catch((e) => { + debug(e) + }) + }, env.CATALOG_CACHE_REFRESH_INTERVAL_MS) + } + })().catch((e) => { + debug(e) + }) }) .on('error', (e) => { console.error(e) lightship?.shutdown() }) lightship?.registerShutdownHandler(() => { + if (catalogRefreshInterval) { + clearInterval(catalogRefreshInterval) + catalogRefreshInterval = undefined + } ;(server as Server).close() }) } @@ -271,6 +309,7 @@ export async function initApp(inOtomiStack?: OtomiStack) { // eslint-disable-next-line @typescript-eslint/no-unsafe-argument app.use('/api-docs/swagger', swaggerUi.serve, swaggerUi.setup(otomiSpec.spec)) + return app } diff --git a/src/openapi/api.yaml b/src/openapi/api.yaml index 3bfd32b2..c685fd74 100644 --- a/src/openapi/api.yaml +++ b/src/openapi/api.yaml @@ -1993,6 +1993,26 @@ paths: application/json: schema: $ref: '#/components/schemas/OpenApiValidationError' + + /v2/catalogs/refresh: + post: + operationId: refreshAplCatalogCache + x-eov-operation-handler: v2/catalogs/refresh + description: Refresh BYO catalog cache(s) + x-aclSchema: AplCatalog + parameters: + - name: catalogId + in: query + description: Optional catalog name to refresh a single cache; when omitted all enabled catalogs are refreshed + required: false + schema: + type: string + responses: + '200': + description: Successfully refreshed catalog cache(s) + '400': + $ref: '#/components/responses/BadRequest' + '/v2/catalogs/{catalogId}/charts/{chartName}': parameters: - $ref: '#/components/parameters/catalogParams' diff --git a/src/otomi-stack.ts b/src/otomi-stack.ts index 25a68fab..1499eca3 100644 --- a/src/otomi-stack.ts +++ b/src/otomi-stack.ts @@ -90,6 +90,7 @@ import { } from 'src/utils' import { deepQuote } from 'src/utils/yamlUtils' import { + CATALOG_CACHE_PATH, cleanEnv, CUSTOM_ROOT_CA, DEFAULT_PLATFORM_ADMIN_EMAIL, @@ -138,7 +139,6 @@ import { defineClusterId, ObjectStorageClient } from './utils/wizardUtils' import { fetchChartYaml, fetchWorkloadCatalog, - fetchWorkloadCatalogChart, isInteralGiteaURL, NewHelmChartValues, sparseCloneChart, @@ -152,6 +152,7 @@ interface ExcludedApp extends App { const debug = Debug('otomi:otomi-stack') const env = cleanEnv({ + CATALOG_CACHE_PATH, CUSTOM_ROOT_CA, DEFAULT_PLATFORM_ADMIN_EMAIL, EDITOR_INACTIVITY_TIMEOUT, @@ -1611,6 +1612,8 @@ export default class OtomiStack { branch: string, teamId?: string, chartsPath?: string, + keepLocalClone: boolean = false, + forceRefresh: boolean = false, ): Promise<{ url: string; helmCharts: any; catalog: any; chartsPath?: string }> { const { cluster } = this.getSettings(['cluster']) try { @@ -1621,13 +1624,14 @@ export default class OtomiStack { cluster?.domainSuffix, teamId, chartsPath, + forceRefresh, ) return { url, helmCharts, catalog, chartsPath } } catch (error) { debug('Error fetching workload catalog') return { url, helmCharts: [], catalog: [], chartsPath } } finally { - if (existsSync(helmChartsDir)) rmSync(helmChartsDir, { recursive: true, force: true }) + if (!keepLocalClone && existsSync(helmChartsDir)) rmSync(helmChartsDir, { recursive: true, force: true }) } } @@ -1648,6 +1652,10 @@ export default class OtomiStack { const aplRecord = await this.saveCatalog(data) await this.doDeployments([aplRecord], false) + const { repositoryUrl, branch, name, chartsPath } = data.spec + void this.getBYOWorkloadCatalog(repositoryUrl, branch, name, chartsPath as string | undefined, true).catch((e) => + debug(`Unable to warm cache for catalog ${data.spec.name}`, e), + ) return aplRecord.content as AplCatalogResponse } @@ -1671,7 +1679,12 @@ export default class OtomiStack { const aplRecord = await this.saveCatalog(platformObject) const catalogResponse = aplRecord.content as AplCatalogResponse await this.doDeployment(aplRecord, false) - + try { + const { repositoryUrl, branch, name: catalogName, chartsPath } = catalogResponse.spec + void this.getBYOWorkloadCatalog(repositoryUrl, branch, catalogName, chartsPath as string | undefined, true) + } catch { + debug(`Unable to warm cache for catalog ${catalogResponse.spec.name}`) + } return catalogResponse } @@ -1680,6 +1693,9 @@ export default class OtomiStack { await this.git.removeFile(filePath) await this.doDeleteDeployment([filePath]) + // delete the cached charts for this catalog + const cacheDir = `${env.CATALOG_CACHE_PATH}/${encodeURIComponent(name)}` + if (existsSync(cacheDir)) rmSync(cacheDir, { recursive: true, force: true }) } async getWorkloadCatalog(data: { @@ -1704,11 +1720,23 @@ export default class OtomiStack { branch: string, catalogName: string, chartsPath?: string, + forceRefresh: boolean = false, ): Promise<{ url: string; helmCharts: any; catalog: any; chartsPath?: string }> { - const uuid = uuidv4() - const helmChartsDir = `/tmp/otomi/charts/${catalogName}/${branch}/charts/${uuid}` + const encodedCatalogName = encodeURIComponent(catalogName) + const encodedBranch = encodeURIComponent(branch) + const helmChartsDir = `${env.CATALOG_CACHE_PATH}/${encodedCatalogName}/${encodedBranch}` + + return this.fetchCatalog(url, helmChartsDir, branch, undefined, chartsPath, true, forceRefresh) + } + + async refreshBYOCatalogCache(catalogName?: string): Promise { + const catalogs = this.getAllAplCatalogs({ enabled: true }) + const selectedCatalogs = catalogName ? catalogs.filter((catalog) => catalog.spec.name === catalogName) : catalogs - return this.fetchCatalog(url, helmChartsDir, branch, undefined, chartsPath) + for (const catalog of selectedCatalogs) { + const { repositoryUrl, branch, name, chartsPath } = catalog.spec + await this.getBYOWorkloadCatalog(repositoryUrl, branch, name, chartsPath as string | undefined, true) + } } async getAplCatalogCharts(name: string): Promise<{ url: string; helmCharts: any; catalog: any; branch: string }> { @@ -1730,27 +1758,20 @@ export default class OtomiStack { const catalog = this.getAplCatalog(name) const { repositoryUrl, branch, chartsPath } = catalog.spec const { cluster } = this.getSettings(['cluster']) - - const uuid = uuidv4() - const helmChartsDir = `/tmp/otomi/charts/${name}/${branch}/chart/${uuid}` + const encodedCatalogName = encodeURIComponent(catalog.spec.name) + const encodedBranch = encodeURIComponent(branch) + const helmChartsDir = `${env.CATALOG_CACHE_PATH}/${encodedCatalogName}/${encodedBranch}` try { - const chart = await fetchWorkloadCatalogChart( - repositoryUrl, - helmChartsDir, - chartName, - branch, - cluster?.domainSuffix, - undefined, - chartsPath as string | undefined, - ) + const singleChart = + ( + await fetchWorkloadCatalog(repositoryUrl, helmChartsDir, branch, cluster?.domainSuffix, undefined, chartsPath) + ).catalog.find((c) => c.name === chartName) || null - return { url: repositoryUrl, branch, chart, chartsPath } + return { url: repositoryUrl, branch, chart: singleChart, chartsPath } } catch (error) { debug(`Error fetching workload chart '${chartName}': ${error.message}`) return { url: repositoryUrl, branch, chart: null, chartsPath } - } finally { - if (existsSync(helmChartsDir)) rmSync(helmChartsDir, { recursive: true, force: true }) } } diff --git a/src/utils/workloadUtils.test.ts b/src/utils/workloadUtils.test.ts index fb1a0e6f..79de2053 100644 --- a/src/utils/workloadUtils.test.ts +++ b/src/utils/workloadUtils.test.ts @@ -638,7 +638,7 @@ describe('fetchWorkloadCatalog', () => { expect(mockGit.clone).toHaveBeenCalledWith( 'https://git-user:git-password@gitea.example.com/otomi/charts.git', helmChartsDir, - ['--branch', 'main', '--single-branch'], + ['--branch', 'main', '--single-branch', '--depth', '1'], ) expect(result).toEqual({ helmCharts: ['chart1', 'chart2'], @@ -757,6 +757,27 @@ describe('fetchWorkloadCatalog', () => { expect(result).toEqual({ helmCharts: [], catalog: [] }) expect(fsPromises.readdir).not.toHaveBeenCalled() }) + + test('ignores folders without Chart.yaml', async () => { + const mockGit = { + clone: jest.fn().mockResolvedValue(undefined), + } + ;(simpleGit as jest.Mock).mockReturnValue(mockGit) + ;(fsPromises.readdir as jest.Mock).mockResolvedValue(['chart1', 'chart2']) + + const baseImpl = (utils.safeReadTextFile as jest.Mock).getMockImplementation()! + ;(utils.safeReadTextFile as jest.Mock).mockImplementation(async (_baseDir, filePath) => { + if (filePath.endsWith('chart2/Chart.yaml') || filePath.endsWith('chart2/chart.yaml')) { + throw new Error('Chart manifest not found') + } + return baseImpl(_baseDir, filePath) + }) + + const result = await fetchWorkloadCatalog(url, helmChartsDir, 'main', undefined, 'admin') + + expect(result.helmCharts).toEqual(['chart1']) + expect(result.helmCharts).not.toContain('chart2') + }) }) // ---------------------------------------------------------------- @@ -790,7 +811,13 @@ describe('chartRepo', () => { const repo = new chartRepo(localPath, chartRepoUrl, gitUser, gitEmail) await repo.clone() - expect(mockGit.clone).toHaveBeenCalledWith(chartRepoUrl, localPath, ['--branch', 'main', '--single-branch']) + expect(mockGit.clone).toHaveBeenCalledWith(chartRepoUrl, localPath, [ + '--branch', + 'main', + '--single-branch', + '--depth', + '1', + ]) }) test('cloneSingleChart method performs sparse checkout', async () => { @@ -974,6 +1001,7 @@ describe('Helper functions integration tests', () => { beforeEach(() => { jest.clearAllMocks() process.env = { ...originalEnv, GIT_USER: 'git-user', GIT_PASSWORD: 'git-password' } + ;(fs.lstatSync as jest.Mock).mockReturnValue({ isDirectory: () => true }) jest.spyOn(utils, 'safeReadTextFile').mockResolvedValue('# README') }) @@ -1006,7 +1034,7 @@ describe('Helper functions integration tests', () => { expect(mockGit.clone).toHaveBeenCalledWith( 'https://git-user:git-password@gitea.cluster.local/otomi/charts.git', helmChartsDir, - ['--branch', 'main', '--single-branch'], + ['--branch', 'main', '--single-branch', '--depth', '1'], ) }) @@ -1030,7 +1058,13 @@ describe('Helper functions integration tests', () => { } // Verify that clone was called with original URL - expect(mockGit.clone).toHaveBeenCalledWith(githubUrl, helmChartsDir, ['--branch', 'main', '--single-branch']) + expect(mockGit.clone).toHaveBeenCalledWith(githubUrl, helmChartsDir, [ + '--branch', + 'main', + '--single-branch', + '--depth', + '1', + ]) }) }) @@ -1201,19 +1235,26 @@ describe('Helper functions integration tests', () => { }) describe('getChartFolders (tested via fetchWorkloadCatalog)', () => { - test('excludes system files from chart list', async () => { + test('excludes hidden entries and non-directory files from chart list', async () => { const mockGit = { clone: jest.fn().mockResolvedValue(undefined) } ;(simpleGit as jest.Mock).mockReturnValue(mockGit) ;(fs.existsSync as jest.Mock).mockReturnValue(false) ;(fsPromises.readdir as jest.Mock).mockResolvedValue([ '.git', + '.github', '.gitignore', '.vscode', 'LICENSE', 'README.md', + 'notes.txt', 'chart1', 'chart2', ]) + ;(fs.lstatSync as jest.Mock).mockImplementation((targetPath: string) => { + const isDirectory = + targetPath.endsWith('/chart1') || targetPath.endsWith('/chart2') || targetPath.endsWith('/.github') + return { isDirectory: () => isDirectory } + }) ;(fsExtra.readFile as unknown as jest.Mock).mockImplementation((filePath) => { if (filePath.endsWith('rbac.yaml')) return Promise.reject(new Error('Not found')) return Promise.reject(new Error('File not found')) @@ -1221,7 +1262,7 @@ describe('Helper functions integration tests', () => { const result = await fetchWorkloadCatalog('https://example.com/charts.git', '/tmp/test', 'main') - // Should only include chart1 and chart2, not system files + // Should only include chart directories, excluding hidden folders and files expect(result.helmCharts).toEqual(['chart1', 'chart2']) }) }) diff --git a/src/utils/workloadUtils.ts b/src/utils/workloadUtils.ts index fa5355e8..b52df623 100644 --- a/src/utils/workloadUtils.ts +++ b/src/utils/workloadUtils.ts @@ -1,19 +1,25 @@ import axios from 'axios' import Debug from 'debug' -import { existsSync, lstatSync, mkdirSync, mkdtempSync, renameSync, rmSync } from 'fs' +import { existsSync, lstatSync, mkdirSync, renameSync, rmSync } from 'fs' import { readFile } from 'fs-extra' import { readdir, writeFile } from 'fs/promises' -import { tmpdir } from 'os' -import path, { join } from 'path' +import path from 'path' import simpleGit, { SimpleGit } from 'simple-git' import { safeReadTextFile } from 'src/utils' -import { cleanEnv, GIT_PROVIDER_URL_PATTERNS } from 'src/validators' +import { + CATALOG_CACHE_REFRESH_INTERVAL_MS, + CATALOG_CACHE_SYNC_MARKER, + cleanEnv, + GIT_PROVIDER_URL_PATTERNS, +} from 'src/validators' import YAML from 'yaml' import { BadRequestError } from '../error' const debug = Debug('apl:workloadUtils') const env = cleanEnv({ + CATALOG_CACHE_REFRESH_INTERVAL_MS, + CATALOG_CACHE_SYNC_MARKER, GIT_PROVIDER_URL_PATTERNS, }) @@ -230,7 +236,53 @@ export class chartRepo { this.git = simpleGit(this.localPath) } async clone(branch: string = 'main') { - await this.git.clone(this.chartRepoUrl, this.localPath, ['--branch', branch, '--single-branch']) + await this.git.clone(this.chartRepoUrl, this.localPath, ['--branch', branch, '--single-branch', '--depth', '1']) + } + async ensureLatest(branch: string = 'main', forceRefresh: boolean = false) { + const gitDir = path.join(this.localPath, '.git') + const cacheSyncMarkerPath = path.join(this.localPath, env.CATALOG_CACHE_SYNC_MARKER) + const canRefreshExistingRepo = + typeof this.git.cwd === 'function' && + typeof this.git.fetch === 'function' && + typeof this.git.checkout === 'function' && + typeof this.git.pull === 'function' + + if (!existsSync(gitDir) || !canRefreshExistingRepo) { + debug(`Catalog cache miss at ${this.localPath}; cloning branch '${branch}'`) + await this.clone(branch) + await writeFile(cacheSyncMarkerPath, new Date().toISOString(), 'utf-8') + debug(`Catalog cache initialized at ${this.localPath}`) + return + } + + if (!forceRefresh && existsSync(cacheSyncMarkerPath)) { + const cacheAgeMs = Date.now() - lstatSync(cacheSyncMarkerPath).mtimeMs + if (cacheAgeMs < env.CATALOG_CACHE_REFRESH_INTERVAL_MS) { + debug( + `Catalog cache hit at ${this.localPath}; age=${Math.round(cacheAgeMs / 1000)}s, ttl=${Math.round(env.CATALOG_CACHE_REFRESH_INTERVAL_MS / 1000)}s`, + ) + return + } + debug( + `Catalog cache expired at ${this.localPath}; age=${Math.round(cacheAgeMs / 1000)}s, ttl=${Math.round(env.CATALOG_CACHE_REFRESH_INTERVAL_MS / 1000)}s`, + ) + } else if (forceRefresh) { + debug(`Catalog cache force-refresh requested at ${this.localPath}`) + } else { + debug(`Catalog cache marker missing at ${this.localPath}; refreshing`) + } + + debug(`Refreshing catalog cache at ${this.localPath} for branch '${branch}'`) + await this.git.cwd(this.localPath) + await this.git.fetch('origin', branch) + try { + await this.git.checkout(branch) + } catch { + await this.git.checkout(['-B', branch, `origin/${branch}`]) + } + await this.git.pull('origin', branch, { '--ff-only': null }) + await writeFile(cacheSyncMarkerPath, new Date().toISOString(), 'utf-8') + debug(`Catalog cache refreshed at ${this.localPath}`) } async cloneSingleChart(refAndPath: string, finalDestinationPath: string) { const remoteResult = await this.git.listRemote([this.chartRepoUrl]) @@ -340,40 +392,6 @@ export async function sparseCloneChart( return true } -export async function sparseCheckoutPath( - gitCloneUrl: string, - ref: string, - sparsePath: string, - targetBaseDir: string, - targetDirName: string, -): Promise<{ success: true; checkoutPath: string } | { success: false; error: string }> { - if (!existsSync(targetBaseDir)) mkdirSync(targetBaseDir, { recursive: true }) - - const tempCloneDir = mkdtempSync(join(tmpdir(), 'sparse-checkout-')) - const finalDestinationPath = join(targetBaseDir, targetDirName) - - try { - rmSync(finalDestinationPath, { recursive: true, force: true }) - - const normalizedSparsePath = sparsePath.replace(/^\/+/, '').replace(/\/+$/, '') - const refAndPath = `${ref}/${normalizedSparsePath}` - - const repo = new chartRepo(tempCloneDir, gitCloneUrl) - await repo.cloneSingleChart(refAndPath, finalDestinationPath) - - rmSync(join(finalDestinationPath, '.git'), { recursive: true, force: true }) - - return { success: true, checkoutPath: finalDestinationPath } - } catch (error) { - return { - success: false, - error: error instanceof Error ? error.message : 'Unknown sparse checkout error.', - } - } finally { - rmSync(tempCloneDir, { recursive: true, force: true }) - } -} - /** * Encodes Git credentials into the URL for internal Gitea repositories */ @@ -480,8 +498,27 @@ async function processChartFolder( */ async function getChartFolders(helmChartsDir: string): Promise { const files = await readdir(helmChartsDir, 'utf-8') - const filesToExclude = ['.git', '.gitignore', '.vscode', 'LICENSE', 'README.md'] - return files.filter((f) => !filesToExclude.includes(f)) + const chartFolders = await Promise.all( + files.map(async (fileName) => { + try { + if (fileName.startsWith('.')) return null + const filePath = path.join(helmChartsDir, fileName) + if (!lstatSync(filePath).isDirectory()) return null + + try { + await safeReadTextFile(helmChartsDir, `${fileName}/Chart.yaml`) + return fileName + } catch { + await safeReadTextFile(helmChartsDir, `${fileName}/chart.yaml`) + return fileName + } + } catch { + return null + } + }), + ) + + return chartFolders.filter((folder): folder is string => folder !== null) } /** @@ -501,6 +538,7 @@ export async function fetchWorkloadCatalog( clusterDomainSuffix?: string, teamId?: string, chartsPath?: string, + forceRefresh: boolean = false, ): Promise<{ helmCharts: string[]; catalog: any[] }> { const resolvedHelmChartsDir = path.resolve(helmChartsDir) @@ -510,7 +548,7 @@ export async function fetchWorkloadCatalog( // Clone repository const gitUrl = encodeGitCredentials(url, clusterDomainSuffix) const gitRepo = new chartRepo(resolvedHelmChartsDir, gitUrl) - await gitRepo.clone(branch) + await gitRepo.ensureLatest(branch, forceRefresh) // Determine the charts directory path const chartsDir = chartsPath ? path.resolve(resolvedHelmChartsDir, chartsPath) : resolvedHelmChartsDir @@ -542,7 +580,10 @@ export async function fetchWorkloadCatalog( // Get chart folders const folders = await getChartFolders(chartsDir) - + if (!folders.length) { + debug(`No chart folders found in '${chartsDir}' at '${url}'`) + return { helmCharts: [], catalog: [] } + } // Read RBAC configuration (try chartsDir first, fallback to root) let rbacConfig = await readRbacConfig(chartsDir) if (!rbacConfig.rbac || Object.keys(rbacConfig.rbac).length === 0) { @@ -569,66 +610,3 @@ export async function fetchWorkloadCatalog( return { helmCharts, catalog } } - -export async function fetchWorkloadCatalogChart( - url: string, - helmChartsDir: string, - chartName: string, - branch: string = 'main', - clusterDomainSuffix?: string, - teamId?: string, - chartsPath?: string, -): Promise { - const resolvedHelmChartsDir = path.resolve(helmChartsDir) - - if (!existsSync(resolvedHelmChartsDir)) { - mkdirSync(resolvedHelmChartsDir, { recursive: true }) - } - - const gitUrl = encodeGitCredentials(url, clusterDomainSuffix) - - const sparsePath = chartsPath ? `${chartsPath}/${chartName}` : chartName - const checkoutResult = await sparseCheckoutPath(gitUrl, branch, sparsePath, resolvedHelmChartsDir, chartName) - - if (!checkoutResult.success) { - debug(`Sparse checkout failed for chart '${chartName}' from '${url}': ${checkoutResult.error}`) - return null - } - - const chartDir = checkoutResult.checkoutPath - - try { - const values = await safeReadTextFile(chartDir, 'values.yaml') - - let valuesSchema = '{}' - try { - const schemaContent = await safeReadTextFile(chartDir, 'values.schema.json') - valuesSchema = schemaContent || '{}' - } catch { - // optional - } - - const chartYaml = await safeReadTextFile(chartDir, 'Chart.yaml') - const chartMetadata = YAML.parse(chartYaml) - - let readme = 'There is no `README` for this chart.' - try { - readme = await safeReadTextFile(chartDir, 'README.md') - } catch { - // optional - } - - return { - name: chartName, - values: values || '{}', - valuesSchema, - icon: chartMetadata?.icon, - chartVersion: chartMetadata?.version, - chartDescription: chartMetadata?.description, - readme, - } - } catch (error) { - debug(`Error parsing chart '${chartName}' in '${chartDir}': ${error.message}`) - return null - } -} diff --git a/src/validators.ts b/src/validators.ts index d97f7874..cc1b85e1 100644 --- a/src/validators.ts +++ b/src/validators.ts @@ -9,6 +9,18 @@ export const AUTHZ_MOCK_IS_TEAM_ADMIN = bool({ default: true, }) export const AUTHZ_MOCK_TEAM = str({ desc: 'Comma separated list of teams a user belongs to', default: undefined }) +export const CATALOG_CACHE_REFRESH_INTERVAL_MS = num({ + desc: 'Interval in milliseconds for refreshing the BYO catalog cache', + default: 600000, // 10 minutes +}) +export const CATALOG_CACHE_PATH = str({ + desc: 'The file path for the BYO catalog cache', + default: '/tmp/otomi/charts-cache', +}) +export const CATALOG_CACHE_SYNC_MARKER = str({ + desc: 'The file name for the BYO catalog cache sync marker', + default: '.apl-cache-last-sync', +}) export const DEFAULT_PLATFORM_ADMIN_EMAIL = str({ desc: 'The email address for the default platform admin user.', devDefault: 'platform-admin@dev.linode-apl.net',