From e27e10c5599dd6d46f9145b8aa7be2e105cd3f4f Mon Sep 17 00:00:00 2001 From: ElderMatt <18527012+ElderMatt@users.noreply.github.com> Date: Tue, 10 Mar 2026 10:01:15 +0100 Subject: [PATCH 01/13] feat: added caching for byo catalog --- [object Object]/bitnami/main | 1 + [object Object]/cert-manager/master | 1 + [object Object]/default/main | 1 + [object Object]/empty/no-k8s | 1 + [object Object]/exporter/main | 1 + package-lock.json | 40 ++++++++++++-- src/api/v2/catalogs/refresh.ts | 16 ++++++ src/app.ts | 47 +++++++++++++++-- src/openapi/api.yaml | 19 +++++++ src/otomi-stack.ts | 41 +++++++++++++-- src/utils/workloadUtils.test.ts | 33 +++++++++++- src/utils/workloadUtils.ts | 81 ++++++++++++++++++++++++++--- src/validators.ts | 12 +++++ 13 files changed, 273 insertions(+), 21 deletions(-) create mode 160000 [object Object]/bitnami/main create mode 160000 [object Object]/cert-manager/master create mode 160000 [object Object]/default/main create mode 160000 [object Object]/empty/no-k8s create mode 160000 [object Object]/exporter/main create mode 100644 src/api/v2/catalogs/refresh.ts diff --git a/[object Object]/bitnami/main b/[object Object]/bitnami/main new file mode 160000 index 00000000..f9057073 --- /dev/null +++ b/[object Object]/bitnami/main @@ -0,0 +1 @@ +Subproject commit f90570733f27531dcac1bf81700934ca9f24fc4f diff --git a/[object Object]/cert-manager/master b/[object Object]/cert-manager/master new file mode 160000 index 00000000..6eaa3a79 --- /dev/null +++ b/[object Object]/cert-manager/master @@ -0,0 +1 @@ +Subproject commit 6eaa3a794234bf88300b4185fc4c6d1bc069c929 diff --git a/[object Object]/default/main b/[object Object]/default/main new file mode 160000 index 00000000..305fe68d --- /dev/null +++ b/[object Object]/default/main @@ -0,0 +1 @@ +Subproject commit 305fe68d6f7c46f34ba07748a8be44e5e379a91f diff --git a/[object Object]/empty/no-k8s b/[object Object]/empty/no-k8s new file mode 160000 index 00000000..baa7656e --- /dev/null +++ b/[object Object]/empty/no-k8s @@ -0,0 +1 @@ +Subproject commit baa7656e9707b4e95132bf5c4fd3e3d9b8aa127d diff --git a/[object Object]/exporter/main b/[object Object]/exporter/main new file mode 160000 index 00000000..1c0c69b0 --- /dev/null +++ b/[object Object]/exporter/main @@ -0,0 +1 @@ +Subproject commit 1c0c69b0ad194558c1884f5af496a1bb4eb17e61 diff --git a/package-lock.json b/package-lock.json index 85d72a9f..9fa046a6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -196,6 +196,7 @@ "integrity": "sha512-CGOfOJqWjg2qW/Mb6zNsDm+u5vFQ8DxXfbM09z69p5Z6+mE1ikP2jUXw+j42Pf1XTYED2Rni5f95npYeuwMDQA==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@babel/code-frame": "^7.29.0", "@babel/generator": "^7.29.0", @@ -2218,6 +2219,7 @@ "integrity": "sha512-itvL5h8RETACmOTFc4UfIyB2RfEHi71Ax6E/PivVxq9NseKbOWpeyHEOIbmAw1rs8Ak0VursQNww7lf7YtUwzg==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "env-paths": "^2.2.1", "import-fresh": "^3.3.0", @@ -2677,6 +2679,7 @@ "integrity": "sha512-Tdfx4eH2uS+gv9V9NCr3Rz+c7RSS6ntXp3Blliud18ibRUlRxO9dTaOjG4iv4x0nAmMeedP1ORkEpeXSkh2QiQ==", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=20" } @@ -2758,7 +2761,8 @@ "resolved": "https://registry.npmjs.org/@cspell/dict-css/-/dict-css-4.0.19.tgz", "integrity": "sha512-VYHtPnZt/Zd/ATbW3rtexWpBnHUohUrQOHff/2JBhsVgxOrksAxJnLAO43Q1ayLJBJUUwNVo+RU0sx0aaysZfg==", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/@cspell/dict-dart": { "version": "2.3.2", @@ -2898,14 +2902,16 @@ "resolved": "https://registry.npmjs.org/@cspell/dict-html/-/dict-html-4.0.14.tgz", "integrity": "sha512-2bf7n+kS92g+cMKV0wr9o/Oq9n8JzU7CcrB96gIh2GHgnF+0xDOqO2W/1KeFAqOfqosoOVE48t+4dnEMkkoJ2Q==", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/@cspell/dict-html-symbol-entities": { "version": "4.0.5", "resolved": "https://registry.npmjs.org/@cspell/dict-html-symbol-entities/-/dict-html-symbol-entities-4.0.5.tgz", "integrity": "sha512-429alTD4cE0FIwpMucvSN35Ld87HCyuM8mF731KU5Rm4Je2SG6hmVx7nkBsLyrmH3sQukTcr1GaiZsiEg8svPA==", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/@cspell/dict-java": { "version": "5.0.12", @@ -3103,7 +3109,8 @@ "resolved": "https://registry.npmjs.org/@cspell/dict-typescript/-/dict-typescript-3.2.3.tgz", "integrity": "sha512-zXh1wYsNljQZfWWdSPYwQhpwiuW0KPW1dSd8idjMRvSD0aSvWWHoWlrMsmZeRl4qM4QCEAjua8+cjflm41cQBg==", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/@cspell/dict-vue": { "version": "3.0.5", @@ -4315,6 +4322,7 @@ "integrity": "sha512-b63wmnKPaK+6ZZfpYhz9K61oybvbI1aMcIs80++JI1O1rR1vaxHUCNqo3ITu6NU0d4V34yZFoHMn/uoKr/Rwfw==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@jest/environment": "30.2.0", "@jest/expect": "30.2.0", @@ -4850,6 +4858,7 @@ "integrity": "sha512-DhGl4xMVFGVIyMwswXeyzdL4uXD5OGILGX5N8Y+f6W7LhC1Ze2poSNrkF/fedpVDHEEZ+PHFW0vL14I+mm8K3Q==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@octokit/auth-token": "^6.0.0", "@octokit/graphql": "^9.0.3", @@ -6304,7 +6313,8 @@ "version": "7.0.15", "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz", "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==", - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/@types/json5": { "version": "0.0.29", @@ -6386,6 +6396,7 @@ "resolved": "https://registry.npmjs.org/@types/node/-/node-24.10.1.tgz", "integrity": "sha512-GNWcUTRBgIRJD5zj+Tq0fKOJ5XZajIiBroOF0yvj2bSU1WvNdYS/dn9UxwsujGW4JX06dnHyjV2y9rRaybH0iQ==", "license": "MIT", + "peer": true, "dependencies": { "undici-types": "~7.16.0" } @@ -6552,6 +6563,7 @@ "integrity": "sha512-klQbnPAAiGYFyI02+znpBRLyjL4/BrBd0nyWkdC0s/6xFLkXYQ8OoRrSkqacS1ddVxf/LDyODIKbQ5TgKAf/Fg==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@typescript-eslint/scope-manager": "8.56.1", "@typescript-eslint/types": "8.56.1", @@ -7120,6 +7132,7 @@ "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", "dev": true, "license": "MIT", + "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -8148,6 +8161,7 @@ } ], "license": "MIT", + "peer": true, "dependencies": { "baseline-browser-mapping": "^2.9.0", "caniuse-lite": "^1.0.30001759", @@ -10841,6 +10855,7 @@ "integrity": "sha512-LEyamqS7W5HB3ujJyvi0HQK/dtVINZvd5mAAp9eT5S/ujByGjiZLCzPcHVzuXbpJDJF/cxwHlfceVUDZ2lnSTw==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.8.0", "@eslint-community/regexpp": "^4.12.1", @@ -10901,6 +10916,7 @@ "integrity": "sha512-82GZUjRS0p/jganf6q1rEO25VSoHH0hKPCTrgillPjdI/3bgBhAE1QzHrHTizjpRvy6pGAvKjDJtk2pF9NDq8w==", "dev": true, "license": "MIT", + "peer": true, "bin": { "eslint-config-prettier": "bin/cli.js" }, @@ -11027,6 +11043,7 @@ "integrity": "sha512-whOE1HFo/qJDyX4SnXzP4N6zOWn79WhnCUY/iDR0mPfQZO8wcYE4JClzI2oZrhBnnMUCBCHZhO6VQyoBU95mZA==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@rtsao/scc": "^1.1.0", "array-includes": "^3.1.9", @@ -11561,6 +11578,7 @@ "resolved": "https://registry.npmjs.org/express/-/express-5.2.1.tgz", "integrity": "sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw==", "license": "MIT", + "peer": true, "dependencies": { "accepts": "^2.0.0", "body-parser": "^2.2.1", @@ -11647,6 +11665,7 @@ "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.17.1.tgz", "integrity": "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==", "license": "MIT", + "peer": true, "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", @@ -14307,6 +14326,7 @@ "integrity": "sha512-F26gjC0yWN8uAA5m5Ss8ZQf5nDHWGlN/xWZIh8S5SRbsEKBovwZhxGd6LJlbZYxBgCYOtreSUyb8hpXyGC5O4A==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@jest/core": "30.2.0", "@jest/types": "30.2.0", @@ -15190,6 +15210,7 @@ "resolved": "https://registry.npmjs.org/jsep/-/jsep-1.4.0.tgz", "integrity": "sha512-B7qPcEVE3NVkmSJbaYxvv4cHkVW7DQsZz13pUMrfS8z8Q/BuShN+gcTXrUlPiGqM2/t/EEaI030bpxMqY8gMlw==", "license": "MIT", + "peer": true, "engines": { "node": ">= 10.16.0" } @@ -16550,6 +16571,7 @@ "integrity": "sha512-8dD6FusOQSrpv9Z1rdNMdlSgQOIP880DHqnohobOmYLElGEqAL/JvxvuxZO16r4HtjTlfPRDC1hbvxC9dPN2nA==", "dev": true, "license": "MIT", + "peer": true, "bin": { "marked": "bin/marked.js" }, @@ -19808,6 +19830,7 @@ "dev": true, "inBundle": true, "license": "MIT", + "peer": true, "engines": { "node": ">=12" }, @@ -21064,6 +21087,7 @@ "integrity": "sha512-UOnG6LftzbdaHZcKoPFtOcCKztrQ57WkHDeRD9t/PTQtmT0NHSeWWepj6pS0z/N7+08BHFDQVUrfmfMRcZwbMg==", "dev": true, "license": "MIT", + "peer": true, "bin": { "prettier": "bin/prettier.cjs" }, @@ -22183,6 +22207,7 @@ "integrity": "sha512-WRgl5GcypwramYX4HV+eQGzUbD7UUbljVmS+5G1uMwX/wLgYuJAxGeerXJDMO2xshng4+FXqCgyB5QfClV6WjA==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@semantic-release/commit-analyzer": "^13.0.1", "@semantic-release/error": "^4.0.0", @@ -24355,6 +24380,7 @@ "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=12" }, @@ -24577,6 +24603,7 @@ "integrity": "sha512-f0FFpIdcHgn8zcPSbf1dRevwt047YMnaiJM3u2w2RewrB+fob/zePZcrOyQoLMMO7aBIddLcQIEK5dYjkLnGrQ==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@cspotcode/source-map-support": "^0.8.0", "@tsconfig/node10": "^1.0.7", @@ -24841,6 +24868,7 @@ "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", "dev": true, "license": "Apache-2.0", + "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -25020,6 +25048,7 @@ "dev": true, "hasInstallScript": true, "license": "MIT", + "peer": true, "dependencies": { "napi-postinstall": "^0.3.0" }, @@ -25452,6 +25481,7 @@ "version": "7.5.7", "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.7.tgz", "integrity": "sha512-KMvVuFzpKBuiIXW3E4u3mySRO2/mCHSyZDJQM5NQ9Q9KHWHWh0NHgfbRMLLrceUK5qAL4ytALJbpRMjixFZh8A==", + "peer": true, "engines": { "node": ">=8.3.0" }, diff --git a/src/api/v2/catalogs/refresh.ts b/src/api/v2/catalogs/refresh.ts new file mode 100644 index 00000000..76d7bb28 --- /dev/null +++ b/src/api/v2/catalogs/refresh.ts @@ -0,0 +1,16 @@ +import Debug from 'debug' +import { Response } from 'express' +import { OpenApiRequestExt } from 'src/otomi-models' + +const debug = Debug('otomi:api:v2:catalogs:refresh') + +/** + * POST /v2/catalogs/refresh + * Refresh BYO catalog cache(s) immediately + */ +export const refreshAplCatalogCache = async (req: OpenApiRequestExt, res: Response): Promise => { + const catalogId = typeof req.query.catalogId === 'string' ? decodeURIComponent(req.query.catalogId) : undefined + debug(`refreshAplCatalogCache(${catalogId || 'all'})`) + const data = await req.otomi.refreshBYOCatalogCache(catalogId) + res.json(data) +} diff --git a/src/app.ts b/src/app.ts index 7d98f311..bccaced7 100644 --- a/src/app.ts +++ b/src/app.ts @@ -24,6 +24,7 @@ import { AplResponseObject, OpenAPIDoc, Schema, SealedSecretManifestResponse } f import { default as OtomiStack } from 'src/otomi-stack' import { extract, getPaths, getValuesSchema } from 'src/utils' import { + CATALOG_CACHE_REFRESH_INTERVAL_MS, CHECK_LATEST_COMMIT_INTERVAL, cleanEnv, EXPRESS_PAYLOAD_LIMIT, @@ -35,6 +36,7 @@ import getLatestRemoteCommitSha from './git/connect' import { getBuildStatus, getSealedSecretStatus, getServiceStatus, getWorkloadStatus } from './k8s_operations' const env = cleanEnv({ + CATALOG_CACHE_REFRESH_INTERVAL_MS, CHECK_LATEST_COMMIT_INTERVAL, EXPRESS_PAYLOAD_LIMIT, GIT_PUSH_RETRIES, @@ -146,6 +148,24 @@ export const getAppList = (): string[] => { return appsSchema.enum as string[] } +let isCatalogRefreshRunning = false +let catalogRefreshInterval: ReturnType | undefined + +export const cloneCatalogRepositories = async (): Promise => { + if (isCatalogRefreshRunning) { + debug('Catalog cache refresh is already running, skipping scheduled run') + return + } + + isCatalogRefreshRunning = true + try { + const otomiStack = await getSessionStack() + await otomiStack.refreshBYOCatalogCache() + } finally { + isCatalogRefreshRunning = false + } +} + export async function initApp(inOtomiStack?: OtomiStack) { // Only create lightship in production (not in tests) const lightship = env.isTest ? null : createLightship() @@ -158,7 +178,6 @@ export async function initApp(inOtomiStack?: OtomiStack) { app.set('trust proxy', env.TRUST_PROXY) } - const apiRoutesPath = path.resolve(__dirname, 'api') await loadSpec() const authz = new Authz(otomiSpec.spec) app.use(logger('dev')) @@ -221,14 +240,35 @@ export async function initApp(inOtomiStack?: OtomiStack) { .listen(PORT, async () => { debug(`Listening on :::${PORT}`) lightship?.signalReady() - // Clone repo after the application is ready to avoid Pod NotReady phenomenon, and thus infinite Pod crash loopback - ;(await getSessionStack()).initGit() + // // Clone repo after the application is ready to avoid Pod NotReady phenomenon, and thus infinite Pod crash loopback + // ;(await getSessionStack()).initGit() + + // Initialize git, warm catalog cache, and then start periodic refresh in one sequenced background task + void (async () => { + const sessionStack = await getSessionStack() + await sessionStack.initGit() + void cloneCatalogRepositories() + + if (!catalogRefreshInterval) { + catalogRefreshInterval = setInterval(() => { + void cloneCatalogRepositories().catch((e) => { + debug(e) + }) + }, env.CATALOG_CACHE_REFRESH_INTERVAL_MS) + } + })().catch((e) => { + debug(e) + }) }) .on('error', (e) => { console.error(e) lightship?.shutdown() }) lightship?.registerShutdownHandler(() => { + if (catalogRefreshInterval) { + clearInterval(catalogRefreshInterval) + catalogRefreshInterval = undefined + } ;(server as Server).close() }) } @@ -271,6 +311,7 @@ export async function initApp(inOtomiStack?: OtomiStack) { // eslint-disable-next-line @typescript-eslint/no-unsafe-argument app.use('/api-docs/swagger', swaggerUi.serve, swaggerUi.setup(otomiSpec.spec)) + return app } diff --git a/src/openapi/api.yaml b/src/openapi/api.yaml index 9bdcc82b..98166d81 100644 --- a/src/openapi/api.yaml +++ b/src/openapi/api.yaml @@ -1994,6 +1994,25 @@ paths: schema: $ref: '#/components/schemas/OpenApiValidationError' + /v2/catalogs/refresh: + post: + operationId: refreshAplCatalogCache + x-eov-operation-handler: v2/catalogs/refresh + description: Refresh BYO catalog cache(s) + x-aclSchema: AplCatalog + parameters: + - name: catalogId + in: query + description: Optional catalog name to refresh a single cache; when omitted all enabled catalogs are refreshed + required: false + schema: + type: string + responses: + '200': + description: Successfully refreshed catalog cache(s) + '400': + $ref: '#/components/responses/BadRequest' + /v1/coderepos: get: operationId: getAllCodeRepos diff --git a/src/otomi-stack.ts b/src/otomi-stack.ts index e6ed9d2d..dd77cceb 100644 --- a/src/otomi-stack.ts +++ b/src/otomi-stack.ts @@ -90,6 +90,7 @@ import { } from 'src/utils' import { deepQuote } from 'src/utils/yamlUtils' import { + CATALOG_CACHE_PATH, cleanEnv, CUSTOM_ROOT_CA, DEFAULT_PLATFORM_ADMIN_EMAIL, @@ -151,6 +152,7 @@ interface ExcludedApp extends App { const debug = Debug('otomi:otomi-stack') const env = cleanEnv({ + CATALOG_CACHE_PATH, CUSTOM_ROOT_CA, DEFAULT_PLATFORM_ADMIN_EMAIL, EDITOR_INACTIVITY_TIMEOUT, @@ -1610,6 +1612,8 @@ export default class OtomiStack { branch: string, teamId?: string, chartsPath?: string, + keepLocalClone: boolean = false, + forceRefresh: boolean = false, ): Promise<{ url: string; helmCharts: any; catalog: any; chartsPath?: string }> { const { cluster } = this.getSettings(['cluster']) try { @@ -1620,13 +1624,14 @@ export default class OtomiStack { cluster?.domainSuffix, teamId, chartsPath, + forceRefresh, ) return { url, helmCharts, catalog, chartsPath } } catch (error) { debug('Error fetching workload catalog') return { url, helmCharts: [], catalog: [], chartsPath } } finally { - if (existsSync(helmChartsDir)) rmSync(helmChartsDir, { recursive: true, force: true }) + if (!keepLocalClone && existsSync(helmChartsDir)) rmSync(helmChartsDir, { recursive: true, force: true }) } } @@ -1647,6 +1652,12 @@ export default class OtomiStack { const aplRecord = await this.saveCatalog(data) await this.doDeployments([aplRecord], false) + try { + const { repositoryUrl, branch, name, chartsPath } = data.spec + await this.getBYOWorkloadCatalog(repositoryUrl, branch, name, chartsPath as string | undefined, true) + } catch { + debug(`Unable to warm cache for catalog ${data.spec.name}`) + } return aplRecord.content as AplCatalogResponse } @@ -1670,7 +1681,12 @@ export default class OtomiStack { const aplRecord = await this.saveCatalog(platformObject) const catalogResponse = aplRecord.content as AplCatalogResponse await this.doDeployment(aplRecord, false) - + try { + const { repositoryUrl, branch, name: catalogName, chartsPath } = catalogResponse.spec + await this.getBYOWorkloadCatalog(repositoryUrl, branch, catalogName, chartsPath as string | undefined, true) + } catch { + debug(`Unable to warm cache for catalog ${catalogResponse.spec.name}`) + } return catalogResponse } @@ -1679,6 +1695,9 @@ export default class OtomiStack { await this.git.removeFile(filePath) await this.doDeleteDeployment([filePath]) + // delete the cached charts for this catalog + const cacheDir = `${env.CATALOG_CACHE_PATH}/${encodeURIComponent(name)}` + if (existsSync(cacheDir)) rmSync(cacheDir, { recursive: true, force: true }) } async getWorkloadCatalog(data: { @@ -1703,11 +1722,23 @@ export default class OtomiStack { branch: string, catalogName: string, chartsPath?: string, + forceRefresh: boolean = false, ): Promise<{ url: string; helmCharts: any; catalog: any; chartsPath?: string }> { - const uuid = uuidv4() - const helmChartsDir = `/tmp/otomi/charts/${catalogName}/${branch}/charts/${uuid}` + const encodedCatalogName = encodeURIComponent(catalogName) + const encodedBranch = encodeURIComponent(branch) + const helmChartsDir = `${env.CATALOG_CACHE_PATH}/${encodedCatalogName}/${encodedBranch}` + + return this.fetchCatalog(url, helmChartsDir, branch, undefined, chartsPath, true, forceRefresh) + } - return this.fetchCatalog(url, helmChartsDir, branch, undefined, chartsPath) + async refreshBYOCatalogCache(catalogName?: string): Promise { + const catalogs = this.getAllAplCatalogs({ enabled: true }) + const selectedCatalogs = catalogName ? catalogs.filter((catalog) => catalog.spec.name === catalogName) : catalogs + + for (const catalog of selectedCatalogs) { + const { repositoryUrl, branch, name, chartsPath } = catalog.spec + await this.getBYOWorkloadCatalog(repositoryUrl, branch, name, chartsPath as string | undefined, true) + } } async getAplCatalogCharts(name: string): Promise<{ url: string; helmCharts: any; catalog: any; branch: string }> { diff --git a/src/utils/workloadUtils.test.ts b/src/utils/workloadUtils.test.ts index fb1a0e6f..c2b18066 100644 --- a/src/utils/workloadUtils.test.ts +++ b/src/utils/workloadUtils.test.ts @@ -757,6 +757,27 @@ describe('fetchWorkloadCatalog', () => { expect(result).toEqual({ helmCharts: [], catalog: [] }) expect(fsPromises.readdir).not.toHaveBeenCalled() }) + + test('ignores folders without Chart.yaml', async () => { + const mockGit = { + clone: jest.fn().mockResolvedValue(undefined), + } + ;(simpleGit as jest.Mock).mockReturnValue(mockGit) + ;(fsPromises.readdir as jest.Mock).mockResolvedValue(['chart1', 'chart2']) + + const baseImpl = (utils.safeReadTextFile as jest.Mock).getMockImplementation()! + ;(utils.safeReadTextFile as jest.Mock).mockImplementation(async (_baseDir, filePath) => { + if (filePath.endsWith('chart2/Chart.yaml') || filePath.endsWith('chart2/chart.yaml')) { + throw new Error('Chart manifest not found') + } + return baseImpl(_baseDir, filePath) + }) + + const result = await fetchWorkloadCatalog(url, helmChartsDir, 'main', undefined, 'admin') + + expect(result.helmCharts).toEqual(['chart1']) + expect(result.helmCharts).not.toContain('chart2') + }) }) // ---------------------------------------------------------------- @@ -974,6 +995,7 @@ describe('Helper functions integration tests', () => { beforeEach(() => { jest.clearAllMocks() process.env = { ...originalEnv, GIT_USER: 'git-user', GIT_PASSWORD: 'git-password' } + ;(fs.lstatSync as jest.Mock).mockReturnValue({ isDirectory: () => true }) jest.spyOn(utils, 'safeReadTextFile').mockResolvedValue('# README') }) @@ -1201,19 +1223,26 @@ describe('Helper functions integration tests', () => { }) describe('getChartFolders (tested via fetchWorkloadCatalog)', () => { - test('excludes system files from chart list', async () => { + test('excludes hidden entries and non-directory files from chart list', async () => { const mockGit = { clone: jest.fn().mockResolvedValue(undefined) } ;(simpleGit as jest.Mock).mockReturnValue(mockGit) ;(fs.existsSync as jest.Mock).mockReturnValue(false) ;(fsPromises.readdir as jest.Mock).mockResolvedValue([ '.git', + '.github', '.gitignore', '.vscode', 'LICENSE', 'README.md', + 'notes.txt', 'chart1', 'chart2', ]) + ;(fs.lstatSync as jest.Mock).mockImplementation((targetPath: string) => { + const isDirectory = + targetPath.endsWith('/chart1') || targetPath.endsWith('/chart2') || targetPath.endsWith('/.github') + return { isDirectory: () => isDirectory } + }) ;(fsExtra.readFile as unknown as jest.Mock).mockImplementation((filePath) => { if (filePath.endsWith('rbac.yaml')) return Promise.reject(new Error('Not found')) return Promise.reject(new Error('File not found')) @@ -1221,7 +1250,7 @@ describe('Helper functions integration tests', () => { const result = await fetchWorkloadCatalog('https://example.com/charts.git', '/tmp/test', 'main') - // Should only include chart1 and chart2, not system files + // Should only include chart directories, excluding hidden folders and files expect(result.helmCharts).toEqual(['chart1', 'chart2']) }) }) diff --git a/src/utils/workloadUtils.ts b/src/utils/workloadUtils.ts index b512ba49..89ecada5 100644 --- a/src/utils/workloadUtils.ts +++ b/src/utils/workloadUtils.ts @@ -1,18 +1,25 @@ import axios from 'axios' import Debug from 'debug' -import { existsSync, lstatSync, mkdirSync, renameSync, rmSync } from 'fs' +import { existsSync, lstatSync, mkdirSync, renameSync, rmSync, statSync } from 'fs' import { readFile } from 'fs-extra' import { readdir, writeFile } from 'fs/promises' import path from 'path' import simpleGit, { SimpleGit } from 'simple-git' import { safeReadTextFile } from 'src/utils' -import { cleanEnv, GIT_PROVIDER_URL_PATTERNS } from 'src/validators' +import { + CATALOG_CACHE_REFRESH_INTERVAL_MS, + CATALOG_CACHE_SYNC_MARKER, + cleanEnv, + GIT_PROVIDER_URL_PATTERNS, +} from 'src/validators' import YAML from 'yaml' import { BadRequestError } from '../error' const debug = Debug('apl:workloadUtils') const env = cleanEnv({ + CATALOG_CACHE_REFRESH_INTERVAL_MS, + CATALOG_CACHE_SYNC_MARKER, GIT_PROVIDER_URL_PATTERNS, }) @@ -231,6 +238,46 @@ export class chartRepo { async clone(branch: string = 'main') { await this.git.clone(this.chartRepoUrl, this.localPath, ['--branch', branch, '--single-branch']) } + async ensureLatest(branch: string = 'main', forceRefresh: boolean = false) { + const gitDir = path.join(this.localPath, '.git') + const cacheSyncMarkerPath = path.join(this.localPath, env.CATALOG_CACHE_SYNC_MARKER) + if (!existsSync(gitDir)) { + debug(`Catalog cache miss at ${this.localPath}; cloning branch '${branch}'`) + await this.clone(branch) + await writeFile(cacheSyncMarkerPath, new Date().toISOString(), 'utf-8') + debug(`Catalog cache initialized at ${this.localPath}`) + return + } + + if (!forceRefresh && existsSync(cacheSyncMarkerPath)) { + const cacheAgeMs = Date.now() - statSync(cacheSyncMarkerPath).mtimeMs + if (cacheAgeMs < env.CATALOG_CACHE_REFRESH_INTERVAL_MS) { + debug( + `Catalog cache hit at ${this.localPath}; age=${Math.round(cacheAgeMs / 1000)}s, ttl=${Math.round(env.CATALOG_CACHE_REFRESH_INTERVAL_MS / 1000)}s`, + ) + return + } + debug( + `Catalog cache expired at ${this.localPath}; age=${Math.round(cacheAgeMs / 1000)}s, ttl=${Math.round(env.CATALOG_CACHE_REFRESH_INTERVAL_MS / 1000)}s`, + ) + } else if (forceRefresh) { + debug(`Catalog cache force-refresh requested at ${this.localPath}`) + } else { + debug(`Catalog cache marker missing at ${this.localPath}; refreshing`) + } + + debug(`Refreshing catalog cache at ${this.localPath} for branch '${branch}'`) + await this.git.cwd(this.localPath) + await this.git.fetch('origin', branch) + try { + await this.git.checkout(branch) + } catch { + await this.git.checkout(['-B', branch, `origin/${branch}`]) + } + await this.git.pull('origin', branch, { '--ff-only': null }) + await writeFile(cacheSyncMarkerPath, new Date().toISOString(), 'utf-8') + debug(`Catalog cache refreshed at ${this.localPath}`) + } async cloneSingleChart(refAndPath: string, finalDestinationPath: string) { const remoteResult = await this.git.listRemote([this.chartRepoUrl]) const { branches, tags } = getBranchesAndTags(remoteResult) @@ -445,8 +492,26 @@ async function processChartFolder( */ async function getChartFolders(helmChartsDir: string): Promise { const files = await readdir(helmChartsDir, 'utf-8') - const filesToExclude = ['.git', '.gitignore', '.vscode', 'LICENSE', 'README.md'] - return files.filter((f) => !filesToExclude.includes(f)) + const chartFolders = await Promise.all( + files.map(async (fileName) => { + try { + const filePath = path.join(helmChartsDir, fileName) + if (!lstatSync(filePath).isDirectory()) return null + + try { + await safeReadTextFile(helmChartsDir, `${fileName}/Chart.yaml`) + return fileName + } catch { + await safeReadTextFile(helmChartsDir, `${fileName}/chart.yaml`) + return fileName + } + } catch { + return null + } + }), + ) + + return chartFolders.filter((folder): folder is string => folder !== null) } /** @@ -466,6 +531,7 @@ export async function fetchWorkloadCatalog( clusterDomainSuffix?: string, teamId?: string, chartsPath?: string, + forceRefresh: boolean = false, ): Promise<{ helmCharts: string[]; catalog: any[] }> { const resolvedHelmChartsDir = path.resolve(helmChartsDir) @@ -475,7 +541,7 @@ export async function fetchWorkloadCatalog( // Clone repository const gitUrl = encodeGitCredentials(url, clusterDomainSuffix) const gitRepo = new chartRepo(resolvedHelmChartsDir, gitUrl) - await gitRepo.clone(branch) + await gitRepo.ensureLatest(branch, forceRefresh) // Determine the charts directory path const chartsDir = chartsPath ? path.resolve(resolvedHelmChartsDir, chartsPath) : resolvedHelmChartsDir @@ -507,7 +573,10 @@ export async function fetchWorkloadCatalog( // Get chart folders const folders = await getChartFolders(chartsDir) - + if (!folders.length) { + debug(`No chart folders found in '${chartsDir}' at '${url}'`) + return { helmCharts: [], catalog: [] } + } // Read RBAC configuration (try chartsDir first, fallback to root) let rbacConfig = await readRbacConfig(chartsDir) if (!rbacConfig.rbac || Object.keys(rbacConfig.rbac).length === 0) { diff --git a/src/validators.ts b/src/validators.ts index d97f7874..cc1b85e1 100644 --- a/src/validators.ts +++ b/src/validators.ts @@ -9,6 +9,18 @@ export const AUTHZ_MOCK_IS_TEAM_ADMIN = bool({ default: true, }) export const AUTHZ_MOCK_TEAM = str({ desc: 'Comma separated list of teams a user belongs to', default: undefined }) +export const CATALOG_CACHE_REFRESH_INTERVAL_MS = num({ + desc: 'Interval in milliseconds for refreshing the BYO catalog cache', + default: 600000, // 10 minutes +}) +export const CATALOG_CACHE_PATH = str({ + desc: 'The file path for the BYO catalog cache', + default: '/tmp/otomi/charts-cache', +}) +export const CATALOG_CACHE_SYNC_MARKER = str({ + desc: 'The file name for the BYO catalog cache sync marker', + default: '.apl-cache-last-sync', +}) export const DEFAULT_PLATFORM_ADMIN_EMAIL = str({ desc: 'The email address for the default platform admin user.', devDefault: 'platform-admin@dev.linode-apl.net', From 43d4b2db2440e331d243e210a8016640a51befb4 Mon Sep 17 00:00:00 2001 From: ElderMatt <18527012+ElderMatt@users.noreply.github.com> Date: Tue, 10 Mar 2026 10:02:32 +0100 Subject: [PATCH 02/13] fix: deleted object files --- [object Object]/bitnami/main | 1 - [object Object]/cert-manager/master | 1 - [object Object]/default/main | 1 - [object Object]/empty/no-k8s | 1 - [object Object]/exporter/main | 1 - 5 files changed, 5 deletions(-) delete mode 160000 [object Object]/bitnami/main delete mode 160000 [object Object]/cert-manager/master delete mode 160000 [object Object]/default/main delete mode 160000 [object Object]/empty/no-k8s delete mode 160000 [object Object]/exporter/main diff --git a/[object Object]/bitnami/main b/[object Object]/bitnami/main deleted file mode 160000 index f9057073..00000000 --- a/[object Object]/bitnami/main +++ /dev/null @@ -1 +0,0 @@ -Subproject commit f90570733f27531dcac1bf81700934ca9f24fc4f diff --git a/[object Object]/cert-manager/master b/[object Object]/cert-manager/master deleted file mode 160000 index 6eaa3a79..00000000 --- a/[object Object]/cert-manager/master +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 6eaa3a794234bf88300b4185fc4c6d1bc069c929 diff --git a/[object Object]/default/main b/[object Object]/default/main deleted file mode 160000 index 305fe68d..00000000 --- a/[object Object]/default/main +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 305fe68d6f7c46f34ba07748a8be44e5e379a91f diff --git a/[object Object]/empty/no-k8s b/[object Object]/empty/no-k8s deleted file mode 160000 index baa7656e..00000000 --- a/[object Object]/empty/no-k8s +++ /dev/null @@ -1 +0,0 @@ -Subproject commit baa7656e9707b4e95132bf5c4fd3e3d9b8aa127d diff --git a/[object Object]/exporter/main b/[object Object]/exporter/main deleted file mode 160000 index 1c0c69b0..00000000 --- a/[object Object]/exporter/main +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 1c0c69b0ad194558c1884f5af496a1bb4eb17e61 From 41a14dcd3d4a97332c0e3fb6bf70450b8d054a9d Mon Sep 17 00:00:00 2001 From: ElderMatt <18527012+ElderMatt@users.noreply.github.com> Date: Tue, 10 Mar 2026 11:19:45 +0100 Subject: [PATCH 03/13] feat: return object when successfully refreshed and testing 1 minute refresh --- src/api/v2/catalogs/refresh.ts | 10 ++++++++-- src/validators.ts | 2 +- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/api/v2/catalogs/refresh.ts b/src/api/v2/catalogs/refresh.ts index 76d7bb28..7b30c27f 100644 --- a/src/api/v2/catalogs/refresh.ts +++ b/src/api/v2/catalogs/refresh.ts @@ -1,5 +1,6 @@ import Debug from 'debug' import { Response } from 'express' +import { BadRequestError } from 'src/error' import { OpenApiRequestExt } from 'src/otomi-models' const debug = Debug('otomi:api:v2:catalogs:refresh') @@ -11,6 +12,11 @@ const debug = Debug('otomi:api:v2:catalogs:refresh') export const refreshAplCatalogCache = async (req: OpenApiRequestExt, res: Response): Promise => { const catalogId = typeof req.query.catalogId === 'string' ? decodeURIComponent(req.query.catalogId) : undefined debug(`refreshAplCatalogCache(${catalogId || 'all'})`) - const data = await req.otomi.refreshBYOCatalogCache(catalogId) - res.json(data) + try { + await req.otomi.refreshBYOCatalogCache(catalogId) + res.status(200).json({ code: 200, message: 'Successfully refreshed catalog cache' }) + } catch (e) { + debug('refreshAplCatalogCache failed', e) + res.status(400).json(new BadRequestError('Failed to refresh catalog cache')) + } } diff --git a/src/validators.ts b/src/validators.ts index cc1b85e1..008ccfd0 100644 --- a/src/validators.ts +++ b/src/validators.ts @@ -11,7 +11,7 @@ export const AUTHZ_MOCK_IS_TEAM_ADMIN = bool({ export const AUTHZ_MOCK_TEAM = str({ desc: 'Comma separated list of teams a user belongs to', default: undefined }) export const CATALOG_CACHE_REFRESH_INTERVAL_MS = num({ desc: 'Interval in milliseconds for refreshing the BYO catalog cache', - default: 600000, // 10 minutes + default: 60000, // 1 minute }) export const CATALOG_CACHE_PATH = str({ desc: 'The file path for the BYO catalog cache', From 4a15357c1d4fcd29acb5c7a2fba688bbc263c388 Mon Sep 17 00:00:00 2001 From: ElderMatt <18527012+ElderMatt@users.noreply.github.com> Date: Tue, 10 Mar 2026 13:34:43 +0100 Subject: [PATCH 04/13] fix: tests --- src/utils/workloadUtils.ts | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/utils/workloadUtils.ts b/src/utils/workloadUtils.ts index 89ecada5..c6cff327 100644 --- a/src/utils/workloadUtils.ts +++ b/src/utils/workloadUtils.ts @@ -1,6 +1,6 @@ import axios from 'axios' import Debug from 'debug' -import { existsSync, lstatSync, mkdirSync, renameSync, rmSync, statSync } from 'fs' +import { existsSync, lstatSync, mkdirSync, renameSync, rmSync } from 'fs' import { readFile } from 'fs-extra' import { readdir, writeFile } from 'fs/promises' import path from 'path' @@ -241,7 +241,13 @@ export class chartRepo { async ensureLatest(branch: string = 'main', forceRefresh: boolean = false) { const gitDir = path.join(this.localPath, '.git') const cacheSyncMarkerPath = path.join(this.localPath, env.CATALOG_CACHE_SYNC_MARKER) - if (!existsSync(gitDir)) { + const canRefreshExistingRepo = + typeof this.git.cwd === 'function' && + typeof this.git.fetch === 'function' && + typeof this.git.checkout === 'function' && + typeof this.git.pull === 'function' + + if (!existsSync(gitDir) || !canRefreshExistingRepo) { debug(`Catalog cache miss at ${this.localPath}; cloning branch '${branch}'`) await this.clone(branch) await writeFile(cacheSyncMarkerPath, new Date().toISOString(), 'utf-8') @@ -250,7 +256,7 @@ export class chartRepo { } if (!forceRefresh && existsSync(cacheSyncMarkerPath)) { - const cacheAgeMs = Date.now() - statSync(cacheSyncMarkerPath).mtimeMs + const cacheAgeMs = Date.now() - lstatSync(cacheSyncMarkerPath).mtimeMs if (cacheAgeMs < env.CATALOG_CACHE_REFRESH_INTERVAL_MS) { debug( `Catalog cache hit at ${this.localPath}; age=${Math.round(cacheAgeMs / 1000)}s, ttl=${Math.round(env.CATALOG_CACHE_REFRESH_INTERVAL_MS / 1000)}s`, @@ -495,6 +501,7 @@ async function getChartFolders(helmChartsDir: string): Promise { const chartFolders = await Promise.all( files.map(async (fileName) => { try { + if (fileName.startsWith('.')) return null const filePath = path.join(helmChartsDir, fileName) if (!lstatSync(filePath).isDirectory()) return null From 4d7340d493efe5935a25e860830fcbb301d1926f Mon Sep 17 00:00:00 2001 From: ElderMatt <18527012+ElderMatt@users.noreply.github.com> Date: Tue, 10 Mar 2026 14:47:21 +0100 Subject: [PATCH 05/13] fix: change clone strategy --- src/utils/workloadUtils.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/workloadUtils.ts b/src/utils/workloadUtils.ts index c6cff327..b52df623 100644 --- a/src/utils/workloadUtils.ts +++ b/src/utils/workloadUtils.ts @@ -236,7 +236,7 @@ export class chartRepo { this.git = simpleGit(this.localPath) } async clone(branch: string = 'main') { - await this.git.clone(this.chartRepoUrl, this.localPath, ['--branch', branch, '--single-branch']) + await this.git.clone(this.chartRepoUrl, this.localPath, ['--branch', branch, '--single-branch', '--depth', '1']) } async ensureLatest(branch: string = 'main', forceRefresh: boolean = false) { const gitDir = path.join(this.localPath, '.git') From ad73ca7e572d4c035659e0ef0da9a0c7a1887f69 Mon Sep 17 00:00:00 2001 From: ElderMatt <18527012+ElderMatt@users.noreply.github.com> Date: Tue, 10 Mar 2026 14:49:13 +0100 Subject: [PATCH 06/13] fix: tests --- src/utils/workloadUtils.test.ts | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/src/utils/workloadUtils.test.ts b/src/utils/workloadUtils.test.ts index c2b18066..79de2053 100644 --- a/src/utils/workloadUtils.test.ts +++ b/src/utils/workloadUtils.test.ts @@ -638,7 +638,7 @@ describe('fetchWorkloadCatalog', () => { expect(mockGit.clone).toHaveBeenCalledWith( 'https://git-user:git-password@gitea.example.com/otomi/charts.git', helmChartsDir, - ['--branch', 'main', '--single-branch'], + ['--branch', 'main', '--single-branch', '--depth', '1'], ) expect(result).toEqual({ helmCharts: ['chart1', 'chart2'], @@ -811,7 +811,13 @@ describe('chartRepo', () => { const repo = new chartRepo(localPath, chartRepoUrl, gitUser, gitEmail) await repo.clone() - expect(mockGit.clone).toHaveBeenCalledWith(chartRepoUrl, localPath, ['--branch', 'main', '--single-branch']) + expect(mockGit.clone).toHaveBeenCalledWith(chartRepoUrl, localPath, [ + '--branch', + 'main', + '--single-branch', + '--depth', + '1', + ]) }) test('cloneSingleChart method performs sparse checkout', async () => { @@ -1028,7 +1034,7 @@ describe('Helper functions integration tests', () => { expect(mockGit.clone).toHaveBeenCalledWith( 'https://git-user:git-password@gitea.cluster.local/otomi/charts.git', helmChartsDir, - ['--branch', 'main', '--single-branch'], + ['--branch', 'main', '--single-branch', '--depth', '1'], ) }) @@ -1052,7 +1058,13 @@ describe('Helper functions integration tests', () => { } // Verify that clone was called with original URL - expect(mockGit.clone).toHaveBeenCalledWith(githubUrl, helmChartsDir, ['--branch', 'main', '--single-branch']) + expect(mockGit.clone).toHaveBeenCalledWith(githubUrl, helmChartsDir, [ + '--branch', + 'main', + '--single-branch', + '--depth', + '1', + ]) }) }) From b62f2f96d8d019d09490e94e6f8099cb26772fad Mon Sep 17 00:00:00 2001 From: ElderMatt <18527012+ElderMatt@users.noreply.github.com> Date: Tue, 10 Mar 2026 14:57:49 +0100 Subject: [PATCH 07/13] feat: create and edit catalog change to void for getbyoworkloadcatalog --- src/otomi-stack.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/otomi-stack.ts b/src/otomi-stack.ts index dd77cceb..27adb688 100644 --- a/src/otomi-stack.ts +++ b/src/otomi-stack.ts @@ -1654,7 +1654,7 @@ export default class OtomiStack { await this.doDeployments([aplRecord], false) try { const { repositoryUrl, branch, name, chartsPath } = data.spec - await this.getBYOWorkloadCatalog(repositoryUrl, branch, name, chartsPath as string | undefined, true) + void this.getBYOWorkloadCatalog(repositoryUrl, branch, name, chartsPath as string | undefined, true) } catch { debug(`Unable to warm cache for catalog ${data.spec.name}`) } @@ -1683,7 +1683,7 @@ export default class OtomiStack { await this.doDeployment(aplRecord, false) try { const { repositoryUrl, branch, name: catalogName, chartsPath } = catalogResponse.spec - await this.getBYOWorkloadCatalog(repositoryUrl, branch, catalogName, chartsPath as string | undefined, true) + void this.getBYOWorkloadCatalog(repositoryUrl, branch, catalogName, chartsPath as string | undefined, true) } catch { debug(`Unable to warm cache for catalog ${catalogResponse.spec.name}`) } From 24a3f00e82b30fadb605071d95785bb1c972c15f Mon Sep 17 00:00:00 2001 From: ElderMatt <18527012+ElderMatt@users.noreply.github.com> Date: Wed, 11 Mar 2026 16:35:03 +0100 Subject: [PATCH 08/13] feat: enabled caching check for sparsecheckout --- src/otomi-stack.ts | 10 +++++----- src/utils/workloadUtils.ts | 4 +++- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/otomi-stack.ts b/src/otomi-stack.ts index 19400f1f..801a11b7 100644 --- a/src/otomi-stack.ts +++ b/src/otomi-stack.ts @@ -1761,9 +1761,11 @@ export default class OtomiStack { const catalog = this.getAplCatalog(name) const { repositoryUrl, branch, chartsPath } = catalog.spec const { cluster } = this.getSettings(['cluster']) - - const uuid = uuidv4() - const helmChartsDir = `/tmp/otomi/charts/${name}/${branch}/chart/${uuid}` + const encodedCatalogName = encodeURIComponent(catalog.spec.name) + const encodedBranch = encodeURIComponent(branch) + const helmChartsDir = chartsPath + ? `${env.CATALOG_CACHE_PATH}/${encodedCatalogName}/${encodedBranch}/${chartsPath}` + : `${env.CATALOG_CACHE_PATH}/${encodedCatalogName}/${encodedBranch}` try { const chart = await fetchWorkloadCatalogChart( @@ -1780,8 +1782,6 @@ export default class OtomiStack { } catch (error) { debug(`Error fetching workload chart '${chartName}': ${error.message}`) return { url: repositoryUrl, branch, chart: null, chartsPath } - } finally { - if (existsSync(helmChartsDir)) rmSync(helmChartsDir, { recursive: true, force: true }) } } diff --git a/src/utils/workloadUtils.ts b/src/utils/workloadUtils.ts index 8fe48617..b8d9f156 100644 --- a/src/utils/workloadUtils.ts +++ b/src/utils/workloadUtils.ts @@ -406,7 +406,9 @@ export async function sparseCheckoutPath( const finalDestinationPath = join(targetBaseDir, targetDirName) try { - rmSync(finalDestinationPath, { recursive: true, force: true }) + if (existsSync(finalDestinationPath)) { + return { success: true, checkoutPath: finalDestinationPath } + } const normalizedSparsePath = sparsePath.replace(/^\/+/, '').replace(/\/+$/, '') const refAndPath = `${ref}/${normalizedSparsePath}` From 64f14722a161fb04aae4c6f28bbe26933d5551fa Mon Sep 17 00:00:00 2001 From: ElderMatt <18527012+ElderMatt@users.noreply.github.com> Date: Wed, 11 Mar 2026 16:39:17 +0100 Subject: [PATCH 09/13] fix: remove commented lines --- src/app.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/app.ts b/src/app.ts index bccaced7..f6ec68da 100644 --- a/src/app.ts +++ b/src/app.ts @@ -240,8 +240,6 @@ export async function initApp(inOtomiStack?: OtomiStack) { .listen(PORT, async () => { debug(`Listening on :::${PORT}`) lightship?.signalReady() - // // Clone repo after the application is ready to avoid Pod NotReady phenomenon, and thus infinite Pod crash loopback - // ;(await getSessionStack()).initGit() // Initialize git, warm catalog cache, and then start periodic refresh in one sequenced background task void (async () => { From c0d6f073253a81564f26f2300be26b17099b9003 Mon Sep 17 00:00:00 2001 From: ElderMatt <18527012+ElderMatt@users.noreply.github.com> Date: Wed, 11 Mar 2026 16:57:47 +0100 Subject: [PATCH 10/13] feat: remove redundant feature for cloning single chart --- src/otomi-stack.ts | 11 +---------- src/utils/workloadUtils.ts | 22 +++------------------- 2 files changed, 4 insertions(+), 29 deletions(-) diff --git a/src/otomi-stack.ts b/src/otomi-stack.ts index 801a11b7..8d6a88af 100644 --- a/src/otomi-stack.ts +++ b/src/otomi-stack.ts @@ -1760,7 +1760,6 @@ export default class OtomiStack { ): Promise<{ url: string; branch: string; chart: any | null; chartsPath?: string }> { const catalog = this.getAplCatalog(name) const { repositoryUrl, branch, chartsPath } = catalog.spec - const { cluster } = this.getSettings(['cluster']) const encodedCatalogName = encodeURIComponent(catalog.spec.name) const encodedBranch = encodeURIComponent(branch) const helmChartsDir = chartsPath @@ -1768,15 +1767,7 @@ export default class OtomiStack { : `${env.CATALOG_CACHE_PATH}/${encodedCatalogName}/${encodedBranch}` try { - const chart = await fetchWorkloadCatalogChart( - repositoryUrl, - helmChartsDir, - chartName, - branch, - cluster?.domainSuffix, - undefined, - chartsPath as string | undefined, - ) + const chart = await fetchWorkloadCatalogChart(helmChartsDir, chartName) return { url: repositoryUrl, branch, chart, chartsPath } } catch (error) { diff --git a/src/utils/workloadUtils.ts b/src/utils/workloadUtils.ts index b8d9f156..6919269a 100644 --- a/src/utils/workloadUtils.ts +++ b/src/utils/workloadUtils.ts @@ -648,32 +648,16 @@ export async function fetchWorkloadCatalog( return { helmCharts, catalog } } -export async function fetchWorkloadCatalogChart( - url: string, - helmChartsDir: string, - chartName: string, - branch: string = 'main', - clusterDomainSuffix?: string, - teamId?: string, - chartsPath?: string, -): Promise { +export async function fetchWorkloadCatalogChart(helmChartsDir: string, chartName: string): Promise { const resolvedHelmChartsDir = path.resolve(helmChartsDir) if (!existsSync(resolvedHelmChartsDir)) { mkdirSync(resolvedHelmChartsDir, { recursive: true }) } - const gitUrl = encodeGitCredentials(url, clusterDomainSuffix) - - const sparsePath = chartsPath ? `${chartsPath}/${chartName}` : chartName - const checkoutResult = await sparseCheckoutPath(gitUrl, branch, sparsePath, resolvedHelmChartsDir, chartName) - - if (!checkoutResult.success) { - debug(`Sparse checkout failed for chart '${chartName}' from '${url}': ${checkoutResult.error}`) - return null - } + const finalDestinationPath = join(resolvedHelmChartsDir, chartName) - const chartDir = checkoutResult.checkoutPath + const chartDir = finalDestinationPath try { const values = await safeReadTextFile(chartDir, 'values.yaml') From 5f55c23ba1f4d4ba616c882bc5d053384bacd816 Mon Sep 17 00:00:00 2001 From: ElderMatt <18527012+ElderMatt@users.noreply.github.com> Date: Wed, 11 Mar 2026 17:00:20 +0100 Subject: [PATCH 11/13] chore: change back refresh cache to minutes --- src/validators.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/validators.ts b/src/validators.ts index 008ccfd0..cc1b85e1 100644 --- a/src/validators.ts +++ b/src/validators.ts @@ -11,7 +11,7 @@ export const AUTHZ_MOCK_IS_TEAM_ADMIN = bool({ export const AUTHZ_MOCK_TEAM = str({ desc: 'Comma separated list of teams a user belongs to', default: undefined }) export const CATALOG_CACHE_REFRESH_INTERVAL_MS = num({ desc: 'Interval in milliseconds for refreshing the BYO catalog cache', - default: 60000, // 1 minute + default: 600000, // 10 minutes }) export const CATALOG_CACHE_PATH = str({ desc: 'The file path for the BYO catalog cache', From 19dd7bd143fa68302ec5f76505767cf953089e37 Mon Sep 17 00:00:00 2001 From: ElderMatt <18527012+ElderMatt@users.noreply.github.com> Date: Thu, 12 Mar 2026 09:43:51 +0100 Subject: [PATCH 12/13] fix: removed getworkloadcatalogchart and remove try catch on create catalog --- src/otomi-stack.ts | 23 +++++++++---------- src/utils/workloadUtils.ts | 47 -------------------------------------- 2 files changed, 11 insertions(+), 59 deletions(-) diff --git a/src/otomi-stack.ts b/src/otomi-stack.ts index 8d6a88af..1499eca3 100644 --- a/src/otomi-stack.ts +++ b/src/otomi-stack.ts @@ -139,7 +139,6 @@ import { defineClusterId, ObjectStorageClient } from './utils/wizardUtils' import { fetchChartYaml, fetchWorkloadCatalog, - fetchWorkloadCatalogChart, isInteralGiteaURL, NewHelmChartValues, sparseCloneChart, @@ -1653,12 +1652,10 @@ export default class OtomiStack { const aplRecord = await this.saveCatalog(data) await this.doDeployments([aplRecord], false) - try { - const { repositoryUrl, branch, name, chartsPath } = data.spec - void this.getBYOWorkloadCatalog(repositoryUrl, branch, name, chartsPath as string | undefined, true) - } catch { - debug(`Unable to warm cache for catalog ${data.spec.name}`) - } + const { repositoryUrl, branch, name, chartsPath } = data.spec + void this.getBYOWorkloadCatalog(repositoryUrl, branch, name, chartsPath as string | undefined, true).catch((e) => + debug(`Unable to warm cache for catalog ${data.spec.name}`, e), + ) return aplRecord.content as AplCatalogResponse } @@ -1760,16 +1757,18 @@ export default class OtomiStack { ): Promise<{ url: string; branch: string; chart: any | null; chartsPath?: string }> { const catalog = this.getAplCatalog(name) const { repositoryUrl, branch, chartsPath } = catalog.spec + const { cluster } = this.getSettings(['cluster']) const encodedCatalogName = encodeURIComponent(catalog.spec.name) const encodedBranch = encodeURIComponent(branch) - const helmChartsDir = chartsPath - ? `${env.CATALOG_CACHE_PATH}/${encodedCatalogName}/${encodedBranch}/${chartsPath}` - : `${env.CATALOG_CACHE_PATH}/${encodedCatalogName}/${encodedBranch}` + const helmChartsDir = `${env.CATALOG_CACHE_PATH}/${encodedCatalogName}/${encodedBranch}` try { - const chart = await fetchWorkloadCatalogChart(helmChartsDir, chartName) + const singleChart = + ( + await fetchWorkloadCatalog(repositoryUrl, helmChartsDir, branch, cluster?.domainSuffix, undefined, chartsPath) + ).catalog.find((c) => c.name === chartName) || null - return { url: repositoryUrl, branch, chart, chartsPath } + return { url: repositoryUrl, branch, chart: singleChart, chartsPath } } catch (error) { debug(`Error fetching workload chart '${chartName}': ${error.message}`) return { url: repositoryUrl, branch, chart: null, chartsPath } diff --git a/src/utils/workloadUtils.ts b/src/utils/workloadUtils.ts index 6919269a..39c389e3 100644 --- a/src/utils/workloadUtils.ts +++ b/src/utils/workloadUtils.ts @@ -647,50 +647,3 @@ export async function fetchWorkloadCatalog( return { helmCharts, catalog } } - -export async function fetchWorkloadCatalogChart(helmChartsDir: string, chartName: string): Promise { - const resolvedHelmChartsDir = path.resolve(helmChartsDir) - - if (!existsSync(resolvedHelmChartsDir)) { - mkdirSync(resolvedHelmChartsDir, { recursive: true }) - } - - const finalDestinationPath = join(resolvedHelmChartsDir, chartName) - - const chartDir = finalDestinationPath - - try { - const values = await safeReadTextFile(chartDir, 'values.yaml') - - let valuesSchema = '{}' - try { - const schemaContent = await safeReadTextFile(chartDir, 'values.schema.json') - valuesSchema = schemaContent || '{}' - } catch { - // optional - } - - const chartYaml = await safeReadTextFile(chartDir, 'Chart.yaml') - const chartMetadata = YAML.parse(chartYaml) - - let readme = 'There is no `README` for this chart.' - try { - readme = await safeReadTextFile(chartDir, 'README.md') - } catch { - // optional - } - - return { - name: chartName, - values: values || '{}', - valuesSchema, - icon: chartMetadata?.icon, - chartVersion: chartMetadata?.version, - chartDescription: chartMetadata?.description, - readme, - } - } catch (error) { - debug(`Error parsing chart '${chartName}' in '${chartDir}': ${error.message}`) - return null - } -} From 9c776a07aa5077989dcc35d7cd31ad8143b5d18e Mon Sep 17 00:00:00 2001 From: ElderMatt <18527012+ElderMatt@users.noreply.github.com> Date: Thu, 12 Mar 2026 09:47:07 +0100 Subject: [PATCH 13/13] fix: remove sparsecheckoutpath --- src/utils/workloadUtils.ts | 41 ++------------------------------------ 1 file changed, 2 insertions(+), 39 deletions(-) diff --git a/src/utils/workloadUtils.ts b/src/utils/workloadUtils.ts index 39c389e3..b52df623 100644 --- a/src/utils/workloadUtils.ts +++ b/src/utils/workloadUtils.ts @@ -1,10 +1,9 @@ import axios from 'axios' import Debug from 'debug' -import { existsSync, lstatSync, mkdirSync, mkdtempSync, renameSync, rmSync } from 'fs' +import { existsSync, lstatSync, mkdirSync, renameSync, rmSync } from 'fs' import { readFile } from 'fs-extra' import { readdir, writeFile } from 'fs/promises' -import { tmpdir } from 'os' -import path, { join } from 'path' +import path from 'path' import simpleGit, { SimpleGit } from 'simple-git' import { safeReadTextFile } from 'src/utils' import { @@ -393,42 +392,6 @@ export async function sparseCloneChart( return true } -export async function sparseCheckoutPath( - gitCloneUrl: string, - ref: string, - sparsePath: string, - targetBaseDir: string, - targetDirName: string, -): Promise<{ success: true; checkoutPath: string } | { success: false; error: string }> { - if (!existsSync(targetBaseDir)) mkdirSync(targetBaseDir, { recursive: true }) - - const tempCloneDir = mkdtempSync(join(tmpdir(), 'sparse-checkout-')) - const finalDestinationPath = join(targetBaseDir, targetDirName) - - try { - if (existsSync(finalDestinationPath)) { - return { success: true, checkoutPath: finalDestinationPath } - } - - const normalizedSparsePath = sparsePath.replace(/^\/+/, '').replace(/\/+$/, '') - const refAndPath = `${ref}/${normalizedSparsePath}` - - const repo = new chartRepo(tempCloneDir, gitCloneUrl) - await repo.cloneSingleChart(refAndPath, finalDestinationPath) - - rmSync(join(finalDestinationPath, '.git'), { recursive: true, force: true }) - - return { success: true, checkoutPath: finalDestinationPath } - } catch (error) { - return { - success: false, - error: error instanceof Error ? error.message : 'Unknown sparse checkout error.', - } - } finally { - rmSync(tempCloneDir, { recursive: true, force: true }) - } -} - /** * Encodes Git credentials into the URL for internal Gitea repositories */