diff --git a/packages/realm-server/lib/realm-config-card-backfill.ts b/packages/realm-server/lib/realm-config-card-backfill.ts new file mode 100644 index 0000000000..ae69d27d54 --- /dev/null +++ b/packages/realm-server/lib/realm-config-card-backfill.ts @@ -0,0 +1,307 @@ +import { existsSync, readdirSync, readFileSync, writeFileSync } from 'fs'; +import { join } from 'path'; +import { + PUBLISHED_DIRECTORY_NAME, + logger, + param, + query, + type DBAdapter, +} from '@cardstack/runtime-common'; +import type { PgAdapter } from '@cardstack/postgres'; +import type { BootstrapRealmSeed } from './realm-registry-backfill'; + +const log = logger('realm-server:config-card-backfill'); + +// Distinct from REGISTRY_BACKFILL_LOCK_ID (7331011) and +// METADATA_BACKFILL_LOCK_ID (7331012) so the three boot-time backfills +// don't serialize on each other. +export const CONFIG_CARD_BACKFILL_LOCK_ID = 7331013; + +// CS-11150 creates a RealmConfig card instance at /realm.json for every +// realm that doesn't have one yet, populating it from the legacy +// .realm.json sidecar. After CS-10051, parseRealmInfo prefers the card +// file but falls back to the sidecar; the fallback can be removed +// (CS-11131) once this backfill has run in every environment. +// +// Skips when /realm.json already exists — the card is the source of +// truth, and the backfill never overwrites it. +// +// Fields owned by the RealmConfig card (see REALM_CONFIG_CARD_PROPERTIES +// in runtime-common/realm.ts). Anything outside this set stays in the +// sidecar (today: hostHome / interactHome, until CS-10055 lands). +const CARD_KEYS_TO_MIGRATE = [ + 'name', + 'backgroundURL', + 'iconURL', + 'hostRoutingRules', + 'includePrerenderedDefaultRealmIndex', +] as const; + +// Canonical RealmConfig adopts-from module. patchRealmConfig writes the +// same absolute URL and rejects anything else on subsequent edits, so +// the migrated card matches what the running server would have written +// itself. Resolving cross-realm to packages/base/realm-config.gts means +// per-realm copies of that file are not needed. +const REALM_CONFIG_MODULE = 'https://cardstack.com/base/realm-config'; +const REALM_CONFIG_NAME = 'RealmConfig'; + +export interface RealmConfigCardBackfillOpts { + dbAdapter: DBAdapter; + realmsRootPath: string; + serverURL: URL; + bootstrapRealms: BootstrapRealmSeed[]; +} + +export async function runRealmConfigCardBackfill( + opts: RealmConfigCardBackfillOpts, +): Promise { + const started = Date.now(); + log.info('starting realm.json card backfill'); + + const sourceCount = await safeStep('source', () => + backfillSourceRealms(opts), + ); + const publishedCount = await safeStep('published', () => + backfillPublishedRealms(opts), + ); + const bootstrapCount = await safeStep('bootstrap', () => + backfillBootstrapRealms(opts), + ); + + log.info( + `realm.json card backfill complete in ${Date.now() - started}ms ` + + `(source=${sourceCount ?? 0}, ` + + `published=${publishedCount ?? 0}, ` + + `bootstrap=${bootstrapCount ?? 0})`, + ); +} + +async function safeStep( + name: string, + fn: () => Promise, +): Promise { + try { + return await fn(); + } catch (err: unknown) { + log.warn( + `realm.json card backfill step "${name}" failed; continuing: ${String(err)}`, + ); + return undefined; + } +} + +// Materializes a RealmConfig card at cardPath from the migratable keys +// in the sidecar. Returns true when a card was actually written. +// +// Pre-existing card → leave both files alone (the card is source of +// truth; trimming the sidecar without reading the card would risk +// losing a value the card doesn't have). +// Pre-existing card absent, sidecar has zero migratable keys → no-op +// (don't write an empty card; a card with no attributes is not +// equivalent to "no card"). +function migrateOne(sidecarPath: string, cardPath: string, url: string): boolean { + if (existsSync(cardPath)) { + return false; + } + if (!existsSync(sidecarPath)) { + return false; + } + let raw: string; + try { + raw = readFileSync(sidecarPath, 'utf8'); + } catch (err: unknown) { + log.warn(`could not read ${sidecarPath}: ${String(err)}`); + return false; + } + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch (err: unknown) { + log.warn(`could not parse ${sidecarPath}: ${String(err)}`); + return false; + } + if (parsed === null || typeof parsed !== 'object' || Array.isArray(parsed)) { + return false; + } + const sidecar = parsed as Record; + + const cardAttrs: Record = {}; + const migratedKeys: string[] = []; + for (const key of CARD_KEYS_TO_MIGRATE) { + if (!(key in sidecar)) { + continue; + } + const value = sidecar[key]; + if (key === 'name') { + // `name` is stored under cardInfo.name on the card, matching what + // patchRealmConfig writes (see REALM_CONFIG_CARD_PROPERTIES handling). + cardAttrs.cardInfo = { name: value }; + } else { + cardAttrs[key] = value; + } + migratedKeys.push(key); + } + if (migratedKeys.length === 0) { + return false; + } + + const cardDoc = { + data: { + type: 'card', + attributes: cardAttrs, + meta: { + adoptsFrom: { + module: REALM_CONFIG_MODULE, + name: REALM_CONFIG_NAME, + }, + }, + }, + }; + + try { + writeFileSync(cardPath, JSON.stringify(cardDoc, null, 2) + '\n'); + } catch (err: unknown) { + log.warn(`could not write ${cardPath} for ${url}: ${String(err)}`); + return false; + } + + // Trim migrated keys from the sidecar. Leave non-migrated keys + // (notably hostHome / interactHome) so they can be picked up by a + // later migration (CS-10055). + const trimmed: Record = {}; + for (const [k, v] of Object.entries(sidecar)) { + if (!(migratedKeys as readonly string[]).includes(k)) { + trimmed[k] = v; + } + } + try { + writeFileSync(sidecarPath, JSON.stringify(trimmed, null, 2) + '\n'); + } catch (err: unknown) { + log.warn( + `could not trim migrated keys from ${sidecarPath}: ${String(err)}`, + ); + } + return true; +} + +async function backfillSourceRealms( + opts: RealmConfigCardBackfillOpts, +): Promise { + if (!existsSync(opts.realmsRootPath)) { + return 0; + } + let count = 0; + for (const ownerEntry of readdirSync(opts.realmsRootPath, { + withFileTypes: true, + })) { + if (!ownerEntry.isDirectory()) { + continue; + } + if (ownerEntry.name === PUBLISHED_DIRECTORY_NAME) { + continue; + } + const owner = ownerEntry.name; + const ownerDir = join(opts.realmsRootPath, owner); + for (const realmEntry of readdirSync(ownerDir, { withFileTypes: true })) { + if (!realmEntry.isDirectory()) { + continue; + } + const endpoint = realmEntry.name; + const realmDir = join(ownerDir, endpoint); + const sidecarPath = join(realmDir, '.realm.json'); + const cardPath = join(realmDir, 'realm.json'); + const url = new URL( + `${opts.serverURL.pathname.replace(/\/$/, '')}/${owner}/${endpoint}/`, + opts.serverURL, + ).href; + if (migrateOne(sidecarPath, cardPath, url)) { + count += 1; + } + } + } + return count; +} + +async function backfillPublishedRealms( + opts: RealmConfigCardBackfillOpts, +): Promise { + const publishedRoot = join(opts.realmsRootPath, PUBLISHED_DIRECTORY_NAME); + if (!existsSync(publishedRoot)) { + return 0; + } + // Map disk uuid → published URL via realm_registry. Mirrors the + // realm-metadata-backfill lookup; disk alone doesn't carry the URL. + const rows = (await query(opts.dbAdapter, [ + `SELECT disk_id, url FROM realm_registry WHERE kind = 'published'`, + ])) as Array<{ disk_id: string; url: string }>; + const byId = new Map(rows.map((r) => [r.disk_id, r.url])); + + let count = 0; + for (const entry of readdirSync(publishedRoot, { withFileTypes: true })) { + if (!entry.isDirectory()) { + continue; + } + const url = byId.get(entry.name); + if (!url) { + continue; + } + const realmDir = join(publishedRoot, entry.name); + const sidecarPath = join(realmDir, '.realm.json'); + const cardPath = join(realmDir, 'realm.json'); + if (migrateOne(sidecarPath, cardPath, url)) { + count += 1; + } + } + return count; +} + +async function backfillBootstrapRealms( + opts: RealmConfigCardBackfillOpts, +): Promise { + let count = 0; + for (const { diskPath, url } of opts.bootstrapRealms) { + const sidecarPath = join(diskPath, '.realm.json'); + const cardPath = join(diskPath, 'realm.json'); + if (migrateOne(sidecarPath, cardPath, url)) { + count += 1; + } + } + return count; +} + +// Multi-instance safety: if a peer process is mid-backfill, skip rather +// than racing. Mirrors the registry / metadata backfill advisory-lock +// pattern with a distinct lock id. +export async function runRealmConfigCardBackfillWithAdvisoryLock( + dbAdapter: PgAdapter, + opts: RealmConfigCardBackfillOpts, +): Promise { + await dbAdapter.withConnection(async (queryFn) => { + const rows = (await queryFn([ + `SELECT pg_try_advisory_lock(`, + param(CONFIG_CARD_BACKFILL_LOCK_ID), + `) AS acquired`, + ])) as [{ acquired: boolean }]; + if (!rows[0]?.acquired) { + log.info( + 'peer process holds the realm.json card backfill advisory lock; skipping', + ); + return; + } + try { + await runRealmConfigCardBackfill(opts); + } finally { + try { + await queryFn([ + `SELECT pg_advisory_unlock(`, + param(CONFIG_CARD_BACKFILL_LOCK_ID), + `)`, + ]); + } catch (err: unknown) { + log.warn( + `failed to release realm.json card backfill advisory lock: ${String(err)}`, + ); + } + } + }); +} diff --git a/packages/realm-server/main.ts b/packages/realm-server/main.ts index b7855d84a9..6121c71dbf 100644 --- a/packages/realm-server/main.ts +++ b/packages/realm-server/main.ts @@ -31,6 +31,7 @@ import { import { writeRuntimeMetadataFile } from './lib/runtime-metadata-file'; import { runRegistryBackfillWithAdvisoryLock } from './lib/realm-registry-backfill'; import { runRealmMetadataBackfillWithAdvisoryLock } from './lib/realm-metadata-backfill'; +import { runRealmConfigCardBackfillWithAdvisoryLock } from './lib/realm-config-card-backfill'; import { RealmRegistryReconciler, type RealmRegistryRow, @@ -371,6 +372,21 @@ const getIndexHTML = async () => { })), }); + // CS-11150: materialize a RealmConfig card at /realm.json from the + // legacy .realm.json sidecar wherever the card is still absent, then + // trim the migrated keys from the sidecar. Unblocks the sidecar + // removal in CS-11131. Idempotent — once a realm has a card, future + // boots skip it. + await runRealmConfigCardBackfillWithAdvisoryLock(dbAdapter, { + dbAdapter, + realmsRootPath, + serverURL: new URL(String(serverURL)), + bootstrapRealms: paths.map((p, i) => ({ + diskPath: String(p), + url: hrefs[i][0], + })), + }); + // Validate per-CLI-path username invariant. Phase 3 no longer constructs // realms here — the reconciler does it via mountFromRow once registry // rows are read — but we still want the misconfiguration to fail fast. diff --git a/packages/realm-server/tests/index.ts b/packages/realm-server/tests/index.ts index 0aed30e1bc..d22c63eed6 100644 --- a/packages/realm-server/tests/index.ts +++ b/packages/realm-server/tests/index.ts @@ -197,6 +197,7 @@ const ALL_TEST_FILES: string[] = [ './realm-endpoints/dependencies-test', './realm-advisory-locks-test', './realm-cleanup-transaction-test', + './realm-config-card-backfill-test', './realm-registry-backfill-test', './realm-registry-reconciler-test', './realm-registry-writes-test', diff --git a/packages/realm-server/tests/realm-config-card-backfill-test.ts b/packages/realm-server/tests/realm-config-card-backfill-test.ts new file mode 100644 index 0000000000..d434315d65 --- /dev/null +++ b/packages/realm-server/tests/realm-config-card-backfill-test.ts @@ -0,0 +1,377 @@ +import { module, test } from 'qunit'; +import { basename, join } from 'path'; +import { dirSync, type DirResult } from 'tmp'; +import { + ensureDirSync, + existsSync, + readFileSync, + writeFileSync, +} from 'fs-extra'; +import type { PgAdapter } from '@cardstack/postgres'; +import { + param, + query, + PUBLISHED_DIRECTORY_NAME, +} from '@cardstack/runtime-common'; +import { setupDB } from './helpers'; +import { runRealmConfigCardBackfill } from '../lib/realm-config-card-backfill'; + +function seedSidecar(realmDir: string, payload: Record) { + ensureDirSync(realmDir); + writeFileSync( + join(realmDir, '.realm.json'), + JSON.stringify(payload, null, 2), + ); +} + +function readSidecar(realmDir: string): unknown { + return JSON.parse(readFileSync(join(realmDir, '.realm.json'), 'utf8')); +} + +function readCard(realmDir: string): unknown { + return JSON.parse(readFileSync(join(realmDir, 'realm.json'), 'utf8')); +} + +function cardExists(realmDir: string): boolean { + return existsSync(join(realmDir, 'realm.json')); +} + +module(basename(__filename), function () { + module('runRealmConfigCardBackfill', function (hooks) { + let dbAdapter: PgAdapter; + let dir: DirResult; + let realmsRootPath: string; + const serverURL = new URL('http://localhost:4201/'); + + setupDB(hooks, { + beforeEach: async (_dbAdapter) => { + dbAdapter = _dbAdapter; + dir = dirSync({ unsafeCleanup: true }); + realmsRootPath = join(dir.name, 'realms'); + ensureDirSync(realmsRootPath); + }, + afterEach: async () => { + dir.removeCallback(); + }, + }); + + test('writes realm.json card from a source realm sidecar and trims migrated keys', async function (assert) { + const realmDir = join(realmsRootPath, 'luke', 'my-realm'); + seedSidecar(realmDir, { + name: 'My Realm', + backgroundURL: 'https://example.com/bg.png', + iconURL: 'https://example.com/icon.svg', + hostHome: 'https://hosted.example.com/', + }); + + await runRealmConfigCardBackfill({ + dbAdapter, + realmsRootPath, + serverURL, + bootstrapRealms: [], + }); + + assert.deepEqual( + readCard(realmDir), + { + data: { + type: 'card', + attributes: { + cardInfo: { name: 'My Realm' }, + backgroundURL: 'https://example.com/bg.png', + iconURL: 'https://example.com/icon.svg', + }, + meta: { + adoptsFrom: { + module: 'https://cardstack.com/base/realm-config', + name: 'RealmConfig', + }, + }, + }, + }, + 'realm.json card written in canonical RealmConfig shape', + ); + assert.deepEqual( + readSidecar(realmDir), + { hostHome: 'https://hosted.example.com/' }, + 'migrated keys trimmed; non-migrated keys (hostHome) preserved', + ); + }); + + test('skips when realm.json already exists and leaves both files untouched', async function (assert) { + const realmDir = join(realmsRootPath, 'luke', 'preserved'); + seedSidecar(realmDir, { + name: 'Sidecar Name', + backgroundURL: 'https://example.com/sidecar-bg.png', + }); + const existingCard = { + data: { + type: 'card', + attributes: { + cardInfo: { name: 'Card Name' }, + backgroundURL: 'https://example.com/card-bg.png', + }, + meta: { + adoptsFrom: { + module: 'https://cardstack.com/base/realm-config', + name: 'RealmConfig', + }, + }, + }, + }; + writeFileSync( + join(realmDir, 'realm.json'), + JSON.stringify(existingCard, null, 2), + ); + + await runRealmConfigCardBackfill({ + dbAdapter, + realmsRootPath, + serverURL, + bootstrapRealms: [], + }); + + assert.deepEqual( + readCard(realmDir), + existingCard, + 'existing card preserved', + ); + assert.deepEqual( + readSidecar(realmDir), + { + name: 'Sidecar Name', + backgroundURL: 'https://example.com/sidecar-bg.png', + }, + 'sidecar untouched when card already exists', + ); + }); + + test('skips realms whose sidecar has no migratable keys', async function (assert) { + const realmDir = join(realmsRootPath, 'luke', 'hosthome-only'); + seedSidecar(realmDir, { + hostHome: 'https://hosted.example.com/', + }); + + await runRealmConfigCardBackfill({ + dbAdapter, + realmsRootPath, + serverURL, + bootstrapRealms: [], + }); + + assert.notOk(cardExists(realmDir), 'no card written'); + assert.deepEqual( + readSidecar(realmDir), + { hostHome: 'https://hosted.example.com/' }, + 'sidecar untouched', + ); + }); + + test('skips realms whose sidecar is empty {}', async function (assert) { + const realmDir = join(realmsRootPath, 'luke', 'empty'); + seedSidecar(realmDir, {}); + + await runRealmConfigCardBackfill({ + dbAdapter, + realmsRootPath, + serverURL, + bootstrapRealms: [], + }); + + assert.notOk(cardExists(realmDir)); + assert.deepEqual(readSidecar(realmDir), {}); + }); + + test('skips realms with no sidecar at all', async function (assert) { + const realmDir = join(realmsRootPath, 'luke', 'no-sidecar'); + ensureDirSync(realmDir); + + await runRealmConfigCardBackfill({ + dbAdapter, + realmsRootPath, + serverURL, + bootstrapRealms: [], + }); + + assert.notOk(cardExists(realmDir)); + assert.notOk(existsSync(join(realmDir, '.realm.json'))); + }); + + test('migrates hostRoutingRules array verbatim', async function (assert) { + const realmDir = join(realmsRootPath, 'luke', 'with-routes'); + const routes = [ + { path: '/', instance: 'https://example.com/home' }, + { path: '/about', instance: 'https://example.com/about' }, + ]; + seedSidecar(realmDir, { + name: 'With Routes', + hostRoutingRules: routes, + }); + + await runRealmConfigCardBackfill({ + dbAdapter, + realmsRootPath, + serverURL, + bootstrapRealms: [], + }); + + const card = readCard(realmDir) as { + data: { attributes: Record }; + }; + assert.deepEqual(card.data.attributes.cardInfo, { name: 'With Routes' }); + assert.deepEqual( + card.data.attributes.hostRoutingRules, + routes, + 'hostRoutingRules migrated verbatim into card attributes', + ); + assert.deepEqual(readSidecar(realmDir), {}, 'sidecar fully trimmed'); + }); + + test('writes a card from a bootstrap realm sidecar', async function (assert) { + const bootstrapDir = join(dir.name, 'base'); + seedSidecar(bootstrapDir, { + name: 'Base Workspace', + backgroundURL: 'https://example.com/base-bg.jpg', + iconURL: 'https://example.com/base-icon.png', + }); + + await runRealmConfigCardBackfill({ + dbAdapter, + realmsRootPath, + serverURL, + bootstrapRealms: [ + { + diskPath: bootstrapDir, + url: 'https://cardstack.com/base/', + }, + ], + }); + + const card = readCard(bootstrapDir) as { + data: { attributes: Record }; + }; + assert.deepEqual(card.data.attributes.cardInfo, { + name: 'Base Workspace', + }); + assert.strictEqual( + card.data.attributes.backgroundURL, + 'https://example.com/base-bg.jpg', + ); + assert.deepEqual(readSidecar(bootstrapDir), {}); + }); + + test('writes a card from a published realm sidecar correlated via realm_registry', async function (assert) { + const publishedRoot = join(realmsRootPath, PUBLISHED_DIRECTORY_NAME); + const uuid = '00000000-0000-0000-0000-000000000001'; + const publishedRealmUrl = 'http://localhost:4201/_published/abc/'; + const publishedDir = join(publishedRoot, uuid); + seedSidecar(publishedDir, { + name: 'Published Realm', + iconURL: 'https://example.com/pub.svg', + }); + await query(dbAdapter, [ + `INSERT INTO realm_registry (url, kind, disk_id, owner_username, source_url, last_published_at, pinned) VALUES (`, + param(publishedRealmUrl), + `, 'published', `, + param(uuid), + `,`, + param('luke'), + `,`, + param('http://localhost:4201/luke/my-realm/'), + `,`, + param(Date.now()), + `, false)`, + ]); + + await runRealmConfigCardBackfill({ + dbAdapter, + realmsRootPath, + serverURL, + bootstrapRealms: [], + }); + + const card = readCard(publishedDir) as { + data: { attributes: Record }; + }; + assert.deepEqual(card.data.attributes.cardInfo, { + name: 'Published Realm', + }); + assert.strictEqual( + card.data.attributes.iconURL, + 'https://example.com/pub.svg', + ); + assert.deepEqual(readSidecar(publishedDir), {}); + }); + + test('is idempotent across reruns', async function (assert) { + const realmDir = join(realmsRootPath, 'luke', 'rerun'); + seedSidecar(realmDir, { + name: 'Rerun', + backgroundURL: 'https://example.com/bg.png', + }); + + await runRealmConfigCardBackfill({ + dbAdapter, + realmsRootPath, + serverURL, + bootstrapRealms: [], + }); + const firstCard = readCard(realmDir); + const firstSidecar = readSidecar(realmDir); + + await runRealmConfigCardBackfill({ + dbAdapter, + realmsRootPath, + serverURL, + bootstrapRealms: [], + }); + + assert.deepEqual(readCard(realmDir), firstCard, 'card unchanged on rerun'); + assert.deepEqual( + readSidecar(realmDir), + firstSidecar, + 'sidecar unchanged on rerun', + ); + }); + + test('tolerates malformed sidecar JSON without crashing', async function (assert) { + const realmDir = join(realmsRootPath, 'luke', 'broken'); + ensureDirSync(realmDir); + writeFileSync(join(realmDir, '.realm.json'), '{ not valid'); + + await runRealmConfigCardBackfill({ + dbAdapter, + realmsRootPath, + serverURL, + bootstrapRealms: [], + }); + + assert.notOk(cardExists(realmDir), 'no card written from broken sidecar'); + assert.strictEqual( + readFileSync(join(realmDir, '.realm.json'), 'utf8'), + '{ not valid', + 'malformed sidecar left untouched', + ); + }); + + test('tolerates a non-object sidecar JSON (array) without crashing', async function (assert) { + const realmDir = join(realmsRootPath, 'luke', 'array'); + ensureDirSync(realmDir); + writeFileSync(join(realmDir, '.realm.json'), '[1, 2, 3]'); + + await runRealmConfigCardBackfill({ + dbAdapter, + realmsRootPath, + serverURL, + bootstrapRealms: [], + }); + + assert.notOk(cardExists(realmDir)); + assert.strictEqual( + readFileSync(join(realmDir, '.realm.json'), 'utf8'), + '[1, 2, 3]', + 'array sidecar left untouched', + ); + }); + }); +});