From ea5c6f311b16622a8e2527466dfa1d6b94836e51 Mon Sep 17 00:00:00 2001 From: "Thomas F. K. Jorna" Date: Thu, 2 Apr 2026 12:16:46 +0200 Subject: [PATCH 1/5] feat: analytics exp --- analytics_schema.sql | 60 +++++++++ infra/docker-compose.dev.yml | 19 +++ server/analytics/__tests__/api.test.ts | 91 +++++++++++--- server/analytics/api.ts | 46 +++---- server/analytics/model.ts | 161 +++++++++++++++++++++++++ server/models.ts | 3 + 6 files changed, 338 insertions(+), 42 deletions(-) create mode 100644 analytics_schema.sql create mode 100644 server/analytics/model.ts diff --git a/analytics_schema.sql b/analytics_schema.sql new file mode 100644 index 0000000000..4dae156928 --- /dev/null +++ b/analytics_schema.sql @@ -0,0 +1,60 @@ +CREATE TABLE pubpub_analytics.data( + __sdc_primary_key character varying(128) ENCODE lzo distkey, + _sdc_batched_at timestamp without time zone ENCODE az64, + _sdc_received_at timestamp without time zone ENCODE az64, + _sdc_sequence bigint ENCODE az64, + _sdc_table_version bigint ENCODE az64, + collectionid character varying(128) ENCODE lzo, + collectionkind character varying(128) ENCODE lzo, + communityid character varying(128) ENCODE lzo, + country character varying(128) ENCODE lzo, + countrycode character varying(128) ENCODE lzo, + event character varying(128) ENCODE lzo, + height bigint ENCODE az64, + isprod boolean ENCODE raw, + primarycollectionid character varying(128) ENCODE lzo, + pubid character varying(128) ENCODE lzo, + type character varying(128) ENCODE lzo, + unique boolean ENCODE raw, + width bigint ENCODE az64, + timestamp bigint ENCODE az64, + utmcontent character varying(128) ENCODE lzo, + utmmedium character varying(128) ENCODE lzo, + utmterm character varying(128) ENCODE lzo, + release__string character varying(128) ENCODE lzo, + path character varying(256) ENCODE lzo, + collectiontitle character varying(256) ENCODE lzo, + collectionslug character varying(256) ENCODE lzo, + pubslug character varying(256) ENCODE lzo, + communityname character varying(256) ENCODE lzo, + collectionids character varying(1024) ENCODE lzo, + release__bigint bigint ENCODE az64, + pagetitle character varying(256) ENCODE lzo, + referrer character varying(4096) ENCODE lzo, + utmcampaign character varying(256) ENCODE lzo, + utmsource character varying(512) ENCODE lzo, + timezone character varying(256) ENCODE lzo, + os character varying(256) ENCODE lzo, + pageid character varying(256) ENCODE lzo, + locale character varying(256) ENCODE lzo, + pageslug character varying(256) ENCODE lzo, + communitysubdomain character varying(256) ENCODE lzo, + format character varying(256) ENCODE lzo, + useragent character varying(8192) ENCODE lzo, + pubtitle character varying(1024) ENCODE lzo, + url character varying(4096) ENCODE lzo, + search character varying(4096) ENCODE lzo, + title character varying(16384) ENCODE lzo, + hash character varying(8192) ENCODE lzo) DISTSTYLE AUTO SORTKEY( + __sdc_primary_key +); + +CREATE TABLE pubpub_analytics.data__collectionids( + _sdc_batched_at timestamp without time zone ENCODE az64, + _sdc_level_0_id bigint ENCODE az64, + _sdc_received_at timestamp without time zone ENCODE az64, + _sdc_sequence bigint ENCODE az64, + _sdc_source_key___sdc_primary_key character varying(128) ENCODE lzo distkey, + _sdc_table_version bigint ENCODE az64, + value character varying(128) ENCODE lzo) DISTSTYLE AUTO; + diff --git a/infra/docker-compose.dev.yml b/infra/docker-compose.dev.yml index 05ae4843fa..c590199cf5 100644 --- a/infra/docker-compose.dev.yml +++ b/infra/docker-compose.dev.yml @@ -85,6 +85,24 @@ services: ports: - "${DB_PORT:-5439}:5432" networks: [appnet] + + analytics_db: + image: postgres:16 + environment: + - POSTGRES_USER=appuser + - POSTGRES_PASSWORD=apppassword + - POSTGRES_DB=analyticsdb + command: > + -c shared_buffers=2GB + -c effective_cache_size=6GB + -c work_mem=16MB + -c maintenance_work_mem=512MB + volumes: + - analytics_pgdata:/var/lib/postgresql/data + - ./analytics_backup.sql:/docker-entrypoint-initdb.d/analytics_backup.sql + ports: + - "${ANALYTICS_DB_PORT:-5440}:5432" + networks: [appnet] # cron: # build: # context: .. @@ -110,3 +128,4 @@ networks: volumes: pgdata: rabbitmqdata: + analytics_pgdata: \ No newline at end of file diff --git a/server/analytics/__tests__/api.test.ts b/server/analytics/__tests__/api.test.ts index 9ab800ace0..525f17f17d 100644 --- a/server/analytics/__tests__/api.test.ts +++ b/server/analytics/__tests__/api.test.ts @@ -1,5 +1,3 @@ -import { vi } from 'vitest'; - import { login, setup, teardown } from 'stubstub'; import { analyticsEventSchema, @@ -8,6 +6,8 @@ import { type sharedEventPayloadSchema, } from 'utils/api/schemas/analytics'; +import { AnalyticsEvent } from '../model'; + const baseTestPayload = { type: 'page', height: 0, @@ -72,22 +72,9 @@ const makeTestOtherPageViewPayload = (options?: Partial) => { } satisfies OtherPageView; }; -setup(beforeAll, async () => { - // mock fetch, we don't actually want to send api calls - vi.spyOn(global, 'fetch').mockImplementation( - () => - Promise.resolve({ - json: () => Promise.resolve({ status: 'ok', id: 'id' }), - }) as unknown as Promise, - ); - - // to be safe, do not actually send any requests to stitch - process.env.STITCH_WEBHOOK_URL = 'http://localhost:9876'; -}); +setup(beforeAll, () => undefined); -teardown(afterAll, () => { - vi.restoreAllMocks(); -}); +teardown(afterAll); describe('analytics schema', () => { describe('pub page view', () => { @@ -99,14 +86,16 @@ describe('analytics schema', () => { expect(analyticsEventSchema.safeParse(pubViewNumber)).toBeTruthy(); }); - // this is needed otherwise redshift will create two colunms, release__bigint and release__string it('should convert number releases into strings', () => { const pubViewNumber = makeTestPubPageViewPayload({ release: 1 }); const parsed = analyticsEventSchema.safeParse(pubViewNumber); + expect(parsed.success).toBeTruthy(); + if (!parsed.success) { throw new Error('parsed failed'); } + expect(parsed.data).toEqual({ ...pubViewNumber, release: '1', @@ -116,11 +105,22 @@ describe('analytics schema', () => { }); describe('analytics', () => { + afterEach(async () => { + await AnalyticsEvent.destroy({ where: {} }); + }); + test('pub page view', async () => { const payload = makeTestPubPageViewPayload(); const agent = await login(); await agent.post('/api/analytics/track').send(payload).expect(204); + + const events = await AnalyticsEvent.findAll({ where: { pubId: payload.pubId } }); + + expect(events).toHaveLength(1); + expect(events[0].event).toBe('pub'); + expect(events[0].pubSlug).toBe('string'); + expect(events[0].release).toBe('draft'); }); test('page page view', async () => { @@ -128,6 +128,11 @@ describe('analytics', () => { const agent = await login(); await agent.post('/api/analytics/track').send(payload).expect(204); + + const events = await AnalyticsEvent.findAll({ where: { event: 'page' } }); + + expect(events).toHaveLength(1); + expect(events[0].pageTitle).toBe('string'); }); test('collection page view', async () => { @@ -135,6 +140,13 @@ describe('analytics', () => { const agent = await login(); await agent.post('/api/analytics/track').send(payload).expect(204); + + const events = await AnalyticsEvent.findAll({ + where: { collectionId: payload.collectionId }, + }); + + expect(events).toHaveLength(1); + expect(events[0].collectionKind).toBe('issue'); }); test('other page view', async () => { @@ -142,12 +154,51 @@ describe('analytics', () => { const agent = await login(); await agent.post('/api/analytics/track').send(payload).expect(204); + + const events = await AnalyticsEvent.findAll({ where: { event: 'other' } }); + + expect(events).toHaveLength(1); + }); + + test('stores country from timezone', async () => { + const payload = makeTestPubPageViewPayload({ timezone: 'Europe/Amsterdam' }); + const agent = await login(); + + await agent.post('/api/analytics/track').send(payload).expect(204); + + const events = await AnalyticsEvent.findAll({ where: { pubId: payload.pubId } }); + + expect(events).toHaveLength(1); + expect(events[0].country).toBe('Netherlands'); + expect(events[0].countryCode).toBe('NL'); }); - test('page page view with optional fields', async () => { - const payload = makeTestPagePageViewPayload(); + test('stores collectionIds as array', async () => { + const id1 = 'de3a36ab-26d9-4b76-aaab-f1bffc18b102'; + const id2 = 'ae3a36ab-26d9-4b76-aaab-f1bffc18b103'; + const payload = makeTestPubPageViewPayload({ + collectionIds: `${id1},${id2}`, + }); const agent = await login(); await agent.post('/api/analytics/track').send(payload).expect(204); + + const events = await AnalyticsEvent.findAll({ where: { pubId: payload.pubId } }); + + expect(events).toHaveLength(1); + expect(events[0].collectionIds).toEqual([id1, id2]); + }); + + test('converts timestamp to date', async () => { + const now = Date.now(); + const payload = makeTestPubPageViewPayload({ timestamp: now }); + const agent = await login(); + + await agent.post('/api/analytics/track').send(payload).expect(204); + + const events = await AnalyticsEvent.findAll({ where: { pubId: payload.pubId } }); + + expect(events).toHaveLength(1); + expect(new Date(events[0].timestamp).getTime()).toBe(now); }); }); diff --git a/server/analytics/api.ts b/server/analytics/api.ts index 78f660f9a0..e7bbeb6f6a 100644 --- a/server/analytics/api.ts +++ b/server/analytics/api.ts @@ -1,6 +1,4 @@ -/** This should all be moved to an AWS lambda */ - -import type { AnalyticsEvent } from 'utils/api/schemas/analytics'; +import type { AnalyticsEvent as AnalyticsEventPayload } from 'utils/api/schemas/analytics'; import { initServer } from '@ts-rest/express'; import { getCountryForTimezone } from 'countries-and-timezones'; @@ -8,25 +6,24 @@ import express from 'express'; import { contract } from 'utils/api/contract'; +import { AnalyticsEvent } from './model'; + const s = initServer(); -const sendToStitch = async ( - payload: AnalyticsEvent & { country: string | null; countryCode: string | null }, +const toEventRecord = ( + payload: AnalyticsEventPayload, + enrichment: { country: string | null; countryCode: string | null }, ) => { - if (!process.env.STITCH_WEBHOOK_URL) { - // throw new Error('Missing STITCH_WEBHOOK_URL'); - return null; - } - - const response = await fetch(process.env.STITCH_WEBHOOK_URL, { - method: 'POST', - body: JSON.stringify(payload), - headers: { - 'Content-Type': 'application/json', - }, - }); + const raw = payload as Record; + const { unique, collectionIds, timestamp, ...fields } = raw; - return response; + return { + ...fields, + ...enrichment, + timestamp: new Date(timestamp as number), + isUnique: (unique as boolean | undefined) ?? null, + collectionIds: typeof collectionIds === 'string' ? collectionIds.split(',') : null, + }; }; export const analyticsServer = s.router(contract.analytics, { @@ -40,7 +37,6 @@ export const analyticsServer = s.router(contract.analytics, { req.body = JSON.parse(req.body); } catch (err) { console.error(err); - // do nothing } } next(); @@ -48,10 +44,16 @@ export const analyticsServer = s.router(contract.analytics, { ], handler: async ({ body: payload }) => { const { timezone } = payload; + const { name: country = null, id: countryCode = null } = + getCountryForTimezone(timezone) || {}; - const { name: country = null, id = null } = getCountryForTimezone(timezone) || {}; - - await sendToStitch({ country, countryCode: id, ...payload }); + try { + await AnalyticsEvent.create( + toEventRecord(payload, { country, countryCode }) as any, + ); + } catch (err) { + console.error('Failed to store analytics event:', err); + } return { status: 204, diff --git a/server/analytics/model.ts b/server/analytics/model.ts new file mode 100644 index 0000000000..935f0089fb --- /dev/null +++ b/server/analytics/model.ts @@ -0,0 +1,161 @@ +import type { CreationOptional, InferAttributes, InferCreationAttributes } from 'sequelize'; + +import { + AllowNull, + Column, + DataType, + Default, + Index, + Model, + PrimaryKey, + Table, +} from 'sequelize-typescript'; + +@Table({ + updatedAt: false, + indexes: [{ fields: ['communityId', 'timestamp'] }, { fields: ['pubId', 'timestamp'] }], +}) +export class AnalyticsEvent extends Model< + InferAttributes, + InferCreationAttributes +> { + @Default(DataType.UUIDV4) + @PrimaryKey + @Column(DataType.UUID) + declare id: CreationOptional; + + @AllowNull(false) + @Column(DataType.TEXT) + declare type: string; + + @AllowNull(false) + @Index + @Column(DataType.TEXT) + declare event: string; + + @AllowNull(false) + @Column(DataType.DATE) + declare timestamp: Date; + + @Column(DataType.TEXT) + declare referrer: string | null; + + @Column(DataType.BOOLEAN) + declare isUnique: boolean | null; + + @Column(DataType.TEXT) + declare search: string | null; + + @Column(DataType.TEXT) + declare utmSource: string | null; + + @Column(DataType.TEXT) + declare utmMedium: string | null; + + @Column(DataType.TEXT) + declare utmCampaign: string | null; + + @Column(DataType.TEXT) + declare utmTerm: string | null; + + @Column(DataType.TEXT) + declare utmContent: string | null; + + @AllowNull(false) + @Column(DataType.TEXT) + declare timezone: string; + + @AllowNull(false) + @Column(DataType.TEXT) + declare locale: string; + + @AllowNull(false) + @Column(DataType.TEXT) + declare userAgent: string; + + @AllowNull(false) + @Column(DataType.TEXT) + declare os: string; + + @Index + @Column(DataType.UUID) + declare communityId: string | null; + + @Column(DataType.TEXT) + declare communitySubdomain: string | null; + + @Column(DataType.TEXT) + declare communityName: string | null; + + @AllowNull(false) + @Column(DataType.BOOLEAN) + declare isProd: boolean; + + @Column(DataType.TEXT) + declare country: string | null; + + @Column(DataType.TEXT) + declare countryCode: string | null; + + @Column(DataType.TEXT) + declare url: string | null; + + @Column(DataType.TEXT) + declare title: string | null; + + @Column(DataType.TEXT) + declare hash: string | null; + + @Column(DataType.INTEGER) + declare height: number | null; + + @Column(DataType.INTEGER) + declare width: number | null; + + @Column(DataType.TEXT) + declare path: string | null; + + @Column(DataType.TEXT) + declare pageTitle: string | null; + + @Column(DataType.UUID) + declare pageId: string | null; + + @Column(DataType.TEXT) + declare pageSlug: string | null; + + @Column(DataType.TEXT) + declare collectionTitle: string | null; + + @Column(DataType.TEXT) + declare collectionKind: string | null; + + @Index + @Column(DataType.UUID) + declare collectionId: string | null; + + @Column(DataType.TEXT) + declare collectionSlug: string | null; + + @Column(DataType.TEXT) + declare pubTitle: string | null; + + @Index + @Column(DataType.UUID) + declare pubId: string | null; + + @Column(DataType.TEXT) + declare pubSlug: string | null; + + @Column(DataType.ARRAY(DataType.TEXT)) + declare collectionIds: string[] | null; + + @Column(DataType.UUID) + declare primaryCollectionId: string | null; + + @Column(DataType.TEXT) + declare release: string | null; + + @Column(DataType.TEXT) + declare format: string | null; +} diff --git a/server/models.ts b/server/models.ts index 925e1b1b75..ee4a3f6fe8 100644 --- a/server/models.ts +++ b/server/models.ts @@ -3,6 +3,7 @@ import passportLocalSequelize from 'passport-local-sequelize'; /* Import and create all models. */ /* Also import them to make them available to other modules */ import { ActivityItem } from './activityItem/model'; +import { AnalyticsEvent } from './analytics/model'; import { AuthToken } from './authToken/model'; import { Collection } from './collection/model'; import { CollectionAttribution } from './collectionAttribution/model'; @@ -64,6 +65,7 @@ import { ZoteroIntegration } from './zoteroIntegration/model'; sequelize.addModels([ ActivityItem, + AnalyticsEvent, AuthToken, Collection, CollectionAttribution, @@ -159,6 +161,7 @@ export const includeUserModel = (() => { export { ActivityItem, + AnalyticsEvent, AuthToken, Collection, CollectionAttribution, From 8b321d09d6b5b566fb560b43e2577cc61fe531fc Mon Sep 17 00:00:00 2001 From: "Thomas F. K. Jorna" Date: Thu, 2 Apr 2026 15:18:45 +0200 Subject: [PATCH 2/5] feat: local analytics --- .../AdminDashboard/AdminDashboard.tsx | 2 +- .../DashboardImpact/DashboardImpact.tsx | 2 +- infra/Caddyfile | 10 +- infra/docker-compose.dev.yml | 38 +- infra/docker-compose.prod.yml | 28 + infra/stack.yml | 36 ++ server/analytics/api.ts | 18 +- server/routes/dashboardImpact.tsx | 1 + server/server.ts | 4 + .../migrate-metabase-queries.ts | 505 ++++++++++++++++++ tools/analytics-migration/migrate.sh | 266 +++++++++ 11 files changed, 887 insertions(+), 23 deletions(-) create mode 100644 tools/analytics-migration/migrate-metabase-queries.ts create mode 100755 tools/analytics-migration/migrate.sh diff --git a/client/containers/AdminDashboard/AdminDashboard.tsx b/client/containers/AdminDashboard/AdminDashboard.tsx index f1b7974551..8aeb766499 100644 --- a/client/containers/AdminDashboard/AdminDashboard.tsx +++ b/client/containers/AdminDashboard/AdminDashboard.tsx @@ -11,7 +11,7 @@ type Props = { const AdminDashboard = (props: Props) => { const { impactData } = props; const { baseToken } = impactData; - const dashUrl = `https://metabase.pubpub.org/embed/dashboard/${baseToken}#bordered=false&titled=false`; + const dashUrl = `http://localhost:3030/embed/dashboard/${baseToken}#bordered=false&titled=false`; const getOffset = (width) => { return width < 960 ? 45 : 61; }; diff --git a/client/containers/DashboardImpact/DashboardImpact.tsx b/client/containers/DashboardImpact/DashboardImpact.tsx index f87aa4792c..b0a455f0ce 100644 --- a/client/containers/DashboardImpact/DashboardImpact.tsx +++ b/client/containers/DashboardImpact/DashboardImpact.tsx @@ -28,7 +28,7 @@ const DashboardImpact = (props: Props) => { const displayDataWarning = activeTarget?.createdAt < '2020-04-29'; const isCollection = activeTargetType === 'collection'; const genUrl = (token) => { - return `https://metabase.pubpub.org/embed/dashboard/${token}#bordered=false&titled=false`; + return `http://localhost:3030/embed/dashboard/${token}#bordered=false&titled=false`; }; const getOffset = (width) => { return width < 960 ? 45 : 61; diff --git a/infra/Caddyfile b/infra/Caddyfile index 14846e6fc2..fcd956c38f 100644 --- a/infra/Caddyfile +++ b/infra/Caddyfile @@ -7,10 +7,18 @@ respond "OK" 200 } +{$METABASE_HOST:analytics.localhost} { + tls internal { + on_demand + } + encode gzip + reverse_proxy pubpub_metabase:3001 +} + :443 { tls internal { on_demand } encode gzip reverse_proxy pubpub_app:3000 -} \ No newline at end of file +} diff --git a/infra/docker-compose.dev.yml b/infra/docker-compose.dev.yml index c590199cf5..dbe4f78478 100644 --- a/infra/docker-compose.dev.yml +++ b/infra/docker-compose.dev.yml @@ -85,24 +85,36 @@ services: ports: - "${DB_PORT:-5439}:5432" networks: [appnet] - - analytics_db: - image: postgres:16 + + metabase: + image: metabase/metabase:v0.48.1 + environment: + MB_DB_TYPE: postgres + MB_DB_DBNAME: metabasedb + MB_DB_PORT: "5432" + MB_DB_HOST: metabase_db + MB_DB_USER: appuser + MB_DB_PASS: apppassword + MB_JETTY_PORT: "3001" + depends_on: + - metabase_db + - db + ports: + - "${METABASE_PORT:-3030}:3001" + networks: [appnet] + + metabase_db: + image: postgres:18 environment: - POSTGRES_USER=appuser - POSTGRES_PASSWORD=apppassword - - POSTGRES_DB=analyticsdb - command: > - -c shared_buffers=2GB - -c effective_cache_size=6GB - -c work_mem=16MB - -c maintenance_work_mem=512MB + - POSTGRES_DB=metabasedb volumes: - - analytics_pgdata:/var/lib/postgresql/data - - ./analytics_backup.sql:/docker-entrypoint-initdb.d/analytics_backup.sql + - metabase_pgdata:/var/lib/postgresql ports: - - "${ANALYTICS_DB_PORT:-5440}:5432" + - "${METABASE_DB_PORT:-5440}:5432" networks: [appnet] + # cron: # build: # context: .. @@ -128,4 +140,4 @@ networks: volumes: pgdata: rabbitmqdata: - analytics_pgdata: \ No newline at end of file + metabase_pgdata: diff --git a/infra/docker-compose.prod.yml b/infra/docker-compose.prod.yml index 2819c25fcc..1a24980379 100644 --- a/infra/docker-compose.prod.yml +++ b/infra/docker-compose.prod.yml @@ -64,6 +64,33 @@ services: limits: memory: 2G + metabase: + image: metabase/metabase:latest + environment: + MB_DB_TYPE: postgres + MB_DB_DBNAME: metabasedb + MB_DB_PORT: "5432" + MB_DB_HOST: metabase_db + MB_DB_USER: appuser + MB_DB_PASS: apppassword + MB_JETTY_PORT: "3001" + depends_on: + - metabase_db + - db + ports: + - "${METABASE_PORT:-3030}:3001" + networks: [appnet] + + metabase_db: + image: postgres:16 + environment: + - POSTGRES_USER=appuser + - POSTGRES_PASSWORD=apppassword + - POSTGRES_DB=metabasedb + volumes: + - metabase_pgdata:/var/lib/postgresql/data + networks: [appnet] + rabbitmq: image: rabbitmq:3.13-alpine environment: @@ -102,3 +129,4 @@ networks: volumes: # pgdata: rabbitmqdata: + metabase_pgdata: diff --git a/infra/stack.yml b/infra/stack.yml index 14ca48d79b..b7ea1777f0 100644 --- a/infra/stack.yml +++ b/infra/stack.yml @@ -126,6 +126,41 @@ services: restart_policy: condition: any + metabase: + image: metabase/metabase:latest + environment: + MB_DB_TYPE: postgres + MB_DB_DBNAME: metabasedb + MB_DB_PORT: '5432' + MB_DB_HOST: metabase_db + MB_DB_USER: '${METABASE_DB_USER:-appuser}' + MB_DB_PASS: '${METABASE_DB_PASS}' + MB_JETTY_PORT: '3001' + networks: [appnet] + deploy: + replicas: 1 + resources: + limits: + memory: 2G + reservations: + memory: 1G + restart_policy: + condition: any + + metabase_db: + image: postgres:16 + environment: + POSTGRES_USER: '${METABASE_DB_USER:-appuser}' + POSTGRES_PASSWORD: '${METABASE_DB_PASS}' + POSTGRES_DB: metabasedb + volumes: + - metabase_pgdata:/var/lib/postgresql/data + networks: [appnet] + deploy: + replicas: 1 + restart_policy: + condition: any + pubstash: image: ghcr.io/knowledgefutures/pubpub:${IMAGE_TAG} env_file: [.env] @@ -163,5 +198,6 @@ networks: volumes: pgdata: rabbitmqdata: + metabase_pgdata: caddy_data: caddy_config: diff --git a/server/analytics/api.ts b/server/analytics/api.ts index e7bbeb6f6a..af27a9e017 100644 --- a/server/analytics/api.ts +++ b/server/analytics/api.ts @@ -6,6 +6,7 @@ import express from 'express'; import { contract } from 'utils/api/contract'; +import { abortStorage } from '../abort'; import { AnalyticsEvent } from './model'; const s = initServer(); @@ -47,13 +48,16 @@ export const analyticsServer = s.router(contract.analytics, { const { name: country = null, id: countryCode = null } = getCountryForTimezone(timezone) || {}; - try { - await AnalyticsEvent.create( - toEventRecord(payload, { country, countryCode }) as any, - ); - } catch (err) { - console.error('Failed to store analytics event:', err); - } + const record = toEventRecord(payload, { country, countryCode }); + + // sendBeacon closes the connection immediately, which triggers the + // request abort middleware. run the insert in a fresh context so it + // isn't killed by the abort signal. + abortStorage.run({ abortController: new AbortController() }, () => { + AnalyticsEvent.create(record as any).catch((err) => { + console.error('Failed to store analytics event:', err); + }); + }); return { status: 204, diff --git a/server/routes/dashboardImpact.tsx b/server/routes/dashboardImpact.tsx index d082304f21..4d6b071779 100644 --- a/server/routes/dashboardImpact.tsx +++ b/server/routes/dashboardImpact.tsx @@ -23,6 +23,7 @@ router.get( throw new NotFoundError(); } const { activeTargetType, activeTarget } = initialData.scopeData.elements; + console.log('activeTargetType', activeTargetType, initialData.locationData.isProd); const impactData = { baseToken: generateMetabaseToken(activeTargetType, activeTarget.id, 'base'), newToken: generateMetabaseToken( diff --git a/server/server.ts b/server/server.ts index 5df9cd7be7..06b7122967 100755 --- a/server/server.ts +++ b/server/server.ts @@ -195,6 +195,10 @@ appRouter.use((req, res, next) => { return next(); } + if (req.path.includes('/api/analytics')) { + return next(); + } + const abortController = new AbortController(); abortStorage.enterWith({ abortController }); diff --git a/tools/analytics-migration/migrate-metabase-queries.ts b/tools/analytics-migration/migrate-metabase-queries.ts new file mode 100644 index 0000000000..f50416d6a7 --- /dev/null +++ b/tools/analytics-migration/migrate-metabase-queries.ts @@ -0,0 +1,505 @@ +// migrates metabase saved questions from a redshift-backed database to +// the new postgres database. uses the metabase API exclusively -- +// no parquet, no direct db access. + +// ── column/table name mapping ── + +const ANALYTICS_COLUMN_MAP: Record = { + collectionid: 'collectionId', + collectionkind: 'collectionKind', + communityid: 'communityId', + communityname: 'communityName', + communitysubdomain: 'communitySubdomain', + countrycode: 'countryCode', + isprod: 'isProd', + primarycollectionid: 'primaryCollectionId', + pubid: 'pubId', + pubtitle: 'pubTitle', + pubslug: 'pubSlug', + pagetitle: 'pageTitle', + pageid: 'pageId', + pageslug: 'pageSlug', + useragent: 'userAgent', + utmsource: 'utmSource', + utmmedium: 'utmMedium', + utmcampaign: 'utmCampaign', + utmterm: 'utmTerm', + utmcontent: 'utmContent', + collectionids: 'collectionIds', + collectiontitle: 'collectionTitle', + collectionslug: 'collectionSlug', + unique: 'isUnique', + release__string: 'release', + + // dropped columns + release__bigint: null, + __sdc_primary_key: null, + _sdc_batched_at: null, + _sdc_received_at: null, + _sdc_sequence: null, + _sdc_table_version: null, +}; + +const ANALYTICS_TABLE_OLD = 'data'; +const ANALYTICS_TABLE_NEW = 'AnalyticsEvents'; + +// ── types ── + +type Table = { + id: number; + name: string; + schema: string; + db_id: number; + fields: Field[]; +}; + +type Field = { + id: number; + name: string; + table_id: number; +}; + +type Card = { + id: number; + name: string; + description: string | null; + display: string; + collection_id: number; + dataset_query: DatasetQuery; + visualization_settings: Record; +}; + +type DatasetQuery = { + type: 'query' | 'native'; + database: number; + query?: Record; + native?: { query: string; 'template-tags'?: Record }; +}; + +type IdMaps = { + tableIdMap: Record; + fieldIdMap: Record; + cardIdMap: Record; +}; + +// ── metabase api ── + +async function metabaseRequest( + baseUrl: string, + session: string, + method: string, + path: string, + body?: unknown, +): Promise { + const headers: Record = { 'X-Metabase-Session': session }; + + if (body) { + headers['Content-Type'] = 'application/json'; + } + + const res = await fetch(`${baseUrl}/api${path}`, { + method, + headers, + ...(body ? { body: JSON.stringify(body) } : {}), + }); + + if (!res.ok) { + const text = await res.text(); + throw new Error(`${method} ${path}: ${res.status} ${text}`); + } + + return res.json(); +} + +// ── field mapping ── + +function buildFieldMapping(oldTables: Table[], newTables: Table[]) { + const tableIdMap: Record = {}; + const fieldIdMap: Record = {}; + const warnings: string[] = []; + + const newByName = new Map(); + for (const t of newTables) { + newByName.set(t.name.toLowerCase(), t); + } + + for (const oldTable of oldTables) { + const isAnalytics = oldTable.name.toLowerCase() === ANALYTICS_TABLE_OLD; + const targetName = isAnalytics ? ANALYTICS_TABLE_NEW : oldTable.name; + const newTable = newByName.get(targetName.toLowerCase()); + + if (!newTable) { + if (oldTable.name.startsWith('_sdc_')) continue; + warnings.push(`table not mapped: ${oldTable.name}`); + continue; + } + + tableIdMap[oldTable.id] = newTable.id; + + const newFieldsByName = new Map(); + for (const f of newTable.fields) { + newFieldsByName.set(f.name.toLowerCase(), f); + } + + for (const oldField of oldTable.fields) { + let targetFieldName: string | null; + + if (isAnalytics) { + const mapped = ANALYTICS_COLUMN_MAP[oldField.name.toLowerCase()]; + targetFieldName = mapped === undefined ? oldField.name : mapped; + } else { + targetFieldName = oldField.name; + } + + if (targetFieldName === null) continue; + + const newField = + newFieldsByName.get(targetFieldName) || + newFieldsByName.get(targetFieldName.toLowerCase()); + + if (newField) { + fieldIdMap[oldField.id] = newField.id; + } + } + } + + return { tableIdMap, fieldIdMap, warnings }; +} + +// ── mbql query remapping ── + +function remapQuery(obj: unknown, maps: IdMaps): unknown { + if (Array.isArray(obj)) { + if (obj.length >= 2 && obj[0] === 'field' && typeof obj[1] === 'number') { + const mapped = maps.fieldIdMap[obj[1]] ?? obj[1]; + return ['field', mapped, ...obj.slice(2).map((x) => remapQuery(x, maps))]; + } + + return obj.map((item) => remapQuery(item, maps)); + } + + if (obj && typeof obj === 'object') { + const result: Record = {}; + + for (const [k, v] of Object.entries(obj)) { + if (k === 'source-table' && typeof v === 'number') { + result[k] = maps.tableIdMap[v] ?? v; + } else if (k === 'source-table' && typeof v === 'string' && v.startsWith('card__')) { + const oldId = parseInt(v.replace('card__', ''), 10); + result[k] = `card__${maps.cardIdMap[oldId] ?? oldId}`; + } else { + result[k] = remapQuery(v, maps); + } + } + + return result; + } + + return obj; +} + +// ── native sql remapping ── + +function remapNativeSql(sql: string): string { + let result = sql; + + // schema-qualified analytics table references + result = result.replace(/"pubpub_analytics"\s*\.\s*"data"/gi, `"${ANALYTICS_TABLE_NEW}"`); + result = result.replace(/pubpub_analytics\s*\.\s*data/gi, `"${ANALYTICS_TABLE_NEW}"`); + result = result.replace(/"pubpub_analytics"\s*\.\s*/g, ''); + + // unqualified analytics table + result = result.replace(/\bFROM\s+"data"/gi, `FROM "${ANALYTICS_TABLE_NEW}"`); + result = result.replace(/\bJOIN\s+"data"/gi, `JOIN "${ANALYTICS_TABLE_NEW}"`); + + // analytics column names (quoted) + for (const [oldName, newName] of Object.entries(ANALYTICS_COLUMN_MAP)) { + if (newName === null) continue; + if (oldName === newName) continue; + + result = result.replace(new RegExp(`"${oldName}"`, 'gi'), `"${newName}"`); + } + + // redshift timestamp conversion pattern -> just use the column directly + result = result.replace( + /TIMESTAMP\s+'1970-01-01T00:00:00Z'\s*\+\s*"?timestamp"?\s*\*\s*INTERVAL\s+'1 second'\s*\/\s*1000/gi, + '"timestamp"', + ); + + // strip _sdc_ column references (with optional leading comma) + result = result.replace(/,?\s*"?_sdc_\w+"?\s*/g, ' '); + + return result; +} + +// ── topological sort (for card->card references) ── + +function findCardRefs(obj: unknown): number[] { + const refs: number[] = []; + + const walk = (o: unknown) => { + if (!o) return; + + if (typeof o === 'string' && o.startsWith('card__')) { + refs.push(parseInt(o.replace('card__', ''), 10)); + } + + if (Array.isArray(o)) { + o.forEach(walk); + } else if (typeof o === 'object') { + Object.values(o as Record).forEach(walk); + } + }; + + walk(obj); + return refs; +} + +function topologicalSort(cards: Card[], targetIds: Set): number[] { + const sorted: number[] = []; + const visited = new Set(); + const visiting = new Set(); + + const cardById = new Map(cards.map((c) => [c.id, c])); + + const visit = (id: number) => { + if (visited.has(id) || visiting.has(id)) return; + + visiting.add(id); + + const card = cardById.get(id); + if (card) { + for (const ref of findCardRefs(card.dataset_query)) { + if (targetIds.has(ref)) visit(ref); + } + } + + visiting.delete(id); + visited.add(id); + sorted.push(id); + }; + + for (const id of targetIds) visit(id); + + return sorted; +} + +// ── arg parsing ── + +function parseArgs(argv: string[]) { + const result: Record = {}; + + for (let i = 0; i < argv.length; i++) { + if (!argv[i].startsWith('--')) continue; + + const key = argv[i].replace(/^--/, ''); + const next = argv[i + 1]; + + if (next && !next.startsWith('--')) { + result[key] = next; + i++; + } else { + result[key] = 'true'; + } + } + + return result; +} + +// ── main ── + +async function main() { + const raw = parseArgs(process.argv.slice(2)); + + const metabaseUrl = raw['metabase-url'] ?? 'http://localhost:3030'; + const session = raw.session ?? ''; + const oldDbId = parseInt(raw['old-db-id'] ?? '0', 10); + const newDbId = parseInt(raw['new-db-id'] ?? '0', 10); + const collectionId = parseInt(raw.collection ?? '14', 10); + const dryRun = raw['dry-run'] === 'true'; + + const isMissingArgs = !session || !oldDbId || !newDbId; + if (isMissingArgs) { + console.error('required: --session, --old-db-id, --new-db-id'); + process.exit(1); + } + + const get = (path: string) => metabaseRequest(metabaseUrl, session, 'GET', path); + const post = (path: string, body: unknown) => + metabaseRequest(metabaseUrl, session, 'POST', path, body); + + // fetch metadata for old and new databases in parallel + const [oldDb, newDb] = await Promise.all([ + get(`/database/${oldDbId}/metadata?include_hidden=true`), + get(`/database/${newDbId}/metadata?include_hidden=true`), + ]); + + const { tableIdMap, fieldIdMap, warnings } = buildFieldMapping( + oldDb.tables || [], + newDb.tables || [], + ); + + console.log( + ` mapped ${Object.keys(tableIdMap).length} tables, ${Object.keys(fieldIdMap).length} fields`, + ); + + for (const w of warnings) { + console.log(` warning: ${w}`); + } + + // fetch cards in the target collection + const collectionItems = await get( + `/collection/${collectionId}/items?models=card&models=dataset`, + ); + + const itemList = collectionItems.data || collectionItems; + const cardIds: number[] = itemList.map((item: { id: number }) => item.id); + + if (cardIds.length === 0) { + console.log(' no cards found in collection'); + return; + } + + const cards: Card[] = await Promise.all(cardIds.map((id) => get(`/card/${id}`))); + const sortedIds = topologicalSort(cards, new Set(cardIds)); + const cardById = new Map(cards.map((c) => [c.id, c])); + + // create target collection + const { name: oldCollectionName } = await get(`/collection/${collectionId}`); + const newCollectionName = `Migrated - ${oldCollectionName}`; + + if (dryRun) { + console.log(`\n dry run: would create collection '${newCollectionName}'`); + console.log(` dry run: would migrate ${sortedIds.length} cards\n`); + + for (const id of sortedIds) { + const card = cardById.get(id)!; + const dq = card.dataset_query; + + console.log(`--- ${card.name} (id=${id}, ${dq.type}) ---`); + + if (dq.type === 'native') { + console.log(remapNativeSql(dq.native?.query || '')); + } else { + const remapped = remapQuery(dq.query, { tableIdMap, fieldIdMap, cardIdMap: {} }); + console.log(JSON.stringify(remapped, null, 2)); + } + + console.log(); + } + + return; + } + + const newCollection = await post('/collection', { + name: newCollectionName, + description: `auto-migrated from '${oldCollectionName}'`, + }); + + // migrate cards + const cardIdMap: Record = {}; + const maps: IdMaps = { tableIdMap, fieldIdMap, cardIdMap }; + let succeeded = 0; + let failed = 0; + + for (const oldId of sortedIds) { + const card = cardById.get(oldId); + if (!card) continue; + + const dq = card.dataset_query; + let newDatasetQuery: DatasetQuery; + + if (dq.type === 'native') { + newDatasetQuery = { + type: 'native', + database: newDbId, + native: { + query: remapNativeSql(dq.native?.query || ''), + 'template-tags': dq.native?.['template-tags'] || {}, + }, + }; + } else { + newDatasetQuery = { + type: 'query', + database: newDbId, + query: remapQuery(dq.query, maps) as Record, + }; + } + + try { + // biome-ignore lint/correctness/noAwaitInLoop: sequential creation required for card->card references + const newCard = await post('/card', { + name: card.name, + description: card.description, + display: card.display, + dataset_query: newDatasetQuery, + collection_id: newCollection.id, + visualization_settings: card.visualization_settings, + }); + + cardIdMap[oldId] = newCard.id; + console.log(` created: ${card.name} (${oldId} -> ${newCard.id})`); + succeeded++; + } catch (e) { + console.error(` FAILED: ${card.name} (${oldId}): ${e}`); + cardIdMap[oldId] = oldId; + failed++; + } + } + + // migrate dashboards in the collection + const dashboardItems = await get(`/collection/${collectionId}/items?models=dashboard`); + const dashList = dashboardItems.data || dashboardItems; + + for (const item of dashList) { + try { + // biome-ignore lint/correctness/noAwaitInLoop: dashboards must be created one at a time + const oldDash = await get(`/dashboard/${item.id}`); + + const newDash = await post('/dashboard', { + name: oldDash.name, + description: oldDash.description, + collection_id: newCollection.id, + parameters: oldDash.parameters || [], + }); + + const oldCards = oldDash.ordered_cards || oldDash.dashcards || []; + const dashCards = []; + + for (let i = 0; i < oldCards.length; i++) { + const dc = oldCards[i]; + const oldCardId = dc.card_id; + const newCardId = cardIdMap[oldCardId] ?? oldCardId; + const remappedParams = remapQuery(dc.parameter_mappings || [], maps); + + dashCards.push({ + id: -(i + 1), + card_id: newCardId, + row: dc.row ?? 0, + col: dc.col ?? 0, + size_x: dc.size_x ?? 6, + size_y: dc.size_y ?? 4, + parameter_mappings: remappedParams, + visualization_settings: dc.visualization_settings || {}, + }); + } + + if (dashCards.length > 0) { + await metabaseRequest(metabaseUrl, session, 'PUT', `/dashboard/${newDash.id}`, { + dashcards: dashCards, + }); + } + + console.log(` created dashboard: ${oldDash.name} (${oldCards.length} cards)`); + } catch (e) { + console.error(` FAILED dashboard: ${item.name}: ${e}`); + } + } + + console.log(`\n migration complete: ${succeeded} succeeded, ${failed} failed`); +} + +main().catch((e) => { + console.error(e); + process.exit(1); +}); diff --git a/tools/analytics-migration/migrate.sh b/tools/analytics-migration/migrate.sh new file mode 100755 index 0000000000..852257d711 --- /dev/null +++ b/tools/analytics-migration/migrate.sh @@ -0,0 +1,266 @@ +#!/bin/bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +REDSHIFT_BAK_DIR="${PROJECT_ROOT}/redshift_bak_data" +S3_REDSHIFT_PATH="${S3_REDSHIFT_PATH:-s3://temp-migration-redshift/pubpub_analytics/backup_20260402/}" +DATABASE_URL="${DATABASE_URL:-postgres://appuser:apppassword@localhost:5439/appdb}" + +echo "Analytics Migration" +echo "===================" +echo " database: $DATABASE_URL" +echo " backup: $REDSHIFT_BAK_DIR" +echo "" + +# ── step 1: download from s3 if not already present ── + +if [ ! -f "$REDSHIFT_BAK_DIR/data000" ]; then + echo "[1/5] downloading redshift backup from s3..." + mkdir -p "$REDSHIFT_BAK_DIR" + aws s3 cp "$S3_REDSHIFT_PATH" "$REDSHIFT_BAK_DIR" --recursive +else + echo "[1/5] backup already present, skipping download" +fi + +# ── step 2: ensure the AnalyticsEvents table exists ── + +echo "[2/5] ensuring AnalyticsEvents table exists..." +psql "$DATABASE_URL" <<'SQL' +CREATE TABLE IF NOT EXISTS "AnalyticsEvents" ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + type text NOT NULL, + event text NOT NULL, + "timestamp" timestamptz NOT NULL, + referrer text, + "isUnique" boolean, + search text, + "utmSource" text, + "utmMedium" text, + "utmCampaign" text, + "utmTerm" text, + "utmContent" text, + timezone text NOT NULL, + locale text NOT NULL, + "userAgent" text NOT NULL, + os text NOT NULL, + "communityId" uuid, + "communitySubdomain" text, + "communityName" text, + "isProd" boolean NOT NULL, + country text, + "countryCode" text, + url text, + title text, + hash text, + height integer, + width integer, + path text, + "pageTitle" text, + "pageId" uuid, + "pageSlug" text, + "collectionTitle" text, + "collectionKind" text, + "collectionId" uuid, + "collectionSlug" text, + "pubTitle" text, + "pubId" uuid, + "pubSlug" text, + "collectionIds" text[], + "primaryCollectionId" uuid, + release text, + format text, + "createdAt" timestamptz NOT NULL DEFAULT now() +); + +CREATE INDEX IF NOT EXISTS "analytics_events_community_ts" + ON "AnalyticsEvents" ("communityId", "timestamp"); +CREATE INDEX IF NOT EXISTS "analytics_events_pub_ts" + ON "AnalyticsEvents" ("pubId", "timestamp"); +CREATE INDEX IF NOT EXISTS "analytics_events_event" + ON "AnalyticsEvents" (event); +CREATE INDEX IF NOT EXISTS "analytics_events_collection" + ON "AnalyticsEvents" ("collectionId"); +SQL + +# ── step 3: create staging table and import csv ── + +echo "[3/5] creating staging table..." +psql "$DATABASE_URL" <<'SQL' +DROP TABLE IF EXISTS analytics_staging; +CREATE UNLOGGED TABLE analytics_staging ( + __sdc_primary_key text, + _sdc_batched_at text, + _sdc_received_at text, + _sdc_sequence text, + _sdc_table_version text, + collectionid text, + collectionkind text, + communityid text, + country text, + countrycode text, + event text, + height text, + isprod text, + primarycollectionid text, + pubid text, + type text, + "unique" text, + width text, + "timestamp" text, + utmcontent text, + utmmedium text, + utmterm text, + release__string text, + path text, + collectiontitle text, + collectionslug text, + pubslug text, + communityname text, + collectionids text, + release__bigint text, + pagetitle text, + referrer text, + utmcampaign text, + utmsource text, + timezone text, + os text, + pageid text, + locale text, + pageslug text, + communitysubdomain text, + format text, + useragent text, + pubtitle text, + url text, + search text, + title text, + hash text +); +SQL + +echo "[4/5] importing csv files into staging..." +for f in "$REDSHIFT_BAK_DIR"/data[0-9][0-9][0-9]; do + echo " $(basename "$f")..." + psql "$DATABASE_URL" -c "\copy analytics_staging FROM '$f' CSV HEADER" +done + +ROW_COUNT=$(psql "$DATABASE_URL" -tA -c "SELECT count(*) FROM analytics_staging") +echo " staged $ROW_COUNT rows" + +# ── step 5: transform and insert ── + +echo "[5/5] transforming into AnalyticsEvents..." +psql "$DATABASE_URL" -v ON_ERROR_STOP=1 <<'SQL' +CREATE OR REPLACE FUNCTION pg_temp.safe_uuid(val text) RETURNS uuid AS $$ +SELECT CASE + WHEN val ~ '^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$' + THEN val::uuid + ELSE NULL +END; +$$ LANGUAGE sql IMMUTABLE; + +-- cast through double precision to handle scientific notation (e.g. 1.71974e+14) +CREATE OR REPLACE FUNCTION pg_temp.safe_ts(val text) RETURNS timestamptz AS $$ +SELECT CASE + WHEN val IS NULL OR val = '' THEN NULL + WHEN val::double precision BETWEEN 1e12 AND 2e13 + THEN to_timestamp(val::double precision / 1000.0) + ELSE NULL +END; +$$ LANGUAGE sql IMMUTABLE; + +INSERT INTO "AnalyticsEvents" ( + id, type, event, "timestamp", + referrer, "isUnique", search, + "utmSource", "utmMedium", "utmCampaign", "utmTerm", "utmContent", + timezone, locale, "userAgent", os, + "communityId", "communitySubdomain", "communityName", "isProd", + country, "countryCode", + url, title, hash, height, width, path, + "pageTitle", "pageId", "pageSlug", + "collectionTitle", "collectionKind", "collectionId", "collectionSlug", + "pubTitle", "pubId", "pubSlug", + "collectionIds", "primaryCollectionId", release, format, + "createdAt" +) +SELECT + COALESCE(pg_temp.safe_uuid(s.__sdc_primary_key), gen_random_uuid()), + s.type, + s.event, + pg_temp.safe_ts(s."timestamp"), + + NULLIF(s.referrer, ''), + CASE WHEN s."unique" = 't' THEN true + WHEN s."unique" = 'f' THEN false + ELSE NULL END, + NULLIF(s.search, ''), + + NULLIF(s.utmsource, ''), + NULLIF(s.utmmedium, ''), + NULLIF(s.utmcampaign, ''), + NULLIF(s.utmterm, ''), + NULLIF(s.utmcontent, ''), + + COALESCE(NULLIF(s.timezone, ''), 'UTC'), + COALESCE(NULLIF(s.locale, ''), 'en-US'), + COALESCE(NULLIF(s.useragent, ''), 'unknown'), + COALESCE(NULLIF(s.os, ''), 'unknown'), + + pg_temp.safe_uuid(s.communityid), + NULLIF(s.communitysubdomain, ''), + NULLIF(s.communityname, ''), + COALESCE(s.isprod = 't', false), + + NULLIF(s.country, ''), + NULLIF(s.countrycode, ''), + + NULLIF(s.url, ''), + NULLIF(s.title, ''), + NULLIF(s.hash, ''), + CASE WHEN s.height ~ '^\d{1,9}$' THEN s.height::int ELSE NULL END, + CASE WHEN s.width ~ '^\d{1,9}$' THEN s.width::int ELSE NULL END, + NULLIF(s.path, ''), + + NULLIF(s.pagetitle, ''), + pg_temp.safe_uuid(s.pageid), + NULLIF(s.pageslug, ''), + + NULLIF(s.collectiontitle, ''), + NULLIF(s.collectionkind, ''), + pg_temp.safe_uuid(s.collectionid), + NULLIF(s.collectionslug, ''), + + NULLIF(s.pubtitle, ''), + pg_temp.safe_uuid(s.pubid), + NULLIF(s.pubslug, ''), + + CASE WHEN NULLIF(s.collectionids, '') IS NOT NULL + THEN string_to_array(s.collectionids, ',') + ELSE NULL END, + pg_temp.safe_uuid(s.primarycollectionid), + COALESCE(NULLIF(s.release__string, ''), NULLIF(s.release__bigint, '')), + NULLIF(s.format, ''), + + COALESCE( + CASE WHEN NULLIF(s._sdc_received_at, '') IS NOT NULL + THEN s._sdc_received_at::timestamptz + ELSE NULL END, + pg_temp.safe_ts(s."timestamp"), + now() + ) + +FROM analytics_staging s +WHERE s.type IS NOT NULL + AND s.event IS NOT NULL + AND pg_temp.safe_ts(s."timestamp") IS NOT NULL; +SQL + +echo " cleaning up staging table..." +psql "$DATABASE_URL" -c "DROP TABLE IF EXISTS analytics_staging;" +psql "$DATABASE_URL" -c 'ANALYZE "AnalyticsEvents";' + +FINAL_COUNT=$(psql "$DATABASE_URL" -tA -c 'SELECT count(*) FROM "AnalyticsEvents"') +echo "" +echo "done. $FINAL_COUNT rows in AnalyticsEvents." From d3ea07878d617ab7eb5ed642d95f6c5abc663d3a Mon Sep 17 00:00:00 2001 From: "Thomas F. K. Jorna" Date: Tue, 7 Apr 2026 15:39:14 +0200 Subject: [PATCH 3/5] fix: use duckdb! --- .gitignore | 3 +- infra/docker-compose.dev.yml | 25 +- infra/duckdb-sync/Dockerfile | 20 ++ infra/duckdb-sync/sync.sh | 65 +++++ infra/metabase/Dockerfile | 34 +++ infra/stack.yml | 30 ++- server/analytics/model.ts | 16 +- .../migrate-metabase-queries.ts | 26 +- tools/analytics-migration/migrate.sh | 14 +- tools/analytics-migration/setup-metabase.sh | 243 ++++++++++++++++++ 10 files changed, 453 insertions(+), 23 deletions(-) create mode 100644 infra/duckdb-sync/Dockerfile create mode 100644 infra/duckdb-sync/sync.sh create mode 100644 infra/metabase/Dockerfile create mode 100755 tools/analytics-migration/setup-metabase.sh diff --git a/.gitignore b/.gitignore index 86680b3bff..0c3805041c 100755 --- a/.gitignore +++ b/.gitignore @@ -67,4 +67,5 @@ pubpub-localdb/ tsconfig.tsbuildinfo .jest/secret-env.js -infra/pgdata/ \ No newline at end of file +infra/pgdata/ +infra/metabase-plugins/ \ No newline at end of file diff --git a/infra/docker-compose.dev.yml b/infra/docker-compose.dev.yml index dbe4f78478..5b84969836 100644 --- a/infra/docker-compose.dev.yml +++ b/infra/docker-compose.dev.yml @@ -87,7 +87,8 @@ services: networks: [appnet] metabase: - image: metabase/metabase:v0.48.1 + build: + context: ./metabase environment: MB_DB_TYPE: postgres MB_DB_DBNAME: metabasedb @@ -96,11 +97,32 @@ services: MB_DB_USER: appuser MB_DB_PASS: apppassword MB_JETTY_PORT: "3001" + MB_ENABLE_QUERY_CACHING: "true" + MB_QUERY_CACHING_TTL_RATIO: "10" + MB_QUERY_CACHING_MIN_TTL: "120" depends_on: - metabase_db - db ports: - "${METABASE_PORT:-3030}:3001" + volumes: + - analytics_duckdb:/duckdb:ro + networks: [appnet] + + analytics_sync: + build: + context: ./duckdb-sync + environment: + PG_HOST: db + PG_PORT: "5432" + PG_DB: appdb + PG_USER: appuser + PG_PASS: apppassword + SYNC_INTERVAL: "${ANALYTICS_SYNC_INTERVAL:-43200}" + volumes: + - analytics_duckdb:/data + depends_on: + - db networks: [appnet] metabase_db: @@ -141,3 +163,4 @@ volumes: pgdata: rabbitmqdata: metabase_pgdata: + analytics_duckdb: diff --git a/infra/duckdb-sync/Dockerfile b/infra/duckdb-sync/Dockerfile new file mode 100644 index 0000000000..edbc4338b8 --- /dev/null +++ b/infra/duckdb-sync/Dockerfile @@ -0,0 +1,20 @@ +FROM debian:bookworm-slim + +ARG DUCKDB_VERSION=v1.4.4 +ARG TARGETARCH + +RUN apt-get update && \ + apt-get install -y --no-install-recommends curl unzip ca-certificates && \ + rm -rf /var/lib/apt/lists/* && \ + curl -fsSL \ + "https://github.com/duckdb/duckdb/releases/download/${DUCKDB_VERSION}/duckdb_cli-linux-${TARGETARCH}.zip" \ + -o /tmp/duckdb.zip && \ + unzip /tmp/duckdb.zip -d /usr/local/bin/ && \ + rm /tmp/duckdb.zip && \ + chmod +x /usr/local/bin/duckdb && \ + duckdb -c "INSTALL postgres" + +COPY sync.sh /usr/local/bin/sync.sh +RUN chmod +x /usr/local/bin/sync.sh + +CMD ["/usr/local/bin/sync.sh"] diff --git a/infra/duckdb-sync/sync.sh b/infra/duckdb-sync/sync.sh new file mode 100644 index 0000000000..736bd9ea19 --- /dev/null +++ b/infra/duckdb-sync/sync.sh @@ -0,0 +1,65 @@ +#!/bin/bash +set -euo pipefail + +DUCKDB_FILE="${DUCKDB_FILE:-/data/analytics.duckdb}" +PG_HOST="${PG_HOST:-db}" +PG_PORT="${PG_PORT:-5432}" +PG_DB="${PG_DB:-appdb}" +PG_USER="${PG_USER:-appuser}" +PG_PASS="${PG_PASS:-apppassword}" +SYNC_INTERVAL="${SYNC_INTERVAL:-43200}" +MODE="${1:-loop}" + +PG_CONN="dbname=${PG_DB} host=${PG_HOST} port=${PG_PORT} user=${PG_USER} password=${PG_PASS}" + +sync_analytics() { + local start + start=$(date +%s) + + duckdb "$DUCKDB_FILE" < ( + SELECT COALESCE(MAX("createdAt"), '1970-01-01'::TIMESTAMPTZ) + FROM "AnalyticsEvents" + ); + +DETACH pg; +SQL + + local count elapsed + count=$(duckdb "$DUCKDB_FILE" -csv -noheader "SELECT count(*) FROM \"AnalyticsEvents\"") + elapsed=$(( $(date +%s) - start )) + echo "[$(date -Iseconds)] sync complete: ${count} rows (${elapsed}s)" +} + +if [ "$MODE" = "--full" ]; then + echo "full resync requested, removing existing data..." + rm -f "$DUCKDB_FILE" +fi + +echo "duckdb analytics sync" +echo " postgres: ${PG_HOST}:${PG_PORT}/${PG_DB}" +echo " duckdb: ${DUCKDB_FILE}" +echo "" + +sync_analytics + +if [ "$MODE" = "--once" ] || [ "$MODE" = "--full" ]; then + exit 0 +fi + +echo "entering sync loop (interval: ${SYNC_INTERVAL}s)" + +while true; do + sleep "$SYNC_INTERVAL" + sync_analytics +done diff --git a/infra/metabase/Dockerfile b/infra/metabase/Dockerfile new file mode 100644 index 0000000000..38c1176e6a --- /dev/null +++ b/infra/metabase/Dockerfile @@ -0,0 +1,34 @@ +FROM eclipse-temurin:21-jre-jammy + +ARG METABASE_VERSION=0.58.9 +ARG METABASE_DUCKDB_DRIVER_VERSION=1.4.4.0 + +ENV MB_PLUGINS_DIR=/home/metabase/plugins/ + +RUN groupadd -r metabase && useradd -r -g metabase metabase + +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +RUN mkdir -p /home/metabase/plugins /home/metabase/data && \ + chown -R metabase:metabase /home/metabase + +WORKDIR /home/metabase + +ADD --chown=metabase:metabase \ + https://downloads.metabase.com/v${METABASE_VERSION}/metabase.jar \ + /home/metabase/ + +ADD --chown=metabase:metabase \ + https://github.com/motherduckdb/metabase_duckdb_driver/releases/download/${METABASE_DUCKDB_DRIVER_VERSION}/duckdb.metabase-driver.jar \ + /home/metabase/plugins/ + +RUN chmod 755 /home/metabase/metabase.jar && \ + chmod 755 /home/metabase/plugins/duckdb.metabase-driver.jar + +EXPOSE 3000 + +USER metabase + +CMD ["java", "-jar", "/home/metabase/metabase.jar"] diff --git a/infra/stack.yml b/infra/stack.yml index b7ea1777f0..3a5b4e909f 100644 --- a/infra/stack.yml +++ b/infra/stack.yml @@ -127,7 +127,7 @@ services: condition: any metabase: - image: metabase/metabase:latest + image: ghcr.io/knowledgefutures/pubpub-metabase:${IMAGE_TAG:-latest} environment: MB_DB_TYPE: postgres MB_DB_DBNAME: metabasedb @@ -136,6 +136,11 @@ services: MB_DB_USER: '${METABASE_DB_USER:-appuser}' MB_DB_PASS: '${METABASE_DB_PASS}' MB_JETTY_PORT: '3001' + MB_ENABLE_QUERY_CACHING: 'true' + MB_QUERY_CACHING_TTL_RATIO: '10' + MB_QUERY_CACHING_MIN_TTL: '120' + volumes: + - analytics_duckdb:/duckdb:ro networks: [appnet] deploy: replicas: 1 @@ -147,6 +152,28 @@ services: restart_policy: condition: any + analytics_sync: + image: ghcr.io/knowledgefutures/pubpub-duckdb-sync:${IMAGE_TAG:-latest} + environment: + PG_HOST: db + PG_PORT: '5432' + PG_DB: appdb + PG_USER: appuser + PG_PASS: '${DB_PASS}' + SYNC_INTERVAL: '${ANALYTICS_SYNC_INTERVAL:-43200}' + volumes: + - analytics_duckdb:/data + networks: [appnet] + deploy: + replicas: 1 + resources: + limits: + memory: 4G + reservations: + memory: 1G + restart_policy: + condition: any + metabase_db: image: postgres:16 environment: @@ -199,5 +226,6 @@ volumes: pgdata: rabbitmqdata: metabase_pgdata: + analytics_duckdb: caddy_data: caddy_config: diff --git a/server/analytics/model.ts b/server/analytics/model.ts index 935f0089fb..2332ada25f 100644 --- a/server/analytics/model.ts +++ b/server/analytics/model.ts @@ -13,7 +13,17 @@ import { @Table({ updatedAt: false, - indexes: [{ fields: ['communityId', 'timestamp'] }, { fields: ['pubId', 'timestamp'] }], + indexes: [ + { + name: 'analytics_events_community_event_ts', + fields: ['communityId', 'event', 'timestamp'], + }, + { name: 'analytics_events_pub_event_ts', fields: ['pubId', 'event', 'timestamp'] }, + { + name: 'analytics_events_collection_event_ts', + fields: ['collectionId', 'event', 'timestamp'], + }, + ], }) export class AnalyticsEvent extends Model< InferAttributes, @@ -29,7 +39,6 @@ export class AnalyticsEvent extends Model< declare type: string; @AllowNull(false) - @Index @Column(DataType.TEXT) declare event: string; @@ -77,7 +86,6 @@ export class AnalyticsEvent extends Model< @Column(DataType.TEXT) declare os: string; - @Index @Column(DataType.UUID) declare communityId: string | null; @@ -130,7 +138,6 @@ export class AnalyticsEvent extends Model< @Column(DataType.TEXT) declare collectionKind: string | null; - @Index @Column(DataType.UUID) declare collectionId: string | null; @@ -140,7 +147,6 @@ export class AnalyticsEvent extends Model< @Column(DataType.TEXT) declare pubTitle: string | null; - @Index @Column(DataType.UUID) declare pubId: string | null; diff --git a/tools/analytics-migration/migrate-metabase-queries.ts b/tools/analytics-migration/migrate-metabase-queries.ts index f50416d6a7..d802db5b93 100644 --- a/tools/analytics-migration/migrate-metabase-queries.ts +++ b/tools/analytics-migration/migrate-metabase-queries.ts @@ -168,14 +168,17 @@ function buildFieldMapping(oldTables: Table[], newTables: Table[]) { // ── mbql query remapping ── -function remapQuery(obj: unknown, maps: IdMaps): unknown { +function remapQuery( + obj: Record | unknown[] | string | undefined, + maps: IdMaps, +): Record | unknown[] | string | undefined { if (Array.isArray(obj)) { if (obj.length >= 2 && obj[0] === 'field' && typeof obj[1] === 'number') { const mapped = maps.fieldIdMap[obj[1]] ?? obj[1]; - return ['field', mapped, ...obj.slice(2).map((x) => remapQuery(x, maps))]; + return ['field', mapped, ...obj.slice(2).map((x) => remapQuery(x as any, maps))]; } - return obj.map((item) => remapQuery(item, maps)); + return obj.map((item) => remapQuery(item as any, maps)); } if (obj && typeof obj === 'object') { @@ -188,7 +191,7 @@ function remapQuery(obj: unknown, maps: IdMaps): unknown { const oldId = parseInt(v.replace('card__', ''), 10); result[k] = `card__${maps.cardIdMap[oldId] ?? oldId}`; } else { - result[k] = remapQuery(v, maps); + result[k] = remapQuery(v as any, maps); } } @@ -427,7 +430,7 @@ async function main() { } try { - // biome-ignore lint/correctness/noAwaitInLoop: sequential creation required for card->card references + // biome-ignore lint/performance/noAwaitInLoops: who cares const newCard = await post('/card', { name: card.name, description: card.description, @@ -453,7 +456,7 @@ async function main() { for (const item of dashList) { try { - // biome-ignore lint/correctness/noAwaitInLoop: dashboards must be created one at a time + // biome-ignore lint/performance/noAwaitInLoops: who cares const oldDash = await get(`/dashboard/${item.id}`); const newDash = await post('/dashboard', { @@ -464,7 +467,16 @@ async function main() { }); const oldCards = oldDash.ordered_cards || oldDash.dashcards || []; - const dashCards = []; + const dashCards = [] as { + id: number; + card_id: number; + row: number; + col: number; + size_x: number; + size_y: number; + parameter_mappings: unknown; + visualization_settings: unknown; + }[]; for (let i = 0; i < oldCards.length; i++) { const dc = oldCards[i]; diff --git a/tools/analytics-migration/migrate.sh b/tools/analytics-migration/migrate.sh index 852257d711..2f38823465 100755 --- a/tools/analytics-migration/migrate.sh +++ b/tools/analytics-migration/migrate.sh @@ -74,14 +74,12 @@ CREATE TABLE IF NOT EXISTS "AnalyticsEvents" ( "createdAt" timestamptz NOT NULL DEFAULT now() ); -CREATE INDEX IF NOT EXISTS "analytics_events_community_ts" - ON "AnalyticsEvents" ("communityId", "timestamp"); -CREATE INDEX IF NOT EXISTS "analytics_events_pub_ts" - ON "AnalyticsEvents" ("pubId", "timestamp"); -CREATE INDEX IF NOT EXISTS "analytics_events_event" - ON "AnalyticsEvents" (event); -CREATE INDEX IF NOT EXISTS "analytics_events_collection" - ON "AnalyticsEvents" ("collectionId"); +CREATE INDEX IF NOT EXISTS "analytics_events_community_event_ts" + ON "AnalyticsEvents" ("communityId", event, "timestamp"); +CREATE INDEX IF NOT EXISTS "analytics_events_pub_event_ts" + ON "AnalyticsEvents" ("pubId", event, "timestamp"); +CREATE INDEX IF NOT EXISTS "analytics_events_collection_event_ts" + ON "AnalyticsEvents" ("collectionId", event, "timestamp"); SQL # ── step 3: create staging table and import csv ── diff --git a/tools/analytics-migration/setup-metabase.sh b/tools/analytics-migration/setup-metabase.sh new file mode 100755 index 0000000000..970e293c69 --- /dev/null +++ b/tools/analytics-migration/setup-metabase.sh @@ -0,0 +1,243 @@ +#!/bin/bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +COMPOSE_FILE="${PROJECT_ROOT}/infra/docker-compose.dev.yml" + +DUMP_FILE="${1:?usage: setup-metabase.sh [collection-id]}" +COLLECTION_ID="${2:-14}" + +MB_DB_PORT="${METABASE_DB_PORT:-5440}" +MB_DB_USER="${METABASE_DB_USER:-appuser}" +MB_DB_PASS="${METABASE_DB_PASS:-apppassword}" +MB_DB_NAME="${METABASE_DB_NAME:-metabasedb}" +MB_PORT="${METABASE_PORT:-3030}" +MB_URL="http://localhost:${MB_PORT}" + +MAIN_DB_HOST="${MAIN_DB_DOCKER_HOST:-db}" +MAIN_DB_PORT="${MAIN_DB_PORT:-5432}" +MAIN_DB_NAME="${MAIN_DB_NAME:-appdb}" +MAIN_DB_USER="${MAIN_DB_USER:-appuser}" +MAIN_DB_PASS="${MAIN_DB_PASS:-apppassword}" + +: "${METABASE_ADMIN_EMAIL:?set METABASE_ADMIN_EMAIL}" +: "${METABASE_ADMIN_PASSWORD:?set METABASE_ADMIN_PASSWORD}" + +MB_CONN="postgres://${MB_DB_USER}:${MB_DB_PASS}@localhost:${MB_DB_PORT}" +MB_DB_URL="${MB_CONN}/${MB_DB_NAME}" +MB_ADMIN_URL="${MB_CONN}/postgres" + +json_val() { + node -e "let d='';process.stdin.on('data',c=>d+=c);process.stdin.on('end',()=>{const o=JSON.parse(d);console.log($1)})" +} + +echo "Metabase Setup" +echo "==============" +echo " dump: $DUMP_FILE" +echo " metabase: $MB_URL" +echo " database: localhost:${MB_DB_PORT}/${MB_DB_NAME}" +echo "" + +# ── 1. stop metabase ── + +echo "[1/10] stopping metabase..." +docker compose -f "$COMPOSE_FILE" stop metabase 2>/dev/null || true + +# ── 2. ensure metabase_db is running ── + +echo "[2/10] ensuring metabase_db is running..." +docker compose -f "$COMPOSE_FILE" up -d metabase_db +until PGPASSWORD="$MB_DB_PASS" pg_isready -h localhost -p "$MB_DB_PORT" -U "$MB_DB_USER" -q 2>/dev/null; do + sleep 1 +done + +# ── 3. reset the database ── + +echo "[3/10] resetting database..." +psql "$MB_ADMIN_URL" -q -c \ + "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = '${MB_DB_NAME}' AND pid != pg_backend_pid();" \ + 2>/dev/null || true + +psql "$MB_ADMIN_URL" -q -c "DROP DATABASE IF EXISTS \"${MB_DB_NAME}\";" +psql "$MB_ADMIN_URL" -q -c "CREATE DATABASE \"${MB_DB_NAME}\" OWNER \"${MB_DB_USER}\";" + +# ── 4. import dump ── + +echo "[4/10] importing dump..." +psql "$MB_DB_URL" -q < "$DUMP_FILE" + +CARD_COUNT=$(psql "$MB_DB_URL" -tA -c "SELECT count(*) FROM report_card") +echo " $CARD_COUNT saved questions imported" + +# ── 5. fix settings ── + +echo "[5/10] patching settings..." +psql "$MB_DB_URL" -q <<'SQL' +UPDATE setting +SET value = REPLACE(value, 'US/Eastern', 'America/New_York') +WHERE value LIKE '%US/Eastern%'; + +UPDATE metabase_database +SET details = REPLACE(details, 'US/Eastern', 'America/New_York') +WHERE details LIKE '%US/Eastern%'; + +UPDATE core_user +SET settings = REPLACE(settings, 'US/Eastern', 'America/New_York') +WHERE settings LIKE '%US/Eastern%'; +SQL +echo " timezone references fixed" + +# ── 6. initial DuckDB sync ── + +echo "[6/10] running initial DuckDB sync..." +docker compose -f "$COMPOSE_FILE" build analytics_sync +docker compose -f "$COMPOSE_FILE" run --rm analytics_sync sync.sh --once + +# ── 7. start metabase ── + +echo "[7/10] starting metabase..." +docker compose -f "$COMPOSE_FILE" up -d metabase + +echo -n " waiting for ready" +for i in $(seq 1 180); do + if curl -sf "${MB_URL}/api/health" >/dev/null 2>&1; then + echo " (${i}s)" + break + fi + + if [ "$i" -eq 180 ]; then + echo "" + echo " ERROR: metabase not ready after 180s" + exit 1 + fi + + echo -n "." + sleep 1 +done + +# ── 8. add Postgres database connection ── + +echo "[8/10] configuring Postgres database connection..." + +SESSION=$(curl -sf "${MB_URL}/api/session" \ + -H 'Content-Type: application/json' \ + -d "{\"username\":\"${METABASE_ADMIN_EMAIL}\",\"password\":\"${METABASE_ADMIN_PASSWORD}\"}" \ + | json_val 'o.id') + +EXISTING_DB=$(curl -sf "${MB_URL}/api/database" \ + -H "X-Metabase-Session: ${SESSION}" \ + | json_val '(o.data || o).find(d => d.name === "PubPub")?.id ?? ""') + +if [ -n "$EXISTING_DB" ] && [ "$EXISTING_DB" != "undefined" ]; then + echo " removing existing 'PubPub' database (id=$EXISTING_DB)..." + curl -sf "${MB_URL}/api/database/${EXISTING_DB}" \ + -H "X-Metabase-Session: ${SESSION}" \ + -X DELETE >/dev/null +fi + +NEW_DB_ID=$(curl -sf "${MB_URL}/api/database" \ + -H "X-Metabase-Session: ${SESSION}" \ + -H 'Content-Type: application/json' \ + -X POST \ + -d "{ + \"name\": \"PubPub\", + \"engine\": \"postgres\", + \"details\": { + \"host\": \"${MAIN_DB_HOST}\", + \"port\": ${MAIN_DB_PORT}, + \"dbname\": \"${MAIN_DB_NAME}\", + \"user\": \"${MAIN_DB_USER}\", + \"password\": \"${MAIN_DB_PASS}\", + \"ssl\": false + }, + \"is_full_sync\": true, + \"auto_run_queries\": true + }" | json_val 'o.id') + +echo " created database 'PubPub' (id=${NEW_DB_ID})" + +curl -sf "${MB_URL}/api/database/${NEW_DB_ID}/sync_schema" \ + -H "X-Metabase-Session: ${SESSION}" \ + -X POST >/dev/null + +echo -n " waiting for sync" +for i in $(seq 1 120); do + STATUS=$(curl -sf "${MB_URL}/api/database/${NEW_DB_ID}" \ + -H "X-Metabase-Session: ${SESSION}" \ + | json_val 'o.initial_sync_status || "incomplete"' 2>/dev/null || echo "incomplete") + + if [ "$STATUS" = "complete" ]; then + echo " (${i}s)" + break + fi + + if [ "$i" -eq 120 ]; then + echo "" + echo " WARNING: sync not complete after 120s, continuing anyway" + fi + + echo -n "." + sleep 2 +done + +# ── 9. add DuckDB database connection ── + +echo "[9/10] configuring DuckDB database..." + +EXISTING_DUCKDB=$(curl -sf "${MB_URL}/api/database" \ + -H "X-Metabase-Session: ${SESSION}" \ + | json_val '(o.data || o).find(d => d.name === "Analytics (DuckDB)")?.id ?? ""') + +if [ -n "$EXISTING_DUCKDB" ] && [ "$EXISTING_DUCKDB" != "undefined" ]; then + echo " removing existing 'Analytics (DuckDB)' database (id=$EXISTING_DUCKDB)..." + curl -sf "${MB_URL}/api/database/${EXISTING_DUCKDB}" \ + -H "X-Metabase-Session: ${SESSION}" \ + -X DELETE >/dev/null +fi + +DUCKDB_ID=$(curl -sf "${MB_URL}/api/database" \ + -H "X-Metabase-Session: ${SESSION}" \ + -H 'Content-Type: application/json' \ + -X POST \ + -d '{ + "name": "Analytics (DuckDB)", + "engine": "duckdb", + "details": { + "database_file": "/duckdb/analytics.duckdb", + "read_only": true + }, + "is_full_sync": true, + "auto_run_queries": true + }' | json_val 'o.id') + +echo " created database 'Analytics (DuckDB)' (id=${DUCKDB_ID})" + +curl -sf "${MB_URL}/api/database/${DUCKDB_ID}/sync_schema" \ + -H "X-Metabase-Session: ${SESSION}" \ + -X POST >/dev/null + +echo " sync triggered" + +# ── 10. migrate queries ── + +OLD_DB_ID=$(psql "$MB_DB_URL" -tA -c "SELECT id FROM metabase_database WHERE engine = 'redshift' ORDER BY id LIMIT 1") + +if [ -z "$OLD_DB_ID" ]; then + echo "[10/10] no redshift database found, skipping query migration" + echo "" + echo "done. open ${MB_URL} to verify." + exit 0 +fi + +echo "[10/10] migrating saved questions (redshift db=$OLD_DB_ID -> postgres db=$NEW_DB_ID)..." + +npx tsx "${SCRIPT_DIR}/migrate-metabase-queries.ts" \ + --metabase-url "$MB_URL" \ + --session "$SESSION" \ + --old-db-id "$OLD_DB_ID" \ + --new-db-id "$NEW_DB_ID" \ + --collection "$COLLECTION_ID" + +echo "" +echo "done. open ${MB_URL} to verify." From d44e62254314b174c608ebe965256de07644d281 Mon Sep 17 00:00:00 2001 From: "Thomas F. K. Jorna" Date: Tue, 7 Apr 2026 15:42:40 +0200 Subject: [PATCH 4/5] fix: rename --- tools/analytics-migration/{migrate.sh => migrate-redshift.sh} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tools/analytics-migration/{migrate.sh => migrate-redshift.sh} (100%) diff --git a/tools/analytics-migration/migrate.sh b/tools/analytics-migration/migrate-redshift.sh similarity index 100% rename from tools/analytics-migration/migrate.sh rename to tools/analytics-migration/migrate-redshift.sh From c53a71ab75502273e283fc81c4ea2531d68c1fa9 Mon Sep 17 00:00:00 2001 From: "Thomas F. K. Jorna" Date: Tue, 7 Apr 2026 18:05:19 +0200 Subject: [PATCH 5/5] feat: lets go --- infra/.env.dev.enc | 106 +++--- infra/.env.enc | 110 +++--- infra/docker-compose.dev.yml | 39 ++- infra/duckdb-sync/sync.sh | 11 +- infra/env.example | 13 + tools/analytics-migration/Dockerfile | 37 ++ tools/analytics-migration/README.md | 34 ++ tools/analytics-migration/migrate-metabase.sh | 316 ++++++++++++++++++ tools/analytics-migration/migrate-redshift.sh | 61 ++-- tools/analytics-migration/setup-metabase.sh | 243 -------------- 10 files changed, 596 insertions(+), 374 deletions(-) create mode 100644 infra/env.example create mode 100644 tools/analytics-migration/Dockerfile create mode 100644 tools/analytics-migration/README.md create mode 100755 tools/analytics-migration/migrate-metabase.sh delete mode 100755 tools/analytics-migration/setup-metabase.sh diff --git a/infra/.env.dev.enc b/infra/.env.dev.enc index 3ed84c1eed..d7491e1299 100644 --- a/infra/.env.dev.enc +++ b/infra/.env.dev.enc @@ -1,52 +1,68 @@ -AES_ENCRYPTION_KEY=ENC[AES256_GCM,data:Kz4/KiHal4JIP5kmXBSVg+VI/nvNGfPtRv8Rrf9mUdY7bDuc5k1In1oSPHsjjNQLDN8bs6YuzCSBzWyyouDUAg==,iv:QC65R/DVjioNsTb6EnChdSaugtlOqvEx+m4C57eZStM=,tag:kc9NcW6faLAdJIy17wcjtQ==,type:str] -ALGOLIA_ID=ENC[AES256_GCM,data:Pu/lBy/Vo/9p1g==,iv:4HhG6IjEyheW+Ug1mI9m+6xzxRuVf2xvJ+dPtONp130=,tag:9LNs+U1klSpxqVgAnQJaug==,type:str] -ALGOLIA_KEY=ENC[AES256_GCM,data:bY2wVEofNkVMlvwhEhGr4punAQ/JMShYYPpKfR/kyK8=,iv:NIrux9ntBLGphMQ0yem+puX9gjnvlGb3jUiDupiyths=,tag:8DkKbPwvQtPPWHGmH0zRWw==,type:str] -ALGOLIA_SEARCH_KEY=ENC[AES256_GCM,data:MWIU9p3KmmsDjdxk+IRUJgzB7SQw479ZUG9WdlXLXk8=,iv:ujK8AHJPvvbCi7HxhL7cGvxQ/HYFJRriCFbsNkCAHXk=,tag:KaU7v4CrQJRKmH+QfJwGlA==,type:str] -AWS_ACCESS_KEY_ID=ENC[AES256_GCM,data:nnIlGdEBlwi268NlCFi8d5Ugjmg=,iv:uP5wFBOWZJrSH50uYNzJ4kJdv3jYUcEFlMRVrDMlTng=,tag:Uv0D0mKNKebrNwuySxIM7g==,type:str] -AWS_BACKUP_ACCESS_KEY_ID=ENC[AES256_GCM,data:aWJiiVfgnam1My2J/J3k1p/8j4g=,iv:XD+OHujRRMUyyZ8cMGJ2mw1TCtwELrR16txekGpwMi8=,tag:HEsGvUYMNEiv1mk2mb4GYw==,type:str] -AWS_BACKUP_SECRET_ACCESS_KEY=ENC[AES256_GCM,data:fYAtsyYDbTp3EBvm6vtabfmBupk9ryfpcu4P3ROoyTEOBnZyPrhPnQ==,iv:L/VZB/DccoB55GFWPfsVbQ9uzdIk3nwadkKZ15YzFGE=,tag:WEbCfpCYN0jHrfzWsc5TNg==,type:str] -AWS_SECRET_ACCESS_KEY=ENC[AES256_GCM,data:DGbljDUXr1ehACzOOWFFN2fCIrzq5aChbZg++7IXzLwBxWNldm1vqQ==,iv:G4lKHMPP4BL0kaZz4oH1CulYBiRh7SyZAu3pKhGoUys=,tag:LIVhg5BYgGuH7B7Eyk82Dw==,type:str] -BACKUPS_SECRET=ENC[AES256_GCM,data:eQLNXNHy2Uv6TJt0nn2aZFCa/cyUFQFp490E+sv4+oMY2c0zZ1VpYSuIQCg=,iv:oFQ6xrc6v0z9K0t9m9EU3nXCwsW0lYQNoRbXclnAIFU=,tag:nO+JnRZKD5wBlIoMtyqnuw==,type:str] -DATABASE_URL=ENC[AES256_GCM,data:+uZ8Kkhyxa6KersbX7p6hnqSpp9KcjCEttWRnxDWatf8jr3h+MhyP1SW/6c=,iv:3HQ97+1Gj+7+UjE798q3B4tF9CH7Oi8T4217dgiSygE=,tag:SXFgzJRgDXECYqx1yW6h1w==,type:str] -DATACITE_DEPOSIT_URL=ENC[AES256_GCM,data:Q+H2KthVe4xO5RChejW0YU1TgGtJqXW4I7kc+5WXpM40GQ==,iv:jZjvyIZf3p/a4hC2LhqW6eo4wDGPLFgTn4YTWPVSrxI=,tag:WVguZkbwTPQ5kot//hkSew==,type:str] -DOI_LOGIN_ID=ENC[AES256_GCM,data:MnV5UPz2,iv:WILeu6evYxrmctdiCKU9xuuW1E4u7RdFSjG5abdBZZs=,tag:u5gWtVGfeuXrJeqRREMPBg==,type:str] -DOI_LOGIN_PASSWORD=ENC[AES256_GCM,data:gNfWFreLj6v6dXP/hxKvD7M7jr0=,iv:VAVh9dBRQP/ggQmKHOLepqDN8+pCmN5kIm+Xe7fpYis=,tag:w4dAIBjRMKUBnoX3FqjtXw==,type:str] -DOI_SUBMISSION_URL=ENC[AES256_GCM,data:BjI1Ebz5t00X3n8PSmjPUk6IaIFgWOqEDDA+SheOHLDNotjabDjUE2k=,iv:FB5IXGUm4c/rdVfzD6S8WnmA7CW8y91YsvDr+EUf1NA=,tag:Q+OwRyQPNaBfNLb21AQVfg==,type:str] -FASTLY_PURGE_TOKEN_DUQDUQ=ENC[AES256_GCM,data:MN2kB/F4ztrOXpuDVV3qDBotky/1suF4NsZbtTmE9gY=,iv:VzvRIdYjX4Tw2op5qTsdLg2l3ABeGQnTRp2EOf4F1S0=,tag:3Ua3WcAWJHVHxyRIfNTDCw==,type:str] -FASTLY_SERVICE_ID_DUQDUQ=ENC[AES256_GCM,data:BpF94SQhZj9+gwlWtGT23VFFQaZBMw==,iv:HRVbeeO7QCYi9wO841TCwS8BZYTvb//3diy9u07RL7c=,tag:EqtpAL5cWLYNulhMupkYOw==,type:str] -FIREBASE_SERVICE_ACCOUNT_BASE64=ENC[AES256_GCM,data:PcVGtS2RrxRkPGNUWT/LF3NfiIzdLVJGgiDpqRqczMjuluc7pUAWP5iUseBEKy6lM7W/ehJGakQhDvyFNhnWAmp+HTebN6XdXEznuf688ZylhGD7jnXSvTmZiN42nx4zLhnX6uUZJCu7Gi/4GPnQyLofzBNjgr82/jvph8ZzaEhuaAYokUnKEp/ks60o/oWAMSf/raMtXdPAaopws24n/YYlHEqu4kfSroWBYcHxhi0DysGDjPn2BR/EFF7G8PiWYgYXUD4cKikZXncpW0Zf+tF4RXA6VXFX3UYNT0AzgjMBeG4yW4xviz170N7t9AFDCzRdhPaplaMDQLJtW3w2oDVXrymIp53LMIeHzs19n1j5A5XyE0uZEWqhVWO8MkJQynF3NEcpYX864K6yA6qm7tnkQGRx9sFvSfFdutpsOpEYR7r18Jq2k6NSdPJEcyLz68ICW+zeKXJTBAN0xBzM/YWDG/dfChdvWLnKS1RXQNpeFqtT7MzwIg6yo5wPsyz/B+09VKPC+m/dyQkr3XpF56cnOgi4CZcRsZf2T6e2+BI3VfbHVr3weNJFIrW6DKthc7maUXexBDU00q0NSCG/evdvumePrDuXOyNUhONTj/29fXLNqyzxFJ/f8YlHcHgHdylUcCgK/8QNYW9UKeFlQbYEX3r2IUpCHbsu1Sz01W3yOrFvZiPgvkj0E2I25kEZz0tyYZsGuNVWr0sOsgit+kpA+0tZOeWLrwJE9dKeB5ILJ8I31fgaCnR6BqltYpkYDKFIXPQjmf9sx6RmK5A7peVbfm6QO3XEVXFxnTiIIo338mqBxmaoYxWGoQqnTTHhNWaPUI5d51JVRY7UDGTUNoqT8u3+aqlCEVMOprvSutj4KSvPEIcOg9gEBy8mlG30elwIhmSfldlctHbwLOGVzMzoake5I8xTFyf2WD4YzP6uB94i4dfhj2wRQYssnVStlJV0DcFe4xJNhoHITo+cExVGzrn6cKuRyvVaFrZdqfMQSWig64Rs0JzTMZrAyeSvlDHfHn3BrEJ6HOipOPhGdtLsc7BNqgUQAGDe0unzfmtTwqz0MZg4pjoMNbtj7lFD1ngVvOxxv8ye8dPbd8UqB+NU+IJIoj1GugapPsSTL0ao3L96NLQBrVLguU9RVZTNmbpX2JXbkONy1l9IHrvQc8ygeLIXwp9UWGzoSzFpsn4iXx8ausW/VnvCBdrAScxYrfBBufLFolIzEi/EROIpdFS0p9E6QwuD8UzDMLi1xS6tw6JN0VKnlwqQRdFUjojVB76kDxGEj0u64+hRhNqDHmDQYw7hvwHq/QAJbjcQAYCRMIjB3LoHckgNF+c8IWLuSbcb4dqjkXxWGEz40HTMlC/cPTZ99/QNosIF7DvnTn3RGzZcaas2zJZJOhktBt+PoGUgongxFLlYlxgnpUAFSDbKqQ0Pm2KLp1eGmwyyNbTYwhPoJaBWLdcqCluPCZnn3gJp4WwwcD7h9OjozunFjxZVbjEknlfQEbCq5ndsaHcQLbQVEeOfyVK0EVJdO0FFLbgaokoW97lgSAtYdD0n32Tqfz/oPgzzhX63gTtZTix7vnDy/1xYmDYMktlIg7NtkN2piGUZLyARy5OgY6B2kGjVyyDsd3MB2YswCxyLd/TsfVj+xcRBlnY0hrF/W1m//yiz1YhKLJGWoR2Qq+zB1YXI/UbfFAII+OT6nIs8+oCfBVrIDJjj2QDtKwm4loJKJ3dkIIFIEzLDfUm6EFv2kOljnlafbOZyQFbTvfT8btwTV1Sc4OFi0qW/6yENHB80uNZi+N0hWf3TlwD/QZHYi6Q2NBydYOdtnC/965cKKhQLN9aoJ4mirOz8LiarhJS5Gdd5aVhGE7XkDsVPHlBe+XTSxMtr4OIoTfEu/zZMblNgAzO+qQKsYdb1a9FoFYHXUr5PK81wZA2oWA5z8aaWSgiLne1cFWmAugs8XJEpmlRnyc76pBsLhrAmZcii2iyYb/y1ZpDLVUrL8BvsUJWZMfLiUKlbf3kbgdp3MNxC3qiUMbiY+bDL+o6sgiOiPVeyv0GJTD1sQ5HLbr0H2BDF4QH+bXASWazyloPMfz/ObAIHBhp96OYGAwefFdRvV8fc3I6T6Uykm77alE2TWW1AF/6Pw4zPvBsiqJ9hea9bTFCpb8DoOm9yJaeS7ZkgJXn8OZd/ylyizT2j2y0TDf+LMM2M78BE7xDD/q6iuzMY4gtbKU5Orl+t8WhCP37ir6cUYlPN4f4llaiASM6IDqkcqSw7kbQ/T4zOvAfd0FoUa2ZZFrj8C9FiY5sFR8mqTmjcxYpK8Jy3uWpAY85xgvTlI+XKvFz3AiVNFi3zJyiKy8PwAX7wPE+6k+y9hUOfw/8+P++vm3F5CtFC6T5lehjFo3mKFbU/xH/UaJsklksWzrrFx8sKd2vvTK9Wu8RQEoJUuJ1ulRtXaO3M0dIsKmCaV0LRKcDuIXSI1sd0WdUOGtSSNfKTt4SioXGRvocfb8bWRfzI/XJjpVgj2pMDwY5+4IcGWTWVJpE4RUC7h49kuc9n7FvP4PH0af11mYHX8Vt9qnv78ge/vsAK0sSB+fRxxMdSEdYDGJlrqUYCmqx7JmeFqKFfZIEZmj2PrMkb9JvU2cv/6N0ur3Ah2ppk6BysMFl/cWtkjVdNIW7X8tx59Jv48sIf/x85ivsNkN2croekCqjF4uJV2jdhDrUbnywEwty1tM6BKS62AqS+Rigz/z/+CmG3mWsEM78SnL7OU1Qiw/rr1JX0SGA8f1S9FozJao70+jsZBp/6dNNotvKKmyVG3Jr1vDojWLW3zhM9bZidR1nHVlyTjbNWA0aNrVPB4dlkOnF3WrS1YnRUnk7xY7kyEIPYaHvPu+DgcUk7juyC6tjUYpb1ex9f/0WRw67n8Ohebulbg1JIaCdkgPraFe8C00RBGvy3B76fXi1efslFlwGU/rXD5xv7MFw4R8VEehWb5Rw6ZS73vbRbZ4M7Jabu1tN7/wO6bWTv4XOhPX3O2zJEI4RlZZt+L5tRgacm94IvbMYZ3E+vDT7YAARsMMF1uDqwLDFanA/geEZO0mHi0SN5X4/9aEb9ZE9CxdzUCRNUJ5GqHLSlGojKkLDK7XSQORFTVrgcI1ofrIOVQ+8YYsp6oEDzCuLPPkv/LjOfmcfBURe45FrAhjYtfkNKXhy7P15ivncTKQaAY1F6+23ryCzmkoUhsKmxf27qfKkA4fsrnYSVej/hmBey7GsEuq/GAiyhjle89aIlOucKV/6XlkXWNAzAgBMvcLCPU/q4VSCND7Uj4BV0uunwUFx22P6FJfrZ8YIVtypOIfyuRykmZaKrPJetLLlrq4zojudjdyr+OYMYjn6I1ijbycJX8MF9e9e5NXK3YQ78pTvTQoSXAe06vooPWoBiqyK2pSLKcb80rQWar5zYZKamS0KDcjdkL8VvBC+/+KTYx11w/qk9SvOwKsK7fqag0TL+mn30N77mMtcMXi9+2SYGC6ldbkLVFBAUDe/kOxeyRkTDmJAZDp9IbdMQn+S9qQF2WPdHt7XvWuwRpw3Apw03k24UQFP+mQ1nDyeWaf5x+vuyfWtHJWgAyTrcvxiDkMdo8D4cenn7K9h8wC09rDyYyBQKUDpYPvTxjSJ4/toaTZq7daXfokcP9p5GEZin6m2QIQ2oAT+LeDXY9A5KN9dXdVzkOAqQy8X1XyfFvhWxhL/txrwIcA+eWuD4j2VgKQ4U4ZWTKgVyDEbQQRRRKSnCs80vBAdCgY6+WpsRMPdRWylB5fZOz4bSHwP0yu9QjyxHoI+FdjtY/F2j4KMt3SjVw7kxVEDYLklqjNcCBf6r4Dy9N1n2nvg3fkB/XQk0RPTDKG40GGPx/uq6HhTT76NnU4K6mBGCaT/K9THQyAFNrnaverAkFX18+QIML7oJtgTV1SM+7NMwIiRP3p2OoPlbjgjSVs/B3VCBKWzlIhcVNSxyO7EpJnzO3Tkdxfp4Q37XbKBCgFuZe57VQiUEuUJ/j9SBOOQt2F5bPo47rPOhk9hcjWpPfuGSuhkNhM3gY+yQHPOYliWi7UqSqhR6WXuIJkbvZH7i8Cb65jFltJtyUGlZjk17DtuKK6IIQtLdJM7SP1VQ6/6Dk6plMIZmbmtJ80cwfvE=,iv:gApN9Y2dF+3z6KbQcR5TmRIBO0DK5B907WaPhD9LIuE=,tag:Hl5fty+5t8g221MjMyfCqA==,type:str] -IS_DUQDUQ=ENC[AES256_GCM,data:NQOkmw==,iv:0tqLjo3HE/XZnR6UgZZBFZtn2vq6mj01XgO9j5B0r9s=,tag:dX45cwa5tWLgIzjxVe1Log==,type:str] -JWT_SIGNING_SECRET=ENC[AES256_GCM,data:PjgE+THClW+wcA1gvnT5Ej4sOmIjYJCXtSw7//wpxOaeuKs5Ic9Jt45qFErknu4lL6RtHEHXvQuuxgYEKkVqzpYuH2ckxFXRW/UChWMj/xw1ye8ZHjca1CqY8kHyN5wMGgxAAaNZAuK9IvRGV8DEWdepRR9JKM8f3RwsZSE4mw0/U/ClBbc9NywFFAGotouVPxexqWZgb8K1qA7i59FXYMP8HqBu0Z9XZzSPJ1Jj9lWe0HbRyE5IC5DX4sqCDuC2UdboDRfp5RwpJhMYWkKMf3q+3xccTm/jMuMX1BgybG26r4xgmko13g7aTcznLj0qzzY3OWilk0jZWJ5zIXp7C8282BxnFZmWfE224zi+TEaJBu8aAn/WsZznqp6dHv1PqqhXEKf109/fHKOlxFjQKx4qL4/AwS9rUBWWQZ34Ldm9Y/7F17/Hmo+DMztL9xfRyu3NkzwjOCWKXL+PX5A9AszKELOPkaJxT4MIdtNZp1CLKl3RQtpZ33l7gqb3kJuYA9+WGgya7HN0QX9IbkX9D1YdAV/cA2V9dMd7tb3pvnxybbEODig2eDwxEp0fVAsULwxTF72PRgczGTiza13zCE31NEwqEYEBRE6Nq1UIJSus2aiBocrETDtfvELzvGSTOtlJUOM+ARoZiGyUvg/BpTuNoqEkCB2lYmSiX4b1jrxiE70qgMdbG85Bi+POqcfrg68cI5rk/zIbj08mBw5QTteELNI0EFeTzd013nxYXWkAlpC+OtlaZk/Pi1n8qhqHmkPfSlZVjxL/hqTwSviybwIJDh/cZCPanSpZ+Sn7fgEG8KzaQQHXRVqxXJp/2FRt80dY2v+nX54K664DPMFWgMHlqVQYIDoWn8Yv8NVVlP2lAxAajuOAC0sRRVgbZ6IOM6Gw/MD/u7mG+cxm/cvNZZfhT2kjzotEPqsGOGJk1tBbNuxcosi1sCmFlhZsFm1gjy09kwbRTVF68U/dpfjC+bl2+e7uMWeFcJOOJfI+AczUNu8MI+Perzs1C+P8bCne+27yvNAnPnerXUdpnJyfaRsbD4kOqm89681iMEhJNA2XToK7DTBslgXlLa2YS+Cp2jQAttsoTDKsEfqZIDN11JxXYgRoBjb3usglxGZRSp3pqGv3rrqXyqx4umL77mreb2xdil4kOBySFSKG4l1hHaK/NRJqsiaHX36F1ELFy3P0V+z0hW9QujyWDlt13gymBZalCgiLfawSDplhgULcFFoEQGc/NvFQvuvlo8ZvFycsBlZaam52WXM0BsoDvfGGyunTg27KCjx0+eB2hqsIABTU4qKNIxFFdWCeT325NAO82oO65TgfSglkXRDujjAh/nj3VgSeLU+SGGobM30+Bg==,iv:JHBkoIafd0nzqnzZNYhBx5Y12q94fDdQktC6iW43cyc=,tag:wV01DXrn1ypimNFrzzJLcQ==,type:str] -MAILCHIMP_API_KEY=ENC[AES256_GCM,data:6Co8J9KOFq1Q4qIVk3h8gayLsmP/kJ8Unczd56XjT/lC4Hkl,iv:2sNxyJmFZMigRKzqsaiDM09N/Dank783OE9SxvpUZfU=,tag:n8AzKw/BKoXciipBZkb6VA==,type:str] -MAILGUN_API_KEY=ENC[AES256_GCM,data:+JBNQ4LxSOkZWtB2AQPSDA4juXF5jUKZrTAQo8xGB0boQEW7,iv:AxtIwHYoNnSN9QjgRQNDK85WK+sCyYB+Rlzks106eyg=,tag:G70UlOuspR+BDsx/5r697w==,type:str] -METABASE_SECRET_KEY=ENC[AES256_GCM,data:GW/O7+RQWlC+rBb2A/xIHKyYyrEcmUMy7dxqDC5nQ/UpkzOHK1SOpRvandIa62YivcNNvxwEzAFHu7BASjZK6A==,iv:GPPmRbHHys6I5FGddg/AZsnysFQ3PVqPquupbhiAtng=,tag:XoN4puLeTMwVB4zQ72Qpqw==,type:str] -NODE_ENV=ENC[AES256_GCM,data:7zwIoxkdMtNzBg==,iv:b8n1uKjI71k1dc1kMYJphEQnnefqS6+C8VzGgOldUm0=,tag:rN6nFQjBY4cghk/n0aeBuw==,type:str] -#ENC[AES256_GCM,data:AQh6rlnCeWROveEJsICN/veNdoKD22C++/M=,iv:Sw5zGhSVxTzs0m5aRWoGiPRARwNnCujwdlEN1i9ZMQg=,tag:oSE0CuDZoN3hJNhykICfbA==,type:comment] -#ENC[AES256_GCM,data:189wvfDMP4Wv/4CUSOi/ls4jIZpU7PBTa8mcDYxqnU5N9NhSjq8=,iv:ohTmqGFpcn4cbIos8TTemUwNGzTXFRZHAEo3qf8TqEc=,tag:NrWA7r7wP4MVSwJU8A42Dg==,type:comment] -S3_BACKUP_ENDPOINT=ENC[AES256_GCM,data:kinsYfgL36qIqGou0TaalR1HGiWvX/6nCcNe/gYPp9ypSGo=,iv:Nz5Bub7iBiXlo5S5vpiv1v7OrX/2dx3VOcXNSNtigaY=,tag:rpAu5dDTZnveOMMjWCBI+A==,type:str] -S3_BACKUP_ACCESS_KEY=ENC[AES256_GCM,data:4/yI1dtcnuRx+wzb3F+ygBDsKxs=,iv:ybu3IGbI8K98byov4TR9iy3FcG59Yr7EZC5oVHbUpUk=,tag:+pDV21jYmde6KlGa9HfV1w==,type:str] -S3_BACKUP_SECRET_KEY=ENC[AES256_GCM,data:wEFLWIh6UL+eI9eMsgqad5lI5dppWIzac/E7B8Fa3PUSLQejPQgpeA==,iv:1r3DFRl7i0QPZ7ITr0jhKV08b1ZbnDmnmIw7FCpOkzo=,tag:rIeDFnAM4M4VaoagzalxOQ==,type:str] -S3_BACKUP_BUCKET=ENC[AES256_GCM,data:aJ9K549KzJSlgNs=,iv:Ca9+3SxNXvyd7pvX9ShtGnjPUn8LARnMhOfSaauUAEg=,tag:0sRvNSahQTOsw3Xk9YrOPw==,type:str] -SENTRY_AUTH_TOKEN=ENC[AES256_GCM,data:OTK6lJ6IK46n8wXKVLTH5WwKXn47iLIZjEPQasLPW29RfBjOfihuZN2kEf4T8pBwDQh0fGaY6crs1dewF4ama4hXvmg13kWqO9MwSU+NHiuroX4gUZ+Fg8xkBOjnarEe7NMiRMP0BrlntD1LwwcTDV3p/Df6SphgyYfqeXDiJP1O4AosPuv+4egBKqqY2KCy3czc47PcMWapSxWJfMU7KlWggYoSiU3BR3EhbP8cRVZ1tYUttrb0brI28A==,iv:c+h68p57wTshNPaAzd0EiEBVXcAPvBUvgozWJEpZ7n4=,tag:aV3vlkzWJCAp1n6kLXCFNA==,type:str] -SENTRY_ORG=ENC[AES256_GCM,data:m8Qr,iv:RdVMbGg1oRNC6t+Ly27+wbjR7oI6uGYmPgGagNFE3cs=,tag:/xqPyr8e2SZgMw4dM9eRrQ==,type:str] -SEQUELIZE_MAX_CONNECTIONS=ENC[AES256_GCM,data:ius=,iv:SgH1AcWawWQRDHcB0E5aWCfp3UXDSOvCXocgsV1eeKo=,tag:1QfV5Jl9bQvV64Ty+IcNwA==,type:str] -SLACK_WEBHOOK_URL=ENC[AES256_GCM,data:eLJUhmgQ+zOYRHnYKJrp60DQpKR6Li3iOCeJ1VaHWv9IwItuakAEIRbVrC8hp8Q0c02M76a6nlObomuRFSqK/zOMc3aqomg9mCpObYsgQw==,iv:vCCjOU36LdEwEHaA1Rqzqfs0Po3uodY4qJSbbih2BlQ=,tag:er56v92xiIMel1MjqRL/cQ==,type:str] -STITCH_WEBHOOK_URL=ENC[AES256_GCM,data:FFBgY/8j+ZkVDJlx20UsSc7OiR7V4yg38AC6jJWKZXZ/KnRhCjMgsqwgGHwxUBbZ1YgB/ZKLscAyhlGWeF9An9BsFueUJk1v9W+w8IOBpiczjftfC4VEqVWTqpiWT09iGbiiHYZTJKUvdxeNnsStoNAmcN7R,iv:GUY0HRWcMHKlYhVDo/LsXrS0UXMdIuak8bn0gdtAVFs=,tag:Vo5Qvjt9JHpRVbG01s01kw==,type:str] -ZOTERO_CLIENT_KEY=ENC[AES256_GCM,data:whlgZOo3vAFqntlSdO+vvrBIws8=,iv:yiBuRlqWtTD7VZuylXFs8c+rDCZbce3N0c3kG8i50Ss=,tag:V7mbPzU7QGJcv98IUc/Mig==,type:str] -ZOTERO_CLIENT_SECRET=ENC[AES256_GCM,data:oFi5Jd45an5EUeIfbNL2cmcMa2U=,iv:Qda3XG/A/3ilj4Ak0AL2LBSq+N5TClAtIZ15w6//U1I=,tag:5tx1ESB4JOQNGM18L4Y0xA==,type:str] -sops_age__list_0__map_enc=-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAzZWZlN0NjRG9aZHlKWHVR\nOElRZlVscVFseGRORHZoSEpHS1ViUHpnWFVNCkR5NUU0ZUJlR2VOZHV5aHRJTi9G\nK29CVHhvOUt6OUs2aWNOSG1RNHdzMlUKLS0tIDVHcUF3Z3dzKzhXeHNERUlUb0Na\nUXBaS1doRkx0aVdFWGJVUGFpNlN4aG8K0cjHGDgqdu4DnvrU1QIZAkaMIoZA02aE\nVlURBU9Y4MInhk3xs/9MSxNLaqlOPDu5sCXRI9ATO02fkiWNDIiDRg==\n-----END AGE ENCRYPTED FILE-----\n +AES_ENCRYPTION_KEY=ENC[AES256_GCM,data:751MRbeh64vjtiQ2jxp1bq8IRtRixwPGYTfDMpj2lic32BLg6UtlWkeH2GytlzE+KRYsIsiGx72lm3ZrZ1fMhw==,iv:3wgol96YJTLcYw76R85RLy/v/ReXnDJYs6HrOeXg9y0=,tag:6HHc2Uiv/ExPVG3i14ZSbQ==,type:str] +ALGOLIA_ID=ENC[AES256_GCM,data:kBCQeIuUlqbS+g==,iv:p5cEC52oqS8pU+mUaEN8OJy/81EihZB5xr80gBor7JQ=,tag:ApZz0/ou6z5Gbsc/8YRaog==,type:str] +ALGOLIA_KEY=ENC[AES256_GCM,data:6n9R94NaX6G0w4hCE76je8ul9WvyMs7p7nr973E10a4=,iv:pVmd5+t3O5M1qy02m6qZVXVaY++XdSPHVE//5wj8nDQ=,tag:Vbd8ajqX+fMYQ3p5v4kCQg==,type:str] +ALGOLIA_SEARCH_KEY=ENC[AES256_GCM,data:JeHKOBgr8M6drd7wtU8SfKCpSX931f8hO9xpmSPDmRY=,iv:Bwf5WkvVIiOrpKT/HGw6WZpJfTosUgtILluJKzRuBsY=,tag:7PkZS2M0XuxZCqd2BjCyyw==,type:str] +AWS_ACCESS_KEY_ID=ENC[AES256_GCM,data:LZeSGTlDiQK/V/8aZuMNDpdDKtw=,iv:Xd72OG34Z1Sk2YyFz7gtZLXv7hRH5BDqOp8RKLfS93k=,tag:dt6RXQ1G+8CCOH9F/pfJQQ==,type:str] +AWS_BACKUP_ACCESS_KEY_ID=ENC[AES256_GCM,data:LpA/VZ5/vO9yTV9EpGPDsMvwYNs=,iv:Ogx5wNtEp5CFdoRZUnGRpsrcs3Z1zM3A5GOzRYJ4UDs=,tag:VkdkZTgQfKEomdH4s66Gdg==,type:str] +AWS_BACKUP_SECRET_ACCESS_KEY=ENC[AES256_GCM,data:PZdQ0rzvqo2OL9xyqbQFqa/ePQDhtZuIMjoww0rTUoQIDyl1CzISGg==,iv:o4W8EFEEwzdsqMMmrgPKaLdWTydGILaw0CV1QdrHw4A=,tag:Q3Pz6ppBiiHd9RHzzdaZ1g==,type:str] +AWS_SECRET_ACCESS_KEY=ENC[AES256_GCM,data:96A6q02ifiAqvLA/mQEWyPlcEEC4txeyu0njUeDtwOT/1o2LWmAa/A==,iv:QpJkInEPh1gtGQq0XtsB5sR1kgLiV9oyylEsd4ccsCw=,tag:oNJoIQxWy/mfJVo1kkZo2w==,type:str] +BACKUPS_SECRET=ENC[AES256_GCM,data:QbNhevI8OwrxrjsG5elQCDNkYSbszzt07dyfFO3MP95l7zYoWpLQA2oR5eo=,iv:N4CfzpS9ba+XTbnrNVlQfVxOypT+gcoPZX9/idaQ4YQ=,tag:1CsEbtdZAIeadF5IfppgtQ==,type:str] +CLOUDFLARE_ANALYTICS_API_TOKEN=ENC[AES256_GCM,data:6/u2EmJ4YUB8PJ4dPwhqvA5OPGnWT/ydFs7F7ELw5hwzsy0A6RF90AwVR3Ef+t7QXUf1F6s=,iv:T76mkS5Q+peXlnt5QYQFKABXYXujb4ac6HY4lbTodAI=,tag:aZxiRBxOuk5Dn9qPcx7uZA==,type:str] +CLOUDFLARE_ZONE_TAG=ENC[AES256_GCM,data:JJYHegytraHTH4pHMzogNvYLk4ZLVgabn2ilWenEbgo=,iv:jOd2ohUO/nPcQ2OLC0I+MUuT8FczGpUY+LQQ/zkbAJ8=,tag:68gw9ZyodNRgmw9jwvGpmw==,type:str] +DATABASE_URL=ENC[AES256_GCM,data:6uaTcNlZvgqfr5sdjv2zV0M6XTzz81+WaKpGobtm3GQRf7eYwxB10NFy2e4=,iv:yXJBRJugdINKw4vEf7Smq/YQYTcaecb6bAhkw7GYktY=,tag:/RB11xiG+2zzEUcxYJ6/lA==,type:str] +DATACITE_DEPOSIT_URL=ENC[AES256_GCM,data:xgVjAO7ICCud4Z32SoLMvPsMIVpDFtAXPn++8SqyrRFUww==,iv:+jtVgxesr4VgEHiy6Jems/rcQa64gUFGIKBOapIJ3Sw=,tag:fE7xazMZpzJa7Hk9m2hplw==,type:str] +DOI_LOGIN_ID=ENC[AES256_GCM,data:rBmFG4TN,iv:IPt6UCcoZS4HOxMpFRusShyw5zQtGZMl1AhH7qYsrg4=,tag:TdZ/F2NrnFMfS44USlvfGA==,type:str] +DOI_LOGIN_PASSWORD=ENC[AES256_GCM,data:35wEuCwiyxp2N9fBUCDegNq8O60=,iv:I1iD5qnltA8IyWPUcp1jBelgN0D2TQ7lG7B2JCdlpvM=,tag:MImbyzW7V7UpJY5aFLPfMw==,type:str] +DOI_SUBMISSION_URL=ENC[AES256_GCM,data:BFNOSgbCFPktK7br1G4kbj2whbaglEvNTmJ3HLxdUYzN41himICbk84=,iv:V/1d30Hp7P8gu0nYveOd6FBknM4KiCQ1iPzZK3KSLq0=,tag:tU9tGnSCVx+Lzmf28yd7ZA==,type:str] +FASTLY_PURGE_TOKEN_DUQDUQ=ENC[AES256_GCM,data:4wy1YftUnRu62Jj3lbN8ylczGF5kAwWmUZfl0rdHztc=,iv:1SibM0+li/9lbCZIbi2U5MdD40Zj0/a3XxL5i1PQ6ck=,tag:/ELz9FvaqLy0cqR0ELo+Bw==,type:str] +FASTLY_SERVICE_ID_DUQDUQ=ENC[AES256_GCM,data:NX4dyxYOGtpMqzMctNDdfEDlMqf1jQ==,iv:tq1kSBmN6Jx00V5M1C8rfZVNbD7M+odY9May5rxnqCE=,tag:QeNLOFU3wjJ5XT5+isbfVQ==,type:str] +FIREBASE_SERVICE_ACCOUNT_BASE64=ENC[AES256_GCM,data://GXciSdjlC4sbhuLQZtpgA74Hv6uvyAMHNc0N4Zw8tE+0wHWzQSaFicupza7na5ShQJbFivyJara63zQXSHp2LHEy+5DIhjuqLVNyyuOxWCbuDZnlk+vYUlAhB1H4fmldO983PKZbywgSU0PbANPtScX3iqv8OWI3RBknIsYI2bHXU8GCO1nS4VhW8i71XyP7jHq4HpMBnNq8MqM/m9jxyLGLwiLGyv4h2/o4p4mB+zMp7n5lLNdI++DFmxaFwFLPuNsePL8dELJAnShdhEO8ktUdnfOz5asAmpnbBsvrPDig3pNwvpFwJTcSnh0otnhCmVvMJdrmJPADf8BvgHtJYUS0a5DYIKg7Y7h6KC9i+KiaLliUceAjaZrxluD4Cog+aGLRiiNZ9vFXBmmiOh9KaSQjCKaiipd5otc8UYaX/uBhFr/4rUJ5voqorcSeds8kVRfhXz8jwS+4SyzOBDQUjXF6i+qdQ7gRRdW2CcLVchIQ62Lu7Gy70lS9aWalsNAbVZ9A4LZJ5SLUHG13KoYz9FXip+Vn2LK/R5LRHxLDInZjiDG7yMENLByqAAJ8f+ddNy0M4hU+4EpVc3xh9Zw8qz0eROLj7jPw3LyrMmT1GR0ZntpVoYI9+zow6RoLA9HNdy36ZyzS5xSm3YryRRiOPMQPSlrmAMu9Dy/TCjM7A1V1HA+2PPYn0GlOSgxK1KjUVesl1Bq7AzJcgJ7THb3L0Z+ZA315GuyESeVXiV5CoFIXN7xXHawMYjO5+AzW0W5W3pkl4jEGDY919xnUWSUVDR6wB31ywRWaRjrXzJVjE7Ici4KSFqR01+ebtj9GFkmsiAonGDSAsXRi1P2b4DAJff30+DBg6U/fog3hA1Ew5Puwhukz+EZgDJuysPcUVYzrm+LCgp7aZ5nCK0dRaQ3KQSPa+F3lIEjs/3JWoRy8XRTJxvWdseTkwO/Ex7GL+pf2Jq33AG9+0jLeisVghQBvRNhY74sZogWvPvq9s2Y/dL3bwgVYy5O7lSvsJiDkPa89lPH78Syaxhz+DaAboeAXHE3ukgHbTdVERN+dOzljXqMSQviofagpULBkm+32vwIZhSps/QUo9mZx6uitn8RKg1aCF/MvcMpspr0F1JP20YDUDjMSllelDJ0DBfiTysDGL5LxRjq5/VPhXHHIvG6+Kwe7ELVqlrUt1CQmkttnPZQN/FJtaDTwJ/2JHGnbvtJZ+9mRHo2K9TdqUgx7n8faxNgGtgs8Lb5lUlRdeszefuSdhapL9BZ0Sh4L1b8tTVyIUZXO1PGALe2ha8AnAOKmhVbrFrYzAM4vQ5kp6kNYOyMNh9SjL0kVwK7sEizntLqrcWIaywXUoKdV/jZQ84m8wBq4vWXOghMOUO2K/ULXbaHC1BgXqWdLNRNy0+iMlcuaIkMjpBVs/40wlWwQZSbUXDjUx6tBlhVRItP2AgFiuqu15Canjnkfb3zX+cnAbjpHOLNk6HoFofjBTFuS1tnty6Hh8sXBdxuBmW7D6l/oJ+erS9sYgESJewFwmm0jtnxrXU9C9rMVw9STXQa9Ew4mN2ocQtsHL8ivJgVs5+/CnNwFq/UKAhjJx13TzEX194zurMPV/g7A4K+YWsS1ZE/f9n02s+6FXcfEHvSXHdE17pbkCk14q3ChaUruj6xzMqgxKAdRSIO0szhv7ZcnCMz9eFBkLszjGJBfCy78NROdCeQyZhWWCE7NBFrSgNIeGUMFiDpKdiISXPqqDEDV2nnXjw2bnFtQv0lyMn9FJJa6Tl1tGA9yorN11UrHOX5rUm/P71tRfRnR03BtaGqWcK/Ut/FiYtCznCvTASBBQkE+9rFKhL0uxjm3xTcqP26SumSNPrex+BtV5Mm451ztgs4T7yJQpJBudngVppf9WAGR9P+cZellUeva1kT1KlJjEmEcxkhonWXVcDWOzk4+dQZlp2kglK2UwxWAhruC33tVZwEo1z2DcuQ7q56kfP2XVdc1RACyyhebgA9rkBAWiNVAEq22m6GkJrs3KoBHbC7jCY44Awa4Etf/k4cPK1K8NnIPPQwKPO+Y+otyTQc3TDk3+MaMLp6Z4ou4+bRm9gfIdR/DV2E4yDBO/yjUyUa3CsUu+6xMV0RpH7Hq7iJ38yGTJkzkwi2e4Ia3AC7gAusyiXfVBvfvRfiShdLoAoSxBxE3CkPWJKOjexfvX5t2bJa+MjuVO77sbW6sejbnxbf8IkX9gRzoixIUILq7VSxzi28vONVizKCYa2B9LibQ2LLNX9Rqx1EXMn7NvtcPCiSZ1wN9csgka11S4+mt5YSM/6YY6HHBGmSdHNDpC6VrIUh7Sa8nQyHQCphUUnTno/mKL02yDsJsju3LTl646DJ1y7LiZb2qggE/7GvCZtsZ10pn5vndGi9K9AJp9RX7n0RhX4/jjqOrcrYbXuP9fCkJ/uZLwdlcPzAmcEVjHvCnx2dsKaeXFBlngJyqEHn+rj85b8nfTHAfz4PlPXJZH/f/EKSZiWOg1bn5nn3Uzr6LWFK2Gz/TB3u52pcLOmwmB8Q7SQIcnYMx9JH8Rif2KwevxJcDcCfHhr892mZtzmt16j+sxJoMAT9Fw9LvAejgI88AH65Gi6meWgxAmMLFTo4x5dE273kF2kAXSs2nsIFJ0ThTdo0FPVxvl/wHRHptx/6Wbf+37tO9C4lVpw5vPPSgWU0gus+dc9ZZe8n5+g5RXDfoYWhYZEdAa3F+Z7ClZlNKmsyRMG713lcfJoeRdL5/D2OfIF9MFC5mTomxD8W/NX05rm7pRWEYZLCVX/JC/8rlro3YjUBbxig5stk1fUpAfvgtYupNBj5qiqM1b43lMarfGgkG2CYsG3TgBF0May2WgRZK84KsRA0l47K5Kn6Qb3uMvQGytGq3FXONL18saQJtL5NbktXYoM583Th6nGugTpHaZZYZ5tn/8Boj10hPmdMJW48g+UY5odN4Qfzf6ySWT40rZqGf07P5sTxdni5fu6zeEJziBO2oHosdYPOk/wTwCVKbKUJFSKS2zRfGyXpaZDAEoNVH54Tcug6T9Nh8QjGxCMgpAgVIuFhYk7DIT1ETK0kSZPiUsD8fpDeL7BpDQVdoKb6undj70n9LWJcZrSAGmTLZgN2gs5VBgXiQt4yNBEgMCO0isxSdiKPSR/INWoRLM/xxvpP5kA0HDOybizwlp5rlN/1xOeeFf26xY1vp05+/42ec7lAmF6rh5AOotOvklGFqbIMEMTB5U3mEyr/b1c7fK19ZSZ5Fw7GQSKL45QXABnWfhYyamJVpFTkZNwuzEZCfzumF3lWSqFyqCLdGiag3gKMhiP2nfdeLH9w+UdF18XrC9I58W5s83LYbdykeWGbFSmF6yswRNWuarTMpfX5xGe8sksMcvCyYbbb6LavS/eEPeBoqJeofqrgGvPDVORm1QcdFNF9qCG+8poh6oLdNtxcM/5gjJL9g5bwEbOQuhu7ay+r4GOp8G/ofelPsj9o/0t6ZDgIkdOPUIaX4O0T7SU4bBLQkgQ+UxAYqJpkg+B/Reh2aIfJsZ62ThsUDTl3+C6yA167oNJtyCARka185mRxVNJe2/o/LbeoKFO/9AB6iIwj/1k/X8Awkr8fmQMsY7twS3jF4BV9jYYaRGN55AH9wm938cHJf5UZplnn37fTn4MZxwXoeanJs4V4d4sk8aVFIWu/wz4qB7mHgCQ0zQhiwTtO0+4T3rBAvPJeAymEl/kD5CyoD+x40pp1XUbtLA/0kT4I/WOgu46Wnlq7fwY2FoDpnkQaA3ymF66ZpU9kfmkHmHTTr78z+w/9zD6rQn8heKGgRsWNx/zZ+Ja2rqMw6LzEwOd7oR6Rck4oOMajPhZ5xywR9IQUQoPbv8FSKk3irL0eiZhb/nFM/Y2mzHD7zKw/YslbW3evr6aetTp2ACrQZZXp0pOybtGFsKqSBIxkY8jjK7DPvNIB7F3xtFNBZ/YX2gAOdkbMY5B/RLIF9sRlf/dA+db9C/PHNnusjEfBIGWKwiViC1237FNOQhuiWmbmjm6s/Mvya78Io+Xrapik0BubBD2cZNLn8V3W/8up/k2wDO67/wO2R+d3SSc+MMLhqrXvfJomn18vlzRY8gqHktQCcCTBLXgTO0oCSiE7ntjD7OauQk=,iv:oCQ7ftTXEJsBdZtUjGmcqoyz3JlBtDHTk/aAUdYsYqk=,tag:TXkZBrMkUhrlLgMGmtVaxQ==,type:str] +IS_DUQDUQ=ENC[AES256_GCM,data:3C9xTg==,iv:icmSqoh6vo7nhF+lefMRM8eUMKRO42vd4aCyVsLC9CA=,tag:mmV4hbOGp+HlMqrbBErgSw==,type:str] +JWT_SIGNING_SECRET=ENC[AES256_GCM,data:xG4dmP96hzsxRwyoyuNwFmZ+2HcqNt6QVwCSp2ZnnJg6o1gWX6i8KXrdEl00eRF0mjCkVxEFvQyvzHJBBclY0cpvoIOc3JP1My9k2/nGzlwvjpoPe0XtCqVcbj0yAc/FEjeCm+QfcQRuyGC8jXYfx0j061M8ehtvFG0z9U3OrhxC20o3cNf2liyGwycubIhwmxOTCkh7XXn5X9v4I1MtKgo1Y+fG2B3g9Edy7tD9TVlMFwrnYA0NcScejM8xbW9mNhga+Znq40q88Kv5GdSQ9zGCMHSPthmydVbqkMqc98vLkpgoZlsJQubLj9EIgNF7Vo9ZwrCE3jGKs5RwjkRib/kdFCvyJjyBz05IeSJOctLS8aMj5A29s+VxboTvnXrbHSpuML5nfpjY3N08Nr3m+LsikEzRSS2J1ZIpwszdVFoCmKmDVJD1Qp1zvp4lyi9a/MxkM2PCkySuIAdZhrnYuoEztKX/I/PooeBJUYj0aqDE7oq8YSXNdGaDkIFVgjmtGetLBoIixaerguT13zMCoJvuWhoKkwPS/erBQvBUZ3ujnWMUvvBbXgHaiJUSkSGZ61pf6JJwW2obFH2FG6tLoQl33y7beA33vhO0cBXbFC6hCoC067A8kU5hPvsqx8Ex9zaHxZRmaOzFu56aqascXSGvlRAOjap3z+MHPeyvfuC2VPCpqNQrmWdGkHZXRQSGhVmLxf9c+L+L5ypI6Fgfd26ZquUuKpnGl3WJ7Qb8N/sJAZU7VbjfRI8bgfsXsgBecch007bhM1GoX/ARTdHgu3mvpHH9BBatdb/5OruSm9DjSNb8NAFC90lHERmZnRPPPLazrqdTO9HcRePMetH+hfhS4WoZzJY2INHjKevr0eaWz2AHkZ239gUJGbuKA0WtimulH3J1iS/NW2nft5X6IGKfsOWdDF/kXmiKyZeSy2YGLJtkO4ILrVAOZnz/HmGu8rmn7pdtoCcSYiTPorHu+e2QD6wSdD6h6fZ2O1IESIDhBOCiusUM+s1cxJGmhs5Fr47GEpPHAY6Jm1wF0C3HbHRodPu7lurgICoRV836vnM/NKfCfP/gk2i9fHph0482SScVQMvJjnh4nZPpb8+T15PPLvKaz/jZSw/TzKxMGxjcl9kvh/c3CsYgqpgi1/TWW3Hz/5Hs0p4psRYDZ134pOq/Fxeqoyk9ZI7/eXo4JL0CgupoyxhhckU445SOth6fc7XFVZzGWJnx+CMsGtJpAxpSqpwDd/d1zZkH/xxoiO81+2UTKs6at3ZTF9/TDnWLkJ6C6PIGZypogaMUhRlGk8i2wBR0Xn3GPW9Fvmy+SlsvEiKm5yQKs41NUtodTo+XnzvxZ1ncHPvUHDhe5//OgQ==,iv:Brlr/YKrcI0R8Yl7QTGzRG+hDfCzFi034AdrKaFq28g=,tag:qA7/wxhOBBHyS4cdjmdLOg==,type:str] +MAILCHIMP_API_KEY=ENC[AES256_GCM,data:m1GipxI2llyCbW5pMkTwmEtJ5nPBBMqrJ7Gko4cfCYM4jW4Z,iv:mI57ZdsOQIlPV5h49LM6t69UoihIcH6B/+0d3TEPw9A=,tag:SqfxzP50kn2nSR4c3oXHXg==,type:str] +MAILGUN_API_KEY=ENC[AES256_GCM,data:BtXONR8wOLPTO7awQ26A8vi2XfX2KQi5YEaHE4Wi7LrwU2HX,iv:j2t3dzF9YUv/PetPnVSjKs0BQFznpOS/a/rMRJfn4AM=,tag:Acq3pgoE+Oy+gdatBL/HJg==,type:str] +METABASE_SECRET_KEY=ENC[AES256_GCM,data:P0F+OIhhibJcPQMnzLCkv+Vwrq1d4bABZwebGyJcZxeoam1U287qaoNb2mQPFk4fyi8EqcAMB5FyaN2kMsKIXQ==,iv:RBXNiECgP1WVj+N1fbsxXaDZfieZgQEL+0IWcw33qbQ=,tag:OhB9X8MHGYzeN+1AyoY9kQ==,type:str] +NODE_ENV=ENC[AES256_GCM,data:3I9z8WOG2E2/KA==,iv:tmmcp0vcKsU/Nj7y0E6LtiYySzof/UW+5RJlTG5gVjw=,tag:xbT9L+cvLeQLWkdFFTCE3w==,type:str] +#ENC[AES256_GCM,data:SOPIyEXJK24O20QtmZ2eMPCkAkyf3Hfc57k=,iv:wydoEYkCs1uOym3/J4Wsqr55Jw7tT04dTK1p7O0yOlM=,tag:l1ImXkH0lYRORmRixFSjdw==,type:comment] +#ENC[AES256_GCM,data:t5arAwQOBjMoFOvhkDEQEGJwcEIhNCzm7JMIpHKVwHGxJtAh/xQ=,iv:25v0UsaYFDa7N7tvE7425/3b4zOKYA6Ebp6nY7x4jm4=,tag:mOU3HuryXltdLfVjFqvung==,type:comment] +S3_BACKUP_ENDPOINT=ENC[AES256_GCM,data:/1qILvvXfTkPlxNcbi1m5fbumckadjzXYR0v3TVHthtdqWI=,iv:5V6XMWkCepZ9vyG8okYX7tNfUvolgs2Pk6BYzxGCnL0=,tag:BeOWLrhcE7pi49cZWO94jg==,type:str] +S3_BACKUP_ACCESS_KEY=ENC[AES256_GCM,data:49w8uFBKAUSjZVihs10/yfc8VH8=,iv:PW/wnHTW/+LWINuXCXjXbFz0s3WbOdiJ6Q1NUkGywF8=,tag:sMy9ZkkKXZ/UVTrJw3W2/A==,type:str] +S3_BACKUP_SECRET_KEY=ENC[AES256_GCM,data:X5Klz6J57fn52Jgx9CSZ2ehTzwjO6V6LNlx4Ks/mKCWcvXmnpPvDag==,iv:iHWK5TGbXPBtqLqkWGKUme3rVgXt82z8NRSEeK7BlSA=,tag:t11GmKyKhoQqX8Hdk9UlvQ==,type:str] +S3_BACKUP_BUCKET=ENC[AES256_GCM,data:DbVSSihLmR9cA14=,iv:8PV3RoF/rsShCS2EOjYO0kkdgh5p4VuGu60LzCpzMGQ=,tag:WLvrJtjzJ5PMjCHOGNNa6w==,type:str] +SENTRY_AUTH_TOKEN=ENC[AES256_GCM,data:pcKtvXZf3CYpvfGr77WuwZC3G3YGhQ+fyfvWp2LGOpq/fV8fon0gy0/7tZwJWJjSC8eHy0k+9SjC4tCP6eY58UlOfATntnTl8EMftoENJouX7hwRJR8SXh/1YKzJOD3P961pnxg2m20tH1iaYjhqx+FYeZDTo03BZGrtPG8vSD4d/93kgY7KpalSRm2SSUfjgcojBWckQ46Y0AgaDoYa12bRZ13nb9noA/p7GAFozVZ3omXFGzuzrKD7AQ==,iv:ufWNitusntf94YUR/QiMysucdtIHJue1UeRVcDJvXTA=,tag:v9ZtXE3uIh6m1FP0tm71IA==,type:str] +SENTRY_ORG=ENC[AES256_GCM,data:l8Oc,iv:6ajU/u4V/Q/pJzLgRiU8sQvcvdIta8StUbsVtGK05Hs=,tag:sAXf+/2Yy5NNjeUEogqXxg==,type:str] +SEQUELIZE_MAX_CONNECTIONS=ENC[AES256_GCM,data:vr0=,iv:wdGODpPJDuJJde7kLs8lwA0Gyp9pPhjIm0uOJtKJquI=,tag:9QunRUGIz35fjZNFfyRMMA==,type:str] +SLACK_WEBHOOK_URL=ENC[AES256_GCM,data:tPrLAXRT/mwgn8/lTPj5pD1Odn3JLXR7zc1BU3x1DVFtCuE0LwPhN0tfyDHU2rAsnLgmpPkeTqrnRgzuVVQZU1Dmd0jxpL+yQeIwJJ4bWA==,iv:m93NYt5LoAAIgxcWwgoit269C8Wq52H+IwuGdsHO4G0=,tag:W0A5PQbQdhLrokN1WKtyAg==,type:str] +STITCH_WEBHOOK_URL=ENC[AES256_GCM,data:HiDHZ/2Gp4vNwbjGNfigKGqz+b5u2OpT56zDt6Xl9HEB9OsMZGxeXf+na/qHdzTZiDpMf3Bn/ow6WHl6jlehqXx4GVnb1FNyGVWUdc8MBagOuTmzu+LQ3POn+H4x04XSseofh+hbRt4Qc8i+ZxbmA3K8UNX+,iv:9UUsj+w9mWnYiTOomj2qZVOHnmOw7gGDReV6BydwH8o=,tag:S+Uo9YqOx0hyPsxeYI3KJQ==,type:str] +ZOTERO_CLIENT_KEY=ENC[AES256_GCM,data:SbiFBzCMU214BxacdBEFiKTt9wk=,iv:QlqAGJo+fMFEVQhm8+jm2nsQaBmheeA+BO5S0xkHShY=,tag:NjK3JLxMXGAtf5kbgwHNxg==,type:str] +ZOTERO_CLIENT_SECRET=ENC[AES256_GCM,data:PIYKCQjxL2vzaPA/GXDLPzkGrok=,iv:uUjS/y3SmbUM2Aj0hSKR3uCnHRpHcq43pHTV0Gk/gGs=,tag:+POGsU+kxYqjx+0KG0UPtA==,type:str] +#ENC[AES256_GCM,data:FZmOvrE9EWFH1CDGZ3Zn9oJ02iyZwlDEhNeJvHhAtxf/XaMf7qsJikYjQWvSX+T4AlDIzRXbv8C5sle8CyJR7CC3,iv:wC1Q/pkP3lxp6GZ/yxnkxyMECiBBnASVHuAJ5EtNRSM=,tag:0sSqQkxL0vPeA4BxUapJ+w==,type:comment] +AM_REDSHIFT_PATH=ENC[AES256_GCM,data:RLhfAvQqCIGcw21WrFoAyeriDWDq1zYZ8339baisOv1Ps3wP5omQATDK9Ql9He1CC08mOWXNeTtr1vwaKC4=,iv:0/BzkwG0bZ/GP4IRfOWSSSSrfUyrZuYCFsEZ1zD7chc=,tag:B9bUenkjJhRJf2/Xe0xOXA==,type:str] +AM_REDSHIFT_BACKUP_ACCESS_KEY=ENC[AES256_GCM,data:bSg1P3WWaMWLGnGShzKc1hcL6N4=,iv:FtkZCua1hZYKh1HsXL2rLPlZJgXznQtAYUqeyo82+BQ=,tag:/7qQIVR8ooNCQxudLVAcFQ==,type:str] +AM_REDSHIFT_BACKUP_SECRET_KEY=ENC[AES256_GCM,data:64OsKjx1rJ4fOgkw6bYFY8nlOOLFLK+LPQJwhhto53QMxCu4dmVpBA==,iv:9/hztUvW3sgxqd3sVPLmOWkeXfAC/lm3goZDaEKTBX4=,tag:B1En64y3TevFOqI27Rz/Ng==,type:str] +AM_REDSHIFT_BACKUP_ROLE_ARN=ENC[AES256_GCM,data:l4a0LSfEZ4P0GoT4MLPqPAd6HlkfktvXzH/DFLREGAsP9nYJU7ATIlrC7kPnUyoZ9hSQ,iv:w2V6aP8EBWVKXNCzxgLqp5ukF5eUuaLjSHqFMdIXoGU=,tag:D1a3Y1gHFe6nFjY95YUChg==,type:str] +AM_MB_RDS_URL=ENC[AES256_GCM,data:YrD7jGR8INX+utQIGGrXceseTb3E40L6wFbTuK9z/0PEQm/8vwJuUKNXbti50Wrw9kQL0vg9N1qQdW4gb0d7YsBs1F6IFI6r5dCstheQWt1xIthdy0xoGUIrRn9RANS/cFvDBnbeawQS,iv:liNIxpd4cAd47oR+YJds2CD3BcAXFTFyVekYqh5cS10=,tag:ayvkAUp8q46GthLJeuCPgA==,type:str] +AM_MB_RDS_ADMIN_EMAIL=ENC[AES256_GCM,data:PGAcQzCrjF/ftwgs7/aSO4NmANgXm7XAZrOO,iv:+kmks0PtFE1oMhrKZlAVDXCIoxfJlrRUoVvA0QIUmLM=,tag:TgJo1RJQ9KwK1+ZJTtqdEw==,type:str] +AM_MB_RDS_ADMIN_PASSWORD=ENC[AES256_GCM,data:RojtXCFz94fsX1gkoHIU4cAmeA==,iv:0jRuzMO9hSeuuKVxNw8Rpsl/V3lIs2pq9fpTdALsWuQ=,tag:Hw/Y1gOswEWlaMyKr9bu1g==,type:str] +AM_MB_COLLECTION_ID=ENC[AES256_GCM,data:5Yw=,iv:09zeziTirEHX1CoLYXA9CDquXl5JEgDYBmjjF5NCkoI=,tag:F8YcGx99kASnYo8EeIUtSg==,type:str] +MB_DB_HOST=ENC[AES256_GCM,data:SF9u1X/Jo0ZsJtk=,iv:nl10iP0E1bzEdPYQR3EuiN1IvmvcSkkFAXBW99qvPmE=,tag:zaaKFZb/HI6xrddeZJkCaw==,type:str] +MB_DB_PASS=ENC[AES256_GCM,data:WdDDXujj0cdoQdGOytDqDssTDoymOPGBqE6zj9QX3khajz5ASsZnMuN5UcQY6xq2j7z50b7DaM0+L9FBn4BU+g==,iv:WFGu9BDTgIpaD/Vqc1HLLs+PqmE27xW9zyQA7R8rvZ4=,tag:xsijuzZGQrC3eXd8UReRJA==,type:str] +MB_DB_USER=ENC[AES256_GCM,data:WBz9UNovZ6o=,iv:xZXANxaS463jT6DGFFJdyHmfNVR40wrl5te2VY16+/g=,tag:FXXcAjNxRGhwt13XLO7jhA==,type:str] +MB_DB_NAME=ENC[AES256_GCM,data:Nqn+4XhMeU7dGoM=,iv:56hnF8rySc2qFNxGD6uhSkeko3YM1KKWI5VPV1tkzLc=,tag:TZ9MQ9I5A7E1pAPkjELv3w==,type:str] +MB_DB_PORT=ENC[AES256_GCM,data:Y9KWDg==,iv:N/lw6/4nz6oWH79iRgph+av411emmK1txDQOgc60YPQ=,tag:rKPc28FdvoMBcSSqSvd8pA==,type:str] +sops_age__list_0__map_enc=-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBiNzhhWDU1d3V5cUJGM3hY\nQVV5OTVzLzFudGlKV09OWkoyUXNGVWNNVkNZCnNHUUE2UkxsNjVXZWtsbU5Na21z\nSFd2b3B5YzBvVGFEWThlSmxmMnI4ajgKLS0tIGQ2TzdyWk84RzRXMWUzZDJpc3Vy\nVUZHakFoRVpvNitzZGRWMzhXYWlVdUUKvd2gu+8GnPJrWuBddev2Gusm5nNH/Bv0\njL8wvnbnLTtBkkRsR+SCJCd9OTXv/SmE2Sqt6J/RGE1ZTfMb7OTvmA==\n-----END AGE ENCRYPTED FILE-----\n sops_age__list_0__map_recipient=age1wravpjmed26772xfjhawmnsnc4933htapg6y5xseqml0jdv8z9hqemzhcr -sops_age__list_1__map_enc=-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAvT3RxWDBqTS9LZ2NDWnND\nZ0p1dkxTNHZ3S1dLaTNPK0Jsd3h3L2xmcDFjCmpoNlgzWU50Tk5DWDc5NWtnTW5X\nZ0VMVzEremEwdWFDL1FDUjBKeUdnRjgKLS0tIFJEeXFqMnlMNzk4cjN3bHBiVm8z\nbVgzdmtaVTdzelRJWXRWQ3VnUllvVEkK7+wFnHcWlQ578ZBdYdEfbstSSUHyftzm\no6E9oYEqwH+oxftQN5E2nVQ2QxwdsXKlnThkMJgVxH3ncgfMSUt5zg==\n-----END AGE ENCRYPTED FILE-----\n +sops_age__list_1__map_enc=-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBQVTMzS2Z3WkI1Z2IrcFNi\nK1Q5ZVFFR2k1bS9kTXJRUGNCYndabjRyM1dJCjcrVXVBcWlNeHJGajBDaHJtSGtu\ndEdsRjdOREJxSWdza1praU1PMmJyZTAKLS0tIE44blFNb0VJeGdIWTVHd2Z2U1NT\nK2VWRGhMdzYyelJWUk95WTF4VWxvZ1kKG2VWJD4LxIqReuC5d90pj0Z/FKEdrdjm\nKvLsxFappiAAMDMZYoEPQ4bvvTPF3+0mLl7KBAIx5eeCZb569xWVEg==\n-----END AGE ENCRYPTED FILE-----\n sops_age__list_1__map_recipient=age1vhftscteyrwphx0jpp0yl60xxrjs77jq05mkzv88js7ckc926vcqepp2cj -sops_age__list_2__map_enc=-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA4eFhnQkJVaVFiWmdzYTd6\ncXduWCtZdG42dTU3VVJ4cEhlOEhpRndSN0JBCkdTWWhFQnFCTzBOalM1S3RTbDFo\naHBGdDdUcFB2M3o1R25VcTJ5YzZ1YnMKLS0tIDFnVnhzcVRtZmJZSmdxSE5JSGFK\nVkRRZWRPMi92VFpUNldYZ0MxU0lTSlEKh6gcbf04oOIrmLzfoK/0wagfzxDh/DSb\nQCRvyhkY4cFQgO1fn6fU4UXdOq8Lp0rXPEuaK15L7hq3q3hEo74O3A==\n-----END AGE ENCRYPTED FILE-----\n +sops_age__list_2__map_enc=-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAwWGFmM2lnSmhsMFlwQklX\ncVlkWmk1NFFNWDRPdjBmdUtjQkRkaXgvd2wwCll0Zk50UlNpaUIrUG9NaUpTZytP\nWnV0ZkMxK3NDdnN4TklPWGMwK1paZm8KLS0tIG1qd1N2MHltZmhQNXpPUWo0Rnpl\nc1J2cGNhVzdhQjF4NEZKV253dXcxYVEKnJ/Cil5FOUopr0gAwVSxWpYe7biR/Mla\n9sV2p/Tue1qugZMF5EVFpTX/D5mYyhI4v/xdA1WvI90Rg6gmJpZ/7w==\n-----END AGE ENCRYPTED FILE-----\n sops_age__list_2__map_recipient=age1slx6e48k7fre0ddyu7dtm2wwcqaywn5ke2mkngym3rpazxwvvuyq9qjknk -sops_age__list_3__map_enc=-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA2T1gvVzZKQkx0NkJmek9i\nUmtWUXN6bGt0WVM4V0NIN0pZdzMxNXpUZGtjCk4zRVNRMzlraFNyQjU4Qkl5WkpQ\nbDcxQ3c4cXFtYnBFd3hmM2owNVZaVjgKLS0tIHpVbGlLbDFzZFhHVGlFRU12c2tC\nYktGSm9JRVlCUmxFOURHeENVU1UzMDAKbUeckn/3XgXyPFn/W4Ha0ayo2v5wVMQb\nNrsjhYQFn9cdG8H8hqeGh/yE1KLfIwzI8U/HSXlYs/NtsvH3h5qUPQ==\n-----END AGE ENCRYPTED FILE-----\n +sops_age__list_3__map_enc=-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB4Y0VVdXYxUE1PYmFzUlFG\nWjFxK2FobGg0enlnUnloTElDelNySVlQSlRnCkV3ellHd3VjNUdkN2loUkJncHNF\nQUtwUExOZUFtNlFURWM0bStBSEtuYWsKLS0tIHhITTNCcytDRGMvd2E5MzRFQmhQ\nekpNbXNPNi85aGp2eXRvM2ozWGtIZDAKz3ECm6Krj4teS3XStAsYzg+fOasmDA+b\nSk83WIRJH4jL0OoKmJDwsrh/XA4EL2ADw66c5B5Uzkei9UG9TlhRyQ==\n-----END AGE ENCRYPTED FILE-----\n sops_age__list_3__map_recipient=age1vfhyk6wmt993dezz5wjf6n3ynkd6xptv2dr0qdl6kmttev9lh5dsfjfs3h -sops_age__list_4__map_enc=-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBQWkY5YkJiTUVxQXFES1Q4\nYkY5UXp6N0YxaVZILzFkZGMvaEZ3TU1XVkVBCmV1dVhCQU4yZU10MnFhUGxVTEQ3\nNjFOeGlHdGRBQ003WHZGMHJzbk9zZU0KLS0tIFBCb0NRY3UrazRzM3g0TEw3MUhn\nTThUbzdFZkJONGNYVTF5QitLaTV6cW8KR/S0wl3+auYy9Ag0tLckJ2Xhy92e+s47\nm0lLrUGvjLYSGdA9Ox3KS2nmem+RQp0RCjTzErDlsY7X5Ai7duCJRA==\n-----END AGE ENCRYPTED FILE-----\n +sops_age__list_4__map_enc=-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB6M1NWUEMxVjFaeHlaMkkz\nQjhUTTJsQWZmL3Z3eEk4WGFJYTUrMURteWwwCkZlaThhcmZFRnFhTEIzWlZVOVh1\nbUJwK0RjSURQUjFEZFJpRE0zTTR1MmsKLS0tIGM3V1l4SlRHQ2FXa2dta29SY1hl\nZi9vdmtORHIzeXNqQVRsWlI2TkJMTHcK4D+hzIY7+tcpYln+lyLhVLwdrtHWdwwb\njeNH7SpAEmEJ3t+2zOu4bPo180GATNvP1LBz3+TPiXU7yiZ4enSpwQ==\n-----END AGE ENCRYPTED FILE-----\n sops_age__list_4__map_recipient=age1jwuvzyghfer7rx3qtqa4vs0gxyff0sg6pqgmqvp5zlhnpmvrkdlsdha4kx -sops_age__list_5__map_enc=-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA1RWdBT2pqd0w3UHhtMzU3\nYlU2Z2ZsWUlCd3lIZkNMQmV3aElYTHBLNURVCkloRmZHRTM5T0MxVWxjQklRVDJx\ncDN3SnM3ZFBYWk9LdXFEQmQ2ZGFTd3cKLS0tIEs0SHhEQTdRdWp3MkdNa25mK1Zz\ncEU4ME9sMFVXT0NtMFNoVnZnOURmd3cKii8ocexy9c0xfxPaV5FtBWlWy9KsaIEh\nMpH3eJTuAK0ElMjFrrI2AvjuW3OYp3WQU7ZnqI6ubZvi8mW7iZH51Q==\n-----END AGE ENCRYPTED FILE-----\n +sops_age__list_5__map_enc=-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAvMUFZdm8vR20zMTN5dUY0\nc3c3bWFEZFB1VWJMdXo3MjRiOHZsZmZac0Y4CldEUWh1cGxMdEJmaGYxMmZOMk9w\nNUdjVzJ1bnlDcWVtYTFYanRZT2IzV0EKLS0tIDlic0g1T29tcld1TGVxUG1VOXVF\nNnVlYXc2aVUxQlI3elBLNHBnOEExNUUKNPvVRkL0qAhb6ER3XnDFfW3te+c72zxn\n0++0uZY3TMwHog+fzDquk6fA8/2pCrNjmvq4IWo6Dqa6c6CVybDhKA==\n-----END AGE ENCRYPTED FILE-----\n sops_age__list_5__map_recipient=age1pgxk292zq30wafwg03gge7hu5dlu3h7yfldp2y8kqekfaljjky7s752uwy -sops_lastmodified=2026-03-31T02:48:03Z -sops_mac=ENC[AES256_GCM,data:PtfB+K2c3gueL/a8MnWPsaO5XL/+uk9vgilMCAGHRW53R0ilHzWujyuNTtjsFGbReRP+SXbU0hllj3RkxCQtwZKouci2Qn9PmAKEonhRLjdYl5owSJsgSaXTL97wUS0hAZrCeYYAlLLPNYgP3XE7DqtFx9mY41vcn0ViyK0MPx8=,iv:F09inZuqJ1oRn8IUQE1noenfjdsyMNBkqtW37Z4qogs=,tag:M4Tc4+MD3zl/YZ+VcUxP2w==,type:str] +sops_lastmodified=2026-04-07T16:04:58Z +sops_mac=ENC[AES256_GCM,data:SJ9WySUDpC29NXq16N+hb/BwbOo/r0FisodMihRfVmTgx3x5+G03TzE+zQ/R+sX9IVauYQ/IytgU3qUbollTVqfMp9YbbcCZhk7cnEpGlOyJSem2jvqGG0liqH6bGKv3YkrjG/u+uEamnLcT3d2zKS0UWyAYZ4le750UKkHuBn0=,iv:+9osEtoyu/LNul6wH3g17Wc4BWhpp4ZOuJ+WKkFRAa4=,tag:YPOBDRusEbcSCbeuWntzBw==,type:str] sops_unencrypted_suffix=_unencrypted -sops_version=3.11.0 +sops_version=3.12.1 diff --git a/infra/.env.enc b/infra/.env.enc index 1be76f8416..6eca348a79 100644 --- a/infra/.env.enc +++ b/infra/.env.enc @@ -1,55 +1,69 @@ -AES_ENCRYPTION_KEY=ENC[AES256_GCM,data:vbymPb9pIX3Ar73KrjW62lYM/8A8fWoh7MnJA3IUAusQ6+MpOXNpL5k8eTu7ieGwiV7gY90MnQAra5upymG3Jg==,iv:xY8Lh00p9lcIT9B9LCARxQihN+GrivS7C2QlVKt1J2w=,tag:RDwybIdDYRk7Ecx/5L5vDA==,type:str] -ALGOLIA_ID=ENC[AES256_GCM,data:T9BnAyvLkom50Q==,iv:hntMtsqHiA+1V6D26KA756wRBm1BkFl5AuopGhLhUT8=,tag:GblUMhdJJG+VrGCniCq3zQ==,type:str] -ALGOLIA_KEY=ENC[AES256_GCM,data:72FxZOZkqzMIdbvYpO1yEZFyG0YZY7rVxJjuDPgcGD4=,iv:v5QGHSSljTYWL0/7djTGtPDP/VPGQeq0zPzDHrIfE7k=,tag:QmtvW6czoCHf0QuqZzyOug==,type:str] -ALGOLIA_SEARCH_KEY=ENC[AES256_GCM,data:PhxMicSMXQSqsFVb1K2Z3ByLwpZROrob50lICyVokto=,iv:+BjJtiUur6YEiP3BxNi34wyPw7D5hVZoXYZVs6CtL8g=,tag:78je9z1AKBKwbJbEs7aKbQ==,type:str] -ALTCHA_HMAC_KEY=ENC[AES256_GCM,data:T+llsxSjDyEyDPzQ5ocRshwFLTeqmG/BsQd1P/aTYxd+3IsWbKuDJzuyikAAQozpahKkJZefeN2nmdFhNJlrPPE7xXXvTn2Mm+gnMRJFb5X2H6/OGua+4Ds/wcREkKhCXJJ8PrUgwR6UNagqfvfa5id34ULX3Q4ZyIUCUnE3EtQ=,iv:/Zt+E7wtlGwiMqwIVOgNT8AH74JciranTzaaPi+5esY=,tag:E6LPkAGz+odAlMMsmRbHIg==,type:str] -AWS_ACCESS_KEY_ID=ENC[AES256_GCM,data:lZO5lg0B0KwZZqcNsVkLYkQN+xc=,iv:gZiZvft7AAcfl1NEXbrkB0FSvb7ncYZ9BQ7TWK2IyLw=,tag:XdtmjbCznNYvh1UfJrgbdg==,type:str] -AWS_BACKUP_ACCESS_KEY_ID=ENC[AES256_GCM,data:4VovW11aEAmALfR0GJzQTUqhPoM=,iv:alQBBxvedCD10OPIf8/J/VRriY38LCQDtHkkzHfdxvc=,tag:RK5YxQOPb0A1MVbLK3ho/g==,type:str] -AWS_BACKUP_SECRET_ACCESS_KEY=ENC[AES256_GCM,data:FZn5un+lcsDlePnkGlMTH6lUjncFMYjzl7aV4xyJMtCHhGHx9OOrMg==,iv:zQR6Z1HxiNJAv/sN3/A3UjRr33wbdNZtCIB6EWj0yNM=,tag:Uk7gvphLvDeXACoCzmryjQ==,type:str] -AWS_SECRET_ACCESS_KEY=ENC[AES256_GCM,data:YZb27G49IssZyhiQJIc596Obf2H1g96Nyh4mXr5rNhHypTUSMYCH8w==,iv:r7aMjD/RZBUjTP5q2rsNOlSt6+ncKgfDuBL4OX39Q7A=,tag:t0Q+Wb9Zq8aDiXRTFdS+SQ==,type:str] -BACKUPS_SECRET=ENC[AES256_GCM,data:4BDHdFZ4+N1bdc2Ae/BO8gUIqcBozcAAW8btH6zbkcTViLq0FVa7jqdfK48=,iv:3f9W35UnnR8KDxTQ1nZAuhMX+9ByCIKJctjjsJ5HvDg=,tag:1xWFZOBVZnK1RQxK3BQ4zA==,type:str] -BLOCKLIST_IP_ADDRESSES=ENC[AES256_GCM,data:hyphmo+BecobINiBmCOtn1uiM91/,iv:5Mz9NiC3siH/ne5a0oomvFdJKsXODXuP9C6gx4kyHIs=,tag:oGRKbNCP+/8S6mEzyccF6Q==,type:str] -DATABASE_URL=ENC[AES256_GCM,data:VlGdDtXbuz4Z03la2CGzYPgaWd9jNjbnZL0Hp7nntg/wsThrxsI5gJcxjNw=,iv:1neCIDqq0sdaaP/5YGi9CDh5bppHCqMeTtMcUwbQ8kg=,tag:8oNLRkD2QccR8a2EBNntjQ==,type:str] -DATACITE_DEPOSIT_URL=ENC[AES256_GCM,data:Ffm/UaNqtwglqDY4pJbrVSvbxoV8AS2hpL5/0NI=,iv:rnCV7miKSbOwpDlKpOr076gMhnas/MhZz+YqOSNrzls=,tag:UR3UiN3wZDv3dKyE/A8sig==,type:str] -NEW_ACCOUNT_LINK_COMMENT_WINDOW_MINUTES=ENC[AES256_GCM,data:KiA=,iv:JyFPxlkDeW6mbwFKmIb1V1RmQ79L1NrvJwg8F+ZoDjc=,tag:X7a/MrnxM6Rx41Gj0fxaQg==,type:str] -DOI_LOGIN_ID=ENC[AES256_GCM,data:rnyWq5KO,iv:ES5zyrlIYq+YiM76fbQkQWOkRlHCfCJfvJKFkkvbe6k=,tag:SbakzlP6bPGata6MxQPN6Q==,type:str] -DOI_LOGIN_PASSWORD=ENC[AES256_GCM,data:LQKtRZYGCzbIYHTXou+Ajpk8VMk=,iv:GgF8FClckpq+ZzfzxQKxTz9LXXblxIie5JeBM3d9kk0=,tag:I1gRtcQevT63dUQrWeCtUg==,type:str] -DOI_SUBMISSION_URL=ENC[AES256_GCM,data:FIeG9pi8ABy0OOS15akayMmbyVDu5yS0WHZk18Zf/QQ2In5bsC52nQ==,iv:XPADr6CQ4Wq1SOlDoGgrbVxrVM941/jtS49uNW7rQfs=,tag:amR+BxRepiiTrjPxIvpZFQ==,type:str] -FASTLY_PURGE_TOKEN_PROD=ENC[AES256_GCM,data:tEeV4a6MslOGAHCO8O9exE2OR+FWI7/P/DEZr1JQEtI=,iv:rs2G/2JHcjYej7XubvvazBuysfGvcR3JqIT1KHYdaqo=,tag:i5vbABs45IDWLZA6stIZKw==,type:str] -FASTLY_SERVICE_ID_PROD=ENC[AES256_GCM,data:r7i2ROh2K0SPmvjdg7S9fRLewlZmiw==,iv:tyvLY7zHaHM7g/7RJt0WP/k/iNeFIu8wCaZeGkWDFak=,tag:+S9pjgNt1xhmxujbKIEOUA==,type:str] -FIREBASE_SERVICE_ACCOUNT_BASE64=ENC[AES256_GCM,data:x+82Oh7112KfnwvHAqW0pUbDQuN8bE85cOWEUFzqgKoNQa1uanPLTk/TCFMIZXpV48Xlsp7F3YZB/ly1ngWLHItKbACh8rsOgGCx5DsU+ND642KBMO4QlR5pSNM5TTkkCyQSLOnbWnX1SoH2Q9eTPQSwgrxIZnni4uYWbVOURiKPNs62PkasxtdfQ9kQV97HcbiXKzBhtV86eWYZOnahDKbjf97Y3IXe2ElsA6+n4rRZ/naIaFMb8L0vJrBQLFNd4V5ebuF8FA9s5RwXHV9ucpVuG2smbcY1vKLnSfP6xTSOXODtv4SFKVteiiyNPk/Oe99frflQtv1z1m1a0kaOzefIRK7dS3G0Uiq0OgHWgwGbuiL9ErAsQRgq+w7jZzJIxLmOVsxFAdE6ymKnh+cDssW16rVpO6ncbjOrm9UJeIFOgk/rPugqH15GkdBuancPvACJVPLVeCGAARxPRvIXHwrQNAEX7nhoBQ3s6Mgo01wKo5ZG6WjS0DxEG3BMCfRQ/QMfbwE5Fn2tnHyRgawCzUoWsh/JiOZ+CZ0xK9fzXdLgnU/aW6ncG6iWt/GxGzmbtap49jAZvf9uI2Tt+eRNGM4GOvJqde1FB9gbgtJymzDwq4Z0kVSV4/YmfdWIKAEHjUkLJWhB86ANzjru/9cLzsfi4trqL2zN0m972T23SS+ERAxM3pVdUeD9C2HNCr3ep9RoAVmZ4WR2EPTkRlfPI3MMKzDs/7ae+AteXq/vZwArP1vRhaaCcrWicqk76LagjTGUfaEiPQiGcnKOzklCdApdmcYnGL2D6XZ9rmyZcdjBrHH9e6xz7ar444ov1xxn/cDb4DaQq4wuiTitNPftq3+dUAxAAM1yGiga+xEChkLzbqV9+izsGhrUWh3mJ8WcCy3Z3k97+vq5/brFOzKEPkwpHDDDB1M/RHFes7i0EFpewyz+QmUlc4zdtY9OunW39wazrZMzJB1Pycv6nQ5+PFaGB6ISz3rWLD5ZrTVP9GGVIH9vTuQrQgIgEGixBsieXl5GSzDOqnY2pCqJ1Qukmnr1sK7rktgmZowv0AoiBxXou77axTRJCUSdtyU3LJ8AFM7KvLSxtYcnzGxaVilzB7C9C6Z/tH360Q/vFOLwPDa3U4Ju6W2AAgfNHLma4MuZbM1VLlaxi7F910HS8jWB3u5F8vR9TkEZh8juiPQUeBU1P4K0G8YEwauUAmuV3jlAPkQRJi0JMLY3nWRAT1N5i7f7cE1FdyQLQEPfDvYrJRg0L37rpxWVYAYikXQNMHLeFaHwxqsv+WoI440hFj9VNCkko/91y447XVxNf0o6fkYQU2JRmDHactU+iWTAH5nYmTe61WX8MsOywLq1TDdokWAqcqN7OTJ2JswPOFA77y44J+pM4okvo7QkiJKH4sRlm7fnZwMErpp5WAGMtJpSXTHwvtrTClpjR0tRbP67sOyp57jlW3iwYFDqEuNi6GgP6PQ+5gI4YYslKvpKC+83rc0AsWzUo0Sp84qHDkUuFFYMZQqTsIQ/zC6jBU4WRZjikwOC2cblUkZY204Ehr2a9nqsIU46MnVz4jtpBetYGDOPaLx7alYjljALU23H+s/yk3ZrgV0apFywegDQHGBRHBz1jMRQDPT4MC8WOjbFEpujiCbw/EafCUa9+onZ5mzM6Or3RJGfa7D/tHDITVLRTmxdUdFli6CDo1Kr54uv3qkZmqIoFVaB5Sl3RXUuzKXfsR2xsF3xpU7pcDkvnC9UtMY0zfUXm8CkXiktZ402J527UubRFd0oT9Gf518wltgYLN13nU8gsn87ylRNGupkFFSLegBhKUCrQ1qwivY/FK2UC6vWj1fc3qO7rUhy7sdRQGV+48/9YbH85UZ83dh2tdQA3yZzCkHldxDOq5U4uV75/Xk1mfTeIJy/Ee6NE3lNIJEIPKW97M4TSfpM51Op3y+uWSaxOkxBO/9Etm8Cc9Yy+P+pfyztTZD2b1DT26N1oRK3jR+3pgw5ijmQeehCIwKia0LM57lOjIcBEJbj7+bUKUljpXtXl0R5PrC+JdSxsXeeDnICzi67rAzyfLKjWU93Udmh61vVrYv+kRrVa93OEE+ed7DdusaDuyD9LNTg/ZBAAxf00MSLrthalNKjl3Mrmw0hj4TKEa9zalxwGCbzqx+ih7j3p2Irx5QLSU8PyyerEN40q104vxiq7CY2SarobVRb3gG8/q4XmSbdAm0F8ZgsESEowJCJ9gezZdWzvAeOgcZkULCQ/IPFms4S+vKt0bljThQJiGqQcbYmN2A+Xmppt2xQYa8HsaDlcjx2UAtn/WseCfpiSwl3qoheLwaptyww0emfMiJJcwR3APveMnEbaS++2iMJQaHfB5WZ+38dJU4mslQI06RR8hyuB2xRB9+7Jlg6oUcBWuGpMonShOgLOFI4Z0ngBT0U1QbdY/2hoz8gs2VyGCBKKffYXgdr/yg55O07DIVHd13gYm9VKqUT4zWcwvAgpxokC3y5lBFy9Y6FBu8+3MXBgn4W5ybslDIomAdHulQizjGmCiEf4ZixNjapE5MviUxUevVdHJJ8zKsE6qxnQiFRCj7knWfb5O31Cl+QsRKyXgZ7fcw3yZ1U/4cAg2M+W1qPKGTNNW8kim98DIHFkMItlfK2OG+u+9hanvnNPw1lTqFiHqqLtlgF8XVUyUwGWo4/ubMFI0/Fr8sPQXMMzNwm2IfmO9KV7Ttr90LCkuDamtBu7xQcqvLC37JeOk3nKIvqC8iRMAsD+KMY1Vahxk4PLTfGYySK+VxuBBApLctwFn++xlBMEKqcB8bNr1xrk+XVSo3yCaDHBW/7IyP33ZeLNQRL5PvzLm0L5pYn7jTd914yo2zFIaDndnIjLY+IwleKmQLudOm0StgGgwXjBCbtXpjslU3OApBYd179aCT7EP3MW8zCIrG/XeHTFCoroNJaJLZ1YTsYr8sGDISL9LT5//FgEk81RqnACz1gd6AtS77lo3p5GYDf7tFxb0CoUvod3JzRbwnpSougyiCngEygVqx/8EnW6sTF7e7safvvCIM/C0Rcq2ClZ0oY0BVWYnYxBJtvuZYSfEqpG0UtH2YFyHuVcDYgsmUaosMKr74QlhaP5fsJjfEYci9YgtJ574ivMT19HLBY0dqZ1LOoyygMLPx8jZ6D8XHswLlNlvHS2WdflD5/GPF9pTqSFAyBN/e1dOlz955sX06ZVvGs4T/hDt0Q0xkxmzQ73/HZxmXAgX/EMO9ySn3JFl2cddzDJK6Noh0pl9PekNdBX+/B5SXjIakfy5haOEhOomUaHVKuc54jRvDQBgb3qujSJ7ODwb0K/PnbUmyNvhnlLD6xTVbnzquII3d7Tub1Gz4HwG/TvvYOxwoEA4zwnb0B8789DUkw/38TqqhBisMcii2jM1jcic7ThOfv0RVJXhSwa3A+iGZs5VDd9K8m72gIaxyhOIYe5lVA6nBvNP5nlxTRwbo7WU1JTmErdyxlKt5rS8xL/IzHNPwWk9LEtnO4Rnb8E/WiokCdNFXhR9dugH9Nuk1azBCsxf4h5D+gAGGMy56u/iymPbI3x3snbUWe85TAgNZoVX8uSjoqd1uPvLn9gd8B2K7akuhY10iRkMVcfihMRX6YDsAnLTnXmgxYmC87ry8eYxyWRseb/MOvacIlFC1jRwz/burboKpgvjIUAFoWpzIrcW+zR2q8aGMXMpXSIFG+d395f+yIztDFznCoW1i2NOjgVQ9lnuPzjgR60eGy4Im06VT8/207J+gnnVhJ4Gwje4YzEsyIXJa9IOYZ0O2i4QgfHPJA1PkPAvTjSfTA5Zi0y8ZlVnulSTwWufytFNhTXgbhGu9GI9lE0BBerF1uGqOXi/k4rAEvy1p63eYu+aFr1TBVI17r0Sen1q16iPXnQGAjvBt/czQVzfPJ5MJ7Cfg67etHgySuS73fKQ7w7Wxi4/9p43ViGvbnuwLIcbKr40jnAZxJmO6Mt2XMRhfSW4oxK5d3mQaHGQgnAyu7xOZm5H8lHiaGrrPpsiHJx8mdeVEdeuOMw+iE3Rub6LVQAMKfPbGFRwFaDxjrfARhoT4YUVoFCDV6W9HK8AHE80kiyJSy8hxGPkNkFRq+pKhsv3K481hQoENIBH9S3U+HZ67Edt4=,iv:hscfsdG8NtRkfYYmGDxtiAlkbac2RTyBsKTkl2jr/m8=,tag:bcuZAcwY7JRrysMC9uFu/A==,type:str] -JWT_SIGNING_SECRET=ENC[AES256_GCM,data:7IwVrNGrGdp2uECsH3QOtZ/OvskoO/LGCs/XHVbwMRVtPPo4ydbHxvmxbQiDzk+Q0GHv0sKMSbsd8B3ybYYzEU5jWp4qL8uQnqYqT8UaPVoPqgfXiaugUYLlWN+GVqTyfgOKQFTdyfcr4EGJ3e6njmaZ1rMF66AJyhJPAKHSqi7bgNWqOECjNISV1SZL9vVgo7L3xuwu9t2tUqTE95y5wWlz7XQvOjDCBJB0nuLd3ztfzI6qwMkzkg2nb18tHDTcPDItrA4t+rEcpC7ZeAhjE38536k3WcU60kCXeXnlzn4AP/2boGLGPfYC/qxVJ4fAXVf9WFmmjuhjGNMP4NqcsqufgpwgmxEVJpLC6ZFb0QFNjtPZK+SnLd/qJUzyJVpuIRUdCjK/V587dVBMh53V0Kuc2q88fZKA2cWQtjihJUUyyGayFoBIkgEi5aVmYX04w7szcaEFaCPDphKbV57UZstfrO8oYXAIDLVH4pGBSD97RiIvJkM7u0gItYzk/n3C/8L28g8/Fm1MVTvF+PsEnpb3F1q49fch9/y7ffyqb+i1JdlGdlDKUyVKScs9Ead8BuVe6VyDHxSEUJnNhmEAftTPyLXf/VLUlEtAxNAfsLDZ3tsb8/kZhp27elwvpCLmYZez+zCBnF6m2iCfqDWrextk+Iu4yghzt86g6Qkimqo=,iv:YcUxqeIcGA1lk45lVLThieXDtcfL71tJnmxfGUyut2k=,tag:1sxWXaALcISDV6zhXfafzA==,type:str] -MAILCHIMP_API_KEY=ENC[AES256_GCM,data:KQhcMqowStnITI9nwlMYy4tdRb4HfyEOf429vgMCvh4DNtza,iv:ANcTeRiOVQRPWYADhSuSr4Iec0wD7LPmR2uJkfSaEV4=,tag:Ys18nnsHrPzeFmXirt4/QQ==,type:str] -MAILGUN_API_KEY=ENC[AES256_GCM,data:qZ/mvlCYsb09CUd6oAxausNEYiWsrxvQgFyAWuqmuvnHDZHn,iv:1n4ndvLhR/ZsZG5id4Zc8rThI87gN8NpK+zdguLJ2Pw=,tag:Q5gZsNDyMqjclOv+xWfP9A==,type:str] -METABASE_SECRET_KEY=ENC[AES256_GCM,data:NYstQY07BV9I8vS7bNQ/Cc0MKh0IasFWGldiG+PIzLpg1sicBORDLZlJjD2/9d8v+pZl7d9f8ETjMSJDK2JTQg==,iv:V665xBdAyDoJISCKCGlr3fHz0ukkGPeIFciu4l0/ykI=,tag:10lk2XHC502P2zmOhIMzqw==,type:str] -NODE_ENV=ENC[AES256_GCM,data:d8aLeE89dNwCcw==,iv:O0Hiqq/LRysBJpT5a598UJq4VW37OHf4OBcB7vYWga8=,tag:nlRerWMVGSrBO6ir8SxD5w==,type:str] -PUBPUB_PRODUCTION=ENC[AES256_GCM,data:+YMqvw==,iv:8BX5vv/f5+joMxH5IbRmps+jy1a/PGlWhdocif+LACU=,tag:jPQwqLcBtVrzxgtuX7KZFQ==,type:str] -#ENC[AES256_GCM,data:a1UATWoyeGcPNbqZDI/5t4ZeGHJ4lQGxIw==,iv:gfIL58YSIUuJ2Dd35HWTKqdKHDWlHFxb22etNoqbXzE=,tag:eQeJq8FzglAtgcxTw6GJHw==,type:comment] -#ENC[AES256_GCM,data:2xM1MTnAs4v2RlHq5XcEe1UNIIhV+JCTvdBAGDzH6EUvZEIh0w==,iv:sr9KzmZsbhzXZ2VEA6jeP9LP23JVxv+EwrQQcwtG7Wc=,tag:hB8YG7IuE/syrIylMkDq+Q==,type:comment] -S3_BACKUP_ENDPOINT=ENC[AES256_GCM,data:vtIbmGq13DPp6xmmPnoLZ9yNa08X4mGaa+10yG8xfvXzgKc=,iv:8AG4A52LWFlJ0ENL/iusIM5I6kBBTnYQjpPZrvKWPQY=,tag:tw0Fm3VItc0SduiVfZS2OQ==,type:str] -S3_BACKUP_ACCESS_KEY=ENC[AES256_GCM,data:2Th8rzoPAvzCYE9Wuqb8dT7x8Uk=,iv:bBspSn1lZyl9fXwni8kHBXF3HNqT4N24Q0qWCJJrfkI=,tag:aL3JNfqt3z9EtS4e7a6Kuw==,type:str] -S3_BACKUP_SECRET_KEY=ENC[AES256_GCM,data:Kn+bvOY060HrZ5iKt8LX9FAxUTqE3lsmFQhlX0CV9HOHL1h8JN+i+A==,iv:1EBNgDsy/kAvUJqxFwTVmH3YLlVWb9yyVwhNqaIYpBM=,tag:d1+up1HVO2YbM5UhFRtn3w==,type:str] -S3_BACKUP_BUCKET=ENC[AES256_GCM,data:Q6+2yr3yNTDohG8=,iv:HO7haOuCSn6wqL2n9PZd+wV+yGue1GtZZlu32zvIzKc=,tag:XgrLlDcIfENqw4ILP9pSbQ==,type:str] -SENTRY_AUTH_TOKEN=ENC[AES256_GCM,data:foNoUlRH4NSmcZb8wcJQWLGFYT+M4WhBOxcuKrMO3yk3M9ffyXASYEfcQuMX0JtyUHODCjIcZBEenxAjorXLPxRRA/86BvxrmUaQW3Ytzyr1oG7MMbx7C/0eomzahMONHFLqUitEiv+IzAm28/qwEvT7C0JoVjX5Inq1MXVJBlXNtn4eIvFVwFoIZD0DjhDPhNAbb9J/pZ7QP7/kXLd7O0ekWZAuyUeor0xSkvHu3RusGhkZp5E9k2idkw==,iv:P4dLkW6qYWo3V7Z828gjA5c/2HDO6g2ACA5QDY0YsdQ=,tag:PVzxgpN9V8nrvlsT2OWVRg==,type:str] -SENTRY_ORG=ENC[AES256_GCM,data:uD4Y,iv:xCwzDtvUsME6rkXD9ZjnWEpWlkkQeCHBTWO/KQ3d1tk=,tag:arH2UqCWDEnx63u0hYfGrA==,type:str] -SEQUELIZE_MAX_CONNECTIONS=ENC[AES256_GCM,data:xZc=,iv:+VnTLa4E9ebKgzkL6ZXmJ77/z1Wg4IPchiLomjucoKw=,tag:FxcZKE2WP24MtmMqysZcaQ==,type:str] -SLACK_WEBHOOK_URL=ENC[AES256_GCM,data:14eklwo2DK7XzFCl4B0zKorH2mjDJRPdLOQGlILCrrnEf9R7lNE9tAg6Jcw27UrNArwmHA4OoEUKEV2dyCVO8b3MPjsUNtm4X4lhFeQQKA==,iv:GMQNtmbaZf3fr9Lz1lhOyx3WQpEGt7HY0NhFqogT2Ys=,tag:P7Cmgkz8Fp797vDIIz3oDw==,type:str] -STITCH_WEBHOOK_URL=ENC[AES256_GCM,data:0DTdF0yYSzaR7kjhJlKAUI69vlizcQvVNKutdsgkJXEM7j7r7+Aa//kxWzs2RC4iybgqh0cd9SYNyMGgvcP/5ilOoV01LGhMEV4BpEzbC/y71U4PVpzJCRPrUDNbnW6tHcqsMA5neXXZUK9a0iae5NHV4PST,iv:kDMMDSMSYs2f+vd+rzEhAUYWhtO0ihNIF7W7eJWh4Z4=,tag:r7B+miNzxPb2MrANLj9GSQ==,type:str] -ZOTERO_CLIENT_KEY=ENC[AES256_GCM,data:3qkmTVWiUpP1D7vYahpLdQh1sG4=,iv:trEME/o+ZXXizoRX3zgRm7d00GGDuhIhPXWO/Ck/5WQ=,tag:WsIG8PSLsA5LIOeOZkTCfA==,type:str] -ZOTERO_CLIENT_SECRET=ENC[AES256_GCM,data:tVHxu0XEMCKOai0fZcCiddozBKw=,iv:WALv9GYJDI5eDo4YiOZhkhGt+I6RFN7V7GpF/P3MVFk=,tag:0XG3bv9CpS72tC07qDYeyg==,type:str] -sops_age__list_0__map_enc=-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA1dmQrZ1ZWZ1ozdU96RWs3\nRDU3MEhPbDNXLzRJZ1pKbUl2Y1E4U2Qyd1c4CkQ4TlBCNHM5L28yRDJIZ1N2ZE5D\ndWJQQ3RXQmJjcnZhc0hxYnFhYXpua00KLS0tIEZyeTJMT243cnliVW1TWEFPTVlh\nZS9wTjUybzJ2cFh5YVdsbE1TUmE2K0kK1k8ivaJttK0pzp5UijicG/NT9BaQ8Xog\n0TQpR54x4vtLBgfEMi8vy/V6jBYtCFHoefIUfKw9psNxrYx6NhJz8A==\n-----END AGE ENCRYPTED FILE-----\n +AES_ENCRYPTION_KEY=ENC[AES256_GCM,data:vygyIeOSU4KUHu+/fxzVNsdcHBO+MYuFxmOjoKUkjMnaJh1zUzlnNRhYV0kwKAOnMt9G9AHCHIJSQy2qFVGJ3A==,iv:PbvrO8d9PkDzc6JMbLyuggNt8KccogKgycU8q9yZacQ=,tag:OkxUrar5Cu0pelVnBuV0uQ==,type:str] +ALGOLIA_ID=ENC[AES256_GCM,data:bUHjSRm+AOHkNQ==,iv:qnT7OqcCpL2VYn3wCxwhH2fqyIUvs/jECHpb8zlH8vk=,tag:u2f/vk1G6UC8xEhZiq4K3g==,type:str] +ALGOLIA_KEY=ENC[AES256_GCM,data:tdjWtPWM7cX6hnvt1v4xoqVVifOxFkIZUTL1bAafKyE=,iv:Af3qGbXERA7PDPCqWq0M+BKgI+f0wJkAtXAVogJlCIE=,tag:adyhpIkcATzUuE1bjwlcuw==,type:str] +ALGOLIA_SEARCH_KEY=ENC[AES256_GCM,data:JW7IpGE8QX0wKNzsTxdSxPsED7PE8u6PcM+jWXLao7M=,iv:r0a8KJeMtoB0ZPy804e+59FlFNXSV1d1xNUKTw3mx+s=,tag:P6SHV0+LKpUA038deFLRKQ==,type:str] +ALTCHA_HMAC_KEY=ENC[AES256_GCM,data:zgjxjOtHld02AvBOXI4GCPNuJrPTLiLiSV+61H8Jrm6TZFO4clSqedYeCLLKsCx/z91bFgOpR/o14izffwCoh8CiggDVuMXCjShwQBbDKHRf3GNHoolSUUtDYMHSzsRGq8EBCntyRHLl9e/tXNAbmE2zkwSbNP/J8B0J64EhUkA=,iv:W/cWEFr7cvK/QAwps33eVuaZ4I2vw2W4ecUsfrBD+RQ=,tag:hu3vbVRMM6qunZI8iZm6EA==,type:str] +AWS_ACCESS_KEY_ID=ENC[AES256_GCM,data:WGHmOUbhcOZxUj5wB5sp73DuBdc=,iv:NpgY16GRgSfZjC5VQqOwSyOkkmIG7CSqSkRPnokP9DI=,tag:GqImV7HF/TZcbfao7LIEoA==,type:str] +AWS_BACKUP_ACCESS_KEY_ID=ENC[AES256_GCM,data:XCikIUyQKb4SMDx3w3tnpZ858fA=,iv:UFBsy/8UjGGsgU/QDzPOLqwfxDS5mEmQFuJNRwjDSkE=,tag:swUNf9SFVtKRin9rLRdoHQ==,type:str] +AWS_BACKUP_SECRET_ACCESS_KEY=ENC[AES256_GCM,data:wjtTJXAFZLwnbD8emLjKc3EEfjxVWA/JUZzduvCf+0DWLVn7IB9aAA==,iv:v/uLO7IeD6DDCBNQ4JJ3pgox5YwsVW9LZgnzN34mLQc=,tag:81R3/hFQqmYzQc2X8s2R8g==,type:str] +AWS_SECRET_ACCESS_KEY=ENC[AES256_GCM,data:ledp+eYDE3t06vVdXs/Eywk9LbJ0t6QITwoSQmSRIRk26g9edDP+kg==,iv:faeUjtqNvUxnYlkxTb6tmARPBYN5I1URFTS+1ENcr3U=,tag:pb/zOwdJ38TF8PG696Dj8Q==,type:str] +BACKUPS_SECRET=ENC[AES256_GCM,data:/RsR0gc+zR7el3g04PugFCKL6LSqxTMYJ9JlOjc0wajz4XxsgU1NUavy2UY=,iv:ykuE2Qwz/bhaJ1qM4ff75VVhDqpcxboPVjyvwWgvTTU=,tag:59y2q7RSLvvtPx9NFSN1JA==,type:str] +BLOCKLIST_IP_ADDRESSES=ENC[AES256_GCM,data:ZJkKI7w5FzILMMwE/EZzdSVMTuq9,iv:1ntwAH57FdntSNoj/U1juyjoSvm3B3yjZAXm3mzx63k=,tag:d3xAa5uDQWLgAQ28mclvLw==,type:str] +DATABASE_URL=ENC[AES256_GCM,data:HIK672oRWuDLX3e4pONz8OjEkb7h7cJipOD8EkESrnWNtocsKEgbmMtpusk=,iv:JVEKUyXVm3UMQfLaqPF/IXKNe6K/mL9n7DdjaFSY1eQ=,tag:LqtRLI5UrnWKPZOgZdGEkw==,type:str] +DATACITE_DEPOSIT_URL=ENC[AES256_GCM,data:i7k85Qs/IhjzoPJdGzE3YLUgM2rbeFO61pT87Ig=,iv:xBPkOss4OhO2m+6sURxPUXHzhA06pVFa0zmP7IusD0I=,tag:ZbHinmWqblmQRNUwdb+3ug==,type:str] +NEW_ACCOUNT_LINK_COMMENT_WINDOW_MINUTES=ENC[AES256_GCM,data:bpk=,iv:h0Jff3p54eZmm9akWjZFP/3zmKDVrtJrToZQJMGbkJM=,tag:Qq7w7ujIO+3ig3MFVbxizA==,type:str] +DOI_LOGIN_ID=ENC[AES256_GCM,data:V9egweGB,iv:gjziKGluU7wCwBDhX6Cx3W4MnidEKiru2d0jfy7a86s=,tag:cvIesO03cV/O8fW220ISTQ==,type:str] +DOI_LOGIN_PASSWORD=ENC[AES256_GCM,data:b7xkNnujost9rSnFFSQs6IYRGQg=,iv:XkYzQaVAKp3UqENNW7PMekr7Ulnb4kHp71i0nZB5484=,tag:l9qeuqjbpvJQ6Lf2spdXFQ==,type:str] +DOI_SUBMISSION_URL=ENC[AES256_GCM,data:SWISAP/WDBhdQlP10DElDohvyXOVyJdY4c3EZA+K7HlTLffajx50Pg==,iv:jSE3Ho4qrwdq6bV618y6EGaKFxoSFAAg1WedzC6T3wM=,tag:Y7OmgyvhXtzpmS9Rz0DnNw==,type:str] +FASTLY_PURGE_TOKEN=ENC[AES256_GCM,data:y50/aBt5bDpW6xHCmRMqFq2cwyDRYXP9tJ9loPyWP50=,iv:zzt0CuELNQNxjb5xfx7JNLxksgfIS2OdqGnBmpXk8kw=,tag:Ke0VzdElzlebab4c+7NGDg==,type:str] +FASTLY_SERVICE_ID=ENC[AES256_GCM,data:ntSxO41AbYBMzCOqT5wOXd2XRNT8Dw==,iv:b+5DzbKJnB9IeJ5vaREENj6gdwZ6l3pyeDxSCAgVhJY=,tag:VPyazK66wA5wTZNPVvBDZA==,type:str] +FIREBASE_SERVICE_ACCOUNT_BASE64=ENC[AES256_GCM,data:Lw20ZSvKXCSQDGUKK97DI5TqGVx2bWHfacwLEpNFng1rwIy4hIhC4XXEROEerUGK4jA2Qr6+OCms+2WQVFqQG38b2pm4kIaikDA0KozsXHj7BsZ+IlMgKF2d5LPgwLjr+MCgBFXOgb6eeZhh1hyXUKfKaHeWdYfXjJT1m3jSd1qaNM7abmR8FEe1e6b7RAQqIy8tyOkFQ8qKdtx4g1k9ZZcmEZCpyg6C65UOUJnlkluxk6EdMVax7An6jkxW22yrAABIKDAljXtsqs7xk1/bGecz4/WUQO5yzlQqfjW3yC8y+/xGAScMM0opZmPvp6DqBkmkivpkyIPBMEpOSQzo/AwuNAb2kyfRYzakdLWXEZXCTmAv8/PUzA8Y5uOgAXDZ52T+kOH4ngKia7+mAMUvHOiCzhrTniM8+plU0jUldnP0EP4yrcbhG2K/zETn5md/JseUakLzUJbYjvD9ZaBL8Jt0rcvvJ6OGTuz35Iorm/5qVfkbWXOSVOF6bVc8AX+U9qFZIJokBy9pr0U1rvf4OBh81Jcs1ngsJPZ2atzfs/6yql9pf3vYbIGim1gsKqsu6sD8unL6Fl4qwvAAKLH6UgbA8u+ewbM0vBZY5Ve4dw8LUSeJ2TgN8i4mYacG94E9c/PHBNJ10FhPJSFdZxkjfOHODRI1u/SNQXgZ0tQg6aDOUWrCiPMCyZI4yecR00hB8uMrNMIX54dZ/CBl162c5AzWf57WaFUK9WapbUYeakqLw+E4ETlOoNqpaso5agOcWgiIRe1iBSZXecLEqmihS97KcSfhlilYuW1eSey2+ekwhoPvNwdLwGaTkENtnUmcXLBDNvLNBSCJ8I9jboA9Hz/my3T4sndsp/12eDpk/b9o43CgZN2FjI1/MlOMXEbneEOMfIQf7D1DPQ66WlvQhXmYznQk7V+RoXf/Tle/4pKOxvULi9a7yHVPeS8g6IVpdJpU1XQj4SyKorD3km4bsyGbOSv8F02J+fTDAuY5M+MfwJMFkGbsDhDhnL/XKbSa1/GL+LDJyTVp3e+VTBCp/ZdRnaXYPy76f9BANMejmN4b32no4LT29njdJHKDxEp61YrWMbzQ8G9cH/TSm02LHfPoApmS+qJObXph8ZANAKOatF6PPlbfPa49uRqxtubQfvwH1iw2ptRrlbxpDRytNRWE9Gzq2Mv8aSIWFwFOyXJIIat+vQYsyijNVo0bDHuXhHK1g9KcWWzM8ULsmtVibS3JCHCCpaxqP+VK/KevWCxdVlWPbcmOEEybrnn0SOr1ZN271Iq0k6YvMMHLPC066Qw3QEt0Sk27XwkpWzZ63YLBZUb3uJCzQ6MsSqu6MEB/nSpohVI5VIqbmiWdGMFw2GR8N1xIj50QxatxBdnqUJnTYmGL7knys6EVyCi70cUJTdrFHeDOEPSzQZOH8KFO4jrWQIqddcaJiI18xLLoAxmqm9W7URLx5gMDZy5j5ThcRQvtAEqcF3czE+PNOfqN1S18UzJrJMmAo68+bSEAurVSXBAHdtIoBfhzvFzfSX4xF6HffXYCISBqNDLYNj9iHgmKHJQQAK+ho7QOK1jpI3KvNccdos4EJZACluSIjrFZ62Y9KVCNm7CdP59J/2WdIWTVgIHfnCLcJwx4UDcqLjhSioPo4+TZw822jEbxjdoeeQ/uzk4jAxNWWEBWeEMOCP05H+mzZx2F22BuWjPhMXgq7Mo/cVJV3vlUY2xoie3mSBGatqvh9NjmHhMQiIHsd7Jx6k8hGBdk/IJSqG8P3ZJPHALf06Xud5F3Dm0nm/8bzXP7f4W2eyyObTbfLGsLfh00Au0dv/S9C2lqg47dcvFgGxfZCyr8zWYnEvv/T/+z8SyiWnzhL1gayol+RL8kBOy8WVBwk1rEqcp8+6Z77dbsiH6qMjrcA1a2C72/6dhV0Kx2MY2eKEAGPQpA5DSdFwJOSnVMXABr0VMh4v74URKKpDUwa67CJqzRgiojcz5QiOQg9GhfnvD2+jFzjATqeMq8vS+/bkyhAdW0U4TUuFwkYatLISWGQTeoBGKxGpxal34qfP3x0+/LcGh6L1qLNNCSFOJlD+/0zx3Fp22Ct/+bF7jeGQm29Zsri3gsqnKlT6sXsTycJlcmkh12cdN5/8tsQHz30gHMcKRuUOwzHyEQhk5hqs206D7DOh3fTQat/1VvZjRYTESrM9IZn5SLzE3sSMa6JVCCOYIuOoH55Gz2i0xYHuIzi8dn2j16860xmx6M6UPN8Zfry4MHh/o7nBFUwFF/A7pXU0+DlYRq8ClT9XeOm7xCTVdfnc3rgU8pPj1jEcrCwbVYEoI+mxtLEXPjN5AsbOpdVba78xZDYUwu1pYKwl7tTiHCticFHdcCLnLycKOr7ZxHzq4EhnTh5+wRBIPkSoypVioo6cD84ULWqKvJd6q3hww0XInrISnswVqakBAHU2EWBnXjPpucY3TAlRhviAlTtbRK//+/Q5N6miOkjfWRyqfLqm7HqxVEaC2BPd72N9OynXSL/0WRXoVYuBx7CA2GbRb1hMAjyaq9IPkNpLpDjNx6kssQAfij+AMkdXFtkHeAno8aZF8pJFuibzqySuJKN59wphjk+aATkbns1AJx5yJEthADPhocZVpvIFf5rCfldkd78ocHOIsq9n/5r/T1Bk0iB5z0qTY/TrDicBnp/TWtGy8U7hlPjkeY1vL4qYZCJR5LCx6LvilJeuOjQrRlqzRr+5hJ+YnzBp/6YZnIN6lrUTGe9JqoNqlyRiyGXrSzhLkAMkRSVzQ1p5D3E+n2Eq+2WChV2MqnFxUtkJxvuP79UK8GgUuej4nesZPBNo80RuJx5ijBELIKMAWYHIIDze21T3aKaM7pMADVdSsjgTs1FaybcAESN1gIYoEOTgNqhJwXRWyUUdChCCFoPzYHja2SDv23v0Oq3jy/JbTKl4bbWvNFM9ttQ/oi8YxlNVFfTx67/1teqfAHG76JV1Qe9lTbWIhg8+F/jgWTm28DycFoILul1m0REwCExS6DT9InFz0EOYsMrc6mvXgtn/MWavtiw0i+TiL30Sl6XO4eSbriLhv/gLN6JlErIRz2Czb5isN6z3AeFFPgIFZzlL6Fpnr4kEjlyECdProRGWCx59DxPUNL0nTmCvBja29up2Zzhvtc6s5MJL6MsPCVdq/lVIfAvbMb1vDpirTUbOOBXMSwqx/P9frjPUvNYVKbm2HJMSkWT2izLL74puuqPYwQu1/Bahj0DNdAk63bViDuEA17K5+h0LHKO3IpgNvIHumYvc5XXAn+KV+fAZaeAWVmy5xWxRGQAG7XatFBmcIr/sY6IOR1hkpsDMHAFUvSoUfj5zORQoX/buCpO1PVPXYSPtF0acbKwCYeoauvifqUrNfbDkDRLewhgILBNNYdkA5ulvqGIrgwlInQAL4PT5akiPjjsjypUK9hZVkd6LuOXnRm0IK7vi0juegdGnfLgtx05W03ULKLJDAsji6fOWcrVTUBf16+aGD2KCbLN8gYMkJnwk6X4CIVwwTJy1lJzOaLv5kC2V0h5iCaoTZU8959XRYIzNeN0z7+EdzLbWXyNkqqD9kq8hmjBaFgTUB8oeCGKb9Vk3T50gnHk3+hHvzv+fxJaXlVNNAzCDYD4yL/9KOfwraCENfYF8yWCnEEnZtAm38Z8JP1ZwDxvjz9o3a4pEvnb53kDUWc8wMViCPgL4c1LBHNjB971x2qVPvNTz9rv3LGWXPYXkVzBkEqnQYnZdXRaLBSCkrKcPaDR8sgfklZudoYKnhtE5gKq0F+Qv2aR36qH7qLY+HPIQbG4jMwlWe8aQj/QPvsAr2d1R7RiXZGSGv0msskjJ+mJXVIRxkw2eVRU7Yp4tubDEUVXcSMuakAu3nPBpC97Bnkr3C82c2NL/6C0p/CBuMhP2B9Fdmy8Lim7vV/J1Xu2j0SM0MOz7YEx5O68L/lzRgSA03eM1rAs/QU6785n9GMLO5hgkxexJnJcNnNYUFCU3xewsPQpg10uAzl45GSdYiD7sfX0hCy3nf+0Ymh+yA/epE93gaEDs/wCicJawpY/pzXeb0/XnhLYD1D/fO6vJz1p77DTBhs8x5cUfHpYKBGDAn3sGYlA3WOl9nIH9ShTv1PvlLztEOFHoZZOFE=,iv:X5jJT6sMqBzn4tCxdNLwlbpgnfPYR0lmUOKgN1HhcDE=,tag:YQrj+C24BBJkYp2+ZROSLw==,type:str] +JWT_SIGNING_SECRET=ENC[AES256_GCM,data:ltBeZ7bD3kNw10aMeagzk1sUaADl7DofbtR4cjU7JbjfTzMZAeqnODb/3N7yEyWxCFEenAGQoUyQCynE0FOMGiGvBW0c4OSvUp3seBeVaSVhxnX9EcFltPUPVU2I2xalcDyVvzX7OUnlhtjayYgezqL6ESxlF64JAuAaDYaGBov/QPTDiEXdSvS7S7SXQfKv8008VoiZOdJq6UVZl+tFW3SYFjn69oizYKMcHFKnY1WlLpr/PBqtAs94mYYhdro2O4I6NFSqwvOZiyIne4kUvfr4sKP/xtYnlQnNX/PZn09F8LoNc97wcihM+KO45oWjzKyLJQ5UnrTg+ehfJeyEa+tcCoK9V7B2dcONLIk1B2LY6D4t6KtGT0xEWa+sEQ+OgqVKTwvoQj6vSLKt/SOr54UcTUH8I1GHORhdNhUZrTCruhpheYFz8+fJNJttpaYWq5LHo85/+wzd0IA+j8RG/dfHwl4e5CLHug8CU2SxjtnO8OJ8W0Ba/Z+IdYZFMvkj8KW7vBIuUGtz6R2qzBPUHknZIBIgKC5litYmxqJY9bV6j0CLORoJlLPAR4txLjUqt/Um+CX13Bm8N56YYBKbDiT+2Q6dDiS2C60EZ14GywU2PZn7P13Y2uQUqSyJEy0jnSHaS+M3T8GyPjgaDn81H8smr+p7Rywp5Tz/WAvu+Ks=,iv:HhZBFTWqwgrD63anv57CWZWpURtSeryK7yBOb3XSC3g=,tag:9w6PKSgkQY1o+6Q75ICbaQ==,type:str] +MAILCHIMP_API_KEY=ENC[AES256_GCM,data:eL8cFJAcsegT7jcV7CWYho8xiLD2THC9TsbyEz1O22S40VUX,iv:yl4HlB1gX0roFtjfe18zmM4b0/WH71zf/n1Vvju2G6I=,tag:rR5ZFrMwnD34Yb/km7zjmg==,type:str] +MAILGUN_API_KEY=ENC[AES256_GCM,data:MojQDcCzBxDQCmCcJX3jzQ0rVHZ7Ooc/9AbR9gEq8gFgt85j,iv:WCNzExQsc8obUv8/hP2cbeVGYgxWMPGWX0vzkO16fP0=,tag:Bbucf5m/uJbk9xLJhU0n0Q==,type:str] +METABASE_SECRET_KEY=ENC[AES256_GCM,data:bhiOd5tbfGH7XuMGlNOos0yd7peshSE/eUdN6KPrOSdS5egjs0YKvUiUGpag2lbsTb62rWaEYsoFzK0teFl3/w==,iv:ZyyLhVYMJrHlyLfNHHckhXU+/AcrHcL82OdGXqh319Y=,tag:Ewq8BbqssDlV0vVjZkemWQ==,type:str] +NODE_ENV=ENC[AES256_GCM,data:RAV/1IftnkAcAg==,iv:8TqiRhtJBhucOVfY7eTYPI+hKnjHiiIUGKUAS2Qd8wQ=,tag:fSWBfDrEoOB9Ti0i14cCGQ==,type:str] +PUBPUB_PRODUCTION=ENC[AES256_GCM,data:OGPgLQ==,iv:gNA78W1+EjFvQLbyKrP3Be17uz7bUG1hHZ5oEnPe/BY=,tag:a0W5QGnAmu4rsEMBn1HNnQ==,type:str] +#ENC[AES256_GCM,data:uSLpX7XUPIeSorbyHblVSB0p7kWLTHJwnQ==,iv:2TRabaOiGKuEiYRlJTRv7ERnCokJI/FfsHxtKIYZQw8=,tag:WQA4VDXpg8JFPHt+rh+iUw==,type:comment] +#ENC[AES256_GCM,data:nwOy+yz0Eo9WJ2UkTLmzKAgSoWv6liE6z9lPqWhUAZ/VnlX6kg==,iv:2iD1nHSJjYKANZ1B3SwBgEcsIrZnk52WAFAovJaUBck=,tag:hoM8AHRLEE0LBx43oOWfDw==,type:comment] +S3_BACKUP_ENDPOINT=ENC[AES256_GCM,data:BJD0csMZBtBO9eP630N0D3ISZt7AneuV8acF4ZBDy112Jmg=,iv:lep1yhJMng5+Ki7IvJru09tA3bDNtKDBfBQ/SpHD1Y0=,tag:LZpCApTY19yjWEZfPNNkpw==,type:str] +S3_BACKUP_ACCESS_KEY=ENC[AES256_GCM,data:tJ4WFOb9WtAd7QCo/ndSSHGTyko=,iv:dcTibepCqaZfecHUmZVt1GK38pRokUixUM4MLaiTigA=,tag:Y/+7XKS84fShnzNTaYp8RA==,type:str] +S3_BACKUP_SECRET_KEY=ENC[AES256_GCM,data:Zow8u01eZtBL3zllK23zWDBbdbe77eCxwqDGAZYE6YTrx+ISqF0BsA==,iv:A0Q8UC89I9zJ4C3Ekcz3XwHEH/7XX3DYXyBTafVG0RA=,tag:kQwLNpoN37huRrryUFxywg==,type:str] +S3_BACKUP_BUCKET=ENC[AES256_GCM,data:W9NTPx/byuB/79s=,iv:DfjDrfzj4CtbdOjA2qIEs8n6vbz4zeT8ZP/a6PQKtEY=,tag:DfIR7qHF9fMYvbEB/wd0tQ==,type:str] +SENTRY_AUTH_TOKEN=ENC[AES256_GCM,data:motVr7lGtqerwVm1FNkLApk8fkD5oZQO7DBb3JHiGgWYD9HQ0/K4ApH4ops3mKgvWMN4lR0GN05YdZOuEofCY/2pbi3XL/nX3NfwDHZsADtU934GCK8N648+2ay7hlLU4Kgu7H9Km+kcld9DGdUkhze4Q74qnKvfTMkPy8FpF2dyQCK2q9j3whymGbmFHSEvnvla266MLNkG0jBWF0GD4bdMEf2PgklJB9WqRym3ql//Euq2VZr/aqdVpA==,iv:Tt4VnnlfnruoiDFr9zpWhMn6tSH5N9cxE7AcivMWsnc=,tag:XJrYO3KEfFOtmdmsb0ZY3w==,type:str] +SENTRY_ORG=ENC[AES256_GCM,data:5Ksg,iv:9XYHOiq52rAufw7A6N4/zRCkKuvEvE7r8X+FaOL7mwM=,tag:4/i2tvQQxd5OdSEo3PDArQ==,type:str] +SEQUELIZE_MAX_CONNECTIONS=ENC[AES256_GCM,data:NNA=,iv:BxefB2zJtOMeWFhz5UTU8N5DKSJjki+AXbrB3rDDheM=,tag:c1i310cLxylp+U57jI9xLA==,type:str] +SLACK_WEBHOOK_URL=ENC[AES256_GCM,data:DXyAO5GfrTiUEg977N7uBMvhVgkmxIFqEPGrja6rdR+ZpM6T68nDuI/MDoNyOJ/+M3tbIKkJKbBSu2Oqr4k+KoTgAXt2GgUisL1OMEZM4g==,iv:Os4EwwXM98Ij+Xw7oCNqEYWLXykQ5/+eUb5f7kfSFnk=,tag:dxPthqTllQT8t9H82Z2LhA==,type:str] +STITCH_WEBHOOK_URL=ENC[AES256_GCM,data:dTFTHyDN5W98nhBtz8/x/t2vdp70Z4sSd5KC8Bw+67YNJt+AiksNXTGe2cC44rPweF1lClcaXkJY5qDVMhYMdrVdNpdpVZQOKHb/TPWCBtdbImSi9k3O6nz0WxeqKvr/qp2GYcMal/dlzDnGjoolSAGwxK3o,iv:zFZDBO+1VaAEslICj4gfYAiMn+7Evp1UG/y5uainBIk=,tag:LuvLe9acXVlpiswlc6cQyQ==,type:str] +ZOTERO_CLIENT_KEY=ENC[AES256_GCM,data:OEA8tPW6hZHSX6yFhCbMsLZ4vW8=,iv:tQIjmn3Av6EwfWMf2f+MRwZTb2d2lSSYJUUUbE3i3Q0=,tag:mKwKmoA1/7/AzBW05fmCgA==,type:str] +ZOTERO_CLIENT_SECRET=ENC[AES256_GCM,data:RthkhjEo//QeqE9g4VJShfjSZV8=,iv:r5OEVwL924+YD39OSt73XE4Xapy/eR0lxzsdy/5iKvM=,tag:TYkgzO6A5LeR8BwEOdwcUA==,type:str] +#ENC[AES256_GCM,data:9Hm14d2sntOEpU0/mNR6qjDyVhLlPuryhhB0z8/1fG1ziAJ3mu8ICeiP+9nXjyMdLhKFbc8xDFkGnpq4mJ4e42qR,iv:UdaHaDkF3cSVughBKIrpg9xIL8coKvA0/PI+qzsQLfU=,tag:BHSyncX9nq98aZiAQpsDXw==,type:comment] +AM_REDSHIFT_PATH=ENC[AES256_GCM,data:wwGSNkmT+R6kK3Zbjh9fxI/TIjvTO0JKamnlsmG/WfZ9B57ouO+4u9Rk7U0hmo9LI506gJOzBHLj823Kj7k=,iv:T5N2tX5ys1uM/Ii7H807Hzs4UbwCA/fga0k7a2TWyEw=,tag:THBi+OHYFJx89vLVRqmpUg==,type:str] +AM_REDSHIFT_BACKUP_ACCESS_KEY=ENC[AES256_GCM,data:ecEUwFwgJSh741C/1xfmMR/syXM=,iv:bEEdtDhXfKBM8c/P4inD9njPGjEe4V6ot2ZJXCPP6ZE=,tag:YEWSQtRGxOGyDLbYlqmlxg==,type:str] +AM_REDSHIFT_BACKUP_SECRET_KEY=ENC[AES256_GCM,data:7M8xTIbH+Cp+PRLzpVecKK7aMKG3p5Z/m1DRVQr0/dFuUAKlwPD4LA==,iv:iftRwF7RrflCnMJmFxHNTk92rEWPcVMNtVkpAxQnkgk=,tag:1+Ia9Iw02H7PlgVNBIVj1Q==,type:str] +AM_REDSHIFT_BACKUP_ROLE_ARN=ENC[AES256_GCM,data:fi3PkT4UR+0t/9JNfbwalkfPEGC2zjnjWGD8inevJ+zzRkiQfEBNzzxEHIMBc2/8WgDl,iv:1GPs28DJCn0gTsSGWJ2U2SFnoXh4SvRzs2KAIPTxA6o=,tag:kfmH7hmfK0kFUy7ydX7w1w==,type:str] +AM_MB_RDS_URL=ENC[AES256_GCM,data:opclwIxyRVRfkjMQ7zjeGAzGkOoRTKVpK8bAVrZVmKghGiLsPysejw35Ad7dhVwm3cc6DOcEkdQxaUCPZNSXx8QisP9Uw4KQ+sJyxipomD43JcwHyM6khHgo8QgvY03uQrBu00oVZerU,iv:qraNn0LtXtCGcdkfExJthEJPCe47KiZWBY/EKhiLYDM=,tag:KtKivtwjjYWF1gu7FV8bBg==,type:str] +AM_MB_RDS_ADMIN_EMAIL=ENC[AES256_GCM,data:zY9ZxUv6AsOnAHl9XA3xmN/Iy9Q+AqVDAuKp,iv:/5+JAw/0RnM7NnAOxpiAGmvJy9k84CAQR6zFzshRZ2c=,tag:mTJ0AJXTa3nZ0nfmcPQU8Q==,type:str] +AM_MB_RDS_ADMIN_PASSWORD=ENC[AES256_GCM,data:VPdTqdzEJBdBsh3JLIb2fXl+CA==,iv:TiiSk0U+5UxewH202IrjTkEDrhyanDaudfMt12LMH7Q=,tag:co5tmib1EzodgRjnEthzog==,type:str] +MB_DB_HOST=ENC[AES256_GCM,data:5F+Js06iU06J1B4=,iv:DWx7Xq3/TkiTh0osC8N0OsT++fq1OjXYy7XZGICFKPM=,tag:LpnbrhItRpgoFaSao8zGew==,type:str] +MB_DB_PASS=ENC[AES256_GCM,data:DppjdGWSmbrsEo14ninXYEPJOWN0N1IJumJAOIRL0C/Q+L0Rv1sd2UppKvBY1/zamfBYPjksucXRbL34tMWQ6Q==,iv:5L87zTW0GHG8DbelyRRidQjyys96SrjGv/nZg7gTCd8=,tag:qQmbzaZqdV/vwHwVzEX98A==,type:str] +MB_DB_USER=ENC[AES256_GCM,data:MKD0jM5ue7Q=,iv:qI0YvXzDUnUx+WfQ3NpuMtawdO82MTQw8QUitip0Xv4=,tag:BliQbhzmRh4z6HiUSzsvpw==,type:str] +MB_DB_NAME=ENC[AES256_GCM,data:Qf044hAeeB8WUSs=,iv:+PacaEGtnvzUDJ6zRokg2FbwXuCU0KrNsSCQ7UbsJ0Y=,tag:zBKwpakiLJ7zBwYZ6E3wIQ==,type:str] +MB_DB_PORT=ENC[AES256_GCM,data:13JAOA==,iv:y8UZ5/GRg4i36RnnzzQvCG0coJdIf4ZE/MCfRfvtUzQ=,tag:Jf3AUlMOd5DkXhQUpaHZFQ==,type:str] +AM_MB_COLLECTION_ID=ENC[AES256_GCM,data:99o=,iv:3aPCWBDOOKy5XHMR6xDUmD6XrnpFMSXac2OmaiYCFAo=,tag:LaNVDdspBzFKZObHQNMY7g==,type:str] +sops_age__list_0__map_enc=-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB5a0prL1JjWTdVZVFLcnBF\naGdyblZjak0vQU0rby9qclF4cE8zcFNZM1I0CmxnRUEwV2xyTVl4TFVPU2NSZFl0\nNUVEOENzQkJ3bjJhVUdSTkFtMjBOOFEKLS0tIHZNbmNZRG5pdWk0V1pOS1lFY0R3\nK3JGbzFDRzNUcXVkSm9nWW82d2J5dEUK/Y2rtcyxkVTKyOu59RT+FGOSRn9W9JzZ\nhbd1AA/LStk5513h/VBWbayc8GpeOKcQwUXJq8q0XIguOYGjK6uAYA==\n-----END AGE ENCRYPTED FILE-----\n sops_age__list_0__map_recipient=age1wravpjmed26772xfjhawmnsnc4933htapg6y5xseqml0jdv8z9hqemzhcr -sops_age__list_1__map_enc=-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBqMGFITjNJazkzNGtwQUF2\ndURyRktTTE9VY3l4TWVlam9zc2psQzBSMUNBClpIcU81VitObWtnM2U2TVhGaHo5\nK25Cb01CQjdwSlRjVGNmdGlUY2xHa1kKLS0tIFMwOUtuVTB6WmNrTDhXd09tRGU0\nLzM1bi8wVmpnazMyLzhvOFYxVmNtUmsKSjl6gGc/oBnkd7rGz5HsXVlRtSY6PopE\nOoGXRVElkLdhBzC9LY6HbgkWSJyu+v56mIbYro7euczYX+6dzrrerw==\n-----END AGE ENCRYPTED FILE-----\n +sops_age__list_1__map_enc=-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBTazlkck1KY1pteS9jV2dv\nalJKR1JNS3dCZFltbnNtYlN1THRHTkhnL0hvCnRQMXJDbFhVSGxHaW10UmJPK05Z\nUGpjTXRRbTRJQjZFVVY0ZXBPajE0RVEKLS0tIEpacHh4UXF0cHQydTczeEtUWUpJ\nSVV5OHVtNFRXcWFISjZQMzluN1BPNlUKb8SmOO/egTtdDtZzEq0GWICT44ffNYmf\nNzwGInRrRMgWtiJg6DBArwJ7l4Ej9yRPkPqE7GAw8ZkoPL64cygBoA==\n-----END AGE ENCRYPTED FILE-----\n sops_age__list_1__map_recipient=age1vhftscteyrwphx0jpp0yl60xxrjs77jq05mkzv88js7ckc926vcqepp2cj -sops_age__list_2__map_enc=-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB2V2c3dkxMQnZwTDRrZWZI\nL3A0ck9CaEo1WGx3UE96bEZnWmVBUXJaazJRCkRLYVN4aFMwN3gvaThkUUE2dGpn\nS3ZGMjlTUW1NL1Radk4xOXIvYm42dHcKLS0tIEJxbHRIQUo3ZlU4QUR5K1RCL2RK\nNmc0bllPUmNDUkVuRFdUaGJYZU5yVmcKZKktxJzujZ/C8VogEoRE4l6RrYEUf41q\nObBXtOgawAax/R0gdCfoTkoC5yfNT27yu5ngIz0wlDpusNF4L+fMpQ==\n-----END AGE ENCRYPTED FILE-----\n +sops_age__list_2__map_enc=-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBwL0lVVWEyMkRDMWZwd0kw\nVTJkd0szNXg0dlhUZkVlYWhHYndZVENDbWlzCms4eHF4OUJ2N2dsQ1ViMHYvUHhU\ndWNjQmxlMHZVVFdPWDRTM3JaNmNjclUKLS0tIC8wWHRCajBIT01BZjRyaDdPZkZx\nbDBVNWZ0MTRqYnNTejBiVjZRdHV5OGMKjNKm9X/GVaJPzO/03k0gqVVZi31XxyZC\nfy3LhFeB+lNLzHJcxpBd7Rm2asQPrHhEtDXd6vanrJCOZEwg7TUtpQ==\n-----END AGE ENCRYPTED FILE-----\n sops_age__list_2__map_recipient=age1slx6e48k7fre0ddyu7dtm2wwcqaywn5ke2mkngym3rpazxwvvuyq9qjknk -sops_age__list_3__map_enc=-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBHc2RSV0trMW1qZ2dRaFlS\ndjZ6ZHliM3hVS2FKa2R2R3FiYklwQllrQTFFCjJwNk1ZZEo1UkNDMTBPQy9vcVM2\nbE1MSHg0WXZ3NGd0MmUzY0pmTWRkSlEKLS0tIGxKTThVR016enMvZi8rSjBHNC9V\nL2MwNWhBdlplYnJ2Q3lLMzNDZGJQZVEKkNanfMXN+vxtDXaUSK/w4q4/bc0NwVQw\nVyKvEmwT2IagUVqD7ey+4upLgiGdRuhkooAGfiBtJHyPVsKvCPeMgg==\n-----END AGE ENCRYPTED FILE-----\n +sops_age__list_3__map_enc=-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB3MDlnV0lBNWJXVFEyaThT\nUWY0VzhFYzEyVWpIZUc0alF3VjFWWEJWbGdjCklPdWZwVXJjQlpGait1cllQaHlJ\nY2lTeWx2djF6THJBWFhGY0RaVXNGRmMKLS0tIEZQMEdmSFVMQ2pabnorM01lUWFI\neFdJR3ErNWp0Rnpjd0psdEl2aUhkVG8KowGgog0VUQAJv6S06RNDc8+awgBhNYzx\nyWWKQ2P/leX0jwKigYGsuYaB1mK9rgZ1rEvW5rpg6jv1Nqqmzh7OaA==\n-----END AGE ENCRYPTED FILE-----\n sops_age__list_3__map_recipient=age1vfhyk6wmt993dezz5wjf6n3ynkd6xptv2dr0qdl6kmttev9lh5dsfjfs3h -sops_age__list_4__map_enc=-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA1N0JjWkt1cGxwbzhPVUVk\nVE9MS1R3MHRIL1VLRHlGNGxpSUdGZWNVc21NCnhMamUyd0RwYWw1SHhQQXBzbzh1\nRVlvMXpaUTNId1BwYU1LZFVONExhNzQKLS0tIEZpZ3dhVHJtRHBidU1OZ1NyZFUw\nT04reTVzR0tuK3QxZ1ZzdTJqUThCTzQK3vTB9CQLDcOJShwvYOkmOcLQfJ9QCkZF\n9SNn1wPd1MNGSfxdSOYWl9t1o0z3gz65YSL71KJC8xZ90TaV3Esthw==\n-----END AGE ENCRYPTED FILE-----\n +sops_age__list_4__map_enc=-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBsV1dZbkd2STc1aGtURk1D\nM0ljblVHU0E2SjlqQzNHTDZlWVJIbDhBUEFRCkFFR21mU2tDZmpMQkpwR1NTbHA2\nbjkxbU1lN3Vlbm5wWjhZV3dubEZ3UEUKLS0tIEVKdStFbzBlYkorWTJ5VERyMmFJ\nQ3NoSWtYVHZUY2VaT0pCRWVZOXNmb3MKfs92pq4pzi0gNoAdhwSxg3ocxF0t+ZJx\nNtfMZQLqo1DXAq1HVsTpH6iE5lPhC/inzGsQCzzn8RRUe25o36biPQ==\n-----END AGE ENCRYPTED FILE-----\n sops_age__list_4__map_recipient=age1jwuvzyghfer7rx3qtqa4vs0gxyff0sg6pqgmqvp5zlhnpmvrkdlsdha4kx -sops_age__list_5__map_enc=-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAyeXlpa0xmWkMzSGVNQ2V1\nOXMwWi9ibitBeXk1b0lsdUVYVlZieEVZS25nCkVUcHdkTlpuUjdkTTlwV3BmOW5i\nbEQ0cEY4TEJubGV1YjlnQjdjckRKbjgKLS0tIHRHa252aXVxT3Bwd1JaWW5Ld0xr\nTW84ZHlWNXF0K2dDMEJ2NmxWWndYckUKEsCbco1+C6mhFUxFj9zGQuo1Xs5U2HMb\nFq/OLyclKqJryJbkNRPQTfD0J/vzLKk1TLjiyn6fE74vvHVp0qUOCQ==\n-----END AGE ENCRYPTED FILE-----\n +sops_age__list_5__map_enc=-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBuVHZkR1FORlFla2huVW1X\nN3pOT1dZYk4wbGVQZ3NGSG5XMVArc3ZjRTJJCkRrdWg3b3B1cUtlYXBnUFhGaDdt\nMUZ2U2QxbXk3bnppcXRhbXB6azhncXMKLS0tIFN3UU40T3YyZVlRc09zY081Vkox\nbG1ZaWcrSVFnM2YrNnVzbUFEYU5MR2MKqW3OObpmnCU9fAp89tpba+ygxYYfxM93\nfl9p3HKUJ5UoZPH9mH0dA2PVEkBguaZ5uwTXHKPo/gl86ooOzo0G/Q==\n-----END AGE ENCRYPTED FILE-----\n sops_age__list_5__map_recipient=age1pgxk292zq30wafwg03gge7hu5dlu3h7yfldp2y8kqekfaljjky7s752uwy -sops_lastmodified=2026-04-01T03:03:08Z -sops_mac=ENC[AES256_GCM,data:hQlfCmb8jk2U5+U1cRI0W662mzTMEfaHXhmX2nHLnTSn/n3I3rVnl/tl16XikJjrO6LQpoELhs0Espu+1PYF7VRknDI0vmoP4XYciV/W2LiXZ8Q9au7NSfNrOmkvNEGdXgOedzxO5cCl96ARmmXsz1Yi5SzWWaxDW3mFb6RPSm4=,iv:TLr9e4RhdNm6Yc/YKPXBxQRzb9Q1RqYeW+0dQodcKDo=,tag:Z2TIN79C44dWJWmgzfdd8g==,type:str] +sops_lastmodified=2026-04-07T16:05:01Z +sops_mac=ENC[AES256_GCM,data:82f0QgWIxB3dBcPThB+fq/4s3iwKDpB2QTF2sio+4wzuG11/2MWuZhryz8mF5QNzU8YANu6NZk+IptNd2ZtEQGiiVmfTIMJQvVwlAeFBIw0QqWoh/i3DzMf9zXDdHZiHmVP+7w7vUg1LXVtWUkv/VoRZ5Mu3fVl1UawovvYUTw4=,iv:7hK17kJe/AcSiWMsnNO2CXS+h6CTK5C4XMaIY0jIjrE=,tag:Zht/fZKhKxU+Kb2iX3LvYA==,type:str] sops_unencrypted_suffix=_unencrypted -sops_version=3.11.0 +sops_version=3.12.1 diff --git a/infra/docker-compose.dev.yml b/infra/docker-compose.dev.yml index 5b84969836..bb7fec834f 100644 --- a/infra/docker-compose.dev.yml +++ b/infra/docker-compose.dev.yml @@ -86,16 +86,13 @@ services: - "${DB_PORT:-5439}:5432" networks: [appnet] + metabase: build: context: ./metabase + env_file: + - .env.dev environment: - MB_DB_TYPE: postgres - MB_DB_DBNAME: metabasedb - MB_DB_PORT: "5432" - MB_DB_HOST: metabase_db - MB_DB_USER: appuser - MB_DB_PASS: apppassword MB_JETTY_PORT: "3001" MB_ENABLE_QUERY_CACHING: "true" MB_QUERY_CACHING_TTL_RATIO: "10" @@ -112,13 +109,8 @@ services: analytics_sync: build: context: ./duckdb-sync - environment: - PG_HOST: db - PG_PORT: "5432" - PG_DB: appdb - PG_USER: appuser - PG_PASS: apppassword - SYNC_INTERVAL: "${ANALYTICS_SYNC_INTERVAL:-43200}" + env_file: + - .env.dev volumes: - analytics_duckdb:/data depends_on: @@ -126,6 +118,8 @@ services: networks: [appnet] metabase_db: + env_file: + - .env.dev image: postgres:18 environment: - POSTGRES_USER=appuser @@ -137,6 +131,24 @@ services: - "${METABASE_DB_PORT:-5440}:5432" networks: [appnet] + analytics_migration: + build: + context: ../tools/analytics-migration + env_file: + - .env.dev + environment: + DATABASE_URL: postgres://appuser:apppassword@db:5432/appdb + volumes: + - ../tools/analytics-migration:/migration + - /var/run/docker.sock:/var/run/docker.sock + - analytics_migration_data:/data + - analytics_duckdb:/duckdb + depends_on: + - db + - metabase_db + networks: [appnet] + profiles: [migration] + # cron: # build: # context: .. @@ -164,3 +176,4 @@ volumes: rabbitmqdata: metabase_pgdata: analytics_duckdb: + analytics_migration_data: diff --git a/infra/duckdb-sync/sync.sh b/infra/duckdb-sync/sync.sh index 736bd9ea19..536e24c01f 100644 --- a/infra/duckdb-sync/sync.sh +++ b/infra/duckdb-sync/sync.sh @@ -2,11 +2,12 @@ set -euo pipefail DUCKDB_FILE="${DUCKDB_FILE:-/data/analytics.duckdb}" -PG_HOST="${PG_HOST:-db}" -PG_PORT="${PG_PORT:-5432}" -PG_DB="${PG_DB:-appdb}" -PG_USER="${PG_USER:-appuser}" -PG_PASS="${PG_PASS:-apppassword}" +PG_HOST="${PG_HOST}" +PG_PORT="${PG_PORT}" +PG_DB="${PG_DB}" +PG_USER="${PG_USER}" +PG_PASS="${PG_PASS}" + SYNC_INTERVAL="${SYNC_INTERVAL:-43200}" MODE="${1:-loop}" diff --git a/infra/env.example b/infra/env.example new file mode 100644 index 0000000000..8794bba335 --- /dev/null +++ b/infra/env.example @@ -0,0 +1,13 @@ +AM_REDSHIFT_PATH= +AM_REDSHIFT_BACKUP_ACCESS_KEY= +AM_REDSHIFT_BACKUP_SECRET_KEY= +AM_MB_RDS_URL= +AM_MB_RDS_ADMIN_EMAIL= +AM_MB_RDS_ADMIN_PASSWORD= +AM_MB_COLLECTION_ID=14 + +MB_DB_HOST= +MB_DB_PASS= +MB_DB_USER= +MB_DB_NAME= +MB_DB_PORT= diff --git a/tools/analytics-migration/Dockerfile b/tools/analytics-migration/Dockerfile new file mode 100644 index 0000000000..60b0e8cefa --- /dev/null +++ b/tools/analytics-migration/Dockerfile @@ -0,0 +1,37 @@ +FROM node:22-bookworm-slim + +ARG DUCKDB_VERSION=v1.4.4 + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + curl unzip ca-certificates postgresql-client awscli jq && \ + rm -rf /var/lib/apt/lists/* + +RUN install -m 0755 -d /etc/apt/keyrings && \ + curl -fsSL https://download.docker.com/linux/debian/gpg \ + -o /etc/apt/keyrings/docker.asc && \ + chmod a+r /etc/apt/keyrings/docker.asc && \ + echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] \ + https://download.docker.com/linux/debian bookworm stable" \ + > /etc/apt/sources.list.d/docker.list && \ + apt-get update && \ + apt-get install -y --no-install-recommends docker-ce-cli && \ + rm -rf /var/lib/apt/lists/* + +RUN ARCH=$(dpkg --print-architecture) && \ + DUCKDB_ARCH=$ARCH && \ + curl -fsSL \ + "https://github.com/duckdb/duckdb/releases/download/${DUCKDB_VERSION}/duckdb_cli-linux-${DUCKDB_ARCH}.zip" \ + -o /tmp/duckdb.zip && \ + unzip /tmp/duckdb.zip -d /usr/local/bin/ && \ + rm /tmp/duckdb.zip && \ + chmod +x /usr/local/bin/duckdb && \ + duckdb -c "INSTALL postgres" + +RUN npm install -g tsx + +COPY migrate-redshift.sh migrate-metabase.sh migrate-metabase-queries.ts /migration/ +RUN chmod +x /migration/*.sh + +WORKDIR /migration +CMD ["bash"] diff --git a/tools/analytics-migration/README.md b/tools/analytics-migration/README.md new file mode 100644 index 0000000000..c3e12a9c8a --- /dev/null +++ b/tools/analytics-migration/README.md @@ -0,0 +1,34 @@ +# analytics migration + +Migrates analytics data from AWS Redshift and the old Metabase RDS instance to the local stack (Postgres, DuckDB, Metabase). Run manually, not automated. + +## prerequisites + +1. Whitelist your IP in the AWS RDS security group for the old Metabase database, otherwise `pg_dump` will hang. +3. The dev stack should be running (`db` and `metabase_db` at minimum). + +## running + +Start the migration container: + + docker compose -f infra/docker-compose.dev.yml --profile migration run --rm analytics_migration + +Inside the container, run the two scripts in order: + + ./migrate-redshift.sh + ./migrate-metabase.sh + +The first script imports Redshift CSV data into the Postgres `AnalyticsEvents` table. The second sets up Metabase with the old dump, syncs DuckDB, configures database connections, and migrates saved questions. + +Both scripts cache downloaded data on a persistent volume. To force re-downloading: + + FORCE_DOWNLOAD=1 ./migrate-redshift.sh + FORCE_DOWNLOAD=1 ./migrate-metabase.sh + +## cleanup + +Once the migration is verified and no longer needs to be re-run: + + docker volume rm pubpub_v6_dev_analytics_migration_data + +Remove the `AM_*` variables from `infra/.env.dev`. diff --git a/tools/analytics-migration/migrate-metabase.sh b/tools/analytics-migration/migrate-metabase.sh new file mode 100755 index 0000000000..832ca37fc8 --- /dev/null +++ b/tools/analytics-migration/migrate-metabase.sh @@ -0,0 +1,316 @@ +#!/bin/bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +DATA_DIR="${DATA_DIR:-/data}" +DUMP_FILE="$DATA_DIR/metabase_dump.sql" +FORCE="${FORCE_DOWNLOAD:-0}" +COLLECTION_ID="${AM_MB_COLLECTION_ID:-14}" + +DATABASE_URL="${DATABASE_URL:-postgres://appuser:apppassword@db:5432/appdb}" + +MB_HOST="${MB_DB_HOST:-metabase_db}" +MB_PORT="${MB_DB_PORT:-5432}" +MB_USER="${MB_DB_USER:-appuser}" +MB_PASS="${MB_DB_PASS:-apppassword}" +MB_NAME="${MB_DB_NAME:-metabasedb}" + +ADMIN_EMAIL="${AM_MB_RDS_ADMIN_EMAIL:?set AM_MB_RDS_ADMIN_EMAIL}" +ADMIN_PASSWORD="${AM_MB_RDS_ADMIN_PASSWORD:?set AM_MB_RDS_ADMIN_PASSWORD}" + +MB_DB_URL="postgres://${MB_USER}:${MB_PASS}@${MB_HOST}:${MB_PORT}/${MB_NAME}" +MB_ADMIN_URL="postgres://${MB_USER}:${MB_PASS}@${MB_HOST}:${MB_PORT}/postgres" +MB_URL="${MB_URL:-http://metabase:3001}" + +DUCKDB_FILE="${DUCKDB_FILE:-/duckdb/analytics.duckdb}" + +parse_database_url() { + local url="${1#postgres://}" + url="${url#postgresql://}" + + PG_USER_MAIN="${url%%:*}"; url="${url#*:}" + PG_PASS_MAIN="${url%%@*}"; url="${url#*@}" + PG_HOST_MAIN="${url%%:*}"; url="${url#*:}" + PG_PORT_MAIN="${url%%/*}" + PG_DB_MAIN="${url#*/}" +} + +parse_database_url "$DATABASE_URL" + +METABASE_STOPPED=0 +cleanup() { + if [ "$METABASE_STOPPED" = "1" ] && command -v docker &>/dev/null; then + echo "restarting metabase..." + local cid + cid=$(docker ps -aq --filter "label=com.docker.compose.service=metabase" 2>/dev/null | head -1 || true) + + if [ -n "$cid" ]; then + docker start "$cid" 2>/dev/null || true + fi + fi +} +trap cleanup EXIT + +echo "metabase migration" +echo " metabase: $MB_URL" +echo " metabase db: ${MB_HOST}:${MB_PORT}/${MB_NAME}" +echo " main db: ${PG_HOST_MAIN}:${PG_PORT_MAIN}/${PG_DB_MAIN}" +echo " collection: $COLLECTION_ID" +echo "" + +# ── 1. get the dump ── + +NEEDS_DOWNLOAD=0 +if [ ! -f "$DUMP_FILE" ]; then + NEEDS_DOWNLOAD=1 +elif [ "$FORCE" = "1" ]; then + NEEDS_DOWNLOAD=1 +fi + +if [ "$NEEDS_DOWNLOAD" = "1" ]; then + : "${AM_MB_RDS_URL:?no dump at $DUMP_FILE and AM_MB_RDS_URL is not set}" + echo "[1/10] downloading metabase dump from RDS..." + mkdir -p "$DATA_DIR" + pg_dump "$AM_MB_RDS_URL" --no-owner --no-acl > "$DUMP_FILE" + echo " saved to $DUMP_FILE" +else + echo "[1/10] dump already present at $DUMP_FILE (set FORCE_DOWNLOAD=1 to re-download)" +fi + +# ── 2. stop metabase ── + +echo "[2/10] stopping metabase..." +if command -v docker &>/dev/null; then + METABASE_CID=$(docker ps -q --filter "label=com.docker.compose.service=metabase" 2>/dev/null || true) + + if [ -n "$METABASE_CID" ]; then + docker stop "$METABASE_CID" >/dev/null + METABASE_STOPPED=1 + echo " stopped" + else + echo " not running" + fi +else + echo " docker not available, make sure metabase is stopped" +fi + +# ── 3. wait for metabase_db ── + +echo "[3/10] waiting for metabase_db..." +until PGPASSWORD="$MB_PASS" pg_isready -h "$MB_HOST" -p "$MB_PORT" -U "$MB_USER" -q 2>/dev/null; do + sleep 1 +done +echo " ready" + +# ── 4. reset the database ── + +echo "[4/10] resetting metabase database..." +psql "$MB_ADMIN_URL" -q -c \ + "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = '${MB_NAME}' AND pid != pg_backend_pid();" \ + 2>/dev/null || true + +psql "$MB_ADMIN_URL" -q -c "DROP DATABASE IF EXISTS \"${MB_NAME}\";" +psql "$MB_ADMIN_URL" -q -c "CREATE DATABASE \"${MB_NAME}\" OWNER \"${MB_USER}\";" + +# ── 5. import dump ── + +echo "[5/10] importing dump..." +psql "$MB_DB_URL" -q < "$DUMP_FILE" + +CARD_COUNT=$(psql "$MB_DB_URL" -tA -c "SELECT count(*) FROM report_card") +echo " $CARD_COUNT saved questions imported" + +# ── 6. patch settings ── + +echo "[6/10] patching settings..." +psql "$MB_DB_URL" -q <<'SQL' +UPDATE setting +SET value = REPLACE(value, 'US/Eastern', 'America/New_York') +WHERE value LIKE '%US/Eastern%'; + +UPDATE metabase_database +SET details = REPLACE(details, 'US/Eastern', 'America/New_York') +WHERE details LIKE '%US/Eastern%'; + +UPDATE core_user +SET settings = REPLACE(settings, 'US/Eastern', 'America/New_York') +WHERE settings LIKE '%US/Eastern%'; +SQL +echo " timezone references fixed" + +# ── 7. duckdb sync ── + +echo "[7/10] running full DuckDB sync..." +rm -f "$DUCKDB_FILE" + +duckdb "$DUCKDB_FILE" </dev/null; then + METABASE_CID=$(docker ps -aq --filter "label=com.docker.compose.service=metabase" 2>/dev/null | head -1 || true) + + if [ -n "$METABASE_CID" ]; then + docker start "$METABASE_CID" >/dev/null + METABASE_STOPPED=0 + else + echo " WARNING: could not find metabase container, start it manually" + fi +else + echo " docker not available, start metabase manually" +fi + +echo -n " waiting for ready" +for i in $(seq 1 180); do + if curl -sf "${MB_URL}/api/health" >/dev/null 2>&1; then + echo " (${i}s)" + break + fi + + if [ "$i" -eq 180 ]; then + echo "" + echo " ERROR: metabase not ready after 180s" + exit 1 + fi + + echo -n "." + sleep 1 +done + +# ── 9. configure database connections ── + +echo "[9/10] configuring database connections..." + +SESSION=$(curl -sf "${MB_URL}/api/session" \ + -H 'Content-Type: application/json' \ + -d "{\"username\":\"${ADMIN_EMAIL}\",\"password\":\"${ADMIN_PASSWORD}\"}" \ + | jq -r '.id') + +# postgres connection + +EXISTING_PG=$(curl -sf "${MB_URL}/api/database" \ + -H "X-Metabase-Session: ${SESSION}" \ + | jq -r '[(.data // .)[] | select(.name == "PubPub")] | first | .id // empty') + +if [ -n "$EXISTING_PG" ]; then + echo " removing existing 'PubPub' database (id=$EXISTING_PG)..." + curl -sf "${MB_URL}/api/database/${EXISTING_PG}" \ + -H "X-Metabase-Session: ${SESSION}" \ + -X DELETE >/dev/null +fi + +NEW_DB_ID=$(curl -sf "${MB_URL}/api/database" \ + -H "X-Metabase-Session: ${SESSION}" \ + -H 'Content-Type: application/json' \ + -X POST \ + -d "{ + \"name\": \"PubPub\", + \"engine\": \"postgres\", + \"details\": { + \"host\": \"${PG_HOST_MAIN}\", + \"port\": ${PG_PORT_MAIN}, + \"dbname\": \"${PG_DB_MAIN}\", + \"user\": \"${PG_USER_MAIN}\", + \"password\": \"${PG_PASS_MAIN}\", + \"ssl\": false + }, + \"is_full_sync\": true, + \"auto_run_queries\": true + }" | jq -r '.id') + +echo " created database 'PubPub' (id=${NEW_DB_ID})" + +curl -sf "${MB_URL}/api/database/${NEW_DB_ID}/sync_schema" \ + -H "X-Metabase-Session: ${SESSION}" \ + -X POST >/dev/null + +echo -n " waiting for schema sync" +for i in $(seq 1 120); do + STATUS=$(curl -sf "${MB_URL}/api/database/${NEW_DB_ID}" \ + -H "X-Metabase-Session: ${SESSION}" \ + | jq -r '.initial_sync_status // "incomplete"' 2>/dev/null || echo "incomplete") + + if [ "$STATUS" = "complete" ]; then + echo " (${i}s)" + break + fi + + if [ "$i" -eq 120 ]; then + echo "" + echo " WARNING: sync not complete after 120s, continuing anyway" + fi + + echo -n "." + sleep 2 +done + +# duckdb connection + +EXISTING_DUCK=$(curl -sf "${MB_URL}/api/database" \ + -H "X-Metabase-Session: ${SESSION}" \ + | jq -r '[(.data // .)[] | select(.name == "Analytics (DuckDB)")] | first | .id // empty') + +if [ -n "$EXISTING_DUCK" ]; then + echo " removing existing 'Analytics (DuckDB)' database (id=$EXISTING_DUCK)..." + curl -sf "${MB_URL}/api/database/${EXISTING_DUCK}" \ + -H "X-Metabase-Session: ${SESSION}" \ + -X DELETE >/dev/null +fi + +DUCKDB_ID=$(curl -sf "${MB_URL}/api/database" \ + -H "X-Metabase-Session: ${SESSION}" \ + -H 'Content-Type: application/json' \ + -X POST \ + -d '{ + "name": "Analytics (DuckDB)", + "engine": "duckdb", + "details": { + "database_file": "/duckdb/analytics.duckdb", + "read_only": true + }, + "is_full_sync": true, + "auto_run_queries": true + }' | jq -r '.id') + +echo " created database 'Analytics (DuckDB)' (id=${DUCKDB_ID})" + +curl -sf "${MB_URL}/api/database/${DUCKDB_ID}/sync_schema" \ + -H "X-Metabase-Session: ${SESSION}" \ + -X POST >/dev/null +echo " sync triggered" + +# ── 10. migrate queries ── + +OLD_DB_ID=$(psql "$MB_DB_URL" -tA -c "SELECT id FROM metabase_database WHERE engine = 'redshift' ORDER BY id LIMIT 1") + +if [ -z "$OLD_DB_ID" ]; then + echo "[10/10] no redshift database found in dump, skipping query migration" + echo "" + echo "done. open metabase to verify." + exit 0 +fi + +echo "[10/10] migrating saved questions (redshift db=$OLD_DB_ID -> postgres db=$NEW_DB_ID)..." + +tsx "${SCRIPT_DIR}/migrate-metabase-queries.ts" \ + --metabase-url "$MB_URL" \ + --session "$SESSION" \ + --old-db-id "$OLD_DB_ID" \ + --new-db-id "$NEW_DB_ID" \ + --collection "$COLLECTION_ID" + +echo "" +echo "done. open metabase to verify." diff --git a/tools/analytics-migration/migrate-redshift.sh b/tools/analytics-migration/migrate-redshift.sh index 2f38823465..5aa68653de 100755 --- a/tools/analytics-migration/migrate-redshift.sh +++ b/tools/analytics-migration/migrate-redshift.sh @@ -1,32 +1,43 @@ #!/bin/bash set -euo pipefail -SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" -PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +DATA_DIR="${DATA_DIR:-/data/redshift}" +FORCE="${FORCE_DOWNLOAD:-0}" +DATABASE_URL="${DATABASE_URL:-postgres://appuser:apppassword@db:5432/appdb}" -REDSHIFT_BAK_DIR="${PROJECT_ROOT}/redshift_bak_data" -S3_REDSHIFT_PATH="${S3_REDSHIFT_PATH:-s3://temp-migration-redshift/pubpub_analytics/backup_20260402/}" -DATABASE_URL="${DATABASE_URL:-postgres://appuser:apppassword@localhost:5439/appdb}" +export AWS_ACCESS_KEY_ID="${AM_REDSHIFT_BACKUP_ACCESS_KEY:?set AM_REDSHIFT_BACKUP_ACCESS_KEY}" +export AWS_SECRET_ACCESS_KEY="${AM_REDSHIFT_BACKUP_SECRET_KEY:?set AM_REDSHIFT_BACKUP_SECRET_KEY}" +S3_PATH="${AM_REDSHIFT_PATH:?set AM_REDSHIFT_PATH}" -echo "Analytics Migration" -echo "===================" -echo " database: $DATABASE_URL" -echo " backup: $REDSHIFT_BAK_DIR" +STAGING_CREATED=0 +cleanup() { + if [ "$STAGING_CREATED" = "1" ]; then + echo "cleaning up staging table..." + psql "$DATABASE_URL" -c "DROP TABLE IF EXISTS analytics_staging;" 2>/dev/null || true + fi +} +trap cleanup EXIT + +echo "redshift migration" +echo " database: ${DATABASE_URL%%@*}@..." +echo " s3 path: $S3_PATH" +echo " data dir: $DATA_DIR" echo "" -# ── step 1: download from s3 if not already present ── +# ── step 1: download from s3 ── -if [ ! -f "$REDSHIFT_BAK_DIR/data000" ]; then - echo "[1/5] downloading redshift backup from s3..." - mkdir -p "$REDSHIFT_BAK_DIR" - aws s3 cp "$S3_REDSHIFT_PATH" "$REDSHIFT_BAK_DIR" --recursive +if [ -f "$DATA_DIR/data000" ] && [ "$FORCE" != "1" ]; then + echo "[1/5] backup already present, skipping (set FORCE_DOWNLOAD=1 to re-download)" else - echo "[1/5] backup already present, skipping download" + echo "[1/5] downloading redshift backup from s3..." + mkdir -p "$DATA_DIR" + aws s3 cp "$S3_PATH" "$DATA_DIR" --recursive fi -# ── step 2: ensure the AnalyticsEvents table exists ── +# ── step 2: ensure AnalyticsEvents table exists ── +# this is a safety net; the table may already exist via sequelize migrations. -echo "[2/5] ensuring AnalyticsEvents table exists..." +echo "[2/5] ensuring AnalyticsEvents table..." psql "$DATABASE_URL" <<'SQL' CREATE TABLE IF NOT EXISTS "AnalyticsEvents" ( id uuid PRIMARY KEY DEFAULT gen_random_uuid(), @@ -82,9 +93,17 @@ CREATE INDEX IF NOT EXISTS "analytics_events_collection_event_ts" ON "AnalyticsEvents" ("collectionId", event, "timestamp"); SQL +EXISTING_COUNT=$(psql "$DATABASE_URL" -tA -c 'SELECT count(*) FROM "AnalyticsEvents"' 2>/dev/null || echo "0") +if [ "$EXISTING_COUNT" != "0" ]; then + echo " table already has $EXISTING_COUNT rows, duplicates will be skipped" +fi + # ── step 3: create staging table and import csv ── +# uses an unlogged table for faster bulk import. all columns are text because +# the redshift backup uses different column names and types. echo "[3/5] creating staging table..." +STAGING_CREATED=1 psql "$DATABASE_URL" <<'SQL' DROP TABLE IF EXISTS analytics_staging; CREATE UNLOGGED TABLE analytics_staging ( @@ -139,7 +158,7 @@ CREATE UNLOGGED TABLE analytics_staging ( SQL echo "[4/5] importing csv files into staging..." -for f in "$REDSHIFT_BAK_DIR"/data[0-9][0-9][0-9]; do +for f in "$DATA_DIR"/data[0-9][0-9][0-9]; do echo " $(basename "$f")..." psql "$DATABASE_URL" -c "\copy analytics_staging FROM '$f' CSV HEADER" done @@ -159,7 +178,7 @@ SELECT CASE END; $$ LANGUAGE sql IMMUTABLE; --- cast through double precision to handle scientific notation (e.g. 1.71974e+14) +-- handles scientific notation (e.g. 1.71974e+14) via double precision cast CREATE OR REPLACE FUNCTION pg_temp.safe_ts(val text) RETURNS timestamptz AS $$ SELECT CASE WHEN val IS NULL OR val = '' THEN NULL @@ -252,9 +271,11 @@ SELECT FROM analytics_staging s WHERE s.type IS NOT NULL AND s.event IS NOT NULL - AND pg_temp.safe_ts(s."timestamp") IS NOT NULL; + AND pg_temp.safe_ts(s."timestamp") IS NOT NULL +ON CONFLICT (id) DO NOTHING; SQL +STAGING_CREATED=0 echo " cleaning up staging table..." psql "$DATABASE_URL" -c "DROP TABLE IF EXISTS analytics_staging;" psql "$DATABASE_URL" -c 'ANALYZE "AnalyticsEvents";' diff --git a/tools/analytics-migration/setup-metabase.sh b/tools/analytics-migration/setup-metabase.sh deleted file mode 100755 index 970e293c69..0000000000 --- a/tools/analytics-migration/setup-metabase.sh +++ /dev/null @@ -1,243 +0,0 @@ -#!/bin/bash -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" -PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" -COMPOSE_FILE="${PROJECT_ROOT}/infra/docker-compose.dev.yml" - -DUMP_FILE="${1:?usage: setup-metabase.sh [collection-id]}" -COLLECTION_ID="${2:-14}" - -MB_DB_PORT="${METABASE_DB_PORT:-5440}" -MB_DB_USER="${METABASE_DB_USER:-appuser}" -MB_DB_PASS="${METABASE_DB_PASS:-apppassword}" -MB_DB_NAME="${METABASE_DB_NAME:-metabasedb}" -MB_PORT="${METABASE_PORT:-3030}" -MB_URL="http://localhost:${MB_PORT}" - -MAIN_DB_HOST="${MAIN_DB_DOCKER_HOST:-db}" -MAIN_DB_PORT="${MAIN_DB_PORT:-5432}" -MAIN_DB_NAME="${MAIN_DB_NAME:-appdb}" -MAIN_DB_USER="${MAIN_DB_USER:-appuser}" -MAIN_DB_PASS="${MAIN_DB_PASS:-apppassword}" - -: "${METABASE_ADMIN_EMAIL:?set METABASE_ADMIN_EMAIL}" -: "${METABASE_ADMIN_PASSWORD:?set METABASE_ADMIN_PASSWORD}" - -MB_CONN="postgres://${MB_DB_USER}:${MB_DB_PASS}@localhost:${MB_DB_PORT}" -MB_DB_URL="${MB_CONN}/${MB_DB_NAME}" -MB_ADMIN_URL="${MB_CONN}/postgres" - -json_val() { - node -e "let d='';process.stdin.on('data',c=>d+=c);process.stdin.on('end',()=>{const o=JSON.parse(d);console.log($1)})" -} - -echo "Metabase Setup" -echo "==============" -echo " dump: $DUMP_FILE" -echo " metabase: $MB_URL" -echo " database: localhost:${MB_DB_PORT}/${MB_DB_NAME}" -echo "" - -# ── 1. stop metabase ── - -echo "[1/10] stopping metabase..." -docker compose -f "$COMPOSE_FILE" stop metabase 2>/dev/null || true - -# ── 2. ensure metabase_db is running ── - -echo "[2/10] ensuring metabase_db is running..." -docker compose -f "$COMPOSE_FILE" up -d metabase_db -until PGPASSWORD="$MB_DB_PASS" pg_isready -h localhost -p "$MB_DB_PORT" -U "$MB_DB_USER" -q 2>/dev/null; do - sleep 1 -done - -# ── 3. reset the database ── - -echo "[3/10] resetting database..." -psql "$MB_ADMIN_URL" -q -c \ - "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = '${MB_DB_NAME}' AND pid != pg_backend_pid();" \ - 2>/dev/null || true - -psql "$MB_ADMIN_URL" -q -c "DROP DATABASE IF EXISTS \"${MB_DB_NAME}\";" -psql "$MB_ADMIN_URL" -q -c "CREATE DATABASE \"${MB_DB_NAME}\" OWNER \"${MB_DB_USER}\";" - -# ── 4. import dump ── - -echo "[4/10] importing dump..." -psql "$MB_DB_URL" -q < "$DUMP_FILE" - -CARD_COUNT=$(psql "$MB_DB_URL" -tA -c "SELECT count(*) FROM report_card") -echo " $CARD_COUNT saved questions imported" - -# ── 5. fix settings ── - -echo "[5/10] patching settings..." -psql "$MB_DB_URL" -q <<'SQL' -UPDATE setting -SET value = REPLACE(value, 'US/Eastern', 'America/New_York') -WHERE value LIKE '%US/Eastern%'; - -UPDATE metabase_database -SET details = REPLACE(details, 'US/Eastern', 'America/New_York') -WHERE details LIKE '%US/Eastern%'; - -UPDATE core_user -SET settings = REPLACE(settings, 'US/Eastern', 'America/New_York') -WHERE settings LIKE '%US/Eastern%'; -SQL -echo " timezone references fixed" - -# ── 6. initial DuckDB sync ── - -echo "[6/10] running initial DuckDB sync..." -docker compose -f "$COMPOSE_FILE" build analytics_sync -docker compose -f "$COMPOSE_FILE" run --rm analytics_sync sync.sh --once - -# ── 7. start metabase ── - -echo "[7/10] starting metabase..." -docker compose -f "$COMPOSE_FILE" up -d metabase - -echo -n " waiting for ready" -for i in $(seq 1 180); do - if curl -sf "${MB_URL}/api/health" >/dev/null 2>&1; then - echo " (${i}s)" - break - fi - - if [ "$i" -eq 180 ]; then - echo "" - echo " ERROR: metabase not ready after 180s" - exit 1 - fi - - echo -n "." - sleep 1 -done - -# ── 8. add Postgres database connection ── - -echo "[8/10] configuring Postgres database connection..." - -SESSION=$(curl -sf "${MB_URL}/api/session" \ - -H 'Content-Type: application/json' \ - -d "{\"username\":\"${METABASE_ADMIN_EMAIL}\",\"password\":\"${METABASE_ADMIN_PASSWORD}\"}" \ - | json_val 'o.id') - -EXISTING_DB=$(curl -sf "${MB_URL}/api/database" \ - -H "X-Metabase-Session: ${SESSION}" \ - | json_val '(o.data || o).find(d => d.name === "PubPub")?.id ?? ""') - -if [ -n "$EXISTING_DB" ] && [ "$EXISTING_DB" != "undefined" ]; then - echo " removing existing 'PubPub' database (id=$EXISTING_DB)..." - curl -sf "${MB_URL}/api/database/${EXISTING_DB}" \ - -H "X-Metabase-Session: ${SESSION}" \ - -X DELETE >/dev/null -fi - -NEW_DB_ID=$(curl -sf "${MB_URL}/api/database" \ - -H "X-Metabase-Session: ${SESSION}" \ - -H 'Content-Type: application/json' \ - -X POST \ - -d "{ - \"name\": \"PubPub\", - \"engine\": \"postgres\", - \"details\": { - \"host\": \"${MAIN_DB_HOST}\", - \"port\": ${MAIN_DB_PORT}, - \"dbname\": \"${MAIN_DB_NAME}\", - \"user\": \"${MAIN_DB_USER}\", - \"password\": \"${MAIN_DB_PASS}\", - \"ssl\": false - }, - \"is_full_sync\": true, - \"auto_run_queries\": true - }" | json_val 'o.id') - -echo " created database 'PubPub' (id=${NEW_DB_ID})" - -curl -sf "${MB_URL}/api/database/${NEW_DB_ID}/sync_schema" \ - -H "X-Metabase-Session: ${SESSION}" \ - -X POST >/dev/null - -echo -n " waiting for sync" -for i in $(seq 1 120); do - STATUS=$(curl -sf "${MB_URL}/api/database/${NEW_DB_ID}" \ - -H "X-Metabase-Session: ${SESSION}" \ - | json_val 'o.initial_sync_status || "incomplete"' 2>/dev/null || echo "incomplete") - - if [ "$STATUS" = "complete" ]; then - echo " (${i}s)" - break - fi - - if [ "$i" -eq 120 ]; then - echo "" - echo " WARNING: sync not complete after 120s, continuing anyway" - fi - - echo -n "." - sleep 2 -done - -# ── 9. add DuckDB database connection ── - -echo "[9/10] configuring DuckDB database..." - -EXISTING_DUCKDB=$(curl -sf "${MB_URL}/api/database" \ - -H "X-Metabase-Session: ${SESSION}" \ - | json_val '(o.data || o).find(d => d.name === "Analytics (DuckDB)")?.id ?? ""') - -if [ -n "$EXISTING_DUCKDB" ] && [ "$EXISTING_DUCKDB" != "undefined" ]; then - echo " removing existing 'Analytics (DuckDB)' database (id=$EXISTING_DUCKDB)..." - curl -sf "${MB_URL}/api/database/${EXISTING_DUCKDB}" \ - -H "X-Metabase-Session: ${SESSION}" \ - -X DELETE >/dev/null -fi - -DUCKDB_ID=$(curl -sf "${MB_URL}/api/database" \ - -H "X-Metabase-Session: ${SESSION}" \ - -H 'Content-Type: application/json' \ - -X POST \ - -d '{ - "name": "Analytics (DuckDB)", - "engine": "duckdb", - "details": { - "database_file": "/duckdb/analytics.duckdb", - "read_only": true - }, - "is_full_sync": true, - "auto_run_queries": true - }' | json_val 'o.id') - -echo " created database 'Analytics (DuckDB)' (id=${DUCKDB_ID})" - -curl -sf "${MB_URL}/api/database/${DUCKDB_ID}/sync_schema" \ - -H "X-Metabase-Session: ${SESSION}" \ - -X POST >/dev/null - -echo " sync triggered" - -# ── 10. migrate queries ── - -OLD_DB_ID=$(psql "$MB_DB_URL" -tA -c "SELECT id FROM metabase_database WHERE engine = 'redshift' ORDER BY id LIMIT 1") - -if [ -z "$OLD_DB_ID" ]; then - echo "[10/10] no redshift database found, skipping query migration" - echo "" - echo "done. open ${MB_URL} to verify." - exit 0 -fi - -echo "[10/10] migrating saved questions (redshift db=$OLD_DB_ID -> postgres db=$NEW_DB_ID)..." - -npx tsx "${SCRIPT_DIR}/migrate-metabase-queries.ts" \ - --metabase-url "$MB_URL" \ - --session "$SESSION" \ - --old-db-id "$OLD_DB_ID" \ - --new-db-id "$NEW_DB_ID" \ - --collection "$COLLECTION_ID" - -echo "" -echo "done. open ${MB_URL} to verify."