diff --git a/.changeset/dashboard-provisioner.md b/.changeset/dashboard-provisioner.md new file mode 100644 index 0000000000..ead2be784f --- /dev/null +++ b/.changeset/dashboard-provisioner.md @@ -0,0 +1,5 @@ +--- +"@hyperdx/api": minor +--- + +feat: add file-based dashboard provisioner that watches a directory for JSON files and upserts dashboards into MongoDB diff --git a/packages/api/docs/auto_provision/AUTO_PROVISION.md b/packages/api/docs/auto_provision/AUTO_PROVISION.md index 6f42ccf777..e1e3b38bd2 100644 --- a/packages/api/docs/auto_provision/AUTO_PROVISION.md +++ b/packages/api/docs/auto_provision/AUTO_PROVISION.md @@ -163,6 +163,62 @@ services: For more complex configurations, you can use environment files or Docker secrets to manage these values. +## Dashboard Provisioning + +HyperDX supports file-based dashboard provisioning, similar to Grafana's +provisioning system. A background process watches a directory for `.json` files +and upserts dashboards into MongoDB, matched by name for idempotency. + +### Environment Variables + +| Variable | Required | Default | Description | +| ----------------------------------- | -------- | ------- | --------------------------------------------------------------- | +| `DASHBOARD_PROVISIONER_DIR` | Yes | — | Directory to watch for `.json` dashboard files | +| `DASHBOARD_PROVISIONER_INTERVAL` | No | `30000` | Sync interval in milliseconds (minimum 1000) | +| `DASHBOARD_PROVISIONER_TEAM_ID` | No\* | — | Scope provisioning to a specific team ID | +| `DASHBOARD_PROVISIONER_ALL_TEAMS` | No\* | `false` | Set to `true` to provision dashboards to all teams | + +\*One of `DASHBOARD_PROVISIONER_TEAM_ID` or `DASHBOARD_PROVISIONER_ALL_TEAMS=true` +is required when `DASHBOARD_PROVISIONER_DIR` is set. + +### Dashboard JSON Format + +Each `.json` file in the provisioner directory should contain a dashboard object +with at minimum a `name` and `tiles` array: + +```json +{ + "name": "My Dashboard", + "tiles": [ + { + "id": "tile-1", + "x": 0, + "y": 0, + "w": 6, + "h": 4, + "config": { + "name": "Request Count", + "source": "Metrics", + "displayType": "line", + "select": [{ "aggFn": "count" }] + } + } + ], + "tags": ["provisioned"] +} +``` + +### Behavior + +- Dashboards are matched by name and team for idempotency +- Provisioned dashboards are flagged with `provisioned: true` so they never + overwrite user-created dashboards with the same name +- Removing a file from the directory does **not** delete the dashboard from + MongoDB (safe by default) +- Files are validated against the `DashboardWithoutIdSchema` Zod schema; invalid + files are skipped with a warning + + ## Note on Security While this feature is convenient for development and initial setup, be careful diff --git a/packages/api/src/__tests__/dashboardProvisioner.test.ts b/packages/api/src/__tests__/dashboardProvisioner.test.ts new file mode 100644 index 0000000000..eaddab905a --- /dev/null +++ b/packages/api/src/__tests__/dashboardProvisioner.test.ts @@ -0,0 +1,314 @@ +import fs from 'fs'; +import os from 'os'; +import path from 'path'; + +import { createTeam } from '@/controllers/team'; +import { + readDashboardFiles, + startDashboardProvisioner, + stopDashboardProvisioner, + syncDashboards, +} from '@/dashboardProvisioner'; +import { clearDBCollections, closeDB, connectDB, makeTile } from '@/fixtures'; +import Dashboard from '@/models/dashboard'; + +describe('dashboardProvisioner', () => { + let tmpDir: string; + + beforeAll(async () => { + await connectDB(); + }); + + beforeEach(() => { + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'hdx-dash-test-')); + }); + + afterEach(async () => { + fs.rmSync(tmpDir, { recursive: true, force: true }); + await clearDBCollections(); + }); + + afterAll(async () => { + await closeDB(); + }); + + describe('readDashboardFiles', () => { + it('returns empty array for non-existent directory', () => { + const result = readDashboardFiles('/non/existent/path'); + expect(result).toEqual([]); + }); + + it('returns empty array for directory with no json files', () => { + fs.writeFileSync(path.join(tmpDir, 'readme.txt'), 'not a dashboard'); + const result = readDashboardFiles(tmpDir); + expect(result).toEqual([]); + }); + + it('skips invalid JSON files', () => { + fs.writeFileSync(path.join(tmpDir, 'bad.json'), '{invalid json'); + const result = readDashboardFiles(tmpDir); + expect(result).toEqual([]); + }); + + it('skips files that fail schema validation', () => { + fs.writeFileSync( + path.join(tmpDir, 'no-name.json'), + JSON.stringify({ tiles: [] }), + ); + const result = readDashboardFiles(tmpDir); + expect(result).toEqual([]); + }); + + it('parses valid dashboard files', () => { + fs.writeFileSync( + path.join(tmpDir, 'test.json'), + JSON.stringify({ name: 'Test', tiles: [makeTile()], tags: [] }), + ); + const result = readDashboardFiles(tmpDir); + expect(result).toHaveLength(1); + expect(result[0].name).toBe('Test'); + }); + }); + + describe('syncDashboards', () => { + it('creates a new dashboard', async () => { + const team = await createTeam({ name: 'My Team' }); + fs.writeFileSync( + path.join(tmpDir, 'test.json'), + JSON.stringify({ + name: 'New Dashboard', + tiles: [makeTile()], + tags: [], + }), + ); + + await syncDashboards(team._id.toString(), tmpDir); + + const count = await Dashboard.countDocuments({ team: team._id }); + expect(count).toBe(1); + }); + + it('updates an existing dashboard by name', async () => { + const team = await createTeam({ name: 'My Team' }); + const tile = makeTile(); + await new Dashboard({ + name: 'Existing', + tiles: [tile], + tags: [], + team: team._id, + provisioned: true, + }).save(); + + const newTile = makeTile(); + fs.writeFileSync( + path.join(tmpDir, 'existing.json'), + JSON.stringify({ + name: 'Existing', + tiles: [newTile], + tags: ['updated'], + }), + ); + + await syncDashboards(team._id.toString(), tmpDir); + + const dashboard = (await Dashboard.findOne({ + name: 'Existing', + team: team._id, + })) as any; + expect(dashboard.tiles[0].id).toBe(newTile.id); + expect(dashboard.tags).toEqual(['updated']); + }); + + it('does not create duplicates on repeated sync', async () => { + const team = await createTeam({ name: 'My Team' }); + fs.writeFileSync( + path.join(tmpDir, 'test.json'), + JSON.stringify({ name: 'Dashboard', tiles: [makeTile()], tags: [] }), + ); + + await syncDashboards(team._id.toString(), tmpDir); + await syncDashboards(team._id.toString(), tmpDir); + await syncDashboards(team._id.toString(), tmpDir); + + const count = await Dashboard.countDocuments({ team: team._id }); + expect(count).toBe(1); + }); + + it('provisions to multiple teams', async () => { + const teamA = await createTeam({ name: 'Team A' }); + const teamB = await createTeam({ name: 'Team B' }); + fs.writeFileSync( + path.join(tmpDir, 'shared.json'), + JSON.stringify({ + name: 'Shared Dashboard', + tiles: [makeTile()], + tags: [], + }), + ); + + await syncDashboards(teamA._id.toString(), tmpDir); + await syncDashboards(teamB._id.toString(), tmpDir); + + expect(await Dashboard.countDocuments({ team: teamA._id })).toBe(1); + expect(await Dashboard.countDocuments({ team: teamB._id })).toBe(1); + }); + + it('does not overwrite user-created dashboards', async () => { + const team = await createTeam({ name: 'My Team' }); + const userTile = makeTile(); + await new Dashboard({ + name: 'My Dashboard', + tiles: [userTile], + tags: ['user-tag'], + team: team._id, + // provisioned defaults to false + }).save(); + + fs.writeFileSync( + path.join(tmpDir, 'my-dashboard.json'), + JSON.stringify({ + name: 'My Dashboard', + tiles: [makeTile()], + tags: ['provisioned-tag'], + }), + ); + + await syncDashboards(team._id.toString(), tmpDir); + + // User-created dashboard is untouched + const userDashboard = (await Dashboard.findOne({ + name: 'My Dashboard', + team: team._id, + provisioned: { $ne: true }, + })) as any; + expect(userDashboard).toBeTruthy(); + expect(userDashboard.tiles[0].id).toBe(userTile.id); + expect(userDashboard.tags).toEqual(['user-tag']); + + // Provisioned copy was created alongside it + const provisionedDashboard = await Dashboard.findOne({ + name: 'My Dashboard', + team: team._id, + provisioned: true, + }); + expect(provisionedDashboard).toBeTruthy(); + }); + }); + + describe('startDashboardProvisioner', () => { + const originalEnv = process.env; + + beforeEach(() => { + process.env = { ...originalEnv }; + stopDashboardProvisioner(); + }); + + afterEach(() => { + stopDashboardProvisioner(); + process.env = originalEnv; + }); + + it('is a no-op when DASHBOARD_PROVISIONER_DIR is not set', async () => { + const team = await createTeam({ name: 'My Team' }); + fs.writeFileSync( + path.join(tmpDir, 'test.json'), + JSON.stringify({ name: 'Noop', tiles: [makeTile()], tags: [] }), + ); + delete process.env.DASHBOARD_PROVISIONER_DIR; + process.env.DASHBOARD_PROVISIONER_ALL_TEAMS = 'true'; + await startDashboardProvisioner(); + expect(await Dashboard.countDocuments({ team: team._id })).toBe(0); + }); + + it('rejects invalid interval', async () => { + const team = await createTeam({ name: 'My Team' }); + fs.writeFileSync( + path.join(tmpDir, 'test.json'), + JSON.stringify({ name: 'Noop', tiles: [makeTile()], tags: [] }), + ); + process.env.DASHBOARD_PROVISIONER_DIR = tmpDir; + process.env.DASHBOARD_PROVISIONER_ALL_TEAMS = 'true'; + process.env.DASHBOARD_PROVISIONER_INTERVAL = 'abc'; + await startDashboardProvisioner(); + expect(await Dashboard.countDocuments({ team: team._id })).toBe(0); + }); + + it('rejects interval below 1000ms', async () => { + const team = await createTeam({ name: 'My Team' }); + fs.writeFileSync( + path.join(tmpDir, 'test.json'), + JSON.stringify({ name: 'Noop', tiles: [makeTile()], tags: [] }), + ); + process.env.DASHBOARD_PROVISIONER_DIR = tmpDir; + process.env.DASHBOARD_PROVISIONER_ALL_TEAMS = 'true'; + process.env.DASHBOARD_PROVISIONER_INTERVAL = '500'; + await startDashboardProvisioner(); + expect(await Dashboard.countDocuments({ team: team._id })).toBe(0); + }); + + it('requires team ID or all-teams flag', async () => { + const team = await createTeam({ name: 'My Team' }); + fs.writeFileSync( + path.join(tmpDir, 'test.json'), + JSON.stringify({ name: 'Noop', tiles: [makeTile()], tags: [] }), + ); + process.env.DASHBOARD_PROVISIONER_DIR = tmpDir; + delete process.env.DASHBOARD_PROVISIONER_TEAM_ID; + delete process.env.DASHBOARD_PROVISIONER_ALL_TEAMS; + await startDashboardProvisioner(); + expect(await Dashboard.countDocuments({ team: team._id })).toBe(0); + }); + + it('rejects invalid team ID format', async () => { + const team = await createTeam({ name: 'My Team' }); + fs.writeFileSync( + path.join(tmpDir, 'test.json'), + JSON.stringify({ name: 'Noop', tiles: [makeTile()], tags: [] }), + ); + process.env.DASHBOARD_PROVISIONER_DIR = tmpDir; + process.env.DASHBOARD_PROVISIONER_TEAM_ID = 'not-an-objectid'; + await startDashboardProvisioner(); + expect(await Dashboard.countDocuments({ team: team._id })).toBe(0); + }); + + it('provisions dashboards for all teams', async () => { + const team = await createTeam({ name: 'My Team' }); + fs.writeFileSync( + path.join(tmpDir, 'test.json'), + JSON.stringify({ + name: 'All Teams Dash', + tiles: [makeTile()], + tags: [], + }), + ); + process.env.DASHBOARD_PROVISIONER_DIR = tmpDir; + process.env.DASHBOARD_PROVISIONER_ALL_TEAMS = 'true'; + process.env.DASHBOARD_PROVISIONER_INTERVAL = '60000'; + + await startDashboardProvisioner(); + + const count = await Dashboard.countDocuments({ team: team._id }); + expect(count).toBe(1); + }); + + it('provisions dashboards for a specific team', async () => { + const team = await createTeam({ name: 'My Team' }); + fs.writeFileSync( + path.join(tmpDir, 'test.json'), + JSON.stringify({ + name: 'Team Specific Dash', + tiles: [makeTile()], + tags: [], + }), + ); + process.env.DASHBOARD_PROVISIONER_DIR = tmpDir; + process.env.DASHBOARD_PROVISIONER_TEAM_ID = team._id.toString(); + process.env.DASHBOARD_PROVISIONER_INTERVAL = '60000'; + + await startDashboardProvisioner(); + + const count = await Dashboard.countDocuments({ team: team._id }); + expect(count).toBe(1); + }); + }); +}); diff --git a/packages/api/src/dashboardProvisioner.ts b/packages/api/src/dashboardProvisioner.ts new file mode 100644 index 0000000000..c249083889 --- /dev/null +++ b/packages/api/src/dashboardProvisioner.ts @@ -0,0 +1,190 @@ +import { + DashboardWithoutId, + DashboardWithoutIdSchema, +} from '@hyperdx/common-utils/dist/types'; +import fs from 'fs'; +import path from 'path'; + +import Dashboard from '@/models/dashboard'; +import Team from '@/models/team'; +import logger from '@/utils/logger'; + +let intervalHandle: ReturnType | null = null; +let isSyncing = false; + +export function readDashboardFiles(dir: string): DashboardWithoutId[] { + let files: string[]; + try { + files = fs.readdirSync(dir).filter(f => f.endsWith('.json')); + } catch (err) { + logger.error({ err, dir }, 'Failed to read dashboard directory'); + return []; + } + + const dashboards: DashboardWithoutId[] = []; + for (const file of files) { + try { + const raw = JSON.parse(fs.readFileSync(path.join(dir, file), 'utf8')); + const parsed = DashboardWithoutIdSchema.safeParse(raw); + if (!parsed.success) { + logger.warn( + { file, errors: parsed.error.issues }, + 'Skipping invalid dashboard file', + ); + continue; + } + dashboards.push(parsed.data); + } catch (err) { + logger.error({ err, file }, 'Failed to parse dashboard file'); + } + } + return dashboards; +} + +export async function syncDashboards(teamId: string, dir: string) { + const dashboards = readDashboardFiles(dir); + if (dashboards.length === 0) return; + + for (const dashboard of dashboards) { + try { + // Warn if a user-created dashboard with the same name exists + const userDashboard = await Dashboard.exists({ + name: dashboard.name, + team: teamId, + provisioned: { $ne: true }, + }); + if (userDashboard) { + logger.warn( + { name: dashboard.name }, + 'A user-created dashboard with this name already exists, provisioned copy will coexist', + ); + } + + // Only match provisioned dashboards to avoid overwriting user-created ones + const result = await Dashboard.findOneAndUpdate( + { name: dashboard.name, team: teamId, provisioned: true }, + { + $set: { + tiles: dashboard.tiles || [], + tags: dashboard.tags || [], + filters: dashboard.filters || [], + savedQuery: dashboard.savedQuery ?? null, + savedQueryLanguage: dashboard.savedQueryLanguage ?? null, + savedFilterValues: dashboard.savedFilterValues || [], + containers: dashboard.containers || [], + }, + $setOnInsert: { + name: dashboard.name, + team: teamId, + provisioned: true, + }, + }, + { upsert: true, new: false }, + ); + + if (result === null) { + logger.info({ name: dashboard.name }, 'Created provisioned dashboard'); + } + } catch (err) { + logger.error( + { err, name: dashboard.name }, + 'Failed to provision dashboard', + ); + } + } +} + +export async function startDashboardProvisioner() { + stopDashboardProvisioner(); + + const dir = process.env.DASHBOARD_PROVISIONER_DIR; + if (!dir) return; + + const teamId = process.env.DASHBOARD_PROVISIONER_TEAM_ID; + const provisionAllTeams = + process.env.DASHBOARD_PROVISIONER_ALL_TEAMS === 'true'; + const intervalMs = parseInt( + process.env.DASHBOARD_PROVISIONER_INTERVAL || '30000', + 10, + ); + + if (isNaN(intervalMs) || intervalMs < 1000) { + logger.error( + { value: process.env.DASHBOARD_PROVISIONER_INTERVAL }, + 'Invalid DASHBOARD_PROVISIONER_INTERVAL, must be >= 1000ms', + ); + return; + } + + if (!teamId && !provisionAllTeams) { + logger.error( + 'DASHBOARD_PROVISIONER_TEAM_ID is required (or set DASHBOARD_PROVISIONER_ALL_TEAMS=true)', + ); + return; + } + + if (teamId && !/^[0-9a-fA-F]{24}$/.test(teamId)) { + logger.error( + { teamId }, + 'DASHBOARD_PROVISIONER_TEAM_ID is not a valid ObjectId', + ); + return; + } + + if (!fs.existsSync(dir)) { + logger.warn( + { dir }, + 'Dashboard provisioner directory does not exist, waiting...', + ); + } + + logger.info( + { dir, intervalMs, teamId: teamId || (provisionAllTeams ? 'all' : 'none') }, + 'Dashboard provisioner started', + ); + + const run = async () => { + if (isSyncing) return; + isSyncing = true; + try { + if (!fs.existsSync(dir)) return; + + let teamIds: string[]; + if (teamId) { + const teamExists = await Team.exists({ _id: teamId }); + if (!teamExists) { + logger.warn( + { teamId }, + 'Configured team does not exist, skipping sync', + ); + return; + } + teamIds = [teamId]; + } else if (provisionAllTeams) { + const teams = await Team.find().select('_id').lean(); + teamIds = teams.map(t => t._id.toString()); + } else { + return; + } + + for (const id of teamIds) { + await syncDashboards(id, dir); + } + } catch (err) { + logger.error({ err }, 'Dashboard provisioner sync failed'); + } finally { + isSyncing = false; + } + }; + + await run(); + intervalHandle = setInterval(run, intervalMs); +} + +export function stopDashboardProvisioner() { + if (intervalHandle) { + clearInterval(intervalHandle); + intervalHandle = null; + } + isSyncing = false; +} diff --git a/packages/api/src/models/dashboard.ts b/packages/api/src/models/dashboard.ts index 8a6cfdf882..ac2c51f8b4 100644 --- a/packages/api/src/models/dashboard.ts +++ b/packages/api/src/models/dashboard.ts @@ -7,6 +7,7 @@ import type { ObjectId } from '.'; export interface IDashboard extends z.infer { _id: ObjectId; team: ObjectId; + provisioned: boolean; createdAt: Date; updatedAt: Date; } @@ -32,10 +33,14 @@ export default mongoose.model( savedQueryLanguage: { type: String, required: false }, savedFilterValues: { type: mongoose.Schema.Types.Array, required: false }, containers: { type: mongoose.Schema.Types.Array, required: false }, + provisioned: { type: Boolean, default: false }, }, { timestamps: true, toJSON: { getters: true }, }, + ).index( + { name: 1, team: 1 }, + { unique: true, partialFilterExpression: { provisioned: true } }, ), ); diff --git a/packages/api/src/server.ts b/packages/api/src/server.ts index c005618aef..d140185511 100644 --- a/packages/api/src/server.ts +++ b/packages/api/src/server.ts @@ -5,6 +5,10 @@ import { serializeError } from 'serialize-error'; import app from '@/api-app'; import * as config from '@/config'; import { LOCAL_APP_TEAM } from '@/controllers/team'; +import { + startDashboardProvisioner, + stopDashboardProvisioner, +} from '@/dashboardProvisioner'; import { connectDB, mongooseConnection } from '@/models'; import opampApp from '@/opamp/app'; import { setupTeamDefaults } from '@/setupDefaults'; @@ -26,6 +30,7 @@ export default class Server { protected async shutdown(signal?: string) { let hasError = false; + stopDashboardProvisioner(); logger.info('Closing all db clients...'); const [mongoCloseResult] = await Promise.allSettled([ mongooseConnection.close(false), @@ -105,5 +110,15 @@ export default class Server { // Don't throw - allow server to start even if defaults setup fails } } + + // Start file-based dashboard provisioner (if DASHBOARD_PROVISIONER_DIR is set) + try { + await startDashboardProvisioner(); + } catch (error) { + logger.error( + { err: serializeError(error) }, + 'Failed to start dashboard provisioner', + ); + } } }