feat: Add file-based dashboard provisioner

.changeset/dashboard-provisioner.md

@@ -0,0 +1,5 @@
+---
+"@hyperdx/api": minor
+---
+
+feat: add file-based dashboard provisioner that watches a directory for JSON files and upserts dashboards into MongoDB

packages/api/docs/auto_provision/AUTO_PROVISION.md

@@ -163,6 +163,62 @@ services:
 For more complex configurations, you can use environment files or Docker secrets
 to manage these values.
 
+## Dashboard Provisioning
+
+HyperDX supports file-based dashboard provisioning, similar to Grafana's
+provisioning system. A background process watches a directory for `.json` files
+and upserts dashboards into MongoDB, matched by name for idempotency.
+
+### Environment Variables
+
+| Variable                            | Required | Default | Description                                                     |
+| ----------------------------------- | -------- | ------- | --------------------------------------------------------------- |
+| `DASHBOARD_PROVISIONER_DIR`         | Yes      | —       | Directory to watch for `.json` dashboard files                  |
+| `DASHBOARD_PROVISIONER_INTERVAL`    | No       | `30000` | Sync interval in milliseconds (minimum 1000)                    |
+| `DASHBOARD_PROVISIONER_TEAM_ID`     | No\*     | —       | Scope provisioning to a specific team ID                        |
+| `DASHBOARD_PROVISIONER_ALL_TEAMS`   | No\*     | `false` | Set to `true` to provision dashboards to all teams              |
+
+\*One of `DASHBOARD_PROVISIONER_TEAM_ID` or `DASHBOARD_PROVISIONER_ALL_TEAMS=true`
+is required when `DASHBOARD_PROVISIONER_DIR` is set.
+
+### Dashboard JSON Format
+
+Each `.json` file in the provisioner directory should contain a dashboard object
+with at minimum a `name` and `tiles` array:
+
+```json
+{
+  "name": "My Dashboard",
+  "tiles": [
+    {
+      "id": "tile-1",
+      "x": 0,
+      "y": 0,
+      "w": 6,
+      "h": 4,
+      "config": {
+        "name": "Request Count",
+        "source": "Metrics",
+        "displayType": "line",
+        "select": [{ "aggFn": "count" }]
+      }
+    }
+  ],
+  "tags": ["provisioned"]
+}
+```
+
+### Behavior
+
+- Dashboards are matched by name and team for idempotency
+- Provisioned dashboards are flagged with `provisioned: true` so they never
+  overwrite user-created dashboards with the same name
+- Removing a file from the directory does **not** delete the dashboard from
+  MongoDB (safe by default)
+- Files are validated against the `DashboardWithoutIdSchema` Zod schema; invalid
+  files are skipped with a warning
+
+
 ## Note on Security
 
 While this feature is convenient for development and initial setup, be careful

packages/api/src/__tests__/dashboardProvisioner.test.ts

@@ -0,0 +1,314 @@
+import fs from 'fs';
+import os from 'os';
+import path from 'path';
+
+import { createTeam } from '@/controllers/team';
+import {
+  readDashboardFiles,
+  startDashboardProvisioner,
+  stopDashboardProvisioner,
+  syncDashboards,
+} from '@/dashboardProvisioner';
+import { clearDBCollections, closeDB, connectDB, makeTile } from '@/fixtures';
+import Dashboard from '@/models/dashboard';
+
+describe('dashboardProvisioner', () => {
+  let tmpDir: string;
+
+  beforeAll(async () => {
+    await connectDB();
+  });
+
+  beforeEach(() => {
+    tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'hdx-dash-test-'));
+  });
+
+  afterEach(async () => {
+    fs.rmSync(tmpDir, { recursive: true, force: true });
+    await clearDBCollections();
+  });
+
+  afterAll(async () => {
+    await closeDB();
+  });
+
+  describe('readDashboardFiles', () => {
+    it('returns empty array for non-existent directory', () => {
+      const result = readDashboardFiles('/non/existent/path');
+      expect(result).toEqual([]);
+    });
+
+    it('returns empty array for directory with no json files', () => {
+      fs.writeFileSync(path.join(tmpDir, 'readme.txt'), 'not a dashboard');
+      const result = readDashboardFiles(tmpDir);
+      expect(result).toEqual([]);
+    });
+
+    it('skips invalid JSON files', () => {
+      fs.writeFileSync(path.join(tmpDir, 'bad.json'), '{invalid json');
+      const result = readDashboardFiles(tmpDir);
+      expect(result).toEqual([]);
+    });
+
+    it('skips files that fail schema validation', () => {
+      fs.writeFileSync(
+        path.join(tmpDir, 'no-name.json'),
+        JSON.stringify({ tiles: [] }),
+      );
+      const result = readDashboardFiles(tmpDir);
+      expect(result).toEqual([]);
+    });
+
+    it('parses valid dashboard files', () => {
+      fs.writeFileSync(
+        path.join(tmpDir, 'test.json'),
+        JSON.stringify({ name: 'Test', tiles: [makeTile()], tags: [] }),
+      );
+      const result = readDashboardFiles(tmpDir);
+      expect(result).toHaveLength(1);
+      expect(result[0].name).toBe('Test');
+    });
+  });
+
+  describe('syncDashboards', () => {
+    it('creates a new dashboard', async () => {
+      const team = await createTeam({ name: 'My Team' });
+      fs.writeFileSync(
+        path.join(tmpDir, 'test.json'),
+        JSON.stringify({
+          name: 'New Dashboard',
+          tiles: [makeTile()],
+          tags: [],
+        }),
+      );
+
+      await syncDashboards(team._id.toString(), tmpDir);
+
+      const count = await Dashboard.countDocuments({ team: team._id });
+      expect(count).toBe(1);
+    });
+
+    it('updates an existing dashboard by name', async () => {
+      const team = await createTeam({ name: 'My Team' });
+      const tile = makeTile();
+      await new Dashboard({
+        name: 'Existing',
+        tiles: [tile],
+        tags: [],
+        team: team._id,
+        provisioned: true,
+      }).save();
+
+      const newTile = makeTile();
+      fs.writeFileSync(
+        path.join(tmpDir, 'existing.json'),
+        JSON.stringify({
+          name: 'Existing',
+          tiles: [newTile],
+          tags: ['updated'],
+        }),
+      );
+
+      await syncDashboards(team._id.toString(), tmpDir);
+
+      const dashboard = (await Dashboard.findOne({
+        name: 'Existing',
+        team: team._id,
+      })) as any;
+      expect(dashboard.tiles[0].id).toBe(newTile.id);
+      expect(dashboard.tags).toEqual(['updated']);
+    });
+
+    it('does not create duplicates on repeated sync', async () => {
+      const team = await createTeam({ name: 'My Team' });
+      fs.writeFileSync(
+        path.join(tmpDir, 'test.json'),
+        JSON.stringify({ name: 'Dashboard', tiles: [makeTile()], tags: [] }),
+      );
+
+      await syncDashboards(team._id.toString(), tmpDir);
+      await syncDashboards(team._id.toString(), tmpDir);
+      await syncDashboards(team._id.toString(), tmpDir);
+
+      const count = await Dashboard.countDocuments({ team: team._id });
+      expect(count).toBe(1);
+    });
+
+    it('provisions to multiple teams', async () => {
+      const teamA = await createTeam({ name: 'Team A' });
+      const teamB = await createTeam({ name: 'Team B' });
+      fs.writeFileSync(
+        path.join(tmpDir, 'shared.json'),
+        JSON.stringify({
+          name: 'Shared Dashboard',
+          tiles: [makeTile()],
+          tags: [],
+        }),
+      );
+
+      await syncDashboards(teamA._id.toString(), tmpDir);
+      await syncDashboards(teamB._id.toString(), tmpDir);
+
+      expect(await Dashboard.countDocuments({ team: teamA._id })).toBe(1);
+      expect(await Dashboard.countDocuments({ team: teamB._id })).toBe(1);
+    });
+
+    it('does not overwrite user-created dashboards', async () => {
+      const team = await createTeam({ name: 'My Team' });
+      const userTile = makeTile();
+      await new Dashboard({
+        name: 'My Dashboard',
+        tiles: [userTile],
+        tags: ['user-tag'],
+        team: team._id,
+        // provisioned defaults to false
+      }).save();
+
+      fs.writeFileSync(
+        path.join(tmpDir, 'my-dashboard.json'),
+        JSON.stringify({
+          name: 'My Dashboard',
+          tiles: [makeTile()],
+          tags: ['provisioned-tag'],
+        }),
+      );
+
+      await syncDashboards(team._id.toString(), tmpDir);
+
+      // User-created dashboard is untouched
+      const userDashboard = (await Dashboard.findOne({
+        name: 'My Dashboard',
+        team: team._id,
+        provisioned: { $ne: true },
+      })) as any;
+      expect(userDashboard).toBeTruthy();
+      expect(userDashboard.tiles[0].id).toBe(userTile.id);
+      expect(userDashboard.tags).toEqual(['user-tag']);
+
+      // Provisioned copy was created alongside it
+      const provisionedDashboard = await Dashboard.findOne({
+        name: 'My Dashboard',
+        team: team._id,
+        provisioned: true,
+      });
+      expect(provisionedDashboard).toBeTruthy();
+    });
+  });
+
+  describe('startDashboardProvisioner', () => {
+    const originalEnv = process.env;
+
+    beforeEach(() => {
+      process.env = { ...originalEnv };
+      stopDashboardProvisioner();
+    });
+
+    afterEach(() => {
+      stopDashboardProvisioner();
+      process.env = originalEnv;
+    });
+
+    it('is a no-op when DASHBOARD_PROVISIONER_DIR is not set', async () => {
+      const team = await createTeam({ name: 'My Team' });
+      fs.writeFileSync(
+        path.join(tmpDir, 'test.json'),
+        JSON.stringify({ name: 'Noop', tiles: [makeTile()], tags: [] }),
+      );
+      delete process.env.DASHBOARD_PROVISIONER_DIR;
+      process.env.DASHBOARD_PROVISIONER_ALL_TEAMS = 'true';
+      await startDashboardProvisioner();
+      expect(await Dashboard.countDocuments({ team: team._id })).toBe(0);
+    });
+
+    it('rejects invalid interval', async () => {
+      const team = await createTeam({ name: 'My Team' });
+      fs.writeFileSync(
+        path.join(tmpDir, 'test.json'),
+        JSON.stringify({ name: 'Noop', tiles: [makeTile()], tags: [] }),
+      );
+      process.env.DASHBOARD_PROVISIONER_DIR = tmpDir;
+      process.env.DASHBOARD_PROVISIONER_ALL_TEAMS = 'true';
+      process.env.DASHBOARD_PROVISIONER_INTERVAL = 'abc';
+      await startDashboardProvisioner();
+      expect(await Dashboard.countDocuments({ team: team._id })).toBe(0);
+    });
+
+    it('rejects interval below 1000ms', async () => {
+      const team = await createTeam({ name: 'My Team' });
+      fs.writeFileSync(
+        path.join(tmpDir, 'test.json'),
+        JSON.stringify({ name: 'Noop', tiles: [makeTile()], tags: [] }),
+      );
+      process.env.DASHBOARD_PROVISIONER_DIR = tmpDir;
+      process.env.DASHBOARD_PROVISIONER_ALL_TEAMS = 'true';
+      process.env.DASHBOARD_PROVISIONER_INTERVAL = '500';
+      await startDashboardProvisioner();
+      expect(await Dashboard.countDocuments({ team: team._id })).toBe(0);
+    });
+
+    it('requires team ID or all-teams flag', async () => {
+      const team = await createTeam({ name: 'My Team' });
+      fs.writeFileSync(
+        path.join(tmpDir, 'test.json'),
+        JSON.stringify({ name: 'Noop', tiles: [makeTile()], tags: [] }),
+      );
+      process.env.DASHBOARD_PROVISIONER_DIR = tmpDir;
+      delete process.env.DASHBOARD_PROVISIONER_TEAM_ID;
+      delete process.env.DASHBOARD_PROVISIONER_ALL_TEAMS;
+      await startDashboardProvisioner();
+      expect(await Dashboard.countDocuments({ team: team._id })).toBe(0);
+    });
+
+    it('rejects invalid team ID format', async () => {
+      const team = await createTeam({ name: 'My Team' });
+      fs.writeFileSync(
+        path.join(tmpDir, 'test.json'),
+        JSON.stringify({ name: 'Noop', tiles: [makeTile()], tags: [] }),
+      );
+      process.env.DASHBOARD_PROVISIONER_DIR = tmpDir;
+      process.env.DASHBOARD_PROVISIONER_TEAM_ID = 'not-an-objectid';
+      await startDashboardProvisioner();
+      expect(await Dashboard.countDocuments({ team: team._id })).toBe(0);
+    });
+
+    it('provisions dashboards for all teams', async () => {
+      const team = await createTeam({ name: 'My Team' });
+      fs.writeFileSync(
+        path.join(tmpDir, 'test.json'),
+        JSON.stringify({
+          name: 'All Teams Dash',
+          tiles: [makeTile()],
+          tags: [],
+        }),
+      );
+      process.env.DASHBOARD_PROVISIONER_DIR = tmpDir;
+      process.env.DASHBOARD_PROVISIONER_ALL_TEAMS = 'true';
+      process.env.DASHBOARD_PROVISIONER_INTERVAL = '60000';
+
+      await startDashboardProvisioner();
+
+      const count = await Dashboard.countDocuments({ team: team._id });
+      expect(count).toBe(1);
+    });
+
+    it('provisions dashboards for a specific team', async () => {
+      const team = await createTeam({ name: 'My Team' });
+      fs.writeFileSync(
+        path.join(tmpDir, 'test.json'),
+        JSON.stringify({
+          name: 'Team Specific Dash',
+          tiles: [makeTile()],
+          tags: [],
+        }),
+      );
+      process.env.DASHBOARD_PROVISIONER_DIR = tmpDir;
+      process.env.DASHBOARD_PROVISIONER_TEAM_ID = team._id.toString();
+      process.env.DASHBOARD_PROVISIONER_INTERVAL = '60000';
+
+      await startDashboardProvisioner();
+
+      const count = await Dashboard.countDocuments({ team: team._id });
+      expect(count).toBe(1);
+    });
+  });
+});