chore: work on dev and test environment

.github/workflows/reuse-quality.yml

@@ -18,9 +18,53 @@ jobs:
     - name: Install dependencies
       run: npm ci
 
-    - name: Run quality checks
-      run: npm run quality
+    - name: Lint
+      run: npm run lint
 
-    - name: Docker logs in case of failure
+    - name: Build types
+      run: npm run build-types
+
+    - name: Type check
+      run: npm run check-types
+
+    - name: Build UI
+      run: npm -w ui run build
+
+    - name: Init .env
+      run: ./dev/init-env.sh
+
+    - name: Use localhost as DEV_HOST in CI
+      run: sed -i 's/^DEV_HOST=.*/DEV_HOST=localhost/' .env
+
+    - name: Start docker compose services
+      run: docker compose --profile dev up -d --wait
+
+    - name: Start dev API
+      run: npx dotenv -- npm run dev-api &
+
+    - name: Start dev worker
+      run: npx dotenv -- npm run dev-worker &
+
+    - name: Wait for dev API to be ready
+      run: |
+        set -a; source .env; set +a
+        for i in {1..30}; do
+          curl -sf "http://localhost:${DEV_API_PORT}/api/v1/_ping" && exit 0
+          sleep 2
+        done
+        echo "dev API did not become ready" >&2
+        exit 1
+
+    - name: Run tests
+      run: npm run test-unit && npm run test-api
+
+    - name: Audit
+      run: npm audit --omit=dev --audit-level=critical
+
+    - name: Docker compose logs on failure
       if: failure()
-      run: docker compose logs data-fair
+      run: docker compose --profile dev logs
+
+    - name: Stop docker compose services
+      if: always()
+      run: docker compose --profile dev down

.gitignore

@@ -1,5 +1,9 @@
 data/
+dev/logs/
 node_modules/
 **/config/local-*
 .config/
 .type/
+.env
+playwright-report/
+test-results/

.zellij.kdl

@@ -7,26 +7,26 @@ layout {
       }
       pane name="deps" {
         command "${DEV_SHELL}"
-        args "-ic" "npm run dev-deps && watch -n 4 \"docker compose ps --all --format 'table {{.Name}}\t{{.Status}}'\""
+        args "-ic" "npm run dev-deps"
       }
     }
     pane {
       split_direction "vertical"
       pane name="ui" {
         command "${DEV_SHELL}"
-        args "-ic" "nvm use > /dev/null 2>&1 && npm -w ui run dev"
+        args "-ic" "nvm use > /dev/null 2>&1 && npm run dev-ui"
       }
       pane name="api" {
         command "${DEV_SHELL}"
-        args "-ic" "nvm use > /dev/null 2>&1 && npm -w api run dev"
+        args "-ic" "nvm use > /dev/null 2>&1 && npm run dev-api"
       }
       pane name="worker" {
         command "${DEV_SHELL}"
-        args "-ic" "nvm use > /dev/null 2>&1 && npm -w worker run dev"
+        args "-ic" "nvm use > /dev/null 2>&1 && npm run dev-worker"
       }
     }
     pane size=1 borderless=true {
         command "bash"
-        args "-ic" "echo -n -e \"Dev server available at \\e[1;96mhttp://localhost:5600\\033[0m\""
+        args "-ic" "echo -n -e \"Dev server available at \\e[1;96mhttp://${DEV_HOST}:${NGINX_PORT1}/processings\\033[0m\""
     }
 }

AGENTS.md

@@ -0,0 +1,113 @@
+# Data Fair Processings — Agent Guidelines
+
+## Project Overview
+
+`@data-fair/processings` is a sister service of `data-fair`. It runs scheduled and on-demand
+processings (NPM-installable plugins) that ingest, transform and publish datasets into a
+running `data-fair` instance. The project is a monorepo with four workspaces:
+
+- `api/` — Express REST API
+- `worker/` — Polling worker that runs the processings tasks in child processes
+- `ui/` — Vue 3 + Vuetify SPA (built with Vite)
+- `shared/` — TypeScript types and helpers shared between api, worker and ui
+
+## Dev environment
+
+The dev environment is managed by **zellij** (terminal multiplexer) and **docker compose**.
+**Never start, stop, or restart dev processes yourself** — the user manages them through
+zellij panes (`npm run dev-zellij`).
+
+The dev environment also serves as the test environment: the Playwright test suite hits the
+running dev API, dev worker and dev UI. There is no separate test server.
+
+Each git worktree gets its own `.env` (random ports + `<branch>.localhost` subdomain) so
+multiple worktrees coexist without port collisions. Use the worktree scripts:
+
+```bash
+./dev/worktree.sh feat-xyz       # create ../processings_feat-xyz with a fresh .env
+./dev/delete-worktree.sh feat-xyz # tear down docker compose and remove the worktree
+```
+
+### Checking status
+
+```bash
+bash dev/status.sh
+```
+
+Shows the health of all services (nginx, dev API, dev UI, dev worker, simple-directory,
+data-fair upstream, events, openapi-viewer, mongo, elasticsearch) and lists log files
+with sizes and timestamps.
+
+### Log files
+
+All dev processes write to `dev/logs/`:
+- `dev-api.log` — API server
+- `dev-worker.log` — worker process
+- `dev-ui.log` — UI dev server (Vite)
+- `docker-compose.log` — all docker compose services
+
+### Troubleshooting
+
+1. Run `bash dev/status.sh` to identify which services are down
+2. Read the relevant log file in `dev/logs/` for error details
+3. Report findings to the user — do not attempt to fix infrastructure issues yourself
+
+### Port assignments
+
+Port numbers and the `DEV_HOST` subdomain live in `.env` (gitignored, generated by
+`dev/init-env.sh`). Do not modify port assignments by hand.
+
+### Test users
+
+Test fixtures live in `dev/resources/users.json` and `dev/resources/organizations.json`.
+All test users and orgs are prefixed with `test_` (e.g. `test_admin1@test.com`,
+`test_org1`). The cleanup endpoint `DELETE /api/v1/test-env` only purges documents
+whose `owner.id` matches `^test_` so that interactive dev work under non-prefixed
+accounts (e.g. `superadmin`, `albanm`) survives test runs.
+
+Special accounts:
+- `test_superadmin@test.com` / password `superpasswd` — superadmin
+- all other `test_*@test.com` — password `passwd`
+
+### Test-env API
+
+When `NODE_ENV=development` the API exposes `/api/v1/test-env`:
+- `DELETE /` — delete all `test_*`-owned processings, runs and limits
+- `GET /pending-tasks` — list runs in `triggered`/`scheduled`/`running` status
+- `GET /raw-processing/:id`, `GET /raw-run/:id` — raw mongo docs
+- `POST /set-env`, `POST /set-config` — runtime overrides for testing
+- `DELETE /plugins` — wipe the installed plugins directory
+
+These endpoints are used by the Playwright support helpers in `tests/support/axios.ts`.
+
+### Testing
+
+```bash
+npm test                          # all tests
+npm run test-unit                 # unit tests only
+npm run test-api                  # API tests only
+npm run test-e2e                  # e2e tests only
+npx playwright test path/to/file  # specific file
+```
+
+Tests are organized under `tests/features/<topic>/<name>.{unit,api,e2e}.spec.ts`.
+The `state-setup` project pings `/api/v1/test-env/pending-tasks` and tails the dev API
+and worker logs into the test reporter output.
+
+The full test suite is long — when iterating on changes always run only the related
+test cases. The full suite runs on `git push` via husky.
+
+### Linting & Type Checking
+
+```bash
+npm run lint             # ESLint (root + ui workspace)
+npm run lint-fix         # auto-fix
+npm run check-types      # TypeScript
+```
+
+### Building
+
+```bash
+npm run build-types      # generate type definitions for shared schemas
+npm -w ui run build      # build the Vue SPA
+```

CLAUDE.md

@@ -0,0 +1 @@
+@AGENTS.md

api/config/development.mjs

@@ -1,13 +1,19 @@
+const apiPort = parseInt(process.env.DEV_API_PORT ?? '8082')
+const mongoPort = process.env.MONGO_PORT ?? '27017'
+const sdPort = process.env.SD_PORT ?? '8080'
+const eventsPort = process.env.EVENTS_PORT ?? '8083'
+const observerPort = parseInt(process.env.DEV_API_OBSERVER_PORT ?? '9092')
+
 export default {
   cipherPassword: 'dev',
   dataDir: '../data/development',
-  mongoUrl: 'mongodb://localhost:27017/data-fair-processings-development',
+  mongoUrl: `mongodb://localhost:${mongoPort}/data-fair-processings-development`,
   observer: {
-    port: 9092
+    port: observerPort
   },
-  port: 8082,
-  privateDirectoryUrl: 'http://localhost:8080',
-  privateEventsUrl: 'http://localhost:8083',
+  port: apiPort,
+  privateDirectoryUrl: `http://localhost:${sdPort}`,
+  privateEventsUrl: `http://localhost:${eventsPort}`,
   secretKeys: {
     identities: 'secret-identities',
     events: 'secret-events'

api/package.json

@@ -3,7 +3,7 @@
   "main": "index.ts",
   "type": "module",
   "scripts": {
-    "dev": "NODE_ENV=development DEBUG=upgrade node --watch index.ts"
+    "dev": "NODE_ENV=development DEBUG=upgrade node --env-file-if-exists=../.env --watch index.ts"
   },
   "imports": {
     "#config": "./src/config.ts",

api/src/app.ts

@@ -23,6 +23,10 @@ app.set('json spaces', 2)
 app.use(express.json())
 app.use(express.urlencoded({ extended: true }))
 
+app.get('/api/v1/_ping', (req, res) => {
+  res.send('ok')
+})
+
 app.use('/api/identities', identitiesRouter)
 app.use('/api/v1/plugins-registry', pluginsRegistryRouter)
 app.use('/api/v1/plugins', pluginsRouter)
@@ -31,7 +35,12 @@ app.use('/api/v1/runs', runsRouter)
 app.use('/api/v1/limits', limitsRouter)
 app.use('/api/v1/admin', adminRouter)
 
-if (process.env.NODE_ENV !== 'test') {
+if (process.env.NODE_ENV === 'development') {
+  app.use('/api/v1/test-env', (await import('./misc/routers/test-env.ts')).default)
+}
+
+if (process.env.NODE_ENV !== 'development') {
+  // in development the UI is served by the Vite dev server through nginx
   const cspDirectives = { ...defaultNonceCSPDirectives }
   // necessary to use vjsf without pre-compilation
   cspDirectives['script-src'] = "'unsafe-eval' " + defaultNonceCSPDirectives['script-src']

api/src/config.ts

@@ -2,12 +2,10 @@ import type { ApiConfig } from '../config/type/index.ts'
 import { assertValid } from '../config/type/index.ts'
 import config from 'config'
 
-// we reload the config instead of using the singleton from the config module for testing purposes
-// @ts-ignore
-const apiConfig = process.env.NODE_ENV === 'test' ? config.util.loadFileConfigs(process.env.NODE_CONFIG_DIR, { skipConfigSources: true }) : config
-assertValid(apiConfig, { lang: 'en', name: 'config', internal: true })
+assertValid(config, { lang: 'en', name: 'config', internal: true })
 
-export default apiConfig as ApiConfig
+const apiConfig = config as unknown as ApiConfig
+export default apiConfig
 
 export const uiConfig = {
   pluginCategories: apiConfig.pluginCategories,