From 77a5dcfe58d0a0111fd5a1e7d0f4689e0d1b5b27 Mon Sep 17 00:00:00 2001 From: Warren Lee <5959690+wrn14897@users.noreply.github.com> Date: Wed, 1 Apr 2026 10:55:38 -0700 Subject: [PATCH] feat: fix dev env port resolution and improve multi-worktree support Fix OTel collector endpoint resolution and add multi-worktree improvements: - Fix dotenv-expand infinite recursion from self-referential ${VAR:-default} patterns in .env files that prevented port isolation from working - Fix HDX_COLLECTOR_URL fallback for browser-side OTel telemetry - Set correct HYPERDX_API_KEY in app .env.development - Add slot-specific session cookie names (connect.sid.) so multiple worktrees on localhost don't overwrite each other's sessions - Share NX build cache across worktrees via NX_CACHE_DIRECTORY - Add worktrunk project config (.config/wt.toml) for worktree lifecycle hooks: symlink node_modules, copy .env.local, clean up Docker on remove --- .config/wt.toml | 27 +++++++++++++++++++++++++++ .env | 16 ++++++++-------- packages/api/.env.development | 14 ++++---------- packages/api/src/api-app.ts | 5 +++++ packages/app/.env.development | 8 +++----- packages/app/src/config.ts | 1 + scripts/dev-env.sh | 7 +++++++ 7 files changed, 55 insertions(+), 23 deletions(-) create mode 100644 .config/wt.toml diff --git a/.config/wt.toml b/.config/wt.toml new file mode 100644 index 0000000000..1ea3d91cce --- /dev/null +++ b/.config/wt.toml @@ -0,0 +1,27 @@ +# HyperDX worktrunk project config +# See: https://worktrunk.dev +# +# This file is committed and shared with the team. It configures lifecycle +# hooks that run when creating/switching/removing worktrees via `wt`. + +# --------------------------------------------------------------------------- +# pre-start: blocking — symlink node_modules from primary worktree for +# instant dependency access. If the branch needs different deps: +# rm node_modules && yarn install +# --------------------------------------------------------------------------- +pre-start = "ln -sf {{ primary_worktree_path }}/node_modules {{ worktree_path }}/node_modules" + +# --------------------------------------------------------------------------- +# post-start: background tasks +# --------------------------------------------------------------------------- +[post-start] +env = "cp {{ primary_worktree_path }}/.env.local {{ worktree_path }}/.env.local 2>/dev/null && echo 'Copied .env.local' || true" + +# --------------------------------------------------------------------------- +# post-remove: clean up all Docker resources for this worktree's slot +# (dev stack, integration tests, E2E tests) +# --------------------------------------------------------------------------- +[post-remove] +dev-down = "make dev-down 2>/dev/null || true" +int-down = "make dev-int-down 2>/dev/null || true" +e2e-down = "make dev-e2e-down 2>/dev/null || true" diff --git a/.env b/.env index a5fdc782c9..afcd2de1e8 100644 --- a/.env +++ b/.env @@ -29,14 +29,14 @@ HYPERDX_OPAMP_PORT=${HYPERDX_OPAMP_PORT:-4320} HYPERDX_BASE_PATH= # Docker service ports (overridden by scripts/dev-env.sh for isolation) -HDX_DEV_MONGO_PORT=${HDX_DEV_MONGO_PORT:-27017} -HDX_DEV_CH_HTTP_PORT=${HDX_DEV_CH_HTTP_PORT:-8123} -HDX_DEV_CH_NATIVE_PORT=${HDX_DEV_CH_NATIVE_PORT:-9000} -HDX_DEV_OTEL_HEALTH_PORT=${HDX_DEV_OTEL_HEALTH_PORT:-13133} -HDX_DEV_OTEL_GRPC_PORT=${HDX_DEV_OTEL_GRPC_PORT:-4317} -HDX_DEV_OTEL_HTTP_PORT=${HDX_DEV_OTEL_HTTP_PORT:-4318} -HDX_DEV_OTEL_METRICS_PORT=${HDX_DEV_OTEL_METRICS_PORT:-8888} -HDX_DEV_OTEL_JSON_HTTP_PORT=${HDX_DEV_OTEL_JSON_HTTP_PORT:-14318} +HDX_DEV_MONGO_PORT=27017 +HDX_DEV_CH_HTTP_PORT=8123 +HDX_DEV_CH_NATIVE_PORT=9000 +HDX_DEV_OTEL_HEALTH_PORT=13133 +HDX_DEV_OTEL_GRPC_PORT=4317 +HDX_DEV_OTEL_HTTP_PORT=4318 +HDX_DEV_OTEL_METRICS_PORT=8888 +HDX_DEV_OTEL_JSON_HTTP_PORT=14318 # Otel/Clickhouse config HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE=default diff --git a/packages/api/.env.development b/packages/api/.env.development index 6164e4e93d..52421a6c66 100644 --- a/packages/api/.env.development +++ b/packages/api/.env.development @@ -1,29 +1,23 @@ -HYPERDX_API_KEY="xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -# Ports are overridden by scripts/dev-env.sh for worktree isolation -HYPERDX_API_PORT=${HYPERDX_API_PORT:-8000} -HYPERDX_OPAMP_PORT=${HYPERDX_OPAMP_PORT:-4320} -HYPERDX_APP_PORT=${HYPERDX_APP_PORT:-8080} +# Ports come from scripts/dev-env.sh (worktree isolation) or root .env (defaults) HYPERDX_LOG_LEVEL=debug EXPRESS_SESSION_SECRET="hyperdx is cool 👋" FRONTEND_URL="http://localhost:${HYPERDX_APP_PORT}" HDX_NODE_ADVANCED_NETWORK_CAPTURE=1 HDX_NODE_BETA_MODE=1 HDX_NODE_CONSOLE_CAPTURE=1 -HYPERDX_API_KEY=${HYPERDX_API_KEY} -HYPERDX_LOG_LEVEL=${HYPERDX_LOG_LEVEL} MINER_API_URL="http://localhost:5123" -MONGO_URI="mongodb://localhost:${HDX_DEV_MONGO_PORT:-27017}/hyperdx" +MONGO_URI="mongodb://localhost:${HDX_DEV_MONGO_PORT}/hyperdx" NODE_ENV=development OTEL_SERVICE_NAME="hdx-oss-dev-api" OTEL_RESOURCE_ATTRIBUTES="service.version=dev" -OTEL_EXPORTER_OTLP_ENDPOINT="http://localhost:${HDX_DEV_OTEL_HTTP_PORT:-4318}" +OTEL_EXPORTER_OTLP_ENDPOINT="http://localhost:${HDX_DEV_OTEL_HTTP_PORT}" PORT=${HYPERDX_API_PORT} OPAMP_PORT=${HYPERDX_OPAMP_PORT} REDIS_URL=redis://localhost:6379 USAGE_STATS_ENABLED=false NODE_OPTIONS="--max-http-header-size=131072" ENABLE_SWAGGER=true -DEFAULT_CONNECTIONS=[{"name":"Local ClickHouse","host":"http://localhost:${HDX_DEV_CH_HTTP_PORT:-8123}","username":"default","password":""}] +DEFAULT_CONNECTIONS=[{"name":"Local ClickHouse","host":"http://localhost:${HDX_DEV_CH_HTTP_PORT}","username":"default","password":""}] DEFAULT_SOURCES=[{"from":{"databaseName":"default","tableName":"otel_logs"},"kind":"log","timestampValueExpression":"TimestampTime","name":"Logs","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local ClickHouse","traceSourceId":"Traces","sessionSourceId":"Sessions","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":"otel_traces"},"kind":"trace","timestampValueExpression":"Timestamp","name":"Traces","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"SpanName","serviceNameExpression":"ServiceName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,StatusCode,round(Duration/1e6),SpanName","traceIdExpression":"TraceId","spanIdExpression":"SpanId","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanNameExpression":"SpanName","spanKindExpression":"SpanKind","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage","connection":"Local ClickHouse","logSourceId":"Logs","sessionSourceId":"Sessions","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":""},"kind":"metric","timestampValueExpression":"TimeUnix","name":"Metrics","resourceAttributesExpression":"ResourceAttributes","metricTables":{"gauge":"otel_metrics_gauge","histogram":"otel_metrics_histogram","sum":"otel_metrics_sum","_id":"682586a8b1f81924e628e808","id":"682586a8b1f81924e628e808"},"connection":"Local ClickHouse","logSourceId":"Logs","traceSourceId":"Traces","sessionSourceId":"Sessions"},{"from":{"databaseName":"default","tableName":"hyperdx_sessions"},"kind":"session","timestampValueExpression":"TimestampTime","name":"Sessions","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local ClickHouse","logSourceId":"Logs","traceSourceId":"Traces","metricSourceId":"Metrics"},{"from":{"databaseName":"otel_json","tableName":"otel_logs"},"kind":"log","timestampValueExpression":"Timestamp","name":"JSON Logs","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local ClickHouse","traceSourceId":"JSON Traces","metricSourceId":"JSON Metrics"},{"from":{"databaseName":"otel_json","tableName":"otel_traces"},"kind":"trace","timestampValueExpression":"Timestamp","name":"JSON Traces","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"SpanName","serviceNameExpression":"ServiceName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,StatusCode,round(Duration/1e6),SpanName","traceIdExpression":"TraceId","spanIdExpression":"SpanId","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanNameExpression":"SpanName","spanKindExpression":"SpanKind","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage","connection":"Local ClickHouse","logSourceId":"JSON Logs","metricSourceId":"JSON Metrics"},{"from":{"databaseName":"otel_json","tableName":""},"kind":"metric","timestampValueExpression":"TimeUnix","name":"JSON Metrics","resourceAttributesExpression":"ResourceAttributes","metricTables":{"gauge":"otel_metrics_gauge","histogram":"otel_metrics_histogram","sum":"otel_metrics_sum"},"connection":"Local ClickHouse","logSourceId":"JSON Logs","traceSourceId":"JSON Traces"}] INGESTION_API_KEY="super-secure-ingestion-api-key" HYPERDX_API_KEY=$INGESTION_API_KEY diff --git a/packages/api/src/api-app.ts b/packages/api/src/api-app.ts index 0e9735542b..7102d31384 100644 --- a/packages/api/src/api-app.ts +++ b/packages/api/src/api-app.ts @@ -22,6 +22,11 @@ import passport from './utils/passport'; const app: express.Application = express(); const sess: session.SessionOptions & { cookie: session.CookieOptions } = { + // Use a slot-specific cookie name in dev so multiple worktrees on localhost + // don't overwrite each other's session cookies. + ...(config.IS_DEV && process.env.HDX_DEV_SLOT + ? { name: `connect.sid.${process.env.HDX_DEV_SLOT}` } + : {}), resave: false, saveUninitialized: false, secret: config.EXPRESS_SESSION_SECRET, diff --git a/packages/app/.env.development b/packages/app/.env.development index 364b271409..c88261551e 100644 --- a/packages/app/.env.development +++ b/packages/app/.env.development @@ -1,10 +1,8 @@ -HYPERDX_API_KEY="xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -# Ports are overridden by scripts/dev-env.sh for worktree isolation -HYPERDX_API_PORT=${HYPERDX_API_PORT:-8000} -HYPERDX_APP_PORT=${HYPERDX_APP_PORT:-8080} +HYPERDX_API_KEY="super-secure-ingestion-api-key" +# Ports come from scripts/dev-env.sh (worktree isolation) or root .env (defaults) SERVER_URL="http://localhost:${HYPERDX_API_PORT}" NODE_ENV=development -OTEL_EXPORTER_OTLP_ENDPOINT="http://localhost:${HDX_DEV_OTEL_HTTP_PORT:-4318}" +OTEL_EXPORTER_OTLP_ENDPOINT="http://localhost:${HDX_DEV_OTEL_HTTP_PORT}" OTEL_SERVICE_NAME="hdx-oss-dev-app" PORT=${HYPERDX_APP_PORT} NODE_OPTIONS="--max-http-header-size=131072" diff --git a/packages/app/src/config.ts b/packages/app/src/config.ts index 5d7a0f3378..36fca96c1c 100644 --- a/packages/app/src/config.ts +++ b/packages/app/src/config.ts @@ -16,6 +16,7 @@ export const HDX_EXPORTER_ENABLED = (process.env.HDX_EXPORTER_ENABLED ?? 'true') === 'true'; export const HDX_COLLECTOR_URL = process.env.NEXT_PUBLIC_OTEL_EXPORTER_OTLP_ENDPOINT ?? + process.env.OTEL_EXPORTER_OTLP_ENDPOINT ?? 'http://localhost:4318'; export const IS_DEV = NODE_ENV === 'development'; diff --git a/scripts/dev-env.sh b/scripts/dev-env.sh index 1c1da742af..938aaeb4d7 100755 --- a/scripts/dev-env.sh +++ b/scripts/dev-env.sh @@ -59,6 +59,12 @@ HDX_DEV_OTEL_JSON_HTTP_PORT=$((31100 + HDX_DEV_SLOT)) # --- Docker Compose project name (unique per slot) --- HDX_DEV_PROJECT="hdx-dev-${HDX_DEV_SLOT}" +# --- Shared NX build cache across all worktrees --- +# NX cache is content-hash based so changed files get cache misses (correct +# behavior). Unchanged packages reuse cached output regardless of worktree. +NX_CACHE_DIRECTORY="${HOME}/.config/hyperdx/nx-cache" +mkdir -p "$NX_CACHE_DIRECTORY" + # Export everything export HDX_DEV_SLOT export HDX_DEV_BRANCH @@ -75,6 +81,7 @@ export HDX_DEV_OTEL_HTTP_PORT export HDX_DEV_OTEL_METRICS_PORT export HDX_DEV_OTEL_JSON_HTTP_PORT export HDX_DEV_PROJECT +export NX_CACHE_DIRECTORY # --- Clean up stale Next.js state from previous sessions --- # Nuke the entire .next directory to avoid stale webpack bundles, lock files,