feat: bound default metric cardinality

README.md

@@ -1,35 +1,53 @@
 # Github Prometheus Exporter (promgithub)
 
-`promgithub` is a service that receives GitHub webhook events and exposes Prometheus metrics for repository activity, workflow runs, workflow jobs, commits, and pull requests.
+`promgithub` receives GitHub webhook events and exposes Prometheus metrics for repository activity, workflow runs, workflow jobs, commits, and pull requests.
 
-It is designed to be simple to deploy and can run either:
+It can run either:
 - as a single instance
 - as multiple instances with Redis for shared deduplication and state
 
 ## Metrics exported
 
-`promgithub` exports the following metrics:
-
-| Name                               | Type      | Labels                                                                  | Description                               |
-|------------------------------------|-----------|-------------------------------------------------------------------------|-------------------------------------------|
-| `promgithub_workflow_status`       | Counter   | `repository`, `branch`, `workflow_name`, `workflow_status`, `conclusion` | Total number of workflow runs with status |
-| `promgithub_workflow_duration`     | Histogram | `repository`, `branch`, `workflow_name`, `workflow_status`, `conclusion` | Duration of workflow runs                 |
-| `promgithub_workflow_queued`       | Gauge     | `repository`, `branch`, `workflow_name`                                 | Number of workflow runs queued            |
-| `promgithub_workflow_in_progress`  | Gauge     | `repository`, `branch`, `workflow_name`                                 | Number of workflow runs in progress       |
-| `promgithub_workflow_completed`    | Gauge     | `repository`, `branch`, `workflow_conclusion`, `workflow_name`          | Number of workflow runs completed         |
-| `promgithub_job_status`            | Counter   | `repository`, `branch`, `workflow_name`, `job_status`, `job_conclusion` | Total number of jobs with status          |
-| `promgithub_job_duration`          | Histogram | `repository`, `branch`, `workflow_name`, `job_status`, `job_conclusion` | Duration of jobs runs in seconds          |
-| `promgithub_job_queued`            | Gauge     | `repository`, `branch`, `workflow_name`                                 | Number of jobs queued                     |
-| `promgithub_job_in_progress`       | Gauge     | `repository`, `branch`, `workflow_name`                                 | Number of jobs in progress                |
-| `promgithub_job_completed`         | Gauge     | `repository`, `branch`, `job_conclusion`, `workflow_name`               | Number of jobs completed                  |
-| `promgithub_commit_pushed`         | Counter   | `repository`                                                            | Total number of commits pushed            |
-| `promgithub_pull_request`          | Counter   | `repository`, `base_branch`, `pull_request_status`                      | Total number of pull requests             |
+### Default metrics
 
-## Metric model
+The default metric set is bounded-cardinality and production-safe for larger repository sets.
+
+| Name                              | Type      | Labels                                         | Description                               |
+|-----------------------------------|-----------|------------------------------------------------|-------------------------------------------|
+| `promgithub_workflow_status`      | Counter   | `repository`, `workflow_status`, `conclusion`  | Total number of workflow runs with status |
+| `promgithub_workflow_duration`    | Histogram | `repository`, `workflow_status`, `conclusion`  | Duration of workflow runs                 |
+| `promgithub_workflow_queued`      | Gauge     | `repository`                                   | Number of workflow runs queued            |
+| `promgithub_workflow_in_progress` | Gauge     | `repository`                                   | Number of workflow runs in progress       |
+| `promgithub_workflow_completed`   | Gauge     | `repository`, `workflow_conclusion`            | Number of workflow runs completed         |
+| `promgithub_job_status`           | Counter   | `repository`, `job_status`, `job_conclusion`   | Total number of jobs with status          |
+| `promgithub_job_duration`         | Histogram | `repository`, `job_status`, `job_conclusion`   | Duration of jobs runs in seconds          |
+| `promgithub_job_queued`           | Gauge     | `repository`                                   | Number of jobs queued                     |
+| `promgithub_job_in_progress`      | Gauge     | `repository`                                   | Number of jobs in progress                |
+| `promgithub_job_completed`        | Gauge     | `repository`, `job_conclusion`                 | Number of jobs completed                  |
+| `promgithub_commit_pushed`        | Counter   | `repository`                                   | Total number of commits pushed            |
+| `promgithub_pull_request`         | Counter   | `repository`, `pull_request_status`            | Total number of pull requests             |
+
+### Optional detailed metrics
+
+Set `PROMGITHUB_ENABLE_DETAILED_METRICS=true` to also emit opt-in detailed metric families with higher-cardinality labels:
 
-The exporter focuses on repository and workflow health signals while avoiding noisy per-entity labels such as runner names, job names, commit author identities, and pull request authors.
+- `promgithub_workflow_status_detailed`
+- `promgithub_workflow_duration_detailed`
+- `promgithub_workflow_queued_detailed`
+- `promgithub_workflow_in_progress_detailed`
+- `promgithub_workflow_completed_detailed`
+- `promgithub_job_status_detailed`
+- `promgithub_job_duration_detailed`
+- `promgithub_job_queued_detailed`
+- `promgithub_job_in_progress_detailed`
+- `promgithub_job_completed_detailed`
+- `promgithub_pull_request_detailed`
+
+These detailed metrics preserve labels such as `branch`, `workflow_name`, and `base_branch`. They are disabled by default because they can grow quickly in larger GitHub environments.
+
+## Metric model
 
-This keeps the default metric set compact and practical for Prometheus while still preserving the `branch` label for branch-specific workflow and job visibility.
+The exporter now defaults to repository-level operational metrics and keeps higher-cardinality dimensions as an explicit opt-in. This avoids unbounded series growth from branch churn, workflow-name sprawl, and pull-request base-branch fragmentation while still allowing teams to enable richer labels when they understand the cost.
 
 ## Redis-backed multi-instance mode
 

docs/usage.md

@@ -21,6 +21,7 @@ The service supports the following environment variables:
 - `PROMGITHUB_REDIS_DB` (optional): Redis database number, default `0`.
 - `PROMGITHUB_REDIS_KEY_PREFIX` (optional): Prefix used for Redis keys, default `promgithub`.
 - `PROMGITHUB_REDIS_DELIVERY_TTL` (optional): TTL for webhook delivery dedupe keys, default `24h`.
+- `PROMGITHUB_ENABLE_DETAILED_METRICS` (optional): When `true`, also emits higher-cardinality `*_detailed` metric families with labels such as `branch`, `workflow_name`, and `base_branch`. Default `false`.
 
 If Redis is configured, the service stores delivery and run state in Redis.
 
@@ -43,6 +44,7 @@ PROMGITHUB_REDIS_PASSWORD="<redis password>" \
 PROMGITHUB_REDIS_DB="0" \
 PROMGITHUB_REDIS_KEY_PREFIX="promgithub" \
 PROMGITHUB_REDIS_DELIVERY_TTL="24h" \
+PROMGITHUB_ENABLE_DETAILED_METRICS="true" \
 PROMGITHUB_SERVICE_PORT="8080" \
 /path/to/binary/promgithub
 ```
@@ -67,6 +69,7 @@ docker run \
   -e PROMGITHUB_REDIS_DB=0 \
   -e PROMGITHUB_REDIS_KEY_PREFIX=promgithub \
   -e PROMGITHUB_REDIS_DELIVERY_TTL=24h \
+  -e PROMGITHUB_ENABLE_DETAILED_METRICS=true \
   -e PROMGITHUB_SERVICE_PORT=8080 \
   -p 8080:8080 \
   ghcr.io/darthfork/promgithub:<version>
@@ -91,6 +94,7 @@ services:
       PROMGITHUB_REDIS_DB: 0
       PROMGITHUB_REDIS_KEY_PREFIX: promgithub
       PROMGITHUB_REDIS_DELIVERY_TTL: 24h
+      PROMGITHUB_ENABLE_DETAILED_METRICS: "true"
       PROMGITHUB_SERVICE_PORT: 8080
     ports:
       - "8080:8080"
@@ -129,6 +133,8 @@ promgithub:
     db: 0
     keyPrefix: promgithub
     deliveryTTL: 24h
+  metrics:
+    enableDetailed: false
 ```
 
 ### Values for a bundled Redis deployment
@@ -146,6 +152,8 @@ promgithub:
     db: 0
     keyPrefix: promgithub
     deliveryTTL: 24h
+  metrics:
+    enableDetailed: true
 ```
 
 When `redis.enabled=true`, the chart deploys Redis as a dependency and configures `promgithub` to connect to it automatically.

helm/promgithub/templates/deployment.yaml

@@ -54,6 +54,8 @@ spec:
             - name: PROMGITHUB_REDIS_DELIVERY_TTL
               value: "{{ .Values.redisConfig.deliveryTTL | default "24h" }}"
             {{- end }}
+            - name: PROMGITHUB_ENABLE_DETAILED_METRICS
+              value: "{{ .Values.metrics.enableDetailed | default false }}"
           envFrom:
             - secretRef:
                 name: "{{ include "promgithub.fullname" . }}"

helm/promgithub/values.yaml

@@ -46,6 +46,9 @@ redisConfig:
   keyPrefix: promgithub
   deliveryTTL: 24h
 
+metrics:
+  enableDetailed: false
+
 # This is for setting up the promgithub service
 service:
   # This sets the service type

src/async_test.go

@@ -35,8 +35,16 @@ func TestAsyncProcessorEnqueueAndProcess(t *testing.T) {
 		t.Fatal("timed out waiting for async processing")
 	}
 
-	if got := testutil.ToFloat64(asyncProcessedEventsCounter.WithLabelValues("workflow_run")); got != 1 {
-		t.Fatalf("expected processed counter to be 1, got %v", got)
+	deadline := time.Now().Add(2 * time.Second)
+	for {
+		if got := testutil.ToFloat64(asyncProcessedEventsCounter.WithLabelValues("workflow_run")); got == 1 {
+			break
+		}
+		if time.Now().After(deadline) {
+			got := testutil.ToFloat64(asyncProcessedEventsCounter.WithLabelValues("workflow_run"))
+			t.Fatalf("expected processed counter to be 1, got %v", got)
+		}
+		time.Sleep(10 * time.Millisecond)
 	}
 }
 

src/github.go

@@ -101,6 +101,11 @@ type runMetricSet struct {
 	durationHistogram *prometheus.HistogramVec
 }
 
+type runMetricSets struct {
+	core     runMetricSet
+	detailed runMetricSet
+}
+
 type runStoreMethods struct {
 	get    func(context.Context, int) (RunState, bool, error)
 	update func(context.Context, int, RunState) error
@@ -255,7 +260,18 @@ func shouldApplyStateTransition(previous, next RunState) bool {
 	return true
 }
 
-func applyGaugeDelta(details RunState, delta float64, queuedGauge, inProgressGauge, completedGauge *prometheus.GaugeVec) {
+func applyCoreGaugeDelta(details RunState, delta float64, queuedGauge, inProgressGauge, completedGauge *prometheus.GaugeVec) {
+	switch normalizeStatus(details.Status) {
+	case statusQueued:
+		queuedGauge.WithLabelValues(details.Repository).Add(delta)
+	case statusInProgress:
+		inProgressGauge.WithLabelValues(details.Repository).Add(delta)
+	case statusCompleted:
+		completedGauge.WithLabelValues(details.Repository, details.Conclusion).Add(delta)
+	}
+}
+
+func applyDetailedGaugeDelta(details RunState, delta float64, queuedGauge, inProgressGauge, completedGauge *prometheus.GaugeVec) {
 	switch normalizeStatus(details.Status) {
 	case statusQueued:
 		queuedGauge.WithLabelValues(details.Repository, details.Branch, details.Name).Add(delta)
@@ -266,7 +282,25 @@ func applyGaugeDelta(details RunState, delta float64, queuedGauge, inProgressGau
 	}
 }
 
-func observeDuration(details RunState, durationHistogram *prometheus.HistogramVec) {
+func observeCoreDuration(details RunState, durationHistogram *prometheus.HistogramVec) {
+	if normalizeStatus(details.Status) != statusCompleted {
+		return
+	}
+
+	startedAt, startedOK := parseMetricTime(details.StartedAt)
+	endedAt, endedOK := parseMetricTime(details.EndedAt)
+	if !startedOK || !endedOK || endedAt.Before(startedAt) {
+		return
+	}
+
+	durationHistogram.WithLabelValues(
+		details.Repository,
+		details.Status,
+		details.Conclusion,
+	).Observe(endedAt.Sub(startedAt).Seconds())
+}
+
+func observeDetailedDuration(details RunState, durationHistogram *prometheus.HistogramVec) {
 	if normalizeStatus(details.Status) != statusCompleted {
 		return
 	}
@@ -286,7 +320,24 @@ func observeDuration(details RunState, durationHistogram *prometheus.HistogramVe
 	).Observe(endedAt.Sub(startedAt).Seconds())
 }
 
-func applyStatefulMetrics(details RunState, previous *RunState, metrics runMetricSet) {
+func applyCoreStatefulMetrics(details RunState, previous *RunState, metrics runMetricSet) {
+	metrics.statusCounter.WithLabelValues(
+		details.Repository,
+		details.Status,
+		details.Conclusion,
+	).Inc()
+
+	if previous != nil {
+		applyCoreGaugeDelta(*previous, -1, metrics.queuedGauge, metrics.inProgressGauge, metrics.completedGauge)
+	}
+	applyCoreGaugeDelta(details, 1, metrics.queuedGauge, metrics.inProgressGauge, metrics.completedGauge)
+
+	if previous == nil || normalizeStatus(previous.Status) != statusCompleted {
+		observeCoreDuration(details, metrics.durationHistogram)
+	}
+}
+
+func applyDetailedStatefulMetrics(details RunState, previous *RunState, metrics runMetricSet) {
 	metrics.statusCounter.WithLabelValues(
 		details.Repository,
 		details.Branch,
@@ -296,12 +347,12 @@ func applyStatefulMetrics(details RunState, previous *RunState, metrics runMetri
 	).Inc()
 
 	if previous != nil {
-		applyGaugeDelta(*previous, -1, metrics.queuedGauge, metrics.inProgressGauge, metrics.completedGauge)
+		applyDetailedGaugeDelta(*previous, -1, metrics.queuedGauge, metrics.inProgressGauge, metrics.completedGauge)
 	}
-	applyGaugeDelta(details, 1, metrics.queuedGauge, metrics.inProgressGauge, metrics.completedGauge)
+	applyDetailedGaugeDelta(details, 1, metrics.queuedGauge, metrics.inProgressGauge, metrics.completedGauge)
 
 	if previous == nil || normalizeStatus(previous.Status) != statusCompleted {
-		observeDuration(details, metrics.durationHistogram)
+		observeDetailedDuration(details, metrics.durationHistogram)
 	}
 }
 
@@ -341,12 +392,15 @@ func updateTrackedRunMetrics(
 	details runMetricDetails,
 	store runStoreMethods,
 	entityName string,
-	metrics runMetricSet,
+	metrics runMetricSets,
 ) {
 	nextState := normalizeRunState(details)
 
 	if stateStore == nil {
-		applyStatefulMetrics(nextState, nil, metrics)
+		applyCoreStatefulMetrics(nextState, nil, metrics.core)
+		if enableDetailedMetrics {
+			applyDetailedStatefulMetrics(nextState, nil, metrics.detailed)
+		}
 		return
 	}
 
@@ -362,7 +416,10 @@ func updateTrackedRunMetrics(
 		return
 	}
 
-	applyStatefulMetrics(nextState, previousState, metrics)
+	applyCoreStatefulMetrics(nextState, previousState, metrics.core)
+	if enableDetailedMetrics {
+		applyDetailedStatefulMetrics(nextState, previousState, metrics.detailed)
+	}
 }
 
 func workflowRunStoreMethods() runStoreMethods {
@@ -409,12 +466,21 @@ func updateWorkflowMetrics(ctx context.Context, body []byte) {
 		},
 		workflowRunStoreMethods(),
 		"workflow_run",
-		runMetricSet{
-			statusCounter:     workflowStatusCounter,
-			queuedGauge:       workflowQueuedGauge,
-			inProgressGauge:   workflowInProgressGauge,
-			completedGauge:    workflowCompletedGauge,
-			durationHistogram: workflowDurationHistogram,
+		runMetricSets{
+			core: runMetricSet{
+				statusCounter:     workflowStatusCounter,
+				queuedGauge:       workflowQueuedGauge,
+				inProgressGauge:   workflowInProgressGauge,
+				completedGauge:    workflowCompletedGauge,
+				durationHistogram: workflowDurationHistogram,
+			},
+			detailed: runMetricSet{
+				statusCounter:     workflowStatusDetailedCounter,
+				queuedGauge:       workflowQueuedDetailedGauge,
+				inProgressGauge:   workflowInProgressDetailedGauge,
+				completedGauge:    workflowCompletedDetailedGauge,
+				durationHistogram: workflowDurationDetailedHistogram,
+			},
 		},
 	)
 }
@@ -441,12 +507,21 @@ func updateJobMetrics(ctx context.Context, body []byte) {
 		},
 		workflowJobStoreMethods(),
 		"workflow_job",
-		runMetricSet{
-			statusCounter:     jobStatusCounter,
-			queuedGauge:       jobQueuedGauge,
-			inProgressGauge:   jobInProgressGauge,
-			completedGauge:    jobCompletedGauge,
-			durationHistogram: jobDurationHistogram,
+		runMetricSets{
+			core: runMetricSet{
+				statusCounter:     jobStatusCounter,
+				queuedGauge:       jobQueuedGauge,
+				inProgressGauge:   jobInProgressGauge,
+				completedGauge:    jobCompletedGauge,
+				durationHistogram: jobDurationHistogram,
+			},
+			detailed: runMetricSet{
+				statusCounter:     jobStatusDetailedCounter,
+				queuedGauge:       jobQueuedDetailedGauge,
+				inProgressGauge:   jobInProgressDetailedGauge,
+				completedGauge:    jobCompletedDetailedGauge,
+				durationHistogram: jobDurationDetailedHistogram,
+			},
 		},
 	)
 }
@@ -474,7 +549,14 @@ func updatePullRequestMetrics(body []byte) {
 
 	pullRequestCounter.WithLabelValues(
 		payload.Repository.FullName,
-		payload.PullRequest.Base.Ref,
 		payload.Action,
 	).Inc()
+
+	if enableDetailedMetrics {
+		pullRequestDetailedCounter.WithLabelValues(
+			payload.Repository.FullName,
+			payload.PullRequest.Base.Ref,
+			payload.Action,
+		).Inc()
+	}
 }