From 89a39ed148cd76e2e321842764362fc5637a5b10 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Thu, 19 Mar 2026 15:14:33 -0400 Subject: [PATCH 01/45] Initial PoC --- pkg/beholder/DurableEmitterDesign.md | 188 +++++++++++++++++++++ pkg/beholder/durable_emitter.go | 237 +++++++++++++++++++++++++++ pkg/beholder/durable_emitter_test.go | 222 +++++++++++++++++++++++++ pkg/beholder/durable_event_store.go | 105 ++++++++++++ 4 files changed, 752 insertions(+) create mode 100644 pkg/beholder/DurableEmitterDesign.md create mode 100644 pkg/beholder/durable_emitter.go create mode 100644 pkg/beholder/durable_emitter_test.go create mode 100644 pkg/beholder/durable_event_store.go diff --git a/pkg/beholder/DurableEmitterDesign.md b/pkg/beholder/DurableEmitterDesign.md new file mode 100644 index 0000000000..20dc1d3374 --- /dev/null +++ b/pkg/beholder/DurableEmitterDesign.md @@ -0,0 +1,188 @@ +# Durable Event Buffer for ChIP + +## Problem Statement + +Today there is no persistence in the ChIP pipeline. The `ChipIngressEmitter` calls `chipingress.Client.Publish()` synchronously over gRPC, and the `batch.Client` uses an in-memory channel buffer of 200 messages. If the node crashes, Chip is unreachable, or the buffer fills up, events (including billing records) are silently dropped. + +**Drop points:** +- `batch.Client` returns `"message buffer is full"` and the event is lost. +- `ChipIngressEmitter` propagates the error up, but `DualSourceEmitter` only logs chip-ingress failures. +- Any in-flight events are lost on node crash — nothing is persisted to disk. + +## Functional Requirements + +- Events must not be lost on node restarts. +- Events must be delivered within a reasonable period of time (seconds, not minutes). +- The system must support eventually-consistent billing. +- Node databases must not bloat unboundedly. + +## Non-Functional Requirements + +- Scale to 1k+ TPS. +- 4 nines of availability (99.99%). + +## Architecture Overview + +``` +Workflow Engine + │ + ▼ + DurableEmitter.Emit() + │ + ├─ 1. Serialize event → proto bytes + ├─ 2. INSERT into Postgres (durable guarantee) + ├─ 3. Return nil (caller unblocked) + │ + └─ 4. Async goroutine: chipingress.Client.Publish() + │ + ├─ Success → DELETE from Postgres + └─ Failure → no-op (retransmit loop will handle) + + ┌────────────────────────────┐ + │ Background Retransmit Loop │ (runs every RetransmitInterval) + │ │ + │ ListPending(olderThan) │ + │ → PublishBatch() │ + │ → DELETE on success │ + └────────────────────────────┘ + + ┌────────────────────────────┐ + │ Background Expiry Loop │ (runs every ExpiryInterval) + │ │ + │ DeleteExpired(ttl) │ + │ → GC old events │ + └────────────────────────────┘ +``` + +## Key Design Decision: Use Standard `chipingress.Client`, Not `batch.Client` + +Per Hagen's guidance, we use the standard `chipingress.Client` directly (which supports both `Publish` and `PublishBatch`) since we are implementing our own queuing mechanism with persistence-backed guarantees. The `batch.Client`'s in-memory buffer would be redundant. + +## Components + +### DurableEventStore (interface — `chainlink-common`) + +```go +type DurableEventStore interface { + Insert(ctx context.Context, payload []byte) (int64, error) + Delete(ctx context.Context, id int64) error + ListPending(ctx context.Context, createdBefore time.Time, limit int) ([]DurableEvent, error) + DeleteExpired(ctx context.Context, ttl time.Duration) (int64, error) +} +``` + +- `Insert` persists a serialized `CloudEventPb` and returns an auto-generated ID. +- `Delete` removes a single event after confirmed delivery. +- `ListPending` returns events older than a cutoff (gives the immediate-publish path time to succeed before retransmit picks it up). +- `DeleteExpired` garbage-collects events older than the TTL to bound table growth. + +Two implementations: +- **`MemDurableEventStore`** — in-memory, for unit tests (lives in `chainlink-common`). +- **`PgDurableEventStore`** — Postgres-backed ORM (lives in `chainlink`). + +### DurableEmitter (`chainlink-common`) + +Implements `beholder.Emitter`: +```go +type Emitter interface { + Emit(ctx context.Context, body []byte, attrKVs ...any) error + io.Closer +} +``` + +**`Emit()` flow (synchronous path):** +1. `ExtractSourceAndType(attrKVs...)` — validate and extract source/type. +2. `chipingress.NewEvent(...)` + `EventToProto(...)` — build the CloudEvent proto. +3. `proto.Marshal(eventPb)` — serialize to bytes. +4. `store.Insert(ctx, payload)` — persist. **Emit returns nil here.** + +**Async delivery path (goroutine):** +5. `client.Publish(ctx, eventPb)` — attempt immediate delivery. +6. On success: `store.Delete(id)`. +7. On failure: log at debug level; retransmit loop handles it. + +**Key guarantee:** `Emit()` returns `nil` once the DB insert succeeds. Even if the node crashes immediately after, the event survives in Postgres and will be retransmitted on restart. + +`Emit()` is non-blocking for workflow execution because the expensive operation (gRPC publish) happens asynchronously. The synchronous path is: attribute extraction → proto serialization → one DB insert. At 1k TPS, Postgres handles this trivially. + +### Configuration + +```go +type DurableEmitterConfig struct { + RetransmitInterval time.Duration // default 5s + RetransmitAfter time.Duration // default 10s (min age before retry) + RetransmitBatchSize int // default 100 + ExpiryInterval time.Duration // default 1min + EventTTL time.Duration // default 24h + PublishTimeout time.Duration // default 5s +} +``` + +## Postgres Table + +```sql +CREATE TABLE IF NOT EXISTS cre.chip_durable_events ( + id BIGSERIAL PRIMARY KEY, + payload BYTEA NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE INDEX idx_chip_durable_events_created_at + ON cre.chip_durable_events (created_at ASC); +``` + +The table lives in the existing `cre` schema. The `created_at` index supports efficient `ListPending` and `DeleteExpired` queries. + +## Service Principal & ACK Guarantees + +CRE nodes authenticate to ChIP using the node's CSA Key as the `servicePrincipal`. This is **not** the `oti-telemetry-shared` principal, which uses a fire-and-forget publish path. With the CSA Key principal, the gateway waits for Kafka ACKs before returning a response, so a successful gRPC response (`200`) means the event was durably accepted by the gateway. + +## Files + +| Repo | File | Purpose | +|------|------|---------| +| chainlink-common | `pkg/beholder/durable_event_store.go` | `DurableEventStore` interface + `MemDurableEventStore` | +| chainlink-common | `pkg/beholder/durable_emitter.go` | `DurableEmitter` struct, `Emit`, retransmit loop, expiry loop | +| chainlink-common | `pkg/beholder/durable_emitter_test.go` | Unit tests with in-memory store | +| chainlink | `core/services/beholder/durable_event_store_orm.go` | `PgDurableEventStore` implementing `DurableEventStore` | +| chainlink | `core/store/migrate/migrations/0294_chip_durable_events.sql` | Postgres migration | + +## Wiring (TODO) + +In `core/cmd/shell.go`, where the beholder client is created, replace or wrap the `ChipIngressEmitter` with `DurableEmitter`: + +```go +// After creating chipIngressClient... +pgStore := beholder.NewPgDurableEventStore(ds) +durableEmitter, err := beholder.NewDurableEmitter(pgStore, chipIngressClient, cfg, log) +durableEmitter.Start(ctx) +// Use durableEmitter as the chip-ingress emitter in DualSourceEmitter +``` + +## Metrics to Instrument (Future) + +| Metric | Description | +|--------|-------------| +| `durable_emitter.queue_depth` | Number of events currently in the store | +| `durable_emitter.insert_rate` | Events persisted per second | +| `durable_emitter.publish_rate` | Events successfully delivered per second | +| `durable_emitter.retransmit_rate` | Events retransmitted per second | +| `durable_emitter.publish_latency` | Time from insert to confirmed delivery | +| `durable_emitter.oldest_pending` | Age of the longest-waiting event | +| `durable_emitter.expired_count` | Events expired (dropped after TTL) | +| `durable_emitter.error_rate` | Failed publish attempts per second | + +## Open Questions + +1. **Chip gateway idempotency** — Does the gateway deduplicate re-sent events? If the retransmit loop re-sends an event that the immediate path already delivered (race window), will the gateway de-dup or create a duplicate billing record? The CloudEvent `id` field (UUID) could serve as a dedup key. + +2. **DB load at scale** — At 1k TPS: ~1k inserts/sec + ~1k deletes/sec. The delete-heavy workload will produce dead tuples requiring autovacuum tuning. Potential optimizations: + - Batch deletes (delete by ID list instead of per-row). + - Two-table approach (queued + recently-sent) to reduce churn. + - CDC streaming as an alternative to the insert/delete pattern entirely. + +3. **Exponential backoff** — Current PoC uses a fixed retransmit interval. Production should implement per-event exponential backoff using `attempts` and `last_sent_at` columns (schema extension). + +4. **rmq / Redis alternative** — Patrick raised using [rmq](https://github.com/wellle/rmq) backed by our own DB instead of re-implementing a queue. Worth evaluating if the Postgres-backed approach has scaling issues. + +5. **CDC streaming** — Could stream WAL changes directly rather than polling the table, avoiding the insert/delete churn entirely. Matthew Gardener and Clement can advise on CDC implementation within the existing data analytics pipeline. diff --git a/pkg/beholder/durable_emitter.go b/pkg/beholder/durable_emitter.go new file mode 100644 index 0000000000..beb7161135 --- /dev/null +++ b/pkg/beholder/durable_emitter.go @@ -0,0 +1,237 @@ +package beholder + +import ( + "context" + "fmt" + "sync" + "time" + + "google.golang.org/protobuf/proto" + + "github.com/smartcontractkit/chainlink-common/pkg/chipingress" + "github.com/smartcontractkit/chainlink-common/pkg/logger" +) + +// DurableEmitterConfig configures the DurableEmitter behaviour. +type DurableEmitterConfig struct { + // RetransmitInterval controls how often the retransmit loop ticks. + RetransmitInterval time.Duration + // RetransmitAfter is the minimum age of an event before the retransmit + // loop considers it. This gives the immediate-publish path time to succeed. + RetransmitAfter time.Duration + // RetransmitBatchSize caps the number of events sent per retransmit cycle. + RetransmitBatchSize int + // ExpiryInterval controls how often the expiry loop ticks. + ExpiryInterval time.Duration + // EventTTL is the maximum age of an event before it is expired. + EventTTL time.Duration + // PublishTimeout is the per-RPC deadline for Publish / PublishBatch calls. + PublishTimeout time.Duration +} + +func DefaultDurableEmitterConfig() DurableEmitterConfig { + return DurableEmitterConfig{ + RetransmitInterval: 5 * time.Second, + RetransmitAfter: 10 * time.Second, + RetransmitBatchSize: 100, + ExpiryInterval: 1 * time.Minute, + EventTTL: 24 * time.Hour, + PublishTimeout: 5 * time.Second, + } +} + +// DurableEmitter implements Emitter with persistence-backed delivery guarantees. +// +// On Emit the event is serialized and written to a DurableEventStore. Once the +// insert succeeds Emit returns nil — the caller has a durable guarantee. An +// immediate async Publish is attempted; on success the record is deleted. If +// that fails a background retransmit loop will pick the event up and retry via +// PublishBatch. +// +// A separate expiry loop garbage-collects events older than EventTTL to bound +// table growth. +type DurableEmitter struct { + store DurableEventStore + client chipingress.Client + cfg DurableEmitterConfig + log logger.Logger + + stopCh chan struct{} + wg sync.WaitGroup +} + +var _ Emitter = (*DurableEmitter)(nil) + +func NewDurableEmitter( + store DurableEventStore, + client chipingress.Client, + cfg DurableEmitterConfig, + log logger.Logger, +) (*DurableEmitter, error) { + if store == nil { + return nil, fmt.Errorf("durable event store is nil") + } + if client == nil { + return nil, fmt.Errorf("chipingress client is nil") + } + if log == nil { + return nil, fmt.Errorf("logger is nil") + } + return &DurableEmitter{ + store: store, + client: client, + cfg: cfg, + log: log, + stopCh: make(chan struct{}), + }, nil +} + +// Start launches the retransmit and expiry background loops. +// Cancel the supplied context or call Close to stop them. +func (d *DurableEmitter) Start(ctx context.Context) { + d.wg.Add(2) + go d.retransmitLoop(ctx) + go d.expiryLoop(ctx) +} + +// Emit persists the event then attempts async delivery. +// Returns nil once the store insert succeeds. +func (d *DurableEmitter) Emit(ctx context.Context, body []byte, attrKVs ...any) error { + sourceDomain, entityType, err := ExtractSourceAndType(attrKVs...) + if err != nil { + return err + } + + event, err := chipingress.NewEvent(sourceDomain, entityType, body, newAttributes(attrKVs...)) + if err != nil { + return err + } + + eventPb, err := chipingress.EventToProto(event) + if err != nil { + return fmt.Errorf("failed to convert event to proto: %w", err) + } + + payload, err := proto.Marshal(eventPb) + if err != nil { + return fmt.Errorf("failed to marshal event proto: %w", err) + } + + id, err := d.store.Insert(ctx, payload) + if err != nil { + return fmt.Errorf("failed to persist event: %w", err) + } + + // Fire-and-forget immediate delivery attempt. + go d.publishAndDelete(id, eventPb) + + return nil +} + +// Close signals background loops to stop and waits for them to finish. +func (d *DurableEmitter) Close() error { + close(d.stopCh) + d.wg.Wait() + return nil +} + +// publishAndDelete attempts a single Publish and deletes the record on success. +func (d *DurableEmitter) publishAndDelete(id int64, eventPb *chipingress.CloudEventPb) { + ctx, cancel := context.WithTimeout(context.Background(), d.cfg.PublishTimeout) + defer cancel() + + if _, err := d.client.Publish(ctx, eventPb); err != nil { + d.log.Debugw("immediate publish failed, retransmit loop will retry", + "id", id, "error", err) + return + } + + if err := d.store.Delete(context.Background(), id); err != nil { + d.log.Errorw("failed to delete delivered event", "id", id, "error", err) + } +} + +func (d *DurableEmitter) retransmitLoop(ctx context.Context) { + defer d.wg.Done() + ticker := time.NewTicker(d.cfg.RetransmitInterval) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-d.stopCh: + return + case <-ticker.C: + d.retransmitPending(ctx) + } + } +} + +func (d *DurableEmitter) retransmitPending(ctx context.Context) { + cutoff := time.Now().Add(-d.cfg.RetransmitAfter) + pending, err := d.store.ListPending(ctx, cutoff, d.cfg.RetransmitBatchSize) + if err != nil { + d.log.Errorw("failed to list pending events", "error", err) + return + } + if len(pending) == 0 { + return + } + + events := make([]*chipingress.CloudEventPb, 0, len(pending)) + ids := make([]int64, 0, len(pending)) + + for _, pe := range pending { + var eventPb chipingress.CloudEventPb + if err := proto.Unmarshal(pe.Payload, &eventPb); err != nil { + d.log.Errorw("corrupt pending event, deleting", "id", pe.ID, "error", err) + _ = d.store.Delete(ctx, pe.ID) + continue + } + events = append(events, &eventPb) + ids = append(ids, pe.ID) + } + if len(events) == 0 { + return + } + + publishCtx, cancel := context.WithTimeout(ctx, d.cfg.PublishTimeout) + defer cancel() + + if _, err := d.client.PublishBatch(publishCtx, &chipingress.CloudEventBatch{Events: events}); err != nil { + d.log.Warnw("retransmit batch failed", "count", len(events), "error", err) + return + } + + for _, id := range ids { + if err := d.store.Delete(ctx, id); err != nil { + d.log.Errorw("failed to delete retransmitted event", "id", id, "error", err) + } + } + d.log.Debugw("retransmitted events", "count", len(ids)) +} + +func (d *DurableEmitter) expiryLoop(ctx context.Context) { + defer d.wg.Done() + ticker := time.NewTicker(d.cfg.ExpiryInterval) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-d.stopCh: + return + case <-ticker.C: + deleted, err := d.store.DeleteExpired(ctx, d.cfg.EventTTL) + if err != nil { + d.log.Errorw("failed to delete expired events", "error", err) + continue + } + if deleted > 0 { + d.log.Infow("purged expired events", "count", deleted) + } + } + } +} diff --git a/pkg/beholder/durable_emitter_test.go b/pkg/beholder/durable_emitter_test.go new file mode 100644 index 0000000000..9c3f2bf498 --- /dev/null +++ b/pkg/beholder/durable_emitter_test.go @@ -0,0 +1,222 @@ +package beholder + +import ( + "context" + "errors" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + + "github.com/smartcontractkit/chainlink-common/pkg/chipingress" + "github.com/smartcontractkit/chainlink-common/pkg/logger" +) + +// testChipClient is a minimal chipingress.Client for tests. +type testChipClient struct { + chipingress.NoopClient + + mu sync.Mutex + publishErr error + batchErr error + publishCount atomic.Int64 + batchCount atomic.Int64 +} + +func (c *testChipClient) Publish(_ context.Context, _ *chipingress.CloudEventPb, _ ...grpc.CallOption) (*chipingress.PublishResponse, error) { + c.publishCount.Add(1) + c.mu.Lock() + defer c.mu.Unlock() + return &chipingress.PublishResponse{}, c.publishErr +} + +func (c *testChipClient) PublishBatch(_ context.Context, _ *chipingress.CloudEventBatch, _ ...grpc.CallOption) (*chipingress.PublishResponse, error) { + c.batchCount.Add(1) + c.mu.Lock() + defer c.mu.Unlock() + return &chipingress.PublishResponse{}, c.batchErr +} + +func (c *testChipClient) setPublishErr(err error) { + c.mu.Lock() + defer c.mu.Unlock() + c.publishErr = err +} + +func (c *testChipClient) setBatchErr(err error) { + c.mu.Lock() + defer c.mu.Unlock() + c.batchErr = err +} + +func testEmitAttrs() []any { + return []any{"source", "test-source", "type", "test-type"} +} + +func newTestDurableEmitter(t *testing.T, store DurableEventStore, client chipingress.Client, cfgOverride *DurableEmitterConfig) *DurableEmitter { + t.Helper() + cfg := DefaultDurableEmitterConfig() + if cfgOverride != nil { + cfg = *cfgOverride + } + em, err := NewDurableEmitter(store, client, cfg, logger.Test(t)) + require.NoError(t, err) + return em +} + +func TestDurableEmitter_EmitPersistsAndPublishes(t *testing.T) { + store := NewMemDurableEventStore() + client := &testChipClient{} + em := newTestDurableEmitter(t, store, client, nil) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + em.Start(ctx) + defer em.Close() + + err := em.Emit(ctx, []byte("hello"), testEmitAttrs()...) + require.NoError(t, err) + + // Immediate async publish should fire and delete the record. + require.Eventually(t, func() bool { + return client.publishCount.Load() == 1 + }, 2*time.Second, 10*time.Millisecond) + + require.Eventually(t, func() bool { + return store.Len() == 0 + }, 2*time.Second, 10*time.Millisecond) +} + +func TestDurableEmitter_EmitReturnSuccessEvenWhenPublishFails(t *testing.T) { + store := NewMemDurableEventStore() + client := &testChipClient{} + client.setPublishErr(errors.New("connection refused")) + + em := newTestDurableEmitter(t, store, client, nil) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + em.Start(ctx) + defer em.Close() + + err := em.Emit(ctx, []byte("hello"), testEmitAttrs()...) + require.NoError(t, err, "Emit must succeed once the DB insert succeeds") + + // Wait for the async publish attempt to complete. + require.Eventually(t, func() bool { + return client.publishCount.Load() == 1 + }, 2*time.Second, 10*time.Millisecond) + + // Event must remain in the store for retransmit. + assert.Equal(t, 1, store.Len()) +} + +func TestDurableEmitter_RetransmitLoopDeliversFailedEvents(t *testing.T) { + store := NewMemDurableEventStore() + client := &testChipClient{} + client.setPublishErr(errors.New("connection refused")) + client.setBatchErr(errors.New("connection refused")) + + cfg := DefaultDurableEmitterConfig() + cfg.RetransmitInterval = 100 * time.Millisecond + cfg.RetransmitAfter = 50 * time.Millisecond + + em := newTestDurableEmitter(t, store, client, &cfg) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + em.Start(ctx) + defer em.Close() + + err := em.Emit(ctx, []byte("retry-me"), testEmitAttrs()...) + require.NoError(t, err) + assert.Equal(t, 1, store.Len()) + + // Fix the batch client so retransmit succeeds. + client.setBatchErr(nil) + + require.Eventually(t, func() bool { + return store.Len() == 0 + }, 5*time.Second, 50*time.Millisecond, "retransmit loop should eventually deliver and delete the event") + + assert.GreaterOrEqual(t, client.batchCount.Load(), int64(1)) +} + +func TestDurableEmitter_ExpiryLoopDeletesOldEvents(t *testing.T) { + store := NewMemDurableEventStore() + client := &testChipClient{} + client.setPublishErr(errors.New("always fail")) + + cfg := DefaultDurableEmitterConfig() + cfg.ExpiryInterval = 100 * time.Millisecond + cfg.EventTTL = 50 * time.Millisecond + cfg.RetransmitInterval = 10 * time.Minute // effectively disable retransmit + + em := newTestDurableEmitter(t, store, client, &cfg) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + em.Start(ctx) + defer em.Close() + + err := em.Emit(ctx, []byte("will-expire"), testEmitAttrs()...) + require.NoError(t, err) + assert.Equal(t, 1, store.Len()) + + require.Eventually(t, func() bool { + return store.Len() == 0 + }, 5*time.Second, 50*time.Millisecond, "expiry loop should purge the event") +} + +func TestDurableEmitter_EmitRejectsInvalidAttributes(t *testing.T) { + store := NewMemDurableEventStore() + client := &testChipClient{} + em := newTestDurableEmitter(t, store, client, nil) + + err := em.Emit(context.Background(), []byte("no-attrs")) + require.Error(t, err) + assert.Equal(t, 0, store.Len(), "nothing should be persisted when attributes are invalid") +} + +func TestDurableEmitter_MultipleEvents(t *testing.T) { + store := NewMemDurableEventStore() + client := &testChipClient{} + em := newTestDurableEmitter(t, store, client, nil) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + em.Start(ctx) + defer em.Close() + + const n = 50 + for i := 0; i < n; i++ { + err := em.Emit(ctx, []byte("event"), testEmitAttrs()...) + require.NoError(t, err) + } + + require.Eventually(t, func() bool { + return client.publishCount.Load() == int64(n) + }, 5*time.Second, 10*time.Millisecond) + + require.Eventually(t, func() bool { + return store.Len() == 0 + }, 5*time.Second, 10*time.Millisecond, "all events should be delivered and deleted") +} + +func TestNewDurableEmitter_ValidationErrors(t *testing.T) { + log := logger.Test(t) + cfg := DefaultDurableEmitterConfig() + + _, err := NewDurableEmitter(nil, &testChipClient{}, cfg, log) + assert.ErrorContains(t, err, "store") + + _, err = NewDurableEmitter(NewMemDurableEventStore(), nil, cfg, log) + assert.ErrorContains(t, err, "client") + + _, err = NewDurableEmitter(NewMemDurableEventStore(), &testChipClient{}, cfg, nil) + assert.ErrorContains(t, err, "logger") +} diff --git a/pkg/beholder/durable_event_store.go b/pkg/beholder/durable_event_store.go new file mode 100644 index 0000000000..1186850f2a --- /dev/null +++ b/pkg/beholder/durable_event_store.go @@ -0,0 +1,105 @@ +package beholder + +import ( + "context" + "sort" + "sync" + "sync/atomic" + "time" +) + +// DurableEvent represents a persisted event awaiting delivery to Chip. +type DurableEvent struct { + ID int64 + Payload []byte // serialized CloudEventPb proto + CreatedAt time.Time +} + +// DurableEventStore abstracts the persistence layer for durable chip events. +// Implementations must be safe for concurrent use. +type DurableEventStore interface { + // Insert persists a serialized event and returns its assigned ID. + Insert(ctx context.Context, payload []byte) (int64, error) + // Delete removes a successfully delivered event. + Delete(ctx context.Context, id int64) error + // ListPending returns events created before the given cutoff, ordered by + // creation time ascending, up to limit rows. + ListPending(ctx context.Context, createdBefore time.Time, limit int) ([]DurableEvent, error) + // DeleteExpired removes events older than ttl and returns the count deleted. + DeleteExpired(ctx context.Context, ttl time.Duration) (int64, error) +} + +// MemDurableEventStore is an in-memory DurableEventStore for unit tests. +type MemDurableEventStore struct { + mu sync.Mutex + events map[int64]*DurableEvent + nextID atomic.Int64 +} + +var _ DurableEventStore = (*MemDurableEventStore)(nil) + +func NewMemDurableEventStore() *MemDurableEventStore { + return &MemDurableEventStore{ + events: make(map[int64]*DurableEvent), + } +} + +func (m *MemDurableEventStore) Insert(_ context.Context, payload []byte) (int64, error) { + id := m.nextID.Add(1) + m.mu.Lock() + defer m.mu.Unlock() + m.events[id] = &DurableEvent{ + ID: id, + Payload: append([]byte(nil), payload...), // defensive copy + CreatedAt: time.Now(), + } + return id, nil +} + +func (m *MemDurableEventStore) Delete(_ context.Context, id int64) error { + m.mu.Lock() + defer m.mu.Unlock() + delete(m.events, id) + return nil +} + +func (m *MemDurableEventStore) ListPending(_ context.Context, createdBefore time.Time, limit int) ([]DurableEvent, error) { + m.mu.Lock() + defer m.mu.Unlock() + + var result []DurableEvent + for _, e := range m.events { + if e.CreatedAt.Before(createdBefore) { + result = append(result, *e) + } + } + sort.Slice(result, func(i, j int) bool { + return result[i].CreatedAt.Before(result[j].CreatedAt) + }) + if len(result) > limit { + result = result[:limit] + } + return result, nil +} + +func (m *MemDurableEventStore) DeleteExpired(_ context.Context, ttl time.Duration) (int64, error) { + m.mu.Lock() + defer m.mu.Unlock() + + cutoff := time.Now().Add(-ttl) + var deleted int64 + for id, e := range m.events { + if e.CreatedAt.Before(cutoff) { + delete(m.events, id) + deleted++ + } + } + return deleted, nil +} + +// Len returns the number of events in the store (test helper). +func (m *MemDurableEventStore) Len() int { + m.mu.Lock() + defer m.mu.Unlock() + return len(m.events) +} From cc10b6d11a7e69d9411e735bdd5651a185b9cd94 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Mon, 23 Mar 2026 11:32:53 -0400 Subject: [PATCH 02/45] Create durable_emitter_integration_test.go --- .../durable_emitter_integration_test.go | 386 ++++++++++++++++++ 1 file changed, 386 insertions(+) create mode 100644 pkg/beholder/durable_emitter_integration_test.go diff --git a/pkg/beholder/durable_emitter_integration_test.go b/pkg/beholder/durable_emitter_integration_test.go new file mode 100644 index 0000000000..e022b00a14 --- /dev/null +++ b/pkg/beholder/durable_emitter_integration_test.go @@ -0,0 +1,386 @@ +package beholder_test + +import ( + "context" + "net" + "sync" + "sync/atomic" + "testing" + "time" + + cepb "github.com/cloudevents/sdk-go/binding/format/protobuf/v2/pb" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/status" + + "github.com/smartcontractkit/chainlink-common/pkg/beholder" + "github.com/smartcontractkit/chainlink-common/pkg/chipingress" + "github.com/smartcontractkit/chainlink-common/pkg/chipingress/pb" + "github.com/smartcontractkit/chainlink-common/pkg/logger" +) + +// mockChipServer implements ChipIngressServer with controllable behaviour. +type mockChipServer struct { + pb.UnimplementedChipIngressServer + + mu sync.Mutex + publishErr error + batchErr error + received []*cepb.CloudEvent + batchReceived [][]*cepb.CloudEvent + publishCount atomic.Int64 + batchCount atomic.Int64 + publishDelay time.Duration +} + +func (s *mockChipServer) Publish(_ context.Context, in *cepb.CloudEvent) (*pb.PublishResponse, error) { + if s.publishDelay > 0 { + time.Sleep(s.publishDelay) + } + s.publishCount.Add(1) + s.mu.Lock() + defer s.mu.Unlock() + if s.publishErr != nil { + return nil, s.publishErr + } + s.received = append(s.received, in) + return &pb.PublishResponse{}, nil +} + +func (s *mockChipServer) PublishBatch(_ context.Context, in *pb.CloudEventBatch) (*pb.PublishResponse, error) { + s.batchCount.Add(1) + s.mu.Lock() + defer s.mu.Unlock() + if s.batchErr != nil { + return nil, s.batchErr + } + s.batchReceived = append(s.batchReceived, in.Events) + s.received = append(s.received, in.Events...) + return &pb.PublishResponse{}, nil +} + +func (s *mockChipServer) Ping(context.Context, *pb.EmptyRequest) (*pb.PingResponse, error) { + return &pb.PingResponse{Message: "pong"}, nil +} + +func (s *mockChipServer) setPublishErr(err error) { + s.mu.Lock() + defer s.mu.Unlock() + s.publishErr = err +} + +func (s *mockChipServer) setBatchErr(err error) { + s.mu.Lock() + defer s.mu.Unlock() + s.batchErr = err +} + +func (s *mockChipServer) receivedCount() int { + s.mu.Lock() + defer s.mu.Unlock() + return len(s.received) +} + +func (s *mockChipServer) batchCallCount() int { + s.mu.Lock() + defer s.mu.Unlock() + return len(s.batchReceived) +} + +// startMockServer starts a gRPC server on a random port and returns the +// server, address, and a cleanup function. +func startMockServer(t *testing.T, srv *mockChipServer) (*grpc.Server, string) { + t.Helper() + lis, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + + gs := grpc.NewServer() + pb.RegisterChipIngressServer(gs, srv) + + go func() { + if err := gs.Serve(lis); err != nil { + // Ignore errors from server being stopped during cleanup. + } + }() + + t.Cleanup(func() { gs.GracefulStop() }) + return gs, lis.Addr().String() +} + +func newChipClient(t *testing.T, addr string) chipingress.Client { + t.Helper() + c, err := chipingress.NewClient(addr, chipingress.WithInsecureConnection()) + require.NoError(t, err) + t.Cleanup(func() { _ = c.Close() }) + return c +} + +func emitAttrs() []any { + return []any{"source", "test-domain", "type", "test-entity"} +} + +func fastCfg() beholder.DurableEmitterConfig { + return beholder.DurableEmitterConfig{ + RetransmitInterval: 100 * time.Millisecond, + RetransmitAfter: 50 * time.Millisecond, + RetransmitBatchSize: 50, + ExpiryInterval: 200 * time.Millisecond, + EventTTL: 500 * time.Millisecond, + PublishTimeout: 2 * time.Second, + } +} + +// ---------- Test cases ---------- + +func TestIntegration_HappyPath(t *testing.T) { + srv := &mockChipServer{} + _, addr := startMockServer(t, srv) + client := newChipClient(t, addr) + store := beholder.NewMemDurableEventStore() + + em, err := beholder.NewDurableEmitter(store, client, fastCfg(), logger.Test(t)) + require.NoError(t, err) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + em.Start(ctx) + defer em.Close() + + require.NoError(t, em.Emit(ctx, []byte("billing-record-1"), emitAttrs()...)) + require.NoError(t, em.Emit(ctx, []byte("billing-record-2"), emitAttrs()...)) + + require.Eventually(t, func() bool { + return srv.receivedCount() == 2 + }, 3*time.Second, 10*time.Millisecond, "server should receive both events") + + require.Eventually(t, func() bool { + return store.Len() == 0 + }, 3*time.Second, 10*time.Millisecond, "store should be empty after delivery") +} + +func TestIntegration_ServerUnavailable_RetransmitRecovers(t *testing.T) { + // Start with server returning UNAVAILABLE. + srv := &mockChipServer{} + srv.setPublishErr(status.Error(codes.Unavailable, "chip down")) + srv.setBatchErr(status.Error(codes.Unavailable, "chip down")) + _, addr := startMockServer(t, srv) + client := newChipClient(t, addr) + store := beholder.NewMemDurableEventStore() + + em, err := beholder.NewDurableEmitter(store, client, fastCfg(), logger.Test(t)) + require.NoError(t, err) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + em.Start(ctx) + defer em.Close() + + require.NoError(t, em.Emit(ctx, []byte("will-retry"), emitAttrs()...)) + + // Event should be in the store, not delivered. + time.Sleep(200 * time.Millisecond) + assert.Equal(t, 1, store.Len(), "event persists while server is unavailable") + + // "Recover" the server. + srv.setPublishErr(nil) + srv.setBatchErr(nil) + + require.Eventually(t, func() bool { + return store.Len() == 0 + }, 5*time.Second, 50*time.Millisecond, "retransmit loop should deliver after recovery") + + assert.GreaterOrEqual(t, srv.batchCount.Load(), int64(1), + "retransmit should use PublishBatch") +} + +func TestIntegration_ServerDown_EventsSurvive(t *testing.T) { + // Start server, then stop it to simulate total outage. + srv := &mockChipServer{} + gs, addr := startMockServer(t, srv) + client := newChipClient(t, addr) + store := beholder.NewMemDurableEventStore() + + cfg := fastCfg() + cfg.PublishTimeout = 500 * time.Millisecond + em, err := beholder.NewDurableEmitter(store, client, cfg, logger.Test(t)) + require.NoError(t, err) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + em.Start(ctx) + + // Stop the gRPC server entirely. + gs.Stop() + time.Sleep(100 * time.Millisecond) + + // Emit while server is down — Emit() itself must succeed (DB insert works). + require.NoError(t, em.Emit(ctx, []byte("server-is-down"), emitAttrs()...)) + assert.Equal(t, 1, store.Len(), "event should be persisted even with server down") + + // Stop the emitter to simulate a "node shutdown". + em.Close() + + // Bring up a new server on the same address. + srv2 := &mockChipServer{} + lis, err := net.Listen("tcp", addr) + require.NoError(t, err) + gs2 := grpc.NewServer() + pb.RegisterChipIngressServer(gs2, srv2) + go func() { _ = gs2.Serve(lis) }() + t.Cleanup(func() { gs2.GracefulStop() }) + + // Create a new client and DurableEmitter re-using the same store + // (simulating node restart with Postgres). + client2, err := chipingress.NewClient(addr, chipingress.WithInsecureConnection()) + require.NoError(t, err) + t.Cleanup(func() { _ = client2.Close() }) + + em2, err := beholder.NewDurableEmitter(store, client2, cfg, logger.Test(t)) + require.NoError(t, err) + em2.Start(ctx) + defer em2.Close() + + require.Eventually(t, func() bool { + return srv2.receivedCount() == 1 + }, 5*time.Second, 50*time.Millisecond, "new emitter should retransmit the surviving event") + + require.Eventually(t, func() bool { + return store.Len() == 0 + }, 5*time.Second, 50*time.Millisecond, "store should be empty after retransmit") +} + +func TestIntegration_HighThroughput(t *testing.T) { + srv := &mockChipServer{} + _, addr := startMockServer(t, srv) + client := newChipClient(t, addr) + store := beholder.NewMemDurableEventStore() + + cfg := fastCfg() + cfg.RetransmitBatchSize = 200 + em, err := beholder.NewDurableEmitter(store, client, cfg, logger.Test(t)) + require.NoError(t, err) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + em.Start(ctx) + defer em.Close() + + const n = 500 + for i := 0; i < n; i++ { + require.NoError(t, em.Emit(ctx, []byte("event"), emitAttrs()...)) + } + + require.Eventually(t, func() bool { + return srv.receivedCount() >= n + }, 10*time.Second, 50*time.Millisecond, "all %d events should be received", n) + + require.Eventually(t, func() bool { + return store.Len() == 0 + }, 10*time.Second, 50*time.Millisecond, "store should drain completely") +} + +func TestIntegration_EventExpiry(t *testing.T) { + // Server always rejects — events can never be delivered. + srv := &mockChipServer{} + srv.setPublishErr(status.Error(codes.Internal, "permanent failure")) + srv.setBatchErr(status.Error(codes.Internal, "permanent failure")) + _, addr := startMockServer(t, srv) + client := newChipClient(t, addr) + store := beholder.NewMemDurableEventStore() + + cfg := fastCfg() + cfg.EventTTL = 100 * time.Millisecond + cfg.ExpiryInterval = 100 * time.Millisecond + em, err := beholder.NewDurableEmitter(store, client, cfg, logger.Test(t)) + require.NoError(t, err) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + em.Start(ctx) + defer em.Close() + + require.NoError(t, em.Emit(ctx, []byte("will-expire"), emitAttrs()...)) + assert.Equal(t, 1, store.Len()) + + require.Eventually(t, func() bool { + return store.Len() == 0 + }, 5*time.Second, 50*time.Millisecond, + "expiry loop should purge undeliverable events after TTL") +} + +func TestIntegration_RetransmitUsesBatch(t *testing.T) { + // Immediate publishes fail, only batch succeeds. + srv := &mockChipServer{} + srv.setPublishErr(status.Error(codes.Unavailable, "reject single")) + _, addr := startMockServer(t, srv) + client := newChipClient(t, addr) + store := beholder.NewMemDurableEventStore() + + em, err := beholder.NewDurableEmitter(store, client, fastCfg(), logger.Test(t)) + require.NoError(t, err) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + em.Start(ctx) + defer em.Close() + + for i := 0; i < 5; i++ { + require.NoError(t, em.Emit(ctx, []byte("batch-me"), emitAttrs()...)) + } + + require.Eventually(t, func() bool { + return store.Len() == 0 + }, 5*time.Second, 50*time.Millisecond, + "retransmit via PublishBatch should deliver all events") + + assert.GreaterOrEqual(t, srv.batchCallCount(), 1, + "at least one PublishBatch call should have been made") +} + +// TestIntegration_GRPCConnection verifies the emitter works over a real gRPC +// connection with proper proto serialization round-trip. +func TestIntegration_GRPCConnection(t *testing.T) { + srv := &mockChipServer{} + _, addr := startMockServer(t, srv) + + // Use a raw gRPC dial to prove we're going over the wire. + conn, err := grpc.NewClient(addr, grpc.WithTransportCredentials(insecure.NewCredentials())) + require.NoError(t, err) + t.Cleanup(func() { _ = conn.Close() }) + + // Ping to verify connectivity. + grpcClient := pb.NewChipIngressClient(conn) + pong, err := grpcClient.Ping(context.Background(), &pb.EmptyRequest{}) + require.NoError(t, err) + assert.Equal(t, "pong", pong.Message) + + // Now use the chipingress.Client wrapper with DurableEmitter. + client := newChipClient(t, addr) + store := beholder.NewMemDurableEventStore() + + em, err := beholder.NewDurableEmitter(store, client, fastCfg(), logger.Test(t)) + require.NoError(t, err) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + em.Start(ctx) + defer em.Close() + + payload := []byte("proto-round-trip-test") + require.NoError(t, em.Emit(ctx, payload, emitAttrs()...)) + + require.Eventually(t, func() bool { + return srv.receivedCount() == 1 + }, 3*time.Second, 10*time.Millisecond) + + // Verify the CloudEvent arrived with correct source/type. + srv.mu.Lock() + received := srv.received[0] + srv.mu.Unlock() + + assert.Equal(t, "test-domain", received.Source) + assert.Equal(t, "test-entity", received.Type) +} From 6da9ad1d3738f5d14b53231ae8fd38142390b501 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Mon, 23 Mar 2026 12:05:16 -0400 Subject: [PATCH 03/45] Update DurableEmitterDesign.md --- pkg/beholder/DurableEmitterDesign.md | 372 +++++++++++++++++++-------- 1 file changed, 271 insertions(+), 101 deletions(-) diff --git a/pkg/beholder/DurableEmitterDesign.md b/pkg/beholder/DurableEmitterDesign.md index 20dc1d3374..304ac78dff 100644 --- a/pkg/beholder/DurableEmitterDesign.md +++ b/pkg/beholder/DurableEmitterDesign.md @@ -2,67 +2,104 @@ ## Problem Statement -Today there is no persistence in the ChIP pipeline. The `ChipIngressEmitter` calls `chipingress.Client.Publish()` synchronously over gRPC, and the `batch.Client` uses an in-memory channel buffer of 200 messages. If the node crashes, Chip is unreachable, or the buffer fills up, events (including billing records) are silently dropped. +Today there is no persistence in the ChIP pipeline. The `ChipIngressEmitter` calls `chipingress.Client.Publish()` synchronously over gRPC, and the `batch.Client` uses an in-memory channel buffer of 200 messages. If the node crashes, Chip is unreachable, or the buffer fills up, events — including billing records — are silently dropped. -**Drop points:** -- `batch.Client` returns `"message buffer is full"` and the event is lost. -- `ChipIngressEmitter` propagates the error up, but `DualSourceEmitter` only logs chip-ingress failures. -- Any in-flight events are lost on node crash — nothing is persisted to disk. +**Drop points in the current architecture:** -## Functional Requirements +``` +DualSourceEmitter.Emit() + ├── OTLP (sync) — errors returned to caller + └── ChipIngressEmitter (async goroutine) + │ + └── chipingress.Client.Publish() ← fire-and-forget gRPC + │ + ├── If Chip is down → error logged, event LOST + ├── If node crashes mid-flight → event LOST + └── batch.Client buffer full → "message buffer is full", event LOST +``` + +- `batch.Client` drops messages when its 200-message channel is full. +- `DualSourceEmitter` only logs chip-ingress failures — errors are swallowed. +- No event survives a node restart. Nothing is persisted to disk. +**Impact:** Billing records are silently dropped, leading to inconsistent revenue reconciliation. Any customer-facing observability that flows through ChIP is unreliable. + +## Requirements + +### Functional - Events must not be lost on node restarts. -- Events must be delivered within a reasonable period of time (seconds, not minutes). -- The system must support eventually-consistent billing. +- Events must be delivered within a reasonable period of time (seconds under normal operation). +- System must support eventually-consistent billing. - Node databases must not bloat unboundedly. -## Non-Functional Requirements - -- Scale to 1k+ TPS. +### Non-Functional +- Scale to 1k+ TPS per node. - 4 nines of availability (99.99%). +- `Emit()` must not block workflow execution. -## Architecture Overview +## Architecture + +### High-Level Flow ``` -Workflow Engine +Workflow Engine / Billing / Lifecycle Events │ ▼ - DurableEmitter.Emit() + DualSourceEmitter.Emit() │ - ├─ 1. Serialize event → proto bytes - ├─ 2. INSERT into Postgres (durable guarantee) - ├─ 3. Return nil (caller unblocked) + ├── OTLP MessageEmitter (sync, unchanged) │ - └─ 4. Async goroutine: chipingress.Client.Publish() - │ - ├─ Success → DELETE from Postgres - └─ Failure → no-op (retransmit loop will handle) - - ┌────────────────────────────┐ - │ Background Retransmit Loop │ (runs every RetransmitInterval) - │ │ - │ ListPending(olderThan) │ - │ → PublishBatch() │ - │ → DELETE on success │ - └────────────────────────────┘ - - ┌────────────────────────────┐ - │ Background Expiry Loop │ (runs every ExpiryInterval) - │ │ - │ DeleteExpired(ttl) │ - │ → GC old events │ - └────────────────────────────┘ + └── DurableEmitter.Emit() + │ + ├─ 1. ExtractSourceAndType + build CloudEventPb + ├─ 2. proto.Marshal → bytes + ├─ 3. store.Insert(payload) ← DURABLE GUARANTEE + ├─ 4. return nil (caller unblocked) + │ + └─ 5. goroutine: client.Publish(eventPb) + ├── Success → store.Delete(id) + └── Failure → no-op (retransmit loop handles it) + + ┌─────────────────────────────────────┐ + │ Background Retransmit Loop │ every 5s (configurable) + │ │ + │ store.ListPending(olderThan 10s) │ + │ → client.PublishBatch(events) │ + │ → store.Delete(ids) on success │ + └─────────────────────────────────────┘ + + ┌─────────────────────────────────────┐ + │ Background Expiry Loop │ every 1min (configurable) + │ │ + │ store.DeleteExpired(ttl=24h) │ + │ → GC events that could never be │ + │ delivered (bounds table growth) │ + └─────────────────────────────────────┘ ``` -## Key Design Decision: Use Standard `chipingress.Client`, Not `batch.Client` +### Key Guarantee + +`Emit()` returns `nil` once the Postgres INSERT succeeds. Even if the node crashes immediately after, the event survives in Postgres and will be retransmitted on restart. The gRPC publish is fully asynchronous — `Emit()` latency is dominated by one DB insert (~1ms at typical payloads). + +### Design Decision: Standard `chipingress.Client`, Not `batch.Client` + +Per Hagen's guidance, we use the standard `chipingress.Client` directly (supports both `Publish` and `PublishBatch`) since we are implementing our own queuing with persistence-backed guarantees. The `batch.Client`'s in-memory buffer is redundant when we have Postgres as the durable queue. + +### Service Principal & ACK Guarantees -Per Hagen's guidance, we use the standard `chipingress.Client` directly (which supports both `Publish` and `PublishBatch`) since we are implementing our own queuing mechanism with persistence-backed guarantees. The `batch.Client`'s in-memory buffer would be redundant. +CRE nodes authenticate to ChIP using the node's **CSA Key** as the `servicePrincipal`. This is NOT the `oti-telemetry-shared` principal which uses a fire-and-forget publish path. With the CSA Key, the gateway waits for Kafka ACKs before returning a gRPC response — a successful response means the event was durably accepted. ## Components -### DurableEventStore (interface — `chainlink-common`) +### DurableEventStore Interface (`chainlink-common`) ```go +type DurableEvent struct { + ID int64 + Payload []byte // serialized CloudEventPb proto + CreatedAt time.Time +} + type DurableEventStore interface { Insert(ctx context.Context, payload []byte) (int64, error) Delete(ctx context.Context, id int64) error @@ -71,56 +108,55 @@ type DurableEventStore interface { } ``` -- `Insert` persists a serialized `CloudEventPb` and returns an auto-generated ID. -- `Delete` removes a single event after confirmed delivery. -- `ListPending` returns events older than a cutoff (gives the immediate-publish path time to succeed before retransmit picks it up). -- `DeleteExpired` garbage-collects events older than the TTL to bound table growth. - Two implementations: -- **`MemDurableEventStore`** — in-memory, for unit tests (lives in `chainlink-common`). -- **`PgDurableEventStore`** — Postgres-backed ORM (lives in `chainlink`). +- **`MemDurableEventStore`** — in-memory map, for unit/integration tests. Lives in `chainlink-common`. +- **`PgDurableEventStore`** — Postgres-backed ORM using `sqlutil.DataSource`. Lives in `chainlink`. ### DurableEmitter (`chainlink-common`) -Implements `beholder.Emitter`: +Implements `beholder.Emitter` (`Emit` + `Close`). Core logic: + ```go -type Emitter interface { - Emit(ctx context.Context, body []byte, attrKVs ...any) error - io.Closer +func (d *DurableEmitter) Emit(ctx context.Context, body []byte, attrKVs ...any) error { + // 1. Validate and extract source/type from attributes + sourceDomain, entityType, err := ExtractSourceAndType(attrKVs...) + + // 2. Build CloudEvent and serialize to proto bytes + event, _ := chipingress.NewEvent(sourceDomain, entityType, body, newAttributes(attrKVs...)) + eventPb, _ := chipingress.EventToProto(event) + payload, _ := proto.Marshal(eventPb) + + // 3. Persist — this is the durable guarantee + id, err := d.store.Insert(ctx, payload) + if err != nil { + return fmt.Errorf("failed to persist event: %w", err) + } + + // 4. Async delivery attempt + go d.publishAndDelete(id, eventPb) + return nil } ``` -**`Emit()` flow (synchronous path):** -1. `ExtractSourceAndType(attrKVs...)` — validate and extract source/type. -2. `chipingress.NewEvent(...)` + `EventToProto(...)` — build the CloudEvent proto. -3. `proto.Marshal(eventPb)` — serialize to bytes. -4. `store.Insert(ctx, payload)` — persist. **Emit returns nil here.** - -**Async delivery path (goroutine):** -5. `client.Publish(ctx, eventPb)` — attempt immediate delivery. -6. On success: `store.Delete(id)`. -7. On failure: log at debug level; retransmit loop handles it. - -**Key guarantee:** `Emit()` returns `nil` once the DB insert succeeds. Even if the node crashes immediately after, the event survives in Postgres and will be retransmitted on restart. - -`Emit()` is non-blocking for workflow execution because the expensive operation (gRPC publish) happens asynchronously. The synchronous path is: attribute extraction → proto serialization → one DB insert. At 1k TPS, Postgres handles this trivially. - ### Configuration ```go type DurableEmitterConfig struct { - RetransmitInterval time.Duration // default 5s - RetransmitAfter time.Duration // default 10s (min age before retry) - RetransmitBatchSize int // default 100 - ExpiryInterval time.Duration // default 1min - EventTTL time.Duration // default 24h - PublishTimeout time.Duration // default 5s + RetransmitInterval time.Duration // default 5s — retransmit loop tick rate + RetransmitAfter time.Duration // default 10s — min age before retry + RetransmitBatchSize int // default 100 — max events per batch + ExpiryInterval time.Duration // default 1min — expiry loop tick rate + EventTTL time.Duration // default 24h — max event age + PublishTimeout time.Duration // default 5s — per-RPC deadline } ``` -## Postgres Table +## Postgres Schema + +**Migration `0295_chip_durable_events.sql`** in the existing `cre` schema: ```sql +-- +goose Up CREATE TABLE IF NOT EXISTS cre.chip_durable_events ( id BIGSERIAL PRIMARY KEY, payload BYTEA NOT NULL, @@ -129,60 +165,194 @@ CREATE TABLE IF NOT EXISTS cre.chip_durable_events ( CREATE INDEX idx_chip_durable_events_created_at ON cre.chip_durable_events (created_at ASC); + +-- +goose Down +DROP INDEX IF EXISTS cre.idx_chip_durable_events_created_at; +DROP TABLE IF EXISTS cre.chip_durable_events; ``` -The table lives in the existing `cre` schema. The `created_at` index supports efficient `ListPending` and `DeleteExpired` queries. +The table lives in each node's existing Postgres database. Under normal operation it is **transient** — events are inserted and deleted within milliseconds. Under Chip outage, events accumulate until delivery resumes. -## Service Principal & ACK Guarantees +## Node Wiring -CRE nodes authenticate to ChIP using the node's CSA Key as the `servicePrincipal`. This is **not** the `oti-telemetry-shared` principal, which uses a fire-and-forget publish path. With the CSA Key principal, the gateway waits for Kafka ACKs before returning a response, so a successful gRPC response (`200`) means the event was durably accepted by the gateway. +### Config Flag -## Files +```toml +[Telemetry] +DurableEmitterEnabled = true +``` -| Repo | File | Purpose | -|------|------|---------| -| chainlink-common | `pkg/beholder/durable_event_store.go` | `DurableEventStore` interface + `MemDurableEventStore` | -| chainlink-common | `pkg/beholder/durable_emitter.go` | `DurableEmitter` struct, `Emit`, retransmit loop, expiry loop | -| chainlink-common | `pkg/beholder/durable_emitter_test.go` | Unit tests with in-memory store | -| chainlink | `core/services/beholder/durable_event_store_orm.go` | `PgDurableEventStore` implementing `DurableEventStore` | -| chainlink | `core/store/migrate/migrations/0294_chip_durable_events.sql` | Postgres migration | +Added to `config.Telemetry` interface, `toml.Telemetry` struct, and `telemetryConfig` implementation. -## Wiring (TODO) +### Integration Point (`application.go`) -In `core/cmd/shell.go`, where the beholder client is created, replace or wrap the `ChipIngressEmitter` with `DurableEmitter`: +Wired in `NewApplication` after the DB is available but before CRE services start: ```go -// After creating chipIngressClient... -pgStore := beholder.NewPgDurableEventStore(ds) -durableEmitter, err := beholder.NewDurableEmitter(pgStore, chipIngressClient, cfg, log) -durableEmitter.Start(ctx) -// Use durableEmitter as the chip-ingress emitter in DualSourceEmitter +func setupDurableEmitter(ctx context.Context, ds sqlutil.DataSource, lggr logger.SugaredLogger) error { + client := beholder.GetClient() + chipClient := client.Chip + + pgStore := beholdersvc.NewPgDurableEventStore(ds) + durableEmitter, _ := beholder.NewDurableEmitter(pgStore, chipClient, beholder.DefaultDurableEmitterConfig(), lggr) + + // Preserve OTLP path alongside durable chip delivery + messageLogger := client.MessageLoggerProvider.Logger("durable-emitter") + otlpEmitter := beholder.NewMessageEmitter(messageLogger) + dualEmitter, _ := beholder.NewDualSourceEmitter(durableEmitter, otlpEmitter) + + durableEmitter.Start(ctx) + client.Emitter = dualEmitter + return nil +} +``` + +This replaces the global beholder emitter, covering **all** emission paths: +- `events.emitProtoMessage()` — billing, workflow execution lifecycle +- `custmsg.Labeler.Emit()` — workflow user logs +- `BridgeStatusReporter` — bridge status events +- Any other `beholder.GetEmitter()` caller + +### CRE Environment Auto-Enable + +`system-tests/lib/cre/don/config/config.go` sets `DurableEmitterEnabled = true` for all Docker-based nodesets, so it activates automatically in local CRE environments. + +## File Manifest + +| Repo | File | Purpose | +|------|------|---------| +| chainlink-common | `pkg/beholder/durable_event_store.go` | `DurableEventStore` interface + `MemDurableEventStore` | +| chainlink-common | `pkg/beholder/durable_emitter.go` | `DurableEmitter` — Emit, retransmit loop, expiry loop | +| chainlink-common | `pkg/beholder/durable_emitter_test.go` | Unit tests (in-memory store) | +| chainlink-common | `pkg/beholder/durable_emitter_integration_test.go` | Integration tests (mock gRPC server) | +| chainlink | `core/services/beholder/durable_event_store_orm.go` | `PgDurableEventStore` (Postgres ORM) | +| chainlink | `core/services/beholder/durable_event_store_orm_test.go` | ORM tests + Postgres benchmarks + load tests | +| chainlink | `core/services/beholder/durable_emitter_load_test.go` | Full-stack load tests (Postgres + mock gRPC) | +| chainlink | `core/store/migrate/migrations/0295_chip_durable_events.sql` | Postgres migration | +| chainlink | `core/config/telemetry_config.go` | `DurableEmitterEnabled()` on Telemetry interface | +| chainlink | `core/config/toml/types.go` | TOML field + setFrom merge | +| chainlink | `core/services/chainlink/config_telemetry.go` | Config accessor | +| chainlink | `core/services/chainlink/application.go` | `setupDurableEmitter` wiring | +| chainlink | `system-tests/lib/cre/don/config/config.go` | Auto-enable in CRE Docker envs | +| chainlink | `system-tests/tests/smoke/cre/v2_durable_emitter_test.go` | CRE smoke tests + load test | +| chainlink | `system-tests/tests/smoke/cre/cre_suite_test.go` | Test entry points | + +## Testing + +### Unit Tests (`chainlink-common`, in-memory store) + +| Test | What It Proves | +|------|---------------| +| `TestDurableEmitter_EmitPersistsAndPublishes` | Happy path: emit → publish → delete | +| `TestDurableEmitter_EmitReturnSuccessEvenWhenPublishFails` | Emit succeeds on DB insert even when gRPC fails | +| `TestDurableEmitter_RetransmitLoopDeliversFailedEvents` | Background loop retries failed events | +| `TestDurableEmitter_ExpiryLoopDeletesOldEvents` | TTL-based garbage collection | +| `TestDurableEmitter_EmitRejectsInvalidAttributes` | Validation before DB insert | +| `TestDurableEmitter_MultipleEvents` | 50 concurrent events all delivered | + +### Integration Tests (`chainlink-common`, mock gRPC server) + +Real gRPC server with controllable failure injection: + +| Test | What It Proves | +|------|---------------| +| `TestIntegration_HappyPath` | Events delivered over real gRPC + proto round-trip | +| `TestIntegration_ServerUnavailable_RetransmitRecovers` | Server returns UNAVAILABLE → retransmit delivers via PublishBatch | +| `TestIntegration_ServerDown_EventsSurvive` | **Crash recovery**: server stopped → events persist → new emitter (same store) retransmits on "restart" | +| `TestIntegration_HighThroughput` | 500 events delivered concurrently | +| `TestIntegration_EventExpiry` | Undeliverable events expired after TTL | +| `TestIntegration_RetransmitUsesBatch` | Retransmit path uses PublishBatch, not individual Publish | +| `TestIntegration_GRPCConnection` | Source/type arrive correctly on server side | + +### Postgres ORM Tests + Benchmarks (`chainlink`, real Postgres) + +| Test / Benchmark | What It Measures | +|---|---| +| `TestPgDurableEventStore_*` | ORM correctness (insert, list, delete, expiry) | +| `Benchmark_Insert` | Raw INSERT throughput | +| `Benchmark_InsertDelete` | Insert+delete cycle (happy-path hot loop) | +| `Benchmark_InsertPayloadSizes` | INSERT at 64B, 256B, 1KB, 4KB | +| `Benchmark_ListPending` | Query performance at 100 and 1000 queue depth | +| `TestLoad_SustainedInsertDelete` | 2000 events, 10-way concurrent insert+delete, measures ops/sec | +| `TestLoad_BurstThenDrain` | 1000-event burst, then drain via ListPending+Delete batches | +| `TestLoad_ConcurrentInsertWithListPending` | 3s of concurrent inserts + ListPending (real contention) | + +### Full-Stack Load Tests (`chainlink`, Postgres + mock gRPC) + +| Test / Benchmark | What It Measures | +|---|---| +| `TestFullStack_SustainedThroughput` | 1000 events, 10 concurrent emitters, end-to-end rate | +| `TestFullStack_ChipOutage` | 3-phase: normal → Chip goes UNAVAILABLE → recovery. Measures accumulation and drain rate | +| `TestFullStack_SlowChip` | 50ms gRPC latency. Proves Emit() stays fast while server is slow | +| `Benchmark_FullStack_EmitThroughput` | Upper bound events/sec through full pipeline | +| `Benchmark_FullStack_EmitPayloadSizes` | Full emit at 64B, 256B, 1KB, 4KB | + +### CRE Smoke Tests (live Docker environment) + +Tests connect to the node's Postgres and query `cre.chip_durable_events` directly, using `pg_stat_user_tables` for insert/delete statistics — the same pattern used by the EVM LogTrigger test for `trigger_pending_events`. + +| Test | What It Does | +|------|-------------| +| `Test_CRE_V2_DurableEmitter` | Deploys a cron workflow (every 5s), waits for 30+ insert+delete cycles, verifies queue drains to near-empty | +| `Test_CRE_V2_DurableEmitter_Load` | Deploys 5 cron workflows (every 1s each), runs for 3 minutes. Logs insert/delete rates, max queue depth, and prints summary table | + +**Running CRE smoke tests:** +```bash +# Basic correctness +go test -v -run Test_CRE_V2_DurableEmitter$ -timeout 10m + +# Load test (5 workflows × 1s cron, 3min observation) +go test -v -run Test_CRE_V2_DurableEmitter_Load -timeout 10m +``` + +**Example load test output:** +``` +╔════════════════════════════════════════════════╗ +║ DURABLE EMITTER LOAD TEST RESULTS ║ +╠════════════════════════════════════════════════╣ +║ Workflows deployed: 5 ║ +║ Observation period: 3m0s ║ +║ Total inserts: 1842 ║ +║ Total deletes: 1840 ║ +║ Avg insert rate: 10.2 events/sec ║ +║ Avg delete rate: 10.2 events/sec ║ +║ Max queue depth: 12 ║ +║ Final pending: 2 ║ +╚════════════════════════════════════════════════╝ ``` ## Metrics to Instrument (Future) | Metric | Description | |--------|-------------| -| `durable_emitter.queue_depth` | Number of events currently in the store | +| `durable_emitter.queue_depth` | Current row count in `chip_durable_events` | | `durable_emitter.insert_rate` | Events persisted per second | | `durable_emitter.publish_rate` | Events successfully delivered per second | -| `durable_emitter.retransmit_rate` | Events retransmitted per second | +| `durable_emitter.retransmit_rate` | Events retransmitted via background loop | | `durable_emitter.publish_latency` | Time from insert to confirmed delivery | | `durable_emitter.oldest_pending` | Age of the longest-waiting event | | `durable_emitter.expired_count` | Events expired (dropped after TTL) | | `durable_emitter.error_rate` | Failed publish attempts per second | -## Open Questions +## Open Questions & Future Work + +### 1. Chip Gateway Idempotency +Does the gateway deduplicate re-sent events? If the retransmit loop re-sends an event that the immediate path already delivered (race window), the gateway should de-dup using the CloudEvent `id` (UUID). Needs server-side confirmation. -1. **Chip gateway idempotency** — Does the gateway deduplicate re-sent events? If the retransmit loop re-sends an event that the immediate path already delivered (race window), will the gateway de-dup or create a duplicate billing record? The CloudEvent `id` field (UUID) could serve as a dedup key. +### 2. DB Load at Scale +At 1k TPS: ~1k inserts/sec + ~1k deletes/sec = ~2k write ops/sec on the node's Postgres. This produces dead tuples requiring autovacuum tuning. Potential optimizations: +- **Batch deletes** — delete by ID list instead of per-row. +- **Two-table approach** — queued + recently-sent to reduce churn on the hot table. +- **CDC streaming** — stream WAL changes directly, avoiding the insert/delete pattern entirely. Matthew Gardener and Clement can advise on CDC implementation. -2. **DB load at scale** — At 1k TPS: ~1k inserts/sec + ~1k deletes/sec. The delete-heavy workload will produce dead tuples requiring autovacuum tuning. Potential optimizations: - - Batch deletes (delete by ID list instead of per-row). - - Two-table approach (queued + recently-sent) to reduce churn. - - CDC streaming as an alternative to the insert/delete pattern entirely. +### 3. Exponential Backoff +Current PoC uses a fixed retransmit interval. Production should implement per-event exponential backoff using `attempts` and `last_sent_at` columns (schema extension). -3. **Exponential backoff** — Current PoC uses a fixed retransmit interval. Production should implement per-event exponential backoff using `attempts` and `last_sent_at` columns (schema extension). +### 4. rmq / Redis Alternative +Patrick raised using [rmq](https://github.com/wellle/rmq) backed by our own DB instead of re-implementing a queue. Worth evaluating if the Postgres-backed approach shows scaling issues in load testing. -4. **rmq / Redis alternative** — Patrick raised using [rmq](https://github.com/wellle/rmq) backed by our own DB instead of re-implementing a queue. Worth evaluating if the Postgres-backed approach has scaling issues. +### 5. CDC Streaming +Could stream WAL changes directly rather than polling the table, avoiding the insert/delete churn entirely. This would also enable real-time analytics on event flow. Requires infrastructure coordination with the data analytics pipeline team. -5. **CDC streaming** — Could stream WAL changes directly rather than polling the table, avoiding the insert/delete churn entirely. Matthew Gardener and Clement can advise on CDC implementation within the existing data analytics pipeline. +### 6. DurableEmitter Lifecycle Management +Currently the `DurableEmitter` is started in `application.go` and its background loops are tied to the application context. For production, it should be registered as a proper `services.ServiceCtx` with Start/Close lifecycle management, health checks, and graceful shutdown (flush pending events before stopping). From a268cb0fc04512f38bd5dea377a5721b08cefbcc Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Mon, 23 Mar 2026 13:22:41 -0400 Subject: [PATCH 04/45] Update doc with tests --- pkg/beholder/DurableEmitterDesign.md | 61 +++++++++++++++++++++++++++- 1 file changed, 60 insertions(+), 1 deletion(-) diff --git a/pkg/beholder/DurableEmitterDesign.md b/pkg/beholder/DurableEmitterDesign.md index 304ac78dff..2be3e009b9 100644 --- a/pkg/beholder/DurableEmitterDesign.md +++ b/pkg/beholder/DurableEmitterDesign.md @@ -227,7 +227,7 @@ This replaces the global beholder emitter, covering **all** emission paths: | chainlink-common | `pkg/beholder/durable_emitter_integration_test.go` | Integration tests (mock gRPC server) | | chainlink | `core/services/beholder/durable_event_store_orm.go` | `PgDurableEventStore` (Postgres ORM) | | chainlink | `core/services/beholder/durable_event_store_orm_test.go` | ORM tests + Postgres benchmarks + load tests | -| chainlink | `core/services/beholder/durable_emitter_load_test.go` | Full-stack load tests (Postgres + mock gRPC) | +| chainlink | `core/services/beholder/durable_emitter_load_test.go` | TPS ramp/sustained/payload tests (Postgres + mock or external Chip via `CHIP_INGRESS_TEST_ADDR`) | | chainlink | `core/store/migrate/migrations/0295_chip_durable_events.sql` | Postgres migration | | chainlink | `core/config/telemetry_config.go` | `DurableEmitterEnabled()` on Telemetry interface | | chainlink | `core/config/toml/types.go` | TOML field + setFrom merge | @@ -287,6 +287,65 @@ Real gRPC server with controllable failure injection: | `Benchmark_FullStack_EmitThroughput` | Upper bound events/sec through full pipeline | | `Benchmark_FullStack_EmitPayloadSizes` | Full emit at 64B, 256B, 1KB, 4KB | +### Durable emitter TPS load tests (`chainlink/core/services/beholder/durable_emitter_load_test.go`) + +These tests exercise **Postgres + `DurableEmitter` + Chip Ingress** (in-process mock **or** a real gateway). They are heavier than the ORM benchmarks and require a **real Postgres** (not `txdb`). + +#### Prerequisites + +- **`CL_DATABASE_URL`** — must point at a Postgres instance where migration **`0295_chip_durable_events`** has been applied (`cre.chip_durable_events` exists). Same URL pattern as other chainlink DB tests. +- **Short tests skipped** — if your test runner uses `-short`, these tests are skipped (`SkipShortDB`); run **without** `-short`. + +#### Mock Chip vs real Chip Ingress + +| Mode | How | Notes | +|------|-----|--------| +| **Mock** (default) | Do **not** set `CHIP_INGRESS_TEST_ADDR` | In-process gRPC server; tests can count **Server recv** events and inject failures (outage, slow Chip). | +| **Real Chip** | Set `CHIP_INGRESS_TEST_ADDR=host:port` | Dials external Chip Ingress. Optional: `CHIP_INGRESS_TEST_TLS`, `CHIP_INGRESS_TEST_BASIC_AUTH_*`, `CHIP_INGRESS_TEST_SKIP_BASIC_AUTH`, `CHIP_INGRESS_TEST_SKIP_SCHEMA_REGISTRATION`. You need Kafka/Redpanda, topic **`chip-demo`**, and schema subject **`chip-demo-pb.DemoClientPayload`** (e.g. Atlas `make create-topic-and-schema` under `atlas/chip-ingress`). | + +Tests that **inject** Chip failures or rely on **in-process** receive counts are **skipped** when `CHIP_INGRESS_TEST_ADDR` is set. + +#### How to run + +From the `chainlink` repo root (examples): + +```bash +# All beholder tests including TPS (requires CL_DATABASE_URL) +export CL_DATABASE_URL='postgres://...' +go test -v -count=1 ./core/services/beholder/ -run 'TestTPS_|TestChipIngressExternalPing' + +# Ramp-up only (100 → 500 → 1k → 2k TPS levels) +go test -v -count=1 ./core/services/beholder/ -run TestTPS_RampUp + +# Sustained 1k TPS for 60s + drain check +go test -v -count=1 ./core/services/beholder/ -run TestTPS_Sustained1k + +# Payload size scaling (fixed duration per size) +go test -v -count=1 ./core/services/beholder/ -run TestTPS_PayloadSizeScaling + +# External Chip smoke (with addr set) +export CHIP_INGRESS_TEST_ADDR='localhost:50051' +go test -v -count=1 ./core/services/beholder/ -run TestChipIngressExternalPing +``` + +After a full package run, **`TestMain`** prints a **TPS LOAD TEST SUMMARY** block aggregating result blocks from **`TestTPS_RampUp`**, **`TestTPS_Sustained1k`**, **`TestTPS_1k_WithChipOutage`** (mock only; skipped with external Chip), and **`TestTPS_PayloadSizeScaling`**. + +#### Reading the tables (column glossary) + +| Column | Meaning | +|--------|---------| +| **Target TPS** | Requested emit rate (token-bucket style scheduling across workers). | +| **Achieved TPS** | `Total emits ÷ window duration` — realized successful `Emit()` throughput. | +| **Total emits** | Count of **`Emit()` calls that returned `nil`** in the measurement window (successful Postgres insert path). Does not count failures. | +| **Emit p50 / p99** | Latency of successful `Emit()` calls (dominated by DB insert). | +| **Failures** | `Emit()` calls that returned an error (e.g. DB failure). | +| **Server recv** | **Mock only:** number of events observed by the in-process gRPC server (`Publish` / `PublishBatch`). | +| **Queue depth** | Rows remaining in `cre.chip_durable_events` after the emit phase (+ short settle), i.e. backlog not yet deleted after successful publish. | + +#### Why **Server recv** shows **N/A** with real Chip + +The **Server recv** column is implemented by counting events on the **in-process mock** `ChipIngress` server. When you use **`CHIP_INGRESS_TEST_ADDR`**, there is no mock — the client talks to a **real** gateway — so the test **cannot** count server-side receives in-process. Use **Kafka / Chip / gateway metrics** (or consumer verification) to validate end-to-end delivery instead. **Total emits** and **Achieved TPS** still reflect client-side durable insert success; they are not replaced by N/A. + ### CRE Smoke Tests (live Docker environment) Tests connect to the node's Postgres and query `cre.chip_durable_events` directly, using `pg_stat_user_tables` for insert/delete statistics — the same pattern used by the EVM LogTrigger test for `trigger_pending_events`. From ae0b37bc9f88a0e787c83b117678df25977ff223 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Wed, 25 Mar 2026 10:15:47 -0400 Subject: [PATCH 05/45] Add hooks --- pkg/beholder/durable_emitter.go | 42 ++++++++++++++++++++++--- pkg/beholder/durable_emitter_test.go | 46 ++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+), 4 deletions(-) diff --git a/pkg/beholder/durable_emitter.go b/pkg/beholder/durable_emitter.go index beb7161135..baba1bf43c 100644 --- a/pkg/beholder/durable_emitter.go +++ b/pkg/beholder/durable_emitter.go @@ -27,6 +27,21 @@ type DurableEmitterConfig struct { EventTTL time.Duration // PublishTimeout is the per-RPC deadline for Publish / PublishBatch calls. PublishTimeout time.Duration + // Hooks is optional instrumentation (load tests, profiling). Nil fields are skipped. + // Callbacks may run from many goroutines; implementations must be thread-safe. + Hooks *DurableEmitterHooks +} + +// DurableEmitterHooks records Publish vs Delete latency to locate pipeline bottlenecks. +type DurableEmitterHooks struct { + // OnImmediatePublish is called after each async Publish in publishAndDelete (every attempt). + OnImmediatePublish func(elapsed time.Duration, err error) + // OnImmediateDelete is called after Delete following a successful immediate Publish. + OnImmediateDelete func(elapsed time.Duration, err error) + // OnRetransmitBatchPublish is called after each retransmit PublishBatch. + OnRetransmitBatchPublish func(elapsed time.Duration, eventCount int, err error) + // OnRetransmitBatchDeletes is called once per successful batch with total time for the delete loop. + OnRetransmitBatchDeletes func(elapsed time.Duration, deleteCount int) } func DefaultDurableEmitterConfig() DurableEmitterConfig { @@ -140,14 +155,24 @@ func (d *DurableEmitter) publishAndDelete(id int64, eventPb *chipingress.CloudEv ctx, cancel := context.WithTimeout(context.Background(), d.cfg.PublishTimeout) defer cancel() - if _, err := d.client.Publish(ctx, eventPb); err != nil { + t0 := time.Now() + _, err := d.client.Publish(ctx, eventPb) + if h := d.cfg.Hooks; h != nil && h.OnImmediatePublish != nil { + h.OnImmediatePublish(time.Since(t0), err) + } + if err != nil { d.log.Debugw("immediate publish failed, retransmit loop will retry", "id", id, "error", err) return } - if err := d.store.Delete(context.Background(), id); err != nil { - d.log.Errorw("failed to delete delivered event", "id", id, "error", err) + t1 := time.Now() + delErr := d.store.Delete(context.Background(), id) + if h := d.cfg.Hooks; h != nil && h.OnImmediateDelete != nil { + h.OnImmediateDelete(time.Since(t1), delErr) + } + if delErr != nil { + d.log.Errorw("failed to delete delivered event", "id", id, "error", delErr) } } @@ -199,16 +224,25 @@ func (d *DurableEmitter) retransmitPending(ctx context.Context) { publishCtx, cancel := context.WithTimeout(ctx, d.cfg.PublishTimeout) defer cancel() - if _, err := d.client.PublishBatch(publishCtx, &chipingress.CloudEventBatch{Events: events}); err != nil { + tPub := time.Now() + _, err := d.client.PublishBatch(publishCtx, &chipingress.CloudEventBatch{Events: events}) + if h := d.cfg.Hooks; h != nil && h.OnRetransmitBatchPublish != nil { + h.OnRetransmitBatchPublish(time.Since(tPub), len(events), err) + } + if err != nil { d.log.Warnw("retransmit batch failed", "count", len(events), "error", err) return } + tDel := time.Now() for _, id := range ids { if err := d.store.Delete(ctx, id); err != nil { d.log.Errorw("failed to delete retransmitted event", "id", id, "error", err) } } + if h := d.cfg.Hooks; h != nil && h.OnRetransmitBatchDeletes != nil { + h.OnRetransmitBatchDeletes(time.Since(tDel), len(ids)) + } d.log.Debugw("retransmitted events", "count", len(ids)) } diff --git a/pkg/beholder/durable_emitter_test.go b/pkg/beholder/durable_emitter_test.go index 9c3f2bf498..d6b978c983 100644 --- a/pkg/beholder/durable_emitter_test.go +++ b/pkg/beholder/durable_emitter_test.go @@ -68,6 +68,52 @@ func newTestDurableEmitter(t *testing.T, store DurableEventStore, client chiping return em } +func TestDurableEmitter_HooksImmediatePath(t *testing.T) { + store := NewMemDurableEventStore() + client := &testChipClient{} + var pubCalls, delCalls atomic.Int32 + cfg := DefaultDurableEmitterConfig() + cfg.Hooks = &DurableEmitterHooks{ + OnImmediatePublish: func(time.Duration, error) { pubCalls.Add(1) }, + OnImmediateDelete: func(time.Duration, error) { delCalls.Add(1) }, + } + em, err := NewDurableEmitter(store, client, cfg, logger.Test(t)) + require.NoError(t, err) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + em.Start(ctx) + defer em.Close() + + require.NoError(t, em.Emit(ctx, []byte("hello"), testEmitAttrs()...)) + require.Eventually(t, func() bool { return store.Len() == 0 }, 2*time.Second, 10*time.Millisecond) + assert.Equal(t, int32(1), pubCalls.Load()) + assert.Equal(t, int32(1), delCalls.Load()) +} + +func TestDurableEmitter_HooksPublishFailureSkipsDeleteHook(t *testing.T) { + store := NewMemDurableEventStore() + client := &testChipClient{} + client.setPublishErr(errors.New("down")) + var pubCalls, delCalls atomic.Int32 + cfg := DefaultDurableEmitterConfig() + cfg.Hooks = &DurableEmitterHooks{ + OnImmediatePublish: func(time.Duration, error) { pubCalls.Add(1) }, + OnImmediateDelete: func(time.Duration, error) { delCalls.Add(1) }, + } + em, err := NewDurableEmitter(store, client, cfg, logger.Test(t)) + require.NoError(t, err) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + em.Start(ctx) + defer em.Close() + + require.NoError(t, em.Emit(ctx, []byte("hello"), testEmitAttrs()...)) + require.Eventually(t, func() bool { return pubCalls.Load() == 1 }, 2*time.Second, 10*time.Millisecond) + assert.Equal(t, int32(0), delCalls.Load()) +} + func TestDurableEmitter_EmitPersistsAndPublishes(t *testing.T) { store := NewMemDurableEventStore() client := &testChipClient{} From f588de91c2db0c7037585b31e42eb818c874bce4 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Wed, 25 Mar 2026 10:18:49 -0400 Subject: [PATCH 06/45] Update durable_emitter.go --- pkg/beholder/durable_emitter.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/beholder/durable_emitter.go b/pkg/beholder/durable_emitter.go index baba1bf43c..788dadf643 100644 --- a/pkg/beholder/durable_emitter.go +++ b/pkg/beholder/durable_emitter.go @@ -225,7 +225,7 @@ func (d *DurableEmitter) retransmitPending(ctx context.Context) { defer cancel() tPub := time.Now() - _, err := d.client.PublishBatch(publishCtx, &chipingress.CloudEventBatch{Events: events}) + _, err = d.client.PublishBatch(publishCtx, &chipingress.CloudEventBatch{Events: events}) if h := d.cfg.Hooks; h != nil && h.OnRetransmitBatchPublish != nil { h.OnRetransmitBatchPublish(time.Since(tPub), len(events), err) } From d9220483c788620670cfecfdfbf8574274eb0a31 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Wed, 25 Mar 2026 11:22:11 -0400 Subject: [PATCH 07/45] Add metrics --- pkg/beholder/DurableEmitterDesign.md | 13 +- pkg/beholder/durable_emitter.go | 112 ++++++++++++- pkg/beholder/durable_emitter_cpu_other.go | 9 ++ pkg/beholder/durable_emitter_cpu_unix.go | 22 +++ pkg/beholder/durable_emitter_metric_info.go | 117 ++++++++++++++ pkg/beholder/durable_emitter_metrics.go | 168 ++++++++++++++++++++ pkg/beholder/durable_emitter_store_wrap.go | 58 +++++++ pkg/beholder/durable_emitter_test.go | 54 +++++++ pkg/beholder/durable_event_store.go | 54 ++++++- 9 files changed, 593 insertions(+), 14 deletions(-) create mode 100644 pkg/beholder/durable_emitter_cpu_other.go create mode 100644 pkg/beholder/durable_emitter_cpu_unix.go create mode 100644 pkg/beholder/durable_emitter_metric_info.go create mode 100644 pkg/beholder/durable_emitter_metrics.go create mode 100644 pkg/beholder/durable_emitter_store_wrap.go diff --git a/pkg/beholder/DurableEmitterDesign.md b/pkg/beholder/DurableEmitterDesign.md index 2be3e009b9..7f34cca20b 100644 --- a/pkg/beholder/DurableEmitterDesign.md +++ b/pkg/beholder/DurableEmitterDesign.md @@ -300,7 +300,7 @@ These tests exercise **Postgres + `DurableEmitter` + Chip Ingress** (in-process | Mode | How | Notes | |------|-----|--------| -| **Mock** (default) | Do **not** set `CHIP_INGRESS_TEST_ADDR` | In-process gRPC server; tests can count **Server recv** events and inject failures (outage, slow Chip). | +| **Mock** (default) | Do **not** set `CHIP_INGRESS_TEST_ADDR` | In-process gRPC server; tests can inject failures (outage, slow Chip). | | **Real Chip** | Set `CHIP_INGRESS_TEST_ADDR=host:port` | Dials external Chip Ingress. Optional: `CHIP_INGRESS_TEST_TLS`, `CHIP_INGRESS_TEST_BASIC_AUTH_*`, `CHIP_INGRESS_TEST_SKIP_BASIC_AUTH`, `CHIP_INGRESS_TEST_SKIP_SCHEMA_REGISTRATION`. You need Kafka/Redpanda, topic **`chip-demo`**, and schema subject **`chip-demo-pb.DemoClientPayload`** (e.g. Atlas `make create-topic-and-schema` under `atlas/chip-ingress`). | Tests that **inject** Chip failures or rely on **in-process** receive counts are **skipped** when `CHIP_INGRESS_TEST_ADDR` is set. @@ -338,13 +338,12 @@ After a full package run, **`TestMain`** prints a **TPS LOAD TEST SUMMARY** bloc | **Achieved TPS** | `Total emits ÷ window duration` — realized successful `Emit()` throughput. | | **Total emits** | Count of **`Emit()` calls that returned `nil`** in the measurement window (successful Postgres insert path). Does not count failures. | | **Emit p50 / p99** | Latency of successful `Emit()` calls (dominated by DB insert). | -| **Failures** | `Emit()` calls that returned an error (e.g. DB failure). | -| **Server recv** | **Mock only:** number of events observed by the in-process gRPC server (`Publish` / `PublishBatch`). | -| **Queue depth** | Rows remaining in `cre.chip_durable_events` after the emit phase (+ short settle), i.e. backlog not yet deleted after successful publish. | +| **Pub fail (retry)*** | Failed `Publish` / `PublishBatch` RPCs during the window: immediate failures (one row each, need retransmit) plus, when shown as `a+b`, `b` = total event count in failed `PublishBatch` calls. `Emit()` insert failures are logged separately if non-zero. | +| **Q max (rows)** | Peak row count in `cre.chip_durable_events` sampled during the emit window (~50ms polls). | +| **Q end (rows)** | Row count after a short settle (async publish / retransmit). | +| **Q max (KB)*** | For the peak queue sample: `sum(octet_length(payload))/1024` over queued rows (payload bytes only). **Q end** payload size is omitted from the printed table to keep it narrow. | -#### Why **Server recv** shows **N/A** with real Chip - -The **Server recv** column is implemented by counting events on the **in-process mock** `ChipIngress` server. When you use **`CHIP_INGRESS_TEST_ADDR`**, there is no mock — the client talks to a **real** gateway — so the test **cannot** count server-side receives in-process. Use **Kafka / Chip / gateway metrics** (or consumer verification) to validate end-to-end delivery instead. **Total emits** and **Achieved TPS** still reflect client-side durable insert success; they are not replaced by N/A. +With **`CHIP_INGRESS_TEST_ADDR`** set, there is no in-process mock — validate end-to-end delivery with **Kafka / Chip / gateway metrics** (or consumer checks). **Total emits** and **Achieved TPS** still reflect successful durable inserts on the node. ### CRE Smoke Tests (live Docker environment) diff --git a/pkg/beholder/durable_emitter.go b/pkg/beholder/durable_emitter.go index 788dadf643..ea8d79a5c9 100644 --- a/pkg/beholder/durable_emitter.go +++ b/pkg/beholder/durable_emitter.go @@ -30,6 +30,9 @@ type DurableEmitterConfig struct { // Hooks is optional instrumentation (load tests, profiling). Nil fields are skipped. // Callbacks may run from many goroutines; implementations must be thread-safe. Hooks *DurableEmitterHooks + // Metrics enables OpenTelemetry instruments on beholder.GetMeter() (queue, publish, store, optional process stats). + // Nil disables. + Metrics *DurableEmitterMetricsConfig } // DurableEmitterHooks records Publish vs Delete latency to locate pipeline bottlenecks. @@ -71,6 +74,8 @@ type DurableEmitter struct { cfg DurableEmitterConfig log logger.Logger + metrics *durableEmitterMetrics + stopCh chan struct{} wg sync.WaitGroup } @@ -92,47 +97,82 @@ func NewDurableEmitter( if log == nil { return nil, fmt.Errorf("logger is nil") } + var m *durableEmitterMetrics + if cfg.Metrics != nil { + var err error + m, err = newDurableEmitterMetrics() + if err != nil { + return nil, fmt.Errorf("durable emitter metrics: %w", err) + } + store = newMetricsInstrumentedStore(store, m) + } return &DurableEmitter{ - store: store, - client: client, - cfg: cfg, - log: log, - stopCh: make(chan struct{}), + store: store, + client: client, + cfg: cfg, + log: log, + metrics: m, + stopCh: make(chan struct{}), }, nil } // Start launches the retransmit and expiry background loops. // Cancel the supplied context or call Close to stop them. func (d *DurableEmitter) Start(ctx context.Context) { - d.wg.Add(2) + n := 2 + if d.metrics != nil && d.cfg.Metrics != nil { + n++ + } + d.wg.Add(n) go d.retransmitLoop(ctx) go d.expiryLoop(ctx) + if d.metrics != nil && d.cfg.Metrics != nil { + go d.metricsLoop(ctx) + } } // Emit persists the event then attempts async delivery. // Returns nil once the store insert succeeds. func (d *DurableEmitter) Emit(ctx context.Context, body []byte, attrKVs ...any) error { + emitFail := func() { + if d.metrics != nil { + d.metrics.emitFail.Add(ctx, 1) + } + } sourceDomain, entityType, err := ExtractSourceAndType(attrKVs...) if err != nil { + emitFail() return err } event, err := chipingress.NewEvent(sourceDomain, entityType, body, newAttributes(attrKVs...)) if err != nil { + emitFail() return err } eventPb, err := chipingress.EventToProto(event) if err != nil { + emitFail() return fmt.Errorf("failed to convert event to proto: %w", err) } payload, err := proto.Marshal(eventPb) if err != nil { + emitFail() return fmt.Errorf("failed to marshal event proto: %w", err) } + tIns := time.Now() id, err := d.store.Insert(ctx, payload) + if d.metrics != nil { + d.metrics.emitDuration.Record(ctx, time.Since(tIns).Seconds()) + if err != nil { + d.metrics.emitFail.Add(ctx, 1) + } else { + d.metrics.emitSuccess.Add(ctx, 1) + } + } if err != nil { return fmt.Errorf("failed to persist event: %w", err) } @@ -160,6 +200,14 @@ func (d *DurableEmitter) publishAndDelete(id int64, eventPb *chipingress.CloudEv if h := d.cfg.Hooks; h != nil && h.OnImmediatePublish != nil { h.OnImmediatePublish(time.Since(t0), err) } + mctx := context.Background() + if d.metrics != nil { + if err != nil { + d.metrics.publishImmErr.Add(mctx, 1) + } else { + d.metrics.publishImmOK.Add(mctx, 1) + } + } if err != nil { d.log.Debugw("immediate publish failed, retransmit loop will retry", "id", id, "error", err) @@ -171,6 +219,9 @@ func (d *DurableEmitter) publishAndDelete(id int64, eventPb *chipingress.CloudEv if h := d.cfg.Hooks; h != nil && h.OnImmediateDelete != nil { h.OnImmediateDelete(time.Since(t1), delErr) } + if delErr == nil && d.metrics != nil { + d.metrics.deliverComplete.Add(mctx, 1) + } if delErr != nil { d.log.Errorw("failed to delete delivered event", "id", id, "error", delErr) } @@ -230,9 +281,17 @@ func (d *DurableEmitter) retransmitPending(ctx context.Context) { h.OnRetransmitBatchPublish(time.Since(tPub), len(events), err) } if err != nil { + if d.metrics != nil { + d.metrics.publishBatchErr.Add(ctx, 1) + d.metrics.publishBatchEvErr.Add(ctx, int64(len(events))) + } d.log.Warnw("retransmit batch failed", "count", len(events), "error", err) return } + if d.metrics != nil { + d.metrics.publishBatchOK.Add(ctx, 1) + d.metrics.publishBatchEvOK.Add(ctx, int64(len(events))) + } tDel := time.Now() for _, id := range ids { @@ -243,6 +302,9 @@ func (d *DurableEmitter) retransmitPending(ctx context.Context) { if h := d.cfg.Hooks; h != nil && h.OnRetransmitBatchDeletes != nil { h.OnRetransmitBatchDeletes(time.Since(tDel), len(ids)) } + if d.metrics != nil { + d.metrics.deliverComplete.Add(ctx, int64(len(ids))) + } d.log.Debugw("retransmitted events", "count", len(ids)) } @@ -264,8 +326,46 @@ func (d *DurableEmitter) expiryLoop(ctx context.Context) { continue } if deleted > 0 { + if d.metrics != nil { + d.metrics.expiredPurged.Add(context.Background(), deleted) + } d.log.Infow("purged expired events", "count", deleted) } } } } + +func (d *DurableEmitter) metricsLoop(ctx context.Context) { + defer d.wg.Done() + mc := d.cfg.Metrics + poll := mc.PollInterval + if poll <= 0 { + poll = 10 * time.Second + } + lead := mc.NearExpiryLead + if lead <= 0 { + lead = 5 * time.Minute + } + ticker := time.NewTicker(poll) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-d.stopCh: + return + case <-ticker.C: + if d.metrics == nil { + return + } + bctx := context.Background() + if obs, ok := d.store.(DurableQueueObserver); ok { + d.metrics.pollQueueGauges(bctx, obs, d.cfg.EventTTL, lead, mc.MaxQueuePayloadBytes) + } + if mc.RecordProcessStats { + d.metrics.recordProcessMem(bctx) + d.metrics.recordProcessCPU(bctx) + } + } + } +} diff --git a/pkg/beholder/durable_emitter_cpu_other.go b/pkg/beholder/durable_emitter_cpu_other.go new file mode 100644 index 0000000000..2b7a7f5206 --- /dev/null +++ b/pkg/beholder/durable_emitter_cpu_other.go @@ -0,0 +1,9 @@ +//go:build !unix + +package beholder + +import "context" + +func (m *durableEmitterMetrics) recordProcessCPU(ctx context.Context) { + _ = ctx +} diff --git a/pkg/beholder/durable_emitter_cpu_unix.go b/pkg/beholder/durable_emitter_cpu_unix.go new file mode 100644 index 0000000000..ec46802a63 --- /dev/null +++ b/pkg/beholder/durable_emitter_cpu_unix.go @@ -0,0 +1,22 @@ +//go:build unix + +package beholder + +import ( + "context" + "syscall" +) + +func (m *durableEmitterMetrics) recordProcessCPU(ctx context.Context) { + if m == nil { + return + } + var r syscall.Rusage + if err := syscall.Getrusage(syscall.RUSAGE_SELF, &r); err != nil { + return + } + u := float64(r.Utime.Sec) + float64(r.Utime.Usec)/1e6 + s := float64(r.Stime.Sec) + float64(r.Stime.Usec)/1e6 + m.procCPUUser.Record(ctx, u) + m.procCPUSys.Record(ctx, s) +} diff --git a/pkg/beholder/durable_emitter_metric_info.go b/pkg/beholder/durable_emitter_metric_info.go new file mode 100644 index 0000000000..e4cce08df0 --- /dev/null +++ b/pkg/beholder/durable_emitter_metric_info.go @@ -0,0 +1,117 @@ +package beholder + +// Durable emitter OTel instruments (registered via beholder.GetMeter), matching the +// MetricInfo pattern used with beholder elsewhere in chainlink-common. + +var ( + durableEmitterMetricEmitSuccess = MetricInfo{ + Name: "beholder.durable_emitter.emit.success", + Unit: "{call}", + Description: "Successful durable Emit calls (insert returned)", + } + durableEmitterMetricEmitFailure = MetricInfo{ + Name: "beholder.durable_emitter.emit.failure", + Unit: "{call}", + Description: "Failed Emit calls (before or during insert)", + } + durableEmitterMetricEmitDuration = MetricInfo{ + Name: "beholder.durable_emitter.emit.duration", + Unit: "s", + Description: "Emit insert path duration", + } + durableEmitterMetricPublishImmSuccess = MetricInfo{ + Name: "beholder.durable_emitter.publish.immediate.success", + Unit: "{call}", + Description: "Immediate Publish RPC successes", + } + durableEmitterMetricPublishImmFailure = MetricInfo{ + Name: "beholder.durable_emitter.publish.immediate.failure", + Unit: "{call}", + Description: "Immediate Publish RPC failures (events await retransmit)", + } + durableEmitterMetricPublishBatchSuccess = MetricInfo{ + Name: "beholder.durable_emitter.publish.retransmit.batch.success", + Unit: "{call}", + Description: "Successful retransmit PublishBatch calls", + } + durableEmitterMetricPublishBatchFailure = MetricInfo{ + Name: "beholder.durable_emitter.publish.retransmit.batch.failure", + Unit: "{call}", + Description: "Failed retransmit PublishBatch calls", + } + durableEmitterMetricPublishBatchEvSuccess = MetricInfo{ + Name: "beholder.durable_emitter.publish.retransmit.events.success", + Unit: "{event}", + Description: "Events delivered via successful PublishBatch", + } + durableEmitterMetricPublishBatchEvFailure = MetricInfo{ + Name: "beholder.durable_emitter.publish.retransmit.events.failure", + Unit: "{event}", + Description: "Events in failed PublishBatch attempts", + } + durableEmitterMetricDeliveryCompleted = MetricInfo{ + Name: "beholder.durable_emitter.delivery.completed", + Unit: "{event}", + Description: "Events removed from store after successful publish (immediate or batch)", + } + durableEmitterMetricExpiredPurged = MetricInfo{ + Name: "beholder.durable_emitter.expired_purged", + Unit: "{event}", + Description: "Events deleted by TTL expiry loop", + } + durableEmitterMetricStoreOperations = MetricInfo{ + Name: "beholder.durable_emitter.store.operations", + Unit: "{op}", + Description: "Durable store operations (proxy for DB load / IOPs)", + } + durableEmitterMetricStoreOpDuration = MetricInfo{ + Name: "beholder.durable_emitter.store.operation.duration", + Unit: "s", + Description: "Durable store operation latency", + } + durableEmitterMetricQueueDepth = MetricInfo{ + Name: "beholder.durable_emitter.queue.depth", + Unit: "{row}", + Description: "Pending rows in durable queue", + } + durableEmitterMetricQueuePayloadBytes = MetricInfo{ + Name: "beholder.durable_emitter.queue.payload_bytes", + Unit: "By", + Description: "Sum of payload bytes for pending rows", + } + durableEmitterMetricQueueOldestAgeSec = MetricInfo{ + Name: "beholder.durable_emitter.queue.oldest_pending_age_seconds", + Unit: "s", + Description: "Age of oldest pending row at last poll (longest wait)", + } + durableEmitterMetricQueueNearTTL = MetricInfo{ + Name: "beholder.durable_emitter.queue.near_ttl", + Unit: "{row}", + Description: "Rows within near-expiry window of EventTTL (DLQ pressure proxy; no separate DLQ table)", + } + durableEmitterMetricQueueCapacityRatio = MetricInfo{ + Name: "beholder.durable_emitter.queue.capacity_usage_ratio", + Unit: "1", + Description: "queue.payload_bytes / MaxQueuePayloadBytes when max > 0", + } + durableEmitterMetricProcHeapInuse = MetricInfo{ + Name: "beholder.durable_emitter.process.memory.heap_inuse_bytes", + Unit: "By", + Description: "Go runtime MemStats HeapInuse", + } + durableEmitterMetricProcHeapSys = MetricInfo{ + Name: "beholder.durable_emitter.process.memory.heap_sys_bytes", + Unit: "By", + Description: "Go runtime MemStats HeapSys", + } + durableEmitterMetricProcCPUUser = MetricInfo{ + Name: "beholder.durable_emitter.process.cpu.user_seconds", + Unit: "s", + Description: "Cumulative user CPU seconds (getrusage; Unix only)", + } + durableEmitterMetricProcCPUSys = MetricInfo{ + Name: "beholder.durable_emitter.process.cpu.system_seconds", + Unit: "s", + Description: "Cumulative system CPU seconds (getrusage; Unix only)", + } +) diff --git a/pkg/beholder/durable_emitter_metrics.go b/pkg/beholder/durable_emitter_metrics.go new file mode 100644 index 0000000000..76d1a20c0d --- /dev/null +++ b/pkg/beholder/durable_emitter_metrics.go @@ -0,0 +1,168 @@ +package beholder + +import ( + "context" + "runtime" + "time" + + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" +) + +// DurableEmitterMetricsConfig enables OpenTelemetry metrics for DurableEmitter. +// Set on DurableEmitterConfig.Metrics; nil disables instrumentation. +// +// Instruments are registered on beholder.GetMeter() (same path as capabilities +// and monitoring metrics). Ensure beholder.SetClient has been called with a +// configured client before NewDurableEmitter when metrics are enabled. +type DurableEmitterMetricsConfig struct { + // PollInterval is how often queue and optional process gauges refresh. Zero = 10s. + PollInterval time.Duration + // NearExpiryLead is the window before EventTTL used for queue.near_ttl (DLQ pressure proxy). Zero = 5m. + NearExpiryLead time.Duration + // MaxQueuePayloadBytes, if > 0, records capacity_usage_ratio = queue_payload_bytes / max. + MaxQueuePayloadBytes int64 + // RecordProcessStats records Go heap gauges and, on Unix, cumulative CPU seconds (getrusage). + RecordProcessStats bool +} + +type durableEmitterMetrics struct { + emitSuccess metric.Int64Counter + emitFail metric.Int64Counter + emitDuration metric.Float64Histogram + publishImmOK metric.Int64Counter + publishImmErr metric.Int64Counter + publishBatchOK metric.Int64Counter + publishBatchErr metric.Int64Counter + publishBatchEvOK metric.Int64Counter + publishBatchEvErr metric.Int64Counter + deliverComplete metric.Int64Counter + expiredPurged metric.Int64Counter + storeOps metric.Int64Counter + storeOpDuration metric.Float64Histogram + queueDepth metric.Int64Gauge + queuePayloadBytes metric.Int64Gauge + queueOldestAgeSec metric.Float64Gauge + queueNearTTL metric.Int64Gauge + queueCapacityRatio metric.Float64Gauge + procHeapInuse metric.Int64Gauge + procHeapSys metric.Int64Gauge + procCPUUser metric.Float64Gauge + procCPUSys metric.Float64Gauge +} + +func newDurableEmitterMetrics() (*durableEmitterMetrics, error) { + meter := GetMeter() + m := &durableEmitterMetrics{} + var err error + if m.emitSuccess, err = durableEmitterMetricEmitSuccess.NewInt64Counter(meter); err != nil { + return nil, err + } + if m.emitFail, err = durableEmitterMetricEmitFailure.NewInt64Counter(meter); err != nil { + return nil, err + } + if m.emitDuration, err = durableEmitterMetricEmitDuration.NewFloat64Histogram(meter); err != nil { + return nil, err + } + if m.publishImmOK, err = durableEmitterMetricPublishImmSuccess.NewInt64Counter(meter); err != nil { + return nil, err + } + if m.publishImmErr, err = durableEmitterMetricPublishImmFailure.NewInt64Counter(meter); err != nil { + return nil, err + } + if m.publishBatchOK, err = durableEmitterMetricPublishBatchSuccess.NewInt64Counter(meter); err != nil { + return nil, err + } + if m.publishBatchErr, err = durableEmitterMetricPublishBatchFailure.NewInt64Counter(meter); err != nil { + return nil, err + } + if m.publishBatchEvOK, err = durableEmitterMetricPublishBatchEvSuccess.NewInt64Counter(meter); err != nil { + return nil, err + } + if m.publishBatchEvErr, err = durableEmitterMetricPublishBatchEvFailure.NewInt64Counter(meter); err != nil { + return nil, err + } + if m.deliverComplete, err = durableEmitterMetricDeliveryCompleted.NewInt64Counter(meter); err != nil { + return nil, err + } + if m.expiredPurged, err = durableEmitterMetricExpiredPurged.NewInt64Counter(meter); err != nil { + return nil, err + } + if m.storeOps, err = durableEmitterMetricStoreOperations.NewInt64Counter(meter); err != nil { + return nil, err + } + if m.storeOpDuration, err = durableEmitterMetricStoreOpDuration.NewFloat64Histogram(meter); err != nil { + return nil, err + } + if m.queueDepth, err = durableEmitterMetricQueueDepth.NewInt64Gauge(meter); err != nil { + return nil, err + } + if m.queuePayloadBytes, err = durableEmitterMetricQueuePayloadBytes.NewInt64Gauge(meter); err != nil { + return nil, err + } + if m.queueOldestAgeSec, err = durableEmitterMetricQueueOldestAgeSec.NewFloat64Gauge(meter); err != nil { + return nil, err + } + if m.queueNearTTL, err = durableEmitterMetricQueueNearTTL.NewInt64Gauge(meter); err != nil { + return nil, err + } + if m.queueCapacityRatio, err = durableEmitterMetricQueueCapacityRatio.NewFloat64Gauge(meter); err != nil { + return nil, err + } + if m.procHeapInuse, err = durableEmitterMetricProcHeapInuse.NewInt64Gauge(meter); err != nil { + return nil, err + } + if m.procHeapSys, err = durableEmitterMetricProcHeapSys.NewInt64Gauge(meter); err != nil { + return nil, err + } + if m.procCPUUser, err = durableEmitterMetricProcCPUUser.NewFloat64Gauge(meter); err != nil { + return nil, err + } + if m.procCPUSys, err = durableEmitterMetricProcCPUSys.NewFloat64Gauge(meter); err != nil { + return nil, err + } + return m, nil +} + +func (m *durableEmitterMetrics) recordStoreOp(ctx context.Context, op string, elapsed time.Duration, opErr error) { + if m == nil { + return + } + attrs := metric.WithAttributes( + attribute.String("operation", op), + attribute.Bool("error", opErr != nil), + ) + m.storeOps.Add(ctx, 1, attrs) + m.storeOpDuration.Record(ctx, elapsed.Seconds(), metric.WithAttributes(attribute.String("operation", op))) +} + +func (m *durableEmitterMetrics) pollQueueGauges(ctx context.Context, obs DurableQueueObserver, ttl, lead time.Duration, maxBytes int64) { + if m == nil || obs == nil { + return + } + st, err := obs.ObserveDurableQueue(ctx, ttl, lead) + if err != nil { + return + } + m.queueDepth.Record(ctx, st.Depth) + m.queuePayloadBytes.Record(ctx, st.PayloadBytes) + if st.Depth == 0 { + m.queueOldestAgeSec.Record(ctx, 0) + } else { + m.queueOldestAgeSec.Record(ctx, st.OldestPendingAge.Seconds()) + } + m.queueNearTTL.Record(ctx, st.NearTTLCount) + if maxBytes > 0 { + m.queueCapacityRatio.Record(ctx, float64(st.PayloadBytes)/float64(maxBytes)) + } +} + +func (m *durableEmitterMetrics) recordProcessMem(ctx context.Context) { + if m == nil { + return + } + var ms runtime.MemStats + runtime.ReadMemStats(&ms) + m.procHeapInuse.Record(ctx, int64(ms.HeapInuse)) + m.procHeapSys.Record(ctx, int64(ms.HeapSys)) +} diff --git a/pkg/beholder/durable_emitter_store_wrap.go b/pkg/beholder/durable_emitter_store_wrap.go new file mode 100644 index 0000000000..75faa252de --- /dev/null +++ b/pkg/beholder/durable_emitter_store_wrap.go @@ -0,0 +1,58 @@ +package beholder + +import ( + "context" + "time" +) + +// metricsInstrumentedStore wraps DurableEventStore to record store operation metrics. +type metricsInstrumentedStore struct { + inner DurableEventStore + m *durableEmitterMetrics +} + +var _ DurableEventStore = (*metricsInstrumentedStore)(nil) +var _ DurableQueueObserver = (*metricsInstrumentedStore)(nil) + +func newMetricsInstrumentedStore(inner DurableEventStore, m *durableEmitterMetrics) DurableEventStore { + if m == nil { + return inner + } + return &metricsInstrumentedStore{inner: inner, m: m} +} + +func (s *metricsInstrumentedStore) Insert(ctx context.Context, payload []byte) (int64, error) { + t0 := time.Now() + id, err := s.inner.Insert(ctx, payload) + s.m.recordStoreOp(ctx, "insert", time.Since(t0), err) + return id, err +} + +func (s *metricsInstrumentedStore) Delete(ctx context.Context, id int64) error { + t0 := time.Now() + err := s.inner.Delete(ctx, id) + s.m.recordStoreOp(ctx, "delete", time.Since(t0), err) + return err +} + +func (s *metricsInstrumentedStore) ListPending(ctx context.Context, createdBefore time.Time, limit int) ([]DurableEvent, error) { + t0 := time.Now() + evs, err := s.inner.ListPending(ctx, createdBefore, limit) + s.m.recordStoreOp(ctx, "list_pending", time.Since(t0), err) + return evs, err +} + +func (s *metricsInstrumentedStore) DeleteExpired(ctx context.Context, ttl time.Duration) (int64, error) { + t0 := time.Now() + n, err := s.inner.DeleteExpired(ctx, ttl) + s.m.recordStoreOp(ctx, "delete_expired", time.Since(t0), err) + return n, err +} + +func (s *metricsInstrumentedStore) ObserveDurableQueue(ctx context.Context, eventTTL, nearExpiryLead time.Duration) (DurableQueueStats, error) { + o, ok := s.inner.(DurableQueueObserver) + if !ok { + return DurableQueueStats{}, nil + } + return o.ObserveDurableQueue(ctx, eventTTL, nearExpiryLead) +} diff --git a/pkg/beholder/durable_emitter_test.go b/pkg/beholder/durable_emitter_test.go index d6b978c983..6c32fb9598 100644 --- a/pkg/beholder/durable_emitter_test.go +++ b/pkg/beholder/durable_emitter_test.go @@ -14,8 +14,27 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/chipingress" "github.com/smartcontractkit/chainlink-common/pkg/logger" + sdkmetric "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/metric/metricdata" ) +// withTestBeholderMeter swaps the global beholder client meter for t's lifetime (for metrics assertions). +func withTestBeholderMeter(t *testing.T) *sdkmetric.ManualReader { + t.Helper() + prev := GetClient() + reader := sdkmetric.NewManualReader() + mp := sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader)) + c := NewNoopClient() + c.MeterProvider = mp + c.Meter = mp.Meter(defaultPackageName) + SetClient(c) + t.Cleanup(func() { + SetClient(prev) + _ = mp.Shutdown(context.Background()) + }) + return reader +} + // testChipClient is a minimal chipingress.Client for tests. type testChipClient struct { chipingress.NoopClient @@ -266,3 +285,38 @@ func TestNewDurableEmitter_ValidationErrors(t *testing.T) { _, err = NewDurableEmitter(NewMemDurableEventStore(), &testChipClient{}, cfg, nil) assert.ErrorContains(t, err, "logger") } + +func TestDurableEmitter_MetricsRegistersEmitSuccess(t *testing.T) { + reader := withTestBeholderMeter(t) + + store := NewMemDurableEventStore() + client := &testChipClient{} + cfg := DefaultDurableEmitterConfig() + cfg.RetransmitInterval = time.Hour + cfg.Metrics = &DurableEmitterMetricsConfig{PollInterval: 25 * time.Millisecond} + + em, err := NewDurableEmitter(store, client, cfg, logger.Test(t)) + require.NoError(t, err) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + em.Start(ctx) + defer func() { _ = em.Close() }() + + require.NoError(t, em.Emit(ctx, []byte("m"), testEmitAttrs()...)) + require.Eventually(t, func() bool { return store.Len() == 0 }, 2*time.Second, 10*time.Millisecond) + time.Sleep(50 * time.Millisecond) + + var rm metricdata.ResourceMetrics + require.NoError(t, reader.Collect(ctx, &rm)) + + var found bool + for _, sm := range rm.ScopeMetrics { + for _, m := range sm.Metrics { + if m.Name == "beholder.durable_emitter.emit.success" { + found = true + } + } + } + assert.True(t, found, "expected beholder.durable_emitter.emit.success in exported metrics") +} diff --git a/pkg/beholder/durable_event_store.go b/pkg/beholder/durable_event_store.go index 1186850f2a..ab67d8ebb4 100644 --- a/pkg/beholder/durable_event_store.go +++ b/pkg/beholder/durable_event_store.go @@ -15,6 +15,25 @@ type DurableEvent struct { CreatedAt time.Time } +// DurableQueueStats is a point-in-time snapshot of the pending queue for metrics. +type DurableQueueStats struct { + Depth int64 + PayloadBytes int64 + OldestPendingAge time.Duration // 0 if the queue is empty + // NearTTLCount is the number of rows within nearExpiryLead of EventTTL (still + // pending, not yet removed by expiry). Serves as a DLQ-pressure proxy; there is + // no separate dead-letter table in the default design. + NearTTLCount int64 +} + +// DurableQueueObserver is optionally implemented by DurableEventStore implementations +// so DurableEmitter can export queue depth and age gauges when metrics are enabled. +type DurableQueueObserver interface { + // ObserveDurableQueue returns live queue statistics. eventTTL and nearExpiryLead + // match DurableEmitterConfig (nearExpiryLead should be << eventTTL). + ObserveDurableQueue(ctx context.Context, eventTTL, nearExpiryLead time.Duration) (DurableQueueStats, error) +} + // DurableEventStore abstracts the persistence layer for durable chip events. // Implementations must be safe for concurrent use. type DurableEventStore interface { @@ -36,7 +55,10 @@ type MemDurableEventStore struct { nextID atomic.Int64 } -var _ DurableEventStore = (*MemDurableEventStore)(nil) +var ( + _ DurableEventStore = (*MemDurableEventStore)(nil) + _ DurableQueueObserver = (*MemDurableEventStore)(nil) +) func NewMemDurableEventStore() *MemDurableEventStore { return &MemDurableEventStore{ @@ -103,3 +125,33 @@ func (m *MemDurableEventStore) Len() int { defer m.mu.Unlock() return len(m.events) } + +// ObserveDurableQueue implements DurableQueueObserver. +func (m *MemDurableEventStore) ObserveDurableQueue(_ context.Context, eventTTL, nearExpiryLead time.Duration) (DurableQueueStats, error) { + m.mu.Lock() + defer m.mu.Unlock() + now := time.Now() + var st DurableQueueStats + if len(m.events) == 0 { + return st, nil + } + var oldest time.Time + first := true + for _, e := range m.events { + st.Depth++ + st.PayloadBytes += int64(len(e.Payload)) + if first || e.CreatedAt.Before(oldest) { + oldest = e.CreatedAt + first = false + } + age := now.Sub(e.CreatedAt) + if eventTTL > 0 && nearExpiryLead > 0 && nearExpiryLead < eventTTL { + threshold := eventTTL - nearExpiryLead + if age >= threshold && age < eventTTL { + st.NearTTLCount++ + } + } + } + st.OldestPendingAge = now.Sub(oldest) + return st, nil +} From 6f9c5dd61b089944e8346c5ca81aab1fa3716839 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Wed, 25 Mar 2026 16:43:23 -0400 Subject: [PATCH 08/45] Single publish --- pkg/beholder/durable_emitter.go | 75 ++++++++++--------- .../durable_emitter_integration_test.go | 25 ++++--- pkg/beholder/durable_emitter_metric_info.go | 14 ++-- pkg/beholder/durable_emitter_test.go | 67 ++++++++++++----- 4 files changed, 108 insertions(+), 73 deletions(-) diff --git a/pkg/beholder/durable_emitter.go b/pkg/beholder/durable_emitter.go index ea8d79a5c9..039bff42b5 100644 --- a/pkg/beholder/durable_emitter.go +++ b/pkg/beholder/durable_emitter.go @@ -19,13 +19,14 @@ type DurableEmitterConfig struct { // RetransmitAfter is the minimum age of an event before the retransmit // loop considers it. This gives the immediate-publish path time to succeed. RetransmitAfter time.Duration - // RetransmitBatchSize caps the number of events sent per retransmit cycle. + // RetransmitBatchSize caps how many pending rows are listed per retransmit tick + // (each row is sent with its own Publish RPC). RetransmitBatchSize int // ExpiryInterval controls how often the expiry loop ticks. ExpiryInterval time.Duration // EventTTL is the maximum age of an event before it is expired. EventTTL time.Duration - // PublishTimeout is the per-RPC deadline for Publish / PublishBatch calls. + // PublishTimeout is the per-RPC deadline for each Publish call. PublishTimeout time.Duration // Hooks is optional instrumentation (load tests, profiling). Nil fields are skipped. // Callbacks may run from many goroutines; implementations must be thread-safe. @@ -41,9 +42,9 @@ type DurableEmitterHooks struct { OnImmediatePublish func(elapsed time.Duration, err error) // OnImmediateDelete is called after Delete following a successful immediate Publish. OnImmediateDelete func(elapsed time.Duration, err error) - // OnRetransmitBatchPublish is called after each retransmit PublishBatch. + // OnRetransmitBatchPublish is called after each retransmit Publish (one RPC per queued event). OnRetransmitBatchPublish func(elapsed time.Duration, eventCount int, err error) - // OnRetransmitBatchDeletes is called once per successful batch with total time for the delete loop. + // OnRetransmitBatchDeletes is called after a retransmit tick with total time and successful delete count. OnRetransmitBatchDeletes func(elapsed time.Duration, deleteCount int) } @@ -64,7 +65,7 @@ func DefaultDurableEmitterConfig() DurableEmitterConfig { // insert succeeds Emit returns nil — the caller has a durable guarantee. An // immediate async Publish is attempted; on success the record is deleted. If // that fails a background retransmit loop will pick the event up and retry via -// PublishBatch. +// Publish (one RPC per pending row per tick, up to RetransmitBatchSize). // // A separate expiry loop garbage-collects events older than EventTTL to bound // table growth. @@ -259,53 +260,55 @@ func (d *DurableEmitter) retransmitPending(ctx context.Context) { ids := make([]int64, 0, len(pending)) for _, pe := range pending { - var eventPb chipingress.CloudEventPb - if err := proto.Unmarshal(pe.Payload, &eventPb); err != nil { + ev := new(chipingress.CloudEventPb) + if err := proto.Unmarshal(pe.Payload, ev); err != nil { d.log.Errorw("corrupt pending event, deleting", "id", pe.ID, "error", err) _ = d.store.Delete(ctx, pe.ID) continue } - events = append(events, &eventPb) + events = append(events, ev) ids = append(ids, pe.ID) } if len(events) == 0 { return } - publishCtx, cancel := context.WithTimeout(ctx, d.cfg.PublishTimeout) - defer cancel() - - tPub := time.Now() - _, err = d.client.PublishBatch(publishCtx, &chipingress.CloudEventBatch{Events: events}) - if h := d.cfg.Hooks; h != nil && h.OnRetransmitBatchPublish != nil { - h.OnRetransmitBatchPublish(time.Since(tPub), len(events), err) - } - if err != nil { + // One Publish per row so a single bad or rejected event does not block the rest of the slice. + tDel := time.Now() + var deleted int + for i := range events { + tPub := time.Now() + pubCtx, cancel := context.WithTimeout(context.Background(), d.cfg.PublishTimeout) + _, pubErr := d.client.Publish(pubCtx, events[i]) + cancel() + if h := d.cfg.Hooks; h != nil && h.OnRetransmitBatchPublish != nil { + h.OnRetransmitBatchPublish(time.Since(tPub), 1, pubErr) + } + if pubErr != nil { + if d.metrics != nil { + d.metrics.publishBatchEvErr.Add(ctx, 1) + } + d.log.Debugw("retransmit publish failed", "id", ids[i], "error", pubErr) + continue + } if d.metrics != nil { - d.metrics.publishBatchErr.Add(ctx, 1) - d.metrics.publishBatchEvErr.Add(ctx, int64(len(events))) + d.metrics.publishBatchEvOK.Add(ctx, 1) } - d.log.Warnw("retransmit batch failed", "count", len(events), "error", err) - return - } - if d.metrics != nil { - d.metrics.publishBatchOK.Add(ctx, 1) - d.metrics.publishBatchEvOK.Add(ctx, int64(len(events))) - } - - tDel := time.Now() - for _, id := range ids { - if err := d.store.Delete(ctx, id); err != nil { - d.log.Errorw("failed to delete retransmitted event", "id", id, "error", err) + if delErr := d.store.Delete(ctx, ids[i]); delErr != nil { + d.log.Errorw("failed to delete retransmitted event", "id", ids[i], "error", delErr) + continue + } + deleted++ + if d.metrics != nil { + d.metrics.deliverComplete.Add(ctx, 1) } } - if h := d.cfg.Hooks; h != nil && h.OnRetransmitBatchDeletes != nil { - h.OnRetransmitBatchDeletes(time.Since(tDel), len(ids)) + if deleted > 0 { + d.log.Debugw("retransmitted events", "deleted", deleted, "attempted", len(events)) } - if d.metrics != nil { - d.metrics.deliverComplete.Add(ctx, int64(len(ids))) + if h := d.cfg.Hooks; h != nil && h.OnRetransmitBatchDeletes != nil && deleted > 0 { + h.OnRetransmitBatchDeletes(time.Since(tDel), deleted) } - d.log.Debugw("retransmitted events", "count", len(ids)) } func (d *DurableEmitter) expiryLoop(ctx context.Context) { diff --git a/pkg/beholder/durable_emitter_integration_test.go b/pkg/beholder/durable_emitter_integration_test.go index e022b00a14..7c9fdff252 100644 --- a/pkg/beholder/durable_emitter_integration_test.go +++ b/pkg/beholder/durable_emitter_integration_test.go @@ -165,7 +165,6 @@ func TestIntegration_ServerUnavailable_RetransmitRecovers(t *testing.T) { // Start with server returning UNAVAILABLE. srv := &mockChipServer{} srv.setPublishErr(status.Error(codes.Unavailable, "chip down")) - srv.setBatchErr(status.Error(codes.Unavailable, "chip down")) _, addr := startMockServer(t, srv) client := newChipClient(t, addr) store := beholder.NewMemDurableEventStore() @@ -186,14 +185,14 @@ func TestIntegration_ServerUnavailable_RetransmitRecovers(t *testing.T) { // "Recover" the server. srv.setPublishErr(nil) - srv.setBatchErr(nil) require.Eventually(t, func() bool { return store.Len() == 0 }, 5*time.Second, 50*time.Millisecond, "retransmit loop should deliver after recovery") - assert.GreaterOrEqual(t, srv.batchCount.Load(), int64(1), - "retransmit should use PublishBatch") + assert.GreaterOrEqual(t, srv.publishCount.Load(), int64(2), + "one failed immediate Publish then one retransmit Publish") + assert.Equal(t, int64(0), srv.batchCount.Load(), "retransmit should not use PublishBatch") } func TestIntegration_ServerDown_EventsSurvive(t *testing.T) { @@ -286,7 +285,6 @@ func TestIntegration_EventExpiry(t *testing.T) { // Server always rejects — events can never be delivered. srv := &mockChipServer{} srv.setPublishErr(status.Error(codes.Internal, "permanent failure")) - srv.setBatchErr(status.Error(codes.Internal, "permanent failure")) _, addr := startMockServer(t, srv) client := newChipClient(t, addr) store := beholder.NewMemDurableEventStore() @@ -311,10 +309,10 @@ func TestIntegration_EventExpiry(t *testing.T) { "expiry loop should purge undeliverable events after TTL") } -func TestIntegration_RetransmitUsesBatch(t *testing.T) { - // Immediate publishes fail, only batch succeeds. +func TestIntegration_RetransmitUsesSerialPublish(t *testing.T) { + // Immediate Publish fails; retransmit uses one Publish per queued row. srv := &mockChipServer{} - srv.setPublishErr(status.Error(codes.Unavailable, "reject single")) + srv.setPublishErr(status.Error(codes.Unavailable, "reject immediate")) _, addr := startMockServer(t, srv) client := newChipClient(t, addr) store := beholder.NewMemDurableEventStore() @@ -328,16 +326,19 @@ func TestIntegration_RetransmitUsesBatch(t *testing.T) { defer em.Close() for i := 0; i < 5; i++ { - require.NoError(t, em.Emit(ctx, []byte("batch-me"), emitAttrs()...)) + require.NoError(t, em.Emit(ctx, []byte("retry-me"), emitAttrs()...)) } + srv.setPublishErr(nil) + require.Eventually(t, func() bool { return store.Len() == 0 }, 5*time.Second, 50*time.Millisecond, - "retransmit via PublishBatch should deliver all events") + "retransmit should deliver each event with its own Publish RPC") - assert.GreaterOrEqual(t, srv.batchCallCount(), 1, - "at least one PublishBatch call should have been made") + assert.Equal(t, 0, srv.batchCallCount(), "retransmit should not call PublishBatch") + assert.GreaterOrEqual(t, srv.publishCount.Load(), int64(10), + "five failed immediate attempts plus five retransmit publishes") } // TestIntegration_GRPCConnection verifies the emitter works over a real gRPC diff --git a/pkg/beholder/durable_emitter_metric_info.go b/pkg/beholder/durable_emitter_metric_info.go index e4cce08df0..c7726e8b1a 100644 --- a/pkg/beholder/durable_emitter_metric_info.go +++ b/pkg/beholder/durable_emitter_metric_info.go @@ -17,7 +17,7 @@ var ( durableEmitterMetricEmitDuration = MetricInfo{ Name: "beholder.durable_emitter.emit.duration", Unit: "s", - Description: "Emit insert path duration", + Description: "Emit insert path duration (seconds, fractional; aligns with Prometheus _duration_seconds)", } durableEmitterMetricPublishImmSuccess = MetricInfo{ Name: "beholder.durable_emitter.publish.immediate.success", @@ -32,27 +32,27 @@ var ( durableEmitterMetricPublishBatchSuccess = MetricInfo{ Name: "beholder.durable_emitter.publish.retransmit.batch.success", Unit: "{call}", - Description: "Successful retransmit PublishBatch calls", + Description: "Unused; retransmit uses serial Publish (see retransmit.events.*)", } durableEmitterMetricPublishBatchFailure = MetricInfo{ Name: "beholder.durable_emitter.publish.retransmit.batch.failure", Unit: "{call}", - Description: "Failed retransmit PublishBatch calls", + Description: "Unused; retransmit uses serial Publish (see retransmit.events.*)", } durableEmitterMetricPublishBatchEvSuccess = MetricInfo{ Name: "beholder.durable_emitter.publish.retransmit.events.success", Unit: "{event}", - Description: "Events delivered via successful PublishBatch", + Description: "Retransmit Publish RPC successes (one RPC per queued event)", } durableEmitterMetricPublishBatchEvFailure = MetricInfo{ Name: "beholder.durable_emitter.publish.retransmit.events.failure", Unit: "{event}", - Description: "Events in failed PublishBatch attempts", + Description: "Retransmit Publish RPC failures (event stays queued)", } durableEmitterMetricDeliveryCompleted = MetricInfo{ Name: "beholder.durable_emitter.delivery.completed", Unit: "{event}", - Description: "Events removed from store after successful publish (immediate or batch)", + Description: "Events removed from store after successful publish (immediate or retransmit)", } durableEmitterMetricExpiredPurged = MetricInfo{ Name: "beholder.durable_emitter.expired_purged", @@ -67,7 +67,7 @@ var ( durableEmitterMetricStoreOpDuration = MetricInfo{ Name: "beholder.durable_emitter.store.operation.duration", Unit: "s", - Description: "Durable store operation latency", + Description: "Durable store operation latency (seconds, fractional)", } durableEmitterMetricQueueDepth = MetricInfo{ Name: "beholder.durable_emitter.queue.depth", diff --git a/pkg/beholder/durable_emitter_test.go b/pkg/beholder/durable_emitter_test.go index 6c32fb9598..9429f6ec56 100644 --- a/pkg/beholder/durable_emitter_test.go +++ b/pkg/beholder/durable_emitter_test.go @@ -39,25 +39,25 @@ func withTestBeholderMeter(t *testing.T) *sdkmetric.ManualReader { type testChipClient struct { chipingress.NoopClient - mu sync.Mutex - publishErr error - batchErr error - publishCount atomic.Int64 - batchCount atomic.Int64 + mu sync.Mutex + publishErr error + publishCount atomic.Int64 + publishedIDs []string } -func (c *testChipClient) Publish(_ context.Context, _ *chipingress.CloudEventPb, _ ...grpc.CallOption) (*chipingress.PublishResponse, error) { +func (c *testChipClient) Publish(_ context.Context, ev *chipingress.CloudEventPb, _ ...grpc.CallOption) (*chipingress.PublishResponse, error) { c.publishCount.Add(1) c.mu.Lock() - defer c.mu.Unlock() - return &chipingress.PublishResponse{}, c.publishErr + if ev != nil { + c.publishedIDs = append(c.publishedIDs, ev.Id) + } + err := c.publishErr + c.mu.Unlock() + return &chipingress.PublishResponse{}, err } func (c *testChipClient) PublishBatch(_ context.Context, _ *chipingress.CloudEventBatch, _ ...grpc.CallOption) (*chipingress.PublishResponse, error) { - c.batchCount.Add(1) - c.mu.Lock() - defer c.mu.Unlock() - return &chipingress.PublishResponse{}, c.batchErr + return &chipingress.PublishResponse{}, nil } func (c *testChipClient) setPublishErr(err error) { @@ -66,10 +66,12 @@ func (c *testChipClient) setPublishErr(err error) { c.publishErr = err } -func (c *testChipClient) setBatchErr(err error) { +func (c *testChipClient) getPublishedIDs() []string { c.mu.Lock() defer c.mu.Unlock() - c.batchErr = err + out := make([]string, len(c.publishedIDs)) + copy(out, c.publishedIDs) + return out } func testEmitAttrs() []any { @@ -184,7 +186,6 @@ func TestDurableEmitter_RetransmitLoopDeliversFailedEvents(t *testing.T) { store := NewMemDurableEventStore() client := &testChipClient{} client.setPublishErr(errors.New("connection refused")) - client.setBatchErr(errors.New("connection refused")) cfg := DefaultDurableEmitterConfig() cfg.RetransmitInterval = 100 * time.Millisecond @@ -201,14 +202,44 @@ func TestDurableEmitter_RetransmitLoopDeliversFailedEvents(t *testing.T) { require.NoError(t, err) assert.Equal(t, 1, store.Len()) - // Fix the batch client so retransmit succeeds. - client.setBatchErr(nil) + client.setPublishErr(nil) require.Eventually(t, func() bool { return store.Len() == 0 }, 5*time.Second, 50*time.Millisecond, "retransmit loop should eventually deliver and delete the event") - assert.GreaterOrEqual(t, client.batchCount.Load(), int64(1)) + assert.GreaterOrEqual(t, client.publishCount.Load(), int64(2)) +} + +func TestDurableEmitter_RetransmitSerialDistinctCloudEvents(t *testing.T) { + store := NewMemDurableEventStore() + client := &testChipClient{} + client.setPublishErr(errors.New("immediate fail")) + + cfg := DefaultDurableEmitterConfig() + cfg.RetransmitInterval = 100 * time.Millisecond + cfg.RetransmitAfter = 50 * time.Millisecond + + em := newTestDurableEmitter(t, store, client, &cfg) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + em.Start(ctx) + defer em.Close() + + require.NoError(t, em.Emit(ctx, []byte("first"), testEmitAttrs()...)) + require.NoError(t, em.Emit(ctx, []byte("second"), testEmitAttrs()...)) + + client.setPublishErr(nil) + + require.Eventually(t, func() bool { return store.Len() == 0 }, 5*time.Second, 50*time.Millisecond) + + ids := client.getPublishedIDs() + require.GreaterOrEqual(t, len(ids), 4, "two immediate fails then two retransmit publishes") + a, b := ids[len(ids)-2], ids[len(ids)-1] + assert.NotEmpty(t, a) + assert.NotEmpty(t, b) + assert.NotEqualf(t, a, b, "retransmit must publish two distinct CloudEvents, not one pointer reused for every row") } func TestDurableEmitter_ExpiryLoopDeletesOldEvents(t *testing.T) { From 1fd13eade84de1403de5387182e1bc814a30f66f Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Wed, 25 Mar 2026 17:01:48 -0400 Subject: [PATCH 09/45] Add beholder schema? --- pkg/beholder/durable_emitter.go | 14 ++++++++++++-- pkg/chipingress/client.go | 6 ++++++ pkg/chipingress/client_test.go | 23 +++++++++++++++++++++++ 3 files changed, 41 insertions(+), 2 deletions(-) diff --git a/pkg/beholder/durable_emitter.go b/pkg/beholder/durable_emitter.go index 039bff42b5..eb09185551 100644 --- a/pkg/beholder/durable_emitter.go +++ b/pkg/beholder/durable_emitter.go @@ -210,8 +210,9 @@ func (d *DurableEmitter) publishAndDelete(id int64, eventPb *chipingress.CloudEv } } if err != nil { + ceID, ceSource, ceType := cloudEventDbg(eventPb) d.log.Debugw("immediate publish failed, retransmit loop will retry", - "id", id, "error", err) + "id", id, "ce_id", ceID, "ce_source", ceSource, "ce_type", ceType, "error", err) return } @@ -288,7 +289,9 @@ func (d *DurableEmitter) retransmitPending(ctx context.Context) { if d.metrics != nil { d.metrics.publishBatchEvErr.Add(ctx, 1) } - d.log.Debugw("retransmit publish failed", "id", ids[i], "error", pubErr) + ceID, ceSource, ceType := cloudEventDbg(events[i]) + d.log.Debugw("retransmit publish failed", + "id", ids[i], "ce_id", ceID, "ce_source", ceSource, "ce_type", ceType, "error", pubErr) continue } if d.metrics != nil { @@ -372,3 +375,10 @@ func (d *DurableEmitter) metricsLoop(ctx context.Context) { } } } + +func cloudEventDbg(ev *chipingress.CloudEventPb) (ceID, ceSource, ceType string) { + if ev == nil { + return "", "", "" + } + return ev.GetId(), ev.GetSource(), ev.GetType() +} diff --git a/pkg/chipingress/client.go b/pkg/chipingress/client.go index d22b1807d1..e3d833284b 100644 --- a/pkg/chipingress/client.go +++ b/pkg/chipingress/client.go @@ -262,6 +262,8 @@ func newHeaderInterceptor(provider HeaderProvider) grpc.UnaryClientInterceptor { } // NewEvent creates a new CloudEvent with the specified domain, entity, payload, and optional attributes. +// Recognized optional keys include CloudEvents names (dataschema, subject, time, …) and Beholder's +// beholder_data_schema, which is mapped to the CloudEvent dataschema when dataschema is not set. func NewEvent(domain, entity string, payload []byte, attributes map[string]any) (CloudEvent, error) { event := ce.NewEvent() @@ -274,6 +276,8 @@ func NewEvent(domain, entity string, payload []byte, attributes map[string]any) attributes = make(map[string]any) } + const beholderDataSchemaKey = "beholder_data_schema" + recordedTime := time.Now() if val, ok := attributes["recordedtime"].(time.Time); ok && !val.IsZero() { recordedTime = val @@ -289,6 +293,8 @@ func NewEvent(domain, entity string, payload []byte, attributes map[string]any) } if val, ok := attributes["dataschema"].(string); ok { event.SetDataSchema(val) + } else if val, ok := attributes[beholderDataSchemaKey].(string); ok { + event.SetDataSchema(val) } if val, ok := attributes["subject"].(string); ok { event.SetSubject(val) diff --git a/pkg/chipingress/client_test.go b/pkg/chipingress/client_test.go index 6b259460a6..82f32e749c 100644 --- a/pkg/chipingress/client_test.go +++ b/pkg/chipingress/client_test.go @@ -126,6 +126,29 @@ func TestNewEvent(t *testing.T) { assert.Equal(t, testProto.Message, resultProto.Message) } +func TestNewEventBeholderDataSchema(t *testing.T) { + testProto := pb.PingResponse{Message: "x"} + protoBytes, err := proto.Marshal(&testProto) + require.NoError(t, err) + + t.Run("beholder_data_schema sets CloudEvent dataschema", func(t *testing.T) { + event, err := NewEvent("platform", "workflows.v2.WorkflowUserLog", protoBytes, map[string]any{ + "beholder_data_schema": "/cre-events-user-logs/v2", + }) + require.NoError(t, err) + assert.Equal(t, "/cre-events-user-logs/v2", event.DataSchema()) + }) + + t.Run("dataschema takes precedence over beholder_data_schema", func(t *testing.T) { + event, err := NewEvent("platform", "workflows.v2.WorkflowUserLog", protoBytes, map[string]any{ + "dataschema": "https://explicit.example/schema", + "beholder_data_schema": "/ignored", + }) + require.NoError(t, err) + assert.Equal(t, "https://explicit.example/schema", event.DataSchema()) + }) +} + func TestEventToProto(t *testing.T) { // Create a test protobuf message testProto := pb.PingResponse{Message: "test message"} From 48810bc052468d8b6abb0e564d45493addeb3435 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Wed, 25 Mar 2026 17:11:28 -0400 Subject: [PATCH 10/45] Log publish details --- pkg/beholder/durable_emitter.go | 97 +++++++++++++++++++++++++++++---- 1 file changed, 86 insertions(+), 11 deletions(-) diff --git a/pkg/beholder/durable_emitter.go b/pkg/beholder/durable_emitter.go index eb09185551..3e03702f23 100644 --- a/pkg/beholder/durable_emitter.go +++ b/pkg/beholder/durable_emitter.go @@ -3,9 +3,12 @@ package beholder import ( "context" "fmt" + "slices" + "strings" "sync" "time" + cepb "github.com/cloudevents/sdk-go/binding/format/protobuf/v2/pb" "google.golang.org/protobuf/proto" "github.com/smartcontractkit/chainlink-common/pkg/chipingress" @@ -196,10 +199,14 @@ func (d *DurableEmitter) publishAndDelete(id int64, eventPb *chipingress.CloudEv ctx, cancel := context.WithTimeout(context.Background(), d.cfg.PublishTimeout) defer cancel() + detailKVs := cloudEventPublishKVs(id, "immediate", d.cfg.PublishTimeout, eventPb) + d.log.Infow("DurableEmitter: Chip Ingress publish attempt (immediate)", detailKVs...) + t0 := time.Now() _, err := d.client.Publish(ctx, eventPb) + elapsed := time.Since(t0) if h := d.cfg.Hooks; h != nil && h.OnImmediatePublish != nil { - h.OnImmediatePublish(time.Since(t0), err) + h.OnImmediatePublish(elapsed, err) } mctx := context.Background() if d.metrics != nil { @@ -210,9 +217,13 @@ func (d *DurableEmitter) publishAndDelete(id int64, eventPb *chipingress.CloudEv } } if err != nil { - ceID, ceSource, ceType := cloudEventDbg(eventPb) - d.log.Debugw("immediate publish failed, retransmit loop will retry", - "id", id, "ce_id", ceID, "ce_source", ceSource, "ce_type", ceType, "error", err) + failKVs := append([]any{}, detailKVs...) + failKVs = append(failKVs, + "error", err, + "elapsed", elapsed.String(), + "elapsed_ms", elapsed.Milliseconds(), + ) + d.log.Infow("DurableEmitter: Chip Ingress publish failed (immediate), retransmit loop will retry", failKVs...) return } @@ -278,20 +289,28 @@ func (d *DurableEmitter) retransmitPending(ctx context.Context) { tDel := time.Now() var deleted int for i := range events { + detailKVs := cloudEventPublishKVs(ids[i], "retransmit", d.cfg.PublishTimeout, events[i]) + d.log.Infow("DurableEmitter: Chip Ingress publish attempt (retransmit)", detailKVs...) + tPub := time.Now() pubCtx, cancel := context.WithTimeout(context.Background(), d.cfg.PublishTimeout) _, pubErr := d.client.Publish(pubCtx, events[i]) cancel() + elapsed := time.Since(tPub) if h := d.cfg.Hooks; h != nil && h.OnRetransmitBatchPublish != nil { - h.OnRetransmitBatchPublish(time.Since(tPub), 1, pubErr) + h.OnRetransmitBatchPublish(elapsed, 1, pubErr) } if pubErr != nil { if d.metrics != nil { d.metrics.publishBatchEvErr.Add(ctx, 1) } - ceID, ceSource, ceType := cloudEventDbg(events[i]) - d.log.Debugw("retransmit publish failed", - "id", ids[i], "ce_id", ceID, "ce_source", ceSource, "ce_type", ceType, "error", pubErr) + failKVs := append([]any{}, detailKVs...) + failKVs = append(failKVs, + "error", pubErr, + "elapsed", elapsed.String(), + "elapsed_ms", elapsed.Milliseconds(), + ) + d.log.Infow("DurableEmitter: Chip Ingress publish failed (retransmit)", failKVs...) continue } if d.metrics != nil { @@ -376,9 +395,65 @@ func (d *DurableEmitter) metricsLoop(ctx context.Context) { } } -func cloudEventDbg(ev *chipingress.CloudEventPb) (ceID, ceSource, ceType string) { +// cloudEventPublishKVs returns structured fields for logging a Chip Ingress Publish RPC. +func cloudEventPublishKVs(durableRowID int64, phase string, timeout time.Duration, ev *chipingress.CloudEventPb) []any { if ev == nil { - return "", "", "" + return []any{ + "durable_row_id", durableRowID, + "publish_phase", phase, + "publish_timeout", timeout.String(), + "ce_nil", true, + } + } + + attrs := ev.GetAttributes() + bin := ev.GetBinaryData() + text := ev.GetTextData() + pd := ev.GetProtoData() + var protoTypeURL string + if pd != nil { + protoTypeURL = pd.GetTypeUrl() + } + + attrKeys := make([]string, 0, len(attrs)) + for k := range attrs { + attrKeys = append(attrKeys, k) + } + slices.Sort(attrKeys) + + kvs := []any{ + "durable_row_id", durableRowID, + "publish_phase", phase, + "publish_timeout", timeout.String(), + "ce_id", ev.GetId(), + "ce_source", ev.GetSource(), + "ce_type", ev.GetType(), + "ce_spec_version", ev.GetSpecVersion(), + "ce_data_binary_bytes", len(bin), + "ce_data_text_bytes", len(text), + "ce_proto_data_type_url", protoTypeURL, + "ce_attribute_count", len(attrs), + "ce_attribute_keys", strings.Join(attrKeys, ","), + "ce_attr_datacontenttype", cloudEventAttrString(attrs, "datacontenttype"), + "ce_attr_dataschema", cloudEventAttrString(attrs, "dataschema"), + "ce_attr_subject", cloudEventAttrString(attrs, "subject"), + } + return kvs +} + +func cloudEventAttrString(attrs map[string]*cepb.CloudEventAttributeValue, key string) string { + if attrs == nil { + return "" + } + v := attrs[key] + if v == nil { + return "" + } + if s := v.GetCeString(); s != "" { + return s + } + if s := v.GetCeUri(); s != "" { + return s } - return ev.GetId(), ev.GetSource(), ev.GetType() + return "" } From b787de217620618af4f980b2478d29be49ad7fa6 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Wed, 25 Mar 2026 17:24:11 -0400 Subject: [PATCH 11/45] Add logs --- pkg/beholder/durable_emitter.go | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/pkg/beholder/durable_emitter.go b/pkg/beholder/durable_emitter.go index 3e03702f23..63d5bbf4b7 100644 --- a/pkg/beholder/durable_emitter.go +++ b/pkg/beholder/durable_emitter.go @@ -227,6 +227,13 @@ func (d *DurableEmitter) publishAndDelete(id int64, eventPb *chipingress.CloudEv return } + pubOKKVs := append([]any{}, detailKVs...) + pubOKKVs = append(pubOKKVs, + "publish_rpc_elapsed", elapsed.String(), + "publish_rpc_elapsed_ms", elapsed.Milliseconds(), + ) + d.log.Infow("DurableEmitter: Chip Ingress publish succeeded (immediate)", pubOKKVs...) + t1 := time.Now() delErr := d.store.Delete(context.Background(), id) if h := d.cfg.Hooks; h != nil && h.OnImmediateDelete != nil { @@ -235,9 +242,18 @@ func (d *DurableEmitter) publishAndDelete(id int64, eventPb *chipingress.CloudEv if delErr == nil && d.metrics != nil { d.metrics.deliverComplete.Add(mctx, 1) } + delElapsed := time.Since(t1) if delErr != nil { d.log.Errorw("failed to delete delivered event", "id", id, "error", delErr) + return } + delOKKVs := append([]any{}, detailKVs...) + delOKKVs = append(delOKKVs, + "publish_rpc_elapsed_ms", elapsed.Milliseconds(), + "store_delete_elapsed", delElapsed.String(), + "store_delete_elapsed_ms", delElapsed.Milliseconds(), + ) + d.log.Infow("DurableEmitter: durable row deleted after successful Chip publish (immediate)", delOKKVs...) } func (d *DurableEmitter) retransmitLoop(ctx context.Context) { @@ -313,9 +329,16 @@ func (d *DurableEmitter) retransmitPending(ctx context.Context) { d.log.Infow("DurableEmitter: Chip Ingress publish failed (retransmit)", failKVs...) continue } + pubOKKVs := append([]any{}, detailKVs...) + pubOKKVs = append(pubOKKVs, + "publish_rpc_elapsed", elapsed.String(), + "publish_rpc_elapsed_ms", elapsed.Milliseconds(), + ) + d.log.Infow("DurableEmitter: Chip Ingress publish succeeded (retransmit)", pubOKKVs...) if d.metrics != nil { d.metrics.publishBatchEvOK.Add(ctx, 1) } + tDelOne := time.Now() if delErr := d.store.Delete(ctx, ids[i]); delErr != nil { d.log.Errorw("failed to delete retransmitted event", "id", ids[i], "error", delErr) continue @@ -324,6 +347,14 @@ func (d *DurableEmitter) retransmitPending(ctx context.Context) { if d.metrics != nil { d.metrics.deliverComplete.Add(ctx, 1) } + delElapsed := time.Since(tDelOne) + delOKKVs := append([]any{}, detailKVs...) + delOKKVs = append(delOKKVs, + "publish_rpc_elapsed_ms", elapsed.Milliseconds(), + "store_delete_elapsed", delElapsed.String(), + "store_delete_elapsed_ms", delElapsed.Milliseconds(), + ) + d.log.Infow("DurableEmitter: durable row deleted after successful Chip publish (retransmit)", delOKKVs...) } if deleted > 0 { d.log.Debugw("retransmitted events", "deleted", deleted, "attempted", len(events)) From df410b6cea2c3a15b78eceafe37cb53772bb4fce Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Wed, 25 Mar 2026 17:50:04 -0400 Subject: [PATCH 12/45] Add Persist Sources --- pkg/beholder/durable_emitter.go | 103 ++++++++++++++++++++++++--- pkg/beholder/durable_emitter_test.go | 82 +++++++++++++++++++++ 2 files changed, 176 insertions(+), 9 deletions(-) diff --git a/pkg/beholder/durable_emitter.go b/pkg/beholder/durable_emitter.go index 63d5bbf4b7..092bc3d6bd 100644 --- a/pkg/beholder/durable_emitter.go +++ b/pkg/beholder/durable_emitter.go @@ -37,6 +37,12 @@ type DurableEmitterConfig struct { // Metrics enables OpenTelemetry instruments on beholder.GetMeter() (queue, publish, store, optional process stats). // Nil disables. Metrics *DurableEmitterMetricsConfig + // PersistCloudEventSources limits durable persistence to these CloudEvent Source values + // (the beholder_domain / ce_source). If nil, every source is persisted (library default). + // If non-nil, only matching sources are inserted and retried; others get a single best-effort + // Publish with no store insert. An empty slice persists nothing (all best-effort only). + // A one-element slice containing only "*" is treated like nil (persist all). + PersistCloudEventSources []string } // DurableEmitterHooks records Publish vs Delete latency to locate pipeline bottlenecks. @@ -78,12 +84,41 @@ type DurableEmitter struct { cfg DurableEmitterConfig log logger.Logger - metrics *durableEmitterMetrics + metrics *durableEmitterMetrics + persistFilter persistSourceFilter stopCh chan struct{} wg sync.WaitGroup } +// persistSourceFilter decides whether a CloudEvent source may be written to the durable store. +type persistSourceFilter struct { + allowAll bool + allowed map[string]struct{} +} + +func newPersistSourceFilter(sources []string) persistSourceFilter { + if sources == nil { + return persistSourceFilter{allowAll: true} + } + if len(sources) == 1 && strings.TrimSpace(sources[0]) == "*" { + return persistSourceFilter{allowAll: true} + } + m := make(map[string]struct{}, len(sources)) + for _, s := range sources { + m[strings.TrimSpace(s)] = struct{}{} + } + return persistSourceFilter{allowed: m} +} + +func (f persistSourceFilter) allows(source string) bool { + if f.allowAll { + return true + } + _, ok := f.allowed[source] + return ok +} + var _ Emitter = (*DurableEmitter)(nil) func NewDurableEmitter( @@ -111,12 +146,13 @@ func NewDurableEmitter( store = newMetricsInstrumentedStore(store, m) } return &DurableEmitter{ - store: store, - client: client, - cfg: cfg, - log: log, - metrics: m, - stopCh: make(chan struct{}), + store: store, + client: client, + cfg: cfg, + log: log, + metrics: m, + persistFilter: newPersistSourceFilter(cfg.PersistCloudEventSources), + stopCh: make(chan struct{}), }, nil } @@ -135,8 +171,9 @@ func (d *DurableEmitter) Start(ctx context.Context) { } } -// Emit persists the event then attempts async delivery. -// Returns nil once the store insert succeeds. +// Emit persists the event then attempts async delivery when the CloudEvent source is allowed +// by PersistCloudEventSources; otherwise it performs a single best-effort Publish with no +// persistence. Returns nil once processing is accepted (insert succeeded, or non-persist path started). func (d *DurableEmitter) Emit(ctx context.Context, body []byte, attrKVs ...any) error { emitFail := func() { if d.metrics != nil { @@ -161,6 +198,11 @@ func (d *DurableEmitter) Emit(ctx context.Context, body []byte, attrKVs ...any) return fmt.Errorf("failed to convert event to proto: %w", err) } + if !d.persistFilter.allows(sourceDomain) { + go d.publishBestEffortNoStore(proto.Clone(eventPb)) + return nil + } + payload, err := proto.Marshal(eventPb) if err != nil { emitFail() @@ -187,6 +229,43 @@ func (d *DurableEmitter) Emit(ctx context.Context, body []byte, attrKVs ...any) return nil } +// publishBestEffortNoStore performs one Publish without persisting or retries. +func (d *DurableEmitter) publishBestEffortNoStore(eventPb *chipingress.CloudEventPb) { + ctx, cancel := context.WithTimeout(context.Background(), d.cfg.PublishTimeout) + defer cancel() + + detailKVs := cloudEventPublishKVs(0, "best_effort_no_store", d.cfg.PublishTimeout, eventPb) + d.log.Infow("DurableEmitter: Chip Ingress publish attempt (best-effort, not persisted)", detailKVs...) + + t0 := time.Now() + _, err := d.client.Publish(ctx, eventPb) + elapsed := time.Since(t0) + if h := d.cfg.Hooks; h != nil && h.OnImmediatePublish != nil { + h.OnImmediatePublish(elapsed, err) + } + mctx := context.Background() + if d.metrics != nil { + if err != nil { + d.metrics.publishImmErr.Add(mctx, 1) + } else { + d.metrics.publishImmOK.Add(mctx, 1) + } + } + if err != nil { + failKVs := append([]any{}, detailKVs...) + failKVs = append(failKVs, + "error", err, + "elapsed", elapsed.String(), + "elapsed_ms", elapsed.Milliseconds(), + ) + d.log.Infow("DurableEmitter: best-effort Chip publish failed (not persisted, no retry)", failKVs...) + return + } + okKVs := append([]any{}, detailKVs...) + okKVs = append(okKVs, "publish_rpc_elapsed_ms", elapsed.Milliseconds()) + d.log.Infow("DurableEmitter: best-effort Chip publish succeeded (not persisted)", okKVs...) +} + // Close signals background loops to stop and waits for them to finish. func (d *DurableEmitter) Close() error { close(d.stopCh) @@ -294,6 +373,12 @@ func (d *DurableEmitter) retransmitPending(ctx context.Context) { _ = d.store.Delete(ctx, pe.ID) continue } + if !d.persistFilter.allows(ev.GetSource()) { + d.log.Infow("DurableEmitter: dropping queued event (ce_source not in PersistCloudEventSources)", + "id", pe.ID, "ce_source", ev.GetSource(), "ce_type", ev.GetType()) + _ = d.store.Delete(ctx, pe.ID) + continue + } events = append(events, ev) ids = append(ids, pe.ID) } diff --git a/pkg/beholder/durable_emitter_test.go b/pkg/beholder/durable_emitter_test.go index 9429f6ec56..7846e63031 100644 --- a/pkg/beholder/durable_emitter_test.go +++ b/pkg/beholder/durable_emitter_test.go @@ -11,6 +11,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "google.golang.org/grpc" + "google.golang.org/protobuf/proto" "github.com/smartcontractkit/chainlink-common/pkg/chipingress" "github.com/smartcontractkit/chainlink-common/pkg/logger" @@ -268,6 +269,87 @@ func TestDurableEmitter_ExpiryLoopDeletesOldEvents(t *testing.T) { }, 5*time.Second, 50*time.Millisecond, "expiry loop should purge the event") } +func TestDurableEmitter_PersistSourceFilter_skipsStoreBestEffortPublish(t *testing.T) { + store := NewMemDurableEventStore() + client := &testChipClient{} + cfg := DefaultDurableEmitterConfig() + cfg.PersistCloudEventSources = []string{"only-this"} + em := newTestDurableEmitter(t, store, client, &cfg) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + em.Start(ctx) + defer em.Close() + + require.NoError(t, em.Emit(ctx, []byte("x"), testEmitAttrs()...)) + require.Eventually(t, func() bool { return client.publishCount.Load() == 1 }, 2*time.Second, 10*time.Millisecond) + assert.Equal(t, 0, store.Len()) +} + +func TestDurableEmitter_PersistSourceFilter_persistsAllowedSource(t *testing.T) { + store := NewMemDurableEventStore() + client := &testChipClient{} + cfg := DefaultDurableEmitterConfig() + cfg.PersistCloudEventSources = []string{"test-source"} + em := newTestDurableEmitter(t, store, client, &cfg) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + em.Start(ctx) + defer em.Close() + + require.NoError(t, em.Emit(ctx, []byte("x"), testEmitAttrs()...)) + require.Eventually(t, func() bool { return client.publishCount.Load() == 1 }, 2*time.Second, 10*time.Millisecond) + require.Eventually(t, func() bool { return store.Len() == 0 }, 2*time.Second, 10*time.Millisecond) +} + +func TestDurableEmitter_PersistSourceWildcardStarAllowsAll(t *testing.T) { + store := NewMemDurableEventStore() + client := &testChipClient{} + cfg := DefaultDurableEmitterConfig() + cfg.PersistCloudEventSources = []string{"*"} + em := newTestDurableEmitter(t, store, client, &cfg) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + em.Start(ctx) + defer em.Close() + + require.NoError(t, em.Emit(ctx, []byte("x"), testEmitAttrs()...)) + require.Eventually(t, func() bool { return store.Len() == 0 }, 2*time.Second, 10*time.Millisecond) +} + +func TestDurableEmitter_RetransmitDropsDisallowedSource(t *testing.T) { + store := NewMemDurableEventStore() + client := &testChipClient{} + + ev, err := chipingress.NewEvent("unknown-domain", "t", []byte("b"), nil) + require.NoError(t, err) + evPb, err := chipingress.EventToProto(ev) + require.NoError(t, err) + payload, err := proto.Marshal(evPb) + require.NoError(t, err) + + _, err = store.Insert(context.Background(), payload) + require.NoError(t, err) + + cfg := DefaultDurableEmitterConfig() + cfg.PersistCloudEventSources = []string{"test-source"} + cfg.RetransmitInterval = 50 * time.Millisecond + cfg.RetransmitAfter = 30 * time.Millisecond + + em := newTestDurableEmitter(t, store, client, &cfg) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + em.Start(ctx) + defer em.Close() + + require.Eventually(t, func() bool { + return store.Len() == 0 && client.publishCount.Load() == 0 + }, 3*time.Second, 20*time.Millisecond, "disallowed row should be deleted without Publish") +} + func TestDurableEmitter_EmitRejectsInvalidAttributes(t *testing.T) { store := NewMemDurableEventStore() client := &testChipClient{} From ac292754837c86e8cbe5725c4fd6f931ea68225b Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Wed, 25 Mar 2026 17:53:43 -0400 Subject: [PATCH 13/45] Update durable_emitter.go --- pkg/beholder/durable_emitter.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pkg/beholder/durable_emitter.go b/pkg/beholder/durable_emitter.go index 092bc3d6bd..07ae54ee88 100644 --- a/pkg/beholder/durable_emitter.go +++ b/pkg/beholder/durable_emitter.go @@ -199,7 +199,13 @@ func (d *DurableEmitter) Emit(ctx context.Context, body []byte, attrKVs ...any) } if !d.persistFilter.allows(sourceDomain) { - go d.publishBestEffortNoStore(proto.Clone(eventPb)) + cl := proto.Clone(eventPb) + evCopy, ok := cl.(*chipingress.CloudEventPb) + if !ok { + emitFail() + return fmt.Errorf("proto.Clone event: got %T, want *chipingress.CloudEventPb", cl) + } + go d.publishBestEffortNoStore(evCopy) return nil } From 61a31c146680df5216865197cd14a5b3b75ddbc9 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Tue, 31 Mar 2026 13:33:59 -0400 Subject: [PATCH 14/45] mute logging --- pkg/beholder/durable_emitter.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/beholder/durable_emitter.go b/pkg/beholder/durable_emitter.go index 07ae54ee88..e89022bf10 100644 --- a/pkg/beholder/durable_emitter.go +++ b/pkg/beholder/durable_emitter.go @@ -241,7 +241,7 @@ func (d *DurableEmitter) publishBestEffortNoStore(eventPb *chipingress.CloudEven defer cancel() detailKVs := cloudEventPublishKVs(0, "best_effort_no_store", d.cfg.PublishTimeout, eventPb) - d.log.Infow("DurableEmitter: Chip Ingress publish attempt (best-effort, not persisted)", detailKVs...) + //d.log.Infow("DurableEmitter: Chip Ingress publish attempt (best-effort, not persisted)", detailKVs...) t0 := time.Now() _, err := d.client.Publish(ctx, eventPb) @@ -264,12 +264,12 @@ func (d *DurableEmitter) publishBestEffortNoStore(eventPb *chipingress.CloudEven "elapsed", elapsed.String(), "elapsed_ms", elapsed.Milliseconds(), ) - d.log.Infow("DurableEmitter: best-effort Chip publish failed (not persisted, no retry)", failKVs...) + //d.log.Infow("DurableEmitter: best-effort Chip publish failed (not persisted, no retry)", failKVs...) return } okKVs := append([]any{}, detailKVs...) okKVs = append(okKVs, "publish_rpc_elapsed_ms", elapsed.Milliseconds()) - d.log.Infow("DurableEmitter: best-effort Chip publish succeeded (not persisted)", okKVs...) + //d.log.Infow("DurableEmitter: best-effort Chip publish succeeded (not persisted)", okKVs...) } // Close signals background loops to stop and waits for them to finish. @@ -285,7 +285,7 @@ func (d *DurableEmitter) publishAndDelete(id int64, eventPb *chipingress.CloudEv defer cancel() detailKVs := cloudEventPublishKVs(id, "immediate", d.cfg.PublishTimeout, eventPb) - d.log.Infow("DurableEmitter: Chip Ingress publish attempt (immediate)", detailKVs...) + //d.log.Infow("DurableEmitter: Chip Ingress publish attempt (immediate)", detailKVs...) t0 := time.Now() _, err := d.client.Publish(ctx, eventPb) @@ -317,7 +317,7 @@ func (d *DurableEmitter) publishAndDelete(id int64, eventPb *chipingress.CloudEv "publish_rpc_elapsed", elapsed.String(), "publish_rpc_elapsed_ms", elapsed.Milliseconds(), ) - d.log.Infow("DurableEmitter: Chip Ingress publish succeeded (immediate)", pubOKKVs...) + //d.log.Infow("DurableEmitter: Chip Ingress publish succeeded (immediate)", pubOKKVs...) t1 := time.Now() delErr := d.store.Delete(context.Background(), id) @@ -338,7 +338,7 @@ func (d *DurableEmitter) publishAndDelete(id int64, eventPb *chipingress.CloudEv "store_delete_elapsed", delElapsed.String(), "store_delete_elapsed_ms", delElapsed.Milliseconds(), ) - d.log.Infow("DurableEmitter: durable row deleted after successful Chip publish (immediate)", delOKKVs...) + //d.log.Infow("DurableEmitter: durable row deleted after successful Chip publish (immediate)", delOKKVs...) } func (d *DurableEmitter) retransmitLoop(ctx context.Context) { From 657342c208d287ca3c604f07df54b2eb83b91729 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Tue, 31 Mar 2026 13:36:07 -0400 Subject: [PATCH 15/45] Update durable_emitter.go --- pkg/beholder/durable_emitter.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/beholder/durable_emitter.go b/pkg/beholder/durable_emitter.go index e89022bf10..45f4439a94 100644 --- a/pkg/beholder/durable_emitter.go +++ b/pkg/beholder/durable_emitter.go @@ -425,7 +425,7 @@ func (d *DurableEmitter) retransmitPending(ctx context.Context) { "publish_rpc_elapsed", elapsed.String(), "publish_rpc_elapsed_ms", elapsed.Milliseconds(), ) - d.log.Infow("DurableEmitter: Chip Ingress publish succeeded (retransmit)", pubOKKVs...) + //d.log.Infow("DurableEmitter: Chip Ingress publish succeeded (retransmit)", pubOKKVs...) if d.metrics != nil { d.metrics.publishBatchEvOK.Add(ctx, 1) } @@ -445,7 +445,7 @@ func (d *DurableEmitter) retransmitPending(ctx context.Context) { "store_delete_elapsed", delElapsed.String(), "store_delete_elapsed_ms", delElapsed.Milliseconds(), ) - d.log.Infow("DurableEmitter: durable row deleted after successful Chip publish (retransmit)", delOKKVs...) + //d.log.Infow("DurableEmitter: durable row deleted after successful Chip publish (retransmit)", delOKKVs...) } if deleted > 0 { d.log.Debugw("retransmitted events", "deleted", deleted, "attempted", len(events)) From d18962efd1855a9dc58fc1720d77542b53ede2fd Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Tue, 31 Mar 2026 13:39:16 -0400 Subject: [PATCH 16/45] Update durable_emitter.go --- pkg/beholder/durable_emitter.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/beholder/durable_emitter.go b/pkg/beholder/durable_emitter.go index 45f4439a94..10f4337369 100644 --- a/pkg/beholder/durable_emitter.go +++ b/pkg/beholder/durable_emitter.go @@ -397,7 +397,7 @@ func (d *DurableEmitter) retransmitPending(ctx context.Context) { var deleted int for i := range events { detailKVs := cloudEventPublishKVs(ids[i], "retransmit", d.cfg.PublishTimeout, events[i]) - d.log.Infow("DurableEmitter: Chip Ingress publish attempt (retransmit)", detailKVs...) + //d.log.Infow("DurableEmitter: Chip Ingress publish attempt (retransmit)", detailKVs...) tPub := time.Now() pubCtx, cancel := context.WithTimeout(context.Background(), d.cfg.PublishTimeout) From 7678416be8a4a1df65e58a58397ab0a42587b8d2 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Wed, 1 Apr 2026 09:59:24 -0400 Subject: [PATCH 17/45] Background delete from event store --- pkg/beholder/durable_emitter.go | 137 ++++++++++++++++----- pkg/beholder/durable_emitter_store_wrap.go | 17 ++- pkg/beholder/durable_event_store.go | 18 ++- 3 files changed, 136 insertions(+), 36 deletions(-) diff --git a/pkg/beholder/durable_emitter.go b/pkg/beholder/durable_emitter.go index 10f4337369..327b4f70bd 100644 --- a/pkg/beholder/durable_emitter.go +++ b/pkg/beholder/durable_emitter.go @@ -31,6 +31,11 @@ type DurableEmitterConfig struct { EventTTL time.Duration // PublishTimeout is the per-RPC deadline for each Publish call. PublishTimeout time.Duration + // PurgeInterval is how often the purge loop runs to batch-delete rows that + // were marked delivered (Postgres). Zero defaults to 250ms. + PurgeInterval time.Duration + // PurgeBatchSize is the maximum rows removed per PurgeDelivered call. Zero defaults to 500. + PurgeBatchSize int // Hooks is optional instrumentation (load tests, profiling). Nil fields are skipped. // Callbacks may run from many goroutines; implementations must be thread-safe. Hooks *DurableEmitterHooks @@ -49,12 +54,13 @@ type DurableEmitterConfig struct { type DurableEmitterHooks struct { // OnImmediatePublish is called after each async Publish in publishAndDelete (every attempt). OnImmediatePublish func(elapsed time.Duration, err error) - // OnImmediateDelete is called after Delete following a successful immediate Publish. + // OnImmediateDelete is called after MarkDelivered following a successful immediate Publish. OnImmediateDelete func(elapsed time.Duration, err error) // OnRetransmitBatchPublish is called after each retransmit Publish (one RPC per queued event). OnRetransmitBatchPublish func(elapsed time.Duration, eventCount int, err error) - // OnRetransmitBatchDeletes is called after a retransmit tick with total time and successful delete count. - OnRetransmitBatchDeletes func(elapsed time.Duration, deleteCount int) + // OnRetransmitBatchDeletes is called after a retransmit tick with total time and count of + // successful MarkDelivered calls (mem store may delete rows; Postgres sets delivered_at). + OnRetransmitBatchDeletes func(elapsed time.Duration, markedDeliveredCount int) } func DefaultDurableEmitterConfig() DurableEmitterConfig { @@ -65,6 +71,8 @@ func DefaultDurableEmitterConfig() DurableEmitterConfig { ExpiryInterval: 1 * time.Minute, EventTTL: 24 * time.Hour, PublishTimeout: 5 * time.Second, + PurgeInterval: 250 * time.Millisecond, + PurgeBatchSize: 500, } } @@ -72,9 +80,11 @@ func DefaultDurableEmitterConfig() DurableEmitterConfig { // // On Emit the event is serialized and written to a DurableEventStore. Once the // insert succeeds Emit returns nil — the caller has a durable guarantee. An -// immediate async Publish is attempted; on success the record is deleted. If -// that fails a background retransmit loop will pick the event up and retry via -// Publish (one RPC per pending row per tick, up to RetransmitBatchSize). +// immediate async Publish is attempted; on success the record is MarkDelivered +// (excluded from retries). Postgres stores then purge physical rows in batches; +// in-memory stores remove the row immediately. If Publish fails, a background +// retransmit loop retries via Publish (one RPC per pending row per tick, up to +// RetransmitBatchSize). // // A separate expiry loop garbage-collects events older than EventTTL to bound // table growth. @@ -156,16 +166,17 @@ func NewDurableEmitter( }, nil } -// Start launches the retransmit and expiry background loops. +// Start launches the retransmit, expiry, and purge background loops. // Cancel the supplied context or call Close to stop them. func (d *DurableEmitter) Start(ctx context.Context) { - n := 2 + n := 3 if d.metrics != nil && d.cfg.Metrics != nil { n++ } d.wg.Add(n) go d.retransmitLoop(ctx) go d.expiryLoop(ctx) + go d.purgeLoop(ctx) if d.metrics != nil && d.cfg.Metrics != nil { go d.metricsLoop(ctx) } @@ -320,25 +331,25 @@ func (d *DurableEmitter) publishAndDelete(id int64, eventPb *chipingress.CloudEv //d.log.Infow("DurableEmitter: Chip Ingress publish succeeded (immediate)", pubOKKVs...) t1 := time.Now() - delErr := d.store.Delete(context.Background(), id) + markErr := d.store.MarkDelivered(context.Background(), id) if h := d.cfg.Hooks; h != nil && h.OnImmediateDelete != nil { - h.OnImmediateDelete(time.Since(t1), delErr) + h.OnImmediateDelete(time.Since(t1), markErr) } - if delErr == nil && d.metrics != nil { + if markErr == nil && d.metrics != nil { d.metrics.deliverComplete.Add(mctx, 1) } - delElapsed := time.Since(t1) - if delErr != nil { - d.log.Errorw("failed to delete delivered event", "id", id, "error", delErr) + markElapsed := time.Since(t1) + if markErr != nil { + d.log.Errorw("failed to mark delivered event", "id", id, "error", markErr) return } delOKKVs := append([]any{}, detailKVs...) delOKKVs = append(delOKKVs, "publish_rpc_elapsed_ms", elapsed.Milliseconds(), - "store_delete_elapsed", delElapsed.String(), - "store_delete_elapsed_ms", delElapsed.Milliseconds(), + "store_mark_delivered_elapsed", markElapsed.String(), + "store_mark_delivered_elapsed_ms", markElapsed.Milliseconds(), ) - //d.log.Infow("DurableEmitter: durable row deleted after successful Chip publish (immediate)", delOKKVs...) + //d.log.Infow("DurableEmitter: durable row marked delivered after successful Chip publish (immediate)", delOKKVs...) } func (d *DurableEmitter) retransmitLoop(ctx context.Context) { @@ -365,6 +376,24 @@ func (d *DurableEmitter) retransmitPending(ctx context.Context) { d.log.Errorw("failed to list pending events", "error", err) return } + + if obs, ok := d.store.(DurableQueueObserver); ok { + st, obsErr := obs.ObserveDurableQueue(ctx, d.cfg.EventTTL, d.queueStatsNearExpiryLead()) + if obsErr != nil { + d.log.Warnw("DurableEmitter: retransmit scan ObserveDurableQueue failed", "error", obsErr) + } else { + d.log.Infow("DurableEmitter: retransmit pending scan", + "pending_rows", st.Depth, + "pending_payload_bytes", st.PayloadBytes, + "oldest_pending_age", st.OldestPendingAge.String(), + "near_ttl_rows", st.NearTTLCount, + "retransmit_list_batch", len(pending), + "retransmit_after", d.cfg.RetransmitAfter.String(), + "list_limit", d.cfg.RetransmitBatchSize, + ) + } + } + if len(pending) == 0 { return } @@ -394,7 +423,7 @@ func (d *DurableEmitter) retransmitPending(ctx context.Context) { // One Publish per row so a single bad or rejected event does not block the rest of the slice. tDel := time.Now() - var deleted int + var markedDelivered int for i := range events { detailKVs := cloudEventPublishKVs(ids[i], "retransmit", d.cfg.PublishTimeout, events[i]) //d.log.Infow("DurableEmitter: Chip Ingress publish attempt (retransmit)", detailKVs...) @@ -429,29 +458,65 @@ func (d *DurableEmitter) retransmitPending(ctx context.Context) { if d.metrics != nil { d.metrics.publishBatchEvOK.Add(ctx, 1) } - tDelOne := time.Now() - if delErr := d.store.Delete(ctx, ids[i]); delErr != nil { - d.log.Errorw("failed to delete retransmitted event", "id", ids[i], "error", delErr) + tMarkOne := time.Now() + if markErr := d.store.MarkDelivered(ctx, ids[i]); markErr != nil { + d.log.Errorw("failed to mark retransmitted event delivered", "id", ids[i], "error", markErr) continue } - deleted++ + markedDelivered++ if d.metrics != nil { d.metrics.deliverComplete.Add(ctx, 1) } - delElapsed := time.Since(tDelOne) + markElapsed := time.Since(tMarkOne) delOKKVs := append([]any{}, detailKVs...) delOKKVs = append(delOKKVs, "publish_rpc_elapsed_ms", elapsed.Milliseconds(), - "store_delete_elapsed", delElapsed.String(), - "store_delete_elapsed_ms", delElapsed.Milliseconds(), + "store_mark_delivered_elapsed", markElapsed.String(), + "store_mark_delivered_elapsed_ms", markElapsed.Milliseconds(), ) //d.log.Infow("DurableEmitter: durable row deleted after successful Chip publish (retransmit)", delOKKVs...) } - if deleted > 0 { - d.log.Debugw("retransmitted events", "deleted", deleted, "attempted", len(events)) + if markedDelivered > 0 { + d.log.Infow("retransmitted events", + "marked_delivered", markedDelivered, + "attempted", len(events), + ) } - if h := d.cfg.Hooks; h != nil && h.OnRetransmitBatchDeletes != nil && deleted > 0 { - h.OnRetransmitBatchDeletes(time.Since(tDel), deleted) + if h := d.cfg.Hooks; h != nil && h.OnRetransmitBatchDeletes != nil && markedDelivered > 0 { + h.OnRetransmitBatchDeletes(time.Since(tDel), markedDelivered) + } +} + +func (d *DurableEmitter) purgeLoop(ctx context.Context) { + defer d.wg.Done() + interval := d.cfg.PurgeInterval + if interval <= 0 { + interval = 250 * time.Millisecond + } + batch := d.cfg.PurgeBatchSize + if batch <= 0 { + batch = 500 + } + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-d.stopCh: + return + case <-ticker.C: + for { + n, err := d.store.PurgeDelivered(ctx, batch) + if err != nil { + d.log.Errorw("failed to purge delivered chip durable events", "error", err) + break + } + if n == 0 { + break + } + } + } } } @@ -482,6 +547,14 @@ func (d *DurableEmitter) expiryLoop(ctx context.Context) { } } +func (d *DurableEmitter) queueStatsNearExpiryLead() time.Duration { + lead := 5 * time.Minute + if d.cfg.Metrics != nil && d.cfg.Metrics.NearExpiryLead > 0 { + lead = d.cfg.Metrics.NearExpiryLead + } + return lead +} + func (d *DurableEmitter) metricsLoop(ctx context.Context) { defer d.wg.Done() mc := d.cfg.Metrics @@ -489,10 +562,6 @@ func (d *DurableEmitter) metricsLoop(ctx context.Context) { if poll <= 0 { poll = 10 * time.Second } - lead := mc.NearExpiryLead - if lead <= 0 { - lead = 5 * time.Minute - } ticker := time.NewTicker(poll) defer ticker.Stop() for { @@ -507,7 +576,7 @@ func (d *DurableEmitter) metricsLoop(ctx context.Context) { } bctx := context.Background() if obs, ok := d.store.(DurableQueueObserver); ok { - d.metrics.pollQueueGauges(bctx, obs, d.cfg.EventTTL, lead, mc.MaxQueuePayloadBytes) + d.metrics.pollQueueGauges(bctx, obs, d.cfg.EventTTL, d.queueStatsNearExpiryLead(), mc.MaxQueuePayloadBytes) } if mc.RecordProcessStats { d.metrics.recordProcessMem(bctx) diff --git a/pkg/beholder/durable_emitter_store_wrap.go b/pkg/beholder/durable_emitter_store_wrap.go index 75faa252de..9f68047a76 100644 --- a/pkg/beholder/durable_emitter_store_wrap.go +++ b/pkg/beholder/durable_emitter_store_wrap.go @@ -2,6 +2,7 @@ package beholder import ( "context" + "errors" "time" ) @@ -35,6 +36,20 @@ func (s *metricsInstrumentedStore) Delete(ctx context.Context, id int64) error { return err } +func (s *metricsInstrumentedStore) MarkDelivered(ctx context.Context, id int64) error { + t0 := time.Now() + err := s.inner.MarkDelivered(ctx, id) + s.m.recordStoreOp(ctx, "mark_delivered", time.Since(t0), err) + return err +} + +func (s *metricsInstrumentedStore) PurgeDelivered(ctx context.Context, batchLimit int) (int64, error) { + t0 := time.Now() + n, err := s.inner.PurgeDelivered(ctx, batchLimit) + s.m.recordStoreOp(ctx, "purge_delivered", time.Since(t0), err) + return n, err +} + func (s *metricsInstrumentedStore) ListPending(ctx context.Context, createdBefore time.Time, limit int) ([]DurableEvent, error) { t0 := time.Now() evs, err := s.inner.ListPending(ctx, createdBefore, limit) @@ -52,7 +67,7 @@ func (s *metricsInstrumentedStore) DeleteExpired(ctx context.Context, ttl time.D func (s *metricsInstrumentedStore) ObserveDurableQueue(ctx context.Context, eventTTL, nearExpiryLead time.Duration) (DurableQueueStats, error) { o, ok := s.inner.(DurableQueueObserver) if !ok { - return DurableQueueStats{}, nil + return DurableQueueStats{}, errors.New("inner DurableEventStore does not implement DurableQueueObserver") } return o.ObserveDurableQueue(ctx, eventTTL, nearExpiryLead) } diff --git a/pkg/beholder/durable_event_store.go b/pkg/beholder/durable_event_store.go index ab67d8ebb4..86b46ad7b0 100644 --- a/pkg/beholder/durable_event_store.go +++ b/pkg/beholder/durable_event_store.go @@ -39,8 +39,16 @@ type DurableQueueObserver interface { type DurableEventStore interface { // Insert persists a serialized event and returns its assigned ID. Insert(ctx context.Context, payload []byte) (int64, error) - // Delete removes a successfully delivered event. + // Delete physically removes a row (corrupt payloads, policy drops, tests). Delete(ctx context.Context, id int64) error + // MarkDelivered records successful delivery to Chip. The row must no longer + // appear in ListPending. Postgres implementations typically set delivered_at; + // a background PurgeDelivered removes rows later. MemDurableEventStore removes + // the row immediately (same as Delete). + MarkDelivered(ctx context.Context, id int64) error + // PurgeDelivered deletes up to batchLimit rows already marked delivered. + // Implementations that remove rows in MarkDelivered may return 0, nil always. + PurgeDelivered(ctx context.Context, batchLimit int) (deleted int64, err error) // ListPending returns events created before the given cutoff, ordered by // creation time ascending, up to limit rows. ListPending(ctx context.Context, createdBefore time.Time, limit int) ([]DurableEvent, error) @@ -85,6 +93,14 @@ func (m *MemDurableEventStore) Delete(_ context.Context, id int64) error { return nil } +func (m *MemDurableEventStore) MarkDelivered(ctx context.Context, id int64) error { + return m.Delete(ctx, id) +} + +func (m *MemDurableEventStore) PurgeDelivered(_ context.Context, _ int) (int64, error) { + return 0, nil +} + func (m *MemDurableEventStore) ListPending(_ context.Context, createdBefore time.Time, limit int) ([]DurableEvent, error) { m.mu.Lock() defer m.mu.Unlock() From 734fe94f345d0978d83058ec7a8aac5f57e7e1be Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Thu, 2 Apr 2026 15:39:51 -0400 Subject: [PATCH 18/45] Add metrics --- pkg/beholder/durable_emitter.go | 3 ++ pkg/beholder/durable_emitter_metric_info.go | 5 ++ pkg/beholder/durable_emitter_metrics.go | 58 +++++++++++++-------- 3 files changed, 45 insertions(+), 21 deletions(-) diff --git a/pkg/beholder/durable_emitter.go b/pkg/beholder/durable_emitter.go index 327b4f70bd..cfb48b426c 100644 --- a/pkg/beholder/durable_emitter.go +++ b/pkg/beholder/durable_emitter.go @@ -261,6 +261,7 @@ func (d *DurableEmitter) publishBestEffortNoStore(eventPb *chipingress.CloudEven h.OnImmediatePublish(elapsed, err) } mctx := context.Background() + d.metrics.recordPublish(mctx, elapsed, "best_effort", err) if d.metrics != nil { if err != nil { d.metrics.publishImmErr.Add(mctx, 1) @@ -305,6 +306,7 @@ func (d *DurableEmitter) publishAndDelete(id int64, eventPb *chipingress.CloudEv h.OnImmediatePublish(elapsed, err) } mctx := context.Background() + d.metrics.recordPublish(mctx, elapsed, "immediate", err) if d.metrics != nil { if err != nil { d.metrics.publishImmErr.Add(mctx, 1) @@ -436,6 +438,7 @@ func (d *DurableEmitter) retransmitPending(ctx context.Context) { if h := d.cfg.Hooks; h != nil && h.OnRetransmitBatchPublish != nil { h.OnRetransmitBatchPublish(elapsed, 1, pubErr) } + d.metrics.recordPublish(context.Background(), elapsed, "retransmit", pubErr) if pubErr != nil { if d.metrics != nil { d.metrics.publishBatchEvErr.Add(ctx, 1) diff --git a/pkg/beholder/durable_emitter_metric_info.go b/pkg/beholder/durable_emitter_metric_info.go index c7726e8b1a..7cd92e127e 100644 --- a/pkg/beholder/durable_emitter_metric_info.go +++ b/pkg/beholder/durable_emitter_metric_info.go @@ -29,6 +29,11 @@ var ( Unit: "{call}", Description: "Immediate Publish RPC failures (events await retransmit)", } + durableEmitterMetricPublishDuration = MetricInfo{ + Name: "beholder.durable_emitter.publish.duration", + Unit: "s", + Description: "Chip Ingress Publish RPC duration (seconds); labels: phase={immediate,retransmit,best_effort}, error={true,false}", + } durableEmitterMetricPublishBatchSuccess = MetricInfo{ Name: "beholder.durable_emitter.publish.retransmit.batch.success", Unit: "{call}", diff --git a/pkg/beholder/durable_emitter_metrics.go b/pkg/beholder/durable_emitter_metrics.go index 76d1a20c0d..c9c11a96b3 100644 --- a/pkg/beholder/durable_emitter_metrics.go +++ b/pkg/beholder/durable_emitter_metrics.go @@ -27,28 +27,29 @@ type DurableEmitterMetricsConfig struct { } type durableEmitterMetrics struct { - emitSuccess metric.Int64Counter - emitFail metric.Int64Counter - emitDuration metric.Float64Histogram - publishImmOK metric.Int64Counter - publishImmErr metric.Int64Counter - publishBatchOK metric.Int64Counter - publishBatchErr metric.Int64Counter - publishBatchEvOK metric.Int64Counter - publishBatchEvErr metric.Int64Counter - deliverComplete metric.Int64Counter - expiredPurged metric.Int64Counter - storeOps metric.Int64Counter - storeOpDuration metric.Float64Histogram - queueDepth metric.Int64Gauge - queuePayloadBytes metric.Int64Gauge - queueOldestAgeSec metric.Float64Gauge - queueNearTTL metric.Int64Gauge + emitSuccess metric.Int64Counter + emitFail metric.Int64Counter + emitDuration metric.Float64Histogram + publishImmOK metric.Int64Counter + publishImmErr metric.Int64Counter + publishDuration metric.Float64Histogram + publishBatchOK metric.Int64Counter + publishBatchErr metric.Int64Counter + publishBatchEvOK metric.Int64Counter + publishBatchEvErr metric.Int64Counter + deliverComplete metric.Int64Counter + expiredPurged metric.Int64Counter + storeOps metric.Int64Counter + storeOpDuration metric.Float64Histogram + queueDepth metric.Int64Gauge + queuePayloadBytes metric.Int64Gauge + queueOldestAgeSec metric.Float64Gauge + queueNearTTL metric.Int64Gauge queueCapacityRatio metric.Float64Gauge - procHeapInuse metric.Int64Gauge - procHeapSys metric.Int64Gauge - procCPUUser metric.Float64Gauge - procCPUSys metric.Float64Gauge + procHeapInuse metric.Int64Gauge + procHeapSys metric.Int64Gauge + procCPUUser metric.Float64Gauge + procCPUSys metric.Float64Gauge } func newDurableEmitterMetrics() (*durableEmitterMetrics, error) { @@ -70,6 +71,9 @@ func newDurableEmitterMetrics() (*durableEmitterMetrics, error) { if m.publishImmErr, err = durableEmitterMetricPublishImmFailure.NewInt64Counter(meter); err != nil { return nil, err } + if m.publishDuration, err = durableEmitterMetricPublishDuration.NewFloat64Histogram(meter); err != nil { + return nil, err + } if m.publishBatchOK, err = durableEmitterMetricPublishBatchSuccess.NewInt64Counter(meter); err != nil { return nil, err } @@ -166,3 +170,15 @@ func (m *durableEmitterMetrics) recordProcessMem(ctx context.Context) { m.procHeapInuse.Record(ctx, int64(ms.HeapInuse)) m.procHeapSys.Record(ctx, int64(ms.HeapSys)) } + +func (m *durableEmitterMetrics) recordPublish(ctx context.Context, elapsed time.Duration, phase string, err error) { + if m == nil { + return + } + m.publishDuration.Record(ctx, elapsed.Seconds(), + metric.WithAttributes( + attribute.String("phase", phase), + attribute.Bool("error", err != nil), + ), + ) +} From 5b9f9382669f6741223174c0ba1c949c44c7046c Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Thu, 9 Apr 2026 14:25:48 -0400 Subject: [PATCH 19/45] Add publish workers --- pkg/beholder/durable_emitter.go | 92 +++++++++++++++++---- pkg/beholder/durable_emitter_metric_info.go | 5 ++ pkg/beholder/durable_emitter_metrics.go | 4 + 3 files changed, 86 insertions(+), 15 deletions(-) diff --git a/pkg/beholder/durable_emitter.go b/pkg/beholder/durable_emitter.go index cfb48b426c..9838f07a15 100644 --- a/pkg/beholder/durable_emitter.go +++ b/pkg/beholder/durable_emitter.go @@ -48,6 +48,17 @@ type DurableEmitterConfig struct { // Publish with no store insert. An empty slice persists nothing (all best-effort only). // A one-element slice containing only "*" is treated like nil (persist all). PersistCloudEventSources []string + // PublishWorkers is the number of long-lived goroutines that process immediate + // Publish+MarkDelivered after each Emit. When all workers are busy, the event + // stays in the DB and the retransmit loop delivers it on the next tick — this + // provides natural back-pressure without unbounded goroutine growth. + // Zero defaults to DefaultPublishWorkers (64). + PublishWorkers int + // QuietMode suppresses high-volume INFO-level logs (retransmit scan stats, + // retransmit results, publish failures, expired event purges, etc.). + // Error-level logs are never suppressed. Useful for load tests where the + // logging overhead is measurable. + QuietMode bool } // DurableEmitterHooks records Publish vs Delete latency to locate pipeline bottlenecks. @@ -63,6 +74,8 @@ type DurableEmitterHooks struct { OnRetransmitBatchDeletes func(elapsed time.Duration, markedDeliveredCount int) } +const DefaultPublishWorkers = 64 + func DefaultDurableEmitterConfig() DurableEmitterConfig { return DurableEmitterConfig{ RetransmitInterval: 5 * time.Second, @@ -73,6 +86,7 @@ func DefaultDurableEmitterConfig() DurableEmitterConfig { PublishTimeout: 5 * time.Second, PurgeInterval: 250 * time.Millisecond, PurgeBatchSize: 500, + PublishWorkers: DefaultPublishWorkers, } } @@ -97,8 +111,14 @@ type DurableEmitter struct { metrics *durableEmitterMetrics persistFilter persistSourceFilter - stopCh chan struct{} - wg sync.WaitGroup + publishCh chan publishWork + stopCh chan struct{} + wg sync.WaitGroup +} + +type publishWork struct { + id int64 + eventPb *chipingress.CloudEventPb } // persistSourceFilter decides whether a CloudEvent source may be written to the durable store. @@ -155,6 +175,10 @@ func NewDurableEmitter( } store = newMetricsInstrumentedStore(store, m) } + workers := cfg.PublishWorkers + if workers <= 0 { + workers = DefaultPublishWorkers + } return &DurableEmitter{ store: store, client: client, @@ -162,18 +186,26 @@ func NewDurableEmitter( log: log, metrics: m, persistFilter: newPersistSourceFilter(cfg.PersistCloudEventSources), + publishCh: make(chan publishWork, workers), stopCh: make(chan struct{}), }, nil } -// Start launches the retransmit, expiry, and purge background loops. +// Start launches the publish worker pool, retransmit, expiry, and purge background loops. // Cancel the supplied context or call Close to stop them. func (d *DurableEmitter) Start(ctx context.Context) { - n := 3 + workers := d.cfg.PublishWorkers + if workers <= 0 { + workers = DefaultPublishWorkers + } + n := 3 + workers // retransmit + expiry + purge + N publish workers if d.metrics != nil && d.cfg.Metrics != nil { n++ } d.wg.Add(n) + for i := 0; i < workers; i++ { + go d.publishWorker() + } go d.retransmitLoop(ctx) go d.expiryLoop(ctx) go d.purgeLoop(ctx) @@ -240,8 +272,18 @@ func (d *DurableEmitter) Emit(ctx context.Context, body []byte, attrKVs ...any) return fmt.Errorf("failed to persist event: %w", err) } - // Fire-and-forget immediate delivery attempt. - go d.publishAndDelete(id, eventPb) + // Try to dispatch to a publish worker; if all workers are busy the event + // stays in the DB and will be picked up by the retransmit loop. + select { + case d.publishCh <- publishWork{id: id, eventPb: eventPb}: + default: + if d.metrics != nil { + d.metrics.publishPoolFull.Add(ctx, 1) + } + if !d.cfg.QuietMode { + d.log.Debugw("DurableEmitter: publish worker pool full, deferring to retransmit loop", "event_id", id) + } + } return nil } @@ -284,13 +326,25 @@ func (d *DurableEmitter) publishBestEffortNoStore(eventPb *chipingress.CloudEven //d.log.Infow("DurableEmitter: best-effort Chip publish succeeded (not persisted)", okKVs...) } -// Close signals background loops to stop and waits for them to finish. +// Close signals background loops and publish workers to stop and waits for +// them to finish. Workers drain any buffered work before exiting. func (d *DurableEmitter) Close() error { - close(d.stopCh) + close(d.publishCh) // workers drain remaining items then exit + close(d.stopCh) // background loops exit d.wg.Wait() return nil } +// publishWorker is a long-lived goroutine that pulls work items from publishCh. +// Each worker processes one Publish+MarkDelivered at a time, providing bounded +// concurrency over the gRPC and DB connections. +func (d *DurableEmitter) publishWorker() { + defer d.wg.Done() + for w := range d.publishCh { + d.publishAndDelete(w.id, w.eventPb) + } +} + // publishAndDelete attempts a single Publish and deletes the record on success. func (d *DurableEmitter) publishAndDelete(id int64, eventPb *chipingress.CloudEventPb) { ctx, cancel := context.WithTimeout(context.Background(), d.cfg.PublishTimeout) @@ -321,7 +375,9 @@ func (d *DurableEmitter) publishAndDelete(id int64, eventPb *chipingress.CloudEv "elapsed", elapsed.String(), "elapsed_ms", elapsed.Milliseconds(), ) - d.log.Infow("DurableEmitter: Chip Ingress publish failed (immediate), retransmit loop will retry", failKVs...) + if !d.cfg.QuietMode { + d.log.Infow("DurableEmitter: Chip Ingress publish failed (immediate), retransmit loop will retry", failKVs...) + } return } @@ -383,7 +439,7 @@ func (d *DurableEmitter) retransmitPending(ctx context.Context) { st, obsErr := obs.ObserveDurableQueue(ctx, d.cfg.EventTTL, d.queueStatsNearExpiryLead()) if obsErr != nil { d.log.Warnw("DurableEmitter: retransmit scan ObserveDurableQueue failed", "error", obsErr) - } else { + } else if !d.cfg.QuietMode { d.log.Infow("DurableEmitter: retransmit pending scan", "pending_rows", st.Depth, "pending_payload_bytes", st.PayloadBytes, @@ -411,8 +467,10 @@ func (d *DurableEmitter) retransmitPending(ctx context.Context) { continue } if !d.persistFilter.allows(ev.GetSource()) { - d.log.Infow("DurableEmitter: dropping queued event (ce_source not in PersistCloudEventSources)", - "id", pe.ID, "ce_source", ev.GetSource(), "ce_type", ev.GetType()) + if !d.cfg.QuietMode { + d.log.Infow("DurableEmitter: dropping queued event (ce_source not in PersistCloudEventSources)", + "id", pe.ID, "ce_source", ev.GetSource(), "ce_type", ev.GetType()) + } _ = d.store.Delete(ctx, pe.ID) continue } @@ -449,7 +507,9 @@ func (d *DurableEmitter) retransmitPending(ctx context.Context) { "elapsed", elapsed.String(), "elapsed_ms", elapsed.Milliseconds(), ) - d.log.Infow("DurableEmitter: Chip Ingress publish failed (retransmit)", failKVs...) + if !d.cfg.QuietMode { + d.log.Infow("DurableEmitter: Chip Ingress publish failed (retransmit)", failKVs...) + } continue } pubOKKVs := append([]any{}, detailKVs...) @@ -479,7 +539,7 @@ func (d *DurableEmitter) retransmitPending(ctx context.Context) { ) //d.log.Infow("DurableEmitter: durable row deleted after successful Chip publish (retransmit)", delOKKVs...) } - if markedDelivered > 0 { + if markedDelivered > 0 && !d.cfg.QuietMode { d.log.Infow("retransmitted events", "marked_delivered", markedDelivered, "attempted", len(events), @@ -544,7 +604,9 @@ func (d *DurableEmitter) expiryLoop(ctx context.Context) { if d.metrics != nil { d.metrics.expiredPurged.Add(context.Background(), deleted) } - d.log.Infow("purged expired events", "count", deleted) + if !d.cfg.QuietMode { + d.log.Infow("purged expired events", "count", deleted) + } } } } diff --git a/pkg/beholder/durable_emitter_metric_info.go b/pkg/beholder/durable_emitter_metric_info.go index 7cd92e127e..f895802a91 100644 --- a/pkg/beholder/durable_emitter_metric_info.go +++ b/pkg/beholder/durable_emitter_metric_info.go @@ -119,4 +119,9 @@ var ( Unit: "s", Description: "Cumulative system CPU seconds (getrusage; Unix only)", } + durableEmitterMetricPublishPoolFull = MetricInfo{ + Name: "beholder.durable_emitter.publish.pool_full", + Unit: "{event}", + Description: "Events where the publish worker pool was full; deferred to retransmit loop", + } ) diff --git a/pkg/beholder/durable_emitter_metrics.go b/pkg/beholder/durable_emitter_metrics.go index c9c11a96b3..06001efb62 100644 --- a/pkg/beholder/durable_emitter_metrics.go +++ b/pkg/beholder/durable_emitter_metrics.go @@ -50,6 +50,7 @@ type durableEmitterMetrics struct { procHeapSys metric.Int64Gauge procCPUUser metric.Float64Gauge procCPUSys metric.Float64Gauge + publishPoolFull metric.Int64Counter } func newDurableEmitterMetrics() (*durableEmitterMetrics, error) { @@ -125,6 +126,9 @@ func newDurableEmitterMetrics() (*durableEmitterMetrics, error) { if m.procCPUSys, err = durableEmitterMetricProcCPUSys.NewFloat64Gauge(meter); err != nil { return nil, err } + if m.publishPoolFull, err = durableEmitterMetricPublishPoolFull.NewInt64Counter(meter); err != nil { + return nil, err + } return m, nil } From 9d87637bc58fc9325f34ce5465392875f9d3b8ab Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Thu, 9 Apr 2026 14:45:55 -0400 Subject: [PATCH 20/45] Revert --- pkg/beholder/durable_emitter.go | 65 +++------------------ pkg/beholder/durable_emitter_metric_info.go | 5 -- pkg/beholder/durable_emitter_metrics.go | 4 -- 3 files changed, 8 insertions(+), 66 deletions(-) diff --git a/pkg/beholder/durable_emitter.go b/pkg/beholder/durable_emitter.go index 9838f07a15..58784907a2 100644 --- a/pkg/beholder/durable_emitter.go +++ b/pkg/beholder/durable_emitter.go @@ -48,12 +48,6 @@ type DurableEmitterConfig struct { // Publish with no store insert. An empty slice persists nothing (all best-effort only). // A one-element slice containing only "*" is treated like nil (persist all). PersistCloudEventSources []string - // PublishWorkers is the number of long-lived goroutines that process immediate - // Publish+MarkDelivered after each Emit. When all workers are busy, the event - // stays in the DB and the retransmit loop delivers it on the next tick — this - // provides natural back-pressure without unbounded goroutine growth. - // Zero defaults to DefaultPublishWorkers (64). - PublishWorkers int // QuietMode suppresses high-volume INFO-level logs (retransmit scan stats, // retransmit results, publish failures, expired event purges, etc.). // Error-level logs are never suppressed. Useful for load tests where the @@ -74,8 +68,6 @@ type DurableEmitterHooks struct { OnRetransmitBatchDeletes func(elapsed time.Duration, markedDeliveredCount int) } -const DefaultPublishWorkers = 64 - func DefaultDurableEmitterConfig() DurableEmitterConfig { return DurableEmitterConfig{ RetransmitInterval: 5 * time.Second, @@ -86,7 +78,6 @@ func DefaultDurableEmitterConfig() DurableEmitterConfig { PublishTimeout: 5 * time.Second, PurgeInterval: 250 * time.Millisecond, PurgeBatchSize: 500, - PublishWorkers: DefaultPublishWorkers, } } @@ -111,14 +102,8 @@ type DurableEmitter struct { metrics *durableEmitterMetrics persistFilter persistSourceFilter - publishCh chan publishWork - stopCh chan struct{} - wg sync.WaitGroup -} - -type publishWork struct { - id int64 - eventPb *chipingress.CloudEventPb + stopCh chan struct{} + wg sync.WaitGroup } // persistSourceFilter decides whether a CloudEvent source may be written to the durable store. @@ -175,10 +160,6 @@ func NewDurableEmitter( } store = newMetricsInstrumentedStore(store, m) } - workers := cfg.PublishWorkers - if workers <= 0 { - workers = DefaultPublishWorkers - } return &DurableEmitter{ store: store, client: client, @@ -186,26 +167,18 @@ func NewDurableEmitter( log: log, metrics: m, persistFilter: newPersistSourceFilter(cfg.PersistCloudEventSources), - publishCh: make(chan publishWork, workers), stopCh: make(chan struct{}), }, nil } -// Start launches the publish worker pool, retransmit, expiry, and purge background loops. +// Start launches the retransmit, expiry, and purge background loops. // Cancel the supplied context or call Close to stop them. func (d *DurableEmitter) Start(ctx context.Context) { - workers := d.cfg.PublishWorkers - if workers <= 0 { - workers = DefaultPublishWorkers - } - n := 3 + workers // retransmit + expiry + purge + N publish workers + n := 3 if d.metrics != nil && d.cfg.Metrics != nil { n++ } d.wg.Add(n) - for i := 0; i < workers; i++ { - go d.publishWorker() - } go d.retransmitLoop(ctx) go d.expiryLoop(ctx) go d.purgeLoop(ctx) @@ -272,18 +245,8 @@ func (d *DurableEmitter) Emit(ctx context.Context, body []byte, attrKVs ...any) return fmt.Errorf("failed to persist event: %w", err) } - // Try to dispatch to a publish worker; if all workers are busy the event - // stays in the DB and will be picked up by the retransmit loop. - select { - case d.publishCh <- publishWork{id: id, eventPb: eventPb}: - default: - if d.metrics != nil { - d.metrics.publishPoolFull.Add(ctx, 1) - } - if !d.cfg.QuietMode { - d.log.Debugw("DurableEmitter: publish worker pool full, deferring to retransmit loop", "event_id", id) - } - } + // Fire-and-forget immediate delivery attempt. + go d.publishAndDelete(id, eventPb) return nil } @@ -326,25 +289,13 @@ func (d *DurableEmitter) publishBestEffortNoStore(eventPb *chipingress.CloudEven //d.log.Infow("DurableEmitter: best-effort Chip publish succeeded (not persisted)", okKVs...) } -// Close signals background loops and publish workers to stop and waits for -// them to finish. Workers drain any buffered work before exiting. +// Close signals background loops to stop and waits for them to finish. func (d *DurableEmitter) Close() error { - close(d.publishCh) // workers drain remaining items then exit - close(d.stopCh) // background loops exit + close(d.stopCh) d.wg.Wait() return nil } -// publishWorker is a long-lived goroutine that pulls work items from publishCh. -// Each worker processes one Publish+MarkDelivered at a time, providing bounded -// concurrency over the gRPC and DB connections. -func (d *DurableEmitter) publishWorker() { - defer d.wg.Done() - for w := range d.publishCh { - d.publishAndDelete(w.id, w.eventPb) - } -} - // publishAndDelete attempts a single Publish and deletes the record on success. func (d *DurableEmitter) publishAndDelete(id int64, eventPb *chipingress.CloudEventPb) { ctx, cancel := context.WithTimeout(context.Background(), d.cfg.PublishTimeout) diff --git a/pkg/beholder/durable_emitter_metric_info.go b/pkg/beholder/durable_emitter_metric_info.go index f895802a91..7cd92e127e 100644 --- a/pkg/beholder/durable_emitter_metric_info.go +++ b/pkg/beholder/durable_emitter_metric_info.go @@ -119,9 +119,4 @@ var ( Unit: "s", Description: "Cumulative system CPU seconds (getrusage; Unix only)", } - durableEmitterMetricPublishPoolFull = MetricInfo{ - Name: "beholder.durable_emitter.publish.pool_full", - Unit: "{event}", - Description: "Events where the publish worker pool was full; deferred to retransmit loop", - } ) diff --git a/pkg/beholder/durable_emitter_metrics.go b/pkg/beholder/durable_emitter_metrics.go index 06001efb62..c9c11a96b3 100644 --- a/pkg/beholder/durable_emitter_metrics.go +++ b/pkg/beholder/durable_emitter_metrics.go @@ -50,7 +50,6 @@ type durableEmitterMetrics struct { procHeapSys metric.Int64Gauge procCPUUser metric.Float64Gauge procCPUSys metric.Float64Gauge - publishPoolFull metric.Int64Counter } func newDurableEmitterMetrics() (*durableEmitterMetrics, error) { @@ -126,9 +125,6 @@ func newDurableEmitterMetrics() (*durableEmitterMetrics, error) { if m.procCPUSys, err = durableEmitterMetricProcCPUSys.NewFloat64Gauge(meter); err != nil { return nil, err } - if m.publishPoolFull, err = durableEmitterMetricPublishPoolFull.NewInt64Counter(meter); err != nil { - return nil, err - } return m, nil } From e5477bc5e1df0ce0459da8fddf8de9c92aeb8f7f Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Thu, 9 Apr 2026 15:10:00 -0400 Subject: [PATCH 21/45] Add counter --- pkg/beholder/durable_emitter.go | 62 +++++++++++++++++++++++-- pkg/beholder/durable_emitter_metrics.go | 4 +- 2 files changed, 60 insertions(+), 6 deletions(-) diff --git a/pkg/beholder/durable_emitter.go b/pkg/beholder/durable_emitter.go index 58784907a2..fa0ae6f280 100644 --- a/pkg/beholder/durable_emitter.go +++ b/pkg/beholder/durable_emitter.go @@ -6,6 +6,7 @@ import ( "slices" "strings" "sync" + "sync/atomic" "time" cepb "github.com/cloudevents/sdk-go/binding/format/protobuf/v2/pb" @@ -57,6 +58,9 @@ type DurableEmitterConfig struct { // DurableEmitterHooks records Publish vs Delete latency to locate pipeline bottlenecks. type DurableEmitterHooks struct { + // OnEmitInsert is called after each store.Insert in Emit (the DB write that + // blocks the caller). elapsed covers only the INSERT; err is nil on success. + OnEmitInsert func(elapsed time.Duration, err error) // OnImmediatePublish is called after each async Publish in publishAndDelete (every attempt). OnImmediatePublish func(elapsed time.Duration, err error) // OnImmediateDelete is called after MarkDelivered following a successful immediate Publish. @@ -102,6 +106,12 @@ type DurableEmitter struct { metrics *durableEmitterMetrics persistFilter persistSourceFilter + // pendingCount is an exact, atomic count of rows inserted but not yet + // delivered/deleted. Incremented on successful Insert, decremented on + // MarkDelivered, Delete, or DeleteExpired. No polling required. + pendingCount atomic.Int64 + pendingMax atomic.Int64 + stopCh chan struct{} wg sync.WaitGroup } @@ -233,8 +243,12 @@ func (d *DurableEmitter) Emit(ctx context.Context, body []byte, attrKVs ...any) tIns := time.Now() id, err := d.store.Insert(ctx, payload) + insElapsed := time.Since(tIns) + if h := d.cfg.Hooks; h != nil && h.OnEmitInsert != nil { + h.OnEmitInsert(insElapsed, err) + } if d.metrics != nil { - d.metrics.emitDuration.Record(ctx, time.Since(tIns).Seconds()) + d.metrics.emitDuration.Record(ctx, insElapsed.Seconds()) if err != nil { d.metrics.emitFail.Add(ctx, 1) } else { @@ -245,6 +259,8 @@ func (d *DurableEmitter) Emit(ctx context.Context, body []byte, attrKVs ...any) return fmt.Errorf("failed to persist event: %w", err) } + d.incPending(1) + // Fire-and-forget immediate delivery attempt. go d.publishAndDelete(id, eventPb) @@ -296,6 +312,33 @@ func (d *DurableEmitter) Close() error { return nil } +// PendingDepth returns the current exact pending queue depth (inserted but not +// yet delivered/deleted). Thread-safe; no DB query required. +func (d *DurableEmitter) PendingDepth() int64 { return d.pendingCount.Load() } + +// PendingMax returns the highest pending queue depth observed since Start. +func (d *DurableEmitter) PendingMax() int64 { return d.pendingMax.Load() } + +func (d *DurableEmitter) incPending(n int64) { + cur := d.pendingCount.Add(n) + for { + old := d.pendingMax.Load() + if cur <= old || d.pendingMax.CompareAndSwap(old, cur) { + break + } + } + if d.metrics != nil { + d.metrics.queueDepth.Record(context.Background(), cur) + } +} + +func (d *DurableEmitter) decPending(n int64) { + cur := d.pendingCount.Add(-n) + if d.metrics != nil { + d.metrics.queueDepth.Record(context.Background(), cur) + } +} + // publishAndDelete attempts a single Publish and deletes the record on success. func (d *DurableEmitter) publishAndDelete(id int64, eventPb *chipingress.CloudEventPb) { ctx, cancel := context.WithTimeout(context.Background(), d.cfg.PublishTimeout) @@ -344,8 +387,11 @@ func (d *DurableEmitter) publishAndDelete(id int64, eventPb *chipingress.CloudEv if h := d.cfg.Hooks; h != nil && h.OnImmediateDelete != nil { h.OnImmediateDelete(time.Since(t1), markErr) } - if markErr == nil && d.metrics != nil { - d.metrics.deliverComplete.Add(mctx, 1) + if markErr == nil { + d.decPending(1) + if d.metrics != nil { + d.metrics.deliverComplete.Add(mctx, 1) + } } markElapsed := time.Since(t1) if markErr != nil { @@ -414,7 +460,9 @@ func (d *DurableEmitter) retransmitPending(ctx context.Context) { ev := new(chipingress.CloudEventPb) if err := proto.Unmarshal(pe.Payload, ev); err != nil { d.log.Errorw("corrupt pending event, deleting", "id", pe.ID, "error", err) - _ = d.store.Delete(ctx, pe.ID) + if delErr := d.store.Delete(ctx, pe.ID); delErr == nil { + d.decPending(1) + } continue } if !d.persistFilter.allows(ev.GetSource()) { @@ -422,7 +470,9 @@ func (d *DurableEmitter) retransmitPending(ctx context.Context) { d.log.Infow("DurableEmitter: dropping queued event (ce_source not in PersistCloudEventSources)", "id", pe.ID, "ce_source", ev.GetSource(), "ce_type", ev.GetType()) } - _ = d.store.Delete(ctx, pe.ID) + if delErr := d.store.Delete(ctx, pe.ID); delErr == nil { + d.decPending(1) + } continue } events = append(events, ev) @@ -477,6 +527,7 @@ func (d *DurableEmitter) retransmitPending(ctx context.Context) { d.log.Errorw("failed to mark retransmitted event delivered", "id", ids[i], "error", markErr) continue } + d.decPending(1) markedDelivered++ if d.metrics != nil { d.metrics.deliverComplete.Add(ctx, 1) @@ -552,6 +603,7 @@ func (d *DurableEmitter) expiryLoop(ctx context.Context) { continue } if deleted > 0 { + d.decPending(deleted) if d.metrics != nil { d.metrics.expiredPurged.Add(context.Background(), deleted) } diff --git a/pkg/beholder/durable_emitter_metrics.go b/pkg/beholder/durable_emitter_metrics.go index c9c11a96b3..e513e69549 100644 --- a/pkg/beholder/durable_emitter_metrics.go +++ b/pkg/beholder/durable_emitter_metrics.go @@ -140,6 +140,9 @@ func (m *durableEmitterMetrics) recordStoreOp(ctx context.Context, op string, el m.storeOpDuration.Record(ctx, elapsed.Seconds(), metric.WithAttributes(attribute.String("operation", op))) } +// pollQueueGauges refreshes DB-derived queue statistics (payload bytes, oldest +// pending age, near-TTL count). Queue depth itself is tracked atomically by +// DurableEmitter.incPending/decPending and recorded there. func (m *durableEmitterMetrics) pollQueueGauges(ctx context.Context, obs DurableQueueObserver, ttl, lead time.Duration, maxBytes int64) { if m == nil || obs == nil { return @@ -148,7 +151,6 @@ func (m *durableEmitterMetrics) pollQueueGauges(ctx context.Context, obs Durable if err != nil { return } - m.queueDepth.Record(ctx, st.Depth) m.queuePayloadBytes.Record(ctx, st.PayloadBytes) if st.Depth == 0 { m.queueOldestAgeSec.Record(ctx, 0) From ed355a2c92abe06765c49a45e78bcb6f3cc0b7ec Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Thu, 9 Apr 2026 21:06:06 -0400 Subject: [PATCH 22/45] Match Metrics + Production Rate --- pkg/beholder/durable_emitter.go | 215 +++++++++++++++++++- pkg/beholder/durable_emitter_metric_info.go | 5 + pkg/beholder/durable_emitter_metrics.go | 34 +++- pkg/beholder/durable_emitter_store_wrap.go | 7 + pkg/beholder/durable_event_store.go | 16 ++ 5 files changed, 264 insertions(+), 13 deletions(-) diff --git a/pkg/beholder/durable_emitter.go b/pkg/beholder/durable_emitter.go index fa0ae6f280..17bec54168 100644 --- a/pkg/beholder/durable_emitter.go +++ b/pkg/beholder/durable_emitter.go @@ -37,6 +37,24 @@ type DurableEmitterConfig struct { PurgeInterval time.Duration // PurgeBatchSize is the maximum rows removed per PurgeDelivered call. Zero defaults to 500. PurgeBatchSize int + // PublishBatchSize enables batched publishing via PublishBatch RPC when > 0. + // Events are collected into batches of this size before a single PublishBatch + // call is made. Zero disables batching (each Emit spawns its own goroutine + // with an individual Publish RPC — the legacy behaviour). + PublishBatchSize int + // PublishBatchWorkers is the number of concurrent goroutines that read from + // the batch channel and call PublishBatch. More workers means higher + // throughput (each worker handles one in-flight batch at a time). Only used + // when PublishBatchSize > 0. Zero defaults to 1. + PublishBatchWorkers int + // PublishBatchFlushInterval is the maximum time to wait for a full batch + // before flushing a partial one. Only used when PublishBatchSize > 0. + // Zero defaults to 50ms. + PublishBatchFlushInterval time.Duration + // DisablePruning disables the background purge (PurgeDelivered) and expiry + // (DeleteExpired) loops. Events remain in the DB after delivery. Useful for + // post-test analysis of created_at / delivered_at timestamps. + DisablePruning bool // Hooks is optional instrumentation (load tests, profiling). Nil fields are skipped. // Callbacks may run from many goroutines; implementations must be thread-safe. Hooks *DurableEmitterHooks @@ -62,9 +80,16 @@ type DurableEmitterHooks struct { // blocks the caller). elapsed covers only the INSERT; err is nil on success. OnEmitInsert func(elapsed time.Duration, err error) // OnImmediatePublish is called after each async Publish in publishAndDelete (every attempt). + // Only fires when PublishBatchSize == 0 (legacy per-event goroutine path). OnImmediatePublish func(elapsed time.Duration, err error) // OnImmediateDelete is called after MarkDelivered following a successful immediate Publish. + // Only fires when PublishBatchSize == 0. OnImmediateDelete func(elapsed time.Duration, err error) + // OnBatchPublish is called after each PublishBatch RPC in the batch publish loop. + // batchSize is the number of events in the batch; err is nil on success. + OnBatchPublish func(elapsed time.Duration, batchSize int, err error) + // OnBatchMarkDelivered is called after MarkDeliveredBatch following a successful batch publish. + OnBatchMarkDelivered func(elapsed time.Duration, count int) // OnRetransmitBatchPublish is called after each retransmit Publish (one RPC per queued event). OnRetransmitBatchPublish func(elapsed time.Duration, eventCount int, err error) // OnRetransmitBatchDeletes is called after a retransmit tick with total time and count of @@ -97,6 +122,12 @@ func DefaultDurableEmitterConfig() DurableEmitterConfig { // // A separate expiry loop garbage-collects events older than EventTTL to bound // table growth. +// publishWork is a unit of work for the batch publish channel. +type publishWork struct { + id int64 + event *chipingress.CloudEventPb +} + type DurableEmitter struct { store DurableEventStore client chipingress.Client @@ -112,6 +143,10 @@ type DurableEmitter struct { pendingCount atomic.Int64 pendingMax atomic.Int64 + // publishCh buffers events for the batch publish loop. Nil when + // PublishBatchSize == 0 (legacy per-goroutine mode). + publishCh chan publishWork + stopCh chan struct{} wg sync.WaitGroup } @@ -170,7 +205,7 @@ func NewDurableEmitter( } store = newMetricsInstrumentedStore(store, m) } - return &DurableEmitter{ + d := &DurableEmitter{ store: store, client: client, cfg: cfg, @@ -178,20 +213,45 @@ func NewDurableEmitter( metrics: m, persistFilter: newPersistSourceFilter(cfg.PersistCloudEventSources), stopCh: make(chan struct{}), - }, nil + } + if cfg.PublishBatchSize > 0 { + bufSize := cfg.PublishBatchSize * 2000 + if bufSize < 200_000 { + bufSize = 200_000 + } + d.publishCh = make(chan publishWork, bufSize) + } + return d, nil } -// Start launches the retransmit, expiry, and purge background loops. -// Cancel the supplied context or call Close to stop them. +// Start launches the retransmit, expiry, purge, and (optionally) batch publish +// background loops. Cancel the supplied context or call Close to stop them. func (d *DurableEmitter) Start(ctx context.Context) { - n := 3 + n := 1 // retransmit always runs + if !d.cfg.DisablePruning { + n += 2 // expiry + purge + } + batchWorkers := d.cfg.PublishBatchWorkers + if batchWorkers <= 0 { + batchWorkers = 1 + } + if d.publishCh != nil { + n += batchWorkers + } if d.metrics != nil && d.cfg.Metrics != nil { n++ } d.wg.Add(n) go d.retransmitLoop(ctx) - go d.expiryLoop(ctx) - go d.purgeLoop(ctx) + if !d.cfg.DisablePruning { + go d.expiryLoop(ctx) + go d.purgeLoop(ctx) + } + if d.publishCh != nil { + for i := 0; i < batchWorkers; i++ { + go d.batchPublishLoop(ctx) + } + } if d.metrics != nil && d.cfg.Metrics != nil { go d.metricsLoop(ctx) } @@ -261,8 +321,21 @@ func (d *DurableEmitter) Emit(ctx context.Context, body []byte, attrKVs ...any) d.incPending(1) - // Fire-and-forget immediate delivery attempt. - go d.publishAndDelete(id, eventPb) + if d.publishCh != nil { + // Batch mode: enqueue for batch publish loop. + select { + case d.publishCh <- publishWork{id: id, event: eventPb}: + default: + // Channel full — event is safely in the DB; retransmit loop will deliver it. + if !d.cfg.QuietMode { + d.log.Warnw("DurableEmitter: batch publish channel full, relying on retransmit", + "id", id, "ch_len", len(d.publishCh), "ch_cap", cap(d.publishCh)) + } + } + } else { + // Legacy mode: fire-and-forget immediate delivery attempt. + go d.publishAndDelete(id, eventPb) + } return nil } @@ -306,8 +379,13 @@ func (d *DurableEmitter) publishBestEffortNoStore(eventPb *chipingress.CloudEven } // Close signals background loops to stop and waits for them to finish. +// When batch publishing is enabled the channel is closed so the batch loop +// can drain remaining events before returning. func (d *DurableEmitter) Close() error { close(d.stopCh) + if d.publishCh != nil { + close(d.publishCh) + } d.wg.Wait() return nil } @@ -321,14 +399,22 @@ func (d *DurableEmitter) PendingMax() int64 { return d.pendingMax.Load() } func (d *DurableEmitter) incPending(n int64) { cur := d.pendingCount.Add(n) + updated := false for { old := d.pendingMax.Load() - if cur <= old || d.pendingMax.CompareAndSwap(old, cur) { + if cur <= old { + break + } + if d.pendingMax.CompareAndSwap(old, cur) { + updated = true break } } if d.metrics != nil { d.metrics.queueDepth.Record(context.Background(), cur) + if updated { + d.metrics.queueDepthMax.Record(context.Background(), cur) + } } } @@ -407,6 +493,113 @@ func (d *DurableEmitter) publishAndDelete(id int64, eventPb *chipingress.CloudEv //d.log.Infow("DurableEmitter: durable row marked delivered after successful Chip publish (immediate)", delOKKVs...) } +// batchPublishLoop reads events from publishCh, collects them into batches of +// PublishBatchSize, and sends each batch via PublishBatch RPC. A partial batch +// is flushed when PublishBatchFlushInterval elapses. On channel close (during +// Close), remaining items are drained and published. +func (d *DurableEmitter) batchPublishLoop(ctx context.Context) { + defer d.wg.Done() + + batchSize := d.cfg.PublishBatchSize + flushInterval := d.cfg.PublishBatchFlushInterval + if flushInterval <= 0 { + flushInterval = 50 * time.Millisecond + } + + batch := make([]publishWork, 0, batchSize) + flush := time.NewTicker(flushInterval) + defer flush.Stop() + + for { + select { + case w, ok := <-d.publishCh: + if !ok { + // Channel closed — drain remaining items. + if len(batch) > 0 { + d.flushBatch(batch) + } + return + } + batch = append(batch, w) + if len(batch) >= batchSize { + d.flushBatch(batch) + batch = batch[:0] + } + case <-flush.C: + if len(batch) > 0 { + d.flushBatch(batch) + batch = batch[:0] + } + case <-ctx.Done(): + // Drain channel on context cancellation. + for w := range d.publishCh { + batch = append(batch, w) + if len(batch) >= batchSize { + d.flushBatch(batch) + batch = batch[:0] + } + } + if len(batch) > 0 { + d.flushBatch(batch) + } + return + } + } +} + +// flushBatch publishes a collected batch via PublishBatch and marks all events +// as delivered in a single MarkDeliveredBatch call. +func (d *DurableEmitter) flushBatch(batch []publishWork) { + events := make([]*chipingress.CloudEventPb, len(batch)) + ids := make([]int64, len(batch)) + for i, w := range batch { + events[i] = w.event + ids[i] = w.id + } + + batchPb := &chipingress.CloudEventBatch{Events: events} + pubCtx, cancel := context.WithTimeout(context.Background(), d.cfg.PublishTimeout) + defer cancel() + + t0 := time.Now() + _, err := d.client.PublishBatch(pubCtx, batchPb) + elapsed := time.Since(t0) + + if h := d.cfg.Hooks; h != nil && h.OnBatchPublish != nil { + h.OnBatchPublish(elapsed, len(batch), err) + } + d.metrics.recordPublish(context.Background(), elapsed, "batch", err) + + if err != nil { + if d.metrics != nil { + d.metrics.publishBatchEvErr.Add(context.Background(), int64(len(batch))) + } + d.log.Warnw("DurableEmitter: PublishBatch failed, events will be retransmitted", + "batch_size", len(batch), "error", err, + "elapsed_ms", elapsed.Milliseconds()) + return + } + + if d.metrics != nil { + d.metrics.publishBatchEvOK.Add(context.Background(), int64(len(batch))) + } + + tMark := time.Now() + marked, markErr := d.store.MarkDeliveredBatch(context.Background(), ids) + markElapsed := time.Since(tMark) + if h := d.cfg.Hooks; h != nil && h.OnBatchMarkDelivered != nil { + h.OnBatchMarkDelivered(markElapsed, int(marked)) + } + if markErr != nil { + d.log.Errorw("failed to batch-mark events delivered", "batch_size", len(ids), "error", markErr) + return + } + d.decPending(marked) + if d.metrics != nil { + d.metrics.deliverComplete.Add(context.Background(), marked) + } +} + func (d *DurableEmitter) retransmitLoop(ctx context.Context) { defer d.wg.Done() ticker := time.NewTicker(d.cfg.RetransmitInterval) @@ -643,6 +836,8 @@ func (d *DurableEmitter) metricsLoop(ctx context.Context) { return } bctx := context.Background() + d.metrics.queueDepth.Record(bctx, d.pendingCount.Load()) + d.metrics.queueDepthMax.Record(bctx, d.pendingMax.Load()) if obs, ok := d.store.(DurableQueueObserver); ok { d.metrics.pollQueueGauges(bctx, obs, d.cfg.EventTTL, d.queueStatsNearExpiryLead(), mc.MaxQueuePayloadBytes) } diff --git a/pkg/beholder/durable_emitter_metric_info.go b/pkg/beholder/durable_emitter_metric_info.go index 7cd92e127e..a6d87df14c 100644 --- a/pkg/beholder/durable_emitter_metric_info.go +++ b/pkg/beholder/durable_emitter_metric_info.go @@ -79,6 +79,11 @@ var ( Unit: "{row}", Description: "Pending rows in durable queue", } + durableEmitterMetricQueueDepthMax = MetricInfo{ + Name: "beholder.durable_emitter.queue.depth_max", + Unit: "{row}", + Description: "High-water mark of pending queue depth since start", + } durableEmitterMetricQueuePayloadBytes = MetricInfo{ Name: "beholder.durable_emitter.queue.payload_bytes", Unit: "By", diff --git a/pkg/beholder/durable_emitter_metrics.go b/pkg/beholder/durable_emitter_metrics.go index e513e69549..27c23bd764 100644 --- a/pkg/beholder/durable_emitter_metrics.go +++ b/pkg/beholder/durable_emitter_metrics.go @@ -42,6 +42,7 @@ type durableEmitterMetrics struct { storeOps metric.Int64Counter storeOpDuration metric.Float64Histogram queueDepth metric.Int64Gauge + queueDepthMax metric.Int64Gauge queuePayloadBytes metric.Int64Gauge queueOldestAgeSec metric.Float64Gauge queueNearTTL metric.Int64Gauge @@ -52,6 +53,15 @@ type durableEmitterMetrics struct { procCPUSys metric.Float64Gauge } +// durationBuckets provides histogram boundaries (in seconds) tuned for +// sub-millisecond through multi-second latencies. The OTel SDK defaults are +// designed for millisecond-scale integer values and produce wildly wrong +// quantile estimates when values are recorded in fractional seconds. +var durationBuckets = metric.WithExplicitBucketBoundaries( + 0.0001, 0.0005, 0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, + 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, +) + func newDurableEmitterMetrics() (*durableEmitterMetrics, error) { meter := GetMeter() m := &durableEmitterMetrics{} @@ -62,7 +72,12 @@ func newDurableEmitterMetrics() (*durableEmitterMetrics, error) { if m.emitFail, err = durableEmitterMetricEmitFailure.NewInt64Counter(meter); err != nil { return nil, err } - if m.emitDuration, err = durableEmitterMetricEmitDuration.NewFloat64Histogram(meter); err != nil { + if m.emitDuration, err = meter.Float64Histogram( + durableEmitterMetricEmitDuration.Name, + metric.WithUnit(durableEmitterMetricEmitDuration.Unit), + metric.WithDescription(durableEmitterMetricEmitDuration.Description), + durationBuckets, + ); err != nil { return nil, err } if m.publishImmOK, err = durableEmitterMetricPublishImmSuccess.NewInt64Counter(meter); err != nil { @@ -71,7 +86,12 @@ func newDurableEmitterMetrics() (*durableEmitterMetrics, error) { if m.publishImmErr, err = durableEmitterMetricPublishImmFailure.NewInt64Counter(meter); err != nil { return nil, err } - if m.publishDuration, err = durableEmitterMetricPublishDuration.NewFloat64Histogram(meter); err != nil { + if m.publishDuration, err = meter.Float64Histogram( + durableEmitterMetricPublishDuration.Name, + metric.WithUnit(durableEmitterMetricPublishDuration.Unit), + metric.WithDescription(durableEmitterMetricPublishDuration.Description), + durationBuckets, + ); err != nil { return nil, err } if m.publishBatchOK, err = durableEmitterMetricPublishBatchSuccess.NewInt64Counter(meter); err != nil { @@ -95,12 +115,20 @@ func newDurableEmitterMetrics() (*durableEmitterMetrics, error) { if m.storeOps, err = durableEmitterMetricStoreOperations.NewInt64Counter(meter); err != nil { return nil, err } - if m.storeOpDuration, err = durableEmitterMetricStoreOpDuration.NewFloat64Histogram(meter); err != nil { + if m.storeOpDuration, err = meter.Float64Histogram( + durableEmitterMetricStoreOpDuration.Name, + metric.WithUnit(durableEmitterMetricStoreOpDuration.Unit), + metric.WithDescription(durableEmitterMetricStoreOpDuration.Description), + durationBuckets, + ); err != nil { return nil, err } if m.queueDepth, err = durableEmitterMetricQueueDepth.NewInt64Gauge(meter); err != nil { return nil, err } + if m.queueDepthMax, err = durableEmitterMetricQueueDepthMax.NewInt64Gauge(meter); err != nil { + return nil, err + } if m.queuePayloadBytes, err = durableEmitterMetricQueuePayloadBytes.NewInt64Gauge(meter); err != nil { return nil, err } diff --git a/pkg/beholder/durable_emitter_store_wrap.go b/pkg/beholder/durable_emitter_store_wrap.go index 9f68047a76..00c556f56e 100644 --- a/pkg/beholder/durable_emitter_store_wrap.go +++ b/pkg/beholder/durable_emitter_store_wrap.go @@ -43,6 +43,13 @@ func (s *metricsInstrumentedStore) MarkDelivered(ctx context.Context, id int64) return err } +func (s *metricsInstrumentedStore) MarkDeliveredBatch(ctx context.Context, ids []int64) (int64, error) { + t0 := time.Now() + n, err := s.inner.MarkDeliveredBatch(ctx, ids) + s.m.recordStoreOp(ctx, "mark_delivered_batch", time.Since(t0), err) + return n, err +} + func (s *metricsInstrumentedStore) PurgeDelivered(ctx context.Context, batchLimit int) (int64, error) { t0 := time.Now() n, err := s.inner.PurgeDelivered(ctx, batchLimit) diff --git a/pkg/beholder/durable_event_store.go b/pkg/beholder/durable_event_store.go index 86b46ad7b0..54e654a880 100644 --- a/pkg/beholder/durable_event_store.go +++ b/pkg/beholder/durable_event_store.go @@ -46,6 +46,9 @@ type DurableEventStore interface { // a background PurgeDelivered removes rows later. MemDurableEventStore removes // the row immediately (same as Delete). MarkDelivered(ctx context.Context, id int64) error + // MarkDeliveredBatch marks multiple events as delivered in a single operation. + // Semantically equivalent to calling MarkDelivered for each id. + MarkDeliveredBatch(ctx context.Context, ids []int64) (int64, error) // PurgeDelivered deletes up to batchLimit rows already marked delivered. // Implementations that remove rows in MarkDelivered may return 0, nil always. PurgeDelivered(ctx context.Context, batchLimit int) (deleted int64, err error) @@ -97,6 +100,19 @@ func (m *MemDurableEventStore) MarkDelivered(ctx context.Context, id int64) erro return m.Delete(ctx, id) } +func (m *MemDurableEventStore) MarkDeliveredBatch(_ context.Context, ids []int64) (int64, error) { + m.mu.Lock() + defer m.mu.Unlock() + var n int64 + for _, id := range ids { + if _, ok := m.events[id]; ok { + delete(m.events, id) + n++ + } + } + return n, nil +} + func (m *MemDurableEventStore) PurgeDelivered(_ context.Context, _ int) (int64, error) { return 0, nil } From f569c893730934800c8ed9c9598a30b3cdb6f62b Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Fri, 10 Apr 2026 00:11:52 -0400 Subject: [PATCH 23/45] Async mark delivered --- pkg/beholder/durable_emitter.go | 50 ++++++++++++++++++++++----------- 1 file changed, 33 insertions(+), 17 deletions(-) diff --git a/pkg/beholder/durable_emitter.go b/pkg/beholder/durable_emitter.go index 17bec54168..89836c57dc 100644 --- a/pkg/beholder/durable_emitter.go +++ b/pkg/beholder/durable_emitter.go @@ -51,6 +51,9 @@ type DurableEmitterConfig struct { // before flushing a partial one. Only used when PublishBatchSize > 0. // Zero defaults to 50ms. PublishBatchFlushInterval time.Duration + // PublishBatchChannelSize overrides the publishCh buffer capacity. Only + // used when PublishBatchSize > 0. Zero defaults to max(PublishBatchSize*2000, 200_000). + PublishBatchChannelSize int // DisablePruning disables the background purge (PurgeDelivered) and expiry // (DeleteExpired) loops. Events remain in the DB after delivery. Useful for // post-test analysis of created_at / delivered_at timestamps. @@ -149,6 +152,7 @@ type DurableEmitter struct { stopCh chan struct{} wg sync.WaitGroup + markWg sync.WaitGroup // tracks in-flight async MarkDelivered goroutines } // persistSourceFilter decides whether a CloudEvent source may be written to the durable store. @@ -215,9 +219,12 @@ func NewDurableEmitter( stopCh: make(chan struct{}), } if cfg.PublishBatchSize > 0 { - bufSize := cfg.PublishBatchSize * 2000 - if bufSize < 200_000 { - bufSize = 200_000 + bufSize := cfg.PublishBatchChannelSize + if bufSize <= 0 { + bufSize = cfg.PublishBatchSize * 2000 + if bufSize < 200_000 { + bufSize = 200_000 + } } d.publishCh = make(chan publishWork, bufSize) } @@ -387,6 +394,7 @@ func (d *DurableEmitter) Close() error { close(d.publishCh) } d.wg.Wait() + d.markWg.Wait() return nil } @@ -584,20 +592,28 @@ func (d *DurableEmitter) flushBatch(batch []publishWork) { d.metrics.publishBatchEvOK.Add(context.Background(), int64(len(batch))) } - tMark := time.Now() - marked, markErr := d.store.MarkDeliveredBatch(context.Background(), ids) - markElapsed := time.Since(tMark) - if h := d.cfg.Hooks; h != nil && h.OnBatchMarkDelivered != nil { - h.OnBatchMarkDelivered(markElapsed, int(marked)) - } - if markErr != nil { - d.log.Errorw("failed to batch-mark events delivered", "batch_size", len(ids), "error", markErr) - return - } - d.decPending(marked) - if d.metrics != nil { - d.metrics.deliverComplete.Add(context.Background(), marked) - } + // Async MarkDelivered: the DB UPDATE runs in a background goroutine so + // the batch worker can immediately start collecting the next batch. + // If MarkDelivered fails, events stay pending and the retransmit loop + // delivers them (at-least-once semantics are unchanged). + d.markWg.Add(1) + go func() { + defer d.markWg.Done() + tMark := time.Now() + marked, markErr := d.store.MarkDeliveredBatch(context.Background(), ids) + markElapsed := time.Since(tMark) + if h := d.cfg.Hooks; h != nil && h.OnBatchMarkDelivered != nil { + h.OnBatchMarkDelivered(markElapsed, int(marked)) + } + if markErr != nil { + d.log.Errorw("failed to batch-mark events delivered", "batch_size", len(ids), "error", markErr) + return + } + d.decPending(marked) + if d.metrics != nil { + d.metrics.deliverComplete.Add(context.Background(), marked) + } + }() } func (d *DurableEmitter) retransmitLoop(ctx context.Context) { From 27cac718d8769b0696ac5c78d5fbdd8e7adb2d5a Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Mon, 13 Apr 2026 10:18:11 -0400 Subject: [PATCH 24/45] Update --- pkg/beholder/durable_emitter.go | 111 +++++++++++++++++++++++++++----- 1 file changed, 95 insertions(+), 16 deletions(-) diff --git a/pkg/beholder/durable_emitter.go b/pkg/beholder/durable_emitter.go index 89836c57dc..1e9804398d 100644 --- a/pkg/beholder/durable_emitter.go +++ b/pkg/beholder/durable_emitter.go @@ -691,12 +691,104 @@ func (d *DurableEmitter) retransmitPending(ctx context.Context) { return } - // One Publish per row so a single bad or rejected event does not block the rest of the slice. + if d.publishCh != nil { + d.retransmitBatch(ctx, events, ids) + } else { + d.retransmitSerial(ctx, events, ids) + } +} + +// retransmitBatch publishes pending events via PublishBatch in chunks, +// then marks each chunk delivered asynchronously. Used when batch mode +// is enabled (PublishBatchSize > 0). +func (d *DurableEmitter) retransmitBatch(ctx context.Context, events []*chipingress.CloudEventPb, ids []int64) { + batchSize := d.cfg.PublishBatchSize + if batchSize <= 0 { + batchSize = 100 + } + + tAll := time.Now() + var totalPublished int + + for i := 0; i < len(events); i += batchSize { + end := i + batchSize + if end > len(events) { + end = len(events) + } + chunk := events[i:end] + chunkIDs := ids[i:end] + + batchPb := &chipingress.CloudEventBatch{Events: chunk} + pubCtx, cancel := context.WithTimeout(context.Background(), d.cfg.PublishTimeout) + t0 := time.Now() + _, err := d.client.PublishBatch(pubCtx, batchPb) + elapsed := time.Since(t0) + cancel() + + if h := d.cfg.Hooks; h != nil && h.OnBatchPublish != nil { + h.OnBatchPublish(elapsed, len(chunk), err) + } + d.metrics.recordPublish(context.Background(), elapsed, "retransmit-batch", err) + + if err != nil { + if d.metrics != nil { + d.metrics.publishBatchEvErr.Add(ctx, int64(len(chunk))) + } + if !d.cfg.QuietMode { + d.log.Warnw("DurableEmitter: retransmit PublishBatch failed, will retry next tick", + "batch_size", len(chunk), "error", err, "elapsed_ms", elapsed.Milliseconds()) + } + continue + } + + if d.metrics != nil { + d.metrics.publishBatchEvOK.Add(ctx, int64(len(chunk))) + } + totalPublished += len(chunk) + + // Async MarkDelivered (same pattern as flushBatch). + markIDs := make([]int64, len(chunkIDs)) + copy(markIDs, chunkIDs) + d.markWg.Add(1) + go func() { + defer d.markWg.Done() + tMark := time.Now() + marked, markErr := d.store.MarkDeliveredBatch(context.Background(), markIDs) + markElapsed := time.Since(tMark) + if h := d.cfg.Hooks; h != nil && h.OnBatchMarkDelivered != nil { + h.OnBatchMarkDelivered(markElapsed, int(marked)) + } + if markErr != nil { + d.log.Errorw("failed to batch-mark retransmitted events delivered", + "batch_size", len(markIDs), "error", markErr) + return + } + d.decPending(marked) + if d.metrics != nil { + d.metrics.deliverComplete.Add(context.Background(), marked) + } + }() + } + + if totalPublished > 0 && !d.cfg.QuietMode { + d.log.Infow("retransmitted events (batch)", + "published", totalPublished, + "attempted", len(events), + "elapsed_ms", time.Since(tAll).Milliseconds(), + ) + } + if h := d.cfg.Hooks; h != nil && h.OnRetransmitBatchDeletes != nil && totalPublished > 0 { + h.OnRetransmitBatchDeletes(time.Since(tAll), totalPublished) + } +} + +// retransmitSerial publishes pending events one at a time via individual +// Publish RPCs. Used in legacy mode (PublishBatchSize == 0). +func (d *DurableEmitter) retransmitSerial(ctx context.Context, events []*chipingress.CloudEventPb, ids []int64) { tDel := time.Now() var markedDelivered int for i := range events { detailKVs := cloudEventPublishKVs(ids[i], "retransmit", d.cfg.PublishTimeout, events[i]) - //d.log.Infow("DurableEmitter: Chip Ingress publish attempt (retransmit)", detailKVs...) tPub := time.Now() pubCtx, cancel := context.WithTimeout(context.Background(), d.cfg.PublishTimeout) @@ -722,12 +814,6 @@ func (d *DurableEmitter) retransmitPending(ctx context.Context) { } continue } - pubOKKVs := append([]any{}, detailKVs...) - pubOKKVs = append(pubOKKVs, - "publish_rpc_elapsed", elapsed.String(), - "publish_rpc_elapsed_ms", elapsed.Milliseconds(), - ) - //d.log.Infow("DurableEmitter: Chip Ingress publish succeeded (retransmit)", pubOKKVs...) if d.metrics != nil { d.metrics.publishBatchEvOK.Add(ctx, 1) } @@ -741,14 +827,7 @@ func (d *DurableEmitter) retransmitPending(ctx context.Context) { if d.metrics != nil { d.metrics.deliverComplete.Add(ctx, 1) } - markElapsed := time.Since(tMarkOne) - delOKKVs := append([]any{}, detailKVs...) - delOKKVs = append(delOKKVs, - "publish_rpc_elapsed_ms", elapsed.Milliseconds(), - "store_mark_delivered_elapsed", markElapsed.String(), - "store_mark_delivered_elapsed_ms", markElapsed.Milliseconds(), - ) - //d.log.Infow("DurableEmitter: durable row deleted after successful Chip publish (retransmit)", delOKKVs...) + _ = time.Since(tMarkOne) } if markedDelivered > 0 && !d.cfg.QuietMode { d.log.Infow("retransmitted events", From 983fc8790fa87e69802923532385d45823d147b0 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Mon, 13 Apr 2026 15:14:17 -0400 Subject: [PATCH 25/45] Batching --- pkg/beholder/durable_emitter.go | 75 +++++++++++++++++++++------------ 1 file changed, 47 insertions(+), 28 deletions(-) diff --git a/pkg/beholder/durable_emitter.go b/pkg/beholder/durable_emitter.go index 1e9804398d..88a66f6828 100644 --- a/pkg/beholder/durable_emitter.go +++ b/pkg/beholder/durable_emitter.go @@ -502,9 +502,9 @@ func (d *DurableEmitter) publishAndDelete(id int64, eventPb *chipingress.CloudEv } // batchPublishLoop reads events from publishCh, collects them into batches of -// PublishBatchSize, and sends each batch via PublishBatch RPC. A partial batch -// is flushed when PublishBatchFlushInterval elapses. On channel close (during -// Close), remaining items are drained and published. +// PublishBatchSize, and sends each batch via PublishBatch RPC. It blocks until +// the batch is full or PublishBatchFlushInterval elapses after the first event +// arrives (linger pattern), guaranteeing full batches at high throughput. func (d *DurableEmitter) batchPublishLoop(ctx context.Context) { defer d.wg.Done() @@ -515,43 +515,62 @@ func (d *DurableEmitter) batchPublishLoop(ctx context.Context) { } batch := make([]publishWork, 0, batchSize) - flush := time.NewTicker(flushInterval) - defer flush.Stop() for { + // Stage 1: block until at least one event arrives (or shutdown). select { case w, ok := <-d.publishCh: if !ok { - // Channel closed — drain remaining items. - if len(batch) > 0 { - d.flushBatch(batch) - } return } batch = append(batch, w) - if len(batch) >= batchSize { - d.flushBatch(batch) - batch = batch[:0] - } - case <-flush.C: - if len(batch) > 0 { - d.flushBatch(batch) - batch = batch[:0] - } case <-ctx.Done(): - // Drain channel on context cancellation. - for w := range d.publishCh { - batch = append(batch, w) - if len(batch) >= batchSize { - d.flushBatch(batch) - batch = batch[:0] + d.drainPublishCh(batch) + return + } + + // Stage 2: linger — keep reading until batch is full or deadline. + linger := time.NewTimer(flushInterval) + fill: + for len(batch) < batchSize { + select { + case w, ok := <-d.publishCh: + if !ok { + linger.Stop() + if len(batch) > 0 { + d.flushBatch(batch) + } + return } + batch = append(batch, w) + case <-linger.C: + break fill + case <-ctx.Done(): + linger.Stop() + d.drainPublishCh(batch) + return } - if len(batch) > 0 { - d.flushBatch(batch) - } - return } + linger.Stop() + + d.flushBatch(batch) + batch = batch[:0] + } +} + +// drainPublishCh flushes the given partial batch plus any remaining items on +// publishCh in batchSize chunks. Called during shutdown (ctx cancelled or +// channel closed). +func (d *DurableEmitter) drainPublishCh(batch []publishWork) { + for w := range d.publishCh { + batch = append(batch, w) + if len(batch) >= d.cfg.PublishBatchSize { + d.flushBatch(batch) + batch = batch[:0] + } + } + if len(batch) > 0 { + d.flushBatch(batch) } } From 1be34c0b8081158fdae8524f39ce42383b9fdda3 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Wed, 15 Apr 2026 12:00:20 -0400 Subject: [PATCH 26/45] Use publish workers for re-transmit --- pkg/beholder/durable_emitter.go | 104 ++++++-------------------------- 1 file changed, 20 insertions(+), 84 deletions(-) diff --git a/pkg/beholder/durable_emitter.go b/pkg/beholder/durable_emitter.go index 88a66f6828..f2ae939d1b 100644 --- a/pkg/beholder/durable_emitter.go +++ b/pkg/beholder/durable_emitter.go @@ -711,93 +711,29 @@ func (d *DurableEmitter) retransmitPending(ctx context.Context) { } if d.publishCh != nil { - d.retransmitBatch(ctx, events, ids) - } else { - d.retransmitSerial(ctx, events, ids) - } -} - -// retransmitBatch publishes pending events via PublishBatch in chunks, -// then marks each chunk delivered asynchronously. Used when batch mode -// is enabled (PublishBatchSize > 0). -func (d *DurableEmitter) retransmitBatch(ctx context.Context, events []*chipingress.CloudEventPb, ids []int64) { - batchSize := d.cfg.PublishBatchSize - if batchSize <= 0 { - batchSize = 100 - } - - tAll := time.Now() - var totalPublished int - - for i := 0; i < len(events); i += batchSize { - end := i + batchSize - if end > len(events) { - end = len(events) - } - chunk := events[i:end] - chunkIDs := ids[i:end] - - batchPb := &chipingress.CloudEventBatch{Events: chunk} - pubCtx, cancel := context.WithTimeout(context.Background(), d.cfg.PublishTimeout) - t0 := time.Now() - _, err := d.client.PublishBatch(pubCtx, batchPb) - elapsed := time.Since(t0) - cancel() - - if h := d.cfg.Hooks; h != nil && h.OnBatchPublish != nil { - h.OnBatchPublish(elapsed, len(chunk), err) - } - d.metrics.recordPublish(context.Background(), elapsed, "retransmit-batch", err) - - if err != nil { - if d.metrics != nil { - d.metrics.publishBatchEvErr.Add(ctx, int64(len(chunk))) - } - if !d.cfg.QuietMode { - d.log.Warnw("DurableEmitter: retransmit PublishBatch failed, will retry next tick", - "batch_size", len(chunk), "error", err, "elapsed_ms", elapsed.Milliseconds()) + // Enqueue to the same batch workers used by Emit(). This gives + // retransmit automatic parallelism (PublishBatchWorkers) and reuses + // all batching/flush/mark-delivered logic with zero duplication. + var enqueued int + for i := range events { + select { + case d.publishCh <- publishWork{id: ids[i], event: events[i]}: + enqueued++ + default: + // Channel full — event stays pending in DB, will retry next tick. } - continue } - - if d.metrics != nil { - d.metrics.publishBatchEvOK.Add(ctx, int64(len(chunk))) + if !d.cfg.QuietMode { + d.log.Infow("DurableEmitter: retransmit enqueued to batch workers", + "enqueued", enqueued, + "skipped_ch_full", len(events)-enqueued, + "total_pending", len(events), + "ch_len", len(d.publishCh), + "ch_cap", cap(d.publishCh), + ) } - totalPublished += len(chunk) - - // Async MarkDelivered (same pattern as flushBatch). - markIDs := make([]int64, len(chunkIDs)) - copy(markIDs, chunkIDs) - d.markWg.Add(1) - go func() { - defer d.markWg.Done() - tMark := time.Now() - marked, markErr := d.store.MarkDeliveredBatch(context.Background(), markIDs) - markElapsed := time.Since(tMark) - if h := d.cfg.Hooks; h != nil && h.OnBatchMarkDelivered != nil { - h.OnBatchMarkDelivered(markElapsed, int(marked)) - } - if markErr != nil { - d.log.Errorw("failed to batch-mark retransmitted events delivered", - "batch_size", len(markIDs), "error", markErr) - return - } - d.decPending(marked) - if d.metrics != nil { - d.metrics.deliverComplete.Add(context.Background(), marked) - } - }() - } - - if totalPublished > 0 && !d.cfg.QuietMode { - d.log.Infow("retransmitted events (batch)", - "published", totalPublished, - "attempted", len(events), - "elapsed_ms", time.Since(tAll).Milliseconds(), - ) - } - if h := d.cfg.Hooks; h != nil && h.OnRetransmitBatchDeletes != nil && totalPublished > 0 { - h.OnRetransmitBatchDeletes(time.Since(tAll), totalPublished) + } else { + d.retransmitSerial(ctx, events, ids) } } From c3a4edc1651874fb54748569bad50664868d5458 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Wed, 15 Apr 2026 13:20:15 -0400 Subject: [PATCH 27/45] Payload optimization --- pkg/beholder/durable_emitter.go | 218 +++++++++++++++++++++++++++----- pkg/chipingress/client.go | 6 + 2 files changed, 191 insertions(+), 33 deletions(-) diff --git a/pkg/beholder/durable_emitter.go b/pkg/beholder/durable_emitter.go index f2ae939d1b..212fb9519e 100644 --- a/pkg/beholder/durable_emitter.go +++ b/pkg/beholder/durable_emitter.go @@ -10,9 +10,13 @@ import ( "time" cepb "github.com/cloudevents/sdk-go/binding/format/protobuf/v2/pb" + "google.golang.org/grpc" + grpcEncoding "google.golang.org/grpc/encoding" + "google.golang.org/protobuf/encoding/protowire" "google.golang.org/protobuf/proto" "github.com/smartcontractkit/chainlink-common/pkg/chipingress" + "github.com/smartcontractkit/chainlink-common/pkg/chipingress/pb" "github.com/smartcontractkit/chainlink-common/pkg/logger" ) @@ -127,8 +131,9 @@ func DefaultDurableEmitterConfig() DurableEmitterConfig { // table growth. // publishWork is a unit of work for the batch publish channel. type publishWork struct { - id int64 - event *chipingress.CloudEventPb + id int64 + payload []byte // serialized CloudEvent proto (always set) + event *chipingress.CloudEventPb // original struct; set from Emit(), nil from retransmit } type DurableEmitter struct { @@ -140,6 +145,10 @@ type DurableEmitter struct { metrics *durableEmitterMetrics persistFilter persistSourceFilter + // rawConn is the underlying gRPC connection when the client exposes it. + // Non-nil enables zero-copy batch publishing (protowire + ForceCodec). + rawConn *grpc.ClientConn + // pendingCount is an exact, atomic count of rows inserted but not yet // delivered/deleted. Incremented on successful Insert, decremented on // MarkDelivered, Delete, or DeleteExpired. No polling required. @@ -155,6 +164,55 @@ type DurableEmitter struct { markWg sync.WaitGroup // tracks in-flight async MarkDelivered goroutines } +// grpcConnProvider is an optional interface for clients that expose the +// underlying gRPC connection. When satisfied, DurableEmitter uses zero-copy +// batch publishing to avoid protobuf marshal/unmarshal overhead. +type grpcConnProvider interface { + Conn() *grpc.ClientConn +} + +// rawBytesCodec is a gRPC codec that passes []byte through without +// marshal/unmarshal. Name returns "proto" so the wire content-type is +// application/grpc+proto, matching what Chip Ingress expects. +type rawBytesCodec struct{} + +func (rawBytesCodec) Marshal(v any) ([]byte, error) { + b, ok := v.([]byte) + if !ok { + return nil, fmt.Errorf("rawBytesCodec.Marshal: expected []byte, got %T", v) + } + return b, nil +} + +func (rawBytesCodec) Unmarshal(data []byte, v any) error { + bp, ok := v.(*[]byte) + if !ok { + return fmt.Errorf("rawBytesCodec.Unmarshal: expected *[]byte, got %T", v) + } + *bp = data + return nil +} + +func (rawBytesCodec) Name() string { return "proto" } + +var _ grpcEncoding.Codec = rawBytesCodec{} + +// buildBatchBytes constructs the protobuf wire format for a CloudEventBatch +// directly from already-serialized CloudEvent payloads. Each payload is +// wrapped with the field-1 tag and length prefix — no unmarshal/re-marshal. +func buildBatchBytes(payloads [][]byte) []byte { + size := 0 + for _, p := range payloads { + size += protowire.SizeTag(1) + protowire.SizeBytes(len(p)) + } + buf := make([]byte, 0, size) + for _, p := range payloads { + buf = protowire.AppendTag(buf, 1, protowire.BytesType) + buf = protowire.AppendBytes(buf, p) + } + return buf +} + // persistSourceFilter decides whether a CloudEvent source may be written to the durable store. type persistSourceFilter struct { allowAll bool @@ -218,6 +276,10 @@ func NewDurableEmitter( persistFilter: newPersistSourceFilter(cfg.PersistCloudEventSources), stopCh: make(chan struct{}), } + if cp, ok := client.(grpcConnProvider); ok { + d.rawConn = cp.Conn() + log.Infow("DurableEmitter: raw-codec batch publishing enabled (zero-copy protowire)") + } if cfg.PublishBatchSize > 0 { bufSize := cfg.PublishBatchChannelSize if bufSize <= 0 { @@ -330,8 +392,14 @@ func (d *DurableEmitter) Emit(ctx context.Context, body []byte, attrKVs ...any) if d.publishCh != nil { // Batch mode: enqueue for batch publish loop. + // Only carry the struct when needed for the typed fallback path; + // the raw path uses payload bytes directly. + work := publishWork{id: id, payload: payload} + if d.rawConn == nil { + work.event = eventPb + } select { - case d.publishCh <- publishWork{id: id, event: eventPb}: + case d.publishCh <- work: default: // Channel full — event is safely in the DB; retransmit loop will deliver it. if !d.cfg.QuietMode { @@ -576,20 +644,26 @@ func (d *DurableEmitter) drainPublishCh(batch []publishWork) { // flushBatch publishes a collected batch via PublishBatch and marks all events // as delivered in a single MarkDeliveredBatch call. +// +// When rawConn is available, batch wire bytes are built directly from the +// already-serialized payloads using protowire — zero unmarshal/re-marshal. +// Otherwise, falls back to the typed PublishBatch RPC. func (d *DurableEmitter) flushBatch(batch []publishWork) { - events := make([]*chipingress.CloudEventPb, len(batch)) ids := make([]int64, len(batch)) for i, w := range batch { - events[i] = w.event ids[i] = w.id } - batchPb := &chipingress.CloudEventBatch{Events: events} pubCtx, cancel := context.WithTimeout(context.Background(), d.cfg.PublishTimeout) defer cancel() t0 := time.Now() - _, err := d.client.PublishBatch(pubCtx, batchPb) + var err error + if d.rawConn != nil { + err = d.flushBatchRaw(pubCtx, batch) + } else { + err = d.flushBatchTyped(pubCtx, batch) + } elapsed := time.Since(t0) if h := d.cfg.Hooks; h != nil && h.OnBatchPublish != nil { @@ -635,6 +709,45 @@ func (d *DurableEmitter) flushBatch(batch []publishWork) { }() } +// flushBatchRaw builds the CloudEventBatch wire format directly from +// already-serialized payloads and sends it via raw gRPC Invoke — zero +// protobuf unmarshal/re-marshal overhead. +func (d *DurableEmitter) flushBatchRaw(ctx context.Context, batch []publishWork) error { + payloads := make([][]byte, len(batch)) + for i, w := range batch { + payloads[i] = w.payload + } + batchBytes := buildBatchBytes(payloads) + var respBytes []byte + return d.rawConn.Invoke( + ctx, + pb.ChipIngress_PublishBatch_FullMethodName, + batchBytes, + &respBytes, + grpc.ForceCodec(rawBytesCodec{}), + ) +} + +// flushBatchTyped builds a typed CloudEventBatch and sends it via the +// standard gRPC client. Used as fallback when rawConn is not available. +func (d *DurableEmitter) flushBatchTyped(ctx context.Context, batch []publishWork) error { + events := make([]*chipingress.CloudEventPb, len(batch)) + for i, w := range batch { + if w.event != nil { + events[i] = w.event + } else { + ev := new(chipingress.CloudEventPb) + if err := proto.Unmarshal(w.payload, ev); err != nil { + return fmt.Errorf("unmarshal event %d for typed publish: %w", i, err) + } + events[i] = ev + } + } + batchPb := &chipingress.CloudEventBatch{Events: events} + _, err := d.client.PublishBatch(ctx, batchPb) + return err +} + func (d *DurableEmitter) retransmitLoop(ctx context.Context) { defer d.wg.Done() ticker := time.NewTicker(d.cfg.RetransmitInterval) @@ -681,6 +794,70 @@ func (d *DurableEmitter) retransmitPending(ctx context.Context) { return } + if d.publishCh != nil { + d.retransmitViaBatchWorkers(ctx, pending) + } else { + d.retransmitSerialFromPending(ctx, pending) + } +} + +// retransmitViaBatchWorkers enqueues pending DB rows to publishCh so the +// existing batch workers handle publishing. When rawConn is set and the +// persist filter accepts all sources, payloads are passed through without +// any proto.Unmarshal — the batch workers will use buildBatchBytes to +// construct the wire format directly. +func (d *DurableEmitter) retransmitViaBatchWorkers(ctx context.Context, pending []DurableEvent) { + var enqueued int + needsFilter := !d.persistFilter.allowAll + + for _, pe := range pending { + var work publishWork + work.id = pe.ID + work.payload = pe.Payload + + if needsFilter { + ev := new(chipingress.CloudEventPb) + if err := proto.Unmarshal(pe.Payload, ev); err != nil { + d.log.Errorw("corrupt pending event, deleting", "id", pe.ID, "error", err) + if delErr := d.store.Delete(ctx, pe.ID); delErr == nil { + d.decPending(1) + } + continue + } + if !d.persistFilter.allows(ev.GetSource()) { + if !d.cfg.QuietMode { + d.log.Infow("DurableEmitter: dropping queued event (ce_source not in PersistCloudEventSources)", + "id", pe.ID, "ce_source", ev.GetSource(), "ce_type", ev.GetType()) + } + if delErr := d.store.Delete(ctx, pe.ID); delErr == nil { + d.decPending(1) + } + continue + } + work.event = ev + } + + select { + case d.publishCh <- work: + enqueued++ + default: + } + } + + if !d.cfg.QuietMode { + d.log.Infow("DurableEmitter: retransmit enqueued to batch workers", + "enqueued", enqueued, + "skipped_ch_full", len(pending)-enqueued, + "total_pending", len(pending), + "ch_len", len(d.publishCh), + "ch_cap", cap(d.publishCh), + ) + } +} + +// retransmitSerialFromPending unmarshals events and publishes them one at a +// time. Used in legacy mode (PublishBatchSize == 0). +func (d *DurableEmitter) retransmitSerialFromPending(ctx context.Context, pending []DurableEvent) { events := make([]*chipingress.CloudEventPb, 0, len(pending)) ids := make([]int64, 0, len(pending)) @@ -706,33 +883,8 @@ func (d *DurableEmitter) retransmitPending(ctx context.Context) { events = append(events, ev) ids = append(ids, pe.ID) } - if len(events) == 0 { - return - } - if d.publishCh != nil { - // Enqueue to the same batch workers used by Emit(). This gives - // retransmit automatic parallelism (PublishBatchWorkers) and reuses - // all batching/flush/mark-delivered logic with zero duplication. - var enqueued int - for i := range events { - select { - case d.publishCh <- publishWork{id: ids[i], event: events[i]}: - enqueued++ - default: - // Channel full — event stays pending in DB, will retry next tick. - } - } - if !d.cfg.QuietMode { - d.log.Infow("DurableEmitter: retransmit enqueued to batch workers", - "enqueued", enqueued, - "skipped_ch_full", len(events)-enqueued, - "total_pending", len(events), - "ch_len", len(d.publishCh), - "ch_cap", cap(d.publishCh), - ) - } - } else { + if len(events) > 0 { d.retransmitSerial(ctx, events, ids) } } diff --git a/pkg/chipingress/client.go b/pkg/chipingress/client.go index e3d833284b..586fb158a8 100644 --- a/pkg/chipingress/client.go +++ b/pkg/chipingress/client.go @@ -154,6 +154,12 @@ func (c *client) Close() error { return c.conn.Close() } +// Conn returns the underlying gRPC connection for advanced use cases such as +// raw-codec PublishBatch calls that bypass protobuf marshal/unmarshal overhead. +func (c *client) Conn() *grpc.ClientConn { + return c.conn +} + // RegisterSchemas registers one or more schemas with the Chip Ingress service. func (c *client) RegisterSchemas(ctx context.Context, schemas ...*pb.Schema) (map[string]int, error) { request := &pb.RegisterSchemaRequest{Schemas: schemas} From e27e4b6ada74ee91d539b0a2ce977cb268960bf7 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Mon, 20 Apr 2026 11:46:44 -0400 Subject: [PATCH 28/45] Batch insert --- pkg/beholder/durable_emitter.go | 190 ++++++++++++++++++-- pkg/beholder/durable_emitter_metric_info.go | 5 + pkg/beholder/durable_emitter_metrics.go | 9 + pkg/beholder/durable_emitter_store_wrap.go | 11 ++ pkg/beholder/durable_event_store.go | 30 +++- 5 files changed, 229 insertions(+), 16 deletions(-) diff --git a/pkg/beholder/durable_emitter.go b/pkg/beholder/durable_emitter.go index 212fb9519e..85ae337db5 100644 --- a/pkg/beholder/durable_emitter.go +++ b/pkg/beholder/durable_emitter.go @@ -74,6 +74,17 @@ type DurableEmitterConfig struct { // Publish with no store insert. An empty slice persists nothing (all best-effort only). // A one-element slice containing only "*" is treated like nil (persist all). PersistCloudEventSources []string + // InsertBatchSize enables write coalescing when > 0 and the store implements + // BatchInserter. Multiple concurrent Emit() calls are grouped into a single + // multi-row INSERT, dramatically reducing per-event transaction overhead. + // Each coalescer worker collects up to InsertBatchSize payloads before flushing. + InsertBatchSize int + // InsertBatchFlushInterval is the linger time after the first payload arrives + // in a coalescing batch. Zero defaults to 2ms. + InsertBatchFlushInterval time.Duration + // InsertBatchWorkers is the number of concurrent batch-insert goroutines. + // Zero defaults to 4. + InsertBatchWorkers int // QuietMode suppresses high-volume INFO-level logs (retransmit scan stats, // retransmit results, publish failures, expired event purges, etc.). // Error-level logs are never suppressed. Useful for load tests where the @@ -136,6 +147,17 @@ type publishWork struct { event *chipingress.CloudEventPb // original struct; set from Emit(), nil from retransmit } +// insertRequest is a single Emit() caller waiting for a coalesced batch INSERT. +type insertRequest struct { + payload []byte + result chan insertResult +} + +type insertResult struct { + id int64 + err error +} + type DurableEmitter struct { store DurableEventStore client chipingress.Client @@ -149,6 +171,13 @@ type DurableEmitter struct { // Non-nil enables zero-copy batch publishing (protowire + ForceCodec). rawConn *grpc.ClientConn + // batchInserter is non-nil when the store supports multi-row INSERTs + // and InsertBatchSize > 0. + batchInserter BatchInserter + // insertCh buffers payloads for the write coalescer. Nil when batch + // inserting is disabled. + insertCh chan *insertRequest + // pendingCount is an exact, atomic count of rows inserted but not yet // delivered/deleted. Incremented on successful Insert, decremented on // MarkDelivered, Delete, or DeleteExpired. No polling required. @@ -280,6 +309,20 @@ func NewDurableEmitter( d.rawConn = cp.Conn() log.Infow("DurableEmitter: raw-codec batch publishing enabled (zero-copy protowire)") } + if cfg.InsertBatchSize > 0 { + if bi, ok := store.(BatchInserter); ok { + d.batchInserter = bi + chanSize := cfg.InsertBatchSize * 200 + if chanSize < 10_000 { + chanSize = 10_000 + } + d.insertCh = make(chan *insertRequest, chanSize) + log.Infow("DurableEmitter: write coalescing enabled", + "insertBatchSize", cfg.InsertBatchSize, + "insertBatchWorkers", cfg.InsertBatchWorkers, + "insertBatchFlushInterval", cfg.InsertBatchFlushInterval) + } + } if cfg.PublishBatchSize > 0 { bufSize := cfg.PublishBatchChannelSize if bufSize <= 0 { @@ -307,6 +350,13 @@ func (d *DurableEmitter) Start(ctx context.Context) { if d.publishCh != nil { n += batchWorkers } + insertWorkers := d.cfg.InsertBatchWorkers + if insertWorkers <= 0 { + insertWorkers = 4 + } + if d.insertCh != nil { + n += insertWorkers + } if d.metrics != nil && d.cfg.Metrics != nil { n++ } @@ -316,6 +366,11 @@ func (d *DurableEmitter) Start(ctx context.Context) { go d.expiryLoop(ctx) go d.purgeLoop(ctx) } + if d.insertCh != nil { + for i := 0; i < insertWorkers; i++ { + go d.insertBatchLoop(ctx) + } + } if d.publishCh != nil { for i := 0; i < batchWorkers; i++ { go d.batchPublishLoop(ctx) @@ -330,6 +385,12 @@ func (d *DurableEmitter) Start(ctx context.Context) { // by PersistCloudEventSources; otherwise it performs a single best-effort Publish with no // persistence. Returns nil once processing is accepted (insert succeeded, or non-persist path started). func (d *DurableEmitter) Emit(ctx context.Context, body []byte, attrKVs ...any) error { + tEmitTotal := time.Now() + defer func() { + if d.metrics != nil { + d.metrics.emitTotalDuration.Record(ctx, time.Since(tEmitTotal).Seconds()) + } + }() emitFail := func() { if d.metrics != nil { d.metrics.emitFail.Add(ctx, 1) @@ -370,23 +431,59 @@ func (d *DurableEmitter) Emit(ctx context.Context, body []byte, attrKVs ...any) return fmt.Errorf("failed to marshal event proto: %w", err) } - tIns := time.Now() - id, err := d.store.Insert(ctx, payload) - insElapsed := time.Since(tIns) - if h := d.cfg.Hooks; h != nil && h.OnEmitInsert != nil { - h.OnEmitInsert(insElapsed, err) - } - if d.metrics != nil { - d.metrics.emitDuration.Record(ctx, insElapsed.Seconds()) + var id int64 + var insElapsed time.Duration + + if d.insertCh != nil { + // Write coalescing: send payload to the batch insert loop and block + // until the multi-row INSERT completes. + req := &insertRequest{ + payload: payload, + result: make(chan insertResult, 1), + } + tIns := time.Now() + select { + case d.insertCh <- req: + case <-ctx.Done(): + emitFail() + return ctx.Err() + } + res := <-req.result + insElapsed = time.Since(tIns) + if h := d.cfg.Hooks; h != nil && h.OnEmitInsert != nil { + h.OnEmitInsert(insElapsed, res.err) + } + if d.metrics != nil { + d.metrics.emitDuration.Record(ctx, insElapsed.Seconds()) + if res.err != nil { + d.metrics.emitFail.Add(ctx, 1) + } else { + d.metrics.emitSuccess.Add(ctx, 1) + } + } + if res.err != nil { + return fmt.Errorf("failed to persist event: %w", res.err) + } + id = res.id + } else { + tIns := time.Now() + id, err = d.store.Insert(ctx, payload) + insElapsed = time.Since(tIns) + if h := d.cfg.Hooks; h != nil && h.OnEmitInsert != nil { + h.OnEmitInsert(insElapsed, err) + } + if d.metrics != nil { + d.metrics.emitDuration.Record(ctx, insElapsed.Seconds()) + if err != nil { + d.metrics.emitFail.Add(ctx, 1) + } else { + d.metrics.emitSuccess.Add(ctx, 1) + } + } if err != nil { - d.metrics.emitFail.Add(ctx, 1) - } else { - d.metrics.emitSuccess.Add(ctx, 1) + return fmt.Errorf("failed to persist event: %w", err) } } - if err != nil { - return fmt.Errorf("failed to persist event: %w", err) - } d.incPending(1) @@ -458,6 +555,9 @@ func (d *DurableEmitter) publishBestEffortNoStore(eventPb *chipingress.CloudEven // can drain remaining events before returning. func (d *DurableEmitter) Close() error { close(d.stopCh) + if d.insertCh != nil { + close(d.insertCh) + } if d.publishCh != nil { close(d.publishCh) } @@ -466,6 +566,68 @@ func (d *DurableEmitter) Close() error { return nil } +// insertBatchLoop collects insertRequest items from insertCh and flushes them +// as multi-row INSERTs via BatchInserter.InsertBatch. Uses a linger pattern: +// blocks for the first request, then collects more until the batch is full or +// the flush interval elapses. +func (d *DurableEmitter) insertBatchLoop(ctx context.Context) { + defer d.wg.Done() + batchSize := d.cfg.InsertBatchSize + linger := d.cfg.InsertBatchFlushInterval + if linger <= 0 { + linger = 2 * time.Millisecond + } + batch := make([]*insertRequest, 0, batchSize) + + for { + batch = batch[:0] + + // Block until first request or shutdown. + select { + case req, ok := <-d.insertCh: + if !ok { + return + } + batch = append(batch, req) + case <-ctx.Done(): + return + case <-d.stopCh: + return + } + + // Linger to collect more. + timer := time.NewTimer(linger) + collecting: + for len(batch) < batchSize { + select { + case req, ok := <-d.insertCh: + if !ok { + timer.Stop() + break collecting + } + batch = append(batch, req) + case <-timer.C: + break collecting + } + } + timer.Stop() + + // Flush: multi-row INSERT. + payloads := make([][]byte, len(batch)) + for i, r := range batch { + payloads[i] = r.payload + } + ids, batchErr := d.batchInserter.InsertBatch(ctx, payloads) + for i, r := range batch { + if batchErr != nil { + r.result <- insertResult{err: batchErr} + } else { + r.result <- insertResult{id: ids[i]} + } + } + } +} + // PendingDepth returns the current exact pending queue depth (inserted but not // yet delivered/deleted). Thread-safe; no DB query required. func (d *DurableEmitter) PendingDepth() int64 { return d.pendingCount.Load() } diff --git a/pkg/beholder/durable_emitter_metric_info.go b/pkg/beholder/durable_emitter_metric_info.go index a6d87df14c..30cda04503 100644 --- a/pkg/beholder/durable_emitter_metric_info.go +++ b/pkg/beholder/durable_emitter_metric_info.go @@ -19,6 +19,11 @@ var ( Unit: "s", Description: "Emit insert path duration (seconds, fractional; aligns with Prometheus _duration_seconds)", } + durableEmitterMetricEmitTotalDuration = MetricInfo{ + Name: "beholder.durable_emitter.emit.total_duration", + Unit: "s", + Description: "Full Emit() wall time including event construction, DB insert, and channel enqueue (seconds)", + } durableEmitterMetricPublishImmSuccess = MetricInfo{ Name: "beholder.durable_emitter.publish.immediate.success", Unit: "{call}", diff --git a/pkg/beholder/durable_emitter_metrics.go b/pkg/beholder/durable_emitter_metrics.go index 27c23bd764..92cbfe689c 100644 --- a/pkg/beholder/durable_emitter_metrics.go +++ b/pkg/beholder/durable_emitter_metrics.go @@ -30,6 +30,7 @@ type durableEmitterMetrics struct { emitSuccess metric.Int64Counter emitFail metric.Int64Counter emitDuration metric.Float64Histogram + emitTotalDuration metric.Float64Histogram publishImmOK metric.Int64Counter publishImmErr metric.Int64Counter publishDuration metric.Float64Histogram @@ -80,6 +81,14 @@ func newDurableEmitterMetrics() (*durableEmitterMetrics, error) { ); err != nil { return nil, err } + if m.emitTotalDuration, err = meter.Float64Histogram( + durableEmitterMetricEmitTotalDuration.Name, + metric.WithUnit(durableEmitterMetricEmitTotalDuration.Unit), + metric.WithDescription(durableEmitterMetricEmitTotalDuration.Description), + durationBuckets, + ); err != nil { + return nil, err + } if m.publishImmOK, err = durableEmitterMetricPublishImmSuccess.NewInt64Counter(meter); err != nil { return nil, err } diff --git a/pkg/beholder/durable_emitter_store_wrap.go b/pkg/beholder/durable_emitter_store_wrap.go index 00c556f56e..75fdb2e0a9 100644 --- a/pkg/beholder/durable_emitter_store_wrap.go +++ b/pkg/beholder/durable_emitter_store_wrap.go @@ -78,3 +78,14 @@ func (s *metricsInstrumentedStore) ObserveDurableQueue(ctx context.Context, even } return o.ObserveDurableQueue(ctx, eventTTL, nearExpiryLead) } + +func (s *metricsInstrumentedStore) InsertBatch(ctx context.Context, payloads [][]byte) ([]int64, error) { + bi, ok := s.inner.(BatchInserter) + if !ok { + return nil, errors.New("inner DurableEventStore does not implement BatchInserter") + } + t0 := time.Now() + ids, err := bi.InsertBatch(ctx, payloads) + s.m.recordStoreOp(ctx, "insert_batch", time.Since(t0), err) + return ids, err +} diff --git a/pkg/beholder/durable_event_store.go b/pkg/beholder/durable_event_store.go index 54e654a880..8a9170879b 100644 --- a/pkg/beholder/durable_event_store.go +++ b/pkg/beholder/durable_event_store.go @@ -34,6 +34,14 @@ type DurableQueueObserver interface { ObserveDurableQueue(ctx context.Context, eventTTL, nearExpiryLead time.Duration) (DurableQueueStats, error) } +// BatchInserter is optionally implemented by DurableEventStore implementations +// to support multi-row inserts for higher throughput. When the store implements +// this interface and InsertBatchSize > 0, DurableEmitter coalesces Emit() calls +// into batched INSERTs, dramatically reducing per-event transaction overhead. +type BatchInserter interface { + InsertBatch(ctx context.Context, payloads [][]byte) ([]int64, error) +} + // DurableEventStore abstracts the persistence layer for durable chip events. // Implementations must be safe for concurrent use. type DurableEventStore interface { @@ -67,8 +75,9 @@ type MemDurableEventStore struct { } var ( - _ DurableEventStore = (*MemDurableEventStore)(nil) - _ DurableQueueObserver = (*MemDurableEventStore)(nil) + _ DurableEventStore = (*MemDurableEventStore)(nil) + _ DurableQueueObserver = (*MemDurableEventStore)(nil) + _ BatchInserter = (*MemDurableEventStore)(nil) ) func NewMemDurableEventStore() *MemDurableEventStore { @@ -89,6 +98,23 @@ func (m *MemDurableEventStore) Insert(_ context.Context, payload []byte) (int64, return id, nil } +func (m *MemDurableEventStore) InsertBatch(_ context.Context, payloads [][]byte) ([]int64, error) { + now := time.Now() + ids := make([]int64, len(payloads)) + m.mu.Lock() + defer m.mu.Unlock() + for i, p := range payloads { + id := m.nextID.Add(1) + m.events[id] = &DurableEvent{ + ID: id, + Payload: append([]byte(nil), p...), + CreatedAt: now, + } + ids[i] = id + } + return ids, nil +} + func (m *MemDurableEventStore) Delete(_ context.Context, id int64) error { m.mu.Lock() defer m.mu.Unlock() From ae84c712559951b23dbfff1c0e362f0cb0f14f03 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Thu, 30 Apr 2026 11:24:26 -0400 Subject: [PATCH 29/45] Clean up files --- pkg/beholder/DurableEmitterDesign.md | 416 ----------------- pkg/beholder/durable_emitter.go | 1 - .../durable_emitter_integration_test.go | 385 ---------------- pkg/beholder/durable_emitter_metric_info.go | 132 ------ pkg/beholder/durable_emitter_metrics.go | 128 ++++++ pkg/beholder/durable_emitter_store_wrap.go | 91 ---- pkg/beholder/durable_emitter_test.go | 433 +++++++++++++++++- pkg/beholder/durable_event_store.go | 85 ++++ 8 files changed, 635 insertions(+), 1036 deletions(-) delete mode 100644 pkg/beholder/DurableEmitterDesign.md delete mode 100644 pkg/beholder/durable_emitter_metric_info.go delete mode 100644 pkg/beholder/durable_emitter_store_wrap.go diff --git a/pkg/beholder/DurableEmitterDesign.md b/pkg/beholder/DurableEmitterDesign.md deleted file mode 100644 index 7f34cca20b..0000000000 --- a/pkg/beholder/DurableEmitterDesign.md +++ /dev/null @@ -1,416 +0,0 @@ -# Durable Event Buffer for ChIP - -## Problem Statement - -Today there is no persistence in the ChIP pipeline. The `ChipIngressEmitter` calls `chipingress.Client.Publish()` synchronously over gRPC, and the `batch.Client` uses an in-memory channel buffer of 200 messages. If the node crashes, Chip is unreachable, or the buffer fills up, events — including billing records — are silently dropped. - -**Drop points in the current architecture:** - -``` -DualSourceEmitter.Emit() - ├── OTLP (sync) — errors returned to caller - └── ChipIngressEmitter (async goroutine) - │ - └── chipingress.Client.Publish() ← fire-and-forget gRPC - │ - ├── If Chip is down → error logged, event LOST - ├── If node crashes mid-flight → event LOST - └── batch.Client buffer full → "message buffer is full", event LOST -``` - -- `batch.Client` drops messages when its 200-message channel is full. -- `DualSourceEmitter` only logs chip-ingress failures — errors are swallowed. -- No event survives a node restart. Nothing is persisted to disk. - -**Impact:** Billing records are silently dropped, leading to inconsistent revenue reconciliation. Any customer-facing observability that flows through ChIP is unreliable. - -## Requirements - -### Functional -- Events must not be lost on node restarts. -- Events must be delivered within a reasonable period of time (seconds under normal operation). -- System must support eventually-consistent billing. -- Node databases must not bloat unboundedly. - -### Non-Functional -- Scale to 1k+ TPS per node. -- 4 nines of availability (99.99%). -- `Emit()` must not block workflow execution. - -## Architecture - -### High-Level Flow - -``` -Workflow Engine / Billing / Lifecycle Events - │ - ▼ - DualSourceEmitter.Emit() - │ - ├── OTLP MessageEmitter (sync, unchanged) - │ - └── DurableEmitter.Emit() - │ - ├─ 1. ExtractSourceAndType + build CloudEventPb - ├─ 2. proto.Marshal → bytes - ├─ 3. store.Insert(payload) ← DURABLE GUARANTEE - ├─ 4. return nil (caller unblocked) - │ - └─ 5. goroutine: client.Publish(eventPb) - ├── Success → store.Delete(id) - └── Failure → no-op (retransmit loop handles it) - - ┌─────────────────────────────────────┐ - │ Background Retransmit Loop │ every 5s (configurable) - │ │ - │ store.ListPending(olderThan 10s) │ - │ → client.PublishBatch(events) │ - │ → store.Delete(ids) on success │ - └─────────────────────────────────────┘ - - ┌─────────────────────────────────────┐ - │ Background Expiry Loop │ every 1min (configurable) - │ │ - │ store.DeleteExpired(ttl=24h) │ - │ → GC events that could never be │ - │ delivered (bounds table growth) │ - └─────────────────────────────────────┘ -``` - -### Key Guarantee - -`Emit()` returns `nil` once the Postgres INSERT succeeds. Even if the node crashes immediately after, the event survives in Postgres and will be retransmitted on restart. The gRPC publish is fully asynchronous — `Emit()` latency is dominated by one DB insert (~1ms at typical payloads). - -### Design Decision: Standard `chipingress.Client`, Not `batch.Client` - -Per Hagen's guidance, we use the standard `chipingress.Client` directly (supports both `Publish` and `PublishBatch`) since we are implementing our own queuing with persistence-backed guarantees. The `batch.Client`'s in-memory buffer is redundant when we have Postgres as the durable queue. - -### Service Principal & ACK Guarantees - -CRE nodes authenticate to ChIP using the node's **CSA Key** as the `servicePrincipal`. This is NOT the `oti-telemetry-shared` principal which uses a fire-and-forget publish path. With the CSA Key, the gateway waits for Kafka ACKs before returning a gRPC response — a successful response means the event was durably accepted. - -## Components - -### DurableEventStore Interface (`chainlink-common`) - -```go -type DurableEvent struct { - ID int64 - Payload []byte // serialized CloudEventPb proto - CreatedAt time.Time -} - -type DurableEventStore interface { - Insert(ctx context.Context, payload []byte) (int64, error) - Delete(ctx context.Context, id int64) error - ListPending(ctx context.Context, createdBefore time.Time, limit int) ([]DurableEvent, error) - DeleteExpired(ctx context.Context, ttl time.Duration) (int64, error) -} -``` - -Two implementations: -- **`MemDurableEventStore`** — in-memory map, for unit/integration tests. Lives in `chainlink-common`. -- **`PgDurableEventStore`** — Postgres-backed ORM using `sqlutil.DataSource`. Lives in `chainlink`. - -### DurableEmitter (`chainlink-common`) - -Implements `beholder.Emitter` (`Emit` + `Close`). Core logic: - -```go -func (d *DurableEmitter) Emit(ctx context.Context, body []byte, attrKVs ...any) error { - // 1. Validate and extract source/type from attributes - sourceDomain, entityType, err := ExtractSourceAndType(attrKVs...) - - // 2. Build CloudEvent and serialize to proto bytes - event, _ := chipingress.NewEvent(sourceDomain, entityType, body, newAttributes(attrKVs...)) - eventPb, _ := chipingress.EventToProto(event) - payload, _ := proto.Marshal(eventPb) - - // 3. Persist — this is the durable guarantee - id, err := d.store.Insert(ctx, payload) - if err != nil { - return fmt.Errorf("failed to persist event: %w", err) - } - - // 4. Async delivery attempt - go d.publishAndDelete(id, eventPb) - return nil -} -``` - -### Configuration - -```go -type DurableEmitterConfig struct { - RetransmitInterval time.Duration // default 5s — retransmit loop tick rate - RetransmitAfter time.Duration // default 10s — min age before retry - RetransmitBatchSize int // default 100 — max events per batch - ExpiryInterval time.Duration // default 1min — expiry loop tick rate - EventTTL time.Duration // default 24h — max event age - PublishTimeout time.Duration // default 5s — per-RPC deadline -} -``` - -## Postgres Schema - -**Migration `0295_chip_durable_events.sql`** in the existing `cre` schema: - -```sql --- +goose Up -CREATE TABLE IF NOT EXISTS cre.chip_durable_events ( - id BIGSERIAL PRIMARY KEY, - payload BYTEA NOT NULL, - created_at TIMESTAMPTZ NOT NULL DEFAULT now() -); - -CREATE INDEX idx_chip_durable_events_created_at - ON cre.chip_durable_events (created_at ASC); - --- +goose Down -DROP INDEX IF EXISTS cre.idx_chip_durable_events_created_at; -DROP TABLE IF EXISTS cre.chip_durable_events; -``` - -The table lives in each node's existing Postgres database. Under normal operation it is **transient** — events are inserted and deleted within milliseconds. Under Chip outage, events accumulate until delivery resumes. - -## Node Wiring - -### Config Flag - -```toml -[Telemetry] -DurableEmitterEnabled = true -``` - -Added to `config.Telemetry` interface, `toml.Telemetry` struct, and `telemetryConfig` implementation. - -### Integration Point (`application.go`) - -Wired in `NewApplication` after the DB is available but before CRE services start: - -```go -func setupDurableEmitter(ctx context.Context, ds sqlutil.DataSource, lggr logger.SugaredLogger) error { - client := beholder.GetClient() - chipClient := client.Chip - - pgStore := beholdersvc.NewPgDurableEventStore(ds) - durableEmitter, _ := beholder.NewDurableEmitter(pgStore, chipClient, beholder.DefaultDurableEmitterConfig(), lggr) - - // Preserve OTLP path alongside durable chip delivery - messageLogger := client.MessageLoggerProvider.Logger("durable-emitter") - otlpEmitter := beholder.NewMessageEmitter(messageLogger) - dualEmitter, _ := beholder.NewDualSourceEmitter(durableEmitter, otlpEmitter) - - durableEmitter.Start(ctx) - client.Emitter = dualEmitter - return nil -} -``` - -This replaces the global beholder emitter, covering **all** emission paths: -- `events.emitProtoMessage()` — billing, workflow execution lifecycle -- `custmsg.Labeler.Emit()` — workflow user logs -- `BridgeStatusReporter` — bridge status events -- Any other `beholder.GetEmitter()` caller - -### CRE Environment Auto-Enable - -`system-tests/lib/cre/don/config/config.go` sets `DurableEmitterEnabled = true` for all Docker-based nodesets, so it activates automatically in local CRE environments. - -## File Manifest - -| Repo | File | Purpose | -|------|------|---------| -| chainlink-common | `pkg/beholder/durable_event_store.go` | `DurableEventStore` interface + `MemDurableEventStore` | -| chainlink-common | `pkg/beholder/durable_emitter.go` | `DurableEmitter` — Emit, retransmit loop, expiry loop | -| chainlink-common | `pkg/beholder/durable_emitter_test.go` | Unit tests (in-memory store) | -| chainlink-common | `pkg/beholder/durable_emitter_integration_test.go` | Integration tests (mock gRPC server) | -| chainlink | `core/services/beholder/durable_event_store_orm.go` | `PgDurableEventStore` (Postgres ORM) | -| chainlink | `core/services/beholder/durable_event_store_orm_test.go` | ORM tests + Postgres benchmarks + load tests | -| chainlink | `core/services/beholder/durable_emitter_load_test.go` | TPS ramp/sustained/payload tests (Postgres + mock or external Chip via `CHIP_INGRESS_TEST_ADDR`) | -| chainlink | `core/store/migrate/migrations/0295_chip_durable_events.sql` | Postgres migration | -| chainlink | `core/config/telemetry_config.go` | `DurableEmitterEnabled()` on Telemetry interface | -| chainlink | `core/config/toml/types.go` | TOML field + setFrom merge | -| chainlink | `core/services/chainlink/config_telemetry.go` | Config accessor | -| chainlink | `core/services/chainlink/application.go` | `setupDurableEmitter` wiring | -| chainlink | `system-tests/lib/cre/don/config/config.go` | Auto-enable in CRE Docker envs | -| chainlink | `system-tests/tests/smoke/cre/v2_durable_emitter_test.go` | CRE smoke tests + load test | -| chainlink | `system-tests/tests/smoke/cre/cre_suite_test.go` | Test entry points | - -## Testing - -### Unit Tests (`chainlink-common`, in-memory store) - -| Test | What It Proves | -|------|---------------| -| `TestDurableEmitter_EmitPersistsAndPublishes` | Happy path: emit → publish → delete | -| `TestDurableEmitter_EmitReturnSuccessEvenWhenPublishFails` | Emit succeeds on DB insert even when gRPC fails | -| `TestDurableEmitter_RetransmitLoopDeliversFailedEvents` | Background loop retries failed events | -| `TestDurableEmitter_ExpiryLoopDeletesOldEvents` | TTL-based garbage collection | -| `TestDurableEmitter_EmitRejectsInvalidAttributes` | Validation before DB insert | -| `TestDurableEmitter_MultipleEvents` | 50 concurrent events all delivered | - -### Integration Tests (`chainlink-common`, mock gRPC server) - -Real gRPC server with controllable failure injection: - -| Test | What It Proves | -|------|---------------| -| `TestIntegration_HappyPath` | Events delivered over real gRPC + proto round-trip | -| `TestIntegration_ServerUnavailable_RetransmitRecovers` | Server returns UNAVAILABLE → retransmit delivers via PublishBatch | -| `TestIntegration_ServerDown_EventsSurvive` | **Crash recovery**: server stopped → events persist → new emitter (same store) retransmits on "restart" | -| `TestIntegration_HighThroughput` | 500 events delivered concurrently | -| `TestIntegration_EventExpiry` | Undeliverable events expired after TTL | -| `TestIntegration_RetransmitUsesBatch` | Retransmit path uses PublishBatch, not individual Publish | -| `TestIntegration_GRPCConnection` | Source/type arrive correctly on server side | - -### Postgres ORM Tests + Benchmarks (`chainlink`, real Postgres) - -| Test / Benchmark | What It Measures | -|---|---| -| `TestPgDurableEventStore_*` | ORM correctness (insert, list, delete, expiry) | -| `Benchmark_Insert` | Raw INSERT throughput | -| `Benchmark_InsertDelete` | Insert+delete cycle (happy-path hot loop) | -| `Benchmark_InsertPayloadSizes` | INSERT at 64B, 256B, 1KB, 4KB | -| `Benchmark_ListPending` | Query performance at 100 and 1000 queue depth | -| `TestLoad_SustainedInsertDelete` | 2000 events, 10-way concurrent insert+delete, measures ops/sec | -| `TestLoad_BurstThenDrain` | 1000-event burst, then drain via ListPending+Delete batches | -| `TestLoad_ConcurrentInsertWithListPending` | 3s of concurrent inserts + ListPending (real contention) | - -### Full-Stack Load Tests (`chainlink`, Postgres + mock gRPC) - -| Test / Benchmark | What It Measures | -|---|---| -| `TestFullStack_SustainedThroughput` | 1000 events, 10 concurrent emitters, end-to-end rate | -| `TestFullStack_ChipOutage` | 3-phase: normal → Chip goes UNAVAILABLE → recovery. Measures accumulation and drain rate | -| `TestFullStack_SlowChip` | 50ms gRPC latency. Proves Emit() stays fast while server is slow | -| `Benchmark_FullStack_EmitThroughput` | Upper bound events/sec through full pipeline | -| `Benchmark_FullStack_EmitPayloadSizes` | Full emit at 64B, 256B, 1KB, 4KB | - -### Durable emitter TPS load tests (`chainlink/core/services/beholder/durable_emitter_load_test.go`) - -These tests exercise **Postgres + `DurableEmitter` + Chip Ingress** (in-process mock **or** a real gateway). They are heavier than the ORM benchmarks and require a **real Postgres** (not `txdb`). - -#### Prerequisites - -- **`CL_DATABASE_URL`** — must point at a Postgres instance where migration **`0295_chip_durable_events`** has been applied (`cre.chip_durable_events` exists). Same URL pattern as other chainlink DB tests. -- **Short tests skipped** — if your test runner uses `-short`, these tests are skipped (`SkipShortDB`); run **without** `-short`. - -#### Mock Chip vs real Chip Ingress - -| Mode | How | Notes | -|------|-----|--------| -| **Mock** (default) | Do **not** set `CHIP_INGRESS_TEST_ADDR` | In-process gRPC server; tests can inject failures (outage, slow Chip). | -| **Real Chip** | Set `CHIP_INGRESS_TEST_ADDR=host:port` | Dials external Chip Ingress. Optional: `CHIP_INGRESS_TEST_TLS`, `CHIP_INGRESS_TEST_BASIC_AUTH_*`, `CHIP_INGRESS_TEST_SKIP_BASIC_AUTH`, `CHIP_INGRESS_TEST_SKIP_SCHEMA_REGISTRATION`. You need Kafka/Redpanda, topic **`chip-demo`**, and schema subject **`chip-demo-pb.DemoClientPayload`** (e.g. Atlas `make create-topic-and-schema` under `atlas/chip-ingress`). | - -Tests that **inject** Chip failures or rely on **in-process** receive counts are **skipped** when `CHIP_INGRESS_TEST_ADDR` is set. - -#### How to run - -From the `chainlink` repo root (examples): - -```bash -# All beholder tests including TPS (requires CL_DATABASE_URL) -export CL_DATABASE_URL='postgres://...' -go test -v -count=1 ./core/services/beholder/ -run 'TestTPS_|TestChipIngressExternalPing' - -# Ramp-up only (100 → 500 → 1k → 2k TPS levels) -go test -v -count=1 ./core/services/beholder/ -run TestTPS_RampUp - -# Sustained 1k TPS for 60s + drain check -go test -v -count=1 ./core/services/beholder/ -run TestTPS_Sustained1k - -# Payload size scaling (fixed duration per size) -go test -v -count=1 ./core/services/beholder/ -run TestTPS_PayloadSizeScaling - -# External Chip smoke (with addr set) -export CHIP_INGRESS_TEST_ADDR='localhost:50051' -go test -v -count=1 ./core/services/beholder/ -run TestChipIngressExternalPing -``` - -After a full package run, **`TestMain`** prints a **TPS LOAD TEST SUMMARY** block aggregating result blocks from **`TestTPS_RampUp`**, **`TestTPS_Sustained1k`**, **`TestTPS_1k_WithChipOutage`** (mock only; skipped with external Chip), and **`TestTPS_PayloadSizeScaling`**. - -#### Reading the tables (column glossary) - -| Column | Meaning | -|--------|---------| -| **Target TPS** | Requested emit rate (token-bucket style scheduling across workers). | -| **Achieved TPS** | `Total emits ÷ window duration` — realized successful `Emit()` throughput. | -| **Total emits** | Count of **`Emit()` calls that returned `nil`** in the measurement window (successful Postgres insert path). Does not count failures. | -| **Emit p50 / p99** | Latency of successful `Emit()` calls (dominated by DB insert). | -| **Pub fail (retry)*** | Failed `Publish` / `PublishBatch` RPCs during the window: immediate failures (one row each, need retransmit) plus, when shown as `a+b`, `b` = total event count in failed `PublishBatch` calls. `Emit()` insert failures are logged separately if non-zero. | -| **Q max (rows)** | Peak row count in `cre.chip_durable_events` sampled during the emit window (~50ms polls). | -| **Q end (rows)** | Row count after a short settle (async publish / retransmit). | -| **Q max (KB)*** | For the peak queue sample: `sum(octet_length(payload))/1024` over queued rows (payload bytes only). **Q end** payload size is omitted from the printed table to keep it narrow. | - -With **`CHIP_INGRESS_TEST_ADDR`** set, there is no in-process mock — validate end-to-end delivery with **Kafka / Chip / gateway metrics** (or consumer checks). **Total emits** and **Achieved TPS** still reflect successful durable inserts on the node. - -### CRE Smoke Tests (live Docker environment) - -Tests connect to the node's Postgres and query `cre.chip_durable_events` directly, using `pg_stat_user_tables` for insert/delete statistics — the same pattern used by the EVM LogTrigger test for `trigger_pending_events`. - -| Test | What It Does | -|------|-------------| -| `Test_CRE_V2_DurableEmitter` | Deploys a cron workflow (every 5s), waits for 30+ insert+delete cycles, verifies queue drains to near-empty | -| `Test_CRE_V2_DurableEmitter_Load` | Deploys 5 cron workflows (every 1s each), runs for 3 minutes. Logs insert/delete rates, max queue depth, and prints summary table | - -**Running CRE smoke tests:** -```bash -# Basic correctness -go test -v -run Test_CRE_V2_DurableEmitter$ -timeout 10m - -# Load test (5 workflows × 1s cron, 3min observation) -go test -v -run Test_CRE_V2_DurableEmitter_Load -timeout 10m -``` - -**Example load test output:** -``` -╔════════════════════════════════════════════════╗ -║ DURABLE EMITTER LOAD TEST RESULTS ║ -╠════════════════════════════════════════════════╣ -║ Workflows deployed: 5 ║ -║ Observation period: 3m0s ║ -║ Total inserts: 1842 ║ -║ Total deletes: 1840 ║ -║ Avg insert rate: 10.2 events/sec ║ -║ Avg delete rate: 10.2 events/sec ║ -║ Max queue depth: 12 ║ -║ Final pending: 2 ║ -╚════════════════════════════════════════════════╝ -``` - -## Metrics to Instrument (Future) - -| Metric | Description | -|--------|-------------| -| `durable_emitter.queue_depth` | Current row count in `chip_durable_events` | -| `durable_emitter.insert_rate` | Events persisted per second | -| `durable_emitter.publish_rate` | Events successfully delivered per second | -| `durable_emitter.retransmit_rate` | Events retransmitted via background loop | -| `durable_emitter.publish_latency` | Time from insert to confirmed delivery | -| `durable_emitter.oldest_pending` | Age of the longest-waiting event | -| `durable_emitter.expired_count` | Events expired (dropped after TTL) | -| `durable_emitter.error_rate` | Failed publish attempts per second | - -## Open Questions & Future Work - -### 1. Chip Gateway Idempotency -Does the gateway deduplicate re-sent events? If the retransmit loop re-sends an event that the immediate path already delivered (race window), the gateway should de-dup using the CloudEvent `id` (UUID). Needs server-side confirmation. - -### 2. DB Load at Scale -At 1k TPS: ~1k inserts/sec + ~1k deletes/sec = ~2k write ops/sec on the node's Postgres. This produces dead tuples requiring autovacuum tuning. Potential optimizations: -- **Batch deletes** — delete by ID list instead of per-row. -- **Two-table approach** — queued + recently-sent to reduce churn on the hot table. -- **CDC streaming** — stream WAL changes directly, avoiding the insert/delete pattern entirely. Matthew Gardener and Clement can advise on CDC implementation. - -### 3. Exponential Backoff -Current PoC uses a fixed retransmit interval. Production should implement per-event exponential backoff using `attempts` and `last_sent_at` columns (schema extension). - -### 4. rmq / Redis Alternative -Patrick raised using [rmq](https://github.com/wellle/rmq) backed by our own DB instead of re-implementing a queue. Worth evaluating if the Postgres-backed approach shows scaling issues in load testing. - -### 5. CDC Streaming -Could stream WAL changes directly rather than polling the table, avoiding the insert/delete churn entirely. This would also enable real-time analytics on event flow. Requires infrastructure coordination with the data analytics pipeline team. - -### 6. DurableEmitter Lifecycle Management -Currently the `DurableEmitter` is started in `application.go` and its background loops are tied to the application context. For production, it should be registered as a proper `services.ServiceCtx` with Start/Close lifecycle management, health checks, and graceful shutdown (flush pending events before stopping). diff --git a/pkg/beholder/durable_emitter.go b/pkg/beholder/durable_emitter.go index 85ae337db5..a19f580d17 100644 --- a/pkg/beholder/durable_emitter.go +++ b/pkg/beholder/durable_emitter.go @@ -704,7 +704,6 @@ func (d *DurableEmitter) publishAndDelete(id int64, eventPb *chipingress.CloudEv "publish_rpc_elapsed", elapsed.String(), "publish_rpc_elapsed_ms", elapsed.Milliseconds(), ) - //d.log.Infow("DurableEmitter: Chip Ingress publish succeeded (immediate)", pubOKKVs...) t1 := time.Now() markErr := d.store.MarkDelivered(context.Background(), id) diff --git a/pkg/beholder/durable_emitter_integration_test.go b/pkg/beholder/durable_emitter_integration_test.go index 7c9fdff252..fffcbd62dc 100644 --- a/pkg/beholder/durable_emitter_integration_test.go +++ b/pkg/beholder/durable_emitter_integration_test.go @@ -1,387 +1,2 @@ package beholder_test -import ( - "context" - "net" - "sync" - "sync/atomic" - "testing" - "time" - - cepb "github.com/cloudevents/sdk-go/binding/format/protobuf/v2/pb" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "google.golang.org/grpc" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/credentials/insecure" - "google.golang.org/grpc/status" - - "github.com/smartcontractkit/chainlink-common/pkg/beholder" - "github.com/smartcontractkit/chainlink-common/pkg/chipingress" - "github.com/smartcontractkit/chainlink-common/pkg/chipingress/pb" - "github.com/smartcontractkit/chainlink-common/pkg/logger" -) - -// mockChipServer implements ChipIngressServer with controllable behaviour. -type mockChipServer struct { - pb.UnimplementedChipIngressServer - - mu sync.Mutex - publishErr error - batchErr error - received []*cepb.CloudEvent - batchReceived [][]*cepb.CloudEvent - publishCount atomic.Int64 - batchCount atomic.Int64 - publishDelay time.Duration -} - -func (s *mockChipServer) Publish(_ context.Context, in *cepb.CloudEvent) (*pb.PublishResponse, error) { - if s.publishDelay > 0 { - time.Sleep(s.publishDelay) - } - s.publishCount.Add(1) - s.mu.Lock() - defer s.mu.Unlock() - if s.publishErr != nil { - return nil, s.publishErr - } - s.received = append(s.received, in) - return &pb.PublishResponse{}, nil -} - -func (s *mockChipServer) PublishBatch(_ context.Context, in *pb.CloudEventBatch) (*pb.PublishResponse, error) { - s.batchCount.Add(1) - s.mu.Lock() - defer s.mu.Unlock() - if s.batchErr != nil { - return nil, s.batchErr - } - s.batchReceived = append(s.batchReceived, in.Events) - s.received = append(s.received, in.Events...) - return &pb.PublishResponse{}, nil -} - -func (s *mockChipServer) Ping(context.Context, *pb.EmptyRequest) (*pb.PingResponse, error) { - return &pb.PingResponse{Message: "pong"}, nil -} - -func (s *mockChipServer) setPublishErr(err error) { - s.mu.Lock() - defer s.mu.Unlock() - s.publishErr = err -} - -func (s *mockChipServer) setBatchErr(err error) { - s.mu.Lock() - defer s.mu.Unlock() - s.batchErr = err -} - -func (s *mockChipServer) receivedCount() int { - s.mu.Lock() - defer s.mu.Unlock() - return len(s.received) -} - -func (s *mockChipServer) batchCallCount() int { - s.mu.Lock() - defer s.mu.Unlock() - return len(s.batchReceived) -} - -// startMockServer starts a gRPC server on a random port and returns the -// server, address, and a cleanup function. -func startMockServer(t *testing.T, srv *mockChipServer) (*grpc.Server, string) { - t.Helper() - lis, err := net.Listen("tcp", "127.0.0.1:0") - require.NoError(t, err) - - gs := grpc.NewServer() - pb.RegisterChipIngressServer(gs, srv) - - go func() { - if err := gs.Serve(lis); err != nil { - // Ignore errors from server being stopped during cleanup. - } - }() - - t.Cleanup(func() { gs.GracefulStop() }) - return gs, lis.Addr().String() -} - -func newChipClient(t *testing.T, addr string) chipingress.Client { - t.Helper() - c, err := chipingress.NewClient(addr, chipingress.WithInsecureConnection()) - require.NoError(t, err) - t.Cleanup(func() { _ = c.Close() }) - return c -} - -func emitAttrs() []any { - return []any{"source", "test-domain", "type", "test-entity"} -} - -func fastCfg() beholder.DurableEmitterConfig { - return beholder.DurableEmitterConfig{ - RetransmitInterval: 100 * time.Millisecond, - RetransmitAfter: 50 * time.Millisecond, - RetransmitBatchSize: 50, - ExpiryInterval: 200 * time.Millisecond, - EventTTL: 500 * time.Millisecond, - PublishTimeout: 2 * time.Second, - } -} - -// ---------- Test cases ---------- - -func TestIntegration_HappyPath(t *testing.T) { - srv := &mockChipServer{} - _, addr := startMockServer(t, srv) - client := newChipClient(t, addr) - store := beholder.NewMemDurableEventStore() - - em, err := beholder.NewDurableEmitter(store, client, fastCfg(), logger.Test(t)) - require.NoError(t, err) - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - em.Start(ctx) - defer em.Close() - - require.NoError(t, em.Emit(ctx, []byte("billing-record-1"), emitAttrs()...)) - require.NoError(t, em.Emit(ctx, []byte("billing-record-2"), emitAttrs()...)) - - require.Eventually(t, func() bool { - return srv.receivedCount() == 2 - }, 3*time.Second, 10*time.Millisecond, "server should receive both events") - - require.Eventually(t, func() bool { - return store.Len() == 0 - }, 3*time.Second, 10*time.Millisecond, "store should be empty after delivery") -} - -func TestIntegration_ServerUnavailable_RetransmitRecovers(t *testing.T) { - // Start with server returning UNAVAILABLE. - srv := &mockChipServer{} - srv.setPublishErr(status.Error(codes.Unavailable, "chip down")) - _, addr := startMockServer(t, srv) - client := newChipClient(t, addr) - store := beholder.NewMemDurableEventStore() - - em, err := beholder.NewDurableEmitter(store, client, fastCfg(), logger.Test(t)) - require.NoError(t, err) - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - em.Start(ctx) - defer em.Close() - - require.NoError(t, em.Emit(ctx, []byte("will-retry"), emitAttrs()...)) - - // Event should be in the store, not delivered. - time.Sleep(200 * time.Millisecond) - assert.Equal(t, 1, store.Len(), "event persists while server is unavailable") - - // "Recover" the server. - srv.setPublishErr(nil) - - require.Eventually(t, func() bool { - return store.Len() == 0 - }, 5*time.Second, 50*time.Millisecond, "retransmit loop should deliver after recovery") - - assert.GreaterOrEqual(t, srv.publishCount.Load(), int64(2), - "one failed immediate Publish then one retransmit Publish") - assert.Equal(t, int64(0), srv.batchCount.Load(), "retransmit should not use PublishBatch") -} - -func TestIntegration_ServerDown_EventsSurvive(t *testing.T) { - // Start server, then stop it to simulate total outage. - srv := &mockChipServer{} - gs, addr := startMockServer(t, srv) - client := newChipClient(t, addr) - store := beholder.NewMemDurableEventStore() - - cfg := fastCfg() - cfg.PublishTimeout = 500 * time.Millisecond - em, err := beholder.NewDurableEmitter(store, client, cfg, logger.Test(t)) - require.NoError(t, err) - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - em.Start(ctx) - - // Stop the gRPC server entirely. - gs.Stop() - time.Sleep(100 * time.Millisecond) - - // Emit while server is down — Emit() itself must succeed (DB insert works). - require.NoError(t, em.Emit(ctx, []byte("server-is-down"), emitAttrs()...)) - assert.Equal(t, 1, store.Len(), "event should be persisted even with server down") - - // Stop the emitter to simulate a "node shutdown". - em.Close() - - // Bring up a new server on the same address. - srv2 := &mockChipServer{} - lis, err := net.Listen("tcp", addr) - require.NoError(t, err) - gs2 := grpc.NewServer() - pb.RegisterChipIngressServer(gs2, srv2) - go func() { _ = gs2.Serve(lis) }() - t.Cleanup(func() { gs2.GracefulStop() }) - - // Create a new client and DurableEmitter re-using the same store - // (simulating node restart with Postgres). - client2, err := chipingress.NewClient(addr, chipingress.WithInsecureConnection()) - require.NoError(t, err) - t.Cleanup(func() { _ = client2.Close() }) - - em2, err := beholder.NewDurableEmitter(store, client2, cfg, logger.Test(t)) - require.NoError(t, err) - em2.Start(ctx) - defer em2.Close() - - require.Eventually(t, func() bool { - return srv2.receivedCount() == 1 - }, 5*time.Second, 50*time.Millisecond, "new emitter should retransmit the surviving event") - - require.Eventually(t, func() bool { - return store.Len() == 0 - }, 5*time.Second, 50*time.Millisecond, "store should be empty after retransmit") -} - -func TestIntegration_HighThroughput(t *testing.T) { - srv := &mockChipServer{} - _, addr := startMockServer(t, srv) - client := newChipClient(t, addr) - store := beholder.NewMemDurableEventStore() - - cfg := fastCfg() - cfg.RetransmitBatchSize = 200 - em, err := beholder.NewDurableEmitter(store, client, cfg, logger.Test(t)) - require.NoError(t, err) - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - em.Start(ctx) - defer em.Close() - - const n = 500 - for i := 0; i < n; i++ { - require.NoError(t, em.Emit(ctx, []byte("event"), emitAttrs()...)) - } - - require.Eventually(t, func() bool { - return srv.receivedCount() >= n - }, 10*time.Second, 50*time.Millisecond, "all %d events should be received", n) - - require.Eventually(t, func() bool { - return store.Len() == 0 - }, 10*time.Second, 50*time.Millisecond, "store should drain completely") -} - -func TestIntegration_EventExpiry(t *testing.T) { - // Server always rejects — events can never be delivered. - srv := &mockChipServer{} - srv.setPublishErr(status.Error(codes.Internal, "permanent failure")) - _, addr := startMockServer(t, srv) - client := newChipClient(t, addr) - store := beholder.NewMemDurableEventStore() - - cfg := fastCfg() - cfg.EventTTL = 100 * time.Millisecond - cfg.ExpiryInterval = 100 * time.Millisecond - em, err := beholder.NewDurableEmitter(store, client, cfg, logger.Test(t)) - require.NoError(t, err) - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - em.Start(ctx) - defer em.Close() - - require.NoError(t, em.Emit(ctx, []byte("will-expire"), emitAttrs()...)) - assert.Equal(t, 1, store.Len()) - - require.Eventually(t, func() bool { - return store.Len() == 0 - }, 5*time.Second, 50*time.Millisecond, - "expiry loop should purge undeliverable events after TTL") -} - -func TestIntegration_RetransmitUsesSerialPublish(t *testing.T) { - // Immediate Publish fails; retransmit uses one Publish per queued row. - srv := &mockChipServer{} - srv.setPublishErr(status.Error(codes.Unavailable, "reject immediate")) - _, addr := startMockServer(t, srv) - client := newChipClient(t, addr) - store := beholder.NewMemDurableEventStore() - - em, err := beholder.NewDurableEmitter(store, client, fastCfg(), logger.Test(t)) - require.NoError(t, err) - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - em.Start(ctx) - defer em.Close() - - for i := 0; i < 5; i++ { - require.NoError(t, em.Emit(ctx, []byte("retry-me"), emitAttrs()...)) - } - - srv.setPublishErr(nil) - - require.Eventually(t, func() bool { - return store.Len() == 0 - }, 5*time.Second, 50*time.Millisecond, - "retransmit should deliver each event with its own Publish RPC") - - assert.Equal(t, 0, srv.batchCallCount(), "retransmit should not call PublishBatch") - assert.GreaterOrEqual(t, srv.publishCount.Load(), int64(10), - "five failed immediate attempts plus five retransmit publishes") -} - -// TestIntegration_GRPCConnection verifies the emitter works over a real gRPC -// connection with proper proto serialization round-trip. -func TestIntegration_GRPCConnection(t *testing.T) { - srv := &mockChipServer{} - _, addr := startMockServer(t, srv) - - // Use a raw gRPC dial to prove we're going over the wire. - conn, err := grpc.NewClient(addr, grpc.WithTransportCredentials(insecure.NewCredentials())) - require.NoError(t, err) - t.Cleanup(func() { _ = conn.Close() }) - - // Ping to verify connectivity. - grpcClient := pb.NewChipIngressClient(conn) - pong, err := grpcClient.Ping(context.Background(), &pb.EmptyRequest{}) - require.NoError(t, err) - assert.Equal(t, "pong", pong.Message) - - // Now use the chipingress.Client wrapper with DurableEmitter. - client := newChipClient(t, addr) - store := beholder.NewMemDurableEventStore() - - em, err := beholder.NewDurableEmitter(store, client, fastCfg(), logger.Test(t)) - require.NoError(t, err) - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - em.Start(ctx) - defer em.Close() - - payload := []byte("proto-round-trip-test") - require.NoError(t, em.Emit(ctx, payload, emitAttrs()...)) - - require.Eventually(t, func() bool { - return srv.receivedCount() == 1 - }, 3*time.Second, 10*time.Millisecond) - - // Verify the CloudEvent arrived with correct source/type. - srv.mu.Lock() - received := srv.received[0] - srv.mu.Unlock() - - assert.Equal(t, "test-domain", received.Source) - assert.Equal(t, "test-entity", received.Type) -} diff --git a/pkg/beholder/durable_emitter_metric_info.go b/pkg/beholder/durable_emitter_metric_info.go deleted file mode 100644 index 30cda04503..0000000000 --- a/pkg/beholder/durable_emitter_metric_info.go +++ /dev/null @@ -1,132 +0,0 @@ -package beholder - -// Durable emitter OTel instruments (registered via beholder.GetMeter), matching the -// MetricInfo pattern used with beholder elsewhere in chainlink-common. - -var ( - durableEmitterMetricEmitSuccess = MetricInfo{ - Name: "beholder.durable_emitter.emit.success", - Unit: "{call}", - Description: "Successful durable Emit calls (insert returned)", - } - durableEmitterMetricEmitFailure = MetricInfo{ - Name: "beholder.durable_emitter.emit.failure", - Unit: "{call}", - Description: "Failed Emit calls (before or during insert)", - } - durableEmitterMetricEmitDuration = MetricInfo{ - Name: "beholder.durable_emitter.emit.duration", - Unit: "s", - Description: "Emit insert path duration (seconds, fractional; aligns with Prometheus _duration_seconds)", - } - durableEmitterMetricEmitTotalDuration = MetricInfo{ - Name: "beholder.durable_emitter.emit.total_duration", - Unit: "s", - Description: "Full Emit() wall time including event construction, DB insert, and channel enqueue (seconds)", - } - durableEmitterMetricPublishImmSuccess = MetricInfo{ - Name: "beholder.durable_emitter.publish.immediate.success", - Unit: "{call}", - Description: "Immediate Publish RPC successes", - } - durableEmitterMetricPublishImmFailure = MetricInfo{ - Name: "beholder.durable_emitter.publish.immediate.failure", - Unit: "{call}", - Description: "Immediate Publish RPC failures (events await retransmit)", - } - durableEmitterMetricPublishDuration = MetricInfo{ - Name: "beholder.durable_emitter.publish.duration", - Unit: "s", - Description: "Chip Ingress Publish RPC duration (seconds); labels: phase={immediate,retransmit,best_effort}, error={true,false}", - } - durableEmitterMetricPublishBatchSuccess = MetricInfo{ - Name: "beholder.durable_emitter.publish.retransmit.batch.success", - Unit: "{call}", - Description: "Unused; retransmit uses serial Publish (see retransmit.events.*)", - } - durableEmitterMetricPublishBatchFailure = MetricInfo{ - Name: "beholder.durable_emitter.publish.retransmit.batch.failure", - Unit: "{call}", - Description: "Unused; retransmit uses serial Publish (see retransmit.events.*)", - } - durableEmitterMetricPublishBatchEvSuccess = MetricInfo{ - Name: "beholder.durable_emitter.publish.retransmit.events.success", - Unit: "{event}", - Description: "Retransmit Publish RPC successes (one RPC per queued event)", - } - durableEmitterMetricPublishBatchEvFailure = MetricInfo{ - Name: "beholder.durable_emitter.publish.retransmit.events.failure", - Unit: "{event}", - Description: "Retransmit Publish RPC failures (event stays queued)", - } - durableEmitterMetricDeliveryCompleted = MetricInfo{ - Name: "beholder.durable_emitter.delivery.completed", - Unit: "{event}", - Description: "Events removed from store after successful publish (immediate or retransmit)", - } - durableEmitterMetricExpiredPurged = MetricInfo{ - Name: "beholder.durable_emitter.expired_purged", - Unit: "{event}", - Description: "Events deleted by TTL expiry loop", - } - durableEmitterMetricStoreOperations = MetricInfo{ - Name: "beholder.durable_emitter.store.operations", - Unit: "{op}", - Description: "Durable store operations (proxy for DB load / IOPs)", - } - durableEmitterMetricStoreOpDuration = MetricInfo{ - Name: "beholder.durable_emitter.store.operation.duration", - Unit: "s", - Description: "Durable store operation latency (seconds, fractional)", - } - durableEmitterMetricQueueDepth = MetricInfo{ - Name: "beholder.durable_emitter.queue.depth", - Unit: "{row}", - Description: "Pending rows in durable queue", - } - durableEmitterMetricQueueDepthMax = MetricInfo{ - Name: "beholder.durable_emitter.queue.depth_max", - Unit: "{row}", - Description: "High-water mark of pending queue depth since start", - } - durableEmitterMetricQueuePayloadBytes = MetricInfo{ - Name: "beholder.durable_emitter.queue.payload_bytes", - Unit: "By", - Description: "Sum of payload bytes for pending rows", - } - durableEmitterMetricQueueOldestAgeSec = MetricInfo{ - Name: "beholder.durable_emitter.queue.oldest_pending_age_seconds", - Unit: "s", - Description: "Age of oldest pending row at last poll (longest wait)", - } - durableEmitterMetricQueueNearTTL = MetricInfo{ - Name: "beholder.durable_emitter.queue.near_ttl", - Unit: "{row}", - Description: "Rows within near-expiry window of EventTTL (DLQ pressure proxy; no separate DLQ table)", - } - durableEmitterMetricQueueCapacityRatio = MetricInfo{ - Name: "beholder.durable_emitter.queue.capacity_usage_ratio", - Unit: "1", - Description: "queue.payload_bytes / MaxQueuePayloadBytes when max > 0", - } - durableEmitterMetricProcHeapInuse = MetricInfo{ - Name: "beholder.durable_emitter.process.memory.heap_inuse_bytes", - Unit: "By", - Description: "Go runtime MemStats HeapInuse", - } - durableEmitterMetricProcHeapSys = MetricInfo{ - Name: "beholder.durable_emitter.process.memory.heap_sys_bytes", - Unit: "By", - Description: "Go runtime MemStats HeapSys", - } - durableEmitterMetricProcCPUUser = MetricInfo{ - Name: "beholder.durable_emitter.process.cpu.user_seconds", - Unit: "s", - Description: "Cumulative user CPU seconds (getrusage; Unix only)", - } - durableEmitterMetricProcCPUSys = MetricInfo{ - Name: "beholder.durable_emitter.process.cpu.system_seconds", - Unit: "s", - Description: "Cumulative system CPU seconds (getrusage; Unix only)", - } -) diff --git a/pkg/beholder/durable_emitter_metrics.go b/pkg/beholder/durable_emitter_metrics.go index 92cbfe689c..e08d1e61a4 100644 --- a/pkg/beholder/durable_emitter_metrics.go +++ b/pkg/beholder/durable_emitter_metrics.go @@ -9,6 +9,134 @@ import ( "go.opentelemetry.io/otel/metric" ) +var ( + durableEmitterMetricEmitSuccess = MetricInfo{ + Name: "beholder.durable_emitter.emit.success", + Unit: "{call}", + Description: "Successful durable Emit calls (insert returned)", + } + durableEmitterMetricEmitFailure = MetricInfo{ + Name: "beholder.durable_emitter.emit.failure", + Unit: "{call}", + Description: "Failed Emit calls (before or during insert)", + } + durableEmitterMetricEmitDuration = MetricInfo{ + Name: "beholder.durable_emitter.emit.duration", + Unit: "s", + Description: "Emit insert path duration (seconds, fractional; aligns with Prometheus _duration_seconds)", + } + durableEmitterMetricEmitTotalDuration = MetricInfo{ + Name: "beholder.durable_emitter.emit.total_duration", + Unit: "s", + Description: "Full Emit() wall time including event construction, DB insert, and channel enqueue (seconds)", + } + durableEmitterMetricPublishImmSuccess = MetricInfo{ + Name: "beholder.durable_emitter.publish.immediate.success", + Unit: "{call}", + Description: "Immediate Publish RPC successes", + } + durableEmitterMetricPublishImmFailure = MetricInfo{ + Name: "beholder.durable_emitter.publish.immediate.failure", + Unit: "{call}", + Description: "Immediate Publish RPC failures (events await retransmit)", + } + durableEmitterMetricPublishDuration = MetricInfo{ + Name: "beholder.durable_emitter.publish.duration", + Unit: "s", + Description: "Chip Ingress Publish RPC duration (seconds); labels: phase={immediate,retransmit,best_effort}, error={true,false}", + } + durableEmitterMetricPublishBatchSuccess = MetricInfo{ + Name: "beholder.durable_emitter.publish.retransmit.batch.success", + Unit: "{call}", + Description: "Unused; retransmit uses serial Publish (see retransmit.events.*)", + } + durableEmitterMetricPublishBatchFailure = MetricInfo{ + Name: "beholder.durable_emitter.publish.retransmit.batch.failure", + Unit: "{call}", + Description: "Unused; retransmit uses serial Publish (see retransmit.events.*)", + } + durableEmitterMetricPublishBatchEvSuccess = MetricInfo{ + Name: "beholder.durable_emitter.publish.retransmit.events.success", + Unit: "{event}", + Description: "Retransmit Publish RPC successes (one RPC per queued event)", + } + durableEmitterMetricPublishBatchEvFailure = MetricInfo{ + Name: "beholder.durable_emitter.publish.retransmit.events.failure", + Unit: "{event}", + Description: "Retransmit Publish RPC failures (event stays queued)", + } + durableEmitterMetricDeliveryCompleted = MetricInfo{ + Name: "beholder.durable_emitter.delivery.completed", + Unit: "{event}", + Description: "Events removed from store after successful publish (immediate or retransmit)", + } + durableEmitterMetricExpiredPurged = MetricInfo{ + Name: "beholder.durable_emitter.expired_purged", + Unit: "{event}", + Description: "Events deleted by TTL expiry loop", + } + durableEmitterMetricStoreOperations = MetricInfo{ + Name: "beholder.durable_emitter.store.operations", + Unit: "{op}", + Description: "Durable store operations (proxy for DB load / IOPs)", + } + durableEmitterMetricStoreOpDuration = MetricInfo{ + Name: "beholder.durable_emitter.store.operation.duration", + Unit: "s", + Description: "Durable store operation latency (seconds, fractional)", + } + durableEmitterMetricQueueDepth = MetricInfo{ + Name: "beholder.durable_emitter.queue.depth", + Unit: "{row}", + Description: "Pending rows in durable queue", + } + durableEmitterMetricQueueDepthMax = MetricInfo{ + Name: "beholder.durable_emitter.queue.depth_max", + Unit: "{row}", + Description: "High-water mark of pending queue depth since start", + } + durableEmitterMetricQueuePayloadBytes = MetricInfo{ + Name: "beholder.durable_emitter.queue.payload_bytes", + Unit: "By", + Description: "Sum of payload bytes for pending rows", + } + durableEmitterMetricQueueOldestAgeSec = MetricInfo{ + Name: "beholder.durable_emitter.queue.oldest_pending_age_seconds", + Unit: "s", + Description: "Age of oldest pending row at last poll (longest wait)", + } + durableEmitterMetricQueueNearTTL = MetricInfo{ + Name: "beholder.durable_emitter.queue.near_ttl", + Unit: "{row}", + Description: "Rows within near-expiry window of EventTTL (DLQ pressure proxy; no separate DLQ table)", + } + durableEmitterMetricQueueCapacityRatio = MetricInfo{ + Name: "beholder.durable_emitter.queue.capacity_usage_ratio", + Unit: "1", + Description: "queue.payload_bytes / MaxQueuePayloadBytes when max > 0", + } + durableEmitterMetricProcHeapInuse = MetricInfo{ + Name: "beholder.durable_emitter.process.memory.heap_inuse_bytes", + Unit: "By", + Description: "Go runtime MemStats HeapInuse", + } + durableEmitterMetricProcHeapSys = MetricInfo{ + Name: "beholder.durable_emitter.process.memory.heap_sys_bytes", + Unit: "By", + Description: "Go runtime MemStats HeapSys", + } + durableEmitterMetricProcCPUUser = MetricInfo{ + Name: "beholder.durable_emitter.process.cpu.user_seconds", + Unit: "s", + Description: "Cumulative user CPU seconds (getrusage; Unix only)", + } + durableEmitterMetricProcCPUSys = MetricInfo{ + Name: "beholder.durable_emitter.process.cpu.system_seconds", + Unit: "s", + Description: "Cumulative system CPU seconds (getrusage; Unix only)", + } +) + // DurableEmitterMetricsConfig enables OpenTelemetry metrics for DurableEmitter. // Set on DurableEmitterConfig.Metrics; nil disables instrumentation. // diff --git a/pkg/beholder/durable_emitter_store_wrap.go b/pkg/beholder/durable_emitter_store_wrap.go deleted file mode 100644 index 75fdb2e0a9..0000000000 --- a/pkg/beholder/durable_emitter_store_wrap.go +++ /dev/null @@ -1,91 +0,0 @@ -package beholder - -import ( - "context" - "errors" - "time" -) - -// metricsInstrumentedStore wraps DurableEventStore to record store operation metrics. -type metricsInstrumentedStore struct { - inner DurableEventStore - m *durableEmitterMetrics -} - -var _ DurableEventStore = (*metricsInstrumentedStore)(nil) -var _ DurableQueueObserver = (*metricsInstrumentedStore)(nil) - -func newMetricsInstrumentedStore(inner DurableEventStore, m *durableEmitterMetrics) DurableEventStore { - if m == nil { - return inner - } - return &metricsInstrumentedStore{inner: inner, m: m} -} - -func (s *metricsInstrumentedStore) Insert(ctx context.Context, payload []byte) (int64, error) { - t0 := time.Now() - id, err := s.inner.Insert(ctx, payload) - s.m.recordStoreOp(ctx, "insert", time.Since(t0), err) - return id, err -} - -func (s *metricsInstrumentedStore) Delete(ctx context.Context, id int64) error { - t0 := time.Now() - err := s.inner.Delete(ctx, id) - s.m.recordStoreOp(ctx, "delete", time.Since(t0), err) - return err -} - -func (s *metricsInstrumentedStore) MarkDelivered(ctx context.Context, id int64) error { - t0 := time.Now() - err := s.inner.MarkDelivered(ctx, id) - s.m.recordStoreOp(ctx, "mark_delivered", time.Since(t0), err) - return err -} - -func (s *metricsInstrumentedStore) MarkDeliveredBatch(ctx context.Context, ids []int64) (int64, error) { - t0 := time.Now() - n, err := s.inner.MarkDeliveredBatch(ctx, ids) - s.m.recordStoreOp(ctx, "mark_delivered_batch", time.Since(t0), err) - return n, err -} - -func (s *metricsInstrumentedStore) PurgeDelivered(ctx context.Context, batchLimit int) (int64, error) { - t0 := time.Now() - n, err := s.inner.PurgeDelivered(ctx, batchLimit) - s.m.recordStoreOp(ctx, "purge_delivered", time.Since(t0), err) - return n, err -} - -func (s *metricsInstrumentedStore) ListPending(ctx context.Context, createdBefore time.Time, limit int) ([]DurableEvent, error) { - t0 := time.Now() - evs, err := s.inner.ListPending(ctx, createdBefore, limit) - s.m.recordStoreOp(ctx, "list_pending", time.Since(t0), err) - return evs, err -} - -func (s *metricsInstrumentedStore) DeleteExpired(ctx context.Context, ttl time.Duration) (int64, error) { - t0 := time.Now() - n, err := s.inner.DeleteExpired(ctx, ttl) - s.m.recordStoreOp(ctx, "delete_expired", time.Since(t0), err) - return n, err -} - -func (s *metricsInstrumentedStore) ObserveDurableQueue(ctx context.Context, eventTTL, nearExpiryLead time.Duration) (DurableQueueStats, error) { - o, ok := s.inner.(DurableQueueObserver) - if !ok { - return DurableQueueStats{}, errors.New("inner DurableEventStore does not implement DurableQueueObserver") - } - return o.ObserveDurableQueue(ctx, eventTTL, nearExpiryLead) -} - -func (s *metricsInstrumentedStore) InsertBatch(ctx context.Context, payloads [][]byte) ([]int64, error) { - bi, ok := s.inner.(BatchInserter) - if !ok { - return nil, errors.New("inner DurableEventStore does not implement BatchInserter") - } - t0 := time.Now() - ids, err := bi.InsertBatch(ctx, payloads) - s.m.recordStoreOp(ctx, "insert_batch", time.Since(t0), err) - return ids, err -} diff --git a/pkg/beholder/durable_emitter_test.go b/pkg/beholder/durable_emitter_test.go index 7846e63031..8a1c14590b 100644 --- a/pkg/beholder/durable_emitter_test.go +++ b/pkg/beholder/durable_emitter_test.go @@ -3,20 +3,26 @@ package beholder import ( "context" "errors" + "net" "sync" "sync/atomic" "testing" "time" + cepb "github.com/cloudevents/sdk-go/binding/format/protobuf/v2/pb" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + sdkmetric "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/metric/metricdata" "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/status" "google.golang.org/protobuf/proto" "github.com/smartcontractkit/chainlink-common/pkg/chipingress" + "github.com/smartcontractkit/chainlink-common/pkg/chipingress/pb" "github.com/smartcontractkit/chainlink-common/pkg/logger" - sdkmetric "go.opentelemetry.io/otel/sdk/metric" - "go.opentelemetry.io/otel/sdk/metric/metricdata" ) // withTestBeholderMeter swaps the global beholder client meter for t's lifetime (for metrics assertions). @@ -40,10 +46,11 @@ func withTestBeholderMeter(t *testing.T) *sdkmetric.ManualReader { type testChipClient struct { chipingress.NoopClient - mu sync.Mutex - publishErr error - publishCount atomic.Int64 - publishedIDs []string + mu sync.Mutex + publishErr error + publishCount atomic.Int64 + batchCount atomic.Int64 + publishedIDs []string } func (c *testChipClient) Publish(_ context.Context, ev *chipingress.CloudEventPb, _ ...grpc.CallOption) (*chipingress.PublishResponse, error) { @@ -57,8 +64,26 @@ func (c *testChipClient) Publish(_ context.Context, ev *chipingress.CloudEventPb return &chipingress.PublishResponse{}, err } -func (c *testChipClient) PublishBatch(_ context.Context, _ *chipingress.CloudEventBatch, _ ...grpc.CallOption) (*chipingress.PublishResponse, error) { - return &chipingress.PublishResponse{}, nil +// PublishBatch mirrors production semantics: respect publishErr and count as a +// separate RPC (batch path / tests that assert Publish only would miss it). +func (c *testChipClient) PublishBatch(_ context.Context, b *chipingress.CloudEventBatch, _ ...grpc.CallOption) (*chipingress.PublishResponse, error) { + c.batchCount.Add(1) + c.mu.Lock() + if b != nil { + for _, ev := range b.Events { + if ev != nil { + c.publishedIDs = append(c.publishedIDs, ev.Id) + } + } + } + err := c.publishErr + c.mu.Unlock() + return &chipingress.PublishResponse{}, err +} + +// totalChipRPCs is unary Publish + PublishBatch for assertions that do not care which path ran. +func (c *testChipClient) totalChipRPCs() int64 { + return c.publishCount.Load() + c.batchCount.Load() } func (c *testChipClient) setPublishErr(err error) { @@ -189,6 +214,7 @@ func TestDurableEmitter_RetransmitLoopDeliversFailedEvents(t *testing.T) { client.setPublishErr(errors.New("connection refused")) cfg := DefaultDurableEmitterConfig() + cfg.PublishBatchSize = 0 // this test keys off unary Publish; batch mode uses PublishBatch cfg.RetransmitInterval = 100 * time.Millisecond cfg.RetransmitAfter = 50 * time.Millisecond @@ -201,7 +227,12 @@ func TestDurableEmitter_RetransmitLoopDeliversFailedEvents(t *testing.T) { err := em.Emit(ctx, []byte("retry-me"), testEmitAttrs()...) require.NoError(t, err) - assert.Equal(t, 1, store.Len()) + + // Wait until the async immediate path has run with the error and the row + // is still pending (not a success race after we clear the error). + require.Eventually(t, func() bool { + return client.publishCount.Load() >= 1 && store.Len() == 1 + }, 2*time.Second, 5*time.Millisecond, "failed immediate publish should leave the row") client.setPublishErr(nil) @@ -209,7 +240,9 @@ func TestDurableEmitter_RetransmitLoopDeliversFailedEvents(t *testing.T) { return store.Len() == 0 }, 5*time.Second, 50*time.Millisecond, "retransmit loop should eventually deliver and delete the event") - assert.GreaterOrEqual(t, client.publishCount.Load(), int64(2)) + // At least: one failed immediate publish + one successful delivery (retransmit + // may be unary Publish or PublishBatch when batching is enabled). + assert.GreaterOrEqual(t, client.totalChipRPCs(), int64(2)) } func TestDurableEmitter_RetransmitSerialDistinctCloudEvents(t *testing.T) { @@ -218,6 +251,7 @@ func TestDurableEmitter_RetransmitSerialDistinctCloudEvents(t *testing.T) { client.setPublishErr(errors.New("immediate fail")) cfg := DefaultDurableEmitterConfig() + cfg.PublishBatchSize = 0 // unary immediate Publish; serial retransmit cfg.RetransmitInterval = 100 * time.Millisecond cfg.RetransmitAfter = 50 * time.Millisecond @@ -231,12 +265,17 @@ func TestDurableEmitter_RetransmitSerialDistinctCloudEvents(t *testing.T) { require.NoError(t, em.Emit(ctx, []byte("first"), testEmitAttrs()...)) require.NoError(t, em.Emit(ctx, []byte("second"), testEmitAttrs()...)) + require.Eventually(t, func() bool { + return client.publishCount.Load() >= 2 && store.Len() == 2 + }, 2*time.Second, 5*time.Millisecond, "both failed immediate publishes should leave two rows") + client.setPublishErr(nil) require.Eventually(t, func() bool { return store.Len() == 0 }, 5*time.Second, 50*time.Millisecond) ids := client.getPublishedIDs() - require.GreaterOrEqual(t, len(ids), 4, "two immediate fails then two retransmit publishes") + require.GreaterOrEqual(t, len(ids), 4, "two failed attempts then two successful deliveries (IDs recorded)") + require.GreaterOrEqual(t, client.totalChipRPCs(), int64(4)) a, b := ids[len(ids)-2], ids[len(ids)-1] assert.NotEmpty(t, a) assert.NotEmpty(t, b) @@ -433,3 +472,375 @@ func TestDurableEmitter_MetricsRegistersEmitSuccess(t *testing.T) { } assert.True(t, found, "expected beholder.durable_emitter.emit.success in exported metrics") } + +// mockChipServer implements ChipIngressServer with controllable behaviour. +type mockChipServer struct { + pb.UnimplementedChipIngressServer + + mu sync.Mutex + publishErr error + batchErr error + received []*cepb.CloudEvent + batchReceived [][]*cepb.CloudEvent + publishCount atomic.Int64 + batchCount atomic.Int64 + publishDelay time.Duration +} + +func (s *mockChipServer) Publish(_ context.Context, in *cepb.CloudEvent) (*pb.PublishResponse, error) { + if s.publishDelay > 0 { + time.Sleep(s.publishDelay) + } + s.publishCount.Add(1) + s.mu.Lock() + defer s.mu.Unlock() + if s.publishErr != nil { + return nil, s.publishErr + } + s.received = append(s.received, in) + return &pb.PublishResponse{}, nil +} + +func (s *mockChipServer) PublishBatch(_ context.Context, in *pb.CloudEventBatch) (*pb.PublishResponse, error) { + s.batchCount.Add(1) + s.mu.Lock() + defer s.mu.Unlock() + if s.batchErr != nil { + return nil, s.batchErr + } + s.batchReceived = append(s.batchReceived, in.Events) + s.received = append(s.received, in.Events...) + return &pb.PublishResponse{}, nil +} + +func (s *mockChipServer) Ping(context.Context, *pb.EmptyRequest) (*pb.PingResponse, error) { + return &pb.PingResponse{Message: "pong"}, nil +} + +func (s *mockChipServer) setPublishErr(err error) { + s.mu.Lock() + defer s.mu.Unlock() + s.publishErr = err +} + +func (s *mockChipServer) setBatchErr(err error) { + s.mu.Lock() + defer s.mu.Unlock() + s.batchErr = err +} + +func (s *mockChipServer) receivedCount() int { + s.mu.Lock() + defer s.mu.Unlock() + return len(s.received) +} + +func (s *mockChipServer) batchCallCount() int { + s.mu.Lock() + defer s.mu.Unlock() + return len(s.batchReceived) +} + +// startMockServer starts a gRPC server on a random port and returns the +// server, address, and a cleanup function. +func startMockServer(t *testing.T, srv *mockChipServer) (*grpc.Server, string) { + t.Helper() + lis, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + + gs := grpc.NewServer() + pb.RegisterChipIngressServer(gs, srv) + + go func() { + if err := gs.Serve(lis); err != nil { + // Ignore errors from server being stopped during cleanup. + } + }() + + t.Cleanup(func() { gs.GracefulStop() }) + return gs, lis.Addr().String() +} + +func newChipClient(t *testing.T, addr string) chipingress.Client { + t.Helper() + c, err := chipingress.NewClient(addr, chipingress.WithInsecureConnection()) + require.NoError(t, err) + t.Cleanup(func() { _ = c.Close() }) + return c +} + +func emitAttrs() []any { + return []any{"source", "test-domain", "type", "test-entity"} +} + +func fastCfg() DurableEmitterConfig { + return DurableEmitterConfig{ + // Retransmit must use unary Publish (not batch enqueue) in these tests. + PublishBatchSize: 0, + RetransmitInterval: 100 * time.Millisecond, + RetransmitAfter: 50 * time.Millisecond, + RetransmitBatchSize: 50, + ExpiryInterval: 200 * time.Millisecond, + EventTTL: 500 * time.Millisecond, + PublishTimeout: 2 * time.Second, + } +} + +// ---------- Test cases ---------- + +func TestIntegration_HappyPath(t *testing.T) { + srv := &mockChipServer{} + _, addr := startMockServer(t, srv) + client := newChipClient(t, addr) + store := NewMemDurableEventStore() + + em, err := NewDurableEmitter(store, client, fastCfg(), logger.Test(t)) + require.NoError(t, err) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + em.Start(ctx) + defer em.Close() + + require.NoError(t, em.Emit(ctx, []byte("billing-record-1"), emitAttrs()...)) + require.NoError(t, em.Emit(ctx, []byte("billing-record-2"), emitAttrs()...)) + + require.Eventually(t, func() bool { + return srv.receivedCount() == 2 + }, 3*time.Second, 10*time.Millisecond, "server should receive both events") + + require.Eventually(t, func() bool { + return store.Len() == 0 + }, 3*time.Second, 10*time.Millisecond, "store should be empty after delivery") +} + +func TestIntegration_ServerUnavailable_RetransmitRecovers(t *testing.T) { + // Start with server returning UNAVAILABLE. + srv := &mockChipServer{} + srv.setPublishErr(status.Error(codes.Unavailable, "chip down")) + _, addr := startMockServer(t, srv) + client := newChipClient(t, addr) + store := NewMemDurableEventStore() + + em, err := NewDurableEmitter(store, client, fastCfg(), logger.Test(t)) + require.NoError(t, err) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + em.Start(ctx) + defer em.Close() + + require.NoError(t, em.Emit(ctx, []byte("will-retry"), emitAttrs()...)) + + require.Eventually(t, func() bool { + return srv.publishCount.Load() >= 1 && store.Len() == 1 + }, 2*time.Second, 10*time.Millisecond, "failed immediate Publish should leave the row pending") + + // "Recover" the server. + srv.setPublishErr(nil) + + require.Eventually(t, func() bool { + return store.Len() == 0 + }, 5*time.Second, 50*time.Millisecond, "retransmit loop should deliver after recovery") + + assert.GreaterOrEqual(t, srv.publishCount.Load(), int64(2), + "one failed immediate Publish then one retransmit Publish") + assert.Equal(t, int64(0), srv.batchCount.Load(), "retransmit should not use PublishBatch") +} + +func TestIntegration_ServerDown_EventsSurvive(t *testing.T) { + // Start server, then stop it to simulate total outage. + srv := &mockChipServer{} + gs, addr := startMockServer(t, srv) + client := newChipClient(t, addr) + store := NewMemDurableEventStore() + + cfg := fastCfg() + cfg.PublishTimeout = 500 * time.Millisecond + em, err := NewDurableEmitter(store, client, cfg, logger.Test(t)) + require.NoError(t, err) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + em.Start(ctx) + + // Stop the gRPC server entirely. + gs.Stop() + time.Sleep(100 * time.Millisecond) + + // Emit while server is down — Emit() itself must succeed (DB insert works). + require.NoError(t, em.Emit(ctx, []byte("server-is-down"), emitAttrs()...)) + assert.Equal(t, 1, store.Len(), "event should be persisted even with server down") + + // Stop the emitter to simulate a "node shutdown". + em.Close() + + // Bring up a new server on the same address. + srv2 := &mockChipServer{} + lis, err := net.Listen("tcp", addr) + require.NoError(t, err) + gs2 := grpc.NewServer() + pb.RegisterChipIngressServer(gs2, srv2) + go func() { _ = gs2.Serve(lis) }() + t.Cleanup(func() { gs2.GracefulStop() }) + + // Create a new client and DurableEmitter re-using the same store + // (simulating node restart with Postgres). + client2, err := chipingress.NewClient(addr, chipingress.WithInsecureConnection()) + require.NoError(t, err) + t.Cleanup(func() { _ = client2.Close() }) + + em2, err := NewDurableEmitter(store, client2, cfg, logger.Test(t)) + require.NoError(t, err) + em2.Start(ctx) + defer em2.Close() + + require.Eventually(t, func() bool { + return srv2.receivedCount() == 1 + }, 5*time.Second, 50*time.Millisecond, "new emitter should retransmit the surviving event") + + require.Eventually(t, func() bool { + return store.Len() == 0 + }, 5*time.Second, 50*time.Millisecond, "store should be empty after retransmit") +} + +func TestIntegration_HighThroughput(t *testing.T) { + srv := &mockChipServer{} + _, addr := startMockServer(t, srv) + client := newChipClient(t, addr) + store := NewMemDurableEventStore() + + cfg := fastCfg() + cfg.RetransmitBatchSize = 200 + em, err := NewDurableEmitter(store, client, cfg, logger.Test(t)) + require.NoError(t, err) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + em.Start(ctx) + defer em.Close() + + const n = 500 + for i := 0; i < n; i++ { + require.NoError(t, em.Emit(ctx, []byte("event"), emitAttrs()...)) + } + + require.Eventually(t, func() bool { + return srv.receivedCount() >= n + }, 10*time.Second, 50*time.Millisecond, "all %d events should be received", n) + + require.Eventually(t, func() bool { + return store.Len() == 0 + }, 10*time.Second, 50*time.Millisecond, "store should drain completely") +} + +func TestIntegration_EventExpiry(t *testing.T) { + // Server always rejects — events can never be delivered. + srv := &mockChipServer{} + srv.setPublishErr(status.Error(codes.Internal, "permanent failure")) + _, addr := startMockServer(t, srv) + client := newChipClient(t, addr) + store := NewMemDurableEventStore() + + cfg := fastCfg() + cfg.EventTTL = 100 * time.Millisecond + cfg.ExpiryInterval = 100 * time.Millisecond + em, err := NewDurableEmitter(store, client, cfg, logger.Test(t)) + require.NoError(t, err) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + em.Start(ctx) + defer em.Close() + + require.NoError(t, em.Emit(ctx, []byte("will-expire"), emitAttrs()...)) + assert.Equal(t, 1, store.Len()) + + require.Eventually(t, func() bool { + return store.Len() == 0 + }, 5*time.Second, 50*time.Millisecond, + "expiry loop should purge undeliverable events after TTL") +} + +func TestIntegration_RetransmitUsesSerialPublish(t *testing.T) { + // Immediate Publish fails; retransmit uses one Publish per queued row. + srv := &mockChipServer{} + srv.setPublishErr(status.Error(codes.Unavailable, "reject immediate")) + _, addr := startMockServer(t, srv) + client := newChipClient(t, addr) + store := NewMemDurableEventStore() + + em, err := NewDurableEmitter(store, client, fastCfg(), logger.Test(t)) + require.NoError(t, err) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + em.Start(ctx) + defer em.Close() + + for i := 0; i < 5; i++ { + require.NoError(t, em.Emit(ctx, []byte("retry-me"), emitAttrs()...)) + } + + // All five async immediate publishes must observe the error before we clear + // it, or they succeed immediately and the retransmit loop has nothing to do. + require.Eventually(t, func() bool { + return srv.publishCount.Load() >= 5 && store.Len() == 5 + }, 3*time.Second, 10*time.Millisecond, "all five immediate Publish RPCs should have failed and left five rows") + + srv.setPublishErr(nil) + + require.Eventually(t, func() bool { + return store.Len() == 0 + }, 5*time.Second, 50*time.Millisecond, + "retransmit should deliver each event with its own Publish RPC") + + assert.Equal(t, 0, srv.batchCallCount(), "retransmit should not call PublishBatch") + assert.GreaterOrEqual(t, srv.publishCount.Load(), int64(10), + "five failed immediate attempts plus five retransmit publishes") +} + +// TestIntegration_GRPCConnection verifies the emitter works over a real gRPC +// connection with proper proto serialization round-trip. +func TestIntegration_GRPCConnection(t *testing.T) { + srv := &mockChipServer{} + _, addr := startMockServer(t, srv) + + // Use a raw gRPC dial to prove we're going over the wire. + conn, err := grpc.NewClient(addr, grpc.WithTransportCredentials(insecure.NewCredentials())) + require.NoError(t, err) + t.Cleanup(func() { _ = conn.Close() }) + + // Ping to verify connectivity. + grpcClient := pb.NewChipIngressClient(conn) + pong, err := grpcClient.Ping(context.Background(), &pb.EmptyRequest{}) + require.NoError(t, err) + assert.Equal(t, "pong", pong.Message) + + // Now use the chipingress.Client wrapper with DurableEmitter. + client := newChipClient(t, addr) + store := NewMemDurableEventStore() + + em, err := NewDurableEmitter(store, client, fastCfg(), logger.Test(t)) + require.NoError(t, err) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + em.Start(ctx) + defer em.Close() + + payload := []byte("proto-round-trip-test") + require.NoError(t, em.Emit(ctx, payload, emitAttrs()...)) + + require.Eventually(t, func() bool { + return srv.receivedCount() == 1 + }, 3*time.Second, 10*time.Millisecond) + + // Verify the CloudEvent arrived with correct source/type. + srv.mu.Lock() + received := srv.received[0] + srv.mu.Unlock() + + assert.Equal(t, "test-domain", received.Source) + assert.Equal(t, "test-entity", received.Type) +} diff --git a/pkg/beholder/durable_event_store.go b/pkg/beholder/durable_event_store.go index 8a9170879b..b00111cd20 100644 --- a/pkg/beholder/durable_event_store.go +++ b/pkg/beholder/durable_event_store.go @@ -2,6 +2,7 @@ package beholder import ( "context" + "errors" "sort" "sync" "sync/atomic" @@ -213,3 +214,87 @@ func (m *MemDurableEventStore) ObserveDurableQueue(_ context.Context, eventTTL, st.OldestPendingAge = now.Sub(oldest) return st, nil } + +// metricsInstrumentedStore wraps DurableEventStore to record store operation metrics. +type metricsInstrumentedStore struct { + inner DurableEventStore + m *durableEmitterMetrics +} + +var _ DurableEventStore = (*metricsInstrumentedStore)(nil) +var _ DurableQueueObserver = (*metricsInstrumentedStore)(nil) + +func newMetricsInstrumentedStore(inner DurableEventStore, m *durableEmitterMetrics) DurableEventStore { + if m == nil { + return inner + } + return &metricsInstrumentedStore{inner: inner, m: m} +} + +func (s *metricsInstrumentedStore) Insert(ctx context.Context, payload []byte) (int64, error) { + t0 := time.Now() + id, err := s.inner.Insert(ctx, payload) + s.m.recordStoreOp(ctx, "insert", time.Since(t0), err) + return id, err +} + +func (s *metricsInstrumentedStore) Delete(ctx context.Context, id int64) error { + t0 := time.Now() + err := s.inner.Delete(ctx, id) + s.m.recordStoreOp(ctx, "delete", time.Since(t0), err) + return err +} + +func (s *metricsInstrumentedStore) MarkDelivered(ctx context.Context, id int64) error { + t0 := time.Now() + err := s.inner.MarkDelivered(ctx, id) + s.m.recordStoreOp(ctx, "mark_delivered", time.Since(t0), err) + return err +} + +func (s *metricsInstrumentedStore) MarkDeliveredBatch(ctx context.Context, ids []int64) (int64, error) { + t0 := time.Now() + n, err := s.inner.MarkDeliveredBatch(ctx, ids) + s.m.recordStoreOp(ctx, "mark_delivered_batch", time.Since(t0), err) + return n, err +} + +func (s *metricsInstrumentedStore) PurgeDelivered(ctx context.Context, batchLimit int) (int64, error) { + t0 := time.Now() + n, err := s.inner.PurgeDelivered(ctx, batchLimit) + s.m.recordStoreOp(ctx, "purge_delivered", time.Since(t0), err) + return n, err +} + +func (s *metricsInstrumentedStore) ListPending(ctx context.Context, createdBefore time.Time, limit int) ([]DurableEvent, error) { + t0 := time.Now() + evs, err := s.inner.ListPending(ctx, createdBefore, limit) + s.m.recordStoreOp(ctx, "list_pending", time.Since(t0), err) + return evs, err +} + +func (s *metricsInstrumentedStore) DeleteExpired(ctx context.Context, ttl time.Duration) (int64, error) { + t0 := time.Now() + n, err := s.inner.DeleteExpired(ctx, ttl) + s.m.recordStoreOp(ctx, "delete_expired", time.Since(t0), err) + return n, err +} + +func (s *metricsInstrumentedStore) ObserveDurableQueue(ctx context.Context, eventTTL, nearExpiryLead time.Duration) (DurableQueueStats, error) { + o, ok := s.inner.(DurableQueueObserver) + if !ok { + return DurableQueueStats{}, errors.New("inner DurableEventStore does not implement DurableQueueObserver") + } + return o.ObserveDurableQueue(ctx, eventTTL, nearExpiryLead) +} + +func (s *metricsInstrumentedStore) InsertBatch(ctx context.Context, payloads [][]byte) ([]int64, error) { + bi, ok := s.inner.(BatchInserter) + if !ok { + return nil, errors.New("inner DurableEventStore does not implement BatchInserter") + } + t0 := time.Now() + ids, err := bi.InsertBatch(ctx, payloads) + s.m.recordStoreOp(ctx, "insert_batch", time.Since(t0), err) + return ids, err +} From 351a495d12ea723b345afba99543db20600e9ddc Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Thu, 30 Apr 2026 11:35:31 -0400 Subject: [PATCH 30/45] Clean up --- pkg/beholder/durable_emitter.go | 9 +-------- pkg/beholder/durable_emitter_integration_test.go | 2 -- 2 files changed, 1 insertion(+), 10 deletions(-) delete mode 100644 pkg/beholder/durable_emitter_integration_test.go diff --git a/pkg/beholder/durable_emitter.go b/pkg/beholder/durable_emitter.go index a19f580d17..a45acb2c4c 100644 --- a/pkg/beholder/durable_emitter.go +++ b/pkg/beholder/durable_emitter.go @@ -669,7 +669,6 @@ func (d *DurableEmitter) publishAndDelete(id int64, eventPb *chipingress.CloudEv defer cancel() detailKVs := cloudEventPublishKVs(id, "immediate", d.cfg.PublishTimeout, eventPb) - //d.log.Infow("DurableEmitter: Chip Ingress publish attempt (immediate)", detailKVs...) t0 := time.Now() _, err := d.client.Publish(ctx, eventPb) @@ -699,12 +698,6 @@ func (d *DurableEmitter) publishAndDelete(id int64, eventPb *chipingress.CloudEv return } - pubOKKVs := append([]any{}, detailKVs...) - pubOKKVs = append(pubOKKVs, - "publish_rpc_elapsed", elapsed.String(), - "publish_rpc_elapsed_ms", elapsed.Milliseconds(), - ) - t1 := time.Now() markErr := d.store.MarkDelivered(context.Background(), id) if h := d.cfg.Hooks; h != nil && h.OnImmediateDelete != nil { @@ -727,7 +720,7 @@ func (d *DurableEmitter) publishAndDelete(id int64, eventPb *chipingress.CloudEv "store_mark_delivered_elapsed", markElapsed.String(), "store_mark_delivered_elapsed_ms", markElapsed.Milliseconds(), ) - //d.log.Infow("DurableEmitter: durable row marked delivered after successful Chip publish (immediate)", delOKKVs...) + d.log.Infow("DurableEmitter: durable row marked delivered after successful Chip publish (immediate)", delOKKVs...) } // batchPublishLoop reads events from publishCh, collects them into batches of diff --git a/pkg/beholder/durable_emitter_integration_test.go b/pkg/beholder/durable_emitter_integration_test.go deleted file mode 100644 index fffcbd62dc..0000000000 --- a/pkg/beholder/durable_emitter_integration_test.go +++ /dev/null @@ -1,2 +0,0 @@ -package beholder_test - From 81c8d1efad66eeba6e64129d4981e0dc92a0b7d9 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Thu, 30 Apr 2026 12:01:04 -0400 Subject: [PATCH 31/45] Clean up --- pkg/beholder/durable_emitter.go | 510 ++++----------------------- pkg/beholder/durable_emitter_test.go | 160 +++------ 2 files changed, 128 insertions(+), 542 deletions(-) diff --git a/pkg/beholder/durable_emitter.go b/pkg/beholder/durable_emitter.go index a45acb2c4c..a38c5837dc 100644 --- a/pkg/beholder/durable_emitter.go +++ b/pkg/beholder/durable_emitter.go @@ -3,13 +3,10 @@ package beholder import ( "context" "fmt" - "slices" - "strings" "sync" "sync/atomic" "time" - cepb "github.com/cloudevents/sdk-go/binding/format/protobuf/v2/pb" "google.golang.org/grpc" grpcEncoding "google.golang.org/grpc/encoding" "google.golang.org/protobuf/encoding/protowire" @@ -25,10 +22,10 @@ type DurableEmitterConfig struct { // RetransmitInterval controls how often the retransmit loop ticks. RetransmitInterval time.Duration // RetransmitAfter is the minimum age of an event before the retransmit - // loop considers it. This gives the immediate-publish path time to succeed. + // loop considers it. This gives the batch publish path time to succeed. RetransmitAfter time.Duration // RetransmitBatchSize caps how many pending rows are listed per retransmit tick - // (each row is sent with its own Publish RPC). + // (each row is enqueued for the batch publish workers). RetransmitBatchSize int // ExpiryInterval controls how often the expiry loop ticks. ExpiryInterval time.Duration @@ -41,22 +38,18 @@ type DurableEmitterConfig struct { PurgeInterval time.Duration // PurgeBatchSize is the maximum rows removed per PurgeDelivered call. Zero defaults to 500. PurgeBatchSize int - // PublishBatchSize enables batched publishing via PublishBatch RPC when > 0. - // Events are collected into batches of this size before a single PublishBatch - // call is made. Zero disables batching (each Emit spawns its own goroutine - // with an individual Publish RPC — the legacy behaviour). + // PublishBatchSize is the target number of events per PublishBatch RPC. Values below 1 are + // clamped to 1 in NewDurableEmitter. PublishBatchSize int // PublishBatchWorkers is the number of concurrent goroutines that read from // the batch channel and call PublishBatch. More workers means higher - // throughput (each worker handles one in-flight batch at a time). Only used - // when PublishBatchSize > 0. Zero defaults to 1. + // throughput (each worker handles one in-flight batch at a time). Zero defaults to 1. PublishBatchWorkers int // PublishBatchFlushInterval is the maximum time to wait for a full batch - // before flushing a partial one. Only used when PublishBatchSize > 0. - // Zero defaults to 50ms. + // before flushing a partial one. Zero defaults to 50ms. PublishBatchFlushInterval time.Duration - // PublishBatchChannelSize overrides the publishCh buffer capacity. Only - // used when PublishBatchSize > 0. Zero defaults to max(PublishBatchSize*2000, 200_000). + // PublishBatchChannelSize overrides the publishCh buffer capacity. Zero + // defaults to max(PublishBatchSize*2000, 200_000). PublishBatchChannelSize int // DisablePruning disables the background purge (PurgeDelivered) and expiry // (DeleteExpired) loops. Events remain in the DB after delivery. Useful for @@ -68,12 +61,6 @@ type DurableEmitterConfig struct { // Metrics enables OpenTelemetry instruments on beholder.GetMeter() (queue, publish, store, optional process stats). // Nil disables. Metrics *DurableEmitterMetricsConfig - // PersistCloudEventSources limits durable persistence to these CloudEvent Source values - // (the beholder_domain / ce_source). If nil, every source is persisted (library default). - // If non-nil, only matching sources are inserted and retried; others get a single best-effort - // Publish with no store insert. An empty slice persists nothing (all best-effort only). - // A one-element slice containing only "*" is treated like nil (persist all). - PersistCloudEventSources []string // InsertBatchSize enables write coalescing when > 0 and the store implements // BatchInserter. Multiple concurrent Emit() calls are grouped into a single // multi-row INSERT, dramatically reducing per-event transaction overhead. @@ -85,11 +72,6 @@ type DurableEmitterConfig struct { // InsertBatchWorkers is the number of concurrent batch-insert goroutines. // Zero defaults to 4. InsertBatchWorkers int - // QuietMode suppresses high-volume INFO-level logs (retransmit scan stats, - // retransmit results, publish failures, expired event purges, etc.). - // Error-level logs are never suppressed. Useful for load tests where the - // logging overhead is measurable. - QuietMode bool } // DurableEmitterHooks records Publish vs Delete latency to locate pipeline bottlenecks. @@ -97,21 +79,18 @@ type DurableEmitterHooks struct { // OnEmitInsert is called after each store.Insert in Emit (the DB write that // blocks the caller). elapsed covers only the INSERT; err is nil on success. OnEmitInsert func(elapsed time.Duration, err error) - // OnImmediatePublish is called after each async Publish in publishAndDelete (every attempt). - // Only fires when PublishBatchSize == 0 (legacy per-event goroutine path). + // OnImmediatePublish is legacy; the durable emitter no longer uses unary Publish for durable emits. OnImmediatePublish func(elapsed time.Duration, err error) - // OnImmediateDelete is called after MarkDelivered following a successful immediate Publish. - // Only fires when PublishBatchSize == 0. + // OnImmediateDelete is unused; retained for API compatibility. OnImmediateDelete func(elapsed time.Duration, err error) // OnBatchPublish is called after each PublishBatch RPC in the batch publish loop. // batchSize is the number of events in the batch; err is nil on success. OnBatchPublish func(elapsed time.Duration, batchSize int, err error) // OnBatchMarkDelivered is called after MarkDeliveredBatch following a successful batch publish. OnBatchMarkDelivered func(elapsed time.Duration, count int) - // OnRetransmitBatchPublish is called after each retransmit Publish (one RPC per queued event). + // OnRetransmitBatchPublish is not invoked (retransmit uses the same batch loop as Emit; use OnBatchPublish). OnRetransmitBatchPublish func(elapsed time.Duration, eventCount int, err error) - // OnRetransmitBatchDeletes is called after a retransmit tick with total time and count of - // successful MarkDelivered calls (mem store may delete rows; Postgres sets delivered_at). + // OnRetransmitBatchDeletes is not invoked (mark delivered is async per batch; hook retained for API compatibility). OnRetransmitBatchDeletes func(elapsed time.Duration, markedDeliveredCount int) } @@ -125,18 +104,18 @@ func DefaultDurableEmitterConfig() DurableEmitterConfig { PublishTimeout: 5 * time.Second, PurgeInterval: 250 * time.Millisecond, PurgeBatchSize: 500, + PublishBatchSize: 1, } } // DurableEmitter implements Emitter with persistence-backed delivery guarantees. // -// On Emit the event is serialized and written to a DurableEventStore. Once the -// insert succeeds Emit returns nil — the caller has a durable guarantee. An -// immediate async Publish is attempted; on success the record is MarkDelivered -// (excluded from retries). Postgres stores then purge physical rows in batches; -// in-memory stores remove the row immediately. If Publish fails, a background -// retransmit loop retries via Publish (one RPC per pending row per tick, up to -// RetransmitBatchSize). +// Emit writes to a DurableEventStore, returns nil after insert, and enqueues the +// row for async PublishBatch delivery. Successful publishes are followed by +// MarkDeliveredBatch; the purge loop removes delivered rows from Postgres. When +// Chip is down or the publish channel is full, a retransmit loop lists stale +// pending rows and re-enqueues them to the same batch workers (up to +// RetransmitBatchSize per tick). // // A separate expiry loop garbage-collects events older than EventTTL to bound // table growth. @@ -164,8 +143,7 @@ type DurableEmitter struct { cfg DurableEmitterConfig log logger.Logger - metrics *durableEmitterMetrics - persistFilter persistSourceFilter + metrics *durableEmitterMetrics // rawConn is the underlying gRPC connection when the client exposes it. // Non-nil enables zero-copy batch publishing (protowire + ForceCodec). @@ -184,8 +162,7 @@ type DurableEmitter struct { pendingCount atomic.Int64 pendingMax atomic.Int64 - // publishCh buffers events for the batch publish loop. Nil when - // PublishBatchSize == 0 (legacy per-goroutine mode). + // publishCh buffers events for the batch publish loop. publishCh chan publishWork stopCh chan struct{} @@ -242,34 +219,6 @@ func buildBatchBytes(payloads [][]byte) []byte { return buf } -// persistSourceFilter decides whether a CloudEvent source may be written to the durable store. -type persistSourceFilter struct { - allowAll bool - allowed map[string]struct{} -} - -func newPersistSourceFilter(sources []string) persistSourceFilter { - if sources == nil { - return persistSourceFilter{allowAll: true} - } - if len(sources) == 1 && strings.TrimSpace(sources[0]) == "*" { - return persistSourceFilter{allowAll: true} - } - m := make(map[string]struct{}, len(sources)) - for _, s := range sources { - m[strings.TrimSpace(s)] = struct{}{} - } - return persistSourceFilter{allowed: m} -} - -func (f persistSourceFilter) allows(source string) bool { - if f.allowAll { - return true - } - _, ok := f.allowed[source] - return ok -} - var _ Emitter = (*DurableEmitter)(nil) func NewDurableEmitter( @@ -287,6 +236,9 @@ func NewDurableEmitter( if log == nil { return nil, fmt.Errorf("logger is nil") } + if cfg.PublishBatchSize < 1 { + cfg.PublishBatchSize = 1 + } var m *durableEmitterMetrics if cfg.Metrics != nil { var err error @@ -297,13 +249,12 @@ func NewDurableEmitter( store = newMetricsInstrumentedStore(store, m) } d := &DurableEmitter{ - store: store, - client: client, - cfg: cfg, - log: log, - metrics: m, - persistFilter: newPersistSourceFilter(cfg.PersistCloudEventSources), - stopCh: make(chan struct{}), + store: store, + client: client, + cfg: cfg, + log: log, + metrics: m, + stopCh: make(chan struct{}), } if cp, ok := client.(grpcConnProvider); ok { d.rawConn = cp.Conn() @@ -323,16 +274,14 @@ func NewDurableEmitter( "insertBatchFlushInterval", cfg.InsertBatchFlushInterval) } } - if cfg.PublishBatchSize > 0 { - bufSize := cfg.PublishBatchChannelSize - if bufSize <= 0 { - bufSize = cfg.PublishBatchSize * 2000 - if bufSize < 200_000 { - bufSize = 200_000 - } + bufSize := cfg.PublishBatchChannelSize + if bufSize <= 0 { + bufSize = cfg.PublishBatchSize * 2000 + if bufSize < 200_000 { + bufSize = 200_000 } - d.publishCh = make(chan publishWork, bufSize) } + d.publishCh = make(chan publishWork, bufSize) return d, nil } @@ -347,9 +296,7 @@ func (d *DurableEmitter) Start(ctx context.Context) { if batchWorkers <= 0 { batchWorkers = 1 } - if d.publishCh != nil { - n += batchWorkers - } + n += batchWorkers insertWorkers := d.cfg.InsertBatchWorkers if insertWorkers <= 0 { insertWorkers = 4 @@ -371,19 +318,16 @@ func (d *DurableEmitter) Start(ctx context.Context) { go d.insertBatchLoop(ctx) } } - if d.publishCh != nil { - for i := 0; i < batchWorkers; i++ { - go d.batchPublishLoop(ctx) - } + for i := 0; i < batchWorkers; i++ { + go d.batchPublishLoop(ctx) } if d.metrics != nil && d.cfg.Metrics != nil { go d.metricsLoop(ctx) } } -// Emit persists the event then attempts async delivery when the CloudEvent source is allowed -// by PersistCloudEventSources; otherwise it performs a single best-effort Publish with no -// persistence. Returns nil once processing is accepted (insert succeeded, or non-persist path started). +// Emit persists the event then enqueues it for async PublishBatch delivery. Returns nil once +// the insert is accepted (or the coalesced insert path completes successfully). func (d *DurableEmitter) Emit(ctx context.Context, body []byte, attrKVs ...any) error { tEmitTotal := time.Now() defer func() { @@ -414,17 +358,6 @@ func (d *DurableEmitter) Emit(ctx context.Context, body []byte, attrKVs ...any) return fmt.Errorf("failed to convert event to proto: %w", err) } - if !d.persistFilter.allows(sourceDomain) { - cl := proto.Clone(eventPb) - evCopy, ok := cl.(*chipingress.CloudEventPb) - if !ok { - emitFail() - return fmt.Errorf("proto.Clone event: got %T, want *chipingress.CloudEventPb", cl) - } - go d.publishBestEffortNoStore(evCopy) - return nil - } - payload, err := proto.Marshal(eventPb) if err != nil { emitFail() @@ -487,80 +420,30 @@ func (d *DurableEmitter) Emit(ctx context.Context, body []byte, attrKVs ...any) d.incPending(1) - if d.publishCh != nil { - // Batch mode: enqueue for batch publish loop. - // Only carry the struct when needed for the typed fallback path; - // the raw path uses payload bytes directly. - work := publishWork{id: id, payload: payload} - if d.rawConn == nil { - work.event = eventPb - } - select { - case d.publishCh <- work: - default: - // Channel full — event is safely in the DB; retransmit loop will deliver it. - if !d.cfg.QuietMode { - d.log.Warnw("DurableEmitter: batch publish channel full, relying on retransmit", - "id", id, "ch_len", len(d.publishCh), "ch_cap", cap(d.publishCh)) - } - } - } else { - // Legacy mode: fire-and-forget immediate delivery attempt. - go d.publishAndDelete(id, eventPb) + // Batch path: enqueue for batch publish loop (PublishBatchSize is always >= 1). + work := publishWork{id: id, payload: payload} + if d.rawConn == nil { + work.event = eventPb + } + select { + case d.publishCh <- work: + default: + // Channel full — event is safely in the DB; retransmit loop will deliver it. + d.log.Warnw("DurableEmitter: batch publish channel full, relying on retransmit", + "id", id, "ch_len", len(d.publishCh), "ch_cap", cap(d.publishCh)) } return nil } -// publishBestEffortNoStore performs one Publish without persisting or retries. -func (d *DurableEmitter) publishBestEffortNoStore(eventPb *chipingress.CloudEventPb) { - ctx, cancel := context.WithTimeout(context.Background(), d.cfg.PublishTimeout) - defer cancel() - - detailKVs := cloudEventPublishKVs(0, "best_effort_no_store", d.cfg.PublishTimeout, eventPb) - //d.log.Infow("DurableEmitter: Chip Ingress publish attempt (best-effort, not persisted)", detailKVs...) - - t0 := time.Now() - _, err := d.client.Publish(ctx, eventPb) - elapsed := time.Since(t0) - if h := d.cfg.Hooks; h != nil && h.OnImmediatePublish != nil { - h.OnImmediatePublish(elapsed, err) - } - mctx := context.Background() - d.metrics.recordPublish(mctx, elapsed, "best_effort", err) - if d.metrics != nil { - if err != nil { - d.metrics.publishImmErr.Add(mctx, 1) - } else { - d.metrics.publishImmOK.Add(mctx, 1) - } - } - if err != nil { - failKVs := append([]any{}, detailKVs...) - failKVs = append(failKVs, - "error", err, - "elapsed", elapsed.String(), - "elapsed_ms", elapsed.Milliseconds(), - ) - //d.log.Infow("DurableEmitter: best-effort Chip publish failed (not persisted, no retry)", failKVs...) - return - } - okKVs := append([]any{}, detailKVs...) - okKVs = append(okKVs, "publish_rpc_elapsed_ms", elapsed.Milliseconds()) - //d.log.Infow("DurableEmitter: best-effort Chip publish succeeded (not persisted)", okKVs...) -} - // Close signals background loops to stop and waits for them to finish. -// When batch publishing is enabled the channel is closed so the batch loop -// can drain remaining events before returning. +// Insert and publish channels are closed so workers can drain. func (d *DurableEmitter) Close() error { close(d.stopCh) if d.insertCh != nil { close(d.insertCh) } - if d.publishCh != nil { - close(d.publishCh) - } + close(d.publishCh) d.wg.Wait() d.markWg.Wait() return nil @@ -663,66 +546,6 @@ func (d *DurableEmitter) decPending(n int64) { } } -// publishAndDelete attempts a single Publish and deletes the record on success. -func (d *DurableEmitter) publishAndDelete(id int64, eventPb *chipingress.CloudEventPb) { - ctx, cancel := context.WithTimeout(context.Background(), d.cfg.PublishTimeout) - defer cancel() - - detailKVs := cloudEventPublishKVs(id, "immediate", d.cfg.PublishTimeout, eventPb) - - t0 := time.Now() - _, err := d.client.Publish(ctx, eventPb) - elapsed := time.Since(t0) - if h := d.cfg.Hooks; h != nil && h.OnImmediatePublish != nil { - h.OnImmediatePublish(elapsed, err) - } - mctx := context.Background() - d.metrics.recordPublish(mctx, elapsed, "immediate", err) - if d.metrics != nil { - if err != nil { - d.metrics.publishImmErr.Add(mctx, 1) - } else { - d.metrics.publishImmOK.Add(mctx, 1) - } - } - if err != nil { - failKVs := append([]any{}, detailKVs...) - failKVs = append(failKVs, - "error", err, - "elapsed", elapsed.String(), - "elapsed_ms", elapsed.Milliseconds(), - ) - if !d.cfg.QuietMode { - d.log.Infow("DurableEmitter: Chip Ingress publish failed (immediate), retransmit loop will retry", failKVs...) - } - return - } - - t1 := time.Now() - markErr := d.store.MarkDelivered(context.Background(), id) - if h := d.cfg.Hooks; h != nil && h.OnImmediateDelete != nil { - h.OnImmediateDelete(time.Since(t1), markErr) - } - if markErr == nil { - d.decPending(1) - if d.metrics != nil { - d.metrics.deliverComplete.Add(mctx, 1) - } - } - markElapsed := time.Since(t1) - if markErr != nil { - d.log.Errorw("failed to mark delivered event", "id", id, "error", markErr) - return - } - delOKKVs := append([]any{}, detailKVs...) - delOKKVs = append(delOKKVs, - "publish_rpc_elapsed_ms", elapsed.Milliseconds(), - "store_mark_delivered_elapsed", markElapsed.String(), - "store_mark_delivered_elapsed_ms", markElapsed.Milliseconds(), - ) - d.log.Infow("DurableEmitter: durable row marked delivered after successful Chip publish (immediate)", delOKKVs...) -} - // batchPublishLoop reads events from publishCh, collects them into batches of // PublishBatchSize, and sends each batch via PublishBatch RPC. It blocks until // the batch is full or PublishBatchFlushInterval elapses after the first event @@ -931,7 +754,7 @@ func (d *DurableEmitter) retransmitPending(ctx context.Context) { st, obsErr := obs.ObserveDurableQueue(ctx, d.cfg.EventTTL, d.queueStatsNearExpiryLead()) if obsErr != nil { d.log.Warnw("DurableEmitter: retransmit scan ObserveDurableQueue failed", "error", obsErr) - } else if !d.cfg.QuietMode { + } else { d.log.Infow("DurableEmitter: retransmit pending scan", "pending_rows", st.Depth, "pending_payload_bytes", st.PayloadBytes, @@ -948,157 +771,39 @@ func (d *DurableEmitter) retransmitPending(ctx context.Context) { return } - if d.publishCh != nil { - d.retransmitViaBatchWorkers(ctx, pending) - } else { - d.retransmitSerialFromPending(ctx, pending) - } + d.retransmit(pending) } -// retransmitViaBatchWorkers enqueues pending DB rows to publishCh so the -// existing batch workers handle publishing. When rawConn is set and the -// persist filter accepts all sources, payloads are passed through without -// any proto.Unmarshal — the batch workers will use buildBatchBytes to -// construct the wire format directly. -func (d *DurableEmitter) retransmitViaBatchWorkers(ctx context.Context, pending []DurableEvent) { +// retransmit enqueues pending DB rows to publishCh so the batch workers handle +// publishing. When rawConn is set, payloads are passed through without +// proto.Unmarshal — the batch workers use buildBatchBytes for the wire format. +func (d *DurableEmitter) retransmit(pending []DurableEvent) { var enqueued int - needsFilter := !d.persistFilter.allowAll for _, pe := range pending { - var work publishWork - work.id = pe.ID - work.payload = pe.Payload - - if needsFilter { - ev := new(chipingress.CloudEventPb) - if err := proto.Unmarshal(pe.Payload, ev); err != nil { - d.log.Errorw("corrupt pending event, deleting", "id", pe.ID, "error", err) - if delErr := d.store.Delete(ctx, pe.ID); delErr == nil { - d.decPending(1) - } - continue - } - if !d.persistFilter.allows(ev.GetSource()) { - if !d.cfg.QuietMode { - d.log.Infow("DurableEmitter: dropping queued event (ce_source not in PersistCloudEventSources)", - "id", pe.ID, "ce_source", ev.GetSource(), "ce_type", ev.GetType()) - } - if delErr := d.store.Delete(ctx, pe.ID); delErr == nil { - d.decPending(1) - } - continue - } - work.event = ev + select { + case <-d.stopCh: + return + default: } + work := publishWork{id: pe.ID, payload: pe.Payload} select { + case <-d.stopCh: + return case d.publishCh <- work: enqueued++ default: } } - if !d.cfg.QuietMode { - d.log.Infow("DurableEmitter: retransmit enqueued to batch workers", - "enqueued", enqueued, - "skipped_ch_full", len(pending)-enqueued, - "total_pending", len(pending), - "ch_len", len(d.publishCh), - "ch_cap", cap(d.publishCh), - ) - } -} - -// retransmitSerialFromPending unmarshals events and publishes them one at a -// time. Used in legacy mode (PublishBatchSize == 0). -func (d *DurableEmitter) retransmitSerialFromPending(ctx context.Context, pending []DurableEvent) { - events := make([]*chipingress.CloudEventPb, 0, len(pending)) - ids := make([]int64, 0, len(pending)) - - for _, pe := range pending { - ev := new(chipingress.CloudEventPb) - if err := proto.Unmarshal(pe.Payload, ev); err != nil { - d.log.Errorw("corrupt pending event, deleting", "id", pe.ID, "error", err) - if delErr := d.store.Delete(ctx, pe.ID); delErr == nil { - d.decPending(1) - } - continue - } - if !d.persistFilter.allows(ev.GetSource()) { - if !d.cfg.QuietMode { - d.log.Infow("DurableEmitter: dropping queued event (ce_source not in PersistCloudEventSources)", - "id", pe.ID, "ce_source", ev.GetSource(), "ce_type", ev.GetType()) - } - if delErr := d.store.Delete(ctx, pe.ID); delErr == nil { - d.decPending(1) - } - continue - } - events = append(events, ev) - ids = append(ids, pe.ID) - } - - if len(events) > 0 { - d.retransmitSerial(ctx, events, ids) - } -} - -// retransmitSerial publishes pending events one at a time via individual -// Publish RPCs. Used in legacy mode (PublishBatchSize == 0). -func (d *DurableEmitter) retransmitSerial(ctx context.Context, events []*chipingress.CloudEventPb, ids []int64) { - tDel := time.Now() - var markedDelivered int - for i := range events { - detailKVs := cloudEventPublishKVs(ids[i], "retransmit", d.cfg.PublishTimeout, events[i]) - - tPub := time.Now() - pubCtx, cancel := context.WithTimeout(context.Background(), d.cfg.PublishTimeout) - _, pubErr := d.client.Publish(pubCtx, events[i]) - cancel() - elapsed := time.Since(tPub) - if h := d.cfg.Hooks; h != nil && h.OnRetransmitBatchPublish != nil { - h.OnRetransmitBatchPublish(elapsed, 1, pubErr) - } - d.metrics.recordPublish(context.Background(), elapsed, "retransmit", pubErr) - if pubErr != nil { - if d.metrics != nil { - d.metrics.publishBatchEvErr.Add(ctx, 1) - } - failKVs := append([]any{}, detailKVs...) - failKVs = append(failKVs, - "error", pubErr, - "elapsed", elapsed.String(), - "elapsed_ms", elapsed.Milliseconds(), - ) - if !d.cfg.QuietMode { - d.log.Infow("DurableEmitter: Chip Ingress publish failed (retransmit)", failKVs...) - } - continue - } - if d.metrics != nil { - d.metrics.publishBatchEvOK.Add(ctx, 1) - } - tMarkOne := time.Now() - if markErr := d.store.MarkDelivered(ctx, ids[i]); markErr != nil { - d.log.Errorw("failed to mark retransmitted event delivered", "id", ids[i], "error", markErr) - continue - } - d.decPending(1) - markedDelivered++ - if d.metrics != nil { - d.metrics.deliverComplete.Add(ctx, 1) - } - _ = time.Since(tMarkOne) - } - if markedDelivered > 0 && !d.cfg.QuietMode { - d.log.Infow("retransmitted events", - "marked_delivered", markedDelivered, - "attempted", len(events), - ) - } - if h := d.cfg.Hooks; h != nil && h.OnRetransmitBatchDeletes != nil && markedDelivered > 0 { - h.OnRetransmitBatchDeletes(time.Since(tDel), markedDelivered) - } + d.log.Infow("DurableEmitter: retransmit enqueued to batch workers", + "enqueued", enqueued, + "skipped_ch_full", len(pending)-enqueued, + "total_pending", len(pending), + "ch_len", len(d.publishCh), + "ch_cap", cap(d.publishCh), + ) } func (d *DurableEmitter) purgeLoop(ctx context.Context) { @@ -1156,9 +861,7 @@ func (d *DurableEmitter) expiryLoop(ctx context.Context) { if d.metrics != nil { d.metrics.expiredPurged.Add(context.Background(), deleted) } - if !d.cfg.QuietMode { - d.log.Infow("purged expired events", "count", deleted) - } + d.log.Infow("purged expired events", "count", deleted) } } } @@ -1204,66 +907,3 @@ func (d *DurableEmitter) metricsLoop(ctx context.Context) { } } } - -// cloudEventPublishKVs returns structured fields for logging a Chip Ingress Publish RPC. -func cloudEventPublishKVs(durableRowID int64, phase string, timeout time.Duration, ev *chipingress.CloudEventPb) []any { - if ev == nil { - return []any{ - "durable_row_id", durableRowID, - "publish_phase", phase, - "publish_timeout", timeout.String(), - "ce_nil", true, - } - } - - attrs := ev.GetAttributes() - bin := ev.GetBinaryData() - text := ev.GetTextData() - pd := ev.GetProtoData() - var protoTypeURL string - if pd != nil { - protoTypeURL = pd.GetTypeUrl() - } - - attrKeys := make([]string, 0, len(attrs)) - for k := range attrs { - attrKeys = append(attrKeys, k) - } - slices.Sort(attrKeys) - - kvs := []any{ - "durable_row_id", durableRowID, - "publish_phase", phase, - "publish_timeout", timeout.String(), - "ce_id", ev.GetId(), - "ce_source", ev.GetSource(), - "ce_type", ev.GetType(), - "ce_spec_version", ev.GetSpecVersion(), - "ce_data_binary_bytes", len(bin), - "ce_data_text_bytes", len(text), - "ce_proto_data_type_url", protoTypeURL, - "ce_attribute_count", len(attrs), - "ce_attribute_keys", strings.Join(attrKeys, ","), - "ce_attr_datacontenttype", cloudEventAttrString(attrs, "datacontenttype"), - "ce_attr_dataschema", cloudEventAttrString(attrs, "dataschema"), - "ce_attr_subject", cloudEventAttrString(attrs, "subject"), - } - return kvs -} - -func cloudEventAttrString(attrs map[string]*cepb.CloudEventAttributeValue, key string) string { - if attrs == nil { - return "" - } - v := attrs[key] - if v == nil { - return "" - } - if s := v.GetCeString(); s != "" { - return s - } - if s := v.GetCeUri(); s != "" { - return s - } - return "" -} diff --git a/pkg/beholder/durable_emitter_test.go b/pkg/beholder/durable_emitter_test.go index 8a1c14590b..ee51fd8b23 100644 --- a/pkg/beholder/durable_emitter_test.go +++ b/pkg/beholder/durable_emitter_test.go @@ -115,14 +115,14 @@ func newTestDurableEmitter(t *testing.T, store DurableEventStore, client chiping return em } -func TestDurableEmitter_HooksImmediatePath(t *testing.T) { +func TestDurableEmitter_HooksBatchPublishPath(t *testing.T) { store := NewMemDurableEventStore() client := &testChipClient{} - var pubCalls, delCalls atomic.Int32 + var pubCalls, markCalls atomic.Int32 cfg := DefaultDurableEmitterConfig() cfg.Hooks = &DurableEmitterHooks{ - OnImmediatePublish: func(time.Duration, error) { pubCalls.Add(1) }, - OnImmediateDelete: func(time.Duration, error) { delCalls.Add(1) }, + OnBatchPublish: func(time.Duration, int, error) { pubCalls.Add(1) }, + OnBatchMarkDelivered: func(time.Duration, int) { markCalls.Add(1) }, } em, err := NewDurableEmitter(store, client, cfg, logger.Test(t)) require.NoError(t, err) @@ -135,18 +135,18 @@ func TestDurableEmitter_HooksImmediatePath(t *testing.T) { require.NoError(t, em.Emit(ctx, []byte("hello"), testEmitAttrs()...)) require.Eventually(t, func() bool { return store.Len() == 0 }, 2*time.Second, 10*time.Millisecond) assert.Equal(t, int32(1), pubCalls.Load()) - assert.Equal(t, int32(1), delCalls.Load()) + require.Eventually(t, func() bool { return markCalls.Load() == 1 }, 2*time.Second, 10*time.Millisecond) } -func TestDurableEmitter_HooksPublishFailureSkipsDeleteHook(t *testing.T) { +func TestDurableEmitter_HooksPublishFailureSkipsMarkHook(t *testing.T) { store := NewMemDurableEventStore() client := &testChipClient{} client.setPublishErr(errors.New("down")) - var pubCalls, delCalls atomic.Int32 + var pubCalls, markCalls atomic.Int32 cfg := DefaultDurableEmitterConfig() cfg.Hooks = &DurableEmitterHooks{ - OnImmediatePublish: func(time.Duration, error) { pubCalls.Add(1) }, - OnImmediateDelete: func(time.Duration, error) { delCalls.Add(1) }, + OnBatchPublish: func(time.Duration, int, error) { pubCalls.Add(1) }, + OnBatchMarkDelivered: func(time.Duration, int) { markCalls.Add(1) }, } em, err := NewDurableEmitter(store, client, cfg, logger.Test(t)) require.NoError(t, err) @@ -158,7 +158,7 @@ func TestDurableEmitter_HooksPublishFailureSkipsDeleteHook(t *testing.T) { require.NoError(t, em.Emit(ctx, []byte("hello"), testEmitAttrs()...)) require.Eventually(t, func() bool { return pubCalls.Load() == 1 }, 2*time.Second, 10*time.Millisecond) - assert.Equal(t, int32(0), delCalls.Load()) + assert.Equal(t, int32(0), markCalls.Load()) } func TestDurableEmitter_EmitPersistsAndPublishes(t *testing.T) { @@ -174,9 +174,9 @@ func TestDurableEmitter_EmitPersistsAndPublishes(t *testing.T) { err := em.Emit(ctx, []byte("hello"), testEmitAttrs()...) require.NoError(t, err) - // Immediate async publish should fire and delete the record. + // Batch publish should fire (PublishBatch with batch size 1) and delete the record. require.Eventually(t, func() bool { - return client.publishCount.Load() == 1 + return client.batchCount.Load() == 1 }, 2*time.Second, 10*time.Millisecond) require.Eventually(t, func() bool { @@ -199,9 +199,9 @@ func TestDurableEmitter_EmitReturnSuccessEvenWhenPublishFails(t *testing.T) { err := em.Emit(ctx, []byte("hello"), testEmitAttrs()...) require.NoError(t, err, "Emit must succeed once the DB insert succeeds") - // Wait for the async publish attempt to complete. + // Wait for the async PublishBatch attempt to complete. require.Eventually(t, func() bool { - return client.publishCount.Load() == 1 + return client.batchCount.Load() == 1 }, 2*time.Second, 10*time.Millisecond) // Event must remain in the store for retransmit. @@ -214,7 +214,6 @@ func TestDurableEmitter_RetransmitLoopDeliversFailedEvents(t *testing.T) { client.setPublishErr(errors.New("connection refused")) cfg := DefaultDurableEmitterConfig() - cfg.PublishBatchSize = 0 // this test keys off unary Publish; batch mode uses PublishBatch cfg.RetransmitInterval = 100 * time.Millisecond cfg.RetransmitAfter = 50 * time.Millisecond @@ -231,8 +230,8 @@ func TestDurableEmitter_RetransmitLoopDeliversFailedEvents(t *testing.T) { // Wait until the async immediate path has run with the error and the row // is still pending (not a success race after we clear the error). require.Eventually(t, func() bool { - return client.publishCount.Load() >= 1 && store.Len() == 1 - }, 2*time.Second, 5*time.Millisecond, "failed immediate publish should leave the row") + return client.batchCount.Load() >= 1 && store.Len() == 1 + }, 2*time.Second, 5*time.Millisecond, "failed PublishBatch should leave the row") client.setPublishErr(nil) @@ -240,9 +239,8 @@ func TestDurableEmitter_RetransmitLoopDeliversFailedEvents(t *testing.T) { return store.Len() == 0 }, 5*time.Second, 50*time.Millisecond, "retransmit loop should eventually deliver and delete the event") - // At least: one failed immediate publish + one successful delivery (retransmit - // may be unary Publish or PublishBatch when batching is enabled). - assert.GreaterOrEqual(t, client.totalChipRPCs(), int64(2)) + // At least: one failed PublishBatch + one successful delivery (retransmit uses PublishBatch too). + assert.GreaterOrEqual(t, client.batchCount.Load(), int64(2)) } func TestDurableEmitter_RetransmitSerialDistinctCloudEvents(t *testing.T) { @@ -251,7 +249,6 @@ func TestDurableEmitter_RetransmitSerialDistinctCloudEvents(t *testing.T) { client.setPublishErr(errors.New("immediate fail")) cfg := DefaultDurableEmitterConfig() - cfg.PublishBatchSize = 0 // unary immediate Publish; serial retransmit cfg.RetransmitInterval = 100 * time.Millisecond cfg.RetransmitAfter = 50 * time.Millisecond @@ -266,8 +263,8 @@ func TestDurableEmitter_RetransmitSerialDistinctCloudEvents(t *testing.T) { require.NoError(t, em.Emit(ctx, []byte("second"), testEmitAttrs()...)) require.Eventually(t, func() bool { - return client.publishCount.Load() >= 2 && store.Len() == 2 - }, 2*time.Second, 5*time.Millisecond, "both failed immediate publishes should leave two rows") + return client.batchCount.Load() >= 2 && store.Len() == 2 + }, 2*time.Second, 5*time.Millisecond, "both failed PublishBatch calls should leave two rows") client.setPublishErr(nil) @@ -275,7 +272,7 @@ func TestDurableEmitter_RetransmitSerialDistinctCloudEvents(t *testing.T) { ids := client.getPublishedIDs() require.GreaterOrEqual(t, len(ids), 4, "two failed attempts then two successful deliveries (IDs recorded)") - require.GreaterOrEqual(t, client.totalChipRPCs(), int64(4)) + require.GreaterOrEqual(t, client.batchCount.Load(), int64(4)) a, b := ids[len(ids)-2], ids[len(ids)-1] assert.NotEmpty(t, a) assert.NotEmpty(t, b) @@ -308,57 +305,7 @@ func TestDurableEmitter_ExpiryLoopDeletesOldEvents(t *testing.T) { }, 5*time.Second, 50*time.Millisecond, "expiry loop should purge the event") } -func TestDurableEmitter_PersistSourceFilter_skipsStoreBestEffortPublish(t *testing.T) { - store := NewMemDurableEventStore() - client := &testChipClient{} - cfg := DefaultDurableEmitterConfig() - cfg.PersistCloudEventSources = []string{"only-this"} - em := newTestDurableEmitter(t, store, client, &cfg) - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - em.Start(ctx) - defer em.Close() - - require.NoError(t, em.Emit(ctx, []byte("x"), testEmitAttrs()...)) - require.Eventually(t, func() bool { return client.publishCount.Load() == 1 }, 2*time.Second, 10*time.Millisecond) - assert.Equal(t, 0, store.Len()) -} - -func TestDurableEmitter_PersistSourceFilter_persistsAllowedSource(t *testing.T) { - store := NewMemDurableEventStore() - client := &testChipClient{} - cfg := DefaultDurableEmitterConfig() - cfg.PersistCloudEventSources = []string{"test-source"} - em := newTestDurableEmitter(t, store, client, &cfg) - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - em.Start(ctx) - defer em.Close() - - require.NoError(t, em.Emit(ctx, []byte("x"), testEmitAttrs()...)) - require.Eventually(t, func() bool { return client.publishCount.Load() == 1 }, 2*time.Second, 10*time.Millisecond) - require.Eventually(t, func() bool { return store.Len() == 0 }, 2*time.Second, 10*time.Millisecond) -} - -func TestDurableEmitter_PersistSourceWildcardStarAllowsAll(t *testing.T) { - store := NewMemDurableEventStore() - client := &testChipClient{} - cfg := DefaultDurableEmitterConfig() - cfg.PersistCloudEventSources = []string{"*"} - em := newTestDurableEmitter(t, store, client, &cfg) - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - em.Start(ctx) - defer em.Close() - - require.NoError(t, em.Emit(ctx, []byte("x"), testEmitAttrs()...)) - require.Eventually(t, func() bool { return store.Len() == 0 }, 2*time.Second, 10*time.Millisecond) -} - -func TestDurableEmitter_RetransmitDropsDisallowedSource(t *testing.T) { +func TestDurableEmitter_RetransmitDeliversManuallyInsertedRow(t *testing.T) { store := NewMemDurableEventStore() client := &testChipClient{} @@ -373,7 +320,6 @@ func TestDurableEmitter_RetransmitDropsDisallowedSource(t *testing.T) { require.NoError(t, err) cfg := DefaultDurableEmitterConfig() - cfg.PersistCloudEventSources = []string{"test-source"} cfg.RetransmitInterval = 50 * time.Millisecond cfg.RetransmitAfter = 30 * time.Millisecond @@ -385,8 +331,8 @@ func TestDurableEmitter_RetransmitDropsDisallowedSource(t *testing.T) { defer em.Close() require.Eventually(t, func() bool { - return store.Len() == 0 && client.publishCount.Load() == 0 - }, 3*time.Second, 20*time.Millisecond, "disallowed row should be deleted without Publish") + return store.Len() == 0 && client.batchCount.Load() >= 1 + }, 3*time.Second, 20*time.Millisecond, "pending row should be delivered via PublishBatch") } func TestDurableEmitter_EmitRejectsInvalidAttributes(t *testing.T) { @@ -416,7 +362,7 @@ func TestDurableEmitter_MultipleEvents(t *testing.T) { } require.Eventually(t, func() bool { - return client.publishCount.Load() == int64(n) + return client.batchCount.Load() == int64(n) }, 5*time.Second, 10*time.Millisecond) require.Eventually(t, func() bool { @@ -575,14 +521,14 @@ func emitAttrs() []any { func fastCfg() DurableEmitterConfig { return DurableEmitterConfig{ - // Retransmit must use unary Publish (not batch enqueue) in these tests. - PublishBatchSize: 0, - RetransmitInterval: 100 * time.Millisecond, - RetransmitAfter: 50 * time.Millisecond, - RetransmitBatchSize: 50, - ExpiryInterval: 200 * time.Millisecond, - EventTTL: 500 * time.Millisecond, - PublishTimeout: 2 * time.Second, + PublishBatchSize: 1, + PublishBatchFlushInterval: 10 * time.Millisecond, + RetransmitInterval: 100 * time.Millisecond, + RetransmitAfter: 50 * time.Millisecond, + RetransmitBatchSize: 50, + ExpiryInterval: 200 * time.Millisecond, + EventTTL: 500 * time.Millisecond, + PublishTimeout: 2 * time.Second, } } @@ -615,9 +561,9 @@ func TestIntegration_HappyPath(t *testing.T) { } func TestIntegration_ServerUnavailable_RetransmitRecovers(t *testing.T) { - // Start with server returning UNAVAILABLE. + // Start with server returning UNAVAILABLE on PublishBatch. srv := &mockChipServer{} - srv.setPublishErr(status.Error(codes.Unavailable, "chip down")) + srv.setBatchErr(status.Error(codes.Unavailable, "chip down")) _, addr := startMockServer(t, srv) client := newChipClient(t, addr) store := NewMemDurableEventStore() @@ -633,19 +579,19 @@ func TestIntegration_ServerUnavailable_RetransmitRecovers(t *testing.T) { require.NoError(t, em.Emit(ctx, []byte("will-retry"), emitAttrs()...)) require.Eventually(t, func() bool { - return srv.publishCount.Load() >= 1 && store.Len() == 1 - }, 2*time.Second, 10*time.Millisecond, "failed immediate Publish should leave the row pending") + return srv.batchCount.Load() >= 1 && store.Len() == 1 + }, 2*time.Second, 10*time.Millisecond, "failed PublishBatch should leave the row pending") // "Recover" the server. - srv.setPublishErr(nil) + srv.setBatchErr(nil) require.Eventually(t, func() bool { return store.Len() == 0 }, 5*time.Second, 50*time.Millisecond, "retransmit loop should deliver after recovery") - assert.GreaterOrEqual(t, srv.publishCount.Load(), int64(2), - "one failed immediate Publish then one retransmit Publish") - assert.Equal(t, int64(0), srv.batchCount.Load(), "retransmit should not use PublishBatch") + assert.GreaterOrEqual(t, srv.batchCount.Load(), int64(2), + "one failed PublishBatch then at least one successful PublishBatch (retransmit)") + assert.Equal(t, int64(0), srv.publishCount.Load(), "unary Publish should not be used for durable path") } func TestIntegration_ServerDown_EventsSurvive(t *testing.T) { @@ -737,7 +683,7 @@ func TestIntegration_HighThroughput(t *testing.T) { func TestIntegration_EventExpiry(t *testing.T) { // Server always rejects — events can never be delivered. srv := &mockChipServer{} - srv.setPublishErr(status.Error(codes.Internal, "permanent failure")) + srv.setBatchErr(status.Error(codes.Internal, "permanent failure")) _, addr := startMockServer(t, srv) client := newChipClient(t, addr) store := NewMemDurableEventStore() @@ -762,10 +708,10 @@ func TestIntegration_EventExpiry(t *testing.T) { "expiry loop should purge undeliverable events after TTL") } -func TestIntegration_RetransmitUsesSerialPublish(t *testing.T) { - // Immediate Publish fails; retransmit uses one Publish per queued row. +func TestIntegration_RetransmitEnqueuesBatchWorkers(t *testing.T) { + // PublishBatch fails for each emit; retransmit recovers via PublishBatch. srv := &mockChipServer{} - srv.setPublishErr(status.Error(codes.Unavailable, "reject immediate")) + srv.setBatchErr(status.Error(codes.Unavailable, "reject batch")) _, addr := startMockServer(t, srv) client := newChipClient(t, addr) store := NewMemDurableEventStore() @@ -782,22 +728,22 @@ func TestIntegration_RetransmitUsesSerialPublish(t *testing.T) { require.NoError(t, em.Emit(ctx, []byte("retry-me"), emitAttrs()...)) } - // All five async immediate publishes must observe the error before we clear + // All five async PublishBatch attempts must observe the error before we clear // it, or they succeed immediately and the retransmit loop has nothing to do. require.Eventually(t, func() bool { - return srv.publishCount.Load() >= 5 && store.Len() == 5 - }, 3*time.Second, 10*time.Millisecond, "all five immediate Publish RPCs should have failed and left five rows") + return srv.batchCount.Load() >= 5 && store.Len() == 5 + }, 3*time.Second, 10*time.Millisecond, "all five PublishBatch RPCs should have failed and left five rows") - srv.setPublishErr(nil) + srv.setBatchErr(nil) require.Eventually(t, func() bool { return store.Len() == 0 }, 5*time.Second, 50*time.Millisecond, - "retransmit should deliver each event with its own Publish RPC") + "retransmit should deliver via PublishBatch") - assert.Equal(t, 0, srv.batchCallCount(), "retransmit should not call PublishBatch") - assert.GreaterOrEqual(t, srv.publishCount.Load(), int64(10), - "five failed immediate attempts plus five retransmit publishes") + assert.GreaterOrEqual(t, srv.batchCount.Load(), int64(10), + "five failed PublishBatch plus five successful PublishBatch (retransmit)") + assert.Equal(t, int64(0), srv.publishCount.Load(), "no unary Publish on durable path") } // TestIntegration_GRPCConnection verifies the emitter works over a real gRPC From f501ebc046b9362fa6694f6cc54d0625b162470b Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Thu, 30 Apr 2026 12:05:18 -0400 Subject: [PATCH 32/45] Remove process stats (only used in local testing) --- pkg/beholder/durable_emitter.go | 4 ---- pkg/beholder/durable_emitter_cpu_other.go | 9 --------- pkg/beholder/durable_emitter_cpu_unix.go | 22 ---------------------- pkg/beholder/durable_emitter_metrics.go | 13 ------------- 4 files changed, 48 deletions(-) delete mode 100644 pkg/beholder/durable_emitter_cpu_other.go delete mode 100644 pkg/beholder/durable_emitter_cpu_unix.go diff --git a/pkg/beholder/durable_emitter.go b/pkg/beholder/durable_emitter.go index a38c5837dc..5cf3ab1687 100644 --- a/pkg/beholder/durable_emitter.go +++ b/pkg/beholder/durable_emitter.go @@ -900,10 +900,6 @@ func (d *DurableEmitter) metricsLoop(ctx context.Context) { if obs, ok := d.store.(DurableQueueObserver); ok { d.metrics.pollQueueGauges(bctx, obs, d.cfg.EventTTL, d.queueStatsNearExpiryLead(), mc.MaxQueuePayloadBytes) } - if mc.RecordProcessStats { - d.metrics.recordProcessMem(bctx) - d.metrics.recordProcessCPU(bctx) - } } } } diff --git a/pkg/beholder/durable_emitter_cpu_other.go b/pkg/beholder/durable_emitter_cpu_other.go deleted file mode 100644 index 2b7a7f5206..0000000000 --- a/pkg/beholder/durable_emitter_cpu_other.go +++ /dev/null @@ -1,9 +0,0 @@ -//go:build !unix - -package beholder - -import "context" - -func (m *durableEmitterMetrics) recordProcessCPU(ctx context.Context) { - _ = ctx -} diff --git a/pkg/beholder/durable_emitter_cpu_unix.go b/pkg/beholder/durable_emitter_cpu_unix.go deleted file mode 100644 index ec46802a63..0000000000 --- a/pkg/beholder/durable_emitter_cpu_unix.go +++ /dev/null @@ -1,22 +0,0 @@ -//go:build unix - -package beholder - -import ( - "context" - "syscall" -) - -func (m *durableEmitterMetrics) recordProcessCPU(ctx context.Context) { - if m == nil { - return - } - var r syscall.Rusage - if err := syscall.Getrusage(syscall.RUSAGE_SELF, &r); err != nil { - return - } - u := float64(r.Utime.Sec) + float64(r.Utime.Usec)/1e6 - s := float64(r.Stime.Sec) + float64(r.Stime.Usec)/1e6 - m.procCPUUser.Record(ctx, u) - m.procCPUSys.Record(ctx, s) -} diff --git a/pkg/beholder/durable_emitter_metrics.go b/pkg/beholder/durable_emitter_metrics.go index e08d1e61a4..d923fc2604 100644 --- a/pkg/beholder/durable_emitter_metrics.go +++ b/pkg/beholder/durable_emitter_metrics.go @@ -2,7 +2,6 @@ package beholder import ( "context" - "runtime" "time" "go.opentelemetry.io/otel/attribute" @@ -150,8 +149,6 @@ type DurableEmitterMetricsConfig struct { NearExpiryLead time.Duration // MaxQueuePayloadBytes, if > 0, records capacity_usage_ratio = queue_payload_bytes / max. MaxQueuePayloadBytes int64 - // RecordProcessStats records Go heap gauges and, on Unix, cumulative CPU seconds (getrusage). - RecordProcessStats bool } type durableEmitterMetrics struct { @@ -328,16 +325,6 @@ func (m *durableEmitterMetrics) pollQueueGauges(ctx context.Context, obs Durable } } -func (m *durableEmitterMetrics) recordProcessMem(ctx context.Context) { - if m == nil { - return - } - var ms runtime.MemStats - runtime.ReadMemStats(&ms) - m.procHeapInuse.Record(ctx, int64(ms.HeapInuse)) - m.procHeapSys.Record(ctx, int64(ms.HeapSys)) -} - func (m *durableEmitterMetrics) recordPublish(ctx context.Context, elapsed time.Duration, phase string, err error) { if m == nil { return From f175829cc854912a1f0124dd4e1ea60e26c6f872 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Thu, 30 Apr 2026 12:16:52 -0400 Subject: [PATCH 33/45] Update durable_emitter.go --- pkg/beholder/durable_emitter.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/pkg/beholder/durable_emitter.go b/pkg/beholder/durable_emitter.go index 5cf3ab1687..31c461b5fa 100644 --- a/pkg/beholder/durable_emitter.go +++ b/pkg/beholder/durable_emitter.go @@ -79,10 +79,6 @@ type DurableEmitterHooks struct { // OnEmitInsert is called after each store.Insert in Emit (the DB write that // blocks the caller). elapsed covers only the INSERT; err is nil on success. OnEmitInsert func(elapsed time.Duration, err error) - // OnImmediatePublish is legacy; the durable emitter no longer uses unary Publish for durable emits. - OnImmediatePublish func(elapsed time.Duration, err error) - // OnImmediateDelete is unused; retained for API compatibility. - OnImmediateDelete func(elapsed time.Duration, err error) // OnBatchPublish is called after each PublishBatch RPC in the batch publish loop. // batchSize is the number of events in the batch; err is nil on success. OnBatchPublish func(elapsed time.Duration, batchSize int, err error) @@ -105,6 +101,7 @@ func DefaultDurableEmitterConfig() DurableEmitterConfig { PurgeInterval: 250 * time.Millisecond, PurgeBatchSize: 500, PublishBatchSize: 1, + Metrics: &DurableEmitterMetricsConfig{}, } } From 3056d793dc02f2e26b4718e9000eb7a327c50d5d Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Thu, 30 Apr 2026 12:19:01 -0400 Subject: [PATCH 34/45] tidy --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index a289bf0fbe..33a59f864f 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/andybalholm/brotli v1.1.1 github.com/atombender/go-jsonschema v0.16.1-0.20240916205339-a74cd4e2851c github.com/bytecodealliance/wasmtime-go/v28 v28.0.0 + github.com/cloudevents/sdk-go/binding/format/protobuf/v2 v2.16.1 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc github.com/dominikbraun/graph v0.23.0 github.com/fxamacker/cbor/v2 v2.9.0 @@ -92,7 +93,6 @@ require ( github.com/buger/jsonparser v1.1.2 // indirect github.com/cenkalti/backoff/v5 v5.0.3 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect - github.com/cloudevents/sdk-go/binding/format/protobuf/v2 v2.16.1 // indirect github.com/cloudevents/sdk-go/v2 v2.16.1 // indirect github.com/fatih/color v1.18.0 // indirect github.com/gabriel-vasile/mimetype v1.4.8 // indirect From 7f08d3dc02c37c9514bc09a744582ddcdf7ffdc7 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Thu, 30 Apr 2026 13:09:02 -0400 Subject: [PATCH 35/45] Fix race in tests --- pkg/beholder/durable_emitter.go | 49 ++++++++++++++++++++++++++++++--- 1 file changed, 45 insertions(+), 4 deletions(-) diff --git a/pkg/beholder/durable_emitter.go b/pkg/beholder/durable_emitter.go index 31c461b5fa..2a3a61ec39 100644 --- a/pkg/beholder/durable_emitter.go +++ b/pkg/beholder/durable_emitter.go @@ -423,6 +423,11 @@ func (d *DurableEmitter) Emit(ctx context.Context, body []byte, attrKVs ...any) work.event = eventPb } select { + case <-d.stopCh: + return fmt.Errorf("durable emitter closed") + default: + } + select { case d.publishCh <- work: default: // Channel full — event is safely in the DB; retransmit loop will deliver it. @@ -434,14 +439,15 @@ func (d *DurableEmitter) Emit(ctx context.Context, body []byte, attrKVs ...any) } // Close signals background loops to stop and waits for them to finish. -// Insert and publish channels are closed so workers can drain. +// stopCh is closed first so workers exit before publishCh is closed after wg.Wait, +// avoiding send vs close races on publishCh. func (d *DurableEmitter) Close() error { close(d.stopCh) if d.insertCh != nil { close(d.insertCh) } - close(d.publishCh) d.wg.Wait() + close(d.publishCh) d.markWg.Wait() return nil } @@ -569,6 +575,9 @@ func (d *DurableEmitter) batchPublishLoop(ctx context.Context) { case <-ctx.Done(): d.drainPublishCh(batch) return + case <-d.stopCh: + d.drainPublishChOnShutdown(batch) + return } // Stage 2: linger — keep reading until batch is full or deadline. @@ -591,6 +600,10 @@ func (d *DurableEmitter) batchPublishLoop(ctx context.Context) { linger.Stop() d.drainPublishCh(batch) return + case <-d.stopCh: + linger.Stop() + d.drainPublishChOnShutdown(batch) + return } } linger.Stop() @@ -600,6 +613,36 @@ func (d *DurableEmitter) batchPublishLoop(ctx context.Context) { } } +// drainPublishChOnShutdown empties publishCh after stopCh has fired but before +// Close closes publishCh; finishes any in-flight batches. +func (d *DurableEmitter) drainPublishChOnShutdown(batch []publishWork) { + bs := d.cfg.PublishBatchSize + if bs < 1 { + bs = 1 + } + for { + select { + case w, ok := <-d.publishCh: + if !ok { + if len(batch) > 0 { + d.flushBatch(batch) + } + return + } + batch = append(batch, w) + if len(batch) >= bs { + d.flushBatch(batch) + batch = batch[:0] + } + default: + if len(batch) > 0 { + d.flushBatch(batch) + } + return + } + } +} + // drainPublishCh flushes the given partial batch plus any remaining items on // publishCh in batchSize chunks. Called during shutdown (ctx cancelled or // channel closed). @@ -786,8 +829,6 @@ func (d *DurableEmitter) retransmit(pending []DurableEvent) { work := publishWork{id: pe.ID, payload: pe.Payload} select { - case <-d.stopCh: - return case d.publishCh <- work: enqueued++ default: From 5d4bf35ad8afb1e65903c8963d010411f990cc97 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Thu, 30 Apr 2026 13:45:50 -0400 Subject: [PATCH 36/45] Trigger CI From 9e61f290c2b0161fb6cacf37963ab87da67d7913 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Tue, 5 May 2026 11:56:24 -0400 Subject: [PATCH 37/45] Fix context --- pkg/beholder/durable_emitter.go | 65 ++++++++++++++------------------- 1 file changed, 28 insertions(+), 37 deletions(-) diff --git a/pkg/beholder/durable_emitter.go b/pkg/beholder/durable_emitter.go index 2a3a61ec39..8f67ec671a 100644 --- a/pkg/beholder/durable_emitter.go +++ b/pkg/beholder/durable_emitter.go @@ -15,6 +15,7 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/chipingress" "github.com/smartcontractkit/chainlink-common/pkg/chipingress/pb" "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-common/pkg/services" ) // DurableEmitterConfig configures the DurableEmitter behaviour. @@ -84,10 +85,6 @@ type DurableEmitterHooks struct { OnBatchPublish func(elapsed time.Duration, batchSize int, err error) // OnBatchMarkDelivered is called after MarkDeliveredBatch following a successful batch publish. OnBatchMarkDelivered func(elapsed time.Duration, count int) - // OnRetransmitBatchPublish is not invoked (retransmit uses the same batch loop as Emit; use OnBatchPublish). - OnRetransmitBatchPublish func(elapsed time.Duration, eventCount int, err error) - // OnRetransmitBatchDeletes is not invoked (mark delivered is async per batch; hook retained for API compatibility). - OnRetransmitBatchDeletes func(elapsed time.Duration, markedDeliveredCount int) } func DefaultDurableEmitterConfig() DurableEmitterConfig { @@ -96,7 +93,7 @@ func DefaultDurableEmitterConfig() DurableEmitterConfig { RetransmitAfter: 10 * time.Second, RetransmitBatchSize: 100, ExpiryInterval: 1 * time.Minute, - EventTTL: 24 * time.Hour, + EventTTL: 72 * time.Hour, PublishTimeout: 5 * time.Second, PurgeInterval: 250 * time.Millisecond, PurgeBatchSize: 500, @@ -162,9 +159,8 @@ type DurableEmitter struct { // publishCh buffers events for the batch publish loop. publishCh chan publishWork - stopCh chan struct{} + stopCh services.StopChan wg sync.WaitGroup - markWg sync.WaitGroup // tracks in-flight async MarkDelivered goroutines } // grpcConnProvider is an optional interface for clients that expose the @@ -305,21 +301,21 @@ func (d *DurableEmitter) Start(ctx context.Context) { n++ } d.wg.Add(n) - go d.retransmitLoop(ctx) + go d.retransmitLoop() if !d.cfg.DisablePruning { go d.expiryLoop(ctx) go d.purgeLoop(ctx) } if d.insertCh != nil { for i := 0; i < insertWorkers; i++ { - go d.insertBatchLoop(ctx) + go d.insertBatchLoop() } } for i := 0; i < batchWorkers; i++ { - go d.batchPublishLoop(ctx) + go d.batchPublishLoop() } if d.metrics != nil && d.cfg.Metrics != nil { - go d.metricsLoop(ctx) + go d.metricsLoop() } } @@ -448,7 +444,6 @@ func (d *DurableEmitter) Close() error { } d.wg.Wait() close(d.publishCh) - d.markWg.Wait() return nil } @@ -456,7 +451,7 @@ func (d *DurableEmitter) Close() error { // as multi-row INSERTs via BatchInserter.InsertBatch. Uses a linger pattern: // blocks for the first request, then collects more until the batch is full or // the flush interval elapses. -func (d *DurableEmitter) insertBatchLoop(ctx context.Context) { +func (d *DurableEmitter) insertBatchLoop() { defer d.wg.Done() batchSize := d.cfg.InsertBatchSize linger := d.cfg.InsertBatchFlushInterval @@ -465,6 +460,9 @@ func (d *DurableEmitter) insertBatchLoop(ctx context.Context) { } batch := make([]*insertRequest, 0, batchSize) + ctx, cancel := d.stopCh.NewCtx() + defer cancel() + for { batch = batch[:0] @@ -475,8 +473,6 @@ func (d *DurableEmitter) insertBatchLoop(ctx context.Context) { return } batch = append(batch, req) - case <-ctx.Done(): - return case <-d.stopCh: return } @@ -553,7 +549,7 @@ func (d *DurableEmitter) decPending(n int64) { // PublishBatchSize, and sends each batch via PublishBatch RPC. It blocks until // the batch is full or PublishBatchFlushInterval elapses after the first event // arrives (linger pattern), guaranteeing full batches at high throughput. -func (d *DurableEmitter) batchPublishLoop(ctx context.Context) { +func (d *DurableEmitter) batchPublishLoop() { defer d.wg.Done() batchSize := d.cfg.PublishBatchSize @@ -572,9 +568,6 @@ func (d *DurableEmitter) batchPublishLoop(ctx context.Context) { return } batch = append(batch, w) - case <-ctx.Done(): - d.drainPublishCh(batch) - return case <-d.stopCh: d.drainPublishChOnShutdown(batch) return @@ -596,10 +589,6 @@ func (d *DurableEmitter) batchPublishLoop(ctx context.Context) { batch = append(batch, w) case <-linger.C: break fill - case <-ctx.Done(): - linger.Stop() - d.drainPublishCh(batch) - return case <-d.stopCh: linger.Stop() d.drainPublishChOnShutdown(batch) @@ -706,9 +695,9 @@ func (d *DurableEmitter) flushBatch(batch []publishWork) { // the batch worker can immediately start collecting the next batch. // If MarkDelivered fails, events stay pending and the retransmit loop // delivers them (at-least-once semantics are unchanged). - d.markWg.Add(1) + d.wg.Add(1) go func() { - defer d.markWg.Done() + defer d.wg.Done() tMark := time.Now() marked, markErr := d.store.MarkDeliveredBatch(context.Background(), ids) markElapsed := time.Since(tMark) @@ -765,24 +754,25 @@ func (d *DurableEmitter) flushBatchTyped(ctx context.Context, batch []publishWor return err } -func (d *DurableEmitter) retransmitLoop(ctx context.Context) { +func (d *DurableEmitter) retransmitLoop() { defer d.wg.Done() ticker := time.NewTicker(d.cfg.RetransmitInterval) defer ticker.Stop() for { select { - case <-ctx.Done(): - return case <-d.stopCh: return case <-ticker.C: - d.retransmitPending(ctx) + d.retransmitPending() } } } -func (d *DurableEmitter) retransmitPending(ctx context.Context) { +func (d *DurableEmitter) retransmitPending() { + ctx, cancel := d.stopCh.NewCtx() + defer cancel() + cutoff := time.Now().Add(-d.cfg.RetransmitAfter) pending, err := d.store.ListPending(ctx, cutoff, d.cfg.RetransmitBatchSize) if err != nil { @@ -913,30 +903,31 @@ func (d *DurableEmitter) queueStatsNearExpiryLead() time.Duration { return lead } -func (d *DurableEmitter) metricsLoop(ctx context.Context) { +func (d *DurableEmitter) metricsLoop() { defer d.wg.Done() mc := d.cfg.Metrics poll := mc.PollInterval if poll <= 0 { poll = 10 * time.Second } + + ctx, cancel := d.stopCh.NewCtx() + defer cancel() + ticker := time.NewTicker(poll) defer ticker.Stop() for { select { - case <-ctx.Done(): - return case <-d.stopCh: return case <-ticker.C: if d.metrics == nil { return } - bctx := context.Background() - d.metrics.queueDepth.Record(bctx, d.pendingCount.Load()) - d.metrics.queueDepthMax.Record(bctx, d.pendingMax.Load()) + d.metrics.queueDepth.Record(ctx, d.pendingCount.Load()) + d.metrics.queueDepthMax.Record(ctx, d.pendingMax.Load()) if obs, ok := d.store.(DurableQueueObserver); ok { - d.metrics.pollQueueGauges(bctx, obs, d.cfg.EventTTL, d.queueStatsNearExpiryLead(), mc.MaxQueuePayloadBytes) + d.metrics.pollQueueGauges(ctx, obs, d.cfg.EventTTL, d.queueStatsNearExpiryLead(), mc.MaxQueuePayloadBytes) } } } From 22889403d3428d4695ebbb4f1ce3e528282ffe4d Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Mon, 11 May 2026 10:32:28 -0400 Subject: [PATCH 38/45] Drain insert on shutdown --- pkg/beholder/durable_emitter.go | 78 +++++++++++++++++----------- pkg/beholder/durable_emitter_test.go | 62 ++++++++++++++++++++++ 2 files changed, 110 insertions(+), 30 deletions(-) diff --git a/pkg/beholder/durable_emitter.go b/pkg/beholder/durable_emitter.go index 8f67ec671a..6c3d381b93 100644 --- a/pkg/beholder/durable_emitter.go +++ b/pkg/beholder/durable_emitter.go @@ -2,6 +2,7 @@ package beholder import ( "context" + "errors" "fmt" "sync" "sync/atomic" @@ -149,6 +150,10 @@ type DurableEmitter struct { // insertCh buffers payloads for the write coalescer. Nil when batch // inserting is disabled. insertCh chan *insertRequest + // insertShutdown stops new coalesced inserts; insertInFlight counts Emit + // callers inside the coalesced path so Close can close(insertCh) after wait + insertShutdown atomic.Bool + insertInFlight atomic.Int32 // pendingCount is an exact, atomic count of rows inserted but not yet // delivered/deleted. Incremented on successful Insert, decremented on @@ -367,15 +372,31 @@ func (d *DurableEmitter) Emit(ctx context.Context, body []byte, attrKVs ...any) payload: payload, result: make(chan insertResult, 1), } - tIns := time.Now() - select { - case d.insertCh <- req: - case <-ctx.Done(): - emitFail() - return ctx.Err() + var res insertResult + var cerr error + func() { + d.insertInFlight.Add(1) + defer d.insertInFlight.Add(-1) + if d.insertShutdown.Load() { + cerr = fmt.Errorf("durable emitter closed") + return + } + tIns := time.Now() + select { + case d.insertCh <- req: + case <-ctx.Done(): + cerr = ctx.Err() + return + } + res = <-req.result + insElapsed = time.Since(tIns) + }() + if cerr != nil { + if errors.Is(cerr, context.Canceled) || errors.Is(cerr, context.DeadlineExceeded) { + emitFail() + } + return cerr } - res := <-req.result - insElapsed = time.Since(tIns) if h := d.cfg.Hooks; h != nil && h.OnEmitInsert != nil { h.OnEmitInsert(insElapsed, res.err) } @@ -419,29 +440,31 @@ func (d *DurableEmitter) Emit(ctx context.Context, body []byte, attrKVs ...any) work.event = eventPb } select { - case <-d.stopCh: - return fmt.Errorf("durable emitter closed") - default: - } - select { case d.publishCh <- work: + return nil default: // Channel full — event is safely in the DB; retransmit loop will deliver it. d.log.Warnw("DurableEmitter: batch publish channel full, relying on retransmit", "id", id, "ch_len", len(d.publishCh), "ch_cap", cap(d.publishCh)) } - return nil } // Close signals background loops to stop and waits for them to finish. -// stopCh is closed first so workers exit before publishCh is closed after wg.Wait, -// avoiding send vs close races on publishCh. +// +// When coalesced inserts are enabled, insertShutdown and insertInFlight drain run +// before close(stopCh) so Emit can finish enqueueing to publishCh after a +// successful insert (receive on a closed stopCh in select would race with default). +// Then stopCh is closed, workers exit, and publishCh is closed after wg.Wait. func (d *DurableEmitter) Close() error { - close(d.stopCh) if d.insertCh != nil { + d.insertShutdown.Store(true) + for d.insertInFlight.Load() > 0 { + time.Sleep(time.Millisecond) + } close(d.insertCh) } + close(d.stopCh) d.wg.Wait() close(d.publishCh) return nil @@ -460,22 +483,16 @@ func (d *DurableEmitter) insertBatchLoop() { } batch := make([]*insertRequest, 0, batchSize) - ctx, cancel := d.stopCh.NewCtx() - defer cancel() - for { batch = batch[:0] - // Block until first request or shutdown. - select { - case req, ok := <-d.insertCh: - if !ok { - return - } - batch = append(batch, req) - case <-d.stopCh: + // Exit only when insertCh is closed and drained; do not exit on stopCh + // or Emit callers blocked on req.result would hang. + req, ok := <-d.insertCh + if !ok { return } + batch = append(batch, req) // Linger to collect more. timer := time.NewTimer(linger) @@ -494,12 +511,13 @@ func (d *DurableEmitter) insertBatchLoop() { } timer.Stop() - // Flush: multi-row INSERT. payloads := make([][]byte, len(batch)) for i, r := range batch { payloads[i] = r.payload } - ids, batchErr := d.batchInserter.InsertBatch(ctx, payloads) + // Detached from stopCh so closing stopCh does not cancel inserts while + // draining insertCh during shutdown. + ids, batchErr := d.batchInserter.InsertBatch(context.Background(), payloads) for i, r := range batch { if batchErr != nil { r.result <- insertResult{err: batchErr} diff --git a/pkg/beholder/durable_emitter_test.go b/pkg/beholder/durable_emitter_test.go index ee51fd8b23..0558196246 100644 --- a/pkg/beholder/durable_emitter_test.go +++ b/pkg/beholder/durable_emitter_test.go @@ -115,6 +115,68 @@ func newTestDurableEmitter(t *testing.T, store DurableEventStore, client chiping return em } +// stallBatchStore wraps MemDurableEventStore so tests can block InsertBatch until stall is unlocked. +type stallBatchStore struct { + *MemDurableEventStore + stall *sync.Mutex +} + +func (s *stallBatchStore) InsertBatch(ctx context.Context, payloads [][]byte) ([]int64, error) { + s.stall.Lock() + defer s.stall.Unlock() + return s.MemDurableEventStore.InsertBatch(ctx, payloads) +} + +func TestDurableEmitter_CloseCoalescedInsertShutdown(t *testing.T) { + stall := new(sync.Mutex) + stall.Lock() + store := &stallBatchStore{ + MemDurableEventStore: NewMemDurableEventStore(), + stall: stall, + } + client := &testChipClient{} + cfg := DefaultDurableEmitterConfig() + cfg.InsertBatchSize = 1 + cfg.InsertBatchWorkers = 1 + cfg.DisablePruning = true + + em := newTestDurableEmitter(t, store, client, &cfg) + ctx := t.Context() + em.Start(ctx) + + emitErr := make(chan error, 1) + go func() { emitErr <- em.Emit(ctx, []byte("during-close"), testEmitAttrs()...) }() + + require.Eventually(t, func() bool { + return em.insertInFlight.Load() == 1 + }, time.Second, 5*time.Millisecond, "Emit should be in-flight waiting on InsertBatch") + + closeErr := make(chan error, 1) + go func() { closeErr <- em.Close() }() + + select { + case err := <-closeErr: + require.NoError(t, err) + t.Fatal("Close returned before coalesced insert finished; shutdown wait is broken") + case <-time.After(150 * time.Millisecond): + } + + stall.Unlock() + + select { + case err := <-closeErr: + require.NoError(t, err) + case <-time.After(5 * time.Second): + t.Fatal("Close timed out after releasing InsertBatch") + } + + require.NoError(t, <-emitErr, "Emit should complete after insert path drains") + + err := em.Emit(ctx, []byte("after-close"), testEmitAttrs()...) + require.Error(t, err) + assert.Contains(t, err.Error(), "durable emitter closed") +} + func TestDurableEmitter_HooksBatchPublishPath(t *testing.T) { store := NewMemDurableEventStore() client := &testChipClient{} From e357765eb663719dcba4de26f639945d22c9174b Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Mon, 11 May 2026 10:52:23 -0400 Subject: [PATCH 39/45] Trigger CI From 6041cf0180fcfed438b989a3e87210bbdf23edc3 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Mon, 11 May 2026 11:37:46 -0400 Subject: [PATCH 40/45] Fix ctx handling --- pkg/beholder/durable_emitter.go | 32 +++++++++++++++++++++----------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/pkg/beholder/durable_emitter.go b/pkg/beholder/durable_emitter.go index 6c3d381b93..bde66d9d23 100644 --- a/pkg/beholder/durable_emitter.go +++ b/pkg/beholder/durable_emitter.go @@ -517,7 +517,9 @@ func (d *DurableEmitter) insertBatchLoop() { } // Detached from stopCh so closing stopCh does not cancel inserts while // draining insertCh during shutdown. - ids, batchErr := d.batchInserter.InsertBatch(context.Background(), payloads) + ctx, cancel := context.WithTimeout(context.Background(), d.cfg.PublishTimeout) + ids, batchErr := d.batchInserter.InsertBatch(ctx, payloads) + cancel() for i, r := range batch { if batchErr != nil { r.result <- insertResult{err: batchErr} @@ -549,9 +551,11 @@ func (d *DurableEmitter) incPending(n int64) { } } if d.metrics != nil { - d.metrics.queueDepth.Record(context.Background(), cur) + ctx, cancel := d.stopCh.NewCtx() + defer cancel() + d.metrics.queueDepth.Record(ctx, cur) if updated { - d.metrics.queueDepthMax.Record(context.Background(), cur) + d.metrics.queueDepthMax.Record(ctx, cur) } } } @@ -559,7 +563,9 @@ func (d *DurableEmitter) incPending(n int64) { func (d *DurableEmitter) decPending(n int64) { cur := d.pendingCount.Add(-n) if d.metrics != nil { - d.metrics.queueDepth.Record(context.Background(), cur) + ctx, cancel := d.stopCh.NewCtx() + defer cancel() + d.metrics.queueDepth.Record(ctx, cur) } } @@ -678,8 +684,10 @@ func (d *DurableEmitter) flushBatch(batch []publishWork) { ids[i] = w.id } - pubCtx, cancel := context.WithTimeout(context.Background(), d.cfg.PublishTimeout) + ctx, cancel := d.stopCh.NewCtx() defer cancel() + pubCtx, pubCancel := context.WithTimeout(ctx, d.cfg.PublishTimeout) + defer pubCancel() t0 := time.Now() var err error @@ -693,11 +701,11 @@ func (d *DurableEmitter) flushBatch(batch []publishWork) { if h := d.cfg.Hooks; h != nil && h.OnBatchPublish != nil { h.OnBatchPublish(elapsed, len(batch), err) } - d.metrics.recordPublish(context.Background(), elapsed, "batch", err) + d.metrics.recordPublish(ctx, elapsed, "batch", err) if err != nil { if d.metrics != nil { - d.metrics.publishBatchEvErr.Add(context.Background(), int64(len(batch))) + d.metrics.publishBatchEvErr.Add(ctx, int64(len(batch))) } d.log.Warnw("DurableEmitter: PublishBatch failed, events will be retransmitted", "batch_size", len(batch), "error", err, @@ -706,7 +714,7 @@ func (d *DurableEmitter) flushBatch(batch []publishWork) { } if d.metrics != nil { - d.metrics.publishBatchEvOK.Add(context.Background(), int64(len(batch))) + d.metrics.publishBatchEvOK.Add(pubCtx, int64(len(batch))) } // Async MarkDelivered: the DB UPDATE runs in a background goroutine so @@ -717,7 +725,7 @@ func (d *DurableEmitter) flushBatch(batch []publishWork) { go func() { defer d.wg.Done() tMark := time.Now() - marked, markErr := d.store.MarkDeliveredBatch(context.Background(), ids) + marked, markErr := d.store.MarkDeliveredBatch(ctx, ids) markElapsed := time.Since(tMark) if h := d.cfg.Hooks; h != nil && h.OnBatchMarkDelivered != nil { h.OnBatchMarkDelivered(markElapsed, int(marked)) @@ -728,7 +736,7 @@ func (d *DurableEmitter) flushBatch(batch []publishWork) { } d.decPending(marked) if d.metrics != nil { - d.metrics.deliverComplete.Add(context.Background(), marked) + d.metrics.deliverComplete.Add(ctx, marked) } }() } @@ -890,6 +898,8 @@ func (d *DurableEmitter) expiryLoop(ctx context.Context) { ticker := time.NewTicker(d.cfg.ExpiryInterval) defer ticker.Stop() + ctx, cancel := d.stopCh.NewCtx() + defer cancel() for { select { case <-ctx.Done(): @@ -905,7 +915,7 @@ func (d *DurableEmitter) expiryLoop(ctx context.Context) { if deleted > 0 { d.decPending(deleted) if d.metrics != nil { - d.metrics.expiredPurged.Add(context.Background(), deleted) + d.metrics.expiredPurged.Add(ctx, deleted) } d.log.Infow("purged expired events", "count", deleted) } From 70a63f8a523d92f31c9ef56bcd895a5f2a7fcdb7 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Mon, 11 May 2026 13:34:55 -0400 Subject: [PATCH 41/45] Add isHostProcess flag --- pkg/beholder/durable_emitter.go | 50 ++++++++--------- pkg/beholder/durable_emitter_test.go | 81 ++++++++++++++++++++++------ 2 files changed, 92 insertions(+), 39 deletions(-) diff --git a/pkg/beholder/durable_emitter.go b/pkg/beholder/durable_emitter.go index bde66d9d23..30ee5f19bd 100644 --- a/pkg/beholder/durable_emitter.go +++ b/pkg/beholder/durable_emitter.go @@ -135,8 +135,11 @@ type insertResult struct { type DurableEmitter struct { store DurableEventStore client chipingress.Client - cfg DurableEmitterConfig - log logger.Logger + // isHostProcess determines if the emitter runs retransmit and cleanup loops. + // Should be set to false when initialized inside LOOP plugins. + isHostProcess bool + cfg DurableEmitterConfig + log logger.Logger metrics *durableEmitterMetrics @@ -222,6 +225,7 @@ var _ Emitter = (*DurableEmitter)(nil) func NewDurableEmitter( store DurableEventStore, client chipingress.Client, + isHostProcess bool, cfg DurableEmitterConfig, log logger.Logger, ) (*DurableEmitter, error) { @@ -247,12 +251,13 @@ func NewDurableEmitter( store = newMetricsInstrumentedStore(store, m) } d := &DurableEmitter{ - store: store, - client: client, - cfg: cfg, - log: log, - metrics: m, - stopCh: make(chan struct{}), + store: store, + client: client, + isHostProcess: isHostProcess, + cfg: cfg, + log: log, + metrics: m, + stopCh: make(chan struct{}), } if cp, ok := client.(grpcConnProvider); ok { d.rawConn = cp.Conn() @@ -286,40 +291,37 @@ func NewDurableEmitter( // Start launches the retransmit, expiry, purge, and (optionally) batch publish // background loops. Cancel the supplied context or call Close to stop them. func (d *DurableEmitter) Start(ctx context.Context) { - n := 1 // retransmit always runs - if !d.cfg.DisablePruning { - n += 2 // expiry + purge + + if d.isHostProcess { + d.wg.Add(1) + go d.retransmitLoop() + if !d.cfg.DisablePruning { + d.wg.Add(2) + go d.expiryLoop(ctx) + go d.purgeLoop(ctx) + } } + batchWorkers := d.cfg.PublishBatchWorkers if batchWorkers <= 0 { batchWorkers = 1 } - n += batchWorkers insertWorkers := d.cfg.InsertBatchWorkers if insertWorkers <= 0 { insertWorkers = 4 } - if d.insertCh != nil { - n += insertWorkers - } - if d.metrics != nil && d.cfg.Metrics != nil { - n++ - } - d.wg.Add(n) - go d.retransmitLoop() - if !d.cfg.DisablePruning { - go d.expiryLoop(ctx) - go d.purgeLoop(ctx) - } if d.insertCh != nil { for i := 0; i < insertWorkers; i++ { + d.wg.Add(1) go d.insertBatchLoop() } } for i := 0; i < batchWorkers; i++ { + d.wg.Add(1) go d.batchPublishLoop() } if d.metrics != nil && d.cfg.Metrics != nil { + d.wg.Add(1) go d.metricsLoop() } } diff --git a/pkg/beholder/durable_emitter_test.go b/pkg/beholder/durable_emitter_test.go index 0558196246..8b88433266 100644 --- a/pkg/beholder/durable_emitter_test.go +++ b/pkg/beholder/durable_emitter_test.go @@ -110,7 +110,7 @@ func newTestDurableEmitter(t *testing.T, store DurableEventStore, client chiping if cfgOverride != nil { cfg = *cfgOverride } - em, err := NewDurableEmitter(store, client, cfg, logger.Test(t)) + em, err := NewDurableEmitter(store, client, true, cfg, logger.Test(t)) require.NoError(t, err) return em } @@ -186,7 +186,7 @@ func TestDurableEmitter_HooksBatchPublishPath(t *testing.T) { OnBatchPublish: func(time.Duration, int, error) { pubCalls.Add(1) }, OnBatchMarkDelivered: func(time.Duration, int) { markCalls.Add(1) }, } - em, err := NewDurableEmitter(store, client, cfg, logger.Test(t)) + em, err := NewDurableEmitter(store, client, true, cfg, logger.Test(t)) require.NoError(t, err) ctx, cancel := context.WithCancel(context.Background()) @@ -210,7 +210,7 @@ func TestDurableEmitter_HooksPublishFailureSkipsMarkHook(t *testing.T) { OnBatchPublish: func(time.Duration, int, error) { pubCalls.Add(1) }, OnBatchMarkDelivered: func(time.Duration, int) { markCalls.Add(1) }, } - em, err := NewDurableEmitter(store, client, cfg, logger.Test(t)) + em, err := NewDurableEmitter(store, client, true, cfg, logger.Test(t)) require.NoError(t, err) ctx, cancel := context.WithCancel(context.Background()) @@ -367,6 +367,57 @@ func TestDurableEmitter_ExpiryLoopDeletesOldEvents(t *testing.T) { }, 5*time.Second, 50*time.Millisecond, "expiry loop should purge the event") } +func TestDurableEmitter_NonHostProcessSkipsRetransmitAndExpiry(t *testing.T) { + store := NewMemDurableEventStore() + client := &testChipClient{} + client.setPublishErr(errors.New("chip unavailable")) + + cfg := DefaultDurableEmitterConfig() + cfg.RetransmitInterval = 40 * time.Millisecond + cfg.RetransmitAfter = 15 * time.Millisecond + cfg.ExpiryInterval = 40 * time.Millisecond + cfg.EventTTL = 25 * time.Millisecond + + em, err := NewDurableEmitter(store, client, false, cfg, logger.Test(t)) + require.NoError(t, err) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + em.Start(ctx) + defer func() { require.NoError(t, em.Close()) }() + + require.NoError(t, em.Emit(ctx, []byte("plugin-row"), testEmitAttrs()...)) + + require.Eventually(t, func() bool { + return client.batchCount.Load() >= 1 && store.Len() == 1 + }, 2*time.Second, 5*time.Millisecond, "initial PublishBatch should fail and leave the row") + + // Several host-only ticks would have cleared or retried by now. + time.Sleep(250 * time.Millisecond) + + assert.Equal(t, 1, store.Len(), "non-host must not run retransmit or expiry loops") + assert.Equal(t, int64(1), client.batchCount.Load(), "non-host must not schedule extra PublishBatch via retransmit") +} + +func TestDurableEmitter_NonHostProcessStillDeliversViaBatchWorkers(t *testing.T) { + store := NewMemDurableEventStore() + client := &testChipClient{} + + em, err := NewDurableEmitter(store, client, false, DefaultDurableEmitterConfig(), logger.Test(t)) + require.NoError(t, err) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + em.Start(ctx) + defer func() { require.NoError(t, em.Close()) }() + + require.NoError(t, em.Emit(ctx, []byte("loop-plugin"), testEmitAttrs()...)) + + require.Eventually(t, func() bool { + return store.Len() == 0 && client.batchCount.Load() >= 1 + }, 2*time.Second, 10*time.Millisecond, "batch publish workers must still run when isHostProcess is false") +} + func TestDurableEmitter_RetransmitDeliversManuallyInsertedRow(t *testing.T) { store := NewMemDurableEventStore() client := &testChipClient{} @@ -436,13 +487,13 @@ func TestNewDurableEmitter_ValidationErrors(t *testing.T) { log := logger.Test(t) cfg := DefaultDurableEmitterConfig() - _, err := NewDurableEmitter(nil, &testChipClient{}, cfg, log) + _, err := NewDurableEmitter(nil, &testChipClient{}, true, cfg, log) assert.ErrorContains(t, err, "store") - _, err = NewDurableEmitter(NewMemDurableEventStore(), nil, cfg, log) + _, err = NewDurableEmitter(NewMemDurableEventStore(), nil, true, cfg, log) assert.ErrorContains(t, err, "client") - _, err = NewDurableEmitter(NewMemDurableEventStore(), &testChipClient{}, cfg, nil) + _, err = NewDurableEmitter(NewMemDurableEventStore(), &testChipClient{}, true, cfg, nil) assert.ErrorContains(t, err, "logger") } @@ -455,7 +506,7 @@ func TestDurableEmitter_MetricsRegistersEmitSuccess(t *testing.T) { cfg.RetransmitInterval = time.Hour cfg.Metrics = &DurableEmitterMetricsConfig{PollInterval: 25 * time.Millisecond} - em, err := NewDurableEmitter(store, client, cfg, logger.Test(t)) + em, err := NewDurableEmitter(store, client, true, cfg, logger.Test(t)) require.NoError(t, err) ctx, cancel := context.WithCancel(context.Background()) @@ -602,7 +653,7 @@ func TestIntegration_HappyPath(t *testing.T) { client := newChipClient(t, addr) store := NewMemDurableEventStore() - em, err := NewDurableEmitter(store, client, fastCfg(), logger.Test(t)) + em, err := NewDurableEmitter(store, client, true, fastCfg(), logger.Test(t)) require.NoError(t, err) ctx, cancel := context.WithCancel(context.Background()) @@ -630,7 +681,7 @@ func TestIntegration_ServerUnavailable_RetransmitRecovers(t *testing.T) { client := newChipClient(t, addr) store := NewMemDurableEventStore() - em, err := NewDurableEmitter(store, client, fastCfg(), logger.Test(t)) + em, err := NewDurableEmitter(store, client, true, fastCfg(), logger.Test(t)) require.NoError(t, err) ctx, cancel := context.WithCancel(context.Background()) @@ -665,7 +716,7 @@ func TestIntegration_ServerDown_EventsSurvive(t *testing.T) { cfg := fastCfg() cfg.PublishTimeout = 500 * time.Millisecond - em, err := NewDurableEmitter(store, client, cfg, logger.Test(t)) + em, err := NewDurableEmitter(store, client, true, cfg, logger.Test(t)) require.NoError(t, err) ctx, cancel := context.WithCancel(context.Background()) @@ -698,7 +749,7 @@ func TestIntegration_ServerDown_EventsSurvive(t *testing.T) { require.NoError(t, err) t.Cleanup(func() { _ = client2.Close() }) - em2, err := NewDurableEmitter(store, client2, cfg, logger.Test(t)) + em2, err := NewDurableEmitter(store, client2, true, cfg, logger.Test(t)) require.NoError(t, err) em2.Start(ctx) defer em2.Close() @@ -720,7 +771,7 @@ func TestIntegration_HighThroughput(t *testing.T) { cfg := fastCfg() cfg.RetransmitBatchSize = 200 - em, err := NewDurableEmitter(store, client, cfg, logger.Test(t)) + em, err := NewDurableEmitter(store, client, true, cfg, logger.Test(t)) require.NoError(t, err) ctx, cancel := context.WithCancel(context.Background()) @@ -753,7 +804,7 @@ func TestIntegration_EventExpiry(t *testing.T) { cfg := fastCfg() cfg.EventTTL = 100 * time.Millisecond cfg.ExpiryInterval = 100 * time.Millisecond - em, err := NewDurableEmitter(store, client, cfg, logger.Test(t)) + em, err := NewDurableEmitter(store, client, true, cfg, logger.Test(t)) require.NoError(t, err) ctx, cancel := context.WithCancel(context.Background()) @@ -778,7 +829,7 @@ func TestIntegration_RetransmitEnqueuesBatchWorkers(t *testing.T) { client := newChipClient(t, addr) store := NewMemDurableEventStore() - em, err := NewDurableEmitter(store, client, fastCfg(), logger.Test(t)) + em, err := NewDurableEmitter(store, client, true, fastCfg(), logger.Test(t)) require.NoError(t, err) ctx, cancel := context.WithCancel(context.Background()) @@ -829,7 +880,7 @@ func TestIntegration_GRPCConnection(t *testing.T) { client := newChipClient(t, addr) store := NewMemDurableEventStore() - em, err := NewDurableEmitter(store, client, fastCfg(), logger.Test(t)) + em, err := NewDurableEmitter(store, client, true, fastCfg(), logger.Test(t)) require.NoError(t, err) ctx, cancel := context.WithCancel(context.Background()) From 750c475f0628989c96b9ae9286e21b9205cdfed4 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Mon, 11 May 2026 13:48:51 -0400 Subject: [PATCH 42/45] Fix wg handling --- pkg/beholder/durable_emitter.go | 44 +++----- pkg/beholder/durable_emitter_test.go | 148 ++++++++++++++++++++++++++ pkg/beholder/durable_event_store.go | 150 --------------------------- 3 files changed, 160 insertions(+), 182 deletions(-) diff --git a/pkg/beholder/durable_emitter.go b/pkg/beholder/durable_emitter.go index bde66d9d23..21c7016541 100644 --- a/pkg/beholder/durable_emitter.go +++ b/pkg/beholder/durable_emitter.go @@ -286,41 +286,33 @@ func NewDurableEmitter( // Start launches the retransmit, expiry, purge, and (optionally) batch publish // background loops. Cancel the supplied context or call Close to stop them. func (d *DurableEmitter) Start(ctx context.Context) { - n := 1 // retransmit always runs - if !d.cfg.DisablePruning { - n += 2 // expiry + purge - } batchWorkers := d.cfg.PublishBatchWorkers if batchWorkers <= 0 { + d.log.Warnw("configured batchWorkers <=0; defaulting to 1") batchWorkers = 1 } - n += batchWorkers insertWorkers := d.cfg.InsertBatchWorkers if insertWorkers <= 0 { + d.log.Warnw("configured insertWorkers <=0; defaulting to 4") insertWorkers = 4 } - if d.insertCh != nil { - n += insertWorkers - } - if d.metrics != nil && d.cfg.Metrics != nil { - n++ - } - d.wg.Add(n) - go d.retransmitLoop() + + // WaitGroup.Go pairs Add/Done with each worker; loop bodies must not call wg.Done. + d.wg.Go(d.retransmitLoop) if !d.cfg.DisablePruning { - go d.expiryLoop(ctx) - go d.purgeLoop(ctx) + d.wg.Go(func() { d.expiryLoop(ctx) }) + d.wg.Go(func() { d.purgeLoop(ctx) }) } if d.insertCh != nil { for i := 0; i < insertWorkers; i++ { - go d.insertBatchLoop() + d.wg.Go(d.insertBatchLoop) } } for i := 0; i < batchWorkers; i++ { - go d.batchPublishLoop() + d.wg.Go(d.batchPublishLoop) } if d.metrics != nil && d.cfg.Metrics != nil { - go d.metricsLoop() + d.wg.Go(d.metricsLoop) } } @@ -475,7 +467,6 @@ func (d *DurableEmitter) Close() error { // blocks for the first request, then collects more until the batch is full or // the flush interval elapses. func (d *DurableEmitter) insertBatchLoop() { - defer d.wg.Done() batchSize := d.cfg.InsertBatchSize linger := d.cfg.InsertBatchFlushInterval if linger <= 0 { @@ -574,8 +565,6 @@ func (d *DurableEmitter) decPending(n int64) { // the batch is full or PublishBatchFlushInterval elapses after the first event // arrives (linger pattern), guaranteeing full batches at high throughput. func (d *DurableEmitter) batchPublishLoop() { - defer d.wg.Done() - batchSize := d.cfg.PublishBatchSize flushInterval := d.cfg.PublishBatchFlushInterval if flushInterval <= 0 { @@ -721,9 +710,7 @@ func (d *DurableEmitter) flushBatch(batch []publishWork) { // the batch worker can immediately start collecting the next batch. // If MarkDelivered fails, events stay pending and the retransmit loop // delivers them (at-least-once semantics are unchanged). - d.wg.Add(1) - go func() { - defer d.wg.Done() + d.wg.Go(func() { tMark := time.Now() marked, markErr := d.store.MarkDeliveredBatch(ctx, ids) markElapsed := time.Since(tMark) @@ -738,7 +725,7 @@ func (d *DurableEmitter) flushBatch(batch []publishWork) { if d.metrics != nil { d.metrics.deliverComplete.Add(ctx, marked) } - }() + }) } // flushBatchRaw builds the CloudEventBatch wire format directly from @@ -781,7 +768,6 @@ func (d *DurableEmitter) flushBatchTyped(ctx context.Context, batch []publishWor } func (d *DurableEmitter) retransmitLoop() { - defer d.wg.Done() ticker := time.NewTicker(d.cfg.RetransmitInterval) defer ticker.Stop() @@ -861,7 +847,6 @@ func (d *DurableEmitter) retransmit(pending []DurableEvent) { } func (d *DurableEmitter) purgeLoop(ctx context.Context) { - defer d.wg.Done() interval := d.cfg.PurgeInterval if interval <= 0 { interval = 250 * time.Millisecond @@ -894,7 +879,6 @@ func (d *DurableEmitter) purgeLoop(ctx context.Context) { } func (d *DurableEmitter) expiryLoop(ctx context.Context) { - defer d.wg.Done() ticker := time.NewTicker(d.cfg.ExpiryInterval) defer ticker.Stop() @@ -932,7 +916,6 @@ func (d *DurableEmitter) queueStatsNearExpiryLead() time.Duration { } func (d *DurableEmitter) metricsLoop() { - defer d.wg.Done() mc := d.cfg.Metrics poll := mc.PollInterval if poll <= 0 { @@ -949,9 +932,6 @@ func (d *DurableEmitter) metricsLoop() { case <-d.stopCh: return case <-ticker.C: - if d.metrics == nil { - return - } d.metrics.queueDepth.Record(ctx, d.pendingCount.Load()) d.metrics.queueDepthMax.Record(ctx, d.pendingMax.Load()) if obs, ok := d.store.(DurableQueueObserver); ok { diff --git a/pkg/beholder/durable_emitter_test.go b/pkg/beholder/durable_emitter_test.go index 0558196246..0ab1e8841f 100644 --- a/pkg/beholder/durable_emitter_test.go +++ b/pkg/beholder/durable_emitter_test.go @@ -4,6 +4,7 @@ import ( "context" "errors" "net" + "sort" "sync" "sync/atomic" "testing" @@ -852,3 +853,150 @@ func TestIntegration_GRPCConnection(t *testing.T) { assert.Equal(t, "test-domain", received.Source) assert.Equal(t, "test-entity", received.Type) } + +// MemDurableEventStore is an in-memory DurableEventStore for unit tests. +type MemDurableEventStore struct { + mu sync.Mutex + events map[int64]*DurableEvent + nextID atomic.Int64 +} + +var ( + _ DurableEventStore = (*MemDurableEventStore)(nil) + _ DurableQueueObserver = (*MemDurableEventStore)(nil) + _ BatchInserter = (*MemDurableEventStore)(nil) +) + +func NewMemDurableEventStore() *MemDurableEventStore { + return &MemDurableEventStore{ + events: make(map[int64]*DurableEvent), + } +} + +func (m *MemDurableEventStore) Insert(_ context.Context, payload []byte) (int64, error) { + id := m.nextID.Add(1) + m.mu.Lock() + defer m.mu.Unlock() + m.events[id] = &DurableEvent{ + ID: id, + Payload: append([]byte(nil), payload...), // defensive copy + CreatedAt: time.Now(), + } + return id, nil +} + +func (m *MemDurableEventStore) InsertBatch(_ context.Context, payloads [][]byte) ([]int64, error) { + now := time.Now() + ids := make([]int64, len(payloads)) + m.mu.Lock() + defer m.mu.Unlock() + for i, p := range payloads { + id := m.nextID.Add(1) + m.events[id] = &DurableEvent{ + ID: id, + Payload: append([]byte(nil), p...), + CreatedAt: now, + } + ids[i] = id + } + return ids, nil +} + +func (m *MemDurableEventStore) Delete(_ context.Context, id int64) error { + m.mu.Lock() + defer m.mu.Unlock() + delete(m.events, id) + return nil +} + +func (m *MemDurableEventStore) MarkDelivered(ctx context.Context, id int64) error { + return m.Delete(ctx, id) +} + +func (m *MemDurableEventStore) MarkDeliveredBatch(_ context.Context, ids []int64) (int64, error) { + m.mu.Lock() + defer m.mu.Unlock() + var n int64 + for _, id := range ids { + if _, ok := m.events[id]; ok { + delete(m.events, id) + n++ + } + } + return n, nil +} + +func (m *MemDurableEventStore) PurgeDelivered(_ context.Context, _ int) (int64, error) { + return 0, nil +} + +func (m *MemDurableEventStore) ListPending(_ context.Context, createdBefore time.Time, limit int) ([]DurableEvent, error) { + m.mu.Lock() + defer m.mu.Unlock() + + var result []DurableEvent + for _, e := range m.events { + if e.CreatedAt.Before(createdBefore) { + result = append(result, *e) + } + } + sort.Slice(result, func(i, j int) bool { + return result[i].CreatedAt.Before(result[j].CreatedAt) + }) + if len(result) > limit { + result = result[:limit] + } + return result, nil +} + +func (m *MemDurableEventStore) DeleteExpired(_ context.Context, ttl time.Duration) (int64, error) { + m.mu.Lock() + defer m.mu.Unlock() + + cutoff := time.Now().Add(-ttl) + var deleted int64 + for id, e := range m.events { + if e.CreatedAt.Before(cutoff) { + delete(m.events, id) + deleted++ + } + } + return deleted, nil +} + +// Len returns the number of events in the store (test helper). +func (m *MemDurableEventStore) Len() int { + m.mu.Lock() + defer m.mu.Unlock() + return len(m.events) +} + +// ObserveDurableQueue implements DurableQueueObserver. +func (m *MemDurableEventStore) ObserveDurableQueue(_ context.Context, eventTTL, nearExpiryLead time.Duration) (DurableQueueStats, error) { + m.mu.Lock() + defer m.mu.Unlock() + now := time.Now() + var st DurableQueueStats + if len(m.events) == 0 { + return st, nil + } + var oldest time.Time + first := true + for _, e := range m.events { + st.Depth++ + st.PayloadBytes += int64(len(e.Payload)) + if first || e.CreatedAt.Before(oldest) { + oldest = e.CreatedAt + first = false + } + age := now.Sub(e.CreatedAt) + if eventTTL > 0 && nearExpiryLead > 0 && nearExpiryLead < eventTTL { + threshold := eventTTL - nearExpiryLead + if age >= threshold && age < eventTTL { + st.NearTTLCount++ + } + } + } + st.OldestPendingAge = now.Sub(oldest) + return st, nil +} diff --git a/pkg/beholder/durable_event_store.go b/pkg/beholder/durable_event_store.go index b00111cd20..cba937c657 100644 --- a/pkg/beholder/durable_event_store.go +++ b/pkg/beholder/durable_event_store.go @@ -3,9 +3,6 @@ package beholder import ( "context" "errors" - "sort" - "sync" - "sync/atomic" "time" ) @@ -68,153 +65,6 @@ type DurableEventStore interface { DeleteExpired(ctx context.Context, ttl time.Duration) (int64, error) } -// MemDurableEventStore is an in-memory DurableEventStore for unit tests. -type MemDurableEventStore struct { - mu sync.Mutex - events map[int64]*DurableEvent - nextID atomic.Int64 -} - -var ( - _ DurableEventStore = (*MemDurableEventStore)(nil) - _ DurableQueueObserver = (*MemDurableEventStore)(nil) - _ BatchInserter = (*MemDurableEventStore)(nil) -) - -func NewMemDurableEventStore() *MemDurableEventStore { - return &MemDurableEventStore{ - events: make(map[int64]*DurableEvent), - } -} - -func (m *MemDurableEventStore) Insert(_ context.Context, payload []byte) (int64, error) { - id := m.nextID.Add(1) - m.mu.Lock() - defer m.mu.Unlock() - m.events[id] = &DurableEvent{ - ID: id, - Payload: append([]byte(nil), payload...), // defensive copy - CreatedAt: time.Now(), - } - return id, nil -} - -func (m *MemDurableEventStore) InsertBatch(_ context.Context, payloads [][]byte) ([]int64, error) { - now := time.Now() - ids := make([]int64, len(payloads)) - m.mu.Lock() - defer m.mu.Unlock() - for i, p := range payloads { - id := m.nextID.Add(1) - m.events[id] = &DurableEvent{ - ID: id, - Payload: append([]byte(nil), p...), - CreatedAt: now, - } - ids[i] = id - } - return ids, nil -} - -func (m *MemDurableEventStore) Delete(_ context.Context, id int64) error { - m.mu.Lock() - defer m.mu.Unlock() - delete(m.events, id) - return nil -} - -func (m *MemDurableEventStore) MarkDelivered(ctx context.Context, id int64) error { - return m.Delete(ctx, id) -} - -func (m *MemDurableEventStore) MarkDeliveredBatch(_ context.Context, ids []int64) (int64, error) { - m.mu.Lock() - defer m.mu.Unlock() - var n int64 - for _, id := range ids { - if _, ok := m.events[id]; ok { - delete(m.events, id) - n++ - } - } - return n, nil -} - -func (m *MemDurableEventStore) PurgeDelivered(_ context.Context, _ int) (int64, error) { - return 0, nil -} - -func (m *MemDurableEventStore) ListPending(_ context.Context, createdBefore time.Time, limit int) ([]DurableEvent, error) { - m.mu.Lock() - defer m.mu.Unlock() - - var result []DurableEvent - for _, e := range m.events { - if e.CreatedAt.Before(createdBefore) { - result = append(result, *e) - } - } - sort.Slice(result, func(i, j int) bool { - return result[i].CreatedAt.Before(result[j].CreatedAt) - }) - if len(result) > limit { - result = result[:limit] - } - return result, nil -} - -func (m *MemDurableEventStore) DeleteExpired(_ context.Context, ttl time.Duration) (int64, error) { - m.mu.Lock() - defer m.mu.Unlock() - - cutoff := time.Now().Add(-ttl) - var deleted int64 - for id, e := range m.events { - if e.CreatedAt.Before(cutoff) { - delete(m.events, id) - deleted++ - } - } - return deleted, nil -} - -// Len returns the number of events in the store (test helper). -func (m *MemDurableEventStore) Len() int { - m.mu.Lock() - defer m.mu.Unlock() - return len(m.events) -} - -// ObserveDurableQueue implements DurableQueueObserver. -func (m *MemDurableEventStore) ObserveDurableQueue(_ context.Context, eventTTL, nearExpiryLead time.Duration) (DurableQueueStats, error) { - m.mu.Lock() - defer m.mu.Unlock() - now := time.Now() - var st DurableQueueStats - if len(m.events) == 0 { - return st, nil - } - var oldest time.Time - first := true - for _, e := range m.events { - st.Depth++ - st.PayloadBytes += int64(len(e.Payload)) - if first || e.CreatedAt.Before(oldest) { - oldest = e.CreatedAt - first = false - } - age := now.Sub(e.CreatedAt) - if eventTTL > 0 && nearExpiryLead > 0 && nearExpiryLead < eventTTL { - threshold := eventTTL - nearExpiryLead - if age >= threshold && age < eventTTL { - st.NearTTLCount++ - } - } - } - st.OldestPendingAge = now.Sub(oldest) - return st, nil -} - // metricsInstrumentedStore wraps DurableEventStore to record store operation metrics. type metricsInstrumentedStore struct { inner DurableEventStore From 79640c4d7c938b7123bce608e4107ea4b69cbda2 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Mon, 11 May 2026 14:00:21 -0400 Subject: [PATCH 43/45] Update durable_emitter.go --- pkg/beholder/durable_emitter.go | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/pkg/beholder/durable_emitter.go b/pkg/beholder/durable_emitter.go index 21c7016541..954e6df23f 100644 --- a/pkg/beholder/durable_emitter.go +++ b/pkg/beholder/durable_emitter.go @@ -135,8 +135,11 @@ type insertResult struct { type DurableEmitter struct { store DurableEventStore client chipingress.Client - cfg DurableEmitterConfig - log logger.Logger + // isHostProcess determines if the emitter runs retransmit and cleanup loops. + // Should be set to false when initialized inside LOOP plugins. + isHostProcess bool + cfg DurableEmitterConfig + log logger.Logger metrics *durableEmitterMetrics @@ -222,6 +225,7 @@ var _ Emitter = (*DurableEmitter)(nil) func NewDurableEmitter( store DurableEventStore, client chipingress.Client, + isHostProcess bool, cfg DurableEmitterConfig, log logger.Logger, ) (*DurableEmitter, error) { @@ -247,12 +251,13 @@ func NewDurableEmitter( store = newMetricsInstrumentedStore(store, m) } d := &DurableEmitter{ - store: store, - client: client, - cfg: cfg, - log: log, - metrics: m, - stopCh: make(chan struct{}), + store: store, + client: client, + isHostProcess: isHostProcess, + cfg: cfg, + log: log, + metrics: m, + stopCh: make(chan struct{}), } if cp, ok := client.(grpcConnProvider); ok { d.rawConn = cp.Conn() @@ -297,11 +302,12 @@ func (d *DurableEmitter) Start(ctx context.Context) { insertWorkers = 4 } - // WaitGroup.Go pairs Add/Done with each worker; loop bodies must not call wg.Done. - d.wg.Go(d.retransmitLoop) - if !d.cfg.DisablePruning { - d.wg.Go(func() { d.expiryLoop(ctx) }) - d.wg.Go(func() { d.purgeLoop(ctx) }) + if d.isHostProcess { + d.wg.Go(d.retransmitLoop) + if !d.cfg.DisablePruning { + d.wg.Go(func() { d.expiryLoop(ctx) }) + d.wg.Go(func() { d.purgeLoop(ctx) }) + } } if d.insertCh != nil { for i := 0; i < insertWorkers; i++ { From 82ac630d3cd79242f2a25fcad1b5305e35283f4b Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Mon, 11 May 2026 16:33:26 -0400 Subject: [PATCH 44/45] Fix host process flag + tests --- pkg/beholder/durable_emitter.go | 44 ++++++++------- pkg/beholder/durable_emitter_test.go | 81 ++++++++++++++++++++++------ 2 files changed, 91 insertions(+), 34 deletions(-) diff --git a/pkg/beholder/durable_emitter.go b/pkg/beholder/durable_emitter.go index 7b6b97e9e1..cd28bca41e 100644 --- a/pkg/beholder/durable_emitter.go +++ b/pkg/beholder/durable_emitter.go @@ -135,8 +135,11 @@ type insertResult struct { type DurableEmitter struct { store DurableEventStore client chipingress.Client - cfg DurableEmitterConfig - log logger.Logger + // isHostProcess determines if the emitter runs retransmit and cleanup loops. + // Should be set to false when initialized inside LOOP plugins. + isHostProcess bool + cfg DurableEmitterConfig + log logger.Logger metrics *durableEmitterMetrics @@ -222,6 +225,7 @@ var _ Emitter = (*DurableEmitter)(nil) func NewDurableEmitter( store DurableEventStore, client chipingress.Client, + isHostProcess bool, cfg DurableEmitterConfig, log logger.Logger, ) (*DurableEmitter, error) { @@ -247,12 +251,13 @@ func NewDurableEmitter( store = newMetricsInstrumentedStore(store, m) } d := &DurableEmitter{ - store: store, - client: client, - cfg: cfg, - log: log, - metrics: m, - stopCh: make(chan struct{}), + store: store, + client: client, + isHostProcess: isHostProcess, + cfg: cfg, + log: log, + metrics: m, + stopCh: make(chan struct{}), } if cp, ok := client.(grpcConnProvider); ok { d.rawConn = cp.Conn() @@ -285,7 +290,7 @@ func NewDurableEmitter( // Start launches the retransmit, expiry, purge, and (optionally) batch publish // background loops. Cancel the supplied context or call Close to stop them. -func (d *DurableEmitter) Start(ctx context.Context) { +func (d *DurableEmitter) Start(_ context.Context) { batchWorkers := d.cfg.PublishBatchWorkers if batchWorkers <= 0 { d.log.Warnw("configured batchWorkers <=0; defaulting to 1") @@ -297,10 +302,12 @@ func (d *DurableEmitter) Start(ctx context.Context) { insertWorkers = 4 } - d.wg.Go(d.retransmitLoop) - if !d.cfg.DisablePruning { - d.wg.Go(func() { d.expiryLoop(ctx) }) - d.wg.Go(func() { d.purgeLoop(ctx) }) + if d.isHostProcess { + d.wg.Go(d.retransmitLoop) + if !d.cfg.DisablePruning { + d.wg.Go(d.expiryLoop) + d.wg.Go(d.purgeLoop) + } } if d.insertCh != nil { for i := 0; i < insertWorkers; i++ { @@ -845,7 +852,7 @@ func (d *DurableEmitter) retransmit(pending []DurableEvent) { ) } -func (d *DurableEmitter) purgeLoop(ctx context.Context) { +func (d *DurableEmitter) purgeLoop() { interval := d.cfg.PurgeInterval if interval <= 0 { interval = 250 * time.Millisecond @@ -854,12 +861,13 @@ func (d *DurableEmitter) purgeLoop(ctx context.Context) { if batch <= 0 { batch = 500 } + + ctx, cancel := d.stopCh.NewCtx() + defer cancel() ticker := time.NewTicker(interval) defer ticker.Stop() for { select { - case <-ctx.Done(): - return case <-d.stopCh: return case <-ticker.C: @@ -877,7 +885,7 @@ func (d *DurableEmitter) purgeLoop(ctx context.Context) { } } -func (d *DurableEmitter) expiryLoop(ctx context.Context) { +func (d *DurableEmitter) expiryLoop() { ticker := time.NewTicker(d.cfg.ExpiryInterval) defer ticker.Stop() @@ -885,8 +893,6 @@ func (d *DurableEmitter) expiryLoop(ctx context.Context) { defer cancel() for { select { - case <-ctx.Done(): - return case <-d.stopCh: return case <-ticker.C: diff --git a/pkg/beholder/durable_emitter_test.go b/pkg/beholder/durable_emitter_test.go index 0ab1e8841f..ad96b53071 100644 --- a/pkg/beholder/durable_emitter_test.go +++ b/pkg/beholder/durable_emitter_test.go @@ -111,7 +111,7 @@ func newTestDurableEmitter(t *testing.T, store DurableEventStore, client chiping if cfgOverride != nil { cfg = *cfgOverride } - em, err := NewDurableEmitter(store, client, cfg, logger.Test(t)) + em, err := NewDurableEmitter(store, client, true, cfg, logger.Test(t)) require.NoError(t, err) return em } @@ -187,7 +187,7 @@ func TestDurableEmitter_HooksBatchPublishPath(t *testing.T) { OnBatchPublish: func(time.Duration, int, error) { pubCalls.Add(1) }, OnBatchMarkDelivered: func(time.Duration, int) { markCalls.Add(1) }, } - em, err := NewDurableEmitter(store, client, cfg, logger.Test(t)) + em, err := NewDurableEmitter(store, client, true, cfg, logger.Test(t)) require.NoError(t, err) ctx, cancel := context.WithCancel(context.Background()) @@ -211,7 +211,7 @@ func TestDurableEmitter_HooksPublishFailureSkipsMarkHook(t *testing.T) { OnBatchPublish: func(time.Duration, int, error) { pubCalls.Add(1) }, OnBatchMarkDelivered: func(time.Duration, int) { markCalls.Add(1) }, } - em, err := NewDurableEmitter(store, client, cfg, logger.Test(t)) + em, err := NewDurableEmitter(store, client, true, cfg, logger.Test(t)) require.NoError(t, err) ctx, cancel := context.WithCancel(context.Background()) @@ -224,6 +224,57 @@ func TestDurableEmitter_HooksPublishFailureSkipsMarkHook(t *testing.T) { assert.Equal(t, int32(0), markCalls.Load()) } +func TestDurableEmitter_NonHostProcessSkipsRetransmitAndExpiry(t *testing.T) { + store := NewMemDurableEventStore() + client := &testChipClient{} + client.setPublishErr(errors.New("chip unavailable")) + + cfg := DefaultDurableEmitterConfig() + cfg.RetransmitInterval = 40 * time.Millisecond + cfg.RetransmitAfter = 15 * time.Millisecond + cfg.ExpiryInterval = 40 * time.Millisecond + cfg.EventTTL = 25 * time.Millisecond + + em, err := NewDurableEmitter(store, client, false, cfg, logger.Test(t)) + require.NoError(t, err) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + em.Start(ctx) + defer func() { require.NoError(t, em.Close()) }() + + require.NoError(t, em.Emit(ctx, []byte("plugin-row"), testEmitAttrs()...)) + + require.Eventually(t, func() bool { + return client.batchCount.Load() >= 1 && store.Len() == 1 + }, 2*time.Second, 5*time.Millisecond, "initial PublishBatch should fail and leave the row") + + // Several host-only ticks would have cleared or retried by now. + time.Sleep(250 * time.Millisecond) + + assert.Equal(t, 1, store.Len(), "non-host must not run retransmit or expiry loops") + assert.Equal(t, int64(1), client.batchCount.Load(), "non-host must not schedule extra PublishBatch via retransmit") +} + +func TestDurableEmitter_NonHostProcessStillDeliversViaBatchWorkers(t *testing.T) { + store := NewMemDurableEventStore() + client := &testChipClient{} + + em, err := NewDurableEmitter(store, client, false, DefaultDurableEmitterConfig(), logger.Test(t)) + require.NoError(t, err) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + em.Start(ctx) + defer func() { require.NoError(t, em.Close()) }() + + require.NoError(t, em.Emit(ctx, []byte("loop-plugin"), testEmitAttrs()...)) + + require.Eventually(t, func() bool { + return store.Len() == 0 && client.batchCount.Load() >= 1 + }, 2*time.Second, 10*time.Millisecond, "batch publish workers must still run when isHostProcess is false") +} + func TestDurableEmitter_EmitPersistsAndPublishes(t *testing.T) { store := NewMemDurableEventStore() client := &testChipClient{} @@ -437,13 +488,13 @@ func TestNewDurableEmitter_ValidationErrors(t *testing.T) { log := logger.Test(t) cfg := DefaultDurableEmitterConfig() - _, err := NewDurableEmitter(nil, &testChipClient{}, cfg, log) + _, err := NewDurableEmitter(nil, &testChipClient{}, true, cfg, log) assert.ErrorContains(t, err, "store") - _, err = NewDurableEmitter(NewMemDurableEventStore(), nil, cfg, log) + _, err = NewDurableEmitter(NewMemDurableEventStore(), nil, true, cfg, log) assert.ErrorContains(t, err, "client") - _, err = NewDurableEmitter(NewMemDurableEventStore(), &testChipClient{}, cfg, nil) + _, err = NewDurableEmitter(NewMemDurableEventStore(), &testChipClient{}, true, cfg, nil) assert.ErrorContains(t, err, "logger") } @@ -456,7 +507,7 @@ func TestDurableEmitter_MetricsRegistersEmitSuccess(t *testing.T) { cfg.RetransmitInterval = time.Hour cfg.Metrics = &DurableEmitterMetricsConfig{PollInterval: 25 * time.Millisecond} - em, err := NewDurableEmitter(store, client, cfg, logger.Test(t)) + em, err := NewDurableEmitter(store, client, true, cfg, logger.Test(t)) require.NoError(t, err) ctx, cancel := context.WithCancel(context.Background()) @@ -603,7 +654,7 @@ func TestIntegration_HappyPath(t *testing.T) { client := newChipClient(t, addr) store := NewMemDurableEventStore() - em, err := NewDurableEmitter(store, client, fastCfg(), logger.Test(t)) + em, err := NewDurableEmitter(store, client, true, fastCfg(), logger.Test(t)) require.NoError(t, err) ctx, cancel := context.WithCancel(context.Background()) @@ -631,7 +682,7 @@ func TestIntegration_ServerUnavailable_RetransmitRecovers(t *testing.T) { client := newChipClient(t, addr) store := NewMemDurableEventStore() - em, err := NewDurableEmitter(store, client, fastCfg(), logger.Test(t)) + em, err := NewDurableEmitter(store, client, true, fastCfg(), logger.Test(t)) require.NoError(t, err) ctx, cancel := context.WithCancel(context.Background()) @@ -666,7 +717,7 @@ func TestIntegration_ServerDown_EventsSurvive(t *testing.T) { cfg := fastCfg() cfg.PublishTimeout = 500 * time.Millisecond - em, err := NewDurableEmitter(store, client, cfg, logger.Test(t)) + em, err := NewDurableEmitter(store, client, true, cfg, logger.Test(t)) require.NoError(t, err) ctx, cancel := context.WithCancel(context.Background()) @@ -699,7 +750,7 @@ func TestIntegration_ServerDown_EventsSurvive(t *testing.T) { require.NoError(t, err) t.Cleanup(func() { _ = client2.Close() }) - em2, err := NewDurableEmitter(store, client2, cfg, logger.Test(t)) + em2, err := NewDurableEmitter(store, client2, true, cfg, logger.Test(t)) require.NoError(t, err) em2.Start(ctx) defer em2.Close() @@ -721,7 +772,7 @@ func TestIntegration_HighThroughput(t *testing.T) { cfg := fastCfg() cfg.RetransmitBatchSize = 200 - em, err := NewDurableEmitter(store, client, cfg, logger.Test(t)) + em, err := NewDurableEmitter(store, client, true, cfg, logger.Test(t)) require.NoError(t, err) ctx, cancel := context.WithCancel(context.Background()) @@ -754,7 +805,7 @@ func TestIntegration_EventExpiry(t *testing.T) { cfg := fastCfg() cfg.EventTTL = 100 * time.Millisecond cfg.ExpiryInterval = 100 * time.Millisecond - em, err := NewDurableEmitter(store, client, cfg, logger.Test(t)) + em, err := NewDurableEmitter(store, client, true, cfg, logger.Test(t)) require.NoError(t, err) ctx, cancel := context.WithCancel(context.Background()) @@ -779,7 +830,7 @@ func TestIntegration_RetransmitEnqueuesBatchWorkers(t *testing.T) { client := newChipClient(t, addr) store := NewMemDurableEventStore() - em, err := NewDurableEmitter(store, client, fastCfg(), logger.Test(t)) + em, err := NewDurableEmitter(store, client, true, fastCfg(), logger.Test(t)) require.NoError(t, err) ctx, cancel := context.WithCancel(context.Background()) @@ -830,7 +881,7 @@ func TestIntegration_GRPCConnection(t *testing.T) { client := newChipClient(t, addr) store := NewMemDurableEventStore() - em, err := NewDurableEmitter(store, client, fastCfg(), logger.Test(t)) + em, err := NewDurableEmitter(store, client, true, fastCfg(), logger.Test(t)) require.NoError(t, err) ctx, cancel := context.WithCancel(context.Background()) From 8a894b35f3f7469dffc8cc9280f6912226f49102 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Tue, 12 May 2026 10:30:58 -0400 Subject: [PATCH 45/45] Remove client changes --- pkg/chipingress/client.go | 6 ------ pkg/chipingress/client_test.go | 23 ----------------------- 2 files changed, 29 deletions(-) diff --git a/pkg/chipingress/client.go b/pkg/chipingress/client.go index 586fb158a8..2c760ab9d7 100644 --- a/pkg/chipingress/client.go +++ b/pkg/chipingress/client.go @@ -268,8 +268,6 @@ func newHeaderInterceptor(provider HeaderProvider) grpc.UnaryClientInterceptor { } // NewEvent creates a new CloudEvent with the specified domain, entity, payload, and optional attributes. -// Recognized optional keys include CloudEvents names (dataschema, subject, time, …) and Beholder's -// beholder_data_schema, which is mapped to the CloudEvent dataschema when dataschema is not set. func NewEvent(domain, entity string, payload []byte, attributes map[string]any) (CloudEvent, error) { event := ce.NewEvent() @@ -282,8 +280,6 @@ func NewEvent(domain, entity string, payload []byte, attributes map[string]any) attributes = make(map[string]any) } - const beholderDataSchemaKey = "beholder_data_schema" - recordedTime := time.Now() if val, ok := attributes["recordedtime"].(time.Time); ok && !val.IsZero() { recordedTime = val @@ -299,8 +295,6 @@ func NewEvent(domain, entity string, payload []byte, attributes map[string]any) } if val, ok := attributes["dataschema"].(string); ok { event.SetDataSchema(val) - } else if val, ok := attributes[beholderDataSchemaKey].(string); ok { - event.SetDataSchema(val) } if val, ok := attributes["subject"].(string); ok { event.SetSubject(val) diff --git a/pkg/chipingress/client_test.go b/pkg/chipingress/client_test.go index 82f32e749c..6b259460a6 100644 --- a/pkg/chipingress/client_test.go +++ b/pkg/chipingress/client_test.go @@ -126,29 +126,6 @@ func TestNewEvent(t *testing.T) { assert.Equal(t, testProto.Message, resultProto.Message) } -func TestNewEventBeholderDataSchema(t *testing.T) { - testProto := pb.PingResponse{Message: "x"} - protoBytes, err := proto.Marshal(&testProto) - require.NoError(t, err) - - t.Run("beholder_data_schema sets CloudEvent dataschema", func(t *testing.T) { - event, err := NewEvent("platform", "workflows.v2.WorkflowUserLog", protoBytes, map[string]any{ - "beholder_data_schema": "/cre-events-user-logs/v2", - }) - require.NoError(t, err) - assert.Equal(t, "/cre-events-user-logs/v2", event.DataSchema()) - }) - - t.Run("dataschema takes precedence over beholder_data_schema", func(t *testing.T) { - event, err := NewEvent("platform", "workflows.v2.WorkflowUserLog", protoBytes, map[string]any{ - "dataschema": "https://explicit.example/schema", - "beholder_data_schema": "/ignored", - }) - require.NoError(t, err) - assert.Equal(t, "https://explicit.example/schema", event.DataSchema()) - }) -} - func TestEventToProto(t *testing.T) { // Create a test protobuf message testProto := pb.PingResponse{Message: "test message"}