From 1ef971ffe50052eb5c7ce6e6444e7152227e967d Mon Sep 17 00:00:00 2001 From: Krish-vemula Date: Wed, 11 Feb 2026 10:42:34 -0800 Subject: [PATCH 1/8] add configurable historical balance health check for finalized block support --- CONFIG.md | 19 ++++ pkg/client/helpers_test.go | 10 ++ pkg/client/rpc_client.go | 29 +++++ pkg/client/rpc_client_internal_test.go | 132 ++++++++++++++++++++++ pkg/config/chain_scoped_node_pool.go | 16 +++ pkg/config/config.go | 2 + pkg/config/config_test.go | 2 + pkg/config/toml/config.go | 11 ++ pkg/config/toml/config_test.go | 2 + pkg/config/toml/defaults/fallback.toml | 2 + pkg/config/toml/docs.toml | 9 ++ pkg/config/toml/testdata/config-full.toml | 2 + 12 files changed, 236 insertions(+) diff --git a/CONFIG.md b/CONFIG.md index b5eaaf1354..93c9d7faf3 100644 --- a/CONFIG.md +++ b/CONFIG.md @@ -911,6 +911,8 @@ SyncThreshold = 5 # Default LeaseDuration = '0s' # Default NodeIsSyncingEnabled = false # Default FinalizedBlockPollInterval = '5s' # Default +HistoricalBalanceCheckEnabled = false # Default +HistoricalBalanceCheckAddress = '0x0000000000000000000000000000000000000000' # Default EnforceRepeatableRead = true # Default DeathDeclarationDelay = '1m' # Default NewHeadsPollInterval = '0s' # Default @@ -989,6 +991,23 @@ reported based on latest block and finality depth. Set to 0 to disable. +### HistoricalBalanceCheckEnabled +```toml +HistoricalBalanceCheckEnabled = false # Default +``` +HistoricalBalanceCheckEnabled controls whether NodePool health polling verifies historical state availability +via `eth_getBalance` against `HistoricalBalanceCheckAddress`. + +When enabled, the probe reads balance at the latest finalized block: +- if `FinalityTagEnabled = true`, it uses the `finalized` block tag +- if `FinalityTagEnabled = false`, it uses `latest - FinalityDepth` + +### HistoricalBalanceCheckAddress +```toml +HistoricalBalanceCheckAddress = '0x0000000000000000000000000000000000000000' # Default +``` +HistoricalBalanceCheckAddress is the account used by the historical balance health check probe. + ### EnforceRepeatableRead ```toml EnforceRepeatableRead = true # Default diff --git a/pkg/client/helpers_test.go b/pkg/client/helpers_test.go index 7734ea1bc6..ddaa96203b 100644 --- a/pkg/client/helpers_test.go +++ b/pkg/client/helpers_test.go @@ -95,6 +95,8 @@ type TestNodePoolConfig struct { NodeLeaseDuration time.Duration NodeIsSyncingEnabledVal bool NodeFinalizedBlockPollInterval time.Duration + HistoricalBalanceCheckEnabledVal bool + HistoricalBalanceCheckAddressVal string NodeErrors config.ClientErrors EnforceRepeatableReadVal bool NodeDeathDeclarationDelay time.Duration @@ -118,6 +120,14 @@ func (tc TestNodePoolConfig) FinalizedBlockPollInterval() time.Duration { return tc.NodeFinalizedBlockPollInterval } +func (tc TestNodePoolConfig) HistoricalBalanceCheckEnabled() bool { + return tc.HistoricalBalanceCheckEnabledVal +} + +func (tc TestNodePoolConfig) HistoricalBalanceCheckAddress() string { + return tc.HistoricalBalanceCheckAddressVal +} + func (tc TestNodePoolConfig) NewHeadsPollInterval() time.Duration { return tc.NodeNewHeadsPollInterval } diff --git a/pkg/client/rpc_client.go b/pkg/client/rpc_client.go index e66765d574..aa2649791e 100644 --- a/pkg/client/rpc_client.go +++ b/pkg/client/rpc_client.go @@ -105,6 +105,8 @@ type RPCClient struct { finalityTagEnabled bool finalityDepth uint32 safeDepth uint32 + historicalBalanceCheckEnabled bool + historicalBalanceCheckAddress common.Address externalRequestMaxResponseSize uint32 ws atomic.Pointer[rawclient] @@ -142,6 +144,8 @@ func NewRPCClient( finalityTagEnabled: supportsFinalityTags, finalityDepth: finalityDepth, safeDepth: safeDepth, + historicalBalanceCheckEnabled: cfg.HistoricalBalanceCheckEnabled(), + historicalBalanceCheckAddress: common.HexToAddress(cfg.HistoricalBalanceCheckAddress()), externalRequestMaxResponseSize: externalRequestMaxResponseSize, } r.cfg = cfg @@ -180,10 +184,35 @@ func (r *RPCClient) ClientVersion(ctx context.Context) (version string, err erro if err != nil { return "", fmt.Errorf("fetching client version failed: %w", err) } + if r.historicalBalanceCheckEnabled { + if err = r.checkHistoricalStateAtFinalized(ctx); err != nil { + return "", fmt.Errorf("historical balance health check failed: %w", err) + } + } r.rpcLog.Debugf("client version: %s", version) return version, nil } +func (r *RPCClient) checkHistoricalStateAtFinalized(ctx context.Context) error { + var blockNumber *big.Int + if r.finalityTagEnabled { + blockNumber = big.NewInt(rpc.FinalizedBlockNumber.Int64()) + } else { + latestBlock, err := r.BlockNumber(ctx) + if err != nil { + return fmt.Errorf("fetching latest block number failed: %w", err) + } + latest := int64(latestBlock) + finalizedHeight := max(int64(0), latest-int64(r.finalityDepth)) + blockNumber = big.NewInt(finalizedHeight) + } + _, err := r.BalanceAt(ctx, r.historicalBalanceCheckAddress, blockNumber) + if err != nil { + return fmt.Errorf("fetching balance for address %s at block %s failed: %w", r.historicalBalanceCheckAddress.String(), blockNumber.String(), err) + } + return nil +} + func (r *RPCClient) Dial(callerCtx context.Context) error { ctx, cancel, _ := r.AcquireQueryCtx(callerCtx, r.rpcTimeout) defer cancel() diff --git a/pkg/client/rpc_client_internal_test.go b/pkg/client/rpc_client_internal_test.go index b8a3b1fa75..1e75637403 100644 --- a/pkg/client/rpc_client_internal_test.go +++ b/pkg/client/rpc_client_internal_test.go @@ -436,3 +436,135 @@ func NewTestRPCClient(t *testing.T, opts RPCClientOpts) *RPCClient { func ptr[T any](v T) *T { return &v } + +func TestRPCClient_ClientVersion_HistoricalBalanceCheck(t *testing.T) { + t.Parallel() + chainID := big.NewInt(1337) + probeAddress := "0x0000000000000000000000000000000000000001" + + t.Run("disabled only calls web3_clientVersion", func(t *testing.T) { + t.Parallel() + methodCalls := make([]string, 0) + wsURL := testutils.NewWSServer(t, chainID, func(method string, _ gjson.Result) (resp testutils.JSONRPCResponse) { + methodCalls = append(methodCalls, method) + switch method { + case "web3_clientVersion": + resp.Result = `"test-client"` + default: + require.Fail(t, "unexpected method: "+method) + } + return + }).WSURL() + + rpcClient := NewDialedTestRPCClient(t, RPCClientOpts{ + HTTP: wsURL, + Cfg: &TestNodePoolConfig{ + NodeFinalizedBlockPollInterval: 1 * time.Second, + }, + FinalityTagsEnabled: true, + ChainID: chainID, + }) + + version, err := rpcClient.ClientVersion(t.Context()) + require.NoError(t, err) + require.Equal(t, "test-client", version) + require.Equal(t, []string{"web3_clientVersion"}, methodCalls) + }) + + t.Run("enabled uses finalized tag", func(t *testing.T) { + t.Parallel() + wsURL := testutils.NewWSServer(t, chainID, func(method string, params gjson.Result) (resp testutils.JSONRPCResponse) { + switch method { + case "web3_clientVersion": + resp.Result = `"test-client"` + case "eth_getBalance": + require.Equal(t, probeAddress, params.Array()[0].String()) + require.Equal(t, "finalized", params.Array()[1].String()) + resp.Result = `"0x0"` + default: + require.Fail(t, "unexpected method: "+method) + } + return + }).WSURL() + + rpcClient := NewDialedTestRPCClient(t, RPCClientOpts{ + HTTP: wsURL, + Cfg: &TestNodePoolConfig{ + NodeFinalizedBlockPollInterval: 1 * time.Second, + HistoricalBalanceCheckEnabledVal: true, + HistoricalBalanceCheckAddressVal: probeAddress, + }, + FinalityTagsEnabled: true, + ChainID: chainID, + }) + + version, err := rpcClient.ClientVersion(t.Context()) + require.NoError(t, err) + require.Equal(t, "test-client", version) + }) + + t.Run("enabled in depth mode uses latest-finalityDepth", func(t *testing.T) { + t.Parallel() + wsURL := testutils.NewWSServer(t, chainID, func(method string, params gjson.Result) (resp testutils.JSONRPCResponse) { + switch method { + case "web3_clientVersion": + resp.Result = `"test-client"` + case "eth_blockNumber": + resp.Result = `"0x14"` // 20 + case "eth_getBalance": + require.Equal(t, probeAddress, params.Array()[0].String()) + require.Equal(t, "0x10", params.Array()[1].String()) // 20 - 4 + resp.Result = `"0x0"` + default: + require.Fail(t, "unexpected method: "+method) + } + return + }).WSURL() + + rpcClient := NewDialedTestRPCClient(t, RPCClientOpts{ + HTTP: wsURL, + Cfg: &TestNodePoolConfig{ + NodeFinalizedBlockPollInterval: 1 * time.Second, + HistoricalBalanceCheckEnabledVal: true, + HistoricalBalanceCheckAddressVal: probeAddress, + }, + FinalityTagsEnabled: false, + FinalityDepth: 4, + ChainID: chainID, + }) + + version, err := rpcClient.ClientVersion(t.Context()) + require.NoError(t, err) + require.Equal(t, "test-client", version) + }) + + t.Run("probe failure returns health check error", func(t *testing.T) { + t.Parallel() + wsURL := testutils.NewWSServer(t, chainID, func(method string, _ gjson.Result) (resp testutils.JSONRPCResponse) { + switch method { + case "web3_clientVersion": + resp.Result = `"test-client"` + case "eth_getBalance": + resp.Error.Message = "balance failure" + default: + require.Fail(t, "unexpected method: "+method) + } + return + }).WSURL() + + rpcClient := NewDialedTestRPCClient(t, RPCClientOpts{ + HTTP: wsURL, + Cfg: &TestNodePoolConfig{ + NodeFinalizedBlockPollInterval: 1 * time.Second, + HistoricalBalanceCheckEnabledVal: true, + HistoricalBalanceCheckAddressVal: probeAddress, + }, + FinalityTagsEnabled: true, + ChainID: chainID, + }) + + _, err := rpcClient.ClientVersion(t.Context()) + require.Error(t, err) + require.ErrorContains(t, err, "historical balance health check failed") + }) +} diff --git a/pkg/config/chain_scoped_node_pool.go b/pkg/config/chain_scoped_node_pool.go index d9cd17ee72..7ed6304902 100644 --- a/pkg/config/chain_scoped_node_pool.go +++ b/pkg/config/chain_scoped_node_pool.go @@ -3,6 +3,8 @@ package config import ( "time" + "github.com/ethereum/go-ethereum/common" + "github.com/smartcontractkit/chainlink-evm/pkg/config/toml" ) @@ -38,6 +40,20 @@ func (n *NodePoolConfig) FinalizedBlockPollInterval() time.Duration { return n.C.FinalizedBlockPollInterval.Duration() } +func (n *NodePoolConfig) HistoricalBalanceCheckEnabled() bool { + if n.C.HistoricalBalanceCheckEnabled == nil { + return false + } + return *n.C.HistoricalBalanceCheckEnabled +} + +func (n *NodePoolConfig) HistoricalBalanceCheckAddress() string { + if n.C.HistoricalBalanceCheckAddress == nil { + return common.Address{}.String() + } + return n.C.HistoricalBalanceCheckAddress.String() +} + func (n *NodePoolConfig) NewHeadsPollInterval() time.Duration { return n.C.NewHeadsPollInterval.Duration() } diff --git a/pkg/config/config.go b/pkg/config/config.go index 9593c96d96..0a08a0fd85 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -208,6 +208,8 @@ type NodePool interface { LeaseDuration() time.Duration NodeIsSyncingEnabled() bool FinalizedBlockPollInterval() time.Duration + HistoricalBalanceCheckEnabled() bool + HistoricalBalanceCheckAddress() string Errors() ClientErrors EnforceRepeatableRead() bool DeathDeclarationDelay() time.Duration diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 27ff81c9b5..f6450a1138 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -369,6 +369,8 @@ func TestNodePoolConfig(t *testing.T) { require.False(t, cfg.EVM().NodePool().NodeIsSyncingEnabled()) require.True(t, cfg.EVM().NodePool().EnforceRepeatableRead()) require.Equal(t, time.Minute, cfg.EVM().NodePool().DeathDeclarationDelay()) + require.False(t, cfg.EVM().NodePool().HistoricalBalanceCheckEnabled()) + require.Equal(t, "0x0000000000000000000000000000000000000000", cfg.EVM().NodePool().HistoricalBalanceCheckAddress()) } func TestClientErrorsConfig(t *testing.T) { diff --git a/pkg/config/toml/config.go b/pkg/config/toml/config.go index 1e70fe46a1..ec8f20c75f 100644 --- a/pkg/config/toml/config.go +++ b/pkg/config/toml/config.go @@ -1085,6 +1085,8 @@ type NodePool struct { LeaseDuration *commonconfig.Duration NodeIsSyncingEnabled *bool FinalizedBlockPollInterval *commonconfig.Duration + HistoricalBalanceCheckEnabled *bool + HistoricalBalanceCheckAddress *types.EIP55Address Errors ClientErrors `toml:",omitempty"` EnforceRepeatableRead *bool DeathDeclarationDelay *commonconfig.Duration @@ -1115,6 +1117,12 @@ func (p *NodePool) setFrom(f *NodePool) { if v := f.FinalizedBlockPollInterval; v != nil { p.FinalizedBlockPollInterval = v } + if v := f.HistoricalBalanceCheckEnabled; v != nil { + p.HistoricalBalanceCheckEnabled = v + } + if v := f.HistoricalBalanceCheckAddress; v != nil { + p.HistoricalBalanceCheckAddress = v + } if v := f.EnforceRepeatableRead; v != nil { p.EnforceRepeatableRead = v @@ -1150,6 +1158,9 @@ func (p *NodePool) ValidateConfig(finalityTagEnabled *bool) (err error) { Msg: "must be greater than 0"}) } } + if p.HistoricalBalanceCheckEnabled != nil && *p.HistoricalBalanceCheckEnabled && p.HistoricalBalanceCheckAddress == nil { + err = multierr.Append(err, commonconfig.ErrMissing{Name: "HistoricalBalanceCheckAddress", Msg: "required when HistoricalBalanceCheckEnabled is true"}) + } return } diff --git a/pkg/config/toml/config_test.go b/pkg/config/toml/config_test.go index ff92277074..4ee0cc44b3 100644 --- a/pkg/config/toml/config_test.go +++ b/pkg/config/toml/config_test.go @@ -306,6 +306,8 @@ var fullConfig = EVMConfig{ LeaseDuration: config.MustNewDuration(0), NodeIsSyncingEnabled: ptr(true), FinalizedBlockPollInterval: config.MustNewDuration(time.Second), + HistoricalBalanceCheckEnabled: ptr(true), + HistoricalBalanceCheckAddress: ptr(types.MustEIP55Address("0x0000000000000000000000000000000000000001")), EnforceRepeatableRead: ptr(true), DeathDeclarationDelay: config.MustNewDuration(time.Minute), VerifyChainID: ptr(true), diff --git a/pkg/config/toml/defaults/fallback.toml b/pkg/config/toml/defaults/fallback.toml index 7d8ea77644..17bd1f5d47 100644 --- a/pkg/config/toml/defaults/fallback.toml +++ b/pkg/config/toml/defaults/fallback.toml @@ -84,6 +84,8 @@ SyncThreshold = 5 LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' +HistoricalBalanceCheckEnabled = false +HistoricalBalanceCheckAddress = '0x0000000000000000000000000000000000000000' EnforceRepeatableRead = true DeathDeclarationDelay = '1m' NewHeadsPollInterval = '0s' diff --git a/pkg/config/toml/docs.toml b/pkg/config/toml/docs.toml index 72947f8394..679197bb87 100644 --- a/pkg/config/toml/docs.toml +++ b/pkg/config/toml/docs.toml @@ -452,6 +452,15 @@ NodeIsSyncingEnabled = false # Default # # Set to 0 to disable. FinalizedBlockPollInterval = '5s' # Default +# HistoricalBalanceCheckEnabled controls whether NodePool health polling also verifies historical state availability +# by executing `eth_getBalance` for HistoricalBalanceCheckAddress at the latest finalized block. +# Finalized block selection follows chain finality settings: +# - `FinalityTagEnabled = true`: use `finalized` tag +# - `FinalityTagEnabled = false`: use `latest - FinalityDepth` +HistoricalBalanceCheckEnabled = false # Default +# HistoricalBalanceCheckAddress is the probe account used by the historical balance health check. +# This check is only active when `HistoricalBalanceCheckEnabled = true`. +HistoricalBalanceCheckAddress = '0x0000000000000000000000000000000000000000' # Default # EnforceRepeatableRead defines if Core should only use RPCs whose most recently finalized block is greater or equal to # `highest finalized block - FinalizedBlockOffset`. In other words, exclude RPCs lagging on latest finalized # block. diff --git a/pkg/config/toml/testdata/config-full.toml b/pkg/config/toml/testdata/config-full.toml index 0284665631..caec9035e3 100644 --- a/pkg/config/toml/testdata/config-full.toml +++ b/pkg/config/toml/testdata/config-full.toml @@ -118,6 +118,8 @@ SyncThreshold = 13 LeaseDuration = '0s' NodeIsSyncingEnabled = true FinalizedBlockPollInterval = '1s' +HistoricalBalanceCheckEnabled = true +HistoricalBalanceCheckAddress = '0x0000000000000000000000000000000000000001' EnforceRepeatableRead = true DeathDeclarationDelay = '1m0s' NewHeadsPollInterval = '0s' From e4db36934ed9cd779385e8697343afd5851a9f36 Mon Sep 17 00:00:00 2001 From: Krish-vemula Date: Wed, 18 Feb 2026 13:27:35 -0800 Subject: [PATCH 2/8] added fix for build issues --- CONFIG.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/CONFIG.md b/CONFIG.md index 93c9d7faf3..2f8e742cac 100644 --- a/CONFIG.md +++ b/CONFIG.md @@ -995,18 +995,18 @@ Set to 0 to disable. ```toml HistoricalBalanceCheckEnabled = false # Default ``` -HistoricalBalanceCheckEnabled controls whether NodePool health polling verifies historical state availability -via `eth_getBalance` against `HistoricalBalanceCheckAddress`. - -When enabled, the probe reads balance at the latest finalized block: -- if `FinalityTagEnabled = true`, it uses the `finalized` block tag -- if `FinalityTagEnabled = false`, it uses `latest - FinalityDepth` +HistoricalBalanceCheckEnabled controls whether NodePool health polling also verifies historical state availability +by executing `eth_getBalance` for HistoricalBalanceCheckAddress at the latest finalized block. +Finalized block selection follows chain finality settings: +- `FinalityTagEnabled = true`: use `finalized` tag +- `FinalityTagEnabled = false`: use `latest - FinalityDepth` ### HistoricalBalanceCheckAddress ```toml HistoricalBalanceCheckAddress = '0x0000000000000000000000000000000000000000' # Default ``` -HistoricalBalanceCheckAddress is the account used by the historical balance health check probe. +HistoricalBalanceCheckAddress is the probe account used by the historical balance health check. +This check is only active when `HistoricalBalanceCheckEnabled = true`. ### EnforceRepeatableRead ```toml From f3dbf9d8750935e8e7583f431112628b644f4a00 Mon Sep 17 00:00:00 2001 From: Krish-vemula Date: Mon, 2 Mar 2026 09:50:15 -0600 Subject: [PATCH 3/8] Make CheckFinalizedStateAvailability public for multinode polling, Accept probeAddress parameter, fall back to EVM config if empty. --- pkg/client/helpers_test.go | 48 ++++++++++++----- pkg/client/rpc_client.go | 24 ++++++--- pkg/client/rpc_client_internal_test.go | 73 ++++++-------------------- 3 files changed, 66 insertions(+), 79 deletions(-) diff --git a/pkg/client/helpers_test.go b/pkg/client/helpers_test.go index ddaa96203b..2d749faa3e 100644 --- a/pkg/client/helpers_test.go +++ b/pkg/client/helpers_test.go @@ -88,20 +88,24 @@ func (c *TestClientErrors) TooManyResults() string { return c.tooManyRe func (c *TestClientErrors) MissingBlocks() string { return c.missingBlocks } type TestNodePoolConfig struct { - NodePollFailureThreshold uint32 - NodePollInterval time.Duration - NodeSelectionMode string - NodeSyncThreshold uint32 - NodeLeaseDuration time.Duration - NodeIsSyncingEnabledVal bool - NodeFinalizedBlockPollInterval time.Duration - HistoricalBalanceCheckEnabledVal bool - HistoricalBalanceCheckAddressVal string - NodeErrors config.ClientErrors - EnforceRepeatableReadVal bool - NodeDeathDeclarationDelay time.Duration - NodeNewHeadsPollInterval time.Duration - ExternalRequestMaxResponseSizeVal uint32 + NodePollFailureThreshold uint32 + NodePollInterval time.Duration + NodeSelectionMode string + NodeSyncThreshold uint32 + NodeLeaseDuration time.Duration + NodeIsSyncingEnabledVal bool + NodeFinalizedBlockPollInterval time.Duration + HistoricalBalanceCheckEnabledVal bool + HistoricalBalanceCheckAddressVal string + NodeErrors config.ClientErrors + EnforceRepeatableReadVal bool + NodeDeathDeclarationDelay time.Duration + NodeNewHeadsPollInterval time.Duration + ExternalRequestMaxResponseSizeVal uint32 + FinalizedStateCheckEnabledVal bool + FinalizedStateCheckAddressVal string + FinalizedStateCheckFailureThresholdVal uint32 + FinalizedStateUnavailableRegexVal string } func (tc TestNodePoolConfig) PollFailureThreshold() uint32 { return tc.NodePollFailureThreshold } @@ -152,6 +156,22 @@ func (tc TestNodePoolConfig) ExternalRequestMaxResponseSize() uint32 { return tc.ExternalRequestMaxResponseSizeVal } +func (tc TestNodePoolConfig) FinalizedStateCheckEnabled() bool { + return tc.FinalizedStateCheckEnabledVal +} + +func (tc TestNodePoolConfig) FinalizedStateCheckAddress() string { + return tc.FinalizedStateCheckAddressVal +} + +func (tc TestNodePoolConfig) FinalizedStateCheckFailureThreshold() uint32 { + return tc.FinalizedStateCheckFailureThresholdVal +} + +func (tc TestNodePoolConfig) FinalizedStateUnavailableRegex() string { + return tc.FinalizedStateUnavailableRegexVal +} + func NewChainClientWithTestNode( t *testing.T, nodeCfg multinode.NodeConfig, diff --git a/pkg/client/rpc_client.go b/pkg/client/rpc_client.go index aa2649791e..2b780a7e9f 100644 --- a/pkg/client/rpc_client.go +++ b/pkg/client/rpc_client.go @@ -184,16 +184,24 @@ func (r *RPCClient) ClientVersion(ctx context.Context) (version string, err erro if err != nil { return "", fmt.Errorf("fetching client version failed: %w", err) } - if r.historicalBalanceCheckEnabled { - if err = r.checkHistoricalStateAtFinalized(ctx); err != nil { - return "", fmt.Errorf("historical balance health check failed: %w", err) - } - } r.rpcLog.Debugf("client version: %s", version) return version, nil } -func (r *RPCClient) checkHistoricalStateAtFinalized(ctx context.Context) error { +// CheckFinalizedStateAvailability verifies if the RPC can serve historical state at the finalized block. +// This is used to detect non-archive nodes that cannot serve state queries for older blocks. +// If probeAddress is provided, it uses that address; otherwise falls back to the configured HistoricalBalanceCheckAddress. +// Returns nil immediately if historical balance check is not enabled and no probeAddress is provided. +func (r *RPCClient) CheckFinalizedStateAvailability(ctx context.Context, probeAddress string) error { + var addr common.Address + if probeAddress != "" { + addr = common.HexToAddress(probeAddress) + } else if r.historicalBalanceCheckEnabled { + addr = r.historicalBalanceCheckAddress + } else { + return nil + } + var blockNumber *big.Int if r.finalityTagEnabled { blockNumber = big.NewInt(rpc.FinalizedBlockNumber.Int64()) @@ -206,9 +214,9 @@ func (r *RPCClient) checkHistoricalStateAtFinalized(ctx context.Context) error { finalizedHeight := max(int64(0), latest-int64(r.finalityDepth)) blockNumber = big.NewInt(finalizedHeight) } - _, err := r.BalanceAt(ctx, r.historicalBalanceCheckAddress, blockNumber) + _, err := r.BalanceAt(ctx, addr, blockNumber) if err != nil { - return fmt.Errorf("fetching balance for address %s at block %s failed: %w", r.historicalBalanceCheckAddress.String(), blockNumber.String(), err) + return fmt.Errorf("fetching balance for address %s at block %s failed: %w", addr.String(), blockNumber.String(), err) } return nil } diff --git a/pkg/client/rpc_client_internal_test.go b/pkg/client/rpc_client_internal_test.go index 1e75637403..517bedfbbe 100644 --- a/pkg/client/rpc_client_internal_test.go +++ b/pkg/client/rpc_client_internal_test.go @@ -10,6 +10,7 @@ import ( "testing" "time" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" ethtypes "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/rpc" @@ -437,48 +438,18 @@ func ptr[T any](v T) *T { return &v } -func TestRPCClient_ClientVersion_HistoricalBalanceCheck(t *testing.T) { +func TestRPCClient_CheckFinalizedStateAvailability(t *testing.T) { t.Parallel() chainID := big.NewInt(1337) probeAddress := "0x0000000000000000000000000000000000000001" + expectedAddress := common.HexToAddress(probeAddress).String() - t.Run("disabled only calls web3_clientVersion", func(t *testing.T) { - t.Parallel() - methodCalls := make([]string, 0) - wsURL := testutils.NewWSServer(t, chainID, func(method string, _ gjson.Result) (resp testutils.JSONRPCResponse) { - methodCalls = append(methodCalls, method) - switch method { - case "web3_clientVersion": - resp.Result = `"test-client"` - default: - require.Fail(t, "unexpected method: "+method) - } - return - }).WSURL() - - rpcClient := NewDialedTestRPCClient(t, RPCClientOpts{ - HTTP: wsURL, - Cfg: &TestNodePoolConfig{ - NodeFinalizedBlockPollInterval: 1 * time.Second, - }, - FinalityTagsEnabled: true, - ChainID: chainID, - }) - - version, err := rpcClient.ClientVersion(t.Context()) - require.NoError(t, err) - require.Equal(t, "test-client", version) - require.Equal(t, []string{"web3_clientVersion"}, methodCalls) - }) - - t.Run("enabled uses finalized tag", func(t *testing.T) { + t.Run("uses finalized tag when enabled", func(t *testing.T) { t.Parallel() wsURL := testutils.NewWSServer(t, chainID, func(method string, params gjson.Result) (resp testutils.JSONRPCResponse) { switch method { - case "web3_clientVersion": - resp.Result = `"test-client"` case "eth_getBalance": - require.Equal(t, probeAddress, params.Array()[0].String()) + require.Equal(t, expectedAddress, params.Array()[0].String()) require.Equal(t, "finalized", params.Array()[1].String()) resp.Result = `"0x0"` default: @@ -490,29 +461,24 @@ func TestRPCClient_ClientVersion_HistoricalBalanceCheck(t *testing.T) { rpcClient := NewDialedTestRPCClient(t, RPCClientOpts{ HTTP: wsURL, Cfg: &TestNodePoolConfig{ - NodeFinalizedBlockPollInterval: 1 * time.Second, - HistoricalBalanceCheckEnabledVal: true, - HistoricalBalanceCheckAddressVal: probeAddress, + NodeFinalizedBlockPollInterval: 1 * time.Second, }, FinalityTagsEnabled: true, ChainID: chainID, }) - version, err := rpcClient.ClientVersion(t.Context()) + err := rpcClient.CheckFinalizedStateAvailability(t.Context(), probeAddress) require.NoError(t, err) - require.Equal(t, "test-client", version) }) - t.Run("enabled in depth mode uses latest-finalityDepth", func(t *testing.T) { + t.Run("uses latest-finalityDepth when finality tags disabled", func(t *testing.T) { t.Parallel() wsURL := testutils.NewWSServer(t, chainID, func(method string, params gjson.Result) (resp testutils.JSONRPCResponse) { switch method { - case "web3_clientVersion": - resp.Result = `"test-client"` case "eth_blockNumber": resp.Result = `"0x14"` // 20 case "eth_getBalance": - require.Equal(t, probeAddress, params.Array()[0].String()) + require.Equal(t, expectedAddress, params.Array()[0].String()) require.Equal(t, "0x10", params.Array()[1].String()) // 20 - 4 resp.Result = `"0x0"` default: @@ -524,28 +490,23 @@ func TestRPCClient_ClientVersion_HistoricalBalanceCheck(t *testing.T) { rpcClient := NewDialedTestRPCClient(t, RPCClientOpts{ HTTP: wsURL, Cfg: &TestNodePoolConfig{ - NodeFinalizedBlockPollInterval: 1 * time.Second, - HistoricalBalanceCheckEnabledVal: true, - HistoricalBalanceCheckAddressVal: probeAddress, + NodeFinalizedBlockPollInterval: 1 * time.Second, }, FinalityTagsEnabled: false, FinalityDepth: 4, ChainID: chainID, }) - version, err := rpcClient.ClientVersion(t.Context()) + err := rpcClient.CheckFinalizedStateAvailability(t.Context(), probeAddress) require.NoError(t, err) - require.Equal(t, "test-client", version) }) - t.Run("probe failure returns health check error", func(t *testing.T) { + t.Run("returns error when RPC fails", func(t *testing.T) { t.Parallel() wsURL := testutils.NewWSServer(t, chainID, func(method string, _ gjson.Result) (resp testutils.JSONRPCResponse) { switch method { - case "web3_clientVersion": - resp.Result = `"test-client"` case "eth_getBalance": - resp.Error.Message = "balance failure" + resp.Error.Message = "missing trie node" default: require.Fail(t, "unexpected method: "+method) } @@ -555,16 +516,14 @@ func TestRPCClient_ClientVersion_HistoricalBalanceCheck(t *testing.T) { rpcClient := NewDialedTestRPCClient(t, RPCClientOpts{ HTTP: wsURL, Cfg: &TestNodePoolConfig{ - NodeFinalizedBlockPollInterval: 1 * time.Second, - HistoricalBalanceCheckEnabledVal: true, - HistoricalBalanceCheckAddressVal: probeAddress, + NodeFinalizedBlockPollInterval: 1 * time.Second, }, FinalityTagsEnabled: true, ChainID: chainID, }) - _, err := rpcClient.ClientVersion(t.Context()) + err := rpcClient.CheckFinalizedStateAvailability(t.Context(), probeAddress) require.Error(t, err) - require.ErrorContains(t, err, "historical balance health check failed") + require.ErrorContains(t, err, "fetching balance") }) } From 798c519f9dd499d58a6f021416513baca86d65c9 Mon Sep 17 00:00:00 2001 From: Krish-vemula Date: Wed, 4 Mar 2026 23:30:22 -0600 Subject: [PATCH 4/8] Address review feedback for finalized state check --- pkg/client/helpers_test.go | 23 ++------ pkg/client/rpc_client.go | 37 ++++++++---- pkg/client/rpc_client_internal_test.go | 73 +++++++++++++++++++++--- pkg/config/chain_scoped_client_errors.go | 3 + pkg/config/chain_scoped_node_pool.go | 15 +++-- pkg/config/config.go | 2 + pkg/config/toml/config.go | 21 +++++-- pkg/config/toml/docs.toml | 2 + 8 files changed, 126 insertions(+), 50 deletions(-) diff --git a/pkg/client/helpers_test.go b/pkg/client/helpers_test.go index 2d749faa3e..ff83fd60f6 100644 --- a/pkg/client/helpers_test.go +++ b/pkg/client/helpers_test.go @@ -39,6 +39,7 @@ type TestClientErrors struct { serviceUnavailable string tooManyResults string missingBlocks string + finalizedStateUnavailable string } func NewTestClientErrors() TestClientErrors { @@ -83,9 +84,10 @@ func (c *TestClientErrors) L2FeeTooHigh() string { return c.l2FeeTooH func (c *TestClientErrors) L2Full() string { return c.l2Full } func (c *TestClientErrors) TransactionAlreadyMined() string { return c.transactionAlreadyMined } func (c *TestClientErrors) Fatal() string { return c.fatal } -func (c *TestClientErrors) ServiceUnavailable() string { return c.serviceUnavailable } -func (c *TestClientErrors) TooManyResults() string { return c.tooManyResults } -func (c *TestClientErrors) MissingBlocks() string { return c.missingBlocks } +func (c *TestClientErrors) ServiceUnavailable() string { return c.serviceUnavailable } +func (c *TestClientErrors) TooManyResults() string { return c.tooManyResults } +func (c *TestClientErrors) MissingBlocks() string { return c.missingBlocks } +func (c *TestClientErrors) FinalizedStateUnavailable() string { return c.finalizedStateUnavailable } type TestNodePoolConfig struct { NodePollFailureThreshold uint32 @@ -102,10 +104,7 @@ type TestNodePoolConfig struct { NodeDeathDeclarationDelay time.Duration NodeNewHeadsPollInterval time.Duration ExternalRequestMaxResponseSizeVal uint32 - FinalizedStateCheckEnabledVal bool - FinalizedStateCheckAddressVal string FinalizedStateCheckFailureThresholdVal uint32 - FinalizedStateUnavailableRegexVal string } func (tc TestNodePoolConfig) PollFailureThreshold() uint32 { return tc.NodePollFailureThreshold } @@ -156,22 +155,10 @@ func (tc TestNodePoolConfig) ExternalRequestMaxResponseSize() uint32 { return tc.ExternalRequestMaxResponseSizeVal } -func (tc TestNodePoolConfig) FinalizedStateCheckEnabled() bool { - return tc.FinalizedStateCheckEnabledVal -} - -func (tc TestNodePoolConfig) FinalizedStateCheckAddress() string { - return tc.FinalizedStateCheckAddressVal -} - func (tc TestNodePoolConfig) FinalizedStateCheckFailureThreshold() uint32 { return tc.FinalizedStateCheckFailureThresholdVal } -func (tc TestNodePoolConfig) FinalizedStateUnavailableRegex() string { - return tc.FinalizedStateUnavailableRegexVal -} - func NewChainClientWithTestNode( t *testing.T, nodeCfg multinode.NodeConfig, diff --git a/pkg/client/rpc_client.go b/pkg/client/rpc_client.go index 2b780a7e9f..a1bb69ba25 100644 --- a/pkg/client/rpc_client.go +++ b/pkg/client/rpc_client.go @@ -9,6 +9,7 @@ import ( "math/big" "net/http" "net/url" + "regexp" "strconv" "sync/atomic" "time" @@ -190,15 +191,10 @@ func (r *RPCClient) ClientVersion(ctx context.Context) (version string, err erro // CheckFinalizedStateAvailability verifies if the RPC can serve historical state at the finalized block. // This is used to detect non-archive nodes that cannot serve state queries for older blocks. -// If probeAddress is provided, it uses that address; otherwise falls back to the configured HistoricalBalanceCheckAddress. -// Returns nil immediately if historical balance check is not enabled and no probeAddress is provided. -func (r *RPCClient) CheckFinalizedStateAvailability(ctx context.Context, probeAddress string) error { - var addr common.Address - if probeAddress != "" { - addr = common.HexToAddress(probeAddress) - } else if r.historicalBalanceCheckEnabled { - addr = r.historicalBalanceCheckAddress - } else { +// Returns nil immediately if historical balance check is not enabled. +// Returns multinode.ErrFinalizedStateUnavailable if the error matches the FinalizedStateUnavailable pattern. +func (r *RPCClient) CheckFinalizedStateAvailability(ctx context.Context) error { + if !r.historicalBalanceCheckEnabled { return nil } @@ -214,13 +210,32 @@ func (r *RPCClient) CheckFinalizedStateAvailability(ctx context.Context, probeAd finalizedHeight := max(int64(0), latest-int64(r.finalityDepth)) blockNumber = big.NewInt(finalizedHeight) } - _, err := r.BalanceAt(ctx, addr, blockNumber) + _, err := r.BalanceAt(ctx, r.historicalBalanceCheckAddress, blockNumber) if err != nil { - return fmt.Errorf("fetching balance for address %s at block %s failed: %w", addr.String(), blockNumber.String(), err) + if r.isFinalizedStateUnavailableError(err) { + return fmt.Errorf("%w: %w", multinode.ErrFinalizedStateUnavailable, err) + } + return fmt.Errorf("fetching balance for address %s at block %s failed: %w", r.historicalBalanceCheckAddress.String(), blockNumber.String(), err) } return nil } +// isFinalizedStateUnavailableError checks if the error matches the FinalizedStateUnavailable regex pattern. +func (r *RPCClient) isFinalizedStateUnavailableError(err error) bool { + if err == nil { + return false + } + pattern := r.clientErrors.FinalizedStateUnavailable() + if pattern == "" { + return false + } + re, compileErr := regexp.Compile(pattern) + if compileErr != nil { + return false + } + return re.MatchString(err.Error()) +} + func (r *RPCClient) Dial(callerCtx context.Context) error { ctx, cancel, _ := r.AcquireQueryCtx(callerCtx, r.rpcTimeout) defer cancel() diff --git a/pkg/client/rpc_client_internal_test.go b/pkg/client/rpc_client_internal_test.go index 517bedfbbe..1f17f463bf 100644 --- a/pkg/client/rpc_client_internal_test.go +++ b/pkg/client/rpc_client_internal_test.go @@ -444,6 +444,22 @@ func TestRPCClient_CheckFinalizedStateAvailability(t *testing.T) { probeAddress := "0x0000000000000000000000000000000000000001" expectedAddress := common.HexToAddress(probeAddress).String() + t.Run("returns nil when historical balance check is disabled", func(t *testing.T) { + t.Parallel() + rpcClient := NewDialedTestRPCClient(t, RPCClientOpts{ + HTTP: &url.URL{Scheme: "http", Host: "localhost:8545"}, + Cfg: &TestNodePoolConfig{ + NodeFinalizedBlockPollInterval: 1 * time.Second, + HistoricalBalanceCheckEnabledVal: false, + }, + FinalityTagsEnabled: true, + ChainID: chainID, + }) + + err := rpcClient.CheckFinalizedStateAvailability(t.Context()) + require.NoError(t, err) + }) + t.Run("uses finalized tag when enabled", func(t *testing.T) { t.Parallel() wsURL := testutils.NewWSServer(t, chainID, func(method string, params gjson.Result) (resp testutils.JSONRPCResponse) { @@ -461,13 +477,15 @@ func TestRPCClient_CheckFinalizedStateAvailability(t *testing.T) { rpcClient := NewDialedTestRPCClient(t, RPCClientOpts{ HTTP: wsURL, Cfg: &TestNodePoolConfig{ - NodeFinalizedBlockPollInterval: 1 * time.Second, + NodeFinalizedBlockPollInterval: 1 * time.Second, + HistoricalBalanceCheckEnabledVal: true, + HistoricalBalanceCheckAddressVal: probeAddress, }, FinalityTagsEnabled: true, ChainID: chainID, }) - err := rpcClient.CheckFinalizedStateAvailability(t.Context(), probeAddress) + err := rpcClient.CheckFinalizedStateAvailability(t.Context()) require.NoError(t, err) }) @@ -490,18 +508,20 @@ func TestRPCClient_CheckFinalizedStateAvailability(t *testing.T) { rpcClient := NewDialedTestRPCClient(t, RPCClientOpts{ HTTP: wsURL, Cfg: &TestNodePoolConfig{ - NodeFinalizedBlockPollInterval: 1 * time.Second, + NodeFinalizedBlockPollInterval: 1 * time.Second, + HistoricalBalanceCheckEnabledVal: true, + HistoricalBalanceCheckAddressVal: probeAddress, }, FinalityTagsEnabled: false, FinalityDepth: 4, ChainID: chainID, }) - err := rpcClient.CheckFinalizedStateAvailability(t.Context(), probeAddress) + err := rpcClient.CheckFinalizedStateAvailability(t.Context()) require.NoError(t, err) }) - t.Run("returns error when RPC fails", func(t *testing.T) { + t.Run("returns ErrFinalizedStateUnavailable when error matches regex", func(t *testing.T) { t.Parallel() wsURL := testutils.NewWSServer(t, chainID, func(method string, _ gjson.Result) (resp testutils.JSONRPCResponse) { switch method { @@ -513,17 +533,56 @@ func TestRPCClient_CheckFinalizedStateAvailability(t *testing.T) { return }).WSURL() + clientErrors := NewTestClientErrors() + clientErrors.finalizedStateUnavailable = "missing trie node" + + rpcClient := NewDialedTestRPCClient(t, RPCClientOpts{ + HTTP: wsURL, + Cfg: &TestNodePoolConfig{ + NodeFinalizedBlockPollInterval: 1 * time.Second, + HistoricalBalanceCheckEnabledVal: true, + HistoricalBalanceCheckAddressVal: probeAddress, + NodeErrors: &clientErrors, + }, + FinalityTagsEnabled: true, + ChainID: chainID, + }) + + err := rpcClient.CheckFinalizedStateAvailability(t.Context()) + require.Error(t, err) + require.ErrorIs(t, err, multinode.ErrFinalizedStateUnavailable) + }) + + t.Run("returns generic error when error does not match regex", func(t *testing.T) { + t.Parallel() + wsURL := testutils.NewWSServer(t, chainID, func(method string, _ gjson.Result) (resp testutils.JSONRPCResponse) { + switch method { + case "eth_getBalance": + resp.Error.Message = "connection reset" + default: + require.Fail(t, "unexpected method: "+method) + } + return + }).WSURL() + + clientErrors := NewTestClientErrors() + clientErrors.finalizedStateUnavailable = "missing trie node" + rpcClient := NewDialedTestRPCClient(t, RPCClientOpts{ HTTP: wsURL, Cfg: &TestNodePoolConfig{ - NodeFinalizedBlockPollInterval: 1 * time.Second, + NodeFinalizedBlockPollInterval: 1 * time.Second, + HistoricalBalanceCheckEnabledVal: true, + HistoricalBalanceCheckAddressVal: probeAddress, + NodeErrors: &clientErrors, }, FinalityTagsEnabled: true, ChainID: chainID, }) - err := rpcClient.CheckFinalizedStateAvailability(t.Context(), probeAddress) + err := rpcClient.CheckFinalizedStateAvailability(t.Context()) require.Error(t, err) require.ErrorContains(t, err, "fetching balance") + require.NotErrorIs(t, err, multinode.ErrFinalizedStateUnavailable) }) } diff --git a/pkg/config/chain_scoped_client_errors.go b/pkg/config/chain_scoped_client_errors.go index f0b95b0b3c..03ddad0157 100644 --- a/pkg/config/chain_scoped_client_errors.go +++ b/pkg/config/chain_scoped_client_errors.go @@ -50,3 +50,6 @@ func (c *clientErrorsConfig) ServiceUnavailable() string { } func (c *clientErrorsConfig) TooManyResults() string { return derefOrDefault(c.c.TooManyResults) } func (c *clientErrorsConfig) MissingBlocks() string { return derefOrDefault(c.c.MissingBlocks) } +func (c *clientErrorsConfig) FinalizedStateUnavailable() string { + return derefOrDefault(c.c.FinalizedStateUnavailable) +} diff --git a/pkg/config/chain_scoped_node_pool.go b/pkg/config/chain_scoped_node_pool.go index 7ed6304902..466899da64 100644 --- a/pkg/config/chain_scoped_node_pool.go +++ b/pkg/config/chain_scoped_node_pool.go @@ -3,8 +3,6 @@ package config import ( "time" - "github.com/ethereum/go-ethereum/common" - "github.com/smartcontractkit/chainlink-evm/pkg/config/toml" ) @@ -41,16 +39,10 @@ func (n *NodePoolConfig) FinalizedBlockPollInterval() time.Duration { } func (n *NodePoolConfig) HistoricalBalanceCheckEnabled() bool { - if n.C.HistoricalBalanceCheckEnabled == nil { - return false - } return *n.C.HistoricalBalanceCheckEnabled } func (n *NodePoolConfig) HistoricalBalanceCheckAddress() string { - if n.C.HistoricalBalanceCheckAddress == nil { - return common.Address{}.String() - } return n.C.HistoricalBalanceCheckAddress.String() } @@ -78,3 +70,10 @@ func (n *NodePoolConfig) ExternalRequestMaxResponseSize() uint32 { } return *n.C.ExternalRequestMaxResponseSize } + +func (n *NodePoolConfig) FinalizedStateCheckFailureThreshold() uint32 { + if n.C.FinalizedStateCheckFailureThreshold == nil { + return 0 + } + return *n.C.FinalizedStateCheckFailureThreshold +} diff --git a/pkg/config/config.go b/pkg/config/config.go index 0a08a0fd85..a9298fa2dc 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -107,6 +107,7 @@ type ClientErrors interface { ServiceUnavailable() string TooManyResults() string MissingBlocks() string + FinalizedStateUnavailable() string } type Transactions interface { @@ -216,6 +217,7 @@ type NodePool interface { NewHeadsPollInterval() time.Duration VerifyChainID() bool ExternalRequestMaxResponseSize() uint32 + FinalizedStateCheckFailureThreshold() uint32 } type ChainScopedConfig interface { diff --git a/pkg/config/toml/config.go b/pkg/config/toml/config.go index ec8f20c75f..63ed555c4e 100644 --- a/pkg/config/toml/config.go +++ b/pkg/config/toml/config.go @@ -1023,6 +1023,7 @@ type ClientErrors struct { ServiceUnavailable *string `toml:",omitempty"` TooManyResults *string `toml:",omitempty"` MissingBlocks *string `toml:",omitempty"` + FinalizedStateUnavailable *string `toml:",omitempty"` } func (r *ClientErrors) setFrom(f *ClientErrors) bool { @@ -1074,6 +1075,9 @@ func (r *ClientErrors) setFrom(f *ClientErrors) bool { if v := f.MissingBlocks; v != nil { r.MissingBlocks = v } + if v := f.FinalizedStateUnavailable; v != nil { + r.FinalizedStateUnavailable = v + } return true } @@ -1087,12 +1091,13 @@ type NodePool struct { FinalizedBlockPollInterval *commonconfig.Duration HistoricalBalanceCheckEnabled *bool HistoricalBalanceCheckAddress *types.EIP55Address - Errors ClientErrors `toml:",omitempty"` - EnforceRepeatableRead *bool - DeathDeclarationDelay *commonconfig.Duration - NewHeadsPollInterval *commonconfig.Duration - VerifyChainID *bool - ExternalRequestMaxResponseSize *uint32 + Errors ClientErrors `toml:",omitempty"` + EnforceRepeatableRead *bool + DeathDeclarationDelay *commonconfig.Duration + NewHeadsPollInterval *commonconfig.Duration + VerifyChainID *bool + ExternalRequestMaxResponseSize *uint32 + FinalizedStateCheckFailureThreshold *uint32 } func (p *NodePool) setFrom(f *NodePool) { @@ -1144,6 +1149,10 @@ func (p *NodePool) setFrom(f *NodePool) { p.ExternalRequestMaxResponseSize = v } + if v := f.FinalizedStateCheckFailureThreshold; v != nil { + p.FinalizedStateCheckFailureThreshold = v + } + p.Errors.setFrom(&f.Errors) } diff --git a/pkg/config/toml/docs.toml b/pkg/config/toml/docs.toml index 679197bb87..a273d6302f 100644 --- a/pkg/config/toml/docs.toml +++ b/pkg/config/toml/docs.toml @@ -521,6 +521,8 @@ ServiceUnavailable = '(: |^)service unavailable' # Example TooManyResults = '(: |^)too many results' # Example # MissingBlocks is a regex pattern to match an eth_getLogs error indicating the rpc server is permanently missing some blocks in the requested block range MissingBlocks = '(: |^)invalid block range' # Example +# FinalizedStateUnavailable is a regex pattern to match errors indicating the RPC cannot serve historical state at the finalized block (e.g., pruned/non-archive node) +FinalizedStateUnavailable = '(missing trie node|state not available|historical state unavailable)' # Example [OCR] # ContractConfirmations sets `OCR.ContractConfirmations` for this EVM chain. From b5a8541306d59d2743071bd7b9368619ee0ca272 Mon Sep 17 00:00:00 2001 From: Krish-vemula Date: Wed, 11 Mar 2026 15:43:28 -0500 Subject: [PATCH 5/8] added default to fallback.toml --- pkg/config/toml/defaults/fallback.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/config/toml/defaults/fallback.toml b/pkg/config/toml/defaults/fallback.toml index 17bd1f5d47..88f6eecd4a 100644 --- a/pkg/config/toml/defaults/fallback.toml +++ b/pkg/config/toml/defaults/fallback.toml @@ -86,6 +86,7 @@ NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' HistoricalBalanceCheckEnabled = false HistoricalBalanceCheckAddress = '0x0000000000000000000000000000000000000000' +FinalizedStateCheckFailureThreshold = 0 EnforceRepeatableRead = true DeathDeclarationDelay = '1m' NewHeadsPollInterval = '0s' From 967013179ae3e6e372b55a09afa2fd0d4b3d259f Mon Sep 17 00:00:00 2001 From: Krish-vemula Date: Tue, 17 Mar 2026 17:59:19 -0500 Subject: [PATCH 6/8] fix: remove HistoricalBalanceCheckEnabled flag, let multinode control polling --- CONFIG.md | 17 +++++------------ pkg/client/helpers_test.go | 5 ----- pkg/client/rpc_client.go | 9 ++------- pkg/client/rpc_client_internal_test.go | 20 -------------------- pkg/config/chain_scoped_node_pool.go | 7 +++---- pkg/config/config.go | 1 - pkg/config/config_test.go | 1 - pkg/config/toml/config.go | 11 ++--------- pkg/config/toml/config_test.go | 6 ++++-- pkg/config/toml/defaults/fallback.toml | 1 - pkg/config/toml/docs.toml | 12 +++++++----- pkg/config/toml/testdata/config-full.toml | 3 ++- 12 files changed, 25 insertions(+), 68 deletions(-) diff --git a/CONFIG.md b/CONFIG.md index 2f8e742cac..0219546c39 100644 --- a/CONFIG.md +++ b/CONFIG.md @@ -911,7 +911,6 @@ SyncThreshold = 5 # Default LeaseDuration = '0s' # Default NodeIsSyncingEnabled = false # Default FinalizedBlockPollInterval = '5s' # Default -HistoricalBalanceCheckEnabled = false # Default HistoricalBalanceCheckAddress = '0x0000000000000000000000000000000000000000' # Default EnforceRepeatableRead = true # Default DeathDeclarationDelay = '1m' # Default @@ -991,22 +990,16 @@ reported based on latest block and finality depth. Set to 0 to disable. -### HistoricalBalanceCheckEnabled -```toml -HistoricalBalanceCheckEnabled = false # Default -``` -HistoricalBalanceCheckEnabled controls whether NodePool health polling also verifies historical state availability -by executing `eth_getBalance` for HistoricalBalanceCheckAddress at the latest finalized block. -Finalized block selection follows chain finality settings: -- `FinalityTagEnabled = true`: use `finalized` tag -- `FinalityTagEnabled = false`: use `latest - FinalityDepth` - ### HistoricalBalanceCheckAddress ```toml HistoricalBalanceCheckAddress = '0x0000000000000000000000000000000000000000' # Default ``` HistoricalBalanceCheckAddress is the probe account used by the historical balance health check. -This check is only active when `HistoricalBalanceCheckEnabled = true`. +The check executes `eth_getBalance` for this address at the latest finalized block. +Finalized block selection follows chain finality settings: +- `FinalityTagEnabled = true`: use `finalized` tag +- `FinalityTagEnabled = false`: use `latest - FinalityDepth` +The check is only active when `FinalizedStateCheckFailureThreshold > 0`. ### EnforceRepeatableRead ```toml diff --git a/pkg/client/helpers_test.go b/pkg/client/helpers_test.go index ff83fd60f6..cd78edfbcc 100644 --- a/pkg/client/helpers_test.go +++ b/pkg/client/helpers_test.go @@ -97,7 +97,6 @@ type TestNodePoolConfig struct { NodeLeaseDuration time.Duration NodeIsSyncingEnabledVal bool NodeFinalizedBlockPollInterval time.Duration - HistoricalBalanceCheckEnabledVal bool HistoricalBalanceCheckAddressVal string NodeErrors config.ClientErrors EnforceRepeatableReadVal bool @@ -123,10 +122,6 @@ func (tc TestNodePoolConfig) FinalizedBlockPollInterval() time.Duration { return tc.NodeFinalizedBlockPollInterval } -func (tc TestNodePoolConfig) HistoricalBalanceCheckEnabled() bool { - return tc.HistoricalBalanceCheckEnabledVal -} - func (tc TestNodePoolConfig) HistoricalBalanceCheckAddress() string { return tc.HistoricalBalanceCheckAddressVal } diff --git a/pkg/client/rpc_client.go b/pkg/client/rpc_client.go index a1bb69ba25..69e5a21020 100644 --- a/pkg/client/rpc_client.go +++ b/pkg/client/rpc_client.go @@ -106,7 +106,6 @@ type RPCClient struct { finalityTagEnabled bool finalityDepth uint32 safeDepth uint32 - historicalBalanceCheckEnabled bool historicalBalanceCheckAddress common.Address externalRequestMaxResponseSize uint32 @@ -145,7 +144,6 @@ func NewRPCClient( finalityTagEnabled: supportsFinalityTags, finalityDepth: finalityDepth, safeDepth: safeDepth, - historicalBalanceCheckEnabled: cfg.HistoricalBalanceCheckEnabled(), historicalBalanceCheckAddress: common.HexToAddress(cfg.HistoricalBalanceCheckAddress()), externalRequestMaxResponseSize: externalRequestMaxResponseSize, } @@ -191,13 +189,9 @@ func (r *RPCClient) ClientVersion(ctx context.Context) (version string, err erro // CheckFinalizedStateAvailability verifies if the RPC can serve historical state at the finalized block. // This is used to detect non-archive nodes that cannot serve state queries for older blocks. -// Returns nil immediately if historical balance check is not enabled. // Returns multinode.ErrFinalizedStateUnavailable if the error matches the FinalizedStateUnavailable pattern. +// The decision to call this method is made by multinode based on its configuration. func (r *RPCClient) CheckFinalizedStateAvailability(ctx context.Context) error { - if !r.historicalBalanceCheckEnabled { - return nil - } - var blockNumber *big.Int if r.finalityTagEnabled { blockNumber = big.NewInt(rpc.FinalizedBlockNumber.Int64()) @@ -231,6 +225,7 @@ func (r *RPCClient) isFinalizedStateUnavailableError(err error) bool { } re, compileErr := regexp.Compile(pattern) if compileErr != nil { + r.rpcLog.Criticalw("FinalizedStateUnavailable regex pattern is invalid; finalized state availability check is effectively disabled", "pattern", pattern, "err", compileErr) return false } return re.MatchString(err.Error()) diff --git a/pkg/client/rpc_client_internal_test.go b/pkg/client/rpc_client_internal_test.go index 1f17f463bf..c407942c38 100644 --- a/pkg/client/rpc_client_internal_test.go +++ b/pkg/client/rpc_client_internal_test.go @@ -444,22 +444,6 @@ func TestRPCClient_CheckFinalizedStateAvailability(t *testing.T) { probeAddress := "0x0000000000000000000000000000000000000001" expectedAddress := common.HexToAddress(probeAddress).String() - t.Run("returns nil when historical balance check is disabled", func(t *testing.T) { - t.Parallel() - rpcClient := NewDialedTestRPCClient(t, RPCClientOpts{ - HTTP: &url.URL{Scheme: "http", Host: "localhost:8545"}, - Cfg: &TestNodePoolConfig{ - NodeFinalizedBlockPollInterval: 1 * time.Second, - HistoricalBalanceCheckEnabledVal: false, - }, - FinalityTagsEnabled: true, - ChainID: chainID, - }) - - err := rpcClient.CheckFinalizedStateAvailability(t.Context()) - require.NoError(t, err) - }) - t.Run("uses finalized tag when enabled", func(t *testing.T) { t.Parallel() wsURL := testutils.NewWSServer(t, chainID, func(method string, params gjson.Result) (resp testutils.JSONRPCResponse) { @@ -478,7 +462,6 @@ func TestRPCClient_CheckFinalizedStateAvailability(t *testing.T) { HTTP: wsURL, Cfg: &TestNodePoolConfig{ NodeFinalizedBlockPollInterval: 1 * time.Second, - HistoricalBalanceCheckEnabledVal: true, HistoricalBalanceCheckAddressVal: probeAddress, }, FinalityTagsEnabled: true, @@ -509,7 +492,6 @@ func TestRPCClient_CheckFinalizedStateAvailability(t *testing.T) { HTTP: wsURL, Cfg: &TestNodePoolConfig{ NodeFinalizedBlockPollInterval: 1 * time.Second, - HistoricalBalanceCheckEnabledVal: true, HistoricalBalanceCheckAddressVal: probeAddress, }, FinalityTagsEnabled: false, @@ -540,7 +522,6 @@ func TestRPCClient_CheckFinalizedStateAvailability(t *testing.T) { HTTP: wsURL, Cfg: &TestNodePoolConfig{ NodeFinalizedBlockPollInterval: 1 * time.Second, - HistoricalBalanceCheckEnabledVal: true, HistoricalBalanceCheckAddressVal: probeAddress, NodeErrors: &clientErrors, }, @@ -572,7 +553,6 @@ func TestRPCClient_CheckFinalizedStateAvailability(t *testing.T) { HTTP: wsURL, Cfg: &TestNodePoolConfig{ NodeFinalizedBlockPollInterval: 1 * time.Second, - HistoricalBalanceCheckEnabledVal: true, HistoricalBalanceCheckAddressVal: probeAddress, NodeErrors: &clientErrors, }, diff --git a/pkg/config/chain_scoped_node_pool.go b/pkg/config/chain_scoped_node_pool.go index 466899da64..36e333c9b1 100644 --- a/pkg/config/chain_scoped_node_pool.go +++ b/pkg/config/chain_scoped_node_pool.go @@ -38,11 +38,10 @@ func (n *NodePoolConfig) FinalizedBlockPollInterval() time.Duration { return n.C.FinalizedBlockPollInterval.Duration() } -func (n *NodePoolConfig) HistoricalBalanceCheckEnabled() bool { - return *n.C.HistoricalBalanceCheckEnabled -} - func (n *NodePoolConfig) HistoricalBalanceCheckAddress() string { + if n.C.HistoricalBalanceCheckAddress == nil { + return "" + } return n.C.HistoricalBalanceCheckAddress.String() } diff --git a/pkg/config/config.go b/pkg/config/config.go index a9298fa2dc..2e3aec4770 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -209,7 +209,6 @@ type NodePool interface { LeaseDuration() time.Duration NodeIsSyncingEnabled() bool FinalizedBlockPollInterval() time.Duration - HistoricalBalanceCheckEnabled() bool HistoricalBalanceCheckAddress() string Errors() ClientErrors EnforceRepeatableRead() bool diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index f6450a1138..510b335053 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -369,7 +369,6 @@ func TestNodePoolConfig(t *testing.T) { require.False(t, cfg.EVM().NodePool().NodeIsSyncingEnabled()) require.True(t, cfg.EVM().NodePool().EnforceRepeatableRead()) require.Equal(t, time.Minute, cfg.EVM().NodePool().DeathDeclarationDelay()) - require.False(t, cfg.EVM().NodePool().HistoricalBalanceCheckEnabled()) require.Equal(t, "0x0000000000000000000000000000000000000000", cfg.EVM().NodePool().HistoricalBalanceCheckAddress()) } diff --git a/pkg/config/toml/config.go b/pkg/config/toml/config.go index 63ed555c4e..8611a790f6 100644 --- a/pkg/config/toml/config.go +++ b/pkg/config/toml/config.go @@ -1089,15 +1089,14 @@ type NodePool struct { LeaseDuration *commonconfig.Duration NodeIsSyncingEnabled *bool FinalizedBlockPollInterval *commonconfig.Duration - HistoricalBalanceCheckEnabled *bool - HistoricalBalanceCheckAddress *types.EIP55Address + HistoricalBalanceCheckAddress *types.EIP55Address + FinalizedStateCheckFailureThreshold *uint32 Errors ClientErrors `toml:",omitempty"` EnforceRepeatableRead *bool DeathDeclarationDelay *commonconfig.Duration NewHeadsPollInterval *commonconfig.Duration VerifyChainID *bool ExternalRequestMaxResponseSize *uint32 - FinalizedStateCheckFailureThreshold *uint32 } func (p *NodePool) setFrom(f *NodePool) { @@ -1122,9 +1121,6 @@ func (p *NodePool) setFrom(f *NodePool) { if v := f.FinalizedBlockPollInterval; v != nil { p.FinalizedBlockPollInterval = v } - if v := f.HistoricalBalanceCheckEnabled; v != nil { - p.HistoricalBalanceCheckEnabled = v - } if v := f.HistoricalBalanceCheckAddress; v != nil { p.HistoricalBalanceCheckAddress = v } @@ -1167,9 +1163,6 @@ func (p *NodePool) ValidateConfig(finalityTagEnabled *bool) (err error) { Msg: "must be greater than 0"}) } } - if p.HistoricalBalanceCheckEnabled != nil && *p.HistoricalBalanceCheckEnabled && p.HistoricalBalanceCheckAddress == nil { - err = multierr.Append(err, commonconfig.ErrMissing{Name: "HistoricalBalanceCheckAddress", Msg: "required when HistoricalBalanceCheckEnabled is true"}) - } return } diff --git a/pkg/config/toml/config_test.go b/pkg/config/toml/config_test.go index 4ee0cc44b3..0752c8d805 100644 --- a/pkg/config/toml/config_test.go +++ b/pkg/config/toml/config_test.go @@ -99,6 +99,7 @@ func TestDefaults_fieldsNotNil(t *testing.T) { ServiceUnavailable: ptr("unavailable"), TooManyResults: ptr("too-many"), MissingBlocks: ptr("missing"), + FinalizedStateUnavailable: ptr("finalized-unavailable"), } configtest.AssertFieldsNotNil(t, unknown) @@ -306,13 +307,13 @@ var fullConfig = EVMConfig{ LeaseDuration: config.MustNewDuration(0), NodeIsSyncingEnabled: ptr(true), FinalizedBlockPollInterval: config.MustNewDuration(time.Second), - HistoricalBalanceCheckEnabled: ptr(true), HistoricalBalanceCheckAddress: ptr(types.MustEIP55Address("0x0000000000000000000000000000000000000001")), EnforceRepeatableRead: ptr(true), DeathDeclarationDelay: config.MustNewDuration(time.Minute), VerifyChainID: ptr(true), NewHeadsPollInterval: config.MustNewDuration(0), - ExternalRequestMaxResponseSize: ptr[uint32](10), + ExternalRequestMaxResponseSize: ptr[uint32](10), + FinalizedStateCheckFailureThreshold: ptr[uint32](3), Errors: ClientErrors{ NonceTooLow: ptr[string]("(: |^)nonce too low"), NonceTooHigh: ptr[string]("(: |^)nonce too high"), @@ -330,6 +331,7 @@ var fullConfig = EVMConfig{ ServiceUnavailable: ptr[string]("(: |^)service unavailable"), TooManyResults: ptr[string]("(: |^)too many results"), MissingBlocks: ptr[string]("(: |^)invalid block range"), + FinalizedStateUnavailable: ptr[string]("(: |^)missing trie node"), }, }, OCR: OCR{ diff --git a/pkg/config/toml/defaults/fallback.toml b/pkg/config/toml/defaults/fallback.toml index 88f6eecd4a..7125f4a227 100644 --- a/pkg/config/toml/defaults/fallback.toml +++ b/pkg/config/toml/defaults/fallback.toml @@ -84,7 +84,6 @@ SyncThreshold = 5 LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' -HistoricalBalanceCheckEnabled = false HistoricalBalanceCheckAddress = '0x0000000000000000000000000000000000000000' FinalizedStateCheckFailureThreshold = 0 EnforceRepeatableRead = true diff --git a/pkg/config/toml/docs.toml b/pkg/config/toml/docs.toml index a273d6302f..eee09c2e9e 100644 --- a/pkg/config/toml/docs.toml +++ b/pkg/config/toml/docs.toml @@ -452,15 +452,17 @@ NodeIsSyncingEnabled = false # Default # # Set to 0 to disable. FinalizedBlockPollInterval = '5s' # Default -# HistoricalBalanceCheckEnabled controls whether NodePool health polling also verifies historical state availability -# by executing `eth_getBalance` for HistoricalBalanceCheckAddress at the latest finalized block. +# HistoricalBalanceCheckAddress is the probe account used by the historical balance health check. +# The check executes `eth_getBalance` for this address at the latest finalized block. # Finalized block selection follows chain finality settings: # - `FinalityTagEnabled = true`: use `finalized` tag # - `FinalityTagEnabled = false`: use `latest - FinalityDepth` -HistoricalBalanceCheckEnabled = false # Default -# HistoricalBalanceCheckAddress is the probe account used by the historical balance health check. -# This check is only active when `HistoricalBalanceCheckEnabled = true`. +# The check is only active when `FinalizedStateCheckFailureThreshold > 0`. HistoricalBalanceCheckAddress = '0x0000000000000000000000000000000000000000' # Default +# FinalizedStateCheckFailureThreshold is the number of consecutive failures of the finalized state availability check +# before the node is marked as FinalizedStateNotAvailable. +# Set to 0 to disable the check. +FinalizedStateCheckFailureThreshold = 0 # Default # EnforceRepeatableRead defines if Core should only use RPCs whose most recently finalized block is greater or equal to # `highest finalized block - FinalizedBlockOffset`. In other words, exclude RPCs lagging on latest finalized # block. diff --git a/pkg/config/toml/testdata/config-full.toml b/pkg/config/toml/testdata/config-full.toml index caec9035e3..b776750e3f 100644 --- a/pkg/config/toml/testdata/config-full.toml +++ b/pkg/config/toml/testdata/config-full.toml @@ -118,8 +118,8 @@ SyncThreshold = 13 LeaseDuration = '0s' NodeIsSyncingEnabled = true FinalizedBlockPollInterval = '1s' -HistoricalBalanceCheckEnabled = true HistoricalBalanceCheckAddress = '0x0000000000000000000000000000000000000001' +FinalizedStateCheckFailureThreshold = 3 EnforceRepeatableRead = true DeathDeclarationDelay = '1m0s' NewHeadsPollInterval = '0s' @@ -143,6 +143,7 @@ Fatal = '(: |^)fatal' ServiceUnavailable = '(: |^)service unavailable' TooManyResults = '(: |^)too many results' MissingBlocks = '(: |^)invalid block range' +FinalizedStateUnavailable = '(: |^)missing trie node' [OCR] ContractConfirmations = 11 From 9e505f508d91685b1582ee18b191543063f5dd69 Mon Sep 17 00:00:00 2001 From: Krish-vemula Date: Wed, 18 Mar 2026 11:16:28 -0500 Subject: [PATCH 7/8] chore: bump chainlink-framework/multinode to PR commit version --- go.mod | 23 ++++++++++++----------- go.sum | 46 ++++++++++++++++++++++++---------------------- 2 files changed, 36 insertions(+), 33 deletions(-) diff --git a/go.mod b/go.mod index b71c372914..3b4a4c8392 100644 --- a/go.mod +++ b/go.mod @@ -25,15 +25,15 @@ require ( github.com/prometheus/client_model v0.6.2 github.com/prometheus/common v0.65.0 github.com/shopspring/decimal v1.4.0 - github.com/smartcontractkit/chain-selectors v1.0.67 - github.com/smartcontractkit/chainlink-common v0.9.6-0.20260119150426-31ad843060e6 + github.com/smartcontractkit/chain-selectors v1.0.89 + github.com/smartcontractkit/chainlink-common v0.10.1-0.20260305114348-b8bbac30bfc7 github.com/smartcontractkit/chainlink-common/keystore v0.1.1-0.20260115164422-897ee18790cd github.com/smartcontractkit/chainlink-evm/gethwrappers v0.0.0-20251022073203-7d8ae8cf67c1 github.com/smartcontractkit/chainlink-framework/capabilities v0.0.0-20250818175541-3389ac08a563 github.com/smartcontractkit/chainlink-framework/chains v0.0.0-20251210101658-1c5c8e4c4f15 - github.com/smartcontractkit/chainlink-framework/metrics v0.0.0-20251020150604-8ab84f7bad1a - github.com/smartcontractkit/chainlink-framework/multinode v0.0.0-20251021173435-e86785845942 - github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20251124151448-0448aefdaab9 + github.com/smartcontractkit/chainlink-framework/metrics v0.0.0-20260310180305-3ee91a6d9ae9 + github.com/smartcontractkit/chainlink-framework/multinode v0.0.0-20260317161400-793ab9bc5b53 + github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260226130359-963f935e0396 github.com/smartcontractkit/chainlink-protos/svr v1.1.1-0.20260203131522-bb8bc5c423b3 github.com/smartcontractkit/chainlink-tron/relayer v0.0.11-0.20250815105909-75499abc4335 github.com/smartcontractkit/freeport v0.1.3-0.20250716200817-cb5dfd0e369e @@ -42,8 +42,8 @@ require ( github.com/theodesp/go-heaps v0.0.0-20190520121037-88e35354fe0a github.com/tidwall/gjson v1.18.0 github.com/ugorji/go/codec v1.2.12 - go.opentelemetry.io/otel v1.39.0 - go.opentelemetry.io/otel/metric v1.39.0 + go.opentelemetry.io/otel v1.41.0 + go.opentelemetry.io/otel/metric v1.41.0 go.uber.org/multierr v1.11.0 go.uber.org/zap v1.27.1 golang.org/x/crypto v0.47.0 @@ -163,6 +163,7 @@ require ( github.com/shirou/gopsutil v3.21.11+incompatible // indirect github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.10 // indirect github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20251002192024-d2ad9222409b // indirect + github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260205130626-db2a2aab956b // indirect github.com/smartcontractkit/grpc-proxy v0.0.0-20240830132753-a7e17fec5ab7 // indirect github.com/stretchr/objx v0.5.2 // indirect github.com/supranational/blst v0.3.16-0.20250831170142-f48500c1fdbe // indirect @@ -190,14 +191,14 @@ require ( go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.36.0 // indirect go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.36.0 // indirect go.opentelemetry.io/otel/log v0.15.0 // indirect - go.opentelemetry.io/otel/sdk v1.39.0 // indirect + go.opentelemetry.io/otel/sdk v1.41.0 // indirect go.opentelemetry.io/otel/sdk/log v0.15.0 // indirect - go.opentelemetry.io/otel/sdk/metric v1.39.0 // indirect - go.opentelemetry.io/otel/trace v1.39.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.41.0 // indirect + go.opentelemetry.io/otel/trace v1.41.0 // indirect go.opentelemetry.io/proto/otlp v1.6.0 // indirect golang.org/x/mod v0.32.0 // indirect golang.org/x/net v0.49.0 // indirect - golang.org/x/sys v0.40.0 // indirect + golang.org/x/sys v0.41.0 // indirect golang.org/x/telemetry v0.0.0-20260109210033-bd525da824e2 // indirect golang.org/x/text v0.33.0 // indirect golang.org/x/time v0.14.0 // indirect diff --git a/go.sum b/go.sum index 20d91a8a68..0ff9c8b793 100644 --- a/go.sum +++ b/go.sum @@ -622,10 +622,10 @@ github.com/shurcooL/vfsgen v0.0.0-20200824052919-0d455de96546/go.mod h1:TrYk7fJV github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/smartcontractkit/chain-selectors v1.0.67 h1:gxTqP/JC40KDe3DE1SIsIKSTKTZEPyEU1YufO1admnw= -github.com/smartcontractkit/chain-selectors v1.0.67/go.mod h1:xsKM0aN3YGcQKTPRPDDtPx2l4mlTN1Djmg0VVXV40b8= -github.com/smartcontractkit/chainlink-common v0.9.6-0.20260119150426-31ad843060e6 h1:7Ybg52RRF7IRfu71E48YF83aJ9KCLonKNmBMJUB8W/A= -github.com/smartcontractkit/chainlink-common v0.9.6-0.20260119150426-31ad843060e6/go.mod h1:Eg5rz/fQINjR9H0TxHw7j+zGZeYxprUpEQZzC5JGHG4= +github.com/smartcontractkit/chain-selectors v1.0.89 h1:L9oWZGqQXWyTPnC6ODXgu3b0DFyLmJ9eHv+uJrE9IZY= +github.com/smartcontractkit/chain-selectors v1.0.89/go.mod h1:qy7whtgG5g+7z0jt0nRyii9bLND9m15NZTzuQPkMZ5w= +github.com/smartcontractkit/chainlink-common v0.10.1-0.20260305114348-b8bbac30bfc7 h1:dpSbRWnEgAx2lXsMbLyzGFoIgiYGMUHgNJPXpuEGMzM= +github.com/smartcontractkit/chainlink-common v0.10.1-0.20260305114348-b8bbac30bfc7/go.mod h1:0ghbAr7tRO0tT5ZqBXhOyzgUO37tNNe33Yn0hskauVM= github.com/smartcontractkit/chainlink-common/keystore v0.1.1-0.20260115164422-897ee18790cd h1:q9wtu29jtE3UgZ/P6qzTD/hK6Zv6xA0EFKmZT9mCIl4= github.com/smartcontractkit/chainlink-common/keystore v0.1.1-0.20260115164422-897ee18790cd/go.mod h1:30GutVApUFKO6DbtxfaIO1sEEQ5jMwkurGcfRkLn7bY= github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.10 h1:FJAFgXS9oqASnkS03RE1HQwYQQxrO4l46O5JSzxqLgg= @@ -636,14 +636,16 @@ github.com/smartcontractkit/chainlink-framework/capabilities v0.0.0-202508181755 github.com/smartcontractkit/chainlink-framework/capabilities v0.0.0-20250818175541-3389ac08a563/go.mod h1:jP5mrOLFEYZZkl7EiCHRRIMSSHCQsYypm1OZSus//iI= github.com/smartcontractkit/chainlink-framework/chains v0.0.0-20251210101658-1c5c8e4c4f15 h1:Mf+IRvrXutcKAKpuOxq5Ae+AAw4Z5vc66q1xI7qimZQ= github.com/smartcontractkit/chainlink-framework/chains v0.0.0-20251210101658-1c5c8e4c4f15/go.mod h1:kGprqyjsz6qFNVszOQoHc24wfvCjyipNZFste/3zcbs= -github.com/smartcontractkit/chainlink-framework/metrics v0.0.0-20251020150604-8ab84f7bad1a h1:pr0VFI7AWlDVJBEkcvzXWd97V8w8QMNjRdfPVa/IQLk= -github.com/smartcontractkit/chainlink-framework/metrics v0.0.0-20251020150604-8ab84f7bad1a/go.mod h1:jo+cUqNcHwN8IF7SInQNXDZ8qzBsyMpnLdYbDswviFc= -github.com/smartcontractkit/chainlink-framework/multinode v0.0.0-20251021173435-e86785845942 h1:T/eCDsUI8EJT4n5zSP4w1mz4RHH+ap8qieA17QYfBhk= -github.com/smartcontractkit/chainlink-framework/multinode v0.0.0-20251021173435-e86785845942/go.mod h1:2JTBNp3FlRdO/nHc4dsc9bfxxMClMO1Qt8sLJgtreBY= -github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20251124151448-0448aefdaab9 h1:QRWXJusIj/IRY5Pl3JclNvDre0cZPd/5NbILwc4RV2M= -github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20251124151448-0448aefdaab9/go.mod h1:jUC52kZzEnWF9tddHh85zolKybmLpbQ1oNA4FjOHt1Q= +github.com/smartcontractkit/chainlink-framework/metrics v0.0.0-20260310180305-3ee91a6d9ae9 h1:GK+2aFpW/Z5ZnMGCa9NU6o7LKHQ/9xJVZx2yMAMudnc= +github.com/smartcontractkit/chainlink-framework/metrics v0.0.0-20260310180305-3ee91a6d9ae9/go.mod h1:HG/aei0MgBOpsyRLexdKGtOUO8yjSJO3iUu0Uu8KBm4= +github.com/smartcontractkit/chainlink-framework/multinode v0.0.0-20260317161400-793ab9bc5b53 h1:xOcX4Mp6udpFErDcRiMghcezT85DXHouJlEp60MskH8= +github.com/smartcontractkit/chainlink-framework/multinode v0.0.0-20260317161400-793ab9bc5b53/go.mod h1:n865LsUxibw9oJM0pH74EBiejJ/x/AgIGHaD99D3SDY= +github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260226130359-963f935e0396 h1:03tbcwjyIEjvHba1IWOj1sfThwebm2XNzyFHSuZtlWc= +github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260226130359-963f935e0396/go.mod h1:Jqt53s27Tr0jDl8mdBXg1xhu6F8Fci8JOuq43tgHOM8= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20251002192024-d2ad9222409b h1:QuI6SmQFK/zyUlVWEf0GMkiUYBPY4lssn26nKSd/bOM= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20251002192024-d2ad9222409b/go.mod h1:qSTSwX3cBP3FKQwQacdjArqv0g6QnukjV4XuzO6UyoY= +github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260205130626-db2a2aab956b h1:36knUpKHHAZ86K4FGWXtx8i/EQftGdk2bqCoEu/Cha8= +github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260205130626-db2a2aab956b/go.mod h1:dkR2uYg9XYJuT1JASkPzWE51jjFkVb86P7a/yXe5/GM= github.com/smartcontractkit/chainlink-protos/svr v1.1.1-0.20260203131522-bb8bc5c423b3 h1:X8Pekpv+cy0eW1laZTwATuYLTLZ6gRTxz1ZWOMtU74o= github.com/smartcontractkit/chainlink-protos/svr v1.1.1-0.20260203131522-bb8bc5c423b3/go.mod h1:TcOliTQU6r59DwG4lo3U+mFM9WWyBHGuFkkxQpvSujo= github.com/smartcontractkit/chainlink-tron/relayer v0.0.11-0.20250815105909-75499abc4335 h1:7bxYNrPpygn8PUSBiEKn8riMd7CXMi/4bjTy0fHhcrY= @@ -746,8 +748,8 @@ go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbE go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0 h1:YH4g8lQroajqUwWbq/tr2QX1JFmEXaDLgG+ew9bLMWo= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0/go.mod h1:fvPi2qXDqFs8M4B4fmJhE92TyQs9Ydjlg3RvfUp+NbQ= go.opentelemetry.io/otel v1.6.3/go.mod h1:7BgNga5fNlF/iZjG06hM3yofffp0ofKCDwSXx1GC4dI= -go.opentelemetry.io/otel v1.39.0 h1:8yPrr/S0ND9QEfTfdP9V+SiwT4E0G7Y5MO7p85nis48= -go.opentelemetry.io/otel v1.39.0/go.mod h1:kLlFTywNWrFyEdH0oj2xK0bFYZtHRYUdv1NklR/tgc8= +go.opentelemetry.io/otel v1.41.0 h1:YlEwVsGAlCvczDILpUXpIpPSL/VPugt7zHThEMLce1c= +go.opentelemetry.io/otel v1.41.0/go.mod h1:Yt4UwgEKeT05QbLwbyHXEwhnjxNO6D8L5PQP51/46dE= go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.12.2 h1:06ZeJRe5BnYXceSM9Vya83XXVaNGe3H1QqsvqRANQq8= go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.12.2/go.mod h1:DvPtKE63knkDVP88qpatBj81JxN+w1bqfVbsbCbj1WY= go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.12.2 h1:tPLwQlXbJ8NSOfZc4OkgU5h2A38M4c9kfHSVc4PFQGs= @@ -770,19 +772,19 @@ go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.36.0 h1:G8Xec/SgZQricwW go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.36.0/go.mod h1:PD57idA/AiFD5aqoxGxCvT/ILJPeHy3MjqU/NS7KogY= go.opentelemetry.io/otel/log v0.15.0 h1:0VqVnc3MgyYd7QqNVIldC3dsLFKgazR6P3P3+ypkyDY= go.opentelemetry.io/otel/log v0.15.0/go.mod h1:9c/G1zbyZfgu1HmQD7Qj84QMmwTp2QCQsZH1aeoWDE4= -go.opentelemetry.io/otel/metric v1.39.0 h1:d1UzonvEZriVfpNKEVmHXbdf909uGTOQjA0HF0Ls5Q0= -go.opentelemetry.io/otel/metric v1.39.0/go.mod h1:jrZSWL33sD7bBxg1xjrqyDjnuzTUB0x1nBERXd7Ftcs= -go.opentelemetry.io/otel/sdk v1.39.0 h1:nMLYcjVsvdui1B/4FRkwjzoRVsMK8uL/cj0OyhKzt18= -go.opentelemetry.io/otel/sdk v1.39.0/go.mod h1:vDojkC4/jsTJsE+kh+LXYQlbL8CgrEcwmt1ENZszdJE= +go.opentelemetry.io/otel/metric v1.41.0 h1:rFnDcs4gRzBcsO9tS8LCpgR0dxg4aaxWlJxCno7JlTQ= +go.opentelemetry.io/otel/metric v1.41.0/go.mod h1:xPvCwd9pU0VN8tPZYzDZV/BMj9CM9vs00GuBjeKhJps= +go.opentelemetry.io/otel/sdk v1.41.0 h1:YPIEXKmiAwkGl3Gu1huk1aYWwtpRLeskpV+wPisxBp8= +go.opentelemetry.io/otel/sdk v1.41.0/go.mod h1:ahFdU0G5y8IxglBf0QBJXgSe7agzjE4GiTJ6HT9ud90= go.opentelemetry.io/otel/sdk/log v0.15.0 h1:WgMEHOUt5gjJE93yqfqJOkRflApNif84kxoHWS9VVHE= go.opentelemetry.io/otel/sdk/log v0.15.0/go.mod h1:qDC/FlKQCXfH5hokGsNg9aUBGMJQsrUyeOiW5u+dKBQ= go.opentelemetry.io/otel/sdk/log/logtest v0.13.0 h1:9yio6AFZ3QD9j9oqshV1Ibm9gPLlHNxurno5BreMtIA= go.opentelemetry.io/otel/sdk/log/logtest v0.13.0/go.mod h1:QOGiAJHl+fob8Nu85ifXfuQYmJTFAvcrxL6w5/tu168= -go.opentelemetry.io/otel/sdk/metric v1.39.0 h1:cXMVVFVgsIf2YL6QkRF4Urbr/aMInf+2WKg+sEJTtB8= -go.opentelemetry.io/otel/sdk/metric v1.39.0/go.mod h1:xq9HEVH7qeX69/JnwEfp6fVq5wosJsY1mt4lLfYdVew= +go.opentelemetry.io/otel/sdk/metric v1.41.0 h1:siZQIYBAUd1rlIWQT2uCxWJxcCO7q3TriaMlf08rXw8= +go.opentelemetry.io/otel/sdk/metric v1.41.0/go.mod h1:HNBuSvT7ROaGtGI50ArdRLUnvRTRGniSUZbxiWxSO8Y= go.opentelemetry.io/otel/trace v1.6.3/go.mod h1:GNJQusJlUgZl9/TQBPKU/Y/ty+0iVB5fjhKeJGZPGFs= -go.opentelemetry.io/otel/trace v1.39.0 h1:2d2vfpEDmCJ5zVYz7ijaJdOF59xLomrvj7bjt6/qCJI= -go.opentelemetry.io/otel/trace v1.39.0/go.mod h1:88w4/PnZSazkGzz/w84VHpQafiU4EtqqlVdxWy+rNOA= +go.opentelemetry.io/otel/trace v1.41.0 h1:Vbk2co6bhj8L59ZJ6/xFTskY+tGAbOnCtQGVVa9TIN0= +go.opentelemetry.io/otel/trace v1.41.0/go.mod h1:U1NU4ULCoxeDKc09yCWdWe+3QoyweJcISEVa1RBzOis= go.opentelemetry.io/proto/otlp v1.6.0 h1:jQjP+AQyTf+Fe7OKj/MfkDrmK4MNVtw2NpXsf9fefDI= go.opentelemetry.io/proto/otlp v1.6.0/go.mod h1:cicgGehlFuNdgZkcALOCh3VE6K/u2tAjzlRhDwmVpZc= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= @@ -1017,8 +1019,8 @@ golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ= -golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k= +golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/telemetry v0.0.0-20260109210033-bd525da824e2 h1:O1cMQHRfwNpDfDJerqRoE2oD+AFlyid87D40L/OkkJo= golang.org/x/telemetry v0.0.0-20260109210033-bd525da824e2/go.mod h1:b7fPSJ0pKZ3ccUh8gnTONJxhn3c/PS6tyzQvyqw4iA8= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= From 1db6705c90a1742bddb8ace8343aa0479ebcd0e6 Mon Sep 17 00:00:00 2001 From: Krish-vemula Date: Wed, 18 Mar 2026 12:05:23 -0500 Subject: [PATCH 8/8] chore: regenerate CONFIG.md with finalized state check fields --- CONFIG.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/CONFIG.md b/CONFIG.md index 25543c7b81..a25776b6ad 100644 --- a/CONFIG.md +++ b/CONFIG.md @@ -926,6 +926,7 @@ LeaseDuration = '0s' # Default NodeIsSyncingEnabled = false # Default FinalizedBlockPollInterval = '5s' # Default HistoricalBalanceCheckAddress = '0x0000000000000000000000000000000000000000' # Default +FinalizedStateCheckFailureThreshold = 0 # Default EnforceRepeatableRead = true # Default DeathDeclarationDelay = '1m' # Default NewHeadsPollInterval = '0s' # Default @@ -1015,6 +1016,14 @@ Finalized block selection follows chain finality settings: - `FinalityTagEnabled = false`: use `latest - FinalityDepth` The check is only active when `FinalizedStateCheckFailureThreshold > 0`. +### FinalizedStateCheckFailureThreshold +```toml +FinalizedStateCheckFailureThreshold = 0 # Default +``` +FinalizedStateCheckFailureThreshold is the number of consecutive failures of the finalized state availability check +before the node is marked as FinalizedStateNotAvailable. +Set to 0 to disable the check. + ### EnforceRepeatableRead ```toml EnforceRepeatableRead = true # Default @@ -1080,6 +1089,7 @@ Fatal = '(: |^)fatal' # Example ServiceUnavailable = '(: |^)service unavailable' # Example TooManyResults = '(: |^)too many results' # Example MissingBlocks = '(: |^)invalid block range' # Example +FinalizedStateUnavailable = '(missing trie node|state not available|historical state unavailable)' # Example ``` Errors enable the node to provide custom regex patterns to match against error messages from RPCs. @@ -1179,6 +1189,12 @@ MissingBlocks = '(: |^)invalid block range' # Example ``` MissingBlocks is a regex pattern to match an eth_getLogs error indicating the rpc server is permanently missing some blocks in the requested block range +### FinalizedStateUnavailable +```toml +FinalizedStateUnavailable = '(missing trie node|state not available|historical state unavailable)' # Example +``` +FinalizedStateUnavailable is a regex pattern to match errors indicating the RPC cannot serve historical state at the finalized block (e.g., pruned/non-archive node) + ## OCR ```toml [OCR]