diff --git a/CONFIG.md b/CONFIG.md index c5f7c0e55c..a25776b6ad 100644 --- a/CONFIG.md +++ b/CONFIG.md @@ -925,6 +925,8 @@ SyncThreshold = 5 # Default LeaseDuration = '0s' # Default NodeIsSyncingEnabled = false # Default FinalizedBlockPollInterval = '5s' # Default +HistoricalBalanceCheckAddress = '0x0000000000000000000000000000000000000000' # Default +FinalizedStateCheckFailureThreshold = 0 # Default EnforceRepeatableRead = true # Default DeathDeclarationDelay = '1m' # Default NewHeadsPollInterval = '0s' # Default @@ -1003,6 +1005,25 @@ reported based on latest block and finality depth. Set to 0 to disable. +### HistoricalBalanceCheckAddress +```toml +HistoricalBalanceCheckAddress = '0x0000000000000000000000000000000000000000' # Default +``` +HistoricalBalanceCheckAddress is the probe account used by the historical balance health check. +The check executes `eth_getBalance` for this address at the latest finalized block. +Finalized block selection follows chain finality settings: +- `FinalityTagEnabled = true`: use `finalized` tag +- `FinalityTagEnabled = false`: use `latest - FinalityDepth` +The check is only active when `FinalizedStateCheckFailureThreshold > 0`. + +### FinalizedStateCheckFailureThreshold +```toml +FinalizedStateCheckFailureThreshold = 0 # Default +``` +FinalizedStateCheckFailureThreshold is the number of consecutive failures of the finalized state availability check +before the node is marked as FinalizedStateNotAvailable. +Set to 0 to disable the check. + ### EnforceRepeatableRead ```toml EnforceRepeatableRead = true # Default @@ -1068,6 +1089,7 @@ Fatal = '(: |^)fatal' # Example ServiceUnavailable = '(: |^)service unavailable' # Example TooManyResults = '(: |^)too many results' # Example MissingBlocks = '(: |^)invalid block range' # Example +FinalizedStateUnavailable = '(missing trie node|state not available|historical state unavailable)' # Example ``` Errors enable the node to provide custom regex patterns to match against error messages from RPCs. @@ -1167,6 +1189,12 @@ MissingBlocks = '(: |^)invalid block range' # Example ``` MissingBlocks is a regex pattern to match an eth_getLogs error indicating the rpc server is permanently missing some blocks in the requested block range +### FinalizedStateUnavailable +```toml +FinalizedStateUnavailable = '(missing trie node|state not available|historical state unavailable)' # Example +``` +FinalizedStateUnavailable is a regex pattern to match errors indicating the RPC cannot serve historical state at the finalized block (e.g., pruned/non-archive node) + ## OCR ```toml [OCR] diff --git a/go.mod b/go.mod index c2f6ce055c..0a0d451c70 100644 --- a/go.mod +++ b/go.mod @@ -33,8 +33,8 @@ require ( github.com/smartcontractkit/chainlink-evm/gethwrappers v0.0.0-20251022073203-7d8ae8cf67c1 github.com/smartcontractkit/chainlink-framework/capabilities v0.0.0-20250818175541-3389ac08a563 github.com/smartcontractkit/chainlink-framework/chains v0.0.0-20260317132927-e8bc2c7b01f1 - github.com/smartcontractkit/chainlink-framework/metrics v0.0.0-20251020150604-8ab84f7bad1a - github.com/smartcontractkit/chainlink-framework/multinode v0.0.0-20251021173435-e86785845942 + github.com/smartcontractkit/chainlink-framework/metrics v0.0.0-20260310180305-3ee91a6d9ae9 + github.com/smartcontractkit/chainlink-framework/multinode v0.0.0-20260317161400-793ab9bc5b53 github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260226130359-963f935e0396 github.com/smartcontractkit/chainlink-protos/svr v1.1.1-0.20260203131522-bb8bc5c423b3 github.com/smartcontractkit/chainlink-tron/relayer v0.0.11-0.20250815105909-75499abc4335 @@ -46,8 +46,8 @@ require ( github.com/tidwall/gjson v1.18.0 github.com/ugorji/go/codec v1.2.12 github.com/umbracle/ethgo v0.1.3 - go.opentelemetry.io/otel v1.39.0 - go.opentelemetry.io/otel/metric v1.39.0 + go.opentelemetry.io/otel v1.41.0 + go.opentelemetry.io/otel/metric v1.41.0 go.uber.org/multierr v1.11.0 go.uber.org/zap v1.27.1 golang.org/x/crypto v0.48.0 @@ -199,10 +199,10 @@ require ( go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.36.0 // indirect go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.36.0 // indirect go.opentelemetry.io/otel/log v0.15.0 // indirect - go.opentelemetry.io/otel/sdk v1.39.0 // indirect + go.opentelemetry.io/otel/sdk v1.41.0 // indirect go.opentelemetry.io/otel/sdk/log v0.15.0 // indirect - go.opentelemetry.io/otel/sdk/metric v1.39.0 // indirect - go.opentelemetry.io/otel/trace v1.39.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.41.0 // indirect + go.opentelemetry.io/otel/trace v1.41.0 // indirect go.opentelemetry.io/proto/otlp v1.9.0 // indirect golang.org/x/mod v0.32.0 // indirect golang.org/x/net v0.49.0 // indirect diff --git a/go.sum b/go.sum index 5cc9b7785d..c4f6173949 100644 --- a/go.sum +++ b/go.sum @@ -662,10 +662,10 @@ github.com/smartcontractkit/chainlink-framework/capabilities v0.0.0-202508181755 github.com/smartcontractkit/chainlink-framework/capabilities v0.0.0-20250818175541-3389ac08a563/go.mod h1:jP5mrOLFEYZZkl7EiCHRRIMSSHCQsYypm1OZSus//iI= github.com/smartcontractkit/chainlink-framework/chains v0.0.0-20260317132927-e8bc2c7b01f1 h1:aUdjMnHpriMkEwsgeqQ/ZuNBjrWw6c46HG57TuPPEbE= github.com/smartcontractkit/chainlink-framework/chains v0.0.0-20260317132927-e8bc2c7b01f1/go.mod h1:kGprqyjsz6qFNVszOQoHc24wfvCjyipNZFste/3zcbs= -github.com/smartcontractkit/chainlink-framework/metrics v0.0.0-20251020150604-8ab84f7bad1a h1:pr0VFI7AWlDVJBEkcvzXWd97V8w8QMNjRdfPVa/IQLk= -github.com/smartcontractkit/chainlink-framework/metrics v0.0.0-20251020150604-8ab84f7bad1a/go.mod h1:jo+cUqNcHwN8IF7SInQNXDZ8qzBsyMpnLdYbDswviFc= -github.com/smartcontractkit/chainlink-framework/multinode v0.0.0-20251021173435-e86785845942 h1:T/eCDsUI8EJT4n5zSP4w1mz4RHH+ap8qieA17QYfBhk= -github.com/smartcontractkit/chainlink-framework/multinode v0.0.0-20251021173435-e86785845942/go.mod h1:2JTBNp3FlRdO/nHc4dsc9bfxxMClMO1Qt8sLJgtreBY= +github.com/smartcontractkit/chainlink-framework/metrics v0.0.0-20260310180305-3ee91a6d9ae9 h1:GK+2aFpW/Z5ZnMGCa9NU6o7LKHQ/9xJVZx2yMAMudnc= +github.com/smartcontractkit/chainlink-framework/metrics v0.0.0-20260310180305-3ee91a6d9ae9/go.mod h1:HG/aei0MgBOpsyRLexdKGtOUO8yjSJO3iUu0Uu8KBm4= +github.com/smartcontractkit/chainlink-framework/multinode v0.0.0-20260317161400-793ab9bc5b53 h1:xOcX4Mp6udpFErDcRiMghcezT85DXHouJlEp60MskH8= +github.com/smartcontractkit/chainlink-framework/multinode v0.0.0-20260317161400-793ab9bc5b53/go.mod h1:n865LsUxibw9oJM0pH74EBiejJ/x/AgIGHaD99D3SDY= github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260226130359-963f935e0396 h1:03tbcwjyIEjvHba1IWOj1sfThwebm2XNzyFHSuZtlWc= github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260226130359-963f935e0396/go.mod h1:Jqt53s27Tr0jDl8mdBXg1xhu6F8Fci8JOuq43tgHOM8= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20251002192024-d2ad9222409b h1:QuI6SmQFK/zyUlVWEf0GMkiUYBPY4lssn26nKSd/bOM= @@ -790,8 +790,8 @@ go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0 h1:YH4g8lQroajqUwWbq/tr2QX1JFmEXaDLgG+ew9bLMWo= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0/go.mod h1:fvPi2qXDqFs8M4B4fmJhE92TyQs9Ydjlg3RvfUp+NbQ= -go.opentelemetry.io/otel v1.39.0 h1:8yPrr/S0ND9QEfTfdP9V+SiwT4E0G7Y5MO7p85nis48= -go.opentelemetry.io/otel v1.39.0/go.mod h1:kLlFTywNWrFyEdH0oj2xK0bFYZtHRYUdv1NklR/tgc8= +go.opentelemetry.io/otel v1.41.0 h1:YlEwVsGAlCvczDILpUXpIpPSL/VPugt7zHThEMLce1c= +go.opentelemetry.io/otel v1.41.0/go.mod h1:Yt4UwgEKeT05QbLwbyHXEwhnjxNO6D8L5PQP51/46dE= go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.12.2 h1:06ZeJRe5BnYXceSM9Vya83XXVaNGe3H1QqsvqRANQq8= go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.12.2/go.mod h1:DvPtKE63knkDVP88qpatBj81JxN+w1bqfVbsbCbj1WY= go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.12.2 h1:tPLwQlXbJ8NSOfZc4OkgU5h2A38M4c9kfHSVc4PFQGs= @@ -814,18 +814,18 @@ go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.36.0 h1:G8Xec/SgZQricwW go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.36.0/go.mod h1:PD57idA/AiFD5aqoxGxCvT/ILJPeHy3MjqU/NS7KogY= go.opentelemetry.io/otel/log v0.15.0 h1:0VqVnc3MgyYd7QqNVIldC3dsLFKgazR6P3P3+ypkyDY= go.opentelemetry.io/otel/log v0.15.0/go.mod h1:9c/G1zbyZfgu1HmQD7Qj84QMmwTp2QCQsZH1aeoWDE4= -go.opentelemetry.io/otel/metric v1.39.0 h1:d1UzonvEZriVfpNKEVmHXbdf909uGTOQjA0HF0Ls5Q0= -go.opentelemetry.io/otel/metric v1.39.0/go.mod h1:jrZSWL33sD7bBxg1xjrqyDjnuzTUB0x1nBERXd7Ftcs= -go.opentelemetry.io/otel/sdk v1.39.0 h1:nMLYcjVsvdui1B/4FRkwjzoRVsMK8uL/cj0OyhKzt18= -go.opentelemetry.io/otel/sdk v1.39.0/go.mod h1:vDojkC4/jsTJsE+kh+LXYQlbL8CgrEcwmt1ENZszdJE= +go.opentelemetry.io/otel/metric v1.41.0 h1:rFnDcs4gRzBcsO9tS8LCpgR0dxg4aaxWlJxCno7JlTQ= +go.opentelemetry.io/otel/metric v1.41.0/go.mod h1:xPvCwd9pU0VN8tPZYzDZV/BMj9CM9vs00GuBjeKhJps= +go.opentelemetry.io/otel/sdk v1.41.0 h1:YPIEXKmiAwkGl3Gu1huk1aYWwtpRLeskpV+wPisxBp8= +go.opentelemetry.io/otel/sdk v1.41.0/go.mod h1:ahFdU0G5y8IxglBf0QBJXgSe7agzjE4GiTJ6HT9ud90= go.opentelemetry.io/otel/sdk/log v0.15.0 h1:WgMEHOUt5gjJE93yqfqJOkRflApNif84kxoHWS9VVHE= go.opentelemetry.io/otel/sdk/log v0.15.0/go.mod h1:qDC/FlKQCXfH5hokGsNg9aUBGMJQsrUyeOiW5u+dKBQ= go.opentelemetry.io/otel/sdk/log/logtest v0.13.0 h1:9yio6AFZ3QD9j9oqshV1Ibm9gPLlHNxurno5BreMtIA= go.opentelemetry.io/otel/sdk/log/logtest v0.13.0/go.mod h1:QOGiAJHl+fob8Nu85ifXfuQYmJTFAvcrxL6w5/tu168= -go.opentelemetry.io/otel/sdk/metric v1.39.0 h1:cXMVVFVgsIf2YL6QkRF4Urbr/aMInf+2WKg+sEJTtB8= -go.opentelemetry.io/otel/sdk/metric v1.39.0/go.mod h1:xq9HEVH7qeX69/JnwEfp6fVq5wosJsY1mt4lLfYdVew= -go.opentelemetry.io/otel/trace v1.39.0 h1:2d2vfpEDmCJ5zVYz7ijaJdOF59xLomrvj7bjt6/qCJI= -go.opentelemetry.io/otel/trace v1.39.0/go.mod h1:88w4/PnZSazkGzz/w84VHpQafiU4EtqqlVdxWy+rNOA= +go.opentelemetry.io/otel/sdk/metric v1.41.0 h1:siZQIYBAUd1rlIWQT2uCxWJxcCO7q3TriaMlf08rXw8= +go.opentelemetry.io/otel/sdk/metric v1.41.0/go.mod h1:HNBuSvT7ROaGtGI50ArdRLUnvRTRGniSUZbxiWxSO8Y= +go.opentelemetry.io/otel/trace v1.41.0 h1:Vbk2co6bhj8L59ZJ6/xFTskY+tGAbOnCtQGVVa9TIN0= +go.opentelemetry.io/otel/trace v1.41.0/go.mod h1:U1NU4ULCoxeDKc09yCWdWe+3QoyweJcISEVa1RBzOis= go.opentelemetry.io/proto/otlp v1.9.0 h1:l706jCMITVouPOqEnii2fIAuO3IVGBRPV5ICjceRb/A= go.opentelemetry.io/proto/otlp v1.9.0/go.mod h1:xE+Cx5E/eEHw+ISFkwPLwCZefwVjY+pqKg1qcK03+/4= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= diff --git a/pkg/client/helpers_test.go b/pkg/client/helpers_test.go index 7734ea1bc6..cd78edfbcc 100644 --- a/pkg/client/helpers_test.go +++ b/pkg/client/helpers_test.go @@ -39,6 +39,7 @@ type TestClientErrors struct { serviceUnavailable string tooManyResults string missingBlocks string + finalizedStateUnavailable string } func NewTestClientErrors() TestClientErrors { @@ -83,23 +84,26 @@ func (c *TestClientErrors) L2FeeTooHigh() string { return c.l2FeeTooH func (c *TestClientErrors) L2Full() string { return c.l2Full } func (c *TestClientErrors) TransactionAlreadyMined() string { return c.transactionAlreadyMined } func (c *TestClientErrors) Fatal() string { return c.fatal } -func (c *TestClientErrors) ServiceUnavailable() string { return c.serviceUnavailable } -func (c *TestClientErrors) TooManyResults() string { return c.tooManyResults } -func (c *TestClientErrors) MissingBlocks() string { return c.missingBlocks } +func (c *TestClientErrors) ServiceUnavailable() string { return c.serviceUnavailable } +func (c *TestClientErrors) TooManyResults() string { return c.tooManyResults } +func (c *TestClientErrors) MissingBlocks() string { return c.missingBlocks } +func (c *TestClientErrors) FinalizedStateUnavailable() string { return c.finalizedStateUnavailable } type TestNodePoolConfig struct { - NodePollFailureThreshold uint32 - NodePollInterval time.Duration - NodeSelectionMode string - NodeSyncThreshold uint32 - NodeLeaseDuration time.Duration - NodeIsSyncingEnabledVal bool - NodeFinalizedBlockPollInterval time.Duration - NodeErrors config.ClientErrors - EnforceRepeatableReadVal bool - NodeDeathDeclarationDelay time.Duration - NodeNewHeadsPollInterval time.Duration - ExternalRequestMaxResponseSizeVal uint32 + NodePollFailureThreshold uint32 + NodePollInterval time.Duration + NodeSelectionMode string + NodeSyncThreshold uint32 + NodeLeaseDuration time.Duration + NodeIsSyncingEnabledVal bool + NodeFinalizedBlockPollInterval time.Duration + HistoricalBalanceCheckAddressVal string + NodeErrors config.ClientErrors + EnforceRepeatableReadVal bool + NodeDeathDeclarationDelay time.Duration + NodeNewHeadsPollInterval time.Duration + ExternalRequestMaxResponseSizeVal uint32 + FinalizedStateCheckFailureThresholdVal uint32 } func (tc TestNodePoolConfig) PollFailureThreshold() uint32 { return tc.NodePollFailureThreshold } @@ -118,6 +122,10 @@ func (tc TestNodePoolConfig) FinalizedBlockPollInterval() time.Duration { return tc.NodeFinalizedBlockPollInterval } +func (tc TestNodePoolConfig) HistoricalBalanceCheckAddress() string { + return tc.HistoricalBalanceCheckAddressVal +} + func (tc TestNodePoolConfig) NewHeadsPollInterval() time.Duration { return tc.NodeNewHeadsPollInterval } @@ -142,6 +150,10 @@ func (tc TestNodePoolConfig) ExternalRequestMaxResponseSize() uint32 { return tc.ExternalRequestMaxResponseSizeVal } +func (tc TestNodePoolConfig) FinalizedStateCheckFailureThreshold() uint32 { + return tc.FinalizedStateCheckFailureThresholdVal +} + func NewChainClientWithTestNode( t *testing.T, nodeCfg multinode.NodeConfig, diff --git a/pkg/client/rpc_client.go b/pkg/client/rpc_client.go index a1977f7d80..3a8c538bcc 100644 --- a/pkg/client/rpc_client.go +++ b/pkg/client/rpc_client.go @@ -9,6 +9,7 @@ import ( "math/big" "net/http" "net/url" + "regexp" "strconv" "sync/atomic" "time" @@ -105,6 +106,7 @@ type RPCClient struct { finalityTagEnabled bool finalityDepth uint32 safeDepth uint32 + historicalBalanceCheckAddress common.Address externalRequestMaxResponseSize uint32 beholderMetrics *rpcClientMetrics @@ -144,6 +146,7 @@ func NewRPCClient( finalityTagEnabled: supportsFinalityTags, finalityDepth: finalityDepth, safeDepth: safeDepth, + historicalBalanceCheckAddress: common.HexToAddress(cfg.HistoricalBalanceCheckAddress()), externalRequestMaxResponseSize: externalRequestMaxResponseSize, } r.cfg = cfg @@ -193,6 +196,50 @@ func (r *RPCClient) ClientVersion(ctx context.Context) (version string, err erro return version, nil } +// CheckFinalizedStateAvailability verifies if the RPC can serve historical state at the finalized block. +// This is used to detect non-archive nodes that cannot serve state queries for older blocks. +// Returns multinode.ErrFinalizedStateUnavailable if the error matches the FinalizedStateUnavailable pattern. +// The decision to call this method is made by multinode based on its configuration. +func (r *RPCClient) CheckFinalizedStateAvailability(ctx context.Context) error { + var blockNumber *big.Int + if r.finalityTagEnabled { + blockNumber = big.NewInt(rpc.FinalizedBlockNumber.Int64()) + } else { + latestBlock, err := r.BlockNumber(ctx) + if err != nil { + return fmt.Errorf("fetching latest block number failed: %w", err) + } + latest := int64(latestBlock) + finalizedHeight := max(int64(0), latest-int64(r.finalityDepth)) + blockNumber = big.NewInt(finalizedHeight) + } + _, err := r.BalanceAt(ctx, r.historicalBalanceCheckAddress, blockNumber) + if err != nil { + if r.isFinalizedStateUnavailableError(err) { + return fmt.Errorf("%w: %w", multinode.ErrFinalizedStateUnavailable, err) + } + return fmt.Errorf("fetching balance for address %s at block %s failed: %w", r.historicalBalanceCheckAddress.String(), blockNumber.String(), err) + } + return nil +} + +// isFinalizedStateUnavailableError checks if the error matches the FinalizedStateUnavailable regex pattern. +func (r *RPCClient) isFinalizedStateUnavailableError(err error) bool { + if err == nil { + return false + } + pattern := r.clientErrors.FinalizedStateUnavailable() + if pattern == "" { + return false + } + re, compileErr := regexp.Compile(pattern) + if compileErr != nil { + r.rpcLog.Criticalw("FinalizedStateUnavailable regex pattern is invalid; finalized state availability check is effectively disabled", "pattern", pattern, "err", compileErr) + return false + } + return re.MatchString(err.Error()) +} + func (r *RPCClient) Dial(callerCtx context.Context) error { ctx, cancel, _ := r.AcquireQueryCtx(callerCtx, r.rpcTimeout) defer cancel() diff --git a/pkg/client/rpc_client_internal_test.go b/pkg/client/rpc_client_internal_test.go index b8a3b1fa75..c407942c38 100644 --- a/pkg/client/rpc_client_internal_test.go +++ b/pkg/client/rpc_client_internal_test.go @@ -10,6 +10,7 @@ import ( "testing" "time" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" ethtypes "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/rpc" @@ -436,3 +437,132 @@ func NewTestRPCClient(t *testing.T, opts RPCClientOpts) *RPCClient { func ptr[T any](v T) *T { return &v } + +func TestRPCClient_CheckFinalizedStateAvailability(t *testing.T) { + t.Parallel() + chainID := big.NewInt(1337) + probeAddress := "0x0000000000000000000000000000000000000001" + expectedAddress := common.HexToAddress(probeAddress).String() + + t.Run("uses finalized tag when enabled", func(t *testing.T) { + t.Parallel() + wsURL := testutils.NewWSServer(t, chainID, func(method string, params gjson.Result) (resp testutils.JSONRPCResponse) { + switch method { + case "eth_getBalance": + require.Equal(t, expectedAddress, params.Array()[0].String()) + require.Equal(t, "finalized", params.Array()[1].String()) + resp.Result = `"0x0"` + default: + require.Fail(t, "unexpected method: "+method) + } + return + }).WSURL() + + rpcClient := NewDialedTestRPCClient(t, RPCClientOpts{ + HTTP: wsURL, + Cfg: &TestNodePoolConfig{ + NodeFinalizedBlockPollInterval: 1 * time.Second, + HistoricalBalanceCheckAddressVal: probeAddress, + }, + FinalityTagsEnabled: true, + ChainID: chainID, + }) + + err := rpcClient.CheckFinalizedStateAvailability(t.Context()) + require.NoError(t, err) + }) + + t.Run("uses latest-finalityDepth when finality tags disabled", func(t *testing.T) { + t.Parallel() + wsURL := testutils.NewWSServer(t, chainID, func(method string, params gjson.Result) (resp testutils.JSONRPCResponse) { + switch method { + case "eth_blockNumber": + resp.Result = `"0x14"` // 20 + case "eth_getBalance": + require.Equal(t, expectedAddress, params.Array()[0].String()) + require.Equal(t, "0x10", params.Array()[1].String()) // 20 - 4 + resp.Result = `"0x0"` + default: + require.Fail(t, "unexpected method: "+method) + } + return + }).WSURL() + + rpcClient := NewDialedTestRPCClient(t, RPCClientOpts{ + HTTP: wsURL, + Cfg: &TestNodePoolConfig{ + NodeFinalizedBlockPollInterval: 1 * time.Second, + HistoricalBalanceCheckAddressVal: probeAddress, + }, + FinalityTagsEnabled: false, + FinalityDepth: 4, + ChainID: chainID, + }) + + err := rpcClient.CheckFinalizedStateAvailability(t.Context()) + require.NoError(t, err) + }) + + t.Run("returns ErrFinalizedStateUnavailable when error matches regex", func(t *testing.T) { + t.Parallel() + wsURL := testutils.NewWSServer(t, chainID, func(method string, _ gjson.Result) (resp testutils.JSONRPCResponse) { + switch method { + case "eth_getBalance": + resp.Error.Message = "missing trie node" + default: + require.Fail(t, "unexpected method: "+method) + } + return + }).WSURL() + + clientErrors := NewTestClientErrors() + clientErrors.finalizedStateUnavailable = "missing trie node" + + rpcClient := NewDialedTestRPCClient(t, RPCClientOpts{ + HTTP: wsURL, + Cfg: &TestNodePoolConfig{ + NodeFinalizedBlockPollInterval: 1 * time.Second, + HistoricalBalanceCheckAddressVal: probeAddress, + NodeErrors: &clientErrors, + }, + FinalityTagsEnabled: true, + ChainID: chainID, + }) + + err := rpcClient.CheckFinalizedStateAvailability(t.Context()) + require.Error(t, err) + require.ErrorIs(t, err, multinode.ErrFinalizedStateUnavailable) + }) + + t.Run("returns generic error when error does not match regex", func(t *testing.T) { + t.Parallel() + wsURL := testutils.NewWSServer(t, chainID, func(method string, _ gjson.Result) (resp testutils.JSONRPCResponse) { + switch method { + case "eth_getBalance": + resp.Error.Message = "connection reset" + default: + require.Fail(t, "unexpected method: "+method) + } + return + }).WSURL() + + clientErrors := NewTestClientErrors() + clientErrors.finalizedStateUnavailable = "missing trie node" + + rpcClient := NewDialedTestRPCClient(t, RPCClientOpts{ + HTTP: wsURL, + Cfg: &TestNodePoolConfig{ + NodeFinalizedBlockPollInterval: 1 * time.Second, + HistoricalBalanceCheckAddressVal: probeAddress, + NodeErrors: &clientErrors, + }, + FinalityTagsEnabled: true, + ChainID: chainID, + }) + + err := rpcClient.CheckFinalizedStateAvailability(t.Context()) + require.Error(t, err) + require.ErrorContains(t, err, "fetching balance") + require.NotErrorIs(t, err, multinode.ErrFinalizedStateUnavailable) + }) +} diff --git a/pkg/config/chain_scoped_client_errors.go b/pkg/config/chain_scoped_client_errors.go index f0b95b0b3c..03ddad0157 100644 --- a/pkg/config/chain_scoped_client_errors.go +++ b/pkg/config/chain_scoped_client_errors.go @@ -50,3 +50,6 @@ func (c *clientErrorsConfig) ServiceUnavailable() string { } func (c *clientErrorsConfig) TooManyResults() string { return derefOrDefault(c.c.TooManyResults) } func (c *clientErrorsConfig) MissingBlocks() string { return derefOrDefault(c.c.MissingBlocks) } +func (c *clientErrorsConfig) FinalizedStateUnavailable() string { + return derefOrDefault(c.c.FinalizedStateUnavailable) +} diff --git a/pkg/config/chain_scoped_node_pool.go b/pkg/config/chain_scoped_node_pool.go index d9cd17ee72..36e333c9b1 100644 --- a/pkg/config/chain_scoped_node_pool.go +++ b/pkg/config/chain_scoped_node_pool.go @@ -38,6 +38,13 @@ func (n *NodePoolConfig) FinalizedBlockPollInterval() time.Duration { return n.C.FinalizedBlockPollInterval.Duration() } +func (n *NodePoolConfig) HistoricalBalanceCheckAddress() string { + if n.C.HistoricalBalanceCheckAddress == nil { + return "" + } + return n.C.HistoricalBalanceCheckAddress.String() +} + func (n *NodePoolConfig) NewHeadsPollInterval() time.Duration { return n.C.NewHeadsPollInterval.Duration() } @@ -62,3 +69,10 @@ func (n *NodePoolConfig) ExternalRequestMaxResponseSize() uint32 { } return *n.C.ExternalRequestMaxResponseSize } + +func (n *NodePoolConfig) FinalizedStateCheckFailureThreshold() uint32 { + if n.C.FinalizedStateCheckFailureThreshold == nil { + return 0 + } + return *n.C.FinalizedStateCheckFailureThreshold +} diff --git a/pkg/config/config.go b/pkg/config/config.go index bdec19f9c4..dfe3d16562 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -107,6 +107,7 @@ type ClientErrors interface { ServiceUnavailable() string TooManyResults() string MissingBlocks() string + FinalizedStateUnavailable() string } type Transactions interface { @@ -210,12 +211,14 @@ type NodePool interface { LeaseDuration() time.Duration NodeIsSyncingEnabled() bool FinalizedBlockPollInterval() time.Duration + HistoricalBalanceCheckAddress() string Errors() ClientErrors EnforceRepeatableRead() bool DeathDeclarationDelay() time.Duration NewHeadsPollInterval() time.Duration VerifyChainID() bool ExternalRequestMaxResponseSize() uint32 + FinalizedStateCheckFailureThreshold() uint32 } type ChainScopedConfig interface { diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 313f50a88e..dc058c9b53 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -369,6 +369,7 @@ func TestNodePoolConfig(t *testing.T) { require.False(t, cfg.EVM().NodePool().NodeIsSyncingEnabled()) require.True(t, cfg.EVM().NodePool().EnforceRepeatableRead()) require.Equal(t, time.Minute, cfg.EVM().NodePool().DeathDeclarationDelay()) + require.Equal(t, "0x0000000000000000000000000000000000000000", cfg.EVM().NodePool().HistoricalBalanceCheckAddress()) } func TestClientErrorsConfig(t *testing.T) { diff --git a/pkg/config/toml/config.go b/pkg/config/toml/config.go index a43c65a6f6..b47c782809 100644 --- a/pkg/config/toml/config.go +++ b/pkg/config/toml/config.go @@ -1031,6 +1031,7 @@ type ClientErrors struct { ServiceUnavailable *string `toml:",omitempty"` TooManyResults *string `toml:",omitempty"` MissingBlocks *string `toml:",omitempty"` + FinalizedStateUnavailable *string `toml:",omitempty"` } func (r *ClientErrors) setFrom(f *ClientErrors) bool { @@ -1082,6 +1083,9 @@ func (r *ClientErrors) setFrom(f *ClientErrors) bool { if v := f.MissingBlocks; v != nil { r.MissingBlocks = v } + if v := f.FinalizedStateUnavailable; v != nil { + r.FinalizedStateUnavailable = v + } return true } @@ -1093,12 +1097,14 @@ type NodePool struct { LeaseDuration *commonconfig.Duration NodeIsSyncingEnabled *bool FinalizedBlockPollInterval *commonconfig.Duration - Errors ClientErrors `toml:",omitempty"` - EnforceRepeatableRead *bool - DeathDeclarationDelay *commonconfig.Duration - NewHeadsPollInterval *commonconfig.Duration - VerifyChainID *bool - ExternalRequestMaxResponseSize *uint32 + HistoricalBalanceCheckAddress *types.EIP55Address + FinalizedStateCheckFailureThreshold *uint32 + Errors ClientErrors `toml:",omitempty"` + EnforceRepeatableRead *bool + DeathDeclarationDelay *commonconfig.Duration + NewHeadsPollInterval *commonconfig.Duration + VerifyChainID *bool + ExternalRequestMaxResponseSize *uint32 } func (p *NodePool) setFrom(f *NodePool) { @@ -1123,6 +1129,9 @@ func (p *NodePool) setFrom(f *NodePool) { if v := f.FinalizedBlockPollInterval; v != nil { p.FinalizedBlockPollInterval = v } + if v := f.HistoricalBalanceCheckAddress; v != nil { + p.HistoricalBalanceCheckAddress = v + } if v := f.EnforceRepeatableRead; v != nil { p.EnforceRepeatableRead = v @@ -1144,6 +1153,10 @@ func (p *NodePool) setFrom(f *NodePool) { p.ExternalRequestMaxResponseSize = v } + if v := f.FinalizedStateCheckFailureThreshold; v != nil { + p.FinalizedStateCheckFailureThreshold = v + } + p.Errors.setFrom(&f.Errors) } diff --git a/pkg/config/toml/config_test.go b/pkg/config/toml/config_test.go index f71423f289..a61aedb3c2 100644 --- a/pkg/config/toml/config_test.go +++ b/pkg/config/toml/config_test.go @@ -101,6 +101,7 @@ func TestDefaults_fieldsNotNil(t *testing.T) { ServiceUnavailable: ptr("unavailable"), TooManyResults: ptr("too-many"), MissingBlocks: ptr("missing"), + FinalizedStateUnavailable: ptr("finalized-unavailable"), } configtest.AssertFieldsNotNil(t, unknown) @@ -312,11 +313,13 @@ var fullConfig = EVMConfig{ LeaseDuration: config.MustNewDuration(0), NodeIsSyncingEnabled: ptr(true), FinalizedBlockPollInterval: config.MustNewDuration(time.Second), + HistoricalBalanceCheckAddress: ptr(types.MustEIP55Address("0x0000000000000000000000000000000000000001")), EnforceRepeatableRead: ptr(true), DeathDeclarationDelay: config.MustNewDuration(time.Minute), VerifyChainID: ptr(true), NewHeadsPollInterval: config.MustNewDuration(0), - ExternalRequestMaxResponseSize: ptr[uint32](10), + ExternalRequestMaxResponseSize: ptr[uint32](10), + FinalizedStateCheckFailureThreshold: ptr[uint32](3), Errors: ClientErrors{ NonceTooLow: ptr[string]("(: |^)nonce too low"), NonceTooHigh: ptr[string]("(: |^)nonce too high"), @@ -334,6 +337,7 @@ var fullConfig = EVMConfig{ ServiceUnavailable: ptr[string]("(: |^)service unavailable"), TooManyResults: ptr[string]("(: |^)too many results"), MissingBlocks: ptr[string]("(: |^)invalid block range"), + FinalizedStateUnavailable: ptr[string]("(: |^)missing trie node"), }, }, OCR: OCR{ diff --git a/pkg/config/toml/defaults/fallback.toml b/pkg/config/toml/defaults/fallback.toml index 7d8ea77644..7125f4a227 100644 --- a/pkg/config/toml/defaults/fallback.toml +++ b/pkg/config/toml/defaults/fallback.toml @@ -84,6 +84,8 @@ SyncThreshold = 5 LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' +HistoricalBalanceCheckAddress = '0x0000000000000000000000000000000000000000' +FinalizedStateCheckFailureThreshold = 0 EnforceRepeatableRead = true DeathDeclarationDelay = '1m' NewHeadsPollInterval = '0s' diff --git a/pkg/config/toml/docs.toml b/pkg/config/toml/docs.toml index a01aad9330..82c908fca7 100644 --- a/pkg/config/toml/docs.toml +++ b/pkg/config/toml/docs.toml @@ -456,6 +456,17 @@ NodeIsSyncingEnabled = false # Default # # Set to 0 to disable. FinalizedBlockPollInterval = '5s' # Default +# HistoricalBalanceCheckAddress is the probe account used by the historical balance health check. +# The check executes `eth_getBalance` for this address at the latest finalized block. +# Finalized block selection follows chain finality settings: +# - `FinalityTagEnabled = true`: use `finalized` tag +# - `FinalityTagEnabled = false`: use `latest - FinalityDepth` +# The check is only active when `FinalizedStateCheckFailureThreshold > 0`. +HistoricalBalanceCheckAddress = '0x0000000000000000000000000000000000000000' # Default +# FinalizedStateCheckFailureThreshold is the number of consecutive failures of the finalized state availability check +# before the node is marked as FinalizedStateNotAvailable. +# Set to 0 to disable the check. +FinalizedStateCheckFailureThreshold = 0 # Default # EnforceRepeatableRead defines if Core should only use RPCs whose most recently finalized block is greater or equal to # `highest finalized block - FinalizedBlockOffset`. In other words, exclude RPCs lagging on latest finalized # block. @@ -516,6 +527,8 @@ ServiceUnavailable = '(: |^)service unavailable' # Example TooManyResults = '(: |^)too many results' # Example # MissingBlocks is a regex pattern to match an eth_getLogs error indicating the rpc server is permanently missing some blocks in the requested block range MissingBlocks = '(: |^)invalid block range' # Example +# FinalizedStateUnavailable is a regex pattern to match errors indicating the RPC cannot serve historical state at the finalized block (e.g., pruned/non-archive node) +FinalizedStateUnavailable = '(missing trie node|state not available|historical state unavailable)' # Example [OCR] # ContractConfirmations sets `OCR.ContractConfirmations` for this EVM chain. diff --git a/pkg/config/toml/testdata/config-full.toml b/pkg/config/toml/testdata/config-full.toml index 564ea7d668..2ab9f93be3 100644 --- a/pkg/config/toml/testdata/config-full.toml +++ b/pkg/config/toml/testdata/config-full.toml @@ -120,6 +120,8 @@ SyncThreshold = 13 LeaseDuration = '0s' NodeIsSyncingEnabled = true FinalizedBlockPollInterval = '1s' +HistoricalBalanceCheckAddress = '0x0000000000000000000000000000000000000001' +FinalizedStateCheckFailureThreshold = 3 EnforceRepeatableRead = true DeathDeclarationDelay = '1m0s' NewHeadsPollInterval = '0s' @@ -143,6 +145,7 @@ Fatal = '(: |^)fatal' ServiceUnavailable = '(: |^)service unavailable' TooManyResults = '(: |^)too many results' MissingBlocks = '(: |^)invalid block range' +FinalizedStateUnavailable = '(: |^)missing trie node' [OCR] ContractConfirmations = 11