Vault payload size optimizations

.github/workflows/cre-system-tests.yaml

@@ -96,7 +96,8 @@
           PER_TEST_TOPOLOGIES_JSON=${PER_TEST_TOPOLOGIES_JSON:-'{
             "Test_CRE_V2_Suite_Bucket_B": [
                {"topology":"workflow-gateway-capabilities","configs":"configs/workflow-gateway-capabilities-don.toml"},
-               {"topology":"workflow-gateway-capabilities-vault-jwt_auth-enabled","configs":"configs/workflow-gateway-capabilities-don-vault-jwt_auth-enabled.toml"}
+               {"topology":"workflow-gateway-capabilities-vault-jwt_auth-enabled","configs":"configs/workflow-gateway-capabilities-don-vault-jwt_auth-enabled.toml"},
+               {"topology":"workflow-gateway-capabilities-vault-base64-enabled","configs":"configs/workflow-gateway-capabilities-don-vault-base64-enabled.toml"}
              ],
             "Test_CRE_V2_Aptos_Suite": [
                {"topology":"workflow-gateway-aptos","configs":"configs/workflow-gateway-don-aptos.toml"}
@@ -268,7 +269,7 @@
           CTF_CHIP_CONFIG_IMAGE: ${{ secrets.AWS_ACCOUNT_ID_PROD }}.dkr.ecr.${{
             secrets.QA_AWS_REGION
             }}.amazonaws.com/atlas-chip-config:7b4e9ee68fd1c737dd3480b5a3ced0188f29b969
         run: |
           set -u
           set +e
           # GitHub runs bash with errexit (-e); allow obs up / validation paths to fail without aborting retries.
@@ -383,7 +384,7 @@
           CTF_CHIP_CONFIG_IMAGE: ${{ secrets.AWS_ACCOUNT_ID_PROD }}.dkr.ecr.${{
             secrets.QA_AWS_REGION
             }}.amazonaws.com/atlas-chip-config:7b4e9ee68fd1c737dd3480b5a3ced0188f29b969
         run: |
           echo "Starting test: '${TEST_NAME}'"
           gotestsum \
             --jsonfile=/tmp/gotest.log \

core/capabilities/vault/allow_list_based_auth.go

@@ -2,15 +2,19 @@ package vault
 
 import (
 	"context"
+	"encoding/base64"
 	"encoding/hex"
 	"encoding/json"
 	"errors"
 	"fmt"
 	"time"
 
+	vaultcommon "github.com/smartcontractkit/chainlink-common/pkg/capabilities/actions/vault"
 	jsonrpc "github.com/smartcontractkit/chainlink-common/pkg/jsonrpc2"
 	"github.com/smartcontractkit/chainlink-common/pkg/logger"
+	"github.com/smartcontractkit/chainlink-common/pkg/settings/limits"
 	"github.com/smartcontractkit/chainlink-evm/gethwrappers/workflow/generated/workflow_registry_wrapper_v2"
+	"github.com/smartcontractkit/chainlink/v2/core/capabilities/vault/vaulttypes"
 	workflowsyncerv2 "github.com/smartcontractkit/chainlink/v2/core/services/workflows/syncer/v2"
 )
 
@@ -27,13 +31,26 @@ type allowListBasedAuth struct {
 	lggr                   logger.Logger
 	retryCount             int
 	retryInterval          time.Duration
+	// vaultBase64EncodingEnabled, when non-nil and AllowErr(ctx)==nil, indicates the
+	// gateway forwards ciphertext as base64. Allowlist digests are always registered
+	// against the client (hex) wire form, so we normalize before Digest().
+	vaultBase64EncodingEnabled limits.GateLimiter
 }
 
 // AuthorizeRequest authorizes a request using AllowListBasedAuth.
 // It does NOT check if the request method is allowed.
 func (r *allowListBasedAuth) AuthorizeRequest(ctx context.Context, req jsonrpc.Request[json.RawMessage]) (*AuthResult, error) {
 	r.lggr.Debugw("AllowListBasedAuth authorizing request", "method", req.Method, "requestID", req.ID)
-	requestDigest, err := req.Digest()
+	reqForDigest := req
+	if r.vaultBase64EncodingEnabled != nil && r.vaultBase64EncodingEnabled.AllowErr(ctx) == nil {
+		normalized, normErr := vaultRequestEncryptedSecretsBase64ToHexForAllowlistDigest(req)
+		if normErr != nil {
+			r.lggr.Debugw("AllowListBasedAuth failed to normalize ciphertext for digest", "method", req.Method, "requestID", req.ID, "error", normErr)
+			return nil, normErr
+		}
+		reqForDigest = normalized
+	}
+	requestDigest, err := reqForDigest.Digest()
 	if err != nil {
 		r.lggr.Debugw("AllowListBasedAuth failed to create digest", "method", req.Method, "requestID", req.ID, "error", err)
 		return nil, err
@@ -120,15 +137,74 @@ func (r *allowListBasedAuth) fetchAllowlistedItem(allowListedRequests []workflow
 }
 
 // NewAllowListBasedAuth creates the allowlist-backed Vault auth mechanism.
-func NewAllowListBasedAuth(lggr logger.Logger, workflowRegistrySyncer workflowsyncerv2.WorkflowRegistrySyncer) *allowListBasedAuth {
+// vaultBase64EncodingEnabled may be nil; when set and limits allow, ciphertext in
+// create/update params is decoded from base64 to hex before computing the allowlist digest.
+func NewAllowListBasedAuth(lggr logger.Logger, workflowRegistrySyncer workflowsyncerv2.WorkflowRegistrySyncer, vaultBase64EncodingEnabled limits.GateLimiter) *allowListBasedAuth {
 	return &allowListBasedAuth{
-		workflowRegistrySyncer: workflowRegistrySyncer,
-		lggr:                   logger.Named(lggr, "VaultAllowListBasedAuth"),
-		retryCount:             allowListBasedAuthRetryCount,
-		retryInterval:          allowListBasedAuthRetryInterval,
+		workflowRegistrySyncer:     workflowRegistrySyncer,
+		lggr:                       logger.Named(lggr, "VaultAllowListBasedAuth"),
+		retryCount:                 allowListBasedAuthRetryCount,
+		retryInterval:              allowListBasedAuthRetryInterval,
+		vaultBase64EncodingEnabled: vaultBase64EncodingEnabled,
+	}
+}
+
+func vaultRequestEncryptedSecretsBase64ToHexForAllowlistDigest(req jsonrpc.Request[json.RawMessage]) (jsonrpc.Request[json.RawMessage], error) {
+	if req.Params == nil {
+		return req, nil
+	}
+	switch req.Method {
+	case vaulttypes.MethodSecretsCreate:
+		parsed := &vaultcommon.CreateSecretsRequest{}
+		if err := json.Unmarshal(*req.Params, parsed); err != nil {
+			return jsonrpc.Request[json.RawMessage]{}, err
+		}
+		if err := convertEncryptedSecretsBase64WireToHexStrings(parsed.EncryptedSecrets); err != nil {
+			return jsonrpc.Request[json.RawMessage]{}, err
+		}
+		b, err := json.Marshal(parsed)
+		if err != nil {
+			return jsonrpc.Request[json.RawMessage]{}, err
+		}
+		raw := json.RawMessage(b)
+		out := req
+		out.Params = &raw
+		return out, nil
+	case vaulttypes.MethodSecretsUpdate:
+		parsed := &vaultcommon.UpdateSecretsRequest{}
+		if err := json.Unmarshal(*req.Params, parsed); err != nil {
+			return jsonrpc.Request[json.RawMessage]{}, err
+		}
+		if err := convertEncryptedSecretsBase64WireToHexStrings(parsed.EncryptedSecrets); err != nil {
+			return jsonrpc.Request[json.RawMessage]{}, err
+		}
+		b, err := json.Marshal(parsed)
+		if err != nil {
+			return jsonrpc.Request[json.RawMessage]{}, err
+		}
+		raw := json.RawMessage(b)
+		out := req
+		out.Params = &raw
+		return out, nil
+	default:
+		return req, nil
 	}
 }
 
+func convertEncryptedSecretsBase64WireToHexStrings(secrets []*vaultcommon.EncryptedSecret) error {
+	for i, item := range secrets {
+		if item == nil || item.EncryptedValue == "" {
+			continue
+		}
+		raw, err := base64.StdEncoding.DecodeString(item.EncryptedValue)
+		if err != nil {
+			return fmt.Errorf("encrypted secret index %d: %w", i, err)
+		}
+		item.EncryptedValue = hex.EncodeToString(raw)
+	}
+	return nil
+}
+
 func sleepWithContext(ctx context.Context, d time.Duration) error {
 	timer := time.NewTimer(d)
 	defer timer.Stop()

core/capabilities/vault/allow_list_based_auth_test.go

@@ -2,6 +2,7 @@ package vault
 
 import (
 	"context"
+	"encoding/base64"
 	"encoding/hex"
 	"encoding/json"
 	"testing"
@@ -13,6 +14,7 @@ import (
 
 	vaultcommon "github.com/smartcontractkit/chainlink-common/pkg/capabilities/actions/vault"
 	jsonrpc "github.com/smartcontractkit/chainlink-common/pkg/jsonrpc2"
+	"github.com/smartcontractkit/chainlink-common/pkg/settings/limits"
 	"github.com/smartcontractkit/chainlink-evm/gethwrappers/workflow/generated/workflow_registry_wrapper_v2"
 	"github.com/smartcontractkit/chainlink/v2/core/capabilities/vault/vaulttypes"
 	"github.com/smartcontractkit/chainlink/v2/core/logger"
@@ -159,7 +161,7 @@ func testAuthForRequests(t *testing.T, allowlistedRequest, notAllowlistedRequest
 	owner := common.Address{1, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}
 
 	mockSyncer := syncerv2mocks.NewWorkflowRegistrySyncer(t)
-	auth := NewAllowListBasedAuth(lggr, mockSyncer)
+	auth := NewAllowListBasedAuth(lggr, mockSyncer, nil)
 	auth.retryCount = 0
 	auth.retryInterval = time.Millisecond
 
@@ -226,7 +228,7 @@ func TestAllowListBasedAuth_RetriesUntilRequestIsAllowlisted(t *testing.T) {
 	}
 
 	mockSyncer := syncerv2mocks.NewWorkflowRegistrySyncer(t)
-	auth := NewAllowListBasedAuth(lggr, mockSyncer)
+	auth := NewAllowListBasedAuth(lggr, mockSyncer, nil)
 	auth.retryCount = 2
 	auth.retryInterval = time.Millisecond
 
@@ -247,7 +249,7 @@ func TestAllowListBasedAuth_FailsAfterAllowlistReadRetries(t *testing.T) {
 	mockSyncer := syncerv2mocks.NewWorkflowRegistrySyncer(t)
 	mockSyncer.On("GetAllowlistedRequests", mock.Anything).Return([]workflow_registry_wrapper_v2.WorkflowRegistryOwnerAllowlistedRequest{}).Times(3)
 
-	auth := NewAllowListBasedAuth(lggr, mockSyncer)
+	auth := NewAllowListBasedAuth(lggr, mockSyncer, nil)
 	auth.retryCount = 2
 	auth.retryInterval = time.Millisecond
 
@@ -266,7 +268,7 @@ func TestAllowListBasedAuth_StopsRetriesWhenContextCanceled(t *testing.T) {
 	mockSyncer := syncerv2mocks.NewWorkflowRegistrySyncer(t)
 	mockSyncer.On("GetAllowlistedRequests", mock.Anything).Return([]workflow_registry_wrapper_v2.WorkflowRegistryOwnerAllowlistedRequest{}).Once()
 
-	auth := NewAllowListBasedAuth(lggr, mockSyncer)
+	auth := NewAllowListBasedAuth(lggr, mockSyncer, nil)
 	auth.retryCount = 2
 	auth.retryInterval = time.Second
 
@@ -275,6 +277,62 @@ func TestAllowListBasedAuth_StopsRetriesWhenContextCanceled(t *testing.T) {
 	require.ErrorIs(t, err, context.Canceled)
 }
 
+func TestAllowListBasedAuth_Base64WireMatchesHexAllowlistedDigest(t *testing.T) {
+	lggr := logger.TestLogger(t)
+	owner := common.Address{9, 8, 7}
+
+	hexParams, err := json.Marshal(vaultcommon.CreateSecretsRequest{
+		RequestId: "rid-1",
+		EncryptedSecrets: []*vaultcommon.EncryptedSecret{{
+			Id:             &vaultcommon.SecretIdentifier{Key: "k", Namespace: "ns", Owner: "o"},
+			EncryptedValue: "41",
+		}},
+	})
+	require.NoError(t, err)
+	hexRaw := json.RawMessage(hexParams)
+	hexReq := jsonrpc.Request[json.RawMessage]{
+		ID:     "123",
+		Method: vaulttypes.MethodSecretsCreate,
+		Params: &hexRaw,
+	}
+	digest, err := hexReq.Digest()
+	require.NoError(t, err)
+	digestBytes, err := hex.DecodeString(digest)
+	require.NoError(t, err)
+
+	b64Val := base64.StdEncoding.EncodeToString([]byte{0x41})
+	b64Params, err := json.Marshal(vaultcommon.CreateSecretsRequest{
+		RequestId: "rid-1",
+		EncryptedSecrets: []*vaultcommon.EncryptedSecret{{
+			Id:             &vaultcommon.SecretIdentifier{Key: "k", Namespace: "ns", Owner: "o"},
+			EncryptedValue: b64Val,
+		}},
+	})
+	require.NoError(t, err)
+	b64Raw := json.RawMessage(b64Params)
+	b64Req := jsonrpc.Request[json.RawMessage]{
+		ID:     "123",
+		Method: vaulttypes.MethodSecretsCreate,
+		Params: &b64Raw,
+	}
+
+	mockSyncer := syncerv2mocks.NewWorkflowRegistrySyncer(t)
+	expiry := time.Now().UTC().Unix() + 100
+	mockSyncer.On("GetAllowlistedRequests", mock.Anything).Return([]workflow_registry_wrapper_v2.WorkflowRegistryOwnerAllowlistedRequest{{
+		RequestDigest:   [32]byte(digestBytes),
+		Owner:           owner,
+		ExpiryTimestamp: uint32(expiry), //nolint:gosec // it is a safe conversion
+	}})
+
+	auth := NewAllowListBasedAuth(lggr, mockSyncer, limits.NewGateLimiter(true))
+	auth.retryCount = 0
+	auth.retryInterval = time.Millisecond
+
+	authResult, err := auth.AuthorizeRequest(t.Context(), b64Req)
+	require.NoError(t, err)
+	require.Equal(t, owner.Hex(), authResult.AuthorizedOwner())
+}
+
 func makeListSecretsRequest(t *testing.T, id, namespace string) jsonrpc.Request[json.RawMessage] {
 	t.Helper()
 

core/capabilities/vault/capability.go

@@ -21,6 +21,7 @@ import (
 	"github.com/smartcontractkit/chainlink-common/pkg/settings/limits"
 	"github.com/smartcontractkit/chainlink-common/pkg/types/core"
 	"github.com/smartcontractkit/chainlink/v2/core/capabilities/vault/vaulttypes"
+	"github.com/smartcontractkit/chainlink/v2/core/capabilities/vault/vaultutils"
 )
 
 var _ capabilities.ExecutableCapability = (*Capability)(nil)
@@ -33,6 +34,7 @@ type Capability struct {
 	capabilitiesRegistry core.CapabilitiesRegistry
 	publicKey            *LazyPublicKey
 	linker               *OrgIDToWorkflowOwnerLinker
+	base64Gate           limits.GateLimiter
 	*RequestValidator
 }
 
@@ -92,9 +94,20 @@ func (s *Capability) Close() error {
 		err = errors.Join(err, fmt.Errorf("error closing org_id linker: %w", lerr))
 	}
 
+	if lerr := s.base64Gate.Close(); lerr != nil {
+		err = errors.Join(err, fmt.Errorf("error closing base64 encoding gate: %w", lerr))
+	}
+
 	return err
 }
 
+func (s *Capability) ciphertextStringEncoding(ctx context.Context) vaultutils.CiphertextStringEncoding {
+	if s.base64Gate.AllowErr(ctx) == nil {
+		return vaultutils.CiphertextStringEncodingBase64
+	}
+	return vaultutils.CiphertextStringEncodingHex
+}
+
 func (s *Capability) Info(_ context.Context) (capabilities.CapabilityInfo, error) {
 	return capabilities.NewCapabilityInfo(vaultcommon.CapabilityID, capabilities.CapabilityTypeAction, "Vault Capability")
 }
@@ -195,7 +208,7 @@ func (s *Capability) CreateSecrets(ctx context.Context, request *vaultcommon.Cre
 		s.lggr.Debugw("failed identity owner checks", "requestID", request.RequestId, "err", ownerErr)
 		return nil, ownerErr
 	}
-	err = s.ValidateCreateSecretsRequest(ctx, s.publicKey.Get(), request, false)
+	err = s.ValidateCreateSecretsRequest(ctx, s.publicKey.Get(), request, false, s.ciphertextStringEncoding(ctx))
 	if err != nil {
 		s.lggr.Debugw("failed validation checks", "requestID", request.RequestId, "err", err)
 		return nil, err
@@ -215,7 +228,7 @@ func (s *Capability) UpdateSecrets(ctx context.Context, request *vaultcommon.Upd
 		s.lggr.Debugw("failed identity owner checks", "requestID", request.RequestId, "err", ownerErr)
 		return nil, ownerErr
 	}
-	err = s.ValidateUpdateSecretsRequest(ctx, s.publicKey.Get(), request, false)
+	err = s.ValidateUpdateSecretsRequest(ctx, s.publicKey.Get(), request, false, s.ciphertextStringEncoding(ctx))
 	if err != nil {
 		s.lggr.Debugw("failed validation checks", "requestID", request.RequestId, "err", err)
 		return nil, err
@@ -411,6 +424,10 @@ func NewCapability(
 	if err != nil {
 		return nil, fmt.Errorf("could not create identifier namespace length limiter: %w", err)
 	}
+	base64Gate, err := limits.MakeGateLimiter(limitsFactory, cresettings.Default.VaultBase64EncodingEnabled)
+	if err != nil {
+		return nil, fmt.Errorf("could not create base64 encoding gate: %w", err)
+	}
 	return &Capability{
 		lggr:                 logger.Named(lggr, "VaultCapability"),
 		clock:                clock,
@@ -419,6 +436,7 @@ func NewCapability(
 		capabilitiesRegistry: capabilitiesRegistry,
 		publicKey:            publicKey,
 		linker:               linker,
+		base64Gate:           base64Gate,
 		RequestValidator:     NewRequestValidator(limiter, ciphertextLimiter, idKeyLengthLimiter, idOwnerLengthLimiter, idNamespaceLengthLimiter),
 	}, nil
 }