diff --git a/config/config.yaml b/config/config.yaml index 9666b09a..ba5820aa 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -399,6 +399,9 @@ checks: postage-depth: 21 postage-label: test-label type: feed + autotls: + timeout: 5m + type: autotls # simulations defines simulations Beekeeper can execute against the cluster # type filed allows defining same simulation with different names and options diff --git a/config/local.yaml b/config/local.yaml index 9391a4a6..b9c4b194 100644 --- a/config/local.yaml +++ b/config/local.yaml @@ -47,6 +47,38 @@ clusters: config: local-light count: 2 mode: node + local-dns-autotls: + _inherit: "local" + node-groups: + bootnode: + mode: bootnode + bee-config: bootnode-local-dns-autotls + config: local + nodes: + - name: bootnode-0 + bootnodes: /dns4/bootnode-0-headless.%s.svc.cluster.local/tcp/1634/p2p/QmaHzvd3iZduu275CMkMVZKwbsjXSyH3GJRj4UvFJApKcb + libp2p-key: '{"address":"28678fe31f09f722d53e77ca2395569f19959fa5","crypto":{"cipher":"aes-128-ctr","ciphertext":"0ff319684c4f8decf9c998047febe3417cfc45832b8bb62fd818183d54cf5d0183bfa021ed95addce3b33e83ce7ee73e926f00eea8241d96b349266a4d299829d3d22db0d536315b52b34db4a6778bfd3ce7631ad7256ea0bb9c50abea9de35d740b6fdc50caf929b1d19494690d9ed649105d02c14f5ec49d","cipherparams":{"iv":"4e9a50fb5852b5e61964f696be78066b"},"kdf":"scrypt","kdfparams":{"n":32768,"r":8,"p":1,"dklen":32,"salt":"4d513e81647e4150bb648ed8d2dda28d460802336bf24d620119eac66ae0c0c4"},"mac":"9ae71db96e5ddc1c214538d42082212bbbe53aeac09fcc3e3a8eff815648331e"},"version":3,"id":"ae3bc991-d89f-405a-9e6a-60e27347e22d"}' + swarm-key: '{"address":"f176839c150e52fe30e5c2b5c648465c6fdfa532","crypto":{"cipher":"aes-128-ctr","ciphertext":"352af096f0fca9dfbd20a6861bde43d988efe7f179e0a9ffd812a285fdcd63b9","cipherparams":{"iv":"613003f1f1bf93430c92629da33f8828"},"kdf":"scrypt","kdfparams":{"n":32768,"r":8,"p":1,"dklen":32,"salt":"ad1d99a4c64c95c26131e079e8c8a82221d58bf66a7ceb767c33a4c376c564b8"},"mac":"cafda1bc8ca0ffc2b22eb69afd1cf5072fd09412243443be1b0c6832f57924b6"},"version":3}' + bee: + bee-config: bee-local-autotls + config: local + count: 3 + mode: node + bee-autotls: + bee-config: bee-local-autotls + config: local + count: 2 + mode: node + light: + bee-config: bee-local-light-autotls + config: local + count: 2 + mode: node + ultra-light: + bee-config: bee-local-ultralight-autotls + config: local + count: 1 + mode: node local-gc: _inherit: "local" node-groups: @@ -111,6 +143,9 @@ bee-configs: _inherit: "" allow-private-cidrs: true api-addr: ":1633" + autotls-ca-endpoint: "https://pebble:14000/dir" + autotls-domain: "local.test" + autotls-registration-endpoint: http://p2p-forge.local.svc.cluster.local:8080 block-time: 1 blockchain-rpc-endpoint: "ws://geth-swap:8546" bootnode-mode: false @@ -126,9 +161,12 @@ bee-configs: full-node: true mainnet: false nat-addr: "" + nat-wss-addr: "" network-id: 0 p2p-addr: ":1634" + p2p-wss-addr: ":1635" p2p-ws-enable: false + p2p-wss-enable: false password: "beekeeper" payment-early-percent: 50 payment-threshold: 13500000 @@ -147,7 +185,22 @@ bee-configs: warmup-time: 0s welcome-message: "Welcome to the Swarm, this is a local cluster!" withdrawal-addresses-whitelist: "0xec44cb15b1b033e74d55ac5d0e24d861bde54532" - + bootnode-local-dns-autotls: + _inherit: "bee-local-dns" + bootnode-mode: true + p2p-wss-enable: true + bee-local-autotls: + _inherit: "bee-local-dns" + bootnode: /dnsaddr/bootnode-0-headless.local.svc.cluster.local + p2p-wss-enable: true + bee-local-light-autotls: + _inherit: "bee-local-light" + p2p-wss-enable: true + bee-local-ultralight-autotls: + _inherit: "bee-local-dns" + full-node: false + p2p-wss-enable: true + blockchain-rpc-endpoint: # ultralight nodes don't connect to the blockchain bootnode-local: _inherit: "bee-local" bootnode-mode: true @@ -392,3 +445,9 @@ checks: postage-depth: 21 postage-label: test-label type: feed + ci-autotls: + timeout: 15m + type: autotls + options: + ultra-light-group: ultra-light + autotls-group: bee-autotls diff --git a/go.mod b/go.mod index e99353a3..ff930d63 100644 --- a/go.mod +++ b/go.mod @@ -16,6 +16,7 @@ require ( github.com/go-git/go-git/v5 v5.16.5 github.com/google/uuid v1.6.0 github.com/gorilla/websocket v1.5.1 + github.com/multiformats/go-multiaddr v0.12.3 github.com/opentracing/opentracing-go v1.2.0 github.com/prometheus/client_golang v1.21.1 github.com/prometheus/common v0.62.0 @@ -95,7 +96,6 @@ require ( github.com/mr-tron/base58 v1.2.0 // indirect github.com/multiformats/go-base32 v0.1.0 // indirect github.com/multiformats/go-base36 v0.2.0 // indirect - github.com/multiformats/go-multiaddr v0.12.3 // indirect github.com/multiformats/go-multiaddr-dns v0.3.1 // indirect github.com/multiformats/go-multibase v0.2.0 // indirect github.com/multiformats/go-multicodec v0.9.0 // indirect diff --git a/pkg/bee/api/node.go b/pkg/bee/api/node.go index de9423bc..789eeccc 100644 --- a/pkg/bee/api/node.go +++ b/pkg/bee/api/node.go @@ -122,6 +122,24 @@ func (n *NodeService) Peers(ctx context.Context) (resp Peers, err error) { return resp, err } +// ConnectResponse represents the response from the connect endpoint +type ConnectResponse struct { + Address string `json:"address"` +} + +// Connect connects to a peer using the provided multiaddress. +// The multiaddr should be in the format: /ip4/x.x.x.x/tcp/port/... +// Returns the overlay address of the connected peer. +func (n *NodeService) Connect(ctx context.Context, multiaddr string) (resp ConnectResponse, err error) { + err = n.client.requestJSON(ctx, http.MethodPost, "/connect"+multiaddr, nil, &resp) + return resp, err +} + +// Disconnect disconnects from a peer with the given overlay address. +func (n *NodeService) Disconnect(ctx context.Context, overlay swarm.Address) error { + return n.client.requestJSON(ctx, http.MethodDelete, "/peers/"+overlay.String(), nil, nil) +} + // Readiness represents node's readiness type Readiness struct { Status string `json:"status"` diff --git a/pkg/bee/client.go b/pkg/bee/client.go index 5bb6ce61..711ffeb6 100644 --- a/pkg/bee/client.go +++ b/pkg/bee/client.go @@ -318,6 +318,30 @@ func (c *Client) Peers(ctx context.Context) (peers []swarm.Address, err error) { return peers, err } +// Connect connects to a peer using the provided multiaddress. +// Returns the overlay address of the connected peer. +func (c *Client) Connect(ctx context.Context, multiaddr string) (swarm.Address, error) { + resp, err := c.api.Node.Connect(ctx, multiaddr) + if err != nil { + return swarm.ZeroAddress, fmt.Errorf("connect to %s: %w", multiaddr, err) + } + + addr, err := swarm.ParseHexAddress(resp.Address) + if err != nil { + return swarm.ZeroAddress, fmt.Errorf("parse overlay address %s: %w", resp.Address, err) + } + + return addr, nil +} + +// Disconnect disconnects from a peer with the given overlay address. +func (c *Client) Disconnect(ctx context.Context, overlay swarm.Address) error { + if err := c.api.Node.Disconnect(ctx, overlay); err != nil { + return fmt.Errorf("disconnect from %s: %w", overlay, err) + } + return nil +} + // PinRootHash pins root hash of given reference. func (c *Client) PinRootHash(ctx context.Context, ref swarm.Address) error { return c.api.Pinning.PinRootHash(ctx, ref) diff --git a/pkg/check/autotls/autotls.go b/pkg/check/autotls/autotls.go new file mode 100644 index 00000000..95c5a77b --- /dev/null +++ b/pkg/check/autotls/autotls.go @@ -0,0 +1,277 @@ +package autotls + +import ( + "context" + "fmt" + "time" + + "github.com/ethersphere/beekeeper/pkg/bee" + "github.com/ethersphere/beekeeper/pkg/beekeeper" + "github.com/ethersphere/beekeeper/pkg/logging" + "github.com/ethersphere/beekeeper/pkg/orchestration" + ma "github.com/multiformats/go-multiaddr" +) + +type Options struct { + AutoTLSGroup string + UltraLightGroup string +} + +func NewDefaultOptions() Options { + return Options{ + AutoTLSGroup: "bee-autotls", + UltraLightGroup: "ultra-light", + } +} + +const ( + underlayPollInterval = 2 * time.Second + connectTimeout = 30 * time.Second +) + +var _ beekeeper.Action = (*Check)(nil) + +type Check struct { + logger logging.Logger +} + +func NewCheck(logger logging.Logger) beekeeper.Action { + return &Check{ + logger: logger, + } +} + +func (c *Check) Run(ctx context.Context, cluster orchestration.Cluster, opts any) error { + o, ok := opts.(Options) + if !ok { + return fmt.Errorf("invalid options type") + } + + c.logger.Info("starting AutoTLS check") + + clients, err := cluster.NodesClients(ctx) + if err != nil { + return fmt.Errorf("get node clients: %w", err) + } + + autoTLSClients := orchestration.ClientMap(clients).FilterByNodeGroups([]string{o.AutoTLSGroup}) + if len(autoTLSClients) == 0 { + return fmt.Errorf("no nodes found in AutoTLS group %q", o.AutoTLSGroup) + } + + c.logger.Infof("found %d nodes in AutoTLS group %q", len(autoTLSClients), o.AutoTLSGroup) + + wssNodes, err := c.verifyWSSUnderlays(ctx, autoTLSClients, o.UltraLightGroup) + if err != nil { + return fmt.Errorf("verify WSS underlays: %w", err) + } + + if err := c.testWSSConnectivity(ctx, clients, wssNodes, connectTimeout); err != nil { + return fmt.Errorf("WSS connectivity test: %w", err) + } + + if o.UltraLightGroup != "" { + if err := c.testUltraLightConnectivity(ctx, clients, wssNodes, o.UltraLightGroup, connectTimeout); err != nil { + return fmt.Errorf("ultra-light connectivity test: %w", err) + } + } + + if err := c.testCertificateRenewal(ctx, clients, wssNodes, connectTimeout); err != nil { + return fmt.Errorf("certificate renewal test: %w", err) + } + + c.logger.Info("AutoTLS check completed successfully") + return nil +} + +func (c *Check) verifyWSSUnderlays(ctx context.Context, autoTLSClients orchestration.ClientList, excludeNodeGroup string) (map[string][]string, error) { + autoTLS := make(map[string][]string) + + for _, client := range autoTLSClients { + if excludeNodeGroup != "" && client.NodeGroup() == excludeNodeGroup { + c.logger.Debugf("skipping %s (node group %s has no WSS underlays)", client.Name(), excludeNodeGroup) + continue + } + + nodeName := client.Name() + var wssUnderlays []string + for { + addresses, err := client.Addresses(ctx) + if err != nil { + return nil, fmt.Errorf("%s: get addresses: %w", nodeName, err) + } + wssUnderlays = filterWSSUnderlays(addresses.Underlay) + if len(wssUnderlays) > 0 { + break + } + c.logger.Debugf("node %s has no WSS underlays yet, retrying in %v", nodeName, underlayPollInterval) + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(underlayPollInterval): + } + } + + autoTLS[nodeName] = wssUnderlays + c.logger.Debugf("node %s has %d WSS underlay(s)", nodeName, len(wssUnderlays)) + } + + return autoTLS, nil +} + +func filterWSSUnderlays(underlays []string) []string { + var wss []string + for _, u := range underlays { + maddr, err := ma.NewMultiaddr(u) + if err != nil { + continue + } + if _, err := maddr.ValueForProtocol(ma.P_TLS); err != nil { + continue + } + if _, err := maddr.ValueForProtocol(ma.P_WS); err != nil { + continue + } + wss = append(wss, u) + } + return wss +} + +func (c *Check) testWSSConnectivity(ctx context.Context, clients map[string]*bee.Client, wssNodes map[string][]string, timeout time.Duration) error { + var nonWSSSource *bee.Client + var nonWSSName string + var wssSource *bee.Client + var wssSourceName string + + for name, client := range clients { + if _, hasWSS := wssNodes[name]; hasWSS { + if wssSource == nil { + wssSource = client + wssSourceName = name + } + } else { + if nonWSSSource == nil { + nonWSSSource = client + nonWSSName = name + } + } + } + + if nonWSSSource != nil { + c.logger.Infof("testing cross-protocol: %s (non-WSS) to WSS nodes", nonWSSName) + if err := c.testConnectivity(ctx, nonWSSSource, nonWSSName, clients, wssNodes, timeout); err != nil { + return fmt.Errorf("cross-protocol test: %w", err) + } + } else { + c.logger.Warning("no non-WSS nodes available, skipping cross-protocol test") + } + + if wssSource != nil { + c.logger.Infof("testing WSS-to-WSS: %s to WSS nodes", wssSourceName) + if err := c.testConnectivity(ctx, wssSource, wssSourceName, clients, wssNodes, timeout); err != nil { + return fmt.Errorf("WSS-to-WSS test: %w", err) + } + } else { + c.logger.Warning("no WSS source nodes available, skipping WSS-to-WSS test") + } + + return nil +} + +func (c *Check) testUltraLightConnectivity(ctx context.Context, clients map[string]*bee.Client, wssNodes map[string][]string, ultraLightGroup string, timeout time.Duration) error { + ultralightClients := orchestration.ClientMap(clients).FilterByNodeGroups([]string{ultraLightGroup}) + if len(ultralightClients) == 0 { + c.logger.Warningf("no nodes found in ultra-light group %q, skipping ultra-light connectivity test", ultraLightGroup) + return nil + } + + c.logger.Infof("found %d nodes in ultra-light group %q", len(ultralightClients), ultraLightGroup) + + for _, client := range ultralightClients { + nodeName := client.Name() + c.logger.Infof("testing ultra-light to WSS: %s (no listen addr) to WSS nodes", nodeName) + if err := c.testConnectivity(ctx, client, nodeName, clients, wssNodes, timeout); err != nil { + return fmt.Errorf("ultra-light %s to WSS test: %w", nodeName, err) + } + } + + return nil +} + +func (c *Check) testConnectivity(ctx context.Context, sourceClient *bee.Client, sourceName string, clients map[string]*bee.Client, wssNodes map[string][]string, timeout time.Duration) error { + for targetName, underlays := range wssNodes { + if targetName == sourceName { + continue + } + + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + + targetClient := clients[targetName] + targetAddresses, err := targetClient.Addresses(ctx) + if err != nil { + return fmt.Errorf("get target %s addresses: %w", targetName, err) + } + targetOverlay := targetAddresses.Overlay + + // Disconnect first to ensure we test actual WSS connection. + // Bee returns 200 OK for both new connections and existing ones, + // so we must disconnect first to guarantee WSS transport is used. + c.logger.Infof("disconnecting from %s before WSS test", targetName) + if err := sourceClient.Disconnect(ctx, targetOverlay); err != nil { + c.logger.Warningf("failed to disconnect from %s: %v", targetName, err) + } + + for _, underlay := range underlays { + c.logger.Infof("testing WSS connection from %s to %s via %s", sourceName, targetName, underlay) + + connectCtx, cancel := context.WithTimeout(ctx, timeout) + start := time.Now() + + overlay, err := sourceClient.Connect(connectCtx, underlay) + duration := time.Since(start) + cancel() + + if err != nil { + return fmt.Errorf("WSS connection failed from %s to %s via %s: %w", sourceName, targetName, underlay, err) + } + + c.logger.Infof("WSS connection successful: %s to %s (overlay: %s, duration: %v)", + sourceName, targetName, overlay, duration) + + if !overlay.Equal(targetOverlay) { + return fmt.Errorf("overlay mismatch: expected %s, got %s", targetOverlay, overlay) + } + + if err := sourceClient.Disconnect(ctx, overlay); err != nil { + c.logger.Warningf("failed to disconnect from %s: %v", targetName, err) + } + } + } + + return nil +} + +func (c *Check) testCertificateRenewal(ctx context.Context, clients map[string]*bee.Client, wssNodes map[string][]string, connectTimeout time.Duration) error { + const renewalWaitTime = 350 * time.Second // This is configured in beelocal setup (we set certificate to expire in 300 seconds) + + c.logger.Infof("testing certificate renewal: waiting %v then re-testing connectivity", renewalWaitTime) + + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(renewalWaitTime): + } + + c.logger.Info("wait complete, re-testing WSS connectivity to verify certificates were renewed") + + if err := c.testWSSConnectivity(ctx, clients, wssNodes, connectTimeout); err != nil { + return fmt.Errorf("post-renewal connectivity test failed (certificates may not have been renewed): %w", err) + } + + c.logger.Info("certificate renewal test passed: WSS connectivity still works after wait period") + return nil +} diff --git a/pkg/config/bee.go b/pkg/config/bee.go index 6d2b5cdd..34366479 100644 --- a/pkg/config/bee.go +++ b/pkg/config/bee.go @@ -16,48 +16,54 @@ type BeeConfig struct { // parent to inherit settings from *Inherit `yaml:",inline"` // Bee configuration - AllowPrivateCIDRs *bool `yaml:"allow-private-cidrs"` - APIAddr *string `yaml:"api-addr"` - BlockchainRPCEndpoint *string `yaml:"blockchain-rpc-endpoint"` - BlockTime *uint64 `yaml:"block-time"` - BootnodeMode *bool `yaml:"bootnode-mode"` - Bootnodes *string `yaml:"bootnodes"` - CacheCapacity *uint64 `yaml:"cache-capacity"` - ChequebookEnable *bool `yaml:"chequebook-enable"` - CORSAllowedOrigins *string `yaml:"cors-allowed-origins"` - DataDir *string `yaml:"data-dir"` - DbBlockCacheCapacity *int `yaml:"db-block-cache-capacity"` - DbDisableSeeksCompaction *bool `yaml:"db-disable-seeks-compaction"` - DbOpenFilesLimit *int `yaml:"db-open-files-limit"` - DbWriteBufferSize *int `yaml:"db-write-buffer-size"` - FullNode *bool `yaml:"full-node"` - Mainnet *bool `yaml:"mainnet"` - NATAddr *string `yaml:"nat-addr"` - NetworkID *uint64 `yaml:"network-id"` - P2PAddr *string `yaml:"p2p-addr"` - P2PWSEnable *bool `yaml:"p2p-ws-enable"` - Password *string `yaml:"password"` - PaymentEarly *uint64 `yaml:"payment-early-percent"` - PaymentThreshold *uint64 `yaml:"payment-threshold"` - PaymentTolerance *uint64 `yaml:"payment-tolerance-percent"` - PostageContractStartBlock *uint64 `yaml:"postage-stamp-start-block"` - PostageStampAddress *string `yaml:"postage-stamp-address"` - PriceOracleAddress *string `yaml:"price-oracle-address"` - RedistributionAddress *string `yaml:"redistribution-address"` - ResolverOptions *string `yaml:"resolver-options"` - StakingAddress *string `yaml:"staking-address"` - StorageIncentivesEnable *string `yaml:"storage-incentives-enable"` - SwapEnable *bool `yaml:"swap-enable"` - SwapEndpoint *string `yaml:"swap-endpoint"` // deprecated: use blockchain-rpc-endpoint - SwapFactoryAddress *string `yaml:"swap-factory-address"` - SwapInitialDeposit *uint64 `yaml:"swap-initial-deposit"` - TracingEnabled *bool `yaml:"tracing-enabled"` - TracingEndpoint *string `yaml:"tracing-endpoint"` - TracingServiceName *string `yaml:"tracing-service-name"` - Verbosity *uint64 `yaml:"verbosity"` - WarmupTime *time.Duration `yaml:"warmup-time"` - WelcomeMessage *string `yaml:"welcome-message"` - WithdrawAddress *string `yaml:"withdrawal-addresses-whitelist"` + AllowPrivateCIDRs *bool `yaml:"allow-private-cidrs"` + APIAddr *string `yaml:"api-addr"` + AutoTLSCAEndpoint *string `yaml:"autotls-ca-endpoint"` + AutoTLSDomain *string `yaml:"autotls-domain"` + AutoTLSRegistrationEndpoint *string `yaml:"autotls-registration-endpoint"` + BlockchainRPCEndpoint *string `yaml:"blockchain-rpc-endpoint"` + BlockTime *uint64 `yaml:"block-time"` + BootnodeMode *bool `yaml:"bootnode-mode"` + Bootnodes *string `yaml:"bootnodes"` + CacheCapacity *uint64 `yaml:"cache-capacity"` + ChequebookEnable *bool `yaml:"chequebook-enable"` + CORSAllowedOrigins *string `yaml:"cors-allowed-origins"` + DataDir *string `yaml:"data-dir"` + DbBlockCacheCapacity *int `yaml:"db-block-cache-capacity"` + DbDisableSeeksCompaction *bool `yaml:"db-disable-seeks-compaction"` + DbOpenFilesLimit *int `yaml:"db-open-files-limit"` + DbWriteBufferSize *int `yaml:"db-write-buffer-size"` + FullNode *bool `yaml:"full-node"` + Mainnet *bool `yaml:"mainnet"` + NATAddr *string `yaml:"nat-addr"` + NATWSSAddr *string `yaml:"nat-wss-addr"` + NetworkID *uint64 `yaml:"network-id"` + P2PAddr *string `yaml:"p2p-addr"` + P2PWSEnable *bool `yaml:"p2p-ws-enable"` + P2PWSSAddr *string `yaml:"p2p-wss-addr"` + P2PWSSEnable *bool `yaml:"p2p-wss-enable"` + Password *string `yaml:"password"` + PaymentEarly *uint64 `yaml:"payment-early-percent"` + PaymentThreshold *uint64 `yaml:"payment-threshold"` + PaymentTolerance *uint64 `yaml:"payment-tolerance-percent"` + PostageContractStartBlock *uint64 `yaml:"postage-stamp-start-block"` + PostageStampAddress *string `yaml:"postage-stamp-address"` + PriceOracleAddress *string `yaml:"price-oracle-address"` + RedistributionAddress *string `yaml:"redistribution-address"` + ResolverOptions *string `yaml:"resolver-options"` + StakingAddress *string `yaml:"staking-address"` + StorageIncentivesEnable *string `yaml:"storage-incentives-enable"` + SwapEnable *bool `yaml:"swap-enable"` + SwapEndpoint *string `yaml:"swap-endpoint"` // deprecated: use blockchain-rpc-endpoint + SwapFactoryAddress *string `yaml:"swap-factory-address"` + SwapInitialDeposit *uint64 `yaml:"swap-initial-deposit"` + TracingEnabled *bool `yaml:"tracing-enabled"` + TracingEndpoint *string `yaml:"tracing-endpoint"` + TracingServiceName *string `yaml:"tracing-service-name"` + Verbosity *uint64 `yaml:"verbosity"` + WarmupTime *time.Duration `yaml:"warmup-time"` + WelcomeMessage *string `yaml:"welcome-message"` + WithdrawAddress *string `yaml:"withdrawal-addresses-whitelist"` } func (b BeeConfig) GetParentName() string { diff --git a/pkg/config/check.go b/pkg/config/check.go index b130c6e4..8d94e077 100644 --- a/pkg/config/check.go +++ b/pkg/config/check.go @@ -8,6 +8,7 @@ import ( "github.com/ethersphere/beekeeper/pkg/beekeeper" "github.com/ethersphere/beekeeper/pkg/check/act" + "github.com/ethersphere/beekeeper/pkg/check/autotls" "github.com/ethersphere/beekeeper/pkg/check/balances" "github.com/ethersphere/beekeeper/pkg/check/cashout" "github.com/ethersphere/beekeeper/pkg/check/datadurability" @@ -82,6 +83,24 @@ var Checks = map[string]CheckType{ return opts, nil }, }, + "autotls": { + NewAction: autotls.NewCheck, + NewOptions: func(checkGlobalConfig CheckGlobalConfig, check Check) (any, error) { + checkOpts := new(struct { + AutoTLSGroup *string `yaml:"autotls-group"` + UltraLightGroup *string `yaml:"ultra-light-group"` + }) + if err := check.Options.Decode(checkOpts); err != nil { + return nil, fmt.Errorf("decoding check %s options: %w", check.Type, err) + } + opts := autotls.NewDefaultOptions() + + if err := applyCheckConfig(checkGlobalConfig, checkOpts, &opts); err != nil { + return nil, fmt.Errorf("applying options: %w", err) + } + return opts, nil + }, + }, "balances": { NewAction: balances.NewCheck, NewOptions: func(checkGlobalConfig CheckGlobalConfig, check Check) (any, error) { diff --git a/pkg/k8s/containers/env.go b/pkg/k8s/containers/env.go index dd133698..ae2f097f 100644 --- a/pkg/k8s/containers/env.go +++ b/pkg/k8s/containers/env.go @@ -29,16 +29,19 @@ type EnvVar struct { // toK8S converts EnvVar to Kubernetes client object func (ev *EnvVar) toK8S() v1.EnvVar { - return v1.EnvVar{ + envVar := v1.EnvVar{ Name: ev.Name, Value: ev.Value, - ValueFrom: &v1.EnvVarSource{ + } + if ev.ValueFrom.hasValues() { + envVar.ValueFrom = &v1.EnvVarSource{ FieldRef: ev.ValueFrom.Field.toK8S(), ResourceFieldRef: ev.ValueFrom.ResourceField.toK8S(), ConfigMapKeyRef: ev.ValueFrom.ConfigMap.toK8S(), SecretKeyRef: ev.ValueFrom.Secret.toK8S(), - }, + } } + return envVar } // ValueFrom represents Kubernetes ValueFrom @@ -49,6 +52,14 @@ type ValueFrom struct { Secret SecretKey } +// hasValues returns true if any ValueFrom field is configured +func (vf *ValueFrom) hasValues() bool { + return vf.Field.Path != "" || + vf.ResourceField.Resource != "" || + vf.ConfigMap.ConfigMapName != "" || + vf.Secret.SecretName != "" +} + // Field represents Kubernetes ObjectFieldSelector type Field struct { APIVersion string diff --git a/pkg/orchestration/k8s/helpers.go b/pkg/orchestration/k8s/helpers.go index 64ae4350..adc53e81 100644 --- a/pkg/orchestration/k8s/helpers.go +++ b/pkg/orchestration/k8s/helpers.go @@ -1,6 +1,7 @@ package k8s import ( + "fmt" "maps" "strconv" "strings" @@ -15,6 +16,9 @@ const ( configTemplate = ` allow-private-cidrs: {{ .AllowPrivateCIDRs }} api-addr: {{.APIAddr}} +autotls-ca-endpoint: {{.AutoTLSCAEndpoint}} +autotls-domain: {{.AutoTLSDomain}} +autotls-registration-endpoint: {{.AutoTLSRegistrationEndpoint}} block-time: {{ .BlockTime }} blockchain-rpc-endpoint: {{.BlockchainRPCEndpoint}} bootnode-mode: {{.BootnodeMode}} @@ -30,9 +34,12 @@ db-write-buffer-size: {{.DbWriteBufferSize}} full-node: {{.FullNode}} mainnet: {{.Mainnet}} nat-addr: {{.NATAddr}} +nat-wss-addr: {{.NATWSSAddr}} network-id: {{.NetworkID}} p2p-addr: {{.P2PAddr}} p2p-ws-enable: {{.P2PWSEnable}} +p2p-wss-addr: {{.P2PWSSAddr}} +p2p-wss-enable: {{.P2PWSSEnable}} password: {{.Password}} payment-early-percent: {{.PaymentEarly}} payment-threshold: {{.PaymentThreshold}} @@ -57,7 +64,33 @@ withdrawal-addresses-whitelist: {{.WithdrawAddress}} ` ) -func setInitContainers() (inits containers.Containers) { +// https://raw.githubusercontent.com/letsencrypt/pebble/main/test/certs/pebble.minica.pem +const pebbleCertificate = `-----BEGIN CERTIFICATE----- +MIIDPzCCAiegAwIBAgIIU0Xm9UFdQxUwDQYJKoZIhvcNAQELBQAwIDEeMBwGA1UE +AxMVbWluaWNhIHJvb3QgY2EgNTM0NWU2MCAXDTI1MDkwMzIzNDAwNVoYDzIxMjUw +OTAzMjM0MDA1WjAgMR4wHAYDVQQDExVtaW5pY2Egcm9vdCBjYSA1MzQ1ZTYwggEi +MA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC5WgZNoVJandj43kkLyU50vzCZ +alozvdRo3OFiKoDtmqKPNWRNO2hC9AUNxTDJco51Yc42u/WV3fPbbhSznTiOOVtn +Ajm6iq4I5nZYltGGZetGDOQWr78y2gWY+SG078MuOO2hyDIiKtVc3xiXYA+8Hluu +9F8KbqSS1h55yxZ9b87eKR+B0zu2ahzBCIHKmKWgc6N13l7aDxxY3D6uq8gtJRU0 +toumyLbdzGcupVvjbjDP11nl07RESDWBLG1/g3ktJvqIa4BWgU2HMh4rND6y8OD3 +Hy3H8MY6CElL+MOCbFJjWqhtOxeFyZZV9q3kYnk9CAuQJKMEGuN4GU6tzhW1AgMB +AAGjezB5MA4GA1UdDwEB/wQEAwIChDATBgNVHSUEDDAKBggrBgEFBQcDATASBgNV +HRMBAf8ECDAGAQH/AgEAMB0GA1UdDgQWBBSu8RGpErgYUoYnQuwCq+/ggTiEjDAf +BgNVHSMEGDAWgBSu8RGpErgYUoYnQuwCq+/ggTiEjDANBgkqhkiG9w0BAQsFAAOC +AQEAXDVYov1+f6EL7S41LhYQkEX/GyNNzsEvqxE9U0+3Iri5JfkcNOiA9O9L6Z+Y +bqcsXV93s3vi4r4WSWuc//wHyJYrVe5+tK4nlFpbJOvfBUtnoBDyKNxXzZCxFJVh +f9uc8UejRfQMFbDbhWY/x83y9BDufJHHq32OjCIN7gp2UR8rnfYvlz7Zg4qkJBsn +DG4dwd+pRTCFWJOVIG0JoNhK3ZmE7oJ1N4H38XkZ31NPcMksKxpsLLIS9+mosZtg +4olL7tMPJklx5ZaeMFaKRDq4Gdxkbw4+O4vRgNm3Z8AXWKknOdfgdpqLUPPhRcP4 +v1lhy71EhBuXXwRQJry0lTdF+w== +-----END CERTIFICATE-----` + +type setInitContainersOptions struct { + AutoTLSEnabled bool +} + +func setInitContainers(o setInitContainersOptions) (inits containers.Containers) { inits = append(inits, containers.Container{ Name: "init-bee", Image: "ethersphere/busybox:1.33", @@ -72,6 +105,31 @@ echo 'bee initialization done';`}, }, }) + // TODO: this init container is only needed for testing with Pebble + // should not be used in other contexts. + if o.AutoTLSEnabled { + // Install Pebble CA certificates as an init container. + inits = append(inits, containers.Container{ + Name: "install-pebble-ca", + Image: "alpine:latest", + Command: []string{"sh", "-c", fmt.Sprintf(`set -ex +apk add --no-cache ca-certificates +mkdir -p /certs +cat > /certs/pebble-minica.crt << 'CERT' +%s +CERT +cp /certs/pebble-minica.crt /usr/local/share/ca-certificates/ +update-ca-certificates +cp /etc/ssl/certs/ca-certificates.crt /certs/ca-certificates.crt`, pebbleCertificate)}, + VolumeMounts: containers.VolumeMounts{ + { + Name: "pebble-ca-certs", + MountPath: "/certs", + }, + }, + }) + } + return inits } @@ -81,6 +139,7 @@ type setContainersOptions struct { ImagePullPolicy string PortAPI int32 PortP2P int32 + PortP2PWSS int32 PersistenceEnabled bool ResourcesLimitCPU string ResourcesLimitMemory string @@ -88,6 +147,7 @@ type setContainersOptions struct { ResourcesRequestMemory string LibP2PEnabled bool SwarmEnabled bool + AutoTLSEnabled bool } func setContainers(o setContainersOptions) (c containers.Containers) { @@ -96,18 +156,40 @@ func setContainers(o setContainersOptions) (c containers.Containers) { Image: o.Image, ImagePullPolicy: o.ImagePullPolicy, Command: []string{"bee", "start", "--config=.bee.yaml"}, - Ports: containers.Ports{ - { - Name: "api", - ContainerPort: o.PortAPI, - Protocol: "TCP", - }, - { - Name: "p2p", - ContainerPort: o.PortP2P, - Protocol: "TCP", - }, - }, + Env: func() containers.EnvVars { + if o.AutoTLSEnabled { + return containers.EnvVars{ + { + Name: "SSL_CERT_FILE", + Value: "/etc/ssl/certs/pebble-ca-certificates.crt", + }, + } + } + return nil + }(), + Ports: func() containers.Ports { + ports := containers.Ports{ + { + Name: "api", + ContainerPort: o.PortAPI, + Protocol: "TCP", + }, + { + Name: "p2p", + ContainerPort: o.PortP2P, + Protocol: "TCP", + }, + } + // Add p2p-wss port if configured + if o.PortP2PWSS > 0 { + ports = append(ports, containers.Port{ + Name: "p2p-wss", + ContainerPort: o.PortP2PWSS, + Protocol: "TCP", + }) + } + return ports + }(), LivenessProbe: containers.Probe{HTTPGet: &containers.HTTPGetProbe{ InitialDelaySeconds: 5, Handler: containers.HTTPGetHandler{ @@ -140,8 +222,9 @@ func setContainers(o setContainersOptions) (c containers.Containers) { RunAsUser: 999, }, VolumeMounts: setBeeVolumeMounts(setBeeVolumeMountsOptions{ - LibP2PEnabled: o.LibP2PEnabled, - SwarmEnabled: o.SwarmEnabled, + LibP2PEnabled: o.LibP2PEnabled, + SwarmEnabled: o.SwarmEnabled, + AutoTLSEnabled: o.AutoTLSEnabled, }), }) @@ -149,8 +232,9 @@ func setContainers(o setContainersOptions) (c containers.Containers) { } type setBeeVolumeMountsOptions struct { - LibP2PEnabled bool - SwarmEnabled bool + LibP2PEnabled bool + SwarmEnabled bool + AutoTLSEnabled bool } func setBeeVolumeMounts(o setBeeVolumeMountsOptions) (volumeMounts containers.VolumeMounts) { @@ -180,6 +264,14 @@ func setBeeVolumeMounts(o setBeeVolumeMountsOptions) (volumeMounts containers.Vo ReadOnly: true, }) } + if o.AutoTLSEnabled { + volumeMounts = append(volumeMounts, containers.VolumeMount{ + Name: "pebble-ca-certs", + MountPath: "/etc/ssl/certs/pebble-ca-certificates.crt", + SubPath: "ca-certificates.crt", + ReadOnly: true, + }) + } return volumeMounts } @@ -190,6 +282,7 @@ type setVolumesOptions struct { PersistenceEnabled bool LibP2PEnabled bool SwarmEnabled bool + AutoTLSEnabled bool } func setVolumes(o setVolumesOptions) (volumes pod.Volumes) { @@ -230,6 +323,13 @@ func setVolumes(o setVolumesOptions) (volumes pod.Volumes) { }, }) } + if o.AutoTLSEnabled { + volumes = append(volumes, pod.Volume{ + EmptyDir: &pod.EmptyDirVolume{ + Name: "pebble-ca-certs", + }, + }) + } return volumes } @@ -257,33 +357,23 @@ func setPersistentVolumeClaims(o setPersistentVolumeClaimsOptions) (pvcs pvc.Per return pvcs } -type setBeeNodePortOptions struct { - AppProtocol string - Name string - Protocol string - TargetPort string - Port int32 - NodePort int32 -} - -func setBeeNodePort(o setBeeNodePortOptions) (ports service.Ports) { - if o.NodePort > 0 { - return service.Ports{{ - AppProtocol: "TCP", - Name: "p2p", - Protocol: "TCP", - Port: o.Port, - TargetPort: "p2p", - Nodeport: o.NodePort, - }} +// createServicePort creates a service port with optional NodePort. +// If targetPort is empty, it defaults to name. +func createServicePort(name string, port int32, targetPort string, nodePort int32) service.Port { + if targetPort == "" { + targetPort = name } - return service.Ports{{ + p := service.Port{ AppProtocol: "TCP", - Name: "p2p", + Name: name, Protocol: "TCP", - Port: o.Port, - TargetPort: "p2p", - }} + Port: port, + TargetPort: targetPort, + } + if nodePort > 0 { + p.Nodeport = nodePort + } + return p } func parsePort(port string) (int32, error) { diff --git a/pkg/orchestration/k8s/orchestrator.go b/pkg/orchestration/k8s/orchestrator.go index 49bfc33f..bb5712d0 100644 --- a/pkg/orchestration/k8s/orchestrator.go +++ b/pkg/orchestration/k8s/orchestrator.go @@ -196,22 +196,42 @@ func (n *nodeOrchestrator) Create(ctx context.Context, o orchestration.CreateOpt } } + var portP2PWSS int32 + if len(o.Config.P2PWSSAddr) > 0 { + portP2PWSS, err = parsePort(o.Config.P2PWSSAddr) + if err != nil { + return fmt.Errorf("parsing P2P WSS port from config: %w", err) + } + } + + var nodePortP2PWSS int32 + if len(o.Config.NATWSSAddr) > 0 { + nodePortP2PWSS, err = parsePort(o.Config.NATWSSAddr) + if err != nil { + return fmt.Errorf("parsing NAT WSS address from config: %w", err) + } + } + p2pSvc := fmt.Sprintf("%s-p2p", o.Name) + + // Build ports for p2p service + p2pPorts := service.Ports{ + createServicePort("p2p", portP2P, "", nodePortP2P), + } + + // Add p2p-wss port if P2PWSSAddr is configured + if portP2PWSS > 0 { + p2pPorts = append(p2pPorts, createServicePort("p2p-wss", portP2PWSS, "", nodePortP2PWSS)) + } + if _, err := n.k8s.Service.Set(ctx, p2pSvc, o.Namespace, service.Options{ Annotations: o.Annotations, Labels: o.Labels, ServiceSpec: service.Spec{ ExternalTrafficPolicy: "Local", - Ports: setBeeNodePort(setBeeNodePortOptions{ - AppProtocol: "TCP", - Name: "p2p", - Protocol: "TCP", - TargetPort: "p2p", - Port: portP2P, - NodePort: nodePortP2P, - }), - Selector: o.Selector, - Type: "NodePort", + Ports: p2pPorts, + Selector: o.Selector, + Type: "NodePort", }, }); err != nil { return fmt.Errorf("set service in namespace %s: %w", o.Namespace, err) @@ -252,6 +272,7 @@ func (n *nodeOrchestrator) Create(ctx context.Context, o orchestration.CreateOpt sSet := o.Name libP2PEnabled := len(o.LibP2PKey) > 0 swarmEnabled := o.SwarmKey != nil + autoTLSEnabled := o.Config.P2PWSSAddr != "" if _, err := n.k8s.StatefulSet.Set(ctx, sSet, o.Namespace, statefulset.Options{ Annotations: o.Annotations, @@ -267,13 +288,16 @@ func (n *nodeOrchestrator) Create(ctx context.Context, o orchestration.CreateOpt Annotations: o.Annotations, Labels: o.Labels, Spec: pod.PodSpec{ - InitContainers: setInitContainers(), + InitContainers: setInitContainers(setInitContainersOptions{ + AutoTLSEnabled: autoTLSEnabled, + }), Containers: setContainers(setContainersOptions{ Name: sSet, Image: o.Image, ImagePullPolicy: o.ImagePullPolicy, PortAPI: portAPI, PortP2P: portP2P, + PortP2PWSS: portP2PWSS, PersistenceEnabled: o.PersistenceEnabled, ResourcesLimitCPU: o.ResourcesLimitCPU, ResourcesLimitMemory: o.ResourcesLimitMemory, @@ -281,6 +305,7 @@ func (n *nodeOrchestrator) Create(ctx context.Context, o orchestration.CreateOpt ResourcesRequestMemory: o.ResourcesRequestMemory, LibP2PEnabled: libP2PEnabled, SwarmEnabled: swarmEnabled, + AutoTLSEnabled: autoTLSEnabled, }), NodeSelector: o.NodeSelector, PodSecurityContext: pod.PodSecurityContext{ @@ -295,6 +320,7 @@ func (n *nodeOrchestrator) Create(ctx context.Context, o orchestration.CreateOpt PersistenceEnabled: o.PersistenceEnabled, LibP2PEnabled: libP2PEnabled, SwarmEnabled: swarmEnabled, + AutoTLSEnabled: autoTLSEnabled, }), }, }, diff --git a/pkg/orchestration/node.go b/pkg/orchestration/node.go index 30f1f57a..ae06f82f 100644 --- a/pkg/orchestration/node.go +++ b/pkg/orchestration/node.go @@ -74,45 +74,51 @@ type CreateOptions struct { // Config represents Bee configuration type Config struct { - AllowPrivateCIDRs bool // allow to advertise private CIDRs to the public network - APIAddr string // HTTP API listen address - BlockTime uint64 // chain block time - Bootnodes string // initial nodes to connect to - BootnodeMode bool // cause the node to always accept incoming connections - CacheCapacity uint64 // cache capacity in chunks, multiply by 4096 (MaxChunkSize) to get approximate capacity in bytes - CORSAllowedOrigins string // origins with CORS headers enabled - DataDir string // data directory - DbOpenFilesLimit int // number of open files allowed by database - DbBlockCacheCapacity int // size of block cache of the database in bytes - DbWriteBufferSize int // size of the database write buffer in bytes - DbDisableSeeksCompaction bool // disables DB compactions triggered by seeks - FullNode bool // cause the node to start in full mode - Mainnet bool // enable mainnet - NATAddr string // NAT exposed address - NetworkID uint64 // ID of the Swarm network - P2PAddr string // P2P listen address - P2PWSEnable bool // enable P2P WebSocket transport - Password string // password for decrypting keys - PaymentEarly uint64 // amount in BZZ below the peers payment threshold when we initiate settlement - PaymentThreshold uint64 // threshold in BZZ where you expect to get paid from your peers - PaymentTolerance uint64 // excess debt above payment threshold in BZZ where you disconnect from your peer - PostageStampAddress string // postage stamp address - PostageContractStartBlock uint64 // postage stamp address - PriceOracleAddress string // price Oracle address - ResolverOptions string // ENS compatible API endpoint for a TLD and with contract address, can be repeated, format [tld:][contract-addr@]url - ChequebookEnable bool // enable chequebook - SwapEnable bool // enable swap - BlockchainRPCEndpoint string // blockchain RPC endpoint - SwapFactoryAddress string // swap factory address - RedistributionAddress string // redistribution address - StakingAddress string // staking address - StorageIncentivesEnable string // storage incentives enable flag - SwapInitialDeposit uint64 // initial deposit if deploying a new chequebook - TracingEnabled bool // enable tracing - TracingEndpoint string // endpoint to send tracing data - TracingServiceName string // service name identifier for tracing - Verbosity uint64 // log verbosity level 0=silent, 1=error, 2=warn, 3=info, 4=debug, 5=trace - WelcomeMessage string // send a welcome message string during handshakes - WarmupTime time.Duration // warmup time pull/pushsync protocols - WithdrawAddress string // allowed addresses for wallet withdrawal + AllowPrivateCIDRs bool // allow to advertise private CIDRs to the public network + APIAddr string // HTTP API listen address + AutoTLSCAEndpoint string // ACME CA endpoint + AutoTLSDomain string // domain for ACME + AutoTLSRegistrationEndpoint string // ACME registration endpoint + BlockchainRPCEndpoint string // blockchain RPC endpoint + BlockTime uint64 // chain block time + BootnodeMode bool // cause the node to always accept incoming connections + Bootnodes string // initial nodes to connect to + CacheCapacity uint64 // cache capacity in chunks, multiply by 4096 (MaxChunkSize) to get approximate capacity in bytes + ChequebookEnable bool // enable chequebook + CORSAllowedOrigins string // origins with CORS headers enabled + DataDir string // data directory + DbBlockCacheCapacity int // size of block cache of the database in bytes + DbDisableSeeksCompaction bool // disables DB compactions triggered by seeks + DbOpenFilesLimit int // number of open files allowed by database + DbWriteBufferSize int // size of the database write buffer in bytes + FullNode bool // cause the node to start in full mode + Mainnet bool // enable mainnet + NATAddr string // NAT exposed address + NATWSSAddr string // NAT exposed secure WebSocket address + NetworkID uint64 // ID of the Swarm network + P2PAddr string // P2P listen address + P2PWSEnable bool // enable P2P WebSocket transport + P2PWSSAddr string // P2P Secure WebSocket listen address + P2PWSSEnable bool // enable P2P Secure WebSocket transport + Password string // password for decrypting keys + PaymentEarly uint64 // amount in BZZ below the peers payment threshold when we initiate settlement + PaymentThreshold uint64 // threshold in BZZ where you expect to get paid from your peers + PaymentTolerance uint64 // excess debt above payment threshold in BZZ where you disconnect from your peer + PostageContractStartBlock uint64 // postage stamp address + PostageStampAddress string // postage stamp address + PriceOracleAddress string // price Oracle address + RedistributionAddress string // redistribution address + ResolverOptions string // ENS compatible API endpoint for a TLD and with contract address, can be repeated, format [tld:][contract-addr@]url + StakingAddress string // staking address + StorageIncentivesEnable string // storage incentives enable flag + SwapEnable bool // enable swap + SwapFactoryAddress string // swap factory address + SwapInitialDeposit uint64 // initial deposit if deploying a new chequebook + TracingEnabled bool // enable tracing + TracingEndpoint string // endpoint to send tracing data + TracingServiceName string // service name identifier for tracing + Verbosity uint64 // log verbosity level 0=silent, 1=error, 2=warn, 3=info, 4=debug, 5=trace + WarmupTime time.Duration // warmup time pull/pushsync protocols + WelcomeMessage string // send a welcome message string during handshakes + WithdrawAddress string // allowed addresses for wallet withdrawal }