From ca0b0a43d092734fee44831397fbc47d86cf4f8b Mon Sep 17 00:00:00 2001 From: Ljubisa Gacevic Date: Wed, 26 Nov 2025 15:34:25 +0100 Subject: [PATCH 01/10] feat: add new flags --- config/local.yaml | 6 ++ pkg/config/bee.go | 90 ++++++++++++++------------- pkg/orchestration/k8s/helpers.go | 6 ++ pkg/orchestration/k8s/orchestrator.go | 8 +++ pkg/orchestration/node.go | 88 ++++++++++++++------------ 5 files changed, 115 insertions(+), 83 deletions(-) diff --git a/config/local.yaml b/config/local.yaml index 7b0ae478..eb30cb4d 100644 --- a/config/local.yaml +++ b/config/local.yaml @@ -111,6 +111,9 @@ bee-configs: _inherit: "" allow-private-cidrs: true api-addr: ":1633" + autotls-ca-endpoint: "https://pebble.local:14000/dir" + autotls-domain: "localhost" + autotls-registration-endpoint: "http://p2p-forge.local:8080/v1/_acme-challenge" block-time: 1 blockchain-rpc-endpoint: "ws://geth-swap:8546" bootnode-mode: false @@ -126,9 +129,12 @@ bee-configs: full-node: true mainnet: false nat-addr: "" + nat-wss-addr: "" network-id: 0 p2p-addr: ":1634" + p2p-wss-addr: ":1635" p2p-ws-enable: false + p2p-wss-enable: false password: "beekeeper" payment-early-percent: 50 payment-threshold: 13500000 diff --git a/pkg/config/bee.go b/pkg/config/bee.go index 6d2b5cdd..34366479 100644 --- a/pkg/config/bee.go +++ b/pkg/config/bee.go @@ -16,48 +16,54 @@ type BeeConfig struct { // parent to inherit settings from *Inherit `yaml:",inline"` // Bee configuration - AllowPrivateCIDRs *bool `yaml:"allow-private-cidrs"` - APIAddr *string `yaml:"api-addr"` - BlockchainRPCEndpoint *string `yaml:"blockchain-rpc-endpoint"` - BlockTime *uint64 `yaml:"block-time"` - BootnodeMode *bool `yaml:"bootnode-mode"` - Bootnodes *string `yaml:"bootnodes"` - CacheCapacity *uint64 `yaml:"cache-capacity"` - ChequebookEnable *bool `yaml:"chequebook-enable"` - CORSAllowedOrigins *string `yaml:"cors-allowed-origins"` - DataDir *string `yaml:"data-dir"` - DbBlockCacheCapacity *int `yaml:"db-block-cache-capacity"` - DbDisableSeeksCompaction *bool `yaml:"db-disable-seeks-compaction"` - DbOpenFilesLimit *int `yaml:"db-open-files-limit"` - DbWriteBufferSize *int `yaml:"db-write-buffer-size"` - FullNode *bool `yaml:"full-node"` - Mainnet *bool `yaml:"mainnet"` - NATAddr *string `yaml:"nat-addr"` - NetworkID *uint64 `yaml:"network-id"` - P2PAddr *string `yaml:"p2p-addr"` - P2PWSEnable *bool `yaml:"p2p-ws-enable"` - Password *string `yaml:"password"` - PaymentEarly *uint64 `yaml:"payment-early-percent"` - PaymentThreshold *uint64 `yaml:"payment-threshold"` - PaymentTolerance *uint64 `yaml:"payment-tolerance-percent"` - PostageContractStartBlock *uint64 `yaml:"postage-stamp-start-block"` - PostageStampAddress *string `yaml:"postage-stamp-address"` - PriceOracleAddress *string `yaml:"price-oracle-address"` - RedistributionAddress *string `yaml:"redistribution-address"` - ResolverOptions *string `yaml:"resolver-options"` - StakingAddress *string `yaml:"staking-address"` - StorageIncentivesEnable *string `yaml:"storage-incentives-enable"` - SwapEnable *bool `yaml:"swap-enable"` - SwapEndpoint *string `yaml:"swap-endpoint"` // deprecated: use blockchain-rpc-endpoint - SwapFactoryAddress *string `yaml:"swap-factory-address"` - SwapInitialDeposit *uint64 `yaml:"swap-initial-deposit"` - TracingEnabled *bool `yaml:"tracing-enabled"` - TracingEndpoint *string `yaml:"tracing-endpoint"` - TracingServiceName *string `yaml:"tracing-service-name"` - Verbosity *uint64 `yaml:"verbosity"` - WarmupTime *time.Duration `yaml:"warmup-time"` - WelcomeMessage *string `yaml:"welcome-message"` - WithdrawAddress *string `yaml:"withdrawal-addresses-whitelist"` + AllowPrivateCIDRs *bool `yaml:"allow-private-cidrs"` + APIAddr *string `yaml:"api-addr"` + AutoTLSCAEndpoint *string `yaml:"autotls-ca-endpoint"` + AutoTLSDomain *string `yaml:"autotls-domain"` + AutoTLSRegistrationEndpoint *string `yaml:"autotls-registration-endpoint"` + BlockchainRPCEndpoint *string `yaml:"blockchain-rpc-endpoint"` + BlockTime *uint64 `yaml:"block-time"` + BootnodeMode *bool `yaml:"bootnode-mode"` + Bootnodes *string `yaml:"bootnodes"` + CacheCapacity *uint64 `yaml:"cache-capacity"` + ChequebookEnable *bool `yaml:"chequebook-enable"` + CORSAllowedOrigins *string `yaml:"cors-allowed-origins"` + DataDir *string `yaml:"data-dir"` + DbBlockCacheCapacity *int `yaml:"db-block-cache-capacity"` + DbDisableSeeksCompaction *bool `yaml:"db-disable-seeks-compaction"` + DbOpenFilesLimit *int `yaml:"db-open-files-limit"` + DbWriteBufferSize *int `yaml:"db-write-buffer-size"` + FullNode *bool `yaml:"full-node"` + Mainnet *bool `yaml:"mainnet"` + NATAddr *string `yaml:"nat-addr"` + NATWSSAddr *string `yaml:"nat-wss-addr"` + NetworkID *uint64 `yaml:"network-id"` + P2PAddr *string `yaml:"p2p-addr"` + P2PWSEnable *bool `yaml:"p2p-ws-enable"` + P2PWSSAddr *string `yaml:"p2p-wss-addr"` + P2PWSSEnable *bool `yaml:"p2p-wss-enable"` + Password *string `yaml:"password"` + PaymentEarly *uint64 `yaml:"payment-early-percent"` + PaymentThreshold *uint64 `yaml:"payment-threshold"` + PaymentTolerance *uint64 `yaml:"payment-tolerance-percent"` + PostageContractStartBlock *uint64 `yaml:"postage-stamp-start-block"` + PostageStampAddress *string `yaml:"postage-stamp-address"` + PriceOracleAddress *string `yaml:"price-oracle-address"` + RedistributionAddress *string `yaml:"redistribution-address"` + ResolverOptions *string `yaml:"resolver-options"` + StakingAddress *string `yaml:"staking-address"` + StorageIncentivesEnable *string `yaml:"storage-incentives-enable"` + SwapEnable *bool `yaml:"swap-enable"` + SwapEndpoint *string `yaml:"swap-endpoint"` // deprecated: use blockchain-rpc-endpoint + SwapFactoryAddress *string `yaml:"swap-factory-address"` + SwapInitialDeposit *uint64 `yaml:"swap-initial-deposit"` + TracingEnabled *bool `yaml:"tracing-enabled"` + TracingEndpoint *string `yaml:"tracing-endpoint"` + TracingServiceName *string `yaml:"tracing-service-name"` + Verbosity *uint64 `yaml:"verbosity"` + WarmupTime *time.Duration `yaml:"warmup-time"` + WelcomeMessage *string `yaml:"welcome-message"` + WithdrawAddress *string `yaml:"withdrawal-addresses-whitelist"` } func (b BeeConfig) GetParentName() string { diff --git a/pkg/orchestration/k8s/helpers.go b/pkg/orchestration/k8s/helpers.go index 64ae4350..ee720c6a 100644 --- a/pkg/orchestration/k8s/helpers.go +++ b/pkg/orchestration/k8s/helpers.go @@ -15,6 +15,9 @@ const ( configTemplate = ` allow-private-cidrs: {{ .AllowPrivateCIDRs }} api-addr: {{.APIAddr}} +autotls-ca-endpoint: {{.AutoTLSCAEndpoint}} +autotls-domain: {{.AutoTLSDomain}} +autotls-registration-endpoint: {{.AutoTLSRegistrationEndpoint}} block-time: {{ .BlockTime }} blockchain-rpc-endpoint: {{.BlockchainRPCEndpoint}} bootnode-mode: {{.BootnodeMode}} @@ -30,9 +33,12 @@ db-write-buffer-size: {{.DbWriteBufferSize}} full-node: {{.FullNode}} mainnet: {{.Mainnet}} nat-addr: {{.NATAddr}} +nat-wss-addr: {{.NATWSSAddr}} network-id: {{.NetworkID}} p2p-addr: {{.P2PAddr}} p2p-ws-enable: {{.P2PWSEnable}} +p2p-wss-addr: {{.P2PWSSAddr}} +p2p-wss-enable: {{.P2PWSSEnable}} password: {{.Password}} payment-early-percent: {{.PaymentEarly}} payment-threshold: {{.PaymentThreshold}} diff --git a/pkg/orchestration/k8s/orchestrator.go b/pkg/orchestration/k8s/orchestrator.go index 49bfc33f..953fea4b 100644 --- a/pkg/orchestration/k8s/orchestrator.go +++ b/pkg/orchestration/k8s/orchestrator.go @@ -196,6 +196,14 @@ func (n *nodeOrchestrator) Create(ctx context.Context, o orchestration.CreateOpt } } + // var nodePortP2PWSS int32 + // if len(o.Config.NATWSSAddr) > 0 { + // nodePortP2PWSS, err = parsePort(o.Config.NATWSSAddr) + // if err != nil { + // return fmt.Errorf("parsing NAT WSS address from config: %w", err) + // } + // } + p2pSvc := fmt.Sprintf("%s-p2p", o.Name) if _, err := n.k8s.Service.Set(ctx, p2pSvc, o.Namespace, service.Options{ Annotations: o.Annotations, diff --git a/pkg/orchestration/node.go b/pkg/orchestration/node.go index 30f1f57a..ae06f82f 100644 --- a/pkg/orchestration/node.go +++ b/pkg/orchestration/node.go @@ -74,45 +74,51 @@ type CreateOptions struct { // Config represents Bee configuration type Config struct { - AllowPrivateCIDRs bool // allow to advertise private CIDRs to the public network - APIAddr string // HTTP API listen address - BlockTime uint64 // chain block time - Bootnodes string // initial nodes to connect to - BootnodeMode bool // cause the node to always accept incoming connections - CacheCapacity uint64 // cache capacity in chunks, multiply by 4096 (MaxChunkSize) to get approximate capacity in bytes - CORSAllowedOrigins string // origins with CORS headers enabled - DataDir string // data directory - DbOpenFilesLimit int // number of open files allowed by database - DbBlockCacheCapacity int // size of block cache of the database in bytes - DbWriteBufferSize int // size of the database write buffer in bytes - DbDisableSeeksCompaction bool // disables DB compactions triggered by seeks - FullNode bool // cause the node to start in full mode - Mainnet bool // enable mainnet - NATAddr string // NAT exposed address - NetworkID uint64 // ID of the Swarm network - P2PAddr string // P2P listen address - P2PWSEnable bool // enable P2P WebSocket transport - Password string // password for decrypting keys - PaymentEarly uint64 // amount in BZZ below the peers payment threshold when we initiate settlement - PaymentThreshold uint64 // threshold in BZZ where you expect to get paid from your peers - PaymentTolerance uint64 // excess debt above payment threshold in BZZ where you disconnect from your peer - PostageStampAddress string // postage stamp address - PostageContractStartBlock uint64 // postage stamp address - PriceOracleAddress string // price Oracle address - ResolverOptions string // ENS compatible API endpoint for a TLD and with contract address, can be repeated, format [tld:][contract-addr@]url - ChequebookEnable bool // enable chequebook - SwapEnable bool // enable swap - BlockchainRPCEndpoint string // blockchain RPC endpoint - SwapFactoryAddress string // swap factory address - RedistributionAddress string // redistribution address - StakingAddress string // staking address - StorageIncentivesEnable string // storage incentives enable flag - SwapInitialDeposit uint64 // initial deposit if deploying a new chequebook - TracingEnabled bool // enable tracing - TracingEndpoint string // endpoint to send tracing data - TracingServiceName string // service name identifier for tracing - Verbosity uint64 // log verbosity level 0=silent, 1=error, 2=warn, 3=info, 4=debug, 5=trace - WelcomeMessage string // send a welcome message string during handshakes - WarmupTime time.Duration // warmup time pull/pushsync protocols - WithdrawAddress string // allowed addresses for wallet withdrawal + AllowPrivateCIDRs bool // allow to advertise private CIDRs to the public network + APIAddr string // HTTP API listen address + AutoTLSCAEndpoint string // ACME CA endpoint + AutoTLSDomain string // domain for ACME + AutoTLSRegistrationEndpoint string // ACME registration endpoint + BlockchainRPCEndpoint string // blockchain RPC endpoint + BlockTime uint64 // chain block time + BootnodeMode bool // cause the node to always accept incoming connections + Bootnodes string // initial nodes to connect to + CacheCapacity uint64 // cache capacity in chunks, multiply by 4096 (MaxChunkSize) to get approximate capacity in bytes + ChequebookEnable bool // enable chequebook + CORSAllowedOrigins string // origins with CORS headers enabled + DataDir string // data directory + DbBlockCacheCapacity int // size of block cache of the database in bytes + DbDisableSeeksCompaction bool // disables DB compactions triggered by seeks + DbOpenFilesLimit int // number of open files allowed by database + DbWriteBufferSize int // size of the database write buffer in bytes + FullNode bool // cause the node to start in full mode + Mainnet bool // enable mainnet + NATAddr string // NAT exposed address + NATWSSAddr string // NAT exposed secure WebSocket address + NetworkID uint64 // ID of the Swarm network + P2PAddr string // P2P listen address + P2PWSEnable bool // enable P2P WebSocket transport + P2PWSSAddr string // P2P Secure WebSocket listen address + P2PWSSEnable bool // enable P2P Secure WebSocket transport + Password string // password for decrypting keys + PaymentEarly uint64 // amount in BZZ below the peers payment threshold when we initiate settlement + PaymentThreshold uint64 // threshold in BZZ where you expect to get paid from your peers + PaymentTolerance uint64 // excess debt above payment threshold in BZZ where you disconnect from your peer + PostageContractStartBlock uint64 // postage stamp address + PostageStampAddress string // postage stamp address + PriceOracleAddress string // price Oracle address + RedistributionAddress string // redistribution address + ResolverOptions string // ENS compatible API endpoint for a TLD and with contract address, can be repeated, format [tld:][contract-addr@]url + StakingAddress string // staking address + StorageIncentivesEnable string // storage incentives enable flag + SwapEnable bool // enable swap + SwapFactoryAddress string // swap factory address + SwapInitialDeposit uint64 // initial deposit if deploying a new chequebook + TracingEnabled bool // enable tracing + TracingEndpoint string // endpoint to send tracing data + TracingServiceName string // service name identifier for tracing + Verbosity uint64 // log verbosity level 0=silent, 1=error, 2=warn, 3=info, 4=debug, 5=trace + WarmupTime time.Duration // warmup time pull/pushsync protocols + WelcomeMessage string // send a welcome message string during handshakes + WithdrawAddress string // allowed addresses for wallet withdrawal } From f8341f702a700aeb8748e7a9334f4ef0ccb3907a Mon Sep 17 00:00:00 2001 From: Ljubisa Gacevic Date: Wed, 26 Nov 2025 16:40:58 +0100 Subject: [PATCH 02/10] fix(k8s): ensure p2p-wss service port is created when configured --- config/local.yaml | 8 +-- pkg/orchestration/k8s/helpers.go | 74 ++++++++++++++------------- pkg/orchestration/k8s/orchestrator.go | 47 +++++++++++------ 3 files changed, 72 insertions(+), 57 deletions(-) diff --git a/config/local.yaml b/config/local.yaml index eb30cb4d..11f3bc89 100644 --- a/config/local.yaml +++ b/config/local.yaml @@ -111,9 +111,9 @@ bee-configs: _inherit: "" allow-private-cidrs: true api-addr: ":1633" - autotls-ca-endpoint: "https://pebble.local:14000/dir" - autotls-domain: "localhost" - autotls-registration-endpoint: "http://p2p-forge.local:8080/v1/_acme-challenge" + autotls-ca-endpoint: "https://pebble:14000/dir" + autotls-domain: "local.test" + autotls-registration-endpoint: "http://p2p-forge:8080/v1/_acme-challenge" block-time: 1 blockchain-rpc-endpoint: "ws://geth-swap:8546" bootnode-mode: false @@ -134,7 +134,7 @@ bee-configs: p2p-addr: ":1634" p2p-wss-addr: ":1635" p2p-ws-enable: false - p2p-wss-enable: false + p2p-wss-enable: true password: "beekeeper" payment-early-percent: 50 payment-threshold: 13500000 diff --git a/pkg/orchestration/k8s/helpers.go b/pkg/orchestration/k8s/helpers.go index ee720c6a..ae91726d 100644 --- a/pkg/orchestration/k8s/helpers.go +++ b/pkg/orchestration/k8s/helpers.go @@ -87,6 +87,7 @@ type setContainersOptions struct { ImagePullPolicy string PortAPI int32 PortP2P int32 + PortP2PWSS int32 PersistenceEnabled bool ResourcesLimitCPU string ResourcesLimitMemory string @@ -102,18 +103,29 @@ func setContainers(o setContainersOptions) (c containers.Containers) { Image: o.Image, ImagePullPolicy: o.ImagePullPolicy, Command: []string{"bee", "start", "--config=.bee.yaml"}, - Ports: containers.Ports{ - { - Name: "api", - ContainerPort: o.PortAPI, - Protocol: "TCP", - }, - { - Name: "p2p", - ContainerPort: o.PortP2P, - Protocol: "TCP", - }, - }, + Ports: func() containers.Ports { + ports := containers.Ports{ + { + Name: "api", + ContainerPort: o.PortAPI, + Protocol: "TCP", + }, + { + Name: "p2p", + ContainerPort: o.PortP2P, + Protocol: "TCP", + }, + } + // Add p2p-wss port if configured + if o.PortP2PWSS > 0 { + ports = append(ports, containers.Port{ + Name: "p2p-wss", + ContainerPort: o.PortP2PWSS, + Protocol: "TCP", + }) + } + return ports + }(), LivenessProbe: containers.Probe{HTTPGet: &containers.HTTPGetProbe{ InitialDelaySeconds: 5, Handler: containers.HTTPGetHandler{ @@ -263,33 +275,23 @@ func setPersistentVolumeClaims(o setPersistentVolumeClaimsOptions) (pvcs pvc.Per return pvcs } -type setBeeNodePortOptions struct { - AppProtocol string - Name string - Protocol string - TargetPort string - Port int32 - NodePort int32 -} - -func setBeeNodePort(o setBeeNodePortOptions) (ports service.Ports) { - if o.NodePort > 0 { - return service.Ports{{ - AppProtocol: "TCP", - Name: "p2p", - Protocol: "TCP", - Port: o.Port, - TargetPort: "p2p", - Nodeport: o.NodePort, - }} +// createServicePort creates a service port with optional NodePort. +// If targetPort is empty, it defaults to name. +func createServicePort(name string, port int32, targetPort string, nodePort int32) service.Port { + if targetPort == "" { + targetPort = name } - return service.Ports{{ + p := service.Port{ AppProtocol: "TCP", - Name: "p2p", + Name: name, Protocol: "TCP", - Port: o.Port, - TargetPort: "p2p", - }} + Port: port, + TargetPort: targetPort, + } + if nodePort > 0 { + p.Nodeport = nodePort + } + return p } func parsePort(port string) (int32, error) { diff --git a/pkg/orchestration/k8s/orchestrator.go b/pkg/orchestration/k8s/orchestrator.go index 953fea4b..8edf4314 100644 --- a/pkg/orchestration/k8s/orchestrator.go +++ b/pkg/orchestration/k8s/orchestrator.go @@ -196,30 +196,42 @@ func (n *nodeOrchestrator) Create(ctx context.Context, o orchestration.CreateOpt } } - // var nodePortP2PWSS int32 - // if len(o.Config.NATWSSAddr) > 0 { - // nodePortP2PWSS, err = parsePort(o.Config.NATWSSAddr) - // if err != nil { - // return fmt.Errorf("parsing NAT WSS address from config: %w", err) - // } - // } + var portP2PWSS int32 + if len(o.Config.P2PWSSAddr) > 0 { + portP2PWSS, err = parsePort(o.Config.P2PWSSAddr) + if err != nil { + return fmt.Errorf("parsing P2P WSS port from config: %w", err) + } + } + + var nodePortP2PWSS int32 + if len(o.Config.NATWSSAddr) > 0 { + nodePortP2PWSS, err = parsePort(o.Config.NATWSSAddr) + if err != nil { + return fmt.Errorf("parsing NAT WSS address from config: %w", err) + } + } p2pSvc := fmt.Sprintf("%s-p2p", o.Name) + + // Build ports for p2p service + p2pPorts := service.Ports{ + createServicePort("p2p", portP2P, "", nodePortP2P), + } + + // Add p2p-wss port if P2PWSSAddr is configured + if portP2PWSS > 0 { + p2pPorts = append(p2pPorts, createServicePort("p2p-wss", portP2PWSS, "", nodePortP2PWSS)) + } + if _, err := n.k8s.Service.Set(ctx, p2pSvc, o.Namespace, service.Options{ Annotations: o.Annotations, Labels: o.Labels, ServiceSpec: service.Spec{ ExternalTrafficPolicy: "Local", - Ports: setBeeNodePort(setBeeNodePortOptions{ - AppProtocol: "TCP", - Name: "p2p", - Protocol: "TCP", - TargetPort: "p2p", - Port: portP2P, - NodePort: nodePortP2P, - }), - Selector: o.Selector, - Type: "NodePort", + Ports: p2pPorts, + Selector: o.Selector, + Type: "NodePort", }, }); err != nil { return fmt.Errorf("set service in namespace %s: %w", o.Namespace, err) @@ -282,6 +294,7 @@ func (n *nodeOrchestrator) Create(ctx context.Context, o orchestration.CreateOpt ImagePullPolicy: o.ImagePullPolicy, PortAPI: portAPI, PortP2P: portP2P, + PortP2PWSS: portP2PWSS, PersistenceEnabled: o.PersistenceEnabled, ResourcesLimitCPU: o.ResourcesLimitCPU, ResourcesLimitMemory: o.ResourcesLimitMemory, From 42e22f23e9dd7db078f1058428cc98568f50e601 Mon Sep 17 00:00:00 2001 From: Ljubisa Gacevic Date: Thu, 27 Nov 2025 17:14:35 +0100 Subject: [PATCH 03/10] chore: add comments to local.yaml --- config/local.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/local.yaml b/config/local.yaml index 11f3bc89..a84178b7 100644 --- a/config/local.yaml +++ b/config/local.yaml @@ -111,9 +111,9 @@ bee-configs: _inherit: "" allow-private-cidrs: true api-addr: ":1633" - autotls-ca-endpoint: "https://pebble:14000/dir" + autotls-ca-endpoint: "https://pebble:14000/dir" # https://pebble.local.svc.cluster.local:14000/dir autotls-domain: "local.test" - autotls-registration-endpoint: "http://p2p-forge:8080/v1/_acme-challenge" + autotls-registration-endpoint: "http://p2p-forge:8080/v1/_acme-challenge" # http://p2p-forge.local.svc.cluster.local:8080/v1/_acme-challenge block-time: 1 blockchain-rpc-endpoint: "ws://geth-swap:8546" bootnode-mode: false From 8778b3c7735f0c7f7e63ac60f6d9986c43db0b4f Mon Sep 17 00:00:00 2001 From: Ljubisa Gacevic Date: Thu, 8 Jan 2026 10:34:38 +0100 Subject: [PATCH 04/10] fix(config): update autotls-registration-endpoint --- config/local.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/local.yaml b/config/local.yaml index 0e04a04a..7685aa3c 100644 --- a/config/local.yaml +++ b/config/local.yaml @@ -113,7 +113,7 @@ bee-configs: api-addr: ":1633" autotls-ca-endpoint: "https://pebble:14000/dir" # https://pebble.local.svc.cluster.local:14000/dir autotls-domain: "local.test" - autotls-registration-endpoint: "http://p2p-forge:8080/v1/_acme-challenge" # http://p2p-forge.local.svc.cluster.local:8080/v1/_acme-challenge + autotls-registration-endpoint: http://p2p-forge.local.svc.cluster.local # "http://p2p-forge:8080" # http://p2p-forge.local.svc.cluster.local:8080/v1/_acme-challenge #http://p2p-forge:8080/v1/_acme-challenge block-time: 1 blockchain-rpc-endpoint: "ws://geth-swap:8546" bootnode-mode: false From 9a9c822d8b1fe5a102951af16136fd250bbebe0c Mon Sep 17 00:00:00 2001 From: Akrem Chabchoub <121046693+akrem-chabchoub@users.noreply.github.com> Date: Fri, 16 Jan 2026 18:50:23 +0100 Subject: [PATCH 05/10] feat(autotls): add autotls support with pebble and p2p-forge (#559) * feat(autotls): add auto tls support in init containers and configuration options * fix(k8s): fix EnvVar handling by adding hasValues method and improving toK8S conversion * feat(config): add p2p-wss support and update autotls registration endpoint in configuration files * chore(config): remove unneeded p2p-wss settings --- config/local.yaml | 2 +- pkg/k8s/containers/env.go | 17 ++++-- pkg/orchestration/k8s/helpers.go | 78 +++++++++++++++++++++++++-- pkg/orchestration/k8s/orchestrator.go | 7 ++- 4 files changed, 94 insertions(+), 10 deletions(-) diff --git a/config/local.yaml b/config/local.yaml index 7685aa3c..a29fb7df 100644 --- a/config/local.yaml +++ b/config/local.yaml @@ -113,7 +113,7 @@ bee-configs: api-addr: ":1633" autotls-ca-endpoint: "https://pebble:14000/dir" # https://pebble.local.svc.cluster.local:14000/dir autotls-domain: "local.test" - autotls-registration-endpoint: http://p2p-forge.local.svc.cluster.local # "http://p2p-forge:8080" # http://p2p-forge.local.svc.cluster.local:8080/v1/_acme-challenge #http://p2p-forge:8080/v1/_acme-challenge + autotls-registration-endpoint: http://p2p-forge.local.svc.cluster.local:8080 # "http://p2p-forge:8080" # http://p2p-forge.local.svc.cluster.local:8080/v1/_acme-challenge #http://p2p-forge:8080/v1/_acme-challenge block-time: 1 blockchain-rpc-endpoint: "ws://geth-swap:8546" bootnode-mode: false diff --git a/pkg/k8s/containers/env.go b/pkg/k8s/containers/env.go index dd133698..ae2f097f 100644 --- a/pkg/k8s/containers/env.go +++ b/pkg/k8s/containers/env.go @@ -29,16 +29,19 @@ type EnvVar struct { // toK8S converts EnvVar to Kubernetes client object func (ev *EnvVar) toK8S() v1.EnvVar { - return v1.EnvVar{ + envVar := v1.EnvVar{ Name: ev.Name, Value: ev.Value, - ValueFrom: &v1.EnvVarSource{ + } + if ev.ValueFrom.hasValues() { + envVar.ValueFrom = &v1.EnvVarSource{ FieldRef: ev.ValueFrom.Field.toK8S(), ResourceFieldRef: ev.ValueFrom.ResourceField.toK8S(), ConfigMapKeyRef: ev.ValueFrom.ConfigMap.toK8S(), SecretKeyRef: ev.ValueFrom.Secret.toK8S(), - }, + } } + return envVar } // ValueFrom represents Kubernetes ValueFrom @@ -49,6 +52,14 @@ type ValueFrom struct { Secret SecretKey } +// hasValues returns true if any ValueFrom field is configured +func (vf *ValueFrom) hasValues() bool { + return vf.Field.Path != "" || + vf.ResourceField.Resource != "" || + vf.ConfigMap.ConfigMapName != "" || + vf.Secret.SecretName != "" +} + // Field represents Kubernetes ObjectFieldSelector type Field struct { APIVersion string diff --git a/pkg/orchestration/k8s/helpers.go b/pkg/orchestration/k8s/helpers.go index ae91726d..1d745665 100644 --- a/pkg/orchestration/k8s/helpers.go +++ b/pkg/orchestration/k8s/helpers.go @@ -63,7 +63,11 @@ withdrawal-addresses-whitelist: {{.WithdrawAddress}} ` ) -func setInitContainers() (inits containers.Containers) { +type setInitContainersOptions struct { + AutoTLSEnabled bool +} + +func setInitContainers(o setInitContainersOptions) (inits containers.Containers) { inits = append(inits, containers.Container{ Name: "init-bee", Image: "ethersphere/busybox:1.33", @@ -78,6 +82,40 @@ echo 'bee initialization done';`}, }, }) + if o.AutoTLSEnabled { + // Install Pebble CA certificates as an init container. + // Pebble is a testing ACME server (like Let's Encrypt for development). + // The bee container needs to trust Pebble's CA certificates to validate + // TLS certificates issued by Pebble during AutoTLS testing. This init + // container downloads and installs the CA certificates, which are then + // shared with the bee container via the pebble-ca-certs volume mount. + inits = append(inits, containers.Container{ + Name: "install-pebble-ca", + Image: "alpine:latest", + Command: []string{"sh", "-c", `set -ex +apk add --no-cache ca-certificates wget +mkdir -p /certs + +wget -q --no-check-certificate -O /certs/pebble-root.crt https://pebble:15000/roots/0 +wget -q --no-check-certificate -O /certs/pebble-intermediate.crt https://pebble:15000/intermediates/0 || true +wget -q --no-check-certificate -O /certs/pebble-minica.crt https://raw.githubusercontent.com/letsencrypt/pebble/main/test/certs/pebble.minica.pem || true + +cat /certs/*.crt > /certs/pebble-bundle.crt +cp /certs/*.crt /usr/local/share/ca-certificates/ 2>/dev/null || true +update-ca-certificates +cp /etc/ssl/certs/ca-certificates.crt /certs/ca-certificates.crt + +echo "Pebble CA certificates installed successfully" +ls -la /certs/`}, + VolumeMounts: containers.VolumeMounts{ + { + Name: "pebble-ca-certs", + MountPath: "/certs", + }, + }, + }) + } + return inits } @@ -95,6 +133,7 @@ type setContainersOptions struct { ResourcesRequestMemory string LibP2PEnabled bool SwarmEnabled bool + AutoTLSEnabled bool } func setContainers(o setContainersOptions) (c containers.Containers) { @@ -103,6 +142,17 @@ func setContainers(o setContainersOptions) (c containers.Containers) { Image: o.Image, ImagePullPolicy: o.ImagePullPolicy, Command: []string{"bee", "start", "--config=.bee.yaml"}, + Env: func() containers.EnvVars { + if o.AutoTLSEnabled { + return containers.EnvVars{ + { + Name: "SSL_CERT_FILE", + Value: "/etc/ssl/certs/pebble-ca-certificates.crt", + }, + } + } + return nil + }(), Ports: func() containers.Ports { ports := containers.Ports{ { @@ -158,8 +208,9 @@ func setContainers(o setContainersOptions) (c containers.Containers) { RunAsUser: 999, }, VolumeMounts: setBeeVolumeMounts(setBeeVolumeMountsOptions{ - LibP2PEnabled: o.LibP2PEnabled, - SwarmEnabled: o.SwarmEnabled, + LibP2PEnabled: o.LibP2PEnabled, + SwarmEnabled: o.SwarmEnabled, + AutoTLSEnabled: o.AutoTLSEnabled, }), }) @@ -167,8 +218,9 @@ func setContainers(o setContainersOptions) (c containers.Containers) { } type setBeeVolumeMountsOptions struct { - LibP2PEnabled bool - SwarmEnabled bool + LibP2PEnabled bool + SwarmEnabled bool + AutoTLSEnabled bool } func setBeeVolumeMounts(o setBeeVolumeMountsOptions) (volumeMounts containers.VolumeMounts) { @@ -198,6 +250,14 @@ func setBeeVolumeMounts(o setBeeVolumeMountsOptions) (volumeMounts containers.Vo ReadOnly: true, }) } + if o.AutoTLSEnabled { + volumeMounts = append(volumeMounts, containers.VolumeMount{ + Name: "pebble-ca-certs", + MountPath: "/etc/ssl/certs/pebble-ca-certificates.crt", + SubPath: "ca-certificates.crt", + ReadOnly: true, + }) + } return volumeMounts } @@ -208,6 +268,7 @@ type setVolumesOptions struct { PersistenceEnabled bool LibP2PEnabled bool SwarmEnabled bool + AutoTLSEnabled bool } func setVolumes(o setVolumesOptions) (volumes pod.Volumes) { @@ -248,6 +309,13 @@ func setVolumes(o setVolumesOptions) (volumes pod.Volumes) { }, }) } + if o.AutoTLSEnabled { + volumes = append(volumes, pod.Volume{ + EmptyDir: &pod.EmptyDirVolume{ + Name: "pebble-ca-certs", + }, + }) + } return volumes } diff --git a/pkg/orchestration/k8s/orchestrator.go b/pkg/orchestration/k8s/orchestrator.go index 8edf4314..bb5712d0 100644 --- a/pkg/orchestration/k8s/orchestrator.go +++ b/pkg/orchestration/k8s/orchestrator.go @@ -272,6 +272,7 @@ func (n *nodeOrchestrator) Create(ctx context.Context, o orchestration.CreateOpt sSet := o.Name libP2PEnabled := len(o.LibP2PKey) > 0 swarmEnabled := o.SwarmKey != nil + autoTLSEnabled := o.Config.P2PWSSAddr != "" if _, err := n.k8s.StatefulSet.Set(ctx, sSet, o.Namespace, statefulset.Options{ Annotations: o.Annotations, @@ -287,7 +288,9 @@ func (n *nodeOrchestrator) Create(ctx context.Context, o orchestration.CreateOpt Annotations: o.Annotations, Labels: o.Labels, Spec: pod.PodSpec{ - InitContainers: setInitContainers(), + InitContainers: setInitContainers(setInitContainersOptions{ + AutoTLSEnabled: autoTLSEnabled, + }), Containers: setContainers(setContainersOptions{ Name: sSet, Image: o.Image, @@ -302,6 +305,7 @@ func (n *nodeOrchestrator) Create(ctx context.Context, o orchestration.CreateOpt ResourcesRequestMemory: o.ResourcesRequestMemory, LibP2PEnabled: libP2PEnabled, SwarmEnabled: swarmEnabled, + AutoTLSEnabled: autoTLSEnabled, }), NodeSelector: o.NodeSelector, PodSecurityContext: pod.PodSecurityContext{ @@ -316,6 +320,7 @@ func (n *nodeOrchestrator) Create(ctx context.Context, o orchestration.CreateOpt PersistenceEnabled: o.PersistenceEnabled, LibP2PEnabled: libP2PEnabled, SwarmEnabled: swarmEnabled, + AutoTLSEnabled: autoTLSEnabled, }), }, }, From 66313ba7e93bcd73976079ef8c9ada831f9ea258 Mon Sep 17 00:00:00 2001 From: Akrem Chabchoub <121046693+akrem-chabchoub@users.noreply.github.com> Date: Mon, 16 Feb 2026 14:10:10 +0100 Subject: [PATCH 06/10] feat(autotls): implement autotls check functionality (#560) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(autotls): implement autotls check functionality * feat(autotls): enhance WSS connectivity checks and update configuration * feat(autotls): add autotls check in config.yaml * feat(config): add WSS configuration for local setup * fix(autotls): ensure context cancellation is handled correctly during WSS connection tests * fix(autotls): update WSS connectivity test to disconnect everything before connecting * fix(autotls): add WSS group option and enhance error handling for WSS connectivity * feat(autotls): add certificate renewal testing options and enhance configuration * refactor(autotls): simplify certificate renewal options and update related configurations * fix(autotls): increase certificate renewal wait time to 500 seconds for improved testing * fix(autotls): extend timeout for certificate renewal to 15 minutes * feat(autotls): add ultralight group support and connectivity testing * feat(config): update local.yaml for ultralight configuration and enhance autotls checks * fix(autotls): re-enable certificate renewal test * feat(config): add local-dns-autotls and local-light-autotls configura… (#562) * feat(config): add local-dns-autotls and local-light-autotls configurations ons. * chore(config): comment out ultralight configuration * fix(config): update autotls configs --- config/config.yaml | 4 +- config/local.yaml | 82 ++++++++++- go.mod | 2 +- pkg/bee/api/node.go | 18 +++ pkg/bee/client.go | 24 ++++ pkg/check/autotls/autotls.go | 270 +++++++++++++++++++++++++++++++++++ pkg/config/check.go | 20 +++ 7 files changed, 417 insertions(+), 3 deletions(-) create mode 100644 pkg/check/autotls/autotls.go diff --git a/config/config.yaml b/config/config.yaml index 9666b09a..af8a0555 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -399,7 +399,9 @@ checks: postage-depth: 21 postage-label: test-label type: feed - + autotls: + timeout: 5m + type: autotls # simulations defines simulations Beekeeper can execute against the cluster # type filed allows defining same simulation with different names and options simulations: diff --git a/config/local.yaml b/config/local.yaml index a29fb7df..4de64953 100644 --- a/config/local.yaml +++ b/config/local.yaml @@ -25,6 +25,38 @@ clusters: config: local count: 2 mode: node + local-dns-autotls: + _inherit: "local" + node-groups: + bootnode: + mode: bootnode + bee-config: bootnode-local-dns-autotls + config: local-dns-autotls + nodes: + - name: bootnode-0 + bootnodes: /dns4/bootnode-0-headless.%s.svc.cluster.local/tcp/1634/p2p/QmaHzvd3iZduu275CMkMVZKwbsjXSyH3GJRj4UvFJApKcb + libp2p-key: '{"address":"28678fe31f09f722d53e77ca2395569f19959fa5","crypto":{"cipher":"aes-128-ctr","ciphertext":"0ff319684c4f8decf9c998047febe3417cfc45832b8bb62fd818183d54cf5d0183bfa021ed95addce3b33e83ce7ee73e926f00eea8241d96b349266a4d299829d3d22db0d536315b52b34db4a6778bfd3ce7631ad7256ea0bb9c50abea9de35d740b6fdc50caf929b1d19494690d9ed649105d02c14f5ec49d","cipherparams":{"iv":"4e9a50fb5852b5e61964f696be78066b"},"kdf":"scrypt","kdfparams":{"n":32768,"r":8,"p":1,"dklen":32,"salt":"4d513e81647e4150bb648ed8d2dda28d460802336bf24d620119eac66ae0c0c4"},"mac":"9ae71db96e5ddc1c214538d42082212bbbe53aeac09fcc3e3a8eff815648331e"},"version":3,"id":"ae3bc991-d89f-405a-9e6a-60e27347e22d"}' + swarm-key: '{"address":"f176839c150e52fe30e5c2b5c648465c6fdfa532","crypto":{"cipher":"aes-128-ctr","ciphertext":"352af096f0fca9dfbd20a6861bde43d988efe7f179e0a9ffd812a285fdcd63b9","cipherparams":{"iv":"613003f1f1bf93430c92629da33f8828"},"kdf":"scrypt","kdfparams":{"n":32768,"r":8,"p":1,"dklen":32,"salt":"ad1d99a4c64c95c26131e079e8c8a82221d58bf66a7ceb767c33a4c376c564b8"},"mac":"cafda1bc8ca0ffc2b22eb69afd1cf5072fd09412243443be1b0c6832f57924b6"},"version":3}' + bee: + bee-config: bee-local-dns-autotls + config: local-dns-autotls + count: 3 + mode: node + light: + bee-config: bee-local-light-autotls + config: local-light-autotls + count: 2 + mode: node + wss: + bee-config: bee-local-wss + config: local + count: 2 + mode: node + ultralight: + bee-config: bee-local-ultralight + config: local-ultralight + count: 1 + mode: node local-dns: _inherit: "local" node-groups: @@ -42,11 +74,21 @@ clusters: config: local-dns count: 3 mode: node + wss: + bee-config: bee-local-wss + config: local-dns + count: 2 + mode: node light: bee-config: bee-local-light config: local-light count: 2 mode: node + # ultralight: + # bee-config: bee-local-ultralight + # config: local-ultralight + # count: 2 + # mode: node local-gc: _inherit: "local" node-groups: @@ -100,13 +142,37 @@ node-groups: update-strategy: "RollingUpdate" local-dns: _inherit: "local" + local-dns-autotls: + _inherit: "local" + local-light-autotls: + _inherit: "local-light" local-gc: _inherit: "local" local-light: _inherit: "local" + local-ultralight: + _inherit: "local" + labels: + app.kubernetes.io/component: "node" + app.kubernetes.io/name: "bee" + app.kubernetes.io/part-of: "bee" + app.kubernetes.io/version: "latest" + beekeeper.ethswarm.org/node-funder: "false" # bee-configs defines Bee configuration that can be assigned to node-groups bee-configs: + bootnode-local-dns-autotls: + _inherit: "bee-local-dns-autotls" + bootnode-mode: true + bee-local-dns-autotls: + _inherit: "bee-local" + bootnode: /dnsaddr/bootnode-0-headless.local.svc.cluster.local + p2p-wss-enable: true + bee-local-light-autotls: + _inherit: "bee-local" + bootnode: /dnsaddr/bootnode-0-headless.local.svc.cluster.local + full-node: false + p2p-wss-enable: true bee-local: _inherit: "" allow-private-cidrs: true @@ -134,7 +200,7 @@ bee-configs: p2p-addr: ":1634" p2p-wss-addr: ":1635" p2p-ws-enable: false - p2p-wss-enable: true + p2p-wss-enable: false password: "beekeeper" payment-early-percent: 50 payment-threshold: 13500000 @@ -157,6 +223,9 @@ bee-configs: bootnode-local: _inherit: "bee-local" bootnode-mode: true + bee-local-wss: + _inherit: "bee-local" + p2p-wss-enable: true bee-local-dns: _inherit: "bee-local" bootnode: /dnsaddr/localhost @@ -171,6 +240,11 @@ bee-configs: bee-local-gc: _inherit: "bee-local" cache-capacity: 10 + bee-local-ultralight: + _inherit: "bee-local" + blockchain-rpc-endpoint: "" + full-node: false + swap-enable: false bootnode-local-gc: _inherit: "bee-local" cache-capacity: 10 @@ -398,3 +472,9 @@ checks: postage-depth: 21 postage-label: test-label type: feed + ci-autotls: + timeout: 15m + type: autotls + options: + ultralight-group: ultralight + wss-group: wss diff --git a/go.mod b/go.mod index ea704f2f..76f69b8d 100644 --- a/go.mod +++ b/go.mod @@ -16,6 +16,7 @@ require ( github.com/go-git/go-git/v5 v5.13.2 github.com/google/uuid v1.6.0 github.com/gorilla/websocket v1.5.1 + github.com/multiformats/go-multiaddr v0.12.3 github.com/opentracing/opentracing-go v1.2.0 github.com/prometheus/client_golang v1.21.1 github.com/prometheus/common v0.62.0 @@ -94,7 +95,6 @@ require ( github.com/mr-tron/base58 v1.2.0 // indirect github.com/multiformats/go-base32 v0.1.0 // indirect github.com/multiformats/go-base36 v0.2.0 // indirect - github.com/multiformats/go-multiaddr v0.12.3 // indirect github.com/multiformats/go-multiaddr-dns v0.3.1 // indirect github.com/multiformats/go-multibase v0.2.0 // indirect github.com/multiformats/go-multicodec v0.9.0 // indirect diff --git a/pkg/bee/api/node.go b/pkg/bee/api/node.go index de9423bc..789eeccc 100644 --- a/pkg/bee/api/node.go +++ b/pkg/bee/api/node.go @@ -122,6 +122,24 @@ func (n *NodeService) Peers(ctx context.Context) (resp Peers, err error) { return resp, err } +// ConnectResponse represents the response from the connect endpoint +type ConnectResponse struct { + Address string `json:"address"` +} + +// Connect connects to a peer using the provided multiaddress. +// The multiaddr should be in the format: /ip4/x.x.x.x/tcp/port/... +// Returns the overlay address of the connected peer. +func (n *NodeService) Connect(ctx context.Context, multiaddr string) (resp ConnectResponse, err error) { + err = n.client.requestJSON(ctx, http.MethodPost, "/connect"+multiaddr, nil, &resp) + return resp, err +} + +// Disconnect disconnects from a peer with the given overlay address. +func (n *NodeService) Disconnect(ctx context.Context, overlay swarm.Address) error { + return n.client.requestJSON(ctx, http.MethodDelete, "/peers/"+overlay.String(), nil, nil) +} + // Readiness represents node's readiness type Readiness struct { Status string `json:"status"` diff --git a/pkg/bee/client.go b/pkg/bee/client.go index d1404c50..66cbf4d2 100644 --- a/pkg/bee/client.go +++ b/pkg/bee/client.go @@ -318,6 +318,30 @@ func (c *Client) Peers(ctx context.Context) (peers []swarm.Address, err error) { return peers, err } +// Connect connects to a peer using the provided multiaddress. +// Returns the overlay address of the connected peer. +func (c *Client) Connect(ctx context.Context, multiaddr string) (swarm.Address, error) { + resp, err := c.api.Node.Connect(ctx, multiaddr) + if err != nil { + return swarm.ZeroAddress, fmt.Errorf("connect to %s: %w", multiaddr, err) + } + + addr, err := swarm.ParseHexAddress(resp.Address) + if err != nil { + return swarm.ZeroAddress, fmt.Errorf("parse overlay address %s: %w", resp.Address, err) + } + + return addr, nil +} + +// Disconnect disconnects from a peer with the given overlay address. +func (c *Client) Disconnect(ctx context.Context, overlay swarm.Address) error { + if err := c.api.Node.Disconnect(ctx, overlay); err != nil { + return fmt.Errorf("disconnect from %s: %w", overlay, err) + } + return nil +} + // PinRootHash pins root hash of given reference. func (c *Client) PinRootHash(ctx context.Context, ref swarm.Address) error { return c.api.Pinning.PinRootHash(ctx, ref) diff --git a/pkg/check/autotls/autotls.go b/pkg/check/autotls/autotls.go new file mode 100644 index 00000000..f808141a --- /dev/null +++ b/pkg/check/autotls/autotls.go @@ -0,0 +1,270 @@ +package autotls + +import ( + "context" + "fmt" + "time" + + "github.com/ethersphere/beekeeper/pkg/bee" + "github.com/ethersphere/beekeeper/pkg/beekeeper" + "github.com/ethersphere/beekeeper/pkg/logging" + "github.com/ethersphere/beekeeper/pkg/orchestration" + ma "github.com/multiformats/go-multiaddr" +) + +type Options struct { + WSSGroup string + UltraLightGroup string + ConnectTimeout time.Duration +} + +func NewDefaultOptions() Options { + return Options{ + WSSGroup: "wss", + UltraLightGroup: "ultralight", + ConnectTimeout: 30 * time.Second, + } +} + +var _ beekeeper.Action = (*Check)(nil) + +type Check struct { + logger logging.Logger +} + +func NewCheck(logger logging.Logger) beekeeper.Action { + return &Check{ + logger: logger, + } +} + +func (c *Check) Run(ctx context.Context, cluster orchestration.Cluster, opts any) error { + o, ok := opts.(Options) + if !ok { + return fmt.Errorf("invalid options type") + } + + c.logger.Info("starting AutoTLS check") + + clients, err := cluster.NodesClients(ctx) + if err != nil { + return fmt.Errorf("get node clients: %w", err) + } + time.Sleep(5 * time.Second) + wssClients := orchestration.ClientMap(clients).FilterByNodeGroups([]string{o.WSSGroup}) + if len(wssClients) == 0 { + return fmt.Errorf("no nodes found in WSS group %q", o.WSSGroup) + } + + c.logger.Infof("found %d nodes in WSS group %q", len(wssClients), o.WSSGroup) + + wssNodes, err := c.verifyWSSUnderlays(ctx, wssClients, o.UltraLightGroup) + if err != nil { + return fmt.Errorf("verify WSS underlays: %w", err) + } + + if err := c.testWSSConnectivity(ctx, clients, wssNodes, o.ConnectTimeout); err != nil { + return fmt.Errorf("WSS connectivity test: %w", err) + } + + if o.UltraLightGroup != "" { + if err := c.testUltraLightConnectivity(ctx, clients, wssNodes, o.UltraLightGroup, o.ConnectTimeout); err != nil { + return fmt.Errorf("ultra-light connectivity test: %w", err) + } + } + + if err := c.testCertificateRenewal(ctx, clients, wssNodes, o.ConnectTimeout); err != nil { + return fmt.Errorf("certificate renewal test: %w", err) + } + + c.logger.Info("AutoTLS check completed successfully") + return nil +} + +func (c *Check) verifyWSSUnderlays(ctx context.Context, wssClients orchestration.ClientList, excludeNodeGroup string) (map[string][]string, error) { + wssNodes := make(map[string][]string) + + for _, client := range wssClients { + if excludeNodeGroup != "" && client.NodeGroup() == excludeNodeGroup { + c.logger.Debugf("skipping %s (node group %s has no WSS underlays)", client.Name(), excludeNodeGroup) + continue + } + + nodeName := client.Name() + addresses, err := client.Addresses(ctx) + if err != nil { + return nil, fmt.Errorf("%s: get addresses: %w", nodeName, err) + } + time.Sleep(2 * time.Second) + wssUnderlays := filterWSSUnderlays(addresses.Underlay) + if len(wssUnderlays) == 0 { + return nil, fmt.Errorf("node %s in WSS group has no WSS underlay addresses", nodeName) + } + + wssNodes[nodeName] = wssUnderlays + c.logger.Debugf("node %s has %d WSS underlay(s)", nodeName, len(wssUnderlays)) + } + + return wssNodes, nil +} + +func filterWSSUnderlays(underlays []string) []string { + var wss []string + for _, u := range underlays { + maddr, err := ma.NewMultiaddr(u) + if err != nil { + continue + } + if _, err := maddr.ValueForProtocol(ma.P_TLS); err != nil { + continue + } + if _, err := maddr.ValueForProtocol(ma.P_WS); err != nil { + continue + } + wss = append(wss, u) + } + return wss +} + +func (c *Check) testWSSConnectivity(ctx context.Context, clients map[string]*bee.Client, wssNodes map[string][]string, timeout time.Duration) error { + var nonWSSSource *bee.Client + var nonWSSName string + var wssSource *bee.Client + var wssSourceName string + + for name, client := range clients { + if _, hasWSS := wssNodes[name]; hasWSS { + if wssSource == nil { + wssSource = client + wssSourceName = name + } + } else { + if nonWSSSource == nil { + nonWSSSource = client + nonWSSName = name + } + } + } + + if nonWSSSource != nil { + c.logger.Infof("testing cross-protocol: %s (non-WSS) to WSS nodes", nonWSSName) + if err := c.testConnectivity(ctx, nonWSSSource, nonWSSName, clients, wssNodes, timeout); err != nil { + return fmt.Errorf("cross-protocol test: %w", err) + } + } else { + c.logger.Warning("no non-WSS nodes available, skipping cross-protocol test") + } + + if wssSource != nil { + c.logger.Infof("testing WSS-to-WSS: %s to WSS nodes", wssSourceName) + if err := c.testConnectivity(ctx, wssSource, wssSourceName, clients, wssNodes, timeout); err != nil { + return fmt.Errorf("WSS-to-WSS test: %w", err) + } + } else { + c.logger.Warning("no WSS source nodes available, skipping WSS-to-WSS test") + } + + return nil +} + +func (c *Check) testUltraLightConnectivity(ctx context.Context, clients map[string]*bee.Client, wssNodes map[string][]string, ultraLightGroup string, timeout time.Duration) error { + ultralightClients := orchestration.ClientMap(clients).FilterByNodeGroups([]string{ultraLightGroup}) + if len(ultralightClients) == 0 { + c.logger.Warningf("no nodes found in ultra-light group %q, skipping ultra-light connectivity test", ultraLightGroup) + return nil + } + + c.logger.Infof("found %d nodes in ultra-light group %q", len(ultralightClients), ultraLightGroup) + + for _, client := range ultralightClients { + nodeName := client.Name() + c.logger.Infof("testing ultra-light to WSS: %s (no listen addr) to WSS nodes", nodeName) + if err := c.testConnectivity(ctx, client, nodeName, clients, wssNodes, timeout); err != nil { + return fmt.Errorf("ultra-light %s to WSS test: %w", nodeName, err) + } + } + + return nil +} + +func (c *Check) testConnectivity(ctx context.Context, sourceClient *bee.Client, sourceName string, clients map[string]*bee.Client, wssNodes map[string][]string, timeout time.Duration) error { + for targetName, underlays := range wssNodes { + if targetName == sourceName { + continue + } + + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + + targetClient := clients[targetName] + targetAddresses, err := targetClient.Addresses(ctx) + if err != nil { + return fmt.Errorf("get target %s addresses: %w", targetName, err) + } + targetOverlay := targetAddresses.Overlay + + // Disconnect first to ensure we test actual WSS connection. + // Bee returns 200 OK for both new connections and existing ones, + // so we must disconnect first to guarantee WSS transport is used. + c.logger.Infof("disconnecting from %s before WSS test", targetName) + if err := sourceClient.Disconnect(ctx, targetOverlay); err != nil { + c.logger.Warningf("failed to disconnect from %s: %v", targetName, err) + } + + time.Sleep(500 * time.Millisecond) + + for _, underlay := range underlays { + c.logger.Infof("testing WSS connection from %s to %s via %s", sourceName, targetName, underlay) + + connectCtx, cancel := context.WithTimeout(ctx, timeout) + start := time.Now() + + overlay, err := sourceClient.Connect(connectCtx, underlay) + duration := time.Since(start) + cancel() + + if err != nil { + return fmt.Errorf("WSS connection failed from %s to %s via %s: %w", sourceName, targetName, underlay, err) + } + + c.logger.Infof("WSS connection successful: %s to %s (overlay: %s, duration: %v)", + sourceName, targetName, overlay, duration) + + if !overlay.Equal(targetOverlay) { + return fmt.Errorf("overlay mismatch: expected %s, got %s", targetOverlay, overlay) + } + + if err := sourceClient.Disconnect(ctx, overlay); err != nil { + c.logger.Warningf("failed to disconnect from %s: %v", targetName, err) + } + + time.Sleep(500 * time.Millisecond) + } + } + + return nil +} + +func (c *Check) testCertificateRenewal(ctx context.Context, clients map[string]*bee.Client, wssNodes map[string][]string, connectTimeout time.Duration) error { + const renewalWaitTime = 500 * time.Second // This is configured in beelocal setup (we set certificate to expire in 300 seconds) + + c.logger.Infof("testing certificate renewal: waiting %v then re-testing connectivity", renewalWaitTime) + + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(renewalWaitTime): + } + + c.logger.Info("wait complete, re-testing WSS connectivity to verify certificates were renewed") + + if err := c.testWSSConnectivity(ctx, clients, wssNodes, connectTimeout); err != nil { + return fmt.Errorf("post-renewal connectivity test failed (certificates may not have been renewed): %w", err) + } + + c.logger.Info("certificate renewal test passed: WSS connectivity still works after wait period") + return nil +} diff --git a/pkg/config/check.go b/pkg/config/check.go index b130c6e4..5543d172 100644 --- a/pkg/config/check.go +++ b/pkg/config/check.go @@ -8,6 +8,7 @@ import ( "github.com/ethersphere/beekeeper/pkg/beekeeper" "github.com/ethersphere/beekeeper/pkg/check/act" + "github.com/ethersphere/beekeeper/pkg/check/autotls" "github.com/ethersphere/beekeeper/pkg/check/balances" "github.com/ethersphere/beekeeper/pkg/check/cashout" "github.com/ethersphere/beekeeper/pkg/check/datadurability" @@ -82,6 +83,25 @@ var Checks = map[string]CheckType{ return opts, nil }, }, + "autotls": { + NewAction: autotls.NewCheck, + NewOptions: func(checkGlobalConfig CheckGlobalConfig, check Check) (any, error) { + checkOpts := new(struct { + WSSGroup *string `yaml:"wss-group"` + UltraLightGroup *string `yaml:"ultralight-group"` + ConnectTimeout *time.Duration `yaml:"connect-timeout"` + }) + if err := check.Options.Decode(checkOpts); err != nil { + return nil, fmt.Errorf("decoding check %s options: %w", check.Type, err) + } + opts := autotls.NewDefaultOptions() + + if err := applyCheckConfig(checkGlobalConfig, checkOpts, &opts); err != nil { + return nil, fmt.Errorf("applying options: %w", err) + } + return opts, nil + }, + }, "balances": { NewAction: balances.NewCheck, NewOptions: func(checkGlobalConfig CheckGlobalConfig, check Check) (any, error) { From 8d3d9490e3df9b47a0b3f700b113516a4f27ca76 Mon Sep 17 00:00:00 2001 From: Ljubisa Gacevic Date: Mon, 16 Feb 2026 16:19:22 +0100 Subject: [PATCH 07/10] fix(autotls): simplify config --- config/config.yaml | 1 + config/local.yaml | 107 ++++++++++++----------------------- pkg/check/autotls/autotls.go | 27 ++++----- pkg/config/check.go | 4 +- 4 files changed, 53 insertions(+), 86 deletions(-) diff --git a/config/config.yaml b/config/config.yaml index af8a0555..ba5820aa 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -402,6 +402,7 @@ checks: autotls: timeout: 5m type: autotls + # simulations defines simulations Beekeeper can execute against the cluster # type filed allows defining same simulation with different names and options simulations: diff --git a/config/local.yaml b/config/local.yaml index 4de64953..e372b76b 100644 --- a/config/local.yaml +++ b/config/local.yaml @@ -25,70 +25,55 @@ clusters: config: local count: 2 mode: node - local-dns-autotls: + local-dns: _inherit: "local" node-groups: bootnode: mode: bootnode - bee-config: bootnode-local-dns-autotls - config: local-dns-autotls + bee-config: bootnode-local-dns + config: local-dns nodes: - name: bootnode-0 bootnodes: /dns4/bootnode-0-headless.%s.svc.cluster.local/tcp/1634/p2p/QmaHzvd3iZduu275CMkMVZKwbsjXSyH3GJRj4UvFJApKcb libp2p-key: '{"address":"28678fe31f09f722d53e77ca2395569f19959fa5","crypto":{"cipher":"aes-128-ctr","ciphertext":"0ff319684c4f8decf9c998047febe3417cfc45832b8bb62fd818183d54cf5d0183bfa021ed95addce3b33e83ce7ee73e926f00eea8241d96b349266a4d299829d3d22db0d536315b52b34db4a6778bfd3ce7631ad7256ea0bb9c50abea9de35d740b6fdc50caf929b1d19494690d9ed649105d02c14f5ec49d","cipherparams":{"iv":"4e9a50fb5852b5e61964f696be78066b"},"kdf":"scrypt","kdfparams":{"n":32768,"r":8,"p":1,"dklen":32,"salt":"4d513e81647e4150bb648ed8d2dda28d460802336bf24d620119eac66ae0c0c4"},"mac":"9ae71db96e5ddc1c214538d42082212bbbe53aeac09fcc3e3a8eff815648331e"},"version":3,"id":"ae3bc991-d89f-405a-9e6a-60e27347e22d"}' swarm-key: '{"address":"f176839c150e52fe30e5c2b5c648465c6fdfa532","crypto":{"cipher":"aes-128-ctr","ciphertext":"352af096f0fca9dfbd20a6861bde43d988efe7f179e0a9ffd812a285fdcd63b9","cipherparams":{"iv":"613003f1f1bf93430c92629da33f8828"},"kdf":"scrypt","kdfparams":{"n":32768,"r":8,"p":1,"dklen":32,"salt":"ad1d99a4c64c95c26131e079e8c8a82221d58bf66a7ceb767c33a4c376c564b8"},"mac":"cafda1bc8ca0ffc2b22eb69afd1cf5072fd09412243443be1b0c6832f57924b6"},"version":3}' bee: - bee-config: bee-local-dns-autotls - config: local-dns-autotls + bee-config: bee-local-dns + config: local-dns count: 3 mode: node light: - bee-config: bee-local-light-autotls - config: local-light-autotls - count: 2 - mode: node - wss: - bee-config: bee-local-wss - config: local + bee-config: bee-local-light + config: local-light count: 2 mode: node - ultralight: - bee-config: bee-local-ultralight - config: local-ultralight - count: 1 - mode: node - local-dns: + local-dns-autotls: _inherit: "local" node-groups: bootnode: mode: bootnode - bee-config: bootnode-local-dns - config: local-dns + bee-config: bootnode-local-dns-autotls + config: local nodes: - name: bootnode-0 bootnodes: /dns4/bootnode-0-headless.%s.svc.cluster.local/tcp/1634/p2p/QmaHzvd3iZduu275CMkMVZKwbsjXSyH3GJRj4UvFJApKcb libp2p-key: '{"address":"28678fe31f09f722d53e77ca2395569f19959fa5","crypto":{"cipher":"aes-128-ctr","ciphertext":"0ff319684c4f8decf9c998047febe3417cfc45832b8bb62fd818183d54cf5d0183bfa021ed95addce3b33e83ce7ee73e926f00eea8241d96b349266a4d299829d3d22db0d536315b52b34db4a6778bfd3ce7631ad7256ea0bb9c50abea9de35d740b6fdc50caf929b1d19494690d9ed649105d02c14f5ec49d","cipherparams":{"iv":"4e9a50fb5852b5e61964f696be78066b"},"kdf":"scrypt","kdfparams":{"n":32768,"r":8,"p":1,"dklen":32,"salt":"4d513e81647e4150bb648ed8d2dda28d460802336bf24d620119eac66ae0c0c4"},"mac":"9ae71db96e5ddc1c214538d42082212bbbe53aeac09fcc3e3a8eff815648331e"},"version":3,"id":"ae3bc991-d89f-405a-9e6a-60e27347e22d"}' swarm-key: '{"address":"f176839c150e52fe30e5c2b5c648465c6fdfa532","crypto":{"cipher":"aes-128-ctr","ciphertext":"352af096f0fca9dfbd20a6861bde43d988efe7f179e0a9ffd812a285fdcd63b9","cipherparams":{"iv":"613003f1f1bf93430c92629da33f8828"},"kdf":"scrypt","kdfparams":{"n":32768,"r":8,"p":1,"dklen":32,"salt":"ad1d99a4c64c95c26131e079e8c8a82221d58bf66a7ceb767c33a4c376c564b8"},"mac":"cafda1bc8ca0ffc2b22eb69afd1cf5072fd09412243443be1b0c6832f57924b6"},"version":3}' bee: - bee-config: bee-local-dns - config: local-dns + bee-config: bee-local-autotls + config: local count: 3 mode: node - wss: - bee-config: bee-local-wss - config: local-dns + bee-autotls: + bee-config: bee-local-autotls + config: local count: 2 mode: node - light: - bee-config: bee-local-light - config: local-light - count: 2 + ultra-light: + bee-config: bee-local-ultralight-autotls + config: local + count: 1 mode: node - # ultralight: - # bee-config: bee-local-ultralight - # config: local-ultralight - # count: 2 - # mode: node local-gc: _inherit: "local" node-groups: @@ -142,44 +127,20 @@ node-groups: update-strategy: "RollingUpdate" local-dns: _inherit: "local" - local-dns-autotls: - _inherit: "local" - local-light-autotls: - _inherit: "local-light" local-gc: _inherit: "local" local-light: _inherit: "local" - local-ultralight: - _inherit: "local" - labels: - app.kubernetes.io/component: "node" - app.kubernetes.io/name: "bee" - app.kubernetes.io/part-of: "bee" - app.kubernetes.io/version: "latest" - beekeeper.ethswarm.org/node-funder: "false" # bee-configs defines Bee configuration that can be assigned to node-groups bee-configs: - bootnode-local-dns-autotls: - _inherit: "bee-local-dns-autotls" - bootnode-mode: true - bee-local-dns-autotls: - _inherit: "bee-local" - bootnode: /dnsaddr/bootnode-0-headless.local.svc.cluster.local - p2p-wss-enable: true - bee-local-light-autotls: - _inherit: "bee-local" - bootnode: /dnsaddr/bootnode-0-headless.local.svc.cluster.local - full-node: false - p2p-wss-enable: true bee-local: _inherit: "" allow-private-cidrs: true api-addr: ":1633" - autotls-ca-endpoint: "https://pebble:14000/dir" # https://pebble.local.svc.cluster.local:14000/dir + autotls-ca-endpoint: "https://pebble:14000/dir" autotls-domain: "local.test" - autotls-registration-endpoint: http://p2p-forge.local.svc.cluster.local:8080 # "http://p2p-forge:8080" # http://p2p-forge.local.svc.cluster.local:8080/v1/_acme-challenge #http://p2p-forge:8080/v1/_acme-challenge + autotls-registration-endpoint: http://p2p-forge.local.svc.cluster.local:8080 block-time: 1 blockchain-rpc-endpoint: "ws://geth-swap:8546" bootnode-mode: false @@ -219,13 +180,22 @@ bee-configs: warmup-time: 0s welcome-message: "Welcome to the Swarm, this is a local cluster!" withdrawal-addresses-whitelist: "0xec44cb15b1b033e74d55ac5d0e24d861bde54532" - + bootnode-local-dns-autotls: + _inherit: "bee-local-dns" + bootnode-mode: true + p2p-wss-enable: true + bee-local-autotls: + _inherit: "bee-local-dns" + bootnode: /dnsaddr/bootnode-0-headless.local.svc.cluster.local + p2p-wss-enable: true + bee-local-ultralight-autotls: + _inherit: "bee-local-dns" + full-node: false + p2p-wss-enable: true + blockchain-rpc-endpoint: # ultralight nodes don't connect to the blockchain bootnode-local: _inherit: "bee-local" bootnode-mode: true - bee-local-wss: - _inherit: "bee-local" - p2p-wss-enable: true bee-local-dns: _inherit: "bee-local" bootnode: /dnsaddr/localhost @@ -240,11 +210,6 @@ bee-configs: bee-local-gc: _inherit: "bee-local" cache-capacity: 10 - bee-local-ultralight: - _inherit: "bee-local" - blockchain-rpc-endpoint: "" - full-node: false - swap-enable: false bootnode-local-gc: _inherit: "bee-local" cache-capacity: 10 @@ -476,5 +441,5 @@ checks: timeout: 15m type: autotls options: - ultralight-group: ultralight - wss-group: wss + ultra-light-group: ultra-light + autotls-group: bee-autotls diff --git a/pkg/check/autotls/autotls.go b/pkg/check/autotls/autotls.go index f808141a..92a44044 100644 --- a/pkg/check/autotls/autotls.go +++ b/pkg/check/autotls/autotls.go @@ -13,15 +13,15 @@ import ( ) type Options struct { - WSSGroup string + AutoTLSGroup string UltraLightGroup string ConnectTimeout time.Duration } func NewDefaultOptions() Options { return Options{ - WSSGroup: "wss", - UltraLightGroup: "ultralight", + AutoTLSGroup: "bee-autotls", + UltraLightGroup: "ultra-light", ConnectTimeout: 30 * time.Second, } } @@ -50,15 +50,16 @@ func (c *Check) Run(ctx context.Context, cluster orchestration.Cluster, opts any if err != nil { return fmt.Errorf("get node clients: %w", err) } + time.Sleep(5 * time.Second) - wssClients := orchestration.ClientMap(clients).FilterByNodeGroups([]string{o.WSSGroup}) - if len(wssClients) == 0 { - return fmt.Errorf("no nodes found in WSS group %q", o.WSSGroup) + autoTLSClients := orchestration.ClientMap(clients).FilterByNodeGroups([]string{o.AutoTLSGroup}) + if len(autoTLSClients) == 0 { + return fmt.Errorf("no nodes found in AutoTLS group %q", o.AutoTLSGroup) } - c.logger.Infof("found %d nodes in WSS group %q", len(wssClients), o.WSSGroup) + c.logger.Infof("found %d nodes in AutoTLS group %q", len(autoTLSClients), o.AutoTLSGroup) - wssNodes, err := c.verifyWSSUnderlays(ctx, wssClients, o.UltraLightGroup) + wssNodes, err := c.verifyWSSUnderlays(ctx, autoTLSClients, o.UltraLightGroup) if err != nil { return fmt.Errorf("verify WSS underlays: %w", err) } @@ -81,10 +82,10 @@ func (c *Check) Run(ctx context.Context, cluster orchestration.Cluster, opts any return nil } -func (c *Check) verifyWSSUnderlays(ctx context.Context, wssClients orchestration.ClientList, excludeNodeGroup string) (map[string][]string, error) { - wssNodes := make(map[string][]string) +func (c *Check) verifyWSSUnderlays(ctx context.Context, autoTLSClients orchestration.ClientList, excludeNodeGroup string) (map[string][]string, error) { + autoTLS := make(map[string][]string) - for _, client := range wssClients { + for _, client := range autoTLSClients { if excludeNodeGroup != "" && client.NodeGroup() == excludeNodeGroup { c.logger.Debugf("skipping %s (node group %s has no WSS underlays)", client.Name(), excludeNodeGroup) continue @@ -101,11 +102,11 @@ func (c *Check) verifyWSSUnderlays(ctx context.Context, wssClients orchestration return nil, fmt.Errorf("node %s in WSS group has no WSS underlay addresses", nodeName) } - wssNodes[nodeName] = wssUnderlays + autoTLS[nodeName] = wssUnderlays c.logger.Debugf("node %s has %d WSS underlay(s)", nodeName, len(wssUnderlays)) } - return wssNodes, nil + return autoTLS, nil } func filterWSSUnderlays(underlays []string) []string { diff --git a/pkg/config/check.go b/pkg/config/check.go index 5543d172..827817c4 100644 --- a/pkg/config/check.go +++ b/pkg/config/check.go @@ -87,8 +87,8 @@ var Checks = map[string]CheckType{ NewAction: autotls.NewCheck, NewOptions: func(checkGlobalConfig CheckGlobalConfig, check Check) (any, error) { checkOpts := new(struct { - WSSGroup *string `yaml:"wss-group"` - UltraLightGroup *string `yaml:"ultralight-group"` + AutoTLSGroup *string `yaml:"autotls-group"` + UltraLightGroup *string `yaml:"ultra-light-group"` ConnectTimeout *time.Duration `yaml:"connect-timeout"` }) if err := check.Options.Decode(checkOpts); err != nil { From 307bd206a940071a87495ad1001f675145a4198e Mon Sep 17 00:00:00 2001 From: Ljubisa Gacevic Date: Tue, 17 Feb 2026 21:21:43 +0100 Subject: [PATCH 08/10] fix: use test cert from constant --- pkg/orchestration/k8s/helpers.go | 50 ++++++++++++++++++++------------ 1 file changed, 32 insertions(+), 18 deletions(-) diff --git a/pkg/orchestration/k8s/helpers.go b/pkg/orchestration/k8s/helpers.go index 1d745665..adc53e81 100644 --- a/pkg/orchestration/k8s/helpers.go +++ b/pkg/orchestration/k8s/helpers.go @@ -1,6 +1,7 @@ package k8s import ( + "fmt" "maps" "strconv" "strings" @@ -63,6 +64,28 @@ withdrawal-addresses-whitelist: {{.WithdrawAddress}} ` ) +// https://raw.githubusercontent.com/letsencrypt/pebble/main/test/certs/pebble.minica.pem +const pebbleCertificate = `-----BEGIN CERTIFICATE----- +MIIDPzCCAiegAwIBAgIIU0Xm9UFdQxUwDQYJKoZIhvcNAQELBQAwIDEeMBwGA1UE +AxMVbWluaWNhIHJvb3QgY2EgNTM0NWU2MCAXDTI1MDkwMzIzNDAwNVoYDzIxMjUw +OTAzMjM0MDA1WjAgMR4wHAYDVQQDExVtaW5pY2Egcm9vdCBjYSA1MzQ1ZTYwggEi +MA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC5WgZNoVJandj43kkLyU50vzCZ +alozvdRo3OFiKoDtmqKPNWRNO2hC9AUNxTDJco51Yc42u/WV3fPbbhSznTiOOVtn +Ajm6iq4I5nZYltGGZetGDOQWr78y2gWY+SG078MuOO2hyDIiKtVc3xiXYA+8Hluu +9F8KbqSS1h55yxZ9b87eKR+B0zu2ahzBCIHKmKWgc6N13l7aDxxY3D6uq8gtJRU0 +toumyLbdzGcupVvjbjDP11nl07RESDWBLG1/g3ktJvqIa4BWgU2HMh4rND6y8OD3 +Hy3H8MY6CElL+MOCbFJjWqhtOxeFyZZV9q3kYnk9CAuQJKMEGuN4GU6tzhW1AgMB +AAGjezB5MA4GA1UdDwEB/wQEAwIChDATBgNVHSUEDDAKBggrBgEFBQcDATASBgNV +HRMBAf8ECDAGAQH/AgEAMB0GA1UdDgQWBBSu8RGpErgYUoYnQuwCq+/ggTiEjDAf +BgNVHSMEGDAWgBSu8RGpErgYUoYnQuwCq+/ggTiEjDANBgkqhkiG9w0BAQsFAAOC +AQEAXDVYov1+f6EL7S41LhYQkEX/GyNNzsEvqxE9U0+3Iri5JfkcNOiA9O9L6Z+Y +bqcsXV93s3vi4r4WSWuc//wHyJYrVe5+tK4nlFpbJOvfBUtnoBDyKNxXzZCxFJVh +f9uc8UejRfQMFbDbhWY/x83y9BDufJHHq32OjCIN7gp2UR8rnfYvlz7Zg4qkJBsn +DG4dwd+pRTCFWJOVIG0JoNhK3ZmE7oJ1N4H38XkZ31NPcMksKxpsLLIS9+mosZtg +4olL7tMPJklx5ZaeMFaKRDq4Gdxkbw4+O4vRgNm3Z8AXWKknOdfgdpqLUPPhRcP4 +v1lhy71EhBuXXwRQJry0lTdF+w== +-----END CERTIFICATE-----` + type setInitContainersOptions struct { AutoTLSEnabled bool } @@ -82,31 +105,22 @@ echo 'bee initialization done';`}, }, }) + // TODO: this init container is only needed for testing with Pebble + // should not be used in other contexts. if o.AutoTLSEnabled { // Install Pebble CA certificates as an init container. - // Pebble is a testing ACME server (like Let's Encrypt for development). - // The bee container needs to trust Pebble's CA certificates to validate - // TLS certificates issued by Pebble during AutoTLS testing. This init - // container downloads and installs the CA certificates, which are then - // shared with the bee container via the pebble-ca-certs volume mount. inits = append(inits, containers.Container{ Name: "install-pebble-ca", Image: "alpine:latest", - Command: []string{"sh", "-c", `set -ex -apk add --no-cache ca-certificates wget + Command: []string{"sh", "-c", fmt.Sprintf(`set -ex +apk add --no-cache ca-certificates mkdir -p /certs - -wget -q --no-check-certificate -O /certs/pebble-root.crt https://pebble:15000/roots/0 -wget -q --no-check-certificate -O /certs/pebble-intermediate.crt https://pebble:15000/intermediates/0 || true -wget -q --no-check-certificate -O /certs/pebble-minica.crt https://raw.githubusercontent.com/letsencrypt/pebble/main/test/certs/pebble.minica.pem || true - -cat /certs/*.crt > /certs/pebble-bundle.crt -cp /certs/*.crt /usr/local/share/ca-certificates/ 2>/dev/null || true +cat > /certs/pebble-minica.crt << 'CERT' +%s +CERT +cp /certs/pebble-minica.crt /usr/local/share/ca-certificates/ update-ca-certificates -cp /etc/ssl/certs/ca-certificates.crt /certs/ca-certificates.crt - -echo "Pebble CA certificates installed successfully" -ls -la /certs/`}, +cp /etc/ssl/certs/ca-certificates.crt /certs/ca-certificates.crt`, pebbleCertificate)}, VolumeMounts: containers.VolumeMounts{ { Name: "pebble-ca-certs", From 576729966b7ddd4c2f3e8a42910b4de8830dfc1b Mon Sep 17 00:00:00 2001 From: Akrem Chabchoub <121046693+akrem-chabchoub@users.noreply.github.com> Date: Tue, 17 Feb 2026 21:27:05 +0100 Subject: [PATCH 09/10] =?UTF-8?q?fix(autotls):=20handle=20context=20cancel?= =?UTF-8?q?lation=20during=20sleep=20intervals=20in=20c=E2=80=A6=20(#566)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(autotls): handle context cancellation during sleep intervals in checks * feat(autotls): add UnderlayPollInterval option and enhance WSS underlay checks * refactor(config): reorder struct fields for clarity in check options * refactor(autotls): simplify options structure and improve WSS underlay checks --- pkg/check/autotls/autotls.go | 44 ++++++++++++++++++++---------------- pkg/config/check.go | 5 ++-- 2 files changed, 27 insertions(+), 22 deletions(-) diff --git a/pkg/check/autotls/autotls.go b/pkg/check/autotls/autotls.go index 92a44044..95c5a77b 100644 --- a/pkg/check/autotls/autotls.go +++ b/pkg/check/autotls/autotls.go @@ -15,17 +15,20 @@ import ( type Options struct { AutoTLSGroup string UltraLightGroup string - ConnectTimeout time.Duration } func NewDefaultOptions() Options { return Options{ AutoTLSGroup: "bee-autotls", UltraLightGroup: "ultra-light", - ConnectTimeout: 30 * time.Second, } } +const ( + underlayPollInterval = 2 * time.Second + connectTimeout = 30 * time.Second +) + var _ beekeeper.Action = (*Check)(nil) type Check struct { @@ -51,7 +54,6 @@ func (c *Check) Run(ctx context.Context, cluster orchestration.Cluster, opts any return fmt.Errorf("get node clients: %w", err) } - time.Sleep(5 * time.Second) autoTLSClients := orchestration.ClientMap(clients).FilterByNodeGroups([]string{o.AutoTLSGroup}) if len(autoTLSClients) == 0 { return fmt.Errorf("no nodes found in AutoTLS group %q", o.AutoTLSGroup) @@ -64,17 +66,17 @@ func (c *Check) Run(ctx context.Context, cluster orchestration.Cluster, opts any return fmt.Errorf("verify WSS underlays: %w", err) } - if err := c.testWSSConnectivity(ctx, clients, wssNodes, o.ConnectTimeout); err != nil { + if err := c.testWSSConnectivity(ctx, clients, wssNodes, connectTimeout); err != nil { return fmt.Errorf("WSS connectivity test: %w", err) } if o.UltraLightGroup != "" { - if err := c.testUltraLightConnectivity(ctx, clients, wssNodes, o.UltraLightGroup, o.ConnectTimeout); err != nil { + if err := c.testUltraLightConnectivity(ctx, clients, wssNodes, o.UltraLightGroup, connectTimeout); err != nil { return fmt.Errorf("ultra-light connectivity test: %w", err) } } - if err := c.testCertificateRenewal(ctx, clients, wssNodes, o.ConnectTimeout); err != nil { + if err := c.testCertificateRenewal(ctx, clients, wssNodes, connectTimeout); err != nil { return fmt.Errorf("certificate renewal test: %w", err) } @@ -92,14 +94,22 @@ func (c *Check) verifyWSSUnderlays(ctx context.Context, autoTLSClients orchestra } nodeName := client.Name() - addresses, err := client.Addresses(ctx) - if err != nil { - return nil, fmt.Errorf("%s: get addresses: %w", nodeName, err) - } - time.Sleep(2 * time.Second) - wssUnderlays := filterWSSUnderlays(addresses.Underlay) - if len(wssUnderlays) == 0 { - return nil, fmt.Errorf("node %s in WSS group has no WSS underlay addresses", nodeName) + var wssUnderlays []string + for { + addresses, err := client.Addresses(ctx) + if err != nil { + return nil, fmt.Errorf("%s: get addresses: %w", nodeName, err) + } + wssUnderlays = filterWSSUnderlays(addresses.Underlay) + if len(wssUnderlays) > 0 { + break + } + c.logger.Debugf("node %s has no WSS underlays yet, retrying in %v", nodeName, underlayPollInterval) + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(underlayPollInterval): + } } autoTLS[nodeName] = wssUnderlays @@ -215,8 +225,6 @@ func (c *Check) testConnectivity(ctx context.Context, sourceClient *bee.Client, c.logger.Warningf("failed to disconnect from %s: %v", targetName, err) } - time.Sleep(500 * time.Millisecond) - for _, underlay := range underlays { c.logger.Infof("testing WSS connection from %s to %s via %s", sourceName, targetName, underlay) @@ -241,8 +249,6 @@ func (c *Check) testConnectivity(ctx context.Context, sourceClient *bee.Client, if err := sourceClient.Disconnect(ctx, overlay); err != nil { c.logger.Warningf("failed to disconnect from %s: %v", targetName, err) } - - time.Sleep(500 * time.Millisecond) } } @@ -250,7 +256,7 @@ func (c *Check) testConnectivity(ctx context.Context, sourceClient *bee.Client, } func (c *Check) testCertificateRenewal(ctx context.Context, clients map[string]*bee.Client, wssNodes map[string][]string, connectTimeout time.Duration) error { - const renewalWaitTime = 500 * time.Second // This is configured in beelocal setup (we set certificate to expire in 300 seconds) + const renewalWaitTime = 350 * time.Second // This is configured in beelocal setup (we set certificate to expire in 300 seconds) c.logger.Infof("testing certificate renewal: waiting %v then re-testing connectivity", renewalWaitTime) diff --git a/pkg/config/check.go b/pkg/config/check.go index 827817c4..8d94e077 100644 --- a/pkg/config/check.go +++ b/pkg/config/check.go @@ -87,9 +87,8 @@ var Checks = map[string]CheckType{ NewAction: autotls.NewCheck, NewOptions: func(checkGlobalConfig CheckGlobalConfig, check Check) (any, error) { checkOpts := new(struct { - AutoTLSGroup *string `yaml:"autotls-group"` - UltraLightGroup *string `yaml:"ultra-light-group"` - ConnectTimeout *time.Duration `yaml:"connect-timeout"` + AutoTLSGroup *string `yaml:"autotls-group"` + UltraLightGroup *string `yaml:"ultra-light-group"` }) if err := check.Options.Decode(checkOpts); err != nil { return nil, fmt.Errorf("decoding check %s options: %w", check.Type, err) From eef159fc5445c6e1061444b9d5fcd0090a1a97a6 Mon Sep 17 00:00:00 2001 From: akrem-chabchoub Date: Wed, 18 Feb 2026 12:40:43 +0100 Subject: [PATCH 10/10] feat(config): add light node configuration for autotls support in local.yaml --- config/local.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/config/local.yaml b/config/local.yaml index e372b76b..b9c4b194 100644 --- a/config/local.yaml +++ b/config/local.yaml @@ -69,6 +69,11 @@ clusters: config: local count: 2 mode: node + light: + bee-config: bee-local-light-autotls + config: local + count: 2 + mode: node ultra-light: bee-config: bee-local-ultralight-autotls config: local @@ -188,6 +193,9 @@ bee-configs: _inherit: "bee-local-dns" bootnode: /dnsaddr/bootnode-0-headless.local.svc.cluster.local p2p-wss-enable: true + bee-local-light-autotls: + _inherit: "bee-local-light" + p2p-wss-enable: true bee-local-ultralight-autotls: _inherit: "bee-local-dns" full-node: false