OCPBUGS-85414/WIP: fix ote ccm aws e2e resiliency on HCP

.ci-operator.yaml

@@ -1,4 +1,4 @@
 build_root_image:
   name: release
   namespace: openshift
-  tag: rhel-9-release-golang-1.25-openshift-4.22
+  tag: rhel-9-release-golang-1.26-openshift-5.0

Dockerfile

@@ -1,4 +1,4 @@
-FROM registry.ci.openshift.org/ocp/builder:rhel-9-golang-1.25-openshift-4.22 AS builder
+FROM registry.ci.openshift.org/ocp/builder:rhel-9-golang-1.26-openshift-5.0 AS builder
 WORKDIR /go/src/github.com/openshift/cluster-cloud-controller-manager-operator
 COPY . .
 RUN make build &&\

Makefile

@@ -53,7 +53,7 @@ azure-config-credentials-injector:
 cloud-controller-manager-aws-tests-ext:
 	cd openshift-tests/ccm-aws-tests && \
 	mkdir -p ../bin && \
-	go build $(GOGCFLAGS) -o "../bin/cloud-controller-manager-aws-tests-ext" \
+	GOWORK=off go build -mod=vendor $(GOGCFLAGS) -o "../bin/cloud-controller-manager-aws-tests-ext" \
 	      -trimpath -ldflags "$(LD_FLAGS)" .
 
 cluster-cloud-controller-manager-operator-tests-ext:
@@ -68,7 +68,7 @@ run: verify manifests
 
 # Generate manifests e.g. CRD, RBAC etc.
 manifests:
-	$(CONTROLLER_GEN) crd rbac:roleName=manager-role webhook paths="./..." output:crd:artifacts:config=config/crd/bases
+	$(CONTROLLER_GEN) crd rbac:roleName=manager-role webhook paths="./cmd/..." paths="./pkg/..." output:crd:artifacts:config=config/crd/bases
 
 # Run go fmt against code
 .PHONY: fmt
@@ -92,7 +92,7 @@ vendor:
 
 # Generate code
 generate:
-	$(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..."
+	$(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./cmd/..." paths="./pkg/..."
 
 # Build the docker image
 .PHONY: image

go.work

@@ -1,7 +1,6 @@
-go 1.25.0
+go 1.26.0
 
 use (
 	.
-	./openshift-tests/ccm-aws-tests
 	./openshift-tests/operator-tests
 )

hack/go-mod.sh

@@ -2,46 +2,65 @@
 
 set -e
 
-echo "Updating dependencies for Machine API Provider AWS workspace"
-
-go work use -r .
-
-# Discover all modules from the workspace
-echo "Discovering modules from workspace..."
-MODULES=$(go work edit -json | grep -o '"DiskPath": "[^"]*"' | cut -d'"' -f4 | sed 's|^\./||')
-echo "Found modules: $MODULES"
-
-# Pass 1: tidy all modules
-echo "Running go mod tidy for all modules (pass 1)..."
-for module in $MODULES; do
-  if [ -f "$module/go.mod" ]; then
-    echo "Tidying $module"
-    (cd "$module" && go mod tidy)
-  fi
-done
-
-# Sync: propagate highest require versions across all modules
-echo "Syncing Go workspace..."
-go work sync
-
-# Pass 2: re-tidy after sync may have bumped versions
-echo "Running go mod tidy for all modules (pass 2)..."
-for module in $MODULES; do
-  if [ -f "$module/go.mod" ]; then
-    echo "Tidying $module"
-    (cd "$module" && go mod tidy)
-  fi
-done
-
-# Verify all modules
-echo "Verifying all modules..."
-for module in $MODULES; do
-  if [ -f "$module/go.mod" ]; then
-    echo "Verifying $module"
-    (cd "$module" && go mod verify)
-  fi
-done
-
-# Create unified vendor directory
-echo "Creating unified vendor directory..."
-go work vendor -v
+function vendor_workspace() {
+  echo "Updating dependencies for Cluster Cloud Controller Manager Operator workspace"
+
+  go work use -r .
+  go work edit -dropuse ./openshift-tests/ccm-aws-tests
+
+  # Discover all modules from the workspace
+  echo "Discovering modules from workspace..."
+  MODULES=$(go work edit -json | grep -o '"DiskPath": "[^"]*"' | cut -d'"' -f4 | sed 's|^\./||')
+  echo "Found modules: $MODULES"
+
+  # Pass 1: tidy all modules
+  echo "Running go mod tidy for all modules (pass 1)..."
+  for module in $MODULES; do
+    if [ -f "$module/go.mod" ]; then
+      echo "Tidying $module"
+      (cd "$module" && go mod tidy)
+    fi
+  done
+
+  # Sync: propagate highest require versions across all modules
+  echo "Syncing Go workspace..."
+  go work sync
+
+  # Pass 2: re-tidy after sync may have bumped versions
+  echo "Running go mod tidy for all modules (pass 2)..."
+  for module in $MODULES; do
+    if [ -f "$module/go.mod" ]; then
+      echo "Tidying $module"
+      (cd "$module" && go mod tidy)
+    fi
+  done
+
+  # Verify all modules
+  echo "Verifying all modules..."
+  for module in $MODULES; do
+    if [ -f "$module/go.mod" ]; then
+      echo "Verifying $module"
+      (cd "$module" && go mod verify)
+    fi
+  done
+
+  # Create unified vendor directory
+  echo "Creating unified vendor directory..."
+  go work vendor -v
+
+  echo "Done updating dependencies for Cluster Cloud Controller Manager Operator workspace"
+}
+
+# vendor_ote_ccmaws is used to update the dependencies for the OpenShift Tests Extension CCM AWS tests.
+# CCM-AWS OTE is outside of workspace to prevent cross-module dependency conflicts.
+function vendor_ote_ccmaws() {
+  echo "Updating dependencies for OpenShift Tests Extension CCM AWS tests"
+  cd openshift-tests/ccm-aws-tests
+  GOWORK=off go mod tidy
+  GOWORK=off go mod vendor
+  cd ../..
+  echo "Updated dependencies for OpenShift Tests Extension CCM AWS tests"
+}
+
+vendor_ote_ccmaws
+vendor_workspace

openshift-tests/ccm-aws-tests/e2e/aws/helper.go

@@ -4,14 +4,17 @@ import (
 	"context"
 	"fmt"
 	"strings"
+	"time"
 
 	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/aws/retry"
 	"github.com/aws/aws-sdk-go-v2/config"
 	ec2 "github.com/aws/aws-sdk-go-v2/service/ec2"
 	ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types"
 	elbv2 "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2"
 	elbv2types "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2/types"
 	"github.com/openshift/cluster-cloud-controller-manager-operator/openshift-tests/ccm-aws-tests/e2e/common"
+	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/kubernetes/test/e2e/framework"
 )
 
@@ -23,33 +26,45 @@ func createAWSClientLoadBalancer(ctx context.Context) (*elbv2.Client, error) {
 	if err != nil {
 		return nil, fmt.Errorf("unable to load AWS config: %v", err)
 	}
-	return elbv2.NewFromConfig(cfg), nil
+
+	customRetryer := retry.NewStandard(func(o *retry.StandardOptions) {
+		o.MaxAttempts = 10
+		o.MaxBackoff = 30 * time.Second
+	})
+
+	return elbv2.NewFromConfig(cfg, func(o *elbv2.Options) {
+		o.Retryer = customRetryer
+	}), nil
 }
 
 // getAWSLoadBalancerFromDNSName finds a load balancer by DNS name using the AWS ELBv2 client.
 func getAWSLoadBalancerFromDNSName(ctx context.Context, elbClient *elbv2.Client, lbDNSName string) (*elbv2types.LoadBalancer, error) {
 	var foundLB *elbv2types.LoadBalancer
 	framework.Logf("describing load balancers with DNS %s", lbDNSName)
 
-	paginator := elbv2.NewDescribeLoadBalancersPaginator(elbClient, &elbv2.DescribeLoadBalancersInput{})
-	for paginator.HasMorePages() {
-		page, err := paginator.NextPage(ctx)
-		if err != nil {
-			return nil, fmt.Errorf("failed to describe load balancers: %v", err)
-		}
+	err := wait.PollUntilContextTimeout(ctx, 5*time.Second, 15*time.Minute, true, func(ctx context.Context) (bool, error) {
+		paginator := elbv2.NewDescribeLoadBalancersPaginator(elbClient, &elbv2.DescribeLoadBalancersInput{})
+		for paginator.HasMorePages() {
+			page, err := paginator.NextPage(ctx)
+			if err != nil {
+				framework.Logf("transient error describing load balancers (will retry): %v", err)
+				return false, nil
+			}
 
-		framework.Logf("found %d load balancers in page", len(page.LoadBalancers))
-		// Search for the load balancer with matching DNS name in this page
-		for i := range page.LoadBalancers {
-			if aws.ToString(page.LoadBalancers[i].DNSName) == lbDNSName {
-				foundLB = &page.LoadBalancers[i]
-				framework.Logf("found load balancer with DNS %s", aws.ToString(foundLB.DNSName))
-				break
+			framework.Logf("found %d load balancers in page", len(page.LoadBalancers))
+			for i := range page.LoadBalancers {
+				if aws.ToString(page.LoadBalancers[i].DNSName) == lbDNSName {
+					foundLB = &page.LoadBalancers[i]
+					framework.Logf("found load balancer with DNS %s", aws.ToString(foundLB.DNSName))
+					return true, nil
+				}
 			}
 		}
-		if foundLB != nil {
-			break
-		}
+		return false, nil
+	})
+
+	if err != nil {
+		return nil, fmt.Errorf("failed to find load balancer with DNS name %s: %v", lbDNSName, err)
 	}
 
 	if foundLB == nil {