diff --git a/ci-operator/config/trusted-execution-clusters/operator/azure.yaml b/ci-operator/config/trusted-execution-clusters/operator/azure.yaml new file mode 100644 index 0000000000000..e4d6c42472c54 --- /dev/null +++ b/ci-operator/config/trusted-execution-clusters/operator/azure.yaml @@ -0,0 +1,31 @@ +base_images: + telco-runner: + name: telco-runner + namespace: ci + tag: latest +build_root: + image_stream_tag: + name: builder + namespace: ocp + tag: rhel-9-golang-1.25-openshift-4.21 +resources: + '*': + limits: + memory: 4Gi + requests: + cpu: 100m + memory: 200Mi +tests: +- as: operator-lifecycle-azure-verify + capabilities: + - intranet + skip_if_only_changed: ^(\.github|LICENSES|bundle|docs|examples)/|^(README\.md|\.gitignore)$ + steps: + test: + - chain: trusted-execution-clusters-operator-azure-lifecycle + post: + - chain: trusted-execution-clusters-operator-azure-cleanup +zz_generated_metadata: + branch: main + org: trusted-execution-clusters + repo: operator diff --git a/ci-operator/jobs/trusted-execution-clusters/operator/trusted-execution-clusters-operator-main-presubmits.yaml b/ci-operator/jobs/trusted-execution-clusters/operator/trusted-execution-clusters-operator-main-presubmits.yaml index 3f566a011df32..7fa41fc0fe381 100644 --- a/ci-operator/jobs/trusted-execution-clusters/operator/trusted-execution-clusters-operator-main-presubmits.yaml +++ b/ci-operator/jobs/trusted-execution-clusters/operator/trusted-execution-clusters-operator-main-presubmits.yaml @@ -75,3 +75,68 @@ presubmits: secret: secretName: result-aggregator trigger: (?m)^/test( | .* )operator-lifecycle-verify,?($|\s.*) + - agent: kubernetes-azure + always_run: false + branches: + - ^main$ + - ^main- + cluster: build07 + context: ci/prow/operator-lifecycle-azure-verify + decorate: true + decoration_config: + skip_cloning: true + labels: + capability/intranet: intranet + ci.openshift.io/generator: prowgen + pj-rehearse.openshift.io/can-be-rehearsed: "true" + name: pull-ci-trusted-execution-clusters-operator-main-operator-lifecycle-azure-verify + rerun_command: /test operator-lifecycle-azure-verify + skip_if_only_changed: ^(\.github|LICENSES|bundle|docs|examples)/|^(README\.md|\.gitignore)$ + spec: + containers: + - args: + - --gcs-upload-secret=/secrets/gcs/service-account.json + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --report-credentials-file=/etc/report/credentials + - --target=operator-lifecycle-azure-verify + command: + - ci-operator + env: + - name: HTTP_SERVER_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest + imagePullPolicy: Always + name: "" + ports: + - containerPort: 8080 + name: http + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /secrets/gcs + name: gcs-credentials + readOnly: true + - mountPath: /secrets/manifest-tool + name: manifest-tool-local-pusher + readOnly: true + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/report + name: result-aggregator + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: manifest-tool-local-pusher + secret: + secretName: manifest-tool-local-pusher + - name: pull-secret + secret: + secretName: registry-pull-credentials + - name: result-aggregator + secret: + secretName: result-aggregator + trigger: (?m)^/test( | .* )operator-lifecycle-azure-verify,?($|\s.*) diff --git a/ci-operator/step-registry/trusted-execution-clusters/operator-azure/OWNERS b/ci-operator/step-registry/trusted-execution-clusters/operator-azure/OWNERS new file mode 100644 index 0000000000000..ff09338355592 --- /dev/null +++ b/ci-operator/step-registry/trusted-execution-clusters/operator-azure/OWNERS @@ -0,0 +1,6 @@ +reviewers: + - alicefr + - Jakob-Naucke +approvers: + - alicefr + - Jakob-Naucke diff --git a/ci-operator/step-registry/trusted-execution-clusters/operator-azure/cleanup/OWNERS b/ci-operator/step-registry/trusted-execution-clusters/operator-azure/cleanup/OWNERS new file mode 100644 index 0000000000000..ff09338355592 --- /dev/null +++ b/ci-operator/step-registry/trusted-execution-clusters/operator-azure/cleanup/OWNERS @@ -0,0 +1,6 @@ +reviewers: + - alicefr + - Jakob-Naucke +approvers: + - alicefr + - Jakob-Naucke diff --git a/ci-operator/step-registry/trusted-execution-clusters/operator-azure/cleanup/trusted-execution-clusters-operator-azure-cleanup-chain.metadata.json b/ci-operator/step-registry/trusted-execution-clusters/operator-azure/cleanup/trusted-execution-clusters-operator-azure-cleanup-chain.metadata.json new file mode 100644 index 0000000000000..ea636fb73bfed --- /dev/null +++ b/ci-operator/step-registry/trusted-execution-clusters/operator-azure/cleanup/trusted-execution-clusters-operator-azure-cleanup-chain.metadata.json @@ -0,0 +1,13 @@ +{ + "path": "trusted-execution-clusters/operator-azure/cleanup/trusted-execution-clusters-operator-azure-cleanup-chain.yaml", + "owners": { + "approvers": [ + "alicefr", + "Jakob-Naucke" + ], + "reviewers": [ + "alicefr", + "Jakob-Naucke" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/trusted-execution-clusters/operator-azure/cleanup/trusted-execution-clusters-operator-azure-cleanup-chain.yaml b/ci-operator/step-registry/trusted-execution-clusters/operator-azure/cleanup/trusted-execution-clusters-operator-azure-cleanup-chain.yaml new file mode 100644 index 0000000000000..fd0e1f2f0b8d2 --- /dev/null +++ b/ci-operator/step-registry/trusted-execution-clusters/operator-azure/cleanup/trusted-execution-clusters-operator-azure-cleanup-chain.yaml @@ -0,0 +1,6 @@ +chain: + as: trusted-execution-clusters-operator-azure-cleanup + steps: + - ref: trusted-execution-clusters-ref-operator-azure-deprovision + documentation: |- + Azure tests create a Kind VM. Remove its resource group. diff --git a/ci-operator/step-registry/trusted-execution-clusters/operator-azure/lifecycle/OWNERS b/ci-operator/step-registry/trusted-execution-clusters/operator-azure/lifecycle/OWNERS new file mode 100644 index 0000000000000..ff09338355592 --- /dev/null +++ b/ci-operator/step-registry/trusted-execution-clusters/operator-azure/lifecycle/OWNERS @@ -0,0 +1,6 @@ +reviewers: + - alicefr + - Jakob-Naucke +approvers: + - alicefr + - Jakob-Naucke diff --git a/ci-operator/step-registry/trusted-execution-clusters/operator-azure/lifecycle/trusted-execution-clusters-operator-azure-lifecycle-chain.metadata.json b/ci-operator/step-registry/trusted-execution-clusters/operator-azure/lifecycle/trusted-execution-clusters-operator-azure-lifecycle-chain.metadata.json new file mode 100644 index 0000000000000..78823b6d7ba35 --- /dev/null +++ b/ci-operator/step-registry/trusted-execution-clusters/operator-azure/lifecycle/trusted-execution-clusters-operator-azure-lifecycle-chain.metadata.json @@ -0,0 +1,13 @@ +{ + "path": "trusted-execution-clusters/operator-azure/lifecycle/trusted-execution-clusters-operator-azure-lifecycle-chain.yaml", + "owners": { + "approvers": [ + "alicefr", + "Jakob-Naucke" + ], + "reviewers": [ + "alicefr", + "Jakob-Naucke" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/trusted-execution-clusters/operator-azure/lifecycle/trusted-execution-clusters-operator-azure-lifecycle-chain.yaml b/ci-operator/step-registry/trusted-execution-clusters/operator-azure/lifecycle/trusted-execution-clusters-operator-azure-lifecycle-chain.yaml new file mode 100644 index 0000000000000..7290c6d9dca38 --- /dev/null +++ b/ci-operator/step-registry/trusted-execution-clusters/operator-azure/lifecycle/trusted-execution-clusters-operator-azure-lifecycle-chain.yaml @@ -0,0 +1,6 @@ +chain: + as: trusted-execution-clusters-operator-azure-lifecycle + steps: + - ref: trusted-execution-clusters-ref-operator-azure-test + documentation: |- + Create a VM for Kind on Azure. Run integration tests with Azure VMs, testing against the operator on that Kind cluster. diff --git a/ci-operator/step-registry/trusted-execution-clusters/ref/operator/azure-deprovision/OWNERS b/ci-operator/step-registry/trusted-execution-clusters/ref/operator/azure-deprovision/OWNERS new file mode 100644 index 0000000000000..ff09338355592 --- /dev/null +++ b/ci-operator/step-registry/trusted-execution-clusters/ref/operator/azure-deprovision/OWNERS @@ -0,0 +1,6 @@ +reviewers: + - alicefr + - Jakob-Naucke +approvers: + - alicefr + - Jakob-Naucke diff --git a/ci-operator/step-registry/trusted-execution-clusters/ref/operator/azure-deprovision/trusted-execution-clusters-ref-operator-azure-deprovision-commands.sh b/ci-operator/step-registry/trusted-execution-clusters/ref/operator/azure-deprovision/trusted-execution-clusters-ref-operator-azure-deprovision-commands.sh new file mode 100755 index 0000000000000..5da82ffc78ddc --- /dev/null +++ b/ci-operator/step-registry/trusted-execution-clusters/ref/operator/azure-deprovision/trusted-execution-clusters-ref-operator-azure-deprovision-commands.sh @@ -0,0 +1,27 @@ +#!/bin/bash -eu +set -o pipefail + +if [ -z "${SHARED_DIR}" ]; then + echo "[ERROR] SHARED_DIR is not set. This script must run in Prow CI environment." + exit 1 +fi + +if [ ! -f "${SHARED_DIR}/az-resource-group" ]; then + echo "[ERROR] az-resource-group was not placed in SHARED_DIR" + exit 1 +fi + +rpm --import https://packages.microsoft.com/keys/microsoft.asc +dnf install -y https://packages.microsoft.com/config/rhel/9.0/packages-microsoft-prod.rpm +dnf install -y azure-cli + +secret_base=/var/run/azure-upstream-ci +az login --service-principal \ + --username "$(cat $secret_base/client-id)" \ + --password "$(cat $secret_base/client-secret)" \ + --tenant "$(cat $secret_base/tenant-id)" + +az_resource_group=$(cat "${SHARED_DIR}/az-resource-group") +echo "[INFO] Delete Kind VM resource group $az_resource_group" +az group delete --name "$az_resource_group" --yes +echo "[SUCCESS] Deleted Kind VM resource group $az_resource_group" diff --git a/ci-operator/step-registry/trusted-execution-clusters/ref/operator/azure-deprovision/trusted-execution-clusters-ref-operator-azure-deprovision-ref.metadata.json b/ci-operator/step-registry/trusted-execution-clusters/ref/operator/azure-deprovision/trusted-execution-clusters-ref-operator-azure-deprovision-ref.metadata.json new file mode 100644 index 0000000000000..fd659b4deb565 --- /dev/null +++ b/ci-operator/step-registry/trusted-execution-clusters/ref/operator/azure-deprovision/trusted-execution-clusters-ref-operator-azure-deprovision-ref.metadata.json @@ -0,0 +1,13 @@ +{ + "path": "trusted-execution-clusters/ref/operator/azure-deprovision/trusted-execution-clusters-ref-operator-azure-deprovision-ref.yaml", + "owners": { + "approvers": [ + "alicefr", + "Jakob-Naucke" + ], + "reviewers": [ + "alicefr", + "Jakob-Naucke" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/trusted-execution-clusters/ref/operator/azure-deprovision/trusted-execution-clusters-ref-operator-azure-deprovision-ref.yaml b/ci-operator/step-registry/trusted-execution-clusters/ref/operator/azure-deprovision/trusted-execution-clusters-ref-operator-azure-deprovision-ref.yaml new file mode 100644 index 0000000000000..fe452edf88eb7 --- /dev/null +++ b/ci-operator/step-registry/trusted-execution-clusters/ref/operator/azure-deprovision/trusted-execution-clusters-ref-operator-azure-deprovision-ref.yaml @@ -0,0 +1,19 @@ +ref: + as: trusted-execution-clusters-ref-operator-azure-deprovision + from_image: + namespace: ci + name: telco-runner + tag: latest + commands: trusted-execution-clusters-ref-operator-azure-deprovision-commands.sh + credentials: + - namespace: test-credentials + name: azure-upstream-ci + mount_path: /var/run/azure-upstream-ci + resources: + requests: + cpu: 500m + memory: 500Mi + limits: + memory: 1Gi + documentation: |- + Azure tests create a Kind VM. Remove its resource group. diff --git a/ci-operator/step-registry/trusted-execution-clusters/ref/operator/azure-test/OWNERS b/ci-operator/step-registry/trusted-execution-clusters/ref/operator/azure-test/OWNERS new file mode 100644 index 0000000000000..ff09338355592 --- /dev/null +++ b/ci-operator/step-registry/trusted-execution-clusters/ref/operator/azure-test/OWNERS @@ -0,0 +1,6 @@ +reviewers: + - alicefr + - Jakob-Naucke +approvers: + - alicefr + - Jakob-Naucke diff --git a/ci-operator/step-registry/trusted-execution-clusters/ref/operator/azure-test/trusted-execution-clusters-ref-operator-azure-test-commands.sh b/ci-operator/step-registry/trusted-execution-clusters/ref/operator/azure-test/trusted-execution-clusters-ref-operator-azure-test-commands.sh new file mode 100755 index 0000000000000..15718f9bffd71 --- /dev/null +++ b/ci-operator/step-registry/trusted-execution-clusters/ref/operator/azure-test/trusted-execution-clusters-ref-operator-azure-test-commands.sh @@ -0,0 +1,193 @@ +#!/bin/bash -eu +set -o pipefail + +log_info() { + echo "[INFO] $1" +} + +log_warn() { + echo "[WARN] $1" +} + +log_error() { + echo "[ERROR] $1" +} + +log_success() { + echo "[SUCCESS] $1" +} + +if [ -z "${SHARED_DIR}" ]; then + log_error "SHARED_DIR is not set. This script must run in Prow CI environment." + exit 1 +fi + +repository=github.com/trusted-execution-clusters/operator +src_dir=/go/src/$repository +if [ ! -d $src_dir ]; then + log_info "No existing checkout (presumed rehearsal PR), creating checkout" + mkdir -p /go/src + git clone https://$repository $src_dir + log_success "Checked out $repository" +fi +pushd /go/src/$repository/.. + +rpm --import https://packages.microsoft.com/keys/microsoft.asc +dnf install -y https://packages.microsoft.com/config/rhel/9.0/packages-microsoft-prod.rpm +dnf install -y azure-cli cargo g++ jq rustfmt + +log_info "Setup an ephemeral Azure VM for a Kind cluster" +secret_base=/var/run/azure-upstream-ci +test_id=$(uuidgen | cut -d- -f1) + +az_region=eastus +az_resource_group=upstream-ci-$test_id +echo "$az_resource_group" > "$SHARED_DIR/az-resource-group" +kind_vm_user=ci +kind_vm_name=kind-vm +kind_vm_image=$(grep KIND_HOST_URN operator/Makefile | cut -d= -f2 | tr -d ' ') +vm_size=Standard_D2s_v3 + +AZURE_SUBSCRIPTION_ID=$(cat $secret_base/subscription-id) +export AZURE_SUBSCRIPTION_ID +az login --service-principal \ + --username "$(cat $secret_base/client-id)" \ + --password "$(cat $secret_base/client-secret)" \ + --tenant "$(cat $secret_base/tenant-id)" + +log_info "Create Azure resource group $az_resource_group" +az group create \ + --location $az_region \ + --resource-group "$az_resource_group" +log_info "Create Azure VM $kind_vm_name of image $kind_vm_image" +kind_vm_ip=$(az vm create \ + --name $kind_vm_name \ + --resource-group "$az_resource_group" \ + --size $vm_size \ + --image "$kind_vm_image" \ + --admin-username $kind_vm_user \ + --generate-ssh-keys | jq -r .publicIpAddress) +log_success "Created Azure VM $kind_vm_name (public IP $kind_vm_ip), waiting for availability" +az vm wait --created \ + --name $kind_vm_name \ + --resource-group "$az_resource_group" + +SSHOPTS=( + -o ConnectTimeout=30 + -o StrictHostKeyChecking=no + -o UserKnownHostsFile=/dev/null +) + +ssh "${SSHOPTS[@]}" $kind_vm_user@"$kind_vm_ip" echo +log_success "Azure VM $kind_vm_name has SSH access" + +log_info "Open ports on Azure VM" +nsg=$(az vm show \ + --resource-group "$az_resource_group" \ + --name $kind_vm_name \ + --query "networkProfile.networkInterfaces[0].id" -o tsv | \ + xargs az network nic show --query "networkSecurityGroup.id" -o tsv --ids | \ + cut -d/ -f9) +ports=(6443 8000 8080) +for i in "${!ports[@]}"; do + port=${ports[$i]} + az network nsg rule create \ + --resource-group "$az_resource_group" \ + --nsg-name "$nsg" \ + --name "allow-$port" \ + --priority $((1001 + i)) \ + --source-address-prefixes "*" \ + --destination-port-ranges "$port" \ + --protocol Tcp \ + --access Allow \ + --direction Inbound +done +log_success "Opened ports on Azure VM" + +log_info "Transfer source to Azure VM" +src_tarball=tec-src.tgz +tar czf $src_tarball operator +# shellcheck disable=SC2029 +scp "${SSHOPTS[@]}" $src_tarball $kind_vm_user@"$kind_vm_ip":~ +popd +provision=$(mktemp) +cat > "$provision" << 'PROVISION' +#!/bin/bash -eu +set -o pipefail + +sudo dnf config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo +sudo dnf install -y docker-ce docker-ce-cli containerd.io golang + +for lv in home tmp var; do + sudo lvextend -r -L +10G /dev/mapper/rootvg-${lv}lv +done + +mkdir -p ~/go/bin +PATH=$HOME/go/bin:$PATH + +tar xf tec-src.tgz +pushd operator + +go install github.com/mikefarah/yq/v4 +go install sigs.k8s.io/kind + +KUBEADM_CONFIG=$(cat <