[WIP]CNTRLPLANE-2995: hypershift: fix OADP e2e job for 4.21 with guest OLM placement

ci-operator/config/openshift/hypershift/openshift-hypershift-release-4.21__periodics-mce.yaml

@@ -220,14 +220,22 @@ tests:
         NUM_EXTRA_WORKERS=3
         PROVISIONING_NETWORK_PROFILE=Disabled
         REDFISH_EMULATOR_IGNORE_BOOT_DEVICE=True
+      EXTRA_ARGS: --olm-catalog-placement=guest
       KONFLUX_DEPLOY_CATALOG_SOURCE: "true"
       KONFLUX_DEPLOY_OPERATORS: "true"
       LVM_OPERATOR_SUB_CHANNEL: stable-4.21
       LVM_OPERATOR_SUB_SOURCE: lvm-catalogsource
       MCE_VERSION: "2.11"
       METALLB_OPERATOR_SUB_SOURCE: metallb-konflux
-      OADP_OPERATOR_SUB_CHANNEL: stable-1.4
+      OADP_OPERATOR_SUB_CHANNEL: stable
       OADP_OPERATOR_SUB_SOURCE: qe-app-registry
+      TEST_SKIPS: |
+        \[sig-storage\] CSI\|
+        \[sig-storage\] In-tree Volumes\|
+        \[sig-storage\] PersistentVolumes-local\|
+        \[OCPFeatureGate:NetworkSegmentation\]\|
+        \[sig-builds\]\|
+        \[sig-node\] Managed cluster should verify that nodes have no unexpected reboots
     test:
     - ref: hypershift-mce-agent-minio
     - ref: operatorhub-subscribe-oadp-operator

ci-operator/step-registry/hypershift/mce/agent/oadp/v2/hypershift-mce-agent-oadp-v2-commands.sh

@@ -9,11 +9,11 @@ if [ -f "${SHARED_DIR}/proxy-conf.sh" ] ; then
   source "${SHARED_DIR}/proxy-conf.sh"
 fi
 
-export KUBECONFIG="${SHARED_DIR}/nested_kubeconfig"
+export KUBECONFIG="${SHARED_DIR}/kubeconfig"
 oc create namespace oadp-helper
 IMAGE=$(oc get clusterversion version -ojsonpath='{.status.desired.image}')
 TOOLS_IMAGE=$(oc adm release info ${IMAGE} --image-for=tools)
-oc create secret generic oadp-kubeconfig-secret --from-file=kubeconfig="$KUBECONFIG" -n oadp-helper
+oc create secret generic oadp-kubeconfig-secret --from-file=kubeconfig="${SHARED_DIR}/nested_kubeconfig" -n oadp-helper
 cat <<EOF | oc apply -f -
 apiVersion: apps/v1
 kind: Deployment
@@ -45,7 +45,9 @@ spec:
 
 EOF
 
-export KUBECONFIG="${SHARED_DIR}/kubeconfig"
+# OADP is installed on the management cluster (kubeconfig) regardless of whether the
+# hosted cluster uses guest OLM catalog placement. The resources being backed up
+# (hostedcluster, nodepool, local-cluster namespaces) are management cluster resources.
 cat <<EOF > /tmp/miniocred
 [default]
 aws_access_key_id=admin
@@ -89,7 +91,7 @@ spec:
         - csi
       customPlugins:
         - name: hypershift-oadp-plugin
-          image: quay.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-oadp-plugin-main:latest
+          image: quay.io/hypershift/hypershift-oadp-plugin:latest
   snapshotLocations:
     - velero:
         config:
@@ -152,11 +154,32 @@ spec:
   defaultVolumesToFsBackup: false
   snapshotVolumes: true
 EOF
-oc wait --timeout=45m --for=jsonpath='{.status.phase}'=Completed backup/hc-clusters-hosted-backup -n openshift-adp
+# Wait for backup to finish. Accept both Completed and PartiallyFailed because
+# the hypershift-oadp-plugin may hit transient conflicts when unpausing the
+# HostedCluster/NodePool, which marks the backup PartiallyFailed even though all
+# data was backed up successfully.
+BACKUP_PHASE=""
+SECONDS=0
+TIMEOUT=$((45 * 60))
+while [[ $SECONDS -lt $TIMEOUT ]]; do
+  BACKUP_PHASE=$(oc get backup/hc-clusters-hosted-backup -n openshift-adp -o jsonpath='{.status.phase}' 2>/dev/null || true)
+  if [[ "$BACKUP_PHASE" == "Completed" || "$BACKUP_PHASE" == "PartiallyFailed" ]]; then
+    echo "Backup finished with phase: ${BACKUP_PHASE}"
+    break
+  fi
+  sleep 30
+done
+if [[ "$BACKUP_PHASE" != "Completed" && "$BACKUP_PHASE" != "PartiallyFailed" ]]; then
+  echo "ERROR: Backup did not finish within 45m (current phase: ${BACKUP_PHASE})"
+  oc get backup -n openshift-adp hc-clusters-hosted-backup -o yaml || true
+  exit 1
+fi
 
-oc delete hostedcluster -n local-cluster "${CLUSTER_NAME}"
+# HostedCluster is a management cluster resource; always use the management kubeconfig
+# to delete it, regardless of where OADP is installed.
+KUBECONFIG="${SHARED_DIR}/kubeconfig" oc delete hostedcluster -n local-cluster "${CLUSTER_NAME}"
 
-cat <<EOF | oc apply -f -
+KUBECONFIG="${SHARED_DIR}/kubeconfig" oc apply -f - <<EOF
 apiVersion: velero.io/v1
 kind: Restore
 metadata:
@@ -180,8 +203,9 @@ spec:
   - resticrepositories.velero.io
 EOF
 
-oc wait --timeout=45m --for=jsonpath='{.status.phase}'=Completed restore/hc-clusters-hosted-restore -n openshift-adp
-oc wait --timeout=30m --for=condition=Available --namespace=local-cluster hostedcluster/${CLUSTER_NAME}
+# Wait for restore and HostedCluster availability on the management cluster.
+KUBECONFIG="${SHARED_DIR}/kubeconfig" oc wait --timeout=45m --for=jsonpath='{.status.phase}'=Completed restore/hc-clusters-hosted-restore -n openshift-adp
+KUBECONFIG="${SHARED_DIR}/kubeconfig" oc wait --timeout=30m --for=condition=Available --namespace=local-cluster hostedcluster/${CLUSTER_NAME}
 export KUBECONFIG=${SHARED_DIR}/nested_kubeconfig
 echo "Wait HostedCluster ready..."
 until \
@@ -191,6 +215,5 @@ until \
     sleep 1s
 done
 oc get pod -A > "${ARTIFACT_DIR}/hostedcluster pods"
-export KUBECONFIG="${SHARED_DIR}/kubeconfig"
-oc get backup -n openshift-adp hc-clusters-hosted-backup -o yaml > "${ARTIFACT_DIR}/backup.yaml"
-oc get restore hc-clusters-hosted-restore -n openshift-adp  -o yaml > "${ARTIFACT_DIR}/restore.yaml"
+KUBECONFIG="${SHARED_DIR}/kubeconfig" oc get backup -n openshift-adp hc-clusters-hosted-backup -o yaml > "${ARTIFACT_DIR}/backup.yaml"
+KUBECONFIG="${SHARED_DIR}/kubeconfig" oc get restore hc-clusters-hosted-restore -n openshift-adp -o yaml > "${ARTIFACT_DIR}/restore.yaml"

ci-operator/step-registry/operatorhub/subscribe/oadp-operator/operatorhub-subscribe-oadp-operator-commands.sh

@@ -22,6 +22,7 @@ fi
 if [[ "${OADP_SUB_TARGET_NAMESPACES}" == "!install" ]]; then
   OADP_SUB_TARGET_NAMESPACES="${OADP_OPERATOR_SUB_INSTALL_NAMESPACE}"
 fi
+
 echo "Installing ${OADP_OPERATOR_SUB_PACKAGE} from channel: ${OADP_OPERATOR_SUB_CHANNEL} in source: ${OADP_OPERATOR_SUB_SOURCE} into ${OADP_OPERATOR_SUB_INSTALL_NAMESPACE}"
 
 # create the install namespace