Skip to content

temporary workaround for OCPBUGS-78431 #76504

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
jianli-wei wants to merge 1 commit into
base: main
Choose a base branch
from
+110 −37
Open

temporary workaround for OCPBUGS-78431 #76504

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
147 changes: 110 additions & 37 deletions ci-operator/step-registry/ipi/conf/gcp/zones/ipi-conf-gcp-zones-commands.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,9 @@ echo "$(date -u --rfc-3339=seconds) - INFO: CONTROL_PLANE_ZONES '${CONTROL_PLANE
echo "$(date -u --rfc-3339=seconds) - INFO: CONTROL_PLANE_NODE_TYPE '${CONTROL_PLANE_NODE_TYPE}'"

function get_zones_from_region() {
# shellcheck disable=SC2178
local -n ZONES=$1

if [[ -n "${ZONES_EXCLUSION_PATTERN}" ]]; then
echo "$(date -u --rfc-3339=seconds) - INFO: Filtering zones by the exclusion pattern '${ZONES_EXCLUSION_PATTERN}'"
mapfile -t AVAILABILITY_ZONES < <(gcloud compute zones list --filter="region:${GCP_REGION} AND status:UP" --format='value(name)' | grep -v "${ZONES_EXCLUSION_PATTERN}" | shuf)
Expand All @@ -26,14 +29,15 @@ function get_zones_from_region() {

echo "$(date -u --rfc-3339=seconds) - INFO: Take the first ${ZONES_COUNT} zones"
ZONES=("${AVAILABILITY_ZONES[@]:0:${ZONES_COUNT}}")
ZONES_STR="[ $(join_by , "${ZONES[@]}") ]"
echo "$(date -u --rfc-3339=seconds) - INFO: GCP region: ${GCP_REGION} (zones: ${ZONES_STR})"
echo "$(date -u --rfc-3339=seconds) - INFO: GCP region: ${GCP_REGION} (zones: ${ZONES[*]})"
}

function join_by { local IFS="$1"; shift; echo "$*"; }

function get_zones_by_machine_type() {
local machine_type=$1
# shellcheck disable=SC2178
local -n ZONES=$2

if [[ -z "${machine_type}" ]]; then
echo "$(date -u --rfc-3339=seconds) - INFO: Nothing to do for empty machine type"
Expand Down Expand Up @@ -76,8 +80,37 @@ function get_zones_by_machine_type() {

echo "$(date -u --rfc-3339=seconds) - INFO: Take the first ${ZONES_COUNT} zones"
ZONES=("${AVAILABILITY_ZONES[@]:0:${ZONES_COUNT}}")
ZONES_STR="[ $(join_by , "${ZONES[@]}") ]"
echo "$(date -u --rfc-3339=seconds) - INFO: [${machine_type}] GCP region: ${GCP_REGION} (zones: ${ZONES_STR})"
echo "$(date -u --rfc-3339=seconds) - INFO: [${machine_type}] GCP region: ${GCP_REGION} (zones: ${ZONES[*]})"
}

# Function to get intersection or fallback
# Updates the first array (target) with intersection or fallback logic
function array_intersection_or_fallback() {
local -n target_array=$1
local -n source_array=$2

# If target_array is empty, use source_array
if [ ${#target_array[@]} -eq 0 ]; then
target_array=("${source_array[@]}")
return
fi

# If source_array is empty, keep target_array as is
if [ ${#source_array[@]} -eq 0 ]; then
return
fi

# Both arrays have elements, find intersection
local result=()
for elem1 in "${target_array[@]}"; do
for elem2 in "${source_array[@]}"; do
if [ "$elem1" = "$elem2" ]; then
result+=("$elem1")
break
fi
done
done
target_array=("${result[@]}")
}

export GCP_SHARED_CREDENTIALS_FILE="${CLUSTER_PROFILE_DIR}/gce.json"
Expand All @@ -89,64 +122,101 @@ then
gcloud config set project "${GOOGLE_PROJECT_ID}"
fi

ZONES_STR=""
MACHINE_TYPE=""
if [[ -n "${COMPUTE_ZONES}" ]]; then
ZONES_STR="${COMPUTE_ZONES}"
elif [[ -n "${COMPUTE_NODE_TYPE}" ]]; then
MACHINE_TYPE="${COMPUTE_NODE_TYPE}"
elif [[ -n "${ZONES_EXCLUSION_PATTERN}" ]]; then
# the intersection of ZONES_ARRAY1 and ZONES_ARRAY2
CANDIDATE_ZONES_ARRAY=()

# compute/worker zones string, e.g. '[us-central1-a us-central1-b]'
ZONES_STR1=""
# compute/worker zones array, e.g. (us-central1-a us-central1-b)
ZONES_ARRAY1=()

# control-plane zones string, e.g. '[us-central1-b us-central1-c]'
ZONES_STR2=""
# control-plane zones array, e.g. (us-central1-b us-central1-c)
ZONES_ARRAY2=()

# Get compute/worker machine type
MACHINE_TYPE1=""
if [[ -n "${COMPUTE_NODE_TYPE}" ]]; then
MACHINE_TYPE1="${COMPUTE_NODE_TYPE}"
else
echo "$(date -u --rfc-3339=seconds) - INFO: Extracting compute node type from install-config"
MACHINE_TYPE=$(yq-go r "${CONFIG}" compute[0].platform.gcp.type)
MACHINE_TYPE1=$(yq-go r "${CONFIG}" compute[0].platform.gcp.type)
if [[ -z "${MACHINE_TYPE}" ]]; then
MACHINE_TYPE=$(yq-go r "${CONFIG}" platform.gcp.defaultMachinePlatform.type)
MACHINE_TYPE1=$(yq-go r "${CONFIG}" platform.gcp.defaultMachinePlatform.type)
fi
fi
get_zones_by_machine_type "${MACHINE_TYPE}"
if [[ -z "${ZONES_STR}" ]] && [[ -n "${ZONES_EXCLUSION_PATTERN}" ]]; then
echo "$(date -u --rfc-3339=seconds) - INFO: Found no zone for machine type '${MACHINE_TYPE}', possibly a custom machine type. Filtering among all zones instead..."
get_zones_from_region
echo "$(date -u --rfc-3339=seconds) - INFO: compute/worker machine type is '${MACHINE_TYPE1}'"

# Get control-plane machine type
MACHINE_TYPE2=""
if [[ -n "${CONTROL_PLANE_NODE_TYPE}" ]]; then
MACHINE_TYPE2="${CONTROL_PLANE_NODE_TYPE}"
else
echo "$(date -u --rfc-3339=seconds) - INFO: Extracting control-plane node type from install-config"
MACHINE_TYPE2=$(yq-go r "${CONFIG}" controlPlane.platform.gcp.type)
if [[ -z "${MACHINE_TYPE2}" ]]; then
MACHINE_TYPE2=$(yq-go r "${CONFIG}" platform.gcp.defaultMachinePlatform.type)
fi
fi
echo "$(date -u --rfc-3339=seconds) - INFO: control-plane machine type is '${MACHINE_TYPE2}'"

if [[ -z "${COMPUTE_ZONES}" ]]; then
echo "$(date -u --rfc-3339=seconds) - INFO: COMPUTE_ZONES unspecified, getting the candidate zones by compute/worker machine type '${MACHINE_TYPE1}'"
get_zones_by_machine_type "${MACHINE_TYPE1}" ZONES_ARRAY1

if [[ ${#ZONES_ARRAY1[@]} -eq 0 ]] && [[ -n "${ZONES_EXCLUSION_PATTERN}" ]]; then
echo "$(date -u --rfc-3339=seconds) - INFO: Found no zone for machine type '${MACHINE_TYPE1}', possibly a custom machine type. Filtering among all zones instead..."
get_zones_from_region ZONES_ARRAY1
fi
echo "$(date -u --rfc-3339=seconds) - INFO: As a temporary workaround of https://redhat.atlassian.net/browse/OCPBUGS-78431, ensure the zones of compute & controlPlane machines are the same"
array_intersection_or_fallback CANDIDATE_ZONES_ARRAY ZONES_ARRAY1
fi

if [[ -n "${ZONES_STR}" ]]; then
if [[ -z "${CONTROL_PLANE_ZONES}" ]]; then
echo "$(date -u --rfc-3339=seconds) - INFO: CONTROL_PLANE_ZONES unspecified, getting the candidate zones by control-plane machine type '${MACHINE_TYPE2}'"
get_zones_by_machine_type "${MACHINE_TYPE2}" ZONES_ARRAY2

if [[ ${#ZONES_ARRAY2[@]} -eq 0 ]] && [[ -n "${ZONES_EXCLUSION_PATTERN}" ]]; then
echo "$(date -u --rfc-3339=seconds) - INFO: Found no zone for machine type '${MACHINE_TYPE2}', possibly a custom machine type. Filtering among all zones instead..."
get_zones_from_region ZONES_ARRAY2
fi
echo "$(date -u --rfc-3339=seconds) - INFO: As a temporary workaround of https://redhat.atlassian.net/browse/OCPBUGS-78431, ensure the zones of compute & controlPlane machines are the same"
array_intersection_or_fallback CANDIDATE_ZONES_ARRAY ZONES_ARRAY2
fi

if [[ -n "${COMPUTE_ZONES}" ]]; then
ZONES_STR1="${COMPUTE_ZONES}"
else
ZONES_STR1="[ $(join_by , "${CANDIDATE_ZONES_ARRAY[@]}") ]"
fi

if [[ -n "${ZONES_STR1}" ]]; then
cat >> "${PATCH}" << EOF
compute:
- name: worker
platform:
gcp:
zones: ${ZONES_STR}
zones: ${ZONES_STR1}
EOF
yq-go m -x -i "${CONFIG}" "${PATCH}"
else
echo "$(date -u --rfc-3339=seconds) - INFO: Skipped setting zones for compute"
fi

ZONES_STR=""
MACHINE_TYPE=""
if [[ -n "${CONTROL_PLANE_ZONES}" ]]; then
ZONES_STR="${CONTROL_PLANE_ZONES}"
elif [[ -n "${CONTROL_PLANE_NODE_TYPE}" ]]; then
MACHINE_TYPE="${CONTROL_PLANE_NODE_TYPE}"
elif [[ -n "${ZONES_EXCLUSION_PATTERN}" ]]; then
echo "$(date -u --rfc-3339=seconds) - INFO: Extracting control-plane node type from install-config"
MACHINE_TYPE=$(yq-go r "${CONFIG}" controlPlane.platform.gcp.type)
if [[ -z "${MACHINE_TYPE}" ]]; then
MACHINE_TYPE=$(yq-go r "${CONFIG}" platform.gcp.defaultMachinePlatform.type)
fi
fi
get_zones_by_machine_type "${MACHINE_TYPE}"
if [[ -z "${ZONES_STR}" ]] && [[ -n "${ZONES_EXCLUSION_PATTERN}" ]]; then
echo "$(date -u --rfc-3339=seconds) - INFO: Found no zone for machine type '${MACHINE_TYPE}', possibly a custom machine type. Filtering among all zones instead..."
get_zones_from_region
ZONES_STR2="${CONTROL_PLANE_ZONES}"
else
ZONES_STR2="[ $(join_by , "${CANDIDATE_ZONES_ARRAY[@]}") ]"
fi

if [[ -n "${ZONES_STR}" ]]; then
if [[ -n "${ZONES_STR2}" ]]; then
cat >> "${PATCH}" << EOF
controlPlane:
name: master
platform:
gcp:
zones: ${ZONES_STR}
zones: ${ZONES_STR2}
EOF
yq-go m -x -i "${CONFIG}" "${PATCH}"
else
Expand All @@ -160,3 +230,6 @@ yq-go r "${CONFIG}" controlPlane
if [ -f "${PATCH}" ]; then
rm "${PATCH}"
fi

echo "$(date -u --rfc-3339=seconds) - INFO: quit for debugging..."
exit 1