From 3be7d44e8e7fbbaae5d77f4de9db6170120006b3 Mon Sep 17 00:00:00 2001 From: Rohan Kumar Date: Fri, 1 Aug 2025 16:20:47 +0530 Subject: [PATCH 1/6] ci : test k6 install and load test execution in e2e workflow - Add another Makefile target for load test execution - Execute this target in e2e test script Signed-off-by: Rohan Kumar --- .ci/openshift_e2e.sh | 2 + Makefile | 5 + test/load/devworkspace_load_test.js | 484 ++++++++++++++++++++++++++++ test/load/runk6.sh | 341 ++++++++++++++++++++ 4 files changed, 832 insertions(+) create mode 100644 test/load/devworkspace_load_test.js create mode 100644 test/load/runk6.sh diff --git a/.ci/openshift_e2e.sh b/.ci/openshift_e2e.sh index 9b9b175b2..c55418f42 100755 --- a/.ci/openshift_e2e.sh +++ b/.ci/openshift_e2e.sh @@ -72,4 +72,6 @@ make install export CLEAN_UP_AFTER_SUITE="false" make test_e2e bumpLogs + +make test_load ARGS="--mode operator --max-vus 250 --separate-namespaces false --test-duration-minutes 25 --dwo-namespace devworkspace-controller --logs-dir ${ARTIFACT_DIR}/load-testing-logs" make uninstall diff --git a/Makefile b/Makefile index d7053812b..c6f6de6e5 100644 --- a/Makefile +++ b/Makefile @@ -180,6 +180,11 @@ test_e2e_debug: mkdir -p /tmp/artifacts dlv test --listen=:2345 --headless=true --api-version=2 ./test/e2e/cmd/workspaces_test.go -- --ginkgo.fail-fast --ginkgo.junit-report=/tmp/artifacts/junit-workspaces-operator.xml +test_load: + @echo "Starting Load Testing Script..." && \ + bash ./test/load/runk6.sh $(ARGS) && \ + echo "Done" + ### manager: Build manager binary manager: generate fmt vet go build -o bin/manager main.go diff --git a/test/load/devworkspace_load_test.js b/test/load/devworkspace_load_test.js new file mode 100644 index 000000000..45c54ab0f --- /dev/null +++ b/test/load/devworkspace_load_test.js @@ -0,0 +1,484 @@ +// +// Copyright (c) 2019-2025 Red Hat, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import http from 'k6/http'; +import {check, sleep} from 'k6'; +import {Trend, Counter} from 'k6/metrics'; +import {htmlReport} from "https://raw.githubusercontent.com/benc-uk/k6-reporter/main/dist/bundle.js"; +import {textSummary} from "https://jslib.k6.io/k6-summary/0.0.1/index.js"; + +const inCluster = __ENV.IN_CLUSTER === 'true'; +const apiServer = inCluster ? `https://kubernetes.default.svc` : __ENV.KUBE_API; +const token = inCluster ? open('/var/run/secrets/kubernetes.io/serviceaccount/token') : __ENV.KUBE_TOKEN; +const useSeparateNamespaces = __ENV.SEPARATE_NAMESPACES === "true"; +const operatorNamespace = __ENV.DWO_NAMESPACE || 'openshift-operators'; +const externalDevWorkspaceLink = __ENV.DEVWORKSPACE_LINK || ''; +const shouldCreateAutomountResources = (__ENV.CREATE_AUTOMOUNT_RESOURCES || 'false') === 'true'; +const maxVUs = Number(__ENV.MAX_VUS || 50); +const devWorkspaceReadyTimeout = Number(__ENV.DEV_WORKSPACE_READY_TIMEOUT_IN_SECONDS || 600); +const autoMountConfigMapName = 'dwo-load-test-automount-configmap'; +const autoMountSecretName = 'dwo-load-test-automount-secret'; +const labelType = "test-type"; +const labelKey = "load-test"; +const loadTestDurationInMinutes = __ENV.TEST_DURATION_IN_MINUTES || "25"; +const loadTestNamespace = __ENV.LOAD_TEST_NAMESPACE || "loadtest-devworkspaces"; + +const headers = { + Authorization: `Bearer ${token}`, 'Content-Type': 'application/json', +}; + +export const options = { + scenarios: { + create_and_delete_devworkspaces: { + executor: 'ramping-vus', + startVUs: 0, + stages: generateLoadTestStages(maxVUs), + gracefulRampDown: '1m', + }, + final_cleanup: { + executor: 'per-vu-iterations', + vus: 1, + iterations: 1, + startTime: `${loadTestDurationInMinutes}m`, + exec: 'final_cleanup', + }, + }, thresholds: { + 'checks': ['rate>0.95'], + 'devworkspace_create_duration': ['p(95)<15000'], + 'devworkspace_delete_duration': ['p(95)<10000'], + 'devworkspace_ready_duration': ['p(95)<60000'], + 'devworkspace_ready_failed': ['count<5'], + 'operator_cpu_violations': ['count==0'], + 'operator_mem_violations': ['count==0'], + }, insecureSkipTLSVerify: true, // trust self-signed certs like in CRC +}; + +const devworkspaceCreateDuration = new Trend('devworkspace_create_duration'); +const devworkspaceReady = new Counter('devworkspace_ready'); +const devworkspaceDeleteDuration = new Trend('devworkspace_delete_duration'); +const devworkspaceReadyDuration = new Trend('devworkspace_ready_duration'); +const devworkspaceReadyFailed = new Counter('devworkspace_ready_failed'); +const operatorCpu = new Trend('average_operator_cpu'); // in milli cores +const operatorMemory = new Trend('average_operator_memory'); // in Mi +const devworkspacesCreated = new Counter('devworkspace_create_count'); +const operatorCpuViolations = new Counter('operator_cpu_violations'); +const operatorMemViolations = new Counter('operator_mem_violations'); + +const maxCpuMillicores = 250; +const maxMemoryBytes = 200 * 1024 * 1024; + +export function setup() { + if (shouldCreateAutomountResources) { + createNewAutomountConfigMap(); + createNewAutomountSecret(); + } +} + +export default function () { + const vuId = __VU; + const iteration = __ITER; + const crName = `dw-test-${vuId}-${iteration}`; + const namespace = useSeparateNamespaces + ? `load-test-ns-${__VU}-${__ITER}` + : loadTestNamespace; + + if (!apiServer) { + throw new Error('KUBE_API env var is required'); + } + try { + if (useSeparateNamespaces) { + createNewNamespace(namespace); + } + const devWorkspaceCreated = createNewDevWorkspace(namespace, vuId, iteration); + if (devWorkspaceCreated) { + waitUntilDevWorkspaceIsReady(vuId, crName, namespace); + deleteDevWorkspace(crName, namespace); + } + } catch (error) { + console.error(`Load test for ${vuId}-${iteration} failed:`, error.message); + } +} + +export function final_cleanup() { + if (useSeparateNamespaces) { + deleteAllSeparateNamespaces(); + } else { + deleteAllDevWorkspacesInCurrentNamespace(); + } + + if (shouldCreateAutomountResources) { + deleteConfigMap(); + deleteSecret(); + } +} + +export function handleSummary(data) { + const allowed = ['devworkspace_create_count', 'devworkspace_create_duration', 'devworkspace_delete_duration', 'devworkspace_ready_duration', 'devworkspace_ready', 'devworkspace_ready_failed', 'operator_cpu_violations', 'operator_mem_violations', 'average_operator_cpu', 'average_operator_memory']; + + const filteredData = JSON.parse(JSON.stringify(data)); + for (const key of Object.keys(filteredData.metrics)) { + if (!allowed.includes(key)) { + delete filteredData.metrics[key]; + } + } + + let loadTestSummaryReport = { + stdout: textSummary(filteredData, {indent: ' ', enableColors: true}) + } + // Only generate HTML report when running outside the cluster + if (!inCluster) { + loadTestSummaryReport["devworkspace-load-test-report.html"] = htmlReport(data, { + title: "DevWorkspace Operator Load Test Report (HTTP)", + }); + } + return loadTestSummaryReport; +} + +function createNewDevWorkspace(namespace, vuId, iteration) { + const baseUrl = `${apiServer}/apis/workspace.devfile.io/v1alpha2/namespaces/${namespace}/devworkspaces`; + + const manifest = generateDevWorkspaceToCreate(vuId, iteration, namespace); + + const payload = JSON.stringify(manifest); + + const createStart = Date.now(); + const createRes = http.post(baseUrl, payload, {headers}); + check(createRes, { + 'DevWorkspace created': (r) => r.status === 201 || r.status === 409, + }); + + if (createRes.status !== 201 && createRes.status !== 409) { + console.error(`[VU ${vuId}] Failed to create DevWorkspace: ${createRes.status}, ${createRes.body}`); + return false; + } + devworkspaceCreateDuration.add(Date.now() - createStart); + devworkspacesCreated.add(1); + return true; +} + +function waitUntilDevWorkspaceIsReady(vuId, crName, namespace) { + const dwUrl = `${apiServer}/apis/workspace.devfile.io/v1alpha2/namespaces/${namespace}/devworkspaces/${crName}`; + const readyStart = Date.now(); + let isReady = false; + let attempts = 0; + const pollWaitInterval = 5; + const maxAttempts = devWorkspaceReadyTimeout / pollWaitInterval; + let res = {}; + + while (!isReady && attempts < maxAttempts) { + res = http.get(`${dwUrl}`, {headers}); + + if (res.status === 200) { + try { + const body = JSON.parse(res.body); + const phase = body?.status?.phase; + if (phase === 'Ready' || phase === 'Running') { + isReady = true; + break; + } else if (phase === 'Failing' || phase === 'Failed' || phase === 'Error') { + isReady = false; + break; + } + } catch (e) { + console.error(`GET [VU ${vuId}] Failed to parse DevWorkspace from API: ${res.body} : ${e.message}`); + } + } + + checkDevWorkspaceOperatorMetrics(); + sleep(pollWaitInterval); + attempts++; + } + + if (res.status === 200) { + if (isReady) { + devworkspaceReady.add(1); + devworkspaceReadyDuration.add(Date.now() - readyStart); + } else { + devworkspaceReadyFailed.add(1); + } + } +} + +function deleteDevWorkspace(crName, namespace) { + const dwUrl = `${apiServer}/apis/workspace.devfile.io/v1alpha2/namespaces/${namespace}/devworkspaces/${crName}`; + const deleteStart = Date.now(); + const delRes = http.del(`${dwUrl}`, null, {headers}); + devworkspaceDeleteDuration.add(Date.now() - deleteStart); + + check(delRes, { + 'DevWorkspace deleted or not found': (r) => r.status === 200 || r.status === 404, + }); +} + +function checkDevWorkspaceOperatorMetrics() { + const metricsUrl = `${apiServer}/apis/metrics.k8s.io/v1beta1/namespaces/${operatorNamespace}/pods`; + const res = http.get(metricsUrl, {headers}); + + + check(res, { + 'Fetched pod metrics successfully': (r) => r.status === 200, + }); + + if (res.status !== 200) { + console.warn(`[DWO METRICS] Unable to fetch DevWorkspace Operator metrics from Kubernetes, got ${res.status}`); + return; + } + + const data = JSON.parse(res.body); + const operatorPods = data.items.filter(p => p.metadata.name.includes("devworkspace-controller")); + + for (const pod of operatorPods) { + const container = pod.containers[0]; // assuming single container + const name = pod.metadata.name; + + const cpu = parseCpuToMillicores(container.usage.cpu); + const memory = parseMemoryToBytes(container.usage.memory); + + operatorCpu.add(cpu); + operatorMemory.add(memory / 1024 / 1024); + + const cpuOk = cpu <= maxCpuMillicores; + const memOk = memory <= maxMemoryBytes; + + if (!cpuOk) { + operatorCpuViolations.add(1); + } + if (!memOk) { + operatorMemViolations.add(1); + } + + check(null, { + [`[${name}] CPU < ${maxCpuMillicores}m`]: () => cpuOk, + [`[${name}] Memory < ${Math.round(maxMemoryBytes / 1024 / 1024)}Mi`]: () => memOk, + }); + } +} + +function createNewNamespace(namespaceName) { + const url = `${apiServer}/api/v1/namespaces`; + + const namespaceObj = { + apiVersion: 'v1', + kind: 'Namespace', + metadata: { + name: namespaceName, + labels: { + [labelKey]: labelType + } + } + } + const res = http.post(url, JSON.stringify(namespaceObj), {headers}); + + if (res.status !== 201 && res.status !== 409) { + throw new Error(`Failed to create Namespace: ${res.status} - ${namespaceName}`); + } +} + +function createNewAutomountConfigMap() { + const url = `${apiServer}/api/v1/namespaces/${loadTestNamespace}/configmaps`; + + const configMapManifest = { + apiVersion: 'v1', kind: 'ConfigMap', metadata: { + name: autoMountConfigMapName, namespace: loadTestNamespace, labels: { + 'controller.devfile.io/mount-to-devworkspace': 'true', 'controller.devfile.io/watch-configmap': 'true', + }, annotations: { + 'controller.devfile.io/mount-path': '/etc/config/dwo-load-test-configmap', + 'controller.devfile.io/mount-access-mode': '0644', + 'controller.devfile.io/mount-as': 'file', + }, + }, data: { + 'test.key': 'test-value', + }, + }; + + const res = http.post(url, JSON.stringify(configMapManifest), {headers}); + + if (res.status !== 201 && res.status !== 409) { + throw new Error(`Failed to create automount ConfigMap: ${res.status} - ${res.body}`); + } + console.log("Created automount configMap : " + autoMountConfigMapName); +} + +function createNewAutomountSecret() { + const manifest = { + apiVersion: 'v1', kind: 'Secret', metadata: { + name: autoMountSecretName, namespace: loadTestNamespace, labels: { + 'controller.devfile.io/mount-to-devworkspace': 'true', 'controller.devfile.io/watch-secret': 'true', + }, annotations: { + 'controller.devfile.io/mount-path': `/etc/secret/dwo-load-test-secret`, + 'controller.devfile.io/mount-as': 'file', + }, + }, type: 'Opaque', data: { + 'secret.key': __ENV.SECRET_VALUE_BASE64 || 'dGVzdA==', // base64-encoded 'test' + }, + }; + + const res = http.post(`${apiServer}/api/v1/namespaces/${loadTestNamespace}/secrets`, JSON.stringify(manifest), {headers}); + if (res.status !== 201 && res.status !== 409) { + throw new Error(`Failed to create automount Secret: ${res.status} - ${res.body}`); + } +} + +function deleteConfigMap() { + const url = `${apiServer}/api/v1/namespaces/${loadTestNamespace}/configmaps/${autoMountConfigMapName}`; + const res = http.del(url, null, { headers }); + if (res.status !== 200 && res.status !== 404) { + console.warn(`[CLEANUP] Failed to delete ConfigMap ${autoMountConfigMapName}: ${res.status}`); + } +} + +function deleteSecret() { + const url = `${apiServer}/api/v1/namespaces/${loadTestNamespace}/secrets/${autoMountSecretName}`; + const res = http.del(url, null, {headers}); + if (res.status !== 200 && res.status !== 404) { + console.warn(`[CLEANUP] Failed to delete Secret ${autoMountSecretName}: ${res.status}`); + } +} + +function deleteNamespace(name) { + const delRes = http.del(`${apiServer}/api/v1/namespaces/${name}`, null, {headers}); + if (delRes.status !== 200 && delRes.status !== 404) { + console.warn(`[CLEANUP] Failed to delete Namespace ${name}: ${delRes.status}`); + } +} + +function deleteAllDevWorkspacesInCurrentNamespace() { + const deleteByLabelSelectorUrl = `${apiServer}/apis/workspace.devfile.io/v1alpha2/namespaces/${loadTestNamespace}/devworkspaces?labelSelector=${labelKey}%3D${labelType}`; + console.log(`[CLEANUP] Deleting all DevWorkspaces in ${loadTestNamespace} containing label ${labelKey}=${labelType}`); + + const res = http.del(deleteByLabelSelectorUrl, null, {headers}); + if (res.status !== 200) { + console.error(`[CLEANUP] Failed to delete DevWorkspaces: ${res.status}`); + } +} + +function deleteAllSeparateNamespaces() { + const getNamespacesByLabel = `${apiServer}/api/v1/namespaces?labelSelector=${labelKey}%3D${labelType}`; + console.log(`[CLEANUP] Deleting all Namespaces containing label ${labelKey}=${labelType}`); + + const res = http.get(getNamespacesByLabel, {headers}); + if (res.status !== 200) { + console.error(`[CLEANUP] Failed to list DevWorkspaces: ${res.status}`); + return; + } + + const body = JSON.parse(res.body); + if (!body.items || !Array.isArray(body.items)) return; + + for (const item of body.items) { + deleteNamespace(item.metadata.name); + } +} + +function createOpinionatedDevWorkspace() { + return { + apiVersion: "workspace.devfile.io/v1alpha2", kind: "DevWorkspace", metadata: { + name: "minimal-dw", + namespace: loadTestNamespace, + labels: { + [labelKey]: labelType + } + }, spec: { + started: true, template: { + attributes: { + "controller.devfile.io/storage-type": "ephemeral", + }, components: [{ + name: "dev", container: { + image: "registry.access.redhat.com/ubi9/ubi-micro:9.6-1752751762", + command: ["sleep", "3600"], + imagePullPolicy: "IfNotPresent", + memoryLimit: "64Mi", + memoryRequest: "32Mi", + cpuLimit: "200m", + cpuRequest: "100m" + }, + },], + }, + }, + }; +} + +function parseJSONResponseToDevWorkspace(response) { + let devWorkspace; + try { + devWorkspace = response.json(); + } catch (e) { + throw new Error(`[DW CREATE] Failed to parse JSON : ${response.body}: ${e.message}`); + } + return devWorkspace; +} + +function downloadAndParseExternalWorkspace(externalDevWorkspaceLink) { + let manifest; + if (externalDevWorkspaceLink) { + const res = http.get(externalDevWorkspaceLink); + + if (res.status !== 200) { + throw new Error(`[DW CREATE] Failed to fetch JSON content from ${externalDevWorkspaceLink}, got ${res.status}`); + } + manifest = parseJSONResponseToDevWorkspace(res); + } + + return manifest; +} + +function generateDevWorkspaceToCreate(vuId, iteration, namespace) { + const name = `dw-test-${vuId}-${iteration}`; + let devWorkspace = {}; + if (externalDevWorkspaceLink.length > 0) { + devWorkspace = downloadAndParseExternalWorkspace(externalDevWorkspaceLink); + } else { + devWorkspace = createOpinionatedDevWorkspace(); + } + devWorkspace.metadata.name = name; + devWorkspace.metadata.namespace = namespace; + devWorkspace.metadata.labels = { + [labelKey]: labelType + } + return devWorkspace; +} + +function generateLoadTestStages(max) { + const stageDefinitions = [ + { percent: 0.25, target: Math.floor(max * 0.25) }, + { percent: 0.25, target: Math.floor(max * 0.5) }, + { percent: 0.2, target: Math.floor(max * 0.75) }, + { percent: 0.15, target: max }, + { percent: 0.10, target: Math.floor(max * 0.5) }, + { percent: 0.05, target: 0 }, + ]; + + return stageDefinitions.map(({ percent, target }) => ({ + duration: `${Math.round(loadTestDurationInMinutes * percent)}m`, + target, + })); +} + +function parseMemoryToBytes(memStr) { + if (memStr.endsWith("Ki")) return parseInt(memStr) * 1024; + if (memStr.endsWith("Mi")) return parseInt(memStr) * 1024 * 1024; + if (memStr.endsWith("Gi")) return parseInt(memStr) * 1024 * 1024 * 1024; + if (memStr.endsWith("n")) return parseInt(memStr) / 1e9; + if (memStr.endsWith("u")) return parseInt(memStr) / 1e6; + if (memStr.endsWith("m")) return parseInt(memStr) / 1e3; + return parseInt(memStr); // bytes +} + +function parseCpuToMillicores(cpuStr) { + if (cpuStr.endsWith("n")) return Math.round(parseInt(cpuStr) / 1e6); + if (cpuStr.endsWith("u")) return Math.round(parseInt(cpuStr) / 1e3); + if (cpuStr.endsWith("m")) return parseInt(cpuStr); + return Math.round(parseFloat(cpuStr) * 1000); +} \ No newline at end of file diff --git a/test/load/runk6.sh b/test/load/runk6.sh new file mode 100644 index 000000000..2bb16f88b --- /dev/null +++ b/test/load/runk6.sh @@ -0,0 +1,341 @@ +#!/bin/bash + +#!/bin/bash + +MODE="binary" # or 'operator' +LOAD_TEST_NAMESPACE="loadtest-devworkspaces" +DWO_NAMESPACE="openshift-operators" +SA_NAME="k6-devworkspace-tester" +CLUSTERROLE_NAME="k6-devworkspace-role" +ROLEBINDING_NAME="k6-devworkspace-binding" +CONFIGMAP_NAME="k6-test-script" +K6_CR_NAME="k6-test-run" +K6_SCRIPT="test/load/devworkspace_load_test.js" +K6_OPERATOR_VERSION="v0.0.22" +DEVWORKSPACE_LINK="https://gist.githubusercontent.com/rohanKanojia/ecf625afaf3fe817ac7d1db78bd967fc/raw/8c30c0370444040105ca45cd4ac0f7062a644bb7/dw-minimal.json" +MAX_VUS="100" +DEV_WORKSPACE_READY_TIMEOUT_IN_SECONDS="1200" +SEPARATE_NAMESPACES="false" +CREATE_AUTOMOUNT_RESOURCES="false" +LOGS_DIR="logs" +TEST_DURATION_IN_MINUTES="25" +MIN_KUBECTL_VERSION="1.24.0" +MIN_CURL_VERSION="7.0.0" +MIN_K6_VERSION="1.1.0" + +# ----------- Main Execution Flow ----------- +main() { + parse_arguments "$@" + check_prerequisites + create_namespace + create_rbac + start_background_watchers + + if [[ "$MODE" == "operator" ]]; then + install_k6_operator + create_k6_configmap + delete_existing_testruns + create_k6_test_run + wait_for_test_completion + fetch_test_logs + elif [[ "$MODE" == "binary" ]]; then + generate_token_and_api_url + run_k6_binary_test + else + echo "❌ Invalid mode: $MODE" + exit 1 + fi + stop_background_watchers + delete_namespace +} + +# ----------- Helper Functions ----------- +print_help() { + cat < Mode to run the script (default: operator) + --max-vus Number of virtual users for k6 (default: 100) + --separate-namespaces Use separate namespaces for workspaces (default: false) + --devworkspace-ready-timeout-seconds Timeout in seconds for workspace to become ready (default: 1200) + --devworkspace-link DevWorkspace link (default: empty, opinionated DevWorkspace is created) + --create-automount-resources Whether to create automount resources (default: false) + --dwo-namespace DevWorkspace Operator namespace (default: loadtest-devworkspaces) + --logs-dir Directory name where DevWorkspace and event logs would be dumped + --test-duration-minutes Duration in minutes for which to run load tests (default: 25 minutes) + -h, --help Show this help message +EOF +} + +parse_arguments() { + while [[ $# -gt 0 ]]; do + case "$1" in + --mode) + MODE="$2"; shift 2;; + --max-vus) + MAX_VUS="$2"; shift 2;; + --separate-namespaces) + SEPARATE_NAMESPACES="$2"; shift 2;; + --devworkspace-ready-timeout-seconds) + DEV_WORKSPACE_READY_TIMEOUT_IN_SECONDS="$2"; shift 2;; + --devworkspace-link) + DEVWORKSPACE_LINK="$2"; shift 2;; + --create-automount-resources) + CREATE_AUTOMOUNT_RESOURCES="$2"; shift 2;; + --dwo-namespace) + LOAD_TEST_NAMESPACE="$2"; shift 2;; + --logs-dir) + LOGS_DIR="$2"; shift 2;; + --test-duration-minutes) + TEST_DURATION_IN_MINUTES="$2"; shift 2;; + -h|--help) + print_help; exit 0;; + *) + echo "❌ Unknown option: $1" + print_help; exit 1;; + esac + done +} + +create_namespace() { + echo "πŸ”§ Creating Namespace: $LOAD_TEST_NAMESPACE" + cat <> "${LOGS_DIR}/${TIMESTAMP}_events.log" 2>&1 & + PID_EVENTS_WATCH=$! + + kubectl get dw --watch --all-namespaces \ + >> "${LOGS_DIR}/${TIMESTAMP}_dw_watch.log" 2>&1 & + PID_DW_WATCH=$! +} + +stop_background_watchers() { + echo "πŸ›‘ Stopping background watchers..." + kill "$PID_EVENTS_WATCH" "$PID_DW_WATCH" 2>/dev/null || true +} + +install_k6_operator() { + echo "πŸ“¦ Installing k6 operator..." + curl -L "https://raw.githubusercontent.com/grafana/k6-operator/refs/tags/${K6_OPERATOR_VERSION}/bundle.yaml" | kubectl apply -f - + echo "⏳ Waiting until k6 operator deployment is ready..." + kubectl rollout status deployment/k6-operator-controller-manager -n k6-operator-system --timeout=300s +} + +create_k6_configmap() { + echo "🧩 Creating ConfigMap from script file: $K6_SCRIPT" + kubectl create configmap "$CONFIGMAP_NAME" \ + --from-file=script.js="$K6_SCRIPT" \ + --namespace "$LOAD_TEST_NAMESPACE" \ + --dry-run=client -o yaml | kubectl apply -f - +} + +delete_existing_testruns() { + echo "🧹 Deleting any existing K6 TestRun resources in namespace: $LOAD_TEST_NAMESPACE" + kubectl delete testrun --all -n "$LOAD_TEST_NAMESPACE" || true +} + +create_k6_test_run() { + echo "πŸš€ Creating K6 TestRun custom resource..." + cat </dev/null) + + if [[ "$stage" == "finished" ]]; then + echo "TestRun $K6_CR_NAME is finished." + break + fi + + if (( SECONDS >= end )); then + echo "Timeout waiting for TestRun $K6_CR_NAME to finish." + exit 1 + fi + + sleep "$INTERVAL" + done +} + +fetch_test_logs() { + K6_TEST_POD=$(kubectl get pod -l k6_cr=$K6_CR_NAME,runner=true -n "${LOAD_TEST_NAMESPACE}" -o jsonpath='{.items[0].metadata.name}') + echo "πŸ“œ Fetching logs from completed K6 test pod: $K6_TEST_POD" + kubectl logs "$K6_TEST_POD" -n "$LOAD_TEST_NAMESPACE" +} + +check_prerequisites() { + echo "πŸ” Checking prerequisites..." + + check_command "kubectl" "$MIN_KUBECTL_VERSION" + check_command "curl" "$MIN_CURL_VERSION" + + if [[ "$MODE" == "binary" ]]; then + check_command "k6" "$MIN_K6_VERSION" + fi +} + +check_command() { + local cmd="$1" + local min_version="$2" + local version + + if ! command -v "$cmd" &>/dev/null; then + echo "❌ Required command '$cmd' not found in PATH." + exit 1 + fi + + case "$cmd" in + kubectl) + version=$(kubectl version --client -o json | jq -r '.clientVersion.gitVersion' | sed 's/^v//') + ;; + curl) + version=$($cmd --version | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -n1) + ;; + k6) + version=$($cmd version | grep -oE '[0-9]+\.[0-9]+\.[0-9]+') + ;; + *) + version="0.0.0" + ;; + esac + + if ! version_gte "$version" "$min_version"; then + echo "❌ $cmd version $version is less than required $min_version" + exit 1 + else + echo "βœ… $cmd version $version (>= $min_version)" + fi +} + +version_gte() { + [ "$(printf '%s\n' "$2" "$1" | sort -V | head -n1)" = "$2" ] +} + +run_k6_binary_test() { + echo "πŸš€ Running k6 load test..." + IN_CLUSTER='false' \ + KUBE_TOKEN="${KUBE_TOKEN}" \ + KUBE_API="${KUBE_API}" \ + DWO_NAMESPACE="${DWO_NAMESPACE}" \ + CREATE_AUTOMOUNT_RESOURCES="${CREATE_AUTOMOUNT_RESOURCES}" \ + SEPARATE_NAMESPACES="${SEPARATE_NAMESPACES}" \ + LOAD_TEST_NAMESPACE="${LOAD_TEST_NAMESPACE}" \ + DEVWORKSPACE_LINK="${DEVWORKSPACE_LINK}" \ + MAX_VUS="${MAX_VUS}" \ + TEST_DURATION_IN_MINUTES="${TEST_DURATION_IN_MINUTES}" \ + DEV_WORKSPACE_READY_TIMEOUT_IN_SECONDS="${DEV_WORKSPACE_READY_TIMEOUT_IN_SECONDS}" \ + k6 run "${K6_SCRIPT}" + exit_code=$? + if [ $exit_code -ne 0 ]; then + echo "⚠️ k6 load test failed with exit code $exit_code. Proceeding to cleanup." + fi + return 0 +} + +main "$@" From 3a49e7342931cbc9886e2ac7591c1faf154930ad Mon Sep 17 00:00:00 2001 From: Rohan Kumar Date: Tue, 26 Aug 2025 02:22:05 +0530 Subject: [PATCH 2/6] add options for --max-devworkspaces and --delete-devworkspace-after-ready Signed-off-by: Rohan Kumar --- .ci/openshift_e2e.sh | 2 -- test/load/devworkspace_load_test.js | 30 +++++++++++++++++-- test/load/runk6.sh | 46 ++++++++++++++++++++++++++++- 3 files changed, 73 insertions(+), 5 deletions(-) diff --git a/.ci/openshift_e2e.sh b/.ci/openshift_e2e.sh index c55418f42..9b9b175b2 100755 --- a/.ci/openshift_e2e.sh +++ b/.ci/openshift_e2e.sh @@ -72,6 +72,4 @@ make install export CLEAN_UP_AFTER_SUITE="false" make test_e2e bumpLogs - -make test_load ARGS="--mode operator --max-vus 250 --separate-namespaces false --test-duration-minutes 25 --dwo-namespace devworkspace-controller --logs-dir ${ARTIFACT_DIR}/load-testing-logs" make uninstall diff --git a/test/load/devworkspace_load_test.js b/test/load/devworkspace_load_test.js index 45c54ab0f..34c3249b6 100644 --- a/test/load/devworkspace_load_test.js +++ b/test/load/devworkspace_load_test.js @@ -23,10 +23,12 @@ const inCluster = __ENV.IN_CLUSTER === 'true'; const apiServer = inCluster ? `https://kubernetes.default.svc` : __ENV.KUBE_API; const token = inCluster ? open('/var/run/secrets/kubernetes.io/serviceaccount/token') : __ENV.KUBE_TOKEN; const useSeparateNamespaces = __ENV.SEPARATE_NAMESPACES === "true"; +const deleteDevWorkspaceAfterReady = __ENV.DELETE_DEVWORKSPACE_AFTER_READY === "true"; const operatorNamespace = __ENV.DWO_NAMESPACE || 'openshift-operators'; const externalDevWorkspaceLink = __ENV.DEVWORKSPACE_LINK || ''; const shouldCreateAutomountResources = (__ENV.CREATE_AUTOMOUNT_RESOURCES || 'false') === 'true'; const maxVUs = Number(__ENV.MAX_VUS || 50); +const maxDevWorkspaces = Number(__ENV.MAX_DEVWORKSPACES || -1); const devWorkspaceReadyTimeout = Number(__ENV.DEV_WORKSPACE_READY_TIMEOUT_IN_SECONDS || 600); const autoMountConfigMapName = 'dwo-load-test-automount-configmap'; const autoMountSecretName = 'dwo-load-test-automount-secret'; @@ -87,6 +89,12 @@ export function setup() { } export default function () { + if (maxDevWorkspaces > 0) { + const totalDevWorkspaces = getDevWorkspacesFromApiServer().length; + if (totalDevWorkspaces > maxDevWorkspaces) { + return; + } + } const vuId = __VU; const iteration = __ITER; const crName = `dw-test-${vuId}-${iteration}`; @@ -104,7 +112,9 @@ export default function () { const devWorkspaceCreated = createNewDevWorkspace(namespace, vuId, iteration); if (devWorkspaceCreated) { waitUntilDevWorkspaceIsReady(vuId, crName, namespace); - deleteDevWorkspace(crName, namespace); + if (deleteDevWorkspaceAfterReady) { + deleteDevWorkspace(crName, namespace); + } } } catch (error) { console.error(`Load test for ${vuId}-${iteration} failed:`, error.message); @@ -232,7 +242,6 @@ function checkDevWorkspaceOperatorMetrics() { }); if (res.status !== 200) { - console.warn(`[DWO METRICS] Unable to fetch DevWorkspace Operator metrics from Kubernetes, got ${res.status}`); return; } @@ -434,6 +443,23 @@ function downloadAndParseExternalWorkspace(externalDevWorkspaceLink) { return manifest; } +function getDevWorkspacesFromApiServer() { + const basePath = useSeparateNamespaces + ? `${apiServer}/apis/workspace.devfile.io/v1alpha2/devworkspaces` + : `${apiServer}/apis/workspace.devfile.io/v1alpha2/namespaces/${loadTestNamespace}/devworkspaces`; + + const url = `${basePath}?labelSelector=${labelKey}%3D${labelType}`; + const res = http.get(url, { headers }); + + if (res.status !== 200) { + console.error(`Failed to fetch DevWorkspaces: ${res.status} ${res.statusText || ''}`); + return []; + } + + const body = JSON.parse(res.body); + return body.items.map((dw) => dw.metadata.name); +} + function generateDevWorkspaceToCreate(vuId, iteration, namespace) { const name = `dw-test-${vuId}-${iteration}`; let devWorkspace = {}; diff --git a/test/load/runk6.sh b/test/load/runk6.sh index 2bb16f88b..e60e050ec 100644 --- a/test/load/runk6.sh +++ b/test/load/runk6.sh @@ -16,6 +16,8 @@ DEVWORKSPACE_LINK="https://gist.githubusercontent.com/rohanKanojia/ecf625afaf3fe MAX_VUS="100" DEV_WORKSPACE_READY_TIMEOUT_IN_SECONDS="1200" SEPARATE_NAMESPACES="false" +DELETE_DEVWORKSPACE_AFTER_READY="true" +MAX_DEVWORKSPACES="-1" CREATE_AUTOMOUNT_RESOURCES="false" LOGS_DIR="logs" TEST_DURATION_IN_MINUTES="25" @@ -57,7 +59,9 @@ Usage: $0 [options] Options: --mode Mode to run the script (default: operator) --max-vus Number of virtual users for k6 (default: 100) + --max-devworkspaces Maximum number of DevWorkspaces to create (by default, it's not specified) --separate-namespaces Use separate namespaces for workspaces (default: false) + --delete-devworkspace-after-ready Delete DevWorkspace once it becomes Ready (default: true) --devworkspace-ready-timeout-seconds Timeout in seconds for workspace to become ready (default: 1200) --devworkspace-link DevWorkspace link (default: empty, opinionated DevWorkspace is created) --create-automount-resources Whether to create automount resources (default: false) @@ -77,6 +81,10 @@ parse_arguments() { MAX_VUS="$2"; shift 2;; --separate-namespaces) SEPARATE_NAMESPACES="$2"; shift 2;; + --max-devworkspaces) + MAX_DEVWORKSPACES="$2"; shift 2;; + --delete-devworkspace-after-ready) + DELETE_DEVWORKSPACE_AFTER_READY="$2"; shift 2;; --devworkspace-ready-timeout-seconds) DEV_WORKSPACE_READY_TIMEOUT_IN_SECONDS="$2"; shift 2;; --devworkspace-link) @@ -173,11 +181,40 @@ start_background_watchers() { kubectl get dw --watch --all-namespaces \ >> "${LOGS_DIR}/${TIMESTAMP}_dw_watch.log" 2>&1 & PID_DW_WATCH=$! + + log_failed_devworkspaces & + PID_FAILED_DW_POLL=$! +} + +log_failed_devworkspaces() { + echo "πŸ“„ Starting periodic failed DevWorkspaces report (every 10s)..." + + POLL_INTERVAL=10 # in seconds + ITERATIONS=$((((TEST_DURATION_IN_MINUTES-1) * 60) / POLL_INTERVAL)) + + for ((i = 0; i < ITERATIONS; i++)); do + OUTPUT=$(kubectl get devworkspaces --all-namespaces -o json | jq -r ' + .items[] + | select(.status.phase == "Failed") + | [ + .metadata.namespace, + .metadata.name, + .status.phase, + (.status.message // "No message") + ] + | @csv') + + if [ -n "$OUTPUT" ]; then + echo "$OUTPUT" > "${LOGS_DIR}/dw_failure_report.csv" + fi + + sleep "$POLL_INTERVAL" + done } stop_background_watchers() { echo "πŸ›‘ Stopping background watchers..." - kill "$PID_EVENTS_WATCH" "$PID_DW_WATCH" 2>/dev/null || true + kill "$PID_EVENTS_WATCH" "$PID_DW_WATCH" "$PID_FAILED_DW_POLL" 2>/dev/null || true } install_k6_operator() { @@ -235,6 +272,10 @@ spec: value: '${TEST_DURATION_IN_MINUTES}' - name: DEV_WORKSPACE_READY_TIMEOUT_IN_SECONDS value: '${DEV_WORKSPACE_READY_TIMEOUT_IN_SECONDS}' + - name: DELETE_DEVWORKSPACE_AFTER_READY + value: '${DELETE_DEVWORKSPACE_AFTER_READY}' + - name: MAX_DEVWORKSPACES + value: '${MAX_DEVWORKSPACES}' EOF } @@ -330,6 +371,8 @@ run_k6_binary_test() { MAX_VUS="${MAX_VUS}" \ TEST_DURATION_IN_MINUTES="${TEST_DURATION_IN_MINUTES}" \ DEV_WORKSPACE_READY_TIMEOUT_IN_SECONDS="${DEV_WORKSPACE_READY_TIMEOUT_IN_SECONDS}" \ + DELETE_DEVWORKSPACE_AFTER_READY="${DELETE_DEVWORKSPACE_AFTER_READY}" \ + MAX_DEVWORKSPACES="${MAX_DEVWORKSPACES}" \ k6 run "${K6_SCRIPT}" exit_code=$? if [ $exit_code -ne 0 ]; then @@ -338,4 +381,5 @@ run_k6_binary_test() { return 0 } +trap stop_background_watchers EXIT main "$@" From 6ca8290f7077033fb73981da0eac194300011385 Mon Sep 17 00:00:00 2001 From: Rohan Kumar Date: Tue, 25 Nov 2025 23:13:14 +0530 Subject: [PATCH 3/6] fix : add logic to abort test execution when max devworkspaces target is reached Instead of returning from each VU iteration after checking number of currently running DevWorkspaces, abort the test execution to save time and reduce load on K8s apiserver. Signed-off-by: Rohan Kumar --- test/load/devworkspace_load_test.js | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/test/load/devworkspace_load_test.js b/test/load/devworkspace_load_test.js index 34c3249b6..434d469c9 100644 --- a/test/load/devworkspace_load_test.js +++ b/test/load/devworkspace_load_test.js @@ -16,6 +16,7 @@ import http from 'k6/http'; import {check, sleep} from 'k6'; import {Trend, Counter} from 'k6/metrics'; +import { test } from 'k6/execution'; import {htmlReport} from "https://raw.githubusercontent.com/benc-uk/k6-reporter/main/dist/bundle.js"; import {textSummary} from "https://jslib.k6.io/k6-summary/0.0.1/index.js"; @@ -90,8 +91,23 @@ export function setup() { export default function () { if (maxDevWorkspaces > 0) { - const totalDevWorkspaces = getDevWorkspacesFromApiServer().length; - if (totalDevWorkspaces > maxDevWorkspaces) { + const devWorkspaces = getDevWorkspacesFromApiServer(); + const totalDevWorkspaces = devWorkspaces.length; + + const runningDevWorkspaces = devWorkspaces.filter( + (dw) => dw.status && dw.status.phase === 'Running' + ).length; + const startingDevWorkspaces = devWorkspaces.filter( + (dw) => dw.status && dw.status.phase === 'Starting' + ).length; + + if (startingDevWorkspaces === 0 && runningDevWorkspaces >= maxDevWorkspaces) { + test.abort( + 'Max concurrent DevWorkspaces target achieved (no Starting workspaces), stopping test gracefully', + { abortOnFail: false } + ); + } else if (totalDevWorkspaces >= maxDevWorkspaces) { + // stop further creation, but don’t abort the test return; } } @@ -457,7 +473,7 @@ function getDevWorkspacesFromApiServer() { } const body = JSON.parse(res.body); - return body.items.map((dw) => dw.metadata.name); + return body.items; } function generateDevWorkspaceToCreate(vuId, iteration, namespace) { From e948c8a4f6eb2f73efa12a4a2936ea14574c1812 Mon Sep 17 00:00:00 2001 From: Rohan Kumar Date: Wed, 3 Dec 2025 16:13:28 +0530 Subject: [PATCH 4/6] fix : use shared-iterations executor to run an exact number of devworkspace iterations The previous ramping-vus configuration generated inconsistent iteration counts due to VU loop timing. Switched to shared-iterations, configured with `vus: maxVUs` and `iterations: maxDevWorkspaces`, ensuring the test creates exactly the intended number of devworkspaces. Signed-off-by: Rohan Kumar --- test/load/devworkspace_load_test.js | 33 ++++++++++++++++++++--------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/test/load/devworkspace_load_test.js b/test/load/devworkspace_load_test.js index 434d469c9..1bfe43c08 100644 --- a/test/load/devworkspace_load_test.js +++ b/test/load/devworkspace_load_test.js @@ -45,17 +45,18 @@ const headers = { export const options = { scenarios: { create_and_delete_devworkspaces: { - executor: 'ramping-vus', - startVUs: 0, - stages: generateLoadTestStages(maxVUs), - gracefulRampDown: '1m', + executor: 'shared-iterations', + vus: maxVUs, + iterations: maxDevWorkspaces, + maxDuration: '3h', }, final_cleanup: { executor: 'per-vu-iterations', vus: 1, iterations: 1, - startTime: `${loadTestDurationInMinutes}m`, exec: 'final_cleanup', + startTime: '0s', + maxDuration: '3h', }, }, thresholds: { 'checks': ['rate>0.95'], @@ -91,7 +92,10 @@ export function setup() { export default function () { if (maxDevWorkspaces > 0) { - const devWorkspaces = getDevWorkspacesFromApiServer(); + const { error, devWorkspaces } = getDevWorkspacesFromApiServer(); + if (error) { + return; + } const totalDevWorkspaces = devWorkspaces.length; const runningDevWorkspaces = devWorkspaces.filter( @@ -468,12 +472,21 @@ function getDevWorkspacesFromApiServer() { const res = http.get(url, { headers }); if (res.status !== 200) { - console.error(`Failed to fetch DevWorkspaces: ${res.status} ${res.statusText || ''}`); - return []; + const errorMsg = `Failed to fetch DevWorkspaces: ${res.status} ${res.statusText || ''}`; + console.error(errorMsg); + + return { + error: errorMsg, + devWorkspaces: null, + }; } const body = JSON.parse(res.body); - return body.items; + + return { + error: null, + devWorkspaces: body.items, + }; } function generateDevWorkspaceToCreate(vuId, iteration, namespace) { @@ -523,4 +536,4 @@ function parseCpuToMillicores(cpuStr) { if (cpuStr.endsWith("u")) return Math.round(parseInt(cpuStr) / 1e3); if (cpuStr.endsWith("m")) return parseInt(cpuStr); return Math.round(parseFloat(cpuStr) * 1000); -} \ No newline at end of file +} From 0ef22dcc951ced737e34111accc0fca29aa6c5e2 Mon Sep 17 00:00:00 2001 From: Rohan Kumar Date: Mon, 8 Dec 2025 21:12:44 +0530 Subject: [PATCH 5/6] test : add metrics to track etcd cpu and memory usage Signed-off-by: Rohan Kumar --- test/load/devworkspace_load_test.js | 81 ++++++++++++++++++++++++++++- 1 file changed, 80 insertions(+), 1 deletion(-) diff --git a/test/load/devworkspace_load_test.js b/test/load/devworkspace_load_test.js index 1bfe43c08..ea8662625 100644 --- a/test/load/devworkspace_load_test.js +++ b/test/load/devworkspace_load_test.js @@ -76,6 +76,8 @@ const devworkspaceReadyDuration = new Trend('devworkspace_ready_duration'); const devworkspaceReadyFailed = new Counter('devworkspace_ready_failed'); const operatorCpu = new Trend('average_operator_cpu'); // in milli cores const operatorMemory = new Trend('average_operator_memory'); // in Mi +const etcdCpu = new Trend('average_etcd_cpu'); // in milli cores +const etcdMemory = new Trend('average_etcd_memory'); // in Mi const devworkspacesCreated = new Counter('devworkspace_create_count'); const operatorCpuViolations = new Counter('operator_cpu_violations'); const operatorMemViolations = new Counter('operator_mem_violations'); @@ -83,7 +85,32 @@ const operatorMemViolations = new Counter('operator_mem_violations'); const maxCpuMillicores = 250; const maxMemoryBytes = 200 * 1024 * 1024; +let etcdNamespace = 'openshift-etcd'; +let etcdPodNamePattern = 'etcd'; + +function detectClusterType() { + const apiGroupsUrl = `${apiServer}/apis`; + const res = http.get(apiGroupsUrl, {headers}); + + if (res.status === 200) { + try { + const data = JSON.parse(res.body); + const groups = data.groups || []; + const hasOpenShiftRoutes = groups.some(g => g.name === 'route.openshift.io'); + + if (!hasOpenShiftRoutes) { + etcdNamespace = __ENV.ETCD_NAMESPACE || 'kube-system'; + etcdPodNamePattern = __ENV.ETCD_POD_NAME_PATTERN || 'kube-proxy'; + console.log('Detected Kubernetes cluster - using kube-system namespace with kube-proxy'); + } + } catch (e) { + console.warn(`Failed to detect cluster type: ${e.message}, using defaults`); + } + } +} + export function setup() { + detectClusterType(); if (shouldCreateAutomountResources) { createNewAutomountConfigMap(); createNewAutomountSecret(); @@ -155,7 +182,7 @@ export function final_cleanup() { } export function handleSummary(data) { - const allowed = ['devworkspace_create_count', 'devworkspace_create_duration', 'devworkspace_delete_duration', 'devworkspace_ready_duration', 'devworkspace_ready', 'devworkspace_ready_failed', 'operator_cpu_violations', 'operator_mem_violations', 'average_operator_cpu', 'average_operator_memory']; + const allowed = ['devworkspace_create_count', 'devworkspace_create_duration', 'devworkspace_delete_duration', 'devworkspace_ready_duration', 'devworkspace_ready', 'devworkspace_ready_failed', 'operator_cpu_violations', 'operator_mem_violations', 'average_operator_cpu', 'average_operator_memory', 'etcd_cpu_violations', 'etcd_mem_violations', 'average_etcd_cpu', 'average_etcd_memory']; const filteredData = JSON.parse(JSON.stringify(data)); for (const key of Object.keys(filteredData.metrics)) { @@ -227,6 +254,7 @@ function waitUntilDevWorkspaceIsReady(vuId, crName, namespace) { } checkDevWorkspaceOperatorMetrics(); + checkEtcdMetrics(); sleep(pollWaitInterval); attempts++; } @@ -295,6 +323,57 @@ function checkDevWorkspaceOperatorMetrics() { } } +function checkEtcdMetrics() { + if (!etcdNamespace || !etcdPodNamePattern) { + console.warn(`[ETCD METRICS] Variables not initialized: etcdNamespace=${etcdNamespace}, etcdPodNamePattern=${etcdPodNamePattern}`); + return; + } + + const metricsUrl = `${apiServer}/apis/metrics.k8s.io/v1beta1/namespaces/${etcdNamespace}/pods`; + const res = http.get(metricsUrl, {headers}); + + check(res, { + 'Fetched etcd pod metrics successfully': (r) => r.status === 200, + }); + + if (res.status !== 200) { + return; + } + + const data = JSON.parse(res.body); + const etcdPods = data.items.filter(p => p.metadata.name.includes(etcdPodNamePattern)); + + if (etcdPods.length === 0) { + if (data.items && data.items.length > 0) { + const podNames = data.items.map(p => p.metadata.name).join(', '); + console.warn(`[ETCD METRICS] No pods found matching pattern '${etcdPodNamePattern}' in namespace '${etcdNamespace}'. Available pods: ${podNames}`); + } else { + console.warn(`[ETCD METRICS] No pods found in namespace '${etcdNamespace}'`); + } + return; + } + + for (const pod of etcdPods) { + if (!pod.containers || pod.containers.length === 0) { + console.warn(`[ETCD METRICS] Pod ${pod.metadata.name} has no containers`); + continue; + } + const container = pod.containers[0]; + const name = pod.metadata.name; + + if (!container.usage || !container.usage.cpu || !container.usage.memory) { + console.warn(`[ETCD METRICS] Pod ${name} has no usage data:`, JSON.stringify(container.usage)); + continue; + } + + const cpu = parseCpuToMillicores(container.usage.cpu); + const memory = parseMemoryToBytes(container.usage.memory); + + etcdCpu.add(cpu); + etcdMemory.add(memory / 1024 / 1024); + } +} + function createNewNamespace(namespaceName) { const url = `${apiServer}/api/v1/namespaces`; From 95bd4288a54135cf27ff8df1261d8bbccb91d185 Mon Sep 17 00:00:00 2001 From: Rohan Kumar Date: Wed, 17 Dec 2025 16:53:24 +0530 Subject: [PATCH 6/6] test : create large ca-bundle configmap in che installation namespace before running load tests Signed-off-by: Rohan Kumar --- test/load/README.md | 119 ++++++++++ test/load/che-cert-bundle-utils.sh | 223 ++++++++++++++++++ test/load/devworkspace_load_test.js | 20 +- .../load/provision-che-workspace-namespace.sh | 52 ++++ test/load/runk6.sh | 24 +- 5 files changed, 428 insertions(+), 10 deletions(-) create mode 100644 test/load/README.md create mode 100644 test/load/che-cert-bundle-utils.sh create mode 100644 test/load/provision-che-workspace-namespace.sh diff --git a/test/load/README.md b/test/load/README.md new file mode 100644 index 000000000..be188de5a --- /dev/null +++ b/test/load/README.md @@ -0,0 +1,119 @@ +# Load Testing for DevWorkspace Operator + +This directory contains load testing tools for the DevWorkspace Operator using k6. The tests create multiple DevWorkspaces concurrently to measure the operator's performance under load. + +## Prerequisites + +- `kubectl` (version >= 1.24.0) +- `curl` (version >= 7.0.0) +- `k6` (version >= 1.1.0) - Required when using `--mode binary` +- Access to a Kubernetes cluster with DevWorkspace Operator installed +- Proper RBAC permissions to create DevWorkspaces, ConfigMaps, Secrets, and Namespaces + +## Running Load Tests + +The load tests can be run using the `make test_load` target with various arguments. The tests support two modes: +- **binary mode**: Runs k6 locally (default) +- **operator mode**: Runs k6 using the k6-operator in the cluster + +### Running with Eclipse Che + +When running with Eclipse Che, the script automatically provisions additional ConfigMaps for certificates that are required for Che workspaces to function properly. + +```bash +make test_load ARGS=" \ + --mode binary \ + --run-with-eclipse-che true \ + --max-vus ${MAX_VUS} \ + --create-automount-resources true \ + --max-devworkspaces ${MAX_DEVWORKSPACES} \ + --devworkspace-ready-timeout-seconds 3600 \ + --delete-devworkspace-after-ready false \ + --separate-namespaces false \ + --test-duration-minutes 40" +``` + +**Note**: When `--run-with-eclipse-che true` is set, the script will: +- Provision a workspace namespace compatible with Eclipse Che +- Create additional certificate ConfigMaps required by Che + +### Running without Eclipse Che + +When running without Eclipse Che, the standard namespace setup is used without additional certificate ConfigMaps. + +```bash +make test_load ARGS=" \ + --mode binary \ + --max-vus ${MAX_VUS} \ + --create-automount-resources true \ + --max-devworkspaces ${MAX_DEVWORKSPACES} \ + --devworkspace-ready-timeout-seconds 3600 \ + --delete-devworkspace-after-ready false \ + --separate-namespaces false \ + --test-duration-minutes 40" +``` + +## Available Parameters + +| Parameter | Description | Default | Example | +|-----------|-------------|---------|---------| +| `--mode` | Execution mode: `binary` or `operator` | `binary` | `--mode binary` | +| `--max-vus` | Maximum number of virtual users (concurrent DevWorkspace creations) | `100` | `--max-vus 50` | +| `--max-devworkspaces` | Maximum number of DevWorkspaces to create (-1 for unlimited) | `-1` | `--max-devworkspaces 200` | +| `--separate-namespaces` | Create each DevWorkspace in its own namespace | `false` | `--separate-namespaces true` | +| `--delete-devworkspace-after-ready` | Delete DevWorkspace once it becomes Ready | `true` | `--delete-devworkspace-after-ready false` | +| `--devworkspace-ready-timeout-seconds` | Timeout in seconds for workspace to become ready | `1200` | `--devworkspace-ready-timeout-seconds 3600` | +| `--devworkspace-link` | URL to external DevWorkspace JSON to use instead of default | (empty) | `--devworkspace-link https://...` | +| `--create-automount-resources` | Create automount ConfigMap and Secret for testing | `false` | `--create-automount-resources true` | +| `--dwo-namespace` | DevWorkspace Operator namespace | `openshift-operators` | `--dwo-namespace devworkspace-controller` | +| `--logs-dir` | Directory for DevWorkspace and event logs | `logs` | `--logs-dir /tmp/test-logs` | +| `--test-duration-minutes` | Duration in minutes for the load test | `25` | `--test-duration-minutes 40` | +| `--run-with-eclipse-che` | Enable Eclipse Che integration (adds certificate ConfigMaps) | `false` | `--run-with-eclipse-che true` | +| `--che-cluster-name` | Eclipse Che cluster name (when using Che) | `eclipse-che` | `--che-cluster-name my-che` | +| `--che-namespace` | Eclipse Che namespace (when using Che) | `eclipse-che` | `--che-namespace my-che-ns` | + +## What the Tests Do + +1. **Setup**: Creates a test namespace, ServiceAccount, and RBAC resources +2. **Eclipse Che Setup** (if enabled): Provisions Che-compatible namespace and certificate ConfigMaps +3. **Load Generation**: Creates DevWorkspaces concurrently based on `--max-devworkspaces` +4. **Monitoring**: + - Watches DevWorkspace status until Ready + - Monitors operator CPU and memory usage + - Tracks etcd metrics + - Logs events and DevWorkspace state changes +5. **Cleanup**: Removes all created resources and test namespace + +## Test Metrics + +The tests track the following metrics: +- DevWorkspace creation duration +- DevWorkspace ready duration +- DevWorkspace deletion duration +- Operator CPU and memory usage +- etcd CPU and memory usage +- Success/failure rates + +## Output + +- **Logs**: Stored in the `logs/` directory (or custom directory specified by `--logs-dir`) + - `{timestamp}_events.log`: Kubernetes events + - `{timestamp}_dw_watch.log`: DevWorkspace watch logs + - `dw_failure_report.csv`: Failed DevWorkspaces report +- **HTML Report**: Generated when running in binary mode (outside cluster) +- **Console Output**: Real-time test progress and summary + +## Troubleshooting + +- **Permission errors**: Ensure your kubeconfig has sufficient RBAC permissions +- **Timeout errors**: Increase `--devworkspace-ready-timeout-seconds` for slower clusters +- **Resource exhaustion**: Reduce `--max-vus` or `--max-devworkspaces` if cluster resources are limited +- **k6 not found**: Install k6 from https://k6.io/docs/getting-started/installation/ + +## Additional Notes + +- The tests use an opinionated minimal DevWorkspace by default, or you can provide a custom one via `--devworkspace-link` +- When `--separate-namespaces true` is used, each DevWorkspace gets its own namespace +- The `--delete-devworkspace-after-ready false` option is useful for testing sustained load scenarios +- Certificate ConfigMaps are only created when `--run-with-eclipse-che true` is set + diff --git a/test/load/che-cert-bundle-utils.sh b/test/load/che-cert-bundle-utils.sh new file mode 100644 index 000000000..ee0653e29 --- /dev/null +++ b/test/load/che-cert-bundle-utils.sh @@ -0,0 +1,223 @@ +#!/usr/bin/env bash +set -euo pipefail + +log_info() { echo -e "ℹ️ $*" >&2; } +log_success() { echo -e "βœ… $*" >&2; } +log_error() { echo -e "❌ $*" >&2; } + + +run_che_ca_bundle_e2e() { + local che_ns="$1" + local dw_ns="$2" + local dw_name="$3" + local cert_count="${4:-500}" + local bundle_file="${5:-custom-ca-certificates.pem}" + + check_namespaces "${che_ns}" "${dw_ns}" + generate_dummy_certs "${cert_count}" "${bundle_file}" + create_che_ca_configmap "${che_ns}" "${bundle_file}" + patch_checluster_disable_pki_mount "${che_ns}" + restart_che "${che_ns}" + create_devworkspace "${dw_ns}" "${dw_name}" + + local pod + pod=$(wait_for_workspace_pod "${dw_ns}" "${dw_name}") + + verify_ca_bundle_in_workspace "${pod}" "${dw_ns}" "${cert_count}" + cleanup_resources "${dw_ns}" "${dw_name}" +} + +check_namespaces() { + local che_ns="$1" + local dw_ns="$2" + + log_info "Checking namespaces..." + kubectl get ns "${che_ns}" >/dev/null + kubectl get ns "${dw_ns}" >/dev/null +} + +generate_dummy_certs() { + local cert_count="$1" + local bundle_file="$2" + + log_info "Generating ${cert_count} dummy CA certificates..." + rm -f "${bundle_file}" + + for i in $(seq 1 "${cert_count}"); do + openssl req -x509 -newkey rsa:2048 -nodes -days 1 \ + -subj "/CN=dummy-ca-${i}" \ + -keyout "dummy-ca-${i}.key" \ + -out "dummy-ca-${i}.pem" \ + >/dev/null 2>&1 + + cat "dummy-ca-${i}.pem" >> "${bundle_file}" + done + + log_success "Created CA bundle: $(du -h "${bundle_file}" | cut -f1)" +} + +create_che_ca_configmap() { + local che_ns="$1" + local bundle_file="$2" + + log_info "Creating Che CA bundle ConfigMap..." + + kubectl create configmap custom-ca-certificates \ + --from-file=custom-ca-certificates.pem="${bundle_file}" \ + -n "${che_ns}" \ + --dry-run=client -o yaml \ + | kubectl apply --server-side -f - + + kubectl label configmap custom-ca-certificates \ + app.kubernetes.io/component=ca-bundle \ + app.kubernetes.io/part-of=che.eclipse.org \ + -n "${che_ns}" \ + --overwrite +} + +patch_checluster_disable_pki_mount() { + local che_ns="$1" + + log_info "Configuring CheCluster..." + local checluster + checluster=$(kubectl get checluster -n "${che_ns}" -o jsonpath='{.items[0].metadata.name}') + + kubectl patch checluster "${checluster}" \ + -n "${che_ns}" \ + --type=merge \ + -p '{ + "spec": { + "devEnvironments": { + "trustedCerts": { + "disableWorkspaceCaBundleMount": true + } + } + } + }' +} + +restart_che() { + local che_ns="$1" + + log_info "Restarting Che..." + kubectl rollout status deploy/che -n "${che_ns}" --timeout=5m + kubectl wait pod -n "${che_ns}" -l app=che --for=condition=Ready --timeout=5m + + log_success "Che restarted" +} + +create_devworkspace() { + local dw_ns="$1" + local dw_name="$2" + + log_info "Creating DevWorkspace '${dw_name}'..." + cat </dev/stderr + + pod_name=$(kubectl get pod \ + -n "${dw_ns}" \ + -l controller.devfile.io/devworkspace_name="${dw_name}" \ + -o jsonpath='{.items[0].metadata.name}') + + echo "${pod_name}" +} + +verify_ca_bundle_in_workspace() { + local pod_name="$1" + local dw_ns="$2" + local expected_count="$3" + local cert_path="/public-certs/tls-ca-bundle.pem" + + log_info "Verifying CA bundle in workspace..." + + kubectl exec "${pod_name}" -n "${dw_ns}" -- test -f "${cert_path}" + + local mounted_count + mounted_count=$(kubectl exec "${pod_name}" -n "${dw_ns}" -- \ + sh -c "grep -c 'BEGIN CERTIFICATE' ${cert_path}") + + log_info "Generated certificates : ${expected_count}" + log_info "Mounted certificates : ${mounted_count}" + + if [[ "${mounted_count}" -le "${expected_count}" ]]; then + log_error "Mounted certificate count validation failed" + return 1 + fi + + log_success "CA bundle verification passed" +} + +cleanup_resources() { + local dw_ns="$1" + local dw_name="$2" + + log_info "Cleaning up..." + kubectl delete dw "${dw_name}" -n "${dw_ns}" --ignore-not-found + rm -f *.pem *.key +} diff --git a/test/load/devworkspace_load_test.js b/test/load/devworkspace_load_test.js index ea8662625..dc33fe524 100644 --- a/test/load/devworkspace_load_test.js +++ b/test/load/devworkspace_load_test.js @@ -50,14 +50,6 @@ export const options = { iterations: maxDevWorkspaces, maxDuration: '3h', }, - final_cleanup: { - executor: 'per-vu-iterations', - vus: 1, - iterations: 1, - exec: 'final_cleanup', - startTime: '0s', - maxDuration: '3h', - }, }, thresholds: { 'checks': ['rate>0.95'], 'devworkspace_create_duration': ['p(95)<15000'], @@ -203,6 +195,11 @@ export function handleSummary(data) { return loadTestSummaryReport; } +export function teardown(data) { + console.log("Running final cleanup after all DevWorkspace creation finished..."); + final_cleanup(); +} + function createNewDevWorkspace(namespace, vuId, iteration) { const baseUrl = `${apiServer}/apis/workspace.devfile.io/v1alpha2/namespaces/${namespace}/devworkspaces`; @@ -259,6 +256,13 @@ function waitUntilDevWorkspaceIsReady(vuId, crName, namespace) { attempts++; } + if (!isReady && attempts >= maxAttempts) { + console.error( + `GET [VU ${vuId}] Timed out waiting for DevWorkspace '${crName}' in namespace '${namespace}' ` + + `after ${attempts} attempts (${devWorkspaceReadyTimeout}s). Last known phase: '${lastPhase}'` + ); + } + if (res.status === 200) { if (isReady) { devworkspaceReady.add(1); diff --git a/test/load/provision-che-workspace-namespace.sh b/test/load/provision-che-workspace-namespace.sh new file mode 100644 index 000000000..3f011677c --- /dev/null +++ b/test/load/provision-che-workspace-namespace.sh @@ -0,0 +1,52 @@ +provision_che_workspace_namespace() { + local LOAD_TEST_NAMESPACE="$1" + local CHE_NAMESPACE="$2" + local CHE_CLUSTER_NAME="$3" + + if [[ -z "${LOAD_TEST_NAMESPACE}" ]]; then + echo "ERROR: LOAD_TEST_NAMESPACE argument is required" + echo "Usage: provision_che_workspace_namespace " + return 1 + fi + + if ! command -v oc >/dev/null 2>&1; then + echo "ERROR: oc CLI not found" + return 1 + fi + + local USERNAME + USERNAME="$(oc whoami)" + + echo "Provisioning Che workspace namespace" + echo " User : ${USERNAME}" + echo " Namespace : ${LOAD_TEST_NAMESPACE}" + + oc patch checluster "${CHE_CLUSTER_NAME}" \ + -n "${CHE_NAMESPACE}" \ + --type=merge \ + -p '{ + "spec": { + "devEnvironments": { + "defaultNamespace": { + "autoProvision": false + } + } + } + }' >/dev/null + + cat </dev/null + + echo "βœ” Namespace '${LOAD_TEST_NAMESPACE}' provisioned for user '${USERNAME}'" +} diff --git a/test/load/runk6.sh b/test/load/runk6.sh index e60e050ec..6160c2cfa 100644 --- a/test/load/runk6.sh +++ b/test/load/runk6.sh @@ -1,6 +1,8 @@ #!/bin/bash -#!/bin/bash +source test/load/provision-che-workspace-namespace.sh +source test/load/che-cert-bundle-utils.sh + MODE="binary" # or 'operator' LOAD_TEST_NAMESPACE="loadtest-devworkspaces" @@ -19,17 +21,26 @@ SEPARATE_NAMESPACES="false" DELETE_DEVWORKSPACE_AFTER_READY="true" MAX_DEVWORKSPACES="-1" CREATE_AUTOMOUNT_RESOURCES="false" +RUN_WITH_ECLIPSE_CHE="false" LOGS_DIR="logs" TEST_DURATION_IN_MINUTES="25" MIN_KUBECTL_VERSION="1.24.0" MIN_CURL_VERSION="7.0.0" MIN_K6_VERSION="1.1.0" +CHE_NAMESPACE="eclipse-che" +CHE_CLUSTER_NAME="eclipse-che" +TEST_CERTIFICATES_COUNT="500" # ----------- Main Execution Flow ----------- main() { parse_arguments "$@" check_prerequisites - create_namespace + if [[ "$RUN_WITH_ECLIPSE_CHE" == "false" ]]; then + create_namespace + else + provision_che_workspace_namespace "$LOAD_TEST_NAMESPACE" "$CHE_NAMESPACE" "$CHE_CLUSTER_NAME" + run_che_ca_bundle_e2e "$CHE_NAMESPACE" "$LOAD_TEST_NAMESPACE" "test-devworkspace" "$TEST_CERTIFICATES_COUNT" + fi create_rbac start_background_watchers @@ -68,6 +79,9 @@ Options: --dwo-namespace DevWorkspace Operator namespace (default: loadtest-devworkspaces) --logs-dir Directory name where DevWorkspace and event logs would be dumped --test-duration-minutes Duration in minutes for which to run load tests (default: 25 minutes) + --run-with-eclipse-che Whether these tests are supposed to be run with Eclipse Che (If yes additional certificates are mounted) + --che-cluster-name Applicable if running on Eclipse Che, defaults to 'eclipse-che' + --che-namespace Applicable if running on Eclipse Che, defaults to 'eclipse-che' -h, --help Show this help message EOF } @@ -97,6 +111,12 @@ parse_arguments() { LOGS_DIR="$2"; shift 2;; --test-duration-minutes) TEST_DURATION_IN_MINUTES="$2"; shift 2;; + --run-with-eclipse-che) + RUN_WITH_ECLIPSE_CHE="$2"; shift 2;; + --che-cluster-name) + CHE_CLUSTER_NAME="$2"; shift 2;; + --che-namespace) + CHE_NAMESPACE="$2"; shift 2;; -h|--help) print_help; exit 0;; *)