diff --git a/Makefile b/Makefile index d7053812b..c6f6de6e5 100644 --- a/Makefile +++ b/Makefile @@ -180,6 +180,11 @@ test_e2e_debug: mkdir -p /tmp/artifacts dlv test --listen=:2345 --headless=true --api-version=2 ./test/e2e/cmd/workspaces_test.go -- --ginkgo.fail-fast --ginkgo.junit-report=/tmp/artifacts/junit-workspaces-operator.xml +test_load: + @echo "Starting Load Testing Script..." && \ + bash ./test/load/runk6.sh $(ARGS) && \ + echo "Done" + ### manager: Build manager binary manager: generate fmt vet go build -o bin/manager main.go diff --git a/test/load/README.md b/test/load/README.md new file mode 100644 index 000000000..be188de5a --- /dev/null +++ b/test/load/README.md @@ -0,0 +1,119 @@ +# Load Testing for DevWorkspace Operator + +This directory contains load testing tools for the DevWorkspace Operator using k6. The tests create multiple DevWorkspaces concurrently to measure the operator's performance under load. + +## Prerequisites + +- `kubectl` (version >= 1.24.0) +- `curl` (version >= 7.0.0) +- `k6` (version >= 1.1.0) - Required when using `--mode binary` +- Access to a Kubernetes cluster with DevWorkspace Operator installed +- Proper RBAC permissions to create DevWorkspaces, ConfigMaps, Secrets, and Namespaces + +## Running Load Tests + +The load tests can be run using the `make test_load` target with various arguments. The tests support two modes: +- **binary mode**: Runs k6 locally (default) +- **operator mode**: Runs k6 using the k6-operator in the cluster + +### Running with Eclipse Che + +When running with Eclipse Che, the script automatically provisions additional ConfigMaps for certificates that are required for Che workspaces to function properly. + +```bash +make test_load ARGS=" \ + --mode binary \ + --run-with-eclipse-che true \ + --max-vus ${MAX_VUS} \ + --create-automount-resources true \ + --max-devworkspaces ${MAX_DEVWORKSPACES} \ + --devworkspace-ready-timeout-seconds 3600 \ + --delete-devworkspace-after-ready false \ + --separate-namespaces false \ + --test-duration-minutes 40" +``` + +**Note**: When `--run-with-eclipse-che true` is set, the script will: +- Provision a workspace namespace compatible with Eclipse Che +- Create additional certificate ConfigMaps required by Che + +### Running without Eclipse Che + +When running without Eclipse Che, the standard namespace setup is used without additional certificate ConfigMaps. + +```bash +make test_load ARGS=" \ + --mode binary \ + --max-vus ${MAX_VUS} \ + --create-automount-resources true \ + --max-devworkspaces ${MAX_DEVWORKSPACES} \ + --devworkspace-ready-timeout-seconds 3600 \ + --delete-devworkspace-after-ready false \ + --separate-namespaces false \ + --test-duration-minutes 40" +``` + +## Available Parameters + +| Parameter | Description | Default | Example | +|-----------|-------------|---------|---------| +| `--mode` | Execution mode: `binary` or `operator` | `binary` | `--mode binary` | +| `--max-vus` | Maximum number of virtual users (concurrent DevWorkspace creations) | `100` | `--max-vus 50` | +| `--max-devworkspaces` | Maximum number of DevWorkspaces to create (-1 for unlimited) | `-1` | `--max-devworkspaces 200` | +| `--separate-namespaces` | Create each DevWorkspace in its own namespace | `false` | `--separate-namespaces true` | +| `--delete-devworkspace-after-ready` | Delete DevWorkspace once it becomes Ready | `true` | `--delete-devworkspace-after-ready false` | +| `--devworkspace-ready-timeout-seconds` | Timeout in seconds for workspace to become ready | `1200` | `--devworkspace-ready-timeout-seconds 3600` | +| `--devworkspace-link` | URL to external DevWorkspace JSON to use instead of default | (empty) | `--devworkspace-link https://...` | +| `--create-automount-resources` | Create automount ConfigMap and Secret for testing | `false` | `--create-automount-resources true` | +| `--dwo-namespace` | DevWorkspace Operator namespace | `openshift-operators` | `--dwo-namespace devworkspace-controller` | +| `--logs-dir` | Directory for DevWorkspace and event logs | `logs` | `--logs-dir /tmp/test-logs` | +| `--test-duration-minutes` | Duration in minutes for the load test | `25` | `--test-duration-minutes 40` | +| `--run-with-eclipse-che` | Enable Eclipse Che integration (adds certificate ConfigMaps) | `false` | `--run-with-eclipse-che true` | +| `--che-cluster-name` | Eclipse Che cluster name (when using Che) | `eclipse-che` | `--che-cluster-name my-che` | +| `--che-namespace` | Eclipse Che namespace (when using Che) | `eclipse-che` | `--che-namespace my-che-ns` | + +## What the Tests Do + +1. **Setup**: Creates a test namespace, ServiceAccount, and RBAC resources +2. **Eclipse Che Setup** (if enabled): Provisions Che-compatible namespace and certificate ConfigMaps +3. **Load Generation**: Creates DevWorkspaces concurrently based on `--max-devworkspaces` +4. **Monitoring**: + - Watches DevWorkspace status until Ready + - Monitors operator CPU and memory usage + - Tracks etcd metrics + - Logs events and DevWorkspace state changes +5. **Cleanup**: Removes all created resources and test namespace + +## Test Metrics + +The tests track the following metrics: +- DevWorkspace creation duration +- DevWorkspace ready duration +- DevWorkspace deletion duration +- Operator CPU and memory usage +- etcd CPU and memory usage +- Success/failure rates + +## Output + +- **Logs**: Stored in the `logs/` directory (or custom directory specified by `--logs-dir`) + - `{timestamp}_events.log`: Kubernetes events + - `{timestamp}_dw_watch.log`: DevWorkspace watch logs + - `dw_failure_report.csv`: Failed DevWorkspaces report +- **HTML Report**: Generated when running in binary mode (outside cluster) +- **Console Output**: Real-time test progress and summary + +## Troubleshooting + +- **Permission errors**: Ensure your kubeconfig has sufficient RBAC permissions +- **Timeout errors**: Increase `--devworkspace-ready-timeout-seconds` for slower clusters +- **Resource exhaustion**: Reduce `--max-vus` or `--max-devworkspaces` if cluster resources are limited +- **k6 not found**: Install k6 from https://k6.io/docs/getting-started/installation/ + +## Additional Notes + +- The tests use an opinionated minimal DevWorkspace by default, or you can provide a custom one via `--devworkspace-link` +- When `--separate-namespaces true` is used, each DevWorkspace gets its own namespace +- The `--delete-devworkspace-after-ready false` option is useful for testing sustained load scenarios +- Certificate ConfigMaps are only created when `--run-with-eclipse-che true` is set + diff --git a/test/load/che-cert-bundle-utils.sh b/test/load/che-cert-bundle-utils.sh new file mode 100644 index 000000000..ee0653e29 --- /dev/null +++ b/test/load/che-cert-bundle-utils.sh @@ -0,0 +1,223 @@ +#!/usr/bin/env bash +set -euo pipefail + +log_info() { echo -e "ℹ️ $*" >&2; } +log_success() { echo -e "✅ $*" >&2; } +log_error() { echo -e "❌ $*" >&2; } + + +run_che_ca_bundle_e2e() { + local che_ns="$1" + local dw_ns="$2" + local dw_name="$3" + local cert_count="${4:-500}" + local bundle_file="${5:-custom-ca-certificates.pem}" + + check_namespaces "${che_ns}" "${dw_ns}" + generate_dummy_certs "${cert_count}" "${bundle_file}" + create_che_ca_configmap "${che_ns}" "${bundle_file}" + patch_checluster_disable_pki_mount "${che_ns}" + restart_che "${che_ns}" + create_devworkspace "${dw_ns}" "${dw_name}" + + local pod + pod=$(wait_for_workspace_pod "${dw_ns}" "${dw_name}") + + verify_ca_bundle_in_workspace "${pod}" "${dw_ns}" "${cert_count}" + cleanup_resources "${dw_ns}" "${dw_name}" +} + +check_namespaces() { + local che_ns="$1" + local dw_ns="$2" + + log_info "Checking namespaces..." + kubectl get ns "${che_ns}" >/dev/null + kubectl get ns "${dw_ns}" >/dev/null +} + +generate_dummy_certs() { + local cert_count="$1" + local bundle_file="$2" + + log_info "Generating ${cert_count} dummy CA certificates..." + rm -f "${bundle_file}" + + for i in $(seq 1 "${cert_count}"); do + openssl req -x509 -newkey rsa:2048 -nodes -days 1 \ + -subj "/CN=dummy-ca-${i}" \ + -keyout "dummy-ca-${i}.key" \ + -out "dummy-ca-${i}.pem" \ + >/dev/null 2>&1 + + cat "dummy-ca-${i}.pem" >> "${bundle_file}" + done + + log_success "Created CA bundle: $(du -h "${bundle_file}" | cut -f1)" +} + +create_che_ca_configmap() { + local che_ns="$1" + local bundle_file="$2" + + log_info "Creating Che CA bundle ConfigMap..." + + kubectl create configmap custom-ca-certificates \ + --from-file=custom-ca-certificates.pem="${bundle_file}" \ + -n "${che_ns}" \ + --dry-run=client -o yaml \ + | kubectl apply --server-side -f - + + kubectl label configmap custom-ca-certificates \ + app.kubernetes.io/component=ca-bundle \ + app.kubernetes.io/part-of=che.eclipse.org \ + -n "${che_ns}" \ + --overwrite +} + +patch_checluster_disable_pki_mount() { + local che_ns="$1" + + log_info "Configuring CheCluster..." + local checluster + checluster=$(kubectl get checluster -n "${che_ns}" -o jsonpath='{.items[0].metadata.name}') + + kubectl patch checluster "${checluster}" \ + -n "${che_ns}" \ + --type=merge \ + -p '{ + "spec": { + "devEnvironments": { + "trustedCerts": { + "disableWorkspaceCaBundleMount": true + } + } + } + }' +} + +restart_che() { + local che_ns="$1" + + log_info "Restarting Che..." + kubectl rollout status deploy/che -n "${che_ns}" --timeout=5m + kubectl wait pod -n "${che_ns}" -l app=che --for=condition=Ready --timeout=5m + + log_success "Che restarted" +} + +create_devworkspace() { + local dw_ns="$1" + local dw_name="$2" + + log_info "Creating DevWorkspace '${dw_name}'..." + cat </dev/stderr + + pod_name=$(kubectl get pod \ + -n "${dw_ns}" \ + -l controller.devfile.io/devworkspace_name="${dw_name}" \ + -o jsonpath='{.items[0].metadata.name}') + + echo "${pod_name}" +} + +verify_ca_bundle_in_workspace() { + local pod_name="$1" + local dw_ns="$2" + local expected_count="$3" + local cert_path="/public-certs/tls-ca-bundle.pem" + + log_info "Verifying CA bundle in workspace..." + + kubectl exec "${pod_name}" -n "${dw_ns}" -- test -f "${cert_path}" + + local mounted_count + mounted_count=$(kubectl exec "${pod_name}" -n "${dw_ns}" -- \ + sh -c "grep -c 'BEGIN CERTIFICATE' ${cert_path}") + + log_info "Generated certificates : ${expected_count}" + log_info "Mounted certificates : ${mounted_count}" + + if [[ "${mounted_count}" -le "${expected_count}" ]]; then + log_error "Mounted certificate count validation failed" + return 1 + fi + + log_success "CA bundle verification passed" +} + +cleanup_resources() { + local dw_ns="$1" + local dw_name="$2" + + log_info "Cleaning up..." + kubectl delete dw "${dw_name}" -n "${dw_ns}" --ignore-not-found + rm -f *.pem *.key +} diff --git a/test/load/devworkspace_load_test.js b/test/load/devworkspace_load_test.js new file mode 100644 index 000000000..dc33fe524 --- /dev/null +++ b/test/load/devworkspace_load_test.js @@ -0,0 +1,622 @@ +// +// Copyright (c) 2019-2025 Red Hat, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import http from 'k6/http'; +import {check, sleep} from 'k6'; +import {Trend, Counter} from 'k6/metrics'; +import { test } from 'k6/execution'; +import {htmlReport} from "https://raw.githubusercontent.com/benc-uk/k6-reporter/main/dist/bundle.js"; +import {textSummary} from "https://jslib.k6.io/k6-summary/0.0.1/index.js"; + +const inCluster = __ENV.IN_CLUSTER === 'true'; +const apiServer = inCluster ? `https://kubernetes.default.svc` : __ENV.KUBE_API; +const token = inCluster ? open('/var/run/secrets/kubernetes.io/serviceaccount/token') : __ENV.KUBE_TOKEN; +const useSeparateNamespaces = __ENV.SEPARATE_NAMESPACES === "true"; +const deleteDevWorkspaceAfterReady = __ENV.DELETE_DEVWORKSPACE_AFTER_READY === "true"; +const operatorNamespace = __ENV.DWO_NAMESPACE || 'openshift-operators'; +const externalDevWorkspaceLink = __ENV.DEVWORKSPACE_LINK || ''; +const shouldCreateAutomountResources = (__ENV.CREATE_AUTOMOUNT_RESOURCES || 'false') === 'true'; +const maxVUs = Number(__ENV.MAX_VUS || 50); +const maxDevWorkspaces = Number(__ENV.MAX_DEVWORKSPACES || -1); +const devWorkspaceReadyTimeout = Number(__ENV.DEV_WORKSPACE_READY_TIMEOUT_IN_SECONDS || 600); +const autoMountConfigMapName = 'dwo-load-test-automount-configmap'; +const autoMountSecretName = 'dwo-load-test-automount-secret'; +const labelType = "test-type"; +const labelKey = "load-test"; +const loadTestDurationInMinutes = __ENV.TEST_DURATION_IN_MINUTES || "25"; +const loadTestNamespace = __ENV.LOAD_TEST_NAMESPACE || "loadtest-devworkspaces"; + +const headers = { + Authorization: `Bearer ${token}`, 'Content-Type': 'application/json', +}; + +export const options = { + scenarios: { + create_and_delete_devworkspaces: { + executor: 'shared-iterations', + vus: maxVUs, + iterations: maxDevWorkspaces, + maxDuration: '3h', + }, + }, thresholds: { + 'checks': ['rate>0.95'], + 'devworkspace_create_duration': ['p(95)<15000'], + 'devworkspace_delete_duration': ['p(95)<10000'], + 'devworkspace_ready_duration': ['p(95)<60000'], + 'devworkspace_ready_failed': ['count<5'], + 'operator_cpu_violations': ['count==0'], + 'operator_mem_violations': ['count==0'], + }, insecureSkipTLSVerify: true, // trust self-signed certs like in CRC +}; + +const devworkspaceCreateDuration = new Trend('devworkspace_create_duration'); +const devworkspaceReady = new Counter('devworkspace_ready'); +const devworkspaceDeleteDuration = new Trend('devworkspace_delete_duration'); +const devworkspaceReadyDuration = new Trend('devworkspace_ready_duration'); +const devworkspaceReadyFailed = new Counter('devworkspace_ready_failed'); +const operatorCpu = new Trend('average_operator_cpu'); // in milli cores +const operatorMemory = new Trend('average_operator_memory'); // in Mi +const etcdCpu = new Trend('average_etcd_cpu'); // in milli cores +const etcdMemory = new Trend('average_etcd_memory'); // in Mi +const devworkspacesCreated = new Counter('devworkspace_create_count'); +const operatorCpuViolations = new Counter('operator_cpu_violations'); +const operatorMemViolations = new Counter('operator_mem_violations'); + +const maxCpuMillicores = 250; +const maxMemoryBytes = 200 * 1024 * 1024; + +let etcdNamespace = 'openshift-etcd'; +let etcdPodNamePattern = 'etcd'; + +function detectClusterType() { + const apiGroupsUrl = `${apiServer}/apis`; + const res = http.get(apiGroupsUrl, {headers}); + + if (res.status === 200) { + try { + const data = JSON.parse(res.body); + const groups = data.groups || []; + const hasOpenShiftRoutes = groups.some(g => g.name === 'route.openshift.io'); + + if (!hasOpenShiftRoutes) { + etcdNamespace = __ENV.ETCD_NAMESPACE || 'kube-system'; + etcdPodNamePattern = __ENV.ETCD_POD_NAME_PATTERN || 'kube-proxy'; + console.log('Detected Kubernetes cluster - using kube-system namespace with kube-proxy'); + } + } catch (e) { + console.warn(`Failed to detect cluster type: ${e.message}, using defaults`); + } + } +} + +export function setup() { + detectClusterType(); + if (shouldCreateAutomountResources) { + createNewAutomountConfigMap(); + createNewAutomountSecret(); + } +} + +export default function () { + if (maxDevWorkspaces > 0) { + const { error, devWorkspaces } = getDevWorkspacesFromApiServer(); + if (error) { + return; + } + const totalDevWorkspaces = devWorkspaces.length; + + const runningDevWorkspaces = devWorkspaces.filter( + (dw) => dw.status && dw.status.phase === 'Running' + ).length; + const startingDevWorkspaces = devWorkspaces.filter( + (dw) => dw.status && dw.status.phase === 'Starting' + ).length; + + if (startingDevWorkspaces === 0 && runningDevWorkspaces >= maxDevWorkspaces) { + test.abort( + 'Max concurrent DevWorkspaces target achieved (no Starting workspaces), stopping test gracefully', + { abortOnFail: false } + ); + } else if (totalDevWorkspaces >= maxDevWorkspaces) { + // stop further creation, but don’t abort the test + return; + } + } + const vuId = __VU; + const iteration = __ITER; + const crName = `dw-test-${vuId}-${iteration}`; + const namespace = useSeparateNamespaces + ? `load-test-ns-${__VU}-${__ITER}` + : loadTestNamespace; + + if (!apiServer) { + throw new Error('KUBE_API env var is required'); + } + try { + if (useSeparateNamespaces) { + createNewNamespace(namespace); + } + const devWorkspaceCreated = createNewDevWorkspace(namespace, vuId, iteration); + if (devWorkspaceCreated) { + waitUntilDevWorkspaceIsReady(vuId, crName, namespace); + if (deleteDevWorkspaceAfterReady) { + deleteDevWorkspace(crName, namespace); + } + } + } catch (error) { + console.error(`Load test for ${vuId}-${iteration} failed:`, error.message); + } +} + +export function final_cleanup() { + if (useSeparateNamespaces) { + deleteAllSeparateNamespaces(); + } else { + deleteAllDevWorkspacesInCurrentNamespace(); + } + + if (shouldCreateAutomountResources) { + deleteConfigMap(); + deleteSecret(); + } +} + +export function handleSummary(data) { + const allowed = ['devworkspace_create_count', 'devworkspace_create_duration', 'devworkspace_delete_duration', 'devworkspace_ready_duration', 'devworkspace_ready', 'devworkspace_ready_failed', 'operator_cpu_violations', 'operator_mem_violations', 'average_operator_cpu', 'average_operator_memory', 'etcd_cpu_violations', 'etcd_mem_violations', 'average_etcd_cpu', 'average_etcd_memory']; + + const filteredData = JSON.parse(JSON.stringify(data)); + for (const key of Object.keys(filteredData.metrics)) { + if (!allowed.includes(key)) { + delete filteredData.metrics[key]; + } + } + + let loadTestSummaryReport = { + stdout: textSummary(filteredData, {indent: ' ', enableColors: true}) + } + // Only generate HTML report when running outside the cluster + if (!inCluster) { + loadTestSummaryReport["devworkspace-load-test-report.html"] = htmlReport(data, { + title: "DevWorkspace Operator Load Test Report (HTTP)", + }); + } + return loadTestSummaryReport; +} + +export function teardown(data) { + console.log("Running final cleanup after all DevWorkspace creation finished..."); + final_cleanup(); +} + +function createNewDevWorkspace(namespace, vuId, iteration) { + const baseUrl = `${apiServer}/apis/workspace.devfile.io/v1alpha2/namespaces/${namespace}/devworkspaces`; + + const manifest = generateDevWorkspaceToCreate(vuId, iteration, namespace); + + const payload = JSON.stringify(manifest); + + const createStart = Date.now(); + const createRes = http.post(baseUrl, payload, {headers}); + check(createRes, { + 'DevWorkspace created': (r) => r.status === 201 || r.status === 409, + }); + + if (createRes.status !== 201 && createRes.status !== 409) { + console.error(`[VU ${vuId}] Failed to create DevWorkspace: ${createRes.status}, ${createRes.body}`); + return false; + } + devworkspaceCreateDuration.add(Date.now() - createStart); + devworkspacesCreated.add(1); + return true; +} + +function waitUntilDevWorkspaceIsReady(vuId, crName, namespace) { + const dwUrl = `${apiServer}/apis/workspace.devfile.io/v1alpha2/namespaces/${namespace}/devworkspaces/${crName}`; + const readyStart = Date.now(); + let isReady = false; + let attempts = 0; + const pollWaitInterval = 5; + const maxAttempts = devWorkspaceReadyTimeout / pollWaitInterval; + let res = {}; + + while (!isReady && attempts < maxAttempts) { + res = http.get(`${dwUrl}`, {headers}); + + if (res.status === 200) { + try { + const body = JSON.parse(res.body); + const phase = body?.status?.phase; + if (phase === 'Ready' || phase === 'Running') { + isReady = true; + break; + } else if (phase === 'Failing' || phase === 'Failed' || phase === 'Error') { + isReady = false; + break; + } + } catch (e) { + console.error(`GET [VU ${vuId}] Failed to parse DevWorkspace from API: ${res.body} : ${e.message}`); + } + } + + checkDevWorkspaceOperatorMetrics(); + checkEtcdMetrics(); + sleep(pollWaitInterval); + attempts++; + } + + if (!isReady && attempts >= maxAttempts) { + console.error( + `GET [VU ${vuId}] Timed out waiting for DevWorkspace '${crName}' in namespace '${namespace}' ` + + `after ${attempts} attempts (${devWorkspaceReadyTimeout}s). Last known phase: '${lastPhase}'` + ); + } + + if (res.status === 200) { + if (isReady) { + devworkspaceReady.add(1); + devworkspaceReadyDuration.add(Date.now() - readyStart); + } else { + devworkspaceReadyFailed.add(1); + } + } +} + +function deleteDevWorkspace(crName, namespace) { + const dwUrl = `${apiServer}/apis/workspace.devfile.io/v1alpha2/namespaces/${namespace}/devworkspaces/${crName}`; + const deleteStart = Date.now(); + const delRes = http.del(`${dwUrl}`, null, {headers}); + devworkspaceDeleteDuration.add(Date.now() - deleteStart); + + check(delRes, { + 'DevWorkspace deleted or not found': (r) => r.status === 200 || r.status === 404, + }); +} + +function checkDevWorkspaceOperatorMetrics() { + const metricsUrl = `${apiServer}/apis/metrics.k8s.io/v1beta1/namespaces/${operatorNamespace}/pods`; + const res = http.get(metricsUrl, {headers}); + + + check(res, { + 'Fetched pod metrics successfully': (r) => r.status === 200, + }); + + if (res.status !== 200) { + return; + } + + const data = JSON.parse(res.body); + const operatorPods = data.items.filter(p => p.metadata.name.includes("devworkspace-controller")); + + for (const pod of operatorPods) { + const container = pod.containers[0]; // assuming single container + const name = pod.metadata.name; + + const cpu = parseCpuToMillicores(container.usage.cpu); + const memory = parseMemoryToBytes(container.usage.memory); + + operatorCpu.add(cpu); + operatorMemory.add(memory / 1024 / 1024); + + const cpuOk = cpu <= maxCpuMillicores; + const memOk = memory <= maxMemoryBytes; + + if (!cpuOk) { + operatorCpuViolations.add(1); + } + if (!memOk) { + operatorMemViolations.add(1); + } + + check(null, { + [`[${name}] CPU < ${maxCpuMillicores}m`]: () => cpuOk, + [`[${name}] Memory < ${Math.round(maxMemoryBytes / 1024 / 1024)}Mi`]: () => memOk, + }); + } +} + +function checkEtcdMetrics() { + if (!etcdNamespace || !etcdPodNamePattern) { + console.warn(`[ETCD METRICS] Variables not initialized: etcdNamespace=${etcdNamespace}, etcdPodNamePattern=${etcdPodNamePattern}`); + return; + } + + const metricsUrl = `${apiServer}/apis/metrics.k8s.io/v1beta1/namespaces/${etcdNamespace}/pods`; + const res = http.get(metricsUrl, {headers}); + + check(res, { + 'Fetched etcd pod metrics successfully': (r) => r.status === 200, + }); + + if (res.status !== 200) { + return; + } + + const data = JSON.parse(res.body); + const etcdPods = data.items.filter(p => p.metadata.name.includes(etcdPodNamePattern)); + + if (etcdPods.length === 0) { + if (data.items && data.items.length > 0) { + const podNames = data.items.map(p => p.metadata.name).join(', '); + console.warn(`[ETCD METRICS] No pods found matching pattern '${etcdPodNamePattern}' in namespace '${etcdNamespace}'. Available pods: ${podNames}`); + } else { + console.warn(`[ETCD METRICS] No pods found in namespace '${etcdNamespace}'`); + } + return; + } + + for (const pod of etcdPods) { + if (!pod.containers || pod.containers.length === 0) { + console.warn(`[ETCD METRICS] Pod ${pod.metadata.name} has no containers`); + continue; + } + const container = pod.containers[0]; + const name = pod.metadata.name; + + if (!container.usage || !container.usage.cpu || !container.usage.memory) { + console.warn(`[ETCD METRICS] Pod ${name} has no usage data:`, JSON.stringify(container.usage)); + continue; + } + + const cpu = parseCpuToMillicores(container.usage.cpu); + const memory = parseMemoryToBytes(container.usage.memory); + + etcdCpu.add(cpu); + etcdMemory.add(memory / 1024 / 1024); + } +} + +function createNewNamespace(namespaceName) { + const url = `${apiServer}/api/v1/namespaces`; + + const namespaceObj = { + apiVersion: 'v1', + kind: 'Namespace', + metadata: { + name: namespaceName, + labels: { + [labelKey]: labelType + } + } + } + const res = http.post(url, JSON.stringify(namespaceObj), {headers}); + + if (res.status !== 201 && res.status !== 409) { + throw new Error(`Failed to create Namespace: ${res.status} - ${namespaceName}`); + } +} + +function createNewAutomountConfigMap() { + const url = `${apiServer}/api/v1/namespaces/${loadTestNamespace}/configmaps`; + + const configMapManifest = { + apiVersion: 'v1', kind: 'ConfigMap', metadata: { + name: autoMountConfigMapName, namespace: loadTestNamespace, labels: { + 'controller.devfile.io/mount-to-devworkspace': 'true', 'controller.devfile.io/watch-configmap': 'true', + }, annotations: { + 'controller.devfile.io/mount-path': '/etc/config/dwo-load-test-configmap', + 'controller.devfile.io/mount-access-mode': '0644', + 'controller.devfile.io/mount-as': 'file', + }, + }, data: { + 'test.key': 'test-value', + }, + }; + + const res = http.post(url, JSON.stringify(configMapManifest), {headers}); + + if (res.status !== 201 && res.status !== 409) { + throw new Error(`Failed to create automount ConfigMap: ${res.status} - ${res.body}`); + } + console.log("Created automount configMap : " + autoMountConfigMapName); +} + +function createNewAutomountSecret() { + const manifest = { + apiVersion: 'v1', kind: 'Secret', metadata: { + name: autoMountSecretName, namespace: loadTestNamespace, labels: { + 'controller.devfile.io/mount-to-devworkspace': 'true', 'controller.devfile.io/watch-secret': 'true', + }, annotations: { + 'controller.devfile.io/mount-path': `/etc/secret/dwo-load-test-secret`, + 'controller.devfile.io/mount-as': 'file', + }, + }, type: 'Opaque', data: { + 'secret.key': __ENV.SECRET_VALUE_BASE64 || 'dGVzdA==', // base64-encoded 'test' + }, + }; + + const res = http.post(`${apiServer}/api/v1/namespaces/${loadTestNamespace}/secrets`, JSON.stringify(manifest), {headers}); + if (res.status !== 201 && res.status !== 409) { + throw new Error(`Failed to create automount Secret: ${res.status} - ${res.body}`); + } +} + +function deleteConfigMap() { + const url = `${apiServer}/api/v1/namespaces/${loadTestNamespace}/configmaps/${autoMountConfigMapName}`; + const res = http.del(url, null, { headers }); + if (res.status !== 200 && res.status !== 404) { + console.warn(`[CLEANUP] Failed to delete ConfigMap ${autoMountConfigMapName}: ${res.status}`); + } +} + +function deleteSecret() { + const url = `${apiServer}/api/v1/namespaces/${loadTestNamespace}/secrets/${autoMountSecretName}`; + const res = http.del(url, null, {headers}); + if (res.status !== 200 && res.status !== 404) { + console.warn(`[CLEANUP] Failed to delete Secret ${autoMountSecretName}: ${res.status}`); + } +} + +function deleteNamespace(name) { + const delRes = http.del(`${apiServer}/api/v1/namespaces/${name}`, null, {headers}); + if (delRes.status !== 200 && delRes.status !== 404) { + console.warn(`[CLEANUP] Failed to delete Namespace ${name}: ${delRes.status}`); + } +} + +function deleteAllDevWorkspacesInCurrentNamespace() { + const deleteByLabelSelectorUrl = `${apiServer}/apis/workspace.devfile.io/v1alpha2/namespaces/${loadTestNamespace}/devworkspaces?labelSelector=${labelKey}%3D${labelType}`; + console.log(`[CLEANUP] Deleting all DevWorkspaces in ${loadTestNamespace} containing label ${labelKey}=${labelType}`); + + const res = http.del(deleteByLabelSelectorUrl, null, {headers}); + if (res.status !== 200) { + console.error(`[CLEANUP] Failed to delete DevWorkspaces: ${res.status}`); + } +} + +function deleteAllSeparateNamespaces() { + const getNamespacesByLabel = `${apiServer}/api/v1/namespaces?labelSelector=${labelKey}%3D${labelType}`; + console.log(`[CLEANUP] Deleting all Namespaces containing label ${labelKey}=${labelType}`); + + const res = http.get(getNamespacesByLabel, {headers}); + if (res.status !== 200) { + console.error(`[CLEANUP] Failed to list DevWorkspaces: ${res.status}`); + return; + } + + const body = JSON.parse(res.body); + if (!body.items || !Array.isArray(body.items)) return; + + for (const item of body.items) { + deleteNamespace(item.metadata.name); + } +} + +function createOpinionatedDevWorkspace() { + return { + apiVersion: "workspace.devfile.io/v1alpha2", kind: "DevWorkspace", metadata: { + name: "minimal-dw", + namespace: loadTestNamespace, + labels: { + [labelKey]: labelType + } + }, spec: { + started: true, template: { + attributes: { + "controller.devfile.io/storage-type": "ephemeral", + }, components: [{ + name: "dev", container: { + image: "registry.access.redhat.com/ubi9/ubi-micro:9.6-1752751762", + command: ["sleep", "3600"], + imagePullPolicy: "IfNotPresent", + memoryLimit: "64Mi", + memoryRequest: "32Mi", + cpuLimit: "200m", + cpuRequest: "100m" + }, + },], + }, + }, + }; +} + +function parseJSONResponseToDevWorkspace(response) { + let devWorkspace; + try { + devWorkspace = response.json(); + } catch (e) { + throw new Error(`[DW CREATE] Failed to parse JSON : ${response.body}: ${e.message}`); + } + return devWorkspace; +} + +function downloadAndParseExternalWorkspace(externalDevWorkspaceLink) { + let manifest; + if (externalDevWorkspaceLink) { + const res = http.get(externalDevWorkspaceLink); + + if (res.status !== 200) { + throw new Error(`[DW CREATE] Failed to fetch JSON content from ${externalDevWorkspaceLink}, got ${res.status}`); + } + manifest = parseJSONResponseToDevWorkspace(res); + } + + return manifest; +} + +function getDevWorkspacesFromApiServer() { + const basePath = useSeparateNamespaces + ? `${apiServer}/apis/workspace.devfile.io/v1alpha2/devworkspaces` + : `${apiServer}/apis/workspace.devfile.io/v1alpha2/namespaces/${loadTestNamespace}/devworkspaces`; + + const url = `${basePath}?labelSelector=${labelKey}%3D${labelType}`; + const res = http.get(url, { headers }); + + if (res.status !== 200) { + const errorMsg = `Failed to fetch DevWorkspaces: ${res.status} ${res.statusText || ''}`; + console.error(errorMsg); + + return { + error: errorMsg, + devWorkspaces: null, + }; + } + + const body = JSON.parse(res.body); + + return { + error: null, + devWorkspaces: body.items, + }; +} + +function generateDevWorkspaceToCreate(vuId, iteration, namespace) { + const name = `dw-test-${vuId}-${iteration}`; + let devWorkspace = {}; + if (externalDevWorkspaceLink.length > 0) { + devWorkspace = downloadAndParseExternalWorkspace(externalDevWorkspaceLink); + } else { + devWorkspace = createOpinionatedDevWorkspace(); + } + devWorkspace.metadata.name = name; + devWorkspace.metadata.namespace = namespace; + devWorkspace.metadata.labels = { + [labelKey]: labelType + } + return devWorkspace; +} + +function generateLoadTestStages(max) { + const stageDefinitions = [ + { percent: 0.25, target: Math.floor(max * 0.25) }, + { percent: 0.25, target: Math.floor(max * 0.5) }, + { percent: 0.2, target: Math.floor(max * 0.75) }, + { percent: 0.15, target: max }, + { percent: 0.10, target: Math.floor(max * 0.5) }, + { percent: 0.05, target: 0 }, + ]; + + return stageDefinitions.map(({ percent, target }) => ({ + duration: `${Math.round(loadTestDurationInMinutes * percent)}m`, + target, + })); +} + +function parseMemoryToBytes(memStr) { + if (memStr.endsWith("Ki")) return parseInt(memStr) * 1024; + if (memStr.endsWith("Mi")) return parseInt(memStr) * 1024 * 1024; + if (memStr.endsWith("Gi")) return parseInt(memStr) * 1024 * 1024 * 1024; + if (memStr.endsWith("n")) return parseInt(memStr) / 1e9; + if (memStr.endsWith("u")) return parseInt(memStr) / 1e6; + if (memStr.endsWith("m")) return parseInt(memStr) / 1e3; + return parseInt(memStr); // bytes +} + +function parseCpuToMillicores(cpuStr) { + if (cpuStr.endsWith("n")) return Math.round(parseInt(cpuStr) / 1e6); + if (cpuStr.endsWith("u")) return Math.round(parseInt(cpuStr) / 1e3); + if (cpuStr.endsWith("m")) return parseInt(cpuStr); + return Math.round(parseFloat(cpuStr) * 1000); +} diff --git a/test/load/provision-che-workspace-namespace.sh b/test/load/provision-che-workspace-namespace.sh new file mode 100644 index 000000000..3f011677c --- /dev/null +++ b/test/load/provision-che-workspace-namespace.sh @@ -0,0 +1,52 @@ +provision_che_workspace_namespace() { + local LOAD_TEST_NAMESPACE="$1" + local CHE_NAMESPACE="$2" + local CHE_CLUSTER_NAME="$3" + + if [[ -z "${LOAD_TEST_NAMESPACE}" ]]; then + echo "ERROR: LOAD_TEST_NAMESPACE argument is required" + echo "Usage: provision_che_workspace_namespace " + return 1 + fi + + if ! command -v oc >/dev/null 2>&1; then + echo "ERROR: oc CLI not found" + return 1 + fi + + local USERNAME + USERNAME="$(oc whoami)" + + echo "Provisioning Che workspace namespace" + echo " User : ${USERNAME}" + echo " Namespace : ${LOAD_TEST_NAMESPACE}" + + oc patch checluster "${CHE_CLUSTER_NAME}" \ + -n "${CHE_NAMESPACE}" \ + --type=merge \ + -p '{ + "spec": { + "devEnvironments": { + "defaultNamespace": { + "autoProvision": false + } + } + } + }' >/dev/null + + cat </dev/null + + echo "✔ Namespace '${LOAD_TEST_NAMESPACE}' provisioned for user '${USERNAME}'" +} diff --git a/test/load/runk6.sh b/test/load/runk6.sh new file mode 100644 index 000000000..6160c2cfa --- /dev/null +++ b/test/load/runk6.sh @@ -0,0 +1,405 @@ +#!/bin/bash + +source test/load/provision-che-workspace-namespace.sh +source test/load/che-cert-bundle-utils.sh + + +MODE="binary" # or 'operator' +LOAD_TEST_NAMESPACE="loadtest-devworkspaces" +DWO_NAMESPACE="openshift-operators" +SA_NAME="k6-devworkspace-tester" +CLUSTERROLE_NAME="k6-devworkspace-role" +ROLEBINDING_NAME="k6-devworkspace-binding" +CONFIGMAP_NAME="k6-test-script" +K6_CR_NAME="k6-test-run" +K6_SCRIPT="test/load/devworkspace_load_test.js" +K6_OPERATOR_VERSION="v0.0.22" +DEVWORKSPACE_LINK="https://gist.githubusercontent.com/rohanKanojia/ecf625afaf3fe817ac7d1db78bd967fc/raw/8c30c0370444040105ca45cd4ac0f7062a644bb7/dw-minimal.json" +MAX_VUS="100" +DEV_WORKSPACE_READY_TIMEOUT_IN_SECONDS="1200" +SEPARATE_NAMESPACES="false" +DELETE_DEVWORKSPACE_AFTER_READY="true" +MAX_DEVWORKSPACES="-1" +CREATE_AUTOMOUNT_RESOURCES="false" +RUN_WITH_ECLIPSE_CHE="false" +LOGS_DIR="logs" +TEST_DURATION_IN_MINUTES="25" +MIN_KUBECTL_VERSION="1.24.0" +MIN_CURL_VERSION="7.0.0" +MIN_K6_VERSION="1.1.0" +CHE_NAMESPACE="eclipse-che" +CHE_CLUSTER_NAME="eclipse-che" +TEST_CERTIFICATES_COUNT="500" + +# ----------- Main Execution Flow ----------- +main() { + parse_arguments "$@" + check_prerequisites + if [[ "$RUN_WITH_ECLIPSE_CHE" == "false" ]]; then + create_namespace + else + provision_che_workspace_namespace "$LOAD_TEST_NAMESPACE" "$CHE_NAMESPACE" "$CHE_CLUSTER_NAME" + run_che_ca_bundle_e2e "$CHE_NAMESPACE" "$LOAD_TEST_NAMESPACE" "test-devworkspace" "$TEST_CERTIFICATES_COUNT" + fi + create_rbac + start_background_watchers + + if [[ "$MODE" == "operator" ]]; then + install_k6_operator + create_k6_configmap + delete_existing_testruns + create_k6_test_run + wait_for_test_completion + fetch_test_logs + elif [[ "$MODE" == "binary" ]]; then + generate_token_and_api_url + run_k6_binary_test + else + echo "❌ Invalid mode: $MODE" + exit 1 + fi + stop_background_watchers + delete_namespace +} + +# ----------- Helper Functions ----------- +print_help() { + cat < Mode to run the script (default: operator) + --max-vus Number of virtual users for k6 (default: 100) + --max-devworkspaces Maximum number of DevWorkspaces to create (by default, it's not specified) + --separate-namespaces Use separate namespaces for workspaces (default: false) + --delete-devworkspace-after-ready Delete DevWorkspace once it becomes Ready (default: true) + --devworkspace-ready-timeout-seconds Timeout in seconds for workspace to become ready (default: 1200) + --devworkspace-link DevWorkspace link (default: empty, opinionated DevWorkspace is created) + --create-automount-resources Whether to create automount resources (default: false) + --dwo-namespace DevWorkspace Operator namespace (default: loadtest-devworkspaces) + --logs-dir Directory name where DevWorkspace and event logs would be dumped + --test-duration-minutes Duration in minutes for which to run load tests (default: 25 minutes) + --run-with-eclipse-che Whether these tests are supposed to be run with Eclipse Che (If yes additional certificates are mounted) + --che-cluster-name Applicable if running on Eclipse Che, defaults to 'eclipse-che' + --che-namespace Applicable if running on Eclipse Che, defaults to 'eclipse-che' + -h, --help Show this help message +EOF +} + +parse_arguments() { + while [[ $# -gt 0 ]]; do + case "$1" in + --mode) + MODE="$2"; shift 2;; + --max-vus) + MAX_VUS="$2"; shift 2;; + --separate-namespaces) + SEPARATE_NAMESPACES="$2"; shift 2;; + --max-devworkspaces) + MAX_DEVWORKSPACES="$2"; shift 2;; + --delete-devworkspace-after-ready) + DELETE_DEVWORKSPACE_AFTER_READY="$2"; shift 2;; + --devworkspace-ready-timeout-seconds) + DEV_WORKSPACE_READY_TIMEOUT_IN_SECONDS="$2"; shift 2;; + --devworkspace-link) + DEVWORKSPACE_LINK="$2"; shift 2;; + --create-automount-resources) + CREATE_AUTOMOUNT_RESOURCES="$2"; shift 2;; + --dwo-namespace) + LOAD_TEST_NAMESPACE="$2"; shift 2;; + --logs-dir) + LOGS_DIR="$2"; shift 2;; + --test-duration-minutes) + TEST_DURATION_IN_MINUTES="$2"; shift 2;; + --run-with-eclipse-che) + RUN_WITH_ECLIPSE_CHE="$2"; shift 2;; + --che-cluster-name) + CHE_CLUSTER_NAME="$2"; shift 2;; + --che-namespace) + CHE_NAMESPACE="$2"; shift 2;; + -h|--help) + print_help; exit 0;; + *) + echo "❌ Unknown option: $1" + print_help; exit 1;; + esac + done +} + +create_namespace() { + echo "🔧 Creating Namespace: $LOAD_TEST_NAMESPACE" + cat <> "${LOGS_DIR}/${TIMESTAMP}_events.log" 2>&1 & + PID_EVENTS_WATCH=$! + + kubectl get dw --watch --all-namespaces \ + >> "${LOGS_DIR}/${TIMESTAMP}_dw_watch.log" 2>&1 & + PID_DW_WATCH=$! + + log_failed_devworkspaces & + PID_FAILED_DW_POLL=$! +} + +log_failed_devworkspaces() { + echo "📄 Starting periodic failed DevWorkspaces report (every 10s)..." + + POLL_INTERVAL=10 # in seconds + ITERATIONS=$((((TEST_DURATION_IN_MINUTES-1) * 60) / POLL_INTERVAL)) + + for ((i = 0; i < ITERATIONS; i++)); do + OUTPUT=$(kubectl get devworkspaces --all-namespaces -o json | jq -r ' + .items[] + | select(.status.phase == "Failed") + | [ + .metadata.namespace, + .metadata.name, + .status.phase, + (.status.message // "No message") + ] + | @csv') + + if [ -n "$OUTPUT" ]; then + echo "$OUTPUT" > "${LOGS_DIR}/dw_failure_report.csv" + fi + + sleep "$POLL_INTERVAL" + done +} + +stop_background_watchers() { + echo "🛑 Stopping background watchers..." + kill "$PID_EVENTS_WATCH" "$PID_DW_WATCH" "$PID_FAILED_DW_POLL" 2>/dev/null || true +} + +install_k6_operator() { + echo "📦 Installing k6 operator..." + curl -L "https://raw.githubusercontent.com/grafana/k6-operator/refs/tags/${K6_OPERATOR_VERSION}/bundle.yaml" | kubectl apply -f - + echo "⏳ Waiting until k6 operator deployment is ready..." + kubectl rollout status deployment/k6-operator-controller-manager -n k6-operator-system --timeout=300s +} + +create_k6_configmap() { + echo "🧩 Creating ConfigMap from script file: $K6_SCRIPT" + kubectl create configmap "$CONFIGMAP_NAME" \ + --from-file=script.js="$K6_SCRIPT" \ + --namespace "$LOAD_TEST_NAMESPACE" \ + --dry-run=client -o yaml | kubectl apply -f - +} + +delete_existing_testruns() { + echo "🧹 Deleting any existing K6 TestRun resources in namespace: $LOAD_TEST_NAMESPACE" + kubectl delete testrun --all -n "$LOAD_TEST_NAMESPACE" || true +} + +create_k6_test_run() { + echo "🚀 Creating K6 TestRun custom resource..." + cat </dev/null) + + if [[ "$stage" == "finished" ]]; then + echo "TestRun $K6_CR_NAME is finished." + break + fi + + if (( SECONDS >= end )); then + echo "Timeout waiting for TestRun $K6_CR_NAME to finish." + exit 1 + fi + + sleep "$INTERVAL" + done +} + +fetch_test_logs() { + K6_TEST_POD=$(kubectl get pod -l k6_cr=$K6_CR_NAME,runner=true -n "${LOAD_TEST_NAMESPACE}" -o jsonpath='{.items[0].metadata.name}') + echo "📜 Fetching logs from completed K6 test pod: $K6_TEST_POD" + kubectl logs "$K6_TEST_POD" -n "$LOAD_TEST_NAMESPACE" +} + +check_prerequisites() { + echo "🔍 Checking prerequisites..." + + check_command "kubectl" "$MIN_KUBECTL_VERSION" + check_command "curl" "$MIN_CURL_VERSION" + + if [[ "$MODE" == "binary" ]]; then + check_command "k6" "$MIN_K6_VERSION" + fi +} + +check_command() { + local cmd="$1" + local min_version="$2" + local version + + if ! command -v "$cmd" &>/dev/null; then + echo "❌ Required command '$cmd' not found in PATH." + exit 1 + fi + + case "$cmd" in + kubectl) + version=$(kubectl version --client -o json | jq -r '.clientVersion.gitVersion' | sed 's/^v//') + ;; + curl) + version=$($cmd --version | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -n1) + ;; + k6) + version=$($cmd version | grep -oE '[0-9]+\.[0-9]+\.[0-9]+') + ;; + *) + version="0.0.0" + ;; + esac + + if ! version_gte "$version" "$min_version"; then + echo "❌ $cmd version $version is less than required $min_version" + exit 1 + else + echo "✅ $cmd version $version (>= $min_version)" + fi +} + +version_gte() { + [ "$(printf '%s\n' "$2" "$1" | sort -V | head -n1)" = "$2" ] +} + +run_k6_binary_test() { + echo "🚀 Running k6 load test..." + IN_CLUSTER='false' \ + KUBE_TOKEN="${KUBE_TOKEN}" \ + KUBE_API="${KUBE_API}" \ + DWO_NAMESPACE="${DWO_NAMESPACE}" \ + CREATE_AUTOMOUNT_RESOURCES="${CREATE_AUTOMOUNT_RESOURCES}" \ + SEPARATE_NAMESPACES="${SEPARATE_NAMESPACES}" \ + LOAD_TEST_NAMESPACE="${LOAD_TEST_NAMESPACE}" \ + DEVWORKSPACE_LINK="${DEVWORKSPACE_LINK}" \ + MAX_VUS="${MAX_VUS}" \ + TEST_DURATION_IN_MINUTES="${TEST_DURATION_IN_MINUTES}" \ + DEV_WORKSPACE_READY_TIMEOUT_IN_SECONDS="${DEV_WORKSPACE_READY_TIMEOUT_IN_SECONDS}" \ + DELETE_DEVWORKSPACE_AFTER_READY="${DELETE_DEVWORKSPACE_AFTER_READY}" \ + MAX_DEVWORKSPACES="${MAX_DEVWORKSPACES}" \ + k6 run "${K6_SCRIPT}" + exit_code=$? + if [ $exit_code -ne 0 ]; then + echo "⚠️ k6 load test failed with exit code $exit_code. Proceeding to cleanup." + fi + return 0 +} + +trap stop_background_watchers EXIT +main "$@"