diff --git a/.github/es-connectivity-monitor/action.yml b/.github/es-connectivity-monitor/action.yml new file mode 100644 index 0000000..2e647b9 --- /dev/null +++ b/.github/es-connectivity-monitor/action.yml @@ -0,0 +1,92 @@ +name: 'ES connectivity monitor' +description: 'Start or stop a background ES connectivity monitor that probes ES every 1s from host and from KBN container/pod' + +inputs: + action: + description: '"start" or "stop"' + required: true + es_url: + description: 'ES URL to probe from host (default: https://localhost:9200)' + required: false + default: 'https://localhost:9200' + es_user: + description: 'ES username' + required: false + default: 'kibana' + es_password: + description: 'ES password' + required: false + default: 'kibana' + +runs: + using: 'composite' + steps: + - name: Start monitor + if: inputs.action == 'start' + shell: bash + run: | + ( + set +e + set +o pipefail + while true; do + echo "===== $(date -Is) =====" + + echo "== ES health from host ==" + curl -sk -u "${{ inputs.es_user }}:${{ inputs.es_password }}" \ + "${{ inputs.es_url }}/_cluster/health?pretty" \ + -w $'\nHTTP %{http_code} in %{time_total}s\n' 2>&1 || true + + echo + echo "== KBN Docker container ==" + KBN_CONTAINER=$(docker ps --filter "name=kbn" --format "{{.Names}}" 2>/dev/null | head -1) + if [ -n "$KBN_CONTAINER" ]; then + STATUS=$(docker inspect --format '{{.State.Status}}' "$KBN_CONTAINER" 2>/dev/null) + echo "Container: $KBN_CONTAINER ($STATUS)" + # TCP connectivity test using bash built-in /dev/tcp (no curl needed in container) + docker exec "$KBN_CONTAINER" bash -c \ + 'timeout 2 bash -c "echo > /dev/tcp/es-ror/9200" 2>/dev/null \ + && echo "TCP to es-ror:9200: OK" \ + || echo "TCP to es-ror:9200: FAIL"' 2>&1 || echo "(exec failed)" + echo "-- last KBN log lines --" + docker logs --tail 5 "$KBN_CONTAINER" 2>&1 || true + else + echo "(no kbn Docker container found)" + fi + + echo + echo "== KBN Kubernetes pod ==" + KBN_POD=$(kubectl get pods -A -l "kibana.k8s.elastic.co/name" \ + --field-selector=status.phase=Running \ + -o jsonpath='{range .items[*]}{.metadata.namespace}/{.metadata.name}{"\n"}{end}' \ + 2>/dev/null | head -1) + if [ -n "$KBN_POD" ]; then + KBN_NS="${KBN_POD%%/*}" + KBN_NAME="${KBN_POD##*/}" + echo "Pod: $KBN_POD" + ES_SVC=$(kubectl get svc -n "$KBN_NS" -l "elasticsearch.k8s.elastic.co/cluster-name" \ + -o jsonpath='{.items[0].metadata.name}' 2>/dev/null) + if [ -n "$ES_SVC" ]; then + kubectl exec -n "$KBN_NS" "$KBN_NAME" -- \ + bash -c "timeout 2 bash -c 'echo > /dev/tcp/${ES_SVC}/9200' 2>/dev/null \ + && echo 'TCP to ${ES_SVC}:9200: OK' \ + || echo 'TCP to ${ES_SVC}:9200: FAIL'" 2>&1 || echo "(exec failed)" + else + echo "(could not resolve ES service name)" + fi + else + echo "(no running kbn pod found)" + fi + + echo + sleep 1 + done + ) >> /tmp/es-connectivity-monitor.log 2>&1 & + echo $! > /tmp/es-connectivity-monitor.pid + + - name: Stop monitor and dump log + if: inputs.action == 'stop' + shell: bash + run: | + kill "$(cat /tmp/es-connectivity-monitor.pid)" 2>/dev/null || true + echo "=== ES connectivity monitor log ===" + cat /tmp/es-connectivity-monitor.log || true diff --git a/.github/workflows/trigger-e2e-tests.yml b/.github/workflows/trigger-e2e-tests.yml index 0be6586..dcef353 100644 --- a/.github/workflows/trigger-e2e-tests.yml +++ b/.github/workflows/trigger-e2e-tests.yml @@ -23,6 +23,8 @@ jobs: matrix: version: ["9.4.1", "9.3.4", "8.19.15", "7.17.29"] env: [docker, eck-2.16.1, eck-3.4.0] + env: + ROR_KBN_VERSION: "1.70.0-pre7" steps: - name: Checkout code uses: actions/checkout@v4 @@ -36,6 +38,10 @@ jobs: uses: ./.github/docker-memory-monitor with: action: start + - name: Start ES connectivity monitor + uses: ./.github/es-connectivity-monitor + with: + action: start # Retry mechanism to handle transient infrastructure issues: # - npm error 429 Too Many Requests from registry.npmjs.org # - Docker image pull failures (e.g., beshultd/kibana-readonlyrest:*-ror-latest not found) @@ -48,7 +54,7 @@ jobs: retry_wait_seconds: 120 retry_on: any command: | - ./main.sh --mode e2e --env ${{ matrix.env }} --elk ${{ matrix.version }} + ./main.sh --mode e2e --env ${{ matrix.env }} --elk ${{ matrix.version }} --ror-kbn ${{ env.ROR_KBN_VERSION }} --dev-kbn env: ROR_ACTIVATION_KEY: ${{ secrets.ROR_KBN_LICENSE }} ELECTRON_EXTRA_LAUNCH_ARGS: '--disable-gpu' @@ -57,6 +63,11 @@ jobs: uses: ./.github/docker-memory-monitor with: action: stop + - name: Stop ES connectivity monitor + if: always() + uses: ./.github/es-connectivity-monitor + with: + action: stop - name: S3 Upload Videos & show logs if: failure() uses: ./.github/upload-videos @@ -90,6 +101,10 @@ jobs: uses: ./.github/docker-memory-monitor with: action: start + - name: Start ES connectivity monitor + uses: ./.github/es-connectivity-monitor + with: + action: start # Retry mechanism to handle transient infrastructure issues: # - npm error 429 Too Many Requests from registry.npmjs.org # - Docker image pull failures (e.g., beshultd/kibana-readonlyrest:*-ror-latest not found) @@ -109,6 +124,11 @@ jobs: uses: ./.github/docker-memory-monitor with: action: stop + - name: Stop ES connectivity monitor + if: always() + uses: ./.github/es-connectivity-monitor + with: + action: stop # ========================================== # E2E TESTS - DEVELOP BRANCH @@ -140,6 +160,10 @@ jobs: uses: ./.github/docker-memory-monitor with: action: start + - name: Start ES connectivity monitor + uses: ./.github/es-connectivity-monitor + with: + action: start # Retry mechanism to handle transient infrastructure issues: # - npm error 429 Too Many Requests from registry.npmjs.org # - Docker image pull failures (e.g., beshultd/kibana-readonlyrest:*-ror-latest not found) @@ -161,6 +185,11 @@ jobs: uses: ./.github/docker-memory-monitor with: action: stop + - name: Stop ES connectivity monitor + if: always() + uses: ./.github/es-connectivity-monitor + with: + action: stop - name: S3 Upload Videos & show logs if: failure() uses: ./.github/upload-videos diff --git a/e2e-tests/cypress/e2e/Tenancy.cy.ts b/e2e-tests/cypress/e2e/Tenancy.cy.ts index 632f01f..5ca4ae2 100644 --- a/e2e-tests/cypress/e2e/Tenancy.cy.ts +++ b/e2e-tests/cypress/e2e/Tenancy.cy.ts @@ -12,6 +12,7 @@ import { Dashboard } from '../support/page-objects/Dashboard'; import { TENANCY_QUERY_STRING_KEY } from '../support/types'; import { Spaces } from '../support/page-objects/Spaces'; import { kbnApiAdvancedClient } from '../support/helpers/KbnApiAdvancedClient'; +import { IndexManagement } from '../support/page-objects/IndexManagement'; describe('Tenancy', () => { describe('should run tests', () => { @@ -159,7 +160,7 @@ function runTests({ finishUrl: urlWithTenancyId, spacePrefix: '' }); - + IndexManagement.waitingForSectionLoadingFinish(); callbackAfterLogin?.(); Spaces.createNewSpace(newSpace); diff --git a/e2e-tests/cypress/support/page-objects/IndexManagement.ts b/e2e-tests/cypress/support/page-objects/IndexManagement.ts index 854fe56..83390ac 100644 --- a/e2e-tests/cypress/support/page-objects/IndexManagement.ts +++ b/e2e-tests/cypress/support/page-objects/IndexManagement.ts @@ -106,4 +106,9 @@ export class IndexManagement { cy.contains('[data-test-subj="title"]', "You don't have any data streams yet"); } + + static waitingForSectionLoadingFinish() { + cy.log('Waiting for section loading'); + cy.get('[data-test-subj="sectionLoading"]', { timeout: 30000 }).should('not.exist'); + } } diff --git a/environments/eck-ror/kind-cluster/ror/base/log4j2.properties.yml b/environments/eck-ror/kind-cluster/ror/base/log4j2.properties.yml index fb5aef6..7e2cd4f 100644 --- a/environments/eck-ror/kind-cluster/ror/base/log4j2.properties.yml +++ b/environments/eck-ror/kind-cluster/ror/base/log4j2.properties.yml @@ -71,7 +71,7 @@ data: appender.header_warning.name = header_warning logger.ror.name=tech.beshu.ror.accesscontrol - logger.ror.level=info + logger.ror.level=debug kind: ConfigMap metadata: name: config-log4j2.properties.yml \ No newline at end of file diff --git a/environments/eck-ror/kind-cluster/ror/base/ror-initial-config.yml b/environments/eck-ror/kind-cluster/ror/base/ror-initial-config.yml index f9e8986..ade6bad 100644 --- a/environments/eck-ror/kind-cluster/ror/base/ror-initial-config.yml +++ b/environments/eck-ror/kind-cluster/ror/base/ror-initial-config.yml @@ -31,13 +31,11 @@ data: access_control_rules: - name: "Kibana service account - token" - verbosity: error token_authentication: token: "Bearer ${KIBANA_SERVICE_ACCOUNT_TOKEN}" username: service_account - name: "Kibana service account - user/pass" - verbosity: error auth_key: kibana:kibana - name: "PROBE" diff --git a/environments/eck-ror/start.sh b/environments/eck-ror/start.sh index 034b924..98a8367 100755 --- a/environments/eck-ror/start.sh +++ b/environments/eck-ror/start.sh @@ -23,7 +23,8 @@ show_help() { echo " --cluster-type Cluster type: 'base' for basic cluster, 'apm' for cluster with APM (default: base)" echo " --ror-es ReadonlyREST ES version (default: latest)" echo " --ror-kbn ReadonlyREST Kibana version (default: latest)" - echo " --dev Use development images" + echo " --dev Use development images (both ES and KBN)" + echo " --dev-kbn Use development image for Kibana only" echo "" echo "Examples:" echo " ./start.sh --es 8.11.0 --kbn 8.11.0 --eck 2.15.0 # Start base cluster" @@ -108,6 +109,10 @@ while [[ $# -gt 0 ]]; do export ROR_KBN_REPO="beshultd/kibana-readonlyrest-dev" shift ;; + --dev-kbn) + export ROR_KBN_REPO="beshultd/kibana-readonlyrest-dev" + shift + ;; --help|-h) show_help ;; diff --git a/environments/elk-ror/start.sh b/environments/elk-ror/start.sh index 967c187..cbbead7 100755 --- a/environments/elk-ror/start.sh +++ b/environments/elk-ror/start.sh @@ -26,7 +26,8 @@ show_help() { echo " --cluster-type Cluster type: 'base' for basic cluster, 'apm' for cluster with APM (default: base)" echo " --ror-es ReadonlyREST ES version (default: latest)" echo " --ror-kbn ReadonlyREST Kibana version (default: latest)" - echo " --dev Use development images" + echo " --dev Use development images (both ES and KBN)" + echo " --dev-kbn Use development image for Kibana only" echo "" echo "Examples:" echo " ./start.sh --es 8.11.0 --kbn 8.11.0 # Start base cluster" @@ -101,6 +102,10 @@ while [[ $# -gt 0 ]]; do export ROR_KBN_REPO="beshultd/kibana-readonlyrest-dev" shift ;; + --dev-kbn) + export ROR_KBN_REPO="beshultd/kibana-readonlyrest-dev" + shift + ;; --help|-h) show_help ;; diff --git a/main.sh b/main.sh index 1ed4b08..01f873a 100755 --- a/main.sh +++ b/main.sh @@ -9,7 +9,8 @@ show_help() { echo " --elk ELK stack version (required)" echo " --ror-es ReadonlyREST ES version (default: latest)" echo " --ror-kbn ReadonlyREST Kibana version (default: latest)" - echo " --dev Use development images" + echo " --dev Use development images (both ES and KBN)" + echo " --dev-kbn Use development image for Kibana only" echo "" echo "Examples:" echo " ./main.sh --env docker --elk 8.11.0 # Run E2E tests with Docker Compose" @@ -23,6 +24,7 @@ OPTIONAL_ECK_ARG="" OPTIONAL_ROR_ES_ARG="" OPTIONAL_ROR_KBN_ARG="" OPTIONAL_DEV_ARG="" +OPTIONAL_DEV_KBN_ARG="" MODE="e2e" CLUSTER_TYPE="apm" @@ -102,6 +104,10 @@ while [[ $# -gt 0 ]]; do OPTIONAL_DEV_ARG="--dev" shift ;; + --dev-kbn) + OPTIONAL_DEV_KBN_ARG="--dev-kbn" + shift + ;; *) echo "Unknown option: $1" show_help @@ -133,7 +139,7 @@ echo -e " echo -e "Running environment...\n" -time ./environments/$ENV_NAME/start.sh --cluster-type "$CLUSTER_TYPE" --es "$ELK_VERSION" --kbn "$ELK_VERSION" $OPTIONAL_ECK_ARG $OPTIONAL_ROR_ES_ARG $OPTIONAL_ROR_KBN_ARG $OPTIONAL_DEV_ARG +time ./environments/$ENV_NAME/start.sh --cluster-type "$CLUSTER_TYPE" --es "$ELK_VERSION" --kbn "$ELK_VERSION" $OPTIONAL_ECK_ARG $OPTIONAL_ROR_ES_ARG $OPTIONAL_ROR_KBN_ARG $OPTIONAL_DEV_ARG $OPTIONAL_DEV_KBN_ARG if [[ "$MODE" == "e2e" ]]; then echo -e "Running E2E tests...\n"