[RORDEV-1944] investigating socket hangs up issue

.github/es-connectivity-monitor/action.yml

@@ -0,0 +1,92 @@
+name: 'ES connectivity monitor'
+description: 'Start or stop a background ES connectivity monitor that probes ES every 1s from host and from KBN container/pod'
+
+inputs:
+  action:
+    description: '"start" or "stop"'
+    required: true
+  es_url:
+    description: 'ES URL to probe from host (default: https://localhost:9200)'
+    required: false
+    default: 'https://localhost:9200'
+  es_user:
+    description: 'ES username'
+    required: false
+    default: 'kibana'
+  es_password:
+    description: 'ES password'
+    required: false
+    default: 'kibana'
+
+runs:
+  using: 'composite'
+  steps:
+    - name: Start monitor
+      if: inputs.action == 'start'
+      shell: bash
+      run: |
+        (
+          set +e
+          set +o pipefail
+          while true; do
+            echo "===== $(date -Is) ====="
+
+            echo "== ES health from host =="
+            curl -sk -u "${{ inputs.es_user }}:${{ inputs.es_password }}" \
+              "${{ inputs.es_url }}/_cluster/health?pretty" \
+              -w $'\nHTTP %{http_code} in %{time_total}s\n' 2>&1 || true
+
+            echo
+            echo "== KBN Docker container =="
+            KBN_CONTAINER=$(docker ps --filter "name=kbn" --format "{{.Names}}" 2>/dev/null | head -1)
+            if [ -n "$KBN_CONTAINER" ]; then
+              STATUS=$(docker inspect --format '{{.State.Status}}' "$KBN_CONTAINER" 2>/dev/null)
+              echo "Container: $KBN_CONTAINER ($STATUS)"
+              # TCP connectivity test using bash built-in /dev/tcp (no curl needed in container)
+              docker exec "$KBN_CONTAINER" bash -c \
+                'timeout 2 bash -c "echo > /dev/tcp/es-ror/9200" 2>/dev/null \
+                  && echo "TCP to es-ror:9200: OK" \
+                  || echo "TCP to es-ror:9200: FAIL"' 2>&1 || echo "(exec failed)"
+              echo "-- last KBN log lines --"
+              docker logs --tail 5 "$KBN_CONTAINER" 2>&1 || true
+            else
+              echo "(no kbn Docker container found)"
+            fi
+
+            echo
+            echo "== KBN Kubernetes pod =="
+            KBN_POD=$(kubectl get pods -A -l "kibana.k8s.elastic.co/name" \
+              --field-selector=status.phase=Running \
+              -o jsonpath='{range .items[*]}{.metadata.namespace}/{.metadata.name}{"\n"}{end}' \
+              2>/dev/null | head -1)
+            if [ -n "$KBN_POD" ]; then
+              KBN_NS="${KBN_POD%%/*}"
+              KBN_NAME="${KBN_POD##*/}"
+              echo "Pod: $KBN_POD"
+              ES_SVC=$(kubectl get svc -n "$KBN_NS" -l "elasticsearch.k8s.elastic.co/cluster-name" \
+                -o jsonpath='{.items[0].metadata.name}' 2>/dev/null)
+              if [ -n "$ES_SVC" ]; then
+                kubectl exec -n "$KBN_NS" "$KBN_NAME" -- \
+                  bash -c "timeout 2 bash -c 'echo > /dev/tcp/${ES_SVC}/9200' 2>/dev/null \
+                    && echo 'TCP to ${ES_SVC}:9200: OK' \
+                    || echo 'TCP to ${ES_SVC}:9200: FAIL'" 2>&1 || echo "(exec failed)"
+              else
+                echo "(could not resolve ES service name)"
+              fi
+            else
+              echo "(no running kbn pod found)"
+            fi
+
+            echo
+            sleep 1
+          done
+        ) >> /tmp/es-connectivity-monitor.log 2>&1 &
+        echo $! > /tmp/es-connectivity-monitor.pid
+
+    - name: Stop monitor and dump log
+      if: inputs.action == 'stop'
+      shell: bash
+      run: |
+        kill "$(cat /tmp/es-connectivity-monitor.pid)" 2>/dev/null || true
+        echo "=== ES connectivity monitor log ==="
+        cat /tmp/es-connectivity-monitor.log || true

.github/workflows/trigger-e2e-tests.yml

@@ -23,6 +23,8 @@ jobs:
       matrix:
         version: ["9.4.1", "9.3.4", "8.19.15", "7.17.29"]
         env: [docker, eck-2.16.1, eck-3.4.0]
+    env:
+      ROR_KBN_VERSION: "1.70.0-pre7"
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
@@ -36,6 +38,10 @@ jobs:
         uses: ./.github/docker-memory-monitor
         with:
           action: start
+      - name: Start ES connectivity monitor
+        uses: ./.github/es-connectivity-monitor
+        with:
+          action: start
       # Retry mechanism to handle transient infrastructure issues:
       # - npm error 429 Too Many Requests from registry.npmjs.org
       # - Docker image pull failures (e.g., beshultd/kibana-readonlyrest:*-ror-latest not found)
@@ -48,7 +54,7 @@ jobs:
           retry_wait_seconds: 120
           retry_on: any
           command: |
-            ./main.sh --mode e2e --env ${{ matrix.env }} --elk ${{ matrix.version }}
+            ./main.sh --mode e2e --env ${{ matrix.env }} --elk ${{ matrix.version }} --ror-kbn ${{ env.ROR_KBN_VERSION }} --dev-kbn
         env:
           ROR_ACTIVATION_KEY: ${{ secrets.ROR_KBN_LICENSE }}
           ELECTRON_EXTRA_LAUNCH_ARGS: '--disable-gpu'
@@ -57,6 +63,11 @@ jobs:
         uses: ./.github/docker-memory-monitor
         with:
           action: stop
+      - name: Stop ES connectivity monitor
+        if: always()
+        uses: ./.github/es-connectivity-monitor
+        with:
+          action: stop
       - name: S3 Upload Videos & show logs
         if: failure()
         uses: ./.github/upload-videos
@@ -90,6 +101,10 @@ jobs:
         uses: ./.github/docker-memory-monitor
         with:
           action: start
+      - name: Start ES connectivity monitor
+        uses: ./.github/es-connectivity-monitor
+        with:
+          action: start
       # Retry mechanism to handle transient infrastructure issues:
       # - npm error 429 Too Many Requests from registry.npmjs.org
       # - Docker image pull failures (e.g., beshultd/kibana-readonlyrest:*-ror-latest not found)
@@ -109,6 +124,11 @@ jobs:
         uses: ./.github/docker-memory-monitor
         with:
           action: stop
+      - name: Stop ES connectivity monitor
+        if: always()
+        uses: ./.github/es-connectivity-monitor
+        with:
+          action: stop
 
   # ==========================================
   # E2E TESTS - DEVELOP BRANCH
@@ -140,6 +160,10 @@ jobs:
         uses: ./.github/docker-memory-monitor
         with:
           action: start
+      - name: Start ES connectivity monitor
+        uses: ./.github/es-connectivity-monitor
+        with:
+          action: start
       # Retry mechanism to handle transient infrastructure issues:
       # - npm error 429 Too Many Requests from registry.npmjs.org
       # - Docker image pull failures (e.g., beshultd/kibana-readonlyrest:*-ror-latest not found)
@@ -161,6 +185,11 @@ jobs:
         uses: ./.github/docker-memory-monitor
         with:
           action: stop
+      - name: Stop ES connectivity monitor
+        if: always()
+        uses: ./.github/es-connectivity-monitor
+        with:
+          action: stop
       - name: S3 Upload Videos & show logs
         if: failure()
         uses: ./.github/upload-videos

e2e-tests/cypress/e2e/Tenancy.cy.ts

@@ -12,6 +12,7 @@ import { Dashboard } from '../support/page-objects/Dashboard';
 import { TENANCY_QUERY_STRING_KEY } from '../support/types';
 import { Spaces } from '../support/page-objects/Spaces';
 import { kbnApiAdvancedClient } from '../support/helpers/KbnApiAdvancedClient';
+import { IndexManagement } from '../support/page-objects/IndexManagement';
 
 describe('Tenancy', () => {
   describe('should run tests', () => {
@@ -159,7 +160,7 @@ function runTests({
       finishUrl: urlWithTenancyId,
       spacePrefix: ''
     });
-
+    IndexManagement.waitingForSectionLoadingFinish();
     callbackAfterLogin?.();
 
     Spaces.createNewSpace(newSpace);

e2e-tests/cypress/support/page-objects/IndexManagement.ts

@@ -106,4 +106,9 @@ export class IndexManagement {
 
     cy.contains('[data-test-subj="title"]', "You don't have any data streams yet");
   }
+
+  static waitingForSectionLoadingFinish() {
+    cy.log('Waiting for section loading');
+    cy.get('[data-test-subj="sectionLoading"]', { timeout: 30000 }).should('not.exist');
+  }
 }

environments/eck-ror/kind-cluster/ror/base/log4j2.properties.yml

@@ -71,7 +71,7 @@ data:
     appender.header_warning.name = header_warning
 
     logger.ror.name=tech.beshu.ror.accesscontrol
-    logger.ror.level=info
+    logger.ror.level=debug
 kind: ConfigMap
 metadata:
   name: config-log4j2.properties.yml

environments/eck-ror/kind-cluster/ror/base/ror-initial-config.yml

@@ -31,13 +31,11 @@ data:
         access_control_rules:
 
         - name: "Kibana service account - token"
-          verbosity: error
           token_authentication:
             token: "Bearer ${KIBANA_SERVICE_ACCOUNT_TOKEN}" 
             username: service_account
 
         - name: "Kibana service account - user/pass"
-          verbosity: error
           auth_key: kibana:kibana
 
         - name: "PROBE"

environments/eck-ror/start.sh

@@ -23,7 +23,8 @@ show_help() {
   echo "  --cluster-type <type>    Cluster type: 'base' for basic cluster, 'apm' for cluster with APM (default: base)"
   echo "  --ror-es <version>       ReadonlyREST ES version (default: latest)"
   echo "  --ror-kbn <version>      ReadonlyREST Kibana version (default: latest)"
-  echo "  --dev                    Use development images"
+  echo "  --dev                    Use development images (both ES and KBN)"
+  echo "  --dev-kbn                Use development image for Kibana only"
   echo ""
   echo "Examples:"
   echo "  ./start.sh --es 8.11.0 --kbn 8.11.0 --eck 2.15.0                    # Start base cluster"
@@ -108,6 +109,10 @@ while [[ $# -gt 0 ]]; do
     export ROR_KBN_REPO="beshultd/kibana-readonlyrest-dev"
     shift
     ;;
+  --dev-kbn)
+    export ROR_KBN_REPO="beshultd/kibana-readonlyrest-dev"
+    shift
+    ;;
   --help|-h)
     show_help
     ;;

environments/elk-ror/start.sh

@@ -26,7 +26,8 @@ show_help() {
   echo "  --cluster-type <type>    Cluster type: 'base' for basic cluster, 'apm' for cluster with APM (default: base)"
   echo "  --ror-es <version>       ReadonlyREST ES version (default: latest)"
   echo "  --ror-kbn <version>      ReadonlyREST Kibana version (default: latest)"
-  echo "  --dev                    Use development images"
+  echo "  --dev                    Use development images (both ES and KBN)"
+  echo "  --dev-kbn                Use development image for Kibana only"
   echo ""
   echo "Examples:"
   echo "  ./start.sh --es 8.11.0 --kbn 8.11.0                    # Start base cluster"
@@ -101,6 +102,10 @@ while [[ $# -gt 0 ]]; do
     export ROR_KBN_REPO="beshultd/kibana-readonlyrest-dev"
     shift
     ;;
+  --dev-kbn)
+    export ROR_KBN_REPO="beshultd/kibana-readonlyrest-dev"
+    shift
+    ;;
   --help|-h)
     show_help
     ;;

main.sh

@@ -9,7 +9,8 @@ show_help() {
   echo "  --elk <version>          ELK stack version (required)"
   echo "  --ror-es <version>       ReadonlyREST ES version (default: latest)"
   echo "  --ror-kbn <version>      ReadonlyREST Kibana version (default: latest)"
-  echo "  --dev                    Use development images"
+  echo "  --dev                    Use development images (both ES and KBN)"
+  echo "  --dev-kbn                Use development image for Kibana only"
   echo ""
   echo "Examples:"
   echo "  ./main.sh --env docker --elk 8.11.0                    # Run E2E tests with Docker Compose"
@@ -23,6 +24,7 @@ OPTIONAL_ECK_ARG=""
 OPTIONAL_ROR_ES_ARG=""
 OPTIONAL_ROR_KBN_ARG=""
 OPTIONAL_DEV_ARG=""
+OPTIONAL_DEV_KBN_ARG=""
 MODE="e2e"
 CLUSTER_TYPE="apm"
 
@@ -102,6 +104,10 @@ while [[ $# -gt 0 ]]; do
     OPTIONAL_DEV_ARG="--dev"
     shift
     ;;
+  --dev-kbn)
+    OPTIONAL_DEV_KBN_ARG="--dev-kbn"
+    shift
+    ;;
   *)
     echo "Unknown option: $1"
     show_help
@@ -133,7 +139,7 @@ echo -e "
 
 echo -e "Running environment...\n"
 
-time ./environments/$ENV_NAME/start.sh --cluster-type "$CLUSTER_TYPE" --es "$ELK_VERSION" --kbn "$ELK_VERSION" $OPTIONAL_ECK_ARG $OPTIONAL_ROR_ES_ARG $OPTIONAL_ROR_KBN_ARG $OPTIONAL_DEV_ARG
+time ./environments/$ENV_NAME/start.sh --cluster-type "$CLUSTER_TYPE" --es "$ELK_VERSION" --kbn "$ELK_VERSION" $OPTIONAL_ECK_ARG $OPTIONAL_ROR_ES_ARG $OPTIONAL_ROR_KBN_ARG $OPTIONAL_DEV_ARG $OPTIONAL_DEV_KBN_ARG
 
 if [[ "$MODE" == "e2e" ]]; then
   echo -e "Running E2E tests...\n"