Add e2e test trigger on PRs

.gitlab/scripts/poll_e2e.sh

@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+# Poll one downstream bridge by name (set E2E_BRIDGE_JOB_NAME in CI, e.g. e2e-test (arm64)).
+
+set -euo pipefail
+
+E2E_BRIDGE_JOB_NAME="${E2E_BRIDGE_JOB_NAME:-e2e-test}"
+
+curl -OL "binaries.ddbuild.io/dd-source/authanywhere/LATEST/authanywhere-linux-amd64" && mv "authanywhere-linux-amd64" /bin/authanywhere && chmod +x /bin/authanywhere
+
+BTI_CI_API_TOKEN=$(authanywhere --audience rapid-devex-ci)
+
+BTI_RESPONSE=$(curl --silent --request GET \
+    --header "$BTI_CI_API_TOKEN" \
+    --header "Content-Type: application/vnd.api+json" \
+    "https://bti-ci-api.us1.ddbuild.io/internal/ci/gitlab/token?owner=DataDog&repository=datadog-lambda-extension")
+
+GITLAB_TOKEN=$(echo "$BTI_RESPONSE" | jq -r '.token // empty')
+if [ -z "$GITLAB_TOKEN" ]; then
+    echo "ERROR: could not obtain GitLab token from BTI"
+    exit 1
+fi
+
+URL="${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/pipelines/${CI_PIPELINE_ID}/bridges"
+
+echo "Polling bridge: ${E2E_BRIDGE_JOB_NAME}"
+echo "Fetching E2E job status from: $URL"
+
+while true; do
+    RESPONSE=$(curl -s --header "PRIVATE-TOKEN: ${GITLAB_TOKEN}" "$URL")
+    E2E_JOB_STATUS=$(echo "$RESPONSE" | jq -r --arg name "$E2E_BRIDGE_JOB_NAME" '.[] | select(.name == $name) | .downstream_pipeline.status')
+    echo -n "E2E job status (${E2E_BRIDGE_JOB_NAME}): $E2E_JOB_STATUS, "
+    if [ "$E2E_JOB_STATUS" == "success" ]; then
+        echo "E2E tests completed successfully"
+        exit 0
+    elif [ "$E2E_JOB_STATUS" == "failed" ]; then
+        echo "E2E tests failed"
+        exit 1
+    elif [ "$E2E_JOB_STATUS" == "running" ]; then
+        echo "E2E tests are still running, retrying in 2 minutes..."
+    elif [ "$E2E_JOB_STATUS" == "canceled" ]; then
+        echo "E2E tests were canceled"
+        exit 1
+    elif [ "$E2E_JOB_STATUS" == "skipped" ]; then
+        echo "E2E tests were skipped"
+        exit 0
+    else
+        echo "Unknown E2E test status: $E2E_JOB_STATUS, retrying in 2 minutes..."
+    fi
+    sleep 120
+done

.gitlab/scripts/publish_layers.sh

@@ -159,3 +159,15 @@ while [ $latest_version -lt $VERSION ]; do
 done
 
 printf "[$REGION] Finished publishing layers...\n"
+
+if [ -n "${DOTENV:-}" ]; then
+    printf "[$REGION] Exporting layer ARN to %s...\n" "$DOTENV"
+    dotenv_key="${DOTENV_VAR_NAME:-EXTENSION_LAYER_ARN}"
+    latest_arn=$(aws lambda get-layer-version \
+        --layer-name "$LAYER_NAME" \
+        --version-number "$VERSION" \
+        --region "$REGION" \
+        --query 'LayerVersionArn' --output text)
+    echo "${dotenv_key}=${latest_arn}" >>"$DOTENV"
+    cat "$DOTENV"
+fi

.gitlab/templates/pipeline.yaml.tpl

@@ -1,9 +1,12 @@
+{{- $e2e_region := "us-west-2" -}}
+
 stages:
   - test
   - compile
   - build
   - integration-tests
   - self-monitoring
+  - e2e
   - sign
   - publish
 
@@ -141,6 +144,7 @@ sign layer ({{ $flavor.name }}):
     - .gitlab/scripts/sign_layers.sh prod
 
 {{ range $environment_name, $environment := (ds "environments").environments }}
+{{- $dotenv := printf "%s_%s.env" $flavor.suffix $environment_name }}
 
 publish layer {{ $environment_name }} ({{ $flavor.name }}):
   stage: publish
@@ -172,12 +176,21 @@ publish layer {{ $environment_name }} ({{ $flavor.name }}):
       - REGION: {{ range (ds "regions").regions }}
           - {{ .code }}
         {{- end}}
+{{- if eq $environment_name "sandbox" }}
+  artifacts:
+    reports:
+      dotenv: {{ $dotenv }}
+{{- end }}
   variables:
     LAYER_NAME_BASE_SUFFIX: {{ $flavor.layer_name_base_suffix }}
     ARCHITECTURE: {{ $flavor.arch }}
     LAYER_FILE: datadog_extension-{{ $flavor.suffix }}.zip
     ADD_LAYER_VERSION_PERMISSIONS: {{ $environment.add_layer_version_permissions }}
     AUTOMATICALLY_BUMP_VERSION: {{ $environment.automatically_bump_version }}
+{{- if eq $environment_name "sandbox" }}
+    LAYER_DESCRIPTION: ${CI_COMMIT_SHORT_SHA}
+    DOTENV: {{ $dotenv }}
+{{- end }}
   before_script:
     - EXTERNAL_ID_NAME={{ $environment.external_id }} ROLE_TO_ASSUME={{ $environment.role_to_assume }} AWS_ACCOUNT={{ $environment.account }} source .gitlab/scripts/get_secrets.sh
   script:
@@ -217,6 +230,84 @@ publish layer [self-monitoring] ({{ $flavor.name }}):
 
 {{ end }}  # end flavors
 
+{{ range $f := (ds "flavors").flavors }}
+{{ if $f.needs_layer_publish }}
+{{- $dotenvE2E := printf "%s_sandbox_e2e.env" $f.suffix }}
+{{ with $environment := (ds "environments").environments.sandbox }}
+
+publish layer e2e sandbox ({{ $f.name }}):
+  stage: e2e
+  tags: ["arch:amd64"]
+  image: ${CI_DOCKER_TARGET_IMAGE}:${CI_DOCKER_TARGET_VERSION}
+  rules:
+    - if: '$CI_COMMIT_TAG =~ /^v.*/'
+      when: on_success
+      variables:
+        LAYER_DESCRIPTION: $CI_COMMIT_TAG
+    - when: on_success
+      variables:
+        LAYER_DESCRIPTION: $CI_COMMIT_SHORT_SHA
+  needs:
+    - layer ({{ $f.name }})
+{{ if and (index $f "max_layer_compressed_size_mb") (index $f "max_layer_uncompressed_size_mb") }}
+    - check layer size ({{ $f.name }})
+{{ end }}
+  dependencies:
+    - layer ({{ $f.name }})
+  artifacts:
+    reports:
+      dotenv: {{ $dotenvE2E }}
+  variables:
+    LAYER_NAME_BASE_SUFFIX: {{ $f.layer_name_base_suffix }}
+    ARCHITECTURE: {{ $f.arch }}
+    LAYER_FILE: datadog_extension-{{ $f.suffix }}.zip
+    REGION: {{ $e2e_region }}
+    ADD_LAYER_VERSION_PERMISSIONS: {{ $environment.add_layer_version_permissions }}
+    AUTOMATICALLY_BUMP_VERSION: {{ $environment.automatically_bump_version }}
+    DOTENV: {{ $dotenvE2E }}
+  before_script:
+    - EXTERNAL_ID_NAME={{ $environment.external_id }} ROLE_TO_ASSUME={{ $environment.role_to_assume }} AWS_ACCOUNT={{ $environment.account }} source .gitlab/scripts/get_secrets.sh
+  script:
+    - .gitlab/scripts/publish_layers.sh
+
+{{ end }}
+
+e2e-test ({{ $f.name }}):
+  stage: e2e
+  trigger:
+    project: DataDog/serverless-e2e-tests
+    strategy: depend
+  rules:
+    - if: '$CI_COMMIT_TAG =~ /^v.*/'
+      when: on_success
+      variables:
+        EXTENSION_VERSION: $CI_COMMIT_TAG
+    - when: on_success
+      variables:
+        EXTENSION_VERSION: $CI_COMMIT_SHORT_SHA
+  needs:
+    - job: "publish layer e2e sandbox ({{ $f.name }})"
+      artifacts: true
+  variables:
+    EXTENSION_LAYER_ARN: ${EXTENSION_LAYER_ARN}
+
+e2e-test-status ({{ $f.name }}):
+  stage: e2e
+  image: registry.ddbuild.io/images/docker:20.10-py3
+  tags: ["arch:amd64"]
+  timeout: 3h
+  rules:
+    - when: on_success
+  needs:
+    - job: "e2e-test ({{ $f.name }})"
+  variables:
+    E2E_BRIDGE_JOB_NAME: "e2e-test ({{ $f.name }})"
+  script:
+    - .gitlab/scripts/poll_e2e.sh
+
+{{ end }}
+{{ end }}
+
 {{ range $multi_arch_image_flavor := (ds "flavors").multi_arch_image_flavors }}
 
 publish private images ({{ $multi_arch_image_flavor.name }}):