From c6a63832e376940f3634742d430d3c01c70bb3ab Mon Sep 17 00:00:00 2001 From: shreyamalpani Date: Tue, 7 Apr 2026 16:06:22 -0400 Subject: [PATCH 1/5] launch e2e runs on PRs --- .gitlab/scripts/poll_e2e.sh | 50 +++++++++++++++++++++++++++ .gitlab/scripts/publish_layers.sh | 12 +++++++ .gitlab/templates/pipeline.yaml.tpl | 53 +++++++++++++++++++++++++++++ 3 files changed, 115 insertions(+) create mode 100644 .gitlab/scripts/poll_e2e.sh diff --git a/.gitlab/scripts/poll_e2e.sh b/.gitlab/scripts/poll_e2e.sh new file mode 100644 index 000000000..dd7679fea --- /dev/null +++ b/.gitlab/scripts/poll_e2e.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# Poll one downstream bridge by name (set E2E_BRIDGE_JOB_NAME in CI, e.g. e2e-test (arm64)). + +set -euo pipefail + +E2E_BRIDGE_JOB_NAME="${E2E_BRIDGE_JOB_NAME:-e2e-test}" + +curl -OL "binaries.ddbuild.io/dd-source/authanywhere/LATEST/authanywhere-linux-amd64" && mv "authanywhere-linux-amd64" /bin/authanywhere && chmod +x /bin/authanywhere + +BTI_CI_API_TOKEN=$(authanywhere --audience rapid-devex-ci) + +BTI_RESPONSE=$(curl --silent --request GET \ + --header "$BTI_CI_API_TOKEN" \ + --header "Content-Type: application/vnd.api+json" \ + "https://bti-ci-api.us1.ddbuild.io/internal/ci/gitlab/token?owner=DataDog&repository=datadog-lambda-extension") + +GITLAB_TOKEN=$(echo "$BTI_RESPONSE" | jq -r '.token // empty') +if [ -z "$GITLAB_TOKEN" ]; then + echo "ERROR: could not obtain GitLab token from BTI" + exit 1 +fi + +URL="${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/pipelines/${CI_PIPELINE_ID}/bridges" + +echo "Polling bridge: ${E2E_BRIDGE_JOB_NAME}" +echo "Fetching E2E job status from: $URL" + +while true; do + RESPONSE=$(curl -s --header "PRIVATE-TOKEN: ${GITLAB_TOKEN}" "$URL") + E2E_JOB_STATUS=$(echo "$RESPONSE" | jq -r --arg name "$E2E_BRIDGE_JOB_NAME" '.[] | select(.name == $name) | .downstream_pipeline.status') + echo -n "E2E job status (${E2E_BRIDGE_JOB_NAME}): $E2E_JOB_STATUS, " + if [ "$E2E_JOB_STATUS" == "success" ]; then + echo "E2E tests completed successfully" + exit 0 + elif [ "$E2E_JOB_STATUS" == "failed" ]; then + echo "E2E tests failed" + exit 1 + elif [ "$E2E_JOB_STATUS" == "running" ]; then + echo "E2E tests are still running, retrying in 2 minutes..." + elif [ "$E2E_JOB_STATUS" == "canceled" ]; then + echo "E2E tests were canceled" + exit 1 + elif [ "$E2E_JOB_STATUS" == "skipped" ]; then + echo "E2E tests were skipped" + exit 0 + else + echo "Unknown E2E test status: $E2E_JOB_STATUS, retrying in 2 minutes..." + fi + sleep 120 +done diff --git a/.gitlab/scripts/publish_layers.sh b/.gitlab/scripts/publish_layers.sh index e16d988e1..ea117f16b 100755 --- a/.gitlab/scripts/publish_layers.sh +++ b/.gitlab/scripts/publish_layers.sh @@ -159,3 +159,15 @@ while [ $latest_version -lt $VERSION ]; do done printf "[$REGION] Finished publishing layers...\n" + +if [ -n "${DOTENV:-}" ]; then + printf "[$REGION] Exporting layer ARN to %s...\n" "$DOTENV" + dotenv_key="${DOTENV_VAR_NAME:-EXTENSION_LAYER_ARN}" + latest_arn=$(aws lambda get-layer-version \ + --layer-name "$LAYER_NAME" \ + --version-number "$VERSION" \ + --region "$REGION" \ + --query 'LayerVersionArn' --output text) + echo "${dotenv_key}=${latest_arn}" >>"$DOTENV" + cat "$DOTENV" +fi diff --git a/.gitlab/templates/pipeline.yaml.tpl b/.gitlab/templates/pipeline.yaml.tpl index 55c51759b..c768d5eab 100644 --- a/.gitlab/templates/pipeline.yaml.tpl +++ b/.gitlab/templates/pipeline.yaml.tpl @@ -1,3 +1,5 @@ +{{- $e2e_region := "us-west-2" -}} + stages: - test - compile @@ -6,6 +8,7 @@ stages: - self-monitoring - sign - publish + - e2e default: retry: @@ -141,12 +144,16 @@ sign layer ({{ $flavor.name }}): - .gitlab/scripts/sign_layers.sh prod {{ range $environment_name, $environment := (ds "environments").environments }} +{{- $dotenv := printf "%s_%s.env" $flavor.suffix $environment_name }} publish layer {{ $environment_name }} ({{ $flavor.name }}): stage: publish tags: ["arch:amd64"] image: ${CI_DOCKER_TARGET_IMAGE}:${CI_DOCKER_TARGET_VERSION} rules: + # MR pipelines: auto-publish all extension flavors to the e2e region only (see $e2e_region). + - if: '"{{ $environment_name }}" == "sandbox" && $REGION == "{{ $e2e_region }}"' + when: on_success - if: '"{{ $environment_name }}" == "sandbox"' when: manual allow_failure: true @@ -172,12 +179,21 @@ publish layer {{ $environment_name }} ({{ $flavor.name }}): - REGION: {{ range (ds "regions").regions }} - {{ .code }} {{- end}} +{{- if eq $environment_name "sandbox" }} + artifacts: + reports: + dotenv: {{ $dotenv }} +{{- end }} variables: LAYER_NAME_BASE_SUFFIX: {{ $flavor.layer_name_base_suffix }} ARCHITECTURE: {{ $flavor.arch }} LAYER_FILE: datadog_extension-{{ $flavor.suffix }}.zip ADD_LAYER_VERSION_PERMISSIONS: {{ $environment.add_layer_version_permissions }} AUTOMATICALLY_BUMP_VERSION: {{ $environment.automatically_bump_version }} +{{- if eq $environment_name "sandbox" }} + LAYER_DESCRIPTION: ${CI_COMMIT_SHORT_SHA} + DOTENV: {{ $dotenv }} +{{- end }} before_script: - EXTERNAL_ID_NAME={{ $environment.external_id }} ROLE_TO_ASSUME={{ $environment.role_to_assume }} AWS_ACCOUNT={{ $environment.account }} source .gitlab/scripts/get_secrets.sh script: @@ -217,6 +233,43 @@ publish layer [self-monitoring] ({{ $flavor.name }}): {{ end }} # end flavors +# MR: one serverless-e2e-tests child pipeline per publishable flavor (amd64, arm64, amd64 fips, arm64 fips). +{{ range $f := (ds "flavors").flavors }} +{{ if $f.needs_layer_publish }} + +e2e-test ({{ $f.name }}): + stage: e2e + trigger: + project: DataDog/serverless-e2e-tests + strategy: depend + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + when: on_success + needs: + - job: "publish layer sandbox ({{ $f.name }}): [{{ $e2e_region }}]" + artifacts: true + variables: + EXTENSION_VERSION: ${CI_COMMIT_SHORT_SHA} + EXTENSION_LAYER_ARN: ${EXTENSION_LAYER_ARN} + +e2e-test-status ({{ $f.name }}): + stage: e2e + image: registry.ddbuild.io/images/docker:20.10-py3 + tags: ["arch:amd64"] + timeout: 3h + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + when: on_success + needs: + - job: "e2e-test ({{ $f.name }})" + variables: + E2E_BRIDGE_JOB_NAME: "e2e-test ({{ $f.name }})" + script: + - .gitlab/scripts/poll_e2e.sh + +{{ end }} +{{ end }} + {{ range $multi_arch_image_flavor := (ds "flavors").multi_arch_image_flavors }} publish private images ({{ $multi_arch_image_flavor.name }}): From fddf68ca5d293893694545e5d39afeea395467c1 Mon Sep 17 00:00:00 2001 From: shreyamalpani Date: Tue, 7 Apr 2026 17:33:59 -0400 Subject: [PATCH 2/5] move e2e stage --- .gitlab/templates/pipeline.yaml.tpl | 61 ++++++++++++++++++++++++----- 1 file changed, 52 insertions(+), 9 deletions(-) diff --git a/.gitlab/templates/pipeline.yaml.tpl b/.gitlab/templates/pipeline.yaml.tpl index c768d5eab..32d628297 100644 --- a/.gitlab/templates/pipeline.yaml.tpl +++ b/.gitlab/templates/pipeline.yaml.tpl @@ -6,9 +6,9 @@ stages: - build - integration-tests - self-monitoring + - e2e - sign - publish - - e2e default: retry: @@ -151,9 +151,6 @@ publish layer {{ $environment_name }} ({{ $flavor.name }}): tags: ["arch:amd64"] image: ${CI_DOCKER_TARGET_IMAGE}:${CI_DOCKER_TARGET_VERSION} rules: - # MR pipelines: auto-publish all extension flavors to the e2e region only (see $e2e_region). - - if: '"{{ $environment_name }}" == "sandbox" && $REGION == "{{ $e2e_region }}"' - when: on_success - if: '"{{ $environment_name }}" == "sandbox"' when: manual allow_failure: true @@ -233,9 +230,48 @@ publish layer [self-monitoring] ({{ $flavor.name }}): {{ end }} # end flavors -# MR: one serverless-e2e-tests child pipeline per publishable flavor (amd64, arm64, amd64 fips, arm64 fips). {{ range $f := (ds "flavors").flavors }} {{ if $f.needs_layer_publish }} +{{- $dotenvE2E := printf "%s_sandbox_e2e.env" $f.suffix }} +{{ with $environment := (ds "environments").environments.sandbox }} + +publish layer e2e sandbox ({{ $f.name }}): + stage: e2e + tags: ["arch:amd64"] + image: ${CI_DOCKER_TARGET_IMAGE}:${CI_DOCKER_TARGET_VERSION} + rules: + - if: '$CI_COMMIT_TAG =~ /^v.*/' + when: on_success + variables: + LAYER_DESCRIPTION: $CI_COMMIT_TAG + - if: '$CI_PIPELINE_SOURCE == "merge_request_event" || $CI_MERGE_REQUEST_IID' + when: on_success + variables: + LAYER_DESCRIPTION: $CI_COMMIT_SHORT_SHA + needs: + - layer ({{ $f.name }}) +{{ if and (index $f "max_layer_compressed_size_mb") (index $f "max_layer_uncompressed_size_mb") }} + - check layer size ({{ $f.name }}) +{{ end }} + dependencies: + - layer ({{ $f.name }}) + artifacts: + reports: + dotenv: {{ $dotenvE2E }} + variables: + LAYER_NAME_BASE_SUFFIX: {{ $f.layer_name_base_suffix }} + ARCHITECTURE: {{ $f.arch }} + LAYER_FILE: datadog_extension-{{ $f.suffix }}.zip + REGION: {{ $e2e_region }} + ADD_LAYER_VERSION_PERMISSIONS: {{ $environment.add_layer_version_permissions }} + AUTOMATICALLY_BUMP_VERSION: {{ $environment.automatically_bump_version }} + DOTENV: {{ $dotenvE2E }} + before_script: + - EXTERNAL_ID_NAME={{ $environment.external_id }} ROLE_TO_ASSUME={{ $environment.role_to_assume }} AWS_ACCOUNT={{ $environment.account }} source .gitlab/scripts/get_secrets.sh + script: + - .gitlab/scripts/publish_layers.sh + +{{ end }} e2e-test ({{ $f.name }}): stage: e2e @@ -243,13 +279,18 @@ e2e-test ({{ $f.name }}): project: DataDog/serverless-e2e-tests strategy: depend rules: - - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + - if: '$CI_COMMIT_TAG =~ /^v.*/' when: on_success + variables: + EXTENSION_VERSION: $CI_COMMIT_TAG + - if: '$CI_PIPELINE_SOURCE == "merge_request_event" || $CI_MERGE_REQUEST_IID' + when: on_success + variables: + EXTENSION_VERSION: $CI_COMMIT_SHORT_SHA needs: - - job: "publish layer sandbox ({{ $f.name }}): [{{ $e2e_region }}]" + - job: "publish layer e2e sandbox ({{ $f.name }})" artifacts: true variables: - EXTENSION_VERSION: ${CI_COMMIT_SHORT_SHA} EXTENSION_LAYER_ARN: ${EXTENSION_LAYER_ARN} e2e-test-status ({{ $f.name }}): @@ -258,7 +299,9 @@ e2e-test-status ({{ $f.name }}): tags: ["arch:amd64"] timeout: 3h rules: - - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + - if: '$CI_COMMIT_TAG =~ /^v.*/' + when: on_success + - if: '$CI_PIPELINE_SOURCE == "merge_request_event" || $CI_MERGE_REQUEST_IID' when: on_success needs: - job: "e2e-test ({{ $f.name }})" From 57f19046a77c4b9eae97189d5bfe735c6ec24bea Mon Sep 17 00:00:00 2001 From: shreyamalpani Date: Tue, 7 Apr 2026 17:42:45 -0400 Subject: [PATCH 3/5] run on open PRs --- .gitlab/templates/pipeline.yaml.tpl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab/templates/pipeline.yaml.tpl b/.gitlab/templates/pipeline.yaml.tpl index 32d628297..e0c0c059f 100644 --- a/.gitlab/templates/pipeline.yaml.tpl +++ b/.gitlab/templates/pipeline.yaml.tpl @@ -244,7 +244,7 @@ publish layer e2e sandbox ({{ $f.name }}): when: on_success variables: LAYER_DESCRIPTION: $CI_COMMIT_TAG - - if: '$CI_PIPELINE_SOURCE == "merge_request_event" || $CI_MERGE_REQUEST_IID' + - if: '$CI_PIPELINE_SOURCE == "merge_request_event" || $CI_MERGE_REQUEST_IID || $CI_OPEN_MERGE_REQUESTS' when: on_success variables: LAYER_DESCRIPTION: $CI_COMMIT_SHORT_SHA @@ -283,7 +283,7 @@ e2e-test ({{ $f.name }}): when: on_success variables: EXTENSION_VERSION: $CI_COMMIT_TAG - - if: '$CI_PIPELINE_SOURCE == "merge_request_event" || $CI_MERGE_REQUEST_IID' + - if: '$CI_PIPELINE_SOURCE == "merge_request_event" || $CI_MERGE_REQUEST_IID || $CI_OPEN_MERGE_REQUESTS' when: on_success variables: EXTENSION_VERSION: $CI_COMMIT_SHORT_SHA @@ -301,7 +301,7 @@ e2e-test-status ({{ $f.name }}): rules: - if: '$CI_COMMIT_TAG =~ /^v.*/' when: on_success - - if: '$CI_PIPELINE_SOURCE == "merge_request_event" || $CI_MERGE_REQUEST_IID' + - if: '$CI_PIPELINE_SOURCE == "merge_request_event" || $CI_MERGE_REQUEST_IID || $CI_OPEN_MERGE_REQUESTS' when: on_success needs: - job: "e2e-test ({{ $f.name }})" From 9d19fe3eb09474966dc0c5d6fa1e008303461d20 Mon Sep 17 00:00:00 2001 From: shreyamalpani Date: Tue, 7 Apr 2026 17:51:02 -0400 Subject: [PATCH 4/5] check parent MRs --- .gitlab-ci.yml | 5 +++++ .gitlab/templates/pipeline.yaml.tpl | 6 +++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2fe746831..1c5ce63c8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -54,6 +54,11 @@ generator: generated pipeline: stage: run + variables: + # Forward MR context to child pipeline since $CI_PIPELINE_SOURCE becomes + # "parent_pipeline" in downstream pipelines and MR variables are not inherited. + PARENT_PIPELINE_SOURCE: $CI_PIPELINE_SOURCE + PARENT_OPEN_MRS: $CI_OPEN_MERGE_REQUESTS trigger: include: - artifact: .gitlab/pipeline.yaml diff --git a/.gitlab/templates/pipeline.yaml.tpl b/.gitlab/templates/pipeline.yaml.tpl index e0c0c059f..027a2a919 100644 --- a/.gitlab/templates/pipeline.yaml.tpl +++ b/.gitlab/templates/pipeline.yaml.tpl @@ -244,7 +244,7 @@ publish layer e2e sandbox ({{ $f.name }}): when: on_success variables: LAYER_DESCRIPTION: $CI_COMMIT_TAG - - if: '$CI_PIPELINE_SOURCE == "merge_request_event" || $CI_MERGE_REQUEST_IID || $CI_OPEN_MERGE_REQUESTS' + - if: '$PARENT_PIPELINE_SOURCE == "merge_request_event" || $PARENT_OPEN_MRS' when: on_success variables: LAYER_DESCRIPTION: $CI_COMMIT_SHORT_SHA @@ -283,7 +283,7 @@ e2e-test ({{ $f.name }}): when: on_success variables: EXTENSION_VERSION: $CI_COMMIT_TAG - - if: '$CI_PIPELINE_SOURCE == "merge_request_event" || $CI_MERGE_REQUEST_IID || $CI_OPEN_MERGE_REQUESTS' + - if: '$PARENT_PIPELINE_SOURCE == "merge_request_event" || $PARENT_OPEN_MRS' when: on_success variables: EXTENSION_VERSION: $CI_COMMIT_SHORT_SHA @@ -301,7 +301,7 @@ e2e-test-status ({{ $f.name }}): rules: - if: '$CI_COMMIT_TAG =~ /^v.*/' when: on_success - - if: '$CI_PIPELINE_SOURCE == "merge_request_event" || $CI_MERGE_REQUEST_IID || $CI_OPEN_MERGE_REQUESTS' + - if: '$PARENT_PIPELINE_SOURCE == "merge_request_event" || $PARENT_OPEN_MRS' when: on_success needs: - job: "e2e-test ({{ $f.name }})" From f64463f39ce5a2b35d791bea6e8e4eadaec59142 Mon Sep 17 00:00:00 2001 From: shreyamalpani Date: Tue, 7 Apr 2026 18:01:14 -0400 Subject: [PATCH 5/5] run on all pushes --- .gitlab-ci.yml | 5 ----- .gitlab/templates/pipeline.yaml.tpl | 11 +++-------- 2 files changed, 3 insertions(+), 13 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1c5ce63c8..2fe746831 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -54,11 +54,6 @@ generator: generated pipeline: stage: run - variables: - # Forward MR context to child pipeline since $CI_PIPELINE_SOURCE becomes - # "parent_pipeline" in downstream pipelines and MR variables are not inherited. - PARENT_PIPELINE_SOURCE: $CI_PIPELINE_SOURCE - PARENT_OPEN_MRS: $CI_OPEN_MERGE_REQUESTS trigger: include: - artifact: .gitlab/pipeline.yaml diff --git a/.gitlab/templates/pipeline.yaml.tpl b/.gitlab/templates/pipeline.yaml.tpl index 027a2a919..65fd4aa08 100644 --- a/.gitlab/templates/pipeline.yaml.tpl +++ b/.gitlab/templates/pipeline.yaml.tpl @@ -244,8 +244,7 @@ publish layer e2e sandbox ({{ $f.name }}): when: on_success variables: LAYER_DESCRIPTION: $CI_COMMIT_TAG - - if: '$PARENT_PIPELINE_SOURCE == "merge_request_event" || $PARENT_OPEN_MRS' - when: on_success + - when: on_success variables: LAYER_DESCRIPTION: $CI_COMMIT_SHORT_SHA needs: @@ -283,8 +282,7 @@ e2e-test ({{ $f.name }}): when: on_success variables: EXTENSION_VERSION: $CI_COMMIT_TAG - - if: '$PARENT_PIPELINE_SOURCE == "merge_request_event" || $PARENT_OPEN_MRS' - when: on_success + - when: on_success variables: EXTENSION_VERSION: $CI_COMMIT_SHORT_SHA needs: @@ -299,10 +297,7 @@ e2e-test-status ({{ $f.name }}): tags: ["arch:amd64"] timeout: 3h rules: - - if: '$CI_COMMIT_TAG =~ /^v.*/' - when: on_success - - if: '$PARENT_PIPELINE_SOURCE == "merge_request_event" || $PARENT_OPEN_MRS' - when: on_success + - when: on_success needs: - job: "e2e-test ({{ $f.name }})" variables: