diff --git a/.gitlab/scripts/poll_e2e.sh b/.gitlab/scripts/poll_e2e.sh new file mode 100644 index 000000000..dd7679fea --- /dev/null +++ b/.gitlab/scripts/poll_e2e.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# Poll one downstream bridge by name (set E2E_BRIDGE_JOB_NAME in CI, e.g. e2e-test (arm64)). + +set -euo pipefail + +E2E_BRIDGE_JOB_NAME="${E2E_BRIDGE_JOB_NAME:-e2e-test}" + +curl -OL "binaries.ddbuild.io/dd-source/authanywhere/LATEST/authanywhere-linux-amd64" && mv "authanywhere-linux-amd64" /bin/authanywhere && chmod +x /bin/authanywhere + +BTI_CI_API_TOKEN=$(authanywhere --audience rapid-devex-ci) + +BTI_RESPONSE=$(curl --silent --request GET \ + --header "$BTI_CI_API_TOKEN" \ + --header "Content-Type: application/vnd.api+json" \ + "https://bti-ci-api.us1.ddbuild.io/internal/ci/gitlab/token?owner=DataDog&repository=datadog-lambda-extension") + +GITLAB_TOKEN=$(echo "$BTI_RESPONSE" | jq -r '.token // empty') +if [ -z "$GITLAB_TOKEN" ]; then + echo "ERROR: could not obtain GitLab token from BTI" + exit 1 +fi + +URL="${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/pipelines/${CI_PIPELINE_ID}/bridges" + +echo "Polling bridge: ${E2E_BRIDGE_JOB_NAME}" +echo "Fetching E2E job status from: $URL" + +while true; do + RESPONSE=$(curl -s --header "PRIVATE-TOKEN: ${GITLAB_TOKEN}" "$URL") + E2E_JOB_STATUS=$(echo "$RESPONSE" | jq -r --arg name "$E2E_BRIDGE_JOB_NAME" '.[] | select(.name == $name) | .downstream_pipeline.status') + echo -n "E2E job status (${E2E_BRIDGE_JOB_NAME}): $E2E_JOB_STATUS, " + if [ "$E2E_JOB_STATUS" == "success" ]; then + echo "E2E tests completed successfully" + exit 0 + elif [ "$E2E_JOB_STATUS" == "failed" ]; then + echo "E2E tests failed" + exit 1 + elif [ "$E2E_JOB_STATUS" == "running" ]; then + echo "E2E tests are still running, retrying in 2 minutes..." + elif [ "$E2E_JOB_STATUS" == "canceled" ]; then + echo "E2E tests were canceled" + exit 1 + elif [ "$E2E_JOB_STATUS" == "skipped" ]; then + echo "E2E tests were skipped" + exit 0 + else + echo "Unknown E2E test status: $E2E_JOB_STATUS, retrying in 2 minutes..." + fi + sleep 120 +done diff --git a/.gitlab/scripts/publish_layers.sh b/.gitlab/scripts/publish_layers.sh index e16d988e1..ea117f16b 100755 --- a/.gitlab/scripts/publish_layers.sh +++ b/.gitlab/scripts/publish_layers.sh @@ -159,3 +159,15 @@ while [ $latest_version -lt $VERSION ]; do done printf "[$REGION] Finished publishing layers...\n" + +if [ -n "${DOTENV:-}" ]; then + printf "[$REGION] Exporting layer ARN to %s...\n" "$DOTENV" + dotenv_key="${DOTENV_VAR_NAME:-EXTENSION_LAYER_ARN}" + latest_arn=$(aws lambda get-layer-version \ + --layer-name "$LAYER_NAME" \ + --version-number "$VERSION" \ + --region "$REGION" \ + --query 'LayerVersionArn' --output text) + echo "${dotenv_key}=${latest_arn}" >>"$DOTENV" + cat "$DOTENV" +fi diff --git a/.gitlab/templates/pipeline.yaml.tpl b/.gitlab/templates/pipeline.yaml.tpl index 55c51759b..65fd4aa08 100644 --- a/.gitlab/templates/pipeline.yaml.tpl +++ b/.gitlab/templates/pipeline.yaml.tpl @@ -1,9 +1,12 @@ +{{- $e2e_region := "us-west-2" -}} + stages: - test - compile - build - integration-tests - self-monitoring + - e2e - sign - publish @@ -141,6 +144,7 @@ sign layer ({{ $flavor.name }}): - .gitlab/scripts/sign_layers.sh prod {{ range $environment_name, $environment := (ds "environments").environments }} +{{- $dotenv := printf "%s_%s.env" $flavor.suffix $environment_name }} publish layer {{ $environment_name }} ({{ $flavor.name }}): stage: publish @@ -172,12 +176,21 @@ publish layer {{ $environment_name }} ({{ $flavor.name }}): - REGION: {{ range (ds "regions").regions }} - {{ .code }} {{- end}} +{{- if eq $environment_name "sandbox" }} + artifacts: + reports: + dotenv: {{ $dotenv }} +{{- end }} variables: LAYER_NAME_BASE_SUFFIX: {{ $flavor.layer_name_base_suffix }} ARCHITECTURE: {{ $flavor.arch }} LAYER_FILE: datadog_extension-{{ $flavor.suffix }}.zip ADD_LAYER_VERSION_PERMISSIONS: {{ $environment.add_layer_version_permissions }} AUTOMATICALLY_BUMP_VERSION: {{ $environment.automatically_bump_version }} +{{- if eq $environment_name "sandbox" }} + LAYER_DESCRIPTION: ${CI_COMMIT_SHORT_SHA} + DOTENV: {{ $dotenv }} +{{- end }} before_script: - EXTERNAL_ID_NAME={{ $environment.external_id }} ROLE_TO_ASSUME={{ $environment.role_to_assume }} AWS_ACCOUNT={{ $environment.account }} source .gitlab/scripts/get_secrets.sh script: @@ -217,6 +230,84 @@ publish layer [self-monitoring] ({{ $flavor.name }}): {{ end }} # end flavors +{{ range $f := (ds "flavors").flavors }} +{{ if $f.needs_layer_publish }} +{{- $dotenvE2E := printf "%s_sandbox_e2e.env" $f.suffix }} +{{ with $environment := (ds "environments").environments.sandbox }} + +publish layer e2e sandbox ({{ $f.name }}): + stage: e2e + tags: ["arch:amd64"] + image: ${CI_DOCKER_TARGET_IMAGE}:${CI_DOCKER_TARGET_VERSION} + rules: + - if: '$CI_COMMIT_TAG =~ /^v.*/' + when: on_success + variables: + LAYER_DESCRIPTION: $CI_COMMIT_TAG + - when: on_success + variables: + LAYER_DESCRIPTION: $CI_COMMIT_SHORT_SHA + needs: + - layer ({{ $f.name }}) +{{ if and (index $f "max_layer_compressed_size_mb") (index $f "max_layer_uncompressed_size_mb") }} + - check layer size ({{ $f.name }}) +{{ end }} + dependencies: + - layer ({{ $f.name }}) + artifacts: + reports: + dotenv: {{ $dotenvE2E }} + variables: + LAYER_NAME_BASE_SUFFIX: {{ $f.layer_name_base_suffix }} + ARCHITECTURE: {{ $f.arch }} + LAYER_FILE: datadog_extension-{{ $f.suffix }}.zip + REGION: {{ $e2e_region }} + ADD_LAYER_VERSION_PERMISSIONS: {{ $environment.add_layer_version_permissions }} + AUTOMATICALLY_BUMP_VERSION: {{ $environment.automatically_bump_version }} + DOTENV: {{ $dotenvE2E }} + before_script: + - EXTERNAL_ID_NAME={{ $environment.external_id }} ROLE_TO_ASSUME={{ $environment.role_to_assume }} AWS_ACCOUNT={{ $environment.account }} source .gitlab/scripts/get_secrets.sh + script: + - .gitlab/scripts/publish_layers.sh + +{{ end }} + +e2e-test ({{ $f.name }}): + stage: e2e + trigger: + project: DataDog/serverless-e2e-tests + strategy: depend + rules: + - if: '$CI_COMMIT_TAG =~ /^v.*/' + when: on_success + variables: + EXTENSION_VERSION: $CI_COMMIT_TAG + - when: on_success + variables: + EXTENSION_VERSION: $CI_COMMIT_SHORT_SHA + needs: + - job: "publish layer e2e sandbox ({{ $f.name }})" + artifacts: true + variables: + EXTENSION_LAYER_ARN: ${EXTENSION_LAYER_ARN} + +e2e-test-status ({{ $f.name }}): + stage: e2e + image: registry.ddbuild.io/images/docker:20.10-py3 + tags: ["arch:amd64"] + timeout: 3h + rules: + - when: on_success + needs: + - job: "e2e-test ({{ $f.name }})" + variables: + E2E_BRIDGE_JOB_NAME: "e2e-test ({{ $f.name }})" + script: + - .gitlab/scripts/poll_e2e.sh + +{{ end }} +{{ end }} + {{ range $multi_arch_image_flavor := (ds "flavors").multi_arch_image_flavors }} publish private images ({{ $multi_arch_image_flavor.name }}):