Merge Master to edge

azure/common/apigee-deployment.yml

@@ -23,20 +23,20 @@ parameters:
   - name: fully_qualified_service_name
     displayName: Supply a custom name for your release (only valid for release pipeline)
     type: string
-    default: ''
+    default: ""
   - name: ping
     displayName: Toggle ping endpoint avaliablility
     type: boolean
     default: true
   - name: hosted_target_connection_path_suffix
     type: string
-    default: ''
+    default: ""
   - name: hosted_target_healthcheck_suffix
     type: string
-    default: '/_status'
+    default: "/_status"
   - name: jinja_templates
     type: object
-    displayName:  Key/values for custom jinja templating
+    displayName: Key/values for custom jinja templating
     default: []
   - name: enable_monitoring
     type: boolean
@@ -85,7 +85,7 @@ parameters:
   - name: manual_approval_env
     displayName: Custom manual approval env overwrite
     type: string
-    default: 'manual-approval'
+    default: "manual-approval"
   - name: _scoped_pipeline_vars
     type: object
     displayName: All the pipeline parameters which support per-environment scoping.
@@ -121,12 +121,12 @@ parameters:
   - name: _pipeline_defaults
     type: object
     default:
-      - name_suffix: '-Pull-Request'
+      - name_suffix: "-Pull-Request"
         deploy_template: pr.yml
         environments:
-        - internal-dev
-        - internal-dev-sandbox
-      - name_suffix: '-Release'
+          - internal-dev
+          - internal-dev-sandbox
+      - name_suffix: "-Release"
         deploy_template: release.yml
         environments:
           - internal-dev
@@ -139,107 +139,106 @@ parameters:
           - prod
     # Deprecated pipeline variables.
   - name: product_display_name
-    default: 'DEPRECATED'
+    default: "DEPRECATED"
   - name: product_description
-    default: 'DEPRECATED'
+    default: "DEPRECATED"
   - name: portal_api_requires_callback_url
-    default: 'DEPRECATED'
+    default: "DEPRECATED"
   - name: make_spec_visible
-    default: 'DEPRECATED'
+    default: "DEPRECATED"
   - name: spec_file
-    default: 'DEPRECATED'
+    default: "DEPRECATED"
   - name: python_version
     type: string
     default: "3.13"
   - name: agent_pool
     type: string
     default: "AWS-ECS"
 
+stages:
+  - template: ./deploy-stages.yml
+    parameters:
+      ${{ each param in parameters }}:
+        ${{ if notIn(param.key, 'apigee_deployments', '_scoped_pipeline_vars', '_scoped_step_list_vars', '_pipeline_defaults') }}:
+          ${{ if not(contains(join(',', parameters._scoped_pipeline_vars), param.key)) }}:
+            ${{ param.key }}: ${{ param.value }}
 
-extends:
-  template: ./deploy-stages.yml
-  parameters:
-    ${{ each param in parameters }}:
-      ${{ if notIn(param.key, 'apigee_deployments', '_scoped_pipeline_vars', '_scoped_step_list_vars', '_pipeline_defaults') }}:
-        ${{ if not(contains(join(',', parameters._scoped_pipeline_vars), param.key)) }}:
-          ${{ param.key }}: ${{ param.value }}
+      ${{ each pipeline_default in parameters._pipeline_defaults }}:
+        ${{ if endsWith(variables['Build.DefinitionName'], pipeline_default.name_suffix) }}:
+          deploy_template: ${{ pipeline_default.deploy_template }}
 
-    ${{ each pipeline_default in parameters._pipeline_defaults }}:
-      ${{ if endsWith(variables['Build.DefinitionName'], pipeline_default.name_suffix) }}:
-        deploy_template: ${{ pipeline_default.deploy_template }}
+          # If apigee_deployments was specified, accept it regardless of which pipeline we are in
+          ${{ if gt(length(parameters.apigee_deployments), 0) }}:
+            apigee_deployments:
+              - ${{ each apigee_deployment in parameters.apigee_deployments }}:
+                  - ${{ if and(eq(apigee_deployment.environment, 'prod'), parameters.prod_requires_approval) }}:
+                      - environment: manual-approval
+                        stage_name: manual_approval
+                        depends_on: []
+                        manual_approval_prod: true
+                        producer_approval: ${{ parameters.prod_producer_approval }}
+                        ${{ each var in parameters._scoped_pipeline_vars }}:
+                          ${{ var }}: ${{ parameters[var] }}
+              - ${{ each apigee_deployment in parameters.apigee_deployments }}:
+                  - environment: ${{ apigee_deployment.environment }}
+                    stage_name: ${{ replace(coalesce (apigee_deployment.stage_name, apigee_deployment.environment), '-', '_') }}
+                    ${{ each var in parameters._scoped_pipeline_vars }}:
+                      ${{ if apigee_deployment[var] }}:
+                        ${{ if containsValue(parameters._scoped_step_list_vars, var) }}:
+                          ${{ var }}:
+                            - ${{ each step in apigee_deployment[var] }}:
+                                ${{ each pair in step }}:
+                                  # Assume scoped template paths are
+                                  # relative to azure/ in @self repo
+                                  ${{ if and(eq(pair.key, 'template'), not(contains(pair.value, '@'))) }}:
+                                    ${{ pair.key }}: azure/${{ pair.value }}@self
+                                  ${{ if not(and(eq(pair.key, 'template'), not(contains(pair.value, '@')))) }}:
+                                    ${{ pair.key }}: ${{ pair.value }}
+                        ${{ if not(containsValue(parameters._scoped_step_list_vars, var)) }}:
+                          ${{ var }}: ${{ apigee_deployment[var] }}
+                      ${{ if not(apigee_deployment[var]) }}:
+                        ${{ var }}: ${{ parameters[var] }}
+                    ${{ if not(apigee_deployment.depends_on) }}:
+                      ${{ if or(startsWith(apigee_deployment.environment, 'internal-'), eq(apigee_deployment.environment, 'manual-approval')) }}:
+                        depends_on: []
+                      ${{ if and(not(startsWith(apigee_deployment.environment, 'internal-')), ne(apigee_deployment.environment, 'manual-approval')) }}:
+                        depends_on:
+                          - internal_qa
+                          - ${{ if and(eq(apigee_deployment.environment, 'prod'), parameters.prod_requires_approval) }}:
+                              - manual_approval
+                    ${{ if apigee_deployment.depends_on }}:
+                      depends_on:
+                        - ${{ each depend_on in apigee_deployment.depends_on }}:
+                            - ${{ depend_on }}
+                        - ${{ if and(eq(apigee_deployment.environment, 'prod'), parameters.prod_requires_approval) }}:
+                            - manual_approval
 
-        # If apigee_deployments was specified, accept it regardless of which pipeline we are in
-        ${{ if gt(length(parameters.apigee_deployments), 0) }}:
-          apigee_deployments:
-            - ${{ each apigee_deployment in parameters.apigee_deployments }}:
-              - ${{ if and(eq(apigee_deployment.environment, 'prod'), parameters.prod_requires_approval) }}:
-                - environment: manual-approval
-                  stage_name: manual_approval
-                  depends_on: []
-                  manual_approval_prod: true
-                  producer_approval: ${{ parameters.prod_producer_approval }}
-                  ${{ each var in parameters._scoped_pipeline_vars }}:
-                      ${{ var }}: ${{ parameters[var] }}
-            - ${{ each apigee_deployment in parameters.apigee_deployments }}:
-              - environment: ${{ apigee_deployment.environment }}
-                stage_name: ${{ replace(coalesce (apigee_deployment.stage_name, apigee_deployment.environment), '-', '_') }}
-                ${{ each var in parameters._scoped_pipeline_vars }}:
-                  ${{ if apigee_deployment[var] }}:
-                    ${{ if containsValue(parameters._scoped_step_list_vars, var) }}:
-                      ${{ var }}:
-                        - ${{ each step in apigee_deployment[var] }}:
-                            ${{ each pair in step }}:
-                              # Assume scoped template paths are
-                              # relative to azure/ in @self repo
-                              ${{ if and(eq(pair.key, 'template'), not(contains(pair.value, '@'))) }}:
-                                ${{ pair.key }}: azure/${{ pair.value }}@self
-                              ${{ if not(and(eq(pair.key, 'template'), not(contains(pair.value, '@')))) }}:
-                                ${{ pair.key }}: ${{ pair.value }}
-                    ${{ if not(containsValue(parameters._scoped_step_list_vars, var)) }}:
-                      ${{ var }}: ${{ apigee_deployment[var] }}
-                  ${{ if not(apigee_deployment[var]) }}:
-                    ${{ var }}: ${{ parameters[var] }}
-                ${{ if not(apigee_deployment.depends_on) }}:
-                  ${{ if or(startsWith(apigee_deployment.environment, 'internal-'), eq(apigee_deployment.environment, 'manual-approval')) }}:
+          # otherwise set the default parameters for this pipeline
+          ${{ if eq(0, length(parameters.apigee_deployments)) }}:
+            apigee_deployments:
+              - ${{ each environment in pipeline_default.environments }}:
+                  - ${{ if notIn(environment, 'internal-dev-sandbox') }}:
+                      - environment: ${{ environment }}
+                        stage_name: ${{ replace(environment, '-', '_') }}
+                        ${{ if startsWith(environment, 'internal-') }}:
+                          depends_on: []
+                        ${{ if not(startsWith(environment, 'internal-')) }}:
+                          depends_on:
+                            - internal_qa
+                            - internal_qa_sandbox
+                            - ${{ if and(parameters.prod_requires_approval, eq(environment, 'prod')) }}:
+                                - manual_approval
+                        ${{ each var in parameters._scoped_pipeline_vars }}:
+                          ${{ var }}: ${{ parameters[var] }}
+              - ${{ if parameters.deploy_review_sandbox }}:
+                  - environment: internal-dev-sandbox
+                    stage_name: internal_dev_sandbox
                     depends_on: []
-                  ${{ if and(not(startsWith(apigee_deployment.environment, 'internal-')), ne(apigee_deployment.environment, 'manual-approval')) }}:
-                    depends_on:
-                      - internal_qa
-                      - ${{ if and(eq(apigee_deployment.environment, 'prod'), parameters.prod_requires_approval) }}:
-                        - manual_approval
-                ${{ if apigee_deployment.depends_on }}:
-                    depends_on:
-                      - ${{ each depend_on in apigee_deployment.depends_on }}:
-                        - ${{ depend_on }}
-                      - ${{ if and(eq(apigee_deployment.environment, 'prod'), parameters.prod_requires_approval) }}:
-                        - manual_approval
-
-        # otherwise set the default parameters for this pipeline
-        ${{ if eq(0, length(parameters.apigee_deployments)) }}:
-          apigee_deployments:
-            - ${{ each environment in pipeline_default.environments }}:
-              - ${{ if notIn(environment, 'internal-dev-sandbox') }}:
-                - environment: ${{ environment }}
-                  stage_name: ${{ replace(environment, '-', '_') }}
-                  ${{ if startsWith(environment, 'internal-') }}:
+                    ${{ each var in parameters._scoped_pipeline_vars }}:
+                      ${{ var }}: ${{ parameters[var] }}
+              - ${{ if and(parameters.prod_requires_approval, containsValue(pipeline_default.environments, 'prod')) }}:
+                  - environment: manual-approval
+                    stage_name: manual_approval
                     depends_on: []
-                  ${{ if not(startsWith(environment, 'internal-')) }}:
-                    depends_on:
-                      - internal_qa
-                      - internal_qa_sandbox
-                      - ${{ if and(parameters.prod_requires_approval, eq(environment, 'prod')) }}:
-                        - manual_approval
-                  ${{ each var in parameters._scoped_pipeline_vars }}:
-                    ${{ var }}: ${{ parameters[var] }}
-            - ${{ if parameters.deploy_review_sandbox }}:
-              - environment: internal-dev-sandbox
-                stage_name: internal_dev_sandbox
-                depends_on: []
-                ${{ each var in parameters._scoped_pipeline_vars }}:
-                    ${{ var }}: ${{ parameters[var] }}
-            - ${{ if and(parameters.prod_requires_approval, containsValue(pipeline_default.environments, 'prod')) }}:
-              - environment: manual-approval
-                stage_name: manual_approval
-                depends_on: []
-                ${{ each var in parameters._scoped_pipeline_vars }}:
-                    ${{ var }}: ${{ parameters[var] }}
+                    ${{ each var in parameters._scoped_pipeline_vars }}:
+                      ${{ var }}: ${{ parameters[var] }}

azure/common/deploy-stage.yml

@@ -167,11 +167,40 @@ stages:
 
                 - ${{ if parameters.notify }}:
                     - bash: |
-                        if [[ -z $(NOTIFY_COMMIT_SHA) ]]; then
-                            export NOTIFY_COMMIT_SHA=`echo "$(Build.SourceVersionMessage)" | cut -d' ' -f2`
-                            echo "##vso[task.setvariable variable=NOTIFY_COMMIT_SHA]$NOTIFY_COMMIT_SHA"
+                        echo "Build.SourceBranch: $(Build.SourceBranch)"
+                        echo "Build.SourceVersion: $(Build.SourceVersion)"
+                        echo "Build.SourceVersionMessage: $(Build.SourceVersionMessage)"
+                        echo "System.PullRequest.SourceCommitId: $(System.PullRequest.SourceCommitId)"
+
+                        if [[ ! -z "$(NOTIFY_COMMIT_SHA)" ]]; then
+                          echo "##[debug]Using already provided NOTIFY_COMMIT_SHA=$(NOTIFY_COMMIT_SHA)"
                         else
-                            echo "##[debug]Using already provided NOTIFY_COMMIT_SHA=$(NOTIFY_COMMIT_SHA)"
+                          NOTIFY_COMMIT_SHA=""
+
+                          if [[ "$(Build.SourceBranch)" =~ ^refs/pull/.+$ ]]; then
+                            if [[ ! -z "$(System.PullRequest.SourceCommitId)" ]]; then
+                              echo "##[debug]Build appears to be a pull request build"
+                              echo "##[debug]Using System.PullRequest.SourceCommitId as NOTIFY_COMMIT_SHA"
+                              NOTIFY_COMMIT_SHA="$(System.PullRequest.SourceCommitId)"
+                            else
+                              echo "##[debug]Build appears to be a pull request build"
+                              echo "##[debug]Extracting NOTIFY_COMMIT_SHA from Build.SourceVersionMessage"
+                              NOTIFY_COMMIT_SHA=$(echo "$(Build.SourceVersionMessage)" | cut -d' ' -f2)
+                            fi
+                          fi
+
+                          if [[ "$(Build.SourceBranch)" =~ ^refs/tags/.+$ ]] && [[ -z "$NOTIFY_COMMIT_SHA" ]]; then
+                            echo "##[debug]Build appears to be a tag build"
+                            echo "##[debug]Using Build.SourceVersion as NOTIFY_COMMIT_SHA"
+                            NOTIFY_COMMIT_SHA="$(Build.SourceVersion)"
+                          fi
+
+                          if [[ -z "$NOTIFY_COMMIT_SHA" ]]; then
+                            echo "##[debug]Using Build.SourceVersion as NOTIFY_COMMIT_SHA"
+                            NOTIFY_COMMIT_SHA="$(Build.SourceVersion)"
+                          fi
+
+                          echo "##vso[task.setvariable variable=NOTIFY_COMMIT_SHA]$NOTIFY_COMMIT_SHA"
                         fi
                       displayName: Set NOTIFY_COMMIT_SHA
 
@@ -180,6 +209,7 @@ stages:
                         state: pending
                         description: "Deploy to ${{ parameters.environment }} started"
                         environment: ${{ parameters.environment }}
+                        stage_name: ${{ parameters.stage_name }}
 
                 - ${{ each param in parameters }}:
                     - ${{ if not(containsValue(parameters._expose_blacklist, param.key)) }}:
@@ -290,10 +320,12 @@ stages:
                         on_success: true
                         description: "Deploy to ${{ parameters.environment }} succeeded"
                         environment: ${{ parameters.environment }}
+                        stage_name: ${{ parameters.stage_name }}
 
                     - template: "../components/update-github-status.yml"
                       parameters:
                         state: failure
                         on_failure: true
                         description: "Deploy to ${{ parameters.environment }} failed"
                         environment: ${{ parameters.environment }}
+                        stage_name: ${{ parameters.stage_name }}

azure/components/update-github-status.yml

@@ -14,6 +14,10 @@ parameters:
     displayName: Apigee environment deploying against
     type: string
     default: ""
+  - name: stage_name
+    displayName: Azure stage name for unique status context
+    type: string
+    default: ""
   # Been through a lot of pain to try to get these next two to work. Unfortunately due to how templates
   # are expanded, flagging like this is the only way I could find to get it to evaluate properly
   - name: on_success
@@ -33,17 +37,41 @@ steps:
          export NOTIFY_GITHUB_REPOSITORY=$(Build.Repository.Name)
       fi
 
+      BUILD_RESULTS_URL="$(System.CollectionUri)$(System.TeamProject)/_build/results?buildId=$(Build.BuildId)&view=results"
+
+
       if [[ "$(NOTIFY_COMMIT_SHA)" =~ ^[0-9a-f]+$ ]]; then
-          echo "Reporting state ${{ parameters.state }} to ${NOTIFY_GITHUB_REPOSITORY}"
-          echo "##[debug] Hitting https://api.github.com/repos/${NOTIFY_GITHUB_REPOSITORY}/statuses/$(NOTIFY_COMMIT_SHA)"
-          curl --fail -q -X POST "https://api.github.com/repos/${NOTIFY_GITHUB_REPOSITORY}/statuses/$(NOTIFY_COMMIT_SHA)" -d '{"state": "${{ parameters.state }}", "context": "$(Build.DefinitionName) $environment", "description": "$description", "target_url": "https://dev.azure.com/NHSD-APIM/API Platform/_build/results?buildId=$(Build.BuildID)"}' --user $(GITHUB_USER):$(GITHUB_ACCESS_TOKEN)
+        if [[ -n "${environment}" && -n "${stage_name}" ]]; then
+          context_name="$(Build.DefinitionName)[${environment}][${stage_name}]"
+        elif [[ -n "${environment}" ]]; then
+          context_name="$(Build.DefinitionName)[${environment}]"
+        else
+          context_name="$(Build.DefinitionName)"
+        fi
+
+        echo "Reporting state ${{ parameters.state }} to ${NOTIFY_GITHUB_REPOSITORY}"
+        echo "##[debug] Hitting https://api.github.com/repos/${NOTIFY_GITHUB_REPOSITORY}/statuses/$(NOTIFY_COMMIT_SHA)"
+
+        payload=$(jq -nc \
+          --arg state "${{ parameters.state }}" \
+          --arg context "$context_name" \
+          --arg description "${description}" \
+          --arg target_url "${BUILD_RESULTS_URL}" \
+          '{state: $state, context: $context, description: $description, target_url: $target_url}')
+
+        curl --fail -q \
+          -H "Accept: application/vnd.github+json" \
+          -X POST "https://api.github.com/repos/${NOTIFY_GITHUB_REPOSITORY}/statuses/$(NOTIFY_COMMIT_SHA)" \
+          -d "${payload}" \
+          --user $(GITHUB_USER):$(GITHUB_ACCESS_TOKEN)
       else
         echo "##[warning]$(NOTIFY_COMMIT_SHA) doesn't look like a commit hash"
         echo "##[task.logissue type=warning]Task wasn't able to find the right commit sha value to notify github."
       fi
     env:
       description: ${{ parameters.description }}
       environment: ${{ parameters.environment }}
+      stage_name: ${{ parameters.stage_name }}
     displayName: "Notify GitHub: ${{ parameters.state }}"
     condition: |
       or(