From 9b38985a5e8138a7e42108621f36aea60f407a6a Mon Sep 17 00:00:00 2001 From: "Patrick J. Cherry" Date: Thu, 1 Sep 2022 15:07:05 +0100 Subject: [PATCH 01/28] Add workflow for production builds --- .github/workflows/build-and-deploy.yml | 66 ++++++++++++++++ .github/workflows/main.yml | 103 ++++++++++--------------- 2 files changed, 108 insertions(+), 61 deletions(-) create mode 100644 .github/workflows/build-and-deploy.yml diff --git a/.github/workflows/build-and-deploy.yml b/.github/workflows/build-and-deploy.yml new file mode 100644 index 000000000..ee67f37b9 --- /dev/null +++ b/.github/workflows/build-and-deploy.yml @@ -0,0 +1,66 @@ +name: Build and Upload to S3 + +on: + workflow_call: + inputs: + aws_region: + required: false + type: string + default: eu-west-2 + bucket: + required: true + type: string + deploy_dir: + required: true + type: string + public_url: + required: true + type: string + secrets: + aws_access_key_id: + required: true + type: string + aws_secret_access_key: + required: true + type: string + +jobs: + build-deploy: + needs: + - test + # Disabling test-cypress while it is flaky + # - test-cypress + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v1 + + - name: Cache dependencies + uses: actions/setup-node@v3 + with: + node-version: 16 + cache: 'yarn' + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.aws_access_key_id }} + aws-secret-access-key: ${{ secrets.aws_secret_access_key }} + aws-region: ${{ inputs.aws_region }} + + - name: Set PUBLIC_URL for react app build + run: | + echo "Setting PUBLIC_URL to ${{ inputs.PUBLIC_URL }}" + echo PUBLIC_URL=${{ inputs.public_url }} >> $GITHUB_ENV + shell: bash + + - name: Install code + run: yarn install --frozen-lock-file + + - name: Build site and WC bundle + run: | + yarn build + yarn build:wc + + - name: Deploy site to S3 bucket + run: aws s3 sync ./build/ s3://${{ inputs.bucket }}/${{ inputs.deploy_dir }} --delete diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index fc35d0947..ea094510d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -82,68 +82,49 @@ jobs: cypress/screenshots cypress/videos - build-deploy: + + build-and-deploy:release: + if: github.ref_type == 'tag' needs: - test - # Disabling test-cypress while it is flaky - # - test-cypress runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v1 - - - name: Cache dependencies - uses: actions/setup-node@v3 - with: - node-version: 16 - cache: 'yarn' - - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: eu-west-2 - - - name: Set bucket - run: | - if [ ${{ github.ref_type }} == 'tag' ]; then - echo "Deploying to production bucket" - echo "bucket=python-editor-dist-test" >> $GITHUB_ENV - else - echo "Deploying to staging/preview bucket" - echo "bucket=python-editor-dist-test" >> $GITHUB_ENV - fi - shell: bash - - - name: Set deploy directory - run: | - if [ ${{ github.ref_type }} == 'tag' ]; then - echo "Deploying tagged release ${{ github.ref_name }}" - echo "deploy_dir=${{ github.ref_name }}" >> $GITHUB_ENV - elif [ ${{ github.ref }} == 'refs/head/main' ]; then - echo "Deploying staging release" - echo "deploy_dir=staging" >> $GITHUB_ENV - else - echo "Deploying preview release ${{ github.ref_name }}" - echo "deploy_dir=previews/${{ github.ref_name }}" >> $GITHUB_ENV - fi - shell: bash - - - name: Set PUBLIC_URL - run: | - public_url=https://${{ env.bucket }}.s3.eu-west-2.amazonaws.com/${{ env.deploy_dir }} - echo "Setting PUBLIC_URL to $public_url" - echo PUBLIC_URL=$public_url >> $GITHUB_ENV - shell: bash - - - name: Install code - run: yarn install --frozen-lock-file - - - name: Build site and WC bundle - run: | - yarn build - yarn build:wc + uses: .github/workflows/build-and-deploy.yml + with: + aws-region: eu-west-2 + bucket: python-editor-dist-test + deploy_dir: ${{ github.ref_name }} + public_url: editor.raspberrypi.org + secrets: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + + build-and-deploy:staging: + if: github.ref == "refs/heads/main" + needs: + - test + runs-on: ubuntu-latest + uses: .github/workflows/build-and-deploy.yml + with: + aws-region: eu-west-2 + bucket: python-editor-dist-test + deploy_dir: staging + public_url: staging-editor.raspberrypi.org + secrets: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + + + build-and-deploy:preview: + if: github.ref_type == "branch" && github.ref != "refs/heads/main" + needs: + - test + runs-on: ubuntu-latest + uses: .github/workflows/build-and-deploy.yml + with: + aws-region: eu-west-2 + bucket: python-editor-dist-test + deploy_dir: previews/${{ github.ref_name }} + secrets: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - - name: Deploy site to S3 bucket - run: aws s3 sync ./build/ s3://${{ env.bucket }}/${{ env.deploy_dir }} --delete From 80563e26a9feadba797e526fb4836911c972f20b Mon Sep 17 00:00:00 2001 From: "Patrick J. Cherry" Date: Thu, 1 Sep 2022 15:11:15 +0100 Subject: [PATCH 02/28] Fix relative directory path for uses: --- .github/workflows/main.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ea094510d..ace9db315 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -88,7 +88,7 @@ jobs: needs: - test runs-on: ubuntu-latest - uses: .github/workflows/build-and-deploy.yml + uses: ./.github/workflows/build-and-deploy.yml with: aws-region: eu-west-2 bucket: python-editor-dist-test @@ -103,7 +103,7 @@ jobs: needs: - test runs-on: ubuntu-latest - uses: .github/workflows/build-and-deploy.yml + uses: ./.github/workflows/build-and-deploy.yml with: aws-region: eu-west-2 bucket: python-editor-dist-test @@ -119,7 +119,7 @@ jobs: needs: - test runs-on: ubuntu-latest - uses: .github/workflows/build-and-deploy.yml + uses: ./.github/workflows/build-and-deploy.yml with: aws-region: eu-west-2 bucket: python-editor-dist-test From 8b31e37d8d1ce8519af5d6a2ca3717fe9c26c578 Mon Sep 17 00:00:00 2001 From: "Patrick J. Cherry" Date: Thu, 1 Sep 2022 15:12:56 +0100 Subject: [PATCH 03/28] Remove runs-on --- .github/workflows/main.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ace9db315..6491459f9 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -87,7 +87,6 @@ jobs: if: github.ref_type == 'tag' needs: - test - runs-on: ubuntu-latest uses: ./.github/workflows/build-and-deploy.yml with: aws-region: eu-west-2 @@ -102,7 +101,6 @@ jobs: if: github.ref == "refs/heads/main" needs: - test - runs-on: ubuntu-latest uses: ./.github/workflows/build-and-deploy.yml with: aws-region: eu-west-2 @@ -118,7 +116,6 @@ jobs: if: github.ref_type == "branch" && github.ref != "refs/heads/main" needs: - test - runs-on: ubuntu-latest uses: ./.github/workflows/build-and-deploy.yml with: aws-region: eu-west-2 From 9bd42a0c8aaa91f59a8e4cfc2ef3f6744aa7d1fa Mon Sep 17 00:00:00 2001 From: "Patrick J. Cherry" Date: Thu, 1 Sep 2022 15:14:14 +0100 Subject: [PATCH 04/28] Fix job names --- .github/workflows/main.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6491459f9..1fbbee64b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -83,7 +83,7 @@ jobs: cypress/videos - build-and-deploy:release: + build-and-deploy-release: if: github.ref_type == 'tag' needs: - test @@ -97,7 +97,7 @@ jobs: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - build-and-deploy:staging: + build-and-deploy-staging: if: github.ref == "refs/heads/main" needs: - test @@ -112,7 +112,7 @@ jobs: aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - build-and-deploy:preview: + build-and-deploy-preview: if: github.ref_type == "branch" && github.ref != "refs/heads/main" needs: - test From 4920846693d1a09df9db8c031cf360a2890a06ad Mon Sep 17 00:00:00 2001 From: "Patrick J. Cherry" Date: Thu, 1 Sep 2022 15:16:13 +0100 Subject: [PATCH 05/28] Use better inverted commas --- .github/workflows/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 1fbbee64b..4319c091a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -98,7 +98,7 @@ jobs: aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} build-and-deploy-staging: - if: github.ref == "refs/heads/main" + if: github.ref == 'refs/heads/main' needs: - test uses: ./.github/workflows/build-and-deploy.yml @@ -113,7 +113,7 @@ jobs: build-and-deploy-preview: - if: github.ref_type == "branch" && github.ref != "refs/heads/main" + if: github.ref_type == 'branch' && github.ref != 'refs/heads/main' needs: - test uses: ./.github/workflows/build-and-deploy.yml From 14a19436b4d24d8203b51791a8a969dca2340a07 Mon Sep 17 00:00:00 2001 From: "Patrick J. Cherry" Date: Thu, 1 Sep 2022 15:17:21 +0100 Subject: [PATCH 06/28] Remove type from secrets --- .github/workflows/build-and-deploy.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/build-and-deploy.yml b/.github/workflows/build-and-deploy.yml index ee67f37b9..ec889708d 100644 --- a/.github/workflows/build-and-deploy.yml +++ b/.github/workflows/build-and-deploy.yml @@ -19,10 +19,8 @@ on: secrets: aws_access_key_id: required: true - type: string aws_secret_access_key: required: true - type: string jobs: build-deploy: From 684ab73e1f5b0c4c1c1ee0d430796d37e6713ee7 Mon Sep 17 00:00:00 2001 From: "Patrick J. Cherry" Date: Thu, 1 Sep 2022 15:37:25 +0100 Subject: [PATCH 07/28] Add in environments --- .github/workflows/build-and-deploy.yml | 12 +++++------ .github/workflows/main.yml | 30 +++++++++++++------------- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/.github/workflows/build-and-deploy.yml b/.github/workflows/build-and-deploy.yml index ec889708d..51484694d 100644 --- a/.github/workflows/build-and-deploy.yml +++ b/.github/workflows/build-and-deploy.yml @@ -3,7 +3,7 @@ name: Build and Upload to S3 on: workflow_call: inputs: - aws_region: + AWS_REGION: required: false type: string default: eu-west-2 @@ -17,9 +17,9 @@ on: required: true type: string secrets: - aws_access_key_id: + AWS_ACCESS_KEY_ID: required: true - aws_secret_access_key: + AWS_SECRET_ACCESS_KEY: required: true jobs: @@ -42,9 +42,9 @@ jobs: - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v1 with: - aws-access-key-id: ${{ secrets.aws_access_key_id }} - aws-secret-access-key: ${{ secrets.aws_secret_access_key }} - aws-region: ${{ inputs.aws_region }} + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ inputs.AWS_REGION }} - name: Set PUBLIC_URL for react app build run: | diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 4319c091a..de9d0e94d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -89,13 +89,13 @@ jobs: - test uses: ./.github/workflows/build-and-deploy.yml with: - aws-region: eu-west-2 bucket: python-editor-dist-test deploy_dir: ${{ github.ref_name }} - public_url: editor.raspberrypi.org - secrets: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + public_url: https://editor.raspberrypi.org/${{ github.ref_name }} + secrets: inherit + environment: + name: production + url: https://editor.raspberrypi.org/${{ github.ref_name }} build-and-deploy-staging: if: github.ref == 'refs/heads/main' @@ -103,14 +103,13 @@ jobs: - test uses: ./.github/workflows/build-and-deploy.yml with: - aws-region: eu-west-2 bucket: python-editor-dist-test deploy_dir: staging - public_url: staging-editor.raspberrypi.org - secrets: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - + public_url: https://staging-editor.raspberrypi.org + secrets: inherit + environment: + name: staging + url: https://staging-editor.raspberrypi.org build-and-deploy-preview: if: github.ref_type == 'branch' && github.ref != 'refs/heads/main' @@ -118,10 +117,11 @@ jobs: - test uses: ./.github/workflows/build-and-deploy.yml with: - aws-region: eu-west-2 bucket: python-editor-dist-test deploy_dir: previews/${{ github.ref_name }} - secrets: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + public_url: https://python-editor-dist-test.s3.eu-west-2.amazonaws.com/previews/${{ github.ref_name }} + secrets: inherit + environment: + name: previews/${{ github.ref_name }} + public_url: https://python-editor-dist-test.s3.eu-west-2.amazonaws.com/previews/${{ github.ref_name }} From c57f956ddc5591bd100679cf21d04e61f7f06e8c Mon Sep 17 00:00:00 2001 From: "Patrick J. Cherry" Date: Thu, 1 Sep 2022 16:27:11 +0100 Subject: [PATCH 08/28] Move environment setting into shared job --- .github/workflows/build-and-deploy.yml | 10 ++++++---- .github/workflows/main.yml | 12 +++--------- 2 files changed, 9 insertions(+), 13 deletions(-) diff --git a/.github/workflows/build-and-deploy.yml b/.github/workflows/build-and-deploy.yml index 51484694d..2fb66dcb9 100644 --- a/.github/workflows/build-and-deploy.yml +++ b/.github/workflows/build-and-deploy.yml @@ -16,6 +16,9 @@ on: public_url: required: true type: string + environment: + required: true + type: string secrets: AWS_ACCESS_KEY_ID: required: true @@ -24,11 +27,10 @@ on: jobs: build-deploy: - needs: - - test - # Disabling test-cypress while it is flaky - # - test-cypress runs-on: ubuntu-latest + environment: + name: ${{ inputs.environment }} + url: ${{ inputs.public_url }} steps: - name: Checkout uses: actions/checkout@v1 diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index de9d0e94d..29c88c685 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -91,11 +91,9 @@ jobs: with: bucket: python-editor-dist-test deploy_dir: ${{ github.ref_name }} + environment: production public_url: https://editor.raspberrypi.org/${{ github.ref_name }} secrets: inherit - environment: - name: production - url: https://editor.raspberrypi.org/${{ github.ref_name }} build-and-deploy-staging: if: github.ref == 'refs/heads/main' @@ -105,11 +103,9 @@ jobs: with: bucket: python-editor-dist-test deploy_dir: staging + environment: staging public_url: https://staging-editor.raspberrypi.org secrets: inherit - environment: - name: staging - url: https://staging-editor.raspberrypi.org build-and-deploy-preview: if: github.ref_type == 'branch' && github.ref != 'refs/heads/main' @@ -119,9 +115,7 @@ jobs: with: bucket: python-editor-dist-test deploy_dir: previews/${{ github.ref_name }} + environment: previews/${{ github.ref_name }} public_url: https://python-editor-dist-test.s3.eu-west-2.amazonaws.com/previews/${{ github.ref_name }} secrets: inherit - environment: - name: previews/${{ github.ref_name }} - public_url: https://python-editor-dist-test.s3.eu-west-2.amazonaws.com/previews/${{ github.ref_name }} From 4f2e871725d3581730b62a7b9076cbf19317849f Mon Sep 17 00:00:00 2001 From: "Patrick J. Cherry" Date: Thu, 1 Sep 2022 16:57:59 +0100 Subject: [PATCH 09/28] Add in S3 url --- .github/workflows/main.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 29c88c685..922c1c856 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -104,6 +104,7 @@ jobs: bucket: python-editor-dist-test deploy_dir: staging environment: staging + # TODO: Trigger CF cache-busting job public_url: https://staging-editor.raspberrypi.org secrets: inherit @@ -116,6 +117,7 @@ jobs: bucket: python-editor-dist-test deploy_dir: previews/${{ github.ref_name }} environment: previews/${{ github.ref_name }} - public_url: https://python-editor-dist-test.s3.eu-west-2.amazonaws.com/previews/${{ github.ref_name }} + # TODO: Place behind CF too, probably. + public_url: http://python-editor-dist-test.s3-website.eu-west-2.amazonaws.com/previews/${{ github.ref_name }} secrets: inherit From 38e75239d01677b70f217e2a3adc4f8da4a8b967 Mon Sep 17 00:00:00 2001 From: "Patrick J. Cherry" Date: Thu, 1 Sep 2022 17:03:11 +0100 Subject: [PATCH 10/28] Don't bother waiting for the tests when building branches --- .github/workflows/main.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 922c1c856..753be2796 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -110,8 +110,6 @@ jobs: build-and-deploy-preview: if: github.ref_type == 'branch' && github.ref != 'refs/heads/main' - needs: - - test uses: ./.github/workflows/build-and-deploy.yml with: bucket: python-editor-dist-test From 4326b5965225435dcada796a957a9c0bd7f23fd0 Mon Sep 17 00:00:00 2001 From: "Patrick J. Cherry" Date: Thu, 1 Sep 2022 17:27:21 +0100 Subject: [PATCH 11/28] Add record coverage step for GH action --- .github/workflows/main.yml | 8 +++ .github/workflows/record_coverage | 90 +++++++++++++++++++++++++++++++ 2 files changed, 98 insertions(+) create mode 100755 .github/workflows/record_coverage diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 753be2796..3aa74c698 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -47,6 +47,14 @@ jobs: - name: Run tests run: yarn test + - name: Record coverage + run: ./.github/workflows/record_coverage + env: + GITHUB_TOKEN: ${{ github.token }} + REPOSITORY: ${{ github.repository }} + RUN_ID: ${{ github.run_id }} + REF_NAME: ${{ github.ref_name }} + test-cypress: runs-on: ubuntu-latest steps: diff --git a/.github/workflows/record_coverage b/.github/workflows/record_coverage new file mode 100755 index 000000000..3adbd85d4 --- /dev/null +++ b/.github/workflows/record_coverage @@ -0,0 +1,90 @@ +#!/bin/bash -ueo pipefail + +# Record coverage +# +# This script uses the Github APIs to poke a comment into a PR about test coverage. +# +# To work, the GITHUB_TOKEN var must be in the environment +# + +CURL_ARGS="-s -S -f" + +function graceful_exit() { + echo "*** Something failed! Exiting gracefully so the build doesn't fail overall" + exit 0 +} + +# +# Wrapper for the Github GraphQL API +# +function gh_query() { + # Build and escape our JSON + json=$(jq -n --arg q "$*" '{query: $q}') + curl $CURL_ARGS -H "Authorization: bearer $GITHUB_TOKEN" -X POST -d "$json" https://api.github.com/graphql +} + + +# Trap any fails, and force a successful exit. +trap graceful_exit ERR + +clover_xml=coverage/clover.xml +if ! [ -s $clover_xml ] ; then + echo "*** No $clover_xml file found." + exit 0 +fi + +sudo apt update -qq +sudo apt install -qq --no-install-recommends -y xmlstarlet +which jq > /dev/null || sudo apt-get install -y jq +which curl > /dev/null || sudo apt-get install -y curl + +# This is the message that makes it into github +msg="* Github [Run ${RUN_ID}](https://github.com/${REPOSITORY}/runs/${RUN_ID}?check_suite_focus=true)" +msg="$msg* Test coverage: " + +statements=$(xmlstarlet sel -t -v '/coverage/project[@name="All files"]/metrics/@statements' $clover_xml) +coveredstatements=$(xmlstarlet sel -t -v '/coverage/project[@name="All files"]/metrics/@coveredstatements' $clover_xml) +# Bash doesn't do floating point. +coverage=$((coveredstatements*100/statements)) + +if [ "${coverage}" = "null" ] ; then + echo "*** Failed to determine coverage" + exit 0 +fi + +# Find associated PR. *NB* we're assuming that the first, open PR is the one +# to comment on. +q="query { + repository(name: \"${REPOSITORY##*/}\", owner: \"${REPOSITORY%%/*}\") { + ref(qualifiedName: \"${REF_NAME}\") { + associatedPullRequests(first: 1) { + nodes { + id + } + } + } + } +}" + +pr_response=$(gh_query $q) +pr_node=$(echo $pr_response | jq -r ".data.repository.ref.associatedPullRequests.nodes[0].id") + +if [ "$pr_node" = "null" ] ; then + echo "*** No PR found" + exit 0 +fi + + +echo ">>> Posting code coverage comment" +m="mutation { + addComment(input: { + subjectId: \"${pr_node}\", + body: \"${msg}\" + }) { + subject { + id + } + } +}" + +gh_query $m From 2ccca8be4d28eafbd25466a30780c36d4dfb7c38 Mon Sep 17 00:00:00 2001 From: "Patrick J. Cherry" Date: Thu, 1 Sep 2022 17:30:59 +0100 Subject: [PATCH 12/28] Remove pipefail option --- .github/workflows/record_coverage | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/record_coverage b/.github/workflows/record_coverage index 3adbd85d4..51432fe80 100755 --- a/.github/workflows/record_coverage +++ b/.github/workflows/record_coverage @@ -1,4 +1,4 @@ -#!/bin/bash -ueo pipefail +#!/bin/bash -eu # Record coverage # From 7af663abc0eff16a83ad6bf0a302c9b43afd5a92 Mon Sep 17 00:00:00 2001 From: "Patrick J. Cherry" Date: Thu, 1 Sep 2022 17:35:36 +0100 Subject: [PATCH 13/28] Add test output --- .github/workflows/main.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 3aa74c698..d9bb79adf 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -45,7 +45,9 @@ jobs: run: yarn install --frozen-lock-file - name: Run tests - run: yarn test + run: yarn run test --coverage --maxWorkers=4 --reporters=default --reporters=jest-junit + env: + JEST_JUNIT_OUTPUT_DIR: ./coverage/ - name: Record coverage run: ./.github/workflows/record_coverage From 147ccad132fdfe61f96f3e894c251118dbe36c5d Mon Sep 17 00:00:00 2001 From: "Patrick J. Cherry" Date: Thu, 1 Sep 2022 17:59:20 +0100 Subject: [PATCH 14/28] Add coverage into messag --- .github/workflows/record_coverage | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/record_coverage b/.github/workflows/record_coverage index 51432fe80..f1979e76e 100755 --- a/.github/workflows/record_coverage +++ b/.github/workflows/record_coverage @@ -39,7 +39,7 @@ which jq > /dev/null || sudo apt-get install -y jq which curl > /dev/null || sudo apt-get install -y curl # This is the message that makes it into github -msg="* Github [Run ${RUN_ID}](https://github.com/${REPOSITORY}/runs/${RUN_ID}?check_suite_focus=true)" +msg="* Github [Run ${RUN_ID}](https://github.com/${REPOSITORY}/actions/runs/${RUN_ID})\n" msg="$msg* Test coverage: " statements=$(xmlstarlet sel -t -v '/coverage/project[@name="All files"]/metrics/@statements' $clover_xml) @@ -52,6 +52,8 @@ if [ "${coverage}" = "null" ] ; then exit 0 fi +msg="$msg $coverage%\n\n" + # Find associated PR. *NB* we're assuming that the first, open PR is the one # to comment on. q="query { From 8eed751b6dd2d4c7e42fbcdf99a6ad01ada4513a Mon Sep 17 00:00:00 2001 From: "Patrick J. Cherry" Date: Thu, 1 Sep 2022 17:59:57 +0100 Subject: [PATCH 15/28] Add coverage archive --- .github/workflows/main.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d9bb79adf..2237aa5e2 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -49,6 +49,12 @@ jobs: env: JEST_JUNIT_OUTPUT_DIR: ./coverage/ + - name: Archive code coverage results + uses: actions/upload-artifact@v3 + with: + name: code-coverage-report + path: ./coverage + - name: Record coverage run: ./.github/workflows/record_coverage env: From b326f2f060cf7240dd615be537874b20a96e92a4 Mon Sep 17 00:00:00 2001 From: "Patrick J. Cherry" Date: Thu, 1 Sep 2022 18:00:42 +0100 Subject: [PATCH 16/28] Fix YAML --- .github/workflows/main.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2237aa5e2..9ae37f79a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -50,10 +50,10 @@ jobs: JEST_JUNIT_OUTPUT_DIR: ./coverage/ - name: Archive code coverage results - uses: actions/upload-artifact@v3 - with: - name: code-coverage-report - path: ./coverage + uses: actions/upload-artifact@v3 + with: + name: code-coverage-report + path: ./coverage - name: Record coverage run: ./.github/workflows/record_coverage From 117474ba2457c4f8cca782eb77d8d230897a69a0 Mon Sep 17 00:00:00 2001 From: "Patrick J. Cherry" Date: Thu, 1 Sep 2022 18:13:05 +0100 Subject: [PATCH 17/28] Add GH actions reporter for jest --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9ae37f79a..f0c05a8a9 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -45,7 +45,7 @@ jobs: run: yarn install --frozen-lock-file - name: Run tests - run: yarn run test --coverage --maxWorkers=4 --reporters=default --reporters=jest-junit + run: yarn run test --coverage --maxWorkers=4 --reporters=default --reporters=jest-junit --reporters=github-actions env: JEST_JUNIT_OUTPUT_DIR: ./coverage/ From 3224afdb15f00d3d3983a2834d6e97e126a7a017 Mon Sep 17 00:00:00 2001 From: "Patrick J. Cherry" Date: Thu, 1 Sep 2022 18:13:47 +0100 Subject: [PATCH 18/28] Remove artifacts; use sensible env vars --- .github/workflows/main.yml | 9 --------- .github/workflows/record_coverage | 6 +++--- 2 files changed, 3 insertions(+), 12 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f0c05a8a9..cb4cc6635 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -49,19 +49,10 @@ jobs: env: JEST_JUNIT_OUTPUT_DIR: ./coverage/ - - name: Archive code coverage results - uses: actions/upload-artifact@v3 - with: - name: code-coverage-report - path: ./coverage - - name: Record coverage run: ./.github/workflows/record_coverage env: GITHUB_TOKEN: ${{ github.token }} - REPOSITORY: ${{ github.repository }} - RUN_ID: ${{ github.run_id }} - REF_NAME: ${{ github.ref_name }} test-cypress: runs-on: ubuntu-latest diff --git a/.github/workflows/record_coverage b/.github/workflows/record_coverage index f1979e76e..344524f28 100755 --- a/.github/workflows/record_coverage +++ b/.github/workflows/record_coverage @@ -39,7 +39,7 @@ which jq > /dev/null || sudo apt-get install -y jq which curl > /dev/null || sudo apt-get install -y curl # This is the message that makes it into github -msg="* Github [Run ${RUN_ID}](https://github.com/${REPOSITORY}/actions/runs/${RUN_ID})\n" +msg="* Github [Run ${GITHUB_RUN_ID}]($GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID)\n" msg="$msg* Test coverage: " statements=$(xmlstarlet sel -t -v '/coverage/project[@name="All files"]/metrics/@statements' $clover_xml) @@ -57,8 +57,8 @@ msg="$msg $coverage%\n\n" # Find associated PR. *NB* we're assuming that the first, open PR is the one # to comment on. q="query { - repository(name: \"${REPOSITORY##*/}\", owner: \"${REPOSITORY%%/*}\") { - ref(qualifiedName: \"${REF_NAME}\") { + repository(name: \"${GITHUB_REPOSITORY##*/}\", owner: \"${GITHUB_REPOSITORY%%/*}\") { + ref(qualifiedName: \"${GITHUB_REF_NAME}\") { associatedPullRequests(first: 1) { nodes { id From 0cc121b451e40df452d9d61ef3f61710cecce4ca Mon Sep 17 00:00:00 2001 From: "Patrick J. Cherry" Date: Thu, 1 Sep 2022 18:18:24 +0100 Subject: [PATCH 19/28] Add GH actions reporter --- package.json | 1 + yarn.lock | 27 +++++++++++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/package.json b/package.json index 4a8f5787b..fecef7fbc 100644 --- a/package.json +++ b/package.json @@ -116,6 +116,7 @@ "html-webpack-plugin": "4.5.0", "jest": "26.6.0", "jest-circus": "26.6.0", + "jest-github-actions": "^0.2.0", "jest-junit": "^13.0.0", "jest-resolve": "26.6.0", "jest-watch-typeahead": "0.6.1", diff --git a/yarn.lock b/yarn.lock index 864a6280c..7c80e5a52 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,6 +2,21 @@ # yarn lockfile v1 +"@actions/core@^1.2.3": + version "1.9.1" + resolved "https://registry.yarnpkg.com/@actions/core/-/core-1.9.1.tgz#97c0201b1f9856df4f7c3a375cdcdb0c2a2f750b" + integrity sha512-5ad+U2YGrmmiw6du20AQW5XuWo7UKN2052FjSV7MX+Wfjf8sCqcsZe62NfgHys4QI4/Y+vQvLKYL8jWtA1ZBTA== + dependencies: + "@actions/http-client" "^2.0.1" + uuid "^8.3.2" + +"@actions/http-client@^2.0.1": + version "2.0.1" + resolved "https://registry.yarnpkg.com/@actions/http-client/-/http-client-2.0.1.tgz#873f4ca98fe32f6839462a6f046332677322f99c" + integrity sha512-PIXiMVtz6VvyaRsGY268qvj57hXQEpsYogYOu2nrQhlf+XCGmZstmuZBbAybUl1nQGnvS1k1eEsQ69ZoD7xlSw== + dependencies: + tunnel "^0.0.6" + "@allmarkedup/fang@^2.0.0": version "2.0.0" resolved "https://registry.yarnpkg.com/@allmarkedup/fang/-/fang-2.0.0.tgz#c61b0ad9e487fca6f1af4639d7813a91e41d8107" @@ -9876,6 +9891,13 @@ jest-get-type@^28.0.2: resolved "https://registry.yarnpkg.com/jest-get-type/-/jest-get-type-28.0.2.tgz#34622e628e4fdcd793d46db8a242227901fcf203" integrity sha512-ioj2w9/DxSYHfOm5lJKCdcAmPJzQXmbM/Url3rhlghrPvT3tt+7a/+oXc9azkKmLvoiXjtV83bEWqi+vs5nlPA== +jest-github-actions@^0.2.0: + version "0.2.0" + resolved "https://registry.yarnpkg.com/jest-github-actions/-/jest-github-actions-0.2.0.tgz#c27cd5effe18b94fac233ec203c9ef719934e93b" + integrity sha512-6QCeXb2L9QeGaQ9QLX2PxSlRWVmwqrQLdV11sDyG3Aqi8nUY2BM9lE46EFIctBqiOR5Vl8uLMRhjZh0nwf+abg== + dependencies: + "@actions/core" "^1.2.3" + jest-haste-map@^26.6.2: version "26.6.2" resolved "https://registry.yarnpkg.com/jest-haste-map/-/jest-haste-map-26.6.2.tgz#dd7e60fe7dc0e9f911a23d79c5ff7fb5c2cafeaa" @@ -16956,6 +16978,11 @@ tunnel-agent@^0.6.0: dependencies: safe-buffer "^5.0.1" +tunnel@^0.0.6: + version "0.0.6" + resolved "https://registry.yarnpkg.com/tunnel/-/tunnel-0.0.6.tgz#72f1314b34a5b192db012324df2cc587ca47f92c" + integrity sha512-1h/Lnq9yajKY2PEbBadPXj3VxsDDu844OnaAo52UVmIzIvwwtBPIuNvkjuzBlTWpfJyUbG3ez0KSBibQkj4ojg== + tweetnacl@^0.14.3, tweetnacl@~0.14.0: version "0.14.5" resolved "https://registry.yarnpkg.com/tweetnacl/-/tweetnacl-0.14.5.tgz#5ae68177f192d4456269d108afa93ff8743f4f64" From 77affba22f06c881aa37c6465c1391b1e3fe2b0d Mon Sep 17 00:00:00 2001 From: "Patrick J. Cherry" Date: Thu, 1 Sep 2022 18:28:09 +0100 Subject: [PATCH 20/28] Use proper GH acctions reporter --- .github/workflows/main.yml | 2 +- package.json | 2 +- yarn.lock | 12 ++++++------ 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index cb4cc6635..1699433a0 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -45,7 +45,7 @@ jobs: run: yarn install --frozen-lock-file - name: Run tests - run: yarn run test --coverage --maxWorkers=4 --reporters=default --reporters=jest-junit --reporters=github-actions + run: yarn run test --coverage --maxWorkers=4 --reporters=default --reporters=jest-junit --reporters=jest-github-actions-reporter env: JEST_JUNIT_OUTPUT_DIR: ./coverage/ diff --git a/package.json b/package.json index fecef7fbc..282e71f23 100644 --- a/package.json +++ b/package.json @@ -116,7 +116,7 @@ "html-webpack-plugin": "4.5.0", "jest": "26.6.0", "jest-circus": "26.6.0", - "jest-github-actions": "^0.2.0", + "jest-github-actions-reporter": "^1.0.3", "jest-junit": "^13.0.0", "jest-resolve": "26.6.0", "jest-watch-typeahead": "0.6.1", diff --git a/yarn.lock b/yarn.lock index 7c80e5a52..4cfc123ae 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,7 +2,7 @@ # yarn lockfile v1 -"@actions/core@^1.2.3": +"@actions/core@^1.2.0": version "1.9.1" resolved "https://registry.yarnpkg.com/@actions/core/-/core-1.9.1.tgz#97c0201b1f9856df4f7c3a375cdcdb0c2a2f750b" integrity sha512-5ad+U2YGrmmiw6du20AQW5XuWo7UKN2052FjSV7MX+Wfjf8sCqcsZe62NfgHys4QI4/Y+vQvLKYL8jWtA1ZBTA== @@ -9891,12 +9891,12 @@ jest-get-type@^28.0.2: resolved "https://registry.yarnpkg.com/jest-get-type/-/jest-get-type-28.0.2.tgz#34622e628e4fdcd793d46db8a242227901fcf203" integrity sha512-ioj2w9/DxSYHfOm5lJKCdcAmPJzQXmbM/Url3rhlghrPvT3tt+7a/+oXc9azkKmLvoiXjtV83bEWqi+vs5nlPA== -jest-github-actions@^0.2.0: - version "0.2.0" - resolved "https://registry.yarnpkg.com/jest-github-actions/-/jest-github-actions-0.2.0.tgz#c27cd5effe18b94fac233ec203c9ef719934e93b" - integrity sha512-6QCeXb2L9QeGaQ9QLX2PxSlRWVmwqrQLdV11sDyG3Aqi8nUY2BM9lE46EFIctBqiOR5Vl8uLMRhjZh0nwf+abg== +jest-github-actions-reporter@^1.0.3: + version "1.0.3" + resolved "https://registry.yarnpkg.com/jest-github-actions-reporter/-/jest-github-actions-reporter-1.0.3.tgz#6aa2b3a6599352e1043bbe42628a7f73a1ce48c2" + integrity sha512-IwLAKLSWLN8ZVfcfEEv6rfeWb78wKDeOhvOmH9KKXayKsKLSCwceopBcB+KUtwxfB5wYnT8Y9s2eZ+WdhA5yng== dependencies: - "@actions/core" "^1.2.3" + "@actions/core" "^1.2.0" jest-haste-map@^26.6.2: version "26.6.2" From 469bcbc681e5df27cd782d327492e1931c8226c3 Mon Sep 17 00:00:00 2001 From: "Patrick J. Cherry" Date: Thu, 1 Sep 2022 19:03:16 +0100 Subject: [PATCH 21/28] Remove CircleCI --- .circleci/config.yml | 104 ------------------------------------- .circleci/record_coverage | 106 -------------------------------------- 2 files changed, 210 deletions(-) delete mode 100644 .circleci/config.yml delete mode 100644 .circleci/record_coverage diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index 3d7f4ae39..000000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,104 +0,0 @@ -version: '2.1' -jobs: - lint: - docker: - - image: cimg/node:16.13.1 - environment: - NODE_PATH: "./src" - REACT_APP_API_URL: https://learning-admin-staging.herokuapp.com - steps: - - checkout - - node/install-packages: - pkg-manager: yarn - - run: - name: eslint - command: yarn run lint - - stylelint: - docker: - - image: cimg/node:16.13.1 - environment: - NODE_PATH: ./src - REACT_APP_API_URL: https://learning-admin-staging.herokuapp.com - steps: - - checkout - - node/install-packages: - pkg-manager: yarn - - run: - name: stylelint - command: yarn run stylelint - - test: - docker: - - image: cimg/node:16.13.1-browsers - environment: - NODE_PATH: ./src - steps: - - checkout - - restore_cache: - keys: - - v1-deps-{{ .Branch }}-{{ checksum "package.json" }} - - v1-deps-{{ .Branch }} - - v1-deps - - run: - name: Install Dependencies - command: yarn install - - save_cache: - key: v1-deps-{{ .Branch }}-{{ checksum "package.json" }} - # cache NPM modules and the folder with the Cypress binary - paths: - - ~/project/node_modules - - ~/.cache - - run: - command: yarn run test --coverage --maxWorkers=4 --reporters=default --reporters=jest-junit - environment: - JEST_JUNIT_OUTPUT_DIR: ./coverage/ - - run: - name: Start server - command: yarn run start - background: true - environment: - PORT: 3000 - - run: - name: Start web component server - command: yarn run start:wc - environment: - PUBLIC_URL: http://localhost:3000 - background: true - - run: - name: Wait for services to be ready - command: dockerize -wait http://localhost:3000 -wait http://localhost:3001 -timeout 120s - - run: - name: Cypress integration tests - command: yarn exec cypress run - - store_test_results: - path: ./coverage - - store_artifacts: - path: ./coverage - - store_artifacts: - path: ./cypress/videos - - run: - name: Post test coverage to Github - command: bash -ue .circleci/record_coverage - when: always - -orbs: - node: circleci/node@5 - -workflows: - test: - jobs: - - test: - context: raspberrypigithubbot -# code_quality: -# jobs: -# - stylelint: -# filters: -# branches: -# ignore: -# - main -# - lint: -# filters: -# branches: -# ignore: -# - main diff --git a/.circleci/record_coverage b/.circleci/record_coverage deleted file mode 100644 index b0a2207bd..000000000 --- a/.circleci/record_coverage +++ /dev/null @@ -1,106 +0,0 @@ -#!/bin/bash -ueo pipefail - -# Record coverage -# -# This script uses the Circle and Github APIs to poke a comment into a PR about test coverage. -# -# To work, the GITHUB_TOKEN and CIRCLE_TOKEN vars must be in the environment, -# with appropriate API tokens from GH and Circle. -# -# Also to get the magic link to your test coverage, you'll want to store the -# `coverage/` directory. -#``` -# - store_artifacts: -# path: coverage -#``` - -CURL_ARGS="-s -S -f" - -function graceful_exit() { - echo "*** Something failed! Exiting gracefully so the build doesn't fail overall" - exit 0 -} - -# -# Wrapper for the Github GraphQL API -# -function gh_query() { - # Build and escape our JSON - json=$(jq -n --arg q "$*" '{query: $q}') - curl $CURL_ARGS -H "Authorization: bearer $GITHUB_TOKEN" -X POST -d "$json" https://api.github.com/graphql -} - - -# Trap any fails, and force a successful exit. -trap graceful_exit ERR - -clover_xml=coverage/clover.xml -if ! [ -s $clover_xml ] ; then - echo "*** No $clover_xml file found." - exit 0 -fi - -sudo apt update -qq -sudo apt install -qq --no-install-recommends -y xmlstarlet -which jq > /dev/null || sudo apt-get install -y jq - -# This is the message that makes it into github -msg="* CircleCI build [#${CIRCLE_BUILD_NUM}](${CIRCLE_BUILD_URL})\n" -msg="$msg* Test coverage: " - -statements=$(xmlstarlet sel -t -v '/coverage/project[@name="All files"]/metrics/@statements' $clover_xml) -coveredstatements=$(xmlstarlet sel -t -v '/coverage/project[@name="All files"]/metrics/@coveredstatements' $clover_xml) -# Bash doesn't do floating point. -coverage=$((coveredstatements*100/statements)) - -if [ "${coverage}" = "null" ] ; then - echo "*** Failed to determine coverage" - exit 0 -fi - -artifacts_response=$(curl $CURL_ARGS -H "Circle-Token: $CIRCLE_TOKEN" https://circleci.com/api/v1.1/project/gh/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}/${CIRCLE_BUILD_NUM}/artifacts) -coverage_url=$(echo ${artifacts_response} | jq -r '. | map(select(.path == "coverage/lcov-report/index.html"))[0].url') - -if ! [ "${coverage_url}" = "null" ] ; then - msg="$msg [$coverage%]($coverage_url)\n\n" -else - msg="$msg $coverage%\n\n" - msg="$msg > CircleCI didn't store the coverage index (maybe the store_artifacts step is missing?)" -fi - -# Find associated PR. *NB* we're assuming that the first, open PR is the one -# to comment on. -q="query { - repository(name: \"${CIRCLE_PROJECT_REPONAME}\", owner: \"${CIRCLE_PROJECT_USERNAME}\") { - ref(qualifiedName: \"${CIRCLE_BRANCH}\") { - associatedPullRequests(first: 1) { - nodes { - id - } - } - } - } -}" - -pr_response=$(gh_query $q) -pr_node=$(echo $pr_response | jq -r ".data.repository.ref.associatedPullRequests.nodes[0].id") - -if [ "$pr_node" = "null" ] ; then - echo "*** No PR found" - exit 0 -fi - - -echo ">>> Posting code coverage comment" -m="mutation { - addComment(input: { - subjectId: \"${pr_node}\", - body: \"${msg}\" - }) { - subject { - id - } - } -}" - -gh_query $m From 504c79cef50e1d42ef15b79e12e7c32a0cfc5a6c Mon Sep 17 00:00:00 2001 From: "Patrick J. Cherry" Date: Fri, 2 Sep 2022 12:32:38 +0100 Subject: [PATCH 22/28] Use more standard env vars --- .github/workflows/build-and-deploy.yml | 34 ++++++++++++-------------- .github/workflows/main.yml | 10 +------- 2 files changed, 16 insertions(+), 28 deletions(-) diff --git a/.github/workflows/build-and-deploy.yml b/.github/workflows/build-and-deploy.yml index 2fb66dcb9..3a30670dc 100644 --- a/.github/workflows/build-and-deploy.yml +++ b/.github/workflows/build-and-deploy.yml @@ -3,34 +3,29 @@ name: Build and Upload to S3 on: workflow_call: inputs: - AWS_REGION: - required: false - type: string - default: eu-west-2 - bucket: - required: true - type: string deploy_dir: required: true type: string - public_url: - required: true - type: string environment: required: true type: string secrets: + AWS_REGION: + required: false + default: eu-west-2 AWS_ACCESS_KEY_ID: required: true AWS_SECRET_ACCESS_KEY: required: true + PUBLIC_URL: + required: true jobs: build-deploy: runs-on: ubuntu-latest environment: name: ${{ inputs.environment }} - url: ${{ inputs.public_url }} + url: ${{ secrets.PUBLIC_URL }}/${{ inputs.deploy_dir }} steps: - name: Checkout uses: actions/checkout@v1 @@ -46,13 +41,7 @@ jobs: with: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ inputs.AWS_REGION }} - - - name: Set PUBLIC_URL for react app build - run: | - echo "Setting PUBLIC_URL to ${{ inputs.PUBLIC_URL }}" - echo PUBLIC_URL=${{ inputs.public_url }} >> $GITHUB_ENV - shell: bash + aws-region: ${{ secrets.AWS_REGION }} - name: Install code run: yarn install --frozen-lock-file @@ -61,6 +50,13 @@ jobs: run: | yarn build yarn build:wc + env: + COOKIEBOT_DOMAIN_GROUP_ID: ${{ secrets.COOKIEBOT_DOMAIN_GROUP_ID }} + PUBLIC_URL: ${{ secrets.PUBLIC_URL }}/${{ inputs.deploy_dir }} + REACT_APP_API_ENDPOINT: ${{ secrets.REACT_APP_API_ENDPOINT }} + REACT_APP_AUTHENTICATION_CLIENT_ID: ${{ secrets.REACT_APP_AUTHENTICATION_CLIENT_ID }} + REACT_APP_AUTHENTICATION_URL: ${{ secrets.REACT_APP_AUTHENTICATION_URL }} + REACT_APP_LOGIN_ENABLED: ${{ secrets.REACT_APP_LOGIN_ENABLED }} - name: Deploy site to S3 bucket - run: aws s3 sync ./build/ s3://${{ inputs.bucket }}/${{ inputs.deploy_dir }} --delete + run: aws s3 sync ./build/ s3://${{ secrets.AWS_S3_BUCKET }}/${{ inputs.deploy_dir }} --delete diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 1699433a0..6560b9ee7 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -96,10 +96,8 @@ jobs: - test uses: ./.github/workflows/build-and-deploy.yml with: - bucket: python-editor-dist-test deploy_dir: ${{ github.ref_name }} environment: production - public_url: https://editor.raspberrypi.org/${{ github.ref_name }} secrets: inherit build-and-deploy-staging: @@ -108,21 +106,15 @@ jobs: - test uses: ./.github/workflows/build-and-deploy.yml with: - bucket: python-editor-dist-test - deploy_dir: staging + deploy_dir: ${{ github.ref_name }} environment: staging - # TODO: Trigger CF cache-busting job - public_url: https://staging-editor.raspberrypi.org secrets: inherit build-and-deploy-preview: if: github.ref_type == 'branch' && github.ref != 'refs/heads/main' uses: ./.github/workflows/build-and-deploy.yml with: - bucket: python-editor-dist-test deploy_dir: previews/${{ github.ref_name }} environment: previews/${{ github.ref_name }} - # TODO: Place behind CF too, probably. - public_url: http://python-editor-dist-test.s3-website.eu-west-2.amazonaws.com/previews/${{ github.ref_name }} secrets: inherit From 5fe22d54ddc6e6a28f9e61977522ccb04699d0d9 Mon Sep 17 00:00:00 2001 From: "Patrick J. Cherry" Date: Fri, 2 Sep 2022 14:00:54 +0100 Subject: [PATCH 23/28] `secrets` context can't be used in environment.url --- .github/workflows/build-and-deploy.yml | 15 ++++----------- .github/workflows/main.yml | 3 +++ 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/.github/workflows/build-and-deploy.yml b/.github/workflows/build-and-deploy.yml index 3a30670dc..3818bc834 100644 --- a/.github/workflows/build-and-deploy.yml +++ b/.github/workflows/build-and-deploy.yml @@ -9,23 +9,16 @@ on: environment: required: true type: string - secrets: - AWS_REGION: - required: false - default: eu-west-2 - AWS_ACCESS_KEY_ID: - required: true - AWS_SECRET_ACCESS_KEY: - required: true - PUBLIC_URL: + public_url: required: true + type: string jobs: build-deploy: runs-on: ubuntu-latest environment: name: ${{ inputs.environment }} - url: ${{ secrets.PUBLIC_URL }}/${{ inputs.deploy_dir }} + url: ${{ inputs.public_url }} steps: - name: Checkout uses: actions/checkout@v1 @@ -52,7 +45,7 @@ jobs: yarn build:wc env: COOKIEBOT_DOMAIN_GROUP_ID: ${{ secrets.COOKIEBOT_DOMAIN_GROUP_ID }} - PUBLIC_URL: ${{ secrets.PUBLIC_URL }}/${{ inputs.deploy_dir }} + PUBLIC_URL: ${{ inputs.public_url }} REACT_APP_API_ENDPOINT: ${{ secrets.REACT_APP_API_ENDPOINT }} REACT_APP_AUTHENTICATION_CLIENT_ID: ${{ secrets.REACT_APP_AUTHENTICATION_CLIENT_ID }} REACT_APP_AUTHENTICATION_URL: ${{ secrets.REACT_APP_AUTHENTICATION_URL }} diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6560b9ee7..6c88c209f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -98,6 +98,7 @@ jobs: with: deploy_dir: ${{ github.ref_name }} environment: production + public_url: ${{ secrets.PUBLIC_URL }}/${{ github.ref_name }} secrets: inherit build-and-deploy-staging: @@ -108,6 +109,7 @@ jobs: with: deploy_dir: ${{ github.ref_name }} environment: staging + public_url: ${{ secrets.PUBLIC_URL }}/${{ github.ref_name }} secrets: inherit build-and-deploy-preview: @@ -116,5 +118,6 @@ jobs: with: deploy_dir: previews/${{ github.ref_name }} environment: previews/${{ github.ref_name }} + public_url: ${{ secrets.PUBLIC_URL }}/previews/${{ github.ref_name }} secrets: inherit From 7477ec62baf2af03cdef502a4456b7387ebb16a0 Mon Sep 17 00:00:00 2001 From: "Patrick J. Cherry" Date: Fri, 2 Sep 2022 14:09:01 +0100 Subject: [PATCH 24/28] Can't access secrets in job.foo.with either --- .github/workflows/main.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6c88c209f..a8b78c48b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -98,7 +98,7 @@ jobs: with: deploy_dir: ${{ github.ref_name }} environment: production - public_url: ${{ secrets.PUBLIC_URL }}/${{ github.ref_name }} + public_url: https://editor.raspberrypi.org/${{ github.ref_name }} secrets: inherit build-and-deploy-staging: @@ -109,7 +109,7 @@ jobs: with: deploy_dir: ${{ github.ref_name }} environment: staging - public_url: ${{ secrets.PUBLIC_URL }}/${{ github.ref_name }} + public_url: https://staging-editor.raspberrypi.org/${{ github.ref_name }} secrets: inherit build-and-deploy-preview: @@ -118,6 +118,6 @@ jobs: with: deploy_dir: previews/${{ github.ref_name }} environment: previews/${{ github.ref_name }} - public_url: ${{ secrets.PUBLIC_URL }}/previews/${{ github.ref_name }} + public_url: http://python-editor-dist-test.s3-website.eu-west-2.amazonaws.com/previews/${{ github.ref_name }} secrets: inherit From 5487c14e370b789cced27bf2ab3622bf8a99a523 Mon Sep 17 00:00:00 2001 From: "Patrick J. Cherry" Date: Fri, 2 Sep 2022 15:41:06 +0100 Subject: [PATCH 25/28] Move the non-secret env vars back --- .github/workflows/build-and-deploy.yml | 41 +++++++++++++++++++++----- .github/workflows/main.yml | 5 ++++ 2 files changed, 39 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build-and-deploy.yml b/.github/workflows/build-and-deploy.yml index 3818bc834..efe892e4a 100644 --- a/.github/workflows/build-and-deploy.yml +++ b/.github/workflows/build-and-deploy.yml @@ -3,6 +3,17 @@ name: Build and Upload to S3 on: workflow_call: inputs: + aws_region: + required: false + default: eu-west-2 + type: string + aws_s3_bucket: + required: true + type: string + cookiebot_domain_group_id: + required: false + default: "1e9a6bdd-5870-4d54-8e5f-adcf6b5c5499" + type: string deploy_dir: required: true type: string @@ -12,6 +23,22 @@ on: public_url: required: true type: string + react_app_api_endpoint: + required: false + default: "https://staging-editor-api.raspberrypi.org" + type: string + react_app_authentication_client_id: + required: false + default: editor-api + type: string + react_app_authentication_url: + required: false + default: "https://staging-auth-v1.raspberrypi.org" + type: string + react_app_login_enabled: + required: false + default: "true" + type: string jobs: build-deploy: @@ -34,7 +61,7 @@ jobs: with: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ secrets.AWS_REGION }} + aws-region: ${{ inputs.aws_region }} - name: Install code run: yarn install --frozen-lock-file @@ -44,12 +71,12 @@ jobs: yarn build yarn build:wc env: - COOKIEBOT_DOMAIN_GROUP_ID: ${{ secrets.COOKIEBOT_DOMAIN_GROUP_ID }} + COOKIEBOT_DOMAIN_GROUP_ID: ${{ inputs.cookiebot_domain_group_id }} PUBLIC_URL: ${{ inputs.public_url }} - REACT_APP_API_ENDPOINT: ${{ secrets.REACT_APP_API_ENDPOINT }} - REACT_APP_AUTHENTICATION_CLIENT_ID: ${{ secrets.REACT_APP_AUTHENTICATION_CLIENT_ID }} - REACT_APP_AUTHENTICATION_URL: ${{ secrets.REACT_APP_AUTHENTICATION_URL }} - REACT_APP_LOGIN_ENABLED: ${{ secrets.REACT_APP_LOGIN_ENABLED }} + REACT_APP_API_ENDPOINT: ${{ inputs.react_app_api_endpoint }} + REACT_APP_AUTHENTICATION_CLIENT_ID: ${{ inputs.react_app_authentication_client_id }} + REACT_APP_AUTHENTICATION_URL: ${{ inputs.react_app_authentication_url }} + REACT_APP_LOGIN_ENABLED: ${{ inputs.react_app_login_enabled }} - name: Deploy site to S3 bucket - run: aws s3 sync ./build/ s3://${{ secrets.AWS_S3_BUCKET }}/${{ inputs.deploy_dir }} --delete + run: aws s3 sync ./build/ s3://${{ inputs.aws_s3_bucket }}/${{ inputs.deploy_dir }} --delete diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a8b78c48b..43be70834 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -96,9 +96,12 @@ jobs: - test uses: ./.github/workflows/build-and-deploy.yml with: + aws_s3_bucket: python-editor-dist deploy_dir: ${{ github.ref_name }} environment: production public_url: https://editor.raspberrypi.org/${{ github.ref_name }} + react_app_api_endpoint: https://editor-api.raspberrypi.org + react_app_authentication_url: https://auth-v1.raspberrypi.org secrets: inherit build-and-deploy-staging: @@ -107,6 +110,7 @@ jobs: - test uses: ./.github/workflows/build-and-deploy.yml with: + aws_s3_bucket: python-editor-dist-staging deploy_dir: ${{ github.ref_name }} environment: staging public_url: https://staging-editor.raspberrypi.org/${{ github.ref_name }} @@ -116,6 +120,7 @@ jobs: if: github.ref_type == 'branch' && github.ref != 'refs/heads/main' uses: ./.github/workflows/build-and-deploy.yml with: + aws_s3_bucket: python-editor-dist-test deploy_dir: previews/${{ github.ref_name }} environment: previews/${{ github.ref_name }} public_url: http://python-editor-dist-test.s3-website.eu-west-2.amazonaws.com/previews/${{ github.ref_name }} From 8e0948678f2d3a95e5ba45e745f293b370882fca Mon Sep 17 00:00:00 2001 From: "Patrick J. Cherry" Date: Mon, 5 Sep 2022 12:37:32 +0100 Subject: [PATCH 26/28] Move all AWS vars back to the secrets --- .github/workflows/build-and-deploy.yml | 20 +++++++++++--------- .github/workflows/main.yml | 3 --- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/.github/workflows/build-and-deploy.yml b/.github/workflows/build-and-deploy.yml index efe892e4a..e7120890b 100644 --- a/.github/workflows/build-and-deploy.yml +++ b/.github/workflows/build-and-deploy.yml @@ -3,13 +3,6 @@ name: Build and Upload to S3 on: workflow_call: inputs: - aws_region: - required: false - default: eu-west-2 - type: string - aws_s3_bucket: - required: true - type: string cookiebot_domain_group_id: required: false default: "1e9a6bdd-5870-4d54-8e5f-adcf6b5c5499" @@ -39,6 +32,15 @@ on: required: false default: "true" type: string + secrets: + AWS_ACCESS_KEY_ID: + required: true + AWS_REGION: + required: true + AWS_S3_BUCKET: + required: true + AWS_SECRET_ACCESS_KEY: + required: true jobs: build-deploy: @@ -61,7 +63,7 @@ jobs: with: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ inputs.aws_region }} + aws-region: ${{ secrets.AWS_REGION }} - name: Install code run: yarn install --frozen-lock-file @@ -79,4 +81,4 @@ jobs: REACT_APP_LOGIN_ENABLED: ${{ inputs.react_app_login_enabled }} - name: Deploy site to S3 bucket - run: aws s3 sync ./build/ s3://${{ inputs.aws_s3_bucket }}/${{ inputs.deploy_dir }} --delete + run: aws s3 sync ./build/ s3://${{ secrets.AWS_S3_BUCKET }}/${{ inputs.deploy_dir }} --delete diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 43be70834..af26910e0 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -96,7 +96,6 @@ jobs: - test uses: ./.github/workflows/build-and-deploy.yml with: - aws_s3_bucket: python-editor-dist deploy_dir: ${{ github.ref_name }} environment: production public_url: https://editor.raspberrypi.org/${{ github.ref_name }} @@ -110,7 +109,6 @@ jobs: - test uses: ./.github/workflows/build-and-deploy.yml with: - aws_s3_bucket: python-editor-dist-staging deploy_dir: ${{ github.ref_name }} environment: staging public_url: https://staging-editor.raspberrypi.org/${{ github.ref_name }} @@ -120,7 +118,6 @@ jobs: if: github.ref_type == 'branch' && github.ref != 'refs/heads/main' uses: ./.github/workflows/build-and-deploy.yml with: - aws_s3_bucket: python-editor-dist-test deploy_dir: previews/${{ github.ref_name }} environment: previews/${{ github.ref_name }} public_url: http://python-editor-dist-test.s3-website.eu-west-2.amazonaws.com/previews/${{ github.ref_name }} From 7e467eb7d719c8d50818d80888c40d44625c9b57 Mon Sep 17 00:00:00 2001 From: "Patrick J. Cherry" Date: Mon, 5 Sep 2022 12:42:16 +0100 Subject: [PATCH 27/28] Update README --- README.md | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index cc733a51d..6cda53562 100644 --- a/README.md +++ b/README.md @@ -65,6 +65,18 @@ It is possible to add query strings to control how the web component is configur For example, to load the page with the Sense Hat always showing, add [`?sense_hat_always_enabled` to the URL](http://localhost:3001?sense_hat_always_enabled) -## Review apps +## Deployment + +Deployment is managed through Giithub actions. The UI is deployed to staging and production environments via an S3 bucket. This requires the following environment variables to be set + +* `AWS_ACCESS_KEY_ID` +* `AWS_REGION` +* `AWS_S3_BUCKET` +* `AWS_SECRET_ACCESS_KEY` + +Other variables that pertain to the app, rather than its deployment are set with defaults in the [build-and-deploy workflow](./.github/workflows/build-and-deploy.yml). These are also in `.env.example`. + +### Review apps Currently the build is deployed to both S3 and Heroku. The PR should get updated with the Heroku URL, and the web component demo is at `/web-component.html` on the Heroku review app domain. + From 1b0e15df4809c39c6ba0a5f6223d54f020836972 Mon Sep 17 00:00:00 2001 From: "Patrick J. Cherry" Date: Mon, 5 Sep 2022 14:37:55 +0100 Subject: [PATCH 28/28] Update CHANGELOG --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8197bc9b8..4c1bdc699 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## [Unreleased] +### Changed +- Update build workflow with a reusable job to update preview, staging, and prod (#176) + ## [0.3.0] ### Added