From 8f30a6490df5be17fe40fd5cce360de25fd6d4b6 Mon Sep 17 00:00:00 2001 From: Hamish Fagg Date: Mon, 18 Aug 2025 09:21:11 +1200 Subject: [PATCH 01/27] initial --- build-push-deploy/workflow.yml | 107 +++++++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 build-push-deploy/workflow.yml diff --git a/build-push-deploy/workflow.yml b/build-push-deploy/workflow.yml new file mode 100644 index 0000000..615c8d4 --- /dev/null +++ b/build-push-deploy/workflow.yml @@ -0,0 +1,107 @@ +on: + workflow_call: + inputs: + service-name: + description: "Name of the service to build. Used as the default image name and src dir unless 'image-name' or 'src-path' are used." + required: true + environment-name: + description: "The backend environment we are building for (API calls are pointed to). This should be one of (development, staging, production)." + required: true + docker-build-args: + description: "Extra args passed to 'docker build'." + required: false + docker-image-ref: + description: "The version number or sha used in creating image tag" + required: false + secrets: + + +jobs: + changes: + name: Filter changed files + runs-on: mdb-dev + outputs: + not-docs: ${{ steps.filter.outputs.not-docs }} + concurrency: + group: ${{ github.workflow_ref }} + cancel-in-progress: true + steps: + - uses: dorny/paths-filter@v3 + id: filter + with: + predicate-quantifier: "every" + filters: | + not-docs: + - '!docs/**' + - '!**/*.md' + + # Looks for labels like "deploy-to-" attached to a PR so we can deploy to those envs + get-deploy-labels: + name: Get Deploy Envs + runs-on: mdb-dev + concurrency: + group: ${{ github.workflow_ref }} + cancel-in-progress: true + environment: + name: ${{ github.event.pull_request.head.repo.fork && 'manual-approval' || '' }} + outputs: + deploy-envs: ${{ steps.get-labels.outputs.deploy-envs }} + steps: + - id: get-labels + uses: ./github-actions/get-deploy-labels + + # Build our docker images based on our bake file + build: + runs-on: mdb-dev + needs: [get-deploy-labels] + if: needs.get-deploy-labels.outputs.deploy-envs != '[]' + concurrency: + group: ${{ github.workflow_ref }} + cancel-in-progress: true + env: + AWS_REGION: us-east-1 + steps: + - uses: ./github-actions/build-push-ecr + with: + module-name: ${{ inputs.service-name }} + build-for-environment: development + + # Push cache layers to docker registry + # This is separate to the build step so we can do other stuff in parallel + # build-cache: + # name: Push Docker Cache + # runs-on: mdb-dev + # needs: [build] + # concurrency: + # group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}-cache + # cancel-in-progress: true + # steps: + # - uses: actions/checkout@v4 + # with: + # ref: ${{ github.event.pull_request.head.sha }} + # - name: Pull MindsDB Github Actions + # uses: actions/checkout@v4 + # with: + # repository: mindsdb/github-actions + # path: github-actions + # # Build the bakefile and push + # - uses: ./github-actions/docker-bake + # with: + # git-sha: ${{ github.event.pull_request.head.sha }} + # target: cloud-cpu + # platforms: linux/amd64 + # push-cache: true + # cache-only: true + + # Call our deployment workflow, so long as this is not a forked PR + # This will run the deployment workflow in the base branch, not in the PR. + # So if you change the deploy workflow in your PR, the changes won't be reflected in this run. + deploy: + name: Deploy + needs: [build, get-deploy-labels] + if: needs.get-deploy-labels.outputs.deploy-envs != '[]' + uses: ./.github/workflows/deploy.yml + with: + deploy-envs: ${{ needs.get-deploy-labels.outputs.deploy-envs }} + image-tag: ${{ github.event.pull_request.head.sha }} + secrets: inherit From 63df8dd71dde43743476f04874f009dff32686ca Mon Sep 17 00:00:00 2001 From: Hamish Fagg Date: Mon, 18 Aug 2025 09:31:09 +1200 Subject: [PATCH 02/27] fix --- .../workflow.yml => .github/workflows/build-push-deploy.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename build-push-deploy/workflow.yml => .github/workflows/build-push-deploy.yml (100%) diff --git a/build-push-deploy/workflow.yml b/.github/workflows/build-push-deploy.yml similarity index 100% rename from build-push-deploy/workflow.yml rename to .github/workflows/build-push-deploy.yml From b0f0f272bd60b1491b9067cee0f41518cdcf856a Mon Sep 17 00:00:00 2001 From: Hamish Fagg Date: Mon, 18 Aug 2025 09:33:09 +1200 Subject: [PATCH 03/27] fix --- .github/workflows/build-push-deploy.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/build-push-deploy.yml b/.github/workflows/build-push-deploy.yml index 615c8d4..d420a95 100644 --- a/.github/workflows/build-push-deploy.yml +++ b/.github/workflows/build-push-deploy.yml @@ -3,15 +3,19 @@ on: inputs: service-name: description: "Name of the service to build. Used as the default image name and src dir unless 'image-name' or 'src-path' are used." + type: string required: true environment-name: description: "The backend environment we are building for (API calls are pointed to). This should be one of (development, staging, production)." + type: string required: true docker-build-args: description: "Extra args passed to 'docker build'." + type: string required: false docker-image-ref: description: "The version number or sha used in creating image tag" + type: string required: false secrets: From d2736ba62992987b9d863ab4a6c5c016d646541f Mon Sep 17 00:00:00 2001 From: Hamish Fagg Date: Mon, 18 Aug 2025 09:33:51 +1200 Subject: [PATCH 04/27] fix --- .github/workflows/build-push-deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-push-deploy.yml b/.github/workflows/build-push-deploy.yml index d420a95..1dbb7ec 100644 --- a/.github/workflows/build-push-deploy.yml +++ b/.github/workflows/build-push-deploy.yml @@ -17,7 +17,7 @@ on: description: "The version number or sha used in creating image tag" type: string required: false - secrets: + #secrets: jobs: From 10bfe9b6a74b5056f39df7db967718fce7a86725 Mon Sep 17 00:00:00 2001 From: Hamish Fagg Date: Mon, 18 Aug 2025 09:35:17 +1200 Subject: [PATCH 05/27] fix --- .github/workflows/build-push-deploy.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build-push-deploy.yml b/.github/workflows/build-push-deploy.yml index 1dbb7ec..d4e4412 100644 --- a/.github/workflows/build-push-deploy.yml +++ b/.github/workflows/build-push-deploy.yml @@ -100,12 +100,12 @@ jobs: # Call our deployment workflow, so long as this is not a forked PR # This will run the deployment workflow in the base branch, not in the PR. # So if you change the deploy workflow in your PR, the changes won't be reflected in this run. - deploy: - name: Deploy - needs: [build, get-deploy-labels] - if: needs.get-deploy-labels.outputs.deploy-envs != '[]' - uses: ./.github/workflows/deploy.yml - with: - deploy-envs: ${{ needs.get-deploy-labels.outputs.deploy-envs }} - image-tag: ${{ github.event.pull_request.head.sha }} - secrets: inherit + # deploy: + # name: Deploy + # needs: [build, get-deploy-labels] + # if: needs.get-deploy-labels.outputs.deploy-envs != '[]' + # uses: ./.github/workflows/deploy.yml + # with: + # deploy-envs: ${{ needs.get-deploy-labels.outputs.deploy-envs }} + # image-tag: ${{ github.event.pull_request.head.sha }} + # secrets: inherit From d89cc67d4314d75a01424a3782ab036139be33f8 Mon Sep 17 00:00:00 2001 From: Hamish Fagg Date: Mon, 18 Aug 2025 09:36:50 +1200 Subject: [PATCH 06/27] fix --- .github/workflows/build-push-deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-push-deploy.yml b/.github/workflows/build-push-deploy.yml index d4e4412..ab501e6 100644 --- a/.github/workflows/build-push-deploy.yml +++ b/.github/workflows/build-push-deploy.yml @@ -52,7 +52,7 @@ jobs: deploy-envs: ${{ steps.get-labels.outputs.deploy-envs }} steps: - id: get-labels - uses: ./github-actions/get-deploy-labels + uses: mindsdb/github-actions/get-deploy-labels # Build our docker images based on our bake file build: From 7ae5c59c97b78d972489ef0360dffd501b90b1d2 Mon Sep 17 00:00:00 2001 From: Hamish Fagg Date: Mon, 18 Aug 2025 09:37:57 +1200 Subject: [PATCH 07/27] fix --- .github/workflows/build-push-deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-push-deploy.yml b/.github/workflows/build-push-deploy.yml index ab501e6..c03f099 100644 --- a/.github/workflows/build-push-deploy.yml +++ b/.github/workflows/build-push-deploy.yml @@ -52,7 +52,7 @@ jobs: deploy-envs: ${{ steps.get-labels.outputs.deploy-envs }} steps: - id: get-labels - uses: mindsdb/github-actions/get-deploy-labels + uses: mindsdb/github-actions/get-deploy-labels@main # Build our docker images based on our bake file build: From 8facb60ff6a9f893844e7e412adc4bef001ac01c Mon Sep 17 00:00:00 2001 From: Hamish Fagg Date: Mon, 18 Aug 2025 09:41:53 +1200 Subject: [PATCH 08/27] fix --- .github/workflows/build-push-deploy.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-push-deploy.yml b/.github/workflows/build-push-deploy.yml index c03f099..6531d27 100644 --- a/.github/workflows/build-push-deploy.yml +++ b/.github/workflows/build-push-deploy.yml @@ -7,7 +7,11 @@ on: required: true environment-name: description: "The backend environment we are building for (API calls are pointed to). This should be one of (development, staging, production)." - type: string + type: choice + options: + - development + - staging + - production required: true docker-build-args: description: "Extra args passed to 'docker build'." @@ -65,10 +69,10 @@ jobs: env: AWS_REGION: us-east-1 steps: - - uses: ./github-actions/build-push-ecr + - uses: mindsdb/github-actions/build-push-ecr@main with: module-name: ${{ inputs.service-name }} - build-for-environment: development + build-for-environment: ${{ inputs.environment-name }} # Push cache layers to docker registry # This is separate to the build step so we can do other stuff in parallel From 458c9f011039f0e93131ea6761ae3a123c0f0b4c Mon Sep 17 00:00:00 2001 From: Hamish Fagg Date: Mon, 18 Aug 2025 09:43:03 +1200 Subject: [PATCH 09/27] fix --- .github/workflows/build-push-deploy.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/build-push-deploy.yml b/.github/workflows/build-push-deploy.yml index 6531d27..cbbfcc7 100644 --- a/.github/workflows/build-push-deploy.yml +++ b/.github/workflows/build-push-deploy.yml @@ -7,11 +7,7 @@ on: required: true environment-name: description: "The backend environment we are building for (API calls are pointed to). This should be one of (development, staging, production)." - type: choice - options: - - development - - staging - - production + type: string required: true docker-build-args: description: "Extra args passed to 'docker build'." From bfd5d4407784c5c9ef3e8fd38dbcc5c0ceb0da67 Mon Sep 17 00:00:00 2001 From: Hamish Fagg Date: Mon, 18 Aug 2025 09:46:01 +1200 Subject: [PATCH 10/27] fix --- .github/workflows/build-push-deploy.yml | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/.github/workflows/build-push-deploy.yml b/.github/workflows/build-push-deploy.yml index cbbfcc7..44809c3 100644 --- a/.github/workflows/build-push-deploy.yml +++ b/.github/workflows/build-push-deploy.yml @@ -21,24 +21,6 @@ on: jobs: - changes: - name: Filter changed files - runs-on: mdb-dev - outputs: - not-docs: ${{ steps.filter.outputs.not-docs }} - concurrency: - group: ${{ github.workflow_ref }} - cancel-in-progress: true - steps: - - uses: dorny/paths-filter@v3 - id: filter - with: - predicate-quantifier: "every" - filters: | - not-docs: - - '!docs/**' - - '!**/*.md' - # Looks for labels like "deploy-to-" attached to a PR so we can deploy to those envs get-deploy-labels: name: Get Deploy Envs From 7e9a827ba1f7c84d3ceef9f7745e3bcd94fe4e3c Mon Sep 17 00:00:00 2001 From: Hamish Fagg Date: Mon, 18 Aug 2025 09:50:09 +1200 Subject: [PATCH 11/27] fix --- build-push-ecr/action.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/build-push-ecr/action.yml b/build-push-ecr/action.yml index eba4585..227f77a 100644 --- a/build-push-ecr/action.yml +++ b/build-push-ecr/action.yml @@ -28,6 +28,10 @@ runs: - name: Login to Amazon ECR id: login-ecr uses: aws-actions/amazon-ecr-login@v2 + - shell: bash + run: | + ls + pwd - shell: bash run: | # Env var parsing From 8e45f8625a075868db73b1eb0f64890ace90e09b Mon Sep 17 00:00:00 2001 From: Hamish Fagg Date: Mon, 18 Aug 2025 09:50:46 +1200 Subject: [PATCH 12/27] fix --- .github/workflows/build-push-deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-push-deploy.yml b/.github/workflows/build-push-deploy.yml index 44809c3..e5b0c5f 100644 --- a/.github/workflows/build-push-deploy.yml +++ b/.github/workflows/build-push-deploy.yml @@ -47,7 +47,7 @@ jobs: env: AWS_REGION: us-east-1 steps: - - uses: mindsdb/github-actions/build-push-ecr@main + - uses: mindsdb/github-actions/build-push-ecr@add/build-push-deploy with: module-name: ${{ inputs.service-name }} build-for-environment: ${{ inputs.environment-name }} From f97bb172d0cfa0e792b4e1b34454cba443177790 Mon Sep 17 00:00:00 2001 From: Hamish Fagg Date: Mon, 18 Aug 2025 09:53:12 +1200 Subject: [PATCH 13/27] fix --- .github/workflows/build-push-deploy.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build-push-deploy.yml b/.github/workflows/build-push-deploy.yml index e5b0c5f..dadadcb 100644 --- a/.github/workflows/build-push-deploy.yml +++ b/.github/workflows/build-push-deploy.yml @@ -47,6 +47,7 @@ jobs: env: AWS_REGION: us-east-1 steps: + - uses: actions/checkout@v4 - uses: mindsdb/github-actions/build-push-ecr@add/build-push-deploy with: module-name: ${{ inputs.service-name }} From b5a22fd0faa7d8afbb24ef746b6b67e4f365fc45 Mon Sep 17 00:00:00 2001 From: Hamish Fagg Date: Mon, 18 Aug 2025 10:13:03 +1200 Subject: [PATCH 14/27] fix --- .github/workflows/build-push-deploy.yml | 63 ++++++++++++++++++++----- build-push-ecr/action.yml | 4 -- 2 files changed, 50 insertions(+), 17 deletions(-) diff --git a/.github/workflows/build-push-deploy.yml b/.github/workflows/build-push-deploy.yml index dadadcb..93ae146 100644 --- a/.github/workflows/build-push-deploy.yml +++ b/.github/workflows/build-push-deploy.yml @@ -5,7 +5,7 @@ on: description: "Name of the service to build. Used as the default image name and src dir unless 'image-name' or 'src-path' are used." type: string required: true - environment-name: + stage-name: description: "The backend environment we are building for (API calls are pointed to). This should be one of (development, staging, production)." type: string required: true @@ -17,7 +17,10 @@ on: description: "The version number or sha used in creating image tag" type: string required: false - #secrets: + secrets: + GH_ACTIONS_SLACK_BOT_TOKEN: + required: true + jobs: @@ -48,10 +51,10 @@ jobs: AWS_REGION: us-east-1 steps: - uses: actions/checkout@v4 - - uses: mindsdb/github-actions/build-push-ecr@add/build-push-deploy + - uses: mindsdb/github-actions/build-push-ecr@main with: module-name: ${{ inputs.service-name }} - build-for-environment: ${{ inputs.environment-name }} + build-for-environment: ${{ inputs.stage-name }} # Push cache layers to docker registry # This is separate to the build step so we can do other stuff in parallel @@ -83,12 +86,46 @@ jobs: # Call our deployment workflow, so long as this is not a forked PR # This will run the deployment workflow in the base branch, not in the PR. # So if you change the deploy workflow in your PR, the changes won't be reflected in this run. - # deploy: - # name: Deploy - # needs: [build, get-deploy-labels] - # if: needs.get-deploy-labels.outputs.deploy-envs != '[]' - # uses: ./.github/workflows/deploy.yml - # with: - # deploy-envs: ${{ needs.get-deploy-labels.outputs.deploy-envs }} - # image-tag: ${{ github.event.pull_request.head.sha }} - # secrets: inherit + deploy: + runs-on: mdb-dev + needs: [ get-deploy-labels, build ] + strategy: + matrix: + deploy-env: ${{fromJson(needs.get-deploy-labels.outputs.deploy-envs)}} + concurrency: + group: deploy-${{ matrix.deploy-env }} + cancel-in-progress: false + environment: + name: ${{ matrix.deploy-env }} + url: ${{ vars.ENV_URL }} + steps: + - uses: actions/checkout@v4 + - uses: mindsdb/github-actions/setup-env@main + - name: Notify of deployment starting + id: slack + uses: ./github-actions/slack-deploy-msg + with: + channel-id: ${{ secrets.SLACK_DEPLOYMENTS_CHANNEL_ID }} + status: "started" + color: "#0099CC" + env-name: ${{ matrix.deploy-env }} + env-url: ${{ vars.ENV_URL }} + slack-token: ${{ secrets.GH_ACTIONS_SLACK_BOT_TOKEN }} + - uses: DevOps-Nirvana/aws-helm-multi-deploy-nodocker@v4 + with: + environment-slug: ${{matrix.deploy-env}} + k8s-namespace: ${{matrix.deploy-env}} + image-tag: ${{ inputs.stage-name }}-${{ github.sha }} + timeout: 600s + wait: "true" # We need to wait till deployment is finished here, since the calling workflow might test the deployment env + - name: Notify of deployment finish + uses: ./github-actions/slack-deploy-msg + if: always() + with: + channel-id: ${{ secrets.SLACK_DEPLOYMENTS_CHANNEL_ID }} + status: "${{ job.status == 'success' && 'finished' || 'failed' }}" + color: "${{ job.status == 'success' && '#00C851' || '#FF4444' }}" + env-name: ${{ matrix.deploy-env }} + env-url: ${{ vars.ENV_URL }} + slack-token: ${{ secrets.GH_ACTIONS_SLACK_BOT_TOKEN }} + update-message-id: ${{ steps.slack.outputs.ts }} diff --git a/build-push-ecr/action.yml b/build-push-ecr/action.yml index 227f77a..eba4585 100644 --- a/build-push-ecr/action.yml +++ b/build-push-ecr/action.yml @@ -28,10 +28,6 @@ runs: - name: Login to Amazon ECR id: login-ecr uses: aws-actions/amazon-ecr-login@v2 - - shell: bash - run: | - ls - pwd - shell: bash run: | # Env var parsing From b855cbad742b1e51a9f0f96019a9d3e1c1483586 Mon Sep 17 00:00:00 2001 From: Hamish Fagg Date: Mon, 18 Aug 2025 10:17:33 +1200 Subject: [PATCH 15/27] fix --- .github/workflows/build-push-deploy.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/build-push-deploy.yml b/.github/workflows/build-push-deploy.yml index 93ae146..aba22be 100644 --- a/.github/workflows/build-push-deploy.yml +++ b/.github/workflows/build-push-deploy.yml @@ -17,9 +17,6 @@ on: description: "The version number or sha used in creating image tag" type: string required: false - secrets: - GH_ACTIONS_SLACK_BOT_TOKEN: - required: true From f20315e436a13d0296bef8eaecaa670f038f29c3 Mon Sep 17 00:00:00 2001 From: Hamish Fagg Date: Mon, 18 Aug 2025 10:18:11 +1200 Subject: [PATCH 16/27] fix --- .github/workflows/build-push-deploy.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/build-push-deploy.yml b/.github/workflows/build-push-deploy.yml index aba22be..93ae146 100644 --- a/.github/workflows/build-push-deploy.yml +++ b/.github/workflows/build-push-deploy.yml @@ -17,6 +17,9 @@ on: description: "The version number or sha used in creating image tag" type: string required: false + secrets: + GH_ACTIONS_SLACK_BOT_TOKEN: + required: true From 318229df6106f7487b3207733af7366672bcf95f Mon Sep 17 00:00:00 2001 From: Hamish Fagg Date: Mon, 18 Aug 2025 10:19:27 +1200 Subject: [PATCH 17/27] fix --- .github/workflows/build-push-deploy.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/build-push-deploy.yml b/.github/workflows/build-push-deploy.yml index 93ae146..aba22be 100644 --- a/.github/workflows/build-push-deploy.yml +++ b/.github/workflows/build-push-deploy.yml @@ -17,9 +17,6 @@ on: description: "The version number or sha used in creating image tag" type: string required: false - secrets: - GH_ACTIONS_SLACK_BOT_TOKEN: - required: true From c58b909a8db393da6da9aec609915818c095d969 Mon Sep 17 00:00:00 2001 From: Hamish Fagg Date: Mon, 18 Aug 2025 10:22:05 +1200 Subject: [PATCH 18/27] fix --- .github/workflows/build-push-deploy.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-push-deploy.yml b/.github/workflows/build-push-deploy.yml index aba22be..e3724d8 100644 --- a/.github/workflows/build-push-deploy.yml +++ b/.github/workflows/build-push-deploy.yml @@ -100,7 +100,7 @@ jobs: - uses: mindsdb/github-actions/setup-env@main - name: Notify of deployment starting id: slack - uses: ./github-actions/slack-deploy-msg + uses: mindsdb/github-actions/slack-deploy-msg with: channel-id: ${{ secrets.SLACK_DEPLOYMENTS_CHANNEL_ID }} status: "started" @@ -116,7 +116,7 @@ jobs: timeout: 600s wait: "true" # We need to wait till deployment is finished here, since the calling workflow might test the deployment env - name: Notify of deployment finish - uses: ./github-actions/slack-deploy-msg + uses: mindsdb/github-actions/slack-deploy-msg if: always() with: channel-id: ${{ secrets.SLACK_DEPLOYMENTS_CHANNEL_ID }} From e0786b10e967bf7f5ed5fc1b59e5f84f2f57a80c Mon Sep 17 00:00:00 2001 From: Hamish Fagg Date: Mon, 18 Aug 2025 10:22:57 +1200 Subject: [PATCH 19/27] fix --- .github/workflows/build-push-deploy.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-push-deploy.yml b/.github/workflows/build-push-deploy.yml index e3724d8..069336d 100644 --- a/.github/workflows/build-push-deploy.yml +++ b/.github/workflows/build-push-deploy.yml @@ -100,7 +100,7 @@ jobs: - uses: mindsdb/github-actions/setup-env@main - name: Notify of deployment starting id: slack - uses: mindsdb/github-actions/slack-deploy-msg + uses: mindsdb/github-actions/slack-deploy-msg@main with: channel-id: ${{ secrets.SLACK_DEPLOYMENTS_CHANNEL_ID }} status: "started" @@ -116,7 +116,7 @@ jobs: timeout: 600s wait: "true" # We need to wait till deployment is finished here, since the calling workflow might test the deployment env - name: Notify of deployment finish - uses: mindsdb/github-actions/slack-deploy-msg + uses: mindsdb/github-actions/slack-deploy-msg@main if: always() with: channel-id: ${{ secrets.SLACK_DEPLOYMENTS_CHANNEL_ID }} From dc832e8f2f35729f3d0b11e4926df8505e0446c4 Mon Sep 17 00:00:00 2001 From: Hamish Fagg Date: Mon, 18 Aug 2025 17:04:09 +1200 Subject: [PATCH 20/27] fix --- .github/workflows/build-push-deploy.yml | 28 +------------------------ 1 file changed, 1 insertion(+), 27 deletions(-) diff --git a/.github/workflows/build-push-deploy.yml b/.github/workflows/build-push-deploy.yml index 069336d..cdc8014 100644 --- a/.github/workflows/build-push-deploy.yml +++ b/.github/workflows/build-push-deploy.yml @@ -19,7 +19,6 @@ on: required: false - jobs: # Looks for labels like "deploy-to-" attached to a PR so we can deploy to those envs get-deploy-labels: @@ -36,6 +35,7 @@ jobs: - id: get-labels uses: mindsdb/github-actions/get-deploy-labels@main + # Build our docker images based on our bake file build: runs-on: mdb-dev @@ -53,32 +53,6 @@ jobs: module-name: ${{ inputs.service-name }} build-for-environment: ${{ inputs.stage-name }} - # Push cache layers to docker registry - # This is separate to the build step so we can do other stuff in parallel - # build-cache: - # name: Push Docker Cache - # runs-on: mdb-dev - # needs: [build] - # concurrency: - # group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}-cache - # cancel-in-progress: true - # steps: - # - uses: actions/checkout@v4 - # with: - # ref: ${{ github.event.pull_request.head.sha }} - # - name: Pull MindsDB Github Actions - # uses: actions/checkout@v4 - # with: - # repository: mindsdb/github-actions - # path: github-actions - # # Build the bakefile and push - # - uses: ./github-actions/docker-bake - # with: - # git-sha: ${{ github.event.pull_request.head.sha }} - # target: cloud-cpu - # platforms: linux/amd64 - # push-cache: true - # cache-only: true # Call our deployment workflow, so long as this is not a forked PR # This will run the deployment workflow in the base branch, not in the PR. From 7eb5d87baa4023fd88d1b8a38608b8ac4dd5b7da Mon Sep 17 00:00:00 2001 From: Hamish Fagg Date: Mon, 25 Aug 2025 17:59:36 +1200 Subject: [PATCH 21/27] WIP --- .github/workflows/build-push-deploy.yml | 75 ++++++++++++++++++++----- 1 file changed, 62 insertions(+), 13 deletions(-) diff --git a/.github/workflows/build-push-deploy.yml b/.github/workflows/build-push-deploy.yml index cdc8014..e51c816 100644 --- a/.github/workflows/build-push-deploy.yml +++ b/.github/workflows/build-push-deploy.yml @@ -14,20 +14,40 @@ on: type: string required: false docker-image-ref: - description: "The version number or sha used in creating image tag" + description: "The version number or sha used in creating image tag." type: string required: false + default: "${{ github.sha }}" + dockerfiles: + description: "JSON list of dockerfiles to build, e.g. ['Dockerfile1', 'Dockerfile2']" + type: string + required: false + default: "['Dockerfile']" + # docker-bake: + # description: "Run docker bake instead of a regular build." + # type: boolean + # required: false + # default: false + # docker-bake-target: + # description: "The target to build with docker bake." + # type: string + # required: false + # docker-bake-platforms: + # description: "The platforms to build with docker bake." + # type: string + # required: false jobs: - # Looks for labels like "deploy-to-" attached to a PR so we can deploy to those envs + # Looks for PR labels like "deploy-to-" so we can deploy to those envs get-deploy-labels: name: Get Deploy Envs runs-on: mdb-dev concurrency: - group: ${{ github.workflow_ref }} - cancel-in-progress: true + group: ${{ github.workflow_ref }} # workflow_ref contains the workflow name and branch ref + cancel-in-progress: true # Cancel any in-progress runs on this branch - this one is newer environment: + # If this is a forked PR, set environment to 'manual-approval' name: ${{ github.event.pull_request.head.repo.fork && 'manual-approval' || '' }} outputs: deploy-envs: ${{ steps.get-labels.outputs.deploy-envs }} @@ -36,14 +56,17 @@ jobs: uses: mindsdb/github-actions/get-deploy-labels@main - # Build our docker images based on our bake file + # Build a docker image and push it to ECR build: runs-on: mdb-dev needs: [get-deploy-labels] if: needs.get-deploy-labels.outputs.deploy-envs != '[]' + strategy: + matrix: + dockerfile: ${{fromJson(inputs.dockerfiles)}} concurrency: - group: ${{ github.workflow_ref }} - cancel-in-progress: true + group: ${{ github.workflow_ref }} # workflow_ref contains the workflow name and branch ref + cancel-in-progress: true # Cancel any in-progress runs on this branch - this one is newer env: AWS_REGION: us-east-1 steps: @@ -52,11 +75,31 @@ jobs: with: module-name: ${{ inputs.service-name }} build-for-environment: ${{ inputs.stage-name }} + extra-build-args: "-f ${{ matrix.dockerfile }}" + # build-bake: + # runs-on: mdb-dev + # needs: [get-deploy-labels] + # if: inputs.docker-bake && needs.get-deploy-labels.outputs.deploy-envs != '[]' + # concurrency: + # group: ${{ github.workflow_ref }} # workflow_ref contains the workflow name and branch ref + # cancel-in-progress: true # Cancel any in-progress runs on this branch - this one is newer + # env: + # AWS_REGION: us-east-1 + # steps: + # - uses: actions/checkout@v4 + # - uses: mindsdb/github-actions/docker-bake@main + # with: + # - uses: ./github-actions/docker-bake + # with: + # git-sha: ${{ inputs.docker-image-ref }} + # target: ${{ inputs.docker-bake-target }} + # platforms: ${{ inputs.docker-bake-platforms }} + # push-cache: false + - # Call our deployment workflow, so long as this is not a forked PR - # This will run the deployment workflow in the base branch, not in the PR. - # So if you change the deploy workflow in your PR, the changes won't be reflected in this run. + # Deploy the built image to the specified environments + # Deploys to all environments at once deploy: runs-on: mdb-dev needs: [ get-deploy-labels, build ] @@ -64,15 +107,18 @@ jobs: matrix: deploy-env: ${{fromJson(needs.get-deploy-labels.outputs.deploy-envs)}} concurrency: - group: deploy-${{ matrix.deploy-env }} - cancel-in-progress: false + group: deploy-${{ matrix.deploy-env }} # All deployments for this env are grouped together + cancel-in-progress: false # Don't cancel in-progress deployments, it breaks helm environment: + # Assuming that ENV_URL is set in a github environment in the repo + # If not the link in the slack message will be borked, thats all name: ${{ matrix.deploy-env }} url: ${{ vars.ENV_URL }} steps: - uses: actions/checkout@v4 - uses: mindsdb/github-actions/setup-env@main - name: Notify of deployment starting + # This same message will be updated later with the deployment status id: slack uses: mindsdb/github-actions/slack-deploy-msg@main with: @@ -83,13 +129,16 @@ jobs: env-url: ${{ vars.ENV_URL }} slack-token: ${{ secrets.GH_ACTIONS_SLACK_BOT_TOKEN }} - uses: DevOps-Nirvana/aws-helm-multi-deploy-nodocker@v4 + # Do the actual deployment with: environment-slug: ${{matrix.deploy-env}} k8s-namespace: ${{matrix.deploy-env}} image-tag: ${{ inputs.stage-name }}-${{ github.sha }} timeout: 600s - wait: "true" # We need to wait till deployment is finished here, since the calling workflow might test the deployment env + # We need to wait till deployment is finished here, since the calling workflow might test the deployment env once this job is done + wait: "true" - name: Notify of deployment finish + # Update the slack message from before with the deployment status uses: mindsdb/github-actions/slack-deploy-msg@main if: always() with: From 2769466a99c63e20196ff7835735d0a9c71a6c3c Mon Sep 17 00:00:00 2001 From: Hamish Fagg Date: Mon, 15 Sep 2025 16:08:05 +1200 Subject: [PATCH 22/27] add bake --- .github/workflows/build-push-deploy.yml | 60 +++++++++---------------- 1 file changed, 21 insertions(+), 39 deletions(-) diff --git a/.github/workflows/build-push-deploy.yml b/.github/workflows/build-push-deploy.yml index e51c816..88d5c98 100644 --- a/.github/workflows/build-push-deploy.yml +++ b/.github/workflows/build-push-deploy.yml @@ -23,19 +23,14 @@ on: type: string required: false default: "['Dockerfile']" - # docker-bake: - # description: "Run docker bake instead of a regular build." - # type: boolean - # required: false - # default: false - # docker-bake-target: - # description: "The target to build with docker bake." - # type: string - # required: false - # docker-bake-platforms: - # description: "The platforms to build with docker bake." - # type: string - # required: false + docker-bake-target: + description: "The target to build with docker bake." + type: string + required: false + docker-bake-platforms: + description: "The platforms to build with docker bake." + type: string + required: false jobs: @@ -46,9 +41,6 @@ jobs: concurrency: group: ${{ github.workflow_ref }} # workflow_ref contains the workflow name and branch ref cancel-in-progress: true # Cancel any in-progress runs on this branch - this one is newer - environment: - # If this is a forked PR, set environment to 'manual-approval' - name: ${{ github.event.pull_request.head.repo.fork && 'manual-approval' || '' }} outputs: deploy-envs: ${{ steps.get-labels.outputs.deploy-envs }} steps: @@ -56,11 +48,11 @@ jobs: uses: mindsdb/github-actions/get-deploy-labels@main - # Build a docker image and push it to ECR + # Build docker image(s) based on Dockerfile(s) and push to ECR build: runs-on: mdb-dev needs: [get-deploy-labels] - if: needs.get-deploy-labels.outputs.deploy-envs != '[]' + if: ${{ !inputs.docker-bake && needs.get-deploy-labels.outputs.deploy-envs != '[]' }} strategy: matrix: dockerfile: ${{fromJson(inputs.dockerfiles)}} @@ -71,32 +63,22 @@ jobs: AWS_REGION: us-east-1 steps: - uses: actions/checkout@v4 - - uses: mindsdb/github-actions/build-push-ecr@main + # Build via docker-bake if a bakefile is specified + - if: ${{ contains(matrix.dockerfile, '.hcl') }} + uses: mindsdb/github-actions/docker-bake@main + with: + git-sha: ${{ inputs.docker-image-ref }} + target: ${{ inputs.docker-bake-target }} + platforms: ${{ inputs.docker-bake-platforms }} + push-cache: false + # Otherwise build via regular docker + - if: ${{ !contains(matrix.dockerfile, '.hcl') }} + uses: mindsdb/github-actions/build-push-ecr@main with: module-name: ${{ inputs.service-name }} build-for-environment: ${{ inputs.stage-name }} extra-build-args: "-f ${{ matrix.dockerfile }}" - # build-bake: - # runs-on: mdb-dev - # needs: [get-deploy-labels] - # if: inputs.docker-bake && needs.get-deploy-labels.outputs.deploy-envs != '[]' - # concurrency: - # group: ${{ github.workflow_ref }} # workflow_ref contains the workflow name and branch ref - # cancel-in-progress: true # Cancel any in-progress runs on this branch - this one is newer - # env: - # AWS_REGION: us-east-1 - # steps: - # - uses: actions/checkout@v4 - # - uses: mindsdb/github-actions/docker-bake@main - # with: - # - uses: ./github-actions/docker-bake - # with: - # git-sha: ${{ inputs.docker-image-ref }} - # target: ${{ inputs.docker-bake-target }} - # platforms: ${{ inputs.docker-bake-platforms }} - # push-cache: false - # Deploy the built image to the specified environments # Deploys to all environments at once From 0594a3a6f1af963a24633acf0540e34fcd274757 Mon Sep 17 00:00:00 2001 From: Hamish Fagg Date: Tue, 7 Oct 2025 09:34:47 +1300 Subject: [PATCH 23/27] add deploy namespace --- .github/workflows/build-push-deploy.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-push-deploy.yml b/.github/workflows/build-push-deploy.yml index 88d5c98..ea7fdfa 100644 --- a/.github/workflows/build-push-deploy.yml +++ b/.github/workflows/build-push-deploy.yml @@ -9,6 +9,10 @@ on: description: "The backend environment we are building for (API calls are pointed to). This should be one of (development, staging, production)." type: string required: true + deploy-namespace: + description: "The Kubernetes namespace to deploy the service to." + type: string + required: false docker-build-args: description: "Extra args passed to 'docker build'." type: string @@ -63,6 +67,8 @@ jobs: AWS_REGION: us-east-1 steps: - uses: actions/checkout@v4 + with: + ref: ${{ inputs.docker-image-ref }} # Build via docker-bake if a bakefile is specified - if: ${{ contains(matrix.dockerfile, '.hcl') }} uses: mindsdb/github-actions/docker-bake@main @@ -114,7 +120,7 @@ jobs: # Do the actual deployment with: environment-slug: ${{matrix.deploy-env}} - k8s-namespace: ${{matrix.deploy-env}} + k8s-namespace: ${{inputs.deploy-namespace || matrix.deploy-env}} image-tag: ${{ inputs.stage-name }}-${{ github.sha }} timeout: 600s # We need to wait till deployment is finished here, since the calling workflow might test the deployment env once this job is done From 5aef156bc90ed7d799f0a333058944f77b921050 Mon Sep 17 00:00:00 2001 From: Hamish Fagg Date: Thu, 9 Oct 2025 14:21:35 +1300 Subject: [PATCH 24/27] add migration --- .github/workflows/build-push-deploy.yml | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-push-deploy.yml b/.github/workflows/build-push-deploy.yml index ea7fdfa..b6c00f1 100644 --- a/.github/workflows/build-push-deploy.yml +++ b/.github/workflows/build-push-deploy.yml @@ -35,6 +35,11 @@ on: description: "The platforms to build with docker bake." type: string required: false + migrate-command: + description: "The command to run migrations." + type: string + required: false + jobs: @@ -83,14 +88,27 @@ jobs: with: module-name: ${{ inputs.service-name }} build-for-environment: ${{ inputs.stage-name }} + image-ref: ${{ inputs.docker-image-ref }} extra-build-args: "-f ${{ matrix.dockerfile }}" + migrate: + runs-on: mdb-dev + container: ${{ secrets.ECR_REGISTRY }}/${{ inputs.service-name }}:${{ inputs.stage-name }}-${{ inputs.docker-image-ref }} + needs: [get-deploy-labels, build] + if: ${{ needs.get-deploy-labels.outputs.deploy-envs != '[]' }} + steps: + - name: Run migrations + if: ${{ inputs.migrate-command }} + run: | + ${{ inputs.migrate-command }} + + # Deploy the built image to the specified environments # Deploys to all environments at once deploy: runs-on: mdb-dev - needs: [ get-deploy-labels, build ] + needs: [ get-deploy-labels, build, migrate ] strategy: matrix: deploy-env: ${{fromJson(needs.get-deploy-labels.outputs.deploy-envs)}} From c7fd0f79ffb8f4fd4cb1cad6035e22297b522938 Mon Sep 17 00:00:00 2001 From: Hamish Fagg Date: Thu, 9 Oct 2025 14:33:48 +1300 Subject: [PATCH 25/27] secret->var --- .github/workflows/build-push-deploy.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-push-deploy.yml b/.github/workflows/build-push-deploy.yml index b6c00f1..64a4c32 100644 --- a/.github/workflows/build-push-deploy.yml +++ b/.github/workflows/build-push-deploy.yml @@ -94,8 +94,11 @@ jobs: migrate: runs-on: mdb-dev - container: ${{ secrets.ECR_REGISTRY }}/${{ inputs.service-name }}:${{ inputs.stage-name }}-${{ inputs.docker-image-ref }} needs: [get-deploy-labels, build] + strategy: + matrix: + deploy-env: ${{fromJson(needs.get-deploy-labels.outputs.deploy-envs)}} + container: ${{ vars.ECR_REGISTRY }}/${{ inputs.service-name }}:${{ inputs.stage-name }}-${{ inputs.docker-image-ref }} if: ${{ needs.get-deploy-labels.outputs.deploy-envs != '[]' }} steps: - name: Run migrations From 3b204278bb9197b8dc57b4edd87f3129d69aa91f Mon Sep 17 00:00:00 2001 From: Hamish Fagg Date: Thu, 9 Oct 2025 14:37:32 +1300 Subject: [PATCH 26/27] log into ecr --- .github/workflows/build-push-deploy.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/build-push-deploy.yml b/.github/workflows/build-push-deploy.yml index 64a4c32..e980143 100644 --- a/.github/workflows/build-push-deploy.yml +++ b/.github/workflows/build-push-deploy.yml @@ -101,6 +101,9 @@ jobs: container: ${{ vars.ECR_REGISTRY }}/${{ inputs.service-name }}:${{ inputs.stage-name }}-${{ inputs.docker-image-ref }} if: ${{ needs.get-deploy-labels.outputs.deploy-envs != '[]' }} steps: + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v2 - name: Run migrations if: ${{ inputs.migrate-command }} run: | From 833709f0ede55acf825bc4ff672469524d98835d Mon Sep 17 00:00:00 2001 From: Hamish Fagg Date: Thu, 9 Oct 2025 15:06:20 +1300 Subject: [PATCH 27/27] fix migrate --- .github/workflows/build-push-deploy.yml | 27 ++++++++++++++++--------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build-push-deploy.yml b/.github/workflows/build-push-deploy.yml index e980143..eff94bb 100644 --- a/.github/workflows/build-push-deploy.yml +++ b/.github/workflows/build-push-deploy.yml @@ -35,10 +35,11 @@ on: description: "The platforms to build with docker bake." type: string required: false - migrate-command: - description: "The command to run migrations." + migration-job-file: + description: "The file path to the migration k8s job YAML." type: string required: false + default: "deployment/migration-job.yaml" @@ -98,17 +99,23 @@ jobs: strategy: matrix: deploy-env: ${{fromJson(needs.get-deploy-labels.outputs.deploy-envs)}} - container: ${{ vars.ECR_REGISTRY }}/${{ inputs.service-name }}:${{ inputs.stage-name }}-${{ inputs.docker-image-ref }} - if: ${{ needs.get-deploy-labels.outputs.deploy-envs != '[]' }} + concurrency: + group: deploy-${{ matrix.deploy-env }} # All deployments for this env are grouped together + cancel-in-progress: false # Don't cancel in-progress deployments, it breaks helm steps: - - name: Login to Amazon ECR - id: login-ecr - uses: aws-actions/amazon-ecr-login@v2 - - name: Run migrations - if: ${{ inputs.migrate-command }} + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.docker-image-ref }} + - name: Migrate run: | - ${{ inputs.migrate-command }} + export NAMESPACE=${{inputs.deploy-namespace || matrix.deploy-env}} + export IMAGE_TAG=${{ inputs.stage-name }}-${{ inputs.docker-image-ref }} + export JOB_NAME=$(grep -E '^ *name:' ${{ inputs.migration-job-file }} | head -1 | awk '{print $2}') + + kubectl -n $NAMESPACE delete job --ignore-not-found $JOB_NAME + envsubst '${IMAGE_TAG} ${NAMESPACE}' < ${{ inputs.migration-job-file }} | kubectl apply -f - + kubectl -n "$NAMESPACE" wait --for=condition=complete --timeout=1m "job/$JOB_NAME" # Deploy the built image to the specified environments # Deploys to all environments at once