diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml
index 4d5ad536..6f0b6050 100644
--- a/.github/workflows/publish.yaml
+++ b/.github/workflows/publish.yaml
@@ -22,7 +22,40 @@ env:
   GCP_PROJECT_ID: moz-fx-remote-settings-prod
 
 jobs:
+  # Let's figure out which container we have to build based on which files were changed.
+  # On pull-requests that don't touch these files, we don't need to rebuild them.
+  # Of course, on push to main or releases, we always build and publish to GAR/Dockerhub.
+  changes:
+    runs-on: ubuntu-latest
+    outputs:
+      cronjobs: ${{ steps.filter.outputs.cronjobs }}
+      browser_tests: ${{ steps.filter.outputs.browser_tests }}
+      git_reader: ${{ steps.filter.outputs.git_reader }}
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          fetch-depth: 0
+      - id: filter
+        env:
+          BASE: ${{ github.event.pull_request.base.sha }}
+          HEAD: ${{ github.event.pull_request.head.sha }}
+        run: |
+          files=$(git diff --name-only "$BASE" "$HEAD")
+          echo "Changed files:"
+          echo "$files"
+          check() {
+            if echo "$files" | grep -qE "$2"; then
+              echo "$1=true" >> "$GITHUB_OUTPUT"
+            else
+              echo "$1=false" >> "$GITHUB_OUTPUT"
+            fi
+          }
+          check cronjobs       '^.github/workflows/|^cronjobs/'
+          check browser_tests  '^.github/workflows/|^browser-tests/'
+          check git_reader     '^.github/workflows/|^git-reader/'
+
   server_container:
+    # Always build the server (since it depends on lots of files uv.lock, dockerfiles, kinto-slack, etc.)
     env:
       DOCKERHUB_IMAGE_NAME: mozilla/remote-settings
       GAR_IMAGE_NAME: remote-settings
@@ -110,8 +143,8 @@ jobs:
           tags: ${{ steps.metagar.outputs.tags }}
           labels: ${{ steps.metagar.outputs.labels }}
           platforms: linux/amd64,linux/arm64
-          cache-from: type=gha # Load cache from GitHub Actions
-          cache-to: type=gha,mode=max # Save cache to GitHub Actions
+          cache-from: type=gha,scope=server
+          cache-to: type=gha,mode=max,scope=server
       - name: Wait for release base image
         if: ${{ github.event_name == 'release' }}
         env:
@@ -139,8 +172,8 @@ jobs:
           tags: ${{ steps.metagar.outputs.tags }}
           labels: ${{ steps.metagar.outputs.labels }}
           platforms: linux/amd64,linux/arm64
-          cache-from: type=gha # Load cache from GitHub Actions
-          cache-to: type=gha,mode=max # Save cache to GitHub Actions
+          cache-from: type=gha,scope=server-release
+          cache-to: type=gha,mode=max,scope=server-release
           build-args: BASE_IMG=${{ env.GAR_LOCATION }}-docker.pkg.dev/${{ env.GCP_PROJECT_ID }}/${{ env.GAR_REPOSITORY}}/remote-settings:sha-${{ github.sha }}
       - name: Copy from Google Artifact Registry to Docker Hub
         if: github.event_name != 'pull_request'
@@ -162,6 +195,8 @@ jobs:
             text: "⚠️  Build of ${{ env.GAR_IMAGE_NAME }}:${{ env.LATEST_TAG }} failed. Please review logs and correct issues."
 
   cronjobs_container:
+    needs: changes
+    if: ${{ github.event_name != 'pull_request' || needs.changes.outputs.cronjobs == 'true' }}
     env:
       GAR_IMAGE_NAME: remote-settings-core-cronjobs
       LATEST_TAG: ""  # Set after checkout step
@@ -205,8 +240,8 @@ jobs:
           push: ${{ github.event_name == 'push' }}
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
-          cache-from: type=gha # Load cache from GitHub Actions
-          cache-to: type=gha,mode=max # Save cache to GitHub Actions
+          cache-from: type=gha,scope=cronjobs
+          cache-to: type=gha,mode=max,scope=cronjobs
       - name: Notify DEVs of build failure
         if: failure()
         uses: slackapi/slack-github-action@v3.0.3
@@ -217,6 +252,8 @@ jobs:
             text: "⚠️  Build of ${{ env.GAR_IMAGE_NAME }}:${{ env.LATEST_TAG }} failed. Please review logs and correct issues."
 
   browser_test_container:
+    needs: changes
+    if: ${{ github.event_name != 'pull_request' || needs.changes.outputs.browser_tests == 'true' }}
     env:
       GAR_IMAGE_NAME: remote-settings-browser-tests
       LATEST_TAG: ""  # Set after checkout step
@@ -260,8 +297,8 @@ jobs:
           push: ${{ github.event_name == 'push' }}
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
-          cache-from: type=gha # Load cache from GitHub Actions
-          cache-to: type=gha,mode=max # Save cache to GitHub Actions
+          cache-from: type=gha,scope=browser-tests
+          cache-to: type=gha,mode=max,scope=browser-tests
       - name: Notify DEVs of build failure
         if: failure()
         uses: slackapi/slack-github-action@v3.0.3
@@ -272,6 +309,8 @@ jobs:
             text: "⚠️  Build of ${{ env.GAR_IMAGE_NAME }}:${{ env.LATEST_TAG }} failed. Please review logs and correct issues."
 
   git_reader_container:
+    needs: changes
+    if: ${{ github.event_name != 'pull_request' || needs.changes.outputs.git_reader == 'true' }}
     env:
       GAR_IMAGE_NAME: remote-settings-git-reader
       LATEST_TAG: ""  # Set after checkout step
@@ -327,8 +366,8 @@ jobs:
           push: ${{ github.event_name != 'pull_request' }}
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
-          cache-from: type=gha # Load cache from GitHub Actions
-          cache-to: type=gha,mode=max # Save cache to GitHub Actions
+          cache-from: type=gha,scope=git-reader
+          cache-to: type=gha,mode=max,scope=git-reader
       - name: Notify DEVs of build failure
         if: failure()
         uses: slackapi/slack-github-action@v3.0.3