diff --git a/flytekit/image_spec/default_builder.py b/flytekit/image_spec/default_builder.py
index 1a465f0e88..c3f29e185c 100644
--- a/flytekit/image_spec/default_builder.py
+++ b/flytekit/image_spec/default_builder.py
@@ -139,6 +139,11 @@
 USER flytekit
 RUN mkdir -p $$HOME && \
     echo "export PATH=$$PATH" >> $$HOME/.profile
+
+# NOTE: Important to set this env var all the way at the bottom
+# Otherwise the slightest change to an imagespec's ID would unnecessarily invalidate
+# too many layers in the build cache.
+ENV $_F_IMG_ID_ENV
 """)
 
 DEFAULT_UV_IMAGE = "ghcr.io/astral-sh/uv:0.5.1"
@@ -332,7 +337,7 @@ def create_docker_context(image_spec: ImageSpec, tmp_dir: Path):
         raise ValueError(msg)
 
     uv_python_install_command = prepare_python_install(image_spec, tmp_dir)
-    env_dict = {"PYTHONPATH": "/root", _F_IMG_ID: image_spec.id}
+    env_dict = {"PYTHONPATH": "/root"}
 
     if image_spec.env:
         env_dict.update(image_spec.env)
@@ -414,6 +419,8 @@ def create_docker_context(image_spec: ImageSpec, tmp_dir: Path):
         uv_image = image_spec.builder_config.get("uv_image", uv_image)
         micromamba_image = image_spec.builder_config.get("micromamba_image", micromamba_image)
 
+    _f_img_id_env = f"{_F_IMG_ID}={image_spec.id}"
+
     docker_content = DOCKER_FILE_TEMPLATE.substitute(
         UV_PYTHON_INSTALL_COMMAND=uv_python_install_command,
         APT_INSTALL_COMMAND=apt_install_command,
@@ -422,6 +429,7 @@ def create_docker_context(image_spec: ImageSpec, tmp_dir: Path):
         PYTHON_EXEC=python_install_template.python_exec,
         BASE_IMAGE=base_image,
         ENV=env,
+        _F_IMG_ID_ENV=_f_img_id_env,
         COPY_COMMAND_RUNTIME=copy_command_runtime,
         ENTRYPOINT=entrypoint,
         RUN_COMMANDS=run_commands,