From 665912dcf7fc4b3944726a0ee9d5f12814d448a9 Mon Sep 17 00:00:00 2001
From: ptaylor <paul.e.taylor@me.com>
Date: Fri, 22 May 2026 09:35:03 -0700
Subject: [PATCH 01/10] fix s3 envvar names

---
 .../src/utils/opt/devcontainer/bin/creds/s3/persist.sh    | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/features/src/utils/opt/devcontainer/bin/creds/s3/persist.sh b/features/src/utils/opt/devcontainer/bin/creds/s3/persist.sh
index a493ce1a0..9ae1015e9 100755
--- a/features/src/utils/opt/devcontainer/bin/creds/s3/persist.sh
+++ b/features/src/utils/opt/devcontainer/bin/creds/s3/persist.sh
@@ -91,19 +91,19 @@ ________EOF
 
     if ! grep -qE "^$" <<< "${aws_secret_access_key:-}"; then
         if test -w ~/.aws/credentials; then
-            reset_envvar "AWS_SESSION_TOKEN";
+            reset_envvar "AWS_SECRET_ACCESS_KEY";
             cat <<< "aws_secret_access_key=${aws_secret_access_key}" >> ~/.aws/credentials
         else
-            export_envvar "AWS_SESSION_TOKEN" "${aws_secret_access_key}";
+            export_envvar "AWS_SECRET_ACCESS_KEY" "${aws_secret_access_key}";
         fi
     fi
 
     if ! grep -qE "^$" <<< "${aws_session_token:-}"; then
         if test -w ~/.aws/credentials; then
-            reset_envvar "AWS_SECRET_ACCESS_KEY";
+            reset_envvar "AWS_SESSION_TOKEN";
             cat <<< "aws_session_token=${aws_session_token}" >> ~/.aws/credentials
         else
-            export_envvar "AWS_SECRET_ACCESS_KEY" "${aws_session_token}";
+            export_envvar "AWS_SESSION_TOKEN" "${aws_session_token}";
         fi
     fi
 

From b213425445d9190ae09d87d09194cdedde38af7a Mon Sep 17 00:00:00 2001
From: ptaylor <paul.e.taylor@me.com>
Date: Fri, 22 May 2026 09:36:47 -0700
Subject: [PATCH 02/10] kill after 15s of waiting to terminate the sccache
 daemon gracefully

---
 .../src/utils/opt/devcontainer/bin/creds/s3/test.sh  | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/features/src/utils/opt/devcontainer/bin/creds/s3/test.sh b/features/src/utils/opt/devcontainer/bin/creds/s3/test.sh
index ed813d214..e73333a49 100755
--- a/features/src/utils/opt/devcontainer/bin/creds/s3/test.sh
+++ b/features/src/utils/opt/devcontainer/bin/creds/s3/test.sh
@@ -32,7 +32,10 @@ _creds_s3_test() {
     local aws_session_token="${AWS_SESSION_TOKEN:-"$(sed -n 's/aws_session_token=//p' ~/.aws/credentials 2>/dev/null)"}";
     local aws_secret_access_key="${AWS_SECRET_ACCESS_KEY:-"$(sed -n 's/aws_secret_access_key=//p' ~/.aws/credentials 2>/dev/null)"}";
 
-    devcontainer-utils-stop-sccache --kill -p 4220 || true;
+    if ! timeout --preserve-status --kill-after=20s 15s \
+         devcontainer-utils-stop-sccache -p 4220; then
+         devcontainer-utils-stop-sccache -p 4220 --kill || :
+    fi
 
     local result=0;
 
@@ -52,7 +55,12 @@ _creds_s3_test() {
        result=1;
     fi
 
-    devcontainer-utils-stop-sccache --kill -p 4220 || true;
+    devcontainer-utils-stop-sccache --kill -p 4220 || :
+
+    if ! timeout --preserve-status --kill-after=20s 15s \
+         devcontainer-utils-stop-sccache -p 4220; then
+         devcontainer-utils-stop-sccache -p 4220 --kill || :
+    fi
 
     if test "$result" -eq 0; then
         local logfile="${SCCACHE_ERROR_LOG:-/tmp/sccache.log}";

From ed3f69abd51d43d7694090a59a6a08a0c21febe4 Mon Sep 17 00:00:00 2001
From: ptaylor <paul.e.taylor@me.com>
Date: Fri, 22 May 2026 09:37:23 -0700
Subject: [PATCH 03/10] Only modify the ~/.aws files once so the sccache file
 watcher doesn't refresh multiple times

---
 .../devcontainer/bin/creds/s3/gh/generate.sh  | 37 +++++++++++-------
 .../bin/creds/s3/vault/generate.sh            | 39 +++++++++++--------
 2 files changed, 44 insertions(+), 32 deletions(-)

diff --git a/features/src/utils/opt/devcontainer/bin/creds/s3/gh/generate.sh b/features/src/utils/opt/devcontainer/bin/creds/s3/gh/generate.sh
index 5f342376b..48adce724 100755
--- a/features/src/utils/opt/devcontainer/bin/creds/s3/gh/generate.sh
+++ b/features/src/utils/opt/devcontainer/bin/creds/s3/gh/generate.sh
@@ -10,20 +10,9 @@ _creds_github_generate() {
     if ! test -n "${AWS_ROLE_ARN:+x}" \
     || ! test -n "${SCCACHE_BUCKET:+x}" \
     || ! gh nv-gha-aws --help >/dev/null 2>&1; then
-        exit 1;
+        return 1;
     fi
 
-    # Remove existing credentials in case nv-gha-aws declines to issue new ones.
-    if test -w ~/.aws; then
-        rm -rf ~/.aws/{stamp,config,credentials};
-    fi
-
-    SCCACHE_REGION="${SCCACHE_REGION:-${AWS_DEFAULT_REGION:-}}";
-
-    devcontainer-utils-creds-s3-persist - <<< \
-        --bucket="${SCCACHE_BUCKET:-}"        \
-        --region="${SCCACHE_REGION:-}"        ;
-
     # Initialize the GitHub CLI with the appropriate user scopes
     # shellcheck disable=SC1091
     . devcontainer-utils-init-github-cli;
@@ -41,7 +30,7 @@ _creds_github_generate() {
     ))";
 
     if test "${#user_orgs[@]}" -eq 0; then
-        exit 1;
+        return 1;
     fi
 
     local org;
@@ -67,7 +56,25 @@ _creds_github_generate() {
         fi
     done
 
-    exit 1;
+    return 1;
 }
 
-_creds_github_generate "$@" <&0;
+if ! _creds_github_generate "$@" <&0; then
+
+    # Remove existing credentials in case nv-gha-aws declines to issue new ones.
+    if test -w ~/.aws; then
+        rm -rf ~/.aws/{stamp,config,credentials};
+    fi
+
+    devcontainer-utils-creds-s3-persist - <<<                 \
+        --bucket="${SCCACHE_BUCKET:-}"                        \
+        --region="${SCCACHE_REGION:-${AWS_DEFAULT_REGION:-}}" ;
+
+    # shellcheck disable=SC1090
+    . /etc/profile.d/*-devcontainer-utils.sh;
+
+    exit 1;
+fi
+
+# shellcheck disable=SC1090
+. /etc/profile.d/*-devcontainer-utils.sh;
diff --git a/features/src/utils/opt/devcontainer/bin/creds/s3/vault/generate.sh b/features/src/utils/opt/devcontainer/bin/creds/s3/vault/generate.sh
index 61c2b6b49..c678e517a 100755
--- a/features/src/utils/opt/devcontainer/bin/creds/s3/vault/generate.sh
+++ b/features/src/utils/opt/devcontainer/bin/creds/s3/vault/generate.sh
@@ -11,27 +11,17 @@ _creds_vault_generate() {
 
     if ! test -n "${VAULT_HOST:+x}" \
     || ! test -n "${SCCACHE_BUCKET:+x}"; then
-        exit 1;
+        return 1;
     fi
 
     SCCACHE_REGION="${SCCACHE_REGION:-${AWS_DEFAULT_REGION:-}}";
 
-    # Remove existing credentials in case vault declines to issue new ones.
-    if test -w ~/.aws; then
-        rm -rf ~/.aws/{stamp,config,credentials};
-    fi
-
-    devcontainer-utils-creds-s3-persist - <<< " \
-        --bucket '${SCCACHE_BUCKET:-}'          \
-        --region '${SCCACHE_REGION:-}'          \
-    ";
-
     # Initialize the GitHub CLI with the appropriate user scopes
     # shellcheck disable=SC1091
     . devcontainer-utils-init-github-cli;
 
     if ! test -n "${GITHUB_USER:+x}"; then
-        exit 1;
+        return 1;
     fi
 
     # Check whether the user is in one of the allowed GitHub orgs
@@ -47,7 +37,7 @@ _creds_vault_generate() {
     )";
 
     if test "${#user_orgs}" -eq 0; then
-        exit 1;
+        return 1;
     fi
 
     cat <<____EOF | tee -a /var/log/devcontainer-utils/creds-s3.log
@@ -65,7 +55,7 @@ ____EOF
         cat <<________EOF | tee -a /var/log/devcontainer-utils/creds-s3.log >&2
 Your GitHub user was not recognized by vault. Skipping.
 ________EOF
-        exit 1;
+        return 1;
     fi
 
     cat <<____EOF | tee -a /var/log/devcontainer-utils/creds-s3.log
@@ -97,14 +87,14 @@ ____EOF
         cat <<________EOF | tee -a /var/log/devcontainer-utils/creds-s3.log >&2
 Failed to retrieve AWS S3 credentials. Skipping.
 ________EOF
-        exit 1;
+        return 1;
     fi
 
     if grep -qE "^null$" <<< "${aws_secret_access_key:-null}"; then
         cat <<________EOF | tee -a /var/log/devcontainer-utils/creds-s3.log >&2
 Failed to retrieve AWS S3 credentials. Skipping.
 ________EOF
-        exit 1;
+        return 1;
     fi
 
     cat <<____EOF | tee -a /var/log/devcontainer-utils/creds-s3.log
@@ -127,7 +117,22 @@ ____EOF
     fi
 }
 
-_creds_vault_generate "$@";
+if ! _creds_vault_generate "$@" <&0; then
+
+    # Remove existing credentials in case vault declines to issue new ones.
+    if test -w ~/.aws; then
+        rm -rf ~/.aws/{stamp,config,credentials};
+    fi
+
+    devcontainer-utils-creds-s3-persist - <<<                 \
+        --bucket="${SCCACHE_BUCKET:-}"                        \
+        --region="${SCCACHE_REGION:-${AWS_DEFAULT_REGION:-}}" ;
+
+    # shellcheck disable=SC1090
+    . /etc/profile.d/*-devcontainer-utils.sh;
+
+    exit 1;
+fi
 
 # shellcheck disable=SC1090
 . /etc/profile.d/*-devcontainer-utils.sh;

From 8123b1a99a9111e93c34f73da5ae0f5fef10259f Mon Sep 17 00:00:00 2001
From: ptaylor <paul.e.taylor@me.com>
Date: Fri, 22 May 2026 09:37:47 -0700
Subject: [PATCH 04/10] use the new SCCACHE_PID_FILE envvar so the sccache
 daemon doesn't randomly change pids

---
 .../utils/opt/devcontainer/bin/sccache/start.sh   |  8 ++++++--
 .../utils/opt/devcontainer/bin/sccache/stop.sh    | 15 ++++++++-------
 2 files changed, 14 insertions(+), 9 deletions(-)

diff --git a/features/src/utils/opt/devcontainer/bin/sccache/start.sh b/features/src/utils/opt/devcontainer/bin/sccache/start.sh
index 3ea982ede..e07cfe719 100755
--- a/features/src/utils/opt/devcontainer/bin/sccache/start.sh
+++ b/features/src/utils/opt/devcontainer/bin/sccache/start.sh
@@ -58,15 +58,19 @@ _start_sccache() {
     else
         # Start the sccache server in the background
         RUST_LOG_STYLE="never"                  \
+        SCCACHE_PID_FILE="${pidfile}"           \
         SCCACHE_ERROR_LOG="${logfile}"          \
         SCCACHE_SERVER_LOG="${log_lvl}"         \
         SCCACHE_SERVER_PORT="${sccache_port}"   \
         sccache --start-server 1>&2 2>/dev/null \
       | tee "$logfile";
-        # Write the pid to the pidfile
-        pgrep sccache | sort -n | head -n1 | tee "${pidfile}" >/dev/null;
+
+        # Wait till the pidfile exists and is not empty
+        until test -s "${pidfile}"; do sleep 1; done
+
         # Increase the open file limit so users can do `make -j(ulimit -n)`
         prlimit --nofile=$(ulimit -Hn):$(ulimit -Hn) --pid "$(cat "${pidfile}")";
+
         echo "=== sccache logfile: $logfile ===" >&2;
         echo "=== sccache pidfile: $pidfile ===" >&2;
     fi
diff --git a/features/src/utils/opt/devcontainer/bin/sccache/stop.sh b/features/src/utils/opt/devcontainer/bin/sccache/stop.sh
index d70b7f064..0db127e2d 100755
--- a/features/src/utils/opt/devcontainer/bin/sccache/stop.sh
+++ b/features/src/utils/opt/devcontainer/bin/sccache/stop.sh
@@ -33,31 +33,32 @@ _stop_sccache() {
 
     if test -n "${a:-${kill_all:+x}}"; then
         # Shutdown all sccache processes forcefully
-        sudo pkill -9 sccache >/dev/null 2>&1 || true;
+        sudo pkill -9 sccache >/dev/null 2>&1 || :
+        rm -f /tmp/sccache.*.pid 2>/dev/null || :
     elif test -n "${k:-${kill:+x}}" && test -f "${pidfile}"; then
         # Shutdown the sccache process on `$sccache_port` forcefully
-        sudo pkill -9 --pidfile "${pidfile}" >/dev/null 2>&1 || true;
+        sudo pkill -9 --pidfile "${pidfile}" >/dev/null 2>&1 || :
+        rm -f "${pidfile}" 2>/dev/null || :
     else
         # Shutdown gracefully
         SCCACHE_SERVER_PORT="${sccache_port}" \
-        sccache --stop-server >/dev/null 2>&1 || true;
+        sccache --stop-server >/dev/null 2>&1 || :
         if test -f "${pidfile}"; then
             # Wait for the server to shutdown
             if command -V pidwait >/dev/null 2>&1; then
-                pidwait --pidfile "${pidfile}" >/dev/null 2>&1 || true;
+                pidwait --pidfile "${pidfile}" >/dev/null 2>&1 || :
             else
                 while IFS= read -r pid; do
                     if test -n "${pid:+x}"; then
                         while test -e "/proc/${pid}"; do
-                            sleep 0.1;
+                            sleep 1;
                         done
                     fi
                 done < "${pidfile}"
             fi
+            rm -f "${pidfile}" 2>/dev/null || :
         fi
     fi
-
-    rm -f "${pidfile}" 2>/dev/null || true;
 }
 
 _stop_sccache "$@" <&0;

From f552ae28fdbfbaf2d99c0cafa282ce14a7e5b58c Mon Sep 17 00:00:00 2001
From: ptaylor <paul.e.taylor@me.com>
Date: Fri, 22 May 2026 09:38:20 -0700
Subject: [PATCH 05/10] bump feature version

---
 features/src/utils/devcontainer-feature.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/features/src/utils/devcontainer-feature.json b/features/src/utils/devcontainer-feature.json
index 4ea99dc4c..28b4a8b2c 100644
--- a/features/src/utils/devcontainer-feature.json
+++ b/features/src/utils/devcontainer-feature.json
@@ -1,7 +1,7 @@
 {
   "name": "devcontainer-utils",
   "id": "utils",
-  "version": "26.8.0",
+  "version": "26.8.1",
   "description": "A feature to install RAPIDS devcontainer utility scripts",
   "containerEnv": {
     "BASH_ENV": "/etc/bash.bash_env"

From 464099c952ed5d85df63fbcbb7de04c52519f6c3 Mon Sep 17 00:00:00 2001
From: ptaylor <paul.e.taylor@me.com>
Date: Fri, 22 May 2026 09:46:06 -0700
Subject: [PATCH 06/10] use RAPIDS token service in feature tests

---
 .github/actions/build-and-test-feature/action.yml | 2 ++
 .github/workflows/build-and-test-feature.yml      | 3 ++-
 features/test/utils/scenarios.json                | 4 ++++
 features/test/utils/ubuntu18.04.sh                | 3 +++
 features/test/utils/ubuntu20.04.sh                | 3 +++
 features/test/utils/ubuntu22.04.sh                | 3 +++
 features/test/utils/ubuntu24.04.sh                | 3 +++
 7 files changed, 20 insertions(+), 1 deletion(-)

diff --git a/.github/actions/build-and-test-feature/action.yml b/.github/actions/build-and-test-feature/action.yml
index 9dbc8f292..896ffcc66 100644
--- a/.github/actions/build-and-test-feature/action.yml
+++ b/.github/actions/build-and-test-feature/action.yml
@@ -5,6 +5,7 @@ description: Test feature
 inputs:
   args: {type: string, required: true}
   gh_token: {type: string, defaut: '', required: false}
+  aws_idp_url: {type: string, defaut: '', required: false}
   aws_role_arn: {type: string, defaut: '', required: false}
   rw_sccache_bucket: {type: string, defaut: '', required: false}
   rw_sccache_region: {type: string, defaut: '', required: false}
@@ -31,6 +32,7 @@ runs:
         VAULT_S3_TTL: "900" # 15 minutes
         ARGS: ${{ inputs.args }}
         gh_token: "${{ inputs.gh_token }}"
+        aws_idp_url: "${{ inputs.aws_idp_url }}"
         aws_role_arn: "${{ inputs.aws_role_arn }}"
         rw_sccache_bucket: "${{ inputs.rw_sccache_bucket }}"
         rw_sccache_region: "${{ inputs.rw_sccache_region }}"
diff --git a/.github/workflows/build-and-test-feature.yml b/.github/workflows/build-and-test-feature.yml
index d97dd34c8..9abfb46be 100644
--- a/.github/workflows/build-and-test-feature.yml
+++ b/.github/workflows/build-and-test-feature.yml
@@ -37,7 +37,8 @@ jobs:
         with:
           args: "${{ inputs.args }}"
           gh_token: "${{ secrets.GIST_REPO_READ_ORG_GITHUB_TOKEN }}"
-          aws_role_arn: "${{ secrets.GIST_REPO_READ_ORG_GITHUB_TOKEN && 'arn:aws:iam::279114543810:role/nv-gha-token-sccache-devs' || '' }}"
+          aws_idp_url: ${{ secrets.GIST_REPO_READ_ORG_GITHUB_TOKEN && 'https://token.rapids.nvidia.com' || '' }}
+          aws_role_arn: "${{ secrets.GIST_REPO_READ_ORG_GITHUB_TOKEN && 'arn:aws:iam::279114543810:role/rapids-token-sccache-devs' || '' }}"
           rw_sccache_bucket: "${{ secrets.GIST_REPO_READ_ORG_GITHUB_TOKEN && 'rapids-sccache-devs' || '' }}"
           rw_sccache_region: "${{ vars.AWS_REGION }}"
           sccache_dist_scheduler_url: "sccache.rapids.nvidia.com"
diff --git a/features/test/utils/scenarios.json b/features/test/utils/scenarios.json
index e0f0cb38c..90aa6679c 100644
--- a/features/test/utils/scenarios.json
+++ b/features/test/utils/scenarios.json
@@ -5,6 +5,7 @@
     "containerEnv": {
       "gh_token": "${localEnv:gh_token}",
       "vault_host": "${localEnv:vault_host}",
+      "aws_idp_url": "${localEnv:aws_idp_url}",
       "aws_role_arn": "${localEnv:aws_role_arn}",
       "VAULT_S3_TTL": "${localEnv:VAULT_S3_TTL}",
       "rw_sccache_bucket": "${localEnv:rw_sccache_bucket}",
@@ -51,6 +52,7 @@
     "containerEnv": {
       "gh_token": "${localEnv:gh_token}",
       "vault_host": "${localEnv:vault_host}",
+      "aws_idp_url": "${localEnv:aws_idp_url}",
       "aws_role_arn": "${localEnv:aws_role_arn}",
       "VAULT_S3_TTL": "${localEnv:VAULT_S3_TTL}",
       "rw_sccache_bucket": "${localEnv:rw_sccache_bucket}",
@@ -97,6 +99,7 @@
     "containerEnv": {
       "gh_token": "${localEnv:gh_token}",
       "vault_host": "${localEnv:vault_host}",
+      "aws_idp_url": "${localEnv:aws_idp_url}",
       "aws_role_arn": "${localEnv:aws_role_arn}",
       "VAULT_S3_TTL": "${localEnv:VAULT_S3_TTL}",
       "rw_sccache_bucket": "${localEnv:rw_sccache_bucket}",
@@ -143,6 +146,7 @@
     "containerEnv": {
       "gh_token": "${localEnv:gh_token}",
       "vault_host": "${localEnv:vault_host}",
+      "aws_idp_url": "${localEnv:aws_idp_url}",
       "aws_role_arn": "${localEnv:aws_role_arn}",
       "VAULT_S3_TTL": "${localEnv:VAULT_S3_TTL}",
       "rw_sccache_bucket": "${localEnv:rw_sccache_bucket}",
diff --git a/features/test/utils/ubuntu18.04.sh b/features/test/utils/ubuntu18.04.sh
index 58c928b86..9842d317c 100644
--- a/features/test/utils/ubuntu18.04.sh
+++ b/features/test/utils/ubuntu18.04.sh
@@ -214,6 +214,7 @@ if test -n "${gh_token:+x}" \
     no_creds_with_GH_TOKEN_AWS_ROLE_ARN_and_SCCACHE_BUCKET_should_generate_credentials() {
         reset_state;
         GH_TOKEN="${gh_token}" \
+        AWS_IDP_URL="${aws_idp_url:-}" \
         AWS_ROLE_ARN="${aws_role_arn}" \
         SCCACHE_BUCKET="${rw_sccache_bucket}" \
         SCCACHE_REGION="${rw_sccache_region}" \
@@ -229,6 +230,7 @@ if test -n "${gh_token:+x}" \
         reset_state;
         cp -ar /tmp/.aws ~/;
         GH_TOKEN="${gh_token}" \
+        AWS_IDP_URL="${aws_idp_url:-}" \
         AWS_ROLE_ARN="${aws_role_arn}" \
         SCCACHE_BUCKET="${rw_sccache_bucket}" \
         SCCACHE_REGION="${rw_sccache_region}" \
@@ -255,6 +257,7 @@ if test -n "${gh_token:+x}" \
         reset_state;
         write_bad_creds;
         GH_TOKEN="${gh_token}" \
+        AWS_IDP_URL="${aws_idp_url:-}" \
         AWS_ROLE_ARN="${aws_role_arn}" \
         SCCACHE_BUCKET="${rw_sccache_bucket}" \
         SCCACHE_REGION="${rw_sccache_region}" \
diff --git a/features/test/utils/ubuntu20.04.sh b/features/test/utils/ubuntu20.04.sh
index 58c928b86..9842d317c 100644
--- a/features/test/utils/ubuntu20.04.sh
+++ b/features/test/utils/ubuntu20.04.sh
@@ -214,6 +214,7 @@ if test -n "${gh_token:+x}" \
     no_creds_with_GH_TOKEN_AWS_ROLE_ARN_and_SCCACHE_BUCKET_should_generate_credentials() {
         reset_state;
         GH_TOKEN="${gh_token}" \
+        AWS_IDP_URL="${aws_idp_url:-}" \
         AWS_ROLE_ARN="${aws_role_arn}" \
         SCCACHE_BUCKET="${rw_sccache_bucket}" \
         SCCACHE_REGION="${rw_sccache_region}" \
@@ -229,6 +230,7 @@ if test -n "${gh_token:+x}" \
         reset_state;
         cp -ar /tmp/.aws ~/;
         GH_TOKEN="${gh_token}" \
+        AWS_IDP_URL="${aws_idp_url:-}" \
         AWS_ROLE_ARN="${aws_role_arn}" \
         SCCACHE_BUCKET="${rw_sccache_bucket}" \
         SCCACHE_REGION="${rw_sccache_region}" \
@@ -255,6 +257,7 @@ if test -n "${gh_token:+x}" \
         reset_state;
         write_bad_creds;
         GH_TOKEN="${gh_token}" \
+        AWS_IDP_URL="${aws_idp_url:-}" \
         AWS_ROLE_ARN="${aws_role_arn}" \
         SCCACHE_BUCKET="${rw_sccache_bucket}" \
         SCCACHE_REGION="${rw_sccache_region}" \
diff --git a/features/test/utils/ubuntu22.04.sh b/features/test/utils/ubuntu22.04.sh
index 58c928b86..9842d317c 100644
--- a/features/test/utils/ubuntu22.04.sh
+++ b/features/test/utils/ubuntu22.04.sh
@@ -214,6 +214,7 @@ if test -n "${gh_token:+x}" \
     no_creds_with_GH_TOKEN_AWS_ROLE_ARN_and_SCCACHE_BUCKET_should_generate_credentials() {
         reset_state;
         GH_TOKEN="${gh_token}" \
+        AWS_IDP_URL="${aws_idp_url:-}" \
         AWS_ROLE_ARN="${aws_role_arn}" \
         SCCACHE_BUCKET="${rw_sccache_bucket}" \
         SCCACHE_REGION="${rw_sccache_region}" \
@@ -229,6 +230,7 @@ if test -n "${gh_token:+x}" \
         reset_state;
         cp -ar /tmp/.aws ~/;
         GH_TOKEN="${gh_token}" \
+        AWS_IDP_URL="${aws_idp_url:-}" \
         AWS_ROLE_ARN="${aws_role_arn}" \
         SCCACHE_BUCKET="${rw_sccache_bucket}" \
         SCCACHE_REGION="${rw_sccache_region}" \
@@ -255,6 +257,7 @@ if test -n "${gh_token:+x}" \
         reset_state;
         write_bad_creds;
         GH_TOKEN="${gh_token}" \
+        AWS_IDP_URL="${aws_idp_url:-}" \
         AWS_ROLE_ARN="${aws_role_arn}" \
         SCCACHE_BUCKET="${rw_sccache_bucket}" \
         SCCACHE_REGION="${rw_sccache_region}" \
diff --git a/features/test/utils/ubuntu24.04.sh b/features/test/utils/ubuntu24.04.sh
index 58c928b86..9842d317c 100644
--- a/features/test/utils/ubuntu24.04.sh
+++ b/features/test/utils/ubuntu24.04.sh
@@ -214,6 +214,7 @@ if test -n "${gh_token:+x}" \
     no_creds_with_GH_TOKEN_AWS_ROLE_ARN_and_SCCACHE_BUCKET_should_generate_credentials() {
         reset_state;
         GH_TOKEN="${gh_token}" \
+        AWS_IDP_URL="${aws_idp_url:-}" \
         AWS_ROLE_ARN="${aws_role_arn}" \
         SCCACHE_BUCKET="${rw_sccache_bucket}" \
         SCCACHE_REGION="${rw_sccache_region}" \
@@ -229,6 +230,7 @@ if test -n "${gh_token:+x}" \
         reset_state;
         cp -ar /tmp/.aws ~/;
         GH_TOKEN="${gh_token}" \
+        AWS_IDP_URL="${aws_idp_url:-}" \
         AWS_ROLE_ARN="${aws_role_arn}" \
         SCCACHE_BUCKET="${rw_sccache_bucket}" \
         SCCACHE_REGION="${rw_sccache_region}" \
@@ -255,6 +257,7 @@ if test -n "${gh_token:+x}" \
         reset_state;
         write_bad_creds;
         GH_TOKEN="${gh_token}" \
+        AWS_IDP_URL="${aws_idp_url:-}" \
         AWS_ROLE_ARN="${aws_role_arn}" \
         SCCACHE_BUCKET="${rw_sccache_bucket}" \
         SCCACHE_REGION="${rw_sccache_region}" \

From 7e5e0733390f871dbbd041554aae0b2d49c4528d Mon Sep 17 00:00:00 2001
From: ptaylor <paul.e.taylor@me.com>
Date: Fri, 22 May 2026 11:32:56 -0700
Subject: [PATCH 07/10] install sccache client in post-start command

---
 .../src/utils/opt/devcontainer/bin/post-attach-command.sh   | 4 ----
 .../src/utils/opt/devcontainer/bin/post-start-command.sh    | 6 ++++++
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/features/src/utils/opt/devcontainer/bin/post-attach-command.sh b/features/src/utils/opt/devcontainer/bin/post-attach-command.sh
index 40b993066..7b7817194 100755
--- a/features/src/utils/opt/devcontainer/bin/post-attach-command.sh
+++ b/features/src/utils/opt/devcontainer/bin/post-attach-command.sh
@@ -17,10 +17,6 @@ if ! test -n "${SKIP_DEVCONTAINER_UTILS_POST_ATTACH_COMMAND:+x}"; then
 
     # Update sccache client configuration to enable/disable sccache-dist
     if test -n "${DEVCONTAINER_UTILS_ENABLE_SCCACHE_DIST:+x}"; then
-        devcontainer-utils-install-sccache                   \
-            --repo "${SCCACHE_REPOSITORY:-rapidsai/sccache}" \
-            --version "${SCCACHE_VERSION:-rapids}"           \
-        ;
         if test -n "${SCCACHE_DIST_AUTH_TOKEN:+x}"; then
             devcontainer-utils-init-sccache-dist        \
                 --enable-sccache-dist - <<< "           \
diff --git a/features/src/utils/opt/devcontainer/bin/post-start-command.sh b/features/src/utils/opt/devcontainer/bin/post-start-command.sh
index f1e37415f..650de9a2d 100755
--- a/features/src/utils/opt/devcontainer/bin/post-start-command.sh
+++ b/features/src/utils/opt/devcontainer/bin/post-start-command.sh
@@ -12,4 +12,10 @@ if ! test -n "${SKIP_DEVCONTAINER_UTILS_POST_START_COMMAND:+x}"; then
 
     # shellcheck disable=SC1091
     . devcontainer-utils-init-git;
+
+    # Install latest sccache client
+    devcontainer-utils-install-sccache                   \
+        --repo "${SCCACHE_REPOSITORY:-rapidsai/sccache}" \
+        --version "${SCCACHE_VERSION:-rapids}"           \
+    ;
 fi

From 941c49dcb86d6c58ce9240e3c8973e90f12bb739 Mon Sep 17 00:00:00 2001
From: ptaylor <paul.e.taylor@me.com>
Date: Fri, 22 May 2026 11:39:37 -0700
Subject: [PATCH 08/10] overlap installing sccache and changing file ownership

---
 .../utils/opt/devcontainer/bin/post-start-command.sh  | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/features/src/utils/opt/devcontainer/bin/post-start-command.sh b/features/src/utils/opt/devcontainer/bin/post-start-command.sh
index 650de9a2d..5c03c4b87 100755
--- a/features/src/utils/opt/devcontainer/bin/post-start-command.sh
+++ b/features/src/utils/opt/devcontainer/bin/post-start-command.sh
@@ -7,15 +7,16 @@ if ! test -n "${SKIP_DEVCONTAINER_UTILS_POST_START_COMMAND:+x}"; then
 
     # Fast parallel `chown -R`
     find ~/ /var/log/devcontainer-utils/ -not -user coder -print0 2>/dev/null \
-  | sudo xargs -0 -r -n1 -P"$(nproc --all)" chown "$(id -u):$(id -g)" 2>/dev/null \
- || true;
-
-    # shellcheck disable=SC1091
-    . devcontainer-utils-init-git;
+  | sudo xargs -0 -r -n1 -P"$(nproc --all)" chown "$(id -u):$(id -g)" 2>/dev/null &
 
     # Install latest sccache client
     devcontainer-utils-install-sccache                   \
         --repo "${SCCACHE_REPOSITORY:-rapidsai/sccache}" \
         --version "${SCCACHE_VERSION:-rapids}"           \
     ;
+
+    wait || :
+
+    # shellcheck disable=SC1091
+    . devcontainer-utils-init-git;
 fi

From 79188ca1329fe7e4e9ea5707cc7dc0c1180d07dc Mon Sep 17 00:00:00 2001
From: ptaylor <paul.e.taylor@me.com>
Date: Fri, 22 May 2026 11:57:29 -0700
Subject: [PATCH 09/10] ensure ~/.aws dir exists

---
 features/src/utils/opt/devcontainer/bin/creds/s3/gh/generate.sh | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/features/src/utils/opt/devcontainer/bin/creds/s3/gh/generate.sh b/features/src/utils/opt/devcontainer/bin/creds/s3/gh/generate.sh
index 48adce724..3009bb791 100755
--- a/features/src/utils/opt/devcontainer/bin/creds/s3/gh/generate.sh
+++ b/features/src/utils/opt/devcontainer/bin/creds/s3/gh/generate.sh
@@ -44,6 +44,8 @@ _creds_github_generate() {
         --idp-url "${AWS_IDP_URL:-https://token.gha-runners.nvidia.com}"
     );
 
+    mkdir -p ~/.aws;
+
     for org in "${user_orgs[@]}"; do
         generated_at="$(date '+%s')";
         if gh nv-gha-aws org "${org}" "${nv_gha_aws_args[@]}" >"${HOME}/.aws/credentials" 2>>/var/log/devcontainer-utils/creds-s3.log; then

From 98f33c4380020532a46f705461f90889d47ca9d1 Mon Sep 17 00:00:00 2001
From: ptaylor <paul.e.taylor@me.com>
Date: Fri, 22 May 2026 12:50:22 -0700
Subject: [PATCH 10/10] write pidfile if it doesn't exist, only update to
 latest sccache if DEVCONTAINER_UTILS_ENABLE_SCCACHE_DIST is set

---
 .../opt/devcontainer/bin/post-attach-command.sh      |  2 +-
 .../utils/opt/devcontainer/bin/post-start-command.sh | 12 +++++++-----
 .../utils/opt/devcontainer/bin/sccache/dist/init.sh  |  2 +-
 .../src/utils/opt/devcontainer/bin/sccache/start.sh  |  6 ++++--
 4 files changed, 13 insertions(+), 9 deletions(-)

diff --git a/features/src/utils/opt/devcontainer/bin/post-attach-command.sh b/features/src/utils/opt/devcontainer/bin/post-attach-command.sh
index 7b7817194..04e03025c 100755
--- a/features/src/utils/opt/devcontainer/bin/post-attach-command.sh
+++ b/features/src/utils/opt/devcontainer/bin/post-attach-command.sh
@@ -35,6 +35,6 @@ if ! test -n "${SKIP_DEVCONTAINER_UTILS_POST_ATTACH_COMMAND:+x}"; then
         fi
     elif command -V sccache >/dev/null 2>&1; then
         # Start the sccache client
-        devcontainer-utils-start-sccache --kill-all;
+        devcontainer-utils-start-sccache;
     fi
 fi
diff --git a/features/src/utils/opt/devcontainer/bin/post-start-command.sh b/features/src/utils/opt/devcontainer/bin/post-start-command.sh
index 5c03c4b87..268d6dd2f 100755
--- a/features/src/utils/opt/devcontainer/bin/post-start-command.sh
+++ b/features/src/utils/opt/devcontainer/bin/post-start-command.sh
@@ -9,11 +9,13 @@ if ! test -n "${SKIP_DEVCONTAINER_UTILS_POST_START_COMMAND:+x}"; then
     find ~/ /var/log/devcontainer-utils/ -not -user coder -print0 2>/dev/null \
   | sudo xargs -0 -r -n1 -P"$(nproc --all)" chown "$(id -u):$(id -g)" 2>/dev/null &
 
-    # Install latest sccache client
-    devcontainer-utils-install-sccache                   \
-        --repo "${SCCACHE_REPOSITORY:-rapidsai/sccache}" \
-        --version "${SCCACHE_VERSION:-rapids}"           \
-    ;
+    if test -n "${DEVCONTAINER_UTILS_ENABLE_SCCACHE_DIST:+x}"; then
+        # Install latest sccache client
+        devcontainer-utils-install-sccache                   \
+            --repo "${SCCACHE_REPOSITORY:-rapidsai/sccache}" \
+            --version "${SCCACHE_VERSION:-rapids}"           \
+        ;
+    fi
 
     wait || :
 
diff --git a/features/src/utils/opt/devcontainer/bin/sccache/dist/init.sh b/features/src/utils/opt/devcontainer/bin/sccache/dist/init.sh
index 1ea95f3c9..5964c39fc 100755
--- a/features/src/utils/opt/devcontainer/bin/sccache/dist/init.sh
+++ b/features/src/utils/opt/devcontainer/bin/sccache/dist/init.sh
@@ -75,7 +75,7 @@ _init_sccache_dist() {
     done
 
     # Restart the sccache client with the new configuration
-    devcontainer-utils-start-sccache --kill-all;
+    devcontainer-utils-start-sccache;
 
     # Verify sccache-dist status and configuration
     if sccache --dist-status 2>/dev/null | jq -er '.SchedulerStatus? != null' >/dev/null 2>&1; then
diff --git a/features/src/utils/opt/devcontainer/bin/sccache/start.sh b/features/src/utils/opt/devcontainer/bin/sccache/start.sh
index e07cfe719..e98ff2800 100755
--- a/features/src/utils/opt/devcontainer/bin/sccache/start.sh
+++ b/features/src/utils/opt/devcontainer/bin/sccache/start.sh
@@ -65,8 +65,10 @@ _start_sccache() {
         sccache --start-server 1>&2 2>/dev/null \
       | tee "$logfile";
 
-        # Wait till the pidfile exists and is not empty
-        until test -s "${pidfile}"; do sleep 1; done
+        # If the pidfile doesn't exist, write it
+        if ! test -s "${pidfile}"; then
+            pgrep sccache | sort -n | head -n1 | tee "${pidfile}" >/dev/null;
+        fi
 
         # Increase the open file limit so users can do `make -j(ulimit -n)`
         prlimit --nofile=$(ulimit -Hn):$(ulimit -Hn) --pid "$(cat "${pidfile}")";