Stamp delta bundles with base commit SHA and exclude stale artifact DBs

gradlecache/extract_default.go

@@ -225,6 +225,10 @@ func processEntry(
 		return err
 	}
 
+	if name == deltaBaseCommitEntry {
+		return nil
+	}
+
 	target := targetFn(name)
 
 	switch hdr.Typeflag {

gradlecache/ghacache.go

@@ -329,7 +329,7 @@ func (g *ghaCacheStore) createAndFinalize(ctx context.Context, commit, cacheKey
 // put uploads a cache entry from a ReadSeeker of known size.
 // For small bundles (≤ 1 block), uses a single PUT. For larger bundles,
 // uses parallel Azure Block Blob upload (Put Block + Put Block List).
-func (g *ghaCacheStore) put(ctx context.Context, commit, cacheKey string, r io.ReadSeeker, size int64) error {
+func (g *ghaCacheStore) put(ctx context.Context, commit, cacheKey string, r io.ReadSeeker, size int64, _ map[string]string) error {
 	return g.createAndFinalize(ctx, commit, cacheKey, size, func(signedURL string) error {
 		if size <= ghaBlockSize {
 			return g.azurePutSingle(ctx, signedURL, r, size)

gradlecache/gradlecache_test.go

@@ -59,6 +59,8 @@ func TestIsDeltaExcluded(t *testing.T) {
 	excluded := []string{
 		"fileHashes",
 		"module-metadata.bin",
+		"module-artifact.bin",
+		"resource-at-url.bin",
 	}
 	for _, name := range excluded {
 		if !IsDeltaExcluded(name) {
@@ -806,6 +808,67 @@ func TestDeltaTarZstdRoundTrip(t *testing.T) {
 	}
 }
 
+func TestStampedDeltaRoundTrip(t *testing.T) {
+	baseCommit := "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+	gradleHome := t.TempDir()
+	cachesDir := filepath.Join(gradleHome, "caches", "modules-2")
+	must(t, os.MkdirAll(cachesDir, 0o755))
+	must(t, os.WriteFile(filepath.Join(cachesDir, "delta-file.bin"), []byte("delta"), 0o644))
+
+	// Create a stamped delta archive.
+	var buf bytes.Buffer
+	must(t, createStampedDeltaTarZstdMulti(&buf, baseCommit,
+		DeltaSource{BaseDir: gradleHome, RelPaths: []string{"caches/modules-2/delta-file.bin"}}))
+
+	// ReadDeltaBaseCommit should return the embedded stamp.
+	r := bytes.NewReader(buf.Bytes())
+	got, err := ReadDeltaBaseCommit(r)
+	must(t, err)
+	if got != baseCommit {
+		t.Fatalf("ReadDeltaBaseCommit = %q, want %q", got, baseCommit)
+	}
+
+	// The file should still extract correctly (stamp entry is skipped).
+	dstDir := t.TempDir()
+	must(t, extractTarZstd(context.Background(), bytes.NewReader(buf.Bytes()), dstDir))
+
+	data, err := os.ReadFile(filepath.Join(dstDir, "caches", "modules-2", "delta-file.bin"))
+	must(t, err)
+	if string(data) != "delta" {
+		t.Fatalf("extracted content = %q, want %q", string(data), "delta")
+	}
+
+	// __base_commit__ should NOT exist as a file on disk.
+	if _, err := os.Stat(filepath.Join(dstDir, deltaBaseCommitEntry)); err == nil {
+		t.Fatal("__base_commit__ should not be extracted as a real file")
+	}
+}
+
+func TestReadDeltaBaseCommitMissing(t *testing.T) {
+	// An unstamped delta should return empty string.
+	var buf bytes.Buffer
+	must(t, CreateDeltaTarZstdMulti(&buf,
+		DeltaSource{BaseDir: t.TempDir(), RelPaths: nil}))
+
+	r := bytes.NewReader(buf.Bytes())
+	got, err := ReadDeltaBaseCommit(r)
+	must(t, err)
+	if got != "" {
+		t.Fatalf("ReadDeltaBaseCommit on unstamped delta = %q, want empty", got)
+	}
+}
+
+func TestBaseCommitFileRoundTrip(t *testing.T) {
+	dir := t.TempDir()
+	sha := "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
+	must(t, writeBaseCommitFile(dir, sha))
+	got, err := readBaseCommitFile(dir)
+	must(t, err)
+	if got != sha {
+		t.Fatalf("readBaseCommitFile = %q, want %q", got, sha)
+	}
+}
+
 func TestSaveDeltaDefaultsProjectDirToWorkingDirectory(t *testing.T) {
 	ctx := context.Background()
 	gradleHome := t.TempDir()
@@ -817,6 +880,8 @@ func TestSaveDeltaDefaultsProjectDirToWorkingDirectory(t *testing.T) {
 
 	markerPath := filepath.Join(gradleHome, ".cache-restore-marker")
 	must(t, touchMarkerFile(markerPath))
+	// Write a base commit file so SaveDelta can stamp the delta.
+	must(t, writeBaseCommitFile(gradleHome, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"))
 
 	// Sleep to ensure files created below have a strictly newer mtime than
 	// the marker. On Linux ext4 the mtime granularity is 1 ms, but rapid

gradlecache/restore.go

@@ -386,6 +386,15 @@ func Restore(ctx context.Context, cfg RestoreConfig) error {
 				deltaCh <- deltaResult{}
 				return
 			}
+			// Check S3 metadata for base-commit mismatch before downloading.
+			if metaBase := deltaInfo.Metadata[deltaBaseCommitMetaKey]; metaBase != "" && metaBase != hitCommit {
+				log.Info("skipping delta: built on different base",
+					"delta_base", metaBase[:min(8, len(metaBase))],
+					"current_base", hitCommit[:min(8, len(hitCommit))])
+				deltaCh <- deltaResult{}
+				return
+			}
+
 			log.Info("downloading delta bundle", "branch", cfg.Branch)
 			dlStart := time.Now()
 			body, err := store.get(ctx, dc, cfg.CacheKey, deltaInfo)
@@ -412,6 +421,19 @@ func Restore(ctx context.Context, cfg RestoreConfig) error {
 				deltaCh <- deltaResult{err: errors.Wrap(err, "rewind delta temp file")}
 				return
 			}
+
+			// Check tar stamp for base-commit mismatch (covers stores without metadata).
+			tarBase, tarErr := ReadDeltaBaseCommit(tmp)
+			if tarErr == nil && tarBase != "" && tarBase != hitCommit {
+				log.Info("skipping delta: tar stamp shows different base",
+					"delta_base", tarBase[:min(8, len(tarBase))],
+					"current_base", hitCommit[:min(8, len(hitCommit))])
+				tmp.Close()           //nolint:errcheck,gosec
+				os.Remove(tmp.Name()) //nolint:errcheck,gosec
+				deltaCh <- deltaResult{}
+				return
+			}
+
 			deltaCh <- deltaResult{tmpFile: tmp, dlStart: dlStart, n: cb.n, eofAt: cb.eofAt}
 		}()
 	}
@@ -489,6 +511,10 @@ func Restore(ctx context.Context, cfg RestoreConfig) error {
 		cfg.Metrics.Distribution("gradle_cache.restore_base.speed_mbps", mbps, "cache_key:"+cfg.CacheKey)
 	}
 
+	if err := writeBaseCommitFile(cfg.GradleUserHome, hitCommit); err != nil {
+		log.Warn("could not write base commit file", "err", err)
+	}
+
 	if err := touchMarkerFile(filepath.Join(cfg.GradleUserHome, ".cache-restore-marker")); err != nil {
 		log.Warn("could not write restore marker", "err", err)
 	}

gradlecache/s3.go

@@ -63,8 +63,9 @@ func newS3Client(region string) (*s3Client, error) {
 }
 
 type s3ObjInfo struct {
-	Size int64
-	ETag string
+	Size     int64
+	ETag     string
+	Metadata map[string]string
 }
 
 func (c *s3Client) stat(ctx context.Context, bucket, key string) (s3ObjInfo, error) {
@@ -82,10 +83,20 @@ func (c *s3Client) stat(ctx context.Context, bucket, key string) (s3ObjInfo, err
 	if resp.StatusCode != http.StatusOK {
 		return s3ObjInfo{}, errors.Errorf("status %d", resp.StatusCode)
 	}
-	return s3ObjInfo{
+	info := s3ObjInfo{
 		Size: resp.ContentLength,
 		ETag: resp.Header.Get("ETag"),
-	}, nil
+	}
+	const metaPrefix = "X-Amz-Meta-"
+	for k, vs := range resp.Header {
+		if len(vs) > 0 && len(k) > len(metaPrefix) && strings.EqualFold(k[:len(metaPrefix)], metaPrefix) {
+			if info.Metadata == nil {
+				info.Metadata = make(map[string]string)
+			}
+			info.Metadata[strings.ToLower(k[len(metaPrefix):])] = vs[0]
+		}
+	}
+	return info, nil
 }
 
 func (c *s3Client) get(ctx context.Context, bucket, key string, info s3ObjInfo) (io.ReadCloser, error) {