Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 43 additions & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,7 @@ tui-term = "0.3.1"
twox-hash = "2.1.1"
uuid = "1.18.1"
vec1 = "1.12.1"
zstd = "0.13"
vite_glob = { path = "crates/vite_glob" }
vite_graph_ser = { path = "crates/vite_graph_ser" }
vite_path = { path = "crates/vite_path" }
Expand Down
3 changes: 3 additions & 0 deletions crates/vite_task/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ rustc-hash = { workspace = true }
serde = { workspace = true, features = ["derive", "rc"] }
serde_json = { workspace = true }
thiserror = { workspace = true }
tar = { workspace = true }
tokio = { workspace = true, features = ["rt-multi-thread", "io-std", "io-util", "macros", "sync"] }
tokio-util = { workspace = true }
tracing = { workspace = true }
Expand All @@ -41,7 +42,9 @@ vite_str = { workspace = true }
vite_task_graph = { workspace = true }
vite_task_plan = { workspace = true }
vite_workspace = { workspace = true }
uuid = { workspace = true, features = ["v4"] }
wax = { workspace = true }
zstd = { workspace = true }

[dev-dependencies]
tempfile = { workspace = true }
Expand Down
4 changes: 2 additions & 2 deletions crates/vite_task/docs/task-cache.md
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@ The cache entry key uniquely identifies a command execution context:
```rust
pub struct CacheEntryKey {
pub spawn_fingerprint: SpawnFingerprint,
pub input_config: ResolvedInputConfig,
pub input_config: ResolvedGlobConfig,
}
```

Expand Down Expand Up @@ -303,7 +303,7 @@ Cache entries are serialized using `bincode` for efficient storage.
│ ────────────────────── │
│ CacheEntryKey { │
│ spawn_fingerprint: SpawnFingerprint { ... }, │
│ input_config: ResolvedInputConfig { ... }, │
│ input_config: ResolvedGlobConfig { ... }, │
│ } │
│ ExecutionCacheKey::UserTask { │
│ task_name: "build", │
Expand Down
61 changes: 61 additions & 0 deletions crates/vite_task/src/session/cache/archive.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
//! Output archive creation and extraction using tar + zstd compression.

use std::fs::File;

use vite_path::{AbsolutePath, RelativePathBuf};

/// Create a tar.zst archive from workspace-relative output file paths.
///
/// Files that no longer exist are silently skipped (the task may delete
/// temporary files during execution).
///
/// # Errors
///
/// Returns an error if creating the archive file or adding entries fails.
pub fn create_output_archive(
workspace_root: &AbsolutePath,
output_files: &[RelativePathBuf],
archive_path: &AbsolutePath,
) -> anyhow::Result<()> {
let file = File::create(archive_path.as_path())?;
let encoder = zstd::Encoder::new(file, 0)?.auto_finish();
let mut builder = tar::Builder::new(encoder);

for rel_path in output_files {
let abs_path = workspace_root.join(rel_path);
// Skip files that no longer exist (task may delete temp files)
if !abs_path.as_path().exists() {
continue;
}
let metadata = std::fs::metadata(abs_path.as_path())?;
if metadata.is_file() {
let mut file = File::open(abs_path.as_path())?;
let mut header = tar::Header::new_gnu();
header.set_metadata(&metadata);
header.set_cksum();
builder.append_data(&mut header, rel_path.as_str(), &mut file)?;
}
}

builder.finish()?;
Ok(())
}

/// Extract a tar.zst archive, restoring files relative to workspace root.
///
/// Parent directories are created automatically. Existing files are overwritten.
///
/// # Errors
///
/// Returns an error if opening the archive or extracting entries fails.
pub fn extract_output_archive(
workspace_root: &AbsolutePath,
archive_path: &AbsolutePath,
) -> anyhow::Result<()> {
let file = File::open(archive_path.as_path())?;
let decoder = zstd::Decoder::new(file)?;
let mut archive = tar::Archive::new(decoder);

archive.unpack(workspace_root.as_path())?;
Ok(())
}
1 change: 1 addition & 0 deletions crates/vite_task/src/session/cache/display.rs
Original file line number Diff line number Diff line change
Expand Up @@ -174,6 +174,7 @@ pub fn format_cache_status_inline(cache_status: &CacheStatus) -> Option<Str> {
}
}
FingerprintMismatch::InputConfig => "input configuration changed",
FingerprintMismatch::OutputConfig => "output configuration changed",
FingerprintMismatch::InputChanged { kind, path } => {
let desc = format_input_change_str(*kind, path.as_str());
return Some(vite_str::format!("○ cache miss: {desc}, executing"));
Expand Down
43 changes: 33 additions & 10 deletions crates/vite_task/src/session/cache/mod.rs
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
//! Execution cache for storing and retrieving cached command results.

pub mod archive;
pub mod display;

use std::{collections::BTreeMap, fmt::Display, fs::File, io::Write, sync::Arc, time::Duration};
Expand All @@ -15,7 +16,8 @@ use rusqlite::{Connection, OptionalExtension as _, config::DbConfig};
use serde::{Deserialize, Serialize};
use tokio::sync::Mutex;
use vite_path::{AbsolutePath, RelativePathBuf};
use vite_task_graph::config::ResolvedInputConfig;
use vite_str::Str;
use vite_task_graph::config::ResolvedGlobConfig;
use vite_task_plan::cache_metadata::{CacheMetadata, ExecutionCacheKey, SpawnFingerprint};

use super::execute::{fingerprint::PostRunFingerprint, spawn::StdOutput};
Expand All @@ -38,14 +40,18 @@ pub struct CacheEntryKey {
pub spawn_fingerprint: SpawnFingerprint,
/// Resolved input configuration that affects cache behavior.
/// Glob patterns are workspace-root-relative.
pub input_config: ResolvedInputConfig,
pub input_config: ResolvedGlobConfig,
/// Resolved output configuration that affects cache restoration.
/// Glob patterns are workspace-root-relative.
pub output_config: ResolvedGlobConfig,
}

impl CacheEntryKey {
fn from_metadata(cache_metadata: &CacheMetadata) -> Self {
Self {
spawn_fingerprint: cache_metadata.spawn_fingerprint.clone(),
input_config: cache_metadata.input_config.clone(),
output_config: cache_metadata.output_config.clone(),
}
}
}
Expand All @@ -64,6 +70,9 @@ pub struct CacheEntryValue {
/// Path is relative to workspace root, value is `xxHash3_64` of file content.
/// Stored in the value (not the key) so changes can be detected and reported.
pub globbed_inputs: BTreeMap<RelativePathBuf, u64>,
/// Filename of the output archive (e.g. `{uuid}.tar.zst`) stored alongside
/// `cache.db` in the cache directory. `None` if no output files were produced.
pub output_archive: Option<Str>,
}

#[derive(Debug)]
Expand Down Expand Up @@ -105,6 +114,8 @@ pub enum FingerprintMismatch {
},
/// Found a previous cache entry key for the same task, but `input_config` differs.
InputConfig,
/// Found a previous cache entry key for the same task, but `output_config` differs.
OutputConfig,

InputChanged {
kind: InputChangeKind,
Expand All @@ -121,6 +132,9 @@ impl Display for FingerprintMismatch {
Self::InputConfig => {
write!(f, "input configuration changed")
}
Self::OutputConfig => {
write!(f, "output configuration changed")
}
Self::InputChanged { kind, path } => {
write!(f, "{}", display::format_input_change_str(*kind, path.as_str()))
}
Expand Down Expand Up @@ -168,16 +182,16 @@ impl ExecutionCache {
"CREATE TABLE task_fingerprints (key BLOB PRIMARY KEY, value BLOB);",
(),
)?;
conn.execute("PRAGMA user_version = 10", ())?;
conn.execute("PRAGMA user_version = 11", ())?;
}
1..=9 => {
1..=10 => {
// old internal db version. reset
conn.set_db_config(DbConfig::SQLITE_DBCONFIG_RESET_DATABASE, true)?;
conn.execute("VACUUM", ())?;
conn.set_db_config(DbConfig::SQLITE_DBCONFIG_RESET_DATABASE, false)?;
}
10 => break, // current version
11.. => {
11 => break, // current version
12.. => {
return Err(anyhow::anyhow!("Unrecognized database version: {user_version}"));
}
}
Expand Down Expand Up @@ -233,11 +247,20 @@ impl ExecutionCache {
self.get_cache_key_by_execution_key(execution_cache_key).await?
{
// Destructure to ensure we handle all fields when new ones are added
let CacheEntryKey { spawn_fingerprint: old_spawn_fingerprint, input_config: _ } =
old_cache_key;
let CacheEntryKey {
spawn_fingerprint: old_spawn_fingerprint,
input_config: old_input_config,
output_config: old_output_config,
} = old_cache_key;
let mismatch = if old_spawn_fingerprint == *spawn_fingerprint {
// spawn fingerprint is the same but input_config or glob_base changed
FingerprintMismatch::InputConfig
// spawn fingerprint is the same but input_config or output_config changed
if old_input_config != cache_metadata.input_config {
FingerprintMismatch::InputConfig
} else if old_output_config != cache_metadata.output_config {
FingerprintMismatch::OutputConfig
} else {
FingerprintMismatch::InputConfig
}
} else {
FingerprintMismatch::SpawnFingerprint {
old: old_spawn_fingerprint,
Expand Down
52 changes: 52 additions & 0 deletions crates/vite_task/src/session/execute/glob_inputs.rs
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,58 @@ pub fn compute_globbed_inputs(
Ok(result)
}

/// Collect file paths matching positive globs, filtered by negative globs.
///
/// Like [`compute_globbed_inputs`] but only collects paths (no hashing).
/// Used for determining which output files to archive.
pub fn collect_glob_paths(
workspace_root: &AbsolutePath,
positive_globs: &std::collections::BTreeSet<Str>,
negative_globs: &std::collections::BTreeSet<Str>,
) -> anyhow::Result<Vec<RelativePathBuf>> {
if positive_globs.is_empty() {
return Ok(Vec::new());
}

let negatives: Vec<Glob<'static>> = negative_globs
.iter()
.map(|p| Ok(Glob::new(p.as_str())?.into_owned()))
.collect::<anyhow::Result<_>>()?;
let negation = wax::any(negatives)?;

let mut result = Vec::new();

for pattern in positive_globs {
let glob = Glob::new(pattern.as_str())?.into_owned();
let walk = glob.walk(workspace_root.as_path());
for entry in walk.not(negation.clone())? {
let entry = match entry {
Ok(entry) => entry,
Err(err) => {
let io_err: io::Error = err.into();
if io_err.kind() == io::ErrorKind::NotFound {
continue;
}
return Err(io_err.into());
}
};
if !entry.file_type().is_file() {
continue;
}
let path = entry.path();
let Some(stripped) = path.strip_prefix(workspace_root.as_path()).ok() else {
continue;
};
let relative = RelativePathBuf::new(stripped)?;
result.push(relative);
}
}

result.sort();
result.dedup();
Ok(result)
}

#[expect(clippy::disallowed_types, reason = "receives std::path::Path from wax glob walker")]
fn hash_file_content(path: &std::path::Path) -> io::Result<u64> {
super::hash::hash_content(io::BufReader::new(File::open(path)?))
Expand Down
Loading
Loading