Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 24 additions & 12 deletions anneal/Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 2 additions & 1 deletion anneal/v2/toolchain-config/examples/static-toml/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ fn main() {

match cli.command {
Commands::Setup => {
// TODO: Probably use a flag, not an environment variable to activate override.
let local_override = if std::env::var("__TOOLCHAIN_EXAMPLE_STATIC_TOML").is_ok() {
println!("Local testing override active. Assembling mock toolchain archive...");
let manifest_dir = std::env::var("CARGO_MANIFEST_DIR")
Expand All @@ -52,7 +53,7 @@ fn main() {
assert!(status.success(), "build-toolchain.sh script failed");

let archive_path = std::path::Path::new(&manifest_dir).join("toolchain.tar.zst");
Some(LocalOverride::Archive(archive_path))
Some(LocalOverride::<TarZstLibraryExtractor>::archive(archive_path))
} else {
None
};
Expand Down
Binary file not shown.
156 changes: 91 additions & 65 deletions anneal/v2/toolchain-config/src/lib.rs
Original file line number Diff line number Diff line change
@@ -1,50 +1,59 @@
/// Setup configuration binding external platform environments and baseline paths.
/// Setup configuration specifying platform, remote source, and remote checksum.
#[derive(Debug, Clone)]
pub struct Config<'a, E, D> {
pub os: &'a str,
pub arch: &'a str,
pub url: &'a str,
pub sha256: &'a [u8],
pub checksum: &'a [u8],
_extractor: std::marker::PhantomData<E>,
_digest: std::marker::PhantomData<D>,
}

/// Optional runtime override directing installation from local dev builds instead of remote hosts.
#[derive(Debug, Clone)]
pub enum LocalOverride {
/// Local toolchain definition that overrides remote specified in [`Config`].
pub enum LocalOverride<E: Extractor = NoExtractor> {
Dir(std::path::PathBuf),
Archive(std::path::PathBuf),
Archive((std::path::PathBuf, std::marker::PhantomData<E>)),
}

impl<E: Extractor> LocalOverride<E> {
pub fn dir(path: std::path::PathBuf) -> Self {
Self::Dir(path)
}

pub fn archive(path: std::path::PathBuf) -> Self {
Self::Archive((path, std::marker::PhantomData))
}
}

impl<'a, E: Extractor, D: digest::Digest> Config<'a, E, D> {
/// Instantiates static toolchain parameters auto-detecting current runtime OS and Architecture.
pub fn new(url: &'a str, sha256: &'a [u8]) -> Self {
pub fn new(url: &'a str, checksum: &'a [u8]) -> Self {
Self {
os: std::env::consts::OS,
arch: std::env::consts::ARCH,
url,
sha256,
checksum,
_extractor: std::marker::PhantomData,
_digest: std::marker::PhantomData,
}
}

/// Explicitly overrides target platform parameters for specialized configurations.
pub fn new_platform(os: &'a str, arch: &'a str, url: &'a str, sha256: &'a [u8]) -> Self {
pub fn new_platform(os: &'a str, arch: &'a str, url: &'a str, checksum: &'a [u8]) -> Self {
Self {
os,
arch,
url,
sha256,
checksum,
_extractor: std::marker::PhantomData,
_digest: std::marker::PhantomData,
}
}

/// Resolves the deterministic subdirectory path containing the verified toolchain files.
pub fn toolchain_dir(&self, root: &std::path::Path) -> std::path::PathBuf {
let expected_hex = encode_hex(self.sha256);
root.join(&expected_hex[..6])
let expected_hex = encode_hex(self.checksum);
root.join(&format!("{}-{}-{}", self.os, self.arch, &expected_hex[..12]))
}
}

Expand All @@ -53,13 +62,26 @@ impl<'a, E: Extractor, D: digest::Digest> Config<'a, E, D> {
pub trait Extractor {
/// Instantiates a new instance of this extractor type.
///
/// Instantiated and used for extracting archives downloaded from [`Config::url`] when
/// no local override is specified during invocation.
/// Instantiated and used for extracting archives downloaded from [`Config::url`] or archives
/// designated by [`LocalOverride::Archive`].
fn new() -> Self;

/// Unpacks stream bytes directly into the specified target directory synchronously on the calling thread.
fn extract(&self, src: &mut dyn std::io::Read, dst: &std::path::Path) -> std::io::Result<()>;
}

/// Marker type used by [`LocalOverride::Dir`] (which implies no archive extraction).
pub struct NoExtractor;

impl Extractor for NoExtractor {
fn new() -> Self {
panic!("NoExtractor is not constructible");
}
fn extract(&self, _src: &mut dyn std::io::Read, _dst: &std::path::Path) -> std::io::Result<()> {
panic!("Attempt to extract using NoExtractor");
}
}

/// A concrete [`Extractor`] implementation delegating archive extraction duties entirely in-process
/// via synchronous streaming pipelines using the `tar` and `zstd` library crates.
pub struct TarZstLibraryExtractor;
Expand All @@ -84,17 +106,15 @@ fn encode_hex(bytes: &[u8]) -> String {
s
}

/// [`std::io::Read`] abstraction that encapsulates read-and-update-hasher.
struct HashReader<R, D> {
inner: R,
hasher: D,
}

impl<R: std::io::Read, D: digest::Digest> HashReader<R, D> {
fn new(inner: R) -> Self {
Self {
inner,
hasher: D::new(),
}
Self { inner, hasher: D::new() }
}

fn finalize(self) -> Vec<u8> {
Expand Down Expand Up @@ -187,7 +207,8 @@ fn link_or_copy_dir(src: &std::path::Path, dst: &std::path::Path) -> std::io::Re

fn setup_from_directory(src: &std::path::Path, dst: &std::path::Path) -> Result<(), String> {
let parent = dst.parent().expect("toolchains directory has parent");
std::fs::create_dir_all(parent).map_err(|e| format!("Failed to create toolchain parent directory: {e}"))?;
std::fs::create_dir_all(parent)
.map_err(|e| format!("Failed to create toolchain parent directory: {e}"))?;
let old_dir = if dst.symlink_metadata().is_ok() {
let old = tempfile::Builder::new()
.prefix("setup-old-")
Expand All @@ -208,43 +229,43 @@ fn setup_from_directory(src: &std::path::Path, dst: &std::path::Path) -> Result<
Ok(())
}

fn setup_inner<E: Extractor, D: digest::Digest>(
config: &Config<'_, E, D>,
local_override: Option<LocalOverride>,
toolchain_dir: std::path::PathBuf,
fn setup_inner<CE: Extractor, D: digest::Digest, LE: Extractor>(
config: &Config<'_, CE, D>,
local_override: Option<LocalOverride<LE>>,
toolchain_root: std::path::PathBuf,
fetcher: impl FnOnce(&str) -> Result<Box<dyn std::io::Read>, String>,
) -> Result<(), String> {
let target_dir = config.toolchain_dir(&toolchain_dir);
let expected_hex = encode_hex(config.sha256);
let toolchain_dir = config.toolchain_dir(&toolchain_root);
let expected_hex = encode_hex(config.checksum);

if let Some(override_src) = local_override {
match override_src {
LocalOverride::Archive(path) => {
LocalOverride::Archive((path, _)) => {
log::warn!(
"Toolchain contents from local archive may not match expected toolchain hash/version number."
);
let extractor = E::new();
let extractor = LE::new();
let file = std::fs::File::open(path)
.map_err(|e| format!("Failed to open local archive: {e}"))?;
setup_from_archive::<E, D>(file, &target_dir, &extractor)
setup_from_archive::<LE, D>(file, &toolchain_dir, &extractor)
.map_err(|e| format!("Failed to extract archive: {e}"))?;
}
LocalOverride::Dir(path) => {
log::warn!(
"Toolchain contents from local directory may not match expected toolchain hash/version number."
);
setup_from_directory(&path, &target_dir)?;
setup_from_directory(&path, &toolchain_dir)?;
}
}
} else {
let extractor = E::new();
let extractor = CE::new();
let response = fetcher(config.url)?;

let actual_hash = setup_from_archive::<E, D>(response, &target_dir, &extractor)
let actual_hash = setup_from_archive::<CE, D>(response, &toolchain_dir, &extractor)
.map_err(|e| format!("Failed to extract downloaded archive: {e}"))?;

if actual_hash.as_slice() != config.sha256 {
let _ = std::fs::remove_dir_all(&target_dir);
if actual_hash.as_slice() != config.checksum {
let _ = std::fs::remove_dir_all(&toolchain_dir);
return Err(format!(
"Checksum mismatch for downloaded archive. Expected {}, got {}",
expected_hex,
Expand All @@ -263,14 +284,14 @@ fn setup_inner<E: Extractor, D: digest::Digest>(
///
/// When no local override is specified, the configured [`Extractor`] type `E` is instantiated
/// via [`Extractor::new`] and used to extract the downloaded toolchain archive stream.
pub fn setup<E: Extractor, D: digest::Digest>(
config: &Config<'_, E, D>,
local_override: Option<LocalOverride>,
toolchain_dir: std::path::PathBuf,
pub fn setup<CE: Extractor, D: digest::Digest, LE: Extractor>(
config: &Config<'_, CE, D>,
local_override: Option<LocalOverride<LE>>,
toolchain_root: std::path::PathBuf,
) -> Result<(), String> {
setup_inner(config, local_override, toolchain_dir, |url| {
let response = reqwest::blocking::get(url)
.map_err(|e| format!("Failed to download archive: {e}"))?;
setup_inner(config, local_override, toolchain_root, |url| {
let response =
reqwest::blocking::get(url).map_err(|e| format!("Failed to download archive: {e}"))?;
let response = response
.error_for_status()
.map_err(|e| format!("HTTP error downloading archive: {e}"))?;
Expand Down Expand Up @@ -332,7 +353,12 @@ mod tests {

let dst = temp.path().join("dst");
let file = std::fs::File::open(&archive_path).unwrap();
let hash = setup_from_archive::<TarZstLibraryExtractor, sha2::Sha256>(file, &dst, &TarZstLibraryExtractor).unwrap();
let hash = setup_from_archive::<TarZstLibraryExtractor, sha2::Sha256>(
file,
&dst,
&TarZstLibraryExtractor,
)
.unwrap();

assert_eq!(hash, compute_sha256(&archive_path));
assert_eq!(std::fs::read_to_string(dst.join("data.txt")).unwrap(), "archive_content");
Expand All @@ -346,12 +372,15 @@ mod tests {
std::fs::write(src.join("test.txt"), "local_dir").unwrap();

let expected_hash = [1u8; 32];
let config = Config::<TarZstLibraryExtractor, sha2::Sha256>::new("http://example.com", &expected_hash);
let config = Config::<TarZstLibraryExtractor, sha2::Sha256>::new(
"http://example.com",
&expected_hash,
);
let target_dir = config.toolchain_dir(temp.path());

setup_inner(
&config,
Some(LocalOverride::Dir(src)),
Some(LocalOverride::<NoExtractor>::Dir(src)),
temp.path().to_path_buf(),
|_| unreachable!(),
)
Expand All @@ -371,12 +400,15 @@ mod tests {
create_test_archive(&src, &archive_path);

let expected_hash = [2u8; 32];
let config = Config::<TarZstLibraryExtractor, sha2::Sha256>::new("http://example.com", &expected_hash);
let config = Config::<TarZstLibraryExtractor, sha2::Sha256>::new(
"http://example.com",
&expected_hash,
);
let target_dir = config.toolchain_dir(temp.path());

setup_inner(
&config,
Some(LocalOverride::Archive(archive_path)),
Some(LocalOverride::<NoExtractor>::archive(archive_path)),
temp.path().to_path_buf(),
|_| unreachable!(),
)
Expand All @@ -396,19 +428,15 @@ mod tests {
create_test_archive(&src, &archive_path);

let actual_hash = compute_sha256(&archive_path);
let config = Config::<TarZstLibraryExtractor, sha2::Sha256>::new("http://example.com", &actual_hash);
let config =
Config::<TarZstLibraryExtractor, sha2::Sha256>::new("http://example.com", &actual_hash);
let target_dir = config.toolchain_dir(temp.path());

let archive_path_clone = archive_path.clone();
setup_inner(
&config,
None,
temp.path().to_path_buf(),
move |_url| {
let file = std::fs::File::open(&archive_path_clone).unwrap();
Ok(Box::new(file))
},
)
setup_inner::<_, _, NoExtractor>(&config, None, temp.path().to_path_buf(), move |_url| {
let file = std::fs::File::open(&archive_path_clone).unwrap();
Ok(Box::new(file))
})
.unwrap();

assert_eq!(std::fs::read_to_string(target_dir.join("test.txt")).unwrap(), "remote_content");
Expand All @@ -428,19 +456,17 @@ mod tests {
let mut expected_hash = actual_hash;
expected_hash[0] ^= 1; // invalidate checksum

let config = Config::<TarZstLibraryExtractor, sha2::Sha256>::new("http://example.com", &expected_hash);
let config = Config::<TarZstLibraryExtractor, sha2::Sha256>::new(
"http://example.com",
&expected_hash,
);
let target_dir = config.toolchain_dir(temp.path());

let archive_path_clone = archive_path.clone();
let res = setup_inner(
&config,
None,
temp.path().to_path_buf(),
move |_url| {
let file = std::fs::File::open(&archive_path_clone).unwrap();
Ok(Box::new(file))
},
);
let res = setup_inner::<_, _, NoExtractor>(&config, None, temp.path().to_path_buf(), move |_url| {
let file = std::fs::File::open(&archive_path_clone).unwrap();
Ok(Box::new(file))
});

assert!(res.is_err());
assert!(res.unwrap_err().contains("Checksum mismatch"));
Expand Down
Loading