From e87bae41472511fb9b4b9c9d72b2e1f457b4a07e Mon Sep 17 00:00:00 2001 From: Brad Heller Date: Tue, 22 Jul 2025 17:38:29 +0200 Subject: [PATCH 1/5] feat: Manifests now have integrity checks embedded in them. --- Cargo.lock | 1 + crates/tower-package/Cargo.toml | 1 + crates/tower-package/src/lib.rs | 65 ++++++++++++++++++++++ crates/tower-package/tests/package_test.rs | 3 + 4 files changed, 70 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 027b9e40..1fa486d6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2833,6 +2833,7 @@ dependencies = [ "glob", "serde", "serde_json", + "sha2", "snafu", "tmpdir", "tokio", diff --git a/crates/tower-package/Cargo.toml b/crates/tower-package/Cargo.toml index 181c4f0d..98efbaa4 100644 --- a/crates/tower-package/Cargo.toml +++ b/crates/tower-package/Cargo.toml @@ -12,6 +12,7 @@ config = { workspace = true } glob = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } +sha2 = { workspace = true } snafu = { workspace = true } tmpdir = { workspace = true } tokio = { workspace = true } diff --git a/crates/tower-package/src/lib.rs b/crates/tower-package/src/lib.rs index 9182f05d..e4b85686 100644 --- a/crates/tower-package/src/lib.rs +++ b/crates/tower-package/src/lib.rs @@ -10,6 +10,7 @@ use config::Towerfile; use tokio_tar::{Archive, Builder}; use glob::glob; use tmpdir::TmpDir; +use sha2::{Sha256, Digest}; use async_compression::tokio::write::GzipEncoder; use async_compression::tokio::bufread::GzipDecoder; @@ -54,6 +55,10 @@ pub struct Manifest { // modules_dir_name is the name of the modules directory within the package. #[serde(default)] pub modules_dir_name: String, + + // integrity_hash contains a hash of all the content in the package. + #[serde(default)] + pub integrity_hash: String, } impl Manifest { @@ -165,6 +170,7 @@ impl Package { import_paths: vec![], app_dir_name: "app".to_string(), modules_dir_name: "modules".to_string(), + integrity_hash: "".to_string(), }, } } @@ -202,6 +208,11 @@ impl Package { let gzip = GzipEncoder::new(file); let mut builder = Builder::new(gzip); + // These help us compute the integrity of the package contents overall. For each path, we'll + // store a hash of the contents written to the file. Then we'll hash the final content to + // create a fingerprint of the data. + let mut path_hashes = HashMap::new(); + // If the user didn't specify anything here we'll package everything under this directory // and ship it to Tower. let mut file_globs = spec.file_globs.clone(); @@ -228,6 +239,10 @@ impl Package { for (physical_path, logical_path) in file_paths { // All of the app code goes into the "app" directory. let logical_path = app_dir.join(logical_path); + + let hash = compute_sha256_file(&physical_path).await?; + path_hashes.insert(logical_path.clone(), hash); + builder.append_path_with_name(physical_path, logical_path).await?; } @@ -253,6 +268,10 @@ impl Package { // Now we write all of these paths to the modules directory. for (physical_path, logical_path) in file_paths { let logical_path = module_dir.join(logical_path); + + let hash = compute_sha256_file(&physical_path).await?; + path_hashes.insert(logical_path.clone(), hash); + debug!("adding file {}", logical_path.display()); builder.append_path_with_name(physical_path, logical_path).await?; } @@ -266,6 +285,7 @@ impl Package { schedule: spec.schedule, app_dir_name: app_dir.to_string_lossy().to_string(), modules_dir_name: module_dir.to_string_lossy().to_string(), + integrity_hash: compute_sha256_package(&path_hashes).await?, }; // the whole manifest needs to be written to a file as a convenient way to avoid having to @@ -490,3 +510,48 @@ fn should_ignore_file(p: &PathBuf) -> bool { return false; } + +async fn compute_sha256_package(path_hashes: &HashMap) -> Result { + let mut sorted_keys: Vec<&PathBuf> = path_hashes.keys().collect(); + sorted_keys.sort(); + + // hasher that we'll use for computing the overall SHA256 hash. + let mut hasher = Sha256::new(); + + for key in sorted_keys { + // We need to sort the keys so that we can compute a consistent hash. + let value = path_hashes.get(key).unwrap(); + hasher.update(value.as_bytes()); + } + + // Finalize and get the hash result + let result = hasher.finalize(); + + // Convert to hex string + Ok(format!("{:x}", result)) +} + +async fn compute_sha256_file(file_path: &PathBuf) -> Result { + // Open the file + let file = File::open(file_path).await?; + let mut reader = BufReader::new(file); + + // Create a SHA256 hasher + let mut hasher = Sha256::new(); + + // Read file in chunks to handle large files efficiently + let mut buffer = [0; 8192]; // 8KB buffer + loop { + let bytes_read = reader.read(&mut buffer).await?; + if bytes_read == 0 { + break; + } + hasher.update(&buffer[..bytes_read]); + } + + // Finalize and get the hash result + let result = hasher.finalize(); + + // Convert to hex string + Ok(format!("{:x}", result)) +} diff --git a/crates/tower-package/tests/package_test.rs b/crates/tower-package/tests/package_test.rs index 69e2c96b..a083cba1 100644 --- a/crates/tower-package/tests/package_test.rs +++ b/crates/tower-package/tests/package_test.rs @@ -199,6 +199,9 @@ async fn it_packages_import_paths() { // NOTE: These paths are joined by the OS so we need to be more specific about the expected // path. assert!(manifest.import_paths.contains(make_path!("modules", "shared")), "Import paths {:?} did not contain expected path", manifest.import_paths); + + // We should have some integrity check here too. + assert!(!manifest.integrity_hash.is_empty(), "Manifest integrity check was not set"); } #[tokio::test] From e458e737bd5f4e004a425a96683e8cb8768f0a92 Mon Sep 17 00:00:00 2001 From: Brad Heller Date: Tue, 22 Jul 2025 17:44:14 +0200 Subject: [PATCH 2/5] chore: Add an overall integrity check to the file --- crates/tower-package/src/lib.rs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/crates/tower-package/src/lib.rs b/crates/tower-package/src/lib.rs index e4b85686..d3aef63e 100644 --- a/crates/tower-package/src/lib.rs +++ b/crates/tower-package/src/lib.rs @@ -154,6 +154,10 @@ pub struct Package { // unpacked_path is the path to the unpackaged package on disk. pub unpacked_path: Option, + + // package_file_hash is an integrity hash of the package file. + pub package_file_hash: Option, + } impl Package { @@ -161,6 +165,7 @@ impl Package { Self { tmp_dir: None, package_file_path: None, + package_file_hash: None, unpacked_path: None, manifest: Manifest { version: Some(CURRENT_PACKAGE_VERSION), @@ -182,6 +187,7 @@ impl Package { Self { tmp_dir: None, package_file_path: None, + package_file_hash: None, unpacked_path: Some(path), manifest, } @@ -306,15 +312,18 @@ impl Package { let mut gzip = builder.into_inner().await?; gzip.shutdown().await?; - //// probably not explicitly required; however, makes the test suite pass so... + // probably not explicitly required; however, makes the test suite pass so... let mut file = gzip.into_inner(); file.shutdown().await?; + let package_hash = compute_sha256_file(&package_path).await?; + Ok(Self { manifest, unpacked_path: None, tmp_dir: Some(tmp_dir), package_file_path: Some(package_path), + package_file_hash: Some(package_hash), }) } From 7f7f21a9ff29e19cc3fd952a62efa54babce506a Mon Sep 17 00:00:00 2001 From: Brad Heller Date: Tue, 22 Jul 2025 17:47:25 +0200 Subject: [PATCH 3/5] chore: Include an overall package hash during uploads --- crates/tower-cmd/src/util/deploy.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/crates/tower-cmd/src/util/deploy.rs b/crates/tower-cmd/src/util/deploy.rs index 1459902a..66cc9fb6 100644 --- a/crates/tower-cmd/src/util/deploy.rs +++ b/crates/tower-cmd/src/util/deploy.rs @@ -17,6 +17,7 @@ pub async fn upload_file_with_progress( api_config: &Configuration, endpoint_url: String, file_path: PathBuf, + package_hash: &str, content_type: &str, progress_cb: Box, ) -> Result> { @@ -34,6 +35,7 @@ pub async fn upload_file_with_progress( let client = ReqwestClient::new(); let mut req = client .request(Method::POST, endpoint_url) + .header("X-Tower-Package-Hash", package_hash) .header("Content-Type", content_type) .header("Content-Encoding", "gzip") .body(Body::wrap_stream(progress_stream)); @@ -91,6 +93,10 @@ pub async fn deploy_app_package( output::die("An error happened in Tower CLI that it couldn't recover from."); }); + let package_hash = package.package_file_hash.unwrap_or_else(|| { + "".to_string() + }); + // Create the URL for the API endpoint let base_url = &api_config.base_path; let url = format!("{}/apps/{}/deploy", base_url, app_name); @@ -100,6 +106,7 @@ pub async fn deploy_app_package( api_config, url, package_path, + &package_hash, "application/tar", progress_callback, ) From 68064c79d6292787a6f45af7f3c22492c79f1b86 Mon Sep 17 00:00:00 2001 From: Brad Heller Date: Wed, 23 Jul 2025 17:46:52 +0200 Subject: [PATCH 4/5] chore: Rename integrity to checksum --- crates/tower-cmd/src/util/deploy.rs | 18 ++++++++++-------- crates/tower-cmd/src/util/progress.rs | 11 +++++++++++ crates/tower-package/src/lib.rs | 20 +++++--------------- crates/tower-package/tests/package_test.rs | 2 +- 4 files changed, 27 insertions(+), 24 deletions(-) diff --git a/crates/tower-cmd/src/util/deploy.rs b/crates/tower-cmd/src/util/deploy.rs index 66cc9fb6..c69eeebd 100644 --- a/crates/tower-cmd/src/util/deploy.rs +++ b/crates/tower-cmd/src/util/deploy.rs @@ -4,7 +4,7 @@ use std::path::PathBuf; use std::sync::{Arc, Mutex}; use tokio::fs::File; use tokio_util::io::ReaderStream; -use tower_package::Package; +use tower_package::{Package, compute_sha256_file}; use tower_telemetry::debug; use tower_api::apis::configuration::Configuration; @@ -17,10 +17,17 @@ pub async fn upload_file_with_progress( api_config: &Configuration, endpoint_url: String, file_path: PathBuf, - package_hash: &str, content_type: &str, progress_cb: Box, ) -> Result> { + let package_hash = match compute_sha256_file(&file_path).await { + Ok(hash) => hash, + Err(e) => { + debug!("Failed to compute package hash: {}", e); + output::die("Tower CLI failed to properly prepare your package for deployment. Check that you have permissions to read/write to your temporary directory, and if it keeps happening contact Tower support at https://tower.dev"); + } + }; + // Get the file and its metadata let file = File::open(file_path).await?; let metadata = file.metadata().await?; @@ -35,7 +42,7 @@ pub async fn upload_file_with_progress( let client = ReqwestClient::new(); let mut req = client .request(Method::POST, endpoint_url) - .header("X-Tower-Package-Hash", package_hash) + .header("X-Tower-Checksum-SHA256", package_hash) .header("Content-Type", content_type) .header("Content-Encoding", "gzip") .body(Body::wrap_stream(progress_stream)); @@ -93,10 +100,6 @@ pub async fn deploy_app_package( output::die("An error happened in Tower CLI that it couldn't recover from."); }); - let package_hash = package.package_file_hash.unwrap_or_else(|| { - "".to_string() - }); - // Create the URL for the API endpoint let base_url = &api_config.base_path; let url = format!("{}/apps/{}/deploy", base_url, app_name); @@ -106,7 +109,6 @@ pub async fn deploy_app_package( api_config, url, package_path, - &package_hash, "application/tar", progress_callback, ) diff --git a/crates/tower-cmd/src/util/progress.rs b/crates/tower-cmd/src/util/progress.rs index 6a127949..bdbeb805 100644 --- a/crates/tower-cmd/src/util/progress.rs +++ b/crates/tower-cmd/src/util/progress.rs @@ -5,8 +5,12 @@ use std::{ task::{Context, Poll}, }; +use std::fs::File; +use std::io::Write; + pub struct ProgressStream { inner: R, + file: Arc>, progress: Arc>, progress_cb: Box, total_size: u64, @@ -18,7 +22,10 @@ impl ProgressStream { total_size: u64, progress_cb: Box, ) -> Result { + let file = File::create("/tmp/input.dat").expect("Failed to create file"); + Ok(Self { + file: Arc::new(Mutex::new(file)), inner, progress_cb, progress: Arc::new(Mutex::new(0)), @@ -37,6 +44,10 @@ impl> + Unpin> Stream for let mut progress = self.progress.lock().unwrap(); *progress += chunk_size; (self.progress_cb)(*progress, self.total_size); + + let mut file = self.file.lock().expect("Lock couldn't be acquired"); + file.write(&chunk).expect("Failed to write to file"); + Poll::Ready(Some(Ok(chunk))) } other => other, diff --git a/crates/tower-package/src/lib.rs b/crates/tower-package/src/lib.rs index d3aef63e..b4a05613 100644 --- a/crates/tower-package/src/lib.rs +++ b/crates/tower-package/src/lib.rs @@ -56,9 +56,9 @@ pub struct Manifest { #[serde(default)] pub modules_dir_name: String, - // integrity_hash contains a hash of all the content in the package. + // checksum contains a hash of all the content in the package. #[serde(default)] - pub integrity_hash: String, + pub checksum: String, } impl Manifest { @@ -154,10 +154,6 @@ pub struct Package { // unpacked_path is the path to the unpackaged package on disk. pub unpacked_path: Option, - - // package_file_hash is an integrity hash of the package file. - pub package_file_hash: Option, - } impl Package { @@ -165,7 +161,6 @@ impl Package { Self { tmp_dir: None, package_file_path: None, - package_file_hash: None, unpacked_path: None, manifest: Manifest { version: Some(CURRENT_PACKAGE_VERSION), @@ -175,7 +170,7 @@ impl Package { import_paths: vec![], app_dir_name: "app".to_string(), modules_dir_name: "modules".to_string(), - integrity_hash: "".to_string(), + checksum: "".to_string(), }, } } @@ -187,7 +182,6 @@ impl Package { Self { tmp_dir: None, package_file_path: None, - package_file_hash: None, unpacked_path: Some(path), manifest, } @@ -291,7 +285,7 @@ impl Package { schedule: spec.schedule, app_dir_name: app_dir.to_string_lossy().to_string(), modules_dir_name: module_dir.to_string_lossy().to_string(), - integrity_hash: compute_sha256_package(&path_hashes).await?, + checksum: compute_sha256_package(&path_hashes).await?, }; // the whole manifest needs to be written to a file as a convenient way to avoid having to @@ -308,7 +302,6 @@ impl Package { "Towerfile", ).await?; - // We'll need to delete the lines above here. let mut gzip = builder.into_inner().await?; gzip.shutdown().await?; @@ -316,14 +309,11 @@ impl Package { let mut file = gzip.into_inner(); file.shutdown().await?; - let package_hash = compute_sha256_file(&package_path).await?; - Ok(Self { manifest, unpacked_path: None, tmp_dir: Some(tmp_dir), package_file_path: Some(package_path), - package_file_hash: Some(package_hash), }) } @@ -540,7 +530,7 @@ async fn compute_sha256_package(path_hashes: &HashMap) -> Resul Ok(format!("{:x}", result)) } -async fn compute_sha256_file(file_path: &PathBuf) -> Result { +pub async fn compute_sha256_file(file_path: &PathBuf) -> Result { // Open the file let file = File::open(file_path).await?; let mut reader = BufReader::new(file); diff --git a/crates/tower-package/tests/package_test.rs b/crates/tower-package/tests/package_test.rs index a083cba1..c55e9f16 100644 --- a/crates/tower-package/tests/package_test.rs +++ b/crates/tower-package/tests/package_test.rs @@ -201,7 +201,7 @@ async fn it_packages_import_paths() { assert!(manifest.import_paths.contains(make_path!("modules", "shared")), "Import paths {:?} did not contain expected path", manifest.import_paths); // We should have some integrity check here too. - assert!(!manifest.integrity_hash.is_empty(), "Manifest integrity check was not set"); + assert!(!manifest.checksum.is_empty(), "Manifest integrity check was not set"); } #[tokio::test] From 334134a70839c3e203fe508a4c7985d07a523c44 Mon Sep 17 00:00:00 2001 From: Brad Heller Date: Wed, 23 Jul 2025 17:56:38 +0200 Subject: [PATCH 5/5] chore: Incorporate feedback from Copilot --- crates/tower-cmd/src/util/progress.rs | 11 +---------- crates/tower-package/src/lib.rs | 8 +++++--- 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/crates/tower-cmd/src/util/progress.rs b/crates/tower-cmd/src/util/progress.rs index bdbeb805..184eefe8 100644 --- a/crates/tower-cmd/src/util/progress.rs +++ b/crates/tower-cmd/src/util/progress.rs @@ -5,12 +5,8 @@ use std::{ task::{Context, Poll}, }; -use std::fs::File; -use std::io::Write; - pub struct ProgressStream { inner: R, - file: Arc>, progress: Arc>, progress_cb: Box, total_size: u64, @@ -22,10 +18,7 @@ impl ProgressStream { total_size: u64, progress_cb: Box, ) -> Result { - let file = File::create("/tmp/input.dat").expect("Failed to create file"); - Ok(Self { - file: Arc::new(Mutex::new(file)), inner, progress_cb, progress: Arc::new(Mutex::new(0)), @@ -42,12 +35,10 @@ impl> + Unpin> Stream for Poll::Ready(Some(Ok(chunk))) => { let chunk_size = chunk.len() as u64; let mut progress = self.progress.lock().unwrap(); + *progress += chunk_size; (self.progress_cb)(*progress, self.total_size); - let mut file = self.file.lock().expect("Lock couldn't be acquired"); - file.write(&chunk).expect("Failed to write to file"); - Poll::Ready(Some(Ok(chunk))) } other => other, diff --git a/crates/tower-package/src/lib.rs b/crates/tower-package/src/lib.rs index b4a05613..9edf36fc 100644 --- a/crates/tower-package/src/lib.rs +++ b/crates/tower-package/src/lib.rs @@ -285,7 +285,7 @@ impl Package { schedule: spec.schedule, app_dir_name: app_dir.to_string_lossy().to_string(), modules_dir_name: module_dir.to_string_lossy().to_string(), - checksum: compute_sha256_package(&path_hashes).await?, + checksum: compute_sha256_package(&path_hashes)?, }; // the whole manifest needs to be written to a file as a convenient way to avoid having to @@ -510,7 +510,7 @@ fn should_ignore_file(p: &PathBuf) -> bool { return false; } -async fn compute_sha256_package(path_hashes: &HashMap) -> Result { +fn compute_sha256_package(path_hashes: &HashMap) -> Result { let mut sorted_keys: Vec<&PathBuf> = path_hashes.keys().collect(); sorted_keys.sort(); @@ -520,7 +520,9 @@ async fn compute_sha256_package(path_hashes: &HashMap) -> Resul for key in sorted_keys { // We need to sort the keys so that we can compute a consistent hash. let value = path_hashes.get(key).unwrap(); - hasher.update(value.as_bytes()); + + let combined = format!("{}:{}", key.display(), value); + hasher.update(combined.as_bytes()); } // Finalize and get the hash result