From 0c60b728bb48250679aed3586ddb7e280c17cb44 Mon Sep 17 00:00:00 2001 From: Brad Heller Date: Mon, 21 Jul 2025 16:40:50 +0200 Subject: [PATCH 1/5] chore: Bump version for v0.3.21 release --- Cargo.lock | 22 +++++++++++----------- Cargo.toml | 4 +++- pyproject.toml | 4 +++- uv.lock | 2 +- 4 files changed, 18 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 23bee215..027b9e40 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -387,7 +387,7 @@ dependencies = [ [[package]] name = "config" -version = "0.3.21-rc.2" +version = "0.3.21" dependencies = [ "chrono", "clap", @@ -474,7 +474,7 @@ checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" [[package]] name = "crypto" -version = "0.3.21-rc.2" +version = "0.3.21" dependencies = [ "aes-gcm", "base64", @@ -2513,7 +2513,7 @@ dependencies = [ [[package]] name = "testutils" -version = "0.3.21-rc.2" +version = "0.3.21" dependencies = [ "pem", "rsa", @@ -2751,7 +2751,7 @@ dependencies = [ [[package]] name = "tower" -version = "0.3.21-rc.2" +version = "0.3.21" dependencies = [ "tokio", "tower-api", @@ -2775,7 +2775,7 @@ dependencies = [ [[package]] name = "tower-api" -version = "0.3.21-rc.2" +version = "0.3.21" dependencies = [ "reqwest", "serde", @@ -2787,7 +2787,7 @@ dependencies = [ [[package]] name = "tower-cmd" -version = "0.3.21-rc.2" +version = "0.3.21" dependencies = [ "anyhow", "bytes", @@ -2826,7 +2826,7 @@ checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" [[package]] name = "tower-package" -version = "0.3.21-rc.2" +version = "0.3.21" dependencies = [ "async-compression", "config", @@ -2844,7 +2844,7 @@ dependencies = [ [[package]] name = "tower-runtime" -version = "0.3.21-rc.2" +version = "0.3.21" dependencies = [ "chrono", "config", @@ -2864,7 +2864,7 @@ checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" [[package]] name = "tower-telemetry" -version = "0.3.21-rc.2" +version = "0.3.21" dependencies = [ "tracing", "tracing-appender", @@ -2873,7 +2873,7 @@ dependencies = [ [[package]] name = "tower-uv" -version = "0.3.21-rc.2" +version = "0.3.21" dependencies = [ "async-compression", "async_zip", @@ -2887,7 +2887,7 @@ dependencies = [ [[package]] name = "tower-version" -version = "0.3.21-rc.2" +version = "0.3.21" dependencies = [ "anyhow", "chrono", diff --git a/Cargo.toml b/Cargo.toml index 9e234ba0..eee577e7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,7 +4,9 @@ resolver = "2" [workspace.package] edition = "2021" -version = "0.3.21-rc.2" +version = "0.3.21" + + description = "Tower is the best way to host Python data apps in production" rust-version = "1.81" authors = ["Brad Heller "] diff --git a/pyproject.toml b/pyproject.toml index 41fd16d1..e3f9bb9c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,9 @@ build-backend = "maturin" [project] name = "tower" -version = "0.3.21rc2" +version = "0.3.21" + + description = "Tower CLI and runtime environment for Tower." authors = [{ name = "Tower Computing Inc.", email = "brad@tower.dev" }] readme = "README.md" diff --git a/uv.lock b/uv.lock index 523da39c..0a733e57 100644 --- a/uv.lock +++ b/uv.lock @@ -1201,7 +1201,7 @@ wheels = [ [[package]] name = "tower" -version = "0.3.21rc2" +version = "0.3.21" source = { editable = "." } dependencies = [ { name = "attrs" }, From c73b88746389d9d4ebdff74fc316e391c185efcb Mon Sep 17 00:00:00 2001 From: Brad Heller Date: Thu, 24 Jul 2025 23:42:29 +0200 Subject: [PATCH 2/5] Add integrity checks during package upload (#71) * feat: Manifests now have integrity checks embedded in them. * chore: Add an overall integrity check to the file * chore: Include an overall package hash during uploads * chore: Rename integrity to checksum * chore: Incorporate feedback from Copilot --- Cargo.lock | 1 + crates/tower-cmd/src/util/deploy.rs | 11 +++- crates/tower-cmd/src/util/progress.rs | 2 + crates/tower-package/Cargo.toml | 1 + crates/tower-package/src/lib.rs | 70 +++++++++++++++++++++- crates/tower-package/tests/package_test.rs | 3 + 6 files changed, 85 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 027b9e40..1fa486d6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2833,6 +2833,7 @@ dependencies = [ "glob", "serde", "serde_json", + "sha2", "snafu", "tmpdir", "tokio", diff --git a/crates/tower-cmd/src/util/deploy.rs b/crates/tower-cmd/src/util/deploy.rs index 1459902a..c69eeebd 100644 --- a/crates/tower-cmd/src/util/deploy.rs +++ b/crates/tower-cmd/src/util/deploy.rs @@ -4,7 +4,7 @@ use std::path::PathBuf; use std::sync::{Arc, Mutex}; use tokio::fs::File; use tokio_util::io::ReaderStream; -use tower_package::Package; +use tower_package::{Package, compute_sha256_file}; use tower_telemetry::debug; use tower_api::apis::configuration::Configuration; @@ -20,6 +20,14 @@ pub async fn upload_file_with_progress( content_type: &str, progress_cb: Box, ) -> Result> { + let package_hash = match compute_sha256_file(&file_path).await { + Ok(hash) => hash, + Err(e) => { + debug!("Failed to compute package hash: {}", e); + output::die("Tower CLI failed to properly prepare your package for deployment. Check that you have permissions to read/write to your temporary directory, and if it keeps happening contact Tower support at https://tower.dev"); + } + }; + // Get the file and its metadata let file = File::open(file_path).await?; let metadata = file.metadata().await?; @@ -34,6 +42,7 @@ pub async fn upload_file_with_progress( let client = ReqwestClient::new(); let mut req = client .request(Method::POST, endpoint_url) + .header("X-Tower-Checksum-SHA256", package_hash) .header("Content-Type", content_type) .header("Content-Encoding", "gzip") .body(Body::wrap_stream(progress_stream)); diff --git a/crates/tower-cmd/src/util/progress.rs b/crates/tower-cmd/src/util/progress.rs index 6a127949..184eefe8 100644 --- a/crates/tower-cmd/src/util/progress.rs +++ b/crates/tower-cmd/src/util/progress.rs @@ -35,8 +35,10 @@ impl> + Unpin> Stream for Poll::Ready(Some(Ok(chunk))) => { let chunk_size = chunk.len() as u64; let mut progress = self.progress.lock().unwrap(); + *progress += chunk_size; (self.progress_cb)(*progress, self.total_size); + Poll::Ready(Some(Ok(chunk))) } other => other, diff --git a/crates/tower-package/Cargo.toml b/crates/tower-package/Cargo.toml index 181c4f0d..98efbaa4 100644 --- a/crates/tower-package/Cargo.toml +++ b/crates/tower-package/Cargo.toml @@ -12,6 +12,7 @@ config = { workspace = true } glob = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } +sha2 = { workspace = true } snafu = { workspace = true } tmpdir = { workspace = true } tokio = { workspace = true } diff --git a/crates/tower-package/src/lib.rs b/crates/tower-package/src/lib.rs index 9182f05d..9edf36fc 100644 --- a/crates/tower-package/src/lib.rs +++ b/crates/tower-package/src/lib.rs @@ -10,6 +10,7 @@ use config::Towerfile; use tokio_tar::{Archive, Builder}; use glob::glob; use tmpdir::TmpDir; +use sha2::{Sha256, Digest}; use async_compression::tokio::write::GzipEncoder; use async_compression::tokio::bufread::GzipDecoder; @@ -54,6 +55,10 @@ pub struct Manifest { // modules_dir_name is the name of the modules directory within the package. #[serde(default)] pub modules_dir_name: String, + + // checksum contains a hash of all the content in the package. + #[serde(default)] + pub checksum: String, } impl Manifest { @@ -165,6 +170,7 @@ impl Package { import_paths: vec![], app_dir_name: "app".to_string(), modules_dir_name: "modules".to_string(), + checksum: "".to_string(), }, } } @@ -202,6 +208,11 @@ impl Package { let gzip = GzipEncoder::new(file); let mut builder = Builder::new(gzip); + // These help us compute the integrity of the package contents overall. For each path, we'll + // store a hash of the contents written to the file. Then we'll hash the final content to + // create a fingerprint of the data. + let mut path_hashes = HashMap::new(); + // If the user didn't specify anything here we'll package everything under this directory // and ship it to Tower. let mut file_globs = spec.file_globs.clone(); @@ -228,6 +239,10 @@ impl Package { for (physical_path, logical_path) in file_paths { // All of the app code goes into the "app" directory. let logical_path = app_dir.join(logical_path); + + let hash = compute_sha256_file(&physical_path).await?; + path_hashes.insert(logical_path.clone(), hash); + builder.append_path_with_name(physical_path, logical_path).await?; } @@ -253,6 +268,10 @@ impl Package { // Now we write all of these paths to the modules directory. for (physical_path, logical_path) in file_paths { let logical_path = module_dir.join(logical_path); + + let hash = compute_sha256_file(&physical_path).await?; + path_hashes.insert(logical_path.clone(), hash); + debug!("adding file {}", logical_path.display()); builder.append_path_with_name(physical_path, logical_path).await?; } @@ -266,6 +285,7 @@ impl Package { schedule: spec.schedule, app_dir_name: app_dir.to_string_lossy().to_string(), modules_dir_name: module_dir.to_string_lossy().to_string(), + checksum: compute_sha256_package(&path_hashes)?, }; // the whole manifest needs to be written to a file as a convenient way to avoid having to @@ -282,11 +302,10 @@ impl Package { "Towerfile", ).await?; - // We'll need to delete the lines above here. let mut gzip = builder.into_inner().await?; gzip.shutdown().await?; - //// probably not explicitly required; however, makes the test suite pass so... + // probably not explicitly required; however, makes the test suite pass so... let mut file = gzip.into_inner(); file.shutdown().await?; @@ -490,3 +509,50 @@ fn should_ignore_file(p: &PathBuf) -> bool { return false; } + +fn compute_sha256_package(path_hashes: &HashMap) -> Result { + let mut sorted_keys: Vec<&PathBuf> = path_hashes.keys().collect(); + sorted_keys.sort(); + + // hasher that we'll use for computing the overall SHA256 hash. + let mut hasher = Sha256::new(); + + for key in sorted_keys { + // We need to sort the keys so that we can compute a consistent hash. + let value = path_hashes.get(key).unwrap(); + + let combined = format!("{}:{}", key.display(), value); + hasher.update(combined.as_bytes()); + } + + // Finalize and get the hash result + let result = hasher.finalize(); + + // Convert to hex string + Ok(format!("{:x}", result)) +} + +pub async fn compute_sha256_file(file_path: &PathBuf) -> Result { + // Open the file + let file = File::open(file_path).await?; + let mut reader = BufReader::new(file); + + // Create a SHA256 hasher + let mut hasher = Sha256::new(); + + // Read file in chunks to handle large files efficiently + let mut buffer = [0; 8192]; // 8KB buffer + loop { + let bytes_read = reader.read(&mut buffer).await?; + if bytes_read == 0 { + break; + } + hasher.update(&buffer[..bytes_read]); + } + + // Finalize and get the hash result + let result = hasher.finalize(); + + // Convert to hex string + Ok(format!("{:x}", result)) +} diff --git a/crates/tower-package/tests/package_test.rs b/crates/tower-package/tests/package_test.rs index 69e2c96b..c55e9f16 100644 --- a/crates/tower-package/tests/package_test.rs +++ b/crates/tower-package/tests/package_test.rs @@ -199,6 +199,9 @@ async fn it_packages_import_paths() { // NOTE: These paths are joined by the OS so we need to be more specific about the expected // path. assert!(manifest.import_paths.contains(make_path!("modules", "shared")), "Import paths {:?} did not contain expected path", manifest.import_paths); + + // We should have some integrity check here too. + assert!(!manifest.checksum.is_empty(), "Manifest integrity check was not set"); } #[tokio::test] From c7eeb67372042d32a5b75c8bd9534a0f7c8058d7 Mon Sep 17 00:00:00 2001 From: Brad Heller Date: Fri, 25 Jul 2025 09:18:34 +0200 Subject: [PATCH 3/5] chore: Bump version to v0.3.22-rc.1 --- Cargo.lock | 22 +++++++++++----------- Cargo.toml | 4 +--- pyproject.toml | 4 +--- uv.lock | 2 +- 4 files changed, 14 insertions(+), 18 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1fa486d6..d754ef2f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -387,7 +387,7 @@ dependencies = [ [[package]] name = "config" -version = "0.3.21" +version = "0.3.22-rc.1" dependencies = [ "chrono", "clap", @@ -474,7 +474,7 @@ checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" [[package]] name = "crypto" -version = "0.3.21" +version = "0.3.22-rc.1" dependencies = [ "aes-gcm", "base64", @@ -2513,7 +2513,7 @@ dependencies = [ [[package]] name = "testutils" -version = "0.3.21" +version = "0.3.22-rc.1" dependencies = [ "pem", "rsa", @@ -2751,7 +2751,7 @@ dependencies = [ [[package]] name = "tower" -version = "0.3.21" +version = "0.3.22-rc.1" dependencies = [ "tokio", "tower-api", @@ -2775,7 +2775,7 @@ dependencies = [ [[package]] name = "tower-api" -version = "0.3.21" +version = "0.3.22-rc.1" dependencies = [ "reqwest", "serde", @@ -2787,7 +2787,7 @@ dependencies = [ [[package]] name = "tower-cmd" -version = "0.3.21" +version = "0.3.22-rc.1" dependencies = [ "anyhow", "bytes", @@ -2826,7 +2826,7 @@ checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" [[package]] name = "tower-package" -version = "0.3.21" +version = "0.3.22-rc.1" dependencies = [ "async-compression", "config", @@ -2845,7 +2845,7 @@ dependencies = [ [[package]] name = "tower-runtime" -version = "0.3.21" +version = "0.3.22-rc.1" dependencies = [ "chrono", "config", @@ -2865,7 +2865,7 @@ checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" [[package]] name = "tower-telemetry" -version = "0.3.21" +version = "0.3.22-rc.1" dependencies = [ "tracing", "tracing-appender", @@ -2874,7 +2874,7 @@ dependencies = [ [[package]] name = "tower-uv" -version = "0.3.21" +version = "0.3.22-rc.1" dependencies = [ "async-compression", "async_zip", @@ -2888,7 +2888,7 @@ dependencies = [ [[package]] name = "tower-version" -version = "0.3.21" +version = "0.3.22-rc.1" dependencies = [ "anyhow", "chrono", diff --git a/Cargo.toml b/Cargo.toml index eee577e7..3b08b120 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,9 +4,7 @@ resolver = "2" [workspace.package] edition = "2021" -version = "0.3.21" - - +version = "0.3.22-rc.1" description = "Tower is the best way to host Python data apps in production" rust-version = "1.81" authors = ["Brad Heller "] diff --git a/pyproject.toml b/pyproject.toml index e3f9bb9c..075e5249 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,9 +4,7 @@ build-backend = "maturin" [project] name = "tower" -version = "0.3.21" - - +version = "0.3.22rc1" description = "Tower CLI and runtime environment for Tower." authors = [{ name = "Tower Computing Inc.", email = "brad@tower.dev" }] readme = "README.md" diff --git a/uv.lock b/uv.lock index 0a733e57..e09444a8 100644 --- a/uv.lock +++ b/uv.lock @@ -1201,7 +1201,7 @@ wheels = [ [[package]] name = "tower" -version = "0.3.21" +version = "0.3.22rc1" source = { editable = "." } dependencies = [ { name = "attrs" }, From cc23f5486666e8665b2c77737e5a36be962adc70 Mon Sep 17 00:00:00 2001 From: Brad Heller Date: Fri, 25 Jul 2025 11:46:40 +0200 Subject: [PATCH 4/5] chore: Bump version to Tower v0.3.22 --- Cargo.lock | 22 +++++++++++----------- Cargo.toml | 4 +++- pyproject.toml | 4 +++- uv.lock | 2 +- 4 files changed, 18 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1917f036..acfd899d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -387,7 +387,7 @@ dependencies = [ [[package]] name = "config" -version = "0.3.22-rc.1" +version = "0.3.22" dependencies = [ "chrono", "clap", @@ -474,7 +474,7 @@ checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" [[package]] name = "crypto" -version = "0.3.22-rc.1" +version = "0.3.22" dependencies = [ "aes-gcm", "base64", @@ -2558,7 +2558,7 @@ dependencies = [ [[package]] name = "testutils" -version = "0.3.22-rc.1" +version = "0.3.22" dependencies = [ "pem", "rsa", @@ -2804,7 +2804,7 @@ checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" [[package]] name = "tower" -version = "0.3.22-rc.1" +version = "0.3.22" dependencies = [ "tokio", "tower-api", @@ -2828,7 +2828,7 @@ dependencies = [ [[package]] name = "tower-api" -version = "0.3.22-rc.1" +version = "0.3.22" dependencies = [ "reqwest", "serde", @@ -2840,7 +2840,7 @@ dependencies = [ [[package]] name = "tower-cmd" -version = "0.3.22-rc.1" +version = "0.3.22" dependencies = [ "anyhow", "bytes", @@ -2897,7 +2897,7 @@ checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" [[package]] name = "tower-package" -version = "0.3.22-rc.1" +version = "0.3.22" dependencies = [ "async-compression", "config", @@ -2916,7 +2916,7 @@ dependencies = [ [[package]] name = "tower-runtime" -version = "0.3.22-rc.1" +version = "0.3.22" dependencies = [ "chrono", "config", @@ -2936,7 +2936,7 @@ checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" [[package]] name = "tower-telemetry" -version = "0.3.22-rc.1" +version = "0.3.22" dependencies = [ "tracing", "tracing-appender", @@ -2945,7 +2945,7 @@ dependencies = [ [[package]] name = "tower-uv" -version = "0.3.22-rc.1" +version = "0.3.22" dependencies = [ "async-compression", "async_zip", @@ -2959,7 +2959,7 @@ dependencies = [ [[package]] name = "tower-version" -version = "0.3.22-rc.1" +version = "0.3.22" dependencies = [ "anyhow", "chrono", diff --git a/Cargo.toml b/Cargo.toml index 3b08b120..e763a0b3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,7 +4,9 @@ resolver = "2" [workspace.package] edition = "2021" -version = "0.3.22-rc.1" +version = "0.3.22" + + description = "Tower is the best way to host Python data apps in production" rust-version = "1.81" authors = ["Brad Heller "] diff --git a/pyproject.toml b/pyproject.toml index 075e5249..5d84ebbd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,9 @@ build-backend = "maturin" [project] name = "tower" -version = "0.3.22rc1" +version = "0.3.22" + + description = "Tower CLI and runtime environment for Tower." authors = [{ name = "Tower Computing Inc.", email = "brad@tower.dev" }] readme = "README.md" diff --git a/uv.lock b/uv.lock index e09444a8..bec8211e 100644 --- a/uv.lock +++ b/uv.lock @@ -1201,7 +1201,7 @@ wheels = [ [[package]] name = "tower" -version = "0.3.22rc1" +version = "0.3.22" source = { editable = "." } dependencies = [ { name = "attrs" }, From 6c06b3f29e826be825c97062b3747b7fea35e323 Mon Sep 17 00:00:00 2001 From: Brad Heller Date: Fri, 25 Jul 2025 11:50:37 +0200 Subject: [PATCH 5/5] chore: Bump rust version for builds in CI --- rust-toolchain.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust-toolchain.toml b/rust-toolchain.toml index 4cef0b73..c95c9057 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,2 +1,2 @@ [toolchain] -channel = "1.81" +channel = "1.88"