diff --git a/Cargo.lock b/Cargo.lock index 03039ce9..9b864c92 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,6 +17,41 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" +[[package]] +name = "aead" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" +dependencies = [ + "crypto-common", + "generic-array", +] + +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures", +] + +[[package]] +name = "aes-gcm" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "831010a0f742e1209b3bcea8fab6a8e149051ba6099432c8cb2cc117dec3ead1" +dependencies = [ + "aead", + "aes", + "cipher", + "ctr", + "ghash", + "subtle", +] + [[package]] name = "aho-corasick" version = "1.1.3" @@ -224,6 +259,16 @@ dependencies = [ "windows-link", ] +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common", + "inout", +] + [[package]] name = "clap" version = "4.5.36" @@ -410,11 +455,13 @@ dependencies = [ name = "crypto" version = "0.3.13" dependencies = [ + "aes-gcm", "base64", "pem", "rand", "rsa", "sha2", + "snafu", "testutils", ] @@ -425,6 +472,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" dependencies = [ "generic-array", + "rand_core", "typenum", ] @@ -449,6 +497,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "ctr" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0369ee1ad671834580515889b80f2ea915f23b8be8d0daa4bbaf2ac5c7590835" +dependencies = [ + "cipher", +] + [[package]] name = "darling" version = "0.20.11" @@ -836,6 +893,16 @@ dependencies = [ "wasi 0.14.2+wasi-0.2.4", ] +[[package]] +name = "ghash" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0d8a4362ccb29cb0b265253fb0a2728f592895ee6854fd9bc13f2ffda266ff1" +dependencies = [ + "opaque-debug", + "polyval", +] + [[package]] name = "gimli" version = "0.31.1" @@ -1223,6 +1290,15 @@ dependencies = [ "web-time", ] +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "generic-array", +] + [[package]] name = "ipnet" version = "2.11.0" @@ -1573,6 +1649,12 @@ version = "1.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" +[[package]] +name = "opaque-debug" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" + [[package]] name = "openssl" version = "0.10.72" @@ -1720,6 +1802,18 @@ version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" +[[package]] +name = "polyval" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d1fe60d06143b2430aa532c94cfe9e29783047f06c0d7fd359a9a51b729fa25" +dependencies = [ + "cfg-if", + "cpufeatures", + "opaque-debug", + "universal-hash", +] + [[package]] name = "portable-atomic" version = "1.11.0" @@ -2748,6 +2842,7 @@ dependencies = [ "serde", "serde_json", "simple_logger", + "snafu", "spinners", "tokio", "tokio-util", @@ -2873,6 +2968,16 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd" +[[package]] +name = "universal-hash" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" +dependencies = [ + "crypto-common", + "subtle", +] + [[package]] name = "untrusted" version = "0.9.0" diff --git a/Cargo.toml b/Cargo.toml index 0894f68b..66a2529f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,9 +5,6 @@ resolver = "2" [workspace.package] edition = "2021" version = "0.3.13" - - - description = "Tower is the best way to host Python data apps in production" rust-version = "1.77" authors = ["Brad Heller "] @@ -15,6 +12,7 @@ license = "MIT" repository = "https://github.com/tower/tower-cli" [workspace.dependencies] +aes-gcm = "0.10" anyhow = "1.0.95" async-compression = { version = "0.4", features = ["tokio", "gzip"] } base64 = "0.22" diff --git a/crates/crypto/Cargo.toml b/crates/crypto/Cargo.toml index dffea22f..87977f89 100644 --- a/crates/crypto/Cargo.toml +++ b/crates/crypto/Cargo.toml @@ -4,9 +4,11 @@ version = { workspace = true } edition = "2021" [dependencies] +aes-gcm = { workspace = true } base64 = { workspace = true } pem = { workspace = true } rand = { workspace = true } rsa = { workspace = true } sha2 = { workspace = true } +snafu = { workspace = true } testutils = { workspace = true } diff --git a/crates/crypto/src/errors.rs b/crates/crypto/src/errors.rs new file mode 100644 index 00000000..5edb1796 --- /dev/null +++ b/crates/crypto/src/errors.rs @@ -0,0 +1,40 @@ +use snafu::prelude::*; + +#[derive(Debug, Snafu)] +pub enum Error { + #[snafu(display("invalid message"))] + InvalidMessage, + + #[snafu(display("invalid encoding"))] + InvalidEncoding, + + #[snafu(display("cryptography error"))] + CryptographyError, + + #[snafu(display("base64 error"))] + Base64Error, +} + +impl From for Error { + fn from(_error: std::string::FromUtf8Error) -> Self { + Self::InvalidEncoding + } +} + +impl From for Error { + fn from(_error: aes_gcm::Error) -> Self { + Self::CryptographyError + } +} + +impl From for Error { + fn from(_error: rsa::Error) -> Self { + Self::CryptographyError + } +} + +impl From for Error { + fn from(_error: base64::DecodeError) -> Self { + Self::Base64Error + } +} diff --git a/crates/crypto/src/lib.rs b/crates/crypto/src/lib.rs index 98a38b0d..92bc1fac 100644 --- a/crates/crypto/src/lib.rs +++ b/crates/crypto/src/lib.rs @@ -1,47 +1,81 @@ -use sha2::{Sha256, Digest, digest::DynDigest}; -use rand::rngs::OsRng; +use sha2::Sha256; use base64::prelude::*; +use rand::rngs::OsRng; +use rand::RngCore; +use aes_gcm::{Aes256Gcm, Key, KeyInit, Nonce}; // Or Aes256GcmSiv, Aes256GcmHs +use aes_gcm::aead::Aead; use rsa::{ - RsaPrivateKey, RsaPublicKey, Oaep, + Oaep, RsaPrivateKey, RsaPublicKey, traits::PublicKeyParts, - pkcs1::EncodeRsaPublicKey, + pkcs8::EncodePublicKey, }; -/// encrypt manages the process of encrypting long messages using the RSA algorithm and OAEP -/// padding. It takes a public key and a plaintext message and returns the ciphertext. -pub fn encrypt(key: RsaPublicKey, plaintext: String) -> String { - let mut rng = OsRng; - let hash = Sha256::new(); - let bytes = key.n().bits() / 8; - let step = bytes - 2*hash.output_size() - 2; - let chunks = plaintext.as_bytes().chunks(step); - let mut res = vec![]; - - for chunk in chunks { - let padding = Oaep::new::(); - let encrypted = key.encrypt(&mut rng, padding, chunk).unwrap(); - res.extend(encrypted); - } - - BASE64_STANDARD.encode(res) +mod errors; +pub use errors::Error; + +/// encrypt encryptes plaintext with a randomly-generated AES-256 key and IV, then encrypts the AES +/// key with RSA-OAEP using the provided public key. The result is a non-URL-safe base64-encoded +/// string. +pub fn encrypt( + key: RsaPublicKey, + plaintext: String +) -> Result { + // Generate a random 32-byte AES key + let mut aes_key = [0u8; 32]; + OsRng.fill_bytes(&mut aes_key); + + // Generate a random 12-byte IV + let mut iv = [0u8; 12]; + OsRng.fill_bytes(&mut iv); + + // Create AES cipher (GCM mode) + let aes_cipher = Aes256Gcm::new(Key::::from_slice(&aes_key)); + + // Encrypt the message + let nonce = Nonce::from_slice(&iv); // 12 bytes; unique per message + let ciphertext = aes_cipher.encrypt(nonce, plaintext.as_bytes())?; + + // Encrypt the AES key with RSA-OAEP + let padding = Oaep::new::(); + let encrypted_key = key.encrypt(&mut OsRng, padding, &aes_key)?; + + // Combine encrypted key + IV + ciphertext + let mut result = Vec::new(); + result.extend_from_slice(&encrypted_key); + result.extend_from_slice(&iv); + result.extend_from_slice(&ciphertext); + + // Encode the result as base64 + Ok(BASE64_STANDARD.encode(&result)) } -/// decrypt takes a given RSA Private Key and the relevant ciphertext and decrypts it into -/// plaintext. It's expected that the message was encrypted using OAEP padding and SHA256 digest. -pub fn decrypt(key: RsaPrivateKey, ciphertext: String) -> String { - let decoded = BASE64_STANDARD.decode(ciphertext.as_bytes()).unwrap(); +/// decrypt uses `key` to decrypt an AES-256 key that's prepended to the ciphertext. The decrypted +/// key is then used to decrypt the suffix of `ciphertext` which contains the relevant message. +/// It's expected that the message was encrypted using OAEP padding and SHA256 digest. +pub fn decrypt(key: RsaPrivateKey, ciphertext: String) -> Result { + let decoded = BASE64_STANDARD.decode(ciphertext)?; - let step = key.n().bits() / 8; - let chunks: Vec<&[u8]> = decoded.chunks(step).collect(); - let mut res = vec![]; + let n = key.size(); + let (ciphered_key, suffix) = decoded.split_at(n); - for (_, chunk) in chunks.iter().enumerate() { - let padding = Oaep::new::(); - let decrypted = key.decrypt(padding, chunk).unwrap(); - res.extend(decrypted); + let key = key.decrypt( + Oaep::new::(), + ciphered_key, + )?; + + let aes_key =Key::::from_slice(&key); + let cipher = Aes256Gcm::new(aes_key); + + // Check if the suffix is at least 12 bytes (96 bits) for the IV + if suffix.len() < 12 { + return Err(Error::InvalidMessage); } - String::from_utf8(res).unwrap() + let (iv, ciphertext) = suffix.split_at(12); + let nonce = Nonce::from_slice(iv); + + let plaintext = cipher.decrypt(nonce, ciphertext)?; + Ok(String::from_utf8(plaintext)?) } /// generate_key_pair creates a new 2048-bit public and private key for use in @@ -55,22 +89,22 @@ pub fn generate_key_pair() -> (RsaPrivateKey, RsaPublicKey) { /// serialize_public_key takes an RSA public key and serializes it into a PEM-encoded string. pub fn serialize_public_key(key: RsaPublicKey) -> String { - key.to_pkcs1_pem(rsa::pkcs1::LineEnding::LF).unwrap() + key.to_public_key_pem(rsa::pkcs8::LineEnding::LF).unwrap() } #[cfg(test)] mod test { use super::*; use rand::{distributions::Alphanumeric, Rng}; - use rsa::pkcs1::DecodeRsaPublicKey; + use rsa::pkcs8::DecodePublicKey; #[test] fn test_encrypt_decrypt() { let (private_key, public_key) = testutils::crypto::get_test_keys(); let plaintext = "Hello, World!".to_string(); - let ciphertext = encrypt(public_key, plaintext.clone()); - let decrypted = decrypt(private_key, ciphertext); + let ciphertext = encrypt(public_key, plaintext.clone()).unwrap(); + let decrypted = decrypt(private_key, ciphertext).unwrap(); assert_eq!(plaintext, decrypted); } @@ -85,8 +119,8 @@ mod test { .map(char::from) .collect(); - let ciphertext = encrypt(public_key, plaintext.clone()); - let decrypted = decrypt(private_key, ciphertext); + let ciphertext = encrypt(public_key, plaintext.clone()).unwrap(); + let decrypted = decrypt(private_key, ciphertext).unwrap(); assert_eq!(plaintext, decrypted); } @@ -95,7 +129,7 @@ mod test { fn test_serialize_public_key() { let (_private_key, public_key) = testutils::crypto::get_test_keys(); let serialized = serialize_public_key(public_key.clone()); - let deserialized = RsaPublicKey::from_pkcs1_pem(&serialized).unwrap(); + let deserialized = RsaPublicKey::from_public_key_pem(&serialized).unwrap(); assert_eq!(public_key, deserialized); } diff --git a/crates/tower-cmd/Cargo.toml b/crates/tower-cmd/Cargo.toml index 5afbdf9d..9f57eb04 100644 --- a/crates/tower-cmd/Cargo.toml +++ b/crates/tower-cmd/Cargo.toml @@ -23,6 +23,7 @@ rsa = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } simple_logger = { workspace = true } +snafu = { workspace = true } spinners = { workspace = true } tokio = { workspace = true } tokio-util = { workspace = true } diff --git a/crates/tower-cmd/src/api.rs b/crates/tower-cmd/src/api.rs index f4b11c42..819b8efe 100644 --- a/crates/tower-cmd/src/api.rs +++ b/crates/tower-cmd/src/api.rs @@ -108,6 +108,24 @@ pub async fn export_secrets(config: &Config, env: &str, all: bool, public_key: r unwrap_api_response(tower_api::apis::default_api::export_secrets(api_config, params)).await } +pub async fn export_catalogs(config: &Config, env: &str, all: bool, public_key: rsa::RsaPublicKey) -> Result> { + let api_config = &config.into(); + + let params = tower_api::apis::default_api::ExportCatalogsParams { + export_catalogs_params: tower_api::models::ExportCatalogsParams { + schema: None, + all, + public_key: crypto::serialize_public_key(public_key), + environment: env.to_string(), + page: 1, + page_size: 100, + }, + }; + + unwrap_api_response(tower_api::apis::default_api::export_catalogs(api_config, params)).await +} + + pub async fn list_secrets(config: &Config, env: &str, all: bool) -> Result> { let api_config = &config.into(); @@ -247,6 +265,17 @@ impl ResponseEntity for tower_api::apis::default_api::ExportSecretsSuccess { } } +impl ResponseEntity for tower_api::apis::default_api::ExportCatalogsSuccess { + type Data = tower_api::models::ExportCatalogsResponse; + + fn extract_data(self) -> Option { + match self { + Self::Status200(data) => Some(data), + Self::UnknownValue(_) => None, + } + } +} + impl ResponseEntity for tower_api::apis::default_api::CreateSecretSuccess { type Data = tower_api::models::CreateSecretResponse; diff --git a/crates/tower-cmd/src/error.rs b/crates/tower-cmd/src/error.rs index e69de29b..7ba35b0a 100644 --- a/crates/tower-cmd/src/error.rs +++ b/crates/tower-cmd/src/error.rs @@ -0,0 +1,20 @@ +use snafu::prelude::*; + +#[derive(Debug, Snafu)] +pub enum Error { + #[snafu(display("fetching catalogs failed"))] + FetchingCatalogsFailed, + + #[snafu(display("fetching secrets failed"))] + FetchingSecretsFailed, + + #[snafu(display("cryptography error"))] + CryptographyError, +} + +impl From for Error { + fn from(err: crypto::Error) -> Self { + log::debug!("cryptography error: {:?}", err); + Self::CryptographyError + } +} diff --git a/crates/tower-cmd/src/lib.rs b/crates/tower-cmd/src/lib.rs index 3fe628c9..76906b5c 100644 --- a/crates/tower-cmd/src/lib.rs +++ b/crates/tower-cmd/src/lib.rs @@ -5,6 +5,7 @@ mod apps; mod deploy; pub mod output; pub mod api; +pub mod error; mod run; mod secrets; mod session; @@ -12,6 +13,8 @@ mod teams; mod util; mod version; +pub use error::Error; + pub struct App { session: Option, cmd: Command, diff --git a/crates/tower-cmd/src/run.rs b/crates/tower-cmd/src/run.rs index 7e78ca7f..e46b7c12 100644 --- a/crates/tower-cmd/src/run.rs +++ b/crates/tower-cmd/src/run.rs @@ -8,6 +8,7 @@ use tower_runtime::{local::LocalApp, App, AppLauncher, OutputReceiver}; use crate::{ output, api, + Error, }; pub fn run_cmd() -> Command { @@ -85,8 +86,22 @@ async fn do_run_local(config: Config, path: PathBuf, mut params: HashMap = AppLauncher::default(); if let Err(err) = launcher - .launch(package, env, secrets, params, HashMap::new()) + .launch(package, env, secrets, params, catalogs) .await { output::runtime_error(err); @@ -236,26 +251,51 @@ fn get_app_name(cmd: Option<(&str, &ArgMatches)>) -> Option { /// get_secrets manages the process of getting secrets from the Tower server in a way that can be /// used by the local runtime during local app execution. -async fn get_secrets(config: &Config, env: &str) -> HashMap { +async fn get_secrets(config: &Config, env: &str) -> Result, Error> { let (private_key, public_key) = crypto::generate_key_pair(); - let mut spinner = output::spinner("Getting secrets..."); - match api::export_secrets(&config, env, false, public_key).await { Ok(res) => { - spinner.success(); - res.secrets - .into_iter() - .map(|secret| { - let decrypted_value = crypto::decrypt(private_key.clone(), secret.encrypted_value.to_string()); - (secret.name, decrypted_value) - }) - .collect() + let mut secrets = HashMap::new(); + + for secret in res.secrets { + // we will decrypt each property and inject it into the vals map. + let decrypted_value = crypto::decrypt(private_key.clone(), secret.encrypted_value.to_string())?; + secrets.insert(secret.name, decrypted_value); + } + + Ok(secrets) + }, + Err(err) => { + output::tower_error(err); + Err(Error::FetchingSecretsFailed) + } + } +} + +/// get_catalogs manages the process of exporting catalogs, decrypting their properties, and +/// preparting them for injection into the environment during app execution +async fn get_catalogs(config: &Config, env: &str) -> Result, Error> { + let (private_key, public_key) = crypto::generate_key_pair(); + + match api::export_catalogs(&config, env, false, public_key).await { + Ok(res) => { + let mut vals = HashMap::new(); + + for catalog in res.catalogs { + // we will decrypt each property and inject it into the vals map. + for property in catalog.properties { + let decrypted_value = crypto::decrypt(private_key.clone(), property.encrypted_value.to_string())?; + let name = create_pyiceberg_catalog_property_name(&catalog.name, &property.name); + vals.insert(name, decrypted_value); + } + } + + Ok(vals) }, Err(err) => { - spinner.failure(); output::tower_error(err); - HashMap::new() + Err(Error::FetchingCatalogsFailed) } } } @@ -323,3 +363,11 @@ async fn monitor_status(mut app: LocalApp) { } } } + +fn create_pyiceberg_catalog_property_name(catalog_name: &str, property_name: &str) -> String { + let catalog_name = catalog_name.replace('.', "_").replace(':', "_").to_uppercase(); + let property_name = property_name.replace('.', "_").replace(':', "_").to_uppercase(); + + format!("PYICEBERG_CATALOG__{}__{}", catalog_name, property_name) +} + diff --git a/crates/tower-cmd/src/secrets.rs b/crates/tower-cmd/src/secrets.rs index 614a8595..1cfb16ca 100644 --- a/crates/tower-cmd/src/secrets.rs +++ b/crates/tower-cmd/src/secrets.rs @@ -103,7 +103,7 @@ pub async fn do_list(config: Config, args: &ArgMatches) { let decrypted_value = crypto::decrypt( private_key.clone(), secret.encrypted_value.clone(), - ); + ).unwrap(); vec![ secret.name.clone(), @@ -203,7 +203,7 @@ async fn encrypt_and_create_secret( output::die("Failed to parse public key"); }); - let encrypted_value = encrypt(public_key, value.to_string()); + let encrypted_value = encrypt(public_key, value.to_string()).unwrap(); let preview = create_preview(value); api::create_secret(&config, name, environment, &encrypted_value, &preview).await