diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 04bf92ab..04017ed5 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -46,7 +46,7 @@ jobs: - name: "cargo build" run: cargo build --all-targets - name: "cargo test" - run: cargo test --bins + run: cargo test --bins --lib tests-release-stable: name: "Tests (release), stable toolchain" runs-on: "ubuntu-24.04" @@ -70,7 +70,7 @@ jobs: - name: "cargo build (release)" run: cargo build --lib --release - name: "cargo test (release)" - run: cargo test --bins --release + run: cargo test --bins --lib --release tests-release-msrv: name: "Tests (release), minimum supported toolchain" runs-on: "ubuntu-24.04" @@ -100,7 +100,7 @@ jobs: - name: "cargo build (release)" run: cargo build --lib --release - name: "cargo test (release)" - run: cargo test --bins --release + run: cargo test --bins --lib --release tests-other-channels: name: "Tests, unstable toolchain" runs-on: "ubuntu-24.04" @@ -128,4 +128,4 @@ jobs: - name: "cargo build" run: cargo build --lib - name: "cargo test" - run: cargo test --bins + run: cargo test --bins --lib diff --git a/Cargo.lock b/Cargo.lock index 836536d0..0e258b1d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3940,11 +3940,14 @@ version = "0.2.1" dependencies = [ "anyhow", "compute-pcrs-lib", + "http 1.4.0", "k8s-openapi", "kopium", "kube", "serde", "serde_json", + "tokio", + "trusted-cluster-operator-test-utils", ] [[package]] diff --git a/Makefile b/Makefile index a23134ba..0fadbc67 100644 --- a/Makefile +++ b/Makefile @@ -202,10 +202,10 @@ equal-conditions: lint: fmt-check clippy vet equal-conditions test: crds-rs - cargo test --workspace --bins + cargo test --workspace --bins --lib test-release: crds-rs - cargo test --workspace --bins --release + cargo test --workspace --bins --lib --release integration-tests: generate trusted-cluster-gen crds-rs RUST_LOG=info REGISTRY=$(REGISTRY) TAG=$(TAG) \ diff --git a/api/trusted-cluster-gen.go b/api/trusted-cluster-gen.go index 793f803f..57bb2c0b 100644 --- a/api/trusted-cluster-gen.go +++ b/api/trusted-cluster-gen.go @@ -92,6 +92,24 @@ func generateOperator(args *Args) error { Name: name, Image: args.image, Command: []string{"/usr/bin/operator"}, + Env: []corev1.EnvVar{ + { + Name: "RELATED_IMAGE_TRUSTEE", + Value: args.trusteeImage, + }, + { + Name: "RELATED_IMAGE_COMPUTE_PCRS", + Value: args.pcrsComputeImage, + }, + { + Name: "RELATED_IMAGE_REGISTRATION_SERVER", + Value: args.registerServerImage, + }, + { + Name: "RELATED_IMAGE_ATTESTATION_KEY_REGISTER", + Value: args.attestationKeyRegisterImage, + }, + }, }, }, }, @@ -140,10 +158,6 @@ func generateTrustedExecutionClusterCR(args *Args) error { Namespace: args.namespace, }, Spec: v1alpha1.TrustedExecutionClusterSpec{ - TrusteeImage: &args.trusteeImage, - PcrsComputeImage: &args.pcrsComputeImage, - RegisterServerImage: &args.registerServerImage, - AttestationKeyRegisterImage: &args.attestationKeyRegisterImage, PublicAttestationKeyRegisterAddr: nil, PublicTrusteeAddr: nil, TrusteeKbsPort: 0, diff --git a/api/v1alpha1/crds.go b/api/v1alpha1/crds.go index 68a6ac71..ba11a574 100644 --- a/api/v1alpha1/crds.go +++ b/api/v1alpha1/crds.go @@ -38,30 +38,6 @@ var ( // +kubebuilder:validation:XValidation:rule="!has(oldSelf.publicAttestationKeyRegisterAddr) || has(self.publicAttestationKeyRegisterAddr)", message="Value is required once set" // +kubebuilder:validation:XValidation:rule="!has(oldSelf.publicTrusteeAddr) || has(self.publicTrusteeAddr)", message="Value is required once set" type TrustedExecutionClusterSpec struct { - // Image reference to Trustee all-in-one image. - // If not specified, uses RELATED_IMAGE_TRUSTEE environment variable from operator deployment. - // +optional - // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="Value is immutable" - TrusteeImage *string `json:"trusteeImage,omitempty"` - - // Image reference to trusted-cluster-operator's compute-pcrs image. - // If not specified, uses RELATED_IMAGE_COMPUTE_PCRS environment variable from operator deployment. - // +optional - // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="Value is immutable" - PcrsComputeImage *string `json:"pcrsComputeImage,omitempty"` - - // Image reference to trusted-cluster-operator's register-server image. - // If not specified, uses RELATED_IMAGE_REGISTRATION_SERVER environment variable from operator deployment. - // +optional - // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="Value is immutable" - RegisterServerImage *string `json:"registerServerImage,omitempty"` - - // Image reference to trusted-cluster-operator's attestation-key-register image. - // If not specified, uses RELATED_IMAGE_ATTESTATION_KEY_REGISTER environment variable from operator deployment. - // +optional - // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="Value is immutable" - AttestationKeyRegisterImage *string `json:"attestationKeyRegisterImage,omitempty"` - // Address where attester can connect to Attestation Key Register // +optional // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="Value is immutable" diff --git a/docs/design/reference-values.md b/docs/design/reference-values.md index 240d1a0c..f8a40a5e 100644 --- a/docs/design/reference-values.md +++ b/docs/design/reference-values.md @@ -93,3 +93,16 @@ A reference value listing for Trustee could then look like this: ## Data flow ![](../pics/rv-flow.png) + +## Ownership + +Unlike `reference-values`, `ApprovedImages` can live independent of a `TrustedExecutionCluster` object. +They can be created without one existing, and reference values are written by jobs (that the `ApprovedImages` also own) to the `image-pcrs` ConfigMap, which is created by the operator and is also independent of `TrustedExecutionClusters`. + +However, the `ApprovedImages` are adopted by the `TrustedExecutionCluster` object, both when created with a `TrustedExecutionCluster` existing and retroactively when created before `TrustedExecutionCluster` creation. +This ensures that removal of a `TrustedExecutionCluster` acts as complete uninstallation. +Finalizers on the `ApprovedImages` ensure the PCR values are removed back out of `image-pcrs` again. + +## Ownership flow + +![](../pics/image-flow.png) diff --git a/docs/pics/image-flow.png b/docs/pics/image-flow.png new file mode 100644 index 00000000..4003248c Binary files /dev/null and b/docs/pics/image-flow.png differ diff --git a/docs/usage/os-and-node-lifecycle.md b/docs/usage/os-and-node-lifecycle.md index b6362f50..ae4e66d8 100644 --- a/docs/usage/os-and-node-lifecycle.md +++ b/docs/usage/os-and-node-lifecycle.md @@ -52,6 +52,9 @@ Machines booting this image can now register and attest. **NB:** Updating nodes is not supported yet. Updates incur one intermediary stage of PCR values (assuming no further update on that boot) because kernel update is effective one boot _before_ shim & GRUB update. +**NB:** The TrustedExecutionCluster object adopts ApprovedImages, including those that lived before it. +This ensures that removal of a TrustedExecutionCluster acts as complete uninstallation. + # Disallowing a bootable container image For the example above: diff --git a/lib/Cargo.toml b/lib/Cargo.toml index 4e424953..f6cc3d37 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -21,3 +21,6 @@ serde_json.workspace = true [dev-dependencies] # Only a generate dependency, not a Rust dependency. Included here for auto-updates. kopium = "0.23.0" +http.workspace = true +tokio.workspace = true +trusted-cluster-operator-test-utils = { path = "../test_utils" } diff --git a/lib/src/lib.rs b/lib/src/lib.rs index b5079494..d1d31b52 100644 --- a/lib/src/lib.rs +++ b/lib/src/lib.rs @@ -19,10 +19,10 @@ pub use kopium::trustedexecutionclusters::*; pub use vendor_kopium::virtualmachineinstances; pub use vendor_kopium::virtualmachines; -use anyhow::Context; +use anyhow::{Context, Result, anyhow}; use conditions::*; use k8s_openapi::apimachinery::pkg::apis::meta::v1::{Condition, OwnerReference, Time}; -use kube::Resource; +use kube::{Api, Client, Resource}; #[macro_export] macro_rules! update_status { @@ -105,7 +105,7 @@ pub fn committed_condition( /// Generate an OwnerReference for any Kubernetes resource pub fn generate_owner_reference>( object: &T, -) -> anyhow::Result { +) -> Result { let name = object.meta().name.clone(); let uid = object.meta().uid.clone(); let kind = T::kind(&()).to_string(); @@ -119,32 +119,108 @@ pub fn generate_owner_reference>( }) } -/// Get the single TrustedExecutionCluster in the namespace -/// -/// Returns an error if: -/// - No TrustedExecutionCluster is found -/// - More than one TrustedExecutionCluster is found (not supported) -pub async fn get_trusted_execution_cluster( - client: kube::Client, -) -> anyhow::Result { - use kube::Api; - +pub async fn get_opt_trusted_execution_cluster( + client: Client, +) -> Result> { let namespace = client.default_namespace().to_string(); let clusters: Api = Api::default_namespaced(client); - let params = Default::default(); - let mut list = clusters.list(¶ms).await?; - - if list.items.is_empty() { - return Err(anyhow::Error::msg(format!( - "No TrustedExecutionCluster found in namespace {namespace}. \ - Ensure that this service is in the same namespace as the TrustedExecutionCluster." - ))); - } else if list.items.len() > 1 { - return Err(anyhow::Error::msg(format!( + let list = clusters.list(&Default::default()).await?; + if list.items.len() > 1 { + return Err(anyhow!( "More than one TrustedExecutionCluster found in namespace {namespace}. \ trusted-cluster-operator does not support more than one TrustedExecutionCluster." - ))); + )); + } + Ok(list.items.into_iter().next()) +} + +/// Get the single TrustedExecutionCluster in the namespace +pub async fn get_trusted_execution_cluster(client: Client) -> Result { + let namespace = client.default_namespace().to_string(); + let cluster = get_opt_trusted_execution_cluster(client).await; + let err = anyhow!( + "No TrustedExecutionCluster found in namespace {namespace}. \ + Ensure that this service is in the same namespace as the TrustedExecutionCluster." + ); + cluster.and_then(|c| c.ok_or(err)) +} + +#[cfg(test)] +mod tests { + use super::*; + use http::StatusCode; + use kube::api::ObjectList; + use trusted_cluster_operator_test_utils::mock_client::*; + + #[tokio::test] + async fn test_get_some_trusted_execution_cluster() { + let clos = async |_, _| { + let object_list = ObjectList { + items: vec![dummy_cluster()], + types: Default::default(), + metadata: Default::default(), + }; + Ok(serde_json::to_string(&object_list).unwrap()) + }; + count_check!(1, clos, |client| { + let res = get_opt_trusted_execution_cluster(client).await; + assert!(res.unwrap().is_some()); + }); + } + + #[tokio::test] + async fn test_get_none_trusted_execution_cluster() { + let clos = async |_, _| { + let object_list = ObjectList:: { + items: vec![], + types: Default::default(), + metadata: Default::default(), + }; + Ok(serde_json::to_string(&object_list).unwrap()) + }; + count_check!(1, clos, |client| { + let res = get_opt_trusted_execution_cluster(client).await; + assert!(res.unwrap().is_none()); + }); + } + + #[tokio::test] + async fn test_non_unique_trusted_execution_cluster() { + let clos = async |_, _| { + let object_list = ObjectList { + items: vec![dummy_cluster(), dummy_cluster()], + types: Default::default(), + metadata: Default::default(), + }; + Ok(serde_json::to_string(&object_list).unwrap()) + }; + count_check!(1, clos, |client| { + let err = get_opt_trusted_execution_cluster(client).await.unwrap_err(); + assert!(err.to_string().contains("More than one")); + }); + } + + #[tokio::test] + async fn test_get_opt_trusted_execution_cluster_error() { + let clos = async |_, _| Err(StatusCode::INTERNAL_SERVER_ERROR); + count_check!(1, clos, |client| { + assert!(get_opt_trusted_execution_cluster(client).await.is_err()); + }); } - Ok(list.items.pop().unwrap()) + #[tokio::test] + async fn test_get_no_trusted_execution_cluster() { + let clos = async |_, _| { + let object_list = ObjectList:: { + items: vec![], + types: Default::default(), + metadata: Default::default(), + }; + Ok(serde_json::to_string(&object_list).unwrap()) + }; + count_check!(1, clos, |client| { + let err = get_trusted_execution_cluster(client).await.unwrap_err(); + assert!(err.to_string().contains("No TrustedExecutionCluster found")); + }); + } } diff --git a/operator/src/attestation_key_register.rs b/operator/src/attestation_key_register.rs index 3bda32be..0697cb96 100644 --- a/operator/src/attestation_key_register.rs +++ b/operator/src/attestation_key_register.rs @@ -381,7 +381,9 @@ pub async fn launch_secret_ak_controller(client: Client) { #[cfg(test)] mod tests { use super::*; + use http::{Method, Request, StatusCode}; use trusted_cluster_operator_test_utils::mock_client::*; + use trusted_cluster_operator_test_utils::test_error_method; #[tokio::test] async fn test_create_ak_register_depl_success() { @@ -396,7 +398,7 @@ mod tests { let clos = |client| { create_attestation_key_register_deployment(client, Default::default(), "image") }; - test_create_error(clos).await; + test_error_method!(clos, Method::POST); } #[tokio::test] @@ -410,6 +412,6 @@ mod tests { async fn test_create_ak_register_svc_error() { let clos = |client| create_attestation_key_register_service(client, Default::default(), Some(80)); - test_create_error(clos).await; + test_error_method!(clos, Method::POST); } } diff --git a/operator/src/lib.rs b/operator/src/lib.rs index bd7e4a09..f39c0cf9 100644 --- a/operator/src/lib.rs +++ b/operator/src/lib.rs @@ -8,8 +8,7 @@ // // Use in other crates is not an intended purpose. -use k8s_openapi::apimachinery::pkg::apis::meta::v1::OwnerReference; -use kube::{Client, runtime::controller::Action}; +use kube::runtime::controller::Action; use log::info; use std::fmt::{Debug, Display}; use std::{sync::Arc, time::Duration}; @@ -17,13 +16,6 @@ use std::{sync::Arc, time::Duration}; // Re-export common functions from the lib pub use trusted_cluster_operator_lib::generate_owner_reference; -#[derive(Clone)] -pub struct RvContextData { - pub client: Client, - pub owner_reference: OwnerReference, - pub pcrs_compute_image: String, -} - #[derive(Debug, thiserror::Error)] pub enum ControllerError { #[error("{0}")] diff --git a/operator/src/main.rs b/operator/src/main.rs index 06f4c88a..eac6be88 100644 --- a/operator/src/main.rs +++ b/operator/src/main.rs @@ -4,7 +4,7 @@ // SPDX-License-Identifier: MIT use std::env; -use std::sync::{Arc, Mutex}; +use std::sync::Arc; use std::time::Duration; use anyhow::Result; @@ -34,21 +34,6 @@ use operator::*; /// Default version tag for operator-managed component images const COMPONENT_VERSION: &str = "0.2.0"; -/// Get image from CR spec, falling back to environment variable (set by OLM), then default. -/// This enables disconnected/airgap installations where OLM rewrites RELATED_IMAGE_* env vars. -fn get_image_or_env(cr_image: &Option, env_var: &str, default: &str) -> String { - cr_image - .clone() - .or_else(|| env::var(env_var).ok()) - .unwrap_or_else(|| default.to_string()) -} - -struct ClusterContext { - client: Client, - /// UID of cluster that watchers are based on - uid: Mutex>, -} - fn is_installed(status: Option) -> bool { let chk = |c: &Condition| c.type_ == INSTALLED_CONDITION && c.status == "True"; status @@ -57,50 +42,9 @@ fn is_installed(status: Option) -> bool { .unwrap_or(false) } -/// Launch reference value-related watchers. Is run once per TrustedExecutionCluster and operator -/// process. Returns whether watchers were launched. -async fn launch_rv_watchers( - cluster: Arc, - ctx: Arc, - name: &str, -) -> Result { - let client = ctx.client.clone(); - let mut launch_watchers = false; - if let Ok(mut ctx_uid) = ctx.uid.lock() { - let err = format!("TrustedExecutionCluster {name} had no UID"); - let cluster_uid = cluster.metadata.uid.clone().expect(&err); - if ctx_uid.is_none() || ctx_uid.clone() != Some(cluster_uid.clone()) { - launch_watchers = true; - *ctx_uid = Some(cluster_uid); - } - } else { - warn!("Failed to acquire lock on context UID store"); - } - if launch_watchers { - info!( - "First registration of TrustedExecutionCluster {name} by this operator. \ - Launching reference value watchers." - ); - let owner_reference = generate_owner_reference(&*cluster)?; - let pcrs_compute_image = get_image_or_env( - &cluster.spec.pcrs_compute_image, - "RELATED_IMAGE_COMPUTE_PCRS", - &format!("quay.io/trusted-execution-clusters/compute-pcrs:{COMPONENT_VERSION}"), - ); - let rv_ctx = RvContextData { - client, - owner_reference: owner_reference.clone(), - pcrs_compute_image, - }; - reference_values::launch_rv_image_controller(rv_ctx.clone()).await; - reference_values::launch_rv_job_controller(rv_ctx.clone()).await; - } - Ok(launch_watchers) -} - async fn reconcile( cluster: Arc, - ctx: Arc, + client: Arc, ) -> Result { let generation = cluster.metadata.generation; let known_address = cluster.spec.public_trustee_addr.is_some(); @@ -109,7 +53,7 @@ async fn reconcile( known_trustee_address_condition(known_address, generation, existing_status); let mut conditions = Some(vec![address_condition]); - let kube_client = ctx.client.clone(); + let kube_client = Arc::unwrap_or_clone(client); let err = "trusted execution cluster had no name"; let name = &cluster.metadata.name.clone().expect(err); let clusters: Api = Api::default_namespaced(kube_client.clone()); @@ -123,8 +67,6 @@ async fn reconcile( return Ok(Action::await_change()); } - let _ = launch_rv_watchers(cluster.clone(), ctx.clone(), name).await?; - if is_installed(cluster.status.clone()) { return Ok(Action::await_change()); } @@ -156,7 +98,9 @@ async fn reconcile( install_trustee_configuration(kube_client.clone(), &cluster).await?; install_register_server(kube_client.clone(), &cluster).await?; - install_attestation_key_register(kube_client, &cluster).await?; + install_attestation_key_register(kube_client.clone(), &cluster).await?; + reference_values::adopt_approved_images(kube_client, &cluster).await?; + let condition = installed_condition(INSTALLED_REASON, generation, existing_status); conditions.as_mut().unwrap().push(condition); update_status!(clusters, name, TrustedExecutionClusterStatus { conditions })?; @@ -174,11 +118,6 @@ async fn install_trustee_configuration( Err(e) => error!("Failed to create the KBS configuration configmap: {e}"), } - match reference_values::create_pcrs_config_map(client.clone(), owner_reference.clone()).await { - Ok(_) => info!("Created bare configmap for PCRs"), - Err(e) => error!("Failed to create the PCRs configmap: {e}"), - } - match trustee::generate_attestation_policy(client.clone(), owner_reference.clone()).await { Ok(_) => info!("Generate configmap for the attestation policy",), Err(e) => error!("Failed to create the attestation policy configmap: {e}"), @@ -190,11 +129,9 @@ async fn install_trustee_configuration( Err(e) => error!("Failed to create the KBS service: {e}"), } - let trustee_image = get_image_or_env( - &cluster.spec.trustee_image, - "RELATED_IMAGE_TRUSTEE", - "quay.io/trusted-execution-clusters/key-broker-service:20260106", - ); + let trustee_image = env::var("RELATED_IMAGE_TRUSTEE") + .ok() + .unwrap_or("quay.io/trusted-execution-clusters/key-broker-service:20260106".to_string()); match trustee::generate_kbs_deployment(client, owner_reference, &trustee_image).await { Ok(_) => info!("Generate the KBS deployment"), Err(e) => error!("Failed to create the KBS deployment: {e}"), @@ -206,11 +143,10 @@ async fn install_trustee_configuration( async fn install_register_server(client: Client, cluster: &TrustedExecutionCluster) -> Result<()> { let owner_reference = generate_owner_reference(cluster)?; - let register_server_image = get_image_or_env( - &cluster.spec.register_server_image, - "RELATED_IMAGE_REGISTRATION_SERVER", - &format!("quay.io/trusted-execution-clusters/registration-server:{COMPONENT_VERSION}"), - ); + let env = "RELATED_IMAGE_REGISTRATION_SERVER"; + let default_image = + format!("quay.io/trusted-execution-clusters/registration-server:{COMPONENT_VERSION}"); + let register_server_image = env::var(env).ok().unwrap_or(default_image); match register_server::create_register_server_deployment( client.clone(), owner_reference.clone(), @@ -239,11 +175,10 @@ async fn install_attestation_key_register( ) -> Result<()> { let owner_reference = generate_owner_reference(cluster)?; - let attestation_key_register_image = get_image_or_env( - &cluster.spec.attestation_key_register_image, - "RELATED_IMAGE_ATTESTATION_KEY_REGISTER", - &format!("quay.io/trusted-execution-clusters/attestation-key-register:{COMPONENT_VERSION}"), - ); + let env = "RELATED_IMAGE_ATTESTATION_KEY_REGISTER"; + let default_image = + format!("quay.io/trusted-execution-clusters/attestation-key-register:{COMPONENT_VERSION}"); + let attestation_key_register_image = env::var(env).ok().unwrap_or(default_image); match attestation_key_register::create_attestation_key_register_deployment( client.clone(), owner_reference.clone(), @@ -278,18 +213,16 @@ async fn main() -> Result<()> { info!("trusted execution clusters operator",); let cl: Api = Api::default_namespaced(kube_client.clone()); - // Launch all controllers except reference value-related ones register_server::launch_keygen_controller(kube_client.clone()).await; attestation_key_register::launch_ak_controller(kube_client.clone()).await; attestation_key_register::launch_machine_ak_controller(kube_client.clone()).await; attestation_key_register::launch_secret_ak_controller(kube_client.clone()).await; + reference_values::create_pcrs_config_map(kube_client.clone()).await?; + reference_values::launch_rv_image_controller(kube_client.clone()).await; + reference_values::launch_rv_job_controller(kube_client.clone()).await; - let ctx = Arc::new(ClusterContext { - client: kube_client, - uid: Mutex::new(None), - }); Controller::new(cl, watcher::Config::default()) - .run(reconcile, controller_error_policy, ctx) + .run(reconcile, controller_error_policy, Arc::new(kube_client)) .for_each(controller_info) .await; @@ -306,47 +239,6 @@ mod tests { use super::*; use trusted_cluster_operator_test_utils::mock_client::*; - fn dummy_cluster_ctx(client: Client) -> ClusterContext { - ClusterContext { - client, - uid: Mutex::new(None), - } - } - - #[tokio::test] - async fn test_launch_watchers_create() { - let clos = async |req, ctr| panic!("unexpected API interaction: {req:?}, counter {ctr}"); - count_check!(0, clos, |client| { - let cluster = Arc::new(dummy_cluster()); - let ctx = Arc::new(dummy_cluster_ctx(client)); - assert!(launch_rv_watchers(cluster, ctx, "test").await.unwrap()); - }); - } - - #[tokio::test] - async fn test_launch_watchers_update() { - let clos = async |req, ctr| panic!("unexpected API interaction: {req:?}, counter {ctr}"); - count_check!(0, clos, |client| { - let cluster = Arc::new(dummy_cluster()); - let mut ctx = dummy_cluster_ctx(client); - ctx.uid = Mutex::new(Some("def".to_string())); - let result = launch_rv_watchers(cluster, Arc::new(ctx), "test"); - assert!(result.await.unwrap()); - }); - } - - #[tokio::test] - async fn test_launch_watchers_existing() { - let clos = async |req, ctr| panic!("unexpected API interaction: {req:?}, counter {ctr}"); - count_check!(0, clos, |client| { - let cluster = dummy_cluster(); - let mut ctx = dummy_cluster_ctx(client); - ctx.uid = Mutex::new(cluster.metadata.uid.clone()); - let result = launch_rv_watchers(Arc::new(cluster), Arc::new(ctx), "test"); - assert!(!result.await.unwrap()); - }); - } - #[tokio::test] async fn test_reconcile_uninstalling() { let clos = async |req: Request, ctr| match req.method() { @@ -359,7 +251,7 @@ mod tests { count_check!(1, clos, |client| { let mut cluster = dummy_cluster(); cluster.metadata.deletion_timestamp = Some(Time(Timestamp::now())); - let result = reconcile(Arc::new(cluster), Arc::new(dummy_cluster_ctx(client))).await; + let result = reconcile(Arc::new(cluster), Arc::new(client)).await; assert_eq!(result.unwrap(), Action::await_change()); }); } @@ -374,19 +266,16 @@ mod tests { metadata: Default::default(), }; Ok(serde_json::to_string(&object_list).unwrap()) - } else if 1 < ctr && ctr < 4 { - // Watchers - Ok(serde_json::to_string(&dummy_cluster()).unwrap()) - } else if ctr == 4 && req.method() == Method::PATCH { + } else if ctr == 1 && req.method() == Method::PATCH { assert_body_contains(req, NOT_INSTALLED_REASON_NON_UNIQUE).await; Ok(serde_json::to_string(&dummy_cluster()).unwrap()) } else { panic!("unexpected API interaction: {req:?}, counter {ctr}"); } }; - count_check!(4, clos, |client| { + count_check!(2, clos, |client| { let cluster = Arc::new(dummy_cluster()); - let result = reconcile(cluster, Arc::new(dummy_cluster_ctx(client))).await; + let result = reconcile(cluster, Arc::new(client)).await; assert_eq!(result.unwrap(), Action::requeue(Duration::from_secs(60))); }); } @@ -397,9 +286,9 @@ mod tests { r if r.method() == Method::GET => Err(StatusCode::INTERNAL_SERVER_ERROR), _ => panic!("unexpected API interaction: {req:?}"), }; - count_check!(3, clos, |client| { + count_check!(1, clos, |client| { let cluster = Arc::new(dummy_cluster()); - let result = reconcile(cluster, Arc::new(dummy_cluster_ctx(client))).await; + let result = reconcile(cluster, Arc::new(client)).await; assert!(result.is_err()); }); } diff --git a/operator/src/reference_values.rs b/operator/src/reference_values.rs index bf16fab4..a244685e 100644 --- a/operator/src/reference_values.rs +++ b/operator/src/reference_values.rs @@ -12,10 +12,9 @@ use k8s_openapi::{ core::v1::{ConfigMap, Container, ImageVolumeSource, Volume, VolumeMount}, core::v1::{Pod, PodSpec, PodTemplateSpec}, }, - apimachinery::pkg::apis::meta::v1::OwnerReference, jiff::Timestamp, }; -use kube::api::{DeleteParams, ListParams, ObjectMeta}; +use kube::api::{DeleteParams, ListParams, ObjectMeta, Patch}; use kube::runtime::{ controller::{Action, Controller}, finalizer, @@ -28,13 +27,13 @@ use oci_client::secrets::RegistryAuth; use oci_spec::image::ImageConfiguration; use openssl::hash::{MessageDigest, hash}; use serde::Deserialize; +use serde_json::json; use std::{collections::BTreeMap, sync::Arc, time::Duration}; +use crate::COMPONENT_VERSION; use crate::trustee::{self, get_image_pcrs}; -use operator::{ - ControllerError, RvContextData, controller_error_policy, controller_info, - create_or_info_if_exists, -}; +use operator::ControllerError; +use operator::{controller_error_policy, controller_info, create_or_info_if_exists}; use trusted_cluster_operator_lib::{conditions::*, reference_values::*, *}; const JOB_LABEL_KEY: &str = "kind"; @@ -50,7 +49,7 @@ struct ComputePcrsOutput { pcrs: Vec, } -pub async fn create_pcrs_config_map(client: Client, owner_reference: OwnerReference) -> Result<()> { +pub async fn create_pcrs_config_map(client: Client) -> Result<()> { let empty_data = BTreeMap::from([( PCR_CONFIG_FILE.to_string(), serde_json::to_string(&ImagePcrs::default())?, @@ -58,7 +57,6 @@ pub async fn create_pcrs_config_map(client: Client, owner_reference: OwnerRefere let config_map = ConfigMap { metadata: ObjectMeta { name: Some(PCR_CONFIG_MAP.to_string()), - owner_references: Some(vec![owner_reference]), ..Default::default() }, data: Some(empty_data), @@ -117,32 +115,33 @@ fn build_compute_pcrs_pod_spec( } } -async fn job_reconcile(job: Arc, ctx: Arc) -> Result { +async fn job_reconcile(job: Arc, client: Arc) -> Result { let err = "Job changed, but had no name"; let name = &job.metadata.name.clone().context(err)?; let err = format!("Job {name} changed, but had no status"); let status = &job.status.clone().context(err)?; + let kube_client = Arc::unwrap_or_clone(client); if status.completion_time.is_none() { info!("Job {name} changed, but had not completed"); return Ok(Action::requeue(Duration::from_secs(300))); } - let jobs: Api = Api::default_namespaced(ctx.client.clone()); + let jobs: Api = Api::default_namespaced(kube_client.clone()); // Foreground deletion: Delete the pod too let delete = jobs.delete(name, &DeleteParams::foreground()).await; delete.map_err(Into::::into)?; - trustee::update_reference_values(Arc::unwrap_or_clone(ctx)).await?; + trustee::update_reference_values(kube_client).await?; Ok(Action::await_change()) } -pub async fn launch_rv_job_controller(ctx: RvContextData) { - let jobs: Api = Api::default_namespaced(ctx.client.clone()); +pub async fn launch_rv_job_controller(client: Client) { + let jobs: Api = Api::default_namespaced(client.clone()); let watcher = watcher::Config { label_selector: Some(format!("{JOB_LABEL_KEY}={PCR_COMMAND_NAME}")), ..Default::default() }; tokio::spawn( Controller::new(jobs, watcher) - .run(job_reconcile, controller_error_policy, Arc::new(ctx)) + .run(job_reconcile, controller_error_policy, Arc::new(client)) .for_each(controller_info), ); } @@ -160,13 +159,15 @@ fn get_job_name(boot_image: &str) -> Result { Ok(trimmed) } -async fn compute_fresh_pcrs( - ctx: RvContextData, - resource_name: &str, - boot_image: &str, -) -> anyhow::Result<()> { - let job_name = get_job_name(boot_image)?; - let pod_spec = build_compute_pcrs_pod_spec(resource_name, boot_image, &ctx.pcrs_compute_image); +async fn compute_fresh_pcrs(client: Client, image: &ApprovedImage) -> anyhow::Result<()> { + let job_name = get_job_name(&image.spec.image)?; + let env = "RELATED_IMAGE_COMPUTE_PCRS"; + let default_image = + format!("quay.io/trusted-execution-clusters/compute-pcrs:{COMPONENT_VERSION}"); + let pcrs_compute_image = std::env::var(env).ok().unwrap_or(default_image); + let resource_name = image.metadata.name.as_ref().unwrap(); + let pod_spec = + build_compute_pcrs_pod_spec(resource_name, &image.spec.image, &pcrs_compute_image); let job = Job { metadata: ObjectMeta { name: Some(job_name.clone()), @@ -174,7 +175,7 @@ async fn compute_fresh_pcrs( JOB_LABEL_KEY.to_string(), PCR_COMMAND_NAME.to_string(), )])), - owner_references: Some(vec![ctx.owner_reference]), + owner_references: Some(vec![generate_owner_reference(image)?]), ..Default::default() }, spec: Some(JobSpec { @@ -192,121 +193,98 @@ async fn compute_fresh_pcrs( }), ..Default::default() }; - create_or_info_if_exists!(ctx.client, Job, job); + create_or_info_if_exists!(client, Job, job); + Ok(()) +} + +async fn adopt_approved_image( + client: Client, + image_name: &str, + cluster: &TrustedExecutionCluster, +) -> Result<()> { + let images: Api = Api::default_namespaced(client.clone()); + let default = "".to_string(); + let cluster_name = cluster.metadata.name.as_ref().unwrap_or(&default); + info!( + "Adding owner reference from TrustedExecutionCluster {cluster_name} \ + to ApprovedImage {image_name}" + ); + let json = json!({ + "metadata": { + "ownerReferences": [generate_owner_reference(cluster)?], + } + }); + let patch = Patch::Merge(&json); + let params = Default::default(); + images.patch(image_name, ¶ms, &patch).await?; + Ok(()) +} + +pub async fn adopt_approved_images( + client: Client, + cluster: &TrustedExecutionCluster, +) -> Result<()> { + let images: Api = Api::default_namespaced(client.clone()); + let images_list = images.list(&Default::default()).await?; + for image in images_list.items.iter() { + if image.metadata.deletion_timestamp.is_none() + && let Some(name) = image.metadata.name.as_ref() + { + adopt_approved_image(client.clone(), name, cluster).await?; + } + } Ok(()) } async fn image_reconcile( image: Arc, - ctx: Arc, + client: Arc, ) -> Result { - let kube_client = ctx.client.clone(); + let kube_client = Arc::::unwrap_or_clone(client); let err = "ApprovedImage had no name"; - let name = image.metadata.name.clone().expect(err); + let name = image.metadata.name.clone().context(err)?; + let cluster = get_opt_trusted_execution_cluster(kube_client.clone()) + .await + .map_err(|e| -> ControllerError { e.into() })?; let images: Api = Api::default_namespaced(kube_client.clone()); - let finalizer_ctx = Arc::unwrap_or_clone(ctx); finalizer(&images, APPROVED_IMAGE_FINALIZER, image, |ev| async { match ev { - Event::Apply(image) => image_add_reconcile(finalizer_ctx, &image).await, - Event::Cleanup(_) => { - // Check if the TrustedExecutionCluster is being deleted - // If so, skip updating reference values as everything will be cleaned up - let tec_name = finalizer_ctx.owner_reference.name.clone(); - let tecs: Api = - Api::default_namespaced(kube_client.clone()); - - match tecs.get(&tec_name).await { - Ok(tec) if tec.metadata.deletion_timestamp.is_some() => { - // TEC is being deleted, skip disallow_image - info!( - "TrustedExecutionCluster {tec_name} is being deleted, \ - skipping disallow_image for {name}" - ); - Ok(Action::await_change()) - } - Err(kube::Error::Api(ae)) if ae.code == 404 => { - // TEC already deleted, skip disallow_image - info!( - "TrustedExecutionCluster {tec_name} not found, \ - skipping disallow_image for {name}" - ); - Ok(Action::await_change()) - } - Ok(_) => { - // TEC exists and is not being deleted, proceed with disallow_image - disallow_image(finalizer_ctx, &name) - .await - .map(|_| Action::await_change()) - .map_err(|e| { - finalizer::Error::::CleanupFailed(e.into()) - }) - } - Err(e) => { - // Some other error occurred - Err(finalizer::Error::::CleanupFailed( - anyhow::Error::from(e).into(), - )) - } - } - } + Event::Apply(image) => image_add_reconcile(kube_client, &image, cluster) + .await + .map_err(|e| finalizer::Error::::ApplyFailed(e.into())), + Event::Cleanup(image) => image_remove_reconcile(kube_client, image, cluster) + .await + .map_err(|e| finalizer::Error::::CleanupFailed(e.into())), } }) .await - .map_err(|e| anyhow!("failed to reconcile on image: {e}").into()) + .map_err(|e| anyhow!("failed to reconcile on image {name}: {e}").into()) } async fn image_add_reconcile( - ctx: RvContextData, + client: Client, image: &ApprovedImage, -) -> Result> { - let kube_client = ctx.client.clone(); + cluster: Option, +) -> Result { let name = image.metadata.name.as_ref().unwrap(); - + let uid_owns = |uid: &String| { + let refs = image.metadata.owner_references.as_ref(); + refs.map(|os| os.iter().any(|o| o.uid == *uid)) + }; + let cluster_owns = |cluster: &TrustedExecutionCluster| { + let uid = cluster.metadata.uid.as_ref(); + uid.and_then(uid_owns).unwrap_or(false) + }; // Adopt the image by adding TEC as owner reference if not already owned - let tec_uid = &ctx.owner_reference.uid; - let already_owned = image - .metadata - .owner_references - .as_ref() - .map(|owners| { - owners - .iter() - .any(|owner| owner.kind == "TrustedExecutionCluster" && owner.uid == *tec_uid) - }) - .unwrap_or(false); - - if !already_owned { - use kube::api::{Patch, PatchParams}; - use serde_json::json; - - info!("Adding owner reference from ApprovedImage {name} to TrustedExecutionCluster"); - - let patch = json!({ - "metadata": { - "ownerReferences": [ctx.owner_reference] - } - }); - - let images: Api = Api::default_namespaced(kube_client.clone()); - images - .patch(name, &PatchParams::default(), &Patch::Merge(&patch)) - .await - .map_err(|e| { - finalizer::Error::::ApplyFailed(anyhow::Error::from(e).into()) - })?; + if let Some(cluster) = cluster + && cluster_owns(&cluster) + { + adopt_approved_image(client.clone(), name, &cluster).await?; } - let (action, reason) = match handle_new_image(ctx, name, &image.spec.image).await { - Ok(reason) => { - let action = match reason { - NOT_COMMITTED_REASON_COMPUTING | NOT_COMMITTED_REASON_PENDING => { - Action::requeue(Duration::from_secs(10)) - } - _ => Action::await_change(), - }; - (action, reason) - } + let (action, reason) = match handle_new_image(client.clone(), image).await { + Ok(reason) => (Action::await_change(), reason), Err(e) => { warn!("PCR computation for {name} failed: {e}"); let action = Action::requeue(Duration::from_secs(60)); @@ -315,17 +293,40 @@ async fn image_add_reconcile( }; let committed = committed_condition(reason, image.metadata.generation, &image.status); let conditions = Some(vec![committed]); - let images: Api = Api::default_namespaced(kube_client); - update_status!(images, &name, ApprovedImageStatus { conditions }) - .map_err(|e| finalizer::Error::::ApplyFailed(e.into()))?; + let images: Api = Api::default_namespaced(client); + update_status!(images, &name, ApprovedImageStatus { conditions })?; Ok(action) } -pub async fn launch_rv_image_controller(ctx: RvContextData) { - let images: Api = Api::default_namespaced(ctx.client.clone()); +async fn image_remove_reconcile( + client: Client, + image: Arc, + cluster: Option, +) -> Result { + let default = "".to_string(); + let name = image.metadata.name.as_ref().unwrap_or(&default); + if cluster.is_none() { + info!("No TrustedExecutionCluster found, skipping disallow_image for {name}"); + return Ok(Action::await_change()); + } + let cluster = cluster.unwrap(); + let tec_name = cluster.metadata.name.unwrap_or("".to_string()); + if cluster.metadata.deletion_timestamp.is_some() { + info!( + "TrustedExecutionCluster {tec_name} is being deleted, \ + skipping disallow_image for {name}" + ); + return Ok(Action::await_change()); + } + disallow_image(client, name).await?; + Ok(Action::await_change()) +} + +pub async fn launch_rv_image_controller(client: Client) { + let images: Api = Api::default_namespaced(client.clone()); tokio::spawn( Controller::new(images, Default::default()) - .run(image_reconcile, controller_error_policy, Arc::new(ctx)) + .run(image_reconcile, controller_error_policy, Arc::new(client)) .for_each(controller_info), ); } @@ -341,21 +342,18 @@ async fn is_pending(client: &Client, resource_name: &str) -> Result { .is_some_and(|phase| phase == "Pending")) } -pub async fn handle_new_image( - ctx: RvContextData, - resource_name: &str, - boot_image: &str, -) -> Result<&'static str> { - let config_maps: Api = Api::default_namespaced(ctx.client.clone()); +pub async fn handle_new_image(client: Client, image: &ApprovedImage) -> Result<&'static str> { + let resource_name = image.metadata.name.as_ref().unwrap(); + let boot_image = image.spec.image.as_ref(); + let config_maps: Api = Api::default_namespaced(client.clone()); let mut image_pcrs_map = config_maps.get(PCR_CONFIG_MAP).await?; let mut image_pcrs = get_image_pcrs(image_pcrs_map.clone())?; if let Some(pcr) = image_pcrs.0.get(resource_name) && pcr.reference == boot_image { info!("Image {boot_image} was to be allowed, but already was allowed"); - return trustee::update_reference_values(ctx) - .await - .map(|_| COMMITTED_REASON); + let res = trustee::update_reference_values(client).await; + return res.map(|_| COMMITTED_REASON); } let image_ref: oci_client::Reference = boot_image.parse()?; if image_ref.digest().is_none() { @@ -369,7 +367,7 @@ pub async fn handle_new_image( let compute_pcrs = match label { Err(ref e) => { warn!("Fetching PCR label for {image_ref} failed: {e}. Falling back to computation."); - if is_pending(&ctx.client, resource_name).await? { + if is_pending(&client, resource_name).await? { return Ok(NOT_COMMITTED_REASON_PENDING); } true @@ -381,9 +379,8 @@ pub async fn handle_new_image( _ => false, }; if compute_pcrs { - return compute_fresh_pcrs(ctx, resource_name, boot_image) - .await - .map(|_| NOT_COMMITTED_REASON_COMPUTING); + let res = compute_fresh_pcrs(client, image).await; + return res.map(|_| NOT_COMMITTED_REASON_COMPUTING); } let image_pcr = ImagePcr { @@ -393,47 +390,67 @@ pub async fn handle_new_image( }; image_pcrs.0.insert(resource_name.to_string(), image_pcr); update_image_pcrs!(config_maps, image_pcrs_map, image_pcrs); - trustee::update_reference_values(ctx) + trustee::update_reference_values(client) .await .map(|_| COMMITTED_REASON) } -pub async fn disallow_image(ctx: RvContextData, resource_name: &str) -> Result<()> { - let config_maps: Api = Api::default_namespaced(ctx.client.clone()); +pub async fn disallow_image(client: Client, resource_name: &str) -> Result<()> { + let config_maps: Api = Api::default_namespaced(client.clone()); let mut image_pcrs_map = config_maps.get(PCR_CONFIG_MAP).await?; let mut image_pcrs = get_image_pcrs(image_pcrs_map.clone())?; if image_pcrs.0.remove(resource_name).is_none() { info!("Image {resource_name} was to be disallowed, but already was not allowed"); } update_image_pcrs!(config_maps, image_pcrs_map, image_pcrs); - trustee::update_reference_values(ctx).await + trustee::update_reference_values(client).await } #[cfg(test)] mod tests { use super::*; use crate::test_utils::*; - use http::{Method, Request}; + use http::{Method, Request, StatusCode}; use k8s_openapi::api::batch::v1::JobStatus; use k8s_openapi::apimachinery::pkg::apis::meta::v1::Time; + use kube::api::ObjectList; + use kube::client::Body; use trusted_cluster_operator_test_utils::mock_client::*; + use trusted_cluster_operator_test_utils::test_error_method; + + const DUMMY_IMAGE_REF: &str = + "quay.io/some-ref@sha256:e71dad00aa0e3d70540e726a0c66407e3004d96e045ab6c253186e327a2419e5"; #[tokio::test] async fn test_create_pcrs_cm_success() { - let clos = |client| create_pcrs_config_map(client, Default::default()); + let clos = |client| create_pcrs_config_map(client); test_create_success::<_, _, ConfigMap>(clos).await; } #[tokio::test] async fn test_create_pcrs_cm_exists() { - let clos = |client| create_pcrs_config_map(client, Default::default()); + let clos = |client| create_pcrs_config_map(client); test_create_already_exists(clos).await; } #[tokio::test] async fn test_create_pcrs_cm_error() { - let clos = |client| create_pcrs_config_map(client, Default::default()); - test_create_error(clos).await; + let clos = |client| create_pcrs_config_map(client); + test_error_method!(clos, Method::POST); + } + + fn dummy_image() -> ApprovedImage { + ApprovedImage { + metadata: ObjectMeta { + name: Some("test".to_string()), + uid: Some("test".to_string()), + ..Default::default() + }, + spec: ApprovedImageSpec { + image: DUMMY_IMAGE_REF.to_string(), + }, + status: None, + } } fn dummy_job() -> Job { @@ -465,9 +482,8 @@ mod tests { _ => panic!("unexpected API interaction: {req:?}, counter {ctr}"), }; count_check!(4, clos, |client| { - let ctx = Arc::new(generate_rv_ctx(client)); let job = Arc::new(dummy_job()); - let result = job_reconcile(job, ctx).await.unwrap(); + let result = job_reconcile(job, Arc::new(client)).await.unwrap(); assert_eq!(result, Action::await_change()); }); } @@ -476,12 +492,11 @@ mod tests { async fn test_job_reconcile_begun_deletion() { let clos = async |req: Request<_>, _| panic!("unexpected API interaction: {req:?}"); count_check!(0, clos, |client| { - let ctx = Arc::new(generate_rv_ctx(client)); let mut job = dummy_job(); let status = job.status.as_mut().unwrap(); status.completion_time = None; - let result = job_reconcile(Arc::new(job), ctx).await.unwrap(); - assert_eq!(result, Action::requeue(Duration::from_secs(300))); + let result = job_reconcile(Arc::new(job), Arc::new(client)).await; + assert_eq!(result.unwrap(), Action::requeue(Duration::from_secs(300))); }); } @@ -493,7 +508,7 @@ mod tests { #[test] fn test_get_job_name_sha() { - let name = get_job_name("quay.io/some-ref@sha256:e71dad00aa0e3d70540e726a0c66407e3004d96e045ab6c253186e327a2419e5").unwrap(); + let name = get_job_name(DUMMY_IMAGE_REF).unwrap(); assert_eq!( name, "compute-pcrs-6c57e93939-quay-io-some-ref-sha256-e71dad00aa0e3d7" @@ -502,20 +517,75 @@ mod tests { #[tokio::test] async fn test_compute_fresh_pcrs_success() { - let clos = |client| compute_fresh_pcrs(generate_rv_ctx(client), "image", "registry"); + let image = dummy_image(); + let clos = |client| compute_fresh_pcrs(client, &image); test_create_success::<_, _, Job>(clos).await; } #[tokio::test] async fn test_compute_fresh_pcrs_error() { - let clos = |client| compute_fresh_pcrs(generate_rv_ctx(client), "image", "registry"); - test_create_error(clos).await; + let image = dummy_image(); + let clos = |client| compute_fresh_pcrs(client, &image); + test_error_method!(clos, Method::POST); + } + + #[tokio::test] + async fn test_adopt_approved_image() { + let cluster = dummy_cluster(); + let clos = async |req: Request, _| { + assert_body_contains(req, TEST_UID).await; + Ok(serde_json::to_string(&dummy_image()).unwrap()) + }; + count_check!(1, clos, |client| { + assert!(adopt_approved_image(client, "test", &cluster).await.is_ok()); + }); + } + + #[tokio::test] + async fn test_adopt_approved_image_error() { + let cluster = dummy_cluster(); + let clos = |client| adopt_approved_image(client, "test", &cluster); + test_error_method!(clos, Method::PATCH); + } + + #[tokio::test] + async fn test_adopt_approved_images() { + let cluster = dummy_cluster(); + let clos = async |req: Request<_>, ctr| { + if ctr == 0 && req.method() == Method::GET { + let mut deleted = dummy_image(); + deleted.metadata.deletion_timestamp = Some(Time(Timestamp::now())); + let list = ObjectList { + items: vec![dummy_image(), deleted, dummy_image()], + types: Default::default(), + metadata: Default::default(), + }; + Ok(serde_json::to_string(&list).unwrap()) + } else if ctr < 3 && req.method() == Method::PATCH { + Ok(serde_json::to_string(&dummy_image()).unwrap()) + } else { + panic!("unexpected API interaction: {req:?}, counter {ctr}") + } + }; + count_check!(3, clos, |client| { + assert!(adopt_approved_images(client, &cluster).await.is_ok()); + }); + } + + #[tokio::test] + async fn test_adopt_approved_images_error() { + let cluster = dummy_cluster(); + let clos = |client| adopt_approved_images(client, &cluster); + test_error_method!(clos, Method::GET); } - // handle_new_image is an inherently online function and not tested here. + // handle_new_image and its caller image_add_reconcile are + // inherently online functions and not tested here #[tokio::test] - async fn test_disallow_image() { + async fn test_image_remove_reconcile() { + let image = Arc::new(dummy_image()); + let cluster = Some(dummy_cluster()); let clos = async |req: Request<_>, ctr| match (ctr, req.method()) { // fetched & updated for removal, then fetched for recomputation (0, &Method::GET) | (1, &Method::PUT) | (2, &Method::GET) => { @@ -529,8 +599,7 @@ mod tests { _ => panic!("unexpected API interaction: {req:?}, counter {ctr}"), }; count_check!(5, clos, |client| { - let ctx = generate_rv_ctx(client); - assert!(disallow_image(ctx, "registry").await.is_ok()); + assert!(image_remove_reconcile(client, image, cluster).await.is_ok()); }); } } diff --git a/operator/src/register_server.rs b/operator/src/register_server.rs index 959799ec..743823f1 100644 --- a/operator/src/register_server.rs +++ b/operator/src/register_server.rs @@ -200,7 +200,9 @@ pub async fn launch_keygen_controller(client: Client) { #[cfg(test)] mod tests { use super::*; + use http::{Method, Request, StatusCode}; use trusted_cluster_operator_test_utils::mock_client::*; + use trusted_cluster_operator_test_utils::test_error_method; #[tokio::test] async fn test_create_reg_server_depl_success() { @@ -211,7 +213,7 @@ mod tests { #[tokio::test] async fn test_create_reg_server_depl_error() { let clos = |client| create_register_server_deployment(client, Default::default(), "image"); - test_create_error(clos).await; + test_error_method!(clos, Method::POST); } #[tokio::test] @@ -223,6 +225,6 @@ mod tests { #[tokio::test] async fn test_create_reg_server_svc_error() { let clos = |client| create_register_server_service(client, Default::default(), Some(80)); - test_create_error(clos).await; + test_error_method!(clos, Method::POST); } } diff --git a/operator/src/test_utils.rs b/operator/src/test_utils.rs index a9c0241d..af4ed28c 100644 --- a/operator/src/test_utils.rs +++ b/operator/src/test_utils.rs @@ -4,8 +4,6 @@ use compute_pcrs_lib::Pcr; use k8s_openapi::{api::core::v1::ConfigMap, jiff::Timestamp}; -use kube::Client; -use operator::RvContextData; use std::collections::BTreeMap; use crate::trustee; @@ -53,11 +51,3 @@ pub fn dummy_pcrs_map() -> ConfigMap { ..Default::default() } } - -pub fn generate_rv_ctx(client: Client) -> RvContextData { - RvContextData { - client, - owner_reference: Default::default(), - pcrs_compute_image: String::new(), - } -} diff --git a/operator/src/trustee.rs b/operator/src/trustee.rs index aba4c542..a1388246 100644 --- a/operator/src/trustee.rs +++ b/operator/src/trustee.rs @@ -23,7 +23,7 @@ use kube::{ api::{ObjectMeta, Patch, PatchParams}, }; use log::info; -use operator::{RvContextData, create_or_info_if_exists}; +use operator::create_or_info_if_exists; use serde::{Serialize, Serializer}; use serde_json::{Value::String as JsonString, json}; use std::collections::BTreeMap; @@ -90,8 +90,8 @@ fn recompute_reference_values(image_pcrs: ImagePcrs) -> Vec { .collect() } -pub async fn update_reference_values(ctx: RvContextData) -> Result<()> { - let config_maps: Api = Api::default_namespaced(ctx.client); +pub async fn update_reference_values(client: Client) -> Result<()> { + let config_maps: Api = Api::default_namespaced(client); let image_pcrs_map = config_maps.get(PCR_CONFIG_MAP).await?; let reference_values = recompute_reference_values(get_image_pcrs(image_pcrs_map)?); @@ -550,6 +550,7 @@ mod tests { use http::{Method, Request, StatusCode}; use kube::client::Body; use trusted_cluster_operator_test_utils::mock_client::*; + use trusted_cluster_operator_test_utils::test_error_method; #[test] fn test_get_image_pcrs_success() { @@ -610,8 +611,7 @@ mod tests { _ => panic!("unexpected API interaction: {req:?}, counter {ctr}"), }; count_check!(3, clos, |client| { - let ctx = generate_rv_ctx(client); - assert!(update_reference_values(ctx).await.is_ok()); + assert!(update_reference_values(client).await.is_ok()); }); } @@ -622,8 +622,7 @@ mod tests { _ => panic!("unexpected API interaction: {req:?}"), }; count_check!(1, clos, |client| { - let ctx = generate_rv_ctx(client); - assert!(update_reference_values(ctx).await.is_err()); + assert!(update_reference_values(client).await.is_err()); }); } @@ -637,8 +636,7 @@ mod tests { _ => panic!("unexpected API interaction: {req:?}, counter {ctr}"), }; count_check!(2, clos, |client| { - let ctx = generate_rv_ctx(client); - assert!(update_reference_values(ctx).await.is_err()) + assert!(update_reference_values(client).await.is_err()) }); } @@ -654,8 +652,7 @@ mod tests { _ => panic!("unexpected API interaction: {req:?}, counter {ctr}"), }; count_check!(2, clos, |client| { - let ctx = generate_rv_ctx(client); - let err = update_reference_values(ctx).await.err().unwrap(); + let err = update_reference_values(client).await.err().unwrap(); assert!(err.to_string().contains("but had no data")); }); } @@ -792,7 +789,7 @@ mod tests { #[tokio::test] async fn test_generate_att_policy_error() { let clos = |client| generate_attestation_policy(client, Default::default()); - test_create_error(clos).await; + test_error_method!(clos, Method::POST); } #[tokio::test] @@ -810,7 +807,7 @@ mod tests { #[tokio::test] async fn test_generate_secret_error() { let clos = |client| generate_secret(client, "id", Default::default()); - test_create_error(clos).await; + test_error_method!(clos, Method::POST); } #[tokio::test] @@ -828,7 +825,7 @@ mod tests { #[tokio::test] async fn test_generate_trustee_data_error() { let clos = |client| generate_trustee_data(client, Default::default()); - test_create_error(clos).await; + test_error_method!(clos, Method::POST); } #[tokio::test] @@ -840,7 +837,7 @@ mod tests { #[tokio::test] async fn test_generate_kbs_service_error() { let clos = |client| generate_kbs_service(client, Default::default(), Some(80)); - test_create_error(clos).await; + test_error_method!(clos, Method::POST); } #[tokio::test] @@ -852,6 +849,6 @@ mod tests { #[tokio::test] async fn test_generate_kbs_depl_error() { let clos = |client| generate_kbs_deployment(client, Default::default(), "image"); - test_create_error(clos).await; + test_error_method!(clos, Method::POST); } } diff --git a/register-server/src/main.rs b/register-server/src/main.rs index 002283af..94601278 100644 --- a/register-server/src/main.rs +++ b/register-server/src/main.rs @@ -215,10 +215,15 @@ async fn main() { #[cfg(test)] mod tests { - use super::*; + use super::{create_machine, EndpointInfo, Machine}; + use http::{Method, Request, StatusCode}; + use k8s_openapi::apimachinery::pkg::apis::meta::v1::OwnerReference; use kube::api::ObjectList; + use kube::api::ObjectMeta; + use trusted_cluster_operator_lib::MachineSpec; use trusted_cluster_operator_lib::TrustedExecutionCluster; use trusted_cluster_operator_test_utils::mock_client::*; + use trusted_cluster_operator_test_utils::test_error_method; fn dummy_clusters() -> ObjectList { ObjectList { @@ -280,7 +285,8 @@ mod tests { #[tokio::test] async fn test_get_public_trustee_error() { - test_get_error(async |c| EndpointInfo::create(c).await.map(|_| ())).await; + let clos = async |c| EndpointInfo::create(c).await.map(|_| ()); + test_error_method!(clos, Method::GET); } fn dummy_machine() -> Machine { @@ -301,7 +307,7 @@ mod tests { api_version: "trusted-execution-clusters.io/v1alpha1".to_string(), kind: "TrustedExecutionCluster".to_string(), name: "test-cluster".to_string(), - uid: "test-uid".to_string(), + uid: TEST_UID.to_string(), controller: Some(true), block_owner_deletion: Some(true), } @@ -319,11 +325,10 @@ mod tests { #[tokio::test] async fn test_create_machine_error() { - test_create_error(async |c| { - create_machine(c, "test", dummy_owner_reference()) - .await - .map(|_| ()) - }) - .await; + let clos = async |c| { + let machine = create_machine(c, "test", dummy_owner_reference()); + machine.await.map(|_| ()) + }; + test_error_method!(clos, Method::POST); } } diff --git a/test_utils/src/lib.rs b/test_utils/src/lib.rs index 644b6aa6..a2b2fda9 100644 --- a/test_utils/src/lib.rs +++ b/test_utils/src/lib.rs @@ -825,7 +825,7 @@ where let name = resource_name.to_string(); async move { match api.get(&name).await { - Ok(_) => Err("{name} still exists, retrying..."), + Ok(_) => Err(format!("{name} still exists, retrying...")), Err(kube::Error::Api(ae)) if ae.code == 404 => Ok(()), Err(e) => { panic!("Unexpected error while fetching {name}: {e:?}"); diff --git a/test_utils/src/mock_client.rs b/test_utils/src/mock_client.rs index 1390cc56..c0f12609 100644 --- a/test_utils/src/mock_client.rs +++ b/test_utils/src/mock_client.rs @@ -14,6 +14,8 @@ use std::{convert::Infallible, sync::Arc}; use tower::service_fn; use trusted_cluster_operator_lib::{TrustedExecutionCluster, TrustedExecutionClusterSpec}; +pub const TEST_UID: &str = "test-uid"; + #[macro_export] macro_rules! assert_kube_api_error { ($err:expr, $code:expr, $reason:expr, $message:expr, $status:expr) => {{ @@ -43,6 +45,17 @@ macro_rules! count_check { } } +#[macro_export] +macro_rules! test_error_method { + ($action:ident, $method:path) => { + let clos = async |req: Request<_>, _| match req.method() { + &$method => Err(StatusCode::INTERNAL_SERVER_ERROR), + _ => panic!("unexpected API interaction: {req:?}"), + }; + test_error($action, clos).await; + }; +} + pub use count_check; async fn create_response>>( @@ -141,7 +154,7 @@ pub async fn test_create_already_exists< }); } -async fn test_error< +pub async fn test_error< F: Fn(Client) -> S, S: Future>, T: Debug, @@ -158,40 +171,18 @@ async fn test_error< }); } -pub async fn test_create_error S, S: Future>>( - create: F, -) { - let clos = async |req: Request<_>, _| match req.method() { - &Method::POST => Err(StatusCode::INTERNAL_SERVER_ERROR), - _ => panic!("unexpected API interaction: {req:?}"), - }; - test_error(create, clos).await; -} - -pub async fn test_get_error S, S: Future>>(get: F) { - let clos = async |req: Request<_>, _| match req.method() { - &Method::GET => Err(StatusCode::INTERNAL_SERVER_ERROR), - _ => panic!("unexpected API interaction: {req:?}"), - }; - test_error(get, clos).await; -} - pub fn dummy_cluster() -> TrustedExecutionCluster { TrustedExecutionCluster { metadata: ObjectMeta { name: Some("test".to_string()), - uid: Some("uid".to_string()), + uid: Some(TEST_UID.to_string()), ..Default::default() }, status: None, spec: TrustedExecutionClusterSpec { - trustee_image: Some("".to_string()), - pcrs_compute_image: Some("".to_string()), - register_server_image: Some("".to_string()), public_trustee_addr: Some("::".to_string()), register_server_port: None, trustee_kbs_port: None, - attestation_key_register_image: Some("".to_string()), attestation_key_register_port: None, public_attestation_key_register_addr: Some("::".to_string()), }, diff --git a/tests/trusted_execution_cluster.rs b/tests/trusted_execution_cluster.rs index c9263ce3..86d21f01 100644 --- a/tests/trusted_execution_cluster.rs +++ b/tests/trusted_execution_cluster.rs @@ -1,10 +1,13 @@ // SPDX-FileCopyrightText: Alice Frosi +// SPDX-FileCopyrightText: Jakob Naucke // // SPDX-License-Identifier: MIT +use anyhow::anyhow; use compute_pcrs_lib::{Part, Pcr}; use k8s_openapi::api::apps::v1::Deployment; use k8s_openapi::api::core::v1::{ConfigMap, Secret}; +use kube::api::ObjectMeta; use kube::{Api, api::DeleteParams}; use std::time::Duration; use trusted_cluster_operator_lib::conditions::NOT_COMMITTED_REASON_PENDING; @@ -15,18 +18,19 @@ use trusted_cluster_operator_lib::{ use trusted_cluster_operator_test_utils::*; const EXPECTED_PCR4: &str = "ff2b357be4a4bc66be796d4e7b2f1f27077dc89b96220aae60b443bcf4672525"; +const TEC_NAME: &str = "trusted-execution-cluster"; +const APPROVED_IMAGE_NAME: &str = "coreos"; +const TRUSTEE_CONFIG_MAP: &str = "trustee-data"; +const RV_JSON_KEY: &str = "reference-values.json"; named_test!( async fn test_trusted_execution_cluster_uninstall() -> anyhow::Result<()> { let test_ctx = setup!().await?; let client = test_ctx.client(); let namespace = test_ctx.namespace(); - let name = "trusted-execution-cluster"; - - let configmap_api: Api = Api::namespaced(client.clone(), namespace); let tec_api: Api = Api::namespaced(client.clone(), namespace); - let tec = tec_api.get(name).await?; + let tec = tec_api.get(TEC_NAME).await?; let owner_reference = generate_owner_reference(&tec)?; @@ -103,7 +107,7 @@ named_test!( .unwrap_or(false); if !has_approved_condition { - return Err(anyhow::anyhow!( + return Err(anyhow!( "AttestationKey does not have Approved condition yet" )); } @@ -118,18 +122,17 @@ named_test!( // Delete the cluster cr let api: Api = Api::namespaced(client.clone(), namespace); let dp = DeleteParams::default(); - api.delete(name, &dp).await?; + api.delete(TEC_NAME, &dp).await?; // Wait until it disappears - wait_for_resource_deleted(&api, name, 120, 5).await?; + wait_for_resource_deleted(&api, TEC_NAME, 120, 5).await?; let deployments_api: Api = Api::namespaced(client.clone(), namespace); wait_for_resource_deleted(&deployments_api, "trustee-deployment", 120, 1).await?; wait_for_resource_deleted(&deployments_api, "register-server", 120, 1).await?; - wait_for_resource_deleted(&configmap_api, "image-pcrs", 120, 1).await?; let images_api: Api = Api::namespaced(client.clone(), namespace); - wait_for_resource_deleted(&images_api, "coreos", 120, 1).await?; + wait_for_resource_deleted(&images_api, APPROVED_IMAGE_NAME, 120, 1).await?; wait_for_resource_deleted(&machines, &machine_name, 120, 1).await?; wait_for_resource_deleted(&attestation_keys, &ak_name, 120, 1).await?; @@ -169,7 +172,7 @@ async fn test_image_pcrs_configmap_updates() -> anyhow::Result<()> { return Ok(()); } - Err(anyhow::anyhow!("image-pcrs ConfigMap not yet populated with image-pcrs.json data")) + Err(anyhow!("image-pcrs ConfigMap not yet populated with image-pcrs.json data")) } }) .await?; @@ -255,7 +258,7 @@ async fn test_image_disallow() -> anyhow::Result<()> { let namespace = test_ctx.namespace(); let images: Api = Api::namespaced(client.clone(), namespace); - images.delete("coreos", &DeleteParams::default()).await?; + images.delete(APPROVED_IMAGE_NAME, &DeleteParams::default()).await?; let configmap_api: Api = Api::namespaced(client.clone(), namespace); let poller = Poller::new() @@ -265,14 +268,14 @@ async fn test_image_disallow() -> anyhow::Result<()> { poller.poll_async(|| { let api = configmap_api.clone(); async move { - let cm = api.get("trustee-data").await?; + let cm = api.get(TRUSTEE_CONFIG_MAP).await?; if let Some(data) = &cm.data - && let Some(reference_values_json) = data.get("reference-values.json") + && let Some(reference_values_json) = data.get(RV_JSON_KEY) && !reference_values_json.contains(EXPECTED_PCR4) { return Ok(()); } - Err(anyhow::anyhow!("Reference value not yet removed")) + Err(anyhow!("Reference value not yet removed")) } }).await?; @@ -286,10 +289,9 @@ async fn test_attestation_key_lifecycle() -> anyhow::Result<()> { let test_ctx = setup!().await?; let client = test_ctx.client(); let namespace = test_ctx.namespace(); - let tec_name = "trusted-execution-cluster"; let tec_api: Api = Api::namespaced(client.clone(), namespace); - let tec = tec_api.get(tec_name).await?; + let tec = tec_api.get(TEC_NAME).await?; let owner_reference = generate_owner_reference(&tec)?; let machine_uuid = uuid::Uuid::new_v4().to_string(); @@ -368,7 +370,7 @@ async fn test_attestation_key_lifecycle() -> anyhow::Result<()> { .unwrap_or(false); if !has_approved_condition { - return Err(anyhow::anyhow!( + return Err(anyhow!( "AttestationKey does not have Approved condition yet" )); } @@ -386,7 +388,7 @@ async fn test_attestation_key_lifecycle() -> anyhow::Result<()> { .unwrap_or(false); if !has_machine_owner_ref { - return Err(anyhow::anyhow!( + return Err(anyhow!( "AttestationKey does not have owner reference to Machine yet" )); } @@ -405,7 +407,7 @@ async fn test_attestation_key_lifecycle() -> anyhow::Result<()> { .unwrap_or(false); if !has_ak_owner_ref { - return Err(anyhow::anyhow!( + return Err(anyhow!( "Secret does not have owner reference to AttestationKey yet" )); } @@ -477,3 +479,79 @@ async fn test_nonexistent_approved_image() -> anyhow::Result<()> { Ok(()) } } + +named_test! { +async fn test_approved_image_readoption() -> anyhow::Result<()> { + let test_ctx = setup!().await?; + let client = test_ctx.client(); + let namespace = test_ctx.namespace(); + + let clusters: Api = Api::namespaced(client.clone(), namespace); + let images: Api = Api::namespaced(client.clone(), namespace); + let configmaps: Api = Api::namespaced(client.clone(), namespace); + + let cluster_spec = clusters.get(TEC_NAME).await?.spec; + let image_spec = images.get(APPROVED_IMAGE_NAME).await?.spec; + + test_ctx.info(format!("Deleting TrustedExecuctionCluster {TEC_NAME}")); + clusters.delete(TEC_NAME, &Default::default()).await?; + let removal_poller = Poller::new() + .with_timeout(Duration::from_secs(60)) + .with_interval(Duration::from_secs(5)) + .with_error_message(format!("{TRUSTEE_CONFIG_MAP} configmap not removed")); + removal_poller.poll_async(|| { + let configmaps = configmaps.clone(); + async move { + if configmaps.get(TRUSTEE_CONFIG_MAP).await.is_err() { + return Ok(()); + } + Err(anyhow!("{TRUSTEE_CONFIG_MAP} not yet removed")) + } + }).await?; + test_ctx.info(format!("Configmap {TRUSTEE_CONFIG_MAP} was removed")); + + let image = ApprovedImage { + spec: image_spec, + metadata: ObjectMeta { + name: Some(APPROVED_IMAGE_NAME.to_string()), + ..Default::default() + }, + status: None + }; + let cluster = TrustedExecutionCluster { + spec: cluster_spec, + metadata: ObjectMeta { + name: Some(TEC_NAME.to_string()), + ..Default::default() + }, + status: None + }; + + test_ctx.info("Creating new ApprovedImage and TrustedExecutionCluster"); + images.create(&Default::default(), &image).await?; + // Ensure adoption works even when cluster creation was delayed + tokio::time::sleep(Duration::from_secs(5)).await; + clusters.create(&Default::default(), &cluster).await?; + let regeneration_poller = Poller::new() + .with_timeout(Duration::from_secs(180)) + .with_interval(Duration::from_secs(5)) + .with_error_message("Reference value not regenerated".to_string()); + regeneration_poller.poll_async(|| { + let configmaps = configmaps.clone(); + async move { + let configmap = configmaps.get(TRUSTEE_CONFIG_MAP).await?; + if let Some(data) = &configmap.data + && let Some(json) = data.get(RV_JSON_KEY) + && json.contains(EXPECTED_PCR4) + { + return Ok(()); + } + Err(anyhow!("Reference value not yet regenerated")) + } + }).await?; + test_ctx.info("Reference values regenerated"); + + test_ctx.cleanup().await?; + Ok(()) +} +}