Remove images from TEC definition & run watchers independently of TECs

.github/workflows/rust.yml

@@ -46,7 +46,7 @@ jobs:
       - name: "cargo build"
         run: cargo build --all-targets
       - name: "cargo test"
-        run: cargo test --bins
+        run: cargo test --bins --lib
   tests-release-stable:
     name: "Tests (release), stable toolchain"
     runs-on: "ubuntu-24.04"
@@ -70,7 +70,7 @@ jobs:
       - name: "cargo build (release)"
         run: cargo build --lib --release
       - name: "cargo test (release)"
-        run: cargo test --bins --release
+        run: cargo test --bins --lib --release
   tests-release-msrv:
     name: "Tests (release), minimum supported toolchain"
     runs-on: "ubuntu-24.04"
@@ -100,7 +100,7 @@ jobs:
       - name: "cargo build (release)"
         run: cargo build --lib --release
       - name: "cargo test (release)"
-        run: cargo test --bins --release
+        run: cargo test --bins --lib --release
   tests-other-channels:
     name: "Tests, unstable toolchain"
     runs-on: "ubuntu-24.04"
@@ -128,4 +128,4 @@ jobs:
       - name: "cargo build"
         run: cargo build --lib
       - name: "cargo test"
-        run: cargo test --bins
+        run: cargo test --bins --lib

Makefile

@@ -202,10 +202,10 @@ equal-conditions:
 lint: fmt-check clippy vet equal-conditions
 
 test: crds-rs
-	cargo test --workspace --bins
+	cargo test --workspace --bins --lib
 
 test-release: crds-rs
-	cargo test --workspace --bins --release
+	cargo test --workspace --bins --lib --release
 
 integration-tests: generate trusted-cluster-gen crds-rs
 	RUST_LOG=info REGISTRY=$(REGISTRY) TAG=$(TAG) \

api/trusted-cluster-gen.go

@@ -92,6 +92,24 @@ func generateOperator(args *Args) error {
 					Name:    name,
 					Image:   args.image,
 					Command: []string{"/usr/bin/operator"},
+					Env: []corev1.EnvVar{
+						{
+							Name:  "RELATED_IMAGE_TRUSTEE",
+							Value: args.trusteeImage,
+						},
+						{
+							Name:  "RELATED_IMAGE_COMPUTE_PCRS",
+							Value: args.pcrsComputeImage,
+						},
+						{
+							Name:  "RELATED_IMAGE_REGISTRATION_SERVER",
+							Value: args.registerServerImage,
+						},
+						{
+							Name:  "RELATED_IMAGE_ATTESTATION_KEY_REGISTER",
+							Value: args.attestationKeyRegisterImage,
+						},
+					},
 				},
 			},
 		},
@@ -140,10 +158,6 @@ func generateTrustedExecutionClusterCR(args *Args) error {
 			Namespace: args.namespace,
 		},
 		Spec: v1alpha1.TrustedExecutionClusterSpec{
-			TrusteeImage:                     &args.trusteeImage,
-			PcrsComputeImage:                 &args.pcrsComputeImage,
-			RegisterServerImage:              &args.registerServerImage,
-			AttestationKeyRegisterImage:      &args.attestationKeyRegisterImage,
 			PublicAttestationKeyRegisterAddr: nil,
 			PublicTrusteeAddr:                nil,
 			TrusteeKbsPort:                   0,

api/v1alpha1/crds.go

@@ -38,30 +38,6 @@ var (
 // +kubebuilder:validation:XValidation:rule="!has(oldSelf.publicAttestationKeyRegisterAddr) || has(self.publicAttestationKeyRegisterAddr)", message="Value is required once set"
 // +kubebuilder:validation:XValidation:rule="!has(oldSelf.publicTrusteeAddr) || has(self.publicTrusteeAddr)", message="Value is required once set"
 type TrustedExecutionClusterSpec struct {
-	// Image reference to Trustee all-in-one image.
-	// If not specified, uses RELATED_IMAGE_TRUSTEE environment variable from operator deployment.
-	// +optional
-	// +kubebuilder:validation:XValidation:rule="self == oldSelf",message="Value is immutable"
-	TrusteeImage *string `json:"trusteeImage,omitempty"`
-
-	// Image reference to trusted-cluster-operator's compute-pcrs image.
-	// If not specified, uses RELATED_IMAGE_COMPUTE_PCRS environment variable from operator deployment.
-	// +optional
-	// +kubebuilder:validation:XValidation:rule="self == oldSelf",message="Value is immutable"
-	PcrsComputeImage *string `json:"pcrsComputeImage,omitempty"`
-
-	// Image reference to trusted-cluster-operator's register-server image.
-	// If not specified, uses RELATED_IMAGE_REGISTRATION_SERVER environment variable from operator deployment.
-	// +optional
-	// +kubebuilder:validation:XValidation:rule="self == oldSelf",message="Value is immutable"
-	RegisterServerImage *string `json:"registerServerImage,omitempty"`
-
-	// Image reference to trusted-cluster-operator's attestation-key-register image.
-	// If not specified, uses RELATED_IMAGE_ATTESTATION_KEY_REGISTER environment variable from operator deployment.
-	// +optional
-	// +kubebuilder:validation:XValidation:rule="self == oldSelf",message="Value is immutable"
-	AttestationKeyRegisterImage *string `json:"attestationKeyRegisterImage,omitempty"`
-
 	// Address where attester can connect to Attestation Key Register
 	// +optional
 	// +kubebuilder:validation:XValidation:rule="self == oldSelf",message="Value is immutable"

docs/design/reference-values.md

@@ -93,3 +93,16 @@ A reference value listing for Trustee could then look like this:
 ## Data flow
 
 ![](../pics/rv-flow.png)
+
+## Ownership
+
+Unlike `reference-values`, `ApprovedImages` can live independent of a `TrustedExecutionCluster` object.
+They can be created without one existing, and reference values are written by jobs (that the `ApprovedImages` also own) to the `image-pcrs` ConfigMap, which is created by the operator and is also independent of `TrustedExecutionClusters`.
+
+However, the `ApprovedImages` are adopted by the `TrustedExecutionCluster` object, both when created with a `TrustedExecutionCluster` existing and retroactively when created before `TrustedExecutionCluster` creation.
+This ensures that removal of a `TrustedExecutionCluster` acts as complete uninstallation.
+Finalizers on the `ApprovedImages` ensure the PCR values are removed back out of `image-pcrs` again.
+
+## Ownership flow
+
+![](../pics/image-flow.png)

docs/usage/os-and-node-lifecycle.md

@@ -52,6 +52,9 @@ Machines booting this image can now register and attest.
 
 **NB:** Updating nodes is not supported yet. Updates incur one intermediary stage of PCR values (assuming no further update on that boot) because kernel update is effective one boot _before_ shim & GRUB update.
 
+**NB:** The TrustedExecutionCluster object adopts ApprovedImages, including those that lived before it.
+This ensures that removal of a TrustedExecutionCluster acts as complete uninstallation.
+
 # Disallowing a bootable container image
 
 For the example above:

lib/Cargo.toml

@@ -21,3 +21,6 @@ serde_json.workspace = true
 [dev-dependencies]
 # Only a generate dependency, not a Rust dependency. Included here for auto-updates.
 kopium = "0.23.0"
+http.workspace = true
+tokio.workspace = true
+trusted-cluster-operator-test-utils = { path = "../test_utils" }

lib/src/lib.rs

@@ -19,10 +19,10 @@ pub use kopium::trustedexecutionclusters::*;
 pub use vendor_kopium::virtualmachineinstances;
 pub use vendor_kopium::virtualmachines;
 
-use anyhow::Context;
+use anyhow::{Context, Result, anyhow};
 use conditions::*;
 use k8s_openapi::apimachinery::pkg::apis::meta::v1::{Condition, OwnerReference, Time};
-use kube::Resource;
+use kube::{Api, Client, Resource};
 
 #[macro_export]
 macro_rules! update_status {
@@ -105,7 +105,7 @@ pub fn committed_condition(
 /// Generate an OwnerReference for any Kubernetes resource
 pub fn generate_owner_reference<T: Resource<DynamicType = ()>>(
     object: &T,
-) -> anyhow::Result<OwnerReference> {
+) -> Result<OwnerReference> {
     let name = object.meta().name.clone();
     let uid = object.meta().uid.clone();
     let kind = T::kind(&()).to_string();
@@ -119,32 +119,108 @@ pub fn generate_owner_reference<T: Resource<DynamicType = ()>>(
     })
 }
 
-/// Get the single TrustedExecutionCluster in the namespace
-///
-/// Returns an error if:
-/// - No TrustedExecutionCluster is found
-/// - More than one TrustedExecutionCluster is found (not supported)
-pub async fn get_trusted_execution_cluster(
-    client: kube::Client,
-) -> anyhow::Result<TrustedExecutionCluster> {
-    use kube::Api;
-
+pub async fn get_opt_trusted_execution_cluster(
+    client: Client,
+) -> Result<Option<TrustedExecutionCluster>> {
     let namespace = client.default_namespace().to_string();
     let clusters: Api<TrustedExecutionCluster> = Api::default_namespaced(client);
-    let params = Default::default();
-    let mut list = clusters.list(&params).await?;
-
-    if list.items.is_empty() {
-        return Err(anyhow::Error::msg(format!(
-            "No TrustedExecutionCluster found in namespace {namespace}. \
-             Ensure that this service is in the same namespace as the TrustedExecutionCluster."
-        )));
-    } else if list.items.len() > 1 {
-        return Err(anyhow::Error::msg(format!(
+    let list = clusters.list(&Default::default()).await?;
+    if list.items.len() > 1 {
+        return Err(anyhow!(
             "More than one TrustedExecutionCluster found in namespace {namespace}. \
              trusted-cluster-operator does not support more than one TrustedExecutionCluster."
-        )));
+        ));
+    }
+    Ok(list.items.into_iter().next())
+}
+
+/// Get the single TrustedExecutionCluster in the namespace
+pub async fn get_trusted_execution_cluster(client: Client) -> Result<TrustedExecutionCluster> {
+    let namespace = client.default_namespace().to_string();
+    let cluster = get_opt_trusted_execution_cluster(client).await;
+    let err = anyhow!(
+        "No TrustedExecutionCluster found in namespace {namespace}. \
+         Ensure that this service is in the same namespace as the TrustedExecutionCluster."
+    );
+    cluster.and_then(|c| c.ok_or(err))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use http::StatusCode;
+    use kube::api::ObjectList;
+    use trusted_cluster_operator_test_utils::mock_client::*;
+
+    #[tokio::test]
+    async fn test_get_some_trusted_execution_cluster() {
+        let clos = async |_, _| {
+            let object_list = ObjectList {
+                items: vec![dummy_cluster()],
+                types: Default::default(),
+                metadata: Default::default(),
+            };
+            Ok(serde_json::to_string(&object_list).unwrap())
+        };
+        count_check!(1, clos, |client| {
+            let res = get_opt_trusted_execution_cluster(client).await;
+            assert!(res.unwrap().is_some());
+        });
+    }
+
+    #[tokio::test]
+    async fn test_get_none_trusted_execution_cluster() {
+        let clos = async |_, _| {
+            let object_list = ObjectList::<TrustedExecutionCluster> {
+                items: vec![],
+                types: Default::default(),
+                metadata: Default::default(),
+            };
+            Ok(serde_json::to_string(&object_list).unwrap())
+        };
+        count_check!(1, clos, |client| {
+            let res = get_opt_trusted_execution_cluster(client).await;
+            assert!(res.unwrap().is_none());
+        });
+    }
+
+    #[tokio::test]
+    async fn test_non_unique_trusted_execution_cluster() {
+        let clos = async |_, _| {
+            let object_list = ObjectList {
+                items: vec![dummy_cluster(), dummy_cluster()],
+                types: Default::default(),
+                metadata: Default::default(),
+            };
+            Ok(serde_json::to_string(&object_list).unwrap())
+        };
+        count_check!(1, clos, |client| {
+            let err = get_opt_trusted_execution_cluster(client).await.unwrap_err();
+            assert!(err.to_string().contains("More than one"));
+        });
+    }
+
+    #[tokio::test]
+    async fn test_get_opt_trusted_execution_cluster_error() {
+        let clos = async |_, _| Err(StatusCode::INTERNAL_SERVER_ERROR);
+        count_check!(1, clos, |client| {
+            assert!(get_opt_trusted_execution_cluster(client).await.is_err());
+        });
     }
 
-    Ok(list.items.pop().unwrap())
+    #[tokio::test]
+    async fn test_get_no_trusted_execution_cluster() {
+        let clos = async |_, _| {
+            let object_list = ObjectList::<TrustedExecutionCluster> {
+                items: vec![],
+                types: Default::default(),
+                metadata: Default::default(),
+            };
+            Ok(serde_json::to_string(&object_list).unwrap())
+        };
+        count_check!(1, clos, |client| {
+            let err = get_trusted_execution_cluster(client).await.unwrap_err();
+            assert!(err.to_string().contains("No TrustedExecutionCluster found"));
+        });
+    }
 }

operator/src/attestation_key_register.rs

@@ -381,7 +381,9 @@ pub async fn launch_secret_ak_controller(client: Client) {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use http::{Method, Request, StatusCode};
     use trusted_cluster_operator_test_utils::mock_client::*;
+    use trusted_cluster_operator_test_utils::test_error_method;
 
     #[tokio::test]
     async fn test_create_ak_register_depl_success() {
@@ -396,7 +398,7 @@ mod tests {
         let clos = |client| {
             create_attestation_key_register_deployment(client, Default::default(), "image")
         };
-        test_create_error(clos).await;
+        test_error_method!(clos, Method::POST);
     }
 
     #[tokio::test]
@@ -410,6 +412,6 @@ mod tests {
     async fn test_create_ak_register_svc_error() {
         let clos =
             |client| create_attestation_key_register_service(client, Default::default(), Some(80));
-        test_create_error(clos).await;
+        test_error_method!(clos, Method::POST);
     }
 }

operator/src/lib.rs

@@ -8,22 +8,14 @@
 //
 // Use in other crates is not an intended purpose.
 
-use k8s_openapi::apimachinery::pkg::apis::meta::v1::OwnerReference;
-use kube::{Client, runtime::controller::Action};
+use kube::runtime::controller::Action;
 use log::info;
 use std::fmt::{Debug, Display};
 use std::{sync::Arc, time::Duration};
 
 // Re-export common functions from the lib
 pub use trusted_cluster_operator_lib::generate_owner_reference;
 
-#[derive(Clone)]
-pub struct RvContextData {
-    pub client: Client,
-    pub owner_reference: OwnerReference,
-    pub pcrs_compute_image: String,
-}
-
 #[derive(Debug, thiserror::Error)]
 pub enum ControllerError {
     #[error("{0}")]