diff --git a/docs/content/.pages b/docs/content/.pages index 8648c38..14ccb27 100644 --- a/docs/content/.pages +++ b/docs/content/.pages @@ -2,6 +2,7 @@ nav: - Documentation: - README.md - setup.md + - readiness.md - faq.md - Init Sources: init-sources - Reference: reference diff --git a/docs/content/readiness.md b/docs/content/readiness.md new file mode 100644 index 0000000..5db702c --- /dev/null +++ b/docs/content/readiness.md @@ -0,0 +1,100 @@ +# Waiting for Readiness + +In some scenarios, simply creating objects during initialization is not enough. You may need to wait for +certain resources to become ready before the init-agent marks the workspace as initialized. For example, +a CRD must be `Established` before custom resources using it can be created by other initializers +further down the chain. + +The `initialization.kcp.io/wait-for-ready` annotation allows you to express this requirement on +individual manifests. + +## How It Works + +When the init-agent encounters a manifest (from any of its [init sources](./init-sources/)) with +the `initialization.kcp.io/wait-for-ready` annotation, it will: + +1. Create (or confirm the existence of) the object as usual. +2. Re-fetch the object's current state from the API server. +3. Check whether the condition type specified in the annotation's value has `status: "True"` in the + object's `status.conditions` list. +4. If the condition is not yet `True`, the agent requeues reconciliation and tries again after a few + seconds. +5. Only once **all** annotated objects across **all** sources have their required conditions met will + the agent remove the initializer from the workspace, completing initialization. + +The annotation value must be the **name of a condition type** (e.g. `Established`, `Ready`, +`Available`). This condition must appear in the standard Kubernetes `status.conditions` array of +the resource. + +!!! warning "Important" + Waiting for a condition to become `True` inherently means that **some process must set that + condition**. In some cases this happens automatically (e.g. the Kubernetes API server sets + `Established` on CRDs), but in other cases you may need a dedicated controller or operator to + act on the resource and update its status. + + Due to the nature of kcp's workspace initialization, the workspace is not accessible through + the regular API while it still has initializers. Only processes that work through the same + `initializingworkspaces` virtual workspace – i.e. processes that are registered for the **same + initializer** as the init-agent – can see and modify objects in the workspace during + initialization. + + This means that if you need an external controller to make a resource "ready", that controller + must also operate on the same initializer's `initializingworkspaces` view. Without this, the + controller will not be able to access the workspace and therefore cannot set the condition the + init-agent is waiting for. The initialization would be stuck indefinitely. + + In practice, this is most relevant for custom operators that need to reconcile resources created + by the init-agent. Make sure these operators have the appropriate kcp permissions and are + configured to watch the same initializing workspaces. + +## Usage + +Add the annotation to any manifest inside an `InitTemplate`'s `spec.template`. The following example +creates a CRD and waits for it to become `Established` before initialization is considered complete: + +```yaml +apiVersion: initialization.kcp.io/v1alpha1 +kind: InitTemplate +metadata: + name: widgets-crd +spec: + template: | + apiVersion: apiextensions.k8s.io/v1 + kind: CustomResourceDefinition + metadata: + name: widgets.example.com + annotations: + initialization.kcp.io/wait-for-ready: "Established" + spec: + group: example.com + names: + kind: Widget + listKind: WidgetList + plural: widgets + singular: widget + scope: Cluster + versions: + - name: v1alpha1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object +``` + +In this example the init-agent will create the CRD and then wait until its `Established` condition +is `True` before considering this source complete. CRDs are just a nice example of a Kube-native +resource that on its own becomes ready. + +The agent will keep retrying indefinitely. If a condition is never set, the workspace will remain +in the initializing state. Use kcp's workspace lifecycle management to handle stuck workspaces if +necessary. + +The annotation works with **any Kubernetes resource** that follows the standard conditions pattern +in its status. Common examples include: + +| Resource | Typical Condition | +| -------- | ----------------- | +| `CustomResourceDefinition` | `Established` | +| `Deployment` | `Available` | +| `APIBinding` (kcp) | `Ready` | diff --git a/internal/manifest/applier.go b/internal/manifest/applier.go index e4868ad..00e8b67 100644 --- a/internal/manifest/applier.go +++ b/internal/manifest/applier.go @@ -21,7 +21,10 @@ import ( "errors" "strings" + "go.uber.org/zap" + "github.com/kcp-dev/init-agent/internal/log" + "github.com/kcp-dev/init-agent/sdk/types" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" @@ -53,18 +56,38 @@ func (a *applier) Apply(ctx context.Context, client ctrlruntimeclient.Client, ob } } - return false, nil -} + // After creating objects, check readiness of annotated ones + for _, object := range objs { + conditionType := object.GetAnnotations()[types.WaitForReadyAnnotation] + if conditionType == "" { + continue + } -func (a *applier) applyObject(ctx context.Context, client ctrlruntimeclient.Client, obj *unstructured.Unstructured) error { - gvk := obj.GroupVersionKind() + // Fetch current state + current := &unstructured.Unstructured{} + current.SetGroupVersionKind(object.GroupVersionKind()) - key := ctrlruntimeclient.ObjectKeyFromObject(obj).String() - // make key look prettier for cluster-scoped objects - key = strings.TrimLeft(key, "/") + if err := client.Get(ctx, ctrlruntimeclient.ObjectKeyFromObject(object), current); err != nil { + if apierrors.IsNotFound(err) { + requeue = true + continue + } + return false, err + } + + if !HasCondition(current, conditionType) { + logger := a.objectLogger(ctx, object) + logger.Debugw("Waiting for condition", "condition", conditionType) + requeue = true + } + } - logger := log.FromContext(ctx) - logger.Debugw("Applying object", "obj-key", key, "obj-gvk", gvk) + return requeue, nil +} + +func (a *applier) applyObject(ctx context.Context, client ctrlruntimeclient.Client, obj *unstructured.Unstructured) error { + logger := a.objectLogger(ctx, obj) + logger.Debugw("Applying object") if err := client.Create(ctx, obj); err != nil { if !apierrors.IsAlreadyExists(err) { @@ -74,3 +97,13 @@ func (a *applier) applyObject(ctx context.Context, client ctrlruntimeclient.Clie return nil } + +func (a *applier) objectLogger(ctx context.Context, obj *unstructured.Unstructured) *zap.SugaredLogger { + gvk := obj.GroupVersionKind() + + key := ctrlruntimeclient.ObjectKeyFromObject(obj).String() + // make key look prettier for cluster-scoped objects + key = strings.TrimLeft(key, "/") + + return log.FromContext(ctx).With("obj-key", key, "obj-gvk", gvk) +} diff --git a/internal/manifest/readiness.go b/internal/manifest/readiness.go new file mode 100644 index 0000000..cb643d2 --- /dev/null +++ b/internal/manifest/readiness.go @@ -0,0 +1,45 @@ +/* +Copyright 2026 The kcp Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package manifest + +import ( + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +// HasCondition checks if an unstructured object has the specified condition +// type with status "True". +func HasCondition(obj *unstructured.Unstructured, conditionType string) bool { + conditions, found, err := unstructured.NestedSlice(obj.Object, "status", "conditions") + if err != nil || !found { + return false + } + + for _, c := range conditions { + condition, ok := c.(map[string]any) + if !ok { + continue + } + + cType, _, _ := unstructured.NestedString(condition, "type") + cStatus, _, _ := unstructured.NestedString(condition, "status") + if cType == conditionType && cStatus == "True" { + return true + } + } + + return false +} diff --git a/internal/manifest/readiness_test.go b/internal/manifest/readiness_test.go new file mode 100644 index 0000000..cf99f61 --- /dev/null +++ b/internal/manifest/readiness_test.go @@ -0,0 +1,151 @@ +/* +Copyright 2026 The kcp Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package manifest + +import ( + "testing" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +func TestHasCondition(t *testing.T) { + testcases := []struct { + name string + obj *unstructured.Unstructured + conditionType string + expected bool + }{ + { + name: "no status", + obj: newUnstructured("v1", "ConfigMap", "test"), + conditionType: "Ready", + expected: false, + }, + { + name: "no conditions", + obj: newUnstructuredWithStatus("v1", "ConfigMap", "test", map[string]any{}), + conditionType: "Ready", + expected: false, + }, + { + name: "empty conditions", + obj: newUnstructuredWithConditions("v1", "ConfigMap", "test", []any{}), + conditionType: "Ready", + expected: false, + }, + { + name: "condition type not found", + obj: newUnstructuredWithConditions("v1", "ConfigMap", "test", []any{ + map[string]any{"type": "Available", "status": "True"}, + }), + conditionType: "Ready", + expected: false, + }, + { + name: "condition found but status is False", + obj: newUnstructuredWithConditions("v1", "ConfigMap", "test", []any{ + map[string]any{"type": "Ready", "status": "False"}, + }), + conditionType: "Ready", + expected: false, + }, + { + name: "condition found but status is Unknown", + obj: newUnstructuredWithConditions("v1", "ConfigMap", "test", []any{ + map[string]any{"type": "Ready", "status": "Unknown"}, + }), + conditionType: "Ready", + expected: false, + }, + { + name: "condition found with status True", + obj: newUnstructuredWithConditions("v1", "ConfigMap", "test", []any{ + map[string]any{"type": "Ready", "status": "True"}, + }), + conditionType: "Ready", + expected: true, + }, + { + name: "multiple conditions - target is True", + obj: newUnstructuredWithConditions("v1", "ConfigMap", "test", []any{ + map[string]any{"type": "Available", "status": "True"}, + map[string]any{"type": "Ready", "status": "True"}, + map[string]any{"type": "Progressing", "status": "False"}, + }), + conditionType: "Ready", + expected: true, + }, + { + name: "multiple conditions - target is False", + obj: newUnstructuredWithConditions("v1", "ConfigMap", "test", []any{ + map[string]any{"type": "Available", "status": "True"}, + map[string]any{"type": "Ready", "status": "False"}, + map[string]any{"type": "Progressing", "status": "True"}, + }), + conditionType: "Ready", + expected: false, + }, + { + name: "CRD Established condition True", + obj: newUnstructuredWithConditions("apiextensions.k8s.io/v1", "CustomResourceDefinition", "test", []any{ + map[string]any{"type": "NamesAccepted", "status": "True"}, + map[string]any{"type": "Established", "status": "True"}, + }), + conditionType: "Established", + expected: true, + }, + { + name: "CRD Established condition False", + obj: newUnstructuredWithConditions("apiextensions.k8s.io/v1", "CustomResourceDefinition", "test", []any{ + map[string]any{"type": "NamesAccepted", "status": "True"}, + map[string]any{"type": "Established", "status": "False"}, + }), + conditionType: "Established", + expected: false, + }, + { + name: "malformed condition entry (not a map)", + obj: newUnstructuredWithConditions("v1", "ConfigMap", "test", []any{ + "not a map", + map[string]any{"type": "Ready", "status": "True"}, + }), + conditionType: "Ready", + expected: true, + }, + } + + for _, tt := range testcases { + t.Run(tt.name, func(t *testing.T) { + result := HasCondition(tt.obj, tt.conditionType) + if result != tt.expected { + t.Fatalf("Expected %v.", tt.expected) + } + }) + } +} + +func newUnstructuredWithStatus(apiVersion, kind, name string, status map[string]any) *unstructured.Unstructured { + obj := newUnstructured(apiVersion, kind, name) + obj.Object["status"] = status + return obj +} + +func newUnstructuredWithConditions(apiVersion, kind, name string, conditions []any) *unstructured.Unstructured { + return newUnstructuredWithStatus(apiVersion, kind, name, map[string]any{ + "conditions": conditions, + }) +} diff --git a/sdk/types/annotations.go b/sdk/types/annotations.go new file mode 100644 index 0000000..d4a7499 --- /dev/null +++ b/sdk/types/annotations.go @@ -0,0 +1,24 @@ +/* +Copyright 2026 The kcp Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package types + +const ( + // WaitForReadyAnnotation specifies a condition type to wait for. + // The value is the condition type name (e.g., "Ready", "Established"). + // The agent will wait until that condition has status=True. + WaitForReadyAnnotation = "initialization.kcp.io/wait-for-ready" +) diff --git a/test/e2e/clusterinit/wait_for_ready_test.go b/test/e2e/clusterinit/wait_for_ready_test.go new file mode 100644 index 0000000..442a5dd --- /dev/null +++ b/test/e2e/clusterinit/wait_for_ready_test.go @@ -0,0 +1,261 @@ +//go:build e2e + +/* +Copyright 2026 The kcp Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package clusterinit + +import ( + "context" + "slices" + "strings" + "testing" + "time" + + "github.com/go-logr/logr" + + initializationv1alpha1 "github.com/kcp-dev/init-agent/sdk/apis/initialization/v1alpha1" + initagenttypes "github.com/kcp-dev/init-agent/sdk/types" + "github.com/kcp-dev/init-agent/test/utils" + + kcptenancyinitialization "github.com/kcp-dev/sdk/apis/tenancy/initialization" + kcptenancyv1alpha1 "github.com/kcp-dev/sdk/apis/tenancy/v1alpha1" + + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/wait" + ctrlruntime "sigs.k8s.io/controller-runtime" +) + +// TestWaitForReadyAnnotation tests that when a manifest object has the +// initialization.kcp.io/wait-for-ready annotation, the agent will wait +// for the specified condition to become True before removing the initializer. +func TestWaitForReadyAnnotation(t *testing.T) { + const ( + targetWorkspace = "wait-for-ready-workspace" + initAgentWorkspace = "wait-for-ready-init-agent" + wstWorkspace = "wait-for-ready-wst" + ) + + ctx := t.Context() + ctrlruntime.SetLogger(logr.Discard()) + + // create dummy workspace and WST in it + t.Log("Creating WorkspaceType…") + kcpClusterClient := utils.GetKcpAdminClusterClient(t) + rootClient := kcpClusterClient.Cluster(rootCluster) + + wstCluster := utils.CreateAndWaitForWorkspace(t, ctx, rootClient, wstWorkspace) + wstClient := kcpClusterClient.Cluster(wstCluster.Path()) + + wst := &kcptenancyv1alpha1.WorkspaceType{ + ObjectMeta: metav1.ObjectMeta{ + Name: "wait-for-ready-type", + }, + Spec: kcptenancyv1alpha1.WorkspaceTypeSpec{ + Initializer: true, + }, + } + + if err := wstClient.Create(ctx, wst); err != nil { + t.Fatalf("Failed to create WorkspaceType: %v", err) + } + + initializer := kcptenancyinitialization.InitializerForType(wst) + + utils.GrantWorkspaceAccess(t, ctx, wstClient, utils.Subject(), rbacv1.PolicyRule{ + APIGroups: []string{"tenancy.kcp.io"}, + Resources: []string{"workspacetypes"}, + Verbs: []string{"list", "watch"}, + }, rbacv1.PolicyRule{ + APIGroups: []string{"tenancy.kcp.io"}, + Resources: []string{"workspacetypes"}, + ResourceNames: []string{wst.Name}, + Verbs: []string{"get", "initialize"}, + }) + + // create init-agent ws + t.Log("Creating init-agent workspace…") + initAgentCluster := utils.CreateAndWaitForWorkspace(t, ctx, rootClient, initAgentWorkspace) + + initAgentClient := kcpClusterClient.Cluster(initAgentCluster.Path()) + utils.GrantWorkspaceAccess(t, ctx, initAgentClient, utils.Subject(), rbacv1.PolicyRule{ + APIGroups: []string{"initialization.kcp.io"}, + Resources: []string{"inittargets", "inittemplates"}, + Verbs: []string{"get", "list", "watch"}, + }) + + // install CRDs there + t.Log("Installing CRDs…") + utils.ApplyCRD(t, ctx, initAgentClient, "deploy/crd/kcp.io/initialization.kcp.io_inittargets.yaml") + utils.ApplyCRD(t, ctx, initAgentClient, "deploy/crd/kcp.io/initialization.kcp.io_inittemplates.yaml") + + // Create InitTarget and InitTemplate with a CRD that has the wait-for-ready annotation. + // CRDs have an "Established" condition that becomes True when the CRD is ready. + t.Logf("Creating init-agent configuration…") + + initTemplate := &initializationv1alpha1.InitTemplate{ + ObjectMeta: metav1.ObjectMeta{ + Name: "wait-for-ready-template", + }, + Spec: initializationv1alpha1.InitTemplateSpec{ + Template: strings.TrimSpace(` +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: widgets.example.com + annotations: + ` + initagenttypes.WaitForReadyAnnotation + `: "Established" +spec: + group: example.com + names: + kind: Widget + listKind: WidgetList + plural: widgets + singular: widget + scope: Namespaced + versions: + - name: v1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + color: + type: string +`), + }, + } + + if err := initAgentClient.Create(ctx, initTemplate); err != nil { + t.Fatalf("Failed to create InitTemplate: %v", err) + } + + initTarget := &initializationv1alpha1.InitTarget{ + ObjectMeta: metav1.ObjectMeta{ + Name: "init-wait-for-ready-type", + }, + Spec: initializationv1alpha1.InitTargetSpec{ + WorkspaceTypeReference: initializationv1alpha1.WorkspaceTypeReference{ + Path: rootCluster.Join(wstWorkspace).String(), + Name: wst.Name, + }, + Sources: []initializationv1alpha1.InitSource{ + { + Template: &initializationv1alpha1.TemplateInitSource{ + Name: initTemplate.Name, + }, + }, + }, + }, + } + + if err := initAgentClient.Create(ctx, initTarget); err != nil { + t.Fatalf("Failed to create InitTarget: %v", err) + } + + // start agent + agentKubeconfig := utils.CreateKcpAgentKubeconfig(t, "") + utils.RunAgent(ctx, t, agentKubeconfig, rootCluster.Join(initAgentWorkspace).String(), "") + + // create final target workspace using that WST from the earlier step + targetWs := &kcptenancyv1alpha1.Workspace{ + ObjectMeta: metav1.ObjectMeta{ + Name: targetWorkspace, + }, + Spec: kcptenancyv1alpha1.WorkspaceSpec{ + Type: &kcptenancyv1alpha1.WorkspaceTypeReference{ + Path: rootCluster.Join(wstWorkspace).String(), + Name: kcptenancyv1alpha1.WorkspaceTypeName(wst.Name), + }, + }, + } + + t.Logf("Creating workspace %s…", targetWorkspace) + if err := rootClient.Create(ctx, targetWs); err != nil { + t.Fatalf("Failed to create %q workspace: %v", targetWorkspace, err) + } + + // Wait for the agent to do its work and initialize the cluster. + // The initializer should only be removed AFTER the CRD's Established condition is True. + err := wait.PollUntilContextTimeout(ctx, 500*time.Millisecond, 60*time.Second, false, func(ctx context.Context) (done bool, err error) { + err = rootClient.Get(ctx, types.NamespacedName{Name: targetWorkspace}, targetWs) + if err != nil { + return false, err + } + + return !slices.Contains(targetWs.Status.Initializers, initializer), nil + }) + if err != nil { + t.Fatalf("Failed to wait for workspace to be initialized: %v", err) + } + + // Verify the CRD exists in the target workspace and is Established + targetClient := kcpClusterClient.Cluster(rootCluster.Join(targetWorkspace)) + + crd := utils.YAMLToUnstructured(t, ` +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: widgets.example.com +`) + + if err := targetClient.Get(ctx, types.NamespacedName{Name: crd.GetName()}, crd); err != nil { + t.Fatalf("Failed to find CRD in target workspace: %v", err) + } + + // Verify the CRD has the Established condition set to True + conditions, found, err := getConditions(crd.Object) + if err != nil || !found { + t.Fatal("CRD does not have conditions in status") + } + + established := false + for _, c := range conditions { + condition, ok := c.(map[string]any) + if !ok { + continue + } + + cType := condition["type"].(string) + cStatus := condition["status"].(string) + if cType == "Established" && cStatus == "True" { + established = true + break + } + } + + if !established { + t.Fatal("Expected CRD to have Established=True condition, but it was not found or not True") + } +} + +func getConditions(obj map[string]any) ([]any, bool, error) { + status, ok := obj["status"].(map[string]any) + if !ok { + return nil, false, nil + } + conditions, ok := status["conditions"].([]any) + if !ok { + return nil, false, nil + } + return conditions, true, nil +}