Phase 1 changes for MCS Firstboot Pivot Failure Reporting

cmd/machine-config-server/bootstrap.go

@@ -22,6 +22,7 @@ var (
 		serverBaseDir    string
 		serverKubeConfig string
 		certificates     []string
+		mcsURL           string
 	}
 )
 
@@ -30,6 +31,7 @@ func init() {
 	bootstrapCmd.PersistentFlags().StringVar(&bootstrapOpts.serverBaseDir, "server-basedir", "/etc/mcs/bootstrap", "base directory on the host, relative to which machine-configs and pools can be found.")
 	bootstrapCmd.PersistentFlags().StringVar(&bootstrapOpts.serverKubeConfig, "bootstrap-kubeconfig", "/etc/kubernetes/kubeconfig", "path to bootstrap kubeconfig served by the bootstrap server.")
 	bootstrapCmd.PersistentFlags().StringArrayVar(&bootstrapOpts.certificates, "bootstrap-certs", []string{}, "a certificate bundle formatted in a string array with the format key=value,key=value")
+	bootstrapCmd.PersistentFlags().StringVar(&bootstrapOpts.mcsURL, "mcs-url", "", "Base URL for Machine Config Server; Used for status reporting")
 }
 
 func runBootstrapCmd(_ *cobra.Command, _ []string) {
@@ -39,10 +41,9 @@ func runBootstrapCmd(_ *cobra.Command, _ []string) {
 	// To help debugging, immediately log version
 	klog.Infof("Version: %+v (%s)", version.Raw, version.Hash)
 
-	bs, err := server.NewBootstrapServer(bootstrapOpts.serverBaseDir, bootstrapOpts.serverKubeConfig, bootstrapOpts.certificates)
-
+	bs, err := server.NewBootstrapServer(bootstrapOpts.serverBaseDir, bootstrapOpts.serverKubeConfig, bootstrapOpts.mcsURL, bootstrapOpts.certificates)
 	if err != nil {
-		klog.Exitf("Machine Config Server exited with error: %v", err)
+		klog.Fatal(err)
 	}
 
 	// Read-in bootstrap apiserver file /etc/mcs/bootstrap/api-server/apiserver.yaml.
@@ -59,9 +60,12 @@ func runBootstrapCmd(_ *cobra.Command, _ []string) {
 	klog.Infof("Launching bootstrap server with tls min version: %v & cipher suites %v", tlsminversion, tlsciphersuites)
 	tlsConfig := ctrlcommon.GetGoTLSConfig(tlsminversion, tlsciphersuites)
 
+	failureReporter := server.NewBootstrapFailureReporter()
+	failureHandler := server.NewNodeFailureHandler(failureReporter)
+
 	apiHandler := server.NewServerAPIHandler(bs)
-	secureServer := server.NewAPIServer(apiHandler, rootOpts.sport, false, rootOpts.cert, rootOpts.key, tlsConfig)
-	insecureServer := server.NewAPIServer(apiHandler, rootOpts.isport, true, "", "", tlsConfig)
+	secureServer := server.NewAPIServer(apiHandler, failureHandler, rootOpts.sport, false, rootOpts.cert, rootOpts.key, tlsConfig)
+	insecureServer := server.NewAPIServer(apiHandler, failureHandler, rootOpts.isport, true, "", "", tlsConfig)
 
 	stopCh := make(chan struct{})
 	go secureServer.Serve()

cmd/machine-config-server/start.go

@@ -23,13 +23,15 @@ var (
 	startOpts struct {
 		kubeconfig   string
 		apiserverURL string
+		mcsURL       string
 	}
 )
 
 func init() {
 	rootCmd.AddCommand(startCmd)
 	startCmd.PersistentFlags().StringVar(&startOpts.kubeconfig, "kubeconfig", "", "Kubeconfig file to access a remote cluster (testing only)")
 	startCmd.PersistentFlags().StringVar(&startOpts.apiserverURL, "apiserver-url", "", "URL for apiserver; Used to generate kubeconfig")
+	startCmd.PersistentFlags().StringVar(&startOpts.mcsURL, "mcs-url", "", "Base URL for Machine Config Server; Used for status reporting")
 
 }
 
@@ -47,17 +49,20 @@ func runStartCmd(_ *cobra.Command, _ []string) {
 		klog.Exitf("--apiserver-url cannot be empty")
 	}
 
-	cs, err := server.NewClusterServer(startOpts.kubeconfig, startOpts.apiserverURL)
+	cs, err := server.NewClusterServer(startOpts.kubeconfig, startOpts.apiserverURL, startOpts.mcsURL)
 	if err != nil {
 		ctrlcommon.WriteTerminationError(err)
 	}
 
 	klog.Infof("Launching server with tls min version: %v & cipher suites %v", rootOpts.tlsminversion, rootOpts.tlsciphersuites)
 	tlsConfig := ctrlcommon.GetGoTLSConfig(rootOpts.tlsminversion, rootOpts.tlsciphersuites)
 
+	failureReporter := server.NewClusterFailureReporter(cs.GetKubeClient())
+	failureHandler := server.NewNodeFailureHandler(failureReporter)
+
 	apiHandler := server.NewServerAPIHandler(cs)
-	secureServer := server.NewAPIServer(apiHandler, rootOpts.sport, false, rootOpts.cert, rootOpts.key, tlsConfig)
-	insecureServer := server.NewAPIServer(apiHandler, rootOpts.isport, true, "", "", tlsConfig)
+	secureServer := server.NewAPIServer(apiHandler, failureHandler, rootOpts.sport, false, rootOpts.cert, rootOpts.key, tlsConfig)
+	insecureServer := server.NewAPIServer(apiHandler, failureHandler, rootOpts.isport, true, "", "", tlsConfig)
 
 	stopCh := make(chan struct{})
 	go secureServer.Serve()

manifests/bootstrap-pod-v2.yaml

@@ -37,6 +37,7 @@ spec:
     command: ["/usr/bin/machine-config-server"]
     args:
       - "bootstrap"
+      - "--mcs-url={{.MCSURL}}"
       - "--payload-version={{.ReleaseVersion}}"
     terminationMessagePolicy: FallbackToLogsOnError
     volumeMounts:

manifests/machineconfigserver/daemonset.yaml

@@ -23,6 +23,7 @@ spec:
         args:
           - "start"
           - "--apiserver-url={{.APIServerURL}}"
+          - "--mcs-url={{.MCSURL}}"
           - "--payload-version={{.ReleaseVersion}}"
           - "--tls-cipher-suites={{join .TLSCipherSuites ","}}"
           - "--tls-min-version={{.TLSMinVersion}}"

pkg/daemon/constants/constants.go

@@ -12,7 +12,8 @@ const (
 	CurrentImageAnnotationKey = "machineconfiguration.openshift.io/currentImage"
 	// DesiredImageAnnotationKey is used to specify the desired OS image pullspec for a machine
 	DesiredImageAnnotationKey = "machineconfiguration.openshift.io/desiredImage"
-
+	// MachineConfigServerURLAnnotationKey stores the MCS base URL for status reporting
+	MachineConfigServerURLAnnotationKey = "machineconfiguration.openshift.io/machineConfigServerURL"
 	// CurrentMachineConfigAnnotationKey is used to fetch current MachineConfig for a machine
 	CurrentMachineConfigAnnotationKey = "machineconfiguration.openshift.io/currentConfig"
 	// DesiredMachineConfigAnnotationKey is used to specify the desired MachineConfig for a machine

pkg/operator/bootstrap.go

@@ -147,8 +147,14 @@ func buildSpec(dependencies *BootstrapDependencies, imgs *ctrlcommon.Images, rel
 		templatectrl.DockerRegistryKey:      imgs.DockerRegistry,
 	}
 
+	ignitionHost, err := getIgnitionHost(&dependencies.Infrastructure.Status)
+	if err != nil {
+		return nil, err
+	}
+	mcsURL := fmt.Sprintf("https://%s", ignitionHost)
+
 	config := getRenderConfig("", dependencies.KubeAPIServerServingCA, spec,
-		&imgs.RenderConfigImages, dependencies.Infrastructure, nil, nil, "2")
+		&imgs.RenderConfigImages, dependencies.Infrastructure, nil, nil, mcsURL, "2")
 	return config, nil
 }
 

pkg/operator/render.go

@@ -45,6 +45,7 @@ type renderConfig struct {
 	ControllerConfig       mcfgv1.ControllerConfigSpec
 	KubeAPIServerServingCA string
 	APIServerURL           string
+	MCSURL                 string
 	Images                 *ctrlcommon.RenderConfigImages
 	Infra                  configv1.Infrastructure
 	Constants              map[string]string

pkg/operator/sync.go

@@ -625,6 +625,8 @@ func (optr *Operator) syncRenderConfig(_ *renderConfig, _ *configv1.ClusterOpera
 		return err
 	}
 
+	mcsURL := fmt.Sprintf("https://%s", ignitionHost)
+
 	pointerConfig, err := ctrlcommon.PointerConfig(ignitionHost, machineConfigServerCABundle)
 	if err != nil {
 		return err
@@ -650,7 +652,7 @@ func (optr *Operator) syncRenderConfig(_ *renderConfig, _ *configv1.ClusterOpera
 		optr.setOperatorLogLevel(mcop.Spec.OperatorLogLevel)
 	}
 
-	optr.renderConfig = getRenderConfig(optr.namespace, string(kubeAPIServerServingCABytes), spec, &imgs.RenderConfigImages, infra, pointerConfigData, apiServer, fmt.Sprintf("%d", optr.logLevel))
+	optr.renderConfig = getRenderConfig(optr.namespace, string(kubeAPIServerServingCABytes), spec, &imgs.RenderConfigImages, infra, pointerConfigData, apiServer, mcsURL, fmt.Sprintf("%d", optr.logLevel))
 
 	return nil
 }
@@ -2150,7 +2152,7 @@ func setGVK(obj runtime.Object, scheme *runtime.Scheme) error {
 	return nil
 }
 
-func getRenderConfig(tnamespace, kubeAPIServerServingCA string, ccSpec *mcfgv1.ControllerConfigSpec, imgs *ctrlcommon.RenderConfigImages, infra *configv1.Infrastructure, pointerConfigData []byte, apiServer *configv1.APIServer, logLevel string) *renderConfig {
+func getRenderConfig(tnamespace, kubeAPIServerServingCA string, ccSpec *mcfgv1.ControllerConfigSpec, imgs *ctrlcommon.RenderConfigImages, infra *configv1.Infrastructure, pointerConfigData []byte, apiServer *configv1.APIServer, mcsURL, logLevel string) *renderConfig {
 	tlsMinVersion, tlsCipherSuites := ctrlcommon.GetSecurityProfileCiphersFromAPIServer(apiServer)
 	return &renderConfig{
 		TargetNamespace:        tnamespace,
@@ -2160,6 +2162,7 @@ func getRenderConfig(tnamespace, kubeAPIServerServingCA string, ccSpec *mcfgv1.C
 		Images:                 imgs,
 		KubeAPIServerServingCA: kubeAPIServerServingCA,
 		APIServerURL:           infra.Status.APIServerInternalURL,
+		MCSURL:                 mcsURL,
 		PointerConfig:          string(pointerConfigData),
 		Infra:                  *infra,
 		TLSMinVersion:          tlsMinVersion,

pkg/server/api.go

@@ -45,10 +45,11 @@ type APIServer struct {
 // NewAPIServer initializes a new API server
 // that runs the Machine Config Server as a
 // handler.
-func NewAPIServer(a *APIHandler, p int, is bool, c, k string, t *tls.Config) *APIServer {
+func NewAPIServer(a *APIHandler, failureHandler *NodeFailureHandler, p int, is bool, c, k string, t *tls.Config) *APIServer {
 	mux := http.NewServeMux()
 	mux.Handle("/config/", a)
 	mux.Handle("/healthz", &healthHandler{})
+	mux.Handle("/v1/node-failure", failureHandler) // NEW
 	mux.Handle("/", &defaultHandler{})
 
 	return &APIServer{
@@ -182,6 +183,71 @@ func (sh *APIHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 
 type healthHandler struct{}
 
+// NodeFailureHandler handles POST requests to /v1/node-failure
+type NodeFailureHandler struct {
+	reporter FailureReporter
+}
+
+// NewNodeFailureHandler creates a handler for node failure reports
+func NewNodeFailureHandler(reporter FailureReporter) *NodeFailureHandler {
+	return &NodeFailureHandler{
+		reporter: reporter,
+	}
+}
+
+// ServeHTTP processes firstboot failure reports from nodes
+func (h *NodeFailureHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	// Only accept POST
+	if r.Method != http.MethodPost {
+		w.Header().Set("Content-Length", "0")
+		w.WriteHeader(http.StatusMethodNotAllowed)
+		return
+	}
+
+	// Read and decode JSON body
+	var report FirstbootFailureReport
+	decoder := json.NewDecoder(r.Body)
+	if err := decoder.Decode(&report); err != nil {
+		klog.Errorf("Failed to decode failure report: %v", err)
+		w.Header().Set("Content-Type", "application/json")
+		w.WriteHeader(http.StatusBadRequest)
+		json.NewEncoder(w).Encode(map[string]string{
+			"error": "Invalid JSON payload",
+		})
+		return
+	}
+
+	// Validate required fields
+	if report.Pool == "" || report.NodeID == "" || report.Stage == "" {
+		klog.Errorf("Missing required fields in failure report: pool=%s node=%s stage=%s",
+			report.Pool, report.NodeID, report.Stage)
+		w.Header().Set("Content-Type", "application/json")
+		w.WriteHeader(http.StatusBadRequest)
+		json.NewEncoder(w).Encode(map[string]string{
+			"error": "Missing required fields: pool, nodeID, and stage are required",
+		})
+		return
+	}
+
+	klog.Infof("Received failure report: pool=%s node=%s stage=%s",
+		report.Pool, report.NodeID, report.Stage)
+
+	// Report the failure (best-effort, errors are logged but don't fail the request)
+	ctx := r.Context()
+	if err := h.reporter.ReportFailure(ctx, &report); err != nil {
+		// Log error but still return 202 - we don't want the node to retry-loop
+		klog.Errorf("Failed to process failure report (still returning 202): %v", err)
+	}
+
+	// Always return 202 Accepted (best-effort delivery)
+	// Never return 5xx - the node must not retry-loop on reporter errors
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(http.StatusAccepted)
+	json.NewEncoder(w).Encode(map[string]string{
+		"status": "accepted",
+	})
+}
+
 type acceptHeaderValue struct {
 	MIMEType    string
 	MIMESubtype string

pkg/server/api_test.go

@@ -15,6 +15,7 @@ import (
 	"github.com/stretchr/testify/require"
 	"golang.org/x/net/http2"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/client-go/kubernetes"
 
 	ctrlcommon "github.com/openshift/machine-config-operator/pkg/controller/common"
 	"github.com/openshift/machine-config-operator/test/helpers"
@@ -28,6 +29,10 @@ func (ms *mockServer) GetConfig(pr poolRequest) (*runtime.RawExtension, error) {
 	return ms.GetConfigFn(pr)
 }
 
+func (ms *mockServer) GetKubeClient() kubernetes.Interface {
+	return nil
+}
+
 type checkResponse func(t *testing.T, response *http.Response)
 
 type scenario struct {
@@ -717,7 +722,7 @@ func TestAPIServer(t *testing.T) {
 			ms := &mockServer{
 				GetConfigFn: scenario.serverFunc,
 			}
-			server := NewAPIServer(NewServerAPIHandler(ms), 0, false, "", "", nil)
+			server := NewAPIServer(NewServerAPIHandler(ms), nil, 0, false, "", "", nil)
 			server.handler.ServeHTTP(w, scenario.request)
 
 			resp := w.Result()

pkg/server/bootstrap_server.go

@@ -8,6 +8,7 @@ import (
 
 	yaml "github.com/ghodss/yaml"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/client-go/kubernetes"
 	clientcmd "k8s.io/client-go/tools/clientcmd/api/v1"
 	"k8s.io/klog/v2"
 
@@ -28,18 +29,26 @@ type bootstrapServer struct {
 	kubeconfigFunc kubeconfigFunc
 
 	certs []string
+
+	mcsURL string
+}
+
+// GetKubeClient returns nil for bootstrap server (no k8s access during bootstrap)
+func (bsc *bootstrapServer) GetKubeClient() kubernetes.Interface {
+	return nil
 }
 
 // NewBootstrapServer initializes a new Bootstrap server that implements
 // the Server interface.
-func NewBootstrapServer(dir, kubeconfig string, ircerts []string) (Server, error) {
+func NewBootstrapServer(dir, kubeconfig, mcsURL string, ircerts []string) (Server, error) {
 	if _, err := os.Stat(kubeconfig); err != nil {
 		return nil, fmt.Errorf("kubeconfig not found at location: %s", kubeconfig)
 	}
 	return &bootstrapServer{
 		serverBaseDir:  dir,
 		kubeconfigFunc: func() ([]byte, []byte, error) { return kubeconfigFromFile(kubeconfig) },
 		certs:          ircerts,
+		mcsURL:         mcsURL,
 	}, nil
 }
 
@@ -136,7 +145,7 @@ func (bsc *bootstrapServer) GetConfig(cr poolRequest) (*runtime.RawExtension, er
 	addDataAndMaybeAppendToIgnition(cloudProviderCAPath, cc.Spec.CloudProviderCAData, &ignConf)
 
 	appenders := newAppendersBuilder(nil, bsc.kubeconfigFunc, bsc.certs, bsc.serverBaseDir).
-		WithNodeAnnotations(currConf, "").
+		WithNodeAnnotations(currConf, "", bsc.mcsURL).
 		build()
 
 	for _, a := range appenders {

pkg/server/cluster_server.go

@@ -22,6 +22,7 @@ import (
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/client-go/informers"
+	"k8s.io/client-go/kubernetes"
 	clientset "k8s.io/client-go/kubernetes"
 	corelisterv1 "k8s.io/client-go/listers/core/v1"
 	"k8s.io/client-go/tools/cache"
@@ -56,6 +57,7 @@ type clusterServer struct {
 
 	kubeconfigFunc kubeconfigFunc
 	apiserverURL   string
+	mcsURL         string
 }
 
 const minResyncPeriod = 20 * time.Minute
@@ -76,7 +78,7 @@ func resyncPeriod() func() time.Duration {
 // It accepts a kubeConfig, which is not required when it's
 // run from within a cluster(useful in testing).
 // It accepts the apiserverURL which is the location of the KubeAPIServer.
-func NewClusterServer(kubeConfig, apiserverURL string) (Server, error) {
+func NewClusterServer(kubeConfig, apiserverURL, mcsURL string) (Server, error) {
 	clientsBuilder, err := clients.NewBuilder(kubeConfig)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create Kubernetes rest client: %w", err)
@@ -123,6 +125,7 @@ func NewClusterServer(kubeConfig, apiserverURL string) (Server, error) {
 		routeclient:             routeClient,
 		kubeconfigFunc:          func() ([]byte, []byte, error) { return kubeconfigFromSecret(bootstrapTokenDir, apiserverURL, nil) },
 		apiserverURL:            apiserverURL,
+		mcsURL:                  mcsURL,
 	}, nil
 }
 
@@ -187,7 +190,7 @@ func (cs *clusterServer) GetConfig(cr poolRequest) (*runtime.RawExtension, error
 	desiredImage := cs.resolveDesiredImageForPool(mp)
 
 	appenders := newAppendersBuilder(cr.version, cs.kubeconfigFunc, []string{}, "").
-		WithNodeAnnotations(currConf, desiredImage).
+		WithNodeAnnotations(currConf, desiredImage, cs.mcsURL).
 		WithCustomAppender(appendDesiredOSImage(desiredImage)).
 		build()
 
@@ -369,3 +372,8 @@ func appendDesiredOSImage(desired string) appenderFunc {
 		return nil
 	}
 }
+
+// GetKubeClient returns the Kubernetes client for this cluster server
+func (cs *clusterServer) GetKubeClient() kubernetes.Interface {
+	return cs.kubeclient
+}