From ffce562e7d37faf23ab1489da9d80e41d96354b0 Mon Sep 17 00:00:00 2001 From: Prachiti Talgulkar Date: Fri, 8 May 2026 18:57:44 +0530 Subject: [PATCH] MCO-2209 MCO-2213 MCO-2233: Migrate security, daemon, and kernel TCs from otp3 mco.go Migrates 13 test cases from openshift-tests-private/test/extended/mco/mco.go into existing destination test suite files: - mco_security.go: 43278, 46965, 47062, 62084, 65208, 66436, 67395 - mco_daemon.go: 68684, 82299, 83134, 83943 - mco_kernel.go: 72132, 72135 New helpers/methods added to support these TCs: - getCommitID, getGoVersion (util.go) - getCoreDNSWorkerPodCreationTime (mco.go) - Controller.RemovePod (controller.go) - MachineConfigPool.GetCertsExpiry (machineconfigpool.go) - NewMachineConfigList (machineconfig.go) - exutil.GetClusterVersion (util/clusters.go) - MCDCrioReloadedRegexp constant (const.go) Template files added: add-gpg-pub-key.yaml, change-policy-json.yaml, change-fips.yaml Co-Authored-By: Claude Sonnet 4.5 --- test/extended-priv/const.go | 2 + test/extended-priv/controller.go | 18 + test/extended-priv/machineconfig.go | 5 + test/extended-priv/machineconfigpool.go | 19 + test/extended-priv/mco.go | 47 ++ test/extended-priv/mco_daemon.go | 193 +++++++++ test/extended-priv/mco_kernel.go | 70 +++ test/extended-priv/mco_security.go | 409 ++++++++++++++++++ .../testdata/files/add-gpg-pub-key.yaml | 23 + .../testdata/files/change-fips.yaml | 17 + .../testdata/files/change-policy-json.yaml | 23 + test/extended-priv/util.go | 57 +++ test/extended-priv/util/clusters.go | 25 ++ 13 files changed, 908 insertions(+) create mode 100644 test/extended-priv/testdata/files/add-gpg-pub-key.yaml create mode 100644 test/extended-priv/testdata/files/change-fips.yaml create mode 100644 test/extended-priv/testdata/files/change-policy-json.yaml diff --git a/test/extended-priv/const.go b/test/extended-priv/const.go index 83fa5b5d2e..aaf7e5e034 100644 --- a/test/extended-priv/const.go +++ b/test/extended-priv/const.go @@ -118,6 +118,8 @@ const ( AlpineImage = "quay.io/openshifttest/alpine@sha256:dc1536cbff0ba235d4219462aeccd4caceab9def96ae8064257d049166890083" // TestSSLImage the testssl image stored in openshiftest TestSSLImage = "quay.io/openshifttest/testssl@sha256:ad6fb8002cb9cfce3ddc8829fd6e7e0d997aeb1faf972650f3e5d7603f90c6ef" + // MCDCrioReloadedRegexp is the regexp used to verify that crio was reloaded by the MCD + MCDCrioReloadedRegexp = "crio.* reloaded successfully" // Constants for NodeDisruptionPolicy NodeDisruptionPolicyActionNone = "None" diff --git a/test/extended-priv/controller.go b/test/extended-priv/controller.go index 59ce5f2d43..cbd4d448e6 100644 --- a/test/extended-priv/controller.go +++ b/test/extended-priv/controller.go @@ -173,6 +173,24 @@ func (mcc *Controller) GetPreviousLogs() (string, error) { return prevLogs, nil } +// RemovePod removes the controller pod forcing the creation of a new one +func (mcc *Controller) RemovePod() error { + cachedPodName, err := mcc.GetCachedPodName() + if err != nil { + return err + } + if cachedPodName == "" { + err := fmt.Errorf("Cannot get controller pod name. Failed getting MCO controller logs") + logger.Errorf("Error getting controller pod name. Error: %s", err) + return err + } + + // remove the cached podname, since it will not be valid anymore + mcc.podName = "" + + return mcc.oc.WithoutNamespace().Run("delete").Args("pod", "-n", MachineConfigNamespace, cachedPodName).Execute() +} + // checkMCCPanic checks the machine-config-controller logs for panics func checkMCCPanic(oc *exutil.CLI) { var ( diff --git a/test/extended-priv/machineconfig.go b/test/extended-priv/machineconfig.go index a2efcf77d5..67f34ad61a 100644 --- a/test/extended-priv/machineconfig.go +++ b/test/extended-priv/machineconfig.go @@ -26,6 +26,11 @@ type MachineConfig struct { skipWaitForMcp bool } +// NewMachineConfigList construct a new machine config list struct to handle all existing machine configs +func NewMachineConfigList(oc *exutil.CLI) *MachineConfigList { + return &MachineConfigList{ResourceList: *NewResourceList(oc, "mc")} +} + // NewMachineConfig create a NewMachineConfig struct func NewMachineConfig(oc *exutil.CLI, name, pool string) *MachineConfig { mc := &MachineConfig{Resource: *NewResource(oc, "mc", name), pool: pool} diff --git a/test/extended-priv/machineconfigpool.go b/test/extended-priv/machineconfigpool.go index bc3da020e7..90833377c3 100644 --- a/test/extended-priv/machineconfigpool.go +++ b/test/extended-priv/machineconfigpool.go @@ -2,6 +2,7 @@ package extended import ( "context" + "encoding/json" "errors" "fmt" "strconv" @@ -1075,6 +1076,24 @@ func (mcp MachineConfigPool) GetMOSC() (*MachineOSConfig, error) { return moscs[0], nil } +// GetCertsExpiry returns the information about the certificates tracked by the MCP +func (mcp *MachineConfigPool) GetCertsExpiry() ([]CertExpiry, error) { + expiryString, err := mcp.Get(`{.status.certExpirys}`) + if err != nil { + return nil, err + } + + var certsExp []CertExpiry + + jsonerr := json.Unmarshal([]byte(expiryString), &certsExp) + + if jsonerr != nil { + return nil, jsonerr + } + + return certsExp, nil +} + // GetAll returns a []*MachineConfigPool list with all existing machine config pools sorted by creation time func (mcpl *MachineConfigPoolList) GetAll() ([]*MachineConfigPool, error) { mcpl.ResourceList.SortByTimestamp() diff --git a/test/extended-priv/mco.go b/test/extended-priv/mco.go index d50522ae4f..52a78a15eb 100644 --- a/test/extended-priv/mco.go +++ b/test/extended-priv/mco.go @@ -152,3 +152,50 @@ func createMcAndVerifyMCValue(oc *exutil.CLI, stepText, mcName string, node *Nod o.Expect(podOut).Should(o.MatchRegexp(textToVerify.textToVerifyForNode)) logger.Infof("%s is verified in the machine config daemon!", stepText) } + +// getCoreDNSWorkerPodCreationTime returns the latest creation timestamp for CoreDNS worker pods +// this func help to verify CoreDNS pods are recreated when MCP starts updating +func getCoreDNSWorkerPodCreationTime(oc *exutil.CLI) (string, error) { + vsphereInfraNamespace := OnPremPlatforms[VspherePlatform] + + // Get all CoreDNS pods + coreDNSPods := NewNamespacedResourceList(oc.AsAdmin(), "pod", vsphereInfraNamespace) + coreDNSPods.ByLabel("app=vsphere-infra-coredns") + + // Get pod names, node names, and creation timestamps + podNames, err := coreDNSPods.Get(`{.items[*].metadata.name}`) + if err != nil { + return "", fmt.Errorf("failed to get pod names: %w", err) + } + + nodeNames, err := coreDNSPods.Get(`{.items[*].spec.nodeName}`) + if err != nil { + return "", fmt.Errorf("failed to get node names: %w", err) + } + + creationTimes, err := coreDNSPods.Get(`{.items[*].metadata.creationTimestamp}`) + if err != nil { + return "", fmt.Errorf("failed to get creation timestamps: %w", err) + } + + // Parse the results + pods := strings.Fields(podNames) + nodes := strings.Fields(nodeNames) + timestamps := strings.Fields(creationTimes) + + // Find the latest creation timestamp for CoreDNS pods on worker nodes + var latestTimestamp string + for i, pod := range pods { + if i >= len(nodes) || i >= len(timestamps) { + break + } + // Only check CoreDNS pods on worker nodes + if strings.HasPrefix(pod, "coredns-") && strings.Contains(nodes[i], "worker") { + if latestTimestamp == "" || timestamps[i] > latestTimestamp { + latestTimestamp = timestamps[i] + } + } + } + + return latestTimestamp, nil +} diff --git a/test/extended-priv/mco_daemon.go b/test/extended-priv/mco_daemon.go index c5c36876ef..5ab3dc8f07 100644 --- a/test/extended-priv/mco_daemon.go +++ b/test/extended-priv/mco_daemon.go @@ -313,4 +313,197 @@ var _ = g.Describe("[sig-mco][Suite:openshift/machine-config-operator/longdurati logger.Infof("OK!\n") }) + + g.It("[PolarionID:68684][OTP] machine-config-controller pod restart should not make nodes unschedulable [Disruptive]", func() { + var ( + controller = NewController(oc.AsAdmin()) + masterNode = NewMachineConfigPool(oc.AsAdmin(), MachineConfigPoolMaster).GetNodesOrFail()[0] + ) + + exutil.By("Check that nodes are not modified when the controller pod is removed") + labels, err := masterNode.Get(`{.metadata.labels}`) + o.Expect(err).NotTo(o.HaveOccurred(), "Error getting the labels in node %s", masterNode.GetName()) + + masterNode.oc.NotShowInfo() // avoid spamming the logs + o.Consistently(func(gm o.Gomega) { // Passing o.Gomega as parameter we can use assertions inside the Consistently function without breaking the retries. + logger.Infof("Remove controller pod") + gm.Expect(controller.RemovePod()).To(o.Succeed(), "Could not remove the controller pod") + + logger.Infof("Check that the node was not modified") + gm.Consistently(func(gm o.Gomega) { + gm.Expect(masterNode.Get(`{.metadata.labels}`)).To(o.MatchJSON(labels), + "Labels in node %s have changed after removing the controller pod, and they should not change", masterNode.GetName()) + gm.Expect(masterNode.IsCordoned()).To(o.BeFalse(), + "The node %s was cordoned after removing the controller pod. Node: \n%s", + masterNode.GetName(), masterNode.PrettyString()) + }, "10s", "0s"). + Should(o.Succeed(), + "The node %s was modified when the controller pod was removed") + }, "4m", "1s"). + Should(o.Succeed(), + "When we remove the controller pod the node %s is modified") + + logger.Infof("OK!\n") + }) + + g.It("[PolarionID:82299][OTP] Check race condition in rpm-ostree update logic [Disruptive]", func() { + mcp, cleanup, err := GetCompactCompatibleOrCustomPool(oc.AsAdmin(), 1) + defer cleanup() + o.Expect(err).NotTo(o.HaveOccurred(), "Error getting pool for testing") + + var ( + node = mcp.GetSortedNodesOrFail()[0] + + overrideContent = `[Service] +ExecStartPre=/bin/bash -c "echo 'exec start pre'; /bin/sleep 15; echo 'exec start pre end'"` + overridePath = "/etc/systemd/system/ostree-finalize-staged-hold.service.d/override.conf" + overrideMode = "0600" + + fileConfig = getBase64EncodedFileConfig(overridePath, overrideContent, overrideMode) + + mcOverrideName = fmt.Sprintf("99-%s-tc-%s-override", mcp.GetName(), GetCurrentTestPolarionIDNumber()) + mcOverride = NewMachineConfig(oc.AsAdmin(), mcOverrideName, mcp.GetName()) + + kernelArgs = "abc=def" + + mcKernelArgsName = fmt.Sprintf("99-%s-tc-%s-kernelargs", mcp.GetName(), GetCurrentTestPolarionIDNumber()) + mcKernelArgs = NewMachineConfig(oc.AsAdmin(), mcKernelArgsName, mcp.GetName()) + ) + + exutil.By("Override ostree finalizer") + mcOverride.parameters = []string{fmt.Sprintf("FILES=[%s]", fileConfig)} + mcOverride.skipWaitForMcp = true + defer mcOverride.DeleteWithWait() + mcOverride.create() + logger.Infof("OK!\n") + + exutil.By("Wait for the override MC to be applied") + mcp.waitForComplete() + logger.Infof("OK!\n") + + exutil.By("Check that the file was correctly deployed") + o.Eventually(NewRemoteFile(node, overridePath), "2m", "10s").Should(HaveContent(overrideContent), + "Wrong content in file %s", overridePath) + logger.Infof("OK!\n") + + exutil.By("Configure kernel args") + mcKernelArgs.parameters = []string{fmt.Sprintf(`KERNEL_ARGS=["%s"]`, kernelArgs)} + mcKernelArgs.skipWaitForMcp = true + defer mcKernelArgs.Delete() + mcKernelArgs.create() + logger.Infof("OK!\n") + + exutil.By("Wait for the kernelargs MC to be applied") + mcp.waitForComplete() + logger.Infof("OK!\n") + + exutil.By("Check that the kernel args were correctly applied") + o.Expect(node.IsKernelArgEnabled(kernelArgs)).To(o.BeTrue(), + "Kernel argument %s was not enabled in the node %s", kernelArgs, node.GetName()) + logger.Infof("OK!\n") + }) + + g.It("[PolarionID:83134][OTP] Check that MCC can find all requested resources [Serial]", func() { + var ( + mcc = NewController(oc.AsAdmin()) + resourceNotFoundErrorMsg = "the server could not find the requested resource" + listFailureErrorMsg = "failed to list" + ) + + exutil.By("Check that MCC can find all requested resources") + o.Eventually(mcc.GetLogs, "1m", "20s").ShouldNot(o.And( + o.ContainSubstring(resourceNotFoundErrorMsg), + o.ContainSubstring(listFailureErrorMsg)), + "MCC is reporting that some resources cannot be found or listed") + logger.Infof("OK!\n") + + }) + + g.It("[PolarionID:83943][OTP] CoreDNS Static Pod Redeploy Causes DNS Failure and rpm-ostree disable on vSphere IPI [Disruptive]", func() { + skipTestIfSupportedPlatformNotMatched(oc, VspherePlatform) + var ( + mcName = "custom-coredns-and-osimage" + coreDNSManifestPath = "/etc/kubernetes/manifests/coredns.yaml" + newCPU = "50m" + newMemory = "50Mi" + mcp = GetCompactCompatiblePool(oc.AsAdmin()) + node = mcp.GetNodesOrFail()[0] + dockerFileCommands = `RUN echo "hello" > /etc/test.txt` + ) + + exutil.By("Get current CoreDNS manifest from the node") + coreDNSFile := NewRemoteFile(node, coreDNSManifestPath) + o.Expect(coreDNSFile.Fetch()).NotTo(o.HaveOccurred(), + "Failed to get CoreDNS manifest from node %s", node.GetName()) + coreDNSContent := coreDNSFile.GetTextContent() + o.Expect(coreDNSContent).NotTo(o.BeEmpty(), "CoreDNS manifest is empty") + logger.Infof("OK!\n") + + exutil.By("Modify CoreDNS manifest to change CPU and memory") + cpuRegex := regexp.MustCompile(`cpu:\s*\d+m`) + memoryRegex := regexp.MustCompile(`memory:\s*\d+Mi`) + modifiedCoreDNS := cpuRegex.ReplaceAllString(coreDNSContent, "cpu: "+newCPU) + modifiedCoreDNS = memoryRegex.ReplaceAllString(modifiedCoreDNS, "memory: "+newMemory) + logger.Infof("OK!\n") + + exutil.By("Pause the MCP") + mcp.pause(true) + defer mcp.pause(false) + o.Expect(mcp.IsPaused()).Should(o.BeTrue(), "%s pool is not paused", mcp.GetName()) + logger.Infof("OK!\n") + + // Build the new osImage + osImageBuilder := OsImageBuilderInNode{ + node: node, + dockerFileCommands: dockerFileCommands, + } + digestedImage, err := osImageBuilder.CreateAndDigestOsImage() + o.Expect(err).NotTo(o.HaveOccurred(), + "Error creating the new osImage") + logger.Infof("OK\n") + logger.Infof("Digested image: %s\n", digestedImage) + + exutil.By("Create MC with modified CoreDNS manifest and custom osImage") + mc := NewMachineConfig(oc.AsAdmin(), mcName, mcp.GetName()) + fileConfig := getBase64EncodedFileConfig(coreDNSManifestPath, modifiedCoreDNS, "420") + mc.parameters = []string{fmt.Sprintf("FILES=[%s]", fileConfig), "OS_IMAGE=" + digestedImage} + defer mc.DeleteWithWait() + mc.skipWaitForMcp = true + mc.create() + logger.Infof("OK!\n") + + exutil.By("Get initial CoreDNS pod creation time before unpause") + initialCreationTime, err := getCoreDNSWorkerPodCreationTime(oc) + o.Expect(err).NotTo(o.HaveOccurred(), "Failed to get initial creation time") + o.Expect(initialCreationTime).NotTo(o.BeEmpty(), "No CoreDNS pods found on worker nodes") + logger.Infof("OK!\n") + + exutil.By("Unpause the MCP") + mcp.pause(false) + logger.Infof("OK!\n") + + exutil.By("Verify CoreDNS pods are recreated when MCP starts updating") + o.Eventually(getCoreDNSWorkerPodCreationTime, "15m", "30s"). + WithArguments(oc). + ShouldNot(o.Or(o.Equal(initialCreationTime), o.BeEmpty()), + "CoreDNS pods were not recreated after MCP update started") + logger.Infof("OK!\n") + + exutil.By("Verify MCP completes update without getting stuck") + mcp.waitForComplete() + logger.Infof("OK!\n") + + exutil.By("Verify CoreDNS manifest changes are applied on the node") + o.Expect(coreDNSFile.Fetch()).NotTo(o.HaveOccurred(), + "Failed to re-fetch CoreDNS manifest from node %s", node.GetName()) + o.Expect(coreDNSFile).To(HaveContent(o.ContainSubstring("cpu: "+newCPU)), + "CPU value not updated in CoreDNS manifest on node %s", node.GetName()) + o.Expect(coreDNSFile).To(HaveContent(o.ContainSubstring("memory: "+newMemory)), + "Memory value not updated in CoreDNS manifest on node %s", node.GetName()) + logger.Infof("OK!\n") + + exutil.By("Verify osImage is applied on the node") + o.Expect(node.GetCurrentBootOSImage()).Should(o.Equal(digestedImage), "Custom osImage not applied on the node") + logger.Infof("OK!\n") + }) }) diff --git a/test/extended-priv/mco_kernel.go b/test/extended-priv/mco_kernel.go index 72ad49a47b..d8562e1055 100644 --- a/test/extended-priv/mco_kernel.go +++ b/test/extended-priv/mco_kernel.go @@ -335,4 +335,74 @@ var _ = g.Describe("[sig-mco][Suite:openshift/machine-config-operator/longdurati "The kernel arguments are not the expected ones") logger.Infof("OK!\n") }) + + g.It("[PolarionID:72132][OTP] enable FIPS by Machine-Config-Operator not supported [Disruptive]", func() { + var ( + mcTemplate = "change-fips.yaml" + mcName = "mco-tc-25819-master-fips" + wMcp = NewMachineConfigPool(oc.AsAdmin(), MachineConfigPoolWorker) + mMcp = NewMachineConfigPool(oc.AsAdmin(), MachineConfigPoolMaster) + + expectedRDMessage = regexp.QuoteMeta("detected change to FIPS flag; refusing to modify FIPS on a running cluster") + expectedRDReason = "" + ) + + // If FIPS is already enabled, we skip the test case + skipTestIfFIPSIsEnabled(oc.AsAdmin()) + + exutil.By("Try to enable FIPS in master pool") + mMc := NewMachineConfig(oc.AsAdmin(), mcName, MachineConfigPoolMaster).SetMCOTemplate(mcTemplate) + mMc.parameters = []string{"FIPS=true"} + mMc.skipWaitForMcp = true + + validateMcpRenderDegraded(mMc, mMcp, expectedRDMessage, expectedRDReason) + logger.Infof("OK!\n") + + exutil.By("Try to enable FIPS in worker pool") + wMc := NewMachineConfig(oc.AsAdmin(), mcName, MachineConfigPoolWorker).SetMCOTemplate(mcTemplate) + wMc.parameters = []string{"FIPS=true"} + wMc.skipWaitForMcp = true + + validateMcpRenderDegraded(wMc, wMcp, expectedRDMessage, expectedRDReason) + logger.Infof("OK!\n") + + }) + + g.It("[PolarionID:72135][OTP] Refuse to disable FIPS mode by Machine-Config-Operator [Disruptive]", func() { + var ( + mMcName = "99-master-fips" + wMcName = "99-worker-fips" + wMcp = NewMachineConfigPool(oc.AsAdmin(), MachineConfigPoolWorker) + mMcp = NewMachineConfigPool(oc.AsAdmin(), MachineConfigPoolMaster) + mMc = NewMachineConfig(oc.AsAdmin(), mMcName, MachineConfigPoolMaster) + wMc = NewMachineConfig(oc.AsAdmin(), wMcName, MachineConfigPoolWorker) + + expectedRDMessage = regexp.QuoteMeta("detected change to FIPS flag; refusing to modify FIPS on a running cluster") + expectedRDReason = "" + ) + + // If FIPS is already disabled, we skip the test case + skipTestIfFIPSIsNotEnabled(oc.AsAdmin()) + + defer func() { + logger.Infof("Starting defer logic") + o.Expect(mMc.Patch("merge", `{"spec":{"fips": true}}`)).To(o.Succeed(), + "Failed to restore FIPS=true in master MC") + o.Expect(wMc.Patch("merge", `{"spec":{"fips": true}}`)).To(o.Succeed(), + "Failed to restore FIPS=true in worker MC") + wMcp.RecoverFromDegraded() + mMcp.RecoverFromDegraded() + }() + + exutil.By("Patch the master-fips MC and set fips=false") + o.Expect(mMc.Patch("merge", `{"spec":{"fips": false}}`)).To(o.Succeed(), + "Failed to patch master MC with fips=false") + checkDegraded(mMcp, expectedRDMessage, expectedRDReason, "RenderDegraded", false, 1) + + exutil.By("Try to disasble FIPS in worker pool") + o.Expect(wMc.Patch("merge", `{"spec":{"fips": false}}`)).To(o.Succeed(), + "Failed to patch worker MC with fips=false") + checkDegraded(wMcp, expectedRDMessage, expectedRDReason, "RenderDegraded", false, 1) + + }) }) diff --git a/test/extended-priv/mco_security.go b/test/extended-priv/mco_security.go index 6c65897757..34eb804084 100644 --- a/test/extended-priv/mco_security.go +++ b/test/extended-priv/mco_security.go @@ -987,6 +987,415 @@ var _ = g.Describe("[sig-mco][Suite:openshift/machine-config-operator/longdurati o.Eventually(newMs.GetIsReady, "20m", "2m").Should(o.BeTrue(), "MachineSet %s is not ready.", newMs.GetName()) logger.Infof("OK!\n") }) + + g.It("[PolarionID:43278][OTP] security fix for unsafe cipher [Serial]", func() { + exutil.By("check go version >= 1.15") + _, clusterVersion, cvErr := exutil.GetClusterVersion(oc) + o.Expect(cvErr).NotTo(o.HaveOccurred()) + o.Expect(clusterVersion).NotTo(o.BeEmpty()) + logger.Infof("cluster version is %s", clusterVersion) + commitID, commitErr := getCommitID(oc, "machine-config", clusterVersion) + o.Expect(commitErr).NotTo(o.HaveOccurred()) + // there is a case that in the payload no commit id from mco + if commitID == "" { + g.Skip("No code change from MCO, skip this case") + } + logger.Infof("machine config commit id is %s", commitID) + goVersion, verErr := getGoVersion("machine-config-operator", commitID) + o.Expect(verErr).NotTo(o.HaveOccurred()) + logger.Infof("go version is: %f", goVersion) + o.Expect(goVersion).Should(o.BeNumerically(">=", 1.15)) + + exutil.By("verify TLS protocol version is 1.3") + intAPIServerURI, err := GetAPIServerInternalURI(oc.AsAdmin()) + o.Expect(err).NotTo(o.HaveOccurred()) + masterNode := NewNodeList(oc.AsAdmin()).GetAllMasterNodesOrFail()[0] + sslOutput, sslErr := masterNode.DebugNodeWithChroot("bash", "-c", "echo 'Q'|openssl s_client -connect "+intAPIServerURI+":6443") + logger.Infof("ssl protocol version is:\n %s", sslOutput) + o.Expect(sslErr).NotTo(o.HaveOccurred()) + o.Expect(sslOutput).Should(o.ContainSubstring("TLSv1.3")) + + exutil.By("verify whether the unsafe cipher is disabled") + cipherOutput, cipherErr := masterNode.DebugNodeWithOptions([]string{"--image=" + TestSSLImage, "-n", MachineConfigNamespace}, "testssl.sh", "--quiet", "--sweet32", "localhost:6443") + logger.Infof("test ssh script output:\n %s", cipherOutput) + o.Expect(cipherErr).NotTo(o.HaveOccurred()) + o.Expect(cipherOutput).Should(o.ContainSubstring("not vulnerable (OK)")) + }) + + g.It("[PolarionID:46965][OTP] Avoid workload disruption for GPG Public Key Rotation [Serial]", func() { + + exutil.By("create new machine config with base64 encoded gpg public key") + workerNode := NewNodeList(oc.AsAdmin()).GetAllLinuxWorkerNodesOrFail()[0] + startTime := workerNode.GetDateOrFail() + mcName := "add-gpg-pub-key" + mcTemplate := "add-gpg-pub-key.yaml" + mc := NewMachineConfig(oc.AsAdmin(), mcName, MachineConfigPoolWorker).SetMCOTemplate(mcTemplate) + defer mc.DeleteWithWait() + mc.create() + + exutil.By("checkout machine config daemon logs to verify ") + log, err := exutil.GetSpecificPodLogs(oc, MachineConfigNamespace, MachineConfigDaemon, workerNode.GetMachineConfigDaemon(), "") + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(log).Should(o.ContainSubstring("/etc/machine-config-daemon/no-reboot/containers-gpg.pub")) + o.Expect(log).Should(o.ContainSubstring("Changes do not require drain, skipping")) + o.Expect(log).Should(o.MatchRegexp(MCDCrioReloadedRegexp)) + + o.Expect(workerNode.GetUptime()).Should(o.BeTemporally("<", startTime), + "The node %s must NOT be rebooted, but it was rebooted. Uptime date happened after the start config time.", workerNode.GetName()) + + exutil.By("verify crio.service status") + cmdOut, cmdErr := workerNode.DebugNodeWithChroot("systemctl", "is-active", "crio.service") + o.Expect(cmdErr).NotTo(o.HaveOccurred()) + o.Expect(cmdOut).Should(o.ContainSubstring("active")) + + }) + + g.It("[PolarionID:47062][OTP] change policy.json on worker nodes [Serial]", func() { + + exutil.By("create new machine config to change /etc/containers/policy.json") + workerNode := NewNodeList(oc.AsAdmin()).GetAllLinuxWorkerNodesOrFail()[0] + startTime := workerNode.GetDateOrFail() + mcName := "change-policy-json" + mcTemplate := "change-policy-json.yaml" + mc := NewMachineConfig(oc.AsAdmin(), mcName, MachineConfigPoolWorker).SetMCOTemplate(mcTemplate) + defer mc.DeleteWithWait() + mc.create() + + exutil.By("verify file content changes") + fileContent, fileErr := workerNode.DebugNodeWithChroot("cat", "/etc/containers/policy.json") + o.Expect(fileErr).NotTo(o.HaveOccurred()) + logger.Infof(fileContent) + o.Expect(fileContent).Should(o.ContainSubstring(`{"default": [{"type": "insecureAcceptAnything"}]}`)) + o.Expect(fileContent).ShouldNot(o.ContainSubstring("transports")) + + exutil.By("checkout machine config daemon logs to make sure node drain/reboot are skipped") + log, err := exutil.GetSpecificPodLogs(oc, MachineConfigNamespace, MachineConfigDaemon, workerNode.GetMachineConfigDaemon(), "") + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(log).Should(o.ContainSubstring("/etc/containers/policy.json")) + o.Expect(log).Should(o.ContainSubstring("Changes do not require drain, skipping")) + o.Expect(log).Should(o.MatchRegexp(MCDCrioReloadedRegexp)) + + o.Expect(workerNode.GetUptime()).Should(o.BeTemporally("<", startTime), + "The node %s must NOT be rebooted, but it was rebooted. Uptime date happened after the start config time.", workerNode.GetName()) + + exutil.By("verify crio.service status") + cmdOut, cmdErr := workerNode.DebugNodeWithChroot("systemctl", "is-active", "crio.service") + o.Expect(cmdErr).NotTo(o.HaveOccurred()) + o.Expect(cmdOut).Should(o.ContainSubstring("active")) + + }) + + g.It("[PolarionID:62084][OTP] Certificate rotation in paused pools [Disruptive]", func() { + var ( + wMcp = NewMachineConfigPool(oc.AsAdmin(), MachineConfigPoolWorker) + mMcp = NewMachineConfigPool(oc.AsAdmin(), MachineConfigPoolMaster) + certSecret = NewSecret(oc.AsAdmin(), "openshift-kube-apiserver-operator", "kube-apiserver-to-kubelet-signer") + ) + + exutil.By("Pause MachineConfigPools") + defer mMcp.waitForComplete() + defer wMcp.waitForComplete() + + defer wMcp.pause(false) + wMcp.pause(true) + defer mMcp.pause(false) + mMcp.pause(true) + logger.Infof("OK!\n") + + exutil.By("Get current kube-apiserver certificate") + initialCert := certSecret.GetDataValueOrFail("tls.crt") + logger.Infof("Current certificate length: %d", len(initialCert)) + logger.Infof("OK!\n") + + exutil.By("Rotate certificate") + o.Expect( + certSecret.Patch("merge", `{"metadata": {"annotations": {"auth.openshift.io/certificate-not-after": null}}}`), + ).To(o.Succeed(), + "The secret could not be patched in order to rotate the certificate") + logger.Infof("OK!\n") + + exutil.By("Get current kube-apiserver certificate") + logger.Infof("Wait for certificate rotation") + o.Eventually(certSecret.GetDataValueOrFail, "5m", "10s").WithArguments("tls.crt"). + ShouldNot(o.Equal(initialCert), + "The certificate was not rotated") + + newCert := certSecret.GetDataValueOrFail("tls.crt") + logger.Infof("New certificate length: %d", len(newCert)) + logger.Infof("OK!\n") + + o.Expect(initialCert).NotTo(o.Equal(newCert), + "The certificate was not rotated") + logger.Infof("OK!\n") + + // We verify all nodes in the pools (be aware that windows nodes do not belong to any pool, we are skipping them) + for _, node := range append(wMcp.GetNodesOrFail(), mMcp.GetNodesOrFail()...) { + logger.Infof("Checking certificate in node: %s", node.GetName()) + + rfCert := NewRemoteFile(node, "/etc/kubernetes/kubelet-ca.crt") + + // Eventually the certificate file in all nodes should contain the new rotated certificate + o.Eventually(func(gm o.Gomega) string { // Passing o.Gomega as parameter we can use assertions inside the Eventually function without breaking the retries. + gm.Expect(rfCert.Fetch()).To(o.Succeed(), + "Cannot read the certificate file in node:%s ", node.GetName()) + return rfCert.GetTextContent() + }, "5m", "10s"). + Should(o.ContainSubstring(newCert), + "The certificate file %s in node %s does not contain the new rotated certificate.", rfCert.GetFullPath(), node.GetName()) + logger.Infof("OK!\n") + } + + exutil.By("Unpause MachineConfigPools") + logger.Infof("Check that once we unpause the pools the pending config can be applied without problems") + wMcp.pause(false) + mMcp.pause(false) + wMcp.waitForComplete() + mMcp.waitForComplete() + + logger.Infof("OK!\n") + }) + + g.It("[PolarionID:65208][OTP] Check the visibility of certificates [Serial]", func() { + var ( + cc = NewControllerConfig(oc.AsAdmin(), "machine-config-controller") + mMcp = NewMachineConfigPool(oc.AsAdmin(), MachineConfigPoolMaster) + ) + + exutil.By("Check that the ControllerConfig resource is storing the right kube-apiserver-client-ca information") + kubeAPIServerClientCACM := NewNamespacedResource(oc.AsAdmin(), "ConfigMap", "openshift-config-managed", "kube-apiserver-client-ca") + kubeAPIServerClientCA := kubeAPIServerClientCACM.GetOrFail(`{.data.ca-bundle\.crt}`) + + ccKubeAPIServerClientCA, err := cc.GetKubeAPIServerServingCAData() + o.Expect(err).NotTo(o.HaveOccurred(), + "Error getting the kubeAPIServerServingCAData information from the ControllerConfig") + + // We write the Expect command so that the certificates are not printed in case of failure + o.Expect(strings.Trim(ccKubeAPIServerClientCA, "\n") == strings.Trim(kubeAPIServerClientCA, "\n")).To(o.BeTrue(), + "The value of kubeAPIServerServingCAData in the ControllerConfig does not equal the value of configmap -n openshift-config-managed kube-apiserver-client-ca") + logger.Infof("OK!\n") + + exutil.By("Check that the ControllerConfig resource is storing the right rootCAData information") + rootCADataCM := NewNamespacedResource(oc.AsAdmin(), "ConfigMap", "kube-system", "root-ca") + rootCAData := rootCADataCM.GetOrFail(`{.data.ca\.crt}`) + + ccRootCAData, err := cc.GetRootCAData() + o.Expect(err).NotTo(o.HaveOccurred(), + "Error getting the rootCAData information from the ControllerConfig") + + // We write the Expect command so that the certificates are not printed in case of failure + o.Expect(strings.Trim(ccRootCAData, "\n") == strings.Trim(rootCAData, "\n")).To(o.BeTrue(), + "The value of rootCAData in the ControllerConfig does not equal the value of configmap -n kube-system root-ca") + logger.Infof("OK!\n") + + exutil.By("Check the information from the KubeAPIServerServingCAData certificates") + + ccKCertsInfo, err := cc.GetCertificatesInfoByBundleFileName("KubeAPIServerServingCAData") + o.Expect(err).NotTo(o.HaveOccurred(), + "Error getting the controller config information for KubeAPIServerServingCAData certificates") + + kubeAPIServerCertsInfo, err := GetCertificatesInfoFromPemBundle("KubeAPIServerServingCAData", []byte(ccKubeAPIServerClientCA)) + o.Expect(err).NotTo(o.HaveOccurred(), + "Error extracting certificate info from KubeAPIServerServingCAData pem bundle") + + o.Expect(kubeAPIServerCertsInfo).To(o.Equal(ccKCertsInfo), + "The ControllerConfig is not reporting the right information about the certificates in KubeAPIServerServingCAData bundle") + + logger.Infof("OK!\n") + + exutil.By("Check the information from the rootCAData certificates") + + ccRCertsInfo, err := cc.GetCertificatesInfoByBundleFileName("RootCAData") + o.Expect(err).NotTo(o.HaveOccurred(), + "Error getting the controller config information for rootCAData certificates") + + rootCACertsInfo, err := GetCertificatesInfoFromPemBundle("RootCAData", []byte(ccRootCAData)) + o.Expect(err).NotTo(o.HaveOccurred(), + "Error extracting certificate info from RootCAData pem bundle") + + o.Expect(rootCACertsInfo).To(o.Equal(ccRCertsInfo), + "The ControllerConfig is not reporting the right information about the certificates in rootCAData bundle") + logger.Infof("OK!\n") + + exutil.By("Check that MCPs are reporting information regarding kubeapiserverserviccadata certificates") + certsExpiry, err := mMcp.GetCertsExpiry() + o.Expect(err).NotTo(o.HaveOccurred(), + "Error getting the certificates expiry information from master MCP") + + o.Expect(certsExpiry).To(o.HaveLen(len(ccKCertsInfo)), + "The expry certs info reported in master MCP has len %d, but the list of kubeAPIServer certs has len %d.\nExpiry:%s\nKubeAPIServer:%s", + len(certsExpiry), len(ccKCertsInfo), certsExpiry, ccKCertsInfo) + + expiryByBundle := map[string]struct { + Expiry string + Subject string + }{} + for _, ce := range certsExpiry { + expiryByBundle[ce.Bundle] = struct { + Expiry string + Subject string + }{Expiry: ce.Expiry, Subject: ce.Subject} + } + + for _, certInfo := range ccKCertsInfo { + ce, ok := expiryByBundle[certInfo.BundleFile] + o.Expect(ok).To(o.BeTrue(), + "Missing expiry data for bundle %s", certInfo.BundleFile) + // Date fields have been temporarily removed by devs: https://github.com/openshift/machine-config-operator/pull/3866 + o.Expect(ce.Expiry).To(o.Equal(certInfo.NotAfter), + "Expiry information does not match ControllerConfig") + o.Expect(ce.Subject).To(o.Equal(certInfo.Subject), + "Subject information does not match ControllerConfig") + } + + logger.Infof("OK!\n") + + exutil.By("Check that the description of ControllerConfig includes the certificates info") + ccDesc, err := cc.Describe() + o.Expect(err).NotTo(o.HaveOccurred(), + "Error describing the ControllerConfig resource") + + o.Expect(ccDesc).To(o.And( + o.ContainSubstring("Controller Certificates:"), + o.ContainSubstring("Bundle File"), + // Date fields have been temporarily removed by devs: https://github.com/openshift/machine-config-operator/pull/3866 + o.ContainSubstring("Not After"), + o.ContainSubstring("Not Before"), + o.ContainSubstring("Signer"), + o.ContainSubstring("Subject"), + ), + "The ControllerConfig description should include information about the certificate, but it does not:\n%s", ccDesc) + logger.Infof("OK!\n") + + exutil.By("Check that the description of MCP includes the certificates info") + mMcpDesc, err := mMcp.Describe() + o.Expect(err).NotTo(o.HaveOccurred(), + "Error describing the master MCP resource") + + o.Expect(mMcpDesc).To(o.And( + o.ContainSubstring("Cert Expirys"), + o.ContainSubstring("Bundle"), + // Date fields have been temporarily removed by devs: https://github.com/openshift/machine-config-operator/pull/3866 + o.ContainSubstring("Expiry"), + ), + "The master MCP description should include information about the certificate, but it does not:\n%s", ccDesc) + logger.Infof("OK!\n") + }) + + g.It("[PolarionID:66436][OTP] disable weak SSH cipher suites [Serial]", func() { + + var ( + // the list of weak cipher suites can be found here: https://issues.redhat.com/browse/OCPBUGS-15202 + weakSuites = []string{"TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA", + "TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA", + "TLS_RSA_WITH_AES_128_CBC_SHA", + "TLS_RSA_WITH_AES_256_CBC_SHA", + "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256"} + ) + + exutil.By("Verify that the controller pod is not using weakSuites") + ccRbacProxyArgs, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("pod", "-n", MachineConfigNamespace, "-l", ControllerLabel+"="+ControllerLabelValue, + "-o", `jsonpath={.items[0].spec.containers[?(@.name=="kube-rbac-proxy")].args}`).Output() + + o.Expect(err).NotTo(o.HaveOccurred(), + "Error getting the arguments used in kube-rbac-proxy container in the controller pod") + + o.Expect(ccRbacProxyArgs).To(o.ContainSubstring("--tls-cipher-suites"), + "Controller's kube-rbac-proxy container is not declaring the list of allowed cipher suites") + + for _, weakSuite := range weakSuites { + logger.Infof("Verifying that %s is not used", weakSuite) + o.Expect(ccRbacProxyArgs).NotTo(o.ContainSubstring(weakSuite), + "Controller's kube-rbac-proxy container is using the weak cipher suite %s, and it should not", weakSuite) + logger.Infof("Suite ok") + } + logger.Infof("OK!\n") + + exutil.By("Connect to the rbac-proxy service to verify the cipher") + mMcp := NewMachineConfigPool(oc.AsAdmin(), MachineConfigPoolMaster) + masterNode := mMcp.GetNodesOrFail()[0] + cipherOutput, cipherErr := masterNode.DebugNodeWithOptions([]string{"--image=" + TestSSLImage, "-n", MachineConfigNamespace}, "testssl.sh", "--color", "0", "localhost:9001") + logger.Infof("test ssh script output:\n %s", cipherOutput) + o.Expect(cipherErr).NotTo(o.HaveOccurred()) + o.Expect(cipherOutput).Should(o.MatchRegexp(`Obsoleted CBC ciphers \(AES, ARIA etc.\) +not offered`)) + + for _, weakSuite := range weakSuites { + logger.Infof("Verifying that %s is not used", weakSuite) + o.Expect(cipherOutput).NotTo(o.ContainSubstring(weakSuite), + "The rbac-proxy service cipher test is reporting weak cipher suite: %s", weakSuite) + logger.Infof("Suite ok") + } + logger.Infof("OK!\n") + }) + + g.It("[PolarionID:67395][OTP] rotate kubernetes certificate authority. Certificates managed via non-MC path [Disruptive]", func() { + + var ( + wMcp = NewMachineConfigPool(oc.AsAdmin(), MachineConfigPoolWorker) + mMcp = NewMachineConfigPool(oc.AsAdmin(), MachineConfigPoolMaster) + mcList = NewMachineConfigList(oc.AsAdmin()) + certSecret = NewSecret(oc.AsAdmin(), "openshift-kube-apiserver-operator", "kube-apiserver-to-kubelet-signer") + ) + + logger.Infof("Get currently rendered MCs") + initialMCs, err := mcList.GetAll() + o.Expect(err).NotTo(o.HaveOccurred(), + "Error getting the list of rendered MCs") + logger.Infof("%d rendered MCs", len(initialMCs)) + logger.Infof("OK!\n") + + exutil.By("Get start time and start collecting events.") + // be aware that windows nodes do not belong to any pool, we are skipping them + checkedNodes := append(mMcp.GetNodesOrFail(), wMcp.GetNodesOrFail()...) + + startTime, dErr := checkedNodes[0].GetDate() + o.Expect(dErr).ShouldNot(o.HaveOccurred(), "Error getting date in node %s", checkedNodes[0].GetName()) + + for i := range checkedNodes { + o.Expect(checkedNodes[i].IgnoreEventsBeforeNow()).NotTo(o.HaveOccurred(), + "Error getting the latest event in node %s", checkedNodes[i].GetName()) + } + logger.Infof("OK!\n") + + exutil.By("Rotate certificate") + newCert := rotateTLSSecretOrFail(certSecret) + logger.Infof("OK!\n") + + exutil.By("Check that no new MC is created") + o.Consistently(mcList.GetAll, "3m", "1m").Should(o.HaveLen(len(initialMCs)), + "New machine configs have been created, but they should not be created") + logger.Infof("OK!\n") + + for _, node := range checkedNodes { + exutil.By(fmt.Sprintf("Checking that the certificate is rotated in node: %s", node.GetName())) + + rfCert := NewRemoteFile(node, "/etc/kubernetes/kubelet-ca.crt") + + // Eventually the certificate file in all nodes should contain the new rotated certificate + o.Eventually(func(gm o.Gomega) string { // Passing o.Gomega as parameter we can use assertions inside the Eventually function without breaking the retries. + gm.Expect(rfCert.Fetch()).To(o.Succeed(), + "Cannot read the certificate file in node:%s ", node.GetName()) + return rfCert.GetTextContent() + }, "5m", "10s"). + Should(o.ContainSubstring(newCert), + "The certificate file %s in node %s does not contain the new rotated certificate.", rfCert.GetFullPath(), node.GetName()) + logger.Infof("OK!\n") + + exutil.By(fmt.Sprintf("Checking that node: %s was not rebooted", node.GetName())) + o.Expect(node.GetUptime()).Should(o.BeTemporally("<", startTime), + "The node %s must NOT be rebooted after rotating the certificate, but it was rebooted. Uptime date happened after the start config time.", node.GetName()) + + logger.Infof("OK!\n") + + exutil.By(fmt.Sprintf("Checking events in node: %s", node.GetName())) + o.Expect(node.GetEvents()).NotTo(HaveEventsSequence("Drain"), + "Error, a Drain event was triggered but it shouldn't") + o.Expect(node.GetEvents()).NotTo(HaveEventsSequence("Reboot"), + "Error, a Reboot event was triggered but it shouldn't") + + logger.Infof("OK!\n") + + } + }) }) // EventuallyFileExistsInNode fails the test if the certificate file does not exist in the node after the time specified as parameters diff --git a/test/extended-priv/testdata/files/add-gpg-pub-key.yaml b/test/extended-priv/testdata/files/add-gpg-pub-key.yaml new file mode 100644 index 0000000000..ac02a217d3 --- /dev/null +++ b/test/extended-priv/testdata/files/add-gpg-pub-key.yaml @@ -0,0 +1,23 @@ +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + name: add-gpg-pub-key +objects: + - kind: MachineConfig + apiVersion: machineconfiguration.openshift.io/v1 + metadata: + name: "${NAME}" + labels: + machineconfiguration.openshift.io/role: "${POOL}" + spec: + config: + ignition: + version: 3.2.0 + storage: + files: + - contents: + source: data:;base64,LS0tLS1CRUdJTiBQR1AgUFVCTElDIEtFWSBCTE9DSy0tLS0tCgptUUdOQkdHNFBDTUJEQURzVzVURjNpRFhJMWlRMkYvakZ0TlF5Z2V6bSszOFQra2tvWDYzTVIzN1pJZXVGUHN0CjdmNU95N0tWQTZWcXJFbGsrSytkWmFVekYyWXRvVXlDSGQ2aXdRTmNOSGJMVHA5dDJya2kvQjFjT1RjZHpRNGcKREhrcXNyTDY2Mm5HUHk2bStmdXM5NGtuY0Q2VjJZVmVJZm9lVi8raTROL2hsbU5aNnVKLzNyL1BJclQ2WWVHeApRYm0xRkM0Q1hEY1FCL1lqUFM2TUs5UE1tdy95eXdhTysrVUpYeGtkTjkrbXd2bUlla1JNdU1OZG1SY1dFSlZWCmJJZjlvVitwclBtS2lyWVVrY0lKVEs3YVYvRlBwNkplajVlMyszQmhuVnlCOFE2QVAwd3RPRTByM3hMdFN1SnIKZ1JZWm1qQThSSm82UXpTR0pFODBZTEFvRmRWb0N4TGFvQlFvVGppSjVvVjNyTzgySVV5c2U3M2gvQktudno5egpMQk1YNDB5WVY2b1lHcFE4eW9hY3dmVTh6b2k3WStmcnBnS1NGL2YrQ05oZ0w0Z2plbkp4NFg2SjZSMG1hY1RoCmFQdjJyb0NRdDJZenQ5Nys5NW1PWldMdkhtVDZNa01NVjdBWm5VVnozUkc4SDVUUk81RmJtZGdFM29vUm56OVQKazdzTmRsMTYyZUE2Z1NrQUVRRUFBYlFlYldOdkxYUmxjM1FnUEcxamJ5MTBaWE4wUUhKbFpHaGhkQzVqYjIwKwppUUhVQkJNQkNBQStGaUVFM0lmRHNDQ2Z0ZkFva2I0ZUpiUGpGbWtTa2lzRkFtRzRQQ01DR3dNRkNRUENad0FGCkN3a0lCd0lHRlFvSkNBc0NCQllDQXdFQ0hnRUNGNEFBQ2drUUpiUGpGbWtTa2lzTkh3d0ExR0U2OFdrb3Q4NWIKS21OMjlaTnJGWmY5aXA1cHpENVFVTzNxaXE0R25ZbXNRMW1hWFBhd0hoa0VQREFuZTY1ZUM2QWxlbXhtV2RSbQpDZzBNTmF4endUUnhPaUI2ckVnNEpVRjdINUtFcUx6cHI1SUNNRWZORWcxaDREWkhKQXdOZWNDb2NWdHA5aFlhCmdwZzdyYU4xZG5qeVRSQXV0anpMTVp0RHVYTHM0ZU5nSWR4aTlrS003U3pIU0xOVmRtMExoU0FSSmN6ckhCQnQKQXdhS1JTbGN2aWl1L1pHVlJyUm9ML3oyNStZcEF3UlNYS1c2cXlFaWR1N2hiY2tYNkdmVzdXL3J3YUFyaFlQbAppOG9vclFib0hBY3hxYjU5VlZjN0dGY09BSjM2U050a2FIcjM3M0s0bG5QWDZ5V3lhL0VEb2RVSTY5Y3A3SitkCnhoYi9EZWJKVXFnRGJvaTVubHNad3pQZWduQXNURWF2a21VdkV0UVlJaTNOTE1DSVpMUHhlZGVIaVhwWHk5VkQKZG0yUHB5TDZ1b1V3S0tpNnZHRjFDRUszODJBYTJqUnhCY1VKTTh1UE9JR25jSmt6MElCV0lTY3FZeG1WQzVuUwoxQ0JxVDVYVGt1dGJFUlhKWkVoRWJzVVpQTVhRNEZhdUl0bEhaQVc3UDdlak9PWXhENlVHdVFHTkJHRzRQQ01CCkRBREFoL282M05tLzdFRmhSbEpldjZHamtHaXFyV3FKWVJXYXYyaFJPaVUzNWlSOHlndnJ2UkhVMHY4ZmFBenIKY3N1RTVuanJzd3dBdnhFbTNDY25Hb3g0dDFUVHkzMjRyRWMwVExTdlpuaG5XVWY2TmttSVl5aGFibkdsQUJFNApFNjJ5Y1Qwa3BCRVp1RktRZEN5Ry9WNkRhMXQ2cEF3djh2bjlEcHJtQlA4NnlYNUVBdjZGRHo3Nmk4S3YxQ1dCCmhoMjdnUDAvM2FVcVdxSEtnaUlLMllCZ0J2R2VZa3ltWkJ5SFBYZEkxSDVVRzYxaVF0SksyOTZHcDVjRit2K2wKZHZXTnc0YUFkTWdibkJQTkw5SlFnanIrcXhhL2VzbHpwWjhBZWMxVmFlRkFnL2YrdDMzOUNXb2JjblJESEh4ZApjNFg3c2l5dWlIKzdBZXJDZzRqdHJvZWY1VDAzdXU0dmFZSENBVENSUVU1aVlIQUNuY1VuYWhxQVFHVkR5aDRQCnF5cUs2OFRkbmFuak5tZE9DaUhqMis1T2JndFBJQmhRMjJUdVNydE9wKzg3d2ZJK3BNY3pnTFFOVVMwdzEzMlUKK2Z0azJSYnNyRTVIY3RWc2VPelZYODFVYzlLRzc3VnlJZks5WWNUb2pway9zdisyNjgrVFhxc25TcWxxdWZWWAowRjhBRVFFQUFZa0J2QVFZQVFnQUpoWWhCTnlIdzdBZ243WHdLSkcrSGlXejR4WnBFcElyQlFKaHVEd2pBaHNNCkJRa0R3bWNBQUFvSkVDV3o0eFpwRXBJclBTQUwvakRYOFNENy9iZEdHT0kxZnN1OENaWDAvOExUUklidTc0THcKVUsxT09IN29ySjgwMnJlczRwRjVpZDFBS3dTa2tQSHJXR25zemxGQnhLWEp4cFFkU292T2N1blJtUi84aVRzMQpOQnV2WEFma1lLdzkyL0RBd1JiYkxBejFMTHRJRU4xMFliYnU1eXgzQXdrZVRtb0xGVm1OZWV1SXNMMXFQU0h4Cm9FM1BlWE4vZFlzUm9KRTJtWGRvdDV3QklqRE8rZEVpTWw5WTdXVFFxaW5uUnVCbXU4WHNnNEVCUldWVi9iZXoKNU1KUkd5SlhkcmU5NmpUb0RxOStlRUNFMmtFVTc2djY4QjhqMUZjZ05HWDZFN1JEcEIrcnptdkpTaUUzWXhQZgpweEE0Y1puN25aTnU0RWJWY0l4cUgwWjdudmpicDhXRlhhTGN0VkUwS1NSTlNDeW5LYUdhU3o5bUZMOVBjMnhPCllsN0w0ZjUvdVBmSVliVFhKTlJUbmVEWEs4N0JlVUtpWGVDMlV5T3ZpZTZZbGNkeDdwdDVMdVV4eGp4azE1cm0KbkNvZko1c1B5QW1qQ0x4YURHVnJ2a1l6a3JMYlRNRURRcUpDQVd0ZURiL05oMXdpQ2pkTFA2enNENnZkdEJvUgpNa01nK3VoUHhEOXJaWWtoQm5QcE1zT0wrdVRONHc9PQo9TTRDbAotLS0tLUVORCBQR1AgUFVCTElDIEtFWSBCTE9DSy0tLS0tCg== + path: /etc/machine-config-daemon/no-reboot/containers-gpg.pub +parameters: + - name: NAME + - name: POOL diff --git a/test/extended-priv/testdata/files/change-fips.yaml b/test/extended-priv/testdata/files/change-fips.yaml new file mode 100644 index 0000000000..c8ab0d13a0 --- /dev/null +++ b/test/extended-priv/testdata/files/change-fips.yaml @@ -0,0 +1,17 @@ +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + name: change-fips-machine-config-template +objects: + - kind: MachineConfig + apiVersion: machineconfiguration.openshift.io/v1 + metadata: + labels: + machineconfiguration.openshift.io/role: "${POOL}" + name: "${NAME}" + spec: + fips: ${{FIPS}} +parameters: + - name: NAME + - name: POOL + - name: FIPS diff --git a/test/extended-priv/testdata/files/change-policy-json.yaml b/test/extended-priv/testdata/files/change-policy-json.yaml new file mode 100644 index 0000000000..5c1464db54 --- /dev/null +++ b/test/extended-priv/testdata/files/change-policy-json.yaml @@ -0,0 +1,23 @@ +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + name: change-policy-json +objects: + - kind: MachineConfig + apiVersion: machineconfiguration.openshift.io/v1 + metadata: + name: "${NAME}" + labels: + machineconfiguration.openshift.io/role: "${POOL}" + spec: + config: + ignition: + version: 3.2.0 + storage: + files: + - contents: + source: data:;base64,eyJkZWZhdWx0IjogW3sidHlwZSI6ICJpbnNlY3VyZUFjY2VwdEFueXRoaW5nIn1dfQo= + path: /etc/containers/policy.json +parameters: + - name: NAME + - name: POOL diff --git a/test/extended-priv/util.go b/test/extended-priv/util.go index 5e630ff08a..c7b5c398e9 100644 --- a/test/extended-priv/util.go +++ b/test/extended-priv/util.go @@ -9,6 +9,8 @@ import ( "encoding/json" "encoding/pem" "fmt" + "io" + "net/http" "net/url" "os" "os/exec" @@ -1312,3 +1314,58 @@ func skipTestIfOsIsNotCoreOs(oc *exutil.CLI) *Node { } return allCoreOs[0] } + +func getCommitID(oc *exutil.CLI, component, clusterVersion string) (string, error) { + secretFile, secretErr := getPullSecret(oc) + if secretErr != nil { + return "", secretErr + } + outFilePath, ocErr := oc.AsAdmin().WithoutNamespace().Run("adm").Args("release", "info", "--registry-config="+secretFile, "--commits", clusterVersion, "--insecure=true").OutputToFile("commitIdLogs.txt") + if ocErr != nil { + return "", ocErr + } + rawBytes, cmdErr := os.ReadFile(outFilePath) + if cmdErr != nil { + return "", cmdErr + } + for _, line := range strings.Split(string(rawBytes), "\n") { + if strings.Contains(line, component) { + fields := strings.Fields(line) + if len(fields) >= 3 { + return fields[2], nil + } + } + } + return "", fmt.Errorf("component %q not found in release info output", component) +} + +func getGoVersion(component, commitID string) (float64, error) { + url := fmt.Sprintf("https://raw.githubusercontent.com/openshift/%s/%s/go.mod", component, commitID) + resp, err := http.Get(url) //nolint:noctx + if err != nil { + return 0, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return 0, fmt.Errorf("unexpected HTTP status %d fetching go.mod for %s@%s", resp.StatusCode, component, commitID) + } + body, err := io.ReadAll(resp.Body) + if err != nil { + return 0, err + } + for _, line := range strings.Split(string(body), "\n") { + if !strings.HasPrefix(line, "go ") { + continue + } + parts := strings.Fields(line) // ["go", "X.Y.Z"] + if len(parts) < 2 { + return 0, fmt.Errorf("malformed go directive: %q", line) + } + segs := strings.SplitN(parts[1], ".", 3) + if len(segs) < 2 { + return 0, fmt.Errorf("wrong go version string %q", parts[1]) + } + return strconv.ParseFloat(segs[0]+"."+segs[1], 64) + } + return 0, fmt.Errorf("go directive not found in go.mod for %s@%s", component, commitID) +} diff --git a/test/extended-priv/util/clusters.go b/test/extended-priv/util/clusters.go index 69d2fd0a2e..e97180b18f 100644 --- a/test/extended-priv/util/clusters.go +++ b/test/extended-priv/util/clusters.go @@ -2,6 +2,7 @@ package util import ( "context" + "fmt" "strings" o "github.com/onsi/gomega" @@ -89,3 +90,27 @@ func SkipOnSingleNodeTopology(oc *CLI) { e2eskipper.Skipf("This test does not apply to single-node topologies") } } + +// GetClusterVersion returns (short version like "4.15", full build version, error) +func GetClusterVersion(oc *CLI) (string, string, error) { + clusterBuild, err := oc.AsAdmin().WithoutNamespace().Run("get").Args( + "clusterversion", "cluster", + "-o", "jsonpath={.status.desired.version}", + ).Output() + if err != nil { + return "", "", err + } + clusterBuild = strings.TrimSpace(clusterBuild) + if clusterBuild == "" { + return "", "", fmt.Errorf("clusterversion cluster returned an empty desired.version") + } + if strings.ContainsAny(clusterBuild, " \t\n") { + return "", "", fmt.Errorf("unexpected multi-token clusterversion output: %q", clusterBuild) + } + splitValues := strings.Split(clusterBuild, ".") + if len(splitValues) < 2 { + return "", "", fmt.Errorf("clusterversion %q does not match expected X.Y[.Z] format", clusterBuild) + } + clusterVersion := splitValues[0] + "." + splitValues[1] + return clusterVersion, clusterBuild, nil +}