From 65e5b305233cd12f0c2651d38dbe6227e15e16f9 Mon Sep 17 00:00:00 2001 From: Kasia Koziol Date: Tue, 24 Feb 2026 15:47:40 +0100 Subject: [PATCH 1/2] CSPL-4354 logs for CM and IC --- pkg/splunk/client/enterprise.go | 21 +- pkg/splunk/client/enterprise_test.go | 12 +- pkg/splunk/enterprise/indexercluster.go | 284 ++++++++++++------- pkg/splunk/enterprise/indexercluster_test.go | 24 +- pkg/splunk/enterprise/ingestorcluster.go | 15 +- 5 files changed, 213 insertions(+), 143 deletions(-) diff --git a/pkg/splunk/client/enterprise.go b/pkg/splunk/client/enterprise.go index 9291484cb..526dd6a83 100644 --- a/pkg/splunk/client/enterprise.go +++ b/pkg/splunk/client/enterprise.go @@ -17,17 +17,18 @@ package client import ( "bytes" + "context" "crypto/tls" "encoding/json" "fmt" "io" + "log/slog" "net/http" "regexp" "strconv" "strings" "time" - "github.com/go-logr/logr" splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" ) @@ -1035,23 +1036,23 @@ func (c *SplunkClient) RestartSplunk() error { // Updates conf files and their properties // See https://help.splunk.com/en/splunk-enterprise/leverage-rest-apis/rest-api-reference/10.0/configuration-endpoints/configuration-endpoint-descriptions -func (c *SplunkClient) UpdateConfFile(scopedLog logr.Logger, fileName, property string, propertyKVList [][]string) error { +func (c *SplunkClient) UpdateConfFile(ctx context.Context, logger *slog.Logger, fileName, property string, propertyKVList [][]string) error { // Creates an object in a conf file if it doesn't exist endpoint := fmt.Sprintf("%s/servicesNS/nobody/system/configs/conf-%s", c.ManagementURI, fileName) body := fmt.Sprintf("name=%s", property) - scopedLog.Info("Creating conf file object if it does not exist", "fileName", fileName, "property", property) + logger.InfoContext(ctx, "Creating conf file object if it does not exist", "fileName", fileName, "property", property) request, err := http.NewRequest("POST", endpoint, strings.NewReader(body)) if err != nil { - scopedLog.Error(err, "Failed to create conf file object if it does not exist", "fileName", fileName, "property", property) + logger.ErrorContext(ctx, "Failed to create conf file object if it does not exist", "fileName", fileName, "property", property, "error", err.Error()) return err } - scopedLog.Info("Validating conf file object creation", "fileName", fileName, "property", property) + logger.InfoContext(ctx, "Validating conf file object creation", "fileName", fileName, "property", property) expectedStatus := []int{200, 201, 409} err = c.Do(request, expectedStatus, nil) if err != nil { - scopedLog.Error(err, fmt.Sprintf("Status not in %v for conf file object creation", expectedStatus), "fileName", fileName, "property", property) + logger.ErrorContext(ctx, fmt.Sprintf("Status not in %v for conf file object creation", expectedStatus), "fileName", fileName, "property", property, "error", err.Error()) return err } @@ -1065,18 +1066,18 @@ func (c *SplunkClient) UpdateConfFile(scopedLog logr.Logger, fileName, property body = body[:len(body)-1] } - scopedLog.Info("Updating conf file object", "fileName", fileName, "property", property, "body", body) + logger.InfoContext(ctx, "Updating conf file object", "fileName", fileName, "property", property, "body", body) request, err = http.NewRequest("POST", endpoint, strings.NewReader(body)) if err != nil { - scopedLog.Error(err, "Failed to update conf file object", "fileName", fileName, "property", property, "body", body) + logger.ErrorContext(ctx, "Failed to update conf file object", "fileName", fileName, "property", property, "error", err.Error()) return err } - scopedLog.Info("Validating conf file object update", "fileName", fileName, "property", property) + logger.InfoContext(ctx, "Validating conf file object update", "fileName", fileName, "property", property) expectedStatus = []int{200, 201} err = c.Do(request, expectedStatus, nil) if err != nil { - scopedLog.Error(err, fmt.Sprintf("Status not in %v for conf file object update", expectedStatus), "fileName", fileName, "property", property, "body", body) + logger.ErrorContext(ctx, fmt.Sprintf("Status not in %v for conf file object update", expectedStatus), "fileName", fileName, "property", property, "error", err.Error()) } return err } diff --git a/pkg/splunk/client/enterprise_test.go b/pkg/splunk/client/enterprise_test.go index 2c902d537..bfbfa180a 100644 --- a/pkg/splunk/client/enterprise_test.go +++ b/pkg/splunk/client/enterprise_test.go @@ -19,13 +19,13 @@ import ( "bytes" "context" "fmt" + "log/slog" "net/http" "net/url" "strings" "testing" splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" - "sigs.k8s.io/controller-runtime/pkg/log" spltest "github.com/splunk/splunk-operator/pkg/splunk/test" ) @@ -704,8 +704,8 @@ func TestUpdateConfFile(t *testing.T) { value := "myvalue" fileName := "outputs" - reqLogger := log.FromContext(context.TODO()) - scopedLog := reqLogger.WithName("TestUpdateConfFile") + ctx := context.Background() + logger := slog.With("func", "TestUpdateConfFile", "fileName", fileName, "property", property) // First request: create the property (object) if it doesn't exist createBody := strings.NewReader(fmt.Sprintf("name=%s", property)) @@ -722,7 +722,7 @@ func TestUpdateConfFile(t *testing.T) { c := NewSplunkClient("https://localhost:8089", "admin", "p@ssw0rd") c.Client = mockSplunkClient - err := c.UpdateConfFile(scopedLog, fileName, property, [][]string{{key, value}}) + err := c.UpdateConfFile(ctx, logger, fileName, property, [][]string{{key, value}}) if err != nil { t.Errorf("UpdateConfFile err = %v", err) } @@ -732,7 +732,7 @@ func TestUpdateConfFile(t *testing.T) { mockSplunkClient = &spltest.MockHTTPClient{} mockSplunkClient.AddHandler(wantCreateRequest, 500, "", nil) c.Client = mockSplunkClient - err = c.UpdateConfFile(scopedLog, fileName, property, [][]string{{key, value}}) + err = c.UpdateConfFile(ctx, logger, fileName, property, [][]string{{key, value}}) if err == nil { t.Errorf("UpdateConfFile expected error on create, got nil") } @@ -742,7 +742,7 @@ func TestUpdateConfFile(t *testing.T) { mockSplunkClient.AddHandler(wantCreateRequest, 201, "", nil) mockSplunkClient.AddHandler(wantUpdateRequest, 500, "", nil) c.Client = mockSplunkClient - err = c.UpdateConfFile(scopedLog, fileName, property, [][]string{{key, value}}) + err = c.UpdateConfFile(ctx, logger, fileName, property, [][]string{{key, value}}) if err == nil { t.Errorf("UpdateConfFile expected error on update, got nil") } diff --git a/pkg/splunk/enterprise/indexercluster.go b/pkg/splunk/enterprise/indexercluster.go index 6cb3aa332..a48559f2f 100644 --- a/pkg/splunk/enterprise/indexercluster.go +++ b/pkg/splunk/enterprise/indexercluster.go @@ -19,6 +19,7 @@ import ( "context" "errors" "fmt" + "log/slog" "regexp" "sort" "strconv" @@ -27,7 +28,6 @@ import ( enterpriseApi "github.com/splunk/splunk-operator/api/v4" - "github.com/go-logr/logr" enterpriseApiV3 "github.com/splunk/splunk-operator/api/v3" splclient "github.com/splunk/splunk-operator/pkg/splunk/client" splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" @@ -37,7 +37,6 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" rclient "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/reconcile" ) @@ -52,8 +51,7 @@ func ApplyIndexerClusterManager(ctx context.Context, client splcommon.Controller Requeue: true, RequeueAfter: time.Second * 5, } - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("ApplyIndexerClusterManager").WithValues("name", cr.GetName(), "namespace", cr.GetNamespace()) + logger := slog.With("func", "ApplyIndexerClusterManager", "name", cr.GetName(), "namespace", cr.GetNamespace()) eventPublisher := GetEventPublisher(ctx, cr) ctx = context.WithValue(ctx, splcommon.EventPublisherKey, eventPublisher) @@ -69,14 +67,15 @@ func ApplyIndexerClusterManager(ctx context.Context, client splcommon.Controller // validate and updates defaults for CR err = validateIndexerClusterSpec(ctx, client, cr) if err != nil { - eventPublisher.Warning(ctx, "validateIndexerClusterSpec", fmt.Sprintf("validate indexercluster spec failed %s", err.Error())) - scopedLog.Error(err, "Failed to validate indexercluster spec") + eventPublisher.Warning(ctx, "IndexerClusterSpecValidationFailure", fmt.Sprintf("validation of Indexer Cluster spec failed with error %s", err.Error())) + logger.ErrorContext(ctx, "Failed to validate Indexer Cluster spec", "error", err.Error()) return result, err } // updates status after function completes cr.Status.ClusterManagerPhase = enterpriseApi.PhaseError if cr.Status.Replicas < cr.Spec.Replicas { + logger.InfoContext(ctx, "Scaling up Indexer Cluster", "previousReplicas", cr.Status.Replicas, "newReplicas", cr.Spec.Replicas) cr.Status.CredentialSecretVersion = "0" cr.Status.ServiceAccount = "" } @@ -95,8 +94,8 @@ func ApplyIndexerClusterManager(ctx context.Context, client splcommon.Controller // create or update general config resources namespaceScopedSecret, err := ApplySplunkConfig(ctx, client, cr, cr.Spec.CommonSplunkSpec, SplunkIndexer) if err != nil { - scopedLog.Error(err, "create or update general config failed", "error", err.Error()) - eventPublisher.Warning(ctx, "ApplySplunkConfig", fmt.Sprintf("create or update general config failed with error %s", err.Error())) + logger.ErrorContext(ctx, "Create or update of general config failed", "error", err.Error()) + eventPublisher.Warning(ctx, "ApplySplunkConfigFailure", fmt.Sprintf("create or update of general config failed with error %s", err.Error())) return result, err } @@ -115,16 +114,17 @@ func ApplyIndexerClusterManager(ctx context.Context, client splcommon.Controller cr.Status.ClusterManagerPhase = managerIdxCluster.Status.Phase } } else { - scopedLog.Error(nil, "The configured clusterMasterRef doesn't exist", "clusterManagerRef", cr.Spec.ClusterManagerRef.Name) + logger.WarnContext(ctx, "The configured clusterMasterRef doesn't exist", "clusterManagerRef", cr.Spec.ClusterManagerRef.Name) cr.Status.ClusterManagerPhase = enterpriseApi.PhaseError } - mgr := newIndexerClusterPodManager(scopedLog, cr, namespaceScopedSecret, splclient.NewSplunkClient, client) + mgr := newIndexerClusterPodManager(logger, cr, namespaceScopedSecret, splclient.NewSplunkClient, client) // Check if we have configured enough number(<= RF) of replicas if mgr.cr.Status.ClusterManagerPhase == enterpriseApi.PhaseReady { err = VerifyRFPeers(ctx, mgr, client) if err != nil { - eventPublisher.Warning(ctx, "verifyRFPeers", fmt.Sprintf("verify RF peer failed %s", err.Error())) + eventPublisher.Warning(ctx, "VerifyRFPeersFailure", fmt.Sprintf("verification of RF peer failed with error %s", err.Error())) + logger.ErrorContext(ctx, "Verification of RF peer failed", "error", err.Error()) return result, err } } @@ -141,28 +141,32 @@ func ApplyIndexerClusterManager(ctx context.Context, client splcommon.Controller result.Requeue = false } if err != nil { - eventPublisher.Warning(ctx, "Delete", fmt.Sprintf("delete custom resource failed %s", err.Error())) + eventPublisher.Warning(ctx, "DeletionFailure", fmt.Sprintf("deletion of custom resource failed with error %s", err.Error())) + logger.ErrorContext(ctx, "Deletion of custom resource failed", "error", err.Error()) } return result, err } // create or update a headless service for indexer cluster err = splctrl.ApplyService(ctx, client, getSplunkService(ctx, cr, &cr.Spec.CommonSplunkSpec, SplunkIndexer, true)) if err != nil { - eventPublisher.Warning(ctx, "ApplyService", fmt.Sprintf("create/update headless service for indexer cluster failed %s", err.Error())) + eventPublisher.Warning(ctx, "ApplyServiceFailure", fmt.Sprintf("create/update of headless service for Indexer Cluster failed with error %s", err.Error())) + logger.ErrorContext(ctx, "Create/update of headless service for Indexer Cluster failed", "error", err.Error()) return result, err } // create or update a regular service for indexer cluster (ingestion) err = splctrl.ApplyService(ctx, client, getSplunkService(ctx, cr, &cr.Spec.CommonSplunkSpec, SplunkIndexer, false)) if err != nil { - eventPublisher.Warning(ctx, "ApplyService", fmt.Sprintf("create/update service for indexer cluster failed %s", err.Error())) + eventPublisher.Warning(ctx, "ApplyServiceFailure", fmt.Sprintf("create/update of service for Indexer Cluster failed with error %s", err.Error())) + logger.ErrorContext(ctx, "Create/update of service for Indexer Cluster failed", "error", err.Error()) return result, err } // create or update statefulset for the indexers statefulSet, err := getIndexerStatefulSet(ctx, client, cr) if err != nil { - eventPublisher.Warning(ctx, "getIndexerStatefulSet", fmt.Sprintf("get indexer stateful set failed %s", err.Error())) + eventPublisher.Warning(ctx, "GetIndexerStatefulSetFailure", fmt.Sprintf("get Indexer stateful set failed with error %s", err.Error())) + logger.ErrorContext(ctx, "Get Indexer stateful set failed", "error", err.Error()) return result, err } @@ -221,15 +225,16 @@ func ApplyIndexerClusterManager(ctx context.Context, client splcommon.Controller if !versionUpgrade { phase, err = mgr.Update(ctx, client, statefulSet, cr.Spec.Replicas) if err != nil { - eventPublisher.Warning(ctx, "UpdateManager", fmt.Sprintf("update statefulset failed %s", err.Error())) + eventPublisher.Warning(ctx, "UpdateManagerFailure", fmt.Sprintf("update of statefulset failed with error %s", err.Error())) + logger.ErrorContext(ctx, "Update of statefulset failed", "error", err.Error()) return result, err } } else { // Delete the statefulset and recreate new one err = client.Delete(ctx, statefulSet) if err != nil { - eventPublisher.Warning(ctx, "UpdateManager", fmt.Sprintf("version mismatch for indexer cluster and indexer container, delete statefulset failed. Error=%s", err.Error())) - eventPublisher.Warning(ctx, "UpdateManager", fmt.Sprintf("%s-%s, %s-%s", "indexer-image", cr.Spec.Image, "container-image", statefulSet.Spec.Template.Spec.Containers[0].Image)) + eventPublisher.Warning(ctx, "DeleteManagerFailure", fmt.Sprintf(fmt.Sprintf("version mismatch for Indexer Cluster %s and Indexer container %s - delete of stateful set failed with error=%s", cr.Spec.Image, statefulSet.Spec.Template.Spec.Containers[0].Image, err.Error()))) + logger.ErrorContext(ctx, "Delete of stateful set failed", "error", err.Error()) return result, err } time.Sleep(1 * time.Second) @@ -237,7 +242,8 @@ func ApplyIndexerClusterManager(ctx context.Context, client splcommon.Controller statefulSet.ResourceVersion = "" phase, err = mgr.Update(ctx, client, statefulSet, cr.Spec.Replicas) if err != nil { - eventPublisher.Warning(ctx, "UpdateManager", fmt.Sprintf("update statefulset failed %s", err.Error())) + eventPublisher.Warning(ctx, "UpdateManagerFailure", fmt.Sprintf("update statefulset failed %s", err.Error())) + logger.ErrorContext(ctx, "Update of statefulset failed", "error", err.Error()) return result, err } } @@ -247,35 +253,47 @@ func ApplyIndexerClusterManager(ctx context.Context, client splcommon.Controller if cr.Status.Phase == enterpriseApi.PhaseReady { qosCfg, err := ResolveQueueAndObjectStorage(ctx, client, cr, cr.Spec.QueueRef, cr.Spec.ObjectStorageRef, cr.Spec.ServiceAccount) if err != nil { - scopedLog.Error(err, "Failed to resolve Queue/ObjectStorage config") + logger.ErrorContext(ctx, "Failed to resolve Queue/ObjectStorage config", "error", err.Error()) return result, err } + logger.DebugContext(ctx, "Resolved Queue/ObjectStorage config", "queue", qosCfg.Queue, "objectStorage", qosCfg.OS, "version", qosCfg.Version, "serviceAccount", cr.Spec.ServiceAccount) secretChanged := cr.Status.CredentialSecretVersion != qosCfg.Version serviceAccountChanged := cr.Status.ServiceAccount != cr.Spec.ServiceAccount + logger.DebugContext(ctx, "Checking for changes", "previousCredentialSecretVersion", cr.Status.CredentialSecretVersion, "previousServiceAccount", cr.Status.ServiceAccount, "secretChanged", secretChanged, "serviceAccountChanged", serviceAccountChanged) + // If queue is updated if cr.Spec.QueueRef.Name != "" { if secretChanged || serviceAccountChanged { - mgr := newIndexerClusterPodManager(scopedLog, cr, namespaceScopedSecret, splclient.NewSplunkClient, client) + mgr := newIndexerClusterPodManager(logger, cr, namespaceScopedSecret, splclient.NewSplunkClient, client) err = mgr.updateIndexerConfFiles(ctx, cr, &qosCfg.Queue, &qosCfg.OS, qosCfg.AccessKey, qosCfg.SecretKey, client) if err != nil { - eventPublisher.Warning(ctx, "ApplyIndexerClusterManager", fmt.Sprintf("Failed to update conf file for Queue/Pipeline config change after pod creation: %s", err.Error())) - scopedLog.Error(err, "Failed to update conf file for Queue/Pipeline config change after pod creation") + eventPublisher.Warning(ctx, "UpdateConfFilesFailure", fmt.Sprintf("failed to update conf file for Queue/Pipeline config due to %s", err.Error())) + logger.ErrorContext(ctx, "Failed to update conf file for Queue/Pipeline config", "error", err.Error()) return result, err } + eventPublisher.Normal(ctx, "QueueConfigUpdated", + fmt.Sprintf("Queue/Pipeline configuration updated for %d indexers", cr.Spec.Replicas)) + logger.InfoContext(ctx, "Queue/Pipeline configuration updated", "readyReplicas", cr.Status.ReadyReplicas) + for i := int32(0); i < cr.Spec.Replicas; i++ { idxcClient := mgr.getClient(ctx, i) err = idxcClient.RestartSplunk() if err != nil { return result, err } - scopedLog.Info("Restarted splunk", "indexer", i) + logger.InfoContext(ctx, "Restarted splunk", "indexer", i) } + eventPublisher.Normal(ctx, "IndexersRestarted", + fmt.Sprintf("Restarted Splunk on %d indexer pods", cr.Spec.Replicas)) + cr.Status.CredentialSecretVersion = qosCfg.Version cr.Status.ServiceAccount = cr.Spec.ServiceAccount + + logger.InfoContext(ctx, "Updated status", "credentialSecretVersion", cr.Status.CredentialSecretVersion, "serviceAccount", cr.Status.ServiceAccount) } } @@ -283,7 +301,8 @@ func ApplyIndexerClusterManager(ctx context.Context, client splcommon.Controller //Retrieve monitoring console ref from CM Spec cmMonitoringConsoleConfigRef, err := RetrieveCMSpec(ctx, client, cr) if err != nil { - eventPublisher.Warning(ctx, "RetrieveCMSpec", fmt.Sprintf("retrieve cluster manager spec failed %s", err.Error())) + eventPublisher.Warning(ctx, "RetrieveCMSpecFailure", fmt.Sprintf("retrieval of Cluster Manager spec failed with %s", err.Error())) + logger.ErrorContext(ctx, "Retrieval of Cluster Manager spec failed", "error", err.Error()) return result, err } if cmMonitoringConsoleConfigRef != "" { @@ -294,12 +313,13 @@ func ApplyIndexerClusterManager(ctx context.Context, client splcommon.Controller c := mgr.getMonitoringConsoleClient(cr, cmMonitoringConsoleConfigRef) err := c.AutomateMCApplyChanges() if err != nil { - eventPublisher.Warning(ctx, "AutomateMCApplyChanges", fmt.Sprintf("get monitoring console client failed %s", err.Error())) + eventPublisher.Warning(ctx, "AutomateMCApplyChangesFailure", fmt.Sprintf("get Monitoring Console client failed with %s", err.Error())) + logger.ErrorContext(ctx, "get Monitoring Console client failed", "error", err.Error()) return result, err } } if len(cr.Spec.MonitoringConsoleRef.Name) > 0 && (cr.Spec.MonitoringConsoleRef.Name != cmMonitoringConsoleConfigRef) { - scopedLog.Info("Indexer Cluster CR should not specify monitoringConsoleRef and if specified, should be similar to cluster manager spec") + logger.WarnContext(ctx, "Indexer Cluster CR should not specify monitoringConsoleRef and if specified, should be similar to Cluster Manager spec") } } if len(cr.Status.IndexerSecretChanged) > 0 { @@ -314,7 +334,8 @@ func ApplyIndexerClusterManager(ctx context.Context, client splcommon.Controller // Disable maintenance mode err = SetClusterMaintenanceMode(ctx, client, cr, false, cmPodName, podExecClient) if err != nil { - eventPublisher.Warning(ctx, "SetClusterMaintenanceMode", fmt.Sprintf("set cluster maintenance mode failed %s", err.Error())) + eventPublisher.Warning(ctx, "ClusterMaintenanceModeFailure", fmt.Sprintf("set Cluster maintenance mode failed with %s", err.Error())) + logger.ErrorContext(ctx, "set Cluster maintenance mode failed", "error", err.Error()) return result, err } } @@ -326,13 +347,14 @@ func ApplyIndexerClusterManager(ctx context.Context, client splcommon.Controller result.Requeue = false // Set indexer cluster CR as owner reference for clustermanager - scopedLog.Info("Setting indexer cluster as owner for cluster manager") + logger.DebugContext(ctx, "Setting Indexer Cluster as owner for Cluster Manager") if len(cr.Spec.ClusterManagerRef.Name) > 0 { namespacedName = types.NamespacedName{Namespace: cr.GetNamespace(), Name: GetSplunkStatefulsetName(SplunkClusterManager, cr.Spec.ClusterManagerRef.Name)} } err = splctrl.SetStatefulSetOwnerRef(ctx, client, cr, namespacedName) if err != nil { - eventPublisher.Warning(ctx, "SetStatefulSetOwnerRef", fmt.Sprintf("set stateful set owner reference failed %s", err.Error())) + eventPublisher.Warning(ctx, "SetStatefulSetOwnerRefFailure", fmt.Sprintf("set stateful set owner reference failed with %s", err.Error())) + logger.ErrorContext(ctx, "Set stateful set owner reference failed", "error", err.Error()) result.Requeue = true return result, err } @@ -353,8 +375,7 @@ func ApplyIndexerCluster(ctx context.Context, client splcommon.ControllerClient, Requeue: true, RequeueAfter: time.Second * 5, } - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("ApplyIndexerCluster") + logger := slog.With("func", "ApplyIndexerCluster", "name", cr.GetName(), "namespace", cr.GetNamespace()) eventPublisher := GetEventPublisher(ctx, cr) cr.Kind = "IndexerCluster" @@ -362,6 +383,8 @@ func ApplyIndexerCluster(ctx context.Context, client splcommon.ControllerClient, // validate and updates defaults for CR err := validateIndexerClusterSpec(ctx, client, cr) if err != nil { + eventPublisher.Warning(ctx, "ValidateIndexerClusterSpecFailure", fmt.Sprintf("validate Indexer Cluster spec failed with %s", err.Error())) + logger.ErrorContext(ctx, "Failed to validate Indexer Cluster spec", "error", err.Error()) return result, err } @@ -369,6 +392,7 @@ func ApplyIndexerCluster(ctx context.Context, client splcommon.ControllerClient, cr.Status.Phase = enterpriseApi.PhaseError cr.Status.ClusterMasterPhase = enterpriseApi.PhaseError if cr.Status.Replicas < cr.Spec.Replicas { + logger.InfoContext(ctx, "Scaling up Indexer Cluster", "previousReplicas", cr.Status.Replicas, "newReplicas", cr.Spec.Replicas) cr.Status.CredentialSecretVersion = "0" cr.Status.ServiceAccount = "" } @@ -390,8 +414,8 @@ func ApplyIndexerCluster(ctx context.Context, client splcommon.ControllerClient, // create or update general config resources namespaceScopedSecret, err := ApplySplunkConfig(ctx, client, cr, cr.Spec.CommonSplunkSpec, SplunkIndexer) if err != nil { - scopedLog.Error(err, "create or update general config failed", "error", err.Error()) - eventPublisher.Warning(ctx, "ApplySplunkConfig", fmt.Sprintf("create or update general config failed with error %s", err.Error())) + logger.ErrorContext(ctx, "Create or update general config failed", "error", err.Error()) + eventPublisher.Warning(ctx, "ApplySplunkConfigFailure", fmt.Sprintf("create or update general config failed with %s", err.Error())) return result, err } @@ -413,12 +437,13 @@ func ApplyIndexerCluster(ctx context.Context, client splcommon.ControllerClient, cr.Status.ClusterMasterPhase = enterpriseApi.PhaseError } - mgr := newIndexerClusterPodManager(scopedLog, cr, namespaceScopedSecret, splclient.NewSplunkClient, client) + mgr := newIndexerClusterPodManager(logger, cr, namespaceScopedSecret, splclient.NewSplunkClient, client) // Check if we have configured enough number(<= RF) of replicas if mgr.cr.Status.ClusterMasterPhase == enterpriseApi.PhaseReady { err = VerifyRFPeers(ctx, mgr, client) if err != nil { - eventPublisher.Warning(ctx, "verifyRFPeers", fmt.Sprintf("verify RF peer failed %s", err.Error())) + eventPublisher.Warning(ctx, "VerifyRFPeersFailure", fmt.Sprintf("verify RF peer failed with %s", err.Error())) + logger.ErrorContext(ctx, "Verify RF peer failed", "error", err.Error()) return result, err } } @@ -435,7 +460,8 @@ func ApplyIndexerCluster(ctx context.Context, client splcommon.ControllerClient, result.Requeue = false } if err != nil { - eventPublisher.Warning(ctx, "Delete", fmt.Sprintf("delete custom resource failed %s", err.Error())) + eventPublisher.Warning(ctx, "DeleteFailure", fmt.Sprintf("delete custom resource failed %s", err.Error())) + logger.ErrorContext(ctx, "Delete custom resource failed", "error", err.Error()) } return result, err } @@ -443,21 +469,24 @@ func ApplyIndexerCluster(ctx context.Context, client splcommon.ControllerClient, // create or update a headless service for indexer cluster err = splctrl.ApplyService(ctx, client, getSplunkService(ctx, cr, &cr.Spec.CommonSplunkSpec, SplunkIndexer, true)) if err != nil { - eventPublisher.Warning(ctx, "ApplyService", fmt.Sprintf("create/update headless service for indexer cluster failed %s", err.Error())) + eventPublisher.Warning(ctx, "ApplyServiceFailure", fmt.Sprintf("create/update of headless service for Indexer Cluster failed with %s", err.Error())) + logger.ErrorContext(ctx, "Apply headless service failed", "error", err.Error()) return result, err } // create or update a regular service for indexer cluster (ingestion) err = splctrl.ApplyService(ctx, client, getSplunkService(ctx, cr, &cr.Spec.CommonSplunkSpec, SplunkIndexer, false)) if err != nil { - eventPublisher.Warning(ctx, "ApplyService", fmt.Sprintf("create/update service for indexer cluster failed %s", err.Error())) + eventPublisher.Warning(ctx, "ApplyServiceFailure", fmt.Sprintf("create/update of service for Indexer Cluster failed with %s", err.Error())) + logger.ErrorContext(ctx, "Apply service failed", "error", err.Error()) return result, err } // create or update statefulset for the indexers statefulSet, err := getIndexerStatefulSet(ctx, client, cr) if err != nil { - eventPublisher.Warning(ctx, "getIndexerStatefulSet", fmt.Sprintf("get indexer stateful set failed %s", err.Error())) + eventPublisher.Warning(ctx, "GetIndexerStatefulSetFailure", fmt.Sprintf("get Indexer stateful set failed with %s", err.Error())) + logger.ErrorContext(ctx, "Get Indexer stateful set failed", "error", err.Error()) return result, err } @@ -516,15 +545,16 @@ func ApplyIndexerCluster(ctx context.Context, client splcommon.ControllerClient, if !versionUpgrade { phase, err = mgr.Update(ctx, client, statefulSet, cr.Spec.Replicas) if err != nil { - eventPublisher.Warning(ctx, "UpdateManager", fmt.Sprintf("update statefulset failed %s", err.Error())) + eventPublisher.Warning(ctx, "UpdateManager", fmt.Sprintf("update of stateful set failed with %s", err.Error())) + logger.ErrorContext(ctx, "Update of stateful set failed", "error", err.Error()) return result, err } } else { // Delete the statefulset and recreate new one err = client.Delete(ctx, statefulSet) if err != nil { - eventPublisher.Warning(ctx, "UpdateManager", fmt.Sprintf("version mitmatch for indexer clustre and indexer container, delete statefulset failed %s", err.Error())) - eventPublisher.Warning(ctx, "UpdateManager", fmt.Sprintf("%s-%s, %s-%s", "indexer-image", cr.Spec.Image, "container-image", statefulSet.Spec.Template.Spec.Containers[0].Image)) + eventPublisher.Warning(ctx, "DeleteFailure", fmt.Sprintf(fmt.Sprintf("version mismatch for Indexer Cluster %s and Indexer container %s - delete of stateful set failed with error=%s", cr.Spec.Image, statefulSet.Spec.Template.Spec.Containers[0].Image, err.Error()))) + logger.ErrorContext(ctx, "Delete of stateful set failed", "error", err.Error()) return result, err } time.Sleep(1 * time.Second) @@ -532,7 +562,8 @@ func ApplyIndexerCluster(ctx context.Context, client splcommon.ControllerClient, statefulSet.ResourceVersion = "" phase, err = mgr.Update(ctx, client, statefulSet, cr.Spec.Replicas) if err != nil { - eventPublisher.Warning(ctx, "UpdateManager", fmt.Sprintf("update statefulset failed %s", err.Error())) + eventPublisher.Warning(ctx, "UpdateFailure", fmt.Sprintf("update of stateful set failed %s", err.Error())) + logger.ErrorContext(ctx, "Update of stateful set failed", "error", err.Error()) return result, err } } @@ -542,34 +573,46 @@ func ApplyIndexerCluster(ctx context.Context, client splcommon.ControllerClient, if cr.Status.Phase == enterpriseApi.PhaseReady { qosCfg, err := ResolveQueueAndObjectStorage(ctx, client, cr, cr.Spec.QueueRef, cr.Spec.ObjectStorageRef, cr.Spec.ServiceAccount) if err != nil { - scopedLog.Error(err, "Failed to resolve Queue/ObjectStorage config") + logger.ErrorContext(ctx, "Failed to resolve Queue/ObjectStorage config", "error", err.Error()) return result, err } + logger.DebugContext(ctx, "Resolved Queue/ObjectStorage config", "queue", qosCfg.Queue, "objectStorage", qosCfg.OS, "version", qosCfg.Version, "serviceAccount", cr.Spec.ServiceAccount) secretChanged := cr.Status.CredentialSecretVersion != qosCfg.Version serviceAccountChanged := cr.Status.ServiceAccount != cr.Spec.ServiceAccount + logger.DebugContext(ctx, "Checking for changes", "previousCredentialSecretVersion", cr.Status.CredentialSecretVersion, "previousServiceAccount", cr.Status.ServiceAccount, "secretChanged", secretChanged, "serviceAccountChanged", serviceAccountChanged) + if cr.Spec.QueueRef.Name != "" { if secretChanged || serviceAccountChanged { - mgr := newIndexerClusterPodManager(scopedLog, cr, namespaceScopedSecret, splclient.NewSplunkClient, client) + mgr := newIndexerClusterPodManager(logger, cr, namespaceScopedSecret, splclient.NewSplunkClient, client) err = mgr.updateIndexerConfFiles(ctx, cr, &qosCfg.Queue, &qosCfg.OS, qosCfg.AccessKey, qosCfg.SecretKey, client) if err != nil { - eventPublisher.Warning(ctx, "ApplyIndexerClusterManager", fmt.Sprintf("Failed to update conf file for Queue/Pipeline config change after pod creation: %s", err.Error())) - scopedLog.Error(err, "Failed to update conf file for Queue/Pipeline config change after pod creation") + eventPublisher.Warning(ctx, "UpdateIndexerConfFilesFailure", fmt.Sprintf("Failed to update conf file for Queue/Pipeline config change after pod creation with %s", err.Error())) + logger.ErrorContext(ctx, "Failed to update conf file for Queue/Pipeline config change after pod creation", "error", err.Error()) return result, err } + eventPublisher.Normal(ctx, "QueueConfigUpdated", + fmt.Sprintf("Queue/Pipeline configuration updated for %d indexers", cr.Spec.Replicas)) + logger.InfoContext(ctx, "Queue/Pipeline configuration updated", "readyReplicas", cr.Status.ReadyReplicas) + for i := int32(0); i < cr.Spec.Replicas; i++ { idxcClient := mgr.getClient(ctx, i) err = idxcClient.RestartSplunk() if err != nil { return result, err } - scopedLog.Info("Restarted splunk", "indexer", i) + logger.InfoContext(ctx, "Restarted splunk", "indexer", i) } + eventPublisher.Normal(ctx, "IndexersRestarted", + fmt.Sprintf("Restarted Splunk on %d indexer pods", cr.Spec.Replicas)) + cr.Status.CredentialSecretVersion = qosCfg.Version cr.Status.ServiceAccount = cr.Spec.ServiceAccount + + logger.InfoContext(ctx, "Updated status", "credentialSecretVersion", cr.Status.CredentialSecretVersion, "serviceAccount", cr.Status.ServiceAccount) } } @@ -577,7 +620,8 @@ func ApplyIndexerCluster(ctx context.Context, client splcommon.ControllerClient, //Retrieve monitoring console ref from CM Spec cmMonitoringConsoleConfigRef, err := RetrieveCMSpec(ctx, client, cr) if err != nil { - eventPublisher.Warning(ctx, "RetrieveCMSpec", fmt.Sprintf("retrieve cluster master spec failed %s", err.Error())) + eventPublisher.Warning(ctx, "RetrieveCMSpecFailure", fmt.Sprintf("retrieve Cluster Master spec failed with %s", err.Error())) + logger.ErrorContext(ctx, "Retrieve Cluster Master spec failed", "error", err.Error()) return result, err } if cmMonitoringConsoleConfigRef != "" { @@ -588,12 +632,13 @@ func ApplyIndexerCluster(ctx context.Context, client splcommon.ControllerClient, c := mgr.getMonitoringConsoleClient(cr, cmMonitoringConsoleConfigRef) err := c.AutomateMCApplyChanges() if err != nil { - eventPublisher.Warning(ctx, "AutomateMCApplyChanges", fmt.Sprintf("get monitoring console client failed %s", err.Error())) + eventPublisher.Warning(ctx, "AutomateMCApplyChangesFailure", fmt.Sprintf("get Monitoring Console client failed with %s", err.Error())) + logger.ErrorContext(ctx, "Get Monitoring Console client failed", "error", err.Error()) return result, err } } if len(cr.Spec.MonitoringConsoleRef.Name) > 0 && (cr.Spec.MonitoringConsoleRef.Name != cmMonitoringConsoleConfigRef) { - scopedLog.Info("Indexer Cluster CR should not specify monitoringConsoleRef and if specified, should be similar to cluster master spec") + logger.WarnContext(ctx, "Indexer Cluster CR should not specify monitoringConsoleRef and if specified, should be similar to Cluster Master spec") } } if len(cr.Status.IndexerSecretChanged) > 0 { @@ -601,14 +646,15 @@ func ApplyIndexerCluster(ctx context.Context, client splcommon.ControllerClient, if len(cr.Spec.ClusterMasterRef.Name) > 0 { managerIdxcName = cr.Spec.ClusterMasterRef.Name } else { - return result, errors.New("empty cluster master reference") + return result, errors.New("empty Cluster Master reference") } cmPodName := fmt.Sprintf("splunk-%s-cluster-master-%s", managerIdxcName, "0") podExecClient := splutil.GetPodExecClient(client, cr, cmPodName) // Disable maintenance mode err = SetClusterMaintenanceMode(ctx, client, cr, false, cmPodName, podExecClient) if err != nil { - eventPublisher.Warning(ctx, "SetClusterMaintenanceMode", fmt.Sprintf("set cluster maintenance mode failed %s", err.Error())) + eventPublisher.Warning(ctx, "SetClusterMaintenanceModeFailure", fmt.Sprintf("set Cluster Master maintenance mode failed with %s", err.Error())) + logger.ErrorContext(ctx, "Set Cluster Master maintenance mode failed", "error", err.Error()) return result, err } } @@ -620,11 +666,12 @@ func ApplyIndexerCluster(ctx context.Context, client splcommon.ControllerClient, result.Requeue = false // Set indexer cluster CR as owner reference for clustermaster - scopedLog.Info("Setting indexer cluster as owner for cluster master") + logger.DebugContext(ctx, "Setting Indexer Cluster as owner for Cluster Master") namespacedName = types.NamespacedName{Namespace: cr.GetNamespace(), Name: GetSplunkStatefulsetName(SplunkClusterMaster, cr.Spec.ClusterMasterRef.Name)} err = splctrl.SetStatefulSetOwnerRef(ctx, client, cr, namespacedName) if err != nil { - eventPublisher.Warning(ctx, "SetStatefulSetOwnerRef", fmt.Sprintf("set stateful set owner reference failed %s", err.Error())) + eventPublisher.Warning(ctx, "SetStatefulSetOwnerRefFailure", fmt.Sprintf("set stateful set owner reference failed %s", err.Error())) + logger.ErrorContext(ctx, "Set stateful set owner reference failed", "error", err.Error()) result.Requeue = true return result, err } @@ -645,14 +692,14 @@ var VerifyRFPeers = func(ctx context.Context, mgr indexerClusterPodManager, clie // indexerClusterPodManager is used to manage the pods within an indexer cluster type indexerClusterPodManager struct { c splcommon.ControllerClient - log logr.Logger + log *slog.Logger cr *enterpriseApi.IndexerCluster secrets *corev1.Secret newSplunkClient func(managementURI, username, password string) *splclient.SplunkClient } // newIndexerClusterPodManager function to create pod manager this is added to write unit test case -var newIndexerClusterPodManager = func(log logr.Logger, cr *enterpriseApi.IndexerCluster, secret *corev1.Secret, newSplunkClient NewSplunkClientFunc, c splcommon.ControllerClient) indexerClusterPodManager { +var newIndexerClusterPodManager = func(log *slog.Logger, cr *enterpriseApi.IndexerCluster, secret *corev1.Secret, newSplunkClient NewSplunkClientFunc, c splcommon.ControllerClient) indexerClusterPodManager { return indexerClusterPodManager{ log: log, cr: cr, @@ -712,25 +759,29 @@ func ApplyIdxcSecret(ctx context.Context, mgr *indexerClusterPodManager, replica return err } - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("ApplyIdxcSecret").WithValues("Desired replicas", replicas, "IdxcSecretChanged", mgr.cr.Status.IndexerSecretChanged, "CrStatusNamespaceSecretResourceVersion", mgr.cr.Status.NamespaceSecretResourceVersion, "NamespaceSecretResourceVersion", namespaceSecret.GetObjectMeta().GetResourceVersion()) + logger := slog.With("func", "ApplyIdxcSecret", "name", mgr.cr.GetName(), "namespace", mgr.cr.GetNamespace()) + logger.InfoContext(ctx, "Applying idxc secret to indexers", "desiredReplicas", replicas, "idxcSecretChanged", mgr.cr.Status.IndexerSecretChanged, "crStatusNamespaceSecretResourceVersion", mgr.cr.Status.NamespaceSecretResourceVersion, "namespaceSecretResourceVersion", namespaceSecret.GetObjectMeta().GetResourceVersion()) // If namespace scoped secret revision is the same ignore if len(mgr.cr.Status.NamespaceSecretResourceVersion) == 0 { // First time, set resource version in CR mgr.cr.Status.NamespaceSecretResourceVersion = namespaceSecret.ObjectMeta.ResourceVersion - scopedLog.Info("Setting CrStatusNamespaceSecretResourceVersion for the first time") + logger.DebugContext(ctx, "Setting CrStatusNamespaceSecretResourceVersion for the first time") return nil } else if mgr.cr.Status.NamespaceSecretResourceVersion == namespaceSecret.ObjectMeta.ResourceVersion { // If resource version hasn't changed don't return return nil } - scopedLog.Info("Namespaced scoped secret revision has changed") + logger.InfoContext(ctx, "Namespaced scoped secret revision has changed") // Retrieve idxc_secret password from secret data nsIdxcSecret := string(namespaceSecret.Data[splcommon.IdxcSecret]) + // Log configuration push start + pushStartTime := time.Now() + logger.InfoContext(ctx, "Starting configuration push to peers", "peerCount", replicas, "configVersion", namespaceSecret.ObjectMeta.ResourceVersion) + // Loop over all indexer pods and get individual pod's idxc password howManyPodsHaveSecretChanged := 0 for i := int32(0); i <= replicas-1; i++ { @@ -740,10 +791,10 @@ func ApplyIdxcSecret(ctx context.Context, mgr *indexerClusterPodManager, replica // Check if pod exists before updating secrets pod := &corev1.Pod{} namespacedName := types.NamespacedName{Namespace: mgr.cr.GetNamespace(), Name: indexerPodName} - scopedLog.Info("Check if pod is created before updating its secrets") + logger.DebugContext(ctx, "Check if pod is created before updating its secrets") err := mgr.c.Get(ctx, namespacedName, pod) if err != nil { - mgr.log.Info("Peer doesn't exists", "peerName", indexerPodName) + logger.WarnContext(ctx, "Peer doesn't exists", "peerName", indexerPodName) continue } @@ -762,7 +813,7 @@ func ApplyIdxcSecret(ctx context.Context, mgr *indexerClusterPodManager, replica // If idxc secret is different from namespace scoped secret change it if indIdxcSecret != nsIdxcSecret { - scopedLog.Info("idxc Secret different from namespace scoped secret") + logger.InfoContext(ctx, "IDXC Secret is different from namespace scoped secret") // Enable maintenance mode if len(mgr.cr.Status.IndexerSecretChanged) == 0 && !mgr.cr.Status.MaintenanceMode { @@ -782,7 +833,7 @@ func ApplyIdxcSecret(ctx context.Context, mgr *indexerClusterPodManager, replica if err != nil { return err } - scopedLog.Info("Set CM in maintenance mode") + logger.InfoContext(ctx, "Set CM in maintenance mode") } // If idxc secret already changed, ignore @@ -803,9 +854,10 @@ func ApplyIdxcSecret(ctx context.Context, mgr *indexerClusterPodManager, replica eventPublisher.Warning(ctx, "PasswordSyncFailed", fmt.Sprintf("Password sync failed for pod '%s': %s. Check pod logs and secret format.", indexerPodName, err.Error())) } + mgr.log.ErrorContext(ctx, "Configuration push failed", "failedPeer", indexerPodName, "error", err.Error()) return err } - scopedLog.Info("Changed idxc secret") + logger.InfoContext(ctx, "Changed idxc secret") howManyPodsHaveSecretChanged += 1 @@ -817,9 +869,10 @@ func ApplyIdxcSecret(ctx context.Context, mgr *indexerClusterPodManager, replica eventPublisher.Warning(ctx, "PasswordSyncFailed", fmt.Sprintf("Password sync failed for pod '%s': %s. Check pod logs and secret format.", indexerPodName, err.Error())) } + logger.ErrorContext(ctx, "Configuration push failed during restart", "failedPeer", indexerPodName, "error", err.Error()) return err } - scopedLog.Info("Restarted splunk") + logger.InfoContext(ctx, "Restarted splunk") // Keep a track of all the secrets on pods to change their idxc secret below mgr.cr.Status.IdxcPasswordChangedSecrets[podSecret.GetName()] = true @@ -862,7 +915,7 @@ func ApplyIdxcSecret(ctx context.Context, mgr *indexerClusterPodManager, replica if err != nil { return err } - scopedLog.Info("idxc password changed on the secret mounted on pod", "Secret on Pod:", podSecretName) + logger.InfoContext(ctx, "IDXC password changed on the secret mounted on pod", "podSecretName", podSecretName) // Set to false marking the idxc password change in the secret mgr.cr.Status.IdxcPasswordChangedSecrets[podSecretName] = false @@ -876,6 +929,9 @@ func ApplyIdxcSecret(ctx context.Context, mgr *indexerClusterPodManager, replica fmt.Sprintf("Password synchronized for %d pods", howManyPodsHaveSecretChanged)) } + // Log configuration push completion + logger.InfoContext(ctx, "Configuration push completed", "successCount", howManyPodsHaveSecretChanged, "duration", time.Since(pushStartTime)) + return nil } @@ -887,8 +943,8 @@ func (mgr *indexerClusterPodManager) Update(ctx context.Context, c splcommon.Con // Get event publisher from context eventPublisher := GetEventPublisher(ctx, mgr.cr) - // Track last successful replica count to emit scale events after completion - previousReplicas := mgr.cr.Status.Replicas + // Track previous ready replicas for scaling events + previousReadyReplicas := mgr.cr.Status.ReadyReplicas // Assign client if mgr.c == nil { @@ -901,7 +957,7 @@ func (mgr *indexerClusterPodManager) Update(ctx context.Context, c splcommon.Con return enterpriseApi.PhaseError, err } } else { - mgr.log.Info("Cluster Manager is not ready yet", "reason ", err) + mgr.log.InfoContext(ctx, "Cluster Manager is not ready yet", "error", err) return enterpriseApi.PhaseError, err } @@ -917,7 +973,7 @@ func (mgr *indexerClusterPodManager) Update(ctx context.Context, c splcommon.Con // update CR status with IDXC information err = mgr.updateStatus(ctx, statefulSet) if err != nil || mgr.cr.Status.ReadyReplicas == 0 || !mgr.cr.Status.Initialized || !mgr.cr.Status.IndexingReady || !mgr.cr.Status.ServiceReady { - mgr.log.Info("Indexer cluster is not ready", "reason ", err) + mgr.log.InfoContext(ctx, "Indexer Cluster is not ready", "error ", err) return enterpriseApi.PhasePending, nil } @@ -927,17 +983,19 @@ func (mgr *indexerClusterPodManager) Update(ctx context.Context, c splcommon.Con return phase, err } - // Emit ScaledUp event only after a successful scale-up has completed + // Emit scale events when phase is ready and ready replicas changed to match desired if phase == enterpriseApi.PhaseReady { - if desiredReplicas > previousReplicas && mgr.cr.Status.Replicas == desiredReplicas { - if eventPublisher != nil { - eventPublisher.Normal(ctx, "ScaledUp", - fmt.Sprintf("Successfully scaled %s up from %d to %d replicas", mgr.cr.GetName(), previousReplicas, desiredReplicas)) - } - } else if desiredReplicas < previousReplicas && mgr.cr.Status.Replicas == desiredReplicas { - if eventPublisher != nil { - eventPublisher.Normal(ctx, "ScaledDown", - fmt.Sprintf("Successfully scaled %s down from %d to %d replicas", mgr.cr.GetName(), previousReplicas, desiredReplicas)) + if mgr.cr.Status.ReadyReplicas == desiredReplicas && previousReadyReplicas != desiredReplicas { + if desiredReplicas > previousReadyReplicas { + if eventPublisher != nil { + eventPublisher.Normal(ctx, "ScaledUp", + fmt.Sprintf("Successfully scaled %s up from %d to %d replicas", mgr.cr.GetName(), previousReadyReplicas, desiredReplicas)) + } + } else if desiredReplicas < previousReadyReplicas { + if eventPublisher != nil { + eventPublisher.Normal(ctx, "ScaledDown", + fmt.Sprintf("Successfully scaled %s down from %d to %d replicas", mgr.cr.GetName(), previousReadyReplicas, desiredReplicas)) + } } } } @@ -958,6 +1016,9 @@ func (mgr *indexerClusterPodManager) PrepareScaleDown(ctx context.Context, n int // next, remove the peer c := mgr.getClusterManagerClient(ctx) + peerName := GetSplunkStatefulsetPodName(SplunkIndexer, mgr.cr.GetName(), n) + remainingPeers := int32(len(mgr.cr.Status.Peers)) - 1 + mgr.log.InfoContext(ctx, "Deregistering peer from cluster manager", "peerName", peerName, "remainingPeers", remainingPeers) return true, c.RemoveIndexerClusterPeer(mgr.cr.Status.Peers[n].ID) } @@ -990,31 +1051,31 @@ func (mgr *indexerClusterPodManager) decommission(ctx context.Context, n int32, // Don't return error here. We may be reconciling several times, and the actual Pod status is down, but // not yet reflecting on the Cluster Master, in which case, the podExec fails, though the decommission is // going fine. - mgr.log.Info("Unable to lower the liveness probe level", "peerName", peerName, "enforceCounts", enforceCounts) + mgr.log.WarnContext(ctx, "Unable to lower the liveness probe level", "peerName", peerName, "enforceCounts", enforceCounts) } - mgr.log.Info("Decommissioning indexer cluster peer", "peerName", peerName, "enforceCounts", enforceCounts) + mgr.log.InfoContext(ctx, "Decommissioning indexer cluster peer", "peerName", peerName, "enforceCounts", enforceCounts) c := mgr.getClient(ctx, n) return false, c.DecommissionIndexerClusterPeer(enforceCounts) case "Decommissioning": - mgr.log.Info("Waiting for decommission to complete", "peerName", peerName) + mgr.log.InfoContext(ctx, "Waiting for decommission to complete", "peerName", peerName) return false, nil case "ReassigningPrimaries": - mgr.log.Info("Waiting for decommission to complete", "peerName", peerName) + mgr.log.InfoContext(ctx, "Waiting for decommission to complete", "peerName", peerName) return false, nil case "GracefulShutdown": - mgr.log.Info("Decommission complete", "peerName", peerName, "Status", mgr.cr.Status.Peers[n].Status) + mgr.log.InfoContext(ctx, "Decommission complete", "peerName", peerName, "status", mgr.cr.Status.Peers[n].Status) return true, nil case "Down": - mgr.log.Info("Decommission complete", "peerName", peerName, "Status", mgr.cr.Status.Peers[n].Status) + mgr.log.InfoContext(ctx, "Decommission complete", "peerName", peerName, "status", mgr.cr.Status.Peers[n].Status) return true, nil case "": // this can happen after the peer has been removed from the indexer cluster - mgr.log.Info("Peer has empty ID", "peerName", peerName) + mgr.log.InfoContext(ctx, "Peer has empty ID", "peerName", peerName) return false, nil } @@ -1024,8 +1085,7 @@ func (mgr *indexerClusterPodManager) decommission(ctx context.Context, n int32, // getClient for indexerClusterPodManager returns a SplunkClient for the member n func (mgr *indexerClusterPodManager) getClient(ctx context.Context, n int32) *splclient.SplunkClient { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("indexerClusterPodManager.getClient").WithValues("name", mgr.cr.GetName(), "namespace", mgr.cr.GetNamespace()) + logger := slog.With("func", "indexerClusterPodManager.getClient", "name", mgr.cr.GetName(), "namespace", mgr.cr.GetNamespace()) // Get Pod Name memberName := GetSplunkStatefulsetPodName(SplunkIndexer, mgr.cr.GetName(), n) @@ -1037,7 +1097,7 @@ func (mgr *indexerClusterPodManager) getClient(ctx context.Context, n int32) *sp // Retrieve admin password from Pod adminPwd, err := splutil.GetSpecificSecretTokenFromPod(ctx, mgr.c, memberName, mgr.cr.GetNamespace(), "password") if err != nil { - scopedLog.Error(err, "Couldn't retrieve the admin password from pod") + logger.WarnContext(ctx, "Couldn't retrieve the admin password from pod", "error", err) } return mgr.newSplunkClient(fmt.Sprintf("https://%s:8089", fqdnName), "admin", adminPwd) @@ -1045,8 +1105,7 @@ func (mgr *indexerClusterPodManager) getClient(ctx context.Context, n int32) *sp // getClusterManagerClient for indexerClusterPodManager returns a SplunkClient for cluster manager func (mgr *indexerClusterPodManager) getClusterManagerClient(ctx context.Context) *splclient.SplunkClient { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("indexerClusterPodManager.getClusterManagerClient") + logger := slog.With("func", "indexerClusterPodManager.getClusterManagerClient", "name", mgr.cr.GetName(), "namespace", mgr.cr.GetNamespace()) // Retrieve admin password from Pod var managerIdxcName string @@ -1058,7 +1117,7 @@ func (mgr *indexerClusterPodManager) getClusterManagerClient(ctx context.Context managerIdxcName = mgr.cr.Spec.ClusterMasterRef.Name cm = SplunkClusterMaster } else { - mgr.log.Info("Empty cluster manager reference") + mgr.log.InfoContext(ctx, "Empty cluster manager reference") } // Get Fully Qualified Domain Name @@ -1068,7 +1127,7 @@ func (mgr *indexerClusterPodManager) getClusterManagerClient(ctx context.Context podName := fmt.Sprintf("splunk-%s-%s-%s", managerIdxcName, cm, "0") adminPwd, err := splutil.GetSpecificSecretTokenFromPod(ctx, mgr.c, podName, mgr.cr.GetNamespace(), "password") if err != nil { - scopedLog.Error(err, "Couldn't retrieve the admin password from pod") + logger.WarnContext(ctx, "Couldn't retrieve the admin password from pod", "error", err.Error()) } return mgr.newSplunkClient(fmt.Sprintf("https://%s:8089", fqdnName), "admin", adminPwd) @@ -1109,7 +1168,7 @@ func (mgr *indexerClusterPodManager) verifyRFPeers(ctx context.Context, c splcom requestedReplicas := mgr.cr.Spec.Replicas if requestedReplicas < replicationFactor { - mgr.log.Info("Changing number of replicas as it is less than RF number of peers", "replicas", requestedReplicas) + mgr.log.InfoContext(ctx, "Changing number of replicas as it is less than RF number of peers", "replicas", requestedReplicas) // Emit event indicating scaling below RF is blocked/adjusted if eventPublisher != nil { eventPublisher.Warning(ctx, "ScalingBlockedRF", @@ -1160,6 +1219,8 @@ func (mgr *indexerClusterPodManager) updateStatus(ctx context.Context, statefulS if err != nil { return err } + totalPeerCount := len(peers) + clusterName := mgr.cr.GetName() for n := int32(0); n < statefulSet.Status.Replicas; n++ { peerName := GetSplunkStatefulsetPodName(SplunkIndexer, mgr.cr.GetName(), n) peerStatus := enterpriseApi.IndexerClusterMemberStatus{Name: peerName} @@ -1170,8 +1231,12 @@ func (mgr *indexerClusterPodManager) updateStatus(ctx context.Context, statefulS peerStatus.ActiveBundleID = peerInfo.ActiveBundleID peerStatus.BucketCount = peerInfo.BucketCount peerStatus.Searchable = peerInfo.Searchable + slog.InfoContext(ctx, "Peer registered with cluster manager", + "peerName", peerName, + "clusterName", clusterName, + "totalPeerCount", totalPeerCount) } else { - mgr.log.Info("Peer is not known by cluster manager", "peerName", peerName) + mgr.log.InfoContext(ctx, "Peer is not known by Cluster Manager", "peerName", peerName) } if n < int32(len(mgr.cr.Status.Peers)) { mgr.cr.Status.Peers[n] = peerStatus @@ -1255,14 +1320,13 @@ func validateIndexerClusterSpec(ctx context.Context, c splcommon.ControllerClien // helper function to get the list of IndexerCluster types in the current namespace func getIndexerClusterList(ctx context.Context, c splcommon.ControllerClient, cr splcommon.MetaObject, listOpts []rclient.ListOption) (enterpriseApi.IndexerClusterList, error) { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("getIndexerClusterList").WithValues("name", cr.GetName(), "namespace", cr.GetNamespace()) + logger := slog.With("func", "getIndexerClusterList", "name", cr.GetName(), "namespace", cr.GetNamespace()) objectList := enterpriseApi.IndexerClusterList{} err := c.List(context.TODO(), &objectList, listOpts...) if err != nil { - scopedLog.Error(err, "IndexerCluster types not found in namespace", "namsespace", cr.GetNamespace()) + logger.ErrorContext(ctx, fmt.Sprintf("IndexerCluster types not found in namespace %s", cr.GetNamespace()), "error", err.Error()) return objectList, err } @@ -1331,8 +1395,7 @@ var newSplunkClientForQueuePipeline = splclient.NewSplunkClient // updateIndexerConfFiles checks if Queue or Pipeline inputs are created for the first time and updates the conf file if so func (mgr *indexerClusterPodManager) updateIndexerConfFiles(ctx context.Context, newCR *enterpriseApi.IndexerCluster, queue *enterpriseApi.QueueSpec, os *enterpriseApi.ObjectStorageSpec, accessKey, secretKey string, k8s rclient.Client) error { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("updateIndexerConfFiles").WithValues("name", newCR.GetName(), "namespace", newCR.GetNamespace()) + logger := slog.With("func", "updateIndexerConfFiles", "name", newCR.GetName(), "namespace", newCR.GetNamespace()) // Only update config for pods that exist readyReplicas := newCR.Status.ReadyReplicas @@ -1351,19 +1414,26 @@ func (mgr *indexerClusterPodManager) updateIndexerConfFiles(ctx context.Context, queueInputs, queueOutputs, pipelineInputs := getQueueAndPipelineInputsForIndexerConfFiles(queue, os, accessKey, secretKey) for _, pbVal := range queueOutputs { - if err := splunkClient.UpdateConfFile(scopedLog, "outputs", fmt.Sprintf("remote_queue:%s", queue.SQS.Name), [][]string{pbVal}); err != nil { + if !strings.Contains(pbVal[0], "access_key") && !strings.Contains(pbVal[0], "secret_key") { + logger.DebugContext(ctx, "Updating queue input in outputs.conf", "input", pbVal) + } + if err := splunkClient.UpdateConfFile(ctx, logger, "outputs", fmt.Sprintf("remote_queue:%s", queue.SQS.Name), [][]string{pbVal}); err != nil { updateErr = err } } for _, pbVal := range queueInputs { - if err := splunkClient.UpdateConfFile(scopedLog, "inputs", fmt.Sprintf("remote_queue:%s", queue.SQS.Name), [][]string{pbVal}); err != nil { + if !strings.Contains(pbVal[0], "access_key") && !strings.Contains(pbVal[0], "secret_key") { + logger.DebugContext(ctx, "Updating queue input in inputs.conf", "input", pbVal) + } + if err := splunkClient.UpdateConfFile(ctx, logger, "inputs", fmt.Sprintf("remote_queue:%s", queue.SQS.Name), [][]string{pbVal}); err != nil { updateErr = err } } for _, field := range pipelineInputs { - if err := splunkClient.UpdateConfFile(scopedLog, "default-mode", field[0], [][]string{{field[1], field[2]}}); err != nil { + logger.DebugContext(ctx, "Updating pipeline input in default-mode.conf", "input", field) + if err := splunkClient.UpdateConfFile(ctx, logger, "default-mode", field[0], [][]string{{field[1], field[2]}}); err != nil { updateErr = err } } diff --git a/pkg/splunk/enterprise/indexercluster_test.go b/pkg/splunk/enterprise/indexercluster_test.go index 6aa5dc5f6..91b7665ec 100644 --- a/pkg/splunk/enterprise/indexercluster_test.go +++ b/pkg/splunk/enterprise/indexercluster_test.go @@ -19,6 +19,7 @@ import ( "context" "encoding/json" "fmt" + "log/slog" "net/http" "os" "path/filepath" @@ -44,7 +45,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" - "github.com/go-logr/logr" splclient "github.com/splunk/splunk-operator/pkg/splunk/client" splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" spltest "github.com/splunk/splunk-operator/pkg/splunk/test" @@ -307,7 +307,7 @@ func TestGetMonitoringConsoleClient(t *testing.T) { }, }, } - scopedLog := logt.WithName("TestGetMonitoringConsoleClient") + logger := slog.With("TestGetMonitoringConsoleClient") secrets := &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ @@ -320,7 +320,7 @@ func TestGetMonitoringConsoleClient(t *testing.T) { } mockSplunkClient := &spltest.MockHTTPClient{} mgr := &indexerClusterPodManager{ - log: scopedLog, + log: logger, cr: ¤t, secrets: secrets, newSplunkClient: func(managementURI, username, password string) *splclient.SplunkClient { @@ -336,7 +336,7 @@ func TestGetClusterManagerClient(t *testing.T) { os.Setenv("SPLUNK_GENERAL_TERMS", "--accept-sgt-current-at-splunk-com") ctx := context.TODO() - scopedLog := logt.WithName("TestGetClusterManagerClient") + logger := slog.With("TestGetClusterManagerClient") cr := enterpriseApi.IndexerCluster{ TypeMeta: metav1.TypeMeta{ Kind: "IndexerCluster", @@ -368,7 +368,7 @@ func TestGetClusterManagerClient(t *testing.T) { } mockSplunkClient := &spltest.MockHTTPClient{} mgr := &indexerClusterPodManager{ - log: scopedLog, + log: logger, cr: &cr, secrets: secrets, newSplunkClient: func(managementURI, username, password string) *splclient.SplunkClient { @@ -387,7 +387,7 @@ func TestGetClusterManagerClient(t *testing.T) { func getIndexerClusterPodManager(method string, mockHandlers []spltest.MockHTTPHandler, mockSplunkClient *spltest.MockHTTPClient, replicas int32) *indexerClusterPodManager { os.Setenv("SPLUNK_GENERAL_TERMS", "--accept-sgt-current-at-splunk-com") - scopedLog := logt.WithName(method) + logger := slog.With(method) cr := enterpriseApi.IndexerCluster{ TypeMeta: metav1.TypeMeta{ Kind: "IndexerCluster", @@ -421,7 +421,7 @@ func getIndexerClusterPodManager(method string, mockHandlers []spltest.MockHTTPH } mgr := &indexerClusterPodManager{ - log: scopedLog, + log: logger, cr: &cr, secrets: secrets, newSplunkClient: func(managementURI, username, password string) *splclient.SplunkClient { @@ -1029,7 +1029,7 @@ func TestSetClusterMaintenanceMode(t *testing.T) { func TestApplyIdxcSecret(t *testing.T) { os.Setenv("SPLUNK_GENERAL_TERMS", "--accept-sgt-current-at-splunk-com") method := "ApplyIdxcSecret" - scopedLog := logt.WithName(method) + logger := slog.With(method) var initObjectList []client.Object ctx := context.TODO() @@ -1154,7 +1154,7 @@ func TestApplyIdxcSecret(t *testing.T) { mockSplunkClient.AddHandlers(mockHandlers...) mgr := &indexerClusterPodManager{ c: c, - log: scopedLog, + log: logger, cr: &cr, secrets: secrets, newSplunkClient: func(managementURI, username, password string) *splclient.SplunkClient { @@ -1598,7 +1598,7 @@ func TestIndexerClusterWithReadyState(t *testing.T) { savedNewIndexerClusterPodManager := newIndexerClusterPodManager defer func() { newIndexerClusterPodManager = savedNewIndexerClusterPodManager }() - newIndexerClusterPodManager = func(log logr.Logger, cr *enterpriseApi.IndexerCluster, secret *corev1.Secret, newSplunkClient NewSplunkClientFunc, c splcommon.ControllerClient) indexerClusterPodManager { + newIndexerClusterPodManager = func(log *slog.Logger, cr *enterpriseApi.IndexerCluster, secret *corev1.Secret, newSplunkClient NewSplunkClientFunc, c splcommon.ControllerClient) indexerClusterPodManager { return indexerClusterPodManager{ log: log, cr: cr, @@ -2744,7 +2744,7 @@ func TestPasswordSyncCompleted(t *testing.T) { // Initialize a minimal pod manager for ApplyIdxcSecret mgr := &indexerClusterPodManager{ c: client, - log: logt.WithName("TestPasswordSyncCompleted"), + log: slog.With("TestPasswordSyncCompleted"), cr: &idxc, } @@ -3276,7 +3276,7 @@ func TestIdxcPasswordSyncFailedEvent(t *testing.T) { mgr := &indexerClusterPodManager{ c: c, - log: logt.WithName("TestIdxcPasswordSyncFailedEvent"), + log: slog.With("TestIdxcPasswordSyncFailedEvent"), cr: &idxc, newSplunkClient: func(managementURI, username, password string) *splclient.SplunkClient { sc := splclient.NewSplunkClient(managementURI, username, password) diff --git a/pkg/splunk/enterprise/ingestorcluster.go b/pkg/splunk/enterprise/ingestorcluster.go index 86ea854f2..ad643047f 100644 --- a/pkg/splunk/enterprise/ingestorcluster.go +++ b/pkg/splunk/enterprise/ingestorcluster.go @@ -17,6 +17,7 @@ package enterprise import ( "context" "fmt" + "log/slog" "reflect" "strings" "time" @@ -45,8 +46,7 @@ func ApplyIngestorCluster(ctx context.Context, client client.Client, cr *enterpr RequeueAfter: time.Second * 5, } - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("ApplyIngestorCluster") + logger := slog.With("func", "ApplyIngestorCluster", "name", cr.GetName(), "namespace", cr.GetNamespace()) if cr.Status.ResourceRevMap == nil { cr.Status.ResourceRevMap = make(map[string]string) @@ -60,8 +60,8 @@ func ApplyIngestorCluster(ctx context.Context, client client.Client, cr *enterpr // Validate and updates defaults for CR err = validateIngestorClusterSpec(ctx, client, cr) if err != nil { - eventPublisher.Warning(ctx, "validateIngestorClusterSpec", fmt.Sprintf("validate ingestor cluster spec failed %s", err.Error())) - scopedLog.Error(err, "Failed to validate ingestor cluster spec") + eventPublisher.Warning(ctx, "ValidateIngestorClusterSpecFailure", fmt.Sprintf("validate Ingestor Cluster spec failed %s", err.Error())) + logger.ErrorContext(ctx, "Failed to validate Ingestor Cluster spec", "error", err.Error()) return result, err } @@ -334,8 +334,7 @@ func getIngestorStatefulSet(ctx context.Context, client splcommon.ControllerClie // updateIngestorConfFiles checks if Queue or Pipeline inputs are created for the first time and updates the conf file if so func (mgr *ingestorClusterPodManager) updateIngestorConfFiles(ctx context.Context, newCR *enterpriseApi.IngestorCluster, queue *enterpriseApi.QueueSpec, os *enterpriseApi.ObjectStorageSpec, accessKey, secretKey string, k8s client.Client) error { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("updateIngestorConfFiles").WithValues("name", newCR.GetName(), "namespace", newCR.GetNamespace()) + logger := slog.With("func", "updateIngestorConfFiles", "name", newCR.GetName(), "namespace", newCR.GetNamespace()) // Only update config for pods that exist readyReplicas := newCR.Status.Replicas @@ -354,13 +353,13 @@ func (mgr *ingestorClusterPodManager) updateIngestorConfFiles(ctx context.Contex queueInputs, pipelineInputs := getQueueAndPipelineInputsForIngestorConfFiles(queue, os, accessKey, secretKey) for _, input := range queueInputs { - if err := splunkClient.UpdateConfFile(scopedLog, "outputs", fmt.Sprintf("remote_queue:%s", queue.SQS.Name), [][]string{input}); err != nil { + if err := splunkClient.UpdateConfFile(ctx, logger, "outputs", fmt.Sprintf("remote_queue:%s", queue.SQS.Name), [][]string{input}); err != nil { updateErr = err } } for _, input := range pipelineInputs { - if err := splunkClient.UpdateConfFile(scopedLog, "default-mode", input[0], [][]string{{input[1], input[2]}}); err != nil { + if err := splunkClient.UpdateConfFile(ctx, logger, "default-mode", input[0], [][]string{{input[1], input[2]}}); err != nil { updateErr = err } } From 87c8d79ffcadcbbfb1c10225f5dcca40e93d0232 Mon Sep 17 00:00:00 2001 From: Kasia Koziol Date: Wed, 25 Feb 2026 09:56:53 +0100 Subject: [PATCH 2/2] CSPL-4354 Refactoring --- pkg/splunk/client/enterprise_test.go | 2 +- pkg/splunk/enterprise/indexercluster.go | 20 ++++++++++---------- pkg/splunk/enterprise/ingestorcluster.go | 2 +- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/pkg/splunk/client/enterprise_test.go b/pkg/splunk/client/enterprise_test.go index af6a4e2c0..a18d3bc1f 100644 --- a/pkg/splunk/client/enterprise_test.go +++ b/pkg/splunk/client/enterprise_test.go @@ -705,7 +705,7 @@ func TestUpdateConfFile(t *testing.T) { fileName := "outputs" ctx := context.TODO() - logger := slog.With("func", "TestUpdateConfFile", "fileName", fileName, "property", property) + logger := slog.With("func", "TestUpdateConfFile") // First request: create the property (object) if it doesn't exist createBody := strings.NewReader(fmt.Sprintf("name=%s", property)) diff --git a/pkg/splunk/enterprise/indexercluster.go b/pkg/splunk/enterprise/indexercluster.go index 569b3870e..b92b0ca32 100644 --- a/pkg/splunk/enterprise/indexercluster.go +++ b/pkg/splunk/enterprise/indexercluster.go @@ -225,15 +225,15 @@ func ApplyIndexerClusterManager(ctx context.Context, client splcommon.Controller if !versionUpgrade { phase, err = mgr.Update(ctx, client, statefulSet, cr.Spec.Replicas) if err != nil { - eventPublisher.Warning(ctx, "UpdateManagerFailure", fmt.Sprintf("update of statefulset failed with error %s", err.Error())) - logger.ErrorContext(ctx, "Update of statefulset failed", "error", err.Error()) + eventPublisher.Warning(ctx, "UpdateFailure", fmt.Sprintf("update of stateful set failed with error %s", err.Error())) + logger.ErrorContext(ctx, "Update of stateful set failed", "error", err.Error()) return result, err } } else { // Delete the statefulset and recreate new one err = client.Delete(ctx, statefulSet) if err != nil { - eventPublisher.Warning(ctx, "DeleteManagerFailure", fmt.Sprintf(fmt.Sprintf("version mismatch for Indexer Cluster %s and Indexer container %s - delete of stateful set failed with error=%s", cr.Spec.Image, statefulSet.Spec.Template.Spec.Containers[0].Image, err.Error()))) + eventPublisher.Warning(ctx, "DeleteFailure", fmt.Sprintf("version mismatch for Indexer Cluster %s and Indexer container %s - delete of stateful set failed with %s", cr.Spec.Image, statefulSet.Spec.Template.Spec.Containers[0].Image, err.Error())) logger.ErrorContext(ctx, "Delete of stateful set failed", "error", err.Error()) return result, err } @@ -242,8 +242,8 @@ func ApplyIndexerClusterManager(ctx context.Context, client splcommon.Controller statefulSet.ResourceVersion = "" phase, err = mgr.Update(ctx, client, statefulSet, cr.Spec.Replicas) if err != nil { - eventPublisher.Warning(ctx, "UpdateManagerFailure", fmt.Sprintf("update statefulset failed %s", err.Error())) - logger.ErrorContext(ctx, "Update of statefulset failed", "error", err.Error()) + eventPublisher.Warning(ctx, "UpdateFailure", fmt.Sprintf("update of stateful set failed with %s", err.Error())) + logger.ErrorContext(ctx, "Update of stateful set failed", "error", err.Error()) return result, err } } @@ -284,7 +284,7 @@ func ApplyIndexerClusterManager(ctx context.Context, client splcommon.Controller if err != nil { return result, err } - logger.InfoContext(ctx, "Restarted splunk", "indexer", i) + logger.DebugContext(ctx, "Restarted splunk", "indexer", i) } eventPublisher.Normal(ctx, "IndexersRestarted", @@ -460,7 +460,7 @@ func ApplyIndexerCluster(ctx context.Context, client splcommon.ControllerClient, result.Requeue = false } if err != nil { - eventPublisher.Warning(ctx, "DeleteFailure", fmt.Sprintf("delete custom resource failed %s", err.Error())) + eventPublisher.Warning(ctx, "DeleteFailure", fmt.Sprintf("delete custom resource failed with %s", err.Error())) logger.ErrorContext(ctx, "Delete custom resource failed", "error", err.Error()) } return result, err @@ -545,7 +545,7 @@ func ApplyIndexerCluster(ctx context.Context, client splcommon.ControllerClient, if !versionUpgrade { phase, err = mgr.Update(ctx, client, statefulSet, cr.Spec.Replicas) if err != nil { - eventPublisher.Warning(ctx, "UpdateManager", fmt.Sprintf("update of stateful set failed with %s", err.Error())) + eventPublisher.Warning(ctx, "UpdateFailure", fmt.Sprintf("update of stateful set failed with %s", err.Error())) logger.ErrorContext(ctx, "Update of stateful set failed", "error", err.Error()) return result, err } @@ -553,7 +553,7 @@ func ApplyIndexerCluster(ctx context.Context, client splcommon.ControllerClient, // Delete the statefulset and recreate new one err = client.Delete(ctx, statefulSet) if err != nil { - eventPublisher.Warning(ctx, "DeleteFailure", fmt.Sprintf(fmt.Sprintf("version mismatch for Indexer Cluster %s and Indexer container %s - delete of stateful set failed with error=%s", cr.Spec.Image, statefulSet.Spec.Template.Spec.Containers[0].Image, err.Error()))) + eventPublisher.Warning(ctx, "DeleteFailure", fmt.Sprintf("version mismatch for Indexer Cluster %s and Indexer container %s - delete of stateful set failed with error=%s", cr.Spec.Image, statefulSet.Spec.Template.Spec.Containers[0].Image, err.Error())) logger.ErrorContext(ctx, "Delete of stateful set failed", "error", err.Error()) return result, err } @@ -603,7 +603,7 @@ func ApplyIndexerCluster(ctx context.Context, client splcommon.ControllerClient, if err != nil { return result, err } - logger.InfoContext(ctx, "Restarted splunk", "indexer", i) + logger.DebugContext(ctx, "Restarted splunk", "indexer", i) } eventPublisher.Normal(ctx, "IndexersRestarted", diff --git a/pkg/splunk/enterprise/ingestorcluster.go b/pkg/splunk/enterprise/ingestorcluster.go index 5bc272a26..7a5524f6e 100644 --- a/pkg/splunk/enterprise/ingestorcluster.go +++ b/pkg/splunk/enterprise/ingestorcluster.go @@ -235,7 +235,7 @@ func ApplyIngestorCluster(ctx context.Context, client client.Client, cr *enterpr secretChanged := cr.Status.CredentialSecretVersion != qosCfg.Version serviceAccountChanged := cr.Status.ServiceAccount != cr.Spec.ServiceAccount - logger.InfoContext(ctx, "Checking for changes", "previousCredentialSecretVersion", cr.Status.CredentialSecretVersion, "previousServiceAccount", cr.Status.ServiceAccount, "secretChanged", secretChanged, "serviceAccountChanged", serviceAccountChanged) + logger.DebugContext(ctx, "Checking for changes", "previousCredentialSecretVersion", cr.Status.CredentialSecretVersion, "previousServiceAccount", cr.Status.ServiceAccount, "secretChanged", secretChanged, "serviceAccountChanged", serviceAccountChanged) // If queue is updated if secretChanged || serviceAccountChanged {