From b49cdf98e11b180b84d166be8873fbb10f07d4a0 Mon Sep 17 00:00:00 2001 From: Michael Marod Date: Wed, 18 Feb 2026 14:21:57 -0500 Subject: [PATCH 1/3] Generate a mapping from spec fields to Splunk config coordinates and publish a manifest --- .gitignore | 3 +- Makefile | 3 + tools/spec-config-mapping/main.go | 247 +++++++++++++ tools/spec-config-mapping/mappings.go | 400 +++++++++++++++++++++ tools/spec-config-mapping/mappings_test.go | 210 +++++++++++ 5 files changed, 862 insertions(+), 1 deletion(-) create mode 100644 tools/spec-config-mapping/main.go create mode 100644 tools/spec-config-mapping/mappings.go create mode 100644 tools/spec-config-mapping/mappings_test.go diff --git a/.gitignore b/.gitignore index 4b6582d68..40a66e71e 100644 --- a/.gitignore +++ b/.gitignore @@ -98,4 +98,5 @@ bundle_*/ test/secret/*.log kubeconfig .devcontainer/devcontainer.json -kuttl-artifacts/* \ No newline at end of file +kuttl-artifacts/* +spec-config-mapping.json \ No newline at end of file diff --git a/Makefile b/Makefile index b5d4a6178..4eee1561b 100644 --- a/Makefile +++ b/Makefile @@ -138,6 +138,9 @@ vet: setup/ginkgo ## Run go vet against code. test: manifests generate fmt vet envtest ## Run tests. KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use ${ENVTEST_K8S_VERSION} --bin-dir $(LOCALBIN) -p path)" ginkgo --junit-report=unit_test.xml --output-dir=`pwd` -vv --trace --keep-going --timeout=3h --cover --covermode=count --coverprofile=coverage.out ./pkg/splunk/common ./pkg/splunk/enterprise ./pkg/splunk/client ./pkg/splunk/util ./internal/controller ./pkg/splunk/splkcontroller +.PHONY: spec-config-mapping +spec-config-mapping: ## Generate CRD spec-to-Splunk-config mapping JSON. + go run ./tools/spec-config-mapping -o spec-config-mapping.json ##@ Build diff --git a/tools/spec-config-mapping/main.go b/tools/spec-config-mapping/main.go new file mode 100644 index 000000000..7d2e5b0a7 --- /dev/null +++ b/tools/spec-config-mapping/main.go @@ -0,0 +1,247 @@ +// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. + +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "encoding/json" + "flag" + "fmt" + "os" + "reflect" + "sort" + "strings" + "time" + + enterpriseApi "github.com/splunk/splunk-operator/api/v4" +) + +// ── JSON output types ───────────────────────────────────────────────── + +// Manifest is the top-level output structure. +type Manifest struct { + Version string `json:"version"` + GeneratedAt string `json:"generatedAt"` + OperatorVersion string `json:"operatorVersion"` + APIVersion string `json:"apiVersion"` + CRDs map[string]CRDMapping `json:"crds"` +} + +// CRDMapping holds the field mappings for one CRD kind. +type CRDMapping struct { + SpecFields map[string]FieldMapping `json:"specFields"` +} + +// FieldMapping is the per-field entry in the JSON output. +type FieldMapping struct { + JSONPath string `json:"jsonPath"` + GoType string `json:"goType"` + ConfFile string `json:"confFile"` + Stanza string `json:"stanza"` + Key string `json:"key"` + Description string `json:"description"` +} + + +// ── CRD registry ────────────────────────────────────────────────────── + +type crdEntry struct { + kind string + specType reflect.Type +} + +func crdRegistry() []crdEntry { + return []crdEntry{ + {"Standalone", reflect.TypeOf(enterpriseApi.StandaloneSpec{})}, + {"ClusterManager", reflect.TypeOf(enterpriseApi.ClusterManagerSpec{})}, + {"IndexerCluster", reflect.TypeOf(enterpriseApi.IndexerClusterSpec{})}, + {"SearchHeadCluster", reflect.TypeOf(enterpriseApi.SearchHeadClusterSpec{})}, + {"LicenseManager", reflect.TypeOf(enterpriseApi.LicenseManagerSpec{})}, + {"MonitoringConsole", reflect.TypeOf(enterpriseApi.MonitoringConsoleSpec{})}, + } +} + +// ── Reflection walker ───────────────────────────────────────────────── + +// fieldInfo holds metadata about a discovered struct field. +type fieldInfo struct { + jsonPath string + goType string +} + +// WalkSpecFields recursively walks a struct type and collects leaf field +// JSON paths. Embedded structs are flattened (their prefix is inherited). +// Slice and struct fields are recursed into; primitive fields are leaves. +func WalkSpecFields(t reflect.Type, prefix string) []fieldInfo { + if t.Kind() == reflect.Ptr { + t = t.Elem() + } + if t.Kind() != reflect.Struct { + return nil + } + + var fields []fieldInfo + for i := 0; i < t.NumField(); i++ { + sf := t.Field(i) + + // Determine JSON name + jsonTag := sf.Tag.Get("json") + if jsonTag == "-" { + continue + } + jsonName := strings.Split(jsonTag, ",")[0] + + // Handle embedded (anonymous) structs: flatten, keep parent prefix + if sf.Anonymous { + fields = append(fields, WalkSpecFields(sf.Type, prefix)...) + continue + } + + if jsonName == "" { + jsonName = sf.Name + } + + fullPath := jsonName + if prefix != "" { + fullPath = prefix + "." + jsonName + } + + ft := sf.Type + if ft.Kind() == reflect.Ptr { + ft = ft.Elem() + } + + switch ft.Kind() { + case reflect.Struct: + // Check if this is a well-known Kubernetes type we treat as a leaf + if isLeafType(ft) { + fields = append(fields, fieldInfo{jsonPath: fullPath, goType: ft.String()}) + } else { + fields = append(fields, WalkSpecFields(ft, fullPath)...) + } + case reflect.Slice: + elemType := ft.Elem() + if elemType.Kind() == reflect.Ptr { + elemType = elemType.Elem() + } + if elemType.Kind() == reflect.Struct && !isLeafType(elemType) { + // Recurse into the element type (represents array items) + fields = append(fields, WalkSpecFields(elemType, fullPath)...) + } else { + fields = append(fields, fieldInfo{jsonPath: fullPath, goType: ft.String()}) + } + default: + fields = append(fields, fieldInfo{jsonPath: fullPath, goType: ft.String()}) + } + } + return fields +} + +// isLeafType returns true for complex types we don't want to recurse into +// (Kubernetes API types, etc.). +func isLeafType(t reflect.Type) bool { + pkg := t.PkgPath() + // Treat all k8s.io types as opaque leaves + if strings.HasPrefix(pkg, "k8s.io/") { + return true + } + return false +} + +// ── Main ────────────────────────────────────────────────────────────── + +func main() { + outPath := flag.String("o", "spec-config-mapping.json", "Output JSON file path") + strict := flag.Bool("strict", true, "Exit non-zero if any spec fields are unmapped") + flag.Parse() + + registry := crdRegistry() + manifest := Manifest{ + Version: "1.0.0", + GeneratedAt: time.Now().UTC().Format(time.RFC3339), + OperatorVersion: "3.0.0", + APIVersion: enterpriseApi.APIVersion, + CRDs: make(map[string]CRDMapping), + } + + var unmapped []string + + for _, crd := range registry { + fields := WalkSpecFields(crd.specType, "") + specFields := make(map[string]FieldMapping) + + for _, f := range fields { + target, ok := fieldMappings[f.jsonPath] + if !ok { + unmapped = append(unmapped, fmt.Sprintf("%s.%s", crd.kind, f.jsonPath)) + continue + } + + // Only include fields with a full conf coordinate (confFile + stanza + key) + if target.ConfFile == "" || target.ConfFile == "env" || target.Stanza == "" || target.Key == "" { + continue + } + + specFields[f.jsonPath] = FieldMapping{ + JSONPath: "spec." + f.jsonPath, + GoType: f.goType, + ConfFile: target.ConfFile, + Stanza: target.Stanza, + Key: target.Key, + Description: target.Description, + } + + } + + manifest.CRDs[crd.kind] = CRDMapping{SpecFields: specFields} + } + + // Write output using Encoder so we can disable HTML escaping + // (json.MarshalIndent escapes <> as \u003c/\u003e by default) + f, err := os.Create(*outPath) + if err != nil { + fmt.Fprintf(os.Stderr, "error: failed to create %s: %v\n", *outPath, err) + os.Exit(1) + } + enc := json.NewEncoder(f) + enc.SetIndent("", " ") + enc.SetEscapeHTML(false) + if err := enc.Encode(manifest); err != nil { + f.Close() + fmt.Fprintf(os.Stderr, "error: failed to write JSON: %v\n", err) + os.Exit(1) + } + f.Close() + + // Summary + totalFields := 0 + for _, crd := range manifest.CRDs { + totalFields += len(crd.SpecFields) + } + fmt.Fprintf(os.Stderr, "Generated %s: %d CRDs, %d total field mappings\n", + *outPath, len(manifest.CRDs), totalFields) + + if len(unmapped) > 0 { + fmt.Fprintf(os.Stderr, "\nWARNING: %d unmapped field(s):\n", len(unmapped)) + sort.Strings(unmapped) + for _, u := range unmapped { + fmt.Fprintf(os.Stderr, " - %s\n", u) + } + if *strict { + fmt.Fprintf(os.Stderr, "\nAdd mappings to tools/spec-config-mapping/mappings.go to fix this.\n") + os.Exit(1) + } + } +} diff --git a/tools/spec-config-mapping/mappings.go b/tools/spec-config-mapping/mappings.go new file mode 100644 index 000000000..da1004b17 --- /dev/null +++ b/tools/spec-config-mapping/mappings.go @@ -0,0 +1,400 @@ +// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. + +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +// ConfigTarget describes how a CRD spec field maps to a Splunk configuration. +type ConfigTarget struct { + // Which Splunk conf file this maps to: "indexes.conf", "server.conf", + // "env" for environment variables, or "" for kubernetes-only fields. + ConfFile string `json:"confFile"` + + // The stanza in the conf file (e.g. "cachemanager", "volume:"). + Stanza string `json:"stanza,omitempty"` + + // The key within the stanza (e.g. "eviction_policy"). + Key string `json:"key,omitempty"` + + // For env var mappings, the environment variable name. + EnvVar string `json:"envVar,omitempty"` + + // Human-readable description of how this field is consumed. + Description string `json:"description"` +} + + +// fieldMappings maps each CRD spec field JSON path to its Splunk config target. +// Maintainers: update this table when CRD spec fields are added or removed. +// The drift detection test in mappings_test.go will fail if a field is missing. +var fieldMappings = map[string]ConfigTarget{ + // ── CommonSplunkSpec fields (shared by all CRDs) ────────────────── + + // Container / image + "image": {Description: "Docker image for Splunk pod containers"}, + "imagePullPolicy": {Description: "Image pull policy (Always or IfNotPresent)"}, + + // Scheduling + "schedulerName": {Description: "Kubernetes scheduler name for pod placement"}, + "affinity": {Description: "Kubernetes affinity rules for pod scheduling"}, + "tolerations": {Description: "Kubernetes tolerations for node taints"}, + "topologySpreadConstraints": {Description: "Kubernetes topology spread constraints"}, + + // Resources + "resources": {Description: "CPU and memory requests/limits for pod containers"}, + "serviceTemplate": {Description: "Template for Kubernetes Service customization (ports, LB config)"}, + + // Storage — etcVolumeStorageConfig (StorageClassSpec sub-fields) + "etcVolumeStorageConfig.storageClassName": {Description: "StorageClass name for /opt/splunk/etc PVC"}, + "etcVolumeStorageConfig.storageCapacity": {Description: "Storage capacity for /opt/splunk/etc PVC (default: 10Gi)"}, + "etcVolumeStorageConfig.ephemeralStorage": {Description: "Use emptyDir instead of PVC for /opt/splunk/etc"}, + + // Storage — varVolumeStorageConfig (StorageClassSpec sub-fields) + "varVolumeStorageConfig.storageClassName": {Description: "StorageClass name for /opt/splunk/var PVC"}, + "varVolumeStorageConfig.storageCapacity": {Description: "Storage capacity for /opt/splunk/var PVC (default: 100Gi)"}, + "varVolumeStorageConfig.ephemeralStorage": {Description: "Use emptyDir instead of PVC for /opt/splunk/var"}, + + // Volumes + "volumes": {Description: "Additional Kubernetes volumes mounted at /mnt/"}, + + // Splunk defaults (Ansible) + "defaults": { + ConfFile: "env", + EnvVar: "SPLUNK_DEFAULTS_URL", + Description: "Inline default.yml overrides; mounted at /mnt/splunk-defaults/default.yml and referenced via SPLUNK_DEFAULTS_URL", + }, + "defaultsUrl": { + ConfFile: "env", + EnvVar: "SPLUNK_DEFAULTS_URL", + Description: "External URL(s) for default.yml files; appended to SPLUNK_DEFAULTS_URL", + }, + "defaultsUrlApps": { + ConfFile: "env", + EnvVar: "SPLUNK_DEFAULTS_URL", + Description: "App-specific defaults URL(s); prepended to SPLUNK_DEFAULTS_URL (not applied to IDXC/SHC members)", + }, + + // License + "licenseUrl": { + ConfFile: "env", + EnvVar: "SPLUNK_LICENSE_URI", + Description: "URL to Splunk Enterprise license file", + }, + "licenseMasterRef": { + ConfFile: "env", + EnvVar: "SPLUNK_LICENSE_MASTER_URL", + Description: "(Deprecated v3 field) Reference to LicenseMaster; resolved to service FQDN", + }, + "licenseManagerRef": { + ConfFile: "env", + EnvVar: "SPLUNK_LICENSE_MASTER_URL", + Description: "Reference to LicenseManager; resolved to service FQDN", + }, + + // Cluster manager + "clusterMasterRef": { + ConfFile: "env", + EnvVar: "SPLUNK_CLUSTER_MASTER_URL", + Description: "(Deprecated v3 field) Reference to ClusterMaster; resolved to service FQDN or 'localhost' for CM", + }, + "clusterManagerRef": { + ConfFile: "env", + EnvVar: "SPLUNK_CLUSTER_MASTER_URL", + Description: "Reference to ClusterManager; resolved to service FQDN or 'localhost' for CM", + }, + + // Monitoring console + "monitoringConsoleRef": { + ConfFile: "env", + EnvVar: "SPLUNK_MONITORING_CONSOLE_REF", + Description: "Reference to MonitoringConsole; passed as CR name to register instance", + }, + + // Service account + "serviceAccount": {Description: "Kubernetes ServiceAccount for pod identity"}, + + // Extra environment + "extraEnv": { + ConfFile: "env", + Description: "Additional environment variables passed to Splunk containers; may override operator-set vars", + }, + + // Probes — top-level delay overrides + "readinessInitialDelaySeconds": {Description: "Initial delay for readiness probe"}, + "livenessInitialDelaySeconds": {Description: "Initial delay for liveness probe"}, + + // Probes — livenessProbe (Probe sub-fields) + "livenessProbe.initialDelaySeconds": {Description: "Liveness probe initial delay seconds"}, + "livenessProbe.timeoutSeconds": {Description: "Liveness probe timeout seconds"}, + "livenessProbe.periodSeconds": {Description: "Liveness probe period seconds"}, + "livenessProbe.failureThreshold": {Description: "Liveness probe failure threshold"}, + + // Probes — readinessProbe (Probe sub-fields) + "readinessProbe.initialDelaySeconds": {Description: "Readiness probe initial delay seconds"}, + "readinessProbe.timeoutSeconds": {Description: "Readiness probe timeout seconds"}, + "readinessProbe.periodSeconds": {Description: "Readiness probe period seconds"}, + "readinessProbe.failureThreshold": {Description: "Readiness probe failure threshold"}, + + // Probes — startupProbe (Probe sub-fields) + "startupProbe.initialDelaySeconds": {Description: "Startup probe initial delay seconds"}, + "startupProbe.timeoutSeconds": {Description: "Startup probe timeout seconds"}, + "startupProbe.periodSeconds": {Description: "Startup probe period seconds"}, + "startupProbe.failureThreshold": {Description: "Startup probe failure threshold"}, + + // Image pull secrets + "imagePullSecrets": {Description: "Secrets for pulling images from private registries"}, + + // Mock (internal) + "Mock": {Description: "Internal flag to differentiate unit tests from actual reconciliation"}, + + // ── Standalone-specific fields ──────────────────────────────────── + + "replicas": {Description: "Number of pods (StatefulSet replica count)"}, + + // ── SearchHeadCluster-specific fields ───────────────────────────── + + "deployerResourceSpec": {Description: "Resource requirements for the SHC Deployer pod"}, + "deployerNodeAffinity": {Description: "Node affinity rules for the SHC Deployer pod"}, + + // ── SmartStore fields (Standalone, ClusterManager) ──────────────── + + // smartstore.volumes[] + "smartstore.volumes.name": { + ConfFile: "indexes.conf", + Stanza: "volume:", + Description: "Remote storage volume name; becomes stanza name in indexes.conf", + }, + "smartstore.volumes.endpoint": { + ConfFile: "indexes.conf", + Stanza: "volume:", + Key: "remote.s3.endpoint", + Description: "Remote storage endpoint URL", + }, + "smartstore.volumes.path": { + ConfFile: "indexes.conf", + Stanza: "volume:", + Key: "path", + Description: "Remote volume path; rendered as s3://", + }, + "smartstore.volumes.secretRef": { + ConfFile: "indexes.conf", + Stanza: "volume:", + Key: "remote.s3.access_key, remote.s3.secret_key", + Description: "Kubernetes Secret containing remote storage credentials", + }, + "smartstore.volumes.storageType": { + ConfFile: "indexes.conf", + Stanza: "volume:", + Key: "storageType", + Description: "Remote storage type (s3, blob, gcs); always rendered as 'remote'", + }, + "smartstore.volumes.provider": { + ConfFile: "indexes.conf", + Stanza: "volume:", + Description: "Storage provider (aws, minio, azure, gcp); used for credential handling", + }, + "smartstore.volumes.region": { + ConfFile: "indexes.conf", + Stanza: "volume:", + Key: "remote.s3.auth_region", + Description: "AWS region for remote storage volume", + }, + + // smartstore.indexes[] + "smartstore.indexes.name": { + ConfFile: "indexes.conf", + Stanza: "", + Description: "Splunk index name; becomes stanza name in indexes.conf", + }, + "smartstore.indexes.remotePath": { + ConfFile: "indexes.conf", + Stanza: "", + Key: "remotePath", + Description: "Index location relative to volume path; rendered as volume:/", + }, + "smartstore.indexes.volumeName": { + ConfFile: "indexes.conf", + Stanza: "", + Key: "remotePath", + Description: "Volume name for index remote path; combined with remotePath", + }, + "smartstore.indexes.hotlistRecencySecs": { + ConfFile: "indexes.conf", + Stanza: "", + Key: "hotlist_recency_secs", + Description: "Time period (seconds) during which bucket is protected from cache eviction", + }, + "smartstore.indexes.hotlistBloomFilterRecencyHours": { + ConfFile: "indexes.conf", + Stanza: "", + Key: "hotlist_bloom_filter_recency_hours", + Description: "Time period (hours) during which bloom filter is protected from cache eviction", + }, + "smartstore.indexes.maxGlobalDataSizeMB": { + ConfFile: "indexes.conf", + Stanza: "", + Key: "maxGlobalDataSizeMB", + Description: "Maximum space for warm and cold buckets of an index", + }, + "smartstore.indexes.maxGlobalRawDataSizeMB": { + ConfFile: "indexes.conf", + Stanza: "", + Key: "maxGlobalRawDataSizeMB", + Description: "Maximum cumulative space for warm and cold buckets of an index", + }, + + // smartstore.defaults + "smartstore.defaults.volumeName": { + ConfFile: "indexes.conf", + Stanza: "default", + Key: "remotePath", + Description: "Default volume for all indexes; rendered as remotePath = volume:/$_index_name", + }, + "smartstore.defaults.maxGlobalDataSizeMB": { + ConfFile: "indexes.conf", + Stanza: "default", + Key: "maxGlobalDataSizeMB", + Description: "Default maximum space for warm and cold buckets", + }, + "smartstore.defaults.maxGlobalRawDataSizeMB": { + ConfFile: "indexes.conf", + Stanza: "default", + Key: "maxGlobalRawDataSizeMB", + Description: "Default maximum cumulative space for warm and cold buckets", + }, + + // smartstore.cacheManager + "smartstore.cacheManager.hotlistRecencySecs": { + ConfFile: "server.conf", + Stanza: "cachemanager", + Key: "hotlist_recency_secs", + Description: "Global cache: time period (seconds) bucket is protected from eviction", + }, + "smartstore.cacheManager.hotlistBloomFilterRecencyHours": { + ConfFile: "server.conf", + Stanza: "cachemanager", + Key: "hotlist_bloom_filter_recency_hours", + Description: "Global cache: time period (hours) bloom filter is protected from eviction", + }, + "smartstore.cacheManager.evictionPolicy": { + ConfFile: "server.conf", + Stanza: "cachemanager", + Key: "eviction_policy", + Description: "Cache eviction policy (e.g. lru)", + }, + "smartstore.cacheManager.maxCacheSize": { + ConfFile: "server.conf", + Stanza: "cachemanager", + Key: "max_cache_size", + Description: "Maximum cache size in MB", + }, + "smartstore.cacheManager.evictionPadding": { + ConfFile: "server.conf", + Stanza: "cachemanager", + Key: "eviction_padding", + Description: "Additional size beyond minFreeSize before eviction kicks in", + }, + "smartstore.cacheManager.maxConcurrentDownloads": { + ConfFile: "server.conf", + Stanza: "cachemanager", + Key: "max_concurrent_downloads", + Description: "Maximum parallel bucket downloads from remote storage", + }, + "smartstore.cacheManager.maxConcurrentUploads": { + ConfFile: "server.conf", + Stanza: "cachemanager", + Key: "max_concurrent_uploads", + Description: "Maximum parallel bucket uploads to remote storage", + }, + + // ── App Framework fields ────────────────────────────────────────── + + // appRepo.defaults + "appRepo.defaults.volumeName": { + Description: "Default remote storage volume name for app sources", + }, + "appRepo.defaults.scope": { + Description: "Default app deployment scope: local, cluster, clusterWithPreConfig, or premiumApps", + }, + "appRepo.defaults.premiumAppsProps.type": { + Description: "Premium app type (e.g. enterpriseSecurity)", + }, + "appRepo.defaults.premiumAppsProps.esDefaults.sslEnablement": { + ConfFile: "web.conf", + Stanza: "settings", + Key: "enableSplunkWebSSL", + Description: "SSL enablement mode for Enterprise Security app (strict, auto, ignore)", + }, + + // appRepo.volumes[] + "appRepo.volumes.name": { + Description: "App repository remote volume name", + }, + "appRepo.volumes.endpoint": { + Description: "App repository remote storage endpoint URL", + }, + "appRepo.volumes.path": { + Description: "App repository remote volume path (S3 bucket / Azure container)", + }, + "appRepo.volumes.secretRef": { + Description: "Kubernetes Secret for app repository remote storage credentials", + }, + "appRepo.volumes.storageType": { + Description: "App repository storage type (s3, blob, gcs)", + }, + "appRepo.volumes.provider": { + Description: "App repository storage provider (aws, minio, azure, gcp)", + }, + "appRepo.volumes.region": { + Description: "AWS region for app repository remote storage", + }, + + // appRepo.appSources[] + "appRepo.appSources.name": { + Description: "Logical name for a set of apps at a remote location", + }, + "appRepo.appSources.location": { + Description: "Path relative to volume where app packages reside", + }, + "appRepo.appSources.volumeName": { + Description: "Remote storage volume name for this app source", + }, + "appRepo.appSources.scope": { + Description: "App deployment scope: local (per-pod), cluster (bundle push), clusterWithPreConfig, premiumApps", + }, + "appRepo.appSources.premiumAppsProps.type": { + Description: "Premium app type for this app source", + }, + "appRepo.appSources.premiumAppsProps.esDefaults.sslEnablement": { + ConfFile: "web.conf", + Stanza: "settings", + Key: "enableSplunkWebSSL", + Description: "SSL enablement for Enterprise Security in this app source", + }, + + // appRepo top-level settings + "appRepo.appsRepoPollIntervalSeconds": { + Description: "Interval in seconds to poll remote storage for app changes (0 = disabled)", + }, + "appRepo.appInstallPeriodSeconds": { + Description: "Time window in seconds within a reconcile cycle to install apps", + }, + "appRepo.installMaxRetries": { + Description: "Maximum retry attempts for failed app installations", + }, + "appRepo.maxConcurrentAppDownloads": { + Description: "Maximum number of apps downloaded in parallel", + }, +} + diff --git a/tools/spec-config-mapping/mappings_test.go b/tools/spec-config-mapping/mappings_test.go new file mode 100644 index 000000000..79ea9152a --- /dev/null +++ b/tools/spec-config-mapping/mappings_test.go @@ -0,0 +1,210 @@ +// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. + +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "fmt" + "sort" + "strings" + "testing" +) + +// TestAllFieldsHaveMappings verifies that every CRD spec field discovered +// via reflection has a corresponding entry in fieldMappings. +func TestAllFieldsHaveMappings(t *testing.T) { + registry := crdRegistry() + allDiscovered := make(map[string]bool) + + for _, crd := range registry { + fields := WalkSpecFields(crd.specType, "") + for _, f := range fields { + allDiscovered[f.jsonPath] = true + if _, ok := fieldMappings[f.jsonPath]; !ok { + t.Errorf("CRD %s: field %q has no entry in fieldMappings", crd.kind, f.jsonPath) + } + } + } + + // Also check for stale entries: mapping keys that don't match any discovered field + var stale []string + for key := range fieldMappings { + if !allDiscovered[key] { + stale = append(stale, key) + } + } + if len(stale) > 0 { + sort.Strings(stale) + t.Errorf("Stale mapping entries (not found in any CRD spec):\n %s", strings.Join(stale, "\n ")) + } +} + +// TestNoDuplicateFieldPaths ensures there are no duplicate field paths +// within a single CRD's discovered fields. +func TestNoDuplicateFieldPaths(t *testing.T) { + registry := crdRegistry() + for _, crd := range registry { + fields := WalkSpecFields(crd.specType, "") + seen := make(map[string]int) + for _, f := range fields { + seen[f.jsonPath]++ + } + for path, count := range seen { + if count > 1 { + t.Errorf("CRD %s: field path %q discovered %d times (expected 1)", crd.kind, path, count) + } + } + } +} + +// TestFieldMappingDescriptions ensures every mapping has a non-empty description. +func TestFieldMappingDescriptions(t *testing.T) { + for path, target := range fieldMappings { + if target.Description == "" { + t.Errorf("fieldMappings[%q] has an empty Description", path) + } + } +} + +// TestConfFileMappingsHaveStanza checks that entries with a non-empty ConfFile +// (other than "env" or "") have a non-empty Stanza. +func TestConfFileMappingsHaveStanza(t *testing.T) { + for path, target := range fieldMappings { + if target.ConfFile != "" && target.ConfFile != "env" { + if target.Stanza == "" { + t.Errorf("fieldMappings[%q] has confFile=%q but no Stanza", path, target.ConfFile) + } + } + } +} + +// TestEnvMappingsHaveEnvVar checks that entries with ConfFile="env" have EnvVar set. +func TestEnvMappingsHaveEnvVar(t *testing.T) { + for path, target := range fieldMappings { + if target.ConfFile == "env" && target.EnvVar == "" { + // extraEnv is a special case - it's a pass-through for arbitrary env vars + if path == "extraEnv" { + continue + } + t.Errorf("fieldMappings[%q] has confFile=env but no EnvVar", path) + } + } +} + +// TestWalkSpecFieldsOutput is a sanity check that the walker produces expected paths. +func TestWalkSpecFieldsOutput(t *testing.T) { + registry := crdRegistry() + + // Find the Standalone CRD and check some expected fields + for _, crd := range registry { + if crd.kind != "Standalone" { + continue + } + fields := WalkSpecFields(crd.specType, "") + paths := make(map[string]bool) + for _, f := range fields { + paths[f.jsonPath] = true + } + + expected := []string{ + "replicas", + "image", + "licenseUrl", + "smartstore.cacheManager.evictionPolicy", + "smartstore.indexes.name", + "smartstore.volumes.endpoint", + "appRepo.appsRepoPollIntervalSeconds", + } + for _, e := range expected { + if !paths[e] { + t.Errorf("Standalone: expected field path %q not found. Got paths: %v", e, sortedKeys(paths)) + } + } + } +} + +func sortedKeys(m map[string]bool) []string { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + sort.Strings(keys) + return keys +} + +// TestGenerateManifestIntegration runs the full generation logic and checks the output structure. +func TestGenerateManifestIntegration(t *testing.T) { + registry := crdRegistry() + fieldToCRDs := make(map[string][]string) + + for _, crd := range registry { + fields := WalkSpecFields(crd.specType, "") + for _, f := range fields { + fieldToCRDs[f.jsonPath] = append(fieldToCRDs[f.jsonPath], crd.kind) + } + } + + // CommonSplunkSpec fields should appear in all 6 CRDs + commonFields := []string{"image", "licenseUrl", "defaults", "clusterManagerRef"} + for _, cf := range commonFields { + crds, ok := fieldToCRDs[cf] + if !ok { + t.Errorf("expected common field %q to be discovered", cf) + continue + } + if len(crds) != 6 { + t.Errorf("common field %q found in %d CRDs, expected 6: %v", cf, len(crds), crds) + } + } + + // SmartStore fields should only appear in Standalone and ClusterManager + ssFields := []string{"smartstore.cacheManager.evictionPolicy", "smartstore.indexes.name"} + for _, sf := range ssFields { + crds := fieldToCRDs[sf] + if len(crds) != 2 { + t.Errorf("smartstore field %q found in %d CRDs, expected 2: %v", sf, len(crds), crds) + } + for _, c := range crds { + if c != "Standalone" && c != "ClusterManager" { + t.Errorf("smartstore field %q unexpectedly in CRD %s", sf, c) + } + } + } + + // appRepo fields: present in Standalone, ClusterManager, SearchHeadCluster, LicenseManager, MonitoringConsole (5) + appFields := []string{"appRepo.appsRepoPollIntervalSeconds"} + for _, af := range appFields { + crds := fieldToCRDs[af] + if len(crds) != 5 { + t.Errorf("appRepo field %q found in %d CRDs, expected 5: %v", af, len(crds), crds) + } + } + + // replicas should appear in Standalone, IndexerCluster, SearchHeadCluster (3) + crds := fieldToCRDs["replicas"] + if len(crds) != 3 { + t.Errorf("replicas found in %d CRDs, expected 3: %v", len(crds), crds) + } + + // Deployer fields only in SearchHeadCluster + for _, df := range []string{"deployerResourceSpec", "deployerNodeAffinity"} { + crds := fieldToCRDs[df] + if len(crds) != 1 || crds[0] != "SearchHeadCluster" { + t.Errorf("field %q expected only in SearchHeadCluster, got: %v", df, crds) + } + } + + fmt.Printf("Integration check passed: %d unique field paths across %d CRDs\n", len(fieldToCRDs), len(registry)) +} From 3b94f402dc25eee302098378c153d244bbccde12 Mon Sep 17 00:00:00 2001 From: Michael Marod Date: Wed, 18 Feb 2026 14:34:21 -0500 Subject: [PATCH 2/3] make the mappings dynamic --- api/v4/common_types.go | 44 +-- tools/spec-config-mapping/main.go | 156 ++++---- tools/spec-config-mapping/mappings.go | 400 --------------------- tools/spec-config-mapping/mappings_test.go | 263 ++++++-------- 4 files changed, 205 insertions(+), 658 deletions(-) delete mode 100644 tools/spec-config-mapping/mappings.go diff --git a/api/v4/common_types.go b/api/v4/common_types.go index 5bba9c0cd..ac11cb8d8 100644 --- a/api/v4/common_types.go +++ b/api/v4/common_types.go @@ -257,16 +257,16 @@ type StorageClassSpec struct { // SmartStoreSpec defines Splunk indexes and remote storage volume configuration type SmartStoreSpec struct { // List of remote storage volumes - VolList []VolumeSpec `json:"volumes,omitempty"` + VolList []VolumeSpec `json:"volumes,omitempty" splunkconf:"indexes.conf,volume:"` // List of Splunk indexes - IndexList []IndexSpec `json:"indexes,omitempty"` + IndexList []IndexSpec `json:"indexes,omitempty" splunkconf:"indexes.conf,"` // Default configuration for indexes - Defaults IndexConfDefaultsSpec `json:"defaults,omitempty"` + Defaults IndexConfDefaultsSpec `json:"defaults,omitempty" splunkconf:"indexes.conf,default"` // Defines Cache manager settings - CacheManagerConf CacheManagerSpec `json:"cacheManager,omitempty"` + CacheManagerConf CacheManagerSpec `json:"cacheManager,omitempty" splunkconf:"server.conf,cachemanager"` } // CacheManagerSpec defines cachemanager specific configuration @@ -274,19 +274,19 @@ type CacheManagerSpec struct { IndexAndCacheManagerCommonSpec `json:",inline"` // Eviction policy to use - EvictionPolicy string `json:"evictionPolicy,omitempty"` + EvictionPolicy string `json:"evictionPolicy,omitempty" splunkconf:"eviction_policy"` // Max cache size per partition - MaxCacheSizeMB uint `json:"maxCacheSize,omitempty"` + MaxCacheSizeMB uint `json:"maxCacheSize,omitempty" splunkconf:"max_cache_size"` // Additional size beyond 'minFreeSize' before eviction kicks in - EvictionPaddingSizeMB uint `json:"evictionPadding,omitempty"` + EvictionPaddingSizeMB uint `json:"evictionPadding,omitempty" splunkconf:"eviction_padding"` // Maximum number of buckets that can be downloaded from remote storage in parallel - MaxConcurrentDownloads uint `json:"maxConcurrentDownloads,omitempty"` + MaxConcurrentDownloads uint `json:"maxConcurrentDownloads,omitempty" splunkconf:"max_concurrent_downloads"` // Maximum number of buckets that can be uploaded to remote storage in parallel - MaxConcurrentUploads uint `json:"maxConcurrentUploads,omitempty"` + MaxConcurrentUploads uint `json:"maxConcurrentUploads,omitempty" splunkconf:"max_concurrent_uploads"` } // IndexConfDefaultsSpec defines Splunk indexes.conf global/defaults @@ -300,22 +300,22 @@ type VolumeSpec struct { Name string `json:"name"` // Remote volume URI - Endpoint string `json:"endpoint"` + Endpoint string `json:"endpoint" splunkconf:"remote.s3.endpoint"` // Remote volume path - Path string `json:"path"` + Path string `json:"path" splunkconf:"path"` // Secret object name - SecretRef string `json:"secretRef"` + SecretRef string `json:"secretRef" splunkconf:"remote.s3.access_key;remote.s3.secret_key"` // Remote Storage type. Supported values: s3, blob, gcs. s3 works with aws or minio providers, whereas blob works with azure provider, gcs works for gcp. - Type string `json:"storageType"` + Type string `json:"storageType" splunkconf:"storageType"` // App Package Remote Store provider. Supported values: aws, minio, azure, gcp. Provider string `json:"provider"` // Region of the remote storage volume where apps reside. Used for aws, if provided. Not used for minio and azure. - Region string `json:"region"` + Region string `json:"region" splunkconf:"remote.s3.auth_region"` } // VolumeAndTypeSpec used to add any custom varaibles for volume implementation @@ -329,7 +329,7 @@ type IndexSpec struct { Name string `json:"name"` // Index location relative to the remote volume path - RemotePath string `json:"remotePath,omitempty"` + RemotePath string `json:"remotePath,omitempty" splunkconf:"remotePath"` IndexAndCacheManagerCommonSpec `json:",inline"` @@ -340,22 +340,22 @@ type IndexSpec struct { type IndexAndGlobalCommonSpec struct { // Remote Volume name - VolName string `json:"volumeName,omitempty"` + VolName string `json:"volumeName,omitempty" splunkconf:"remotePath"` // MaxGlobalDataSizeMB defines the maximum amount of space for warm and cold buckets of an index - MaxGlobalDataSizeMB uint `json:"maxGlobalDataSizeMB,omitempty"` + MaxGlobalDataSizeMB uint `json:"maxGlobalDataSizeMB,omitempty" splunkconf:"maxGlobalDataSizeMB"` // MaxGlobalDataSizeMB defines the maximum amount of cumulative space for warm and cold buckets of an index - MaxGlobalRawDataSizeMB uint `json:"maxGlobalRawDataSizeMB,omitempty"` + MaxGlobalRawDataSizeMB uint `json:"maxGlobalRawDataSizeMB,omitempty" splunkconf:"maxGlobalRawDataSizeMB"` } // IndexAndCacheManagerCommonSpec defines configurations that can be configured at index level or at server level type IndexAndCacheManagerCommonSpec struct { // Time period relative to the bucket's age, during which the bucket is protected from cache eviction - HotlistRecencySecs uint `json:"hotlistRecencySecs,omitempty"` + HotlistRecencySecs uint `json:"hotlistRecencySecs,omitempty" splunkconf:"hotlist_recency_secs"` // Time period relative to the bucket's age, during which the bloom filter file is protected from cache eviction - HotlistBloomFilterRecencyHours uint `json:"hotlistBloomFilterRecencyHours,omitempty"` + HotlistBloomFilterRecencyHours uint `json:"hotlistBloomFilterRecencyHours,omitempty" splunkconf:"hotlist_bloom_filter_recency_hours"` } // AppSourceDefaultSpec defines config common for defaults and App Sources @@ -381,7 +381,7 @@ type PremiumAppsProps struct { // Enterpreise Security App defaults // +optional - EsDefaults EsDefaults `json:"esDefaults,omitempty"` + EsDefaults EsDefaults `json:"esDefaults,omitempty" splunkconf:"web.conf,settings"` } // EsDefaults captures defaults for the Enterprise Security App @@ -397,7 +397,7 @@ type EsDefaults struct { // ignore: Ignores whether SSL is enabled or disabled. // // +optional - SslEnablement string `json:"sslEnablement,omitempty"` + SslEnablement string `json:"sslEnablement,omitempty" splunkconf:"enableSplunkWebSSL"` } // AppSourceSpec defines list of App package (*.spl, *.tgz) locations on remote volumes diff --git a/tools/spec-config-mapping/main.go b/tools/spec-config-mapping/main.go index 7d2e5b0a7..4c6453f22 100644 --- a/tools/spec-config-mapping/main.go +++ b/tools/spec-config-mapping/main.go @@ -21,7 +21,6 @@ import ( "fmt" "os" "reflect" - "sort" "strings" "time" @@ -32,11 +31,11 @@ import ( // Manifest is the top-level output structure. type Manifest struct { - Version string `json:"version"` - GeneratedAt string `json:"generatedAt"` - OperatorVersion string `json:"operatorVersion"` - APIVersion string `json:"apiVersion"` - CRDs map[string]CRDMapping `json:"crds"` + Version string `json:"version"` + GeneratedAt string `json:"generatedAt"` + OperatorVersion string `json:"operatorVersion"` + APIVersion string `json:"apiVersion"` + CRDs map[string]CRDMapping `json:"crds"` } // CRDMapping holds the field mappings for one CRD kind. @@ -46,15 +45,13 @@ type CRDMapping struct { // FieldMapping is the per-field entry in the JSON output. type FieldMapping struct { - JSONPath string `json:"jsonPath"` - GoType string `json:"goType"` - ConfFile string `json:"confFile"` - Stanza string `json:"stanza"` - Key string `json:"key"` - Description string `json:"description"` + JSONPath string `json:"jsonPath"` + GoType string `json:"goType"` + ConfFile string `json:"confFile"` + Stanza string `json:"stanza"` + Key string `json:"key"` } - // ── CRD registry ────────────────────────────────────────────────────── type crdEntry struct { @@ -73,18 +70,20 @@ func crdRegistry() []crdEntry { } } -// ── Reflection walker ───────────────────────────────────────────────── +// ── confContext tracks inherited confFile and stanza from parent tags ── -// fieldInfo holds metadata about a discovered struct field. -type fieldInfo struct { - jsonPath string - goType string +type confContext struct { + confFile string + stanza string } -// WalkSpecFields recursively walks a struct type and collects leaf field -// JSON paths. Embedded structs are flattened (their prefix is inherited). -// Slice and struct fields are recursed into; primitive fields are leaves. -func WalkSpecFields(t reflect.Type, prefix string) []fieldInfo { +// ── Reflection walker ───────────────────────────────────────────────── + +// WalkConfTags recursively walks a struct type and collects fields that +// have a `splunkconf` struct tag. Parent struct/slice fields with a +// 2-part tag ("confFile,stanza") set the context for child leaf fields +// with a 1-part tag ("key"). +func WalkConfTags(t reflect.Type, prefix string, ctx confContext) map[string]FieldMapping { if t.Kind() == reflect.Ptr { t = t.Elem() } @@ -92,7 +91,8 @@ func WalkSpecFields(t reflect.Type, prefix string) []fieldInfo { return nil } - var fields []fieldInfo + result := make(map[string]FieldMapping) + for i := 0; i < t.NumField(); i++ { sf := t.Field(i) @@ -103,9 +103,11 @@ func WalkSpecFields(t reflect.Type, prefix string) []fieldInfo { } jsonName := strings.Split(jsonTag, ",")[0] - // Handle embedded (anonymous) structs: flatten, keep parent prefix + // Handle embedded (anonymous) structs: flatten, keep parent prefix and context if sf.Anonymous { - fields = append(fields, WalkSpecFields(sf.Type, prefix)...) + for k, v := range WalkConfTags(sf.Type, prefix, ctx) { + result[k] = v + } continue } @@ -118,6 +120,9 @@ func WalkSpecFields(t reflect.Type, prefix string) []fieldInfo { fullPath = prefix + "." + jsonName } + // Check for splunkconf tag + splunkTag := sf.Tag.Get("splunkconf") + ft := sf.Type if ft.Kind() == reflect.Ptr { ft = ft.Elem() @@ -125,46 +130,63 @@ func WalkSpecFields(t reflect.Type, prefix string) []fieldInfo { switch ft.Kind() { case reflect.Struct: - // Check if this is a well-known Kubernetes type we treat as a leaf if isLeafType(ft) { - fields = append(fields, fieldInfo{jsonPath: fullPath, goType: ft.String()}) - } else { - fields = append(fields, WalkSpecFields(ft, fullPath)...) + // Opaque k8s type — skip + continue + } + // If this struct field has a 2-part splunkconf tag, it sets context for children + childCtx := ctx + if parts := strings.SplitN(splunkTag, ",", 2); len(parts) == 2 { + childCtx = confContext{confFile: parts[0], stanza: parts[1]} } + for k, v := range WalkConfTags(ft, fullPath, childCtx) { + result[k] = v + } + case reflect.Slice: elemType := ft.Elem() if elemType.Kind() == reflect.Ptr { elemType = elemType.Elem() } if elemType.Kind() == reflect.Struct && !isLeafType(elemType) { - // Recurse into the element type (represents array items) - fields = append(fields, WalkSpecFields(elemType, fullPath)...) - } else { - fields = append(fields, fieldInfo{jsonPath: fullPath, goType: ft.String()}) + childCtx := ctx + if parts := strings.SplitN(splunkTag, ",", 2); len(parts) == 2 { + childCtx = confContext{confFile: parts[0], stanza: parts[1]} + } + for k, v := range WalkConfTags(elemType, fullPath, childCtx) { + result[k] = v + } } + default: - fields = append(fields, fieldInfo{jsonPath: fullPath, goType: ft.String()}) + // Leaf field — only include if it has a splunkconf key tag + if splunkTag == "" { + continue + } + if ctx.confFile == "" || ctx.stanza == "" { + continue + } + result[fullPath] = FieldMapping{ + JSONPath: "spec." + fullPath, + GoType: ft.String(), + ConfFile: ctx.confFile, + Stanza: ctx.stanza, + Key: splunkTag, + } } } - return fields + return result } -// isLeafType returns true for complex types we don't want to recurse into -// (Kubernetes API types, etc.). +// isLeafType returns true for complex types we don't want to recurse into. func isLeafType(t reflect.Type) bool { - pkg := t.PkgPath() - // Treat all k8s.io types as opaque leaves - if strings.HasPrefix(pkg, "k8s.io/") { - return true - } - return false + return strings.HasPrefix(t.PkgPath(), "k8s.io/") } // ── Main ────────────────────────────────────────────────────────────── func main() { outPath := flag.String("o", "spec-config-mapping.json", "Output JSON file path") - strict := flag.Bool("strict", true, "Exit non-zero if any spec fields are unmapped") flag.Parse() registry := crdRegistry() @@ -176,40 +198,12 @@ func main() { CRDs: make(map[string]CRDMapping), } - var unmapped []string - for _, crd := range registry { - fields := WalkSpecFields(crd.specType, "") - specFields := make(map[string]FieldMapping) - - for _, f := range fields { - target, ok := fieldMappings[f.jsonPath] - if !ok { - unmapped = append(unmapped, fmt.Sprintf("%s.%s", crd.kind, f.jsonPath)) - continue - } - - // Only include fields with a full conf coordinate (confFile + stanza + key) - if target.ConfFile == "" || target.ConfFile == "env" || target.Stanza == "" || target.Key == "" { - continue - } - - specFields[f.jsonPath] = FieldMapping{ - JSONPath: "spec." + f.jsonPath, - GoType: f.goType, - ConfFile: target.ConfFile, - Stanza: target.Stanza, - Key: target.Key, - Description: target.Description, - } - - } - - manifest.CRDs[crd.kind] = CRDMapping{SpecFields: specFields} + fields := WalkConfTags(crd.specType, "", confContext{}) + manifest.CRDs[crd.kind] = CRDMapping{SpecFields: fields} } - // Write output using Encoder so we can disable HTML escaping - // (json.MarshalIndent escapes <> as \u003c/\u003e by default) + // Write output f, err := os.Create(*outPath) if err != nil { fmt.Fprintf(os.Stderr, "error: failed to create %s: %v\n", *outPath, err) @@ -232,16 +226,4 @@ func main() { } fmt.Fprintf(os.Stderr, "Generated %s: %d CRDs, %d total field mappings\n", *outPath, len(manifest.CRDs), totalFields) - - if len(unmapped) > 0 { - fmt.Fprintf(os.Stderr, "\nWARNING: %d unmapped field(s):\n", len(unmapped)) - sort.Strings(unmapped) - for _, u := range unmapped { - fmt.Fprintf(os.Stderr, " - %s\n", u) - } - if *strict { - fmt.Fprintf(os.Stderr, "\nAdd mappings to tools/spec-config-mapping/mappings.go to fix this.\n") - os.Exit(1) - } - } } diff --git a/tools/spec-config-mapping/mappings.go b/tools/spec-config-mapping/mappings.go deleted file mode 100644 index da1004b17..000000000 --- a/tools/spec-config-mapping/mappings.go +++ /dev/null @@ -1,400 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -// ConfigTarget describes how a CRD spec field maps to a Splunk configuration. -type ConfigTarget struct { - // Which Splunk conf file this maps to: "indexes.conf", "server.conf", - // "env" for environment variables, or "" for kubernetes-only fields. - ConfFile string `json:"confFile"` - - // The stanza in the conf file (e.g. "cachemanager", "volume:"). - Stanza string `json:"stanza,omitempty"` - - // The key within the stanza (e.g. "eviction_policy"). - Key string `json:"key,omitempty"` - - // For env var mappings, the environment variable name. - EnvVar string `json:"envVar,omitempty"` - - // Human-readable description of how this field is consumed. - Description string `json:"description"` -} - - -// fieldMappings maps each CRD spec field JSON path to its Splunk config target. -// Maintainers: update this table when CRD spec fields are added or removed. -// The drift detection test in mappings_test.go will fail if a field is missing. -var fieldMappings = map[string]ConfigTarget{ - // ── CommonSplunkSpec fields (shared by all CRDs) ────────────────── - - // Container / image - "image": {Description: "Docker image for Splunk pod containers"}, - "imagePullPolicy": {Description: "Image pull policy (Always or IfNotPresent)"}, - - // Scheduling - "schedulerName": {Description: "Kubernetes scheduler name for pod placement"}, - "affinity": {Description: "Kubernetes affinity rules for pod scheduling"}, - "tolerations": {Description: "Kubernetes tolerations for node taints"}, - "topologySpreadConstraints": {Description: "Kubernetes topology spread constraints"}, - - // Resources - "resources": {Description: "CPU and memory requests/limits for pod containers"}, - "serviceTemplate": {Description: "Template for Kubernetes Service customization (ports, LB config)"}, - - // Storage — etcVolumeStorageConfig (StorageClassSpec sub-fields) - "etcVolumeStorageConfig.storageClassName": {Description: "StorageClass name for /opt/splunk/etc PVC"}, - "etcVolumeStorageConfig.storageCapacity": {Description: "Storage capacity for /opt/splunk/etc PVC (default: 10Gi)"}, - "etcVolumeStorageConfig.ephemeralStorage": {Description: "Use emptyDir instead of PVC for /opt/splunk/etc"}, - - // Storage — varVolumeStorageConfig (StorageClassSpec sub-fields) - "varVolumeStorageConfig.storageClassName": {Description: "StorageClass name for /opt/splunk/var PVC"}, - "varVolumeStorageConfig.storageCapacity": {Description: "Storage capacity for /opt/splunk/var PVC (default: 100Gi)"}, - "varVolumeStorageConfig.ephemeralStorage": {Description: "Use emptyDir instead of PVC for /opt/splunk/var"}, - - // Volumes - "volumes": {Description: "Additional Kubernetes volumes mounted at /mnt/"}, - - // Splunk defaults (Ansible) - "defaults": { - ConfFile: "env", - EnvVar: "SPLUNK_DEFAULTS_URL", - Description: "Inline default.yml overrides; mounted at /mnt/splunk-defaults/default.yml and referenced via SPLUNK_DEFAULTS_URL", - }, - "defaultsUrl": { - ConfFile: "env", - EnvVar: "SPLUNK_DEFAULTS_URL", - Description: "External URL(s) for default.yml files; appended to SPLUNK_DEFAULTS_URL", - }, - "defaultsUrlApps": { - ConfFile: "env", - EnvVar: "SPLUNK_DEFAULTS_URL", - Description: "App-specific defaults URL(s); prepended to SPLUNK_DEFAULTS_URL (not applied to IDXC/SHC members)", - }, - - // License - "licenseUrl": { - ConfFile: "env", - EnvVar: "SPLUNK_LICENSE_URI", - Description: "URL to Splunk Enterprise license file", - }, - "licenseMasterRef": { - ConfFile: "env", - EnvVar: "SPLUNK_LICENSE_MASTER_URL", - Description: "(Deprecated v3 field) Reference to LicenseMaster; resolved to service FQDN", - }, - "licenseManagerRef": { - ConfFile: "env", - EnvVar: "SPLUNK_LICENSE_MASTER_URL", - Description: "Reference to LicenseManager; resolved to service FQDN", - }, - - // Cluster manager - "clusterMasterRef": { - ConfFile: "env", - EnvVar: "SPLUNK_CLUSTER_MASTER_URL", - Description: "(Deprecated v3 field) Reference to ClusterMaster; resolved to service FQDN or 'localhost' for CM", - }, - "clusterManagerRef": { - ConfFile: "env", - EnvVar: "SPLUNK_CLUSTER_MASTER_URL", - Description: "Reference to ClusterManager; resolved to service FQDN or 'localhost' for CM", - }, - - // Monitoring console - "monitoringConsoleRef": { - ConfFile: "env", - EnvVar: "SPLUNK_MONITORING_CONSOLE_REF", - Description: "Reference to MonitoringConsole; passed as CR name to register instance", - }, - - // Service account - "serviceAccount": {Description: "Kubernetes ServiceAccount for pod identity"}, - - // Extra environment - "extraEnv": { - ConfFile: "env", - Description: "Additional environment variables passed to Splunk containers; may override operator-set vars", - }, - - // Probes — top-level delay overrides - "readinessInitialDelaySeconds": {Description: "Initial delay for readiness probe"}, - "livenessInitialDelaySeconds": {Description: "Initial delay for liveness probe"}, - - // Probes — livenessProbe (Probe sub-fields) - "livenessProbe.initialDelaySeconds": {Description: "Liveness probe initial delay seconds"}, - "livenessProbe.timeoutSeconds": {Description: "Liveness probe timeout seconds"}, - "livenessProbe.periodSeconds": {Description: "Liveness probe period seconds"}, - "livenessProbe.failureThreshold": {Description: "Liveness probe failure threshold"}, - - // Probes — readinessProbe (Probe sub-fields) - "readinessProbe.initialDelaySeconds": {Description: "Readiness probe initial delay seconds"}, - "readinessProbe.timeoutSeconds": {Description: "Readiness probe timeout seconds"}, - "readinessProbe.periodSeconds": {Description: "Readiness probe period seconds"}, - "readinessProbe.failureThreshold": {Description: "Readiness probe failure threshold"}, - - // Probes — startupProbe (Probe sub-fields) - "startupProbe.initialDelaySeconds": {Description: "Startup probe initial delay seconds"}, - "startupProbe.timeoutSeconds": {Description: "Startup probe timeout seconds"}, - "startupProbe.periodSeconds": {Description: "Startup probe period seconds"}, - "startupProbe.failureThreshold": {Description: "Startup probe failure threshold"}, - - // Image pull secrets - "imagePullSecrets": {Description: "Secrets for pulling images from private registries"}, - - // Mock (internal) - "Mock": {Description: "Internal flag to differentiate unit tests from actual reconciliation"}, - - // ── Standalone-specific fields ──────────────────────────────────── - - "replicas": {Description: "Number of pods (StatefulSet replica count)"}, - - // ── SearchHeadCluster-specific fields ───────────────────────────── - - "deployerResourceSpec": {Description: "Resource requirements for the SHC Deployer pod"}, - "deployerNodeAffinity": {Description: "Node affinity rules for the SHC Deployer pod"}, - - // ── SmartStore fields (Standalone, ClusterManager) ──────────────── - - // smartstore.volumes[] - "smartstore.volumes.name": { - ConfFile: "indexes.conf", - Stanza: "volume:", - Description: "Remote storage volume name; becomes stanza name in indexes.conf", - }, - "smartstore.volumes.endpoint": { - ConfFile: "indexes.conf", - Stanza: "volume:", - Key: "remote.s3.endpoint", - Description: "Remote storage endpoint URL", - }, - "smartstore.volumes.path": { - ConfFile: "indexes.conf", - Stanza: "volume:", - Key: "path", - Description: "Remote volume path; rendered as s3://", - }, - "smartstore.volumes.secretRef": { - ConfFile: "indexes.conf", - Stanza: "volume:", - Key: "remote.s3.access_key, remote.s3.secret_key", - Description: "Kubernetes Secret containing remote storage credentials", - }, - "smartstore.volumes.storageType": { - ConfFile: "indexes.conf", - Stanza: "volume:", - Key: "storageType", - Description: "Remote storage type (s3, blob, gcs); always rendered as 'remote'", - }, - "smartstore.volumes.provider": { - ConfFile: "indexes.conf", - Stanza: "volume:", - Description: "Storage provider (aws, minio, azure, gcp); used for credential handling", - }, - "smartstore.volumes.region": { - ConfFile: "indexes.conf", - Stanza: "volume:", - Key: "remote.s3.auth_region", - Description: "AWS region for remote storage volume", - }, - - // smartstore.indexes[] - "smartstore.indexes.name": { - ConfFile: "indexes.conf", - Stanza: "", - Description: "Splunk index name; becomes stanza name in indexes.conf", - }, - "smartstore.indexes.remotePath": { - ConfFile: "indexes.conf", - Stanza: "", - Key: "remotePath", - Description: "Index location relative to volume path; rendered as volume:/", - }, - "smartstore.indexes.volumeName": { - ConfFile: "indexes.conf", - Stanza: "", - Key: "remotePath", - Description: "Volume name for index remote path; combined with remotePath", - }, - "smartstore.indexes.hotlistRecencySecs": { - ConfFile: "indexes.conf", - Stanza: "", - Key: "hotlist_recency_secs", - Description: "Time period (seconds) during which bucket is protected from cache eviction", - }, - "smartstore.indexes.hotlistBloomFilterRecencyHours": { - ConfFile: "indexes.conf", - Stanza: "", - Key: "hotlist_bloom_filter_recency_hours", - Description: "Time period (hours) during which bloom filter is protected from cache eviction", - }, - "smartstore.indexes.maxGlobalDataSizeMB": { - ConfFile: "indexes.conf", - Stanza: "", - Key: "maxGlobalDataSizeMB", - Description: "Maximum space for warm and cold buckets of an index", - }, - "smartstore.indexes.maxGlobalRawDataSizeMB": { - ConfFile: "indexes.conf", - Stanza: "", - Key: "maxGlobalRawDataSizeMB", - Description: "Maximum cumulative space for warm and cold buckets of an index", - }, - - // smartstore.defaults - "smartstore.defaults.volumeName": { - ConfFile: "indexes.conf", - Stanza: "default", - Key: "remotePath", - Description: "Default volume for all indexes; rendered as remotePath = volume:/$_index_name", - }, - "smartstore.defaults.maxGlobalDataSizeMB": { - ConfFile: "indexes.conf", - Stanza: "default", - Key: "maxGlobalDataSizeMB", - Description: "Default maximum space for warm and cold buckets", - }, - "smartstore.defaults.maxGlobalRawDataSizeMB": { - ConfFile: "indexes.conf", - Stanza: "default", - Key: "maxGlobalRawDataSizeMB", - Description: "Default maximum cumulative space for warm and cold buckets", - }, - - // smartstore.cacheManager - "smartstore.cacheManager.hotlistRecencySecs": { - ConfFile: "server.conf", - Stanza: "cachemanager", - Key: "hotlist_recency_secs", - Description: "Global cache: time period (seconds) bucket is protected from eviction", - }, - "smartstore.cacheManager.hotlistBloomFilterRecencyHours": { - ConfFile: "server.conf", - Stanza: "cachemanager", - Key: "hotlist_bloom_filter_recency_hours", - Description: "Global cache: time period (hours) bloom filter is protected from eviction", - }, - "smartstore.cacheManager.evictionPolicy": { - ConfFile: "server.conf", - Stanza: "cachemanager", - Key: "eviction_policy", - Description: "Cache eviction policy (e.g. lru)", - }, - "smartstore.cacheManager.maxCacheSize": { - ConfFile: "server.conf", - Stanza: "cachemanager", - Key: "max_cache_size", - Description: "Maximum cache size in MB", - }, - "smartstore.cacheManager.evictionPadding": { - ConfFile: "server.conf", - Stanza: "cachemanager", - Key: "eviction_padding", - Description: "Additional size beyond minFreeSize before eviction kicks in", - }, - "smartstore.cacheManager.maxConcurrentDownloads": { - ConfFile: "server.conf", - Stanza: "cachemanager", - Key: "max_concurrent_downloads", - Description: "Maximum parallel bucket downloads from remote storage", - }, - "smartstore.cacheManager.maxConcurrentUploads": { - ConfFile: "server.conf", - Stanza: "cachemanager", - Key: "max_concurrent_uploads", - Description: "Maximum parallel bucket uploads to remote storage", - }, - - // ── App Framework fields ────────────────────────────────────────── - - // appRepo.defaults - "appRepo.defaults.volumeName": { - Description: "Default remote storage volume name for app sources", - }, - "appRepo.defaults.scope": { - Description: "Default app deployment scope: local, cluster, clusterWithPreConfig, or premiumApps", - }, - "appRepo.defaults.premiumAppsProps.type": { - Description: "Premium app type (e.g. enterpriseSecurity)", - }, - "appRepo.defaults.premiumAppsProps.esDefaults.sslEnablement": { - ConfFile: "web.conf", - Stanza: "settings", - Key: "enableSplunkWebSSL", - Description: "SSL enablement mode for Enterprise Security app (strict, auto, ignore)", - }, - - // appRepo.volumes[] - "appRepo.volumes.name": { - Description: "App repository remote volume name", - }, - "appRepo.volumes.endpoint": { - Description: "App repository remote storage endpoint URL", - }, - "appRepo.volumes.path": { - Description: "App repository remote volume path (S3 bucket / Azure container)", - }, - "appRepo.volumes.secretRef": { - Description: "Kubernetes Secret for app repository remote storage credentials", - }, - "appRepo.volumes.storageType": { - Description: "App repository storage type (s3, blob, gcs)", - }, - "appRepo.volumes.provider": { - Description: "App repository storage provider (aws, minio, azure, gcp)", - }, - "appRepo.volumes.region": { - Description: "AWS region for app repository remote storage", - }, - - // appRepo.appSources[] - "appRepo.appSources.name": { - Description: "Logical name for a set of apps at a remote location", - }, - "appRepo.appSources.location": { - Description: "Path relative to volume where app packages reside", - }, - "appRepo.appSources.volumeName": { - Description: "Remote storage volume name for this app source", - }, - "appRepo.appSources.scope": { - Description: "App deployment scope: local (per-pod), cluster (bundle push), clusterWithPreConfig, premiumApps", - }, - "appRepo.appSources.premiumAppsProps.type": { - Description: "Premium app type for this app source", - }, - "appRepo.appSources.premiumAppsProps.esDefaults.sslEnablement": { - ConfFile: "web.conf", - Stanza: "settings", - Key: "enableSplunkWebSSL", - Description: "SSL enablement for Enterprise Security in this app source", - }, - - // appRepo top-level settings - "appRepo.appsRepoPollIntervalSeconds": { - Description: "Interval in seconds to poll remote storage for app changes (0 = disabled)", - }, - "appRepo.appInstallPeriodSeconds": { - Description: "Time window in seconds within a reconcile cycle to install apps", - }, - "appRepo.installMaxRetries": { - Description: "Maximum retry attempts for failed app installations", - }, - "appRepo.maxConcurrentAppDownloads": { - Description: "Maximum number of apps downloaded in parallel", - }, -} - diff --git a/tools/spec-config-mapping/mappings_test.go b/tools/spec-config-mapping/mappings_test.go index 79ea9152a..4842103f8 100644 --- a/tools/spec-config-mapping/mappings_test.go +++ b/tools/spec-config-mapping/mappings_test.go @@ -16,195 +16,160 @@ package main import ( - "fmt" - "sort" - "strings" "testing" ) -// TestAllFieldsHaveMappings verifies that every CRD spec field discovered -// via reflection has a corresponding entry in fieldMappings. -func TestAllFieldsHaveMappings(t *testing.T) { +// TestStandaloneSmartStoreFields verifies SmartStore fields are discovered +// with correct conf coordinates for Standalone. +func TestStandaloneSmartStoreFields(t *testing.T) { registry := crdRegistry() - allDiscovered := make(map[string]bool) for _, crd := range registry { - fields := WalkSpecFields(crd.specType, "") - for _, f := range fields { - allDiscovered[f.jsonPath] = true - if _, ok := fieldMappings[f.jsonPath]; !ok { - t.Errorf("CRD %s: field %q has no entry in fieldMappings", crd.kind, f.jsonPath) - } - } - } - - // Also check for stale entries: mapping keys that don't match any discovered field - var stale []string - for key := range fieldMappings { - if !allDiscovered[key] { - stale = append(stale, key) - } - } - if len(stale) > 0 { - sort.Strings(stale) - t.Errorf("Stale mapping entries (not found in any CRD spec):\n %s", strings.Join(stale, "\n ")) - } -} - -// TestNoDuplicateFieldPaths ensures there are no duplicate field paths -// within a single CRD's discovered fields. -func TestNoDuplicateFieldPaths(t *testing.T) { - registry := crdRegistry() - for _, crd := range registry { - fields := WalkSpecFields(crd.specType, "") - seen := make(map[string]int) - for _, f := range fields { - seen[f.jsonPath]++ + if crd.kind != "Standalone" { + continue } - for path, count := range seen { - if count > 1 { - t.Errorf("CRD %s: field path %q discovered %d times (expected 1)", crd.kind, path, count) + fields := WalkConfTags(crd.specType, "", confContext{}) + + expected := map[string]FieldMapping{ + "smartstore.cacheManager.evictionPolicy": { + ConfFile: "server.conf", Stanza: "cachemanager", Key: "eviction_policy", + }, + "smartstore.cacheManager.maxCacheSize": { + ConfFile: "server.conf", Stanza: "cachemanager", Key: "max_cache_size", + }, + "smartstore.cacheManager.hotlistRecencySecs": { + ConfFile: "server.conf", Stanza: "cachemanager", Key: "hotlist_recency_secs", + }, + "smartstore.indexes.remotePath": { + ConfFile: "indexes.conf", Stanza: "", Key: "remotePath", + }, + "smartstore.indexes.maxGlobalDataSizeMB": { + ConfFile: "indexes.conf", Stanza: "", Key: "maxGlobalDataSizeMB", + }, + "smartstore.volumes.endpoint": { + ConfFile: "indexes.conf", Stanza: "volume:", Key: "remote.s3.endpoint", + }, + "smartstore.defaults.volumeName": { + ConfFile: "indexes.conf", Stanza: "default", Key: "remotePath", + }, + } + + for path, want := range expected { + got, ok := fields[path] + if !ok { + t.Errorf("expected field %q not found in Standalone", path) + continue } - } - } -} - -// TestFieldMappingDescriptions ensures every mapping has a non-empty description. -func TestFieldMappingDescriptions(t *testing.T) { - for path, target := range fieldMappings { - if target.Description == "" { - t.Errorf("fieldMappings[%q] has an empty Description", path) - } - } -} - -// TestConfFileMappingsHaveStanza checks that entries with a non-empty ConfFile -// (other than "env" or "") have a non-empty Stanza. -func TestConfFileMappingsHaveStanza(t *testing.T) { - for path, target := range fieldMappings { - if target.ConfFile != "" && target.ConfFile != "env" { - if target.Stanza == "" { - t.Errorf("fieldMappings[%q] has confFile=%q but no Stanza", path, target.ConfFile) + if got.ConfFile != want.ConfFile { + t.Errorf("%s: confFile = %q, want %q", path, got.ConfFile, want.ConfFile) } - } - } -} - -// TestEnvMappingsHaveEnvVar checks that entries with ConfFile="env" have EnvVar set. -func TestEnvMappingsHaveEnvVar(t *testing.T) { - for path, target := range fieldMappings { - if target.ConfFile == "env" && target.EnvVar == "" { - // extraEnv is a special case - it's a pass-through for arbitrary env vars - if path == "extraEnv" { - continue + if got.Stanza != want.Stanza { + t.Errorf("%s: stanza = %q, want %q", path, got.Stanza, want.Stanza) + } + if got.Key != want.Key { + t.Errorf("%s: key = %q, want %q", path, got.Key, want.Key) } - t.Errorf("fieldMappings[%q] has confFile=env but no EnvVar", path) } } } -// TestWalkSpecFieldsOutput is a sanity check that the walker produces expected paths. -func TestWalkSpecFieldsOutput(t *testing.T) { +// TestContextInheritance verifies that shared embedded types inherit the +// correct context from different parents (e.g., hotlistRecencySecs maps +// to server.conf under cacheManager but indexes.conf under indexes). +func TestContextInheritance(t *testing.T) { registry := crdRegistry() - // Find the Standalone CRD and check some expected fields for _, crd := range registry { if crd.kind != "Standalone" { continue } - fields := WalkSpecFields(crd.specType, "") - paths := make(map[string]bool) - for _, f := range fields { - paths[f.jsonPath] = true - } + fields := WalkConfTags(crd.specType, "", confContext{}) + + cmField := fields["smartstore.cacheManager.hotlistRecencySecs"] + idxField := fields["smartstore.indexes.hotlistRecencySecs"] - expected := []string{ - "replicas", - "image", - "licenseUrl", - "smartstore.cacheManager.evictionPolicy", - "smartstore.indexes.name", - "smartstore.volumes.endpoint", - "appRepo.appsRepoPollIntervalSeconds", + if cmField.ConfFile != "server.conf" { + t.Errorf("cacheManager.hotlistRecencySecs: confFile = %q, want server.conf", cmField.ConfFile) } - for _, e := range expected { - if !paths[e] { - t.Errorf("Standalone: expected field path %q not found. Got paths: %v", e, sortedKeys(paths)) - } + if idxField.ConfFile != "indexes.conf" { + t.Errorf("indexes.hotlistRecencySecs: confFile = %q, want indexes.conf", idxField.ConfFile) + } + if cmField.Stanza != "cachemanager" { + t.Errorf("cacheManager.hotlistRecencySecs: stanza = %q, want cachemanager", cmField.Stanza) + } + if idxField.Stanza != "" { + t.Errorf("indexes.hotlistRecencySecs: stanza = %q, want ", idxField.Stanza) + } + if cmField.Key != idxField.Key { + t.Errorf("hotlistRecencySecs key mismatch: %q vs %q", cmField.Key, idxField.Key) } } } -func sortedKeys(m map[string]bool) []string { - keys := make([]string, 0, len(m)) - for k := range m { - keys = append(keys, k) - } - sort.Strings(keys) - return keys -} - -// TestGenerateManifestIntegration runs the full generation logic and checks the output structure. -func TestGenerateManifestIntegration(t *testing.T) { +// TestSmartStoreOnlyInCorrectCRDs verifies SmartStore fields only appear +// in Standalone and ClusterManager. +func TestSmartStoreOnlyInCorrectCRDs(t *testing.T) { registry := crdRegistry() - fieldToCRDs := make(map[string][]string) + ssField := "smartstore.cacheManager.evictionPolicy" for _, crd := range registry { - fields := WalkSpecFields(crd.specType, "") - for _, f := range fields { - fieldToCRDs[f.jsonPath] = append(fieldToCRDs[f.jsonPath], crd.kind) + fields := WalkConfTags(crd.specType, "", confContext{}) + _, found := fields[ssField] + + switch crd.kind { + case "Standalone", "ClusterManager": + if !found { + t.Errorf("%s should have SmartStore fields but %q not found", crd.kind, ssField) + } + default: + if found { + t.Errorf("%s should NOT have SmartStore fields but %q was found", crd.kind, ssField) + } } } +} - // CommonSplunkSpec fields should appear in all 6 CRDs - commonFields := []string{"image", "licenseUrl", "defaults", "clusterManagerRef"} - for _, cf := range commonFields { - crds, ok := fieldToCRDs[cf] - if !ok { - t.Errorf("expected common field %q to be discovered", cf) - continue - } - if len(crds) != 6 { - t.Errorf("common field %q found in %d CRDs, expected 6: %v", cf, len(crds), crds) - } +// TestNoUntaggedFieldsLeakThrough verifies that fields without a splunkconf +// tag do not appear in the output. +func TestNoUntaggedFieldsLeakThrough(t *testing.T) { + registry := crdRegistry() + + untaggedPaths := []string{ + "image", + "replicas", + "licenseUrl", + "affinity", + "resources", } - // SmartStore fields should only appear in Standalone and ClusterManager - ssFields := []string{"smartstore.cacheManager.evictionPolicy", "smartstore.indexes.name"} - for _, sf := range ssFields { - crds := fieldToCRDs[sf] - if len(crds) != 2 { - t.Errorf("smartstore field %q found in %d CRDs, expected 2: %v", sf, len(crds), crds) - } - for _, c := range crds { - if c != "Standalone" && c != "ClusterManager" { - t.Errorf("smartstore field %q unexpectedly in CRD %s", sf, c) + for _, crd := range registry { + fields := WalkConfTags(crd.specType, "", confContext{}) + for _, path := range untaggedPaths { + if _, found := fields[path]; found { + t.Errorf("%s: untagged field %q should not appear in output", crd.kind, path) } } } +} - // appRepo fields: present in Standalone, ClusterManager, SearchHeadCluster, LicenseManager, MonitoringConsole (5) - appFields := []string{"appRepo.appsRepoPollIntervalSeconds"} - for _, af := range appFields { - crds := fieldToCRDs[af] - if len(crds) != 5 { - t.Errorf("appRepo field %q found in %d CRDs, expected 5: %v", af, len(crds), crds) - } - } +// TestEsDefaultsMapping verifies the web.conf mapping from the app framework. +func TestEsDefaultsMapping(t *testing.T) { + registry := crdRegistry() - // replicas should appear in Standalone, IndexerCluster, SearchHeadCluster (3) - crds := fieldToCRDs["replicas"] - if len(crds) != 3 { - t.Errorf("replicas found in %d CRDs, expected 3: %v", len(crds), crds) - } + for _, crd := range registry { + if crd.kind != "Standalone" { + continue + } + fields := WalkConfTags(crd.specType, "", confContext{}) - // Deployer fields only in SearchHeadCluster - for _, df := range []string{"deployerResourceSpec", "deployerNodeAffinity"} { - crds := fieldToCRDs[df] - if len(crds) != 1 || crds[0] != "SearchHeadCluster" { - t.Errorf("field %q expected only in SearchHeadCluster, got: %v", df, crds) + path := "appRepo.defaults.premiumAppsProps.esDefaults.sslEnablement" + got, ok := fields[path] + if !ok { + t.Fatalf("expected field %q not found", path) + } + if got.ConfFile != "web.conf" || got.Stanza != "settings" || got.Key != "enableSplunkWebSSL" { + t.Errorf("%s: got %s/%s/%s, want web.conf/settings/enableSplunkWebSSL", + path, got.ConfFile, got.Stanza, got.Key) } } - - fmt.Printf("Integration check passed: %d unique field paths across %d CRDs\n", len(fieldToCRDs), len(registry)) } From 99a7c8b8f3cb4a8fb7008f829bfe789cae35c108 Mon Sep 17 00:00:00 2001 From: Michael Marod Date: Wed, 18 Feb 2026 14:38:36 -0500 Subject: [PATCH 3/3] Publish the manifest to the api/v4 dir --- .gitignore | 3 +- Makefile | 2 +- api/v4/spec-config-mapping.json | 395 ++++++++++++++++++++++++++++++++ 3 files changed, 397 insertions(+), 3 deletions(-) create mode 100644 api/v4/spec-config-mapping.json diff --git a/.gitignore b/.gitignore index 40a66e71e..4b6582d68 100644 --- a/.gitignore +++ b/.gitignore @@ -98,5 +98,4 @@ bundle_*/ test/secret/*.log kubeconfig .devcontainer/devcontainer.json -kuttl-artifacts/* -spec-config-mapping.json \ No newline at end of file +kuttl-artifacts/* \ No newline at end of file diff --git a/Makefile b/Makefile index 4eee1561b..0f7a041ca 100644 --- a/Makefile +++ b/Makefile @@ -140,7 +140,7 @@ test: manifests generate fmt vet envtest ## Run tests. .PHONY: spec-config-mapping spec-config-mapping: ## Generate CRD spec-to-Splunk-config mapping JSON. - go run ./tools/spec-config-mapping -o spec-config-mapping.json + go run ./tools/spec-config-mapping -o api/v4/spec-config-mapping.json ##@ Build diff --git a/api/v4/spec-config-mapping.json b/api/v4/spec-config-mapping.json new file mode 100644 index 000000000..746094406 --- /dev/null +++ b/api/v4/spec-config-mapping.json @@ -0,0 +1,395 @@ +{ + "version": "1.0.0", + "generatedAt": "2026-02-18T19:38:20Z", + "operatorVersion": "3.0.0", + "apiVersion": "enterprise.splunk.com/v4", + "crds": { + "ClusterManager": { + "specFields": { + "appRepo.appSources.premiumAppsProps.esDefaults.sslEnablement": { + "jsonPath": "spec.appRepo.appSources.premiumAppsProps.esDefaults.sslEnablement", + "goType": "string", + "confFile": "web.conf", + "stanza": "settings", + "key": "enableSplunkWebSSL" + }, + "appRepo.defaults.premiumAppsProps.esDefaults.sslEnablement": { + "jsonPath": "spec.appRepo.defaults.premiumAppsProps.esDefaults.sslEnablement", + "goType": "string", + "confFile": "web.conf", + "stanza": "settings", + "key": "enableSplunkWebSSL" + }, + "smartstore.cacheManager.evictionPadding": { + "jsonPath": "spec.smartstore.cacheManager.evictionPadding", + "goType": "uint", + "confFile": "server.conf", + "stanza": "cachemanager", + "key": "eviction_padding" + }, + "smartstore.cacheManager.evictionPolicy": { + "jsonPath": "spec.smartstore.cacheManager.evictionPolicy", + "goType": "string", + "confFile": "server.conf", + "stanza": "cachemanager", + "key": "eviction_policy" + }, + "smartstore.cacheManager.hotlistBloomFilterRecencyHours": { + "jsonPath": "spec.smartstore.cacheManager.hotlistBloomFilterRecencyHours", + "goType": "uint", + "confFile": "server.conf", + "stanza": "cachemanager", + "key": "hotlist_bloom_filter_recency_hours" + }, + "smartstore.cacheManager.hotlistRecencySecs": { + "jsonPath": "spec.smartstore.cacheManager.hotlistRecencySecs", + "goType": "uint", + "confFile": "server.conf", + "stanza": "cachemanager", + "key": "hotlist_recency_secs" + }, + "smartstore.cacheManager.maxCacheSize": { + "jsonPath": "spec.smartstore.cacheManager.maxCacheSize", + "goType": "uint", + "confFile": "server.conf", + "stanza": "cachemanager", + "key": "max_cache_size" + }, + "smartstore.cacheManager.maxConcurrentDownloads": { + "jsonPath": "spec.smartstore.cacheManager.maxConcurrentDownloads", + "goType": "uint", + "confFile": "server.conf", + "stanza": "cachemanager", + "key": "max_concurrent_downloads" + }, + "smartstore.cacheManager.maxConcurrentUploads": { + "jsonPath": "spec.smartstore.cacheManager.maxConcurrentUploads", + "goType": "uint", + "confFile": "server.conf", + "stanza": "cachemanager", + "key": "max_concurrent_uploads" + }, + "smartstore.defaults.maxGlobalDataSizeMB": { + "jsonPath": "spec.smartstore.defaults.maxGlobalDataSizeMB", + "goType": "uint", + "confFile": "indexes.conf", + "stanza": "default", + "key": "maxGlobalDataSizeMB" + }, + "smartstore.defaults.maxGlobalRawDataSizeMB": { + "jsonPath": "spec.smartstore.defaults.maxGlobalRawDataSizeMB", + "goType": "uint", + "confFile": "indexes.conf", + "stanza": "default", + "key": "maxGlobalRawDataSizeMB" + }, + "smartstore.defaults.volumeName": { + "jsonPath": "spec.smartstore.defaults.volumeName", + "goType": "string", + "confFile": "indexes.conf", + "stanza": "default", + "key": "remotePath" + }, + "smartstore.indexes.hotlistBloomFilterRecencyHours": { + "jsonPath": "spec.smartstore.indexes.hotlistBloomFilterRecencyHours", + "goType": "uint", + "confFile": "indexes.conf", + "stanza": "", + "key": "hotlist_bloom_filter_recency_hours" + }, + "smartstore.indexes.hotlistRecencySecs": { + "jsonPath": "spec.smartstore.indexes.hotlistRecencySecs", + "goType": "uint", + "confFile": "indexes.conf", + "stanza": "", + "key": "hotlist_recency_secs" + }, + "smartstore.indexes.maxGlobalDataSizeMB": { + "jsonPath": "spec.smartstore.indexes.maxGlobalDataSizeMB", + "goType": "uint", + "confFile": "indexes.conf", + "stanza": "", + "key": "maxGlobalDataSizeMB" + }, + "smartstore.indexes.maxGlobalRawDataSizeMB": { + "jsonPath": "spec.smartstore.indexes.maxGlobalRawDataSizeMB", + "goType": "uint", + "confFile": "indexes.conf", + "stanza": "", + "key": "maxGlobalRawDataSizeMB" + }, + "smartstore.indexes.remotePath": { + "jsonPath": "spec.smartstore.indexes.remotePath", + "goType": "string", + "confFile": "indexes.conf", + "stanza": "", + "key": "remotePath" + }, + "smartstore.indexes.volumeName": { + "jsonPath": "spec.smartstore.indexes.volumeName", + "goType": "string", + "confFile": "indexes.conf", + "stanza": "", + "key": "remotePath" + }, + "smartstore.volumes.endpoint": { + "jsonPath": "spec.smartstore.volumes.endpoint", + "goType": "string", + "confFile": "indexes.conf", + "stanza": "volume:", + "key": "remote.s3.endpoint" + }, + "smartstore.volumes.path": { + "jsonPath": "spec.smartstore.volumes.path", + "goType": "string", + "confFile": "indexes.conf", + "stanza": "volume:", + "key": "path" + }, + "smartstore.volumes.region": { + "jsonPath": "spec.smartstore.volumes.region", + "goType": "string", + "confFile": "indexes.conf", + "stanza": "volume:", + "key": "remote.s3.auth_region" + }, + "smartstore.volumes.secretRef": { + "jsonPath": "spec.smartstore.volumes.secretRef", + "goType": "string", + "confFile": "indexes.conf", + "stanza": "volume:", + "key": "remote.s3.access_key;remote.s3.secret_key" + }, + "smartstore.volumes.storageType": { + "jsonPath": "spec.smartstore.volumes.storageType", + "goType": "string", + "confFile": "indexes.conf", + "stanza": "volume:", + "key": "storageType" + } + } + }, + "IndexerCluster": { + "specFields": {} + }, + "LicenseManager": { + "specFields": { + "appRepo.appSources.premiumAppsProps.esDefaults.sslEnablement": { + "jsonPath": "spec.appRepo.appSources.premiumAppsProps.esDefaults.sslEnablement", + "goType": "string", + "confFile": "web.conf", + "stanza": "settings", + "key": "enableSplunkWebSSL" + }, + "appRepo.defaults.premiumAppsProps.esDefaults.sslEnablement": { + "jsonPath": "spec.appRepo.defaults.premiumAppsProps.esDefaults.sslEnablement", + "goType": "string", + "confFile": "web.conf", + "stanza": "settings", + "key": "enableSplunkWebSSL" + } + } + }, + "MonitoringConsole": { + "specFields": { + "appRepo.appSources.premiumAppsProps.esDefaults.sslEnablement": { + "jsonPath": "spec.appRepo.appSources.premiumAppsProps.esDefaults.sslEnablement", + "goType": "string", + "confFile": "web.conf", + "stanza": "settings", + "key": "enableSplunkWebSSL" + }, + "appRepo.defaults.premiumAppsProps.esDefaults.sslEnablement": { + "jsonPath": "spec.appRepo.defaults.premiumAppsProps.esDefaults.sslEnablement", + "goType": "string", + "confFile": "web.conf", + "stanza": "settings", + "key": "enableSplunkWebSSL" + } + } + }, + "SearchHeadCluster": { + "specFields": { + "appRepo.appSources.premiumAppsProps.esDefaults.sslEnablement": { + "jsonPath": "spec.appRepo.appSources.premiumAppsProps.esDefaults.sslEnablement", + "goType": "string", + "confFile": "web.conf", + "stanza": "settings", + "key": "enableSplunkWebSSL" + }, + "appRepo.defaults.premiumAppsProps.esDefaults.sslEnablement": { + "jsonPath": "spec.appRepo.defaults.premiumAppsProps.esDefaults.sslEnablement", + "goType": "string", + "confFile": "web.conf", + "stanza": "settings", + "key": "enableSplunkWebSSL" + } + } + }, + "Standalone": { + "specFields": { + "appRepo.appSources.premiumAppsProps.esDefaults.sslEnablement": { + "jsonPath": "spec.appRepo.appSources.premiumAppsProps.esDefaults.sslEnablement", + "goType": "string", + "confFile": "web.conf", + "stanza": "settings", + "key": "enableSplunkWebSSL" + }, + "appRepo.defaults.premiumAppsProps.esDefaults.sslEnablement": { + "jsonPath": "spec.appRepo.defaults.premiumAppsProps.esDefaults.sslEnablement", + "goType": "string", + "confFile": "web.conf", + "stanza": "settings", + "key": "enableSplunkWebSSL" + }, + "smartstore.cacheManager.evictionPadding": { + "jsonPath": "spec.smartstore.cacheManager.evictionPadding", + "goType": "uint", + "confFile": "server.conf", + "stanza": "cachemanager", + "key": "eviction_padding" + }, + "smartstore.cacheManager.evictionPolicy": { + "jsonPath": "spec.smartstore.cacheManager.evictionPolicy", + "goType": "string", + "confFile": "server.conf", + "stanza": "cachemanager", + "key": "eviction_policy" + }, + "smartstore.cacheManager.hotlistBloomFilterRecencyHours": { + "jsonPath": "spec.smartstore.cacheManager.hotlistBloomFilterRecencyHours", + "goType": "uint", + "confFile": "server.conf", + "stanza": "cachemanager", + "key": "hotlist_bloom_filter_recency_hours" + }, + "smartstore.cacheManager.hotlistRecencySecs": { + "jsonPath": "spec.smartstore.cacheManager.hotlistRecencySecs", + "goType": "uint", + "confFile": "server.conf", + "stanza": "cachemanager", + "key": "hotlist_recency_secs" + }, + "smartstore.cacheManager.maxCacheSize": { + "jsonPath": "spec.smartstore.cacheManager.maxCacheSize", + "goType": "uint", + "confFile": "server.conf", + "stanza": "cachemanager", + "key": "max_cache_size" + }, + "smartstore.cacheManager.maxConcurrentDownloads": { + "jsonPath": "spec.smartstore.cacheManager.maxConcurrentDownloads", + "goType": "uint", + "confFile": "server.conf", + "stanza": "cachemanager", + "key": "max_concurrent_downloads" + }, + "smartstore.cacheManager.maxConcurrentUploads": { + "jsonPath": "spec.smartstore.cacheManager.maxConcurrentUploads", + "goType": "uint", + "confFile": "server.conf", + "stanza": "cachemanager", + "key": "max_concurrent_uploads" + }, + "smartstore.defaults.maxGlobalDataSizeMB": { + "jsonPath": "spec.smartstore.defaults.maxGlobalDataSizeMB", + "goType": "uint", + "confFile": "indexes.conf", + "stanza": "default", + "key": "maxGlobalDataSizeMB" + }, + "smartstore.defaults.maxGlobalRawDataSizeMB": { + "jsonPath": "spec.smartstore.defaults.maxGlobalRawDataSizeMB", + "goType": "uint", + "confFile": "indexes.conf", + "stanza": "default", + "key": "maxGlobalRawDataSizeMB" + }, + "smartstore.defaults.volumeName": { + "jsonPath": "spec.smartstore.defaults.volumeName", + "goType": "string", + "confFile": "indexes.conf", + "stanza": "default", + "key": "remotePath" + }, + "smartstore.indexes.hotlistBloomFilterRecencyHours": { + "jsonPath": "spec.smartstore.indexes.hotlistBloomFilterRecencyHours", + "goType": "uint", + "confFile": "indexes.conf", + "stanza": "", + "key": "hotlist_bloom_filter_recency_hours" + }, + "smartstore.indexes.hotlistRecencySecs": { + "jsonPath": "spec.smartstore.indexes.hotlistRecencySecs", + "goType": "uint", + "confFile": "indexes.conf", + "stanza": "", + "key": "hotlist_recency_secs" + }, + "smartstore.indexes.maxGlobalDataSizeMB": { + "jsonPath": "spec.smartstore.indexes.maxGlobalDataSizeMB", + "goType": "uint", + "confFile": "indexes.conf", + "stanza": "", + "key": "maxGlobalDataSizeMB" + }, + "smartstore.indexes.maxGlobalRawDataSizeMB": { + "jsonPath": "spec.smartstore.indexes.maxGlobalRawDataSizeMB", + "goType": "uint", + "confFile": "indexes.conf", + "stanza": "", + "key": "maxGlobalRawDataSizeMB" + }, + "smartstore.indexes.remotePath": { + "jsonPath": "spec.smartstore.indexes.remotePath", + "goType": "string", + "confFile": "indexes.conf", + "stanza": "", + "key": "remotePath" + }, + "smartstore.indexes.volumeName": { + "jsonPath": "spec.smartstore.indexes.volumeName", + "goType": "string", + "confFile": "indexes.conf", + "stanza": "", + "key": "remotePath" + }, + "smartstore.volumes.endpoint": { + "jsonPath": "spec.smartstore.volumes.endpoint", + "goType": "string", + "confFile": "indexes.conf", + "stanza": "volume:", + "key": "remote.s3.endpoint" + }, + "smartstore.volumes.path": { + "jsonPath": "spec.smartstore.volumes.path", + "goType": "string", + "confFile": "indexes.conf", + "stanza": "volume:", + "key": "path" + }, + "smartstore.volumes.region": { + "jsonPath": "spec.smartstore.volumes.region", + "goType": "string", + "confFile": "indexes.conf", + "stanza": "volume:", + "key": "remote.s3.auth_region" + }, + "smartstore.volumes.secretRef": { + "jsonPath": "spec.smartstore.volumes.secretRef", + "goType": "string", + "confFile": "indexes.conf", + "stanza": "volume:", + "key": "remote.s3.access_key;remote.s3.secret_key" + }, + "smartstore.volumes.storageType": { + "jsonPath": "spec.smartstore.volumes.storageType", + "goType": "string", + "confFile": "indexes.conf", + "stanza": "volume:", + "key": "storageType" + } + } + } + } +}