Update CLO to use Vector v0.54.0

Makefile

@@ -26,7 +26,7 @@ export VERSION=$(LOGGING_VERSION).0
 export NAMESPACE?=openshift-logging
 export LOKI_OPERATOR_CHANNEL?=stable-6.4
 
-IMAGE_LOGGING_VECTOR?=quay.io/openshift-logging/vector:v0.47.0
+IMAGE_LOGGING_VECTOR?=quay.io/openshift-logging/vector:v0.54.0
 IMAGE_LOGFILEMETRICEXPORTER?=quay.io/openshift-logging/log-file-metric-exporter:latest
 IMAGE_LOGGING_EVENTROUTER?=quay.io/openshift-logging/eventrouter:v0.5.0
 

bundle/image-references

@@ -6,7 +6,7 @@ spec:
   - name: vector-rhel9
     from:
       kind: DockerImage
-      name: quay.io/openshift-logging/vector:v0.47.0
+      name: quay.io/openshift-logging/vector:v0.54.0
   - name: log-file-metric-exporter-rhel9
     from:
       kind: DockerImage

bundle/manifests/cluster-logging.clusterserviceversion.yaml

@@ -2379,7 +2379,7 @@ spec:
                 - name: OPERATOR_NAME
                   value: cluster-logging-operator
                 - name: RELATED_IMAGE_VECTOR
-                  value: quay.io/openshift-logging/vector:v0.47.0
+                  value: quay.io/openshift-logging/vector:v0.54.0
                 - name: RELATED_IMAGE_LOG_FILE_METRIC_EXPORTER
                   value: quay.io/openshift-logging/log-file-metric-exporter:latest
                 image: quay.io/openshift-logging/cluster-logging-operator:latest
@@ -2426,7 +2426,7 @@ spec:
   provider:
     name: Red Hat
   relatedImages:
-  - image: quay.io/openshift-logging/vector:v0.47.0
+  - image: quay.io/openshift-logging/vector:v0.54.0
     name: vector
   - image: quay.io/openshift-logging/log-file-metric-exporter:latest
     name: log-file-metric-exporter

config/manager/manager.yaml

@@ -48,6 +48,6 @@ spec:
           - name: OPERATOR_NAME
             value: "cluster-logging-operator"
           - name: RELATED_IMAGE_VECTOR
-            value: quay.io/openshift-logging/vector:v0.47.0
+            value: quay.io/openshift-logging/vector:v0.54.0
           - name: RELATED_IMAGE_LOG_FILE_METRIC_EXPORTER
             value: quay.io/openshift-logging/log-file-metric-exporter:latest

internal/cmd/functional-benchmarker/config/options.go

@@ -17,7 +17,7 @@ import (
 
 const (
 	LogStressorImage = "quay.io/openshift-logging/cluster-logging-load-client:0.2"
-	imageVector      = "quay.io/openshift-logging/vector:v0.47.0"
+	imageVector      = "quay.io/openshift-logging/vector:v0.54.0"
 )
 
 type Options struct {

internal/generator/vector/api/config.go

@@ -7,6 +7,9 @@ type Config struct {
 	// Api is the set of API keys to values
 	Api *Api `json:"api,omitempty" yaml:"api,omitempty" toml:"api,omitempty"`
 
+	//
+	LogSchema *LogSchema `json:"log_schema,omitempty" yaml:"log_schema,omitempty" toml:"log_schema,omitempty"`
+
 	// Secrets is the set of secret ids to secret configurations
 	Secret map[string]*Secret `json:"secret,omitempty" yaml:"secret,omitempty" toml:"secret,omitempty"`
 

internal/generator/vector/api/config_test.go

@@ -16,6 +16,9 @@ data_dir = "/var/lib/vector/openshift-logging/collector"
 [api]
 enabled = true
 
+[log_schema]
+host_key = "hostname"
+
 # Load sensitive data from files
 [secret.kubernetes_secret]
 type = "file"
@@ -40,6 +43,8 @@ expire_metrics_secs: 60
 data_dir: "/var/lib/vector/openshift-logging/collector"
 api:
   enabled: true
+log_schema: 
+  host_key: "hostname"
 secret:
   kubernetes_secret:
     type: "file"

internal/generator/vector/api/log_schema.go

@@ -0,0 +1,7 @@
+package api
+
+//LogSchema default log schema for all events.
+type LogSchema struct {
+	// The name of the event field to treat as the host which sent the message.
+	HostKey string `json:"host_key,omitempty" yaml:"host_key,omitempty" toml:"host_key,omitempty"`
+}

internal/generator/vector/api/sinks/socket_sink.go

@@ -20,18 +20,19 @@ type Keepalive struct {
 	TimeSecs uint `json:"time_secs,omitempty" yaml:"time_secs,omitempty" toml:"time_secs,omitempty"`
 }
 
+type SyslogEncodingConfig struct {
+	RFC      string `json:"rfc,omitempty" yaml:"rfc,omitempty" toml:"rfc,omitempty"`
+	Facility string `json:"facility,omitempty" yaml:"facility,omitempty" toml:"facility,omitempty"`
+	Severity string `json:"severity,omitempty" yaml:"severity,omitempty" toml:"severity,omitempty"`
+	AppName  string `json:"app_name,omitempty" yaml:"app_name,omitempty" toml:"app_name,omitempty"`
+	MsgID    string `json:"msg_id,omitempty" yaml:"msg_id,omitempty" toml:"msg_id,omitempty"`
+	ProcID   string `json:"proc_id,omitempty" yaml:"proc_id,omitempty" toml:"proc_id,omitempty"`
+}
+
 type SocketEncoding struct {
-	Codec        string   `json:"codec,omitempty" yaml:"codec,omitempty" toml:"codec,omitempty"`
-	ExceptFields []string `json:"except_fields,omitempty" yaml:"except_fields,omitempty" toml:"except_fields,omitempty"`
-	RFC          string   `json:"rfc,omitempty" yaml:"rfc,omitempty" toml:"rfc,omitempty"`
-	Facility     string   `json:"facility,omitempty" yaml:"facility,omitempty" toml:"facility,omitempty"`
-	Severity     string   `json:"severity,omitempty" yaml:"severity,omitempty" toml:"severity,omitempty"`
-	AppName      string   `json:"app_name,omitempty" yaml:"app_name,omitempty" toml:"app_name,omitempty"`
-	MsgID        string   `json:"msg_id,omitempty" yaml:"msg_id,omitempty" toml:"msg_id,omitempty"`
-	ProcID       string   `json:"proc_id,omitempty" yaml:"proc_id,omitempty" toml:"proc_id,omitempty"`
-	Tag          string   `json:"tag,omitempty" yaml:"tag,omitempty" toml:"tag,omitempty"`
-	AddLogSource *bool    `json:"add_log_source,omitempty" yaml:"add_log_source,omitempty" toml:"add_log_source,omitempty"`
-	PayloadKey   string   `json:"payload_key,omitempty" yaml:"payload_key,omitempty" toml:"payload_key,omitempty"`
+	Codec        string                `json:"codec,omitempty" yaml:"codec,omitempty" toml:"codec,omitempty"`
+	ExceptFields []string              `json:"except_fields,omitempty" yaml:"except_fields,omitempty" toml:"except_fields,omitempty"`
+	Syslog       *SyslogEncodingConfig `json:"syslog,omitempty" yaml:"syslog,omitempty" toml:"syslog,omitempty"`
 }
 
 type Socket struct {

internal/generator/vector/conf/complex.toml

@@ -4,6 +4,9 @@ data_dir = "/var/lib/vector/openshift-logging/my-forwarder"
 [api]
 enabled = true
 
+[log_schema]
+host_key = "hostname"
+
 [secret.kubernetes_secret]
 type = "directory"
 path = "/var/run/ocp-collector/secrets"

internal/generator/vector/conf/complex_http_receiver.toml

@@ -4,6 +4,9 @@ data_dir = "/var/lib/vector/openshift-logging/my-forwarder"
 [api]
 enabled = true
 
+[log_schema]
+host_key = "hostname"
+
 [secret.kubernetes_secret]
 type = "directory"
 path = "/var/run/ocp-collector/secrets"

internal/generator/vector/conf/container.toml

@@ -4,6 +4,9 @@ data_dir = "/var/lib/vector/openshift-logging/my-forwarder"
 [api]
 enabled = true
 
+[log_schema]
+host_key = "hostname"
+
 # Load sensitive data from files
 [secret.kubernetes_secret]
 type = "directory"

internal/generator/vector/conf/global.go

@@ -13,6 +13,7 @@ func Global(c *api.Config, namespace, forwarderName string) *api.Config {
 		dataDir = ""
 	}
 	c.Api = &api.Api{Enabled: true}
+	c.LogSchema = &api.LogSchema{HostKey: "hostname"}
 	c.Global = api.Global{
 		ExpireMetricsSec: 60,
 		DataDir:          dataDir,

internal/generator/vector/conf/global_test.go

@@ -20,6 +20,9 @@ data_dir = "/var/lib/vector/openshift-logging/my-forwarder"
 [api]
 enabled = true
 
+[log_schema]
+host_key = "hostname"
+
 [secret.kubernetes_secret]
 type = "directory"
 path = "/var/run/ocp-collector/secrets"
@@ -31,9 +34,12 @@ path = "/var/run/ocp-collector/secrets"
 		It("should not include a global data directory for that forwarder", func() {
 			Expect(`
 	expire_metrics_secs = 60
-	
+
 	[api]
 	enabled = true
+
+    [log_schema]
+    host_key = "hostname"
 
 	[secret.kubernetes_secret]
 	type = "directory"

internal/generator/vector/output/syslog/rfc3164_with_defaults.toml

@@ -2,36 +2,80 @@
 type = "remap"
 inputs = ["application"]
 source = '''
+
+._syslog = {}
 if .log_type == "infrastructure" && .log_source == "node" {
-   ._syslog.tag = to_string!(.systemd.u.SYSLOG_IDENTIFIER || "")
+   ._syslog.app_name = to_string!(.systemd.u.SYSLOG_IDENTIFIER || "")
    ._syslog.proc_id = to_string!(.systemd.t.PID || "")
 }
 if .log_source == "container" {
-   	._syslog.tag, err = join([.kubernetes.namespace_name, .kubernetes.pod_name, .kubernetes.container_name], "")
+   	._syslog.app_name, err = join([.kubernetes.namespace_name, .kubernetes.pod_name, .kubernetes.container_name], "")
    	if err != null {
-   	   log("K8s metadata (namespace, pod, or container) missing; syslog.tag set to empty", level: "error")
-  	   ._syslog.tag = ""
+   	   log("K8s metadata (namespace, pod, or container) missing: unable to calculate syslog.app_name (TAG)", level: "error")
     } else {
-       #Remove non-alphanumeric characters
-       ._syslog.tag = replace(._syslog.tag, r'[^a-zA-Z0-9]', "")
-       #Truncate the sanitized tag to 32 characters
-       ._syslog.tag = truncate(._syslog.tag, 32)
-    }
+  	   #Remove non-alphanumeric characters
+  	   ._syslog.app_name = replace(._syslog.app_name, r'[^a-zA-Z0-9]', "")
+  	   #Truncate the sanitized tag to 32 characters
+  	   ._syslog.app_name = truncate(._syslog.app_name, 32)
+  	}
     ._syslog.severity = .level
     ._syslog.facility = "user"
 }
 if .log_type == "audit" {
-   ._syslog.tag = .log_source
+   ._syslog.app_name = .log_source
    ._syslog.severity = "informational"
    ._syslog.facility = "security"
 }
-proc_id = to_string!(._syslog.proc_id || "-")
-tag = to_string!(._syslog.tag || "")
-if proc_id != "-" && proc_id != "" {
-  ._syslog.tag = to_string(tag||"") + "[" + to_string(proc_id)  + "]"
-} else {
-  ._syslog.tag = to_string(tag)
+._syslog.proc_id = to_string!(._syslog.proc_id || "")
+if exists(._syslog.proc_id) && is_empty(strip_whitespace(string!(._syslog.proc_id))) { del(._syslog.proc_id) }
+._syslog.app_name = to_string!(._syslog.app_name || "")
+
+# try to convert syslog code to the facility, severity names (e.g. 4 -> "warning", "4" -> "warning"  )
+if exists(._syslog.facility) {
+  _, err = to_syslog_facility_code(._syslog.facility)
+  if err != null {
+    # Field is not a valid name — try treating it as a code (int or string int)
+    code, err2 = to_int(._syslog.facility)
+    if err2 == null {
+      facility, err3 = to_syslog_facility(code)
+      if err3 == null {
+        ._syslog.facility = facility
+      } else {
+        log("Invalid syslog facility code: " + to_string!(._syslog.facility) , level: "warn")
+      }
+    } else {
+      log("Invalid syslog facility value: " + to_string!(._syslog.facility), level: "warn")
+    }
+  }
+  # else: already a valid name, leave it as-is
+}
+
+if exists(._syslog.severity) {
+  _, err = to_syslog_severity(._syslog.severity)
+  if err != null {
+    # Field is not a valid name — try treating it as a code (int or string int)
+    code, err2 = to_int(._syslog.severity)
+    if err2 == null {
+      severity, err3 = to_syslog_level(code)
+      if err3 == null {
+        ._syslog.severity = severity
+      } else {
+        log("Invalid syslog severity code: " + to_string!(._syslog.severity), level: "warn")
+      }
+    } else {
+      log("Invalid syslog severity value: " + to_string!(._syslog.severity), level: "warn")
+    }
+  }
+  # else: already a valid name, leave it as-is
+}
+
+# Payload key NOT configured, full payload set to .message field (skipping internal objects)
+excluded_fields = ["_internal", "_syslog"]
+temp = .
+for_each(excluded_fields) -> |_index, field| {
+  temp = remove(temp, [field]) ?? temp
 }
+.message = temp
 '''
 
 [sinks.example]
@@ -42,10 +86,8 @@ mode = "udp"
 
 [sinks.example.encoding]
 codec = "syslog"
-except_fields = ["_internal"]
-rfc = "rfc3164"
-facility = "$$._syslog.facility"
-severity = "$$._syslog.severity"
-proc_id = "$$._syslog.proc_id"
-tag = "$$._syslog.tag"
-add_log_source = false
+syslog.rfc = "rfc3164"
+syslog.facility = "._syslog.facility"
+syslog.severity = "._syslog.severity"
+syslog.app_name = "._syslog.app_name"
+syslog.proc_id = "._syslog.proc_id"