background task to time out incomplete audit logs

Author: david-crespo

dev-tools/omdb/src/bin/omdb/nexus.rs

@@ -52,6 +52,7 @@ use nexus_types::deployment::OximeterReadPolicy;
 use nexus_types::fm;
 use nexus_types::internal_api::background::AbandonedVmmReaperStatus;
 use nexus_types::internal_api::background::AttachedSubnetManagerStatus;
+use nexus_types::internal_api::background::AuditLogTimeoutIncompleteStatus;
 use nexus_types::internal_api::background::BlueprintPlannerStatus;
 use nexus_types::internal_api::background::BlueprintRendezvousStats;
 use nexus_types::internal_api::background::BlueprintRendezvousStatus;
@@ -1220,6 +1221,9 @@ fn print_task_details(bgtask: &BackgroundTask, details: &serde_json::Value) {
         "attached_subnet_manager" => {
             print_task_attached_subnet_manager_status(details);
         }
+        "audit_log_timeout_incomplete" => {
+            print_task_audit_log_timeout_incomplete(details);
+        }
         "blueprint_planner" => {
             print_task_blueprint_planner(details);
         }
@@ -2679,6 +2683,38 @@ fn print_task_saga_recovery(details: &serde_json::Value) {
     }
 }
 
+fn print_task_audit_log_timeout_incomplete(details: &serde_json::Value) {
+    match serde_json::from_value::<AuditLogTimeoutIncompleteStatus>(
+        details.clone(),
+    ) {
+        Err(error) => eprintln!(
+            "warning: failed to interpret task details: {:?}: {:?}",
+            error, details
+        ),
+        Ok(status) => {
+            const TIMED_OUT: &str = "timed_out:";
+            const CUTOFF: &str = "cutoff:";
+            const MAX_UPDATE: &str = "max_update_per_activation:";
+            const ERROR: &str = "error:";
+            const WIDTH: usize =
+                const_max_len(&[TIMED_OUT, CUTOFF, MAX_UPDATE, ERROR]) + 1;
+
+            println!("    {TIMED_OUT:<WIDTH$}{}", status.timed_out);
+            println!(
+                "    {CUTOFF:<WIDTH$}{}",
+                status.cutoff.to_rfc3339_opts(SecondsFormat::AutoSi, true),
+            );
+            println!(
+                "    {MAX_UPDATE:<WIDTH$}{}",
+                status.max_update_per_activation
+            );
+            if let Some(error) = &status.error {
+                println!("    {ERROR:<WIDTH$}{error}");
+            }
+        }
+    };
+}
+
 fn print_task_session_cleanup(details: &serde_json::Value) {
     match serde_json::from_value::<SessionCleanupStatus>(details.clone()) {
         Err(error) => eprintln!(

dev-tools/omdb/tests/env.out

@@ -38,6 +38,11 @@ task: "attached_subnet_manager"
     distributes attached subnets to sleds and switch
 
 
+task: "audit_log_timeout_incomplete"
+    transitions stale incomplete audit log entries to timeout status so they
+    become visible in the audit log
+
+
 task: "bfd_manager"
     Manages bidirectional fowarding detection (BFD) configuration on rack
     switches
@@ -283,6 +288,11 @@ task: "attached_subnet_manager"
     distributes attached subnets to sleds and switch
 
 
+task: "audit_log_timeout_incomplete"
+    transitions stale incomplete audit log entries to timeout status so they
+    become visible in the audit log
+
+
 task: "bfd_manager"
     Manages bidirectional fowarding detection (BFD) configuration on rack
     switches
@@ -515,6 +525,11 @@ task: "attached_subnet_manager"
     distributes attached subnets to sleds and switch
 
 
+task: "audit_log_timeout_incomplete"
+    transitions stale incomplete audit log entries to timeout status so they
+    become visible in the audit log
+
+
 task: "bfd_manager"
     Manages bidirectional fowarding detection (BFD) configuration on rack
     switches

dev-tools/omdb/tests/successes.out

@@ -273,6 +273,11 @@ task: "attached_subnet_manager"
     distributes attached subnets to sleds and switch
 
 
+task: "audit_log_timeout_incomplete"
+    transitions stale incomplete audit log entries to timeout status so they
+    become visible in the audit log
+
+
 task: "bfd_manager"
     Manages bidirectional fowarding detection (BFD) configuration on rack
     switches
@@ -588,6 +593,14 @@ task: "attached_subnet_manager"
    no dendrite instances found
    no sleds found
 
+task: "audit_log_timeout_incomplete"
+  configured period: every <REDACTED_DURATION>m
+  last completed activation: <REDACTED ITERATIONS>, triggered by <TRIGGERED_BY_REDACTED>
+    started at <REDACTED_TIMESTAMP> (<REDACTED DURATION>s ago) and ran for <REDACTED DURATION>ms
+    timed_out:                 0
+    cutoff:                    <REDACTED_TIMESTAMP>
+    max_update_per_activation: 1000
+
 task: "bfd_manager"
   configured period: every <REDACTED_DURATION>s
   last completed activation: <REDACTED ITERATIONS>, triggered by <TRIGGERED_BY_REDACTED>
@@ -1183,6 +1196,14 @@ task: "attached_subnet_manager"
    no dendrite instances found
    no sleds found
 
+task: "audit_log_timeout_incomplete"
+  configured period: every <REDACTED_DURATION>m
+  last completed activation: <REDACTED ITERATIONS>, triggered by <TRIGGERED_BY_REDACTED>
+    started at <REDACTED_TIMESTAMP> (<REDACTED DURATION>s ago) and ran for <REDACTED DURATION>ms
+    timed_out:                 0
+    cutoff:                    <REDACTED_TIMESTAMP>
+    max_update_per_activation: 1000
+
 task: "bfd_manager"
   configured period: every <REDACTED_DURATION>s
   last completed activation: <REDACTED ITERATIONS>, triggered by <TRIGGERED_BY_REDACTED>

nexus-config/src/nexus_config.rs

@@ -437,6 +437,8 @@ pub struct BackgroundTaskConfig {
     pub attached_subnet_manager: AttachedSubnetManagerConfig,
     /// configuration for console session cleanup task
     pub session_cleanup: SessionCleanupConfig,
+    /// configuration for audit log incomplete timeout task
+    pub audit_log_timeout_incomplete: AuditLogTimeoutIncompleteConfig,
 }
 
 #[serde_as]
@@ -450,6 +452,21 @@ pub struct SessionCleanupConfig {
     pub max_delete_per_activation: u32,
 }
 
+#[serde_as]
+#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
+pub struct AuditLogTimeoutIncompleteConfig {
+    /// period (in seconds) for periodic activations of this task
+    #[serde_as(as = "DurationSeconds<u64>")]
+    pub period_secs: Duration,
+
+    /// how old an incomplete entry must be before it is timed out
+    #[serde_as(as = "DurationSeconds<u64>")]
+    pub timeout_secs: Duration,
+
+    /// max rows per SQL statement
+    pub max_update_per_activation: u32,
+}
+
 #[serde_as]
 #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
 pub struct DnsTasksConfig {
@@ -1267,6 +1284,9 @@ mod test {
             attached_subnet_manager.period_secs = 60
             session_cleanup.period_secs = 300
             session_cleanup.max_delete_per_activation = 10000
+            audit_log_timeout_incomplete.period_secs = 600
+            audit_log_timeout_incomplete.timeout_secs = 14400
+            audit_log_timeout_incomplete.max_update_per_activation = 1000
             [default_region_allocation_strategy]
             type = "random"
             seed = 0
@@ -1534,6 +1554,12 @@ mod test {
                             period_secs: Duration::from_secs(300),
                             max_delete_per_activation: 10_000,
                         },
+                        audit_log_timeout_incomplete:
+                            AuditLogTimeoutIncompleteConfig {
+                                period_secs: Duration::from_secs(600),
+                                timeout_secs: Duration::from_secs(14400),
+                                max_update_per_activation: 1000,
+                            },
                     },
                     multicast: MulticastConfig { enabled: false },
                     default_region_allocation_strategy:
@@ -1641,6 +1667,9 @@ mod test {
             attached_subnet_manager.period_secs = 60
             session_cleanup.period_secs = 300
             session_cleanup.max_delete_per_activation = 10000
+            audit_log_timeout_incomplete.period_secs = 600
+            audit_log_timeout_incomplete.timeout_secs = 14400
+            audit_log_timeout_incomplete.max_update_per_activation = 1000
 
             [default_region_allocation_strategy]
             type = "random"

nexus/background-task-interface/src/init.rs

@@ -37,6 +37,7 @@ pub struct BackgroundTasks {
     pub task_instance_reincarnation: Activator,
     pub task_service_firewall_propagation: Activator,
     pub task_abandoned_vmm_reaper: Activator,
+    pub task_audit_log_timeout_incomplete: Activator,
     pub task_vpc_route_manager: Activator,
     pub task_saga_recovery: Activator,
     pub task_lookup_region_port: Activator,

nexus/db-model/src/audit_log.rs

@@ -276,15 +276,15 @@ pub enum AuditLogCompletion {
     /// error, and I don't think we even have API timeouts) but rather that the
     /// attempts to complete the log entry failed (or were never even attempted
     /// because, e.g., Nexus crashed during the operation), and this entry had
-    /// to be cleaned up later by a background job (which doesn't exist yet)
-    /// after a timeout. Note we represent this result status as "Unknown" in
-    /// the external API because timeout is an implementation detail and makes
-    /// it sound like the operation timed out.
+    /// to be cleaned up later by a background job after a timeout. Note we
+    /// represent this result status as "Unknown" in the external API because
+    /// timeout is an implementation detail and makes it sound like the
+    /// operation timed out.
     Timeout,
 }
 
 #[derive(AsChangeset, Clone)]
-#[diesel(table_name = audit_log)]
+#[diesel(table_name = audit_log, treat_none_as_null = true)]
 pub struct AuditLogCompletionUpdate {
     pub time_completed: DateTime<Utc>,
     pub result_kind: AuditLogResultKind,

nexus/db-model/src/schema_versions.rs

@@ -16,7 +16,7 @@ use std::{collections::BTreeMap, sync::LazyLock};
 ///
 /// This must be updated when you change the database schema.  Refer to
 /// schema/crdb/README.adoc in the root of this repository for details.
-pub const SCHEMA_VERSION: Version = Version::new(237, 0, 0);
+pub const SCHEMA_VERSION: Version = Version::new(238, 0, 0);
 
 /// List of all past database schema versions, in *reverse* order
 ///
@@ -28,6 +28,7 @@ static KNOWN_VERSIONS: LazyLock<Vec<KnownVersion>> = LazyLock::new(|| {
         // |  leaving the first copy as an example for the next person.
         // v
         // KnownVersion::new(next_int, "unique-dirname-with-the-sql-files"),
+        KnownVersion::new(238, "audit-log-incomplete-timeout"),
         KnownVersion::new(237, "switch-slot-enum"),
         KnownVersion::new(
             236,