make metrics poll interval configurable

Author: Anyitechs

e2e-tests/src/lib.rs

@@ -143,6 +143,7 @@ client_trusts_lsp = true
 
 [metrics]
 enabled = true
+poll_metrics_interval = 1
 "#,
 			storage_dir = storage_dir.display(),
 		);

e2e-tests/tests/e2e.rs

@@ -856,10 +856,36 @@ async fn test_metrics_endpoint() {
 	assert!(metrics.contains("ldk_server_total_anchor_channels_reserve_sats 0"));
 	assert!(metrics.contains("ldk_server_total_lightning_balance_sats 0"));
 
-	// Set up channel and make a payment to trigger metric update
+	// Set up channel and make a payment to trigger metrics update
 	setup_funded_channel(&bitcoind, &server_a, &server_b, 100_000).await;
 
-	// Make a payment to trigger payment metric updates.
+	// Poll for channel, peer and balance metrics.
+	let timeout = Duration::from_secs(10);
+	let start = std::time::Instant::now();
+	loop {
+		let metrics = client.get_metrics().await.unwrap();
+		if metrics.contains("ldk_server_total_peers_count 1")
+			&& metrics.contains("ldk_server_total_channels_count 1")
+			&& metrics.contains("ldk_server_total_public_channels_count 1")
+			&& metrics.contains("ldk_server_total_payments_count 2")
+			&& !metrics.contains("ldk_server_total_lightning_balance_sats 0")
+			&& !metrics.contains("ldk_server_total_onchain_balance_sats 0")
+			&& !metrics.contains("ldk_server_spendable_onchain_balance_sats 0")
+			&& !metrics.contains("ldk_server_total_anchor_channels_reserve_sats 0")
+		{
+			break;
+		}
+
+		if start.elapsed() > timeout {
+			let current_metrics = client.get_metrics().await.unwrap();
+			panic!(
+				"Timed out waiting for channel, peer and balance metrics to update. Current metrics:\n{}",
+				current_metrics
+			);
+		}
+		tokio::time::sleep(Duration::from_secs(1)).await;
+	}
+
 	let invoice_resp = server_b
 		.client()
 		.bolt11_receive(Bolt11ReceiveRequest {
@@ -880,7 +906,6 @@ async fn test_metrics_endpoint() {
 	loop {
 		let metrics = client.get_metrics().await.unwrap();
 		if metrics.contains("ldk_server_total_successful_payments_count 1")
-			&& metrics.contains("ldk_server_total_channels_count 1")
 			&& !metrics.contains("ldk_server_total_lightning_balance_sats 0")
 			&& !metrics.contains("ldk_server_total_onchain_balance_sats 0")
 			&& !metrics.contains("ldk_server_spendable_onchain_balance_sats 0")

ldk-server-client/src/client.rs

@@ -43,8 +43,7 @@ use ldk_server_protos::endpoints::{
 	GRAPH_LIST_CHANNELS_PATH, GRAPH_LIST_NODES_PATH, LIST_CHANNELS_PATH,
 	LIST_FORWARDED_PAYMENTS_PATH, LIST_PAYMENTS_PATH, LIST_PEERS_PATH, ONCHAIN_RECEIVE_PATH,
 	ONCHAIN_SEND_PATH, OPEN_CHANNEL_PATH, SIGN_MESSAGE_PATH, SPLICE_IN_PATH, SPLICE_OUT_PATH,
-	SPONTANEOUS_SEND_PATH, UNIFIED_SEND_PATH, UPDATE_CHANNEL_CONFIG_PATH,
-	VERIFY_SIGNATURE_PATH,
+	SPONTANEOUS_SEND_PATH, UNIFIED_SEND_PATH, UPDATE_CHANNEL_CONFIG_PATH, VERIFY_SIGNATURE_PATH,
 };
 use ldk_server_protos::error::{ErrorCode, ErrorResponse};
 use prost::bytes::Bytes;

ldk-server/ldk-server-config.toml

@@ -92,3 +92,4 @@ client_trusts_lsp = false
 # Metrics settings
 [metrics]
 enabled = false
+poll_metrics_interval = 60       # The polling interval for metrics in seconds. Defaults to 60secs if unset and metrics enabled.

ldk-server/src/main.rs

@@ -16,7 +16,7 @@ use std::fs;
 use std::os::unix::fs::PermissionsExt;
 use std::path::{Path, PathBuf};
 use std::sync::Arc;
-use std::time::{SystemTime, UNIX_EPOCH};
+use std::time::{Duration, SystemTime, UNIX_EPOCH};
 
 use clap::Parser;
 use hex::DisplayHex;
@@ -50,7 +50,7 @@ use crate::io::persist::{
 use crate::service::NodeService;
 use crate::util::config::{load_config, ArgsConfig, ChainSource};
 use crate::util::logger::ServerLogger;
-use crate::util::metrics::{Metrics, BUILD_METRICS_INTERVAL};
+use crate::util::metrics::Metrics;
 use crate::util::proto_adapter::{forwarded_payment_to_proto, payment_to_proto};
 use crate::util::systemd;
 use crate::util::tls::get_or_generate_tls_config;
@@ -276,8 +276,9 @@ fn main() {
 		let event_node = Arc::clone(&node);
 
 		let metrics: Option<Arc<Metrics>> = if config_file.metrics_enabled {
+			let poll_metrics_interval = Duration::from_secs(config_file.poll_metrics_interval.unwrap_or(60));
 			let metrics_node = Arc::clone(&node);
-			let mut interval = tokio::time::interval(BUILD_METRICS_INTERVAL);
+			let mut interval = tokio::time::interval(poll_metrics_interval);
 			let metrics = Arc::new(Metrics::new());
 			let metrics_bg = Arc::clone(&metrics);
 

ldk-server/src/service.rs

@@ -28,8 +28,7 @@ use ldk_server_protos::endpoints::{
 	GRAPH_LIST_CHANNELS_PATH, GRAPH_LIST_NODES_PATH, LIST_CHANNELS_PATH,
 	LIST_FORWARDED_PAYMENTS_PATH, LIST_PAYMENTS_PATH, LIST_PEERS_PATH, ONCHAIN_RECEIVE_PATH,
 	ONCHAIN_SEND_PATH, OPEN_CHANNEL_PATH, SIGN_MESSAGE_PATH, SPLICE_IN_PATH, SPLICE_OUT_PATH,
-	SPONTANEOUS_SEND_PATH, UNIFIED_SEND_PATH, UPDATE_CHANNEL_CONFIG_PATH,
-	VERIFY_SIGNATURE_PATH,
+	SPONTANEOUS_SEND_PATH, UNIFIED_SEND_PATH, UPDATE_CHANNEL_CONFIG_PATH, VERIFY_SIGNATURE_PATH,
 };
 use prost::Message;
 

ldk-server/src/util/config.rs

@@ -59,6 +59,7 @@ pub struct Config {
 	pub log_file_path: Option<String>,
 	pub pathfinding_scores_source_url: Option<String>,
 	pub metrics_enabled: bool,
+	pub poll_metrics_interval: Option<u64>,
 }
 
 #[derive(Debug, Clone, PartialEq, Eq)]
@@ -105,6 +106,7 @@ struct ConfigBuilder {
 	log_file_path: Option<String>,
 	pathfinding_scores_source_url: Option<String>,
 	metrics_enabled: Option<bool>,
+	poll_metrics_interval: Option<u64>,
 }
 
 impl ConfigBuilder {
@@ -167,6 +169,8 @@ impl ConfigBuilder {
 
 		if let Some(metrics) = toml.metrics {
 			self.metrics_enabled = metrics.enabled.or(self.metrics_enabled);
+			self.poll_metrics_interval =
+				metrics.poll_metrics_interval.or(self.poll_metrics_interval);
 		}
 	}
 
@@ -214,6 +218,10 @@ impl ConfigBuilder {
 		if args.metrics_enabled {
 			self.metrics_enabled = Some(true);
 		}
+
+		if let Some(poll_metrics_interval) = &args.poll_metrics_interval {
+			self.poll_metrics_interval = Some(*poll_metrics_interval);
+		}
 	}
 
 	fn build(self) -> io::Result<Config> {
@@ -374,6 +382,8 @@ impl ConfigBuilder {
 
 		let metrics_enabled = self.metrics_enabled.unwrap_or(false);
 
+		let poll_metrics_interval = self.poll_metrics_interval;
+
 		Ok(Config {
 			network,
 			listening_addrs,
@@ -392,6 +402,7 @@ impl ConfigBuilder {
 			log_file_path: self.log_file_path,
 			pathfinding_scores_source_url,
 			metrics_enabled,
+			poll_metrics_interval,
 		})
 	}
 }
@@ -471,6 +482,7 @@ struct TomlTlsConfig {
 #[derive(Deserialize, Serialize)]
 struct MetricsTomlConfig {
 	enabled: Option<bool>,
+	poll_metrics_interval: Option<u64>,
 }
 
 #[derive(Deserialize, Serialize)]
@@ -638,6 +650,14 @@ pub struct ArgsConfig {
 		help = "The option to enable the metrics endpoint. WARNING: This endpoint is unauthenticated."
 	)]
 	metrics_enabled: bool,
+
+	#[arg(
+		long,
+		env = "LDK_SERVER_POLL_METRICS_INTERVAL",
+		help = "The polling interval for metrics in seconds. Required when
+		metrics is enabled, but defaults to 60secs if unset."
+	)]
+	poll_metrics_interval: Option<u64>,
 }
 
 pub fn load_config(args: &ArgsConfig) -> io::Result<Config> {
@@ -772,6 +792,7 @@ mod tests {
 			node_alias: Some(String::from("LDK Server CLI")),
 			pathfinding_scores_source_url: Some(String::from("https://example.com/")),
 			metrics_enabled: false,
+			poll_metrics_interval: None,
 		}
 	}
 
@@ -789,6 +810,7 @@ mod tests {
 			storage_dir_path: None,
 			pathfinding_scores_source_url: None,
 			metrics_enabled: false,
+			poll_metrics_interval: None,
 		}
 	}
 
@@ -866,6 +888,7 @@ mod tests {
 			log_file_path: Some("/var/log/ldk-server.log".to_string()),
 			pathfinding_scores_source_url: None,
 			metrics_enabled: false,
+			poll_metrics_interval: None,
 		};
 
 		assert_eq!(config.listening_addrs, expected.listening_addrs);
@@ -1192,6 +1215,7 @@ mod tests {
 			log_file_path: Some("/var/log/ldk-server.log".to_string()),
 			pathfinding_scores_source_url: Some("https://example.com/".to_string()),
 			metrics_enabled: false,
+			poll_metrics_interval: None,
 		};
 
 		assert_eq!(config.listening_addrs, expected.listening_addrs);
@@ -1306,6 +1330,7 @@ mod tests {
 			log_file_path: Some("/var/log/ldk-server.log".to_string()),
 			pathfinding_scores_source_url: Some("https://example.com/".to_string()),
 			metrics_enabled: false,
+			poll_metrics_interval: None,
 		};
 
 		assert_eq!(config.listening_addrs, expected.listening_addrs);

ldk-server/src/util/metrics.rs

@@ -14,26 +14,25 @@
 //!
 //! The metrics are updated through two main mechanisms:
 //! 1.  **Periodic Polling**: The `update_all_pollable_metrics` function is called at a regular
-//!     interval (`BUILD_METRICS_INTERVAL`) to perform a full recount of metrics like peer count,
-//!     channels count, and balances.
+//!     interval (`poll_metrics_interval`) configurable via the config file but defaults to 60secs if unset, to perform a full recount of metrics like peer count,
+//!     payments count, and channels metrics.
 //! 2.  **Event-Driven Updates**: For metrics that can change frequently and where a full recount
-//!     would be inefficient (e.g., total_successful_payments_count), a hybrid approach is used.
+//!     would be inefficient (e.g., total_successful_payments_count, balances), a hybrid approach is used.
 //!     - `initialize_payment_metrics` is called once at startup to get the accurate persisted state.
 //!     - `update_payments_count` is called incrementally whenever a relevant event (like
 //!       `PaymentSuccessful` or `PaymentFailed`) occurs.
+//!    - `update_all_balances` is called when we receive a `PaymentSuccessful` event to update all balance metrics.
+//!    - `update_channels_count` is called when we receive a `ChannelReady` or `ChannelClosed` event to update the channels metrics.
 //!
 //! The `gather_metrics` function collects all current metric values and formats them into the
 //! plain-text format that Prometheus scrapers expect. This output is exposed via an
 //! unauthenticated `/metrics` HTTP endpoint on the rest service address.
 
 use std::sync::atomic::{AtomicI64, AtomicU64, Ordering};
-use std::time::Duration;
 
 use ldk_node::payment::PaymentStatus;
 use ldk_node::Node;
 
-pub const BUILD_METRICS_INTERVAL: Duration = Duration::from_secs(60);
-
 /// Holds all the metrics that are tracked for LDK Server.
 ///
 /// These metrics are exposed in a Prometheus-compatible format. The values are stored
@@ -153,6 +152,7 @@ impl Metrics {
 		self.total_private_channels_count.store(private_channels_count, Ordering::Relaxed);
 
 		self.update_peer_count(node);
+		self.update_all_balances(node);
 	}
 
 	/// Gathers all metrics and formats them into the Prometheus text-based format.