Bug 2040030 - Increase pending pings limitations to better support long network outages

CHANGELOG.md

@@ -2,6 +2,12 @@
 
 [Full changelog](https://github.com/mozilla/glean/compare/v67.3.1...main)
 
+* General
+  * Increase pending ping limits to better support long network outages ([bug 2040030](https://bugzilla.mozilla.org/show_bug.cgi?id=2040030))
+    * New `glean.upload.pending_pings_deleted` metric added to differentiate between deletions due to pending ping count or directory size limitations
+    * Default pending pings allowed before deletion raised from 250 to 500, and the directory size before deletion increased from 10MB to 50MB.
+    * New configuration fields added to allow integrating applications to override the maximum pending pings count and directory size limits.
+
 # v67.3.1 (2026-05-08)
 
 [Full changelog](https://github.com/mozilla/glean/compare/v67.3.0...v67.3.1)

glean-core/android/src/main/java/mozilla/telemetry/glean/Glean.kt

@@ -270,6 +270,8 @@ open class GleanInternalAPI internal constructor() {
                 pingSchedule = configuration.pingSchedule,
                 pingLifetimeThreshold = configuration.pingLifetimeThreshold.toULong(),
                 pingLifetimeMaxTime = configuration.pingLifetimeMaxTime.toULong(),
+                maxPendingPingsCount = configuration.maxPendingPingsCount?.toULong(),
+                maxPendingPingsDirectorySize = configuration.maxPendingPingsDirectorySize?.toULong(),
             )
             val clientInfo = getClientInfo(configuration, buildInfo)
             val callbacks = OnGleanEventsImpl(this@GleanInternalAPI)

glean-core/android/src/main/java/mozilla/telemetry/glean/config/Configuration.kt

@@ -30,6 +30,10 @@ import mozilla.telemetry.glean.net.PingUploader
  * @property pingSchedule A ping schedule map.
  *           Maps a ping name to a list of pings to schedule along with it.
  *           Only used if the ping's own ping schedule list is empty.
+ * @property maxPendingPingsCount The maximum number of pending pings stored on disk.
+ *           When this limit is exceeded, the oldest pings are deleted. Defaults to 500.
+ * @property maxPendingPingsDirectorySize The maximum size in bytes of the pending pings directory.
+ *           When this limit is exceeded, the oldest pings are deleted. Defaults to 50 MB.
  */
 data class Configuration
     @JvmOverloads
@@ -50,6 +54,8 @@ data class Configuration
         val pingLifetimeThreshold: Int = 1000,
         val pingLifetimeMaxTime: Int = 0,
         val pingSchedule: Map<String, List<String>> = emptyMap(),
+        val maxPendingPingsCount: Long? = null,
+        val maxPendingPingsDirectorySize: Long? = null,
     ) {
         companion object {
             /**

glean-core/benchmark/benches/dispatcher.rs

@@ -83,6 +83,8 @@ pub fn metric_dispatcher_benchmark(c: &mut Criterion) {
         ping_schedule: Default::default(),
         ping_lifetime_threshold: 0,
         ping_lifetime_max_time: 0,
+        max_pending_pings_count: None,
+        max_pending_pings_directory_size: None,
     };
     let client_info = ClientInfoMetrics::unknown();
 

glean-core/benchmark/benches/lifetime_buffering.rs

@@ -31,6 +31,8 @@ pub fn delay_io_benchmark(c: &mut Criterion) {
             ping_schedule: Default::default(),
             ping_lifetime_threshold: 0,
             ping_lifetime_max_time: 0,
+            max_pending_pings_count: None,
+            max_pending_pings_directory_size: None,
         };
         let glean = Glean::new(cfg).unwrap();
 
@@ -73,6 +75,8 @@ pub fn delay_io_benchmark(c: &mut Criterion) {
             ping_schedule: Default::default(),
             ping_lifetime_threshold: 0,
             ping_lifetime_max_time: 0,
+            max_pending_pings_count: None,
+            max_pending_pings_directory_size: None,
         };
         let glean = Glean::new(cfg).unwrap();
 

glean-core/examples/rkv-open.rs

@@ -51,6 +51,8 @@ fn main() {
         ping_schedule: HashMap::default(),
         ping_lifetime_threshold: 0,
         ping_lifetime_max_time: 0,
+        max_pending_pings_count: None,
+        max_pending_pings_directory_size: None,
     };
 
     let client_info = ClientInfoMetrics::unknown();

glean-core/ios/Glean/Config/Configuration.swift

@@ -17,6 +17,8 @@ public struct Configuration {
     let pingLifetimeMaxTime: Int
     let pingSchedule: [String: [String]]
     let httpClient: PingUploader
+    let maxPendingPingsCount: UInt64?
+    let maxPendingPingsDirectorySize: UInt64?
 
     struct Constants {
         static let defaultTelemetryEndpoint =
@@ -43,6 +45,10 @@ public struct Configuration {
     ///   Maps a ping name to a list of pings to schedule along with it.
     ///   Only used if the ping's own ping schedule list is empty.
     ///   * httpClient An http uploader that supports the `PingUploader` protocol
+    ///   * maxPendingPingsCount The maximum number of pending pings stored on disk.
+    ///   When exceeded, the oldest pings are deleted. Defaults to 500.
+    ///   * maxPendingPingsDirectorySize The maximum size in bytes of the pending pings directory.
+    ///   When exceeded, the oldest pings are deleted. Defaults to 50 MB.
     public init(
         maxEvents: Int32? = nil,
         channel: String? = nil,
@@ -55,7 +61,9 @@ public struct Configuration {
         pingLifetimeThreshold: Int = 0,
         pingLifetimeMaxTime: Int = 0,
         pingSchedule: [String: [String]] = [:],
-        httpClient: PingUploader = HttpPingUploader()
+        httpClient: PingUploader = HttpPingUploader(),
+        maxPendingPingsCount: UInt64? = nil,
+        maxPendingPingsDirectorySize: UInt64? = nil
     ) {
         self.serverEndpoint =
             serverEndpoint ?? Constants.defaultTelemetryEndpoint
@@ -70,5 +78,7 @@ public struct Configuration {
         self.pingLifetimeMaxTime = pingLifetimeMaxTime
         self.pingSchedule = pingSchedule
         self.httpClient = httpClient
+        self.maxPendingPingsCount = maxPendingPingsCount
+        self.maxPendingPingsDirectorySize = maxPendingPingsDirectorySize
     }
 }

glean-core/ios/Glean/Glean.swift

@@ -212,7 +212,9 @@ public final class Glean: @unchecked Sendable {
             enableInternalPings: configuration.enableInternalPings,
             pingSchedule: configuration.pingSchedule,
             pingLifetimeThreshold: UInt64(configuration.pingLifetimeThreshold),
-            pingLifetimeMaxTime: UInt64(configuration.pingLifetimeMaxTime)
+            pingLifetimeMaxTime: UInt64(configuration.pingLifetimeMaxTime),
+            maxPendingPingsCount: configuration.maxPendingPingsCount,
+            maxPendingPingsDirectorySize: configuration.maxPendingPingsDirectorySize
         )
         let clientInfo = getClientInfo(configuration, buildInfo: buildInfo)
         let callbacks = OnGleanEventsImpl(glean: self)

glean-core/metrics.yaml

@@ -808,6 +808,32 @@ glean.upload:
     no_lint:
       - COMMON_PREFIX
 
+  pending_pings_deleted:
+    type: labeled_counter
+    labels:
+      - count_quota
+      - size_quota
+    description: |
+      The number of pings deleted because a pending pings quota was exceeded,
+      labeled by which quota triggered the deletion.
+      `count_quota`: the maximum number of pending pings was reached.
+      `size_quota`: the maximum size of the pending pings directory was reached.
+      Sent on the health ping only; use this metric to verify whether
+      the configured limits are sufficient to survive connectivity outages.
+    send_in_pings:
+      - health
+    bugs:
+      - https://bugzilla.mozilla.org/show_bug.cgi?id=2040030
+    data_reviews:
+      - https://bugzilla.mozilla.org/show_bug.cgi?id=2040030
+    data_sensitivity:
+      - technical
+    notification_emails:
+      - glean-team@mozilla.com
+    expires: never
+    no_lint:
+      - COMMON_PREFIX
+
   pending_pings:
     type: counter
     description: |

glean-core/python/glean/config.py

@@ -35,6 +35,8 @@ def __init__(
         enable_event_timestamps: bool = True,
         experimentation_id: Optional[str] = None,
         enable_internal_pings: bool = True,
+        max_pending_pings_count: Optional[int] = None,
+        max_pending_pings_directory_size: Optional[int] = None,
     ):
         """
         Args:
@@ -53,6 +55,12 @@ def __init__(
             experimentation_id (string): An experimentation identifier derived
                 by the application to be sent with all pings. Default: None.
             enable_internal_pings (bool): Whether to enable internal pings. Default: `True`.
+            max_pending_pings_count (int): Optional. The maximum number of pending
+                pings stored on disk. When exceeded, the oldest pings are deleted.
+                Defaults to 500.
+            max_pending_pings_directory_size (int): Optional. The maximum size in
+                bytes of the pending pings directory. When exceeded, the oldest pings
+                are deleted. Defaults to 50 MB.
         """
         if server_endpoint is None:
             server_endpoint = DEFAULT_TELEMETRY_ENDPOINT
@@ -66,6 +74,8 @@ def __init__(
         self._enable_event_timestamps = enable_event_timestamps
         self._experimentation_id = experimentation_id
         self._enable_internal_pings = enable_internal_pings
+        self._max_pending_pings_count = max_pending_pings_count
+        self._max_pending_pings_directory_size = max_pending_pings_directory_size
 
     @property
     def server_endpoint(self) -> str:
@@ -111,6 +121,16 @@ def enable_internal_pings(self) -> bool:
         """Whether to enable internal pings."""
         return self._enable_internal_pings
 
+    @property
+    def max_pending_pings_count(self) -> Optional[int]:
+        """The maximum number of pending pings stored on disk."""
+        return self._max_pending_pings_count
+
+    @property
+    def max_pending_pings_directory_size(self) -> Optional[int]:
+        """The maximum size in bytes of the pending pings directory."""
+        return self._max_pending_pings_directory_size
+
     @property
     def ping_uploader(self) -> net.BaseUploader:
         """The ping uploader implementation."""

glean-core/python/glean/glean.py

@@ -238,6 +238,8 @@ def initialize(
             ping_schedule={},
             ping_lifetime_threshold=0,
             ping_lifetime_max_time=0,
+            max_pending_pings_count=configuration.max_pending_pings_count,
+            max_pending_pings_directory_size=configuration.max_pending_pings_directory_size,
         )
 
         _uniffi.glean_initialize(cfg, client_info, callbacks)

glean-core/python/glean/net/ping_upload_worker.py

@@ -122,6 +122,8 @@ def _process(data_dir: Path, application_id: str, configuration) -> bool:
             ping_schedule={},
             ping_lifetime_threshold=0,
             ping_lifetime_max_time=0,
+            max_pending_pings_count=configuration.max_pending_pings_count,
+            max_pending_pings_directory_size=configuration.max_pending_pings_directory_size,
         )
         if not glean_initialize_for_subprocess(cfg):
             log.error("Couldn't initialize Glean in subprocess")

glean-core/rlb/src/lib.rs

@@ -129,6 +129,8 @@ fn initialize_internal(cfg: Configuration, client_info: ClientInfoMetrics) -> Op
         ping_schedule: cfg.ping_schedule,
         ping_lifetime_threshold: cfg.ping_lifetime_threshold as u64,
         ping_lifetime_max_time: cfg.ping_lifetime_max_time.as_millis() as u64,
+        max_pending_pings_count: None,
+        max_pending_pings_directory_size: None,
     };
 
     glean_core::glean_initialize(core_cfg, client_info.into(), callbacks);

glean-core/src/core/mod.rs

@@ -139,6 +139,8 @@ where
 ///     ping_schedule: Default::default(),
 ///     ping_lifetime_threshold: 1000,
 ///     ping_lifetime_max_time: 2000,
+///     max_pending_pings_count: None,
+///     max_pending_pings_directory_size: None,
 /// };
 /// let mut glean = Glean::new(cfg).unwrap();
 /// let ping = PingType::new("sample", true, false, true, true, true, vec![], vec![], true, vec![]);
@@ -214,6 +216,12 @@ impl Glean {
             rate_limit.seconds_per_interval,
             rate_limit.pings_per_interval,
         );
+        if let Some(n) = cfg.max_pending_pings_count {
+            upload_manager.set_max_pending_pings_count(n);
+        }
+        if let Some(n) = cfg.max_pending_pings_directory_size {
+            upload_manager.set_max_pending_pings_directory_size(n);
+        }
 
         // We only scan the pending ping directories when calling this from a subprocess,
         // when calling this from ::new we need to scan the directories after dealing with the upload state.
@@ -540,6 +548,8 @@ impl Glean {
             ping_schedule: Default::default(),
             ping_lifetime_threshold: 0,
             ping_lifetime_max_time: 0,
+            max_pending_pings_count: None,
+            max_pending_pings_directory_size: None,
         };
 
         let mut glean = Self::new(cfg).unwrap();

glean-core/src/glean.udl

@@ -111,6 +111,8 @@ dictionary InternalConfiguration {
     record<string, sequence<string>> ping_schedule;
     u64 ping_lifetime_threshold;
     u64 ping_lifetime_max_time; // in millis
+    u64? max_pending_pings_count;
+    u64? max_pending_pings_directory_size;
 };
 
 // How to specify the rate pings may be uploaded before they are throttled.

glean-core/src/internal_metrics.rs

@@ -233,6 +233,7 @@ pub struct UploadMetrics {
     pub discarded_exceeding_pings_size: MemoryDistributionMetric,
     pub pending_pings_directory_size: MemoryDistributionMetric,
     pub deleted_pings_after_quota_hit: CounterMetric,
+    pub pending_pings_deleted: LabeledMetric<CounterMetric>,
     pub pending_pings: CounterMetric,
     pub send_success: TimingDistributionMetric,
     pub send_failure: TimingDistributionMetric,
@@ -297,6 +298,20 @@ impl UploadMetrics {
                 dynamic_label: None,
             }),
 
+            pending_pings_deleted: LabeledMetric::<CounterMetric>::new(
+                LabeledMetricData::Common {
+                    cmd: CommonMetricData {
+                        name: "pending_pings_deleted".into(),
+                        category: "glean.upload".into(),
+                        send_in_pings: vec!["health".into()],
+                        lifetime: Lifetime::Ping,
+                        disabled: false,
+                        dynamic_label: None,
+                    },
+                },
+                Some(vec![Cow::from("count_quota"), Cow::from("size_quota")]),
+            ),
+
             pending_pings: CounterMetric::new(CommonMetricData {
                 name: "pending_pings".into(),
                 category: "glean.upload".into(),

glean-core/src/lib.rs

@@ -166,6 +166,10 @@ pub struct InternalConfiguration {
     pub ping_lifetime_threshold: u64,
     /// After what time to auto-flush. 0 disables it.
     pub ping_lifetime_max_time: u64,
+    /// Maximum number of pending pings on disk. Overrides the default when set.
+    pub max_pending_pings_count: Option<u64>,
+    /// Maximum size in bytes of the pending pings directory. Overrides the default when set.
+    pub max_pending_pings_directory_size: Option<u64>,
 }
 
 /// How to specify the rate at which pings may be uploaded before they are throttled.

glean-core/src/lib_unit_tests.rs

@@ -232,6 +232,8 @@ fn experimentation_id_is_set_correctly() {
         ping_schedule: Default::default(),
         ping_lifetime_threshold: 0,
         ping_lifetime_max_time: 0,
+        max_pending_pings_count: None,
+        max_pending_pings_directory_size: None,
     })
     .unwrap();