Bug 2004779 - Avoid parsing JSON for the packaged data if it's outdated, by storing the timestamp in a separate file. (#7121)

Author: mstange

components/remote_settings/dumps/main/regions.timestamp

@@ -0,0 +1 @@
+1600363002708

components/remote_settings/dumps/main/search-config-icons.timestamp

@@ -0,0 +1 @@
+1763049497744

components/remote_settings/dumps/main/search-config-v2.timestamp

@@ -0,0 +1 @@
+1764082724032

components/remote_settings/dumps/main/search-telemetry-v2.timestamp

@@ -0,0 +1 @@
+1757010621729

components/remote_settings/dumps/main/summarizer-models-config.timestamp

@@ -0,0 +1 @@
+1755604678567

components/remote_settings/dumps/main/translations-models.timestamp

@@ -0,0 +1 @@
+1761148716130

components/remote_settings/dumps/main/translations-wasm.timestamp

@@ -0,0 +1 @@
+1749069444811

components/remote_settings/src/client.rs

@@ -213,10 +213,17 @@ impl<C: ApiClient> RemoteSettingsClient<C> {
         &self.collection_name
     }
 
+    fn load_packaged_timestamp(&self) -> Option<u64> {
+        // Using the macro generated `get_packaged_timestamp` in macros.rs
+        Self::get_packaged_timestamp(&self.collection_name)
+    }
+
     fn load_packaged_data(&self) -> Option<CollectionData> {
         // Using the macro generated `get_packaged_data` in macros.rs
-        Self::get_packaged_data(&self.collection_name)
-            .and_then(|data| serde_json::from_str(data).ok())
+        let str_data = Self::get_packaged_data(&self.collection_name)?;
+        let data: CollectionData = serde_json::from_str(str_data).ok()?;
+        debug_assert_eq!(data.timestamp, self.load_packaged_timestamp().unwrap());
+        Some(data)
     }
 
     fn load_packaged_attachment(&self, filename: &str) -> Option<(&'static [u8], &'static str)> {
@@ -241,30 +248,44 @@ impl<C: ApiClient> RemoteSettingsClient<C> {
             .collect()
     }
 
+    /// Returns the parsed packaged data, but only if it's newer than the data we have
+    /// in storage. This avoids parsing the packaged data if we won't use it.
+    fn get_packaged_data_if_newer(
+        &self,
+        storage: &mut Storage,
+        collection_url: &str,
+    ) -> Result<Option<CollectionData>> {
+        let packaged_ts = self.load_packaged_timestamp();
+        let storage_ts = storage.get_last_modified_timestamp(collection_url)?;
+        let packaged_is_newer = match (packaged_ts, storage_ts) {
+            (Some(packaged_ts), Some(storage_ts)) => packaged_ts > storage_ts,
+            (Some(_), None) => true, // no storage data
+            (None, _) => false,      // no packaged data
+        };
+
+        if packaged_is_newer {
+            Ok(self.load_packaged_data())
+        } else {
+            Ok(None)
+        }
+    }
+
     /// Get the current set of records.
     ///
     /// If records are not present in storage this will normally return None.  Use `sync_if_empty =
     /// true` to change this behavior and perform a network request in this case.
     pub fn get_records(&self, sync_if_empty: bool) -> Result<Option<Vec<RemoteSettingsRecord>>> {
         let mut inner = self.inner.lock();
         let collection_url = inner.api_client.collection_url();
-        let is_prod = inner.api_client.is_prod_server()?;
-        let packaged_data = if is_prod {
-            self.load_packaged_data()
-        } else {
-            None
-        };
 
         // Case 1: The packaged data is more recent than the cache
         //
         // This happens when there's no cached data or when we get new packaged data because of a
         // product update
-        if let Some(packaged_data) = packaged_data {
-            let cached_timestamp = inner
-                .storage
-                .get_last_modified_timestamp(&collection_url)?
-                .unwrap_or(0);
-            if packaged_data.timestamp > cached_timestamp {
+        if inner.api_client.is_prod_server()? {
+            if let Some(packaged_data) =
+                self.get_packaged_data_if_newer(&mut inner.storage, &collection_url)?
+            {
                 // Remove previously cached data (packaged data does not have tombstones like diff responses do).
                 inner.storage.empty()?;
                 // Insert new packaged data.

components/remote_settings/src/macros.rs

@@ -18,6 +18,26 @@ macro_rules! packaged_collections {
                 _ => None,
             }
         }
+
+        /// Get just the timestamp, which is stored separately. This allows
+        /// checking which data is newer without paying the cost of parsing
+        /// the full packaged JSON.
+        fn get_packaged_timestamp(collection_name: &str) -> Option<u64> {
+            match collection_name {
+                $($collection => {
+                    let timestamp_str = include_str!(concat!(
+                        env!("CARGO_MANIFEST_DIR"),
+                        "/dumps/",
+                        $bucket,
+                        "/",
+                        $collection,
+                        ".timestamp"
+                    ));
+                    timestamp_str.trim().parse().ok()
+                }),*
+                _ => None,
+            }
+        }
     };
 }
 

examples/remote-settings-cli/src/dump/client.rs

@@ -432,6 +432,10 @@ impl CollectionDownloader {
             }
             std::fs::write(&dumps_path, serde_json::to_string_pretty(&data)?)?;
 
+            // Write timestamp file for fast timestamp checking without JSON parsing
+            let timestamp_path = dumps_path.with_extension("timestamp");
+            std::fs::write(&timestamp_path, data.timestamp.to_string())?;
+
             // Count attachments needing updates
             for record in &data.data {
                 if let Some(attachment) = record.get("attachment") {