Bug 2004779 - Avoid parsing JSON for the packaged data if it's outdated, by storing the timestamp in a separate file.

Author: mstange

components/remote_settings/dumps/main/regions.timestamp

@@ -0,0 +1 @@
+1600363002708

components/remote_settings/dumps/main/search-config-icons.timestamp

@@ -0,0 +1 @@
+1763049497744

components/remote_settings/dumps/main/search-config-v2.timestamp

@@ -0,0 +1 @@
+1764082724032

components/remote_settings/dumps/main/search-telemetry-v2.timestamp

@@ -0,0 +1 @@
+1757010621729

components/remote_settings/dumps/main/summarizer-models-config.timestamp

@@ -0,0 +1 @@
+1755604678567

components/remote_settings/dumps/main/translations-models.timestamp

@@ -0,0 +1 @@
+1761148716130

components/remote_settings/dumps/main/translations-wasm.timestamp

@@ -0,0 +1 @@
+1749069444811

components/remote_settings/src/client.rs

@@ -219,6 +219,11 @@ impl<C: ApiClient> RemoteSettingsClient<C> {
             .and_then(|data| serde_json::from_str(data).ok())
     }
 
+    fn load_packaged_timestamp(&self) -> Option<u64> {
+        // Using the macro generated `get_packaged_timestamp` in macros.rs
+        Self::get_packaged_timestamp(&self.collection_name)
+    }
+
     fn load_packaged_attachment(&self, filename: &str) -> Option<(&'static [u8], &'static str)> {
         // Using the macro generated `get_packaged_attachment` in macros.rs
         Self::get_packaged_attachment(&self.collection_name, filename)
@@ -241,30 +246,47 @@ impl<C: ApiClient> RemoteSettingsClient<C> {
             .collect()
     }
 
+    /// Returns the parsed packaged data, but only if it's newer than the data we have
+    /// in storage. This avoids parsing the packaged data if we won't use it.
+    ///
+    /// Does not mutate `storage`.
+    fn get_packaged_data_if_newer(
+        &self,
+        storage: &mut Storage,
+        collection_url: &str,
+    ) -> Result<Option<CollectionData>> {
+        let packaged_timestamp = match self.load_packaged_timestamp() {
+            Some(ts) => ts,
+            None => return Ok(None),
+        };
+        match storage.get_last_modified_timestamp(collection_url)? {
+            Some(storage_timestamp) if storage_timestamp >= packaged_timestamp => {
+                // Data in storage is newer, don't parse packaged data.
+                Ok(None)
+            }
+            _ => {
+                // Packaged data is newer, parse packaged JSON
+                Ok(self.load_packaged_data())
+            }
+        }
+    }
+
     /// Get the current set of records.
     ///
     /// If records are not present in storage this will normally return None.  Use `sync_if_empty =
     /// true` to change this behavior and perform a network request in this case.
     pub fn get_records(&self, sync_if_empty: bool) -> Result<Option<Vec<RemoteSettingsRecord>>> {
         let mut inner = self.inner.lock();
         let collection_url = inner.api_client.collection_url();
-        let is_prod = inner.api_client.is_prod_server()?;
-        let packaged_data = if is_prod {
-            self.load_packaged_data()
-        } else {
-            None
-        };
 
         // Case 1: The packaged data is more recent than the cache
         //
         // This happens when there's no cached data or when we get new packaged data because of a
         // product update
-        if let Some(packaged_data) = packaged_data {
-            let cached_timestamp = inner
-                .storage
-                .get_last_modified_timestamp(&collection_url)?
-                .unwrap_or(0);
-            if packaged_data.timestamp > cached_timestamp {
+        if inner.api_client.is_prod_server()? {
+            if let Some(packaged_data) =
+                self.get_packaged_data_if_newer(&mut inner.storage, &collection_url)?
+            {
                 // Remove previously cached data (packaged data does not have tombstones like diff responses do).
                 inner.storage.empty()?;
                 // Insert new packaged data.

components/remote_settings/src/macros.rs

@@ -18,6 +18,26 @@ macro_rules! packaged_collections {
                 _ => None,
             }
         }
+
+        /// Get just the timestamp, which is stored separately. This allows
+        /// checking which data is newer without paying the cost of parsing
+        /// the full packaged JSON.
+        fn get_packaged_timestamp(collection_name: &str) -> Option<u64> {
+            match collection_name {
+                $($collection => {
+                    let timestamp_str = include_str!(concat!(
+                        env!("CARGO_MANIFEST_DIR"),
+                        "/dumps/",
+                        $bucket,
+                        "/",
+                        $collection,
+                        ".timestamp"
+                    ));
+                    timestamp_str.trim().parse().ok()
+                }),*
+                _ => None,
+            }
+        }
     };
 }
 

examples/remote-settings-cli/src/dump/client.rs

@@ -432,6 +432,10 @@ impl CollectionDownloader {
             }
             std::fs::write(&dumps_path, serde_json::to_string_pretty(&data)?)?;
 
+            // Write timestamp file for fast timestamp checking without JSON parsing
+            let timestamp_path = dumps_path.with_extension("timestamp");
+            std::fs::write(&timestamp_path, data.timestamp.to_string())?;
+
             // Count attachments needing updates
             for record in &data.data {
                 if let Some(attachment) = record.get("attachment") {