diff --git a/components/remote_settings/dumps/main/regions.timestamp b/components/remote_settings/dumps/main/regions.timestamp new file mode 100644 index 0000000000..60ae2308d5 --- /dev/null +++ b/components/remote_settings/dumps/main/regions.timestamp @@ -0,0 +1 @@ +1600363002708 \ No newline at end of file diff --git a/components/remote_settings/dumps/main/search-config-icons.timestamp b/components/remote_settings/dumps/main/search-config-icons.timestamp new file mode 100644 index 0000000000..1ec63ac809 --- /dev/null +++ b/components/remote_settings/dumps/main/search-config-icons.timestamp @@ -0,0 +1 @@ +1763049497744 \ No newline at end of file diff --git a/components/remote_settings/dumps/main/search-config-v2.timestamp b/components/remote_settings/dumps/main/search-config-v2.timestamp new file mode 100644 index 0000000000..4451d3c5b2 --- /dev/null +++ b/components/remote_settings/dumps/main/search-config-v2.timestamp @@ -0,0 +1 @@ +1764082724032 \ No newline at end of file diff --git a/components/remote_settings/dumps/main/search-telemetry-v2.timestamp b/components/remote_settings/dumps/main/search-telemetry-v2.timestamp new file mode 100644 index 0000000000..541e3d4dd7 --- /dev/null +++ b/components/remote_settings/dumps/main/search-telemetry-v2.timestamp @@ -0,0 +1 @@ +1757010621729 \ No newline at end of file diff --git a/components/remote_settings/dumps/main/summarizer-models-config.timestamp b/components/remote_settings/dumps/main/summarizer-models-config.timestamp new file mode 100644 index 0000000000..2e0b81c77a --- /dev/null +++ b/components/remote_settings/dumps/main/summarizer-models-config.timestamp @@ -0,0 +1 @@ +1755604678567 \ No newline at end of file diff --git a/components/remote_settings/dumps/main/translations-models.timestamp b/components/remote_settings/dumps/main/translations-models.timestamp new file mode 100644 index 0000000000..a76d3c9f57 --- /dev/null +++ b/components/remote_settings/dumps/main/translations-models.timestamp @@ -0,0 +1 @@ +1761148716130 \ No newline at end of file diff --git a/components/remote_settings/dumps/main/translations-wasm.timestamp b/components/remote_settings/dumps/main/translations-wasm.timestamp new file mode 100644 index 0000000000..dfdef03a43 --- /dev/null +++ b/components/remote_settings/dumps/main/translations-wasm.timestamp @@ -0,0 +1 @@ +1749069444811 \ No newline at end of file diff --git a/components/remote_settings/src/client.rs b/components/remote_settings/src/client.rs index 84a720d3e2..55e903e7dc 100644 --- a/components/remote_settings/src/client.rs +++ b/components/remote_settings/src/client.rs @@ -213,10 +213,17 @@ impl RemoteSettingsClient { &self.collection_name } + fn load_packaged_timestamp(&self) -> Option { + // Using the macro generated `get_packaged_timestamp` in macros.rs + Self::get_packaged_timestamp(&self.collection_name) + } + fn load_packaged_data(&self) -> Option { // Using the macro generated `get_packaged_data` in macros.rs - Self::get_packaged_data(&self.collection_name) - .and_then(|data| serde_json::from_str(data).ok()) + let str_data = Self::get_packaged_data(&self.collection_name)?; + let data: CollectionData = serde_json::from_str(str_data).ok()?; + debug_assert_eq!(data.timestamp, self.load_packaged_timestamp().unwrap()); + Some(data) } fn load_packaged_attachment(&self, filename: &str) -> Option<(&'static [u8], &'static str)> { @@ -241,6 +248,28 @@ impl RemoteSettingsClient { .collect() } + /// Returns the parsed packaged data, but only if it's newer than the data we have + /// in storage. This avoids parsing the packaged data if we won't use it. + fn get_packaged_data_if_newer( + &self, + storage: &mut Storage, + collection_url: &str, + ) -> Result> { + let packaged_ts = self.load_packaged_timestamp(); + let storage_ts = storage.get_last_modified_timestamp(collection_url)?; + let packaged_is_newer = match (packaged_ts, storage_ts) { + (Some(packaged_ts), Some(storage_ts)) => packaged_ts > storage_ts, + (Some(_), None) => true, // no storage data + (None, _) => false, // no packaged data + }; + + if packaged_is_newer { + Ok(self.load_packaged_data()) + } else { + Ok(None) + } + } + /// Get the current set of records. /// /// If records are not present in storage this will normally return None. Use `sync_if_empty = @@ -248,23 +277,15 @@ impl RemoteSettingsClient { pub fn get_records(&self, sync_if_empty: bool) -> Result>> { let mut inner = self.inner.lock(); let collection_url = inner.api_client.collection_url(); - let is_prod = inner.api_client.is_prod_server()?; - let packaged_data = if is_prod { - self.load_packaged_data() - } else { - None - }; // Case 1: The packaged data is more recent than the cache // // This happens when there's no cached data or when we get new packaged data because of a // product update - if let Some(packaged_data) = packaged_data { - let cached_timestamp = inner - .storage - .get_last_modified_timestamp(&collection_url)? - .unwrap_or(0); - if packaged_data.timestamp > cached_timestamp { + if inner.api_client.is_prod_server()? { + if let Some(packaged_data) = + self.get_packaged_data_if_newer(&mut inner.storage, &collection_url)? + { // Remove previously cached data (packaged data does not have tombstones like diff responses do). inner.storage.empty()?; // Insert new packaged data. diff --git a/components/remote_settings/src/macros.rs b/components/remote_settings/src/macros.rs index 34b14661be..dd071b0a98 100644 --- a/components/remote_settings/src/macros.rs +++ b/components/remote_settings/src/macros.rs @@ -18,6 +18,26 @@ macro_rules! packaged_collections { _ => None, } } + + /// Get just the timestamp, which is stored separately. This allows + /// checking which data is newer without paying the cost of parsing + /// the full packaged JSON. + fn get_packaged_timestamp(collection_name: &str) -> Option { + match collection_name { + $($collection => { + let timestamp_str = include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/dumps/", + $bucket, + "/", + $collection, + ".timestamp" + )); + timestamp_str.trim().parse().ok() + }),* + _ => None, + } + } }; } diff --git a/examples/remote-settings-cli/src/dump/client.rs b/examples/remote-settings-cli/src/dump/client.rs index 047a3af22d..a136c71c26 100644 --- a/examples/remote-settings-cli/src/dump/client.rs +++ b/examples/remote-settings-cli/src/dump/client.rs @@ -432,6 +432,10 @@ impl CollectionDownloader { } std::fs::write(&dumps_path, serde_json::to_string_pretty(&data)?)?; + // Write timestamp file for fast timestamp checking without JSON parsing + let timestamp_path = dumps_path.with_extension("timestamp"); + std::fs::write(×tamp_path, data.timestamp.to_string())?; + // Count attachments needing updates for record in &data.data { if let Some(attachment) = record.get("attachment") {