From a066025bc61e438e3ecc62341362dac585e0407b Mon Sep 17 00:00:00 2001 From: Mikhail Date: Wed, 13 Aug 2025 23:16:17 +0400 Subject: [PATCH 1/7] Introduce flat builders --- src/blocker.rs | 6 +- src/engine.rs | 4 +- src/filters/fb_builder.rs | 236 +++--- src/filters/fb_network.rs | 4 +- src/flatbuffers/containers/flat_map.rs | 71 ++ src/flatbuffers/containers/flat_multimap.rs | 52 +- src/flatbuffers/containers/flat_serialize.rs | 109 +++ src/flatbuffers/containers/flat_set.rs | 21 +- src/flatbuffers/containers/mod.rs | 2 + src/flatbuffers/unsafe_tools.rs | 5 +- .../flatbuffers/containers/flat_multimap.rs | 86 +- tests/unit/flatbuffers/containers/flat_set.rs | 60 ++ .../containers/test_containers.fbs | 25 + .../containers/test_containers_generated.rs | 785 ++++++++++++++++++ 14 files changed, 1345 insertions(+), 121 deletions(-) create mode 100644 src/flatbuffers/containers/flat_map.rs create mode 100644 src/flatbuffers/containers/flat_serialize.rs create mode 100644 tests/unit/flatbuffers/containers/test_containers.fbs create mode 100644 tests/unit/flatbuffers/containers/test_containers_generated.rs diff --git a/src/blocker.rs b/src/blocker.rs index fe31fb18..5a9f77bd 100644 --- a/src/blocker.rs +++ b/src/blocker.rs @@ -440,11 +440,9 @@ impl Blocker { network_filters: Vec, options: &BlockerOptions, ) -> Self { - use crate::filters::fb_builder::FlatBufferBuilder; - use crate::filters::fb_network::FilterDataContext; + use crate::filters::{fb_builder::make_flatbuffer, fb_network::FilterDataContext}; - let memory = - FlatBufferBuilder::make_flatbuffer(network_filters, options.enable_optimizations); + let memory = make_flatbuffer(network_filters, options.enable_optimizations); let filter_data_context = FilterDataContext::new(memory); Self::from_context(filter_data_context) } diff --git a/src/engine.rs b/src/engine.rs index 2d12cfec..ce1e6cd5 100644 --- a/src/engine.rs +++ b/src/engine.rs @@ -2,7 +2,7 @@ use crate::blocker::{Blocker, BlockerResult}; use crate::cosmetic_filter_cache::{CosmeticFilterCache, UrlSpecificResources}; -use crate::filters::fb_builder::FlatBufferBuilder; +use crate::filters::fb_builder::make_flatbuffer; use crate::filters::fb_network::{FilterDataContext, FilterDataContextRef}; use crate::lists::{FilterSet, ParseOptions}; use crate::regex_manager::RegexManagerDiscardPolicy; @@ -103,7 +103,7 @@ impl Engine { .. } = set; - let memory = FlatBufferBuilder::make_flatbuffer(network_filters, optimize); + let memory = make_flatbuffer(network_filters, optimize); let filter_data_context = FilterDataContext::new(memory); diff --git a/src/filters/fb_builder.rs b/src/filters/fb_builder.rs index d83833bc..6bd7db1b 100644 --- a/src/filters/fb_builder.rs +++ b/src/filters/fb_builder.rs @@ -6,9 +6,11 @@ use std::collections::{HashMap, HashSet}; use std::vec; -use flatbuffers::WIPOffset; +use flatbuffers::{ForwardsUOffset, Vector, WIPOffset}; use crate::filters::network::{NetworkFilter, NetworkFilterMaskHelper}; +use crate::flatbuffers::containers::flat_multimap::FlatMultiMapBuilder; +use crate::flatbuffers::containers::flat_serialize::{FlatBuilder, FlatSerialize}; use crate::flatbuffers::unsafe_tools::VerifiedFlatbufferMemory; use crate::network_filter_list::token_histogram; use crate::optimizer; @@ -29,65 +31,83 @@ pub(crate) enum NetworkFilterListId { } #[derive(Default, Clone)] -struct FilterListBuilder { +struct NetworkFilterListBuilder { filters: Vec, + optimize: bool, } -pub(crate) struct FlatBufferBuilder { - lists: Vec, - - unique_domains_hashes: Vec, - unique_domains_hashes_map: HashMap, - index: u32, +struct EngineFlatBuilder<'a> { + fb_builder: flatbuffers::FlatBufferBuilder<'a>, + unique_domains_hashes: HashMap, + unique_domains_hashes_vec: Vec, } -impl FlatBufferBuilder { - pub fn new(list_count: usize) -> Self { +impl Default for EngineFlatBuilder<'_> { + fn default() -> Self { Self { - lists: vec![FilterListBuilder::default(); list_count], - unique_domains_hashes: vec![], - unique_domains_hashes_map: HashMap::new(), - index: 0, + fb_builder: flatbuffers::FlatBufferBuilder::new(), + unique_domains_hashes: HashMap::new(), + unique_domains_hashes_vec: Vec::new(), } } +} - fn get_or_insert_unique_domain_hash(&mut self, h: &Hash) -> u32 { - if let Some(&index) = self.unique_domains_hashes_map.get(h) { +impl<'a> EngineFlatBuilder<'a> { + pub fn get_or_insert_unique_domain_hash(&mut self, hash: &Hash) -> u32 { + if let Some(&index) = self.unique_domains_hashes.get(hash) { return index; } - let index = self.unique_domains_hashes.len() as u32; - self.unique_domains_hashes.push(*h); - self.unique_domains_hashes_map.insert(*h, index); + let index = self.unique_domains_hashes_vec.len() as u32; + self.unique_domains_hashes_vec.push(*hash); + self.unique_domains_hashes.insert(*hash, index); index } - pub fn add_filter(&mut self, network_filter: NetworkFilter, list_id: u32) { - self.lists[list_id as usize].filters.push(network_filter); + pub fn write_unique_domains(&mut self) -> WIPOffset> { + self.fb_builder + .create_vector(&self.unique_domains_hashes_vec) } +} - fn write_filter<'a>( - &mut self, - builder: &mut flatbuffers::FlatBufferBuilder<'a>, +impl<'a> FlatBuilder<'a> for EngineFlatBuilder<'a> { + fn create_string(&mut self, s: &str) -> WIPOffset<&'a str> { + self.fb_builder.create_string(s) + } + + fn raw_builder(&mut self) -> &mut flatbuffers::FlatBufferBuilder<'a> { + &mut self.fb_builder + } +} + +struct NetworkRulesBuilder { + lists: Vec, +} + +impl<'a> FlatSerialize<'a, EngineFlatBuilder<'a>> for &NetworkFilter { + type Output = WIPOffset>; + + fn serialize( network_filter: &NetworkFilter, + builder: &mut EngineFlatBuilder<'a>, ) -> WIPOffset> { let opt_domains = network_filter.opt_domains.as_ref().map(|v| { let mut o: Vec = v .iter() - .map(|x| self.get_or_insert_unique_domain_hash(x)) + .map(|x| builder.get_or_insert_unique_domain_hash(x)) .collect(); o.sort_unstable(); o.dedup(); - builder.create_vector(&o) + FlatSerialize::serialize(o, builder) }); let opt_not_domains = network_filter.opt_not_domains.as_ref().map(|v| { let mut o: Vec = v .iter() - .map(|x| self.get_or_insert_unique_domain_hash(x)) + .map(|x| builder.get_or_insert_unique_domain_hash(x)) .collect(); o.sort_unstable(); o.dedup(); - builder.create_vector(&o) + FlatSerialize::serialize(o, builder) }); let modifier_option = network_filter @@ -111,7 +131,7 @@ impl FlatBufferBuilder { .iter() .map(|s| builder.create_string(s)) .collect(); - Some(builder.create_vector(&offsets)) + Some(FlatSerialize::serialize(offsets, builder)) } else { None }; @@ -121,8 +141,8 @@ impl FlatBufferBuilder { .as_ref() .map(|v| builder.create_string(v.as_str())); - let filter = fb::NetworkFilter::create( - builder, + let network_filter = fb::NetworkFilter::create( + &mut builder.fb_builder, &fb::NetworkFilterArgs { mask: network_filter.mask.bits(), patterns, @@ -135,53 +155,32 @@ impl FlatBufferBuilder { }, ); - self.index += 1; - - filter + network_filter } +} - pub fn finish(&mut self, optimize: bool) -> VerifiedFlatbufferMemory { - let mut builder = flatbuffers::FlatBufferBuilder::new(); - let mut flat_network_rules = vec![]; - - let lists = std::mem::take(&mut self.lists); - for (list_id, list) in lists.into_iter().enumerate() { - // Don't optimize removeparam, since it can fuse filters without respecting distinct - let optimize = optimize && list_id != NetworkFilterListId::RemoveParam as usize; - - flat_network_rules.push(self.write_filter_list(&mut builder, list.filters, optimize)); +impl NetworkFilterListBuilder { + fn new(optimize: bool) -> Self { + Self { + filters: vec![], + optimize, } - - // Create vectors first to avoid simultaneous mutable borrows of `builder`. - let network_rules = builder.create_vector(&flat_network_rules); - let unique_vec = builder.create_vector(&self.unique_domains_hashes); - - let root = fb::Engine::create( - &mut builder, - &fb::EngineArgs { - network_rules: Some(network_rules), - unique_domains_hashes: Some(unique_vec), - }, - ); - - builder.finish(root, None); - - // TODO: consider using builder.collapse() to avoid reallocating memory. - VerifiedFlatbufferMemory::from_builder(&builder) } +} - pub fn write_filter_list<'a>( - &mut self, - builder: &mut flatbuffers::FlatBufferBuilder<'a>, - filters: Vec, - optimize: bool, +impl<'a> FlatSerialize<'a, EngineFlatBuilder<'a>> for NetworkFilterListBuilder { + type Output = WIPOffset>; + fn serialize( + value: Self, + builder: &mut EngineFlatBuilder<'a>, ) -> WIPOffset> { let mut filter_map = HashMap::>>>::new(); let mut optimizable = HashMap::>::new(); // Compute tokens for all filters - let filter_tokens: Vec<_> = filters + let filter_tokens: Vec<_> = value + .filters .into_iter() .map(|filter| { let tokens = filter.get_tokens(); @@ -193,11 +192,11 @@ impl FlatBufferBuilder { let (total_number_of_tokens, tokens_histogram) = token_histogram(&filter_tokens); { - for (network_filter, multi_tokens) in filter_tokens { - let flat_filter = if !optimize + for (network_filter, multi_tokens) in filter_tokens.into_iter() { + let flat_filter = if !value.optimize || !optimizer::is_filter_optimizable_by_patterns(&network_filter) { - Some(self.write_filter(builder, &network_filter)) + Some(FlatSerialize::serialize(&network_filter, builder)) } else { None }; @@ -232,7 +231,7 @@ impl FlatBufferBuilder { } } - if optimize { + if value.optimize { // Sort the entries to ensure deterministic iteration order let mut optimizable_entries: Vec<_> = optimizable.drain().collect(); optimizable_entries.sort_unstable_by_key(|(token, _)| *token); @@ -241,7 +240,7 @@ impl FlatBufferBuilder { let optimized = optimizer::optimize(v); for filter in optimized { - let flat_filter = self.write_filter(builder, &filter); + let flat_filter = FlatSerialize::serialize(&filter, builder); filter_map.entry(token).or_default().push(flat_filter); } } @@ -252,49 +251,35 @@ impl FlatBufferBuilder { ); } - let len = filter_map.len(); - - // Convert filter_map keys to a sorted vector of (hash, filter_indices). - let mut entries: Vec<_> = filter_map.drain().collect(); - entries.sort_unstable_by_key(|(k, _)| *k); - - // Convert sorted_entries to two flatbuffers vectors. - let mut flat_index: Vec = Vec::with_capacity(len); - let mut flat_values: Vec<_> = Vec::with_capacity(len); - for (key, filter_indices) in entries { - for &filter_index in &filter_indices { - flat_index.push(key); - flat_values.push(filter_index); - } - } - - let filter_map_index = builder.create_vector(&flat_index); - let filter_map_values = builder.create_vector(&flat_values); + let flat_filter_map_builder = FlatMultiMapBuilder::from_filter_map(filter_map); + let flat_filter_map = FlatMultiMapBuilder::finish(flat_filter_map_builder, builder); fb::NetworkFilterList::create( - builder, + builder.raw_builder(), &fb::NetworkFilterListArgs { - filter_map_index: Some(filter_map_index), - filter_map_values: Some(filter_map_values), + filter_map_index: Some(flat_filter_map.keys), + filter_map_values: Some(flat_filter_map.values), }, ) } +} - pub fn make_flatbuffer( - network_filters: Vec, - optimize: bool, - ) -> VerifiedFlatbufferMemory { - type FilterId = NetworkFilterListId; - let mut builder = FlatBufferBuilder::new(FilterId::Size as usize); +impl NetworkRulesBuilder { + pub fn from_rules(network_filters: Vec, optimize: bool) -> Self { + let mut lists = vec![]; + for list_id in 0..NetworkFilterListId::Size as usize { + // Don't optimize removeparam, since it can fuse filters without respecting distinct + let optimize = optimize && list_id != NetworkFilterListId::RemoveParam as usize; + lists.push(NetworkFilterListBuilder::new(optimize)); + } + let mut self_ = Self { lists }; let mut badfilter_ids: HashSet = HashSet::new(); - for filter in network_filters.iter() { + for filter in network_filters.into_iter() { if filter.is_badfilter() { badfilter_ids.insert(filter.get_id_without_badfilter()); } - } - for filter in network_filters.into_iter() { - // skip any bad filters + let filter_id = filter.get_id(); if badfilter_ids.contains(&filter_id) || filter.is_badfilter() { continue; @@ -302,8 +287,9 @@ impl FlatBufferBuilder { // Redirects are independent of blocking behavior. if filter.is_redirect() { - builder.add_filter(filter.clone(), FilterId::Redirects as u32); + self_.add_filter(filter.clone(), NetworkFilterListId::Redirects); } + type FilterId = NetworkFilterListId; let list_id: FilterId = if filter.is_csp() { FilterId::Csp @@ -326,9 +312,47 @@ impl FlatBufferBuilder { continue; }; - builder.add_filter(filter, list_id as u32); + self_.add_filter(filter, list_id); } - builder.finish(optimize) + self_ + } + + fn add_filter(&mut self, network_filter: NetworkFilter, list_id: NetworkFilterListId) { + self.lists[list_id as usize].filters.push(network_filter); + } +} + +impl<'a> FlatSerialize<'a, EngineFlatBuilder<'a>> for NetworkRulesBuilder { + type Output = WIPOffset>>>; + fn serialize( + value: Self, + builder: &mut EngineFlatBuilder<'a>, + ) -> WIPOffset>>> { + let flat_network_rules: Vec<_> = value + .lists + .into_iter() + .map(|list| FlatSerialize::serialize(list, builder)) + .collect(); + builder.raw_builder().create_vector(&flat_network_rules) } } + +pub fn make_flatbuffer( + network_filters: Vec, + optimize: bool, +) -> VerifiedFlatbufferMemory { + let mut builder = EngineFlatBuilder::default(); + let network_rules_builder = NetworkRulesBuilder::from_rules(network_filters, optimize); + let network_rules = FlatSerialize::serialize(network_rules_builder, &mut builder); + let unique_domains_hashes = Some(builder.write_unique_domains()); + let engine = fb::Engine::create( + builder.raw_builder(), + &fb::EngineArgs { + network_rules: Some(network_rules), + unique_domains_hashes, + }, + ); + builder.raw_builder().finish(engine, None); + VerifiedFlatbufferMemory::from_builder(builder.raw_builder()) +} diff --git a/src/filters/fb_network.rs b/src/filters/fb_network.rs index c6d7940c..325bb2ea 100644 --- a/src/filters/fb_network.rs +++ b/src/filters/fb_network.rs @@ -2,7 +2,7 @@ use std::collections::HashMap; -use crate::filters::fb_builder::FlatBufferBuilder; +use crate::filters::fb_builder::make_flatbuffer; use crate::filters::network::{NetworkFilterMask, NetworkFilterMaskHelper, NetworkMatchable}; use crate::flatbuffers::unsafe_tools::{fb_vector_to_slice, VerifiedFlatbufferMemory}; @@ -84,7 +84,7 @@ pub(crate) struct FilterDataContext { impl Default for FilterDataContext { fn default() -> Self { Self { - memory: FlatBufferBuilder::make_flatbuffer(vec![], false), + memory: make_flatbuffer(vec![], false), unique_domains_hashes_map: HashMap::new(), } } diff --git a/src/flatbuffers/containers/flat_map.rs b/src/flatbuffers/containers/flat_map.rs new file mode 100644 index 00000000..71dc6bcf --- /dev/null +++ b/src/flatbuffers/containers/flat_map.rs @@ -0,0 +1,71 @@ +#![allow(dead_code)] // Unused code is allowed during cosmetic filter migration + +use std::marker::PhantomData; + +use crate::flatbuffers::containers; +use containers::flat_serialize::{FlatBuilder, FlatMapBuilderOutput, FlatSerialize}; +use containers::sorted_index::SortedIndex; +use flatbuffers::{Follow, Vector}; + +pub(crate) struct FlatMapView<'a, I: Ord, V, Keys> +where + Keys: SortedIndex, + V: Follow<'a>, +{ + keys: Keys, + values: Vector<'a, V>, + _phantom: PhantomData, +} + +impl<'a, I: Ord + Copy, V, Keys> FlatMapView<'a, I, V, Keys> +where + Keys: SortedIndex + Clone, + V: flatbuffers::Follow<'a>, +{ + pub fn new(keys: Keys, values: Vector<'a, V>) -> Self { + debug_assert!(keys.len() == values.len()); + Self { + keys, + values, + _phantom: PhantomData, + } + } + + pub fn get(&self, key: I) -> Option<>::Inner> { + let index = self.keys.partition_point(|x| *x < key); + if index < self.keys.len() && self.keys.get(index) == key { + Some(self.values.get(index)) + } else { + None + } + } +} + +pub(crate) struct FlatMapBuilder; + +impl FlatMapBuilder { + pub fn finish<'a, I, V, B: FlatBuilder<'a>>( + value: std::collections::HashMap, + builder: &mut B, + ) -> FlatMapBuilderOutput<'a, I, V, B> + where + I: FlatSerialize<'a, B> + Ord, + V: FlatSerialize<'a, B>, + { + let mut entries: Vec<_> = value.into_iter().collect(); + entries.sort_unstable_by(|(a, _), (b, _)| a.cmp(b)); + + let mut indexes = Vec::with_capacity(entries.len()); + let mut values = Vec::with_capacity(entries.len()); + + for (key, value) in entries.into_iter() { + indexes.push(FlatSerialize::serialize(key, builder)); + values.push(FlatSerialize::serialize(value, builder)); + } + + FlatMapBuilderOutput { + keys: builder.raw_builder().create_vector(&indexes), + values: builder.raw_builder().create_vector(&values), + } + } +} diff --git a/src/flatbuffers/containers/flat_multimap.rs b/src/flatbuffers/containers/flat_multimap.rs index a90dcc59..99b6255f 100644 --- a/src/flatbuffers/containers/flat_multimap.rs +++ b/src/flatbuffers/containers/flat_multimap.rs @@ -1,6 +1,8 @@ -use std::marker::PhantomData; +use std::{collections::HashMap, marker::PhantomData}; -use crate::flatbuffers::containers::sorted_index::SortedIndex; +use crate::flatbuffers::containers; +use containers::flat_serialize::{FlatBuilder, FlatMapBuilderOutput, FlatSerialize}; +use containers::sorted_index::SortedIndex; use flatbuffers::{Follow, Vector}; /// A map-like container that uses flatbuffer references. @@ -81,6 +83,52 @@ where } } +#[derive(Default)] +pub(crate) struct FlatMultiMapBuilder { + map: HashMap>, +} + +impl FlatMultiMapBuilder { + pub fn from_filter_map(map: HashMap>) -> Self { + Self { map } + } + + #[allow(dead_code)] // Unused code is allowed during cosmetic filter migration + pub fn insert(&mut self, key: I, value: V) { + self.map.entry(key).or_default().push(value); + } + + pub fn finish<'a, B: FlatBuilder<'a>>( + value: Self, + builder: &mut B, + ) -> FlatMapBuilderOutput<'a, I, V, B> + where + I: FlatSerialize<'a, B>, + V: FlatSerialize<'a, B>, + { + let mut entries: Vec<_> = value.map.into_iter().collect(); + entries.sort_unstable_by(|(a, _), (b, _)| a.cmp(b)); + let mut indexes = Vec::with_capacity(entries.len()); + let mut values = Vec::with_capacity(entries.len()); + + for (key, mv) in entries.into_iter() { + let index = FlatSerialize::serialize(key, builder); + for value in mv.into_iter() { + indexes.push(index.clone()); + values.push(FlatSerialize::serialize(value, builder)); + } + } + + let indexes_vec = builder.raw_builder().create_vector(&indexes); + let values_vec = builder.raw_builder().create_vector(&values); + + FlatMapBuilderOutput { + keys: indexes_vec, + values: values_vec, + } + } +} + #[cfg(test)] #[path = "../../../tests/unit/flatbuffers/containers/flat_multimap.rs"] mod unit_tests; diff --git a/src/flatbuffers/containers/flat_serialize.rs b/src/flatbuffers/containers/flat_serialize.rs new file mode 100644 index 00000000..09ab1234 --- /dev/null +++ b/src/flatbuffers/containers/flat_serialize.rs @@ -0,0 +1,109 @@ +use flatbuffers::{Vector, WIPOffset}; + +// A generic builder trait that is used to serialize flatbuffers. +// flatbuffers::FlatBufferBuilder can be used as Builder. +// Although, you can extend it to create a custom builder. +pub trait FlatBuilder<'a> { + fn create_string(&mut self, s: &str) -> WIPOffset<&'a str>; + fn raw_builder(&mut self) -> &mut flatbuffers::FlatBufferBuilder<'a>; +} + +// The trait to serialize structure into flatbuffer. +// * Implement the traits directly if the structure has a direct representation +// as a flatbuffer. I.e. for HashSet, Vec, etc. +// * Prefer using implent the traits for MyStruct instead of &MyStruct to +// prevent cloning. I.e. FlatMultiMapBuilder (wrapping HashMap>). +// * Make MyStructBuilder implement FlatSerialize, if extra work is required +// during building. +// * Make MyStructBuilder with finish() INSTEAD of implementing FlatSerialize +// if the output doesn't fit the trait. I.e. FlatMapBuilder, that returns +// a pair of vectors. +// * Axillary functions (i.e. string deduplication) can be implemented +// in a custom builder. +pub trait FlatSerialize<'b, B: FlatBuilder<'b>>: Sized { + type Output: Sized + Clone + flatbuffers::Push + 'b; + fn serialize(value: Self, builder: &mut B) -> Self::Output; +} + +impl<'b> FlatBuilder<'b> for flatbuffers::FlatBufferBuilder<'b> { + fn create_string(&mut self, s: &str) -> WIPOffset<&'b str> { + self.create_string(s) + } + fn raw_builder(&mut self) -> &mut flatbuffers::FlatBufferBuilder<'b> { + self + } +} + +impl<'b, B: FlatBuilder<'b>> FlatSerialize<'b, B> for String { + type Output = WIPOffset<&'b str>; + fn serialize(value: Self, builder: &mut B) -> Self::Output { + builder.create_string(&value) + } +} + +impl<'b, B: FlatBuilder<'b>> FlatSerialize<'b, B> for &str { + type Output = WIPOffset<&'b str>; + fn serialize(value: Self, builder: &mut B) -> Self::Output { + builder.create_string(value) + } +} + +impl<'b, B: FlatBuilder<'b>> FlatSerialize<'b, B> for u32 { + type Output = u32; + fn serialize(value: Self, _builder: &mut B) -> Self::Output { + value + } +} + +impl<'b, B: FlatBuilder<'b>> FlatSerialize<'b, B> for u64 { + type Output = u64; + fn serialize(value: Self, _builder: &mut B) -> Self::Output { + value + } +} + +impl<'b, B: FlatBuilder<'b>, T: 'b> FlatSerialize<'b, B> for WIPOffset { + type Output = WIPOffset; + fn serialize(value: Self, _builder: &mut B) -> Self::Output { + value + } +} + +pub(crate) type WIPFlatVec<'b, T, B> = + WIPOffset>::Output as flatbuffers::Push>::Output>>; + +// Serialize a vector of items if T implements FlatSerialize. +// It puts the items first, then puts the vector of offsets. +impl<'b, B: FlatBuilder<'b>, T: FlatSerialize<'b, B>> FlatSerialize<'b, B> for Vec { + type Output = WIPFlatVec<'b, T, B>; + fn serialize(value: Self, builder: &mut B) -> Self::Output { + let v = value + .into_iter() + .map(|x| FlatSerialize::serialize(x, builder)) + .collect::>(); + builder.raw_builder().create_vector(&v) + } +} + +pub(crate) struct FlatMapBuilderOutput<'b, I, V, B: FlatBuilder<'b>> +where + I: FlatSerialize<'b, B>, + V: FlatSerialize<'b, B>, +{ + pub(crate) keys: WIPFlatVec<'b, I, B>, + pub(crate) values: WIPFlatVec<'b, V, B>, +} + +// A helper function to serialize a vector of items if T implements FlatSerialize. +// It returns None if the vector is empty, otherwise it returns the vector of offsets. +#[allow(dead_code)] // Unused code is allowed during cosmetic filter migration +pub(crate) fn serialize_vec_opt<'b, B: FlatBuilder<'b>, T: FlatSerialize<'b, B>>( + value: Vec, + builder: &mut B, +) -> Option> { + if value.is_empty() { + None + } else { + Some(FlatSerialize::serialize(value, builder)) + } +} diff --git a/src/flatbuffers/containers/flat_set.rs b/src/flatbuffers/containers/flat_set.rs index 48b1199c..a6c4d771 100644 --- a/src/flatbuffers/containers/flat_set.rs +++ b/src/flatbuffers/containers/flat_set.rs @@ -2,7 +2,9 @@ use std::marker::PhantomData; -use crate::flatbuffers::containers::sorted_index::SortedIndex; +use crate::flatbuffers::containers; +use containers::flat_serialize::{FlatBuilder, FlatSerialize, WIPFlatVec}; +use containers::sorted_index::SortedIndex; /// A set-like container that uses flatbuffer references. /// Provides O(log n) lookup time using binary search on the sorted data. @@ -44,6 +46,23 @@ where } } +impl<'b, B: FlatBuilder<'b>, T: FlatSerialize<'b, B> + std::hash::Hash + Ord> FlatSerialize<'b, B> + for std::collections::HashSet +{ + type Output = WIPFlatVec<'b, T, B>; + + fn serialize(value: Self, builder: &mut B) -> Self::Output { + let mut items = value.into_iter().collect::>(); + items.sort_unstable(); + let v = items + .into_iter() + .map(|x| FlatSerialize::serialize(x, builder)) + .collect::>(); + + builder.raw_builder().create_vector(&v) + } +} + #[cfg(test)] #[path = "../../../tests/unit/flatbuffers/containers/flat_set.rs"] mod unit_tests; diff --git a/src/flatbuffers/containers/mod.rs b/src/flatbuffers/containers/mod.rs index 507620de..20eb251d 100644 --- a/src/flatbuffers/containers/mod.rs +++ b/src/flatbuffers/containers/mod.rs @@ -1,3 +1,5 @@ +pub(crate) mod flat_map; pub(crate) mod flat_multimap; +pub(crate) mod flat_serialize; pub(crate) mod flat_set; pub(crate) mod sorted_index; diff --git a/src/flatbuffers/unsafe_tools.rs b/src/flatbuffers/unsafe_tools.rs index f53edaa5..15156546 100644 --- a/src/flatbuffers/unsafe_tools.rs +++ b/src/flatbuffers/unsafe_tools.rs @@ -3,8 +3,7 @@ use crate::filters::fb_network::flat::fb; // Minimum alignment for the beginning of the flatbuffer data. -// Should be 4 while we support armv7 and x86_32. -const MIN_ALIGNMENT: usize = 4; +const MIN_ALIGNMENT: usize = 8; /// Converts a flatbuffers Vector to a slice. /// # Safety @@ -20,7 +19,7 @@ pub fn fb_vector_to_slice(vector: flatbuffers::Vector<'_, T>) -> &[T] { // the alignment of the data must be a divisor of MIN_ALIGNMENT. assert!(MIN_ALIGNMENT % std::mem::size_of::() == 0); } - let _ = static_assert_alignment::; + const { static_assert_alignment::() }; assert!(bytes.len() % std::mem::size_of::() == 0); assert!(bytes.as_ptr() as usize % std::mem::align_of::() == 0); diff --git a/tests/unit/flatbuffers/containers/flat_multimap.rs b/tests/unit/flatbuffers/containers/flat_multimap.rs index 74505bd6..7d39df14 100644 --- a/tests/unit/flatbuffers/containers/flat_multimap.rs +++ b/tests/unit/flatbuffers/containers/flat_multimap.rs @@ -1,6 +1,10 @@ +#[allow(dead_code, clippy::all, unused_imports, unsafe_code)] +#[path = "./test_containers_generated.rs"] +pub mod flat; #[cfg(test)] mod tests { use super::super::*; + use super::flat::fb_test; // Helper function to create a Vector from a slice fn create_vector_u32<'a>( @@ -10,7 +14,7 @@ mod tests { let vec_offset = builder.create_vector(data); builder.finish(vec_offset, None); let buf = builder.finished_data(); - flatbuffers::root::>(buf).expect("OK") + flatbuffers::root::>(buf).unwrap() } #[test] @@ -99,4 +103,84 @@ mod tests { assert!(map.get(2).is_none()); assert!(map.get(4).is_none()); } + + #[test] + fn test_uint_builder() { + let mut builder = flatbuffers::FlatBufferBuilder::new(); + let mut map = FlatMultiMapBuilder::::default(); + map.insert(2, 20); + map.insert(1, 10); + map.insert(2, 30); + let map = FlatMultiMapBuilder::finish(map, &mut builder); + + // Serialize to the test flatbuffer. + let test_map = fb_test::TestUIntMap::create( + &mut builder, + &fb_test::TestUIntMapArgs { + keys: Some(map.keys), + values: Some(map.values), + }, + ); + let mut root_args = fb_test::TestRootArgs::default(); + root_args.test_uint_map = Some(test_map); + let root = fb_test::TestRoot::create(&mut builder, &root_args); + builder.finish(root, None); + + // Load from the serialized test flatbuffer. + use crate::flatbuffers::unsafe_tools::fb_vector_to_slice; + let data = builder.finished_data(); + let root = fb_test::root_as_test_root(data).unwrap(); + let flat_map = root.test_uint_map().unwrap(); + let map = FlatMultiMapView::::new( + fb_vector_to_slice(flat_map.keys()), + flat_map.values(), + ); + + assert_eq!(map.total_size(), 3); + assert_eq!(map.get(1).unwrap().collect::>(), vec![(0, 10)]); + assert_eq!( + map.get(2).unwrap().collect::>(), + vec![(1, 20), (2, 30)] + ); + assert!(map.get(0).is_none()); + assert!(map.get(3).is_none()); + } + + #[test] + fn test_string_builder() { + let mut builder = flatbuffers::FlatBufferBuilder::new(); + let mut map = FlatMultiMapBuilder::<&str, &str>::default(); + map.insert("b", "20"); + map.insert("a", "10"); + map.insert("b", "30"); + let map = FlatMultiMapBuilder::finish(map, &mut builder); + + // Serialize to the test flatbuffer. + let test_map = fb_test::TestStringMap::create( + &mut builder, + &fb_test::TestStringMapArgs { + keys: Some(map.keys), + values: Some(map.values), + }, + ); + let mut root_args = fb_test::TestRootArgs::default(); + root_args.test_string_map = Some(test_map); + let root = fb_test::TestRoot::create(&mut builder, &root_args); + builder.finish(root, None); + + // Load from the serialized test flatbuffer. + let data = builder.finished_data(); + let root = fb_test::root_as_test_root(data).unwrap(); + let flat_map = root.test_string_map().unwrap(); + let map = FlatMultiMapView::new(flat_map.keys(), flat_map.values()); + + assert_eq!(map.total_size(), 3); + assert_eq!(map.get("a").unwrap().collect::>(), vec![(0, "10")]); + assert_eq!( + map.get("b").unwrap().collect::>(), + vec![(1, "20"), (2, "30")] + ); + assert!(map.get("c").is_none()); + assert!(map.get("d").is_none()); + } } diff --git a/tests/unit/flatbuffers/containers/flat_set.rs b/tests/unit/flatbuffers/containers/flat_set.rs index 2fb37813..0a789cd2 100644 --- a/tests/unit/flatbuffers/containers/flat_set.rs +++ b/tests/unit/flatbuffers/containers/flat_set.rs @@ -1,6 +1,12 @@ +#[allow(dead_code, clippy::all, unused_imports, unsafe_code)] +#[path = "./test_containers_generated.rs"] +pub mod flat; #[cfg(test)] mod tests { + use std::collections::HashSet; + use super::super::*; + use super::flat::fb_test; #[test] fn test_flat_set_view() { @@ -16,4 +22,58 @@ mod tests { // Test len assert_eq!(set.len(), 8); } + + #[test] + fn test_uint_builder() { + let mut builder = flatbuffers::FlatBufferBuilder::new(); + let mut set = HashSet::::default(); + set.insert(2); + set.insert(1); + set.insert(2); + let set = FlatSerialize::serialize(set, &mut builder); + + // Serialize to the test flatbuffer. + let mut root_args = fb_test::TestRootArgs::default(); + root_args.test_uint_set = Some(set); + let root = fb_test::TestRoot::create(&mut builder, &root_args); + builder.finish(root, None); + + // Load from the serialized test flatbuffer. + use crate::flatbuffers::unsafe_tools::fb_vector_to_slice; + let data = builder.finished_data(); + let root = fb_test::root_as_test_root(data).unwrap(); + let set = + FlatSetView::::new(fb_vector_to_slice(root.test_uint_set().unwrap())); + + assert_eq!(set.len(), 2); + assert!(set.contains(1)); + assert!(set.contains(2)); + assert!(!set.contains(3)); + } + + #[test] + fn test_string_builder() { + let mut builder = flatbuffers::FlatBufferBuilder::new(); + let mut set = HashSet::<&str>::default(); + set.insert("b"); + set.insert("a"); + set.insert("b"); + let set = FlatSerialize::serialize(set, &mut builder); + + // Serialize to the test flatbuffer. + let mut root_args = fb_test::TestRootArgs::default(); + root_args.test_string_set = Some(set); + let root = fb_test::TestRoot::create(&mut builder, &root_args); + builder.finish(root, None); + + // Load from the serialized test flatbuffer. + let data = builder.finished_data(); + let root = fb_test::root_as_test_root(data).unwrap(); + let set = FlatSetView::new(root.test_string_set().unwrap()); + + assert_eq!(set.len(), 2); + assert!(set.contains("a")); + assert!(set.contains("b")); + assert!(!set.contains("c")); + } } diff --git a/tests/unit/flatbuffers/containers/test_containers.fbs b/tests/unit/flatbuffers/containers/test_containers.fbs new file mode 100644 index 00000000..f44d519d --- /dev/null +++ b/tests/unit/flatbuffers/containers/test_containers.fbs @@ -0,0 +1,25 @@ +// A test flatbuffer that is used in unit_tests. +// To build *_generated.rs run: +// 1. flatc --rust --gen-object-api -o tests/unit/flatbuffers/containers/ tests/unit/flatbuffers/containers/test_containers.fbs +// 2. cargo fmt +namespace fb_test; + +table TestUIntMap { + keys: [uint64] (required); + values: [uint32] (required); +} + +table TestStringMap { + keys: [string] (required); + values: [string] (required); +} + +table TestRoot { + test_uint_map: TestUIntMap; + test_string_map: TestStringMap; + + test_uint_set: [uint64]; + test_string_set: [string]; +} + +root_type TestRoot; diff --git a/tests/unit/flatbuffers/containers/test_containers_generated.rs b/tests/unit/flatbuffers/containers/test_containers_generated.rs new file mode 100644 index 00000000..75ee889d --- /dev/null +++ b/tests/unit/flatbuffers/containers/test_containers_generated.rs @@ -0,0 +1,785 @@ +// automatically generated by the FlatBuffers compiler, do not modify + +// @generated + +use core::cmp::Ordering; +use core::mem; + +extern crate flatbuffers; +use self::flatbuffers::{EndianScalar, Follow}; + +#[allow(unused_imports, dead_code)] +pub mod fb_test { + + use core::cmp::Ordering; + use core::mem; + + extern crate flatbuffers; + use self::flatbuffers::{EndianScalar, Follow}; + + pub enum TestUIntMapOffset {} + #[derive(Copy, Clone, PartialEq)] + + pub struct TestUIntMap<'a> { + pub _tab: flatbuffers::Table<'a>, + } + + impl<'a> flatbuffers::Follow<'a> for TestUIntMap<'a> { + type Inner = TestUIntMap<'a>; + #[inline] + unsafe fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + Self { + _tab: flatbuffers::Table::new(buf, loc), + } + } + } + + impl<'a> TestUIntMap<'a> { + pub const VT_KEYS: flatbuffers::VOffsetT = 4; + pub const VT_VALUES: flatbuffers::VOffsetT = 6; + + #[inline] + pub unsafe fn init_from_table(table: flatbuffers::Table<'a>) -> Self { + TestUIntMap { _tab: table } + } + #[allow(unused_mut)] + pub fn create< + 'bldr: 'args, + 'args: 'mut_bldr, + 'mut_bldr, + A: flatbuffers::Allocator + 'bldr, + >( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr, A>, + args: &'args TestUIntMapArgs<'args>, + ) -> flatbuffers::WIPOffset> { + let mut builder = TestUIntMapBuilder::new(_fbb); + if let Some(x) = args.values { + builder.add_values(x); + } + if let Some(x) = args.keys { + builder.add_keys(x); + } + builder.finish() + } + + pub fn unpack(&self) -> TestUIntMapT { + let keys = { + let x = self.keys(); + x.into_iter().collect() + }; + let values = { + let x = self.values(); + x.into_iter().collect() + }; + TestUIntMapT { keys, values } + } + + #[inline] + pub fn keys(&self) -> flatbuffers::Vector<'a, u64> { + // Safety: + // Created from valid Table for this object + // which contains a valid value in this slot + unsafe { + self._tab + .get::>>( + TestUIntMap::VT_KEYS, + None, + ) + .unwrap() + } + } + #[inline] + pub fn values(&self) -> flatbuffers::Vector<'a, u32> { + // Safety: + // Created from valid Table for this object + // which contains a valid value in this slot + unsafe { + self._tab + .get::>>( + TestUIntMap::VT_VALUES, + None, + ) + .unwrap() + } + } + } + + impl flatbuffers::Verifiable for TestUIntMap<'_> { + #[inline] + fn run_verifier( + v: &mut flatbuffers::Verifier, + pos: usize, + ) -> Result<(), flatbuffers::InvalidFlatbuffer> { + use self::flatbuffers::Verifiable; + v.visit_table(pos)? + .visit_field::>>( + "keys", + Self::VT_KEYS, + true, + )? + .visit_field::>>( + "values", + Self::VT_VALUES, + true, + )? + .finish(); + Ok(()) + } + } + pub struct TestUIntMapArgs<'a> { + pub keys: Option>>, + pub values: Option>>, + } + impl<'a> Default for TestUIntMapArgs<'a> { + #[inline] + fn default() -> Self { + TestUIntMapArgs { + keys: None, // required field + values: None, // required field + } + } + } + + pub struct TestUIntMapBuilder<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a, A>, + start_: flatbuffers::WIPOffset, + } + impl<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> TestUIntMapBuilder<'a, 'b, A> { + #[inline] + pub fn add_keys(&mut self, keys: flatbuffers::WIPOffset>) { + self.fbb_ + .push_slot_always::>(TestUIntMap::VT_KEYS, keys); + } + #[inline] + pub fn add_values(&mut self, values: flatbuffers::WIPOffset>) { + self.fbb_ + .push_slot_always::>(TestUIntMap::VT_VALUES, values); + } + #[inline] + pub fn new( + _fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>, + ) -> TestUIntMapBuilder<'a, 'b, A> { + let start = _fbb.start_table(); + TestUIntMapBuilder { + fbb_: _fbb, + start_: start, + } + } + #[inline] + pub fn finish(self) -> flatbuffers::WIPOffset> { + let o = self.fbb_.end_table(self.start_); + self.fbb_.required(o, TestUIntMap::VT_KEYS, "keys"); + self.fbb_.required(o, TestUIntMap::VT_VALUES, "values"); + flatbuffers::WIPOffset::new(o.value()) + } + } + + impl core::fmt::Debug for TestUIntMap<'_> { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + let mut ds = f.debug_struct("TestUIntMap"); + ds.field("keys", &self.keys()); + ds.field("values", &self.values()); + ds.finish() + } + } + #[non_exhaustive] + #[derive(Debug, Clone, PartialEq)] + pub struct TestUIntMapT { + pub keys: Vec, + pub values: Vec, + } + impl Default for TestUIntMapT { + fn default() -> Self { + Self { + keys: Default::default(), + values: Default::default(), + } + } + } + impl TestUIntMapT { + pub fn pack<'b, A: flatbuffers::Allocator + 'b>( + &self, + _fbb: &mut flatbuffers::FlatBufferBuilder<'b, A>, + ) -> flatbuffers::WIPOffset> { + let keys = Some({ + let x = &self.keys; + _fbb.create_vector(x) + }); + let values = Some({ + let x = &self.values; + _fbb.create_vector(x) + }); + TestUIntMap::create(_fbb, &TestUIntMapArgs { keys, values }) + } + } + pub enum TestStringMapOffset {} + #[derive(Copy, Clone, PartialEq)] + + pub struct TestStringMap<'a> { + pub _tab: flatbuffers::Table<'a>, + } + + impl<'a> flatbuffers::Follow<'a> for TestStringMap<'a> { + type Inner = TestStringMap<'a>; + #[inline] + unsafe fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + Self { + _tab: flatbuffers::Table::new(buf, loc), + } + } + } + + impl<'a> TestStringMap<'a> { + pub const VT_KEYS: flatbuffers::VOffsetT = 4; + pub const VT_VALUES: flatbuffers::VOffsetT = 6; + + #[inline] + pub unsafe fn init_from_table(table: flatbuffers::Table<'a>) -> Self { + TestStringMap { _tab: table } + } + #[allow(unused_mut)] + pub fn create< + 'bldr: 'args, + 'args: 'mut_bldr, + 'mut_bldr, + A: flatbuffers::Allocator + 'bldr, + >( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr, A>, + args: &'args TestStringMapArgs<'args>, + ) -> flatbuffers::WIPOffset> { + let mut builder = TestStringMapBuilder::new(_fbb); + if let Some(x) = args.values { + builder.add_values(x); + } + if let Some(x) = args.keys { + builder.add_keys(x); + } + builder.finish() + } + + pub fn unpack(&self) -> TestStringMapT { + let keys = { + let x = self.keys(); + x.iter().map(|s| s.to_string()).collect() + }; + let values = { + let x = self.values(); + x.iter().map(|s| s.to_string()).collect() + }; + TestStringMapT { keys, values } + } + + #[inline] + pub fn keys(&self) -> flatbuffers::Vector<'a, flatbuffers::ForwardsUOffset<&'a str>> { + // Safety: + // Created from valid Table for this object + // which contains a valid value in this slot + unsafe { + self._tab + .get::>, + >>(TestStringMap::VT_KEYS, None) + .unwrap() + } + } + #[inline] + pub fn values(&self) -> flatbuffers::Vector<'a, flatbuffers::ForwardsUOffset<&'a str>> { + // Safety: + // Created from valid Table for this object + // which contains a valid value in this slot + unsafe { + self._tab + .get::>, + >>(TestStringMap::VT_VALUES, None) + .unwrap() + } + } + } + + impl flatbuffers::Verifiable for TestStringMap<'_> { + #[inline] + fn run_verifier( + v: &mut flatbuffers::Verifier, + pos: usize, + ) -> Result<(), flatbuffers::InvalidFlatbuffer> { + use self::flatbuffers::Verifiable; + v.visit_table(pos)? + .visit_field::>, + >>("keys", Self::VT_KEYS, true)? + .visit_field::>, + >>("values", Self::VT_VALUES, true)? + .finish(); + Ok(()) + } + } + pub struct TestStringMapArgs<'a> { + pub keys: Option< + flatbuffers::WIPOffset>>, + >, + pub values: Option< + flatbuffers::WIPOffset>>, + >, + } + impl<'a> Default for TestStringMapArgs<'a> { + #[inline] + fn default() -> Self { + TestStringMapArgs { + keys: None, // required field + values: None, // required field + } + } + } + + pub struct TestStringMapBuilder<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a, A>, + start_: flatbuffers::WIPOffset, + } + impl<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> TestStringMapBuilder<'a, 'b, A> { + #[inline] + pub fn add_keys( + &mut self, + keys: flatbuffers::WIPOffset< + flatbuffers::Vector<'b, flatbuffers::ForwardsUOffset<&'b str>>, + >, + ) { + self.fbb_ + .push_slot_always::>(TestStringMap::VT_KEYS, keys); + } + #[inline] + pub fn add_values( + &mut self, + values: flatbuffers::WIPOffset< + flatbuffers::Vector<'b, flatbuffers::ForwardsUOffset<&'b str>>, + >, + ) { + self.fbb_ + .push_slot_always::>(TestStringMap::VT_VALUES, values); + } + #[inline] + pub fn new( + _fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>, + ) -> TestStringMapBuilder<'a, 'b, A> { + let start = _fbb.start_table(); + TestStringMapBuilder { + fbb_: _fbb, + start_: start, + } + } + #[inline] + pub fn finish(self) -> flatbuffers::WIPOffset> { + let o = self.fbb_.end_table(self.start_); + self.fbb_.required(o, TestStringMap::VT_KEYS, "keys"); + self.fbb_.required(o, TestStringMap::VT_VALUES, "values"); + flatbuffers::WIPOffset::new(o.value()) + } + } + + impl core::fmt::Debug for TestStringMap<'_> { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + let mut ds = f.debug_struct("TestStringMap"); + ds.field("keys", &self.keys()); + ds.field("values", &self.values()); + ds.finish() + } + } + #[non_exhaustive] + #[derive(Debug, Clone, PartialEq)] + pub struct TestStringMapT { + pub keys: Vec, + pub values: Vec, + } + impl Default for TestStringMapT { + fn default() -> Self { + Self { + keys: Default::default(), + values: Default::default(), + } + } + } + impl TestStringMapT { + pub fn pack<'b, A: flatbuffers::Allocator + 'b>( + &self, + _fbb: &mut flatbuffers::FlatBufferBuilder<'b, A>, + ) -> flatbuffers::WIPOffset> { + let keys = Some({ + let x = &self.keys; + let w: Vec<_> = x.iter().map(|s| _fbb.create_string(s)).collect(); + _fbb.create_vector(&w) + }); + let values = Some({ + let x = &self.values; + let w: Vec<_> = x.iter().map(|s| _fbb.create_string(s)).collect(); + _fbb.create_vector(&w) + }); + TestStringMap::create(_fbb, &TestStringMapArgs { keys, values }) + } + } + pub enum TestRootOffset {} + #[derive(Copy, Clone, PartialEq)] + + pub struct TestRoot<'a> { + pub _tab: flatbuffers::Table<'a>, + } + + impl<'a> flatbuffers::Follow<'a> for TestRoot<'a> { + type Inner = TestRoot<'a>; + #[inline] + unsafe fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + Self { + _tab: flatbuffers::Table::new(buf, loc), + } + } + } + + impl<'a> TestRoot<'a> { + pub const VT_TEST_UINT_MAP: flatbuffers::VOffsetT = 4; + pub const VT_TEST_STRING_MAP: flatbuffers::VOffsetT = 6; + pub const VT_TEST_UINT_SET: flatbuffers::VOffsetT = 8; + pub const VT_TEST_STRING_SET: flatbuffers::VOffsetT = 10; + + #[inline] + pub unsafe fn init_from_table(table: flatbuffers::Table<'a>) -> Self { + TestRoot { _tab: table } + } + #[allow(unused_mut)] + pub fn create< + 'bldr: 'args, + 'args: 'mut_bldr, + 'mut_bldr, + A: flatbuffers::Allocator + 'bldr, + >( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr, A>, + args: &'args TestRootArgs<'args>, + ) -> flatbuffers::WIPOffset> { + let mut builder = TestRootBuilder::new(_fbb); + if let Some(x) = args.test_string_set { + builder.add_test_string_set(x); + } + if let Some(x) = args.test_uint_set { + builder.add_test_uint_set(x); + } + if let Some(x) = args.test_string_map { + builder.add_test_string_map(x); + } + if let Some(x) = args.test_uint_map { + builder.add_test_uint_map(x); + } + builder.finish() + } + + pub fn unpack(&self) -> TestRootT { + let test_uint_map = self.test_uint_map().map(|x| Box::new(x.unpack())); + let test_string_map = self.test_string_map().map(|x| Box::new(x.unpack())); + let test_uint_set = self.test_uint_set().map(|x| x.into_iter().collect()); + let test_string_set = self + .test_string_set() + .map(|x| x.iter().map(|s| s.to_string()).collect()); + TestRootT { + test_uint_map, + test_string_map, + test_uint_set, + test_string_set, + } + } + + #[inline] + pub fn test_uint_map(&self) -> Option> { + // Safety: + // Created from valid Table for this object + // which contains a valid value in this slot + unsafe { + self._tab.get::>( + TestRoot::VT_TEST_UINT_MAP, + None, + ) + } + } + #[inline] + pub fn test_string_map(&self) -> Option> { + // Safety: + // Created from valid Table for this object + // which contains a valid value in this slot + unsafe { + self._tab + .get::>( + TestRoot::VT_TEST_STRING_MAP, + None, + ) + } + } + #[inline] + pub fn test_uint_set(&self) -> Option> { + // Safety: + // Created from valid Table for this object + // which contains a valid value in this slot + unsafe { + self._tab + .get::>>( + TestRoot::VT_TEST_UINT_SET, + None, + ) + } + } + #[inline] + pub fn test_string_set( + &self, + ) -> Option>> { + // Safety: + // Created from valid Table for this object + // which contains a valid value in this slot + unsafe { + self._tab.get::>, + >>(TestRoot::VT_TEST_STRING_SET, None) + } + } + } + + impl flatbuffers::Verifiable for TestRoot<'_> { + #[inline] + fn run_verifier( + v: &mut flatbuffers::Verifier, + pos: usize, + ) -> Result<(), flatbuffers::InvalidFlatbuffer> { + use self::flatbuffers::Verifiable; + v.visit_table(pos)? + .visit_field::>( + "test_uint_map", + Self::VT_TEST_UINT_MAP, + false, + )? + .visit_field::>( + "test_string_map", + Self::VT_TEST_STRING_MAP, + false, + )? + .visit_field::>>( + "test_uint_set", + Self::VT_TEST_UINT_SET, + false, + )? + .visit_field::>, + >>("test_string_set", Self::VT_TEST_STRING_SET, false)? + .finish(); + Ok(()) + } + } + pub struct TestRootArgs<'a> { + pub test_uint_map: Option>>, + pub test_string_map: Option>>, + pub test_uint_set: Option>>, + pub test_string_set: Option< + flatbuffers::WIPOffset>>, + >, + } + impl<'a> Default for TestRootArgs<'a> { + #[inline] + fn default() -> Self { + TestRootArgs { + test_uint_map: None, + test_string_map: None, + test_uint_set: None, + test_string_set: None, + } + } + } + + pub struct TestRootBuilder<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a, A>, + start_: flatbuffers::WIPOffset, + } + impl<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> TestRootBuilder<'a, 'b, A> { + #[inline] + pub fn add_test_uint_map( + &mut self, + test_uint_map: flatbuffers::WIPOffset>, + ) { + self.fbb_ + .push_slot_always::>( + TestRoot::VT_TEST_UINT_MAP, + test_uint_map, + ); + } + #[inline] + pub fn add_test_string_map( + &mut self, + test_string_map: flatbuffers::WIPOffset>, + ) { + self.fbb_ + .push_slot_always::>( + TestRoot::VT_TEST_STRING_MAP, + test_string_map, + ); + } + #[inline] + pub fn add_test_uint_set( + &mut self, + test_uint_set: flatbuffers::WIPOffset>, + ) { + self.fbb_.push_slot_always::>( + TestRoot::VT_TEST_UINT_SET, + test_uint_set, + ); + } + #[inline] + pub fn add_test_string_set( + &mut self, + test_string_set: flatbuffers::WIPOffset< + flatbuffers::Vector<'b, flatbuffers::ForwardsUOffset<&'b str>>, + >, + ) { + self.fbb_.push_slot_always::>( + TestRoot::VT_TEST_STRING_SET, + test_string_set, + ); + } + #[inline] + pub fn new( + _fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>, + ) -> TestRootBuilder<'a, 'b, A> { + let start = _fbb.start_table(); + TestRootBuilder { + fbb_: _fbb, + start_: start, + } + } + #[inline] + pub fn finish(self) -> flatbuffers::WIPOffset> { + let o = self.fbb_.end_table(self.start_); + flatbuffers::WIPOffset::new(o.value()) + } + } + + impl core::fmt::Debug for TestRoot<'_> { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + let mut ds = f.debug_struct("TestRoot"); + ds.field("test_uint_map", &self.test_uint_map()); + ds.field("test_string_map", &self.test_string_map()); + ds.field("test_uint_set", &self.test_uint_set()); + ds.field("test_string_set", &self.test_string_set()); + ds.finish() + } + } + #[non_exhaustive] + #[derive(Debug, Clone, PartialEq)] + pub struct TestRootT { + pub test_uint_map: Option>, + pub test_string_map: Option>, + pub test_uint_set: Option>, + pub test_string_set: Option>, + } + impl Default for TestRootT { + fn default() -> Self { + Self { + test_uint_map: None, + test_string_map: None, + test_uint_set: None, + test_string_set: None, + } + } + } + impl TestRootT { + pub fn pack<'b, A: flatbuffers::Allocator + 'b>( + &self, + _fbb: &mut flatbuffers::FlatBufferBuilder<'b, A>, + ) -> flatbuffers::WIPOffset> { + let test_uint_map = self.test_uint_map.as_ref().map(|x| x.pack(_fbb)); + let test_string_map = self.test_string_map.as_ref().map(|x| x.pack(_fbb)); + let test_uint_set = self.test_uint_set.as_ref().map(|x| _fbb.create_vector(x)); + let test_string_set = self.test_string_set.as_ref().map(|x| { + let w: Vec<_> = x.iter().map(|s| _fbb.create_string(s)).collect(); + _fbb.create_vector(&w) + }); + TestRoot::create( + _fbb, + &TestRootArgs { + test_uint_map, + test_string_map, + test_uint_set, + test_string_set, + }, + ) + } + } + #[inline] + /// Verifies that a buffer of bytes contains a `TestRoot` + /// and returns it. + /// Note that verification is still experimental and may not + /// catch every error, or be maximally performant. For the + /// previous, unchecked, behavior use + /// `root_as_test_root_unchecked`. + pub fn root_as_test_root(buf: &[u8]) -> Result { + flatbuffers::root::(buf) + } + #[inline] + /// Verifies that a buffer of bytes contains a size prefixed + /// `TestRoot` and returns it. + /// Note that verification is still experimental and may not + /// catch every error, or be maximally performant. For the + /// previous, unchecked, behavior use + /// `size_prefixed_root_as_test_root_unchecked`. + pub fn size_prefixed_root_as_test_root( + buf: &[u8], + ) -> Result { + flatbuffers::size_prefixed_root::(buf) + } + #[inline] + /// Verifies, with the given options, that a buffer of bytes + /// contains a `TestRoot` and returns it. + /// Note that verification is still experimental and may not + /// catch every error, or be maximally performant. For the + /// previous, unchecked, behavior use + /// `root_as_test_root_unchecked`. + pub fn root_as_test_root_with_opts<'b, 'o>( + opts: &'o flatbuffers::VerifierOptions, + buf: &'b [u8], + ) -> Result, flatbuffers::InvalidFlatbuffer> { + flatbuffers::root_with_opts::>(opts, buf) + } + #[inline] + /// Verifies, with the given verifier options, that a buffer of + /// bytes contains a size prefixed `TestRoot` and returns + /// it. Note that verification is still experimental and may not + /// catch every error, or be maximally performant. For the + /// previous, unchecked, behavior use + /// `root_as_test_root_unchecked`. + pub fn size_prefixed_root_as_test_root_with_opts<'b, 'o>( + opts: &'o flatbuffers::VerifierOptions, + buf: &'b [u8], + ) -> Result, flatbuffers::InvalidFlatbuffer> { + flatbuffers::size_prefixed_root_with_opts::>(opts, buf) + } + #[inline] + /// Assumes, without verification, that a buffer of bytes contains a TestRoot and returns it. + /// # Safety + /// Callers must trust the given bytes do indeed contain a valid `TestRoot`. + pub unsafe fn root_as_test_root_unchecked(buf: &[u8]) -> TestRoot { + flatbuffers::root_unchecked::(buf) + } + #[inline] + /// Assumes, without verification, that a buffer of bytes contains a size prefixed TestRoot and returns it. + /// # Safety + /// Callers must trust the given bytes do indeed contain a valid size prefixed `TestRoot`. + pub unsafe fn size_prefixed_root_as_test_root_unchecked(buf: &[u8]) -> TestRoot { + flatbuffers::size_prefixed_root_unchecked::(buf) + } + #[inline] + pub fn finish_test_root_buffer<'a, 'b, A: flatbuffers::Allocator + 'a>( + fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>, + root: flatbuffers::WIPOffset>, + ) { + fbb.finish(root, None); + } + + #[inline] + pub fn finish_size_prefixed_test_root_buffer<'a, 'b, A: flatbuffers::Allocator + 'a>( + fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>, + root: flatbuffers::WIPOffset>, + ) { + fbb.finish_size_prefixed(root, None); + } +} // pub mod fb_test From 42769a9af0b998b2ae89fb88f31a819e15908b4f Mon Sep 17 00:00:00 2001 From: Mikhail Atuchin Date: Thu, 14 Aug 2025 18:17:07 +0700 Subject: [PATCH 2/7] Remove flat_map.rs --- src/flatbuffers/containers/flat_map.rs | 71 -------------------------- src/flatbuffers/containers/mod.rs | 1 - 2 files changed, 72 deletions(-) delete mode 100644 src/flatbuffers/containers/flat_map.rs diff --git a/src/flatbuffers/containers/flat_map.rs b/src/flatbuffers/containers/flat_map.rs deleted file mode 100644 index 71dc6bcf..00000000 --- a/src/flatbuffers/containers/flat_map.rs +++ /dev/null @@ -1,71 +0,0 @@ -#![allow(dead_code)] // Unused code is allowed during cosmetic filter migration - -use std::marker::PhantomData; - -use crate::flatbuffers::containers; -use containers::flat_serialize::{FlatBuilder, FlatMapBuilderOutput, FlatSerialize}; -use containers::sorted_index::SortedIndex; -use flatbuffers::{Follow, Vector}; - -pub(crate) struct FlatMapView<'a, I: Ord, V, Keys> -where - Keys: SortedIndex, - V: Follow<'a>, -{ - keys: Keys, - values: Vector<'a, V>, - _phantom: PhantomData, -} - -impl<'a, I: Ord + Copy, V, Keys> FlatMapView<'a, I, V, Keys> -where - Keys: SortedIndex + Clone, - V: flatbuffers::Follow<'a>, -{ - pub fn new(keys: Keys, values: Vector<'a, V>) -> Self { - debug_assert!(keys.len() == values.len()); - Self { - keys, - values, - _phantom: PhantomData, - } - } - - pub fn get(&self, key: I) -> Option<>::Inner> { - let index = self.keys.partition_point(|x| *x < key); - if index < self.keys.len() && self.keys.get(index) == key { - Some(self.values.get(index)) - } else { - None - } - } -} - -pub(crate) struct FlatMapBuilder; - -impl FlatMapBuilder { - pub fn finish<'a, I, V, B: FlatBuilder<'a>>( - value: std::collections::HashMap, - builder: &mut B, - ) -> FlatMapBuilderOutput<'a, I, V, B> - where - I: FlatSerialize<'a, B> + Ord, - V: FlatSerialize<'a, B>, - { - let mut entries: Vec<_> = value.into_iter().collect(); - entries.sort_unstable_by(|(a, _), (b, _)| a.cmp(b)); - - let mut indexes = Vec::with_capacity(entries.len()); - let mut values = Vec::with_capacity(entries.len()); - - for (key, value) in entries.into_iter() { - indexes.push(FlatSerialize::serialize(key, builder)); - values.push(FlatSerialize::serialize(value, builder)); - } - - FlatMapBuilderOutput { - keys: builder.raw_builder().create_vector(&indexes), - values: builder.raw_builder().create_vector(&values), - } - } -} diff --git a/src/flatbuffers/containers/mod.rs b/src/flatbuffers/containers/mod.rs index 20eb251d..50164fd2 100644 --- a/src/flatbuffers/containers/mod.rs +++ b/src/flatbuffers/containers/mod.rs @@ -1,4 +1,3 @@ -pub(crate) mod flat_map; pub(crate) mod flat_multimap; pub(crate) mod flat_serialize; pub(crate) mod flat_set; From 2e3fc0399ca4c42f4c73c6a614ec88499b0ae59b Mon Sep 17 00:00:00 2001 From: Mikhail Atuchin Date: Thu, 14 Aug 2025 18:24:19 +0700 Subject: [PATCH 3/7] minor changes --- src/filters/fb_builder.rs | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/src/filters/fb_builder.rs b/src/filters/fb_builder.rs index 6bd7db1b..5baaf00d 100644 --- a/src/filters/fb_builder.rs +++ b/src/filters/fb_builder.rs @@ -6,11 +6,11 @@ use std::collections::{HashMap, HashSet}; use std::vec; -use flatbuffers::{ForwardsUOffset, Vector, WIPOffset}; +use flatbuffers::{Vector, WIPOffset}; use crate::filters::network::{NetworkFilter, NetworkFilterMaskHelper}; use crate::flatbuffers::containers::flat_multimap::FlatMultiMapBuilder; -use crate::flatbuffers::containers::flat_serialize::{FlatBuilder, FlatSerialize}; +use crate::flatbuffers::containers::flat_serialize::{FlatBuilder, FlatSerialize, WIPFlatVec}; use crate::flatbuffers::unsafe_tools::VerifiedFlatbufferMemory; use crate::network_filter_list::token_histogram; use crate::optimizer; @@ -324,17 +324,9 @@ impl NetworkRulesBuilder { } impl<'a> FlatSerialize<'a, EngineFlatBuilder<'a>> for NetworkRulesBuilder { - type Output = WIPOffset>>>; - fn serialize( - value: Self, - builder: &mut EngineFlatBuilder<'a>, - ) -> WIPOffset>>> { - let flat_network_rules: Vec<_> = value - .lists - .into_iter() - .map(|list| FlatSerialize::serialize(list, builder)) - .collect(); - builder.raw_builder().create_vector(&flat_network_rules) + type Output = WIPFlatVec<'a, NetworkFilterListBuilder, EngineFlatBuilder<'a>>; + fn serialize(value: Self, builder: &mut EngineFlatBuilder<'a>) -> Self::Output { + FlatSerialize::serialize(value.lists, builder) } } From 6e0e572373bba3905cfe4ada9a4bf1d4c3243ab5 Mon Sep 17 00:00:00 2001 From: Mikhail Atuchin Date: Thu, 14 Aug 2025 19:10:06 +0700 Subject: [PATCH 4/7] Simplify some calls --- src/filters/fb_builder.rs | 43 ++++++++++++++++++--------------------- 1 file changed, 20 insertions(+), 23 deletions(-) diff --git a/src/filters/fb_builder.rs b/src/filters/fb_builder.rs index 5baaf00d..6c711f31 100644 --- a/src/filters/fb_builder.rs +++ b/src/filters/fb_builder.rs @@ -36,22 +36,13 @@ struct NetworkFilterListBuilder { optimize: bool, } +#[derive(Default)] struct EngineFlatBuilder<'a> { fb_builder: flatbuffers::FlatBufferBuilder<'a>, unique_domains_hashes: HashMap, unique_domains_hashes_vec: Vec, } -impl Default for EngineFlatBuilder<'_> { - fn default() -> Self { - Self { - fb_builder: flatbuffers::FlatBufferBuilder::new(), - unique_domains_hashes: HashMap::new(), - unique_domains_hashes_vec: Vec::new(), - } - } -} - impl<'a> EngineFlatBuilder<'a> { pub fn get_or_insert_unique_domain_hash(&mut self, hash: &Hash) -> u32 { if let Some(&index) = self.unique_domains_hashes.get(hash) { @@ -63,9 +54,24 @@ impl<'a> EngineFlatBuilder<'a> { index } - pub fn write_unique_domains(&mut self) -> WIPOffset> { - self.fb_builder - .create_vector(&self.unique_domains_hashes_vec) + pub fn finish( + &mut self, + network_rules: WIPFlatVec<'a, NetworkFilterListBuilder, EngineFlatBuilder<'a>>, + ) -> VerifiedFlatbufferMemory { + let unique_domains_hashes = Some( + self.fb_builder + .create_vector(&self.unique_domains_hashes_vec), + ); + let network_rules = Some(network_rules); + let engine = fb::Engine::create( + self.raw_builder(), + &fb::EngineArgs { + network_rules, + unique_domains_hashes, + }, + ); + self.raw_builder().finish(engine, None); + VerifiedFlatbufferMemory::from_builder(self.raw_builder()) } } @@ -337,14 +343,5 @@ pub fn make_flatbuffer( let mut builder = EngineFlatBuilder::default(); let network_rules_builder = NetworkRulesBuilder::from_rules(network_filters, optimize); let network_rules = FlatSerialize::serialize(network_rules_builder, &mut builder); - let unique_domains_hashes = Some(builder.write_unique_domains()); - let engine = fb::Engine::create( - builder.raw_builder(), - &fb::EngineArgs { - network_rules: Some(network_rules), - unique_domains_hashes, - }, - ); - builder.raw_builder().finish(engine, None); - VerifiedFlatbufferMemory::from_builder(builder.raw_builder()) + builder.finish(network_rules) } From e4f92b151e8f2cb5a2166ed65f3be4f0c1045cdb Mon Sep 17 00:00:00 2001 From: Mikhail Atuchin Date: Thu, 14 Aug 2025 19:23:01 +0700 Subject: [PATCH 5/7] Fix clippy and other issues --- src/filters/fb_builder.rs | 2 +- tests/unit/engine.rs | 4 ++-- .../flatbuffers/containers/flat_multimap.rs | 21 +++++++++++++------ tests/unit/flatbuffers/containers/flat_set.rs | 20 ++++++++++++------ 4 files changed, 32 insertions(+), 15 deletions(-) diff --git a/src/filters/fb_builder.rs b/src/filters/fb_builder.rs index 6c711f31..d2518fc6 100644 --- a/src/filters/fb_builder.rs +++ b/src/filters/fb_builder.rs @@ -6,7 +6,7 @@ use std::collections::{HashMap, HashSet}; use std::vec; -use flatbuffers::{Vector, WIPOffset}; +use flatbuffers::WIPOffset; use crate::filters::network::{NetworkFilter, NetworkFilterMaskHelper}; use crate::flatbuffers::containers::flat_multimap::FlatMultiMapBuilder; diff --git a/tests/unit/engine.rs b/tests/unit/engine.rs index ae7b6cea..fdd36f2e 100644 --- a/tests/unit/engine.rs +++ b/tests/unit/engine.rs @@ -219,9 +219,9 @@ mod tests { let data = engine.serialize().unwrap(); let expected_hash = if cfg!(feature = "css-validation") { - 12046041060659687422 + 2449299753308785566 } else { - 11420623023091203502 + 13012908745357887728 }; assert_eq!(hash(&data), expected_hash, "{}", HASH_MISMATCH_MSG); diff --git a/tests/unit/flatbuffers/containers/flat_multimap.rs b/tests/unit/flatbuffers/containers/flat_multimap.rs index 7d39df14..85ef0d62 100644 --- a/tests/unit/flatbuffers/containers/flat_multimap.rs +++ b/tests/unit/flatbuffers/containers/flat_multimap.rs @@ -121,9 +121,14 @@ mod tests { values: Some(map.values), }, ); - let mut root_args = fb_test::TestRootArgs::default(); - root_args.test_uint_map = Some(test_map); - let root = fb_test::TestRoot::create(&mut builder, &root_args); + + let root = fb_test::TestRoot::create( + &mut builder, + &fb_test::TestRootArgs { + test_uint_map: Some(test_map), + ..Default::default() + }, + ); builder.finish(root, None); // Load from the serialized test flatbuffer. @@ -163,9 +168,13 @@ mod tests { values: Some(map.values), }, ); - let mut root_args = fb_test::TestRootArgs::default(); - root_args.test_string_map = Some(test_map); - let root = fb_test::TestRoot::create(&mut builder, &root_args); + let root = fb_test::TestRoot::create( + &mut builder, + &fb_test::TestRootArgs { + test_string_map: Some(test_map), + ..Default::default() + }, + ); builder.finish(root, None); // Load from the serialized test flatbuffer. diff --git a/tests/unit/flatbuffers/containers/flat_set.rs b/tests/unit/flatbuffers/containers/flat_set.rs index 0a789cd2..3807e188 100644 --- a/tests/unit/flatbuffers/containers/flat_set.rs +++ b/tests/unit/flatbuffers/containers/flat_set.rs @@ -33,9 +33,13 @@ mod tests { let set = FlatSerialize::serialize(set, &mut builder); // Serialize to the test flatbuffer. - let mut root_args = fb_test::TestRootArgs::default(); - root_args.test_uint_set = Some(set); - let root = fb_test::TestRoot::create(&mut builder, &root_args); + let root = fb_test::TestRoot::create( + &mut builder, + &fb_test::TestRootArgs { + test_uint_set: Some(set), + ..Default::default() + }, + ); builder.finish(root, None); // Load from the serialized test flatbuffer. @@ -61,9 +65,13 @@ mod tests { let set = FlatSerialize::serialize(set, &mut builder); // Serialize to the test flatbuffer. - let mut root_args = fb_test::TestRootArgs::default(); - root_args.test_string_set = Some(set); - let root = fb_test::TestRoot::create(&mut builder, &root_args); + let root = fb_test::TestRoot::create( + &mut builder, + &fb_test::TestRootArgs { + test_string_set: Some(set), + ..Default::default() + }, + ); builder.finish(root, None); // Load from the serialized test flatbuffer. From e42ff2bb95a0ba7d91271aefe83c94268994a3b5 Mon Sep 17 00:00:00 2001 From: Mikhail Atuchin Date: Thu, 14 Aug 2025 19:47:39 +0700 Subject: [PATCH 6/7] Fix bad filter logic --- src/filters/fb_builder.rs | 13 +++++++++---- tests/unit/engine.rs | 4 ++-- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/src/filters/fb_builder.rs b/src/filters/fb_builder.rs index d2518fc6..c17b78de 100644 --- a/src/filters/fb_builder.rs +++ b/src/filters/fb_builder.rs @@ -281,11 +281,16 @@ impl NetworkRulesBuilder { let mut self_ = Self { lists }; let mut badfilter_ids: HashSet = HashSet::new(); - for filter in network_filters.into_iter() { - if filter.is_badfilter() { - badfilter_ids.insert(filter.get_id_without_badfilter()); - } + // Collect badfilter ids in advance. + for filter in network_filters.iter() { + if filter.is_badfilter() { + badfilter_ids.insert(filter.get_id_without_badfilter()); + } + } + + for filter in network_filters.into_iter() { + // skip any bad filters let filter_id = filter.get_id(); if badfilter_ids.contains(&filter_id) || filter.is_badfilter() { continue; diff --git a/tests/unit/engine.rs b/tests/unit/engine.rs index fdd36f2e..ae7b6cea 100644 --- a/tests/unit/engine.rs +++ b/tests/unit/engine.rs @@ -219,9 +219,9 @@ mod tests { let data = engine.serialize().unwrap(); let expected_hash = if cfg!(feature = "css-validation") { - 2449299753308785566 + 12046041060659687422 } else { - 13012908745357887728 + 11420623023091203502 }; assert_eq!(hash(&data), expected_hash, "{}", HASH_MISMATCH_MSG); From 690a3e54495b1b89967dace529f25ac8fc6b8a4e Mon Sep 17 00:00:00 2001 From: Mikhail Atuchin Date: Thu, 14 Aug 2025 19:50:49 +0700 Subject: [PATCH 7/7] remove some diff --- src/filters/fb_builder.rs | 34 ++++++++++++++++------------------ 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/src/filters/fb_builder.rs b/src/filters/fb_builder.rs index c17b78de..9775fa6b 100644 --- a/src/filters/fb_builder.rs +++ b/src/filters/fb_builder.rs @@ -39,18 +39,18 @@ struct NetworkFilterListBuilder { #[derive(Default)] struct EngineFlatBuilder<'a> { fb_builder: flatbuffers::FlatBufferBuilder<'a>, - unique_domains_hashes: HashMap, - unique_domains_hashes_vec: Vec, + unique_domains_hashes: Vec, + unique_domains_hashes_map: HashMap, } impl<'a> EngineFlatBuilder<'a> { - pub fn get_or_insert_unique_domain_hash(&mut self, hash: &Hash) -> u32 { - if let Some(&index) = self.unique_domains_hashes.get(hash) { + pub fn get_or_insert_unique_domain_hash(&mut self, h: &Hash) -> u32 { + if let Some(&index) = self.unique_domains_hashes_map.get(h) { return index; } - let index = self.unique_domains_hashes_vec.len() as u32; - self.unique_domains_hashes_vec.push(*hash); - self.unique_domains_hashes.insert(*hash, index); + let index = self.unique_domains_hashes.len() as u32; + self.unique_domains_hashes.push(*h); + self.unique_domains_hashes_map.insert(*h, index); index } @@ -58,10 +58,8 @@ impl<'a> EngineFlatBuilder<'a> { &mut self, network_rules: WIPFlatVec<'a, NetworkFilterListBuilder, EngineFlatBuilder<'a>>, ) -> VerifiedFlatbufferMemory { - let unique_domains_hashes = Some( - self.fb_builder - .create_vector(&self.unique_domains_hashes_vec), - ); + let unique_domains_hashes = + Some(self.fb_builder.create_vector(&self.unique_domains_hashes)); let network_rules = Some(network_rules); let engine = fb::Engine::create( self.raw_builder(), @@ -177,7 +175,7 @@ impl NetworkFilterListBuilder { impl<'a> FlatSerialize<'a, EngineFlatBuilder<'a>> for NetworkFilterListBuilder { type Output = WIPOffset>; fn serialize( - value: Self, + rule_list: Self, builder: &mut EngineFlatBuilder<'a>, ) -> WIPOffset> { let mut filter_map = HashMap::>>>::new(); @@ -185,7 +183,7 @@ impl<'a> FlatSerialize<'a, EngineFlatBuilder<'a>> for NetworkFilterListBuilder { let mut optimizable = HashMap::>::new(); // Compute tokens for all filters - let filter_tokens: Vec<_> = value + let filter_tokens: Vec<_> = rule_list .filters .into_iter() .map(|filter| { @@ -199,7 +197,7 @@ impl<'a> FlatSerialize<'a, EngineFlatBuilder<'a>> for NetworkFilterListBuilder { { for (network_filter, multi_tokens) in filter_tokens.into_iter() { - let flat_filter = if !value.optimize + let flat_filter = if !rule_list.optimize || !optimizer::is_filter_optimizable_by_patterns(&network_filter) { Some(FlatSerialize::serialize(&network_filter, builder)) @@ -237,7 +235,7 @@ impl<'a> FlatSerialize<'a, EngineFlatBuilder<'a>> for NetworkFilterListBuilder { } } - if value.optimize { + if rule_list.optimize { // Sort the entries to ensure deterministic iteration order let mut optimizable_entries: Vec<_> = optimizable.drain().collect(); optimizable_entries.sort_unstable_by_key(|(token, _)| *token); @@ -284,9 +282,9 @@ impl NetworkRulesBuilder { // Collect badfilter ids in advance. for filter in network_filters.iter() { - if filter.is_badfilter() { - badfilter_ids.insert(filter.get_id_without_badfilter()); - } + if filter.is_badfilter() { + badfilter_ids.insert(filter.get_id_without_badfilter()); + } } for filter in network_filters.into_iter() {