From ad1ad79172a073ae20ff9e0245e8162df8f35530 Mon Sep 17 00:00:00 2001 From: Aiden Grossman Date: Thu, 1 May 2025 19:40:57 +0000 Subject: [PATCH] Only pull unique values in generate_vocab This patch makes generate_vocab only consider unique values when creating quantile buckets for the individual features. There is no point in mapping multiple of the same value to different values post normalization/bucketization. This only serves to decrease dynamic range. Only looking at unique values preserves the property of increasing dynamic range around areas that have lots of slightly different values. This is especially useful when generating vocab for features that are oftentimes zero or one. In some cases I was seeing <50/1000 actual values with the rest being zeros or ones, which results in models that train poorly. --- compiler_opt/tools/generate_vocab.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/compiler_opt/tools/generate_vocab.py b/compiler_opt/tools/generate_vocab.py index 0ec24feb..23da1a59 100644 --- a/compiler_opt/tools/generate_vocab.py +++ b/compiler_opt/tools/generate_vocab.py @@ -127,7 +127,9 @@ def _generate_vocab(feature_values_arrays, feature_name, sample_length = math.floor( np.shape(feature_values)[0] * FLAGS.sampling_fraction) values = rng.choice(feature_values, sample_length, replace=False) - bin_edges = np.quantile(values, np.linspace(0, 1, FLAGS.num_buckets)) + unique_values = np.unique(values) + num_buckets = min(FLAGS.num_buckets, len(unique_values)) + bin_edges = np.quantile(unique_values, np.linspace(0, 1, num_buckets)) filename = os.path.join(FLAGS.output_dir, f'{feature_name}.buckets') with open(filename, 'w', encoding='utf-8') as f: for edge in bin_edges: