GENAI-4295 Remove normalization, add more data in inferred (#1340)

* Remove normalization, add more data in inferred

* update thresholds

* Fixes

* reverse print

* test fix

Author: rolf-moz

merino/curated_recommendations/ml_backends/static_local_model.py

@@ -169,9 +169,13 @@ def get_topic(topic: str) -> InterestVectorConfig:
 
 
 THRESHOLDS_V3_NORMALIZED = [0.3, 0.5, 0.8]
+THRESHOLDS_V3_NON_NORMALIZED = [0.002, 0.008, 0.017]
+THRESHOLDS_V3_NON_NORMALIZED_ALL_TOPICS = [0.0001, 0.002, 0.004]
 
 SUBTOPIC_TOPIC_BLEND_RATIO = 0.15
 
+SPECIAL_ALL_TOPIC_KEYWOWRD = "all"
+
 
 # Creates a limited model based on topics. Topics features are stored with a t_
 # in telemetry.
@@ -194,12 +198,19 @@ class SuperInferredModel(LocalModelBackend):
         Topic.FOOD.value,
         Topic.TECHNOLOGY.value,
         Topic.SCIENCE.value,
-        Topic.PERSONAL_FINANCE.value,
+        SPECIAL_ALL_TOPIC_KEYWOWRD,
     ]
     limited_topics_set = set(v3_limited_topics)
 
     @staticmethod
     def _get_topic(topic: str, thresholds: list[float]) -> InterestVectorConfig:
+        if topic == SPECIAL_ALL_TOPIC_KEYWOWRD:
+            return InterestVectorConfig(
+                features={f"t_{t}": 1 for t in BASE_TOPICS},
+                thresholds=THRESHOLDS_V3_NON_NORMALIZED_ALL_TOPICS,
+                diff_p=MODEL_P_VALUE_V3,
+                diff_q=MODEL_Q_VALUE_V3,
+            )
         return InterestVectorConfig(
             features={f"t_{topic}": 1},
             thresholds=thresholds,
@@ -218,7 +229,7 @@ def _get_section(section_name: str, thresholds: list[float]) -> InterestVectorCo
         )
 
     def _build_local(self, model_id, surface_id) -> InferredLocalModel | None:
-        model_thresholds = THRESHOLDS_V3_NORMALIZED
+        model_thresholds = THRESHOLDS_V3_NON_NORMALIZED
         private_features: list[str] | None = None
 
         if model_id == SERVER_V3_MODEL_ID:
@@ -236,7 +247,7 @@ def _build_local(self, model_id, surface_id) -> InferredLocalModel | None:
         topic_features = {a: self._get_topic(a, model_thresholds) for a in self.v3_limited_topics}
         model_data: ModelData = ModelData(
             model_type=ModelType.CTR,
-            rescale=True,
+            rescale=False,
             noise_scale=0.0,
             day_time_weighting=DayTimeWeightingConfig(
                 days=[30],

merino/curated_recommendations/rankers/contextual_ranker.py

@@ -44,7 +44,6 @@
 # These topics are in the current interest vector but not being used to determine the
 # cohort selection.
 CONTEXUAL_INFERRED_PER_TOPIC_WEIGHTING = {
-    Topic.PERSONAL_FINANCE: 1.2,
     Topic.TECHNOLOGY: 1.0,
     Topic.POLITICS: 0.3,
     Topic.ARTS: 0.3,

tests/unit/curated_recommendations/ml_backends/test_static_local_model.py

@@ -6,7 +6,7 @@
 from merino.curated_recommendations.corpus_backends.protocol import Topic, SurfaceId
 from merino.curated_recommendations.ml_backends.static_local_model import (
     SERVER_V3_MODEL_ID,
-    THRESHOLDS_V3_NORMALIZED,
+    THRESHOLDS_V3_NON_NORMALIZED,
     FakeLocalModelSections,
     SuperInferredModel,
     CTR_SECTION_MODEL_ID,
@@ -111,7 +111,7 @@ def test_model_returns_default_limited_model(model_limited):
     # test a specific threshold value
     assert (
         result.model_data.interest_vector[Topic.SPORTS.value].thresholds[0]
-        == THRESHOLDS_V3_NORMALIZED[0]
+        == THRESHOLDS_V3_NON_NORMALIZED[0]
     )