Merge pull request #72 from TrueLearnAI/update-interest

fix: use binary skill representation in interest

Author: KD-7

truelearn/learning/_ink_classifier.py

@@ -64,14 +64,14 @@ class INKClassifier(BaseClassifier):
         ...         ink_classifier.predict_proba(event)
         ...     )
         ...
-        True 0.64839...
-        False 0.43767...
-        True 0.65660...
+        True 0.64387...
+        False 0.42658...
+        True 0.65406...
         >>> ink_classifier.get_params(deep=False)  # doctest:+ELLIPSIS
         {...'learner_meta_weights': LearnerMetaWeights(novelty_weights=Weights(\
-mean=0.20461..., variance=0.45871...), interest_weights=Weights(\
-mean=0.66315..., variance=0.42187...), bias_weights=Weights(\
-mean=0.12698..., variance=0.39796...))...}
+mean=0.20787..., variance=0.45787...), interest_weights=Weights(\
+mean=0.66924..., variance=0.42672...), bias_weights=Weights(\
+mean=0.13029..., variance=0.39582...))...}
 
     """
 

truelearn/learning/_interest_classifier.py

@@ -77,12 +77,12 @@ class InterestClassifier(InterestNoveltyKnowledgeBaseClassifier):
         ...         interest_classifier.predict_proba(event)
         ...     )
         ...
-        True 0.88450...
-        True 0.81079...
-        True 0.95872...
+        True 0.87299...
+        True 0.69146...
+        True 0.91941...
         >>> interest_classifier.get_params()  # doctest:+ELLIPSIS
         {..., 'learner_model': LearnerModel(knowledge=Knowledge(knowledge=\
-{1: KnowledgeComponent(mean=0.99556..., variance=0.10483..., ...), ...}), ...}
+{1: KnowledgeComponent(mean=1.3651..., variance=0.07128..., ...), ...}), ...}
     """
 
     _parameter_constraints: Dict[str, Any] = {
@@ -196,6 +196,25 @@ def __get_decay_func(self) -> Callable[[float], float]:
 
         return lambda t_delta: min(math.exp(-self._decay_func_factor * t_delta), 1.0)
 
+    @staticmethod
+    def _content_kc_masks(
+        content_kcs: Iterable[BaseKnowledgeComponent],
+    ) -> Iterable[BaseKnowledgeComponent]:
+        """Return a new iterable of content's knowledge components.
+
+        Args:
+            content_kcs: An iterable of content's knowledge components.
+
+        Returns:
+            A new iterable of content's knowledge components, where
+            the mean of each knowledge component is set to 1,
+            based on the assumption of the TrueLearn Interest model.
+        """
+        return (
+            kc.clone(mean=1.0, variance=kc.variance, timestamp=kc.timestamp)
+            for kc in content_kcs
+        )
+
     def _generate_ratings(
         self,
         env: trueskill.TrueSkill,
@@ -257,7 +276,9 @@ def __apply_interest_decay(
         learner_kcs_decayed = map(__apply_interest_decay, learner_kcs)
 
         team_learner = gather_trueskill_team(env, learner_kcs_decayed)
-        team_content = gather_trueskill_team(env, content_kcs)
+        team_content = gather_trueskill_team(
+            env, InterestClassifier._content_kc_masks(content_kcs)
+        )
 
         # learner always wins in interest
         updated_team_learner, _ = env.rate([team_learner, team_content], ranks=[0, 1])
@@ -269,4 +290,6 @@ def _eval_matching_quality(
         learner_kcs: Iterable[BaseKnowledgeComponent],
         content_kcs: Iterable[BaseKnowledgeComponent],
     ) -> float:
-        return team_sum_quality_from_kcs(learner_kcs, content_kcs, self._beta)
+        return team_sum_quality_from_kcs(
+            learner_kcs, InterestClassifier._content_kc_masks(content_kcs), self._beta
+        )

truelearn/tests/test_learning.py

@@ -618,16 +618,6 @@ def test_interest_get_set_params(self):
             "the draw_proba_static should not be None."
         )
 
-    def test_interest_positive_easy(self):
-        classifier = learning.InterestClassifier(init_skill=0.0, def_var=0.5)
-
-        knowledge = models.Knowledge(
-            {1: models.KnowledgeComponent(mean=0.0, variance=0.5)}
-        )
-        event = models.EventModel(knowledge)
-
-        assert classifier.predict_proba(event) == 0.5
-
     def test_interest_throw(self):
         with pytest.raises(TrueLearnTypeError) as excinfo:
             learning.InterestClassifier(threshold=0)
@@ -749,7 +739,7 @@ def test_interest_classifier(self, train_cases, test_events):
         for event, label in zip(train_events, train_labels):
             classifier.fit(event, label)
 
-        expected_results = [0.8648794445446283, 0.8438279621999456, 0.7777471206958368]
+        expected_results = [0.8245402410711562, 0.7833295255047532, 0.9194176141732581]
         actual_results = [classifier.predict_proba(event) for event in test_events]
 
         check_farray_close(actual_results, expected_results)
@@ -800,9 +790,9 @@ def test_ink_classifier_customize(self, train_cases, test_events):
             classifier.fit(event, label)
 
         expected_results = [
-            0.40575267541878457,
-            0.36519542301026875,
-            0.33362493980730495,
+            0.39583121564200274,
+            0.3542200117164174,
+            0.36177605375601996,
         ]
         actual_results = [classifier.predict_proba(event) for event in test_events]
 
@@ -853,7 +843,7 @@ def test_ink_classifier(self, train_cases, test_events):
         for event, label in zip(train_events, train_labels):
             classifier.fit(event, label)
 
-        expected_results = [0.3844070661899784, 0.3398805698754434, 0.3133264788862059]
+        expected_results = [0.3807588746036166, 0.33099758411287944, 0.3461486335816942]
         actual_results = [classifier.predict_proba(event) for event in test_events]
 
         check_farray_close(actual_results, expected_results)
@@ -869,7 +859,7 @@ def test_ink_classifier_greedy(self):
                 event_time=0,
             )
         ]
-        train_labels = [True]
+        train_labels = [False]
         for event, label in zip(train_events, train_labels):
             classifier.fit(event, label)