Tests for default score, probability floor, and top score tie-breaking (#52)

* tests for default score, probability floor, and top score tie-breaking

* actually add the test case

* fix up banditReferences

* add comments to new test cases

* grammar tweak

Author: aarsilv

ufc/bandit-flags-v1.json

@@ -201,6 +201,31 @@
                 }
             ],
             "totalShards": 10000
+        },
+        "car_bandit_flag": {
+            "key": "car_bandit",
+            "enabled": true,
+            "variationType": "STRING",
+            "variations": {
+                "car_bandit": {
+                    "key": "car_bandit",
+                    "value": "car_bandit"
+                }
+            },
+            "allocations": [
+                {
+                    "key": "all-traffic",
+                    "rules": [],
+                    "splits": [
+                        {
+                            "variationKey": "car_bandit",
+                            "shards": []
+                        }
+                    ],
+                    "doLog": true
+                }
+            ],
+            "totalShards": 10000
         }
     },
     "bandits": {
@@ -218,6 +243,14 @@
                 "variationValue": "banner_bandit"
             }
         ],
+        "car_bandit": [
+            {
+                "key": "car_bandit",
+                "flagKey": "car_bandit_flag",
+                "variationKey": "car_bandit",
+                "variationValue": "car_bandit"
+            }
+        ],
         "cold_start_bandit": [
             {
                 "key": "cold_start_bandit",
@@ -233,20 +266,39 @@
                 {
                     "key": "banner_bandit",
                     "flagKey": "banner_bandit_flag",
-                    "allocationKey": "banner_bandit_allocation",
+                    "allocationKey": "analysis",
+                    "variationKey": "banner_bandit",
+                    "variationValue": "banner_bandit"
+                },
+                {
+                    "key": "banner_bandit",
+                    "flagKey": "banner_bandit_flag",
+                    "allocationKey": "training",
                     "variationKey": "banner_bandit",
                     "variationValue": "banner_bandit"
                 },
                 {
                     "key": "banner_bandit",
                     "flagKey": "banner_bandit_flag_uk_only",
-                    "allocationKey": "banner_bandit_uk_allocation",
+                    "allocationKey": "training",
                     "variationKey": "banner_bandit",
                     "variationValue": "banner_bandit"
                 }
             ],
             "modelVersion": "v123"
         },
+        "car_bandit": {
+            "flagVariations": [
+                {
+                    "key": "car_bandit",
+                    "flagKey": "car_bandit_flag",
+                    "allocationKey": "all-traffic",
+                    "variationKey": "car_bandit",
+                    "variationValue": "car_bandit"
+                }
+            ],
+            "modelVersion": "v456"
+        },
         "cold_start_bandit": {
             "flagVariations": [
                 {

ufc/bandit-models-v1.json

@@ -106,6 +106,31 @@
         }
       }
     },
+    "car_bandit": {
+      "banditKey": "car_bandit",
+      "modelName": "falcon",
+      "updatedAt": "2023-09-13T04:52:06.462Z",
+      "modelVersion": "v456",
+      "modelData": {
+        "gamma": 1.0,
+        "defaultActionScore": 5.0,
+        "actionProbabilityFloor": 0.2,
+        "coefficients": {
+          "toyota": {
+            "actionKey": "toyota",
+            "intercept": 1.0,
+            "actionNumericCoefficients": [{
+              "attributeKey": "speed",
+              "coefficient": 1,
+              "missingValueCoefficient": 0.0
+            }],
+            "actionCategoricalCoefficients": [],
+            "subjectNumericCoefficients": [],
+            "subjectCategoricalCoefficients": []
+          }
+        }
+      }
+    },
     "cold_start_bandit": {
       "banditKey": "cold_start_bandit",
       "modelName": "falcon",

ufc/bandit-tests/test-case-car-bandit.json

@@ -0,0 +1,52 @@
+{
+    "flag": "car_bandit_flag",
+    "defaultValue": "default",
+    "subjects": [
+      {
+        "comment": "Tests that probability floor is applied. These actions shuffle to: Mercedes, Dodge, Toyota. Without minimum probability floors, Toyota gets most of the weight, and that's what gets assigned. However, with a probability floor, the other actions each get a non-trivial shard range, so it is not assigned. If the probability floor is not normalized (divided by the number of actions), then the first action, Mercedes, will be assigned too many shards and get assigned. Only when it's non-zero and normalized does Anastasia get correctly assigned the middle action: Dodge.",
+        "subjectKey": "anastasia",
+        "subjectAttributes": {},
+        "actions": [
+            {
+                "actionKey": "dodge",
+                "numericAttributes": {},
+                "categoricalAttributes": {}
+            },
+            {
+                "actionKey": "mercedes",
+                "numericAttributes": {},
+                "categoricalAttributes": {}
+            },
+            {
+              "actionKey": "toyota",
+              "numericAttributes": {"speed":  1000},
+              "categoricalAttributes": {}
+            }
+        ],
+        "assignment": {"variation": "car_bandit", "action": "dodge"}
+      },
+      {
+        "comment": "Tests that default action scores are applied and top-scoring actions are tie-broken by name. These actions shuffle to: Toyota, Honda, Ford. If the default action score is ignored and the unknown (non-Toyota) actions get scored 0, Toyota takes up too many shards and that is what is assigned. With the default score at play, the two unknown actions are top-scoring and get most of the shards. However, because of the top scoring action weight of 1 - all other action weights, the top scoring action gets the most. When not tie-broken, implementations like JavaScript that iterate over them in order would declare Honda the top-scoring action, and it would end up getting assigned. However, when breaking ties by action name, Ford takes the prize and gets assigned. Thus, for this test to pass with Ford being assigned to Bob, the default action scores need to be respected, and top-scoring actions need to be tie-broken by name.",
+        "subjectKey": "bob",
+        "subjectAttributes": {},
+        "actions": [
+            {
+              "actionKey": "toyota",
+              "numericAttributes": {"speed":  1},
+              "categoricalAttributes": {}
+            },
+            {
+              "actionKey": "honda",
+              "numericAttributes": {},
+              "categoricalAttributes": {}
+            },
+            {
+              "actionKey": "ford",
+              "numericAttributes": {},
+              "categoricalAttributes": {}
+            }
+        ],
+        "assignment": {"variation": "car_bandit", "action": "ford"}
+      }
+    ]
+  }