Score widgets with missing user input as 'invalid' (#2752)

This fixes a bug observed while testing the Perseus service with production
traffic.

Issue: https://khanacademy.atlassian.net/browse/LEMS-3341

## Test plan:

`pnpm test`

Author: benchristel

Reviewers: handeyeco, benchristel, ivyolamit, mark-fitzgerald, SonicScrewdriver, anakaren-rojas, nishasy

Required Reviewers:

Approved By: handeyeco

Checks: ✅ 10 checks were successful

Pull Request URL: #2752

Author: benchristel

.changeset/loud-schools-end.md

@@ -0,0 +1,5 @@
+---
+"@khanacademy/perseus-score": patch
+---
+
+Fix a bug where `scorePerseusItem` would throw an exception if some scorable widgets in the question did not have a corresponding entry in the `UserInputMap`. Such inputs are now scored as "invalid," meaning the learner did not fully answer the question.

packages/perseus-core/src/validation.types.ts

@@ -48,14 +48,14 @@ import type {
 import type {Relationship} from "./types";
 
 export type WidgetValidatorFunction = (
-    userInput: UserInput,
+    userInput: UserInput | undefined,
     validationData: ValidationData,
     locale: string,
 ) => ValidationResult;
 
 export type WidgetScorerFunction = (
     // The user data needed to score
-    userInput: UserInput,
+    userInput: UserInput | undefined,
     // The scoring criteria to score against
     rubric: Rubric,
     // Locale, for math evaluation

packages/perseus-score/src/score.test.ts

@@ -419,6 +419,22 @@ function generateDropdown(): DropdownWidget {
 }
 
 describe("scorePerseusItem", () => {
+    it("returns a score of 'invalid' if some widgets are missing from the user input", () => {
+        // Arrange:
+        const item = {
+            content: "[[☃ dropdown 1]]",
+            widgets: {"dropdown 1": generateDropdown()},
+            images: {},
+        };
+        const userInputMap = {};
+
+        // Act:
+        const score = scorePerseusItem(item, userInputMap, "en");
+
+        // Assert:
+        expect(score).toEqual({type: "invalid", message: null});
+    });
+
     it("should return empty if any validator returns empty", () => {
         // Act
         const score = scorePerseusItem(

packages/perseus-score/src/score.ts

@@ -12,6 +12,7 @@ import type {
     PerseusScore,
     PerseusWidgetsMap,
     UserInputMap,
+    UserInput,
 } from "@khanacademy/perseus-core";
 
 const noScore: PerseusScore = {
@@ -174,7 +175,11 @@ export function scoreWidgetsFunctional(
             return;
         }
 
-        const userInput = userInputMap[id];
+        // TODO(benchristel): Without the explicit type annotation, the type of
+        // userInput would be inferred as `any`. This is because the keys of
+        // userInputMap are strings with a specific format, but `id` is any old
+        // string. Find a way to make this more typesafe.
+        const userInput: UserInput | undefined = userInputMap[id];
         const validator = getWidgetValidator(widget.type);
         const scorer = getWidgetScorer(widget.type);
 

packages/perseus-score/src/widgets/categorizer/score-categorizer.test.ts

@@ -3,6 +3,19 @@ import scoreCategorizer from "./score-categorizer";
 import type {PerseusCategorizerRubric} from "@khanacademy/perseus-core";
 
 describe("scoreCategorizer", () => {
+    it("returns a score of 'invalid' when the user input is undefined", () => {
+        const rubric: PerseusCategorizerRubric = {
+            values: [1, 3],
+            items: ["apples", "oranges"],
+        };
+
+        const userInput = undefined;
+
+        const score = scoreCategorizer(userInput, rubric);
+
+        expect(score).toHaveInvalidInput();
+    });
+
     it("gives points when the answer is correct", () => {
         const rubric: PerseusCategorizerRubric = {
             values: [1, 3],

packages/perseus-score/src/widgets/categorizer/score-categorizer.ts

@@ -5,9 +5,14 @@ import type {
 } from "@khanacademy/perseus-core";
 
 function scoreCategorizer(
-    userInput: PerseusCategorizerUserInput,
+    // NOTE(benchristel): userInput can be undefined if the widget has never
+    // been interacted with.
+    userInput: PerseusCategorizerUserInput | undefined,
     rubric: PerseusCategorizerRubric,
 ): PerseusScore {
+    if (userInput == null) {
+        return {type: "invalid", message: null};
+    }
     let allCorrect = true;
     rubric.values.forEach((value, i) => {
         if (userInput.values[i] !== value) {

packages/perseus-score/src/widgets/categorizer/validate-categorizer.test.ts

@@ -3,6 +3,18 @@ import validateCategorizer from "./validate-categorizer";
 import type {PerseusCategorizerValidationData} from "@khanacademy/perseus-core";
 
 describe("validateCategorizer", () => {
+    it("returns a score of 'invalid' when the user input is undefined", () => {
+        const rubric: PerseusCategorizerValidationData = {
+            items: ["apples", "oranges"],
+        };
+
+        const userInput = undefined;
+
+        const score = validateCategorizer(userInput, rubric);
+
+        expect(score).toHaveInvalidInput();
+    });
+
     it("tells the learner its not complete if not selected", () => {
         const validationData: PerseusCategorizerValidationData = {
             items: ["apples", "oranges"],

packages/perseus-score/src/widgets/categorizer/validate-categorizer.ts

@@ -15,9 +15,15 @@ import type {
  * @param strings - Used to provide a validation message
  */
 function validateCategorizer(
-    userInput: PerseusCategorizerUserInput,
+    // NOTE(benchristel): userInput can be undefined if the widget has never
+    // been interacted with.
+    userInput: PerseusCategorizerUserInput | undefined,
     validationData: PerseusCategorizerValidationData,
 ): ValidationResult {
+    if (userInput == null) {
+        return {type: "invalid", message: null};
+    }
+
     const incomplete = validationData.items.some(
         (_, i) => userInput.values[i] == null,
     );

packages/perseus-score/src/widgets/cs-program/score-cs-program.test.ts

@@ -3,6 +3,18 @@ import scoreCSProgram from "./score-cs-program";
 import type {PerseusCSProgramUserInput} from "@khanacademy/perseus-core";
 
 describe("scoreCSProgram", () => {
+    it("is 'invalid' when the state is undefined", () => {
+        // Arrange
+        const state = undefined;
+
+        // Act
+        const result = scoreCSProgram(state);
+
+        // Assert
+
+        expect(result).toHaveInvalidInput();
+    });
+
     it("is correct when the state from the iframe shows the status is correct", () => {
         // Arrange
         const state: PerseusCSProgramUserInput = {

packages/perseus-score/src/widgets/cs-program/score-cs-program.ts

@@ -3,7 +3,15 @@ import type {
     PerseusScore,
 } from "@khanacademy/perseus-core";
 
-function scoreCSProgram(userInput: PerseusCSProgramUserInput): PerseusScore {
+function scoreCSProgram(
+    // NOTE(benchristel): userInput can be undefined if the widget has never
+    // been interacted with.
+    userInput: PerseusCSProgramUserInput | undefined,
+): PerseusScore {
+    if (userInput == null) {
+        return {type: "invalid", message: null};
+    }
+
     // The CS program can tell us whether it's correct or incorrect,
     // and pass an optional message
     if (userInput.status === "correct") {