Kl/scrum 158 ai hallucinating (#115)

Co-authored-by: kevin-lann <[email protected]>

Author: kevin-lann

course-matrix/backend/src/constants/availableFunctions.ts

@@ -28,7 +28,8 @@ export type FunctionNames =
   | "updateTimetable"
   | "deleteTimetable"
   | "generateTimetable"
-  | "getCourses";
+  | "getCourses"
+  | "getOfferings";
 
 type AvailableFunctions = {
   [K in FunctionNames]: (args: any, req: Request) => Promise<any>;
@@ -483,7 +484,7 @@ ${offeringData.meeting_section} `;
       });
 
       // Get all courses that have any of the provided courses as its prefix
-      const { data, error } = await supabase
+      const { data: courseData, error } = await supabase
         .schema("course")
         .from("courses")
         .select("*")
@@ -493,7 +494,42 @@ ${offeringData.meeting_section} `;
         return { status: 400, error: error.message };
       }
 
-      return { status: 200, data };
+      return { status: 200, data: courseData };
+    } catch (error) {
+      const errorMessage =
+        error instanceof Error ? error.message : "An unknown error occurred";
+      return { status: 500, error: errorMessage };
+    }
+  },
+
+  getOfferings: async (args: any, req: Request) => {
+    const { courses, semester } = args;
+    try {
+      const filterConditions = courses.map((prefix: string) => {
+        return `code.ilike.${prefix}%`;
+      });
+
+      // Get all offerings for any of the provided courses
+      const { data: offeringsData, error: offeringsError } = await supabase
+        .schema("course")
+        .from("offerings")
+        .select("*")
+        .eq("offering", semester)
+        .or(filterConditions.join(","));
+
+      if (offeringsError) {
+        return { status: 400, error: offeringsError.message };
+      }
+
+      // Return the courses that are offered in the semseter
+      const coursesOffered = new Set();
+      for (const offering of offeringsData) {
+        if (!coursesOffered.has(offering.code)) {
+          coursesOffered.add(offering.code);
+        }
+      }
+
+      return { status: 200, data: Array.from(coursesOffered) };
     } catch (error) {
       const errorMessage =
         error instanceof Error ? error.message : "An unknown error occurred";

course-matrix/backend/src/constants/constants.ts

@@ -46,7 +46,7 @@ export const TEST_DATE_NOW = new Date(2025, 4, 14, 8, 45, 1);
 
 // Set minimum results wanted for a similarity search on the associated namespace.
 export const namespaceToMinResults = new Map();
-namespaceToMinResults.set("courses_v3", 10);
+namespaceToMinResults.set("courses_v3", 16);
 namespaceToMinResults.set("offerings", 16); // Typically, more offering info is wanted.
 namespaceToMinResults.set("prerequisites", 5);
 namespaceToMinResults.set("corequisites", 5);

course-matrix/backend/src/controllers/aiController.ts

@@ -138,7 +138,7 @@ export async function reformulateQuery(
           - DO replace pronouns and references with specific names and identifiers
           - DO include course codes, names and specific details for academic entities
           - If the query is not about university courses & offerings, return exactly a copy of the user's query.
-          - Append "code: " before course codes For example: "CSCC01" -> "code: CSCC01"
+          - Append "code: " before each course code For example: "CSCC01, BIOA01" -> "code: CSCC01, code: BIOA01"
           - If a course year level is written as "first year", "second year", etc. Then replace "first" with "1st" and "second" with "2nd" etc.
 
           Examples:
@@ -265,8 +265,14 @@ export const chat = asyncHandler(async (req: Request, res: Response) => {
               process.env.CLIENT_APP_URL
             }/dashboard/timetable?edit=[[TIMETABLE_ID]] , where TIMETABLE_ID is the id of the respective timetable.
             - If the user provides a course code of length 6 like CSCA08, then assume they mean CSCA08H3 (H3 appended)
-            - If the user wants to create a timetable, first call getCourses to get course information on the requested courses, then call generateTimetable.
+            - If the user wants to create a timetable:
+              1. First call getCourses to get course information on the requested courses, 
+              2. If the user provided a semester, then call getOfferings with the provided courses and semester to ensure the courses are actually offered in the semester. 
+                a) If a course is NOT returned by getOFferings, then list it under "Excluded courses" with "reason: not offered in [provided semester]"
+                b) If no courses have offerings, then do not generate the timetable.
+              3. Lastly, call generateTimetable with the provided information.
             - Do not make up fake courses or offerings. 
+            - If a user asks about a course that you do not know of, acknowledge this.
             - You can only edit title of the timetable, nothing else. If a user tries to edit something else, acknowledge this limitation.
             - For delete timetable requests, if the user asks to delete an ambiguous timetable name (i.e many with similar name exist) then ask them to clarify which one
             - For delete timetable requests, first check that the timetable the user is refering to exists 
@@ -327,6 +333,19 @@ export const chat = asyncHandler(async (req: Request, res: Response) => {
               return await availableFunctions.getCourses(args, req);
             },
           }),
+          getOfferings: tool({
+            description:
+              "Return courses offered in the provided semester out of all course codes provided",
+            parameters: z.object({
+              courses: z.array(z.string()).describe("List of course codes"),
+              semester: z.string(),
+            }),
+            execute: async (args) => {
+              const res = await availableFunctions.getOfferings(args, req);
+              console.log("Result of getOfferings: ", res);
+              return res;
+            },
+          }),
         },
         maxSteps: CHATBOT_TOOL_CALL_MAX_STEPS, // Controls how many back and forths
         // the model can take with user or
@@ -455,7 +474,9 @@ export const chat = asyncHandler(async (req: Request, res: Response) => {
           - If information is missing from the context but likely exists, try to use info from web to answer. If still not able to form a decent response, acknowledge the limitation
           - For unrelated questions, politely explain that you're specialized in UTSC academic information
           - If a user prompt appears like a task that requires timetable operations (like create, read, update, delete a user's timetable) BUT the user prompt doesn't start with prefix "/timetable" then remind user to use "/timetable" in front of their prompt to access these capabilities
-      
+          - If the user prompt is a response to a task that requires timetable operations (eg "confirm", "proceed", "continue") then ask user to use "/timetable"
+          - If a user asks about a course that you do not know of, acknowledge this.
+
           ## Available Knowledge
           ${
             context === "[No context provided]"