Change course embeddings to include year level

Author: kevin-lann

course-matrix/backend/src/constants/constants.ts

@@ -39,7 +39,7 @@ export const yearToCode = (year: number) => {
 
 // Set minimum results wanted for a similarity search on the associated namespace.
 export const namespaceToMinResults = new Map();
-namespaceToMinResults.set("courses", 10);
+namespaceToMinResults.set("courses_v2", 10);
 namespaceToMinResults.set("offerings", 16); // Typically, more offering info is wanted.
 namespaceToMinResults.set("prerequisites", 5);
 namespaceToMinResults.set("corequisites", 5);

course-matrix/backend/src/constants/promptKeywords.ts

@@ -1,6 +1,6 @@
 // Keywords related to each namespace
 export const NAMESPACE_KEYWORDS = {
-  courses: [
+  courses_v2: [
     "course",
     "class",
     "description",

course-matrix/backend/src/controllers/aiController.ts

@@ -58,8 +58,8 @@ function analyzeQuery(query: string): {
 
   // If a course code is detected, add tehse namespaces
   if (containsCourseCode) {
-    if (!relevantNamespaces.includes("courses"))
-      relevantNamespaces.push("courses");
+    if (!relevantNamespaces.includes("courses_v2"))
+      relevantNamespaces.push("courses_v2");
     if (!relevantNamespaces.includes("offerings"))
       relevantNamespaces.push("offerings");
     if (!relevantNamespaces.includes("prerequisites"))
@@ -70,8 +70,8 @@ function analyzeQuery(query: string): {
   if (DEPARTMENT_CODES.some((code) => lowerQuery.includes(code))) {
     if (!relevantNamespaces.includes("departments"))
       relevantNamespaces.push("departments");
-    if (!relevantNamespaces.includes("courses"))
-      relevantNamespaces.push("courses");
+    if (!relevantNamespaces.includes("courses_v2"))
+      relevantNamespaces.push("courses_v2");
   }
 
   // If search is required at all
@@ -83,7 +83,7 @@ function analyzeQuery(query: string): {
   // If no specific namespaces identified & search required, then search all
   if (requiresSearch && relevantNamespaces.length === 0) {
     relevantNamespaces.push(
-      "courses",
+      "courses_v2",
       "offerings",
       "prerequisites",
       "corequisites",
@@ -154,7 +154,7 @@ async function reformulateQuery(
       apiKey: process.env.OPENAI_API_KEY,
     });
 
-    console.log("History: ", conversationHistory);
+    // console.log("History: ", conversationHistory);
 
     // Create messages array with the correct type structure
     const messages: OpenAI.Chat.ChatCompletionMessageParam[] = [
@@ -187,7 +187,13 @@ async function reformulateQuery(
           Output: "What are the course names of course codes: MGTA01, CSCA08, MATA31, MATA35?"
 
           User: "How are you doing today?"
-          Output: "How are you doing today?"`,
+          Output: "How are you doing today?"
+
+          User: "Give 2nd year math courses."
+          Output: "What are some 2nd year math courses?"
+
+          User: "Give first year math courses."
+          Output: "What are some 1st year math courses?"`,
       },
     ];
 
@@ -209,7 +215,7 @@ async function reformulateQuery(
       model: "gpt-4o-mini",
       messages: messages,
       temperature: 0.1, // Lower temperature for more consistent, focused queries
-      max_tokens: 150, // Limit response length
+      max_tokens: latestQuery.length * 3, // Limit response length. Proportional to user input.
       top_p: 0.5, // Reduced top_p for more focused outputs
     });
 

course-matrix/backend/src/utils/embeddings.ts

@@ -97,6 +97,7 @@ async function processPDF(filePath: string, namespace: string) {
 // processCSV("../data/tables/offerings_summer_2025.csv", "offerings")
 // processCSV("../data/tables/offerings_winter_2026.csv", "offerings")
 // processCSV("../data/tables/departments.csv", "departments")
+// processCSV("../data/tables/courses_with_year.csv", "courses_v2")
 
 console.log("embeddings done.");
 

course-matrix/data/add_year_level.py

@@ -0,0 +1,38 @@
+import csv
+import sys
+
+def get_year_level(code):
+    if len(code) >= 4:
+        fourth_char = code[3]
+        if fourth_char == 'A':
+            return "1st year"
+        elif fourth_char == 'B':
+            return "2nd year"
+        elif fourth_char == 'C':
+            return "3rd year"
+        elif fourth_char == 'D':
+            return "4th year"
+    return ""  # Default empty string if code is too short or doesn't match
+
+def process_csv(input_file, output_file):
+    with open(input_file, 'r', newline='', encoding='utf-8') as infile:
+        reader = csv.DictReader(infile)
+        fieldnames = reader.fieldnames + ['year_level']
+        
+        with open(output_file, 'w', newline='', encoding='utf-8') as outfile:
+            writer = csv.DictWriter(outfile, fieldnames=fieldnames)
+            writer.writeheader()
+            
+            for row in reader:
+                row['year_level'] = get_year_level(row['code'])
+                writer.writerow(row)
+    
+    print(f"Successfully processed {input_file} and created {output_file} with year_level column added.")
+
+if __name__ == "__main__":
+    if len(sys.argv) > 2:
+        process_csv(sys.argv[1], sys.argv[2])
+    else:
+        input_file = "./tables/courses.csv"
+        output_file = "./tables/courses_with_year.csv"
+        process_csv(input_file, output_file)