Update tool

Author: tingkai-mai

backend/question/tools/qn_cleaner.py

@@ -2,6 +2,7 @@
 import json
 from typing import List
 from pymongo import MongoClient
+import re
 from dotenv import load_dotenv
 import os
 
@@ -17,8 +18,9 @@
 def clean():
     with open(TARGET_FILE, "r") as f:
         data = json.load(f)
-    for entry in data:
+    for entry in data[1:]:
         questionid = entry.get("questionid")
+        print("Processing questionid", questionid)
         try:
             if questionid is not None:
                 qn_doc = collection.find_one({"questionid": questionid})
@@ -28,10 +30,34 @@ def clean():
                 examples = qn_doc.get("examples", [])
                 solution = entry.get("solution")
 
-                cleaned_examples = html.unescape(
-                    examples
-                )  # Converts &quot; to ", &lt; to <, &gt; to >, etc.
-                cleaned_solution = html.unescape(solution)
+                cleaned_examples = []
+                cleaned_solution = ""
+                if examples:
+                    for example in examples:
+                        new_example = {}
+                        new_example["expected_input"] = html.unescape(
+                            example.get("expected_input")
+                        )
+                        new_example["expected_input"] = re.sub(
+                            r"<[^>]+>", "", new_example["expected_input"]
+                        )
+                        new_example["expected_output"] = html.unescape(
+                            example.get("expected_output")
+                        )
+                        new_example["expected_output"] = re.sub(
+                            r"<[^>]+>", "", new_example["expected_output"]
+                        )
+                        if example.get("explanation"):
+                            new_example["explanation"] = html.unescape(
+                                example.get("explanation")
+                            )
+                            new_example["explanation"] = re.sub(
+                                r"<[^>]+>", "", new_example["explanation"]
+                            )
+
+                        cleaned_examples.append(new_example)
+                if solution:
+                    cleaned_solution = html.unescape(solution)
 
                 result = collection.update_one(
                     {"questionid": questionid},