AI-powered marking

.gitignore

@@ -118,3 +118,6 @@ erl_crash.dump
 
 # Generated lexer
 /src/source_lexer.erl
+
+# Ignore log files
+/log

lib/cadet/ai_comments.ex

@@ -0,0 +1,95 @@
+defmodule Cadet.AIComments do
+  @moduledoc """
+  Handles operations related to AI comments, including creation, updates, and retrieval.
+  """
+
+  import Ecto.Query
+  alias Cadet.Repo
+  alias Cadet.AIComments.AIComment
+
+  @doc """
+  Creates a new AI comment log entry.
+  """
+  def create_ai_comment(attrs \\ %{}) do
+    %AIComment{}
+    |> AIComment.changeset(attrs)
+    |> Repo.insert()
+  end
+
+  @doc """
+  Gets an AI comment by ID.
+  """
+  def get_ai_comment!(id), do: Repo.get!(AIComment, id)
+
+  @doc """
+  Retrieves an AI comment for a specific submission and question.
+  Returns `nil` if no comment exists.
+  """
+  def get_ai_comments_for_submission(submission_id, question_id) do
+    Repo.one(
+      from(c in AIComment,
+        where: c.submission_id == ^submission_id and c.question_id == ^question_id
+      )
+    )
+  end
+
+  @doc """
+  Retrieves the latest AI comment for a specific submission and question.
+  Returns `nil` if no comment exists.
+  """
+  def get_latest_ai_comment(submission_id, question_id) do
+    Repo.one(
+      from(c in AIComment,
+        where: c.submission_id == ^submission_id and c.question_id == ^question_id,
+        order_by: [desc: c.inserted_at],
+        limit: 1
+      )
+    )
+  end
+
+  @doc """
+  Updates the final comment for a specific submission and question.
+  Returns the most recent comment entry for that submission/question.
+  """
+  def update_final_comment(submission_id, question_id, final_comment) do
+    comment = get_latest_ai_comment(submission_id, question_id)
+
+    case comment do
+      nil ->
+        {:error, :not_found}
+
+      _ ->
+        comment
+        |> AIComment.changeset(%{final_comment: final_comment})
+        |> Repo.update()
+    end
+  end
+
+  @doc """
+  Updates an existing AI comment with new attributes.
+  """
+  def update_ai_comment(id, attrs) do
+    id
+    |> get_ai_comment!()
+    |> AIComment.changeset(attrs)
+    |> Repo.update()
+  end
+
+  @doc """
+  Updates the chosen comments for a specific submission and question.
+  Accepts an array of comments and replaces the existing array in the database.
+  """
+  def update_chosen_comments(submission_id, question_id, new_comments) do
+    comment = get_latest_ai_comment(submission_id, question_id)
+
+    case comment do
+      nil ->
+        {:error, :not_found}
+
+      _ ->
+        comment
+        |> AIComment.changeset(%{comment_chosen: new_comments})
+        |> Repo.update()
+    end
+  end
+end

lib/cadet/ai_comments/ai_comment.ex

@@ -0,0 +1,36 @@
+defmodule Cadet.AIComments.AIComment do
+  @moduledoc """
+  Defines the schema and changeset for AI comments.
+  """
+
+  use Ecto.Schema
+  import Ecto.Changeset
+
+  schema "ai_comment_logs" do
+    field(:submission_id, :integer)
+    field(:question_id, :integer)
+    field(:raw_prompt, :string)
+    field(:answers_json, :string)
+    field(:response, :string)
+    field(:error, :string)
+    field(:comment_chosen, {:array, :string})
+    field(:final_comment, :string)
+
+    timestamps()
+  end
+
+  def changeset(ai_comment, attrs) do
+    ai_comment
+    |> cast(attrs, [
+      :submission_id,
+      :question_id,
+      :raw_prompt,
+      :answers_json,
+      :response,
+      :error,
+      :comment_chosen,
+      :final_comment
+    ])
+    |> validate_required([:submission_id, :question_id, :raw_prompt, :answers_json])
+  end
+end

lib/cadet/assessments/assessments.ex

@@ -941,7 +941,7 @@
         raw_answer,
         force_submit
       ) do
     with {:ok, team} <- find_team(question.assessment.id, cr_id),
          {:ok, submission} <- find_or_create_submission(cr, question.assessment),
          {:status, true} <- {:status, force_submit or submission.status != :submitted},
          {:ok, _answer} <- insert_or_update_answer(submission, question, raw_answer, cr_id) do
@@ -2289,8 +2289,8 @@
   @spec get_answers_in_submission(integer() | String.t()) ::
           {:ok, {[Answer.t()], Assessment.t()}}
           | {:error, {:bad_request, String.t()}}
-  def get_answers_in_submission(id) when is_ecto_id(id) do
-    answer_query =
+  def get_answers_in_submission(id, question_id \\ nil) when is_ecto_id(id) do
+    base_query =
       Answer
       |> where(submission_id: ^id)
       |> join(:inner, [a], q in assoc(a, :question))
@@ -2312,6 +2312,13 @@
           {s, student: {st, user: u}, team: {t, team_members: {tm, student: {tms, user: tmu}}}}
       )
 
+    answer_query =
+      if is_nil(question_id) do
+        base_query
+      else
+        base_query |> where(question_id: ^question_id)
+      end
+
     answers =
       answer_query
       |> Repo.all()
@@ -2692,7 +2699,7 @@
 
   def has_last_modified_answer?(
         question = %Question{},
         cr = %CourseRegistration{id: cr_id},
         last_modified_at,
         force_submit
       ) do

lib/cadet/assessments/question_types/programming_question.ex

@@ -13,13 +13,14 @@ defmodule Cadet.Assessments.QuestionTypes.ProgrammingQuestion do
     field(:template, :string)
     field(:postpend, :string, default: "")
     field(:solution, :string)
+    field(:llm_prompt, :string)
     embeds_many(:public, Testcase)
     embeds_many(:opaque, Testcase)
     embeds_many(:secret, Testcase)
   end
 
   @required_fields ~w(content template)a
-  @optional_fields ~w(solution prepend postpend)a
+  @optional_fields ~w(solution prepend postpend llm_prompt)a
 
   def changeset(question, params \\ %{}) do
     question

lib/cadet/courses/course.ex

@@ -14,6 +14,8 @@ defmodule Cadet.Courses.Course do
           enable_achievements: boolean(),
           enable_sourcecast: boolean(),
           enable_stories: boolean(),
+          enable_llm_grading: boolean(),
+          llm_api_key: String.t() | nil,
           source_chapter: integer(),
           source_variant: String.t(),
           module_help_text: String.t(),
@@ -28,6 +30,8 @@ defmodule Cadet.Courses.Course do
     field(:enable_achievements, :boolean, default: true)
     field(:enable_sourcecast, :boolean, default: true)
     field(:enable_stories, :boolean, default: false)
+    field(:enable_llm_grading, :boolean)
+    field(:llm_api_key, :string)
     field(:source_chapter, :integer)
     field(:source_variant, :string)
     field(:module_help_text, :string)
@@ -42,13 +46,59 @@ defmodule Cadet.Courses.Course do
 
   @required_fields ~w(course_name viewable enable_game
     enable_achievements enable_sourcecast enable_stories source_chapter source_variant)a
-  @optional_fields ~w(course_short_name module_help_text)a
-
+  @optional_fields ~w(course_short_name module_help_text enable_llm_grading llm_api_key)a
+
+  @spec changeset(
+          {map(), map()}
+          | %{
+              :__struct__ => atom() | %{:__changeset__ => map(), optional(any()) => any()},
+              optional(atom()) => any()
+            },
+          %{optional(:__struct__) => none(), optional(atom() | binary()) => any()}
+        ) :: Ecto.Changeset.t()
   def changeset(course, params) do
     course
     |> cast(params, @required_fields ++ @optional_fields)
     |> validate_required(@required_fields)
     |> validate_sublanguage_combination(params)
+    |> put_encrypted_llm_api_key()
+  end
+
+  def put_encrypted_llm_api_key(changeset) do
+    if llm_api_key = get_change(changeset, :llm_api_key) do
+      if is_binary(llm_api_key) and llm_api_key != "" do
+        secret = Application.get_env(:openai, :encryption_key)
+
+        if is_binary(secret) and byte_size(secret) >= 16 do
+          # Use first 16 bytes for AES-128, 24 for AES-192, or 32 for AES-256
+          key = binary_part(secret, 0, min(32, byte_size(secret)))
+          # Use AES in GCM mode for encryption
+          iv = :crypto.strong_rand_bytes(16)
+
+          {ciphertext, tag} =
+            :crypto.crypto_one_time_aead(
+              :aes_gcm,
+              key,
+              iv,
+              llm_api_key,
+              "",
+              true
+            )
+
+          # Store both the IV, ciphertext and tag
+          encrypted = iv <> tag <> ciphertext
+          put_change(changeset, :llm_api_key, Base.encode64(encrypted))
+        else
+          add_error(changeset, :llm_api_key, "encryption key not configured properly")
+        end
+      else
+        # If empty string or nil is provided, don't encrypt but don't add error
+        changeset
+      end
+    else
+      # The key is not being changed, so we need to preserve the existing value
+      put_change(changeset, :llm_api_key, changeset.data.llm_api_key)
+    end
   end
 
   # Validates combination of Source chapter and variant

lib/cadet/jobs/xml_parser.ex

@@ -202,7 +202,8 @@ defmodule Cadet.Updater.XMLParser do
         prepend: ~x"./SNIPPET/PREPEND/text()" |> transform_by(&process_charlist/1),
         template: ~x"./SNIPPET/TEMPLATE/text()" |> transform_by(&process_charlist/1),
         postpend: ~x"./SNIPPET/POSTPEND/text()" |> transform_by(&process_charlist/1),
-        solution: ~x"./SNIPPET/SOLUTION/text()" |> transform_by(&process_charlist/1)
+        solution: ~x"./SNIPPET/SOLUTION/text()" |> transform_by(&process_charlist/1),
+        llm_prompt: ~x"./LLM_GRADING_PROMPT/text()" |> transform_by(&process_charlist/1)
       ),
       entity
       |> xmap(

lib/cadet_web/admin_controllers/admin_courses_controller.ex

@@ -108,6 +108,8 @@ defmodule CadetWeb.AdminCoursesController do
       enable_achievements(:body, :boolean, "Enable achievements")
       enable_sourcecast(:body, :boolean, "Enable sourcecast")
       enable_stories(:body, :boolean, "Enable stories")
+      enable_llm_grading(:body, :boolean, "Enable LLM grading")
+      llm_api_key(:body, :string, "OpenAI API key for this course")
       sublanguage(:body, Schema.ref(:AdminSublanguage), "sublanguage object")
       module_help_text(:body, :string, "Module help text")
     end

lib/cadet_web/controllers/courses_controller.ex

@@ -56,6 +56,8 @@ defmodule CadetWeb.CoursesController do
       enable_achievements(:body, :boolean, "Enable achievements", required: true)
       enable_sourcecast(:body, :boolean, "Enable sourcecast", required: true)
       enable_stories(:body, :boolean, "Enable stories", required: true)
+      enable_llm_grading(:body, :boolean, "Enable LLM grading", required: false)
+      llm_api_key(:body, :string, "OpenAI API key for this course", required: false)
       source_chapter(:body, :number, "Default source chapter", required: true)
 
       source_variant(:body, Schema.ref(:SourceVariant), "Default source variant name",
@@ -97,6 +99,8 @@ defmodule CadetWeb.CoursesController do
             enable_achievements(:boolean, "Enable achievements", required: true)
             enable_sourcecast(:boolean, "Enable sourcecast", required: true)
             enable_stories(:boolean, "Enable stories", required: true)
+            enable_llm_grading(:boolean, "Enable LLM grading", required: false)
+            llm_api_key(:string, "OpenAI API key for this course", required: false)
             source_chapter(:integer, "Source Chapter number from 1 to 4", required: true)
             source_variant(Schema.ref(:SourceVariant), "Source Variant name", required: true)
             module_help_text(:string, "Module help text", required: true)
@@ -111,6 +115,8 @@ defmodule CadetWeb.CoursesController do
             enable_achievements: true,
             enable_sourcecast: true,
             enable_stories: false,
+            enable_llm_grading: false,
+            llm_api_key: "sk-1234567890",
             source_chapter: 1,
             source_variant: "default",
             module_help_text: "Help text",