AI-powered marking

.gitignore

@@ -118,3 +118,6 @@ erl_crash.dump
 
 # Generated lexer
 /src/source_lexer.erl
+
+# Ignore log files
+/log

config/dev.secrets.exs.example

@@ -99,12 +99,8 @@ config :cadet,
   ]
 
   config :openai,
-  # find it at https://platform.openai.com/account/api-keys
-  api_key: "the actual api key",
-  # For source academy deployment, leave this as empty string.Ingeneral could find it at https://platform.openai.com/account/org-settings under "Organization ID".
-  organization_key: "",
-  # optional, passed to [HTTPoison.Request](https://hexdocs.pm/httpoison/HTTPoison.Request.html) options
-  http_options: [recv_timeout: 170_0000]
+    # TODO: Input your own AES-256 encryption key here for encrypting LLM API keys
+    encryption_key: "<ADD YOUR OWN KEY HERE>"
 
 # config :sentry,
 #   dsn: "https://public_key/sentry.io/somethingsomething"

config/test.exs

@@ -100,3 +100,7 @@ config :cadet, Oban,
   testing: :manual
 
 config :cadet, Cadet.Mailer, adapter: Bamboo.TestAdapter
+
+config :openai,
+  # TODO: Input your own AES-256 encryption key here for encrypting LLM API keys
+  encryption_key: "b4u7g0AyN3Tu2br9WSdZQjLMQ8bed/wgQWrH2x3qPdW8D55iv10+ySgs+bxDirWE"

lib/cadet/ai_comments.ex

@@ -0,0 +1,107 @@
+defmodule Cadet.AIComments do
+  @moduledoc """
+  Handles operations related to AI comments, including creation, updates, and retrieval.
+  """
+
+  import Ecto.Query
+  alias Cadet.Repo
+  alias Cadet.AIComments.AIComment
+
+  @doc """
+  Creates a new AI comment log entry.
+  """
+  def create_ai_comment(attrs \\ %{}) do
+    %AIComment{}
+    |> AIComment.changeset(attrs)
+    |> Repo.insert()
+  end
+
+  @doc """
+  Gets an AI comment by ID.
+  """
+  def get_ai_comment(id) do
+    case Repo.get(AIComment, id) do
+      nil -> {:error, :not_found}
+      comment -> {:ok, comment}
+    end
+  end
+
+  @doc """
+  Retrieves an AI comment for a specific submission and question.
+  Returns `nil` if no comment exists.
+  """
+  def get_ai_comments_for_submission(submission_id, question_id) do
+    Repo.one(
+      from(c in AIComment,
+        where: c.submission_id == ^submission_id and c.question_id == ^question_id
+      )
+    )
+  end
+
+  @doc """
+  Retrieves the latest AI comment for a specific submission and question.
+  Returns `nil` if no comment exists.
+  """
+  def get_latest_ai_comment(submission_id, question_id) do
+    Repo.one(
+      from(c in AIComment,
+        where: c.submission_id == ^submission_id and c.question_id == ^question_id,
+        order_by: [desc: c.inserted_at],
+        limit: 1
+      )
+    )
+  end
+
+  @doc """
+  Updates the final comment for a specific submission and question.
+  Returns the most recent comment entry for that submission/question.
+  """
+  def update_final_comment(submission_id, question_id, final_comment) do
+    comment = get_latest_ai_comment(submission_id, question_id)
+
+    case comment do
+      nil ->
+        {:error, :not_found}
+
+      _ ->
+        comment
+        |> AIComment.changeset(%{final_comment: final_comment})
+        |> Repo.update()
+    end
+  end
+
+  @doc """
+  Updates an existing AI comment with new attributes.
+  """
+  def update_ai_comment(id, attrs) do
+    id
+    |> get_ai_comment()
+    |> case do
+      {:error, :not_found} ->
+        {:error, :not_found}
+
+      {:ok, comment} ->
+        comment
+        |> AIComment.changeset(attrs)
+        |> Repo.update()
+    end
+  end
+
+  @doc """
+  Updates the chosen comments for a specific submission and question.
+  Accepts an array of comments and replaces the existing array in the database.
+  """
+  def update_chosen_comments(submission_id, question_id, new_comments) do
+    comment = get_latest_ai_comment(submission_id, question_id)
+
+    case comment do
+      nil ->
+        {:error, :not_found}
+
+      _ ->
+        comment
+        |> AIComment.changeset(%{comment_chosen: new_comments})
+        |> Repo.update()
+    end
+  end
+end

lib/cadet/ai_comments/ai_comment.ex

@@ -0,0 +1,39 @@
+defmodule Cadet.AIComments.AIComment do
+  @moduledoc """
+  Defines the schema and changeset for AI comments.
+  """
+
+  use Ecto.Schema
+  import Ecto.Changeset
+
+  schema "ai_comment_logs" do
+    field(:raw_prompt, :string)
+    field(:answers_json, :string)
+    field(:response, :string)
+    field(:error, :string)
+    field(:comment_chosen, {:array, :string})
+    field(:final_comment, :string)
+
+    belongs_to(:submission, Cadet.Assessments.Submission)
+    belongs_to(:question, Cadet.Assessments.Question)
+
+    timestamps()
+  end
+
+  def changeset(ai_comment, attrs) do
+    ai_comment
+    |> cast(attrs, [
+      :submission_id,
+      :question_id,
+      :raw_prompt,
+      :answers_json,
+      :response,
+      :error,
+      :comment_chosen,
+      :final_comment
+    ])
+    |> validate_required([:submission_id, :question_id, :raw_prompt, :answers_json])
+    |> foreign_key_constraint(:submission_id)
+    |> foreign_key_constraint(:question_id)
+  end
+end

lib/cadet/assessments/answer.ex

@@ -29,6 +29,7 @@ defmodule Cadet.Assessments.Answer do
     belongs_to(:grader, CourseRegistration)
     belongs_to(:submission, Submission)
     belongs_to(:question, Question)
+    has_many(:ai_comments, through: [:submission, :ai_comments])
 
     timestamps()
   end

lib/cadet/assessments/assessment.ex

@@ -36,6 +36,7 @@ defmodule Cadet.Assessments.Assessment do
     field(:max_team_size, :integer, default: 1)
     field(:has_token_counter, :boolean, default: false)
     field(:has_voting_features, :boolean, default: false)
+    field(:llm_assessment_prompt, :string, default: nil)
 
     belongs_to(:config, AssessmentConfig)
     belongs_to(:course, Course)
@@ -46,7 +47,7 @@ defmodule Cadet.Assessments.Assessment do
 
   @required_fields ~w(title open_at close_at number course_id config_id max_team_size)a
   @optional_fields ~w(reading summary_short summary_long
-    is_published story cover_picture access password has_token_counter has_voting_features)a
+    is_published story cover_picture access password has_token_counter has_voting_features llm_assessment_prompt)a
   @optional_file_fields ~w(mission_pdf)a
 
   def changeset(assessment, params) do

lib/cadet/assessments/assessments.ex

@@ -3046,29 +3046,39 @@ defmodule Cadet.Assessments do
   @spec get_answers_in_submission(integer() | String.t()) ::
           {:ok, {[Answer.t()], Assessment.t()}}
           | {:error, {:bad_request, String.t()}}
-  def get_answers_in_submission(id) when is_ecto_id(id) do
-    answer_query =
+  def get_answers_in_submission(id, question_id \\ nil) when is_ecto_id(id) do
+    base_query =
       Answer
       |> where(submission_id: ^id)
+      # [a] are bindings (in SQL it is similar to FROM answers "AS a"),
+      # this line's alias is INNER JOIN ... "AS q"
       |> join(:inner, [a], q in assoc(a, :question))
       |> join(:inner, [_, q], ast in assoc(q, :assessment))
       |> join(:inner, [..., ast], ac in assoc(ast, :config))
       |> join(:left, [a, ...], g in assoc(a, :grader))
       |> join(:left, [_, ..., g], gu in assoc(g, :user))
       |> join(:inner, [a, ...], s in assoc(a, :submission))
-      |> join(:left, [_, ..., s], st in assoc(s, :student))
+      |> join(:left, [..., s], ai in assoc(s, :ai_comments))
+      |> join(:left, [_, ..., s, _], st in assoc(s, :student))
       |> join(:left, [..., st], u in assoc(st, :user))
-      |> join(:left, [..., s, _, _], t in assoc(s, :team))
+      |> join(:left, [..., s, _, _, _], t in assoc(s, :team))
       |> join(:left, [..., t], tm in assoc(t, :team_members))
       |> join(:left, [..., tm], tms in assoc(tm, :student))
       |> join(:left, [..., tms], tmu in assoc(tms, :user))
-      |> preload([_, q, ast, ac, g, gu, s, st, u, t, tm, tms, tmu],
+      |> preload([_, q, ast, ac, g, gu, s, ai, st, u, t, tm, tms, tmu],
+        ai_comments: ai,
         question: {q, assessment: {ast, config: ac}},
         grader: {g, user: gu},
         submission:
           {s, student: {st, user: u}, team: {t, team_members: {tm, student: {tms, user: tmu}}}}
       )
 
+    answer_query =
+      case question_id do
+        nil -> base_query
+        _ -> base_query |> where(question_id: ^question_id)
+      end
+
     answers =
       answer_query
       |> Repo.all()
@@ -3544,4 +3554,15 @@ defmodule Cadet.Assessments do
         end)
     end
   end
+
+  def get_llm_assessment_prompt(question_id) do
+    query =
+      from(q in Question,
+        where: q.id == ^question_id,
+        join: a in assoc(q, :assessment),
+        select: a.llm_assessment_prompt
+      )
+
+    Repo.one(query)
+  end
 end

lib/cadet/assessments/question_types/programming_question.ex

@@ -13,13 +13,14 @@ defmodule Cadet.Assessments.QuestionTypes.ProgrammingQuestion do
     field(:template, :string)
     field(:postpend, :string, default: "")
     field(:solution, :string)
+    field(:llm_prompt, :string)
     embeds_many(:public, Testcase)
     embeds_many(:opaque, Testcase)
     embeds_many(:secret, Testcase)
   end
 
   @required_fields ~w(content template)a
-  @optional_fields ~w(solution prepend postpend)a
+  @optional_fields ~w(solution prepend postpend llm_prompt)a
 
   def changeset(question, params \\ %{}) do
     question

lib/cadet/assessments/submission.ex

@@ -25,6 +25,7 @@ defmodule Cadet.Assessments.Submission do
     belongs_to(:team, Team)
     belongs_to(:unsubmitted_by, CourseRegistration)
     has_many(:answers, Answer, on_delete: :delete_all)
+    has_many(:ai_comments, Cadet.AIComments.AIComment, on_delete: :delete_all)
 
     timestamps()
   end

lib/cadet/courses/course.ex

@@ -18,6 +18,11 @@ defmodule Cadet.Courses.Course do
           top_contest_leaderboard_display: integer(),
           enable_sourcecast: boolean(),
           enable_stories: boolean(),
+          enable_llm_grading: boolean(),
+          llm_api_key: String.t() | nil,
+          llm_model: String.t() | nil,
+          llm_api_url: String.t() | nil,
+          llm_course_level_prompt: String.t() | nil,
           source_chapter: integer(),
           source_variant: String.t(),
           module_help_text: String.t(),
@@ -36,6 +41,11 @@ defmodule Cadet.Courses.Course do
     field(:top_contest_leaderboard_display, :integer, default: 10)
     field(:enable_sourcecast, :boolean, default: true)
     field(:enable_stories, :boolean, default: false)
+    field(:enable_llm_grading, :boolean, default: false)
+    field(:llm_api_key, :string, default: nil)
+    field(:llm_model, :string, default: nil)
+    field(:llm_api_url, :string, default: nil)
+    field(:llm_course_level_prompt, :string, default: nil)
     field(:source_chapter, :integer)
     field(:source_variant, :string)
     field(:module_help_text, :string)
@@ -50,13 +60,65 @@ defmodule Cadet.Courses.Course do
 
   @required_fields ~w(course_name viewable enable_game
     enable_achievements enable_overall_leaderboard enable_contest_leaderboard top_leaderboard_display top_contest_leaderboard_display enable_sourcecast enable_stories source_chapter source_variant)a
-  @optional_fields ~w(course_short_name module_help_text)a
+  @optional_fields ~w(course_short_name module_help_text enable_llm_grading llm_api_key llm_model llm_api_url llm_course_level_prompt)a
 
   def changeset(course, params) do
     course
     |> cast(params, @required_fields ++ @optional_fields)
     |> validate_required(@required_fields)
     |> validate_sublanguage_combination(params)
+    |> put_encrypted_llm_api_key()
+  end
+
+  def encrypt_llm_api_key(llm_api_key) do
+    secret = Application.get_env(:openai, :encryption_key)
+
+    if is_binary(secret) and byte_size(secret) >= 16 do
+      # Use first 16 bytes for AES-128, 24 for AES-192, or 32 for AES-256
+      key = binary_part(secret, 0, min(32, byte_size(secret)))
+      # Use AES in GCM mode for encryption
+      iv = :crypto.strong_rand_bytes(16)
+
+      {ciphertext, tag} =
+        :crypto.crypto_one_time_aead(
+          :aes_gcm,
+          key,
+          iv,
+          llm_api_key,
+          "",
+          true
+        )
+
+      # Store both the IV, ciphertext and tag
+      encrypted = Base.encode64(iv <> tag <> ciphertext)
+    else
+      nil
+    end
+  end
+
+  def put_encrypted_llm_api_key(changeset) do
+    if llm_api_key = get_change(changeset, :llm_api_key) do
+      if is_binary(llm_api_key) and llm_api_key != "" do
+        encrypted = encrypt_llm_api_key(llm_api_key)
+
+        case encrypted do
+          nil ->
+            add_error(
+              changeset,
+              :llm_api_key,
+              "encryption key is not configured properly, cannot store LLM API key"
+            )
+
+          encrypted ->
+            put_change(changeset, :llm_api_key, encrypted)
+        end
+      else
+        # If empty string or nil is provided, don't encrypt but don't add error
+        changeset
+      end
+    else
+      changeset
+    end
   end
 
   # Validates combination of Source chapter and variant

lib/cadet/jobs/xml_parser.ex

@@ -98,6 +98,8 @@ defmodule Cadet.Updater.XMLParser do
         reading: ~x"//READING/text()" |> transform_by(&process_charlist/1),
         summary_short: ~x"//WEBSUMMARY/text()" |> transform_by(&process_charlist/1),
         summary_long: ~x"./TEXT/text()" |> transform_by(&process_charlist/1),
+        llm_assessment_prompt:
+          ~x"./LLM_ASSESSMENT_PROMPT/text()" |> transform_by(&process_charlist/1),
         password: ~x"//PASSWORD/text()"so |> transform_by(&process_charlist/1)
       )
       |> Map.put(:is_published, false)
@@ -202,7 +204,8 @@ defmodule Cadet.Updater.XMLParser do
         prepend: ~x"./SNIPPET/PREPEND/text()" |> transform_by(&process_charlist/1),
         template: ~x"./SNIPPET/TEMPLATE/text()" |> transform_by(&process_charlist/1),
         postpend: ~x"./SNIPPET/POSTPEND/text()" |> transform_by(&process_charlist/1),
-        solution: ~x"./SNIPPET/SOLUTION/text()" |> transform_by(&process_charlist/1)
+        solution: ~x"./SNIPPET/SOLUTION/text()" |> transform_by(&process_charlist/1),
+        llm_prompt: ~x"./LLM_GRADING_PROMPT/text()" |> transform_by(&process_charlist/1)
       ),
       entity
       |> xmap(