Improve coding challenge verdict messaging

.gitignore

@@ -3,3 +3,4 @@
 /.env
 /.devshell
 result
+/.idea/

challenges/src/endpoints/coding_challenges/submissions.rs

@@ -18,7 +18,9 @@
 use poem_ext::{db::DbTxn, response, responses::ErrorResponse};
 use poem_openapi::{param::Path, payload::Json, OpenApi};
 use sandkasten_client::{schemas::environments::Environment, SandkastenClient};
-use schemas::challenges::coding_challenges::{QueueStatus, Submission, SubmissionContent};
+use schemas::challenges::coding_challenges::{
+    CheckResult, QueueStatus, RunSummary, Submission, SubmissionContent,
+};
 use sea_orm::{
     ActiveModelTrait, ColumnTrait, DatabaseConnection, DatabaseTransaction, DbErr, EntityTrait,
     ModelTrait, QueryFilter, QueryOrder, Set, TransactionTrait,
@@ -37,6 +39,7 @@
             deduct_hearts, get_subtask, get_user_subtask, send_task_rewards, update_user_subtask,
             SendTaskRewardsError, UserSubtaskExt,
         },
+        verdict_message::{build_message, VerdictMessageContext},
     },
 };
 
@@ -50,6 +53,25 @@
     pub queue_positions: Arc<RwLock<QueuePositions>>,
 }
 
+fn attach_verdict_message(result: &mut CheckResult<RunSummary>, limits: Option<(u64, u64)>) {
+    let (time_limit_ms, memory_limit_mb) = limits
+        .map(|(time, mem)| (Some(time), Some(mem)))
+        .unwrap_or((None, None));
+
+    result.message = build_message(VerdictMessageContext {
+        verdict: result.verdict,
+        reason: result.reason.as_deref(),
+        compile_status: result.compile.as_ref().map(|r| r.status),
+        compile_stderr: result.compile.as_ref().map(|r| r.stderr.as_str()),
+        run_status: result.run.as_ref().map(|r| r.status),
+        run_stderr: result.run.as_ref().map(|r| r.stderr.as_str()),
+        run_time_ms: result.run.as_ref().map(|r| r.resource_usage.time),
+        run_memory_kib: result.run.as_ref().map(|r| r.resource_usage.memory),
+        time_limit_ms,
+        memory_limit_mb,
+    });
+}
+
 #[OpenApi(tag = "Tags::CodingChallenges")]
 impl Api {
     /// Return the current judge queue status.
@@ -96,7 +118,14 @@
                 .into_iter()
                 .map(|(submission, result)| {
                     let position = queue_positions.position(submission.id);
-                    Submission::from(&submission, result.map(Into::into), position)
+                    let mut result = result.map(Into::into);
+                    if let Some(ref mut res) = result {
+                        attach_verdict_message(
+                            res,
+                            Some((cc.time_limit as u64, cc.memory_limit as u64)),
+                        );
+                    }
+                    Submission::from(&submission, result, position)
                 })
                 .collect(),
         )
@@ -465,16 +494,16 @@
 }
 
 #[derive(Debug, Error)]
 enum JudgeSubmissionError {
     #[error("failed to judge submission: {0}")]
     Judge(#[from] judge::Error),
     #[error("database error: {0}")]
     Db(#[from] DbErr),
     #[error("check error: {0:?}")]
     Check(Box<CheckError>),
     #[error("could not send task rewards: {0}")]
     TaskRewards(#[from] SendTaskRewardsError),
 }
 
 impl Api {
     async fn get_environments(&self) -> Result<HashMap<String, Environment>, ErrorResponse> {
@@ -597,10 +626,10 @@
     }
 
     pub fn pop(&mut self, key: Uuid) -> bool {
         if !self
             .ids
             .get(&key)
             .is_some_and(|&x| self.id_position(x) == 0)
         {
             return false;
         }

challenges/src/main.rs

@@ -75,6 +75,12 @@ async fn main() -> anyhow::Result<()> {
         );
     }
 
+    info!("Running executor smoke checks");
+    match crate::services::judge::smoke_test_java(&sandkasten).await {
+        Ok(_) => info!("Java executor smoke test succeeded"),
+        Err(err) => warn!("Java executor smoke test failed: {err:?}"),
+    }
+
     let jwt_secret = JwtSecret::try_from(config.jwt_secret.as_str())?;
     let services = Services::from_config(
         jwt_secret.clone(),

challenges/src/services/judge.rs

@@ -1,10 +1,11 @@
+use anyhow::{bail, Context};
 use entity::sea_orm_active_enums::ChallengesVerdict;
 use fnct::{format::JsonFormatter, key};
 use lib::{Cache, CacheError};
 use sandkasten_client::{
     schemas::{
         programs::{
-            BuildRequest, BuildRunError, BuildRunRequest, BuildRunResult, File, LimitsOpt,
+            BuildRequest, BuildRunError, BuildRunRequest, BuildRunResult, EnvVar, File, LimitsOpt,
             MainFile, RunRequest, RunResult,
         },
         ErrorResponse,
@@ -16,8 +17,17 @@
 use serde_json::Value;
 use thiserror::Error;
 
+use super::verdict_message::{build_message, VerdictMessageContext};
+
 pub const EVALUATOR_TEMPLATE: &str = include_str!("../../assets/evaluator/template.py");
 pub const EVALUATOR_LIBRARY: &str = include_str!("../../assets/evaluator/lib.py");
+const JAVA_SMOKE_TEST: &str = r#"
+class Main {
+    public static void main(String[] args) {
+        System.out.println("ok");
+    }
+}
+"#;
 
 pub struct Judge<'a> {
     pub sandkasten: &'a SandkastenClient,
@@ -159,15 +169,29 @@
         let code = match prepare_result.code {
             Some(code) => code,
             None => {
+                let reason = prepare_result.reason;
                 return Ok(CheckResult {
                     verdict: ChallengesVerdict::PreCheckFailed,
-                    reason: Some(prepare_result.reason),
+                    reason: Some(reason.clone()),
                     compile: None,
                     run: None,
-                })
+                    message: build_message(VerdictMessageContext {
+                        verdict: ChallengesVerdict::PreCheckFailed,
+                        reason: Some(reason.as_str()),
+                        compile_status: None,
+                        compile_stderr: None,
+                        run_status: None,
+                        run_stderr: None,
+                        run_time_ms: None,
+                        run_memory_kib: None,
+                        time_limit_ms: time_limit,
+                        memory_limit_mb: memory_limit,
+                    }),
+                });
             }
         };
 
+        let runtime_env = java_env_vars(environment, memory_limit);
         let output = match self
             .sandkasten
             .build_and_run(&BuildRunRequest {
@@ -177,10 +201,12 @@
                         content: code,
                         ..Default::default()
                     },
+                    env_vars: runtime_env.clone(),
                     ..Default::default()
                 },
                 run: RunRequest {
                     stdin: Some(input.input.clone()),
+                    env_vars: runtime_env,
                     run_limits: LimitsOpt {
                         time: time_limit.map(|x| x / 1000 + 1),
                         memory: memory_limit,
@@ -196,12 +222,27 @@
                     ErrorResponse::Inner(BuildRunError::EnvironmentNotFound) => {
                         Err(Error::EnvironmentNotFound)
                     }
-                    ErrorResponse::Inner(BuildRunError::CompileError(result)) => Ok(CheckResult {
-                        verdict: ChallengesVerdict::CompilationError,
-                        reason: None,
-                        compile: Some(result),
-                        run: None,
-                    }),
+                    ErrorResponse::Inner(BuildRunError::CompileError(result)) => {
+                        let message = build_message(VerdictMessageContext {
+                            verdict: ChallengesVerdict::CompilationError,
+                            reason: None,
+                            compile_status: Some(result.status),
+                            compile_stderr: Some(&result.stderr),
+                            run_status: None,
+                            run_stderr: None,
+                            run_time_ms: None,
+                            run_memory_kib: None,
+                            time_limit_ms: time_limit,
+                            memory_limit_mb: memory_limit,
+                        });
+                        Ok(CheckResult {
+                            verdict: ChallengesVerdict::CompilationError,
+                            reason: None,
+                            compile: Some(result),
+                            run: None,
+                            message,
+                        })
+                    }
                     err => Err(Error::Sandkasten(SandkastenError::ErrorResponse(Box::new(
                         err,
                     )))),
@@ -220,11 +261,24 @@
             _ if output.run.stdout.is_empty() => Some(ChallengesVerdict::NoOutput),
             _ => None,
         } {
+            let message = build_message(VerdictMessageContext {
+                verdict,
+                reason: None,
+                compile_status: output.build.as_ref().map(|r| r.status),
+                compile_stderr: output.build.as_ref().map(|r| r.stderr.as_str()),
+                run_status: Some(output.run.status),
+                run_stderr: Some(output.run.stderr.as_str()),
+                run_time_ms: Some(output.run.resource_usage.time),
+                run_memory_kib: Some(output.run.resource_usage.memory),
+                time_limit_ms: time_limit,
+                memory_limit_mb: memory_limit,
+            });
             return Ok(CheckResult {
                 verdict,
                 reason: None,
                 compile: output.build,
                 run: Some(output.run),
+                message,
             });
         }
         let result = self
@@ -236,15 +290,71 @@
                 },
             )
             .await?;
+        let verdict = result.verdict;
+        let reason = result.reason;
+        let message = build_message(VerdictMessageContext {
+            verdict,
+            reason: reason.as_deref(),
+            compile_status: output.build.as_ref().map(|r| r.status),
+            compile_stderr: output.build.as_ref().map(|r| r.stderr.as_str()),
+            run_status: Some(output.run.status),
+            run_stderr: Some(output.run.stderr.as_str()),
+            run_time_ms: Some(output.run.resource_usage.time),
+            run_memory_kib: Some(output.run.resource_usage.memory),
+            time_limit_ms: time_limit,
+            memory_limit_mb: memory_limit,
+        });
         Ok(CheckResult {
-            verdict: result.verdict,
-            reason: result.reason,
+            verdict,
+            reason,
             compile: output.build,
             run: Some(output.run),
+            message,
         })
     }
 }
 
+fn java_env_vars(environment: &str, memory_limit_mb: Option<u64>) -> Vec<EnvVar> {
+    let env = environment.to_ascii_lowercase();
+    if env.starts_with("java") {
+        let mut env_vars = Vec::new();
+
+        let heap_settings = memory_limit_mb.and_then(|limit| {
+            if limit < 64 {
+                None
+            } else {
+                let headroom = limit.saturating_sub(16);
+                let suggested = ((limit as f64) * 0.6).round() as u64;
+                let xmx = suggested.min(headroom).max(32);
+                let xms = (xmx / 2).max(16);
+                Some((xms, xmx))
+            }
+        });
+
+        let tool_options = match heap_settings {
+            Some((xms, xmx)) => format!(
+                "-Xms{}m -Xmx{}m -Xss256k -XX:ThreadStackSize=256 -XX:+UseSerialGC",
+                xms, xmx
+            ),
+            None => "-Xss256k -XX:ThreadStackSize=256 -XX:+UseSerialGC".into(),
+        };
+
+        env_vars.push(EnvVar {
+            name: "JAVA_TOOL_OPTIONS".into(),
+            value: tool_options,
+        });
+
+        env_vars.push(EnvVar {
+            name: "MALLOC_ARENA_MAX".into(),
+            value: "2".into(),
+        });
+
+        env_vars
+    } else {
+        Vec::new()
+    }
+}
+
 pub async fn get_executor_config(
     cache: &Cache<JsonFormatter>,
     sandkasten: &SandkastenClient,
@@ -257,6 +367,39 @@
         .into())
 }
 
+pub async fn smoke_test_java(sandkasten: &SandkastenClient) -> anyhow::Result<()> {
+    let env_vars = java_env_vars("java", Some(256));
+    let result = sandkasten
+        .build_and_run(&BuildRunRequest {
+            build: BuildRequest {
+                environment: "java".into(),
+                main_file: MainFile {
+                    name: Some("Main.java".into()),
+                    content: JAVA_SMOKE_TEST.trim().into(),
+                    ..Default::default()
+                },
+                env_vars: env_vars.clone(),
+                ..Default::default()
+            },
+            run: RunRequest {
+                env_vars,
+                ..Default::default()
+            },
+        })
+        .await
+        .context("java smoke test execution failed")?;
+
+    if result.run.status != 0 {
+        bail!(
+            "java smoke test exited with status {} and stderr: {}",
+            result.run.status,
+            result.run.stderr
+        );
+    }
+
+    Ok(())
+}
+
 #[derive(Debug, Error)]
 pub enum Error {
     #[error("cache error: {0}")]

challenges/src/services/mod.rs

@@ -3,3 +3,4 @@ pub mod judge;
 pub mod leaderboard;
 pub mod subtasks;
 pub mod tasks;
+pub mod verdict_message;