d

Author: markhuang1212

backend/cmd/compile_user_code.go

@@ -80,12 +80,11 @@ func CompileUserCode(ctx context.Context, gr types.GradeRequest, tmpDir string)
 	}
 
 	defer func() {
-		err := cli.ContainerRemove(ctxdl, resp.ID, dockertypes.ContainerRemoveOptions{
+		err := cli.ContainerRemove(ctx, resp.ID, dockertypes.ContainerRemoveOptions{
 			Force: true,
 		})
 		if err != nil {
 			log.Panicln("cannot kill and remove container")
-			panic(err)
 		}
 	}()
 

backend/cmd/exec_user_code.go

@@ -20,10 +20,13 @@ import (
 const imageExec = "markhuang1212/code-grader/runtime-exec:latest"
 
 type ExecUserCodeResult struct {
-	Ok           bool
-	MemoryExceed bool
-	TimeExceed   bool
-	Msg          string
+	Ok             bool
+	WrongAnswer    bool
+	MemoryExceed   bool
+	TimeExceed     bool
+	ExecutionError bool
+	Duration       time.Duration
+	Msg            string
 }
 
 func ExecUserCode(ctx context.Context, gr types.GradeRequest, tmpDir string) (*ExecUserCodeResult, error) {
@@ -76,6 +79,7 @@ func ExecUserCode(ctx context.Context, gr types.GradeRequest, tmpDir string) (*E
 	if err != nil {
 		return nil, errors.Wrap(err, "cannot start container")
 	}
+	startTime := time.Now()
 
 	defer func() {
 		err := cli.ContainerRemove(ctx, resp.ID, dockertypes.ContainerRemoveOptions{
@@ -88,12 +92,14 @@ func ExecUserCode(ctx context.Context, gr types.GradeRequest, tmpDir string) (*E
 
 	select {
 	case status := <-statusCh:
+		endTime := time.Now()
+		result.Duration = endTime.Sub(startTime)
 		switch status.StatusCode {
 		case 0:
 			result.Ok = true
 			result.Msg = "correct answer"
 			return result, nil
-		case 1, 2:
+		case 1:
 			out, err := cli.ContainerLogs(ctxdl, resp.ID, dockertypes.ContainerLogsOptions{
 				ShowStderr: true,
 				ShowStdout: true,
@@ -103,6 +109,20 @@ func ExecUserCode(ctx context.Context, gr types.GradeRequest, tmpDir string) (*E
 			}
 			text, _ := io.ReadAll(out)
 			result.Msg = string(text)
+			result.WrongAnswer = true
+			result.Ok = false
+			return result, nil
+		case 2:
+			out, err := cli.ContainerLogs(ctxdl, resp.ID, dockertypes.ContainerLogsOptions{
+				ShowStderr: true,
+				ShowStdout: true,
+			})
+			if err != nil {
+				return nil, errors.Wrap(err, "error reading stdout")
+			}
+			text, _ := io.ReadAll(out)
+			result.Msg = string(text)
+			result.ExecutionError = true
 			result.Ok = false
 			return result, nil
 		case 3:

backend/cmd/exec_user_code_test.go

@@ -11,7 +11,7 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
-func TestExecUserCode(t *testing.T) {
+func TestExecUserCode1(t *testing.T) {
 
 	ctx := context.Background()
 
@@ -36,3 +36,57 @@ func TestExecUserCode(t *testing.T) {
 	assert.True(t, er.Ok)
 
 }
+
+func TestExecUserCode2(t *testing.T) {
+
+	ctx := context.Background()
+
+	tmpDir, err := ioutil.TempDir("/tmp", "")
+	assert.Nil(t, err)
+	defer os.RemoveAll(tmpDir)
+
+	err = os.Chmod(tmpDir, 0777)
+	assert.Nil(t, err)
+
+	gr := types.GradeRequest{
+		TestCaseName: "example-1",
+		UserCode:     "int main() { return 10; }",
+	}
+
+	cr, err := cmd.CompileUserCode(ctx, gr, tmpDir)
+	assert.Nil(t, err)
+	assert.True(t, cr.Ok)
+
+	er, err := cmd.ExecUserCode(ctx, gr, tmpDir)
+	assert.Nil(t, err)
+	assert.False(t, er.Ok)
+	assert.True(t, er.ExecutionError)
+
+}
+
+func TestExecUserCode3(t *testing.T) {
+
+	ctx := context.Background()
+
+	tmpDir, err := ioutil.TempDir("/tmp", "")
+	assert.Nil(t, err)
+	defer os.RemoveAll(tmpDir)
+
+	err = os.Chmod(tmpDir, 0777)
+	assert.Nil(t, err)
+
+	gr := types.GradeRequest{
+		TestCaseName: "example-1",
+		UserCode:     "int main() { vector<int> data; while(1) { data.push_back(100); } }",
+	}
+
+	cr, err := cmd.CompileUserCode(ctx, gr, tmpDir)
+	assert.Nil(t, err)
+	assert.True(t, cr.Ok)
+
+	er, err := cmd.ExecUserCode(ctx, gr, tmpDir)
+	assert.Nil(t, err)
+	assert.False(t, er.Ok)
+	assert.True(t, er.MemoryExceed)
+
+}

backend/cmd/grade_user_code.go

@@ -37,14 +37,17 @@ func GradeUserCode(ctx context.Context, gr types.GradeRequest) (*types.GradeResu
 		return nil, errors.Wrap(err, "cannot execute")
 	}
 
+	result.Duration = er.Duration
 	if !er.Ok {
 		result.Msg = er.Msg
 		if er.MemoryExceed {
 			result.Status = types.GradeResultMemoryExceed
 		} else if er.TimeExceed {
 			result.Status = types.GradeResultTimeLimitExceed
-		} else {
+		} else if er.WrongAnswer {
 			result.Status = types.GradeResultWrongAnswer
+		} else {
+			return nil, types.ErrInternal
 		}
 		return &result, nil
 	}

backend/cmd/grade_user_code_test.go

@@ -9,6 +9,7 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
+// success
 func TestGradeUserCode1(t *testing.T) {
 	ctx := context.Background()
 	gr := types.GradeRequest{
@@ -17,16 +18,41 @@ func TestGradeUserCode1(t *testing.T) {
 	}
 	result, err := cmd.GradeUserCode(ctx, gr)
 	assert.Nil(t, err)
-	assert.Equal(t, result.Status, types.GradeResultSuccess)
+	assert.Equal(t, types.GradeResultSuccess, result.Status)
 }
 
+// wrong answer
 func TestGradeUserCode2(t *testing.T) {
+	ctx := context.Background()
+	gr := types.GradeRequest{
+		TestCaseName: "example-1",
+		UserCode:     "int main() { cout << \"Helloooo\" << endl; }",
+	}
+	result, err := cmd.GradeUserCode(ctx, gr)
+	assert.Nil(t, err)
+	assert.Equal(t, types.GradeResultWrongAnswer, result.Status)
+}
+
+// time limit exceed
+func TestGradeUserCode3(t *testing.T) {
 	ctx := context.Background()
 	gr := types.GradeRequest{
 		TestCaseName: "example-1",
 		UserCode:     "int main() { while (1) { } }",
 	}
 	result, err := cmd.GradeUserCode(ctx, gr)
 	assert.Nil(t, err)
-	assert.Equal(t, result.Status, types.GradeResultTimeLimitExceed)
+	assert.Equal(t, types.GradeResultTimeLimitExceed, result.Status)
+}
+
+// memory limit exceed
+func TestGradeUserCode4(t *testing.T) {
+	ctx := context.Background()
+	gr := types.GradeRequest{
+		TestCaseName: "example-1",
+		UserCode:     "int main() { vector<int> data; while (1) { data.push_back(1024); } }",
+	}
+	result, err := cmd.GradeUserCode(ctx, gr)
+	assert.Nil(t, err)
+	assert.Equal(t, types.GradeResultMemoryExceed, result.Status)
 }

backend/types/GradeResult.go

@@ -1,17 +1,21 @@
 package types
 
+import "time"
+
 type GradeResultStatus int
 
 const (
 	GradeResultSuccess GradeResultStatus = iota
 	GradeResultWrongAnswer
+	GradeResultExecutionError
 	GradeResultInternalError
 	GradeResultCompilationError
 	GradeResultTimeLimitExceed
 	GradeResultMemoryExceed
 )
 
 type GradeResult struct {
-	Status GradeResultStatus
-	Msg    string
+	Status   GradeResultStatus
+	Duration time.Duration
+	Msg      string
 }

runtime-exec/run.sh

@@ -15,10 +15,20 @@ then
 fi
 
 ${DATA_DIR}/a.out < ${TEST_CASE_DIR}/input.txt > ${DATA_DIR}/output.txt 2>&1
+STATUS=$?
 
-if [[ $? -ne 0 ]]
+if [[ $STATUS -eq 137 ]]
 then
-    cat output.txt
+    exit 137
+fi
+
+if [[ $STATUS -ne 0 ]]
+then
+    echo "program exited with status $STATUS"
+    if [[ -f output.txt ]];
+    then
+        cat output.txt
+    fi
     exit 2
 fi
 

testcases/example-1/testcase.json

@@ -1,6 +1,6 @@
 {
     "RuntimeOptions": {
-        "MemoryLimit": 512,
-        "RuntimeLimit": 10
+        "MemoryLimit": 64,
+        "RuntimeLimit": 5
     }
 }