Report CSV data per case instead of per repository for more granular information

Closes #390

Author: bauersimon

cmd/eval-dev-quality/cmd/evaluate_test.go

@@ -597,12 +597,12 @@ func TestEvaluateExecute(t *testing.T) {
 				filepath.Join("result-directory", "config.json"): nil,
 				filepath.Join("result-directory", "evaluation.csv"): func(t *testing.T, filePath, data string) {
 					// Check if the runs are written to the CSV file.
-					assert.Contains(t, data, "golang,"+filepath.Join("golang", "plain")+",write-tests,1")
-					assert.Contains(t, data, "golang,"+filepath.Join("golang", "plain")+",write-tests,2")
-					assert.Contains(t, data, "golang,"+filepath.Join("golang", "plain")+",write-tests,3")
-					assert.Contains(t, data, "golang,"+filepath.Join("golang", "plain")+",write-tests-symflower-fix,1")
-					assert.Contains(t, data, "golang,"+filepath.Join("golang", "plain")+",write-tests-symflower-fix,2")
-					assert.Contains(t, data, "golang,"+filepath.Join("golang", "plain")+",write-tests-symflower-fix,3")
+					assert.Contains(t, data, "golang,"+filepath.Join("golang", "plain")+",plain.go,write-tests,1")
+					assert.Contains(t, data, "golang,"+filepath.Join("golang", "plain")+",plain.go,write-tests,2")
+					assert.Contains(t, data, "golang,"+filepath.Join("golang", "plain")+",plain.go,write-tests,3")
+					assert.Contains(t, data, "golang,"+filepath.Join("golang", "plain")+",plain.go,write-tests-symflower-fix,1")
+					assert.Contains(t, data, "golang,"+filepath.Join("golang", "plain")+",plain.go,write-tests-symflower-fix,2")
+					assert.Contains(t, data, "golang,"+filepath.Join("golang", "plain")+",plain.go,write-tests-symflower-fix,3")
 
 					_ = validateMetrics(t, data, []metrics.Assessments{
 						metrics.Assessments{

evaluate/evaluate.go

@@ -13,6 +13,7 @@ import (
 	"github.com/symflower/eval-dev-quality/provider"
 	evaltask "github.com/symflower/eval-dev-quality/task"
 	"github.com/symflower/eval-dev-quality/util"
+	"golang.org/x/exp/maps"
 )
 
 // Context holds an evaluation context.
@@ -322,7 +323,12 @@ func withLoadedModel(logger *log.Logger, model evalmodel.Model, modelProvider pr
 }
 
 // succeededPlain checks if the assessments attest that the "plain" repository was successfully solved.
-func succeededPlain(assessment map[evaltask.Identifier]metrics.Assessments) bool {
+func succeededPlain(assessmentPerCase map[string]map[evaltask.Identifier]metrics.Assessments) bool {
+	if len(assessmentPerCase) != 1 { // The "plain" repository only has one case.
+		return false
+	}
+	assessment := assessmentPerCase[maps.Keys(assessmentPerCase)[0]]
+
 	if withoutTemplate, ok := assessment[evaluatetask.IdentifierWriteTests]; ok && withoutTemplate[metrics.AssessmentKeyFilesExecuted] > 0 {
 		return true
 	} else if withTemplate, ok := assessment[evaluatetask.IdentifierWriteTestsSymflowerTemplate]; ok && withTemplate[metrics.AssessmentKeyFilesExecuted] > 0 {

evaluate/evaluate_test.go

@@ -55,6 +55,7 @@ type AssessmentTuple struct {
 	Model          string
 	Language       string
 	RepositoryPath string
+	Case           string
 	Task           task.Identifier
 	Assessment     metrics.Assessments
 }

evaluate/metrics/testing/assessments.go

@@ -5,6 +5,7 @@ import (
 	"encoding/csv"
 	"io"
 	"slices"
+	"sort"
 	"strconv"
 
 	pkgerrors "github.com/pkg/errors"
@@ -39,17 +40,24 @@ func NewEvaluationFile(writer io.Writer) (evaluationFile *EvaluationFile, err er
 }
 
 // WriteEvaluationRecord writes the assessments of a task into the evaluation CSV.
-func (e *EvaluationFile) WriteEvaluationRecord(model model.Model, language language.Language, repositoryName string, run uint, assessmentsPerTask map[task.Identifier]metrics.Assessments) (err error) {
-	tasks := maps.Keys(assessmentsPerTask)
-	slices.SortStableFunc(tasks, func(a, b task.Identifier) int {
-		return cmp.Compare(a, b)
-	})
-
+func (e *EvaluationFile) WriteEvaluationRecord(model model.Model, language language.Language, repositoryName string, run uint, assessmentsPerCasePerTask map[string]map[task.Identifier]metrics.Assessments) (err error) {
 	allRecords := [][]string{}
-	for _, task := range tasks {
-		assessment := assessmentsPerTask[task]
-		row := append([]string{model.ID(), language.ID(), repositoryName, string(task), strconv.FormatUint(uint64(run), 10)}, assessment.StringCSV()...)
-		allRecords = append(allRecords, row)
+
+	cases := maps.Keys(assessmentsPerCasePerTask)
+	sort.Strings(cases)
+	for _, caseName := range cases {
+		assessmentsPerTask := assessmentsPerCasePerTask[caseName]
+
+		tasks := maps.Keys(assessmentsPerTask)
+		slices.SortStableFunc(tasks, func(a, b task.Identifier) int {
+			return cmp.Compare(a, b)
+		})
+
+		for _, task := range tasks {
+			assessment := assessmentsPerTask[task]
+			row := append([]string{model.ID(), language.ID(), repositoryName, caseName, string(task), strconv.FormatUint(uint64(run), 10)}, assessment.StringCSV()...)
+			allRecords = append(allRecords, row)
+		}
 	}
 
 	return e.WriteLines(allRecords)
@@ -72,5 +80,5 @@ func (e *EvaluationFile) WriteLines(records [][]string) (err error) {
 
 // EvaluationHeader returns the CSV header for the evaluation CSV.
 func EvaluationHeader() (header []string) {
-	return append([]string{"model-id", "language", "repository", "task", "run"}, metrics.AllAssessmentKeysStrings...)
+	return append([]string{"model-id", "language", "repository", "case", "task", "run"}, metrics.AllAssessmentKeysStrings...)
 }

evaluate/report/csv.go

@@ -24,7 +24,7 @@ func TestNewEvaluationFile(t *testing.T) {
 	require.NoError(t, err)
 
 	expectedEvaluationFileContent := bytesutil.StringTrimIndentations(`
-		model-id,language,repository,task,run,coverage,files-executed,files-executed-maximum-reachable,generate-tests-for-file-character-count,processing-time,response-character-count,response-no-error,response-no-excess,response-with-code,tests-passing
+		model-id,language,repository,case,task,run,coverage,files-executed,files-executed-maximum-reachable,generate-tests-for-file-character-count,processing-time,response-character-count,response-no-error,response-no-excess,response-with-code,tests-passing
 	`)
 
 	assert.Equal(t, expectedEvaluationFileContent, string(actualEvaluationFileContent))
@@ -34,7 +34,7 @@ func TestWriteEvaluationRecord(t *testing.T) {
 	type testCase struct {
 		Name string
 
-		Assessments map[task.Identifier]metrics.Assessments
+		Assessments map[string]map[task.Identifier]metrics.Assessments
 
 		ExpectedCSV string
 	}
@@ -58,37 +58,41 @@ func TestWriteEvaluationRecord(t *testing.T) {
 	validate(t, &testCase{
 		Name: "Single task with empty assessments",
 
-		Assessments: map[task.Identifier]metrics.Assessments{
-			evaluatetask.IdentifierWriteTests: metrics.NewAssessments(),
+		Assessments: map[string]map[task.Identifier]metrics.Assessments{
+			"plain.go": {
+				evaluatetask.IdentifierWriteTests: metrics.NewAssessments(),
+			},
 		},
 
 		ExpectedCSV: `
-			model-id,language,repository,task,run,coverage,files-executed,files-executed-maximum-reachable,generate-tests-for-file-character-count,processing-time,response-character-count,response-no-error,response-no-excess,response-with-code,tests-passing
-			mocked-model,golang,golang/plain,write-tests,1,0,0,0,0,0,0,0,0,0,0
+			model-id,language,repository,case,task,run,coverage,files-executed,files-executed-maximum-reachable,generate-tests-for-file-character-count,processing-time,response-character-count,response-no-error,response-no-excess,response-with-code,tests-passing
+			mocked-model,golang,golang/plain,plain.go,write-tests,1,0,0,0,0,0,0,0,0,0,0
 		`,
 	})
 	validate(t, &testCase{
 		Name: "Multiple tasks with assessments",
 
-		Assessments: map[task.Identifier]metrics.Assessments{
-			evaluatetask.IdentifierWriteTests: metrics.Assessments{
-				metrics.AssessmentKeyFilesExecuted:                 1,
-				metrics.AssessmentKeyFilesExecutedMaximumReachable: 1,
-				metrics.AssessmentKeyResponseNoError:               1,
-				metrics.AssessmentKeyCoverage:                      0,
-			},
-			evaluatetask.IdentifierWriteTestsSymflowerFix: metrics.Assessments{
-				metrics.AssessmentKeyFilesExecuted:                 1,
-				metrics.AssessmentKeyFilesExecutedMaximumReachable: 1,
-				metrics.AssessmentKeyResponseNoError:               1,
-				metrics.AssessmentKeyCoverage:                      10,
+		Assessments: map[string]map[task.Identifier]metrics.Assessments{
+			"plain.go": {
+				evaluatetask.IdentifierWriteTests: metrics.Assessments{
+					metrics.AssessmentKeyFilesExecuted:                 1,
+					metrics.AssessmentKeyFilesExecutedMaximumReachable: 1,
+					metrics.AssessmentKeyResponseNoError:               1,
+					metrics.AssessmentKeyCoverage:                      0,
+				},
+				evaluatetask.IdentifierWriteTestsSymflowerFix: metrics.Assessments{
+					metrics.AssessmentKeyFilesExecuted:                 1,
+					metrics.AssessmentKeyFilesExecutedMaximumReachable: 1,
+					metrics.AssessmentKeyResponseNoError:               1,
+					metrics.AssessmentKeyCoverage:                      10,
+				},
 			},
 		},
 
 		ExpectedCSV: `
-			model-id,language,repository,task,run,coverage,files-executed,files-executed-maximum-reachable,generate-tests-for-file-character-count,processing-time,response-character-count,response-no-error,response-no-excess,response-with-code,tests-passing
-			mocked-model,golang,golang/plain,write-tests,1,0,1,1,0,0,0,1,0,0,0
-			mocked-model,golang,golang/plain,write-tests-symflower-fix,1,10,1,1,0,0,0,1,0,0,0
+			model-id,language,repository,case,task,run,coverage,files-executed,files-executed-maximum-reachable,generate-tests-for-file-character-count,processing-time,response-character-count,response-no-error,response-no-excess,response-with-code,tests-passing
+			mocked-model,golang,golang/plain,plain.go,write-tests,1,0,1,1,0,0,0,1,0,0,0
+			mocked-model,golang,golang/plain,plain.go,write-tests-symflower-fix,1,10,1,1,0,0,0,1,0,0,0
 		`,
 	})
 }

evaluate/report/csv_test.go

@@ -19,7 +19,7 @@ func atoiUint64(t *testing.T, s string) uint64 {
 }
 
 // extractMetricsCSVMatch is a regular expression to extract metrics from CSV rows.
-var extractMetricsCSVMatch = regexp.MustCompile(`(\S+),(\S+),(\S+),(\S+),\d+,(\d+),(\d+),(\d+),(\d+),(\d+),(\d+),(\d+),(\d+),(\d+),(\d+)`)
+var extractMetricsCSVMatch = regexp.MustCompile(`(\S+),(\S+),(\S+),(\S+),(\S+),\d+,(\d+),(\d+),(\d+),(\d+),(\d+),(\d+),(\d+),(\d+),(\d+),(\d+)`)
 
 // ParseMetrics extracts multiple assessment metrics from the given string.
 func ParseMetrics(t *testing.T, data string) (assessments metricstesting.AssessmentTuples) {
@@ -30,17 +30,18 @@ func ParseMetrics(t *testing.T, data string) (assessments metricstesting.Assessm
 			Model:          match[1],
 			Language:       match[2],
 			RepositoryPath: match[3],
-			Task:           task.Identifier(match[4]),
+			Case:           match[4],
+			Task:           task.Identifier(match[5]),
 			Assessment: metrics.Assessments{
-				metrics.AssessmentKeyCoverage:                           atoiUint64(t, match[5]),
-				metrics.AssessmentKeyFilesExecuted:                      atoiUint64(t, match[6]),
-				metrics.AssessmentKeyFilesExecutedMaximumReachable:      atoiUint64(t, match[7]),
-				metrics.AssessmentKeyGenerateTestsForFileCharacterCount: atoiUint64(t, match[8]),
-				metrics.AssessmentKeyProcessingTime:                     atoiUint64(t, match[9]),
-				metrics.AssessmentKeyResponseCharacterCount:             atoiUint64(t, match[10]),
-				metrics.AssessmentKeyResponseNoError:                    atoiUint64(t, match[11]),
-				metrics.AssessmentKeyResponseNoExcess:                   atoiUint64(t, match[12]),
-				metrics.AssessmentKeyResponseWithCode:                   atoiUint64(t, match[13]),
+				metrics.AssessmentKeyCoverage:                           atoiUint64(t, match[6]),
+				metrics.AssessmentKeyFilesExecuted:                      atoiUint64(t, match[7]),
+				metrics.AssessmentKeyFilesExecutedMaximumReachable:      atoiUint64(t, match[8]),
+				metrics.AssessmentKeyGenerateTestsForFileCharacterCount: atoiUint64(t, match[9]),
+				metrics.AssessmentKeyProcessingTime:                     atoiUint64(t, match[10]),
+				metrics.AssessmentKeyResponseCharacterCount:             atoiUint64(t, match[11]),
+				metrics.AssessmentKeyResponseNoError:                    atoiUint64(t, match[12]),
+				metrics.AssessmentKeyResponseNoExcess:                   atoiUint64(t, match[13]),
+				metrics.AssessmentKeyResponseWithCode:                   atoiUint64(t, match[14]),
 			},
 		})
 	}

evaluate/report/testing/csv.go

@@ -33,7 +33,7 @@ func (t *CodeRepair) Identifier() evaltask.Identifier {
 
 // Run performs source code repairing in a repository with compilation errors.
 // This task requires the repository to consist of multiple packages, with each containing one faulty implementation file and a corresponding test file.
-func (t *CodeRepair) Run(ctx evaltask.Context) (repositoryAssessment map[evaltask.Identifier]metrics.Assessments, problems []error, err error) {
+func (t *CodeRepair) Run(ctx evaltask.Context) (repositoryAssessment map[string]map[evaltask.Identifier]metrics.Assessments, problems []error, err error) {
 	modelCapability, ok := ctx.Model.(model.CapabilityRepairCode)
 	if !ok {
 		return nil, nil, pkgerrors.Wrap(evaltask.ErrTaskUnsupportedByModel, fmt.Sprintf("%q does not support %q", ctx.Model.ID(), string(t.Identifier())))
@@ -54,26 +54,31 @@ func (t *CodeRepair) Run(ctx evaltask.Context) (repositoryAssessment map[evaltas
 	}
 	for _, file := range files {
 		if file.IsDir() && !strings.HasPrefix(file.Name(), ".") { // Ignore hidden directories.
-			packagePaths = append(packagePaths, filepath.Join(ctx.Repository.DataPath(), file.Name()))
+			packagePaths = append(packagePaths, file.Name())
 		}
 	}
 
-	modelAssessment := metrics.NewAssessments()
-	modelAssessment[metrics.AssessmentKeyFilesExecutedMaximumReachable] = uint64(len(packagePaths))
+	repositoryAssessment = map[string]map[evaltask.Identifier]metrics.Assessments{}
 	for _, packagePath := range packagePaths {
+		modelAssessment := metrics.NewAssessments()
+		modelAssessment[metrics.AssessmentKeyFilesExecutedMaximumReachable] = 1
+		repositoryAssessment[packagePath] = map[evaltask.Identifier]metrics.Assessments{
+			IdentifierCodeRepair: modelAssessment,
+		}
+
 		if err := ctx.Repository.Reset(ctx.Logger); err != nil {
 			ctx.Logger.Panicf("ERROR: unable to reset temporary repository path: %s", err)
 		}
 
-		sourceFile, mistakes, err := t.unpackCodeRepairPackage(ctx, taskLogger.Logger, packagePath)
+		sourceFile, mistakes, err := t.unpackCodeRepairPackage(ctx, taskLogger.Logger, filepath.Join(ctx.Repository.DataPath(), packagePath))
 		if err != nil {
 			return nil, nil, err
 		}
 
 		modelContext := model.Context{
 			Language: ctx.Language,
 
-			RepositoryPath: packagePath,
+			RepositoryPath: filepath.Join(ctx.Repository.DataPath(), packagePath),
 			FilePath:       sourceFile,
 
 			Arguments: &ArgumentsCodeRepair{
@@ -94,7 +99,7 @@ func (t *CodeRepair) Run(ctx evaltask.Context) (repositoryAssessment map[evaltas
 		modelAssessment.Add(assessments)
 		modelAssessment.Award(metrics.AssessmentKeyResponseNoError)
 
-		testResult, ps, err := ctx.Language.ExecuteTests(taskLogger.Logger, packagePath)
+		testResult, ps, err := ctx.Language.ExecuteTests(taskLogger.Logger, filepath.Join(ctx.Repository.DataPath(), packagePath))
 		problems = append(problems, ps...)
 		if err != nil {
 			problems = append(problems, pkgerrors.WithMessage(err, sourceFile))
@@ -107,10 +112,6 @@ func (t *CodeRepair) Run(ctx evaltask.Context) (repositoryAssessment map[evaltas
 		modelAssessment.AwardMultiple(metrics.AssessmentKeyTestsPassing, uint64(testsPassing))
 	}
 
-	repositoryAssessment = map[evaltask.Identifier]metrics.Assessments{
-		IdentifierCodeRepair: modelAssessment,
-	}
-
 	return repositoryAssessment, problems, nil
 }