Skip to content

Commit 2fc125f

Browse files
bauersimonzimmski
bauersimon
authored and
zimmski
committed
Add maximum scores per task case to repository.json
Part of #390
1 parent 3b9df3b commit 2fc125f

File tree

14 files changed

+574
-45
lines changed

14 files changed

+574
-45
lines changed

‎task/config.go‎

Lines changed: 31 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -9,27 +9,39 @@ import (
99
"strings"
1010

1111
pkgerrors "github.com/pkg/errors"
12-
"github.com/symflower/eval-dev-quality/util"
1312
"github.com/zimmski/osutil"
13+
14+
"github.com/symflower/eval-dev-quality/evaluate/metrics"
15+
"github.com/symflower/eval-dev-quality/util"
1416
)
1517

1618
// RepositoryConfiguration holds the configuration of a repository.
1719
type RepositoryConfiguration struct {
1820
// Tasks holds the tasks supported by the repository.
19-
Tasks []Identifier
21+
Tasks []Identifier `json:"tasks"`
2022
// IgnorePaths holds the relative paths that should be ignored when searching for cases.
2123
IgnorePaths []string `json:"ignore,omitempty"`
2224

2325
// Prompt holds LLM prompt-related configuration.
24-
Prompt struct {
25-
// TestFramework overwrites the language-specific test framework to use.
26-
TestFramework string `json:"test-framework,omitempty"`
27-
} `json:",omitempty"`
26+
Prompt RepositoryConfigurationPrompt `json:"prompt,omitempty"`
2827

2928
// Validation holds quality gates for evaluation.
30-
Validation struct {
31-
Execution RepositoryConfigurationExecution `json:",omitempty"`
32-
}
29+
Validation RepositoryConfigurationValidation `json:"validation,omitempty"`
30+
31+
// MaxScores holds the maximum scores per task type, case and metric for this repository.
32+
MaxScores map[Identifier]map[string]map[metrics.AssessmentKey]uint64 `json:"scores,omitempty"`
33+
}
34+
35+
// RepositoryConfigurationPrompt holds LLM prompt-related configuration.
36+
type RepositoryConfigurationPrompt struct {
37+
// TestFramework overwrites the language-specific test framework to use.
38+
TestFramework string `json:"test-framework,omitempty"`
39+
}
40+
41+
// RepositoryConfigurationValidation holds quality gates for evaluation.
42+
type RepositoryConfigurationValidation struct {
43+
// Execution holds execution-related validation.
44+
Execution RepositoryConfigurationExecution `json:"execution,omitempty"`
3345
}
3446

3547
// RepositoryConfigurationExecution execution-related quality gates for evaluation.
@@ -67,6 +79,16 @@ func LoadRepositoryConfiguration(path string, defaultTasks []Identifier) (config
6779
return config, nil
6880
}
6981

82+
// Write stores the configuration on the given path.
83+
func (rc *RepositoryConfiguration) Write(path string) (err error) {
84+
data, err := json.Marshal(rc)
85+
if err != nil {
86+
return err
87+
}
88+
89+
return os.WriteFile(path, data, 0666)
90+
}
91+
7092
// validate validates the configuration.
7193
func (rc *RepositoryConfiguration) validate(validTasks []Identifier) (err error) {
7294
if len(rc.Tasks) == 0 {

‎task/config_test.go‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,16 @@
11
package task
22

33
import (
4+
"os"
5+
"path/filepath"
6+
"strings"
47
"testing"
58

69
"github.com/stretchr/testify/assert"
10+
"github.com/stretchr/testify/require"
11+
"github.com/zimmski/osutil/bytesutil"
12+
13+
"github.com/symflower/eval-dev-quality/evaluate/metrics"
714
)
815

916
func TestConfigurationIsFilePathIgnored(t *testing.T) {
@@ -57,3 +64,71 @@ func TestConfigurationIsFilePathIgnored(t *testing.T) {
5764
ExpectedBool: true,
5865
})
5966
}
67+
68+
var configJSON = bytesutil.TrimIndentations([]byte(`
69+
{
70+
"tasks": ["task-identifier"],
71+
"ignore": ["some-path"],
72+
"prompt" : {
73+
"test-framework": "some-framework"
74+
},
75+
"validation": {
76+
"execution": {
77+
"stdout": ".*"
78+
}
79+
},
80+
"scores": {
81+
"task-identifier": {
82+
"case-identifier": {
83+
"metric-identifier": 123
84+
}
85+
}
86+
}
87+
}
88+
`))
89+
90+
var configParsed = &RepositoryConfiguration{
91+
Tasks: []Identifier{
92+
Identifier("task-identifier"),
93+
},
94+
95+
IgnorePaths: []string{
96+
"some-path",
97+
},
98+
99+
Prompt: RepositoryConfigurationPrompt{
100+
TestFramework: "some-framework",
101+
},
102+
103+
Validation: RepositoryConfigurationValidation{
104+
Execution: RepositoryConfigurationExecution{
105+
StdOutRE: ".*",
106+
},
107+
},
108+
109+
MaxScores: map[Identifier]map[string]map[metrics.AssessmentKey]uint64{
110+
Identifier("task-identifier"): {
111+
"case-identifier": {
112+
metrics.AssessmentKey("metric-identifier"): 123,
113+
},
114+
},
115+
},
116+
}
117+
118+
func TestLoadRepositoryConfiguration(t *testing.T) {
119+
configurationPath := filepath.Join(t.TempDir(), "repository.json")
120+
require.NoError(t, os.WriteFile(configurationPath, configJSON, 0666))
121+
122+
configuration, err := LoadRepositoryConfiguration(configurationPath, nil)
123+
assert.NoError(t, err)
124+
assert.Equal(t, configParsed, configuration)
125+
}
126+
127+
func TestWriteRepositoryConfiguration(t *testing.T) {
128+
configurationPath := filepath.Join(t.TempDir(), "repository.json")
129+
assert.NoError(t, configParsed.Write(configurationPath))
130+
131+
data, err := os.ReadFile(configurationPath)
132+
require.NoError(t, err)
133+
assert.Equal(t, strings.NewReplacer("\n", "", "\t", "", " ", "").Replace(string(configJSON)), string(data))
134+
}

‎testdata/golang/light/repository.json‎

Lines changed: 97 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,99 @@
11
{
2-
"tasks": [
3-
"write-tests"
4-
]
2+
"tasks": ["write-tests"],
3+
"scores": {
4+
"write-tests": {
5+
"balancedBrackets.go": {
6+
"coverage": 5,
7+
"files-executed-maximum-reachable": 1
8+
},
9+
"binarySearch.go": {
10+
"coverage": 4,
11+
"files-executed-maximum-reachable": 1
12+
},
13+
"callLoopConditionsOftenEnough.go": {
14+
"coverage": 4,
15+
"files-executed-maximum-reachable": 1
16+
},
17+
"cascadingIfElse.go": {
18+
"coverage": 3,
19+
"files-executed-maximum-reachable": 1
20+
},
21+
"collatzCalculator.go": {
22+
"coverage": 6,
23+
"files-executed-maximum-reachable": 1
24+
},
25+
"conditionsAnd.go": {
26+
"coverage": 2,
27+
"files-executed-maximum-reachable": 1
28+
},
29+
"equilibriumIndices.go": {
30+
"coverage": 4,
31+
"files-executed-maximum-reachable": 1
32+
},
33+
"forLoop.go": {
34+
"coverage": 3,
35+
"files-executed-maximum-reachable": 1
36+
},
37+
"jacobiSymbol.go": {
38+
"coverage": 6,
39+
"files-executed-maximum-reachable": 1
40+
},
41+
"klarnerRadoSequence.go": {
42+
"coverage": 3,
43+
"files-executed-maximum-reachable": 1
44+
},
45+
"knapsack.go": {
46+
"coverage": 5,
47+
"files-executed-maximum-reachable": 1
48+
},
49+
"largestProperDivisor.go": {
50+
"coverage": 4,
51+
"files-executed-maximum-reachable": 1
52+
},
53+
"magicSquareOdd.go": {
54+
"coverage": 8,
55+
"files-executed-maximum-reachable": 1
56+
},
57+
"matchBytes.go": {
58+
"coverage": 2,
59+
"files-executed-maximum-reachable": 1
60+
},
61+
"pascalsTriangle.go": {
62+
"coverage": 4,
63+
"files-executed-maximum-reachable": 1
64+
},
65+
"phoneNumber.go": {
66+
"coverage": 5,
67+
"files-executed-maximum-reachable": 1
68+
},
69+
"simpleIfElse.go": {
70+
"coverage": 2,
71+
"files-executed-maximum-reachable": 1
72+
},
73+
"simpleSwitchWithReturn.go": {
74+
"coverage": 3,
75+
"files-executed-maximum-reachable": 1
76+
},
77+
"sort.go": {
78+
"coverage": 2,
79+
"files-executed-maximum-reachable": 1
80+
},
81+
"typeArrayAccess.go": {
82+
"coverage": 2,
83+
"files-executed-maximum-reachable": 1
84+
},
85+
"typeArrayConsecutiveAccess.go": {
86+
"coverage": 3,
87+
"files-executed-maximum-reachable": 1
88+
},
89+
"typeArrayMultidimensionalArrayLength.go": {
90+
"coverage": 3,
91+
"files-executed-maximum-reachable": 1
92+
},
93+
"validateDate.go": {
94+
"coverage": 7,
95+
"files-executed-maximum-reachable": 1
96+
}
97+
}
98+
}
599
}

‎testdata/golang/mistakes/repository.json‎

Lines changed: 25 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,27 @@
11
{
2-
"tasks": [
3-
"code-repair"
4-
]
2+
"tasks": ["code-repair"],
3+
"scores": {
4+
"code-repair": {
5+
"importMissing": {
6+
"tests-passing": 3,
7+
"files-executed-maximum-reachable": 1
8+
},
9+
"openingBracketMissing": {
10+
"tests-passing": 4,
11+
"files-executed-maximum-reachable": 1
12+
},
13+
"syntaxErrorTypeMissing": {
14+
"tests-passing": 4,
15+
"files-executed-maximum-reachable": 1
16+
},
17+
"typeUnknown": {
18+
"tests-passing": 4,
19+
"files-executed-maximum-reachable": 1
20+
},
21+
"variableUnknown": {
22+
"tests-passing": 4,
23+
"files-executed-maximum-reachable": 1
24+
}
25+
}
26+
}
527
}

‎testdata/golang/plain/repository.json‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,11 @@
11
{
2-
"tasks": [
3-
"write-tests"
4-
]
2+
"tasks": ["write-tests"],
3+
"scores": {
4+
"write-tests": {
5+
"plain.go": {
6+
"coverage": 1,
7+
"files-executed-maximum-reachable": 1
8+
}
9+
}
10+
}
511
}

‎testdata/golang/transpile/repository.json‎

Lines changed: 25 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,27 @@
11
{
2-
"tasks": [
3-
"transpile"
4-
]
2+
"tasks": ["transpile"],
3+
"scores": {
4+
"transpile": {
5+
"balancedBrackets": {
6+
"tests-passing": 6,
7+
"files-executed-maximum-reachable": 2
8+
},
9+
"binarySearch": {
10+
"tests-passing": 6,
11+
"files-executed-maximum-reachable": 2
12+
},
13+
"cascadingIfElse": {
14+
"tests-passing": 4,
15+
"files-executed-maximum-reachable": 2
16+
},
17+
"isSorted": {
18+
"tests-passing": 6,
19+
"files-executed-maximum-reachable": 2
20+
},
21+
"pascalsTriangle": {
22+
"tests-passing": 5,
23+
"files-executed-maximum-reachable": 2
24+
}
25+
}
26+
}
527
}

0 commit comments

Comments
 (0)