Generalize assessments to be of type float64 instead of uint64

Author: ahumenberger

cmd/eval-dev-quality/cmd/evaluate_test.go

@@ -34,7 +34,7 @@ func validateMetrics(t *testing.T, csvData string, expectedAssessments []metrics
 	actualAssessmentTuples := reporttesting.ParseMetrics(t, csvData)
 	actual = make([]metrics.Assessments, len(actualAssessmentTuples))
 	for i, tuple := range actualAssessmentTuples {
-		assert.Greater(t, tuple.Assessment[metrics.AssessmentKeyProcessingTime], uint64(0))
+		assert.Greater(t, tuple.Assessment[metrics.AssessmentKeyProcessingTime], float64(0))
 		actual[i] = tuple.Assessment
 	}
 

evaluate/evaluate_test.go

@@ -62,11 +62,11 @@ var (
 )
 
 // Assessments holds a collection of numerical assessment metrics.
-type Assessments map[AssessmentKey]uint64
+type Assessments map[AssessmentKey]float64
 
 // NewAssessments creates a new assessment collection.
 func NewAssessments() Assessments {
-	return map[AssessmentKey]uint64{}
+	return map[AssessmentKey]float64{}
 }
 
 // Add adds the given assessment collection to the current one.
@@ -98,7 +98,7 @@ func (a Assessments) Award(key AssessmentKey) {
 
 // AwardMultiple yields multiple score points.
 func (a Assessments) AwardMultiple(key AssessmentKey, count uint64) {
-	a[key] += count
+	a[key] += float64(count)
 }
 
 // String returns a string representation of the metrics.
@@ -109,7 +109,7 @@ func (a Assessments) String() string {
 	entries := make([]string, len(AllAssessmentKeys))
 
 	for i, key := range AllAssessmentKeys {
-		entries[i] = fmt.Sprintf("%s=%d", key, a[key])
+		entries[i] = fmt.Sprintf("%s=%v", key, a[key])
 	}
 
 	return strings.Join(entries, ", ")
@@ -123,7 +123,7 @@ func (a Assessments) StringCSV() (row []string) {
 
 	row = make([]string, len(AllAssessmentKeys))
 	for i, key := range AllAssessmentKeys {
-		row[i] = fmt.Sprintf("%d", a[key])
+		row[i] = fmt.Sprintf("%v", a[key])
 	}
 
 	return row

evaluate/metrics/assessment.go

@@ -37,26 +37,26 @@ func TestAssessmentsAdd(t *testing.T) {
 		Name: "Non existing key",
 
 		Assessments: NewAssessments(),
-		X: map[AssessmentKey]uint64{
+		X: map[AssessmentKey]float64{
 			AssessmentKeyResponseNoExcess: 1,
 		},
 
-		ExpectedAssessments: map[AssessmentKey]uint64{
+		ExpectedAssessments: map[AssessmentKey]float64{
 			AssessmentKeyResponseNoExcess: 1,
 		},
 	})
 
 	validate(t, &testCase{
 		Name: "Existing key",
 
-		Assessments: map[AssessmentKey]uint64{
+		Assessments: map[AssessmentKey]float64{
 			AssessmentKeyResponseNoExcess: 1,
 		},
-		X: map[AssessmentKey]uint64{
+		X: map[AssessmentKey]float64{
 			AssessmentKeyResponseNoExcess: 1,
 		},
 
-		ExpectedAssessments: map[AssessmentKey]uint64{
+		ExpectedAssessments: map[AssessmentKey]float64{
 			AssessmentKeyResponseNoExcess: 2,
 		},
 	})
@@ -209,7 +209,7 @@ func TestCombineModelAndSymflowerFixAssessments(t *testing.T) {
 
 		ModelAssessment: Assessments{
 			AssessmentKeyFilesExecuted:                      1,
-			AssessmentKeyProcessingTime:                     uint64(200),
+			AssessmentKeyProcessingTime:                     float64(200),
 			AssessmentKeyCoverage:                           0,
 			AssessmentKeyResponseCharacterCount:             100,
 			AssessmentKeyGenerateTestsForFileCharacterCount: 50,
@@ -219,15 +219,15 @@ func TestCombineModelAndSymflowerFixAssessments(t *testing.T) {
 		},
 		SymflowerFixAssessments: Assessments{
 			AssessmentKeyFilesExecuted:   1,
-			AssessmentKeyProcessingTime:  uint64(100),
+			AssessmentKeyProcessingTime:  float64(100),
 			AssessmentKeyCoverage:        1,
 			AssessmentKeyResponseNoError: 1,
 			AssessmentKeyTestsPassing:    10,
 		},
 
 		ExpectedAssessments: Assessments{
 			AssessmentKeyFilesExecuted:                      1,
-			AssessmentKeyProcessingTime:                     uint64(300),
+			AssessmentKeyProcessingTime:                     float64(300),
 			AssessmentKeyCoverage:                           1,
 			AssessmentKeyResponseCharacterCount:             100,
 			AssessmentKeyGenerateTestsForFileCharacterCount: 50,

evaluate/metrics/assessment_test.go

@@ -11,11 +11,11 @@ import (
 	"github.com/symflower/eval-dev-quality/task"
 )
 
-func atoiUint64(t *testing.T, s string) uint64 {
-	value, err := strconv.ParseUint(s, 10, 64)
+func parseFloat64(t *testing.T, s string) float64 {
+	value, err := strconv.ParseFloat(s, 64)
 	assert.NoErrorf(t, err, "parsing unsigned integer from: %q", s)
 
-	return uint64(value)
+	return value
 }
 
 // ParseMetrics extracts multiple assessment metrics from the given string.
@@ -37,7 +37,7 @@ func ParseMetrics(t *testing.T, data string) (assessments metricstesting.Assessm
 			Assessment:     metrics.Assessments{},
 		}
 		for i, key := range metrics.AllAssessmentKeys {
-			tuple.Assessment[key] = atoiUint64(t, cells[i+6])
+			tuple.Assessment[key] = parseFloat64(t, cells[i+6])
 		}
 
 		assessments = append(assessments, tuple)

evaluate/report/testing/csv.go

@@ -109,7 +109,7 @@ func runModelAndSymflowerFix(ctx evaltask.Context, modelCtx model.Context, runMo
 
 			// Symflower was able to fix a failure so now update the assessment with the improved results.
 			withSymflowerFix := metrics.NewAssessments()
-			withSymflowerFix[metrics.AssessmentKeyProcessingTime] = processingTime
+			withSymflowerFix[metrics.AssessmentKeyProcessingTime] = float64(processingTime)
 			withSymflowerFix.Award(metrics.AssessmentKeyFilesExecuted)
 			withSymflowerFix.AwardMultiple(metrics.AssessmentKeyCoverage, withSymflowerFixTestResult.Coverage)
 

evaluate/task/symflower.go

@@ -69,8 +69,8 @@ func (t *Transpile) Run(ctx evaltask.Context) (repositoryAssessment map[string]m
 		modelAssessments := metrics.NewAssessments()
 		withSymflowerAssessments := metrics.NewAssessments()
 		maximumReachableFiles := uint64(len(language.Languages) - 1) // Transpile repositories contain sub-tasks to transpile from every other supported language minus the one we are transpiling to.
-		modelAssessments[metrics.AssessmentKeyFilesExecutedMaximumReachable] = maximumReachableFiles
-		withSymflowerAssessments[metrics.AssessmentKeyFilesExecutedMaximumReachable] = maximumReachableFiles
+		modelAssessments[metrics.AssessmentKeyFilesExecutedMaximumReachable] = float64(maximumReachableFiles)
+		withSymflowerAssessments[metrics.AssessmentKeyFilesExecutedMaximumReachable] = float64(maximumReachableFiles)
 		repositoryAssessment[packagePath] = map[evaltask.Identifier]metrics.Assessments{
 			IdentifierTranspile:             modelAssessments,
 			IdentifierTranspileSymflowerFix: withSymflowerAssessments,
@@ -131,7 +131,7 @@ func (t *Transpile) Run(ctx evaltask.Context) (repositoryAssessment map[string]m
 
 					// Symflower was able to fix a failure so now update the assessment with the improved results.
 					withSymflowerFixAssessments := metrics.NewAssessments()
-					withSymflowerFixAssessments[metrics.AssessmentKeyProcessingTime] = processingTime
+					withSymflowerFixAssessments[metrics.AssessmentKeyProcessingTime] = float64(processingTime)
 					withSymflowerFixAssessments.Award(metrics.AssessmentKeyFilesExecuted)
 					withSymflowerFixAssessments.AwardMultiple(metrics.AssessmentKeyTestsPassing, uint64(testsPassing))
 

evaluate/task/transpile.go

@@ -491,11 +491,11 @@ func handleQueryResult(queryResult *provider.QueryResult, filePathAbsolute strin
 	if err != nil {
 		return nil, pkgerrors.WithStack(err)
 	}
-	assessment[metrics.AssessmentKeyProcessingTime] = uint64(queryResult.Duration.Milliseconds())
-	assessment[metrics.AssessmentKeyResponseCharacterCount] = uint64(len(queryResult.Message))
-	assessment[metrics.AssessmentKeyGenerateTestsForFileCharacterCount] = uint64(len(sourceFileContent))
-	assessment[metrics.AssessmentKeyTokenInput] = uint64(queryResult.Usage.PromptTokens)
-	assessment[metrics.AssessmentKeyTokenOutput] = uint64(queryResult.Usage.CompletionTokens)
+	assessment[metrics.AssessmentKeyProcessingTime] = float64(queryResult.Duration.Milliseconds())
+	assessment[metrics.AssessmentKeyResponseCharacterCount] = float64(len(queryResult.Message))
+	assessment[metrics.AssessmentKeyGenerateTestsForFileCharacterCount] = float64(len(sourceFileContent))
+	assessment[metrics.AssessmentKeyTokenInput] = float64(queryResult.Usage.PromptTokens)
+	assessment[metrics.AssessmentKeyTokenOutput] = float64(queryResult.Usage.CompletionTokens)
 
 	if err := os.MkdirAll(filepath.Dir(filePathAbsolute), 0755); err != nil {
 		return nil, pkgerrors.WithStack(err)

model/llm/llm.go

@@ -121,7 +121,7 @@ func (m *Model) WriteTests(ctx model.Context) (assessment metrics.Assessments, e
 		return nil, pkgerrors.WithStack(err)
 	}
 
-	processingTime := uint64(time.Since(start).Milliseconds())
+	processingTime := float64(time.Since(start).Milliseconds())
 
 	characterCount, err := countCharactersOfGeneratedFiles(ctx.RepositoryPath, extractGeneratedFilePaths(output))
 	if err != nil {
@@ -130,8 +130,8 @@ func (m *Model) WriteTests(ctx model.Context) (assessment metrics.Assessments, e
 
 	return metrics.Assessments{ // Symflower always generates just source code when it does not fail, so no need to check the assessment properties.
 		metrics.AssessmentKeyProcessingTime:                     processingTime,
-		metrics.AssessmentKeyGenerateTestsForFileCharacterCount: characterCount,
-		metrics.AssessmentKeyResponseCharacterCount:             characterCount,
+		metrics.AssessmentKeyGenerateTestsForFileCharacterCount: float64(characterCount),
+		metrics.AssessmentKeyResponseCharacterCount:             float64(characterCount),
 		metrics.AssessmentKeyResponseNoExcess:                   1,
 		metrics.AssessmentKeyResponseWithCode:                   1,
 	}, nil