fix, Collect assessments if a model responds with an empty message

Closes #427

Author: ahumenberger

evaluate/evaluate_test.go

@@ -161,15 +161,39 @@ func TestEvaluate(t *testing.T) {
 
 	{
 		languageGolang := &golang.Language{}
-		mockedModel := modeltesting.NewMockCapabilityWriteTestsNamed(t, "empty-response-model")
+		mockedModelID := "testing-provider/empty-response-model"
+		mockedQuery := providertesting.NewMockQuery(t)
+		mockedModel := llm.NewModel(mockedQuery, mockedModelID)
 		repositoryPath := filepath.Join("golang", "plain")
 
 		validate(t, &testCase{
-			Name: "Empty model responses are errors",
+			Name: "Empty model response",
 
 			Before: func(t *testing.T, logger *log.Logger, resultPath string) {
+				queryResult1 := &provider.QueryResult{
+					Message: "",
+					GenerationInfo: &provider.GenerationInfo{
+						TotalCost:              0.111111111,
+						NativeTokensPrompt:     111,
+						NativeTokensCompletion: 222,
+					},
+				}
+				// Set up mocks, when test is running.
+				mockedQuery.On("Query", mock.Anything, mock.Anything, mock.Anything).Return(queryResult1, nil).Once().After(10 * time.Millisecond) // Simulate a model response delay because our internal safety measures trigger when a query is done in 0 milliseconds.
+
+				queryResult2 := &provider.QueryResult{
+					Message: "",
+					GenerationInfo: &provider.GenerationInfo{
+						TotalCost:              0.222222222,
+						NativeTokensPrompt:     333,
+						NativeTokensCompletion: 444,
+					},
+				}
 				// Set up mocks, when test is running.
-				mockedModel.MockCapabilityWriteTests.On("WriteTests", mock.Anything).Return(nil, ErrEmptyResponseFromModel)
+				mockedQuery.On("Query", mock.Anything, mock.Anything, mock.Anything).Return(queryResult2, nil).Once().After(10 * time.Millisecond) // Simulate a model response delay because our internal safety measures trigger when a query is done in 0 milliseconds.
+			},
+			After: func(t *testing.T, logger *log.Logger, resultPath string) {
+				mockedQuery.AssertNumberOfCalls(t, "Query", 2)
 			},
 
 			Context: &Context{
@@ -180,6 +204,11 @@ func TestEvaluate(t *testing.T) {
 				Models: []evalmodel.Model{
 					mockedModel,
 				},
+				QueryAttempts: 3,
+
+				RepositoryPaths: []string{
+					repositoryPath,
+				},
 			},
 
 			ExpectedAssessments: []*metricstesting.AssessmentTuple{
@@ -189,8 +218,12 @@ func TestEvaluate(t *testing.T) {
 					RepositoryPath: repositoryPath,
 					Case:           "plain.go",
 					Task:           evaluatetask.IdentifierWriteTests,
-					Assessment: metrics.Assessments{
+					Assessment: map[metrics.AssessmentKey]float64{
 						metrics.AssessmentKeyFilesExecutedMaximumReachable: 1,
+						metrics.AssessmentKeyResponseNoError:               1,
+						metrics.AssessmentKeyCostsTokenActual:              0.111111111,
+						metrics.AssessmentKeyNativeTokenInput:              111,
+						metrics.AssessmentKeyNativeTokenOutput:             222,
 					},
 				},
 				&metricstesting.AssessmentTuple{
@@ -199,8 +232,12 @@ func TestEvaluate(t *testing.T) {
 					RepositoryPath: repositoryPath,
 					Case:           "plain.go",
 					Task:           evaluatetask.IdentifierWriteTestsSymflowerFix,
-					Assessment: metrics.Assessments{
+					Assessment: map[metrics.AssessmentKey]float64{
 						metrics.AssessmentKeyFilesExecutedMaximumReachable: 1,
+						metrics.AssessmentKeyResponseNoError:               1,
+						metrics.AssessmentKeyCostsTokenActual:              0.111111111,
+						metrics.AssessmentKeyNativeTokenInput:              111,
+						metrics.AssessmentKeyNativeTokenOutput:             222,
 					},
 				},
 				&metricstesting.AssessmentTuple{
@@ -209,8 +246,12 @@ func TestEvaluate(t *testing.T) {
 					RepositoryPath: repositoryPath,
 					Case:           "plain.go",
 					Task:           evaluatetask.IdentifierWriteTestsSymflowerTemplate,
-					Assessment: metrics.Assessments{
+					Assessment: map[metrics.AssessmentKey]float64{
 						metrics.AssessmentKeyFilesExecutedMaximumReachable: 1,
+						metrics.AssessmentKeyResponseNoError:               1,
+						metrics.AssessmentKeyCostsTokenActual:              0.222222222,
+						metrics.AssessmentKeyNativeTokenInput:              333,
+						metrics.AssessmentKeyNativeTokenOutput:             444,
 					},
 				},
 				&metricstesting.AssessmentTuple{
@@ -219,15 +260,23 @@ func TestEvaluate(t *testing.T) {
 					RepositoryPath: repositoryPath,
 					Case:           "plain.go",
 					Task:           evaluatetask.IdentifierWriteTestsSymflowerTemplateSymflowerFix,
-					Assessment: metrics.Assessments{
+					Assessment: map[metrics.AssessmentKey]float64{
 						metrics.AssessmentKeyFilesExecutedMaximumReachable: 1,
+						metrics.AssessmentKeyResponseNoError:               1,
+						metrics.AssessmentKeyCostsTokenActual:              0.222222222,
+						metrics.AssessmentKeyNativeTokenInput:              333,
+						metrics.AssessmentKeyNativeTokenOutput:             444,
 					},
 				},
 			},
 			ExpectedResultFiles: map[string]func(t *testing.T, filePath string, data string){
 				"evaluation.log": nil,
-				filepath.Join(string(evaluatetask.IdentifierWriteTests), mockedModel.ID(), "golang", "golang", "plain", "evaluation.log"): nil,
-				"evaluation.csv": nil,
+				filepath.Join(string(evaluatetask.IdentifierWriteTests), log.CleanModelNameForFileSystem(mockedModelID), "golang", "golang", "plain", "evaluation.log"): func(t *testing.T, filePath, data string) {
+					assert.Equal(t, 4, strings.Count(data, "no test files found"), "number of ocurrences of \"no test files found\" not matched")
+				},
+				"evaluation.csv": func(t *testing.T, filePath, data string) {
+					assert.Lenf(t, strings.Split(data, "\n"), 6, "expected 6 lines: header, 4x entries and final new line:\n%s", data)
+				},
 			},
 		})
 	}

model/llm/llm.go

@@ -506,6 +506,10 @@ func handleQueryResult(queryResult *provider.QueryResult, filePathAbsolute strin
 		assessment[metrics.AssessmentKeyCostsTokenActual] = queryResult.GenerationInfo.TotalCost
 	}
 
+	if sourceFileContent == "" {
+		return assessment, nil
+	}
+
 	if err := os.MkdirAll(filepath.Dir(filePathAbsolute), 0755); err != nil {
 		return nil, pkgerrors.WithStack(err)
 	}

model/llm/llm_test.go

@@ -5,6 +5,7 @@ import (
 	"path/filepath"
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/mock"
@@ -35,9 +36,10 @@ func TestModelGenerateTestsForFile(t *testing.T) {
 		SourceFileContent string
 		SourceFilePath    string
 
-		ExpectedAssessment      metrics.Assessments
-		ExpectedTestFileContent string
-		ExpectedTestFilePath    string
+		ExpectedAssessment            metrics.Assessments
+		ExpectedTestFileContent       string
+		ExpectedTestFilePath          string
+		ExpectedTestFilePathNotExists string
 	}
 
 	validate := func(t *testing.T, tc *testCase) {
@@ -74,10 +76,16 @@ func TestModelGenerateTestsForFile(t *testing.T) {
 
 			assert.Equal(t, metricstesting.Clean(tc.ExpectedAssessment), metricstesting.Clean(actualAssessment))
 
-			actualTestFileContent, err := os.ReadFile(filepath.Join(temporaryPath, tc.ExpectedTestFilePath))
-			assert.NoError(t, err)
+			if tc.ExpectedTestFilePath != "" {
+				actualTestFileContent, err := os.ReadFile(filepath.Join(temporaryPath, tc.ExpectedTestFilePath))
+				assert.NoError(t, err)
+
+				assert.Equal(t, strings.TrimSpace(bytesutil.StringTrimIndentations(tc.ExpectedTestFileContent)), string(actualTestFileContent))
+			}
 
-			assert.Equal(t, strings.TrimSpace(bytesutil.StringTrimIndentations(tc.ExpectedTestFileContent)), string(actualTestFileContent))
+			if tc.ExpectedTestFilePathNotExists != "" {
+				assert.NoFileExists(t, filepath.Join(temporaryPath, tc.ExpectedTestFilePathNotExists))
+			}
 		})
 	}
 
@@ -131,6 +139,29 @@ func TestModelGenerateTestsForFile(t *testing.T) {
 		`,
 		ExpectedTestFilePath: "simple_test.go",
 	})
+	validate(t, &testCase{
+		Name: "Empty response",
+
+		SetupMock: func(mockedProvider *providertesting.MockQuery) {
+			queryResult := &provider.QueryResult{
+				Duration: time.Millisecond * 123,
+				GenerationInfo: &provider.GenerationInfo{
+					TotalCost: 0.123456789,
+				},
+			}
+			mockedProvider.On("Query", mock.Anything, mock.Anything, promptMessage).Return(queryResult, nil)
+		},
+
+		Language:          &golang.Language{},
+		ModelID:           "model-id",
+		SourceFileContent: sourceFileContent,
+		SourceFilePath:    sourceFilePath,
+
+		ExpectedAssessment: metrics.Assessments{
+			metrics.AssessmentKeyCostsTokenActual: 0.123456789,
+		},
+		ExpectedTestFilePathNotExists: "simple_test.go",
+	})
 }
 
 func TestModelRepairSourceCodeFile(t *testing.T) {

model/llm/prompt/parse.go

@@ -4,7 +4,6 @@ import (
 	"regexp"
 	"strings"
 
-	pkgerrors "github.com/pkg/errors"
 	"github.com/zimmski/osutil/bytesutil"
 
 	"github.com/symflower/eval-dev-quality/evaluate/metrics"
@@ -21,7 +20,7 @@ func ParseResponse(response string) (assessment metrics.Assessments, code string
 
 	// Check for empty responses.
 	if strings.TrimSpace(response) == "" {
-		return assessment, "", pkgerrors.New("empty response from model")
+		return assessment, "", nil
 	}
 
 	// Some models produce duplicated code tags, so unify them if needed.

model/llm/prompt/parse_test.go

@@ -73,13 +73,12 @@ func TestParseResponse(t *testing.T) {
 	})
 
 	validate(t, &testCase{
-		Name: "Expected error on empty response",
+		Name: "No error on empty response",
 
 		Response: "",
 
 		ExpectedAssessment: metrics.Assessments{},
 		ExpectedCode:       "",
-		ExpectedError:      true,
 	})
 
 	t.Run("Formatted Code", func(t *testing.T) {