fix, Make sure that the test template is not on disk after generating

If the test file is not removed, then a test file exists that is not coming from the model.

Author: ahumenberger

evaluate/task/write-test.go

@@ -144,6 +144,7 @@ func (t *WriteTests) Run(ctx evaltask.Context) (repositoryAssessment map[string]
 	return repositoryAssessment, problems, nil
 }
 
+// symflowerTemplateAsString generates a test template for the given file and makes sure that the repository is in the same state as before.
 func symflowerTemplateAsString(ctx evaltask.Context, taskLogger *taskLogger, dataPath string, filePath string) (testTemplate string, err error) {
 	_, err = symflowerTemplate(taskLogger.Logger, dataPath, ctx.Language, filePath) // TODO Incorporate template processing time. https://github.com/symflower/eval-dev-quality/issues/350
 	if err != nil {
@@ -155,6 +156,10 @@ func symflowerTemplateAsString(ctx evaltask.Context, taskLogger *taskLogger, dat
 		return "", pkgerrors.WithMessagef(err, "reading Symflower template from %q", testTemplateFilePath)
 	}
 
+	if err := ctx.Repository.Reset(ctx.Logger); err != nil {
+		ctx.Logger.Panicf("ERROR: unable to reset temporary repository path: %s", err)
+	}
+
 	return string(testTemplateData), nil
 }
 

evaluate/task/write-test_test.go

@@ -8,6 +8,7 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/mock"
 	"github.com/stretchr/testify/require"
 	"github.com/symflower/eval-dev-quality/evaluate/metrics"
 	metricstesting "github.com/symflower/eval-dev-quality/evaluate/metrics/testing"
@@ -123,6 +124,52 @@ func TestWriteTestsRun(t *testing.T) {
 		})
 	})
 
+	{
+		temporaryDirectoryPath := t.TempDir()
+		repositoryPath := filepath.Join(temporaryDirectoryPath, "golang", "plain")
+		require.NoError(t, osutil.CopyTree(filepath.Join("..", "..", "testdata", "golang", "plain"), repositoryPath))
+
+		modelMock := modeltesting.NewMockCapabilityWriteTestsNamed(t, "mocked-model")
+		// Simulate that a model does not generate anything.
+		modelMock.MockCapabilityWriteTests.On("WriteTests", mock.Anything).Return(metricstesting.AssessmentsWithProcessingTime, nil)
+
+		validate(t, &tasktesting.TestCaseTask{
+			Name: "Reset symflower template so it's not mistaken for model solution",
+
+			Model:          modelMock,
+			Language:       &golang.Language{},
+			TestDataPath:   temporaryDirectoryPath,
+			RepositoryPath: filepath.Join("golang", "plain"),
+
+			ExpectedRepositoryAssessment: map[string]map[evaltask.Identifier]metrics.Assessments{
+				"plain.go": map[evaltask.Identifier]metrics.Assessments{
+					IdentifierWriteTests: metrics.Assessments{
+						metrics.AssessmentKeyFilesExecutedMaximumReachable: 1,
+						metrics.AssessmentKeyResponseNoError:               1,
+					},
+					IdentifierWriteTestsSymflowerFix: metrics.Assessments{
+						metrics.AssessmentKeyFilesExecutedMaximumReachable: 1,
+						metrics.AssessmentKeyResponseNoError:               1,
+					},
+					IdentifierWriteTestsSymflowerTemplate: metrics.Assessments{
+						metrics.AssessmentKeyFilesExecutedMaximumReachable: 1,
+						metrics.AssessmentKeyResponseNoError:               1,
+					},
+					IdentifierWriteTestsSymflowerTemplateSymflowerFix: metrics.Assessments{
+						metrics.AssessmentKeyFilesExecutedMaximumReachable: 1,
+						metrics.AssessmentKeyResponseNoError:               1,
+					},
+				},
+			},
+			ExpectedProblemContains: []string{
+				"ERROR: no test files found",
+				"ERROR: no test files found",
+				"ERROR: no test files found",
+				"ERROR: no test files found",
+			},
+		})
+	}
+
 	t.Run("Symflower Fix", func(t *testing.T) {
 		t.Run("Go", func(t *testing.T) {
 			validateGo := func(t *testing.T, testName string, language language.Language, testFileContent string, expectedAssessments map[string]map[evaltask.Identifier]metrics.Assessments, expectedProblems []string, assertTestsPass bool) {