Merge pull request #425 from symflower/fetch-costs

Munsio · web-flow · commit 12b82ce0ab90 · 2025-03-06T14:54:56.000+01:00
Fetch total costs from OpenRouter after query
diff --git a/cmd/eval-dev-quality/cmd/evaluate_test.go b/cmd/eval-dev-quality/cmd/evaluate_test.go
@@ -34,7 +34,7 @@ func validateMetrics(t *testing.T, csvData string, expectedAssessments []metrics
 	actualAssessmentTuples := reporttesting.ParseMetrics(t, csvData)
 	actual = make([]metrics.Assessments, len(actualAssessmentTuples))
 	for i, tuple := range actualAssessmentTuples {
-		assert.Greater(t, tuple.Assessment[metrics.AssessmentKeyProcessingTime], uint64(0))
+		assert.Greater(t, tuple.Assessment[metrics.AssessmentKeyProcessingTime], float64(0))
 		actual[i] = tuple.Assessment
 	}
 
diff --git a/evaluate/evaluate_test.go b/evaluate/evaluate_test.go
diff --git a/evaluate/metrics/assessment.go b/evaluate/metrics/assessment.go
@@ -59,14 +59,20 @@ var (
 	AssessmentKeyTokenInput = RegisterAssessmentKey("token-input")
 	// AssessmentKeyTokenOutput collects the number of output token.
 	AssessmentKeyTokenOutput = RegisterAssessmentKey("token-output")
+	// AssessmentKeyNativeTokenInput collects the number of input token.
+	AssessmentKeyNativeTokenInput = RegisterAssessmentKey("native-token-input")
+	// AssessmentKeyNativeTokenOutput collects the number of output token.
+	AssessmentKeyNativeTokenOutput = RegisterAssessmentKey("native-token-output")
+	// AssessmentKeyCostsTokenActual collects the number of output token.
+	AssessmentKeyCostsTokenActual = RegisterAssessmentKey("costs-total-actual")
 )
 
 // Assessments holds a collection of numerical assessment metrics.
-type Assessments map[AssessmentKey]uint64
+type Assessments map[AssessmentKey]float64
 
 // NewAssessments creates a new assessment collection.
 func NewAssessments() Assessments {
-	return map[AssessmentKey]uint64{}
+	return map[AssessmentKey]float64{}
 }
 
 // Add adds the given assessment collection to the current one.
@@ -98,7 +104,7 @@ func (a Assessments) Award(key AssessmentKey) {
 
 // AwardMultiple yields multiple score points.
 func (a Assessments) AwardMultiple(key AssessmentKey, count uint64) {
-	a[key] += count
+	a[key] += float64(count)
 }
 
 // String returns a string representation of the metrics.
@@ -109,7 +115,7 @@ func (a Assessments) String() string {
 	entries := make([]string, len(AllAssessmentKeys))
 
 	for i, key := range AllAssessmentKeys {
-		entries[i] = fmt.Sprintf("%s=%d", key, a[key])
+		entries[i] = fmt.Sprintf("%s=%v", key, a[key])
 	}
 
 	return strings.Join(entries, ", ")
@@ -123,7 +129,7 @@ func (a Assessments) StringCSV() (row []string) {
 
 	row = make([]string, len(AllAssessmentKeys))
 	for i, key := range AllAssessmentKeys {
-		row[i] = fmt.Sprintf("%d", a[key])
+		row[i] = fmt.Sprintf("%v", a[key])
 	}
 
 	return row
diff --git a/evaluate/metrics/assessment_test.go b/evaluate/metrics/assessment_test.go
@@ -37,26 +37,26 @@ func TestAssessmentsAdd(t *testing.T) {
 		Name: "Non existing key",
 
 		Assessments: NewAssessments(),
-		X: map[AssessmentKey]uint64{
+		X: map[AssessmentKey]float64{
 			AssessmentKeyResponseNoExcess: 1,
 		},
 
-		ExpectedAssessments: map[AssessmentKey]uint64{
+		ExpectedAssessments: map[AssessmentKey]float64{
 			AssessmentKeyResponseNoExcess: 1,
 		},
 	})
 
 	validate(t, &testCase{
 		Name: "Existing key",
 
-		Assessments: map[AssessmentKey]uint64{
+		Assessments: map[AssessmentKey]float64{
 			AssessmentKeyResponseNoExcess: 1,
 		},
-		X: map[AssessmentKey]uint64{
+		X: map[AssessmentKey]float64{
 			AssessmentKeyResponseNoExcess: 1,
 		},
 
-		ExpectedAssessments: map[AssessmentKey]uint64{
+		ExpectedAssessments: map[AssessmentKey]float64{
 			AssessmentKeyResponseNoExcess: 2,
 		},
 	})
@@ -84,7 +84,7 @@ func TestAssessmentString(t *testing.T) {
 
 		Assessment: NewAssessments(),
 
-		ExpectedString: "coverage=0, files-executed=0, files-executed-maximum-reachable=0, generate-tests-for-file-character-count=0, processing-time=0, response-character-count=0, response-no-error=0, response-no-excess=0, response-with-code=0, tests-passing=0, token-input=0, token-output=0",
+		ExpectedString: "costs-total-actual=0, coverage=0, files-executed=0, files-executed-maximum-reachable=0, generate-tests-for-file-character-count=0, native-token-input=0, native-token-output=0, processing-time=0, response-character-count=0, response-no-error=0, response-no-excess=0, response-with-code=0, tests-passing=0, token-input=0, token-output=0",
 	})
 
 	validate(t, &testCase{
@@ -105,7 +105,7 @@ func TestAssessmentString(t *testing.T) {
 			AssessmentKeyTokenOutput:                        456,
 		},
 
-		ExpectedString: "coverage=1, files-executed=2, files-executed-maximum-reachable=2, generate-tests-for-file-character-count=50, processing-time=200, response-character-count=100, response-no-error=3, response-no-excess=4, response-with-code=5, tests-passing=7, token-input=123, token-output=456",
+		ExpectedString: "costs-total-actual=0, coverage=1, files-executed=2, files-executed-maximum-reachable=2, generate-tests-for-file-character-count=50, native-token-input=0, native-token-output=0, processing-time=200, response-character-count=100, response-no-error=3, response-no-excess=4, response-with-code=5, tests-passing=7, token-input=123, token-output=456",
 	})
 }
 
@@ -209,7 +209,7 @@ func TestCombineModelAndSymflowerFixAssessments(t *testing.T) {
 
 		ModelAssessment: Assessments{
 			AssessmentKeyFilesExecuted:                      1,
-			AssessmentKeyProcessingTime:                     uint64(200),
+			AssessmentKeyProcessingTime:                     float64(200),
 			AssessmentKeyCoverage:                           0,
 			AssessmentKeyResponseCharacterCount:             100,
 			AssessmentKeyGenerateTestsForFileCharacterCount: 50,
@@ -219,15 +219,15 @@ func TestCombineModelAndSymflowerFixAssessments(t *testing.T) {
 		},
 		SymflowerFixAssessments: Assessments{
 			AssessmentKeyFilesExecuted:   1,
-			AssessmentKeyProcessingTime:  uint64(100),
+			AssessmentKeyProcessingTime:  float64(100),
 			AssessmentKeyCoverage:        1,
 			AssessmentKeyResponseNoError: 1,
 			AssessmentKeyTestsPassing:    10,
 		},
 
 		ExpectedAssessments: Assessments{
 			AssessmentKeyFilesExecuted:                      1,
-			AssessmentKeyProcessingTime:                     uint64(300),
+			AssessmentKeyProcessingTime:                     float64(300),
 			AssessmentKeyCoverage:                           1,
 			AssessmentKeyResponseCharacterCount:             100,
 			AssessmentKeyGenerateTestsForFileCharacterCount: 50,
diff --git a/evaluate/report/csv_test.go b/evaluate/report/csv_test.go
@@ -24,7 +24,7 @@ func TestNewEvaluationFile(t *testing.T) {
 	require.NoError(t, err)
 
 	expectedEvaluationFileContent := bytesutil.StringTrimIndentations(`
-		model-id,language,repository,case,task,run,coverage,files-executed,files-executed-maximum-reachable,generate-tests-for-file-character-count,processing-time,response-character-count,response-no-error,response-no-excess,response-with-code,tests-passing,token-input,token-output
+		model-id,language,repository,case,task,run,costs-total-actual,coverage,files-executed,files-executed-maximum-reachable,generate-tests-for-file-character-count,native-token-input,native-token-output,processing-time,response-character-count,response-no-error,response-no-excess,response-with-code,tests-passing,token-input,token-output
 	`)
 
 	assert.Equal(t, expectedEvaluationFileContent, string(actualEvaluationFileContent))
@@ -65,8 +65,8 @@ func TestWriteEvaluationRecord(t *testing.T) {
 		},
 
 		ExpectedCSV: `
-			model-id,language,repository,case,task,run,coverage,files-executed,files-executed-maximum-reachable,generate-tests-for-file-character-count,processing-time,response-character-count,response-no-error,response-no-excess,response-with-code,tests-passing,token-input,token-output
-			mocked-model,golang,golang/plain,plain.go,write-tests,1,0,0,0,0,0,0,0,0,0,0,0,0
+			model-id,language,repository,case,task,run,costs-total-actual,coverage,files-executed,files-executed-maximum-reachable,generate-tests-for-file-character-count,native-token-input,native-token-output,processing-time,response-character-count,response-no-error,response-no-excess,response-with-code,tests-passing,token-input,token-output
+			mocked-model,golang,golang/plain,plain.go,write-tests,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
 		`,
 	})
 	validate(t, &testCase{
@@ -90,9 +90,9 @@ func TestWriteEvaluationRecord(t *testing.T) {
 		},
 
 		ExpectedCSV: `
-			model-id,language,repository,case,task,run,coverage,files-executed,files-executed-maximum-reachable,generate-tests-for-file-character-count,processing-time,response-character-count,response-no-error,response-no-excess,response-with-code,tests-passing,token-input,token-output
-			mocked-model,golang,golang/plain,plain.go,write-tests,1,0,1,1,0,0,0,1,0,0,0,0,0
-			mocked-model,golang,golang/plain,plain.go,write-tests-symflower-fix,1,10,1,1,0,0,0,1,0,0,0,0,0
+			model-id,language,repository,case,task,run,costs-total-actual,coverage,files-executed,files-executed-maximum-reachable,generate-tests-for-file-character-count,native-token-input,native-token-output,processing-time,response-character-count,response-no-error,response-no-excess,response-with-code,tests-passing,token-input,token-output
+			mocked-model,golang,golang/plain,plain.go,write-tests,1,0,0,1,1,0,0,0,0,0,1,0,0,0,0,0
+			mocked-model,golang,golang/plain,plain.go,write-tests-symflower-fix,1,0,10,1,1,0,0,0,0,0,1,0,0,0,0,0
 		`,
 	})
 }
diff --git a/evaluate/report/testing/csv.go b/evaluate/report/testing/csv.go
@@ -11,11 +11,11 @@ import (
 	"github.com/symflower/eval-dev-quality/task"
 )
 
-func atoiUint64(t *testing.T, s string) uint64 {
-	value, err := strconv.ParseUint(s, 10, 64)
+func parseFloat64(t *testing.T, s string) float64 {
+	value, err := strconv.ParseFloat(s, 64)
 	assert.NoErrorf(t, err, "parsing unsigned integer from: %q", s)
 
-	return uint64(value)
+	return value
 }
 
 // ParseMetrics extracts multiple assessment metrics from the given string.
@@ -37,7 +37,7 @@ func ParseMetrics(t *testing.T, data string) (assessments metricstesting.Assessm
 			Assessment:     metrics.Assessments{},
 		}
 		for i, key := range metrics.AllAssessmentKeys {
-			tuple.Assessment[key] = atoiUint64(t, cells[i+6])
+			tuple.Assessment[key] = parseFloat64(t, cells[i+6])
 		}
 
 		assessments = append(assessments, tuple)
diff --git a/evaluate/task/symflower.go b/evaluate/task/symflower.go
@@ -109,7 +109,7 @@ func runModelAndSymflowerFix(ctx evaltask.Context, modelCtx model.Context, runMo
 
 			// Symflower was able to fix a failure so now update the assessment with the improved results.
 			withSymflowerFix := metrics.NewAssessments()
-			withSymflowerFix[metrics.AssessmentKeyProcessingTime] = processingTime
+			withSymflowerFix[metrics.AssessmentKeyProcessingTime] = float64(processingTime)
 			withSymflowerFix.Award(metrics.AssessmentKeyFilesExecuted)
 			withSymflowerFix.AwardMultiple(metrics.AssessmentKeyCoverage, withSymflowerFixTestResult.Coverage)
 
diff --git a/evaluate/task/transpile.go b/evaluate/task/transpile.go
@@ -69,8 +69,8 @@ func (t *Transpile) Run(ctx evaltask.Context) (repositoryAssessment map[string]m
 		modelAssessments := metrics.NewAssessments()
 		withSymflowerAssessments := metrics.NewAssessments()
 		maximumReachableFiles := uint64(len(language.Languages) - 1) // Transpile repositories contain sub-tasks to transpile from every other supported language minus the one we are transpiling to.
-		modelAssessments[metrics.AssessmentKeyFilesExecutedMaximumReachable] = maximumReachableFiles
-		withSymflowerAssessments[metrics.AssessmentKeyFilesExecutedMaximumReachable] = maximumReachableFiles
+		modelAssessments[metrics.AssessmentKeyFilesExecutedMaximumReachable] = float64(maximumReachableFiles)
+		withSymflowerAssessments[metrics.AssessmentKeyFilesExecutedMaximumReachable] = float64(maximumReachableFiles)
 		repositoryAssessment[packagePath] = map[evaltask.Identifier]metrics.Assessments{
 			IdentifierTranspile:             modelAssessments,
 			IdentifierTranspileSymflowerFix: withSymflowerAssessments,
@@ -131,7 +131,7 @@ func (t *Transpile) Run(ctx evaltask.Context) (repositoryAssessment map[string]m
 
 					// Symflower was able to fix a failure so now update the assessment with the improved results.
 					withSymflowerFixAssessments := metrics.NewAssessments()
-					withSymflowerFixAssessments[metrics.AssessmentKeyProcessingTime] = processingTime
+					withSymflowerFixAssessments[metrics.AssessmentKeyProcessingTime] = float64(processingTime)
 					withSymflowerFixAssessments.Award(metrics.AssessmentKeyFilesExecuted)
 					withSymflowerFixAssessments.AwardMultiple(metrics.AssessmentKeyTestsPassing, uint64(testsPassing))
 
diff --git a/model/llm/llm.go b/model/llm/llm.go
@@ -339,7 +339,11 @@ func (m *Model) query(logger *log.Logger, request string) (queryResult *provider
 				return err
 			}
 			duration = time.Since(start)
-			logger.Info("model responded", "model", m.ID(), "id", id, "duration", duration.Milliseconds(), "response-id", queryResult.ResponseID, "token-input", queryResult.Usage.PromptTokens, "token-output", queryResult.Usage.CompletionTokens, "response", string(bytesutil.PrefixLines([]byte(queryResult.Message), []byte("\t"))))
+			totalCosts := float64(-1)
+			if queryResult.GenerationInfo != nil {
+				totalCosts = queryResult.GenerationInfo.TotalCost
+			}
+			logger.Info("model responded", "model", m.ID(), "id", id, "duration", duration.Milliseconds(), "response-id", queryResult.ResponseID, "costs-total", totalCosts, "token-input", queryResult.Usage.PromptTokens, "token-output", queryResult.Usage.CompletionTokens, "response", string(bytesutil.PrefixLines([]byte(queryResult.Message), []byte("\t"))))
 
 			return nil
 		},
@@ -491,11 +495,16 @@ func handleQueryResult(queryResult *provider.QueryResult, filePathAbsolute strin
 	if err != nil {
 		return nil, pkgerrors.WithStack(err)
 	}
-	assessment[metrics.AssessmentKeyProcessingTime] = uint64(queryResult.Duration.Milliseconds())
-	assessment[metrics.AssessmentKeyResponseCharacterCount] = uint64(len(queryResult.Message))
-	assessment[metrics.AssessmentKeyGenerateTestsForFileCharacterCount] = uint64(len(sourceFileContent))
-	assessment[metrics.AssessmentKeyTokenInput] = uint64(queryResult.Usage.PromptTokens)
-	assessment[metrics.AssessmentKeyTokenOutput] = uint64(queryResult.Usage.CompletionTokens)
+	assessment[metrics.AssessmentKeyProcessingTime] = float64(queryResult.Duration.Milliseconds())
+	assessment[metrics.AssessmentKeyResponseCharacterCount] = float64(len(queryResult.Message))
+	assessment[metrics.AssessmentKeyGenerateTestsForFileCharacterCount] = float64(len(sourceFileContent))
+	assessment[metrics.AssessmentKeyTokenInput] = float64(queryResult.Usage.PromptTokens)
+	assessment[metrics.AssessmentKeyTokenOutput] = float64(queryResult.Usage.CompletionTokens)
+	if queryResult.GenerationInfo != nil {
+		assessment[metrics.AssessmentKeyNativeTokenInput] = float64(queryResult.GenerationInfo.NativeTokensPrompt)
+		assessment[metrics.AssessmentKeyNativeTokenOutput] = float64(queryResult.GenerationInfo.NativeTokensCompletion)
+		assessment[metrics.AssessmentKeyCostsTokenActual] = queryResult.GenerationInfo.TotalCost
+	}
 
 	if err := os.MkdirAll(filepath.Dir(filePathAbsolute), 0755); err != nil {
 		return nil, pkgerrors.WithStack(err)
diff --git a/model/symflower/symflower.go b/model/symflower/symflower.go
@@ -121,7 +121,7 @@ func (m *Model) WriteTests(ctx model.Context) (assessment metrics.Assessments, e
 		return nil, pkgerrors.WithStack(err)
 	}
 
-	processingTime := uint64(time.Since(start).Milliseconds())
+	processingTime := float64(time.Since(start).Milliseconds())
 
 	characterCount, err := countCharactersOfGeneratedFiles(ctx.RepositoryPath, extractGeneratedFilePaths(output))
 	if err != nil {
@@ -130,8 +130,8 @@ func (m *Model) WriteTests(ctx model.Context) (assessment metrics.Assessments, e
 
 	return metrics.Assessments{ // Symflower always generates just source code when it does not fail, so no need to check the assessment properties.
 		metrics.AssessmentKeyProcessingTime:                     processingTime,
-		metrics.AssessmentKeyGenerateTestsForFileCharacterCount: characterCount,
-		metrics.AssessmentKeyResponseCharacterCount:             characterCount,
+		metrics.AssessmentKeyGenerateTestsForFileCharacterCount: float64(characterCount),
+		metrics.AssessmentKeyResponseCharacterCount:             float64(characterCount),
 		metrics.AssessmentKeyResponseNoExcess:                   1,
 		metrics.AssessmentKeyResponseWithCode:                   1,
 	}, nil
diff --git a/provider/openrouter/openrouter.go b/provider/openrouter/openrouter.go
@@ -138,7 +138,17 @@ var _ provider.Query = (*Provider)(nil)
 
 // Query queries the provider with the given model name.
 func (p *Provider) Query(ctx context.Context, model model.Model, promptText string) (result *provider.QueryResult, err error) {
-	return openaiapi.QueryOpenAIAPIModel(ctx, p.client(), model.ModelIDWithoutProvider(), model.Attributes(), promptText)
+	queryResult, err := openaiapi.QueryOpenAIAPIModel(ctx, p.client(), model.ModelIDWithoutProvider(), model.Attributes(), promptText)
+	if err != nil {
+		return nil, pkgerrors.WithStack(err)
+	}
+
+	queryResult.GenerationInfo, err = p.fetchGenerationInfo(queryResult.ResponseID)
+	if err != nil {
+		return nil, pkgerrors.WithStack(err)
+	}
+
+	return queryResult, nil
 }
 
 // client returns a new client with the current configuration.
@@ -148,3 +158,54 @@ func (p *Provider) client() (client *openai.Client) {
 
 	return openai.NewClientWithConfig(config)
 }
+
+func (p *Provider) fetchGenerationInfo(generationID string) (generationInfo *provider.GenerationInfo, err error) {
+	request, err := http.NewRequest("GET", "https://openrouter.ai/api/v1/generation?id="+generationID, nil)
+	if err != nil {
+		return nil, pkgerrors.WithStack(err)
+	}
+	request.Header.Set("Accept", "application/json")
+	request.Header.Set("Authorization", "Bearer "+p.token)
+
+	client := &http.Client{}
+	var responseBody []byte
+	if err := retry.Do( // Query available models with a retry logic cause "openrouter.ai" has failed us in the past.
+		func() error {
+			response, err := client.Do(request)
+			if err != nil {
+				return pkgerrors.WithStack(err)
+			}
+			defer func() {
+				if e := response.Body.Close(); e != nil {
+					err = errors.Join(err, pkgerrors.WithStack(e))
+				}
+			}()
+
+			if response.StatusCode != http.StatusOK {
+				return pkgerrors.Errorf("received status code %d when querying provider models", response.StatusCode)
+			}
+
+			responseBody, err = io.ReadAll(response.Body)
+			if err != nil {
+				return pkgerrors.WithStack(err)
+			}
+
+			return nil
+		},
+		retry.Attempts(3),
+		retry.Delay(5*time.Second),
+		retry.DelayType(retry.BackOffDelay),
+		retry.LastErrorOnly(true),
+	); err != nil {
+		return nil, err
+	}
+
+	var dataResponse struct {
+		provider.GenerationInfo `json:"data"`
+	}
+	if err := json.Unmarshal(responseBody, &dataResponse); err != nil {
+		return nil, err
+	}
+
+	return &dataResponse.GenerationInfo, nil
+}
diff --git a/provider/provider.go b/provider/provider.go
@@ -54,6 +54,20 @@ type QueryResult struct {
 	Duration time.Duration
 	// Usage holds the usage metrics of the query.
 	Usage openai.Usage
+	// GenerationInfo holds information about a generation.
+	GenerationInfo *GenerationInfo
+}
+
+// GenerationInfo holds information about a generation.
+// See https://openrouter.ai/docs/api-reference/overview#querying-cost-and-stats for more details.
+type GenerationInfo struct {
+	ID                     string  `json:"id"`
+	TotalCost              float64 `json:"total_cost"`
+	TokensPrompt           int     `json:"tokens_prompt"`
+	TokensCompletion       int     `json:"tokens_completion"`
+	NativeTokensPrompt     int     `json:"native_tokens_prompt"`
+	NativeTokensCompletion int     `json:"native_tokens_completion"`
+	NativeTokensReasoning  int     `json:"native_tokens_reasoning"`
 }
 
 // Query is a provider that allows to query a model directly.

Original file line number	Diff line number	Diff line change
`@@ -34,7 +34,7 @@ func validateMetrics(t *testing.T, csvData string, expectedAssessments []metrics`
`34`	`34`	`actualAssessmentTuples := reporttesting.ParseMetrics(t, csvData)`
`35`	`35`	`actual = make([]metrics.Assessments, len(actualAssessmentTuples))`
`36`	`36`	`for i, tuple := range actualAssessmentTuples {`
`37`		`- assert.Greater(t, tuple.Assessment[metrics.AssessmentKeyProcessingTime], uint64(0))`
	`37`	`+ assert.Greater(t, tuple.Assessment[metrics.AssessmentKeyProcessingTime], float64(0))`
`38`	`38`	`actual[i] = tuple.Assessment`
`39`	`39`	`}`
`40`	`40`
Original file line number	Diff line number	Diff line change
`@@ -11,11 +11,11 @@ import (`
`11`	`11`	`"github.com/symflower/eval-dev-quality/task"`
`12`	`12`	`)`
`13`	`13`
`14`		`-func atoiUint64(t *testing.T, s string) uint64 {`
`15`		`- value, err := strconv.ParseUint(s, 10, 64)`
	`14`	`+func parseFloat64(t *testing.T, s string) float64 {`
	`15`	`+ value, err := strconv.ParseFloat(s, 64)`
`16`	`16`	`assert.NoErrorf(t, err, "parsing unsigned integer from: %q", s)`
`17`	`17`
`18`		`- return uint64(value)`
	`18`	`+ return value`
`19`	`19`	`}`
`20`	`20`
`21`	`21`	`// ParseMetrics extracts multiple assessment metrics from the given string.`
`@@ -37,7 +37,7 @@ func ParseMetrics(t *testing.T, data string) (assessments metricstesting.Assessm`
`37`	`37`	`Assessment: metrics.Assessments{},`
`38`	`38`	`}`
`39`	`39`	`for i, key := range metrics.AllAssessmentKeys {`
`40`		`- tuple.Assessment[key] = atoiUint64(t, cells[i+6])`
	`40`	`+ tuple.Assessment[key] = parseFloat64(t, cells[i+6])`
`41`	`41`	`}`
`42`	`42`
`43`	`43`	`assessments = append(assessments, tuple)`