Refactor testProviderMetrics to support concurrent result handling and add session-based logging and results directories; implement markdown report generation for test results

Author: lemon07r

.gitignore

@@ -15,3 +15,7 @@ build/
 # PAMPAX
 .pampa/
 pampa.*
+
+# Logs
+logs/
+results/

main.go

@@ -42,7 +42,7 @@ type TestResult struct {
 
 // testProviderMetrics runs a full benchmark test against a single provider.
 // It is designed to be run as a goroutine.
-func testProviderMetrics(config ProviderConfig, tke *tiktoken.Tiktoken, wg *sync.WaitGroup, logDir, resultsDir string) {
+func testProviderMetrics(config ProviderConfig, tke *tiktoken.Tiktoken, wg *sync.WaitGroup, logDir, resultsDir string, results *[]TestResult, resultsMutex *sync.Mutex) {
 	// Defer wg.Done() if this is part of a concurrent group
 	if wg != nil {
 		defer wg.Done()
@@ -52,7 +52,7 @@ func testProviderMetrics(config ProviderConfig, tke *tiktoken.Tiktoken, wg *sync
 	timestamp := time.Now().Format("20060102-150405")
 	logFile, err := os.Create(filepath.Join(logDir, fmt.Sprintf("%s-%s.log", config.Name, timestamp)))
 	if err != nil {
-		log.Printf("❌ Error creating log file for %s: %v", config.Name, err)
+		log.Printf("Error creating log file for %s: %v", config.Name, err)
 		return
 	}
 	defer logFile.Close()
@@ -88,18 +88,23 @@ func testProviderMetrics(config ProviderConfig, tke *tiktoken.Tiktoken, wg *sync
 	var firstTokenTime time.Time
 	var fullResponseContent strings.Builder
 
-	ctx := context.Background()
+	// Add timeout context to prevent indefinite hangs (5 minutes)
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
+	defer cancel()
+
 	stream, err := client.CreateChatCompletionStream(ctx, req)
 	if err != nil {
-		providerLogger.Printf("❌ Error creating stream for %s: %v", config.Name, err)
+		providerLogger.Printf("Error creating stream for %s: %v", config.Name, err)
 		// Save error result
-		saveResult(resultsDir, TestResult{
+		result := TestResult{
 			Provider:  config.Name,
 			Model:     config.Model,
 			Timestamp: time.Now(),
 			Success:   false,
 			Error:     err.Error(),
-		})
+		}
+		saveResult(resultsDir, result)
+		appendResult(results, resultsMutex, result)
 		return
 	}
 	defer stream.Close() // IMPORTANT: Always close the stream
@@ -116,20 +121,27 @@ func testProviderMetrics(config ProviderConfig, tke *tiktoken.Tiktoken, wg *sync
 		}
 
 		if err != nil {
-			providerLogger.Printf("Stream error for %s: %v", config.Name, err)
+			errMsg := err.Error()
+			if ctx.Err() == context.DeadlineExceeded {
+				errMsg = "Timeout: stream took longer than 5 minutes"
+			}
+			providerLogger.Printf("Stream error for %s: %v", config.Name, errMsg)
 			// Save error result
-			saveResult(resultsDir, TestResult{
+			result := TestResult{
 				Provider:  config.Name,
 				Model:     config.Model,
 				Timestamp: time.Now(),
 				Success:   false,
-				Error:     err.Error(),
-			})
+				Error:     errMsg,
+			}
+			saveResult(resultsDir, result)
+			appendResult(results, resultsMutex, result)
 			return
 		}
 
 		// Check if Choices array is empty (some APIs send empty chunks)
 		if len(response.Choices) == 0 {
+			providerLogger.Printf("[%s] ... Received empty chunk (no Choices)", config.Name)
 			continue
 		}
 
@@ -155,13 +167,15 @@ func testProviderMetrics(config ProviderConfig, tke *tiktoken.Tiktoken, wg *sync
 	if firstTokenTime.IsZero() {
 		providerLogger.Printf("Error for %s: Did not receive any content from the API.", config.Name)
 		// Save error result
-		saveResult(resultsDir, TestResult{
+		result := TestResult{
 			Provider:  config.Name,
 			Model:     config.Model,
 			Timestamp: time.Now(),
 			Success:   false,
 			Error:     "No content received from API",
-		})
+		}
+		saveResult(resultsDir, result)
+		appendResult(results, resultsMutex, result)
 		return
 	}
 
@@ -173,13 +187,15 @@ func testProviderMetrics(config ProviderConfig, tke *tiktoken.Tiktoken, wg *sync
 	if completionTokens == 0 {
 		providerLogger.Printf("Error for %s: Received response with 0 tokens.", config.Name)
 		// Save error result
-		saveResult(resultsDir, TestResult{
+		result := TestResult{
 			Provider:  config.Name,
 			Model:     config.Model,
 			Timestamp: time.Now(),
 			Success:   false,
 			Error:     "Received 0 tokens",
-		})
+		}
+		saveResult(resultsDir, result)
+		appendResult(results, resultsMutex, result)
 		return
 	}
 
@@ -215,7 +231,7 @@ func testProviderMetrics(config ProviderConfig, tke *tiktoken.Tiktoken, wg *sync
 	// providerLogger.Printf("[%s] Full Response:\n%s\n", config.Name, fullResponse)
 
 	// Save successful result
-	saveResult(resultsDir, TestResult{
+	result := TestResult{
 		Provider:         config.Name,
 		Model:            config.Model,
 		Timestamp:        time.Now(),
@@ -224,7 +240,18 @@ func testProviderMetrics(config ProviderConfig, tke *tiktoken.Tiktoken, wg *sync
 		Throughput:       throughput,
 		CompletionTokens: completionTokens,
 		Success:          true,
-	})
+	}
+	saveResult(resultsDir, result)
+	appendResult(results, resultsMutex, result)
+}
+
+// appendResult safely appends a result to the shared results slice
+func appendResult(results *[]TestResult, mutex *sync.Mutex, result TestResult) {
+	if results != nil && mutex != nil {
+		mutex.Lock()
+		*results = append(*results, result)
+		mutex.Unlock()
+	}
 }
 
 // saveResult saves the test result to a JSON file
@@ -234,16 +261,149 @@ func saveResult(resultsDir string, result TestResult) {
 
 	data, err := json.MarshalIndent(result, "", "  ")
 	if err != nil {
-		log.Printf("❌ Error marshaling result for %s: %v", result.Provider, err)
+		log.Printf("Error marshaling result for %s: %v", result.Provider, err)
 		return
 	}
 
 	if err := os.WriteFile(filename, data, 0644); err != nil {
-		log.Printf("❌ Error writing result file for %s: %v", result.Provider, err)
+		log.Printf("Error writing result file for %s: %v", result.Provider, err)
 		return
 	}
 
-	log.Printf("✅ Result saved: %s", filename)
+	log.Printf("Result saved: %s", filename)
+}
+
+// generateMarkdownReport creates a summary report of all test results
+func generateMarkdownReport(resultsDir string, results []TestResult, sessionTimestamp string) error {
+	filename := filepath.Join(resultsDir, "REPORT.md")
+
+	var report strings.Builder
+	report.WriteString("# LLM API Speed Test Results\n\n")
+	report.WriteString(fmt.Sprintf("**Test Session:** %s\n\n", sessionTimestamp))
+	report.WriteString("---\n\n")
+
+	// Summary statistics
+	successful := 0
+	failed := 0
+	for _, r := range results {
+		if r.Success {
+			successful++
+		} else {
+			failed++
+		}
+	}
+
+	report.WriteString("## Summary\n\n")
+	report.WriteString(fmt.Sprintf("- **Total Providers Tested:** %d\n", len(results)))
+	report.WriteString(fmt.Sprintf("- **Successful:** %d\n", successful))
+	report.WriteString(fmt.Sprintf("- **Failed:** %d\n\n", failed))
+
+	// Successful results table
+	if successful > 0 {
+		report.WriteString("## Successful Tests\n\n")
+		report.WriteString("| Provider | Model | E2E Latency | TTFT | Throughput | Tokens |\n")
+		report.WriteString("|----------|-------|-------------|------|------------|--------|\n")
+
+		for _, r := range results {
+			if r.Success {
+				report.WriteString(fmt.Sprintf("| %s | %s | %v | %v | %.2f tok/s | %d |\n",
+					r.Provider,
+					r.Model,
+					r.E2ELatency,
+					r.TTFT,
+					r.Throughput,
+					r.CompletionTokens))
+			}
+		}
+		report.WriteString("\n")
+	}
+
+	// Failed results
+	if failed > 0 {
+		report.WriteString("## Failed Tests\n\n")
+		report.WriteString("| Provider | Model | Error |\n")
+		report.WriteString("|----------|-------|-------|\n")
+
+		for _, r := range results {
+			if !r.Success {
+				report.WriteString(fmt.Sprintf("| %s | %s | %s |\n",
+					r.Provider,
+					r.Model,
+					r.Error))
+			}
+		}
+		report.WriteString("\n")
+	}
+
+	// Leaderboard (sorted by throughput)
+	if successful > 0 {
+		report.WriteString("## 🏆 Performance Leaderboard\n\n")
+		report.WriteString("### By Throughput (Tokens/sec)\n\n")
+
+		// Sort by throughput
+		successfulResults := make([]TestResult, 0)
+		for _, r := range results {
+			if r.Success {
+				successfulResults = append(successfulResults, r)
+			}
+		}
+
+		// Simple bubble sort by throughput descending
+		for i := 0; i < len(successfulResults); i++ {
+			for j := i + 1; j < len(successfulResults); j++ {
+				if successfulResults[j].Throughput > successfulResults[i].Throughput {
+					successfulResults[i], successfulResults[j] = successfulResults[j], successfulResults[i]
+				}
+			}
+		}
+
+		report.WriteString("| Rank | Provider | Throughput | TTFT | E2E Latency |\n")
+		report.WriteString("|------|----------|------------|------|-------------|\n")
+
+		for i, r := range successfulResults {
+			report.WriteString(fmt.Sprintf("| %d | %s | %.2f tok/s | %v | %v |\n",
+				i+1,
+				r.Provider,
+				r.Throughput,
+				r.TTFT,
+				r.E2ELatency))
+		}
+		report.WriteString("\n")
+
+		// Sort by TTFT
+		report.WriteString("### By Time to First Token (TTFT)\n\n")
+
+		for i := 0; i < len(successfulResults); i++ {
+			for j := i + 1; j < len(successfulResults); j++ {
+				if successfulResults[j].TTFT < successfulResults[i].TTFT {
+					successfulResults[i], successfulResults[j] = successfulResults[j], successfulResults[i]
+				}
+			}
+		}
+
+		report.WriteString("| Rank | Provider | TTFT | Throughput | E2E Latency |\n")
+		report.WriteString("|------|----------|------|------------|-------------|\n")
+
+		for i, r := range successfulResults {
+			report.WriteString(fmt.Sprintf("| %d | %s | %v | %.2f tok/s | %v |\n",
+				i+1,
+				r.Provider,
+				r.TTFT,
+				r.Throughput,
+				r.E2ELatency))
+		}
+		report.WriteString("\n")
+	}
+
+	report.WriteString("---\n\n")
+	report.WriteString(fmt.Sprintf("*Report generated at %s*\n", time.Now().Format("2006-01-02 15:04:05")))
+
+	if err := os.WriteFile(filename, []byte(report.String()), 0644); err != nil {
+		return fmt.Errorf("error writing report: %v", err)
+	}
+
+	log.Printf("Report generated: %s", filename)
+	return nil
 }
 
 func main() {
@@ -269,9 +429,11 @@ func main() {
 	flagGenericModel := flag.String("model", "", "Model name for 'generic' provider (required if --provider is not set)")
 	flag.Parse()
 
-	// 3. Create log and results directories
-	logDir := "logs"
-	resultsDir := "results"
+	// 3. Create session-based folder structure
+	sessionTimestamp := time.Now().Format("20060102-150405")
+	sessionDir := filepath.Join("results", fmt.Sprintf("session-%s", sessionTimestamp))
+	logDir := filepath.Join(sessionDir, "logs")
+	resultsDir := sessionDir
 
 	if err := os.MkdirAll(logDir, 0755); err != nil {
 		log.Fatalf("Error creating logs directory: %v", err)
@@ -281,8 +443,9 @@ func main() {
 		log.Fatalf("Error creating results directory: %v", err)
 	}
 
-	log.Printf("📁 Logs will be saved to: %s/", logDir)
-	log.Printf("📁 Results will be saved to: %s/", resultsDir)
+	log.Printf("Session folder: %s/", sessionDir)
+	log.Printf("Logs will be saved to: %s/", logDir)
+	log.Printf("Results will be saved to: %s/", resultsDir)
 
 	// 4. Initialize Tokenizer
 	tke, err := tiktoken.GetEncoding("cl100k_base")
@@ -395,14 +558,17 @@ func main() {
 
 	// 6. Run Tests
 	var wg sync.WaitGroup
+	var results []TestResult
+	var resultsMutex sync.Mutex
+
 	for _, provider := range providersToTest {
 		if *testAll {
 			// Run all tests concurrently
 			wg.Add(1)
-			go testProviderMetrics(provider, tke, &wg, logDir, resultsDir)
+			go testProviderMetrics(provider, tke, &wg, logDir, resultsDir, &results, &resultsMutex)
 		} else {
 			// Run a single test sequentially
-			testProviderMetrics(provider, tke, nil, logDir, resultsDir)
+			testProviderMetrics(provider, tke, nil, logDir, resultsDir, &results, &resultsMutex)
 		}
 	}
 
@@ -411,4 +577,12 @@ func main() {
 		wg.Wait()
 		log.Println("--- All provider tests complete. ---")
 	}
+
+	// Generate markdown report
+	log.Println("Generating summary report...")
+	if err := generateMarkdownReport(resultsDir, results, sessionTimestamp); err != nil {
+		log.Printf("Warning: Failed to generate report: %v", err)
+	}
+
+	log.Printf("All tests complete. Results saved to: %s/", sessionDir)
 }