Skip to content

Commit fa5fb29

Browse files
author
Qi Chen
committed
write the result in time
1 parent 20fa847 commit fa5fb29

File tree

1 file changed

+67
-81
lines changed

1 file changed

+67
-81
lines changed

‎Test/src/SPFreshTest.cpp‎

Lines changed: 67 additions & 81 deletions
Original file line numberDiff line numberDiff line change
@@ -255,7 +255,7 @@ void BenchmarkQueryPerformance(std::shared_ptr<VectorIndex> &index, std::shared_
255255
std::shared_ptr<VectorSet> &truth, std::shared_ptr<VectorSet> &vecset,
256256
std::shared_ptr<VectorSet> &addvecset, const std::string &truthPath,
257257
SizeType baseVectorCount, int topK, int numThreads, int numQueries, int batches,
258-
std::ostringstream &benchmarkData, std::string prefix = "")
258+
std::ostream &benchmarkData, std::string prefix = "")
259259
{
260260
// Benchmark: Query performance with detailed latency stats
261261
std::vector<float> latencies(numQueries);
@@ -363,7 +363,7 @@ void RunBenchmark(const std::string &vectorPath, const std::string &queryPath, c
363363
int deleteBatchSize = deleteVectorCount / max(batches, 1);
364364

365365
// Variables to collect JSON output data
366-
std::ostringstream tmpbenchmark, benchmark0Data, benchmark1Data;
366+
std::ostringstream tmpbenchmark;
367367

368368
// Generate test data
369369
std::shared_ptr<VectorSet> vecset, addvecset, queryset, truth;
@@ -372,6 +372,45 @@ void RunBenchmark(const std::string &vectorPath, const std::string &queryPath, c
372372
generator.RunBatches(vecset, metaset, addvecset, addmetaset, queryset, N, insertBatchSize, deleteBatchSize,
373373
batches, truth);
374374

375+
376+
std::ofstream jsonFile(outputFile);
377+
BOOST_REQUIRE(jsonFile.is_open());
378+
379+
jsonFile << std::fixed << std::setprecision(4);
380+
381+
// Get current timestamp
382+
auto time_t_now = std::chrono::system_clock::to_time_t(std::chrono::system_clock::now());
383+
std::tm tm_now;
384+
#if defined(_MSC_VER)
385+
localtime_s(&tm_now, &time_t_now);
386+
#else
387+
localtime_r(&time_t_now, &tm_now);
388+
#endif
389+
390+
std::ostringstream timestampStream;
391+
timestampStream << std::put_time(&tm_now, "%Y-%m-%dT%H:%M:%S");
392+
std::string timestamp = timestampStream.str();
393+
394+
jsonFile << "{\n";
395+
jsonFile << " \"timestamp\": \"" << timestamp << "\",\n";
396+
jsonFile << " \"config\": {\n";
397+
jsonFile << " \"vectorPath\": \"" << vectorPath << "\",\n";
398+
jsonFile << " \"queryPath\": \"" << queryPath << "\",\n";
399+
jsonFile << " \"truthPath\": \"" << truthPath << "\",\n";
400+
jsonFile << " \"indexPath\": \"" << indexPath << "\",\n";
401+
jsonFile << " \"ValueType\": \"" << Helper::Convert::ConvertToString(GetEnumValueType<T>()) << "\",\n";
402+
jsonFile << " \"dimension\": " << dimension << ",\n";
403+
jsonFile << " \"baseVectorCount\": " << baseVectorCount << ",\n";
404+
jsonFile << " \"insertVectorCount\": " << insertVectorCount << ",\n";
405+
jsonFile << " \"DeleteVectorCount\": " << deleteVectorCount << ",\n";
406+
jsonFile << " \"BatchNum\": " << batches << ",\n";
407+
jsonFile << " \"topK\": " << topK << ",\n";
408+
jsonFile << " \"numQueries\": " << numQueries << ",\n";
409+
jsonFile << " \"numThreads\": " << numThreads << ",\n";
410+
jsonFile << " \"DistMethod\": \"" << Helper::Convert::ConvertToString(distMethod) << "\"\n";
411+
jsonFile << " },\n";
412+
jsonFile << " \"results\": {\n";
413+
375414
// Build initial index
376415
BOOST_TEST_MESSAGE("\n=== Building Index ===");
377416
std::filesystem::remove_all(indexPath);
@@ -384,8 +423,12 @@ void RunBenchmark(const std::string &vectorPath, const std::string &queryPath, c
384423
BOOST_TEST_MESSAGE("\n=== Benchmark 0: Query Before Insertions ===");
385424
BenchmarkQueryPerformance<T>(index, queryset, truth, vecset, addvecset, truthPath, baseVectorCount, topK,
386425
numThreads, numQueries, 0, tmpbenchmark);
426+
jsonFile << " \"benchmark0_query_before_insert\": ";
387427
BenchmarkQueryPerformance<T>(index, queryset, truth, vecset, addvecset, truthPath, baseVectorCount, topK,
388-
numThreads, numQueries, 0, benchmark0Data);
428+
numThreads, numQueries, 0, jsonFile);
429+
jsonFile << ",\n";
430+
jsonFile.flush();
431+
389432
BOOST_REQUIRE(index->SaveIndex(indexPath) == ErrorCode::Success);
390433
index = nullptr;
391434

@@ -402,12 +445,11 @@ void RunBenchmark(const std::string &vectorPath, const std::string &queryPath, c
402445
{
403446
BOOST_TEST_MESSAGE("\n=== Benchmark 1: Insert Performance ===");
404447
{
405-
benchmark1Data << std::fixed << std::setprecision(4);
406-
benchmark1Data << "{\n";
448+
jsonFile << " \"benchmark1_insert\": {\n";
407449
std::string prevPath = indexPath;
408450
for (int iter = 0; iter < batches; iter++)
409451
{
410-
benchmark1Data << " \"batch_" << iter + 1 << "\": {\n";
452+
jsonFile << " \"batch_" << iter + 1 << "\": {\n";
411453

412454
std::string clonePath = indexPath + "_" + std::to_string(iter);
413455
std::shared_ptr<VectorIndex> prevIndex, clonedIndex;
@@ -424,8 +466,8 @@ void RunBenchmark(const std::string &vectorPath, const std::string &queryPath, c
424466
BOOST_TEST_MESSAGE(" Index vectors after reload: " << vectorCount);
425467

426468
// Collect JSON data for Benchmark 4
427-
benchmark1Data << " \"Load timeSeconds\": " << seconds << ",\n";
428-
benchmark1Data << " \"Load vectorCount\": " << vectorCount << ",\n";
469+
jsonFile << " \"Load timeSeconds\": " << seconds << ",\n";
470+
jsonFile << " \"Load vectorCount\": " << vectorCount << ",\n";
429471

430472
auto cloneIndex = prevIndex->Clone(clonePath);
431473
prevIndex = nullptr;
@@ -445,9 +487,9 @@ void RunBenchmark(const std::string &vectorPath, const std::string &queryPath, c
445487
BOOST_TEST_MESSAGE(" Throughput: " << throughput << " vectors/sec");
446488

447489
// Collect JSON data for Benchmark 1
448-
benchmark1Data << " \"inserted\": " << insertBatchSize << ",\n";
449-
benchmark1Data << " \"insert timeSeconds\": " << seconds << ",\n";
450-
benchmark1Data << " \"insert throughput\": " << throughput << ",\n";
490+
jsonFile << " \"inserted\": " << insertBatchSize << ",\n";
491+
jsonFile << " \"insert timeSeconds\": " << seconds << ",\n";
492+
jsonFile << " \"insert throughput\": " << throughput << ",\n";
451493

452494
if (deleteBatchSize > 0)
453495
{
@@ -492,16 +534,16 @@ void RunBenchmark(const std::string &vectorPath, const std::string &queryPath, c
492534
std::chrono::duration_cast<std::chrono::microseconds>(end - start).count() / 1000000.0f;
493535
double throughput = deleteBatchSize / seconds;
494536

495-
benchmark1Data << " \"deleted\": " << deleteVectorCount << ",\n";
496-
benchmark1Data << " \"delete timeSeconds\": " << seconds << ",\n";
497-
benchmark1Data << " \"delete throughput\": " << throughput << ",\n";
537+
jsonFile << " \"deleted\": " << deleteVectorCount << ",\n";
538+
jsonFile << " \"delete timeSeconds\": " << seconds << ",\n";
539+
jsonFile << " \"delete throughput\": " << throughput << ",\n";
498540
}
499541

500542
BOOST_TEST_MESSAGE("\n=== Benchmark 2: Query After Insertions and Deletions ===");
501-
benchmark1Data << " \"search\":";
543+
jsonFile << " \"search\":";
502544
BenchmarkQueryPerformance<T>(cloneIndex, queryset, truth, vecset, addvecset, truthPath, baseVectorCount,
503-
topK, numThreads, numQueries, iter + 1, benchmark1Data, " ");
504-
benchmark1Data << ",\n";
545+
topK, numThreads, numQueries, iter + 1, jsonFile, " ");
546+
jsonFile << ",\n";
505547

506548
start = std::chrono::high_resolution_clock::now();
507549
BOOST_REQUIRE(cloneIndex->SaveIndex(clonePath) == ErrorCode::Success);
@@ -512,80 +554,24 @@ void RunBenchmark(const std::string &vectorPath, const std::string &queryPath, c
512554
BOOST_TEST_MESSAGE(" Save completed successfully");
513555

514556
// Collect JSON data for Benchmark 3
515-
benchmark1Data << " \"save timeSeconds\": " << seconds << "\n";
557+
jsonFile << " \"save timeSeconds\": " << seconds << "\n";
516558

517559
if (iter != batches - 1)
518-
benchmark1Data << " },\n";
560+
jsonFile << " },\n";
519561
else
520-
benchmark1Data << " }\n";
521-
522-
562+
jsonFile << " }\n";
523563

524564
cloneIndex = nullptr;
525565
prevPath = clonePath;
566+
jsonFile.flush();
526567
}
527568
}
528-
benchmark1Data << " }";
569+
jsonFile << " }\n";
529570
}
530571

531-
532-
BOOST_TEST_MESSAGE("\n=== Benchmark Complete ===");
533-
534-
// Write JSON output
535-
{
536-
std::ofstream jsonFile(outputFile);
537-
if (jsonFile.is_open())
538-
{
539-
jsonFile << std::fixed << std::setprecision(4);
540-
541-
// Get current timestamp
542-
auto time_t_now = std::chrono::system_clock::to_time_t(std::chrono::system_clock::now());
543-
std::tm tm_now;
544-
#if defined(_MSC_VER)
545-
localtime_s(&tm_now, &time_t_now);
546-
#else
547-
localtime_r(&time_t_now, &tm_now);
548-
#endif
549-
550-
std::ostringstream timestampStream;
551-
timestampStream << std::put_time(&tm_now, "%Y-%m-%dT%H:%M:%S");
552-
std::string timestamp = timestampStream.str();
553-
554-
jsonFile << "{\n";
555-
jsonFile << " \"timestamp\": \"" << timestamp << "\",\n";
556-
jsonFile << " \"config\": {\n";
557-
jsonFile << " \"vectorPath\": \"" << vectorPath << "\",\n";
558-
jsonFile << " \"queryPath\": \"" << queryPath << "\",\n";
559-
jsonFile << " \"truthPath\": \"" << truthPath << "\",\n";
560-
jsonFile << " \"indexPath\": \"" << indexPath << "\",\n";
561-
jsonFile << " \"ValueType\": \"" << Helper::Convert::ConvertToString(GetEnumValueType<T>()) << "\",\n";
562-
jsonFile << " \"dimension\": " << dimension << ",\n";
563-
jsonFile << " \"baseVectorCount\": " << baseVectorCount << ",\n";
564-
jsonFile << " \"insertVectorCount\": " << insertVectorCount << ",\n";
565-
jsonFile << " \"DeleteVectorCount\": " << deleteVectorCount << ",\n";
566-
jsonFile << " \"BatchNum\": " << batches << ",\n";
567-
jsonFile << " \"topK\": " << topK << ",\n";
568-
jsonFile << " \"numQueries\": " << numQueries << ",\n";
569-
jsonFile << " \"numThreads\": " << numThreads << ",\n";
570-
jsonFile << " \"DistMethod\": \"" << Helper::Convert::ConvertToString(distMethod) << "\"\n";
571-
jsonFile << " },\n";
572-
jsonFile << " \"results\": {\n";
573-
jsonFile << " \"benchmark0_query_before_insert\": " << benchmark0Data.str() << ",\n";
574-
if (!benchmark1Data.str().empty())
575-
{
576-
jsonFile << " \"benchmark1_insert\": " << benchmark1Data.str() << "\n";
577-
}
578-
jsonFile << " }\n";
579-
jsonFile << "}\n";
580-
581-
jsonFile.close();
582-
BOOST_TEST_MESSAGE("\n=== Benchmark results saved to output.json ===");
583-
}
584-
else
585-
{
586-
BOOST_TEST_MESSAGE("\n=== Failed to create output.json ===");
587-
}
588-
}
572+
jsonFile << " }\n";
573+
jsonFile << "}\n";
574+
jsonFile.close();
589575

590576
M = oldM;
591577
K = oldK;

0 commit comments

Comments
 (0)