automated evaluation result saving

Author: EricBenschneider

create_csv.py evaluation/create_csv.pycreate_csv.py renamed to evaluation/create_csv.py

@@ -1,2 +1,2 @@
 #./bin/daphne --timing --second-read-opt test/api/cli/io/evalReadFrame.daphne
-readFrame("data_1000r_10c_NUMBER.csv");
+readFrame("evaluation/data_1000r_10c_NUMBER.csv");

data_1000r_10c_NUMBER.csv evaluation/data_1000r_10c_NUMBER.csvdata_1000r_10c_NUMBER.csv renamed to evaluation/data_1000r_10c_NUMBER.csv

@@ -0,0 +1 @@
+readFrame("evaluation/data_1000r_1000c_NUMBER.csv");

data_1000r_10c_NUMBER.csv.meta evaluation/data_1000r_10c_NUMBER.csv.metadata_1000r_10c_NUMBER.csv.meta renamed to evaluation/data_1000r_10c_NUMBER.csv.meta

@@ -360,8 +360,6 @@ template <> struct ReadCsvFile<Frame> {
             rawCols[i] = reinterpret_cast<uint8_t *>(res->getColumnRaw(i));
             colTypes[i] = res->getColumnType(i);
         }
-        using clock = std::chrono::high_resolution_clock;
-        auto time = clock::now();
         // Determine if any optimized branch should be used.
         bool useOptimized = false;
         bool useBin = false;
@@ -375,6 +373,8 @@ template <> struct ReadCsvFile<Frame> {
                 fName = daphneFile;
             }
         }
+        using clock = std::chrono::high_resolution_clock;
+        auto time = clock::now();
         if (useOptimized) {
             if (useBin) {
                 try {

test/api/cli/io/evalReadFrame.daphne evaluation/evalReadFrame.daphnetest/api/cli/io/evalReadFrame.daphne renamed to evaluation/evalReadFrame.daphne

@@ -24,6 +24,23 @@
 #include <regex>
 #include <string>
 
+std::string createDaphneScript(const std::string &evaluationDir,
+                        const std::string &csvFilename,
+                        const std::string &daphneScript) {
+    std::filesystem::create_directories(evaluationDir); // ensure directory exists
+    std::string daphneFilePath = evaluationDir + daphneScript;
+    if (std::filesystem::exists(daphneFilePath)) {
+        return daphneFilePath;
+    }
+    std::ofstream ofs(daphneFilePath);
+    if(!ofs) {
+        throw std::runtime_error("Could not create Daphne script file: " + daphneFilePath);
+    }
+    ofs << "readFrame(\"" << evaluationDir + csvFilename << "\");";
+    ofs.close();
+    return daphneFilePath;
+}
+
 template <typename... Args>
 std::string runDaphneEval( const std::string &scriptFilePath, Args... args) {
     std::stringstream out;
@@ -36,7 +53,7 @@ std::string runDaphneEval( const std::string &scriptFilePath, Args... args) {
     // output.
     CHECK(status == StatusCode::SUCCESS);
     //std::cout << out.str() << std::endl;
-    CHECK(err.str() == "");
+    //CHECK(err.str() == "");
     return out.str()+err.str();
 }
 // New data structure for timing values.
@@ -121,21 +138,16 @@ TimingData extractTiming(const std::string &output, bool expectWriteTime = false
     return timingData;
 }
 
-void writeResultsToFile(const std::string& feature, const std::string &csvFilename, bool opt, bool firstRead, std::string timing, std::string readTime,  std::string writeTime) {
-    // default results file where all timings are stored
-    const std::string resultsFile = "evaluation_results_"+ feature + ".csv";
+void writeResultsToFile(const std::string& feature, const std::string &csvFilename, bool opt, bool firstRead, const TimingData &timingData) {
+    const std::string resultsFile = "evaluation/evaluation_results_" + feature + ".csv";
     bool fileExists = std::filesystem::exists(resultsFile);
     std::ofstream ofs(resultsFile, std::ios::app);
     if (!ofs) {
         throw std::runtime_error("Could not open " + resultsFile + " for writing.");
     }
-    // Write CSV header if file did not already exist
     if (!fileExists) {
-        ofs << "CSVFile,OptEnabled,NumCols,NumRows,FileType,Timing,ReadTime,WriteTime\n";
+        ofs << "CSVFile,OptEnabled,FirstRead,NumCols,NumRows,FileType,ReadTime,WriteTime,StartupSeconds,ParsingSeconds,CompilationSeconds,ExecutionSeconds,TotalSeconds,WriteTime\n";
     }
-    // Create string representation for bool and FileType.
-    std::string optStr = opt? "true" : "false";
-    std::string firstReadStr = firstRead? "true" : "false";
 
     // Extract numRows, numCols, and FileType from the filename.
     // Expected format: data_<numRows>r_<numCols>c_<FileType>.csv
@@ -154,51 +166,106 @@ void writeResultsToFile(const std::string& feature, const std::string &csvFilena
     std::string type = "";
     if (parts.size() >= 4) {
         // parts[0] is "data", parts[1] is "<numRows>r", parts[2] is "<numCols>c", parts[3] is "<FileType>"
-        std::string rowToken = parts[1]; // e.g. "100r"
+        std::string rowToken = parts[1]; // e.g. "1000r"
         std::string colToken = parts[2]; // e.g. "10c"
         if (!rowToken.empty() && rowToken.back() == 'r') {
-            rowToken.pop_back(); // remove the trailing 'r'
+            rowToken.pop_back(); // remove trailing 'r'
         }
         if (!colToken.empty() && colToken.back() == 'c') {
-            colToken.pop_back(); // remove the trailing 'c'
+            colToken.pop_back(); // remove trailing 'c'
         }
         numRows = std::stoi(rowToken);
         numCols = std::stoi(colToken);
         type = parts[3];
     }
 
-    // Append the result line. Use csvFilename (i.e. the name of the read CSV file) as identifier.
+    std::string optStr = opt ? "true" : "false";
+    std::string firstReadStr = firstRead ? "true" : "false";
     ofs << csvFilename << ","
         << optStr << ","
         << firstReadStr << ","
         << numCols << ","
         << numRows << ","
         << type << ","
-        << timing << ","
-        << readTime << ","
-        << writeTime << "\n";
+        << timingData.readTime << ","
+        << timingData.writeTime << ","
+        << timingData.startupSeconds << ","
+        << timingData.parsingSeconds << ","
+        << timingData.compilationSeconds << ","
+        << timingData.executionSeconds << ","
+        << timingData.totalSeconds << "\n";
     ofs.close();
 }
 
-const std::string dirPath = "test/api/cli/io/";
+void runEvalTestCase(const std::string &csvFilename,
+                     std::string daphneScript= "",
+                     const std::string &dirPath= "evaluation/"
+                     ) {
+    // Remove potential binary output file.
+    std::filesystem::remove(dirPath + csvFilename + ".dbdf");
+    if (daphneScript.empty()) {
+        daphneScript = createDaphneScript(dirPath, csvFilename, csvFilename+".daphne");
+    }else{
+        daphneScript = dirPath + daphneScript;
+    }
+    
+    // Normal read for comparison.
+    std::string output = runDaphneEval(daphneScript, "--timing");
+    std::cout << output << std::endl;
+    TimingData timingData = extractTiming(output);
+    writeResultsToFile("binopt", csvFilename, false, true, timingData);
+
+    // Build binary file and positional map on first read.
+    output = runDaphneEval(daphneScript, "--timing", "--second-read-opt");
+    std::cout << output << std::endl;
+    timingData = extractTiming(output, true);
+    writeResultsToFile("binopt", csvFilename, true, true, timingData);
+    CHECK(std::filesystem::exists(dirPath + csvFilename + ".dbdf"));
+
+    // Subsequent read.
+    output = runDaphneEval( daphneScript, "--timing", "--second-read-opt");
+    std::cout << output << std::endl;
+    timingData = extractTiming(output);
+    writeResultsToFile("binopt", csvFilename, true, false, timingData);
+}
+
+TEST_CASE("EvalTestCaseVariant60KB", TAG_IO) {
+    // Example instantiation.
+    const std::string csvFilename = "data_1000r_10c_NUMBER.csv";
+    const std::string daphneScript = "evalReadFrame.daphne";
+    runEvalTestCase(csvFilename, daphneScript);
+}
+
+TEST_CASE("EvalTestCaseVariant6MB", TAG_IO) {
+    // Example instantiation.
+    const std::string csvFilename = "data_1000r_1000c_NUMBER.csv";
+    const std::string daphneScript = "evalReadFrame2.daphne";
+    runEvalTestCase(csvFilename);//, daphneScript);
+}
+
+const std::string dirPath = "evaluation/";
 TEST_CASE("evalFrameFromCSVBinOpt", TAG_IO) {
+    //assumes file has been generated via create_csv.py in /daphne
     std::string filename = "data_1000r_10c_NUMBER.csv";
-    std::string filepath = dirPath + filename;
-    std::filesystem::remove(filename + ".dbdf");
+    std::filesystem::remove(dirPath + filename + ".dbdf");
     // normal read for comparison
     std::string output;
     output = runDaphneEval(dirPath + "evalReadFrame.daphne", "--timing");
     std::cout << output << std::endl;
-    //TODO: extract read time from output and clear it
+
     std::string timing, readTime, writeTime;
-    //{timing, readTime, writeTime} = extractTiming(output);
-    writeResultsToFile("binopt", filename, false, true, output, "0", "0");
+    TimingData timingData = extractTiming(output);
+    writeResultsToFile("binopt", filename, false, true, timingData);
+    
     // build binary file and positional map on first read
     output = runDaphneEval(dirPath + "evalReadFrame.daphne", "--timing", "--second-read-opt");
     std::cout << output << std::endl;
-    writeResultsToFile("binopt", filename, true, true,  output, "0", "0");
-    CHECK(std::filesystem::exists(filepath + ".dbdf"));
+    timingData = extractTiming(output, true);
+    writeResultsToFile("binopt", filename, true, true, timingData);
+    REQUIRE(std::filesystem::exists(dirPath + filename + ".dbdf"));
+    
     output= runDaphneEval(dirPath + "evalReadFrame.daphne", "--timing", "--second-read-opt");
-    std::cout << output << std::endl;
-    writeResultsToFile("binopt", filename, true, false,  output, "0", "0");
+        std::cout << output << std::endl;
+    timingData = extractTiming(output);
+    writeResultsToFile("binopt", filename, true, false, timingData);
 }