Merge pull request #1322 from stan-dev/improve-diagnose-tests

Helper for diagnose tests

Author: WardBrian

src/test/interface/csv_header_consistency_test.cpp

@@ -1,4 +1,4 @@
-#include <stan/mcmc/chains.hpp>
+#include <stan/mcmc/chainset.hpp>
 #include <stan/services/error_codes.hpp>
 #include <test/utility.hpp>
 #include <gtest/gtest.h>
@@ -28,7 +28,7 @@ TEST(interface, csv_header_consistency) {
 
   std::ifstream ifstream;
   ifstream.open(samples.c_str());
-  stan::mcmc::chains<> chains(
+  stan::mcmc::chainset chains(
       stan::io::stan_csv_reader::parse(ifstream, &std::cout));
   ifstream.close();
 

src/test/interface/diagnose_test.cpp

@@ -3,10 +3,10 @@
 #include <fstream>
 #include <sstream>
 
+using cmdstan::test::compare_to_stored_output;
 using cmdstan::test::count_matches;
 using cmdstan::test::get_path_separator;
 using cmdstan::test::run_command;
-using cmdstan::test::run_command_output;
 
 TEST(CommandDiagnose, corr_gauss) {
   std::string path_separator;
@@ -16,14 +16,11 @@ TEST(CommandDiagnose, corr_gauss) {
                          + "interface" + path_separator + "example_output"
                          + path_separator + "corr_gauss_output.csv";
 
-  run_command_output out = run_command(command + " " + csv_file);
+  auto out = run_command(command + " " + csv_file);
   ASSERT_FALSE(out.hasError) << "\"" << out.command << "\" quit with an error";
 
-  std::ifstream expected_output(
-      "src/test/interface/example_output/corr_gauss.nom");
-  std::stringstream ss;
-  ss << expected_output.rdbuf();
-  EXPECT_EQ(1, count_matches(ss.str(), out.output));
+  compare_to_stored_output(out.output,
+                           "src/test/interface/example_output/corr_gauss.nom");
 }
 
 TEST(CommandDiagnose, corr_gauss_depth8) {
@@ -34,14 +31,11 @@ TEST(CommandDiagnose, corr_gauss_depth8) {
                          + "interface" + path_separator + "example_output"
                          + path_separator + "corr_gauss_output_depth8.csv";
 
-  run_command_output out = run_command(command + " " + csv_file);
+  auto out = run_command(command + " " + csv_file);
   ASSERT_FALSE(out.hasError) << "\"" << out.command << "\" quit with an error";
 
-  std::ifstream expected_output(
-      "src/test/interface/example_output/corr_gauss_depth8.nom");
-  std::stringstream ss;
-  ss << expected_output.rdbuf();
-  EXPECT_EQ(1, count_matches(ss.str(), out.output));
+  compare_to_stored_output(
+      out.output, "src/test/interface/example_output/corr_gauss_depth8.nom");
 }
 
 TEST(CommandDiagnose, corr_gauss_depth15) {
@@ -52,14 +46,11 @@ TEST(CommandDiagnose, corr_gauss_depth15) {
                          + "interface" + path_separator + "example_output"
                          + path_separator + "corr_gauss_output_depth15.csv";
 
-  run_command_output out = run_command(command + " " + csv_file);
+  auto out = run_command(command + " " + csv_file);
   ASSERT_FALSE(out.hasError) << "\"" << out.command << "\" quit with an error";
 
-  std::ifstream expected_output(
-      "src/test/interface/example_output/corr_gauss_depth15.nom");
-  std::stringstream ss;
-  ss << expected_output.rdbuf();
-  EXPECT_EQ(1, count_matches(ss.str(), out.output));
+  compare_to_stored_output(
+      out.output, "src/test/interface/example_output/corr_gauss_depth15.nom");
 }
 
 TEST(CommandDiagnose, eight_schools) {
@@ -70,14 +61,11 @@ TEST(CommandDiagnose, eight_schools) {
                          + "interface" + path_separator + "example_output"
                          + path_separator + "eight_schools_output.csv";
 
-  run_command_output out = run_command(command + " " + csv_file);
+  auto out = run_command(command + " " + csv_file);
   ASSERT_FALSE(out.hasError) << "\"" << out.command << "\" quit with an error";
 
-  std::ifstream expected_output(
-      "src/test/interface/example_output/eight_schools.nom");
-  std::stringstream ss;
-  ss << expected_output.rdbuf();
-  EXPECT_EQ(1, count_matches(ss.str(), out.output));
+  compare_to_stored_output(
+      out.output, "src/test/interface/example_output/eight_schools.nom");
 }
 
 TEST(CommandDiagnose, mix) {
@@ -88,13 +76,11 @@ TEST(CommandDiagnose, mix) {
                          + "interface" + path_separator + "example_output"
                          + path_separator + "mix_output.*";
 
-  run_command_output out = run_command(command + " " + csv_file);
+  auto out = run_command(command + " " + csv_file);
   ASSERT_FALSE(out.hasError) << "\"" << out.command << "\" quit with an error";
 
-  std::ifstream expected_output("src/test/interface/example_output/mix.nom");
-  std::stringstream ss;
-  ss << expected_output.rdbuf();
-  EXPECT_EQ(1, count_matches(ss.str(), out.output));
+  compare_to_stored_output(out.output,
+                           "src/test/interface/example_output/mix.nom");
 }
 
 TEST(CommandDiagnose, divergences) {
@@ -105,11 +91,9 @@ TEST(CommandDiagnose, divergences) {
                          + "interface" + path_separator + "example_output"
                          + path_separator + "div_output*.csv";
 
-  run_command_output out = run_command(command + " " + csv_file);
+  auto out = run_command(command + " " + csv_file);
   ASSERT_FALSE(out.hasError) << "\"" << out.command << "\" quit with an error";
 
-  std::ifstream expected_output("src/test/interface/example_output/div.nom");
-  std::stringstream ss;
-  ss << expected_output.rdbuf();
-  EXPECT_EQ(1, count_matches(ss.str(), out.output));
+  compare_to_stored_output(out.output,
+                           "src/test/interface/example_output/div.nom");
 }

src/test/utility.hpp

@@ -364,6 +364,34 @@ struct temporary_unwritable_file {
   }
 };
 
+void compare_to_stored_output(const std::string &output,
+                              const std::string &expected_path) {
+  if (getenv("CMDSTAN_UPDATE_EXPECTED_OUTPUT")) {
+    std::ofstream expected_output_file(expected_path);
+    ASSERT_TRUE(expected_output_file.good())
+        << "Could not open expected output file: " << expected_path;
+    expected_output_file << output;
+    expected_output_file.close();
+    std::cout << "Updated expected output file: " << expected_path << std::endl;
+    return;
+  }
+
+  std::ifstream expected_output_file(expected_path);
+  ASSERT_TRUE(expected_output_file.good())
+      << "Could not open expected output file: " << expected_path;
+  std::stringstream ss;
+  ss << expected_output_file.rdbuf();
+  expected_output_file.close();
+  std::string expected_output = ss.str();
+
+  EXPECT_EQ(output, expected_output)
+      << "Output does not match expected output. Expected:\n--------\n"
+      << expected_output << "\n--------\nActual:\n--------\n"
+      << output
+      << "\n--------\n\nSet CMDSTAN_UPDATE_EXPECTED_OUTPUT=1 to update the "
+         "expected output.\n";
+}
+
 }  // namespace test
 }  // namespace cmdstan
 #endif