Improve error reporting in CI check for CSV coverage report comparison

Author: tamasvajk

misc/scripts/library-coverage/compare-files.py

@@ -18,7 +18,7 @@ def ignore_line_ending(ch):
     return difflib.IS_CHARACTER_JUNK(ch, ws=" \r\n")
 
 
-def compare_files(file1, file2):
+def compare_files(file1, file2, path_to_report):
     has_differences = False
     diff = difflib.ndiff(open(file1).readlines(),
                          open(file2).readlines(), None, ignore_line_ending)
@@ -28,13 +28,16 @@ def compare_files(file1, file2):
             has_differences = True
 
     if has_differences:
-        print("Error: The generated file doesn't match the one in the codebase. Please check and fix file '" +
-              file1 + "'.", file=sys.stderr)
-        sys.exit(1)
+        print("The generated file doesn't match the one in the codebase. Please check and fix file '" +
+              path_to_report + "'.", file=sys.stderr)
+        return False
+    return True
 
 
 languages = ['java']
 
+all_ok = True
+
 for lang in languages:
     repo_output_rst = settings.repo_output_rst.format(language=lang)
     repo_output_csv = settings.repo_output_csv.format(language=lang)
@@ -47,8 +50,20 @@ def compare_files(file1, file2):
     check_file_exists(generated_output_rst)
     check_file_exists(generated_output_csv)
 
-    compare_files(repo_output_rst, generated_output_rst)
-    compare_files(repo_output_csv, generated_output_csv)
+    docs_folder = settings.documentation_folder_no_prefix.format(language=lang)
+
+    rst_ok = compare_files(repo_output_rst, generated_output_rst,
+                           docs_folder + settings.output_rst_file_name)
+    csv_ok = compare_files(repo_output_csv, generated_output_csv,
+                           docs_folder + settings.output_csv_file_name)
+
+    if not rst_ok or not csv_ok:
+        print("The generated CSV coverage report files for '" + lang + "' don't match the ones in the codebase. Please update the files in '" +
+              docs_folder + "'. The new files can be downloaded from the artifacts of this job.", file=sys.stderr)
+        all_ok = False
+    else:
+        print("The generated files for '" + lang +
+              "' match the ones in the codebase.")
 
-    print("The generated files for '" + lang +
-          "' match the ones in the codebase.")
+if not all_ok:
+    sys.exit(1)

misc/scripts/library-coverage/settings.py

@@ -13,8 +13,10 @@
 if len(sys.argv) > index:
     data_prefix = sys.argv[index] + "/"
 
-documentation_folder = data_prefix + \
-    "{language}/documentation/library-coverage/"
+documentation_folder_no_prefix = "{language}/documentation/library-coverage/"
+documentation_folder = data_prefix + documentation_folder_no_prefix
 
-repo_output_rst = documentation_folder + "flow-model-coverage.rst"
-repo_output_csv = documentation_folder + "flow-model-coverage.csv"
+output_rst_file_name = "flow-model-coverage.rst"
+output_csv_file_name = "flow-model-coverage.csv"
+repo_output_rst = documentation_folder + output_rst_file_name
+repo_output_csv = documentation_folder + output_csv_file_name