Merge pull request #57 from cbrianpace/main

Added reporting ability

Author: cbrianpace

src/main/java/com/crunchydata/controller/ReconcileController.java

@@ -69,7 +69,7 @@ public static JSONObject reconcileData(Properties Props, Connection connRepo, Co
         // Variables
         ArrayList<Object> binds = new ArrayList<>();
         JSONObject columnMap;
-        Boolean useLoaderThreads = (Integer.parseInt(Props.getProperty("loader-threads")) > 0);
+        boolean useLoaderThreads = (Integer.parseInt(Props.getProperty("loader-threads")) > 0);
 
         BlockingQueue<DataCompare[]> qs;
         BlockingQueue<DataCompare[]> qt;
@@ -81,7 +81,11 @@ public static JSONObject reconcileData(Properties Props, Connection connRepo, Co
            qt = null;
         }
 
+        // Capture the start time for the compare run.
+        long startStopWatch = System.currentTimeMillis();
+
         // Prepare JSON formatted results
+        JSONObject checkResult = new JSONObject();
         JSONObject result = new JSONObject();
         result.put("tableName", dct.getTableAlias());
         result.put("status", "processing");
@@ -143,7 +147,8 @@ public static JSONObject reconcileData(Properties Props, Connection connRepo, Co
             Logging.write("info", THREAD_NAME, String.format("(target) Compare SQL: %s", dctmTarget.getCompareSQL()));
 
             if (check) {
-                threadReconcileCheck.checkRows(Props, connRepo, connSource, connTarget, dct, dctmSource, dctmTarget, ciSource, ciTarget, cid);
+                checkResult = threadReconcileCheck.checkRows(Props, connRepo, connSource, connTarget, dct, dctmSource, dctmTarget, ciSource, ciTarget, cid);
+                result.put("checkResult", checkResult);
             } else {
                 // Execute Compare SQL
                 if (ciTarget.pkList.isBlank() || ciTarget.pkList.isEmpty() || ciSource.pkList.isBlank() || ciSource.pkList.isEmpty()) {
@@ -253,8 +258,16 @@ public static JSONObject reconcileData(Properties Props, Connection connRepo, Co
                 result.put("equal", crsResult.getInt(1));
             }
 
+            result.put("totalRows",notEqual+missingSource+missingTarget+result.getInt("equal"));
+
             crsResult.close();
 
+            long endStopWatch = System.currentTimeMillis();
+            long elapsedTime = (endStopWatch - startStopWatch) / 1000;
+
+            result.put("elapsedTime", elapsedTime);
+            result.put("rowsPerSecond", (result.getInt("elapsedTime") > 0 ) ? result.getInt("totalRows")/elapsedTime : result.getInt("totalRows"));
+
             DecimalFormat formatter = new DecimalFormat("#,###");
 
             String msgFormat = "Reconciliation Complete: Table = %s; Status = %s; Equal = %s; Not Equal = %s; Missing Source = %s; Missing Target = %s";
@@ -264,9 +277,11 @@ public static JSONObject reconcileData(Properties Props, Connection connRepo, Co
 
         }  catch( SQLException e) {
             StackTraceElement[] stackTrace = e.getStackTrace();
+            result.put("status", "failed");
             Logging.write("severe", THREAD_NAME, String.format("Database error at line %s:  %s", stackTrace[0].getLineNumber(), e.getMessage()));
         } catch (Exception e) {
             StackTraceElement[] stackTrace = e.getStackTrace();
+            result.put("status", "failed");
             Logging.write("severe", THREAD_NAME, String.format("Error in reconcile controller at line %s:  %s", stackTrace[0].getLineNumber(), e.getMessage()));
         }
 

src/main/java/com/crunchydata/pgCompare.java

@@ -19,8 +19,11 @@
 import java.sql.Connection;
 import javax.sql.rowset.CachedRowSet;
 import java.text.DecimalFormat;
+import java.util.List;
 
 import static com.crunchydata.controller.TableController.getTableMap;
+import static com.crunchydata.services.Reporter.createSection;
+import static com.crunchydata.services.Reporter.generateHtmlReport;
 import static com.crunchydata.util.Settings.*;
 
 import com.crunchydata.controller.ColumnController;
@@ -44,8 +47,9 @@ public class pgCompare {
     private static final String THREAD_NAME = "main";
 
     private static String action = "reconcile";
+    private static String reportFileName;
     private static Integer batchParameter;
-    private static boolean check;
+    private static boolean check, genReport;
     private static CommandLine cmd;
     private static Integer pid = 1;
     private static Connection connRepo;
@@ -268,6 +272,7 @@ private static CommandLine parseCommandLine(String[] args) {
         options.addOption(Option.builder("h").longOpt("help").argName("help").hasArg(false).desc("Usage and help").build());
         options.addOption(Option.builder("i").longOpt("init").argName("init").hasArg(false).desc("Initialize repository").build());
         options.addOption(Option.builder("p").longOpt("project").argName("project").hasArg(true).desc("Project ID").build());
+        options.addOption(Option.builder("r").longOpt("report").argName("report").hasArg(true).desc("Generate report").build());
         options.addOption(Option.builder("t").longOpt("table").argName("table").hasArg(true).desc("Limit to specified table").build());
         options.addOption(Option.builder("v").longOpt("version").argName("version").hasArg(false).desc("Version").build());
 
@@ -291,6 +296,8 @@ private static CommandLine parseCommandLine(String[] args) {
             // Capture Argument Values
             batchParameter = (cmd.hasOption("batch")) ? Integer.parseInt(cmd.getOptionValue("batch")) : (System.getenv("PGCOMPARE-BATCH") == null) ? 0 : Integer.parseInt(System.getenv("PGCOMPARE-BATCH"));
             check = cmd.hasOption("check");
+            genReport = cmd.hasOption("report");
+            reportFileName = (cmd.hasOption("report")) ? cmd.getOptionValue("report") : "";
 
             // Determine the desired action, reconcile, discovery or init.
             //   reconcile = Perform comparison between source and target databases.
@@ -307,6 +314,17 @@ private static CommandLine parseCommandLine(String[] args) {
         }
     }
 
+    //
+    // Create Column Metadata JSON Object
+    //
+    private static JSONObject createReportColumn(String header, String key, String align, boolean commaFormat) {
+        return new JSONObject()
+                .put("columnHeader", header)
+                .put("columnClass", align)
+                .put("columnKey", key)
+                .put("commaFormat", commaFormat);
+    }
+
     //
     // Create Summary
     //
@@ -351,6 +369,58 @@ private static void createSummary(int tablesProcessed, JSONArray runResult, long
             printSummary(String.format("Total Out-of-Sync              = %s", df.format(outOfSyncRows)), 2);
             printSummary(String.format("Through-put (rows/per second)  = %s", df.format(totalRows / elapsedTime)), 2);
 
+            // Generate Report
+            if (genReport) {
+                // Create JSON report
+                JSONObject jobSummary = new JSONObject()
+                        .put("tablesProcessed", df.format(tablesProcessed))
+                        .put("elapsedTime", df.format(elapsedTime))
+                        .put("totalRows", df.format(totalRows))
+                        .put("outOfSyncRows", df.format(outOfSyncRows))
+                        .put("rowsPerSecond", df.format(totalRows / elapsedTime));
+
+
+                JSONArray jobSummaryLayout = new JSONArray(List.of(
+                        createReportColumn("Tables Processed", "tablesProcessed", "right-align", false),
+                        createReportColumn("Elapsed Time", "elapsedTime", "right-align", false),
+                        createReportColumn("Rows per Second", "rowsPerSecond", "right-align", false),
+                        createReportColumn("Total Rows", "totalRows", "right-align", false),
+                        createReportColumn("Out of Sync Rows", "outOfSyncRows", "right-align", false)
+                ));
+
+
+                JSONArray runResultLayout = new JSONArray(List.of(
+                        createReportColumn("Table", "tableName", "left-align", false),
+                        createReportColumn("Compare Status", "compareStatus", "left-align", false),
+                        createReportColumn("Elapsed Time", "elapsedTime", "right-align", true),
+                        createReportColumn("Rows per Second", "rowsPerSecond", "right-align", true),
+                        createReportColumn("Rows Total", "totalRows", "right-align", true),
+                        createReportColumn("Rows Equal", "equal", "right-align", true),
+                        createReportColumn("Rows Not Equal", "notEqual", "right-align", true),
+                        createReportColumn("Rows Missing on Source", "missingSource", "right-align", true),
+                        createReportColumn("Rows Missing on Target", "missingTarget", "right-align", true)
+                ));
+
+                JSONArray reportArray = new JSONArray()
+                        .put(createSection("Job Summary", new JSONArray().put(jobSummary), jobSummaryLayout)) // Pass JSONObject directly
+                        .put(createSection("Table Summary", runResult, runResultLayout)); // Pass runResult
+
+                if (check) {
+                    JSONArray runCheckResultLayout = new JSONArray(List.of(
+                            createReportColumn("Primary Key", "pk", "left-align", false),
+                            createReportColumn("Status", "compareStatus", "left-align", false),
+                            createReportColumn("Result", "compareResult", "left-align", false)
+                    ));
+
+                    for (int i =0; i<runResult.length(); i++ ) {
+                        reportArray.put(createSection(String.format("Table: %s", runResult.getJSONObject(i).getString("tableName")), runResult.getJSONObject(i).getJSONObject("checkResult").getJSONArray("data"), runCheckResultLayout));
+                    }
+                }
+
+                generateHtmlReport(reportArray, reportFileName,"pgCompare Summary");
+
+            }
+
         } else {
             // Print message if no tables processed or out of sync records found
             String msg = (check) ? "No out of sync records found" : "No tables were processed. Need to do discovery? Used correct batch nbr?";
@@ -375,7 +445,9 @@ public static void showHelp () {
         System.out.println("   -b|--batch <batch nbr>");
         System.out.println("   -c|--check Check out of sync rows");
         System.out.println("   -d|--discovery <schema> Discover tables in database");
+        System.out.println("   -i|--init <schema> Init tables in database");
         System.out.println("   -p|--project Project ID");
+        System.out.println("   -r|--report <file> Create html report of compare");
         System.out.println("   -t|--table <target table>");
         System.out.println("   --help");
         System.out.println();

src/main/java/com/crunchydata/services/Reporter.java

@@ -0,0 +1,78 @@
+package com.crunchydata.services;
+
+import com.crunchydata.util.Logging;
+import org.json.JSONArray;
+import org.json.JSONObject;
+
+import java.io.FileWriter;
+import java.text.DecimalFormat;
+
+public class Reporter {
+
+    public static JSONObject createSection(String title, Object data, JSONArray layout) {
+        return new JSONObject()
+                .put("title", title)
+                .put("data", data)
+                .put("layout", layout);
+    }
+
+    public static void generateHtmlReport(JSONArray report, String filePath, String title) {
+        Logging.write("info","main", String.format("Generating HTML report:  %s...", filePath));
+
+        DecimalFormat df = new DecimalFormat("###,###,###,###,###");
+
+        try {
+            FileWriter writer = new FileWriter(filePath);
+
+            writer.write(String.format("<html><head><title>%s</title>",title));
+            writer.write("""
+                <style>
+                    table {width: 100%; border-collapse: collapse;} th,
+                    td {border: 1px solid black; padding: 8px; text-align: left;}
+                    th {background-color: #f2f2f2;}
+                    .right-align { text-align: right; }
+                    .left-align  { text-align: left;}
+                    .center-align { text-align: center;}
+                </style>
+                """);
+            writer.write("</head><body>");
+
+            for (int i = 0; i < report.length(); i++) {
+                JSONObject section = report.getJSONObject(i);
+                JSONArray sectionData = section.getJSONArray("data");
+
+                writer.write("<div/>");
+                writer.write(String.format("<h2>%s</h2>", section.getString("title")));
+                writer.write("<table>");
+
+                // Header
+                writer.write("<tr>");
+
+                JSONObject firstObject = sectionData.getJSONObject(0);
+                for (int j =0; j < section.getJSONArray("layout").length(); j++) {
+                    JSONObject rowLayout = section.getJSONArray("layout").getJSONObject(j);
+                    writer.write(String.format("<th>%s</th>", rowLayout.getString("columnHeader")));
+                }
+                writer.write("</tr>");
+
+                for (int j = 0; j < sectionData.length(); j++) {
+                    JSONObject row = sectionData.getJSONObject(j);
+                    writer.write("<tr>");
+                    for (int jr =0; jr < section.getJSONArray("layout").length(); jr++) {
+                        JSONObject rowLayout = section.getJSONArray("layout").getJSONObject(jr);
+                        writer.write(String.format("<td class=\"%s\">%s</td>", rowLayout.getString("columnClass"), (rowLayout.getBoolean("commaFormat")) ? df.format(row.getInt(rowLayout.getString("columnKey"))) : row.get(rowLayout.getString("columnKey")).toString()));
+                    }
+                    writer.write("</tr>");
+                }
+                writer.write("</table>");
+            }
+
+            writer.write("</body></html>");
+            writer.close();
+        } catch (Exception e) {
+            StackTraceElement[] stackTrace = e.getStackTrace();
+            Logging.write("severe", "main", String.format("Error generating report at line %s:  %s", stackTrace[0].getLineNumber(), e.getMessage()));
+        }
+    }
+    
+}

src/main/java/com/crunchydata/services/dbCommon.java

@@ -35,9 +35,6 @@
 public class dbCommon {
     private static final String THREAD_NAME = "dbCommon";
 
-
-
-
     /**
      * Utility method to execute a provided SQL query and retrieve a list of tables.
      *

src/main/java/com/crunchydata/services/threadReconcileCheck.java

@@ -59,13 +59,14 @@ public class threadReconcileCheck {
      * @param ciTarget           Column metadata from target database.
      * @param cid                Identifier for the reconciliation process.
      */
-    public static void checkRows (Properties Props, Connection repoConn, Connection sourceConn, Connection targetConn, DCTable dct, DCTableMap dctmSource, DCTableMap dctmTarget, ColumnMetadata ciSource, ColumnMetadata ciTarget, Integer cid) {
+    public static JSONObject checkRows (Properties Props, Connection repoConn, Connection sourceConn, Connection targetConn, DCTable dct, DCTableMap dctmSource, DCTableMap dctmTarget, ColumnMetadata ciSource, ColumnMetadata ciTarget, Integer cid) {
         ArrayList<Object> binds = new ArrayList<>();
         JSONObject result = new JSONObject();
+        JSONArray rows = new JSONArray();
+
         StringBuilder tableFilter;
 
-        result.put("status","failed");
-        result.put("compareStatus","failed");
+        result.put("status","success");
 
         try {
             PreparedStatement stmt = repoConn.prepareStatement(SQL_REPO_SELECT_OUTOFSYNC_ROWS);
@@ -102,17 +103,24 @@ public static void checkRows (Properties Props, Connection repoConn, Connection
                 }
                 Logging.write("info", THREAD_NAME, String.format("Primary Key:  %s (WHERE = '%s')", pk, dctmSource.getTableFilter().substring(6)));
 
-                reCheck(repoConn, sourceConn, targetConn, dctmSource, dctmTarget, ciTarget.pkList, binds, dcRow, cid);
+                JSONObject recheckResult = reCheck(repoConn, sourceConn, targetConn, dctmSource, dctmTarget, ciTarget.pkList, binds, dcRow, cid);
 
+                if ( rows.length() < 1000 ) {
+                    rows.put(recheckResult);
+                }
             }
 
             rs.close();
             stmt.close();
+            result.put("data", rows);
+
         } catch (Exception e) {
+            result.put("status","failed");
             StackTraceElement[] stackTrace = e.getStackTrace();
             Logging.write("severe", THREAD_NAME, String.format("Error performing check of table %s at line %s:  %s", dct.getTableAlias(), stackTrace[0].getLineNumber(), e.getMessage()));
         }
 
+        return result;
     }
 
     /**
@@ -126,12 +134,13 @@ public static void checkRows (Properties Props, Connection repoConn, Connection
      * @param dcRow              DataCompare object with row to be compared.
      * @param cid                Identifier for the reconciliation process.
      */
-    public static void reCheck (Connection repoConn, Connection sourceConn, Connection targetConn, DCTableMap dctmSource, DCTableMap dctmTarget, String pkList, ArrayList<Object> binds, DataCompare dcRow, Integer cid) {
+    public static JSONObject reCheck (Connection repoConn, Connection sourceConn, Connection targetConn, DCTableMap dctmSource, DCTableMap dctmTarget, String pkList, ArrayList<Object> binds, DataCompare dcRow, Integer cid) {
         JSONArray arr = new JSONArray();
         int columnOutofSync = 0;
         JSONObject rowResult = new JSONObject();
 
         rowResult.put("compareStatus","in-sync");
+        rowResult.put("compareResult"," ");
         rowResult.put("equal",0);
         rowResult.put("notEqual",0);
         rowResult.put("missingSource",0);
@@ -141,12 +150,16 @@ public static void reCheck (Connection repoConn, Connection sourceConn, Connecti
         CachedRowSet targetRow = dbCommon.simpleSelect(targetConn, dctmTarget.getCompareSQL() + dctmTarget.getTableFilter(), binds);
 
         try {
+            rowResult.put("pk", dcRow.getPk());
+
             if (sourceRow.size() > 0 && targetRow.size() == 0) {
                 rowResult.put("compareStatus", "out-of-sync");
+                rowResult.put("compareResult", "Missing Target");
                 rowResult.put("missingTarget", 1);
                 rowResult.put("result", new JSONArray().put(0, "Missing Target"));
             } else if (targetRow.size() > 0 && sourceRow.size() == 0 ) {
                 rowResult.put("compareStatus", "out-of-sync");
+                rowResult.put("compareResult", "Missing Source");
                 rowResult.put("missingSource", 1);
                 rowResult.put("result", new JSONArray().put(0, "Missing Source"));
             } else {
@@ -179,6 +192,8 @@ public static void reCheck (Connection repoConn, Connection sourceConn, Connecti
 
                 if (columnOutofSync > 0) {
                     rowResult.put("compareStatus", "out-of-sync");
+                    rowResult.put("compareResult", arr.toString());
+                    rowResult.put("pk", dcRow.getPk());
                     rowResult.put("notEqual", 1);
                     rowResult.put("result", arr);
                 }
@@ -210,6 +225,8 @@ public static void reCheck (Connection repoConn, Connection sourceConn, Connecti
             Logging.write("severe", THREAD_NAME, String.format("Error comparing source and target values at line %s:  %s", stackTrace[0].getLineNumber(), e.getMessage()));
         }
 
+        return rowResult;
+
     }
 
 }