Notice table names that we don't handle yet

Adds a new exception to look for (a subclass of SQLException, so no interfaces
are changed yet), and stops trying to deal with tables that we don't handle (looking
at you, `repcap`).

Author: BryceStevenWilley

proxyserver/src/main/java/edu/suffolk/litlab/efsp/ecfcodes/CodeUpdater.java

@@ -2,6 +2,7 @@
 
 import edu.suffolk.litlab.efsp.db.DatabaseCreator;
 import edu.suffolk.litlab.efsp.ecfcodes.tyler.CodeDatabase;
+import edu.suffolk.litlab.efsp.ecfcodes.tyler.CodeTableConstants.UnsupportedTableException;
 import edu.suffolk.litlab.efsp.server.ecf4.Ecf4Helper;
 import edu.suffolk.litlab.efsp.server.ecf4.SoapClientChooser;
 import edu.suffolk.litlab.efsp.server.utils.HeaderSigner;
@@ -25,15 +26,14 @@
 import java.net.URL;
 import java.sql.SQLException;
 import java.sql.Savepoint;
-import java.time.Clock;
 import java.time.Duration;
 import java.time.Instant;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Optional;
-import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.function.Function;
 import java.util.function.Supplier;
@@ -176,19 +176,17 @@ public static InputStream getCodesZip(String toRead, String authHeader) throws I
    */
   private boolean downloadAndProcessZip(
       String toRead, String signedTime, Function<InputStream, Boolean> process) {
-    Instant startTable = Instant.now(Clock.systemUTC());
+    Instant startTable = Instant.now();
     try (InputStream urlStream = getCodesZip(toRead, signedTime)) {
       // Write out the zip file
-      downloadDuration =
-          downloadDuration.plus(Duration.between(startTable, Instant.now(Clock.systemUTC())));
+      downloadDuration = downloadDuration.plus(Duration.between(startTable, Instant.now()));
 
       ZipInputStream zip = new ZipInputStream(urlStream);
       zip.getNextEntry();
 
-      Instant updateTableLoc = Instant.now(Clock.systemUTC());
+      Instant updateTableLoc = Instant.now();
       boolean success = process.apply(zip);
-      updateDuration =
-          updateDuration.plus(Duration.between(updateTableLoc, Instant.now(Clock.systemUTC())));
+      updateDuration = updateDuration.plus(Duration.between(updateTableLoc, Instant.now()));
       zip.close();
       return success;
     } catch (IOException ex) {
@@ -234,6 +232,8 @@ private boolean downloadSystemTables(String baseUrl, CodeDatabaseAPI cd, HeaderS
     }
 
     for (Map.Entry<String, String> urlSuffix : codeUrls.entrySet()) {
+      // Let SQL exceptions through here; table names are hard coded, so if they break
+      // we need to know.
       cd.deleteFromTable(urlSuffix.getKey());
       final Function<InputStream, Boolean> process =
           (is) -> {
@@ -348,7 +348,7 @@ private boolean downloadCourtTables(
       throws JAXBException, IOException, SQLException {
     MDC.put(MDCWrappers.SESSION_ID, location);
     log.info("Doing updates for: {}, tables: {}", location, tables);
-    Instant downloadStart = Instant.now(Clock.systemUTC());
+    Instant downloadStart = Instant.now();
     // TODO(brycew-later): check that the effective date is later than today
     // JAXBElement<?> obj = ccl.getEffectiveDate().getDateRepresentation();
     Map<String, String> urlMap =
@@ -383,12 +383,12 @@ private boolean downloadCourtTables(
     }
     Map<String, DownloadedCodes> downloaded =
         streamDownload(signedTime.get(), location, toDownload.parallel(), tables);
-    var downloadInc = Duration.between(downloadStart, Instant.now(Clock.systemUTC()));
+    var downloadInc = Duration.between(downloadStart, Instant.now());
     downloadDuration = downloadDuration.plus(downloadInc);
     log.info(
         "Location: {}: Downloads took: {} (total: {})", location, downloadInc, downloadDuration);
 
-    Instant updateStart = Instant.now(Clock.systemUTC());
+    Instant updateStart = Instant.now();
     for (DownloadedCodes down : downloaded.values()) {
       MDC.put(MDCWrappers.REQUEST_ID, down.tableName());
       try {
@@ -400,7 +400,7 @@ private boolean downloadCourtTables(
         down.input().close();
       }
     }
-    var updateInc = Duration.between(updateStart, Instant.now(Clock.systemUTC()));
+    var updateInc = Duration.between(updateStart, Instant.now());
     updateDuration = updateDuration.plus(updateInc);
 
     cd.commit();
@@ -410,6 +410,45 @@ private boolean downloadCourtTables(
     return true;
   }
 
+  /**
+   * Creates all of the tables to update. If it can't create them, it removes them from the list.
+   */
+  private Map<String, List<String>> makeOrRemoveUnsupportedTables(
+      Map<String, List<String>> versionsToUpdate, CodeDatabaseAPI cd) throws SQLException {
+    var allTables = new HashSet<String>();
+    log.info("Making tables (if any are absent)");
+    for (var tables : versionsToUpdate.values()) {
+      allTables.addAll(tables);
+    }
+
+    var brokenTables = new HashSet<String>();
+    for (String table : allTables) {
+      try {
+        cd.createTableIfAbsent(table);
+      } catch (UnsupportedTableException ex) {
+        log.warn("Ignoring table {} from Tyler's report (not in our allow list)", table);
+        brokenTables.add(table);
+      }
+    }
+
+    var toReturn = new HashMap<String, List<String>>();
+    for (var courtAndTables : versionsToUpdate.entrySet()) {
+      String courtLocation = courtAndTables.getKey();
+      if (courtLocation.isBlank()) {
+        log.warn("Ignoring tables with an empty court!");
+        continue;
+      }
+      List<String> tables = courtAndTables.getValue();
+      tables.removeAll(brokenTables);
+      if (tables.isEmpty()) {
+        continue;
+      }
+      toReturn.put(courtLocation, tables);
+    }
+
+    return toReturn;
+  }
+
   /** Returns true if successful, false if not successful */
   public boolean updateAll(String baseUrl, FilingReviewMDEPort filingPort, CodeDatabaseAPI cd)
       throws SQLException, IOException, JAXBException {
@@ -424,49 +463,36 @@ public boolean updateAll(String baseUrl, FilingReviewMDEPort filingPort, CodeDat
 
     // Drop each of tables that need to be updated
     Savepoint sp = cd.setSavepoint("court update savepoint");
-    Map<String, List<String>> versionsToUpdate = cd.getVersionsToUpdate();
-    Instant startDel = Instant.now(Clock.systemUTC());
-    Set<String> allTables = new HashSet<>();
-    int n = 0;
-    log.info("Making tables if absent");
-    for (var tables : versionsToUpdate.values()) {
-      n += tables.size();
-      allTables.addAll(tables);
-    }
-    for (String table : allTables) {
-      cd.createTableIfAbsent(table);
-    }
-    log.info("Removing {} court entries, over {} queries", versionsToUpdate.size(), n);
+    Map<String, List<String>> rawVersionsToUpdate = cd.getVersionsToUpdate();
+    Map<String, List<String>> versionsToUpdate =
+        makeOrRemoveUnsupportedTables(rawVersionsToUpdate, cd);
+    Instant startDel = Instant.now();
+    log.info(
+        "Removing {} court entries, over {} queries",
+        versionsToUpdate.size(),
+        versionsToUpdate.values().stream().map(List::size).reduce(0, (a, b) -> a + b));
     for (Entry<String, List<String>> courtAndTables : versionsToUpdate.entrySet()) {
       final String courtLocation = courtAndTables.getKey();
-      if (courtLocation.isBlank()) {
-        log.warn("Ignoring tables with an empty court!");
-        continue;
-      }
       List<String> tables = courtAndTables.getValue();
       log.debug(
           "In {}, removing entries for court {} for tables: {}",
           cd.getDomain(),
           courtLocation,
           tables);
       for (String table : tables) {
-        Instant deleteFromTable = Instant.now(Clock.systemUTC());
+        Instant delTime = Instant.now();
 
-        // No longer checking for false here -- see PR.
         // Will ignore tables that don't exist.
         cd.deleteFromTable(table, courtLocation);
 
-        updateDuration =
-            updateDuration.plus(Duration.between(deleteFromTable, Instant.now(Clock.systemUTC())));
+        updateDuration = updateDuration.plus(Duration.between(delTime, Instant.now()));
       }
     }
-    log.info(
-        "Took {} to remove existing tables",
-        Duration.between(startDel, Instant.now(Clock.systemUTC())));
-    Instant startPolicy = Instant.now(Clock.systemUTC());
+    log.info("Took {} to remove existing tables", Duration.between(startDel, Instant.now()));
+    Instant startPolicy = Instant.now();
     Map<String, CourtPolicyResponseMessageType> policies =
         streamPolicies(versionsToUpdate.keySet().stream().parallel(), cd.getDomain(), filingPort);
-    var soapInc = Duration.between(startPolicy, Instant.now(Clock.systemUTC()));
+    var soapInc = Duration.between(startPolicy, Instant.now());
     soapDuration = soapDuration.plus(soapInc);
     log.info("Soaps took: {} (total: {})", soapInc, soapDuration);
 
@@ -517,10 +543,10 @@ public boolean replaceSome(
     }
     // Remove the "0" or top level court, which doesn't usually have individual court tables
     locs.remove("0");
-    Instant startPolicy = Instant.now(Clock.systemUTC());
+    Instant startPolicy = Instant.now();
     Map<String, CourtPolicyResponseMessageType> policies =
         streamPolicies(locs.stream(), cd.getDomain(), filingPort);
-    soapDuration = soapDuration.plus(Duration.between(startPolicy, Instant.now(Clock.systemUTC())));
+    soapDuration = soapDuration.plus(Duration.between(startPolicy, Instant.now()));
     log.info("Soaps: {}", soapDuration);
     for (var policy : policies.entrySet()) {
       final String location = policy.getKey();

proxyserver/src/main/java/edu/suffolk/litlab/efsp/ecfcodes/tyler/CodeDatabase.java

@@ -1,6 +1,7 @@
 package edu.suffolk.litlab.efsp.ecfcodes.tyler;
 
 import edu.suffolk.litlab.efsp.ecfcodes.CodeDatabaseAPI;
+import edu.suffolk.litlab.efsp.ecfcodes.tyler.CodeTableConstants.UnsupportedTableException;
 import edu.suffolk.litlab.efsp.stdlib.SQLFunction;
 import edu.suffolk.litlab.efsp.tyler.TylerEnv;
 import jakarta.xml.bind.JAXBException;
@@ -106,10 +107,6 @@ public void createTableIfAbsent(String tableName) throws SQLException {
           OptionalServiceCode.createFromOptionalServiceTable(conn);
         } else {
           String createQuery = CodeTableConstants.getCreateTable(tableName);
-          if (createQuery.isEmpty()) {
-            log.warn("Will not create table with name: {}", tableName);
-            return;
-          }
           try (Statement createSt = conn.createStatement()) {
             log.info("Full statement: {}", createQuery);
             createSt.executeUpdate(createQuery);
@@ -190,6 +187,9 @@ public void updateTable(
       update.setString(4, tylerDomain);
       update.setString(5, newVersion);
       update.executeUpdate();
+    } catch (UnsupportedTableException ex) {
+      log.error("Silently ignoring a missing table exception.", ex);
+      return;
     } catch (SQLException ex) {
       log.error("Tried to execute an insert, but failed! Exception", ex);
       log.error("Going to rollback updates to this table");

proxyserver/src/main/java/edu/suffolk/litlab/efsp/ecfcodes/tyler/CodeTableConstants.java

@@ -1,5 +1,6 @@
 package edu.suffolk.litlab.efsp.ecfcodes.tyler;
 
+import java.sql.SQLException;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
@@ -451,15 +452,22 @@ public static String vacuumAnalyzeAll() {
     return "VACUUM ANALYZE";
   }
 
+  /** This exception is returned when a given table name isn't in the pre-approved list of names. */
+  public static class UnsupportedTableException extends SQLException {
+    public UnsupportedTableException(String message) {
+      super(message);
+    }
+  }
+
   public static boolean tableHasLocation(String tableName) {
     return tableColumns.containsKey(tableName) && tableColumns.get(tableName).needsExtraLocCol;
   }
 
-  public static String getCreateTable(String tableName) {
+  public static String getCreateTable(String tableName) throws UnsupportedTableException {
     if (createQueries.containsKey(tableName)) {
       return createQueries.get(tableName);
     }
-    return "";
+    throw new UnsupportedTableException("No create query for table " + tableName);
   }
 
   public static List<String> getCreateIndex(String tableName) {
@@ -469,23 +477,23 @@ public static List<String> getCreateIndex(String tableName) {
     return tableIndices.get(tableName);
   }
 
-  public static String getInsertInto(String tableName) {
+  public static String getInsertInto(String tableName) throws UnsupportedTableException {
     if (!insertQueries.containsKey(tableName)) {
-      return "";
+      throw new UnsupportedTableException("No insert query for table " + tableName);
     }
     return insertQueries.get(tableName);
   }
 
-  public static String getDeleteFrom(String tableName) {
+  public static String getDeleteFrom(String tableName) throws UnsupportedTableException {
     if (!deleteFromQueries.containsKey(tableName)) {
-      return "";
+      throw new UnsupportedTableException("No 'delete from' query for table " + tableName);
     }
     return deleteFromQueries.get(tableName);
   }
 
-  public static String getDeleteAllCourtsFrom(String tableName) {
+  public static String getDeleteAllCourtsFrom(String tableName) throws UnsupportedTableException {
     if (!deleteAllCourtsFromQueries.containsKey(tableName)) {
-      return "";
+      throw new UnsupportedTableException("No 'delete all courts from' query for table " + tableName);
     }
     return deleteAllCourtsFromQueries.get(tableName);
   }