fix: upload timeout UX, rate limit, specimen warning, deploy pipeline & struts config

.github/scripts/deploy.sh

@@ -7,8 +7,9 @@ cd $HOME/antweb
 
 # Get the latest source code
 git checkout master
-git pull origin master --no-edit
+git pull --rebase origin master
 
 # Restart docker compose services
-docker-compose exec antweb ant deploy
-docker-compose restart antweb
+# -T disables pseudo-TTY allocation (required when running non-interactively via SSH/CI)
+docker compose exec -T antweb ant deploy || docker-compose exec -T antweb ant deploy
+docker compose restart antweb || docker-compose restart antweb

.github/workflows/tests.yaml

@@ -20,10 +20,8 @@ jobs:
 
       - name: Test API Build
         run: |
-          cd docker/api
-          docker build -t antweb/api:dev .
+          docker build -t antweb/api:dev -f docker/api/Dockerfile .
 
       - name: Test Antweb Build
         run: |
-          cd docker/api
-          docker build -t antweb/antweb:dev .
+          docker build -t antweb/antweb:dev -f docker/antweb/Dockerfile .

WEB-INF/struts-configDb.xml

@@ -0,0 +1,99 @@
+<?xml version="1.0" encoding="ISO-8859-1" ?>
+
+<!-- 
+On dev and live server there is a softlink created pointing to the DB config file.
+
+root@antweb:~/antweb# ls -al WEB-INF/struts-configDb*
+lrwxrwxrwx 1 root root   22 Dec 18  2021 WEB-INF/struts-configDb.xml -> struts-configDbAnt.xml
+-rwxr-xr-x 1 root root 6958 Aug 12  2022 WEB-INF/struts-configDbAnt.xml
+
+struts-configDBAnt.xml is the file to be modified - for use on antweb-dev, antweb-stg and antweb-prod.
+
+For local one could use a different struts-configDb.xml.
+
+Another connection is defined in DBSimpleUtil.getDataSource()
+-->
+
+<!DOCTYPE struts-config PUBLIC
+          "-//Apache Software Foundation//DTD Struts Configuration 1.1//EN"
+          "http://jakarta.apache.org/struts/dtds/struts-config_1_1.dtd">
+
+<struts-config>
+  <data-sources>
+
+    <data-source key="conPool" type="com.mchange.v2.c3p0.ComboPooledDataSource">    <!-- add  destroy-method="close" ? -->
+      <set-property property="factory" value="org.apache.naming.factory.BeanFactory"/>
+      <set-property property="auth" value="Container"/>
+      <set-property property="maxPoolSize" value="140"/> <!-- was 15 --> <!-- was 200 --> <!-- was 200 -->
+      <set-property property="maxIdleTime" value="10"/>
+      <set-property property="user" value="antweb"/>
+      <set-property property="password" value="f0rm1c6"/>
+      <set-property property="driverClass" value="com.mysql.cj.jdbc.Driver" />   <!-- was driverClassName-->
+      <set-property property="jdbcUrl" value="jdbc:mysql://mysql:3306/ant?autoReconnect=true&amp;useUnicode=true&amp;characterEncoding=UTF-8&amp;characterSetResults=utf8&amp;connectionCollation=utf8_general_ci&amp;useSSL=false&amp;serverTimezone=UTC"/>
+      <set-property property="name" value="jdbc/antDb"/> <!-- not used elsewhere -->
+      <set-property property="testConnectionOnCheckout" value="true"/>
+      <set-property property="autoCommitOnClose" value="true"/>
+
+      <!-- See DBUtil.java for documentation -->
+      <set-property property="unreturnedConnectionTimeout" value="200"/>  <!-- was 600 == 10 min -->
+      <set-property property="debugUnreturnedConnectionStackTraces" value="false"/>
+
+      <set-property property="testConnectionOnCheckin" value="true"/> 
+      <set-property property="idleConnectionTestPeriod" value="60"/> 
+      <set-property property="maxIdleTimeExcessConnections" value="60"/>
+    </data-source>
+
+    <data-source key="mediumConPool" type="com.mchange.v2.c3p0.ComboPooledDataSource">    <!-- add  destroy-method="close" ? -->
+      <set-property property="factory" value="org.apache.naming.factory.BeanFactory"/>
+      <set-property property="auth" value="Container"/>
+      <set-property property="maxPoolSize" value="6"/> <!-- was 400 -->
+      <!-- set-property property="maxIdleTime" value="10"/ -->
+      <set-property property="user" value="antweb"/>
+      <set-property property="password" value="f0rm1c6"/>
+      <set-property property="driverClass" value="com.mysql.cj.jdbc.Driver" />   <!-- was driverClassName-->
+      <set-property property="jdbcUrl" value="jdbc:mysql://mysql:3306/ant?autoReconnect=true&amp;useUnicode=true&amp;characterEncoding=UTF-8&amp;characterSetResults=utf8&amp;connectionCollation=utf8_general_ci&amp;useSSL=false&amp;serverTimezone=UTC"/>
+      <set-property property="name" value="jdbc/antDb"/> <!-- not used elsewhere -->
+      <set-property property="testConnectionOnCheckout" value="true"/>
+      <set-property property="autoCommitOnClose" value="true"/>
+
+      <!-- See DBUtil.java for documentation -->
+      <set-property property="unreturnedConnectionTimeout" value="600"/>
+      <set-property property="debugUnreturnedConnectionStackTraces" value="false"/>
+
+      <set-property property="testConnectionOnCheckin" value="true"/> 
+      <set-property property="idleConnectionTestPeriod" value="300"/> 
+      <set-property property="maxIdleTimeExcessConnections" value="240"/>
+    </data-source>
+
+    <data-source key="longConPool" type="com.mchange.v2.c3p0.ComboPooledDataSource">    <!-- add  destroy-method="close" ? -->
+      <set-property property="factory" value="org.apache.naming.factory.BeanFactory"/>
+      <set-property property="auth" value="Container"/>
+      <set-property property="maxPoolSize" value="5"/> <!-- was 15 -->
+      <!-- set-property property="maxIdleTime" value="10"/ -->
+      <set-property property="user" value="antweb"/>
+      <set-property property="password" value="f0rm1c6"/>
+      <set-property property="driverClass" value="com.mysql.cj.jdbc.Driver" />   <!-- was com.mysql.jdbc.Driver driverClassName-->
+      <set-property property="jdbcUrl" value="jdbc:mysql://mysql:3306/ant?autoReconnect=true&amp;useUnicode=true&amp;characterEncoding=UTF-8&amp;characterSetResults=utf8&amp;connectionCollation=utf8_general_ci&amp;useSSL=false&amp;serverTimezone=UTC"/>
+      <set-property property="name" value="jdbc/antDb"/> <!-- not used elsewhere -->
+      <set-property property="testConnectionOnCheckout" value="true"/>
+      <set-property property="autoCommitOnClose" value="true"/> 
+
+      <!-- See DBUtil.java for documentation -->
+      <set-property property="unreturnedConnectionTimeout" value="3600"/> <!-- 4200 / 60 is 70 min -->  <!-- 10800 / 60 is 3 hours -->
+      <set-property property="debugUnreturnedConnectionStackTraces" value="false"/>
+
+      <set-property property="testConnectionOnCheckin" value="true"/> 
+      <set-property property="idleConnectionTestPeriod" value="300"/>
+      <set-property property="maxIdleTimeExcessConnections" value="240"/>
+
+    </data-source>
+
+<!--  
+Does not work:
+      <set-property property="zeroDateTimeBehavior" value="convertToNull"/>
+      <set-property property="jdbcUrl" value="jdbc:mysql://mysql:3306/ant?autoReconnect=true&amp;characterEncoding=utf-8"/>
+-->
+
+  </data-sources>
+
+</struts-config>

docker-compose.yml

@@ -67,7 +67,7 @@ services:
     expose:
       - 8009
     ports:
-      - "127.0.0.1:8080:8080"   # expose manager interface to local machine for monitoring
+      - "127.0.0.1:8081:8080"   # expose manager interface to local machine for monitoring
     environment:
       CATALINA_OPTS: "-XX:MaxRAMPercentage=40 -server -XX:+UseParallelGC"
     extra_hosts:

docker/caddy/Caddyfile

@@ -54,7 +54,10 @@ www.antweb.org {
 
         error "Not found" 404
 
-        rate_limit {
+        @not_upload {
+            not path /upload.do
+        }
+        rate_limit @not_upload {
             zone requests {
                 key {http.request.remote.host}
                 window 1m

src/org/calacademy/antweb/upload/MessageMgr.java

@@ -159,6 +159,7 @@ private ArrayList<Test> getTests() {
     public static final String nonAntTaxa = "nonAntTaxa";
     public static final String invalidSpeciesName = "invalidSpeciesName";
     public static final String specialCharacterFound = "specialCharacterFound";
+    public static final String specimenIdSpacesRemoved = "specimenIdSpacesRemoved";
     // public static final String  = "";
 
 
@@ -232,6 +233,7 @@ public void init() {
       testList.add(new Test(invalidSpeciesName, SET, "<b>Invalid species name <font color=red>(not uploaded)</font></b>"));
 
       testList.add(new Test(specialCharacterFound, SET, "<b>Special Character Found <font color=red>(not uploaded)</font></b>"));
+      testList.add(new Test(specimenIdSpacesRemoved, SET, "<b>Spaces removed from specimen code (auto-corrected — please fix your source data)</b>"));
     }      
 
     public static String getMessageDisplay(String key) {

src/org/calacademy/antweb/upload/SpecimenUploadParse.java

@@ -153,8 +153,10 @@ protected String parseLine(String theLine, int lineNum, Hashtable specimenItem,
                         if (col.equals("code")) {
                            String lowerCode = element.toLowerCase();
                            String newCode = new Formatter().removeSpaces(lowerCode);
+                           if (!lowerCode.equals(newCode)) {
+                               getMessageMgr().addToMessages(MessageMgr.specimenIdSpacesRemoved, newCode);
+                           }
                            code = newCode;
-                           //if (!lowerCode.equals(newCode)) ++spacesRemoved;
                            specimenItem.put(col, newCode);
                            //A.log("parseLine() code col:" + col + " element:" + element);
                         } else {

web/curate/curate-body.jsp

@@ -145,6 +145,9 @@ Need Help? Check out the <a href="<%= domainApp %>/documentation.do" target="new
         <!-- Antweb, TaxonWorks, or GBIF Specimen (file or Zip File) Upload -->
 
         <script type="text/javascript">
+            var uploadReportUrl = '<%= domainApp %>/listSpecimenUploads.do?groupId=<%= accessGroup.getId() %>';
+            var uploadTimeoutId = null;
+
             function validateZipUpload(form) {
                 var fileInput = form.elements['theFile'];
                 if (!fileInput || !fileInput.value) {
@@ -155,6 +158,17 @@ Need Help? Check out the <a href="<%= domainApp %>/documentation.do" target="new
                     alert('Error: Only ZIP files are allowed.');
                     return false;
                 }
+                document.getElementById('upload-submit-btn').disabled = true;
+                document.getElementById('upload-processing-msg').style.display = 'block';
+                // Cloudflare times out at 100s — redirect to upload report after 85s
+                // The upload continues processing server-side even after a Cloudflare timeout
+                uploadTimeoutId = setTimeout(function() {
+                    document.getElementById('upload-processing-msg').innerHTML =
+                        '<b>This is taking a while.</b> Your data is still being processed on the server.<br>' +
+                        'You will be redirected to your <a href="' + uploadReportUrl + '">Upload Report</a> shortly, ' +
+                        'where you can check the result.';
+                    setTimeout(function() { window.location.href = uploadReportUrl; }, 5000);
+                }, 85000);
                 return true;
             }
         </script>
@@ -167,10 +181,15 @@ Need Help? Check out the <a href="<%= domainApp %>/documentation.do" target="new
                     <html:file property="theFile" accept=".zip" />
                 </div>
                 <div class="clear"></div>
-
+
+                <div id="upload-processing-msg" style="display:none; color: blue; font-weight: bold; margin: 10px 0;">
+                    Uploading and processing your file&hellip; please wait.
+                </div>
+
                 <div style="color: red; font-weight: bold; margin: 10px 0;">
                     1. Only ZIP Files Allowed<br>
-                    2. If you see Cloudflare timeout just go back to the homepage.
+                    2. If you see a Cloudflare timeout, your data is still processing &mdash;
+                       check your <a href="<%= domainApp %>/listSpecimenUploads.do?groupId=<%= accessGroup.getId() %>">Upload Report</a>.
                 </div>
 
             <%
@@ -223,7 +242,7 @@ Need Help? Check out the <a href="<%= domainApp %>/documentation.do" target="new
              <div class="clear"></div>
 
                 <div class="align_right">
-                    <input border="0" type="image" src="<%= domainApp %>/image/grey_submit.png" width="77" height="23" value="Submit" <%= active %>>
+                    <input id="upload-submit-btn" border="0" type="image" src="<%= domainApp %>/image/grey_submit.png" width="77" height="23" value="Submit" <%= active %>>
                 </div>
 
              <div class="clear"></div>