fix: Add species list validation tool #33

WEB-INF/struts-config.xml

@@ -222,6 +222,8 @@
       <form-property name="toSpecies"     type="java.lang.String"/>
       <form-property name="toSubspecies"     type="java.lang.String"/>
   </form-bean>  
+
+  <form-bean name="validateSpeciesListForm" type="org.calacademy.antweb.curate.speciesList.ValidateSpeciesListForm"/>
 
   </form-beans>
 
@@ -610,6 +612,15 @@ We would like to remove getChildImages.  Not needed now in the new UI
       <forward name="success" path="/uploadHistory.jsp" />
     </action>
 
+    <action path="/validateSpeciesList"
+            type="org.calacademy.antweb.curate.speciesList.ValidateSpeciesListAction"
+            scope="request"
+            validate="false"
+            input="/curate/speciesList/validateSpeciesList.jsp"
+            name="validateSpeciesListForm">
+      <forward name="validateSpeciesList" path="/curate/speciesList/validateSpeciesList.jsp" />
+    </action>
+
     <action path="/speciesListTool"
             type="org.calacademy.antweb.curate.speciesList.SpeciesListToolAction"
             scope="request"

src/org/calacademy/antweb/ValidationParseException.java

@@ -0,0 +1,7 @@
+package org.calacademy.antweb;
+
+public class ValidationParseException extends Exception {
+    public ValidationParseException(String message) {
+        super(message);
+    }
+}

src/org/calacademy/antweb/curate/speciesList/ValidateSpeciesListAction.java

@@ -0,0 +1,113 @@
+package org.calacademy.antweb.curate.speciesList;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.sql.Connection;
+import java.sql.SQLException;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import javax.sql.DataSource;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.struts.action.ActionForm;
+import org.apache.struts.action.ActionForward;
+import org.apache.struts.action.ActionMapping;
+import org.apache.struts.upload.FormFile;
+import org.calacademy.antweb.util.DBUtil;
+import org.calacademy.antweb.util.Check;
+import org.calacademy.antweb.ValidationParseException;
+
+public class ValidateSpeciesListAction extends SpeciesListSuperAction {
+
+    private static final Log s_log = LogFactory.getLog(ValidateSpeciesListAction.class);
+
+    @Override
+    public ActionForward execute(ActionMapping mapping, ActionForm form,
+            HttpServletRequest request, HttpServletResponse response) {
+
+        ActionForward loginCheck = Check.login(request, mapping);
+        if (loginCheck != null) return loginCheck;
+
+        ValidateSpeciesListForm toolForm = (ValidateSpeciesListForm) form;
+        FormFile uploadedFile = toolForm.getFile();
+
+        Connection connection = null;
+        try {
+            // Check downloads first, since they won't have the uploaded file!
+            if ("download".equals(toolForm.getAction()) || "downloadCorrected".equals(toolForm.getAction())) {
+                ValidateSpeciesReport cachedReport = (ValidateSpeciesReport) request.getSession().getAttribute("validationReport");
+                if (cachedReport != null) {
+                    response.setContentType("text/tab-separated-values");
+                    if ("downloadCorrected".equals(toolForm.getAction())) {
+                        response.setHeader("Content-Disposition", "attachment; filename=\"corrected_species_list_for_upload.tsv\"");
+                        response.getWriter().write(cachedReport.generateCorrectedTsvReport());
+                    } else {
+                        response.setHeader("Content-Disposition", "attachment; filename=\"validation_report.tsv\"");
+                        response.getWriter().write(cachedReport.generateTsvReport());
+                    }
+                    return null;
+                } else {
+                    request.setAttribute("message", "Session expired or no report found. Please re-validate your file.");
+                    return mapping.findForward("validateSpeciesList");
+                }
+            }
+
+            // Initial render or empty submission
+            if (uploadedFile == null || uploadedFile.getFileSize() == 0) {
+                return mapping.findForward("validateSpeciesList");
+            }
+
+            String filename = uploadedFile.getFileName().toLowerCase();
+            if (!filename.endsWith(".txt") && !filename.endsWith(".tsv")) {
+                request.setAttribute("message", "File must be a tab-delimited .txt or .tsv file.");
+                return mapping.findForward("validateSpeciesList");
+            }
+
+            // 5MB max
+            if (uploadedFile.getFileSize() > (5 * 1024 * 1024)) {
+                request.setAttribute("message", "File is too large. Maximum size is 5MB (approx 50,000 rows).");
+                return mapping.findForward("validateSpeciesList");
+            }
+
+            InputStream fileStream = uploadedFile.getInputStream();
+
+            DataSource ds = getDataSource(request, "longConPool");
+            connection = DBUtil.getConnection(ds, "ValidateSpeciesListAction.execute()");
+            if (connection == null) {
+                request.setAttribute("message", "Could not obtain database connection.");
+                return mapping.findForward("validateSpeciesList");
+            }
+
+            // Enforce strictly read-only mode to prevent ALL data modifications.
+            connection.setReadOnly(true);
+
+            SpeciesListValidator validator = new SpeciesListValidator(connection);
+            ValidateSpeciesReport report = validator.validate(fileStream, "UTF-8", toolForm.isShowUnmatched());
+
+            request.getSession().setAttribute("validationReport", report);
+            request.setAttribute("validationReport", report);
+            return mapping.findForward("validateSpeciesList");
+
+        } catch (ValidationParseException e) {
+            s_log.warn("ValidateSpeciesListAction parse error: " + e.getMessage());
+            request.setAttribute("message", e.getMessage());
+            return mapping.findForward("validateSpeciesList");
+        } catch (SQLException e) {
+            s_log.error("ValidateSpeciesListAction DB error: " + e);
+            request.setAttribute("message", "Database error occurred during validation: " + e.getMessage());
+            return mapping.findForward("validateSpeciesList");
+        } catch (IOException e) {
+            s_log.error("ValidateSpeciesListAction IO error: " + e);
+            request.setAttribute("message", "Error reading uploaded file: " + e.getMessage());
+            return mapping.findForward("validateSpeciesList");
+        } finally {
+            if (connection != null) {
+                try { 
+                    connection.setReadOnly(false); // Reset to default pool state
+                } catch (SQLException e) { s_log.error("Failed to reset connection readOnly status", e); }
+            }
+            DBUtil.close(connection, this, "ValidateSpeciesListAction.execute()");
+        }
+    }
+}

src/org/calacademy/antweb/curate/speciesList/ValidateSpeciesListForm.java

@@ -0,0 +1,43 @@
+package org.calacademy.antweb.curate.speciesList;
+
+import javax.servlet.http.HttpServletRequest;
+import org.apache.struts.action.ActionForm;
+import org.apache.struts.action.ActionMapping;
+import org.apache.struts.upload.FormFile;
+
+public class ValidateSpeciesListForm extends ActionForm {
+    private FormFile file;
+    private String action;
+    private boolean showUnmatched;
+
+    public FormFile getFile() {
+        return file;
+    }
+
+    public void setFile(FormFile file) {
+        this.file = file;
+    }
+
+    public String getAction() {
+        return action;
+    }
+
+    public void setAction(String action) {
+        this.action = action;
+    }
+
+    public boolean isShowUnmatched() {
+        return showUnmatched;
+    }
+
+    public void setShowUnmatched(boolean showUnmatched) {
+        this.showUnmatched = showUnmatched;
+    }
+
+    @Override
+    public void reset(ActionMapping mapping, HttpServletRequest request) {
+        this.file = null;
+        this.action = null;
+        this.showUnmatched = false;
+    }
+}

src/org/calacademy/antweb/curate/speciesList/ValidateSpeciesReport.java

@@ -0,0 +1,110 @@
+package org.calacademy.antweb.curate.speciesList;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class ValidateSpeciesReport {
+    private final List<ValidateSpeciesResultItem> exactMatches = new ArrayList<>();
+    private final List<ValidateSpeciesResultItem> problems = new ArrayList<>();
+    private final List<ValidateSpeciesResultItem> formatErrors = new ArrayList<>();
+
+    // For "Show Unmatched" feature
+    private final List<String> unmatchedValidTaxa = new ArrayList<>();
+
+    private int totalInputRows = 0;
+
+    // Limits
+    private boolean rowLimitExceeded = false;
+
+    public void addResult(ValidateSpeciesResultItem item) {
+        totalInputRows++;
+        if (item.getStatus() == ValidateSpeciesResultItem.Status.EXACT_MATCH) {
+            exactMatches.add(item);
+        } else if (item.getStatus() == ValidateSpeciesResultItem.Status.FORMAT_ERROR) {
+            formatErrors.add(item);
+        } else {
+            problems.add(item);
+        }
+    }
+
+    public void addUnmatchedValidTaxon(String taxonName) {
+        this.unmatchedValidTaxa.add(taxonName);
+    }
+
+    public List<ValidateSpeciesResultItem> getExactMatches() { return exactMatches; }
+    public List<ValidateSpeciesResultItem> getProblems() { return problems; }
+    public List<ValidateSpeciesResultItem> getFormatErrors() { return formatErrors; }
+    public List<String> getUnmatchedValidTaxa() { return unmatchedValidTaxa; }
+
+    public int getTotalInputRows() { return totalInputRows; }
+    public int getExactMatchCount() { return exactMatches.size(); }
+    public int getProblemCount() { return problems.size(); }
+    public int getFormatErrorCount() { return formatErrors.size(); }
+    public int getUnmatchedCount() { return unmatchedValidTaxa.size(); }
+
+    public void setRowLimitExceeded(boolean exceeded) { this.rowLimitExceeded = exceeded; }
+    public boolean isRowLimitExceeded() { return rowLimitExceeded; }
+
+    public String generateTsvReport() {
+        StringBuilder sb = new StringBuilder();
+        sb.append("Row\tInput Raw\tNormalized Taxon Name\tStatus\tMessage\tSuggestion\n");
+
+        List<ValidateSpeciesResultItem> all = new ArrayList<>();
+        all.addAll(formatErrors);
+        all.addAll(problems);
+        all.addAll(exactMatches);
+
+        // Sort by row number
+        all.sort((a, b) -> Integer.compare(a.getRowNum(), b.getRowNum()));
+
+        for (ValidateSpeciesResultItem item : all) {
+            // Sanitize rawLine: replace embedded tabs to avoid corrupting TSV columns
+            String safeRaw = item.getInputRaw().replace("\t", "  ");
+            sb.append(item.getRowNum()).append("\t")
+              .append(safeRaw).append("\t")
+              .append(item.getNormalizedName()).append("\t")
+              .append(item.getStatus().name()).append("\t")
+              .append(item.getMessage()).append("\t")
+              .append(item.getSuggestion()).append("\n");
+        }
+
+        if (!unmatchedValidTaxa.isEmpty()) {
+            sb.append("\n\n--- UNMATCHED VALID ANTWEB TAXA ---\n");
+            sb.append("Taxon Name\n");
+            for (String t : unmatchedValidTaxa) {
+                sb.append(t).append("\n");
+            }
+        }
+
+        return sb.toString();
+    }
+
+    public String generateCorrectedTsvReport() {
+        StringBuilder sb = new StringBuilder();
+        sb.append("taxon_name\n");
+
+        List<ValidateSpeciesResultItem> all = new ArrayList<>();
+        all.addAll(formatErrors);
+        all.addAll(problems);
+        all.addAll(exactMatches);
+
+        // Sort by row number
+        all.sort((a, b) -> Integer.compare(a.getRowNum(), b.getRowNum()));
+
+        for (ValidateSpeciesResultItem item : all) {
+            String taxonStr = "";
+            if (item.getStatus() == ValidateSpeciesResultItem.Status.EXACT_MATCH) {
+                taxonStr = item.getNormalizedName();
+            } else if (item.getSuggestion() != null && !item.getSuggestion().isEmpty()
+                       && Character.isUpperCase(item.getSuggestion().charAt(0))) {
+                // Only include suggestions that look like actual taxon names (start with uppercase genus).
+                // Excludes messages like "Check spelling.", "No valid species found...", etc.
+                taxonStr = item.getSuggestion();
+            }
+            if (!taxonStr.isEmpty()) {
+                sb.append(taxonStr).append("\n");
+            }
+        }
+        return sb.toString();
+    }
+}

src/org/calacademy/antweb/curate/speciesList/ValidateSpeciesResultItem.java

@@ -0,0 +1,28 @@
+package org.calacademy.antweb.curate.speciesList;
+
+public final class ValidateSpeciesResultItem {
+    public enum Status { EXACT_MATCH, NOT_FOUND, FORMAT_ERROR, AMBIGUOUS }
+
+    private final int rowNum;
+    private final String inputRaw;
+    private final String normalizedName;
+    private final Status status;
+    private final String message;
+    private final String suggestion;
+
+    public ValidateSpeciesResultItem(int rowNum, String inputRaw, String normalizedName, Status status, String message, String suggestion) {
+        this.rowNum = rowNum;
+        this.inputRaw = inputRaw != null ? inputRaw : "";
+        this.normalizedName = normalizedName != null ? normalizedName : "";
+        this.status = status;
+        this.message = message != null ? message : "";
+        this.suggestion = suggestion != null ? suggestion : "";
+    }
+
+    public int getRowNum() { return rowNum; }
+    public String getInputRaw() { return inputRaw; }
+    public String getNormalizedName() { return normalizedName; }
+    public Status getStatus() { return status; }
+    public String getMessage() { return message; }
+    public String getSuggestion() { return suggestion; }
+}

web/curate/curate-body.jsp

@@ -489,6 +489,23 @@ Upload Curator File
 <!-- Projects -->                
 <!-- Download Species List -->   
 
+<% if (accessLogin.isCurator()) { %>
+    <div class="admin_action_module">
+        <div class="admin_action_item">
+            <div style="float:left;">
+                <h2>Read-Only Tools</h2>
+            </div>
+            <div class="clear"></div>
+        </div>
+        <div class="admin_action_item">
+          <div class="action_desc">
+            &nbsp;&nbsp;&nbsp;<a href="<%= AntwebProps.getDomainApp() %>/validateSpeciesList.do">Validate Species List (Pre-flight checker)</a>
+          </div>  
+          <div class="clear"></div>
+        </div>
+    </div>
+<% } %>
+
 <% if (accessLogin.isDeveloper()) { %>  <!-- was isAdmin() -->
 
         <div class="admin_action_item">