Re-gen xref if missing. Fix list.jsp w.r.t. isGenerateHtml().

This also fixes #2092, since stale xrefs will be
ignored and instead the latest content will be
xrefed anew.

Also fixes #2316

Author: idodeclare

opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/AnalyzerGuru.java

@@ -700,6 +700,16 @@ public static Genre getGenre(FileAnalyzerFactory factory) {
         return null;
     }
 
+    /**
+     * Finds a {@code FileAnalyzerFactory} for the specified
+     * {@link FileAnalyzer#getFileTypeName()}.
+     * @param fileTypeName a defined instance
+     * @return a defined instance or {@code null}
+     */
+    public static FileAnalyzerFactory findByFileTypeName(String fileTypeName) {
+        return FILETYPE_FACTORIES.get(fileTypeName);
+    }
+
     /**
      * Find a {@code FileAnalyzerFactory} with the specified class name. If one
      * doesn't exist, create one and register it. Allow specification of either

opengrok-indexer/src/main/java/org/opengrok/indexer/index/IndexDatabase.java

@@ -87,6 +87,7 @@
 import org.opengrok.indexer.analysis.Definitions;
 import org.opengrok.indexer.analysis.FileAnalyzer;
 import org.opengrok.indexer.analysis.FileAnalyzer.Genre;
+import org.opengrok.indexer.analysis.FileAnalyzerFactory;
 import org.opengrok.indexer.configuration.Project;
 import org.opengrok.indexer.configuration.RuntimeEnvironment;
 import org.opengrok.indexer.history.HistoryException;
@@ -313,9 +314,7 @@ private void initialize() throws IOException {
             ignoredNames = env.getIgnoredNames();
             includedNames = env.getIncludedNames();
             analyzerGuru = new AnalyzerGuru();
-            if (env.isGenerateHtml()) {
-                xrefDir = new File(env.getDataRootFile(), XREF_DIR);
-            }
+            xrefDir = new File(env.getDataRootFile(), XREF_DIR);
             listeners = new CopyOnWriteArrayList<>();
             dirtyFile = new File(indexDir, "dirty");
             dirty = dirtyFile.exists();
@@ -677,17 +676,17 @@ private void setDirty() {
         }
     }
 
+    private File whatXrefFile(String path, boolean compress) {
+        return new File(xrefDir, path + (compress ? ".gz" : ""));
+    }
+
     /**
      * Queue the removal of xref file for given path
      * @param path path to file under source root
      */
     private void removeXrefFile(String path) {
-        File xrefFile;
-        if (RuntimeEnvironment.getInstance().isCompressXref()) {
-            xrefFile = new File(xrefDir, path + ".gz");
-        } else {
-            xrefFile = new File(xrefDir, path);
-        }
+        RuntimeEnvironment env = RuntimeEnvironment.getInstance();
+        File xrefFile = whatXrefFile(path, env.isCompressXref());
         PendingFileDeletion pending = new PendingFileDeletion(
             xrefFile.getAbsolutePath());
         completer.add(pending);
@@ -750,7 +749,7 @@ private void addFile(File file, String path, Ctags ctags)
         fa.setFoldingEnabled(RuntimeEnvironment.getInstance().isFoldingEnabled());
 
         Document doc = new Document();
-        try (Writer xrefOut = getXrefWriter(fa, path)) {
+        try (Writer xrefOut = newXrefWriter(fa, path)) {
             analyzerGuru.populateDocument(doc, file, path, fa, xrefOut);
         } catch (InterruptedException e) {
             LOGGER.log(Level.WARNING, "File ''{0}'' interrupted--{1}",
@@ -1076,6 +1075,8 @@ private void indexDown(File dir, String parent, IndexDownArgs args)
         }
         Arrays.sort(files, FILENAME_COMPARATOR);
 
+        final boolean[] outIsXrefWriter =  new boolean[1];
+
         for (File file : files) {
             String path = parent + File.separator + file.getName();
             if (!accept(dir, file, outLocalRelPath)) {
@@ -1127,7 +1128,9 @@ private void indexDown(File dir, String parent, IndexDownArgs args)
                          */
                         if (uidIter != null && uidIter.term() != null
                                 && uidIter.term().bytesEquals(buid)) {
-                            boolean chkres = chkSettings(file, path);
+                            boolean chkres = chkSettings(outIsXrefWriter, file,
+                                    path) && (!outIsXrefWriter[0] ||
+                                    checkXrefExistence(path));
                             if (!chkres) {
                                 removeFile(false);
                             }
@@ -1579,16 +1582,21 @@ public int hashCode() {
         return hash;
     }
 
+    private boolean isXrefWriter(FileAnalyzer fa) {
+        Genre g = fa.getFactory().getGenre();
+        return (g == Genre.PLAIN || g == Genre.XREFABLE);
+    }
+
     /**
      * Get a writer to which the xref can be written, or null if no xref
      * should be produced for files of this type.
      */
-    private Writer getXrefWriter(FileAnalyzer fa, String path) throws IOException {
-        Genre g = fa.getFactory().getGenre();
-        if (xrefDir != null && (g == Genre.PLAIN || g == Genre.XREFABLE)) {
-            RuntimeEnvironment env = RuntimeEnvironment.getInstance();
+    private Writer newXrefWriter(FileAnalyzer fa, String path)
+            throws IOException {
+        RuntimeEnvironment env = RuntimeEnvironment.getInstance();
+        if (env.isGenerateHtml() && isXrefWriter(fa)) {
             boolean compressed = env.isCompressXref();
-            File xrefFile = new File(xrefDir, path + (compressed ? ".gz" : ""));
+            File xrefFile = whatXrefFile(path, compressed);
             File parentFile = xrefFile.getParentFile();
 
             // If mkdirs() returns false, the failure is most likely
@@ -1662,7 +1670,9 @@ private void finishWriting() throws IOException {
      * @param path the source file path
      * @return {@code false} if a mismatch is detected
      */
-    private boolean chkSettings(File file, String path) throws IOException {
+    private boolean chkSettings(boolean[] outIsXrefWriter, File file,
+            String path) throws IOException {
+
         int reqTabSize = project != null && project.hasTabSizeSetting() ?
             project.getTabSize() : 0;
         Integer actTabSize = settings.getTabSize();
@@ -1693,6 +1703,7 @@ private boolean chkSettings(File file, String path) throws IOException {
                 break;
             }
 
+            FileAnalyzer fa = null;
             String fileTypeName;
             if (actGuruVersion.equals(reqGuruVersion)) {
                 fileTypeName = doc.get(QueryBuilder.TYPE);
@@ -1701,6 +1712,12 @@ private boolean chkSettings(File file, String path) throws IOException {
                     LOGGER.log(Level.FINEST, "Missing TYPE field: {0}", path);
                     break;
                 }
+
+                FileAnalyzerFactory fac =
+                        AnalyzerGuru.findByFileTypeName(fileTypeName);
+                if (fac != null) {
+                    fa = fac.getAnalyzer();
+                }
             } else {
                 /**
                  * If the stored guru version does not match, re-verify the
@@ -1709,7 +1726,7 @@ private boolean chkSettings(File file, String path) throws IOException {
                  */
                 LOGGER.log(Level.FINER, "Guru version mismatch: {0}", path);
 
-                FileAnalyzer fa = getAnalyzerFor(file, path);
+                fa = getAnalyzerFor(file, path);
                 fileTypeName = fa.getFileTypeName();
                 String oldTypeName = doc.get(QueryBuilder.TYPE);
                 if (!fileTypeName.equals(oldTypeName)) {
@@ -1732,6 +1749,10 @@ private boolean chkSettings(File file, String path) throws IOException {
                 return false;
             }
 
+            if (fa != null) {
+                outIsXrefWriter[0] = isXrefWriter(fa);
+            }
+
             // The versions checks have passed.
             break;
         }
@@ -1761,6 +1782,24 @@ private IndexAnalysisSettings2 readAnalysisSettings() throws IOException {
         return dao.read(reader);
     }
 
+    private boolean checkXrefExistence(String path) {
+        RuntimeEnvironment env = RuntimeEnvironment.getInstance();
+        boolean chkres = whatXrefFile(path, env.isCompressXref()).exists();
+
+        if (!env.isGenerateHtml()) {
+            if (chkres) {
+                LOGGER.log(Level.FINEST, "Extraneous {0}", path);
+                removeXrefFile(path);
+            }
+            return true;
+        }
+
+        if (!chkres) {
+            LOGGER.log(Level.FINEST, "Missing {0}", path);
+        }
+        return chkres;
+    }
+
     private class IndexDownArgs {
         boolean count_only;
         int cur_count;

opengrok-web/src/main/webapp/list.jsp

@@ -20,7 +20,7 @@ CDDL HEADER END
 
 Copyright (c) 2005, 2018, Oracle and/or its affiliates. All rights reserved.
 Portions Copyright 2011 Jens Elkner.
-Portions Copyright (c) 2017, Chris Fraire <[email protected]>.
+Portions Copyright (c) 2017-2018, Chris Fraire <[email protected]>.
 
 --%>
 <%@page errorPage="error.jsp" import="
@@ -176,7 +176,9 @@ document.pageReady.push(function() { pageReadyList();});
         }
     } else if (rev.length() != 0) {
         // requesting a revision
-        if (cfg.isLatestRevision(rev)) {
+        File xrefFile = null;
+        if (cfg.isLatestRevision(rev) &&
+                (xrefFile = cfg.findDataFile()) != null) {
             if (cfg.annotate()) {
                 // annotate
                 BufferedInputStream bin =
@@ -232,8 +234,6 @@ Click <a href="<%= rawPath %>">download <%= basename %></a><%
                 }
 
             } else {
-                File xrefFile = cfg.findDataFile();
-                if (xrefFile != null) {
 %>
 <div id="src" data-navigate-window-enabled="<%= navigateWindowEnabled %>">
     <pre><%
@@ -242,7 +242,6 @@ Click <a href="<%= rawPath %>">download <%= basename %></a><%
                             request.getContextPath());
     %></pre>
 </div><%
-                }
             }
         } else {
             // requesting a previous revision