Store and verify AnalyzerGuru and Analyzer versions

Author: idodeclare

OpenGrok

@@ -120,6 +120,14 @@ Supported Environment Variables for configuring the default setup:
                                   OPENGROK_IGNORE_PATTERNS="-i d:dummy"
                                   Multiple entries can be joined together:
                                     "-i dummy -i d:tmp -i f:dummy"
+  - OPENGROK_ASSIGNMENTS        Assign/Unassign specialized analyzers using
+                                  (.ext|prefix.):(-|Analyzer) syntax. E.g.,
+                                  .lib:Ignorant,Makefile.:- to set files ending
+                                  with .LIB (case-insensitive) to use the
+                                  IgnorantAnalyzerFactory (case-sensitive due to
+                                  Java limitation), and to clear special
+                                  handling for files starting with MAKEFILE
+                                  (case-insensitive and no full-stop)
   - OPENGROK_SCAN_REPOS         Disable Scan for repositories (*)
   - OPENGROK_SCAN_DEPTH         how deep should scanning for repos go
                                 (by default 3 directories from SRC_ROOT)
@@ -445,6 +453,11 @@ ${BZR:+-Dorg.opensolaris.opengrok.history.Bazaar=$BZR} \
     then
         PROGRESS="--progress"
     fi
+
+    if [ "$OPENGROK_ASSIGNMENTS" != "" ]; then
+        ASSIGNMENTS="`echo $OPENGROK_ASSIGNMENTS | sed 's/[:space:]+/_/g'`"
+        ASSIGNMENTS="-A `echo $ASSIGNMENTS | sed 's/,/ -A /g'`"
+    fi
 }
 
 #
@@ -878,6 +891,7 @@ CommonInvocation()
         ${CTAGS_OPTIONS_FILE:+-o} ${CTAGS_OPTIONS_FILE}         	\
         ${OPENGROK_FLUSH_RAM_BUFFER_SIZE} ${SKIN} ${LEADING_WILDCARD}	\
         ${OPENGROK_PARALLELISM:+--threads} ${OPENGROK_PARALLELISM}	\
+        ${ASSIGNMENTS}							\
         ${READ_XML_CONF}                                        	\
         ${WEBAPP_CONFIG}						\
         ${WEBAPP_CONTEXT}						\

src/org/opensolaris/opengrok/analysis/AnalyzerGuru.java

@@ -41,8 +41,10 @@
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
+import java.util.Objects;
 import java.util.SortedMap;
 import java.util.TreeMap;
+import java.util.TreeSet;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
@@ -52,6 +54,7 @@
 import org.apache.lucene.document.Field.Store;
 import org.apache.lucene.document.FieldType;
 import org.apache.lucene.document.SortedDocValuesField;
+import org.apache.lucene.document.StoredField;
 import org.apache.lucene.document.StringField;
 import org.apache.lucene.document.TextField;
 import org.apache.lucene.util.BytesRef;
@@ -167,6 +170,17 @@ public class AnalyzerGuru {
      */
     private static final Map<String, FileAnalyzerFactory> pre = new HashMap<>();
 
+    /**
+     * Appended when
+     * {@link #addExtension(java.lang.String, org.opensolaris.opengrok.analysis.FileAnalyzerFactory)}
+     * or
+     * {@link #addPrefix(java.lang.String, org.opensolaris.opengrok.analysis.FileAnalyzerFactory)}
+     * are called to augment the value in {@link #getVersionNo()}.
+     */
+    private static final TreeSet<String> CUSTOMIZATION_KEYS = new TreeSet<>();
+
+    private static int customizationHashCode;
+
     /**
      * Descending string length comparator for magics
      */
@@ -220,6 +234,12 @@ public class AnalyzerGuru {
     private static final Map<String, FileAnalyzerFactory> FILETYPE_FACTORIES =
             new HashMap<>();
 
+    /**
+     * Maps from {@link FileAnalyzer#getFileTypeName()} to
+     * {@link FileAnalyzer#getVersionNo()}
+     */
+    private static final Map<String, Long> ANALYZER_VERSIONS = new HashMap<>();
+
     /*
      * If you write your own analyzer please register it here. The order is
      * important for any factory that uses a FileAnalyzerFactory.Matcher
@@ -295,6 +315,45 @@ public class AnalyzerGuru {
         }
     }
 
+    /**
+     * Gets a version number to be used to tag documents examined by the guru so
+     * that analysis can be re-done later if a stored version number is
+     * different from the current implementation or if customization has been
+     * done by the user to change the {@link AnalyzerGuru} operation.
+     * <p>
+     * The static part of the version is bumped in a release when e.g. new
+     * {@link FileAnalyzerFactory} subclasses are registered or when existing
+     * {@link FileAnalyzerFactory} subclasses are revised to target more or
+     * different files.
+     * @return a value whose lower 32-bits are a static value
+     * 20171230_00
+     * for the current implementation and whose higher-32 bits are non-zero if
+     * {@link #addExtension(java.lang.String, org.opensolaris.opengrok.analysis.FileAnalyzerFactory)}
+     * or
+     * {@link #addPrefix(java.lang.String, org.opensolaris.opengrok.analysis.FileAnalyzerFactory)}
+     * has been called.
+     */
+    public static long getVersionNo() {
+        final int ver32 = 20171230_00; // Edit comment above too!
+        long ver = ver32;
+        if (customizationHashCode != 0) {
+            ver |= (long)customizationHashCode << 32;
+        }
+        return ver;
+    }
+
+    /**
+     * Gets a version number according to a registered
+     * {@link FileAnalyzer#getVersionNo()} for a {@code fileTypeName} according
+     * to {@link FileAnalyzer#getFileTypeName()}.
+     * @param fileTypeName a defined instance
+     * @return a registered value or {@link Long#MIN_VALUE} if
+     * {@code fileTypeName} is unknown
+     */
+    public static long getAnalyzerVersionNo(String fileTypeName) {
+        return ANALYZER_VERSIONS.getOrDefault(fileTypeName, Long.MIN_VALUE);
+    }
+
     public static Map<String, FileAnalyzerFactory> getExtensionsMap() {
         return Collections.unmodifiableMap(ext);
     }
@@ -347,7 +406,9 @@ private static void registerAnalyzer(FileAnalyzerFactory factory) {
         factories.add(factory);
 
         FileAnalyzer fa = factory.getAnalyzer();
-        FILETYPE_FACTORIES.put(fa.getFileTypeName(), factory);
+        String fileTypeName = fa.getFileTypeName();
+        FILETYPE_FACTORIES.put(fileTypeName, factory);
+        ANALYZER_VERSIONS.put(fileTypeName, fa.getVersionNo());
     }
 
     /**
@@ -359,12 +420,16 @@ private static void registerAnalyzer(FileAnalyzerFactory factory) {
      * extension (if you pass null as the analyzer, you will disable the
      * analyzer used for that extension)
      */
-    public static void addPrefix(String prefix,
-            FileAnalyzerFactory factory) {
+    public static void addPrefix(String prefix, FileAnalyzerFactory factory) {
+        FileAnalyzerFactory oldFactory;
         if (factory == null) {
-            pre.remove(prefix);
+            oldFactory = pre.remove(prefix);
         } else {
-            pre.put(prefix, factory);
+            oldFactory = pre.put(prefix, factory);
+        }
+
+        if (factoriesDifferent(factory, oldFactory)) {
+            addCustomizationKey("p:" + prefix);
         }
     }
 
@@ -379,10 +444,15 @@ public static void addPrefix(String prefix,
      */
     public static void addExtension(String extension,
             FileAnalyzerFactory factory) {
+        FileAnalyzerFactory oldFactory;
         if (factory == null) {
-            ext.remove(extension);
+            oldFactory = ext.remove(extension);
         } else {
-            ext.put(extension, factory);
+            oldFactory = ext.put(extension, factory);
+        }
+
+        if (factoriesDifferent(factory, oldFactory)) {
+            addCustomizationKey("e:" + extension);
         }
     }
 
@@ -501,6 +571,8 @@ public void populateDocument(Document doc, File file, String path,
 
             String type = fa.getFileTypeName();
             doc.add(new StringField(QueryBuilder.TYPE, type, Store.YES));
+
+            doc.add(new StoredField(QueryBuilder.ZVER, fa.getVersionNo()));
         }
     }
 
@@ -1003,4 +1075,32 @@ private static String readOpening(InputStream in, byte[] sig)
         in.reset();
         return opening.toString();
     }
+
+    private static void addCustomizationKey(String k) {
+        CUSTOMIZATION_KEYS.add(k);
+        Object[] keys = CUSTOMIZATION_KEYS.toArray();
+        customizationHashCode = Objects.hash(keys);
+    }
+
+    private static boolean factoriesDifferent(FileAnalyzerFactory a,
+            FileAnalyzerFactory b) {
+        String a_name = null;
+        if (a != null) {
+            a_name = a.getName();
+            if (a_name == null) {
+                a_name = a.getClass().getSimpleName();
+            }
+        }
+        String b_name = null;
+        if (b != null) {
+            b_name = b.getName();
+            if (b_name == null) {
+                b_name = b.getClass().getSimpleName();
+            }
+        }
+        if (a_name == null && b_name == null) {
+            return false;
+        }
+        return a_name == null || b_name == null || !a_name.equals(b_name);
+    }
 }

src/org/opensolaris/opengrok/analysis/FileAnalyzer.java

@@ -63,6 +63,7 @@ public class FileAnalyzer extends Analyzer {
     private static final Logger LOGGER = LoggerFactory.getLogger(FileAnalyzer.class);
 
     protected Project project;
+    protected Ctags ctags;
     protected boolean scopesEnabled;
     protected boolean foldingEnabled;
     private final FileAnalyzerFactory factory;
@@ -126,7 +127,31 @@ public static Genre get(String typeName) {
             return null;
         }
     }
-    protected Ctags ctags;
+
+    /**
+     * Gets a version number to be used to tag processed documents so that
+     * re-analysis can be re-done later if a stored version number is different
+     * from the current implementation.
+     * <p>
+     * The value is the union of a {@link FileAnalyzer} root version and the
+     * value from {@link #getSpecializedVersionNo()}. Changing the root version
+     * affects all analyzers simultaneously; while subclasses can override
+     * {@link #getSpecializedVersionNo()} to allow changes that affect a few.
+     * @return (20061115_01 &lt;&lt; 32) | {@link #getSpecializedVersionNo()}
+     */
+    public final long getVersionNo() {
+        final int rootVersionNo = 20061115_01; // Edit comment above too!
+        return ((long)rootVersionNo << 32) | getSpecializedVersionNo();
+    }
+
+    /**
+     * Subclasses should override to produce a value relevant for the evolution
+     * of their analysis in each release.
+     * @return 0
+     */
+    protected int getSpecializedVersionNo() {
+        return 0; // FileAnalyzer is not specialized.
+    }
 
     public void setCtags(Ctags ctags) {
         this.ctags = ctags;

src/org/opensolaris/opengrok/analysis/TextAnalyzer.java

@@ -19,7 +19,7 @@
 
 /*
  * Copyright (c) 2005, 2017, Oracle and/or its affiliates. All rights reserved.
- * Portions Copyright (c) 2017, Chris Fraire <[email protected]>.
+ * Portions Copyright (c) 2017-2018, Chris Fraire <[email protected]>.
  */
 package org.opensolaris.opengrok.analysis;
 
@@ -49,6 +49,17 @@ protected TextAnalyzer(FileAnalyzerFactory factory,
         super(factory, symbolTokenizer);
     }
 
+    /**
+     * Gets a version number to be used to tag processed documents so that
+     * re-analysis can be re-done later if a stored version number is different
+     * from the current implementation.
+     * @return 20171223_00
+     */
+    @Override
+    protected int getSpecializedVersionNo() {
+        return 20171223_00; // Edit comment above too!
+    }
+
     /**
      * Write a cross referenced HTML file reads the source from in
      * @param args a defined instance

src/org/opensolaris/opengrok/analysis/ada/AdaAnalyzer.java

@@ -19,7 +19,7 @@
 
 /*
  * Copyright (c) 2010, 2017, Oracle and/or its affiliates. All rights reserved.
- * Portions Copyright (c) 2017, Chris Fraire <[email protected]>.
+ * Portions Copyright (c) 2017-2018, Chris Fraire <[email protected]>.
  */
 package org.opensolaris.opengrok.analysis.ada;
 
@@ -46,6 +46,17 @@ protected AdaAnalyzer(FileAnalyzerFactory factory) {
         super(factory, new JFlexTokenizer(new AdaSymbolTokenizer(
             FileAnalyzer.dummyReader)));
     }
+
+    /**
+     * Gets a version number to be used to tag processed documents so that
+     * re-analysis can be re-done later if a stored version number is different
+     * from the current implementation.
+     * @return 20171218_00
+     */
+    @Override
+    protected int getSpecializedVersionNo() {
+        return 20171218_00; // Edit comment above too!
+    }
 
     /**
      * Creates a wrapped {@link AdaXref} instance.

src/org/opensolaris/opengrok/analysis/archive/BZip2Analyzer.java

@@ -19,7 +19,7 @@
 
 /*
  * Copyright (c) 2005, 2017, Oracle and/or its affiliates. All rights reserved.
- * Portions Copyright (c) 2017, Chris Fraire <[email protected]>.
+ * Portions Copyright (c) 2017-2018, Chris Fraire <[email protected]>.
  */
 package org.opensolaris.opengrok.analysis.archive;
 
@@ -55,11 +55,23 @@ public Genre getGenre() {
     protected BZip2Analyzer(FileAnalyzerFactory factory) {
         super(factory);
     }
-    private FileAnalyzer fa;
+
+    /**
+     * Gets a version number to be used to tag processed documents so that
+     * re-analysis can be re-done later if a stored version number is different
+     * from the current implementation.
+     * @return 20180111_00
+     */
+    @Override
+    protected int getSpecializedVersionNo() {
+        return 20180111_00; // Edit comment above too!
+    }
 
     @Override
     public void analyze(Document doc, StreamSource src, Writer xrefOut)
             throws IOException, InterruptedException {
+        FileAnalyzer fa;
+
         StreamSource bzSrc = wrap(src);
         String path = doc.get("path");
         if (path != null
@@ -69,9 +81,7 @@ public void analyze(Document doc, StreamSource src, Writer xrefOut)
             try (InputStream in = bzSrc.getStream()) {
                 fa = AnalyzerGuru.getAnalyzer(in, newname);
             }
-            if (fa instanceof BZip2Analyzer) {
-                fa = null;
-            } else {
+            if (!(fa instanceof BZip2Analyzer)) {
                 if (fa.getGenre() == Genre.PLAIN || fa.getGenre() == Genre.XREFABLE) {
                     this.g = Genre.XREFABLE;
                 } else {

src/org/opensolaris/opengrok/analysis/archive/GZIPAnalyzer.java

@@ -19,7 +19,7 @@
 
 /*
  * Copyright (c) 2005, 2017, Oracle and/or its affiliates. All rights reserved.
- * Portions Copyright (c) 2017, Chris Fraire <[email protected]>.
+ * Portions Copyright (c) 2017-2018, Chris Fraire <[email protected]>.
  */
 package org.opensolaris.opengrok.analysis.archive;
 
@@ -61,11 +61,23 @@ public Genre getGenre() {
     protected GZIPAnalyzer(FileAnalyzerFactory factory) {
         super(factory);
     }
-    private FileAnalyzer fa;
+
+    /**
+     * Gets a version number to be used to tag processed documents so that
+     * re-analysis can be re-done later if a stored version number is different
+     * from the current implementation.
+     * @return 20180111_00
+     */
+    @Override
+    protected int getSpecializedVersionNo() {
+        return 20180111_00; // Edit comment above too!
+    }
 
     @Override
     public void analyze(Document doc, StreamSource src, Writer xrefOut)
             throws IOException, InterruptedException {
+        FileAnalyzer fa;
+
         StreamSource gzSrc = wrap(src);
         String path = doc.get("path");
         if (path != null