Merge branch 'master' into bugfix/symlink_loop

Author: idodeclare

OpenGrok

@@ -120,6 +120,14 @@ Supported Environment Variables for configuring the default setup:
                                   OPENGROK_IGNORE_PATTERNS="-i d:dummy"
                                   Multiple entries can be joined together:
                                     "-i dummy -i d:tmp -i f:dummy"
+  - OPENGROK_ASSIGNMENTS        Assign/Unassign specialized analyzers using
+                                  (.ext|prefix.):(-|Analyzer) syntax. E.g.,
+                                  .lib:Ignorant,Makefile.:- to set files ending
+                                  with .LIB (case-insensitive) to use the
+                                  IgnorantAnalyzerFactory (case-sensitive due to
+                                  Java limitation), and to clear special
+                                  handling for files starting with MAKEFILE
+                                  (case-insensitive and no full-stop)
   - OPENGROK_SCAN_REPOS         Disable Scan for repositories (*)
   - OPENGROK_SCAN_DEPTH         How deep should scanning for repos go
                                   (by default 3 directories from SRC_ROOT)
@@ -449,6 +457,11 @@ ${BZR:+-Dorg.opensolaris.opengrok.history.Bazaar=$BZR} \
     then
         PROGRESS="--progress"
     fi
+
+    if [ "$OPENGROK_ASSIGNMENTS" != "" ]; then
+        ASSIGNMENTS="`echo $OPENGROK_ASSIGNMENTS | sed 's/[:space:]+/_/g'`"
+        ASSIGNMENTS="-A `echo $ASSIGNMENTS | sed 's/,/ -A /g'`"
+    fi
 }
 
 #
@@ -896,6 +909,7 @@ CommonInvocation()
         ${CTAGS_OPTIONS_FILE:+-o} ${CTAGS_OPTIONS_FILE}         	\
         ${OPENGROK_FLUSH_RAM_BUFFER_SIZE} ${SKIN} ${LEADING_WILDCARD}	\
         ${OPENGROK_PARALLELISM:+--threads} ${OPENGROK_PARALLELISM}	\
+        ${ASSIGNMENTS}							\
         ${READ_XML_CONF}                                        	\
         ${WEBAPP_CONFIG}						\
         ${WEBAPP_CONTEXT}						\

distribution/pom.xml

@@ -7,7 +7,7 @@
   <parent>
     <groupId>org.opensolaris.opengrok</groupId>
     <artifactId>opengrok-top</artifactId>
-    <version>1.1-rc25</version>
+    <version>1.1-rc26</version>
   </parent>
 
   <artifactId>opengrok-dist</artifactId>
@@ -24,13 +24,13 @@
     <dependency>
       <groupId>org.opensolaris.opengrok</groupId>
       <artifactId>opengrok</artifactId>
-      <version>1.1-rc25</version>
+      <version>1.1-rc26</version>
     </dependency>
 
     <dependency>
       <groupId>org.opensolaris.opengrok</groupId>
       <artifactId>opengrok-web</artifactId>
-      <version>1.1-rc25</version>
+      <version>1.1-rc26</version>
       <type>war</type>
     </dependency>
   </dependencies>

jrcs/pom.xml

@@ -5,13 +5,13 @@
     <groupId>org.opensolaris.opengrok</groupId>
     <artifactId>jrcs</artifactId>
     <packaging>jar</packaging>
-    <version>1.1-rc25</version>
+    <version>1.1-rc26</version>
     <name>Java RCS</name>
 
     <parent>
         <groupId>org.opensolaris.opengrok</groupId>
         <artifactId>opengrok-top</artifactId>
-        <version>1.1-rc25</version>
+        <version>1.1-rc26</version>
     </parent>
 
     <dependencies>

opengrok-indexer/pom.xml

@@ -28,13 +28,13 @@ Portions Copyright (c) 2017-2018, Chris Fraire <[email protected]>.
     <groupId>org.opensolaris.opengrok</groupId>
     <artifactId>opengrok</artifactId>
     <packaging>jar</packaging>
-    <version>1.1-rc25</version>
+    <version>1.1-rc26</version>
     <name>OpenGrok Indexer</name>
 
     <parent>
         <groupId>org.opensolaris.opengrok</groupId>
         <artifactId>opengrok-top</artifactId>
-        <version>1.1-rc25</version>
+        <version>1.1-rc26</version>
     </parent>
 
     <build>

opengrok-web/pom.xml

@@ -5,13 +5,13 @@
     <groupId>org.opensolaris.opengrok</groupId>
     <artifactId>opengrok-web</artifactId>
     <packaging>war</packaging>
-    <version>1.1-rc25</version>
+    <version>1.1-rc26</version>
     <name>OpenGrok Web</name>
 
     <parent>
         <groupId>org.opensolaris.opengrok</groupId>
         <artifactId>opengrok-top</artifactId>
-        <version>1.1-rc25</version>
+        <version>1.1-rc26</version>
     </parent>
 
     <build>

plugins/pom.xml

@@ -27,13 +27,13 @@ Copyright (c) 2016, Oracle and/or its affiliates. All rights reserved.
     <groupId>org.opensolaris.opengrok</groupId>
     <artifactId>plugins</artifactId>
     <packaging>jar</packaging>
-    <version>1.1-rc25</version>
+    <version>1.1-rc26</version>
     <name>OpenGrok authorization plugins</name>
 
     <parent>
         <groupId>org.opensolaris.opengrok</groupId>
         <artifactId>opengrok-top</artifactId>
-        <version>1.1-rc25</version>
+        <version>1.1-rc26</version>
     </parent>
 
     <build>

pom.xml

@@ -28,7 +28,7 @@ Portions Copyright (c) 2018, Chris Fraire <[email protected]>.
     <groupId>org.opensolaris.opengrok</groupId>
     <artifactId>opengrok-top</artifactId>
     <packaging>pom</packaging>
-    <version>1.1-rc25</version>
+    <version>1.1-rc26</version>
     <name>OpenGrok</name>
     <url>http://opengrok.github.com/OpenGrok/</url>
 

src/org/opensolaris/opengrok/analysis/AnalyzerGuru.java

@@ -41,8 +41,10 @@
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
+import java.util.Objects;
 import java.util.SortedMap;
 import java.util.TreeMap;
+import java.util.TreeSet;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
@@ -167,6 +169,17 @@ public class AnalyzerGuru {
      */
     private static final Map<String, FileAnalyzerFactory> pre = new HashMap<>();
 
+    /**
+     * Appended when
+     * {@link #addExtension(java.lang.String, org.opensolaris.opengrok.analysis.FileAnalyzerFactory)}
+     * or
+     * {@link #addPrefix(java.lang.String, org.opensolaris.opengrok.analysis.FileAnalyzerFactory)}
+     * are called to augment the value in {@link #getVersionNo()}.
+     */
+    private static final TreeSet<String> CUSTOMIZATION_KEYS = new TreeSet<>();
+
+    private static int customizationHashCode;
+
     /**
      * Descending string length comparator for magics
      */
@@ -220,6 +233,12 @@ public class AnalyzerGuru {
     private static final Map<String, FileAnalyzerFactory> FILETYPE_FACTORIES =
             new HashMap<>();
 
+    /**
+     * Maps from {@link FileAnalyzer#getFileTypeName()} to
+     * {@link FileAnalyzer#getVersionNo()}
+     */
+    private static final Map<String, Long> ANALYZER_VERSIONS = new HashMap<>();
+
     /*
      * If you write your own analyzer please register it here. The order is
      * important for any factory that uses a FileAnalyzerFactory.Matcher
@@ -295,6 +314,49 @@ public class AnalyzerGuru {
         }
     }
 
+    /**
+     * Gets a version number to be used to tag documents examined by the guru so
+     * that analysis can be re-done later if a stored version number is
+     * different from the current implementation or if customization has been
+     * done by the user to change the {@link AnalyzerGuru} operation.
+     * <p>
+     * The static part of the version is bumped in a release when e.g. new
+     * {@link FileAnalyzerFactory} subclasses are registered or when existing
+     * {@link FileAnalyzerFactory} subclasses are revised to target more or
+     * different files.
+     * @return a value whose lower 32-bits are a static value
+     * 20171230_00
+     * for the current implementation and whose higher-32 bits are non-zero if
+     * {@link #addExtension(java.lang.String, org.opensolaris.opengrok.analysis.FileAnalyzerFactory)}
+     * or
+     * {@link #addPrefix(java.lang.String, org.opensolaris.opengrok.analysis.FileAnalyzerFactory)}
+     * has been called.
+     */
+    public static long getVersionNo() {
+        final int ver32 = 20171230_00; // Edit comment above too!
+        long ver = ver32;
+        if (customizationHashCode != 0) {
+            ver |= (long)customizationHashCode << 32;
+        }
+        return ver;
+    }
+
+    /**
+     * Gets a version number according to a registered
+     * {@link FileAnalyzer#getVersionNo()} for a {@code fileTypeName} according
+     * to {@link FileAnalyzer#getFileTypeName()}.
+     * @param fileTypeName a defined instance
+     * @return a registered value or {@link Long#MIN_VALUE} if
+     * {@code fileTypeName} is unknown
+     */
+    public static long getAnalyzerVersionNo(String fileTypeName) {
+        return ANALYZER_VERSIONS.getOrDefault(fileTypeName, Long.MIN_VALUE);
+    }
+
+    public static Map<String, Long> getAnalyzersVersionNos() {
+        return Collections.unmodifiableMap(ANALYZER_VERSIONS);
+    }
+
     public static Map<String, FileAnalyzerFactory> getExtensionsMap() {
         return Collections.unmodifiableMap(ext);
     }
@@ -347,7 +409,9 @@ private static void registerAnalyzer(FileAnalyzerFactory factory) {
         factories.add(factory);
 
         FileAnalyzer fa = factory.getAnalyzer();
-        FILETYPE_FACTORIES.put(fa.getFileTypeName(), factory);
+        String fileTypeName = fa.getFileTypeName();
+        FILETYPE_FACTORIES.put(fileTypeName, factory);
+        ANALYZER_VERSIONS.put(fileTypeName, fa.getVersionNo());
     }
 
     /**
@@ -359,12 +423,16 @@ private static void registerAnalyzer(FileAnalyzerFactory factory) {
      * extension (if you pass null as the analyzer, you will disable the
      * analyzer used for that extension)
      */
-    public static void addPrefix(String prefix,
-            FileAnalyzerFactory factory) {
+    public static void addPrefix(String prefix, FileAnalyzerFactory factory) {
+        FileAnalyzerFactory oldFactory;
         if (factory == null) {
-            pre.remove(prefix);
+            oldFactory = pre.remove(prefix);
         } else {
-            pre.put(prefix, factory);
+            oldFactory = pre.put(prefix, factory);
+        }
+
+        if (factoriesDifferent(factory, oldFactory)) {
+            addCustomizationKey("p:" + prefix);
         }
     }
 
@@ -379,10 +447,15 @@ public static void addPrefix(String prefix,
      */
     public static void addExtension(String extension,
             FileAnalyzerFactory factory) {
+        FileAnalyzerFactory oldFactory;
         if (factory == null) {
-            ext.remove(extension);
+            oldFactory = ext.remove(extension);
         } else {
-            ext.put(extension, factory);
+            oldFactory = ext.put(extension, factory);
+        }
+
+        if (factoriesDifferent(factory, oldFactory)) {
+            addCustomizationKey("e:" + extension);
         }
     }
 
@@ -1003,4 +1076,32 @@ private static String readOpening(InputStream in, byte[] sig)
         in.reset();
         return opening.toString();
     }
+
+    private static void addCustomizationKey(String k) {
+        CUSTOMIZATION_KEYS.add(k);
+        Object[] keys = CUSTOMIZATION_KEYS.toArray();
+        customizationHashCode = Objects.hash(keys);
+    }
+
+    private static boolean factoriesDifferent(FileAnalyzerFactory a,
+            FileAnalyzerFactory b) {
+        String a_name = null;
+        if (a != null) {
+            a_name = a.getName();
+            if (a_name == null) {
+                a_name = a.getClass().getSimpleName();
+            }
+        }
+        String b_name = null;
+        if (b != null) {
+            b_name = b.getName();
+            if (b_name == null) {
+                b_name = b.getClass().getSimpleName();
+            }
+        }
+        if (a_name == null && b_name == null) {
+            return false;
+        }
+        return a_name == null || b_name == null || !a_name.equals(b_name);
+    }
 }

src/org/opensolaris/opengrok/analysis/FileAnalyzer.java

@@ -20,7 +20,7 @@
  /*
  * Copyright (c) 2005, 2017, Oracle and/or its affiliates. All rights reserved.
  * Use is subject to license terms.
- * Portions Copyright (c) 2017, Chris Fraire <[email protected]>.
+ * Portions Copyright (c) 2017-2018, Chris Fraire <[email protected]>.
  */
 package org.opensolaris.opengrok.analysis;
 
@@ -63,6 +63,7 @@ public class FileAnalyzer extends Analyzer {
     private static final Logger LOGGER = LoggerFactory.getLogger(FileAnalyzer.class);
 
     protected Project project;
+    protected Ctags ctags;
     protected boolean scopesEnabled;
     protected boolean foldingEnabled;
     private final FileAnalyzerFactory factory;
@@ -126,7 +127,31 @@ public static Genre get(String typeName) {
             return null;
         }
     }
-    protected Ctags ctags;
+
+    /**
+     * Gets a version number to be used to tag processed documents so that
+     * re-analysis can be re-done later if a stored version number is different
+     * from the current implementation.
+     * <p>
+     * The value is the union of a {@link FileAnalyzer} root version and the
+     * value from {@link #getSpecializedVersionNo()}. Changing the root version
+     * affects all analyzers simultaneously; while subclasses can override
+     * {@link #getSpecializedVersionNo()} to allow changes that affect a few.
+     * @return (20061115_01 &lt;&lt; 32) | {@link #getSpecializedVersionNo()}
+     */
+    public final long getVersionNo() {
+        final int rootVersionNo = 20061115_01; // Edit comment above too!
+        return ((long)rootVersionNo << 32) | getSpecializedVersionNo();
+    }
+
+    /**
+     * Subclasses should override to produce a value relevant for the evolution
+     * of their analysis in each release.
+     * @return 0
+     */
+    protected int getSpecializedVersionNo() {
+        return 0; // FileAnalyzer is not specialized.
+    }
 
     public void setCtags(Ctags ctags) {
         this.ctags = ctags;

src/org/opensolaris/opengrok/analysis/TextAnalyzer.java

@@ -19,7 +19,7 @@
 
 /*
  * Copyright (c) 2005, 2017, Oracle and/or its affiliates. All rights reserved.
- * Portions Copyright (c) 2017, Chris Fraire <[email protected]>.
+ * Portions Copyright (c) 2017-2018, Chris Fraire <[email protected]>.
  */
 package org.opensolaris.opengrok.analysis;
 
@@ -49,6 +49,17 @@ protected TextAnalyzer(FileAnalyzerFactory factory,
         super(factory, symbolTokenizer);
     }
 
+    /**
+     * Gets a version number to be used to tag processed documents so that
+     * re-analysis can be re-done later if a stored version number is different
+     * from the current implementation.
+     * @return 20171223_00
+     */
+    @Override
+    protected int getSpecializedVersionNo() {
+        return 20171223_00; // Edit comment above too!
+    }
+
     /**
      * Write a cross referenced HTML file reads the source from in
      * @param args a defined instance