Introducing an plugin framework for analyzers

 - loading analyzers implementing an interface from a directory
 - augmenting analyzer guru to use the loaded plugins
 - making analyzer guru a property of RuntimeEnvironment
 - loading the default analyzers as it used to be

approaches #2588

Author: tulinkry

dev/checkstyle/suppressions.xml

@@ -37,7 +37,7 @@ Portions Copyright (c) 2018-2019, Chris Fraire <[email protected]>.
 
     <suppress checks="ParameterNumber" files="CtagsReader\.java|Definitions\.java|
         |JFlexXrefUtils\.java|FileAnalyzerFactory\.java|SearchController\.java|
-        |Context\.java|HistoryContext\.java|Suggester\.java" />
+        |Context\.java|HistoryContext\.java|Suggester\.java|AnalyzersInfo\.java" />
 
     <suppress checks="MethodLength" files="Indexer\.java" />
 </suppressions>

opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/AnalyzerFramework.java

@@ -35,40 +35,41 @@
 public class AnalyzerGuruHelp {
     /**
      * Gets a reportable hunk of text that details
-     * {@link AnalyzerGuru#getPrefixesMap()},
-     * {@link AnalyzerGuru#getExtensionsMap()},
-     * {@link AnalyzerGuru#getMagicsMap()}, and
-     * {@link AnalyzerGuru#getAnalyzerFactoryMatchers()}.
+     * {@link AnalyzerFramework#getPrefixesMap()},
+     * {@link AnalyzerFramework#getExtensionsMap()},
+     * {@link AnalyzerFramework#getMagicsMap()}, and
+     * {@link AnalyzerFramework#getAnalyzerFactoryMatchers()}.
      * @return a defined, multi-line String
+     * @param analyzerGuru
      */
-    public static String getUsage() {
+    public static String getUsage(AnalyzerGuru analyzerGuru) {
         StringBuilder b = new StringBuilder();
         b.append("AnalyzerGuru prefixes:\n");
-        byKey(AnalyzerGuru.getPrefixesMap()).forEach((kv) -> {
+        byKey(analyzerGuru.getPrefixesMap()).forEach((kv) -> {
             b.append(String.format("%-10s : %s\n", reportable(kv.key + '*'),
                 reportable(kv.fac)));
         });
 
         b.append("\nAnalyzerGuru extensions:\n");
-        byKey(AnalyzerGuru.getExtensionsMap()).forEach((kv) -> {
+        byKey(analyzerGuru.getExtensionsMap()).forEach((kv) -> {
             b.append(String.format("*.%-7s : %s\n",
                 reportable(kv.key.toLowerCase(Locale.ROOT)),
                 reportable(kv.fac)));
         });
 
         b.append("\nAnalyzerGuru magic strings:\n");
-        byFactory(AnalyzerGuru.getMagicsMap()).forEach((kv) -> {
+        byFactory(analyzerGuru.getMagicsMap()).forEach((kv) -> {
             b.append(String.format("%-23s : %s\n", reportable(kv.key),
                 reportable(kv.fac)));
         });
 
         b.append("\nAnalyzerGuru magic matchers:\n");
-        AnalyzerGuru.getAnalyzerFactoryMatchers().forEach((m) -> {
+        analyzerGuru.getAnalyzerFactoryMatchers().forEach((m) -> {
             if (m.getIsPreciseMagic()) {
                 b.append(reportable(m));
             }
         });
-        AnalyzerGuru.getAnalyzerFactoryMatchers().forEach((m) -> {
+        analyzerGuru.getAnalyzerFactoryMatchers().forEach((m) -> {
             if (!m.getIsPreciseMagic()) {
                 b.append(reportable(m));
             }

opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/AnalyzerGuru.java

@@ -0,0 +1,180 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * See LICENSE.txt included in this distribution for the specific
+ * language governing permissions and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at LICENSE.txt.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+/*
+ * Copyright (c) 2019 Oracle and/or its affiliates. All rights reserved.
+ */
+package org.opengrok.indexer.analysis;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.SortedMap;
+import java.util.SortedSet;
+import java.util.TreeMap;
+import java.util.TreeSet;
+
+/**
+ * A class wrapping information about used analyzers.
+ */
+public class AnalyzersInfo {
+
+    /**
+     * Descending string length comparator for magics
+     */
+    private static final Comparator<String> DESCENDING_LENGTH_COMPARATOR =
+            Comparator.comparingInt(String::length).thenComparing(String::toString);
+
+    /**
+     * Modified when
+     * {@link AnalyzerFramework#addExtension(String, AnalyzerFactory)}
+     * or
+     * {@link AnalyzerFramework#addPrefix(String, AnalyzerFactory)}
+     * are called to augment the value in {@link AnalyzerGuru#getVersionNo()}.
+     */
+    public final SortedSet<String> customizations;
+
+    /**
+     * Map from file names to analyzer factories.
+     */
+    public final Map<String, AnalyzerFactory> fileNames;
+
+    /**
+     * Map from file extensions to analyzer factories.
+     */
+    public final Map<String, AnalyzerFactory> extensions;
+
+    /**
+     * Map from file prefixes to analyzer factories.
+     */
+    public final Map<String, AnalyzerFactory> prefixes;
+
+    /**
+     * Map from magic strings to analyzer factories.
+     */
+    public final SortedMap<String, AnalyzerFactory> magics;
+
+    /**
+     * List of matcher objects which can be used to determine which analyzer
+     * factory to use.
+     */
+    public final List<FileAnalyzerFactory.Matcher> matchers;
+
+    /**
+     * List of all registered {@code FileAnalyzerFactory} instances.
+     */
+    public final List<AnalyzerFactory> factories;
+
+    /**
+     * A map of {@link FileAnalyzer#getFileTypeName()} to human readable analyzer name.
+     */
+    public final Map<String, String> fileTypeDescriptions;
+
+    /**
+     * Maps from {@link FileAnalyzer#getFileTypeName()} to
+     * {@link FileAnalyzerFactory}
+     */
+    public final Map<String, AnalyzerFactory> filetypeFactories;
+
+    /**
+     * Maps from {@link FileAnalyzer#getFileTypeName()} to
+     * {@link FileAnalyzer#getVersionNo()}
+     */
+    public final Map<String, Long> analyzerVersions;
+
+
+    /**
+     * Construct an empty analyzers info.
+     */
+    public AnalyzersInfo() {
+        this(
+                new TreeSet<>(),
+                new HashMap<>(),
+                new HashMap<>(),
+                new HashMap<>(),
+                new TreeMap<>(DESCENDING_LENGTH_COMPARATOR),
+                new ArrayList<>(),
+                new ArrayList<>(),
+                new HashMap<>(),
+                new HashMap<>(),
+                new HashMap<>()
+        );
+    }
+
+    /**
+     * Construct the analyzers info with given collections.
+     *
+     * @param customizations       the customization keys set
+     * @param fileNames            map of filenames to analyzers factories
+     * @param extensions             map of extensions to analyzers factories
+     * @param prefixes             map of prefixes to analyzers factories
+     * @param magics               map of magics to analyzers factories
+     * @param matchers             a list of matchers for analyzers factories
+     * @param factories            a list of analyzers factories
+     * @param fileTypeDescriptions map of analyzer names to analyzer descriptions
+     * @param filetypeFactories    map of file type to analyzers factories
+     * @param analyzerVersions     map of analyzer names to analyzer versions
+     */
+    private AnalyzersInfo(
+            SortedSet<String> customizations,
+            Map<String, AnalyzerFactory> fileNames,
+            Map<String, AnalyzerFactory> extensions,
+            Map<String, AnalyzerFactory> prefixes,
+            SortedMap<String, AnalyzerFactory> magics,
+            List<FileAnalyzerFactory.Matcher> matchers,
+            List<AnalyzerFactory> factories,
+            Map<String, String> fileTypeDescriptions,
+            Map<String, AnalyzerFactory> filetypeFactories,
+            Map<String, Long> analyzerVersions
+    ) {
+        this.customizations = customizations;
+        this.fileNames = fileNames;
+        this.extensions = extensions;
+        this.prefixes = prefixes;
+        this.magics = magics;
+        this.matchers = matchers;
+        this.factories = factories;
+        this.fileTypeDescriptions = fileTypeDescriptions;
+        this.filetypeFactories = filetypeFactories;
+        this.analyzerVersions = analyzerVersions;
+    }
+
+    /**
+     * Make the object unmodifiable.
+     *
+     * @return a new object reference with all properties wrapped as unmodifiable collections
+     */
+    public AnalyzersInfo freeze() {
+        return new AnalyzersInfo(
+                Collections.unmodifiableSortedSet(this.customizations),
+                Collections.unmodifiableMap(this.fileNames),
+                Collections.unmodifiableMap(this.extensions),
+                Collections.unmodifiableMap(this.prefixes),
+                Collections.unmodifiableSortedMap(this.magics),
+                Collections.unmodifiableList(this.matchers),
+                Collections.unmodifiableList(this.factories),
+                Collections.unmodifiableMap(this.fileTypeDescriptions),
+                Collections.unmodifiableMap(this.filetypeFactories),
+                Collections.unmodifiableMap(this.analyzerVersions)
+        );
+    }
+}

opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/AnalyzerGuruHelp.java

@@ -0,0 +1,28 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * See LICENSE.txt included in this distribution for the specific
+ * language governing permissions and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at LICENSE.txt.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+/*
+ * Copyright (c) 2019 Oracle and/or its affiliates. All rights reserved.
+ */
+package org.opengrok.indexer.analysis;
+
+public interface IAnalyzerPlugin {
+
+    AnalyzerFactory getFactory();
+}

opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/AnalyzersInfo.java

@@ -35,6 +35,7 @@
 import org.opengrok.indexer.analysis.AnalyzerGuru;
 import org.opengrok.indexer.analysis.FileAnalyzer;
 import org.opengrok.indexer.analysis.StreamSource;
+import org.opengrok.indexer.configuration.RuntimeEnvironment;
 
 /**
  * Analyzes a BZip2 file Created on September 22, 2005
@@ -80,7 +81,7 @@ public void analyze(Document doc, StreamSource src, Writer xrefOut)
             String newname = path.substring(0, path.lastIndexOf('.'));
             //System.err.println("BZIPPED OF = " + newname);
             try (InputStream in = bzSrc.getStream()) {
-                fa = AnalyzerGuru.getAnalyzer(in, newname);
+                fa = RuntimeEnvironment.getInstance().getAnalyzerGuru().getAnalyzer(in, newname);
             }
             if (!(fa instanceof BZip2Analyzer)) {
                 if (fa.getGenre() == Genre.PLAIN || fa.getGenre() == Genre.XREFABLE) {

opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/IAnalyzerPlugin.java

@@ -38,6 +38,7 @@
 import org.opengrok.indexer.analysis.AnalyzerGuru;
 import org.opengrok.indexer.analysis.FileAnalyzer;
 import org.opengrok.indexer.analysis.StreamSource;
+import org.opengrok.indexer.configuration.RuntimeEnvironment;
 import org.opengrok.indexer.logger.LoggerFactory;
 
 /**
@@ -85,7 +86,7 @@ public void analyze(Document doc, StreamSource src, Writer xrefOut)
             String newname = path.substring(0, path.length() - 3);
             //System.err.println("GZIPPED OF = " + newname);
             try (InputStream gzis = gzSrc.getStream()) {
-                fa = AnalyzerGuru.getAnalyzer(gzis, newname);
+                fa = RuntimeEnvironment.getInstance().getAnalyzerGuru().getAnalyzer(gzis, newname);
             }
             if (fa == null) {
                 this.g = Genre.DATA;

opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/archive/BZip2Analyzer.java

@@ -32,10 +32,10 @@
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field.Store;
 import org.opengrok.indexer.analysis.AnalyzerFactory;
-import org.opengrok.indexer.analysis.AnalyzerGuru;
 import org.opengrok.indexer.analysis.FileAnalyzer;
 import org.opengrok.indexer.analysis.OGKTextField;
 import org.opengrok.indexer.analysis.StreamSource;
+import org.opengrok.indexer.configuration.RuntimeEnvironment;
 import org.opengrok.indexer.search.QueryBuilder;
 import org.opengrok.indexer.web.Util;
 
@@ -82,7 +82,7 @@ public void analyze(Document doc, StreamSource src, Writer xrefOut) throws IOExc
                 fout.write(ename);
                 fout.write("\n");
 
-                AnalyzerFactory fac = AnalyzerGuru.find(ename);
+                AnalyzerFactory fac = RuntimeEnvironment.getInstance().getAnalyzerGuru().find(ename);
                 if (fac instanceof JavaClassAnalyzerFactory) {
                     if (xrefOut != null) {
                         xrefOut.append("<br/>");