Merge pull request #2963 from idodeclare/feature/ctags_opts

Feature/ctags opts

Author: tarzanek

opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/AbstractAnalyzer.java

@@ -20,7 +20,7 @@
 /*
  * Copyright (c) 2005, 2019, Oracle and/or its affiliates. All rights reserved.
  * Use is subject to license terms.
- * Portions Copyright (c) 2017-2018, Chris Fraire <[email protected]>.
+ * Portions Copyright (c) 2017-2019, Chris Fraire <[email protected]>.
  */
 package org.opengrok.indexer.analysis;
 
@@ -34,7 +34,6 @@
 import org.opengrok.indexer.configuration.Project;
 
 /**
- *
  * @author Chandan
  */
 public abstract class AbstractAnalyzer extends Analyzer {
@@ -52,16 +51,25 @@ public AbstractAnalyzer(ReuseStrategy reuseStrategy) {
         super(reuseStrategy);
     }
 
+    /**
+     * Subclasses should override to return the case-insensitive name aligning
+     * with either a built-in Universal Ctags language name or an OpenGrok
+     * custom language name.
+     * @return a defined instance or {@code null} if the analyzer has no aligned
+     * Universal Ctags language
+     */
+    public abstract String getCtagsLang();
+
     public abstract long getVersionNo();
 
     /**
      * Subclasses should override to produce a value relevant for the evolution
      * of their analysis in each release.
      *
-     * @return 0
+     * @return 0 since {@link AbstractAnalyzer} is not specialized
      */
     protected int getSpecializedVersionNo() {
-        return 0; // FileAnalyzer is not specialized.
+        return 0;
     }
 
     public void setCtags(Ctags ctags) {

opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/AnalyzerGuru.java

@@ -240,6 +240,8 @@ public class AnalyzerGuru {
      */
     private static final Map<String, Long> ANALYZER_VERSIONS = new HashMap<>();
 
+    private static final LangTreeMap langMap = new LangTreeMap();
+
     /*
      * If you write your own analyzer please register it here. The order is
      * important for any factory that uses a FileAnalyzerFactory.Matcher
@@ -431,8 +433,10 @@ public static void addPrefix(String prefix, AnalyzerFactory factory) {
         AnalyzerFactory oldFactory;
         if (factory == null) {
             oldFactory = pre.remove(prefix);
+            langMap.exclude(prefix);
         } else {
             oldFactory = pre.put(prefix, factory);
+            langMap.add(prefix, factory.getAnalyzer().getCtagsLang());
         }
 
         if (factoriesDifferent(factory, oldFactory)) {
@@ -448,21 +452,39 @@ public static void addPrefix(String prefix, AnalyzerFactory factory) {
      * @param factory a factory which creates the analyzer to use for the given
      * extension (if you pass null as the analyzer, you will disable the
      * analyzer used for that extension)
+     * @throws IllegalArgumentException if {@code extension} contains a period
      */
-    public static void addExtension(String extension,
-            AnalyzerFactory factory) {
+    public static void addExtension(String extension, AnalyzerFactory factory) {
+        if (extension.contains(".")) {
+            throw new IllegalArgumentException("extension contains a '.'");
+        }
+
+        // LangMap fileSpec requires a leading period to indicate an extension.
+        String langMapExtension = "." + extension;
+
         AnalyzerFactory oldFactory;
         if (factory == null) {
             oldFactory = ext.remove(extension);
+            langMap.exclude(langMapExtension);
         } else {
             oldFactory = ext.put(extension, factory);
+            langMap.add(langMapExtension, factory.getAnalyzer().getCtagsLang());
         }
 
         if (factoriesDifferent(factory, oldFactory)) {
             addCustomizationKey("e:" + extension);
         }
     }
 
+    /**
+     * Gets an unmodifiable view of the language mappings resulting from
+     * {@link #addExtension(String, AnalyzerFactory)} and
+     * {@link #addPrefix(String, AnalyzerFactory)}.
+     */
+    public static LangMap getLangMap() {
+        return langMap.unmodifiable();
+    }
+
     /**
      * Get the default Analyzer.
      *

opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/Ctags.java

@@ -59,6 +59,8 @@ public class Ctags implements Resettable {
     private static final Logger LOGGER = LoggerFactory.getLogger(Ctags.class);
 
     private volatile boolean closing;
+    private final LangTreeMap defaultLangMap = new LangTreeMap();
+    private LangMap langMap;
     private Process ctags;
     private OutputStreamWriter ctagsIn;
     private BufferedReader ctagsOut;
@@ -88,6 +90,10 @@ public void setBinary(String binary) {
         this.binary = binary;
     }
 
+    public void setLangMap(LangMap langMap) {
+        this.langMap = langMap;
+    }
+
     public int getTabSize() {
         return tabSize;
     }
@@ -152,13 +158,15 @@ private void initialize() throws IOException {
         command.add("--filter-terminator=" + CTAGS_FILTER_TERMINATOR + "\n");
         command.add("--fields=-anf+iKnS");
         command.add("--excmd=pattern");
-        command.add("--langmap=sh:+.kshlib"); // RFE #17849
-        command.add("--langmap=sql:+.plb"); // RFE #19208
-        command.add("--langmap=sql:+.pls"); // RFE #19208
-        command.add("--langmap=sql:+.pld"); // RFE #19208
-        command.add("--langmap=sql:+.pks"); // RFE #19208 ?
-        command.add("--langmap=sql:+.pkb"); // # 1763
-        command.add("--langmap=sql:+.pck"); // # 1763
+
+        defaultLangMap.clear();
+        defaultLangMap.add(".KSHLIB", "sh"); // RFE #17849. Upper-case file spec
+        defaultLangMap.add(".PLB", "sql"); // RFE #19208. Upper-case file spec
+        defaultLangMap.add(".PLS", "sql"); // RFE #19208. Upper-case file spec
+        defaultLangMap.add(".PLD", "sql"); // RFE #19208. Upper-case file spec
+        defaultLangMap.add(".PKS", "sql"); // RFE #19208 ? Upper-case file spec
+        defaultLangMap.add(".PKB", "sql"); // # 1763. Upper-case file spec
+        defaultLangMap.add(".PCK", "sql"); // # 1763. Upper-case file spec
 
         //Ideally all below should be in ctags, or in outside config file,
         //we might run out of command line SOON
@@ -185,6 +193,12 @@ private void initialize() throws IOException {
 
         //PLEASE add new languages ONLY with POSIX syntax (see above wiki link)
 
+        if (langMap != null) {
+            command.addAll(langMap.mergeSecondary(defaultLangMap).getCtagsArgs());
+        } else {
+            command.addAll(defaultLangMap.getCtagsArgs());
+        }
+
         /* Add extra command line options for ctags. */
         if (CTagsExtraOptionsFile != null) {
             LOGGER.log(Level.INFO, "Adding extra options to ctags");
@@ -229,7 +243,7 @@ private void initialize() throws IOException {
 
     private void addRustSupport(List<String> command) {
         command.add("--langdef=rust");
-        command.add("--langmap=rust:+.rs");
+        defaultLangMap.add(".RS", "rust"); // Upper-case file spec
 
         // The following are not supported yet in Universal Ctags b13cb551
         command.add("--regex-rust=/^[[:space:]]*(pub[[:space:]]+)?(static|const)[[:space:]]+(mut[[:space:]]+)?" +
@@ -242,26 +256,27 @@ private void addRustSupport(List<String> command) {
     }
 
     private void addPowerShellSupport(List<String> command) {
-        command.add("--langdef=Posh");
-        command.add("--langmap=Posh:+.ps1,Posh:+.psm1");
-        command.add("--regex-Posh=/\\$(\\{[^}]+\\})/\\1/v,variable/");
-        command.add("--regex-Posh=/\\$([[:alnum:]_]+([:.][[:alnum:]_]+)*)/\\1/v,variable/");
-        command.add("--regex-Posh=/^[[:space:]]*(:[^[:space:]]+)/\\1/l,label/");
-
-        command.add("--_fielddef-Posh=signature,signatures");
-        command.add("--fields-Posh=+{signature}");
+        command.add("--langdef=powershell");
+        defaultLangMap.add(".PS1", "powershell"); // Upper-case file spec
+        defaultLangMap.add(".PSM1", "powershell"); // Upper-case file spec
+        command.add("--regex-powershell=/\\$(\\{[^}]+\\})/\\1/v,variable/");
+        command.add("--regex-powershell=/\\$([[:alnum:]_]+([:.][[:alnum:]_]+)*)/\\1/v,variable/");
+        command.add("--regex-powershell=/^[[:space:]]*(:[^[:space:]]+)/\\1/l,label/");
+
+        command.add("--_fielddef-powershell=signature,signatures");
+        command.add("--fields-powershell=+{signature}");
         // escaped variable markers
-        command.add("--regex-Posh=/`\\$([[:alnum:]_]+([:.][[:alnum:]_]+)*)/\\1//{exclusive}");
-        command.add("--regex-Posh=/`\\$(\\{[^}]+\\})/\\1//{exclusive}");
-        command.add("--regex-Posh=/#.*\\$([[:alnum:]_]+([:.][[:alnum:]_]+)*)/\\1//{exclusive}");
-        command.add("--regex-Posh=/#.*\\$(\\{[^}]+\\})/\\1//{exclusive}");
-        command.add("--regex-Posh=/^[[:space:]]*(function|filter)[[:space:]]+([^({[:space:]]+)[[:space:]]*" +
+        command.add("--regex-powershell=/`\\$([[:alnum:]_]+([:.][[:alnum:]_]+)*)/\\1//{exclusive}");
+        command.add("--regex-powershell=/`\\$(\\{[^}]+\\})/\\1//{exclusive}");
+        command.add("--regex-powershell=/#.*\\$([[:alnum:]_]+([:.][[:alnum:]_]+)*)/\\1//{exclusive}");
+        command.add("--regex-powershell=/#.*\\$(\\{[^}]+\\})/\\1//{exclusive}");
+        command.add("--regex-powershell=/^[[:space:]]*(function|filter)[[:space:]]+([^({[:space:]]+)[[:space:]]*" +
                 "(\\(([^)]+)\\))?/\\2/f,function,functions/{icase}{exclusive}{_field=signature:(\\4)}");
     }
 
     private void addPascalSupport(List<String> command) {
         command.add("--langdef=pascal");
-        command.add("--langmap=pascal:+.pas");
+        defaultLangMap.add(".PAS", "pascal"); // Upper-case file spec
         command.add("--regex-pascal=/([[:alnum:]_]+)[[:space:]]*=[[:space:]]*\\([[:space:]]*[[:alnum:]_][[:space:]]*\\)/\\1/t,Type/");
         command.add("--regex-pascal=/([[:alnum:]_]+)[[:space:]]*=[[:space:]]*class[[:space:]]*[^;]*$/\\1/c,Class/");
         command.add("--regex-pascal=/([[:alnum:]_]+)[[:space:]]*=[[:space:]]*interface[[:space:]]*[^;]*$/\\1/i,interface/");
@@ -276,7 +291,7 @@ private void addPascalSupport(List<String> command) {
 
     private void addSwiftSupport(List<String> command) {
         command.add("--langdef=swift");
-        command.add("--langmap=swift:+.swift");
+        defaultLangMap.add(".SWIFT", "swift"); // Upper-case file spec
         command.add("--regex-swift=/enum[[:space:]]+([^\\{\\}]+).*$/\\1/n,enum,enums/");
         command.add("--regex-swift=/typealias[[:space:]]+([^:=]+).*$/\\1/t,typealias,typealiases/");
         command.add("--regex-swift=/protocol[[:space:]]+([^:\\{]+).*$/\\1/p,protocol,protocols/");
@@ -289,8 +304,8 @@ private void addSwiftSupport(List<String> command) {
 
     private void addKotlinSupport(List<String> command) {
         command.add("--langdef=kotlin");
-        command.add("--langmap=kotlin:+.kt");
-        command.add("--langmap=kotlin:+.kts");
+        defaultLangMap.add(".KT", "kotlin"); // Upper-case file spec
+        defaultLangMap.add(".KTS", "kotlin"); // Upper-case file spec
         command.add("--regex-kotlin=/^[[:space:]]*((abstract|final|sealed|implicit|lazy)[[:space:]]*)*" +
                 "(private[^ ]*|protected)?[[:space:]]*class[[:space:]]+([[:alnum:]_:]+)/\\4/c,classes/");
         command.add("--regex-kotlin=/^[[:space:]]*((abstract|final|sealed|implicit|lazy)[[:space:]]*)*" +
@@ -313,9 +328,9 @@ private void addKotlinSupport(List<String> command) {
 
     private void addClojureSupport(List<String> command) {
         command.add("--langdef=clojure"); // clojure support (patterns are from https://gist.github.com/kul/8704283)
-        command.add("--langmap=clojure:+.clj");
-        command.add("--langmap=clojure:+.cljs");
-        command.add("--langmap=clojure:+.cljx");
+        defaultLangMap.add(".CLJ", "clojure"); // Upper-case file spec
+        defaultLangMap.add(".CLJS", "clojure"); // Upper-case file spec
+        defaultLangMap.add(".CLJX", "clojure"); // Upper-case file spec
 
         command.add("--regex-clojure=/\\([[:space:]]*create-ns[[:space:]]+([-[:alnum:]*+!_:\\/.?]+)/\\1/n,namespace/");
         command.add("--regex-clojure=/\\([[:space:]]*def[[:space:]]+([-[:alnum:]*+!_:\\/.?]+)/\\1/d,definition/");
@@ -331,8 +346,8 @@ private void addClojureSupport(List<String> command) {
 
     private void addHaskellSupport(List<String> command) {
         command.add("--langdef=haskell"); // below was added with #912
-        command.add("--langmap=haskell:+.hs");
-        command.add("--langmap=haskell:+.hsc");
+        defaultLangMap.add(".HS", "haskell"); // Upper-case file spec
+        defaultLangMap.add(".HSC", "haskell"); // Upper-case file spec
         command.add("--regex-haskell=/^[[:space:]]*class[[:space:]]+([a-zA-Z0-9_]+)/\\1/c,classes/");
         command.add("--regex-haskell=/^[[:space:]]*data[[:space:]]+([a-zA-Z0-9_]+)/\\1/t,types/");
         command.add("--regex-haskell=/^[[:space:]]*newtype[[:space:]]+([a-zA-Z0-9_]+)/\\1/t,types/");
@@ -345,7 +360,7 @@ private void addHaskellSupport(List<String> command) {
 
     private void addScalaSupport(List<String> command) {
         command.add("--langdef=scala"); // below is bug 61 to get full scala support
-        command.add("--langmap=scala:+.scala");
+        defaultLangMap.add(".SCALA", "scala"); // Upper-case file spec
         command.add("--regex-scala=/^[[:space:]]*((abstract|final|sealed|implicit|lazy)[[:space:]]*)*" +
                 "(private|protected)?[[:space:]]*class[[:space:]]+([a-zA-Z0-9_]+)/\\4/c,classes/");
         command.add("--regex-scala=/^[[:space:]]*((abstract|final|sealed|implicit|lazy)[[:space:]]*)*" +

opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/FileAnalyzer.java

@@ -20,7 +20,7 @@
 /*
  * Copyright (c) 2005, 2019, Oracle and/or its affiliates. All rights reserved.
  * Use is subject to license terms.
- * Portions Copyright (c) 2017-2018, Chris Fraire <[email protected]>.
+ * Portions Copyright (c) 2017-2019, Chris Fraire <[email protected]>.
  */
 package org.opengrok.indexer.analysis;
 
@@ -59,6 +59,14 @@ public class FileAnalyzer extends AbstractAnalyzer {
 
     private static final Logger LOGGER = LoggerFactory.getLogger(FileAnalyzer.class);
 
+    /**
+     * @return {@code null} as there is no aligned language
+     */
+    @Override
+    public String getCtagsLang() {
+        return null;
+    }
+
     /**
      * Gets a version number to be used to tag processed documents so that
      * re-analysis can be re-done later if a stored version number is different

opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/LangMap.java

@@ -0,0 +1,89 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * See LICENSE.txt included in this distribution for the specific
+ * language governing permissions and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at LICENSE.txt.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+/*
+ * Copyright (c) 2019, Chris Fraire <[email protected]>.
+ */
+
+package org.opengrok.indexer.analysis;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Represents an API for mapping file specifications versus languages and
+ * getting the ctags options representation (--langmap-&lt;LANG&gt; or
+ * --map-&lt;LANG&gt;) thereof.
+ */
+public interface LangMap {
+
+    /**
+     * Removes all settings from this map.
+     */
+    void clear();
+
+    /**
+     * Adds the specified mapping of a file specification to a language. Any
+     * matching exclusion via {@link #exclude(String)} is undone.
+     * @param fileSpec a value starting with a period ({@code '.'}) to specify
+     *                 a file extension; otherwise specifying a prefix.
+     * @throws IllegalArgumentException if {@code fileSpec} is {@code null} or
+     * is an extension (i.e. starting with a period) but contains any other
+     * periods, as that is not ctags-compatible
+     */
+    void add(String fileSpec, String ctagsLang);
+
+    /**
+     * Exclude the specified mapping of a file specification to any language.
+     * Any matching addition via {@link #add(String, String)} is undone.
+     * @throws IllegalArgumentException if {@code fileSpec} is {@code null}
+     */
+    void exclude(String fileSpec);
+
+    /**
+     * Gets the transformation of the instance's mappings to ctags arguments.
+     */
+    List<String> getCtagsArgs();
+
+    /**
+     * Creates a new instance, merging the settings from the current instance
+     * overlaying a specified {@code other}. Additions from the current instance
+     * take precedence, and exclusions from the {@code other} only take effect
+     * if the current instance has no matching addition.
+     * @param other a defined instance
+     * @return a defined instance
+     */
+    LangMap mergeSecondary(LangMap other);
+
+    /**
+     * Gets an unmodifiable view of the current instance.
+     */
+    LangMap unmodifiable();
+
+    /**
+     * Gets an unmodifiable view of the current additions.
+     */
+    Map<String, String> getAdditions();
+
+    /**
+     * Gets an unmodifiable view of the current exclusions.
+     */
+    Set<String> getExclusions();
+}