Resolve #2613 : support HCL and a Terraform derivation

Author: idodeclare

opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/AnalyzerGuru.java

@@ -19,7 +19,7 @@
 
 /*
  * Copyright (c) 2005, 2018, Oracle and/or its affiliates. All rights reserved.
- * Portions Copyright (c) 2017-2019, Chris Fraire <[email protected]>.
+ * Portions Copyright (c) 2017-2020, Chris Fraire <[email protected]>.
  */
 package org.opengrok.indexer.analysis;
 
@@ -30,7 +30,6 @@
 import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.io.Reader;
-import java.io.StringReader;
 import java.io.Writer;
 import java.lang.reflect.InvocationTargetException;
 import java.nio.charset.StandardCharsets;
@@ -80,6 +79,7 @@
 import org.opengrok.indexer.analysis.fortran.FortranAnalyzerFactory;
 import org.opengrok.indexer.analysis.golang.GolangAnalyzerFactory;
 import org.opengrok.indexer.analysis.haskell.HaskellAnalyzerFactory;
+import org.opengrok.indexer.analysis.hcl.HCLAnalyzerFactory;
 import org.opengrok.indexer.analysis.java.JavaAnalyzerFactory;
 import org.opengrok.indexer.analysis.javascript.JavaScriptAnalyzerFactory;
 import org.opengrok.indexer.analysis.json.JsonAnalyzerFactory;
@@ -101,6 +101,7 @@
 import org.opengrok.indexer.analysis.sql.SQLAnalyzerFactory;
 import org.opengrok.indexer.analysis.swift.SwiftAnalyzerFactory;
 import org.opengrok.indexer.analysis.tcl.TclAnalyzerFactory;
+import org.opengrok.indexer.analysis.terraform.TerraformAnalyzerFactory;
 import org.opengrok.indexer.analysis.typescript.TypeScriptAnalyzerFactory;
 import org.opengrok.indexer.analysis.uue.UuencodeAnalyzerFactory;
 import org.opengrok.indexer.analysis.vb.VBAnalyzerFactory;
@@ -221,8 +222,6 @@ public class AnalyzerGuru {
      */
     private static final List<String> analysisPkgNames = new ArrayList<>();
 
-    public static final Reader dummyR = new StringReader("");
-    public static final String dummyS = "";
     public static final FieldType string_ft_stored_nanalyzed_norms = new FieldType(StringField.TYPE_STORED);
     public static final FieldType string_ft_nstored_nanalyzed_norms = new FieldType(StringField.TYPE_NOT_STORED);
 
@@ -300,7 +299,9 @@ public class AnalyzerGuru {
                 new EiffelAnalyzerFactory(),
                 new VerilogAnalyzerFactory(),
                 new TypeScriptAnalyzerFactory(),
-                new AsmAnalyzerFactory()
+                new AsmAnalyzerFactory(),
+                new HCLAnalyzerFactory(),
+                new TerraformAnalyzerFactory()
             };
 
             for (AnalyzerFactory analyzer : analyzers) {
@@ -334,15 +335,15 @@ public class AnalyzerGuru {
      * {@link FileAnalyzerFactory} subclasses are revised to target more or
      * different files.
      * @return a value whose lower 32-bits are a static value
-     * 20191120_00
+     * 20200410_00
      * for the current implementation and whose higher-32 bits are non-zero if
      * {@link #addExtension(java.lang.String, AnalyzerFactory)}
      * or
      * {@link #addPrefix(java.lang.String, AnalyzerFactory)}
      * has been called.
      */
     public static long getVersionNo() {
-        final int ver32 = 20191120_00; // Edit comment above too!
+        final int ver32 = 20200410_00; // Edit comment above too!
         long ver = ver32;
         if (customizationHashCode != 0) {
             ver |= (long) customizationHashCode << 32;
@@ -633,26 +634,6 @@ public void populateDocument(Document doc, File file, String path,
         }
     }
 
-    /**
-     * Get the content type for a named file.
-     *
-     * @param in The input stream we want to get the content type for (if we
-     * cannot determine the content type by the filename)
-     * @param file The name of the file
-     * @return The contentType suitable for printing to
-     * response.setContentType() or null if the factory was not found
-     * @throws java.io.IOException If an error occurs while accessing the input
-     * stream.
-     */
-    public static String getContentType(InputStream in, String file) throws IOException {
-        AnalyzerFactory factory = find(in, file);
-        String type = null;
-        if (factory != null) {
-            type = factory.getContentType();
-        }
-        return type;
-    }
-
     /**
      * Write a browse-able version of the file.
      *
@@ -780,7 +761,7 @@ public static AnalyzerFactory findByFileTypeName(String fileTypeName) {
     public static AnalyzerFactory findFactory(String factoryClassName)
             throws ClassNotFoundException, IllegalAccessException, InstantiationException, NoSuchMethodException, 
             InvocationTargetException {
-        Class<?> fcn = null;
+        Class<?> fcn;
         try {
             fcn = Class.forName(factoryClassName);
 
@@ -1031,9 +1012,7 @@ private static AnalyzerFactory findForStream(InputStream in,
         return null;
     }
 
-    private static AnalyzerFactory findMagicString(String opening,
-        String file)
-            throws IOException {
+    private static AnalyzerFactory findMagicString(String opening, String file) {
 
         // first, try to look up two words in magics
         String fragment = getWords(opening, 2);
@@ -1207,6 +1186,6 @@ private static boolean factoriesDifferent(AnalyzerFactory a,
         if (a_name == null && b_name == null) {
             return false;
         }
-        return a_name == null || b_name == null || !a_name.equals(b_name);
+        return a_name == null || !a_name.equals(b_name);
     }
 }

opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/Ctags.java

@@ -19,7 +19,7 @@
 
  /*
  * Copyright (c) 2005, 2018, Oracle and/or its affiliates. All rights reserved.
- * Portions Copyright (c) 2017-2019, Chris Fraire <[email protected]>.
+ * Portions Copyright (c) 2017-2020, Chris Fraire <[email protected]>.
  */
 package org.opengrok.indexer.analysis;
 
@@ -185,22 +185,15 @@ private void initialize() {
         //on Solaris regexp.h used is different than on linux (gnu regexp)
         //http://en.wikipedia.org/wiki/Regular_expression#POSIX_basic_and_extended
         addScalaSupport(command);
-
         addHaskellSupport(command);
-
         //temporarily use our defs until ctags will fix https://github.com/universal-ctags/ctags/issues/988
         addClojureSupport(command);
-
         addKotlinSupport(command);
-
         addSwiftSupport(command);
-
         addRustSupport(command);
-
         addPascalSupport(command);
-
         addPowerShellSupport(command);
-
+        addTerraformSupport(command);
         //PLEASE add new languages ONLY with POSIX syntax (see above wiki link)
 
         if (langMap == null) {
@@ -401,6 +394,30 @@ private void addScalaSupport(List<String> command) {
         command.add("--regex-scala=/^[[:space:]]*package[[:space:]]+([a-zA-Z0-9_.]+)/\\1/p,packages/");
     }
 
+    private void addTerraformSupport(List<String> command) {
+        if (!env.getCtagsLanguages().contains("Terraform")) { // Built-in would be capitalized.
+            command.add("--langdef=terraform"); // Lower-case if user-defined.
+        }
+
+        /*
+         * Ignore Terraform single-line comments with short-form (only two
+         * separators following the pattern), exclusive matches.
+         */
+        command.add("--regex-terraform=,^[[:space:]]*#,,{exclusive}");
+        command.add("--regex-terraform=,^[[:space:]]*//,,{exclusive}");
+
+        /*
+         * Terraform "resource block declares a resource of a given type ...
+         * with a given local name...." Unfortunately there is no Posix
+         * equivalent of {Identifier} from HCL.lexh, so we must approximate with
+         * the possibility of leaving out some matches.
+         */
+        command.add("--regex-terraform=" +
+                "/[[:<:]]resource[[:space:]]*\"([[:alpha:]][-_[:alpha:]]*)\"[[:space:]]*" +
+                "\"([[:alpha:]][-_[:alpha:]]*)\"[[:space:]]*\\{/" +
+                "\\1.\\2/s,struct,resource names/");
+    }
+
     /**
      * Run ctags on a file.
      * @param file file path to process

opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/hcl/Consts.java

@@ -0,0 +1,63 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * See LICENSE.txt included in this distribution for the specific
+ * language governing permissions and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at LICENSE.txt.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+/*
+ * Copyright (c) 2020, Chris Fraire <[email protected]>.
+ */
+
+package org.opengrok.indexer.analysis.hcl;
+
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
+
+/**
+ * Represents a container for HCL keywords and other string constants.
+ */
+public class Consts {
+
+    private static final Set<String> kwd = new HashSet<>();
+
+    public static final Set<String> KEYWORDS = Collections.unmodifiableSet(kwd);
+
+    static {
+        /*
+         * HCL has the irritating aspect that "there are no globally-reserved
+         * words, but in some contexts certain identifiers are reserved to
+         * function as keywords."
+         *
+         * We'll just treat the following as globally reserved and wait to see
+         * if that causes problems for any users.
+         */
+
+        kwd.add("false");
+        kwd.add("true");
+        kwd.add("null");
+        kwd.add("for");
+        kwd.add("endfor");
+        kwd.add("in");
+        kwd.add("if");
+        kwd.add("else");
+        kwd.add("endif");
+    }
+
+    /* private to enforce static */
+    private Consts() {
+    }
+}

opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/hcl/HCLAnalyzer.java

@@ -0,0 +1,78 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * See LICENSE.txt included in this distribution for the specific
+ * language governing permissions and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at LICENSE.txt.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+/*
+ * Copyright (c) 2010, 2018, Oracle and/or its affiliates. All rights reserved.
+ * Portions Copyright (c) 2017-2020, Chris Fraire <[email protected]>.
+ */
+
+package org.opengrok.indexer.analysis.hcl;
+
+import org.opengrok.indexer.analysis.AbstractAnalyzer;
+import org.opengrok.indexer.analysis.AnalyzerFactory;
+import org.opengrok.indexer.analysis.JFlexTokenizer;
+import org.opengrok.indexer.analysis.JFlexXref;
+import org.opengrok.indexer.analysis.plain.AbstractSourceCodeAnalyzer;
+
+import java.io.Reader;
+
+/**
+ * Represents an extension of {@link AbstractSourceCodeAnalyzer} for the HCL
+ * configuration language.
+ */
+public class HCLAnalyzer extends AbstractSourceCodeAnalyzer {
+
+    /**
+     * Creates a new instance of {@link HCLAnalyzer}.
+     * @param factory defined instance for the analyzer
+     */
+    protected HCLAnalyzer(AnalyzerFactory factory) {
+        super(factory, () -> new JFlexTokenizer(new HCLSymbolTokenizer(
+                AbstractAnalyzer.DUMMY_READER)));
+    }
+
+    /**
+     * @return {@code null} as there is no aligned language
+     */
+    @Override
+    public String getCtagsLang() {
+        return null;
+    }
+
+    /**
+     * Gets a version number to be used to tag processed documents so that
+     * re-analysis can be re-done later if a stored version number is different
+     * from the current implementation.
+     * @return 20200409_05
+     */
+    @Override
+    protected int getSpecializedVersionNo() {
+        return 20200409_05; // Edit comment above too!
+    }
+
+    /**
+     * Creates a wrapped instance of {@link HCLXref}.
+     * @param reader an instance passed to the new {@link HCLXref}
+     * @return a defined instance
+     */
+    @Override
+    protected JFlexXref newXref(Reader reader) {
+        return new JFlexXref(new HCLXref(reader));
+    }
+}

opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/hcl/HCLAnalyzerFactory.java

@@ -0,0 +1,55 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * See LICENSE.txt included in this distribution for the specific
+ * language governing permissions and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at LICENSE.txt.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+/*
+ * Copyright (c) 2010, 2018 Oracle and/or its affiliates. All rights reserved.
+ * Portions Copyright (c) 2017, 2020, Chris Fraire <[email protected]>.
+ */
+
+package org.opengrok.indexer.analysis.hcl;
+
+import org.opengrok.indexer.analysis.AbstractAnalyzer;
+import org.opengrok.indexer.analysis.FileAnalyzerFactory;
+
+/**
+ * Represents an extension of {@link FileAnalyzerFactory} to produce
+ * {@link HCLAnalyzer} instances.
+ */
+public class HCLAnalyzerFactory extends FileAnalyzerFactory {
+
+    private static final String name = "HCL";
+
+    private static final String[] SUFFIXES = {"HCL"};
+
+    /**
+     * Creates a new instance of {@link HCLAnalyzerFactory}.
+     */
+    public HCLAnalyzerFactory() {
+        super(null, null, SUFFIXES, null, null, "text/plain", AbstractAnalyzer.Genre.PLAIN, name);
+    }
+
+    /**
+     * Creates a new instance of {@link HCLAnalyzer}.
+     * @return the new instance
+     */
+    @Override
+    protected AbstractAnalyzer newAnalyzer() {
+        return new HCLAnalyzer(this);
+    }
+}