Adding Powershell analyzer #1753

This is the initial introduction of the PowerShell script analyzer for OpenGrok.

Author: shaehn

build.xml

@@ -246,6 +246,8 @@ Copyright (c) 2005, 2017, Oracle and/or its affiliates. All rights reserved.
         <run-jflex dir="${gen.analysis.dir}/javascript" name="JavaScriptXref"/>
         <run-jflex dir="${gen.analysis.dir}/lua" name="LuaSymbolTokenizer"/>
         <run-jflex dir="${gen.analysis.dir}/lua" name="LuaXref"/>
+        <run-jflex dir="${gen.analysis.dir}/posh" name="PoshSymbolTokenizer"/>
+        <run-jflex dir="${gen.analysis.dir}/posh" name="PoshXref"/>
         <run-jflex dir="${gen.analysis.dir}/python" name="PythonSymbolTokenizer"/>
         <run-jflex dir="${gen.analysis.dir}/python" name="PythonXref"/>
         <run-jflex dir="${gen.analysis.dir}/rust" name="RustSymbolTokenizer"/>

src/org/opensolaris/opengrok/analysis/AnalyzerGuru.java

@@ -82,6 +82,7 @@
 import org.opensolaris.opengrok.analysis.json.JsonAnalyzerFactory;
 import org.opensolaris.opengrok.analysis.kotlin.KotlinAnalyzerFactory;
 import org.opensolaris.opengrok.analysis.sh.ShAnalyzerFactory;
+import org.opensolaris.opengrok.analysis.posh.PoshAnalyzerFactory;
 import org.opensolaris.opengrok.analysis.sql.PLSQLAnalyzerFactory;
 import org.opensolaris.opengrok.analysis.sql.SQLAnalyzerFactory;
 import org.opensolaris.opengrok.analysis.swift.SwiftAnalyzerFactory;
@@ -177,6 +178,7 @@ public class AnalyzerGuru {
             new CxxAnalyzerFactory(),
             new ErlangAnalyzerFactory(),
             new ShAnalyzerFactory(),
+            new PoshAnalyzerFactory(),
             PlainAnalyzerFactory.DEFAULT_INSTANCE,
             new UuencodeAnalyzerFactory(),
             new GZIPAnalyzerFactory(),

src/org/opensolaris/opengrok/analysis/Ctags.java

@@ -213,7 +213,8 @@ private void initialize() throws IOException {
 
             command.add("--langdef=rust");
             command.add("--langmap=rust:+.rs");
-            command.add("--regex-rust=/^[[:space:]]*(#\\[[^\\]]\\][[:space:]]*)*(pub[[:space:]]+)?(extern[[:space:]]+)?(\"[^\"]+\"[[:space:]]+)?(unsafe[[:space:]]+)?fn[[:space:]]+([[:alnum:]_]+)/\\6/h,functions,function definitions/");
+          //command.add("--regex-rust=/^[[:space:]]*(#\\[[^\\]]\\][[:space:]]*)*(pub[[:space:]]+)?(extern[[:space:]]+)?(\"[^\"]+\"[[:space:]]+)?(unsafe[[:space:]]+)?fn[[:space:]]+([[:alnum:]_]+)/\\6/h,functions,function definitions/");
+            command.add("--regex-rust=/^[[:space:]]*(#\\[[^]]+\\][[:space:]]*)*(pub[[:space:]]+)?(extern[[:space:]]+)?(\\\"[^\\\"]+\\\"[[:space:]]+)?(unsafe[[:space:]]+)?fn[[:space:]]+([[:alnum:]_]+)/\\6/h,functions,function definitions/");
             command.add("--regex-rust=/^[[:space:]]*(pub[[:space:]]+)?type[[:space:]]+([[:alnum:]_]+)/\\2/T,types,type definitions/");
             command.add("--regex-rust=/^[[:space:]]*(pub[[:space:]]+)?enum[[:space:]]+([[:alnum:]_]+)/\\2/g,enum,enumeration names/");
             command.add("--regex-rust=/^[[:space:]]*(pub[[:space:]]+)?struct[[:space:]]+([[:alnum:]_]+)/\\2/S,structure names/");
@@ -237,6 +238,24 @@ private void initialize() throws IOException {
             command.add("--regex-pascal=/^(uses|interface|implementation)$/\\1/s,Section/");
             command.add("--regex-pascal=/^unit[[:space:]]+([a-zA-Z0-9_<>, ]+)[;(]/\\1/u,unit/");
 
+            // PowerShell
+            command.add("--langdef=Posh");
+            command.add("--langmap=Posh:+.ps1,Posh:+.psm1");
+            command.add("--kinddef-Posh=f,function,functions");
+            command.add("--_fielddef-Posh=signature,signatures");
+            command.add("--fields-Posh=+{signature}");
+
+            // escaped variable markers
+            command.add("--regex-Posh=/`(\\$[[:alnum:]_]+([:.][[:alnum:]_]+)*)/\\1//{exclusive}");
+            command.add("--regex-Posh=/`(\\$\\{[^}]+\\})/\\1//{exclusive}");
+            command.add("--regex-Posh=/#.*(\\$[[:alnum:]_]+([:.][[:alnum:]_]+)*)/\\1//{exclusive}");
+            command.add("--regex-Posh=/#.*(\\$\\{[^}]+\\})/\\1//{exclusive}");
+
+            command.add("--regex-Posh=/(\\$\\{[^}]+\\})/\\1/v,variable/");
+            command.add("--regex-Posh=/(\\$[[:alnum:]_]+([:.][[:alnum:]_]+)*)/\\1/v,variable/");
+            command.add("--regex-Posh=/^[[:space:]]*(function|filter)[[:space:]]+([^({[:space:]]+)[[:space:]]*(\\(([^)]+)\\))?/\\2/f/{icase}{exclusive}{_field=signature:(\\4)}");
+            command.add("--regex-Posh=/^[[:space:]]*(:[^[:space:]]+)/\\1/l,label/");
+            
             //PLEASE add new languages ONLY with POSIX syntax (see above wiki link)
 
             /* Add extra command line options for ctags. */
@@ -516,19 +535,43 @@ private void readTags(Definitions defs) {
                     //TODO if some languages use different character for separating arguments, below needs to be adjusted
                     String[] args = signature.split(",");
                     for (String arg : args) {
-                        //log.fine("Param = "+ arg);
-                        int space = arg.lastIndexOf(' ');//TODO this is not the best way, but works to find the last string(name) in the argument, hence skipping type
-                        if (space > 0 && space < arg.length()) {
-                            String afters = arg.substring(space + 1);
-                            //FIXME this will not work for typeless languages such as python or assignments inside signature ... but since ctags doesn't provide signatures for python yet and assigning stuff in signature is not the case for c or java, we don't care ...
-                            String[] names = afters.split("[\\W]"); //this should just parse out variables, we assume first non empty text is the argument name
-                            for (String name : names) {
-                                if (name.length() > 0) {
-                                    //log.fine("Param Def = "+ string);
-                                    addTag(defs, seenSymbols, lnum, name, "argument",
+                        //TODO this algorithm assumes that data types occur to
+                        //     the left of the argument name, so it will not
+                        //     work for languages like rust, kotlin, etc. which
+                        //     place the data type to the right of the argument name.
+                        //     Need an attribute from ctags to indicate data type location.
+                        // ----------------------------------------------------------------
+                        // When no assignment of default values,
+                        // expecting: <type> <name>, or <name>
+                        //
+                        // When default value assignment applied to parameter,
+                        // expecting: <type> <name> = <value> or
+                        //            <name> = <value>
+                        // (Note whitespace content made irrelevant)
+
+                        // Need to ditch the default assignment value
+                        // so that the extraction loop below will work.
+                        // This assumes all languages use '=' to assign value.
+                        
+                        if (arg.indexOf("=") != -1) {
+                            String[] a = arg.split("=");
+                            arg = a[0];  // throws away assigned value
+                        }
+                        
+                        // Strip out all non 'word' class symbols
+                        // which leaves just names intact.
+                        String [] names = arg.trim().split("[\\W]");  
+                        String name;
+                        
+                        // Walk the array backwards from the end and
+                        // the parameter name should always be the first
+                        // non-empty element encountered.
+                        for (int ii=names.length-1; ii >= 0; ii--) {
+                            name = names[ii];
+                            if (name.length() > 0) {
+                                addTag(defs, seenSymbols, lnum, name, "argument",
                                             def.trim() + signature.trim(), null, signature);
                                     break;
-                                }
                             }
                         }
                     }

src/org/opensolaris/opengrok/analysis/JFlexXref.java

@@ -539,10 +539,10 @@ protected void startNewLine() throws IOException {
      */
     protected void writeSymbol(String symbol, Set<String> keywords, int line)
             throws IOException {
-        writeSymbol(symbol, keywords, line, true);
+        writeSymbol(symbol, keywords, line, true, false);
     }
 
-    /**
+        /**
      * Write a symbol and generate links as appropriate.
      *
      * @param symbol the symbol to write
@@ -555,9 +555,27 @@ protected void writeSymbol(String symbol, Set<String> keywords, int line)
     protected void writeSymbol(
             String symbol, Set<String> keywords, int line, boolean caseSensitive)
             throws IOException {
+        writeSymbol(symbol, keywords, line, caseSensitive, false);
+    }
+    
+    /**
+     * Write a symbol and generate links as appropriate.
+     *
+     * @param symbol the symbol to write
+     * @param keywords a set of keywords recognized by this analyzer (no links
+     * will be generated if the symbol is a keyword)
+     * @param line the line number on which the symbol appears
+     * @param caseSensitive Whether the keyword list is case sensitive
+     * @param quote Whether the symbol gets quoted in links or not
+     * @throws IOException if an error occurs while writing to the stream
+     */
+    protected void writeSymbol(
+            String symbol, Set<String> keywords, int line, boolean caseSensitive, boolean quote)
+            throws IOException {
         String[] strs = new String[1];
         strs[0] = "";
         String jsEscapedSymbol = symbol.replace("'", "\\'");
+        String qt = (quote) ? "&quot;" : "";
 
         String check = caseSensitive ? symbol : symbol.toLowerCase();
         if (keywords != null && keywords.contains( check )) {
@@ -596,7 +614,7 @@ protected void writeSymbol(
             out.append("<a href=\"");
             out.append(urlPrefix);
             out.append("refs=");
-            out.append(symbol);
+            out.append(qt+symbol+qt);
             appendProject();
             out.append("\" class=\"");
             out.append(style_class);
@@ -629,7 +647,7 @@ protected void writeSymbol(
             out.append("<a href=\"");
             out.append(urlPrefix);
             out.append("defs=");
-            out.append(symbol);
+            out.append(qt+symbol+qt);
             appendProject();
             out.append("\"");
             out.append(" class=\"intelliWindow-symbol\"");