Merge pull request #1987 from idodeclare/feature/loc

Feature/LOC

Author: tarzanek

src/org/opensolaris/opengrok/analysis/AnalyzerGuru.java

@@ -426,7 +426,8 @@ public void populateDocument(Document doc, File file, String path,
                 string_ft_stored_nanalyzed_norms));
         doc.add(new Field(QueryBuilder.FULLPATH, file.getAbsolutePath(),
                 string_ft_nstored_nanalyzed_norms));
-        doc.add(new SortedDocValuesField(QueryBuilder.FULLPATH, new BytesRef(file.getAbsolutePath())));
+        doc.add(new SortedDocValuesField(QueryBuilder.FULLPATH,
+                new BytesRef(file.getAbsolutePath())));
 
         if (RuntimeEnvironment.getInstance().isHistoryEnabled()) {
             try {
@@ -441,12 +442,26 @@ public void populateDocument(Document doc, File file, String path,
         }
         doc.add(new Field(QueryBuilder.DATE, date, string_ft_stored_nanalyzed_norms));
         doc.add(new SortedDocValuesField(QueryBuilder.DATE, new BytesRef(date)));
-        if (path != null) {
-            doc.add(new TextField(QueryBuilder.PATH, path, Store.YES));
-            Project project = Project.getProject(path);
-            if (project != null) {
-                doc.add(new TextField(QueryBuilder.PROJECT, project.getPath(), Store.YES));
-            }
+
+        // `path' is not null, as it was passed to Util.path2uid() above.
+        doc.add(new TextField(QueryBuilder.PATH, path, Store.YES));
+        Project project = Project.getProject(path);
+        if (project != null) {
+            doc.add(new TextField(QueryBuilder.PROJECT, project.getPath(), Store.YES));
+        }
+
+        /*
+         * Use the parent of the path -- not the absolute file as is done for
+         * FULLPATH -- so that DIRPATH is the same convention as for PATH
+         * above. A StringField, however, is used instead of a TextField.
+         */
+        File fpath = new File(path);
+        String fileParent = fpath.getParent();
+        if (fileParent != null && fileParent.length() > 0) {
+            String normalizedPath = QueryBuilder.normalizeDirPath(fileParent);
+            StringField npstring = new StringField(QueryBuilder.DIRPATH,
+                normalizedPath, Store.NO);
+            doc.add(npstring);
         }
 
         if (fa != null) {

src/org/opensolaris/opengrok/analysis/CommonXref.lexh

@@ -0,0 +1,29 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * See LICENSE.txt included in this distribution for the specific
+ * language governing permissions and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at LICENSE.txt.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+/*
+ * Copyright (c) 2017, Chris Fraire <[email protected]>.
+ */
+
+%{
+    /**
+     * Calls {@link #onSourceCodeSeen(int)} with {@code yychar}.
+     */
+    public void phLOC() { onSourceCodeSeen(yychar); }
+%}

src/org/opensolaris/opengrok/analysis/FileAnalyzer.java

@@ -33,10 +33,12 @@
 
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.document.Document;
+import org.apache.lucene.document.StoredField;
 import org.opensolaris.opengrok.analysis.plain.PlainFullTokenizer;
 import org.opensolaris.opengrok.analysis.plain.PlainSymbolTokenizer;
 import org.opensolaris.opengrok.configuration.Project;
 import org.opensolaris.opengrok.logger.LoggerFactory;
+import org.opensolaris.opengrok.search.QueryBuilder;
 
 /**
  * Base class for all different File Analyzers
@@ -245,6 +247,24 @@ protected TokenStreamComponents createComponents(String fieldName) {
         }
     }
 
+    /**
+     * Add a field to store document number of lines.
+     * @param doc the target document
+     * @param value the number of lines
+     */
+    protected void addNumLines(Document doc, int value)  {
+        doc.add(new StoredField(QueryBuilder.NUML, value));
+    }
+
+    /**
+     * Add a field to store document lines-of-code.
+     * @param doc the target document
+     * @param value the loc
+     */
+    protected void addLOC(Document doc, int value)  {
+        doc.add(new StoredField(QueryBuilder.LOC, value));
+    }
+
     private JFlexTokenizer createPlainSymbolTokenizer() {
         return new JFlexTokenizer(new PlainSymbolTokenizer(
             FileAnalyzer.dummyReader));

src/org/opensolaris/opengrok/analysis/JFlexJointLexer.java

@@ -79,4 +79,10 @@ boolean offerSymbol(String value, int captureOffset, boolean ignoreKwd)
      * @throws IOException if an output error occurs
      */
     void disjointSpan(String className) throws IOException;
+
+    /**
+     * Indicates that eligible source code was encountered for physical
+     * lines-of-code count (physical LOC).
+     */
+    void phLOC();
 }

src/org/opensolaris/opengrok/analysis/JFlexNonXref.java

@@ -75,6 +75,15 @@ public abstract class JFlexNonXref extends JFlexStateStacker
     protected Scope scope;
     private int scopeLevel;
 
+    /**
+     * The following field is set to {@code true} (via {@link #phLOC()}) when
+     * applicable during lexing before a call to {@link #startNewLine()} so
+     * that the lines-of-code count is also incremented.
+     */
+    protected boolean didSeePhysicalLOC;
+
+    protected int loc;
+
     /**
      * See {@link RuntimeEnvironment#getUserPage()}. Per default initialized in
      * the constructor and here to be consistent and avoid lot of unnecessary
@@ -137,13 +146,23 @@ public void reset() {
         super.reset();
 
         annotation = null;
+        didSeePhysicalLOC = false;
         disjointSpanClassName = null;
+        loc = 0;
         scopes = new Scopes();
         scope = null;
         scopeLevel = 0;
         scopeOpen = false;
+        setLineNumber(1);
     }
 
+    /**
+     * Gets the document physical lines-of-code count.
+     * @return a number greater than or equal to 0
+     */
+    @Override
+    public int getLOC() { return loc; }
+
     @Override
     public void setAnnotation(Annotation annotation) {
         this.annotation = annotation;
@@ -178,6 +197,11 @@ public void setFoldingEnabled(boolean foldingEnabled) {
         this.foldingEnabled = foldingEnabled;
     }
 
+    /**
+     * Sets a value indicating that a physical line-of-code was encountered.
+     */
+    protected void phLOC() { didSeePhysicalLOC = true; }
+
     /**
      * Calls {@link #appendLink(java.lang.String, boolean)} with {@code url}
      * and false.
@@ -347,6 +371,11 @@ public void startNewLine() throws IOException {
         boolean skipNl = false;
         setLineNumber(line + 1);
 
+        if (didSeePhysicalLOC) {
+            ++loc;
+            didSeePhysicalLOC = false;
+        }
+
         if (scopesEnabled) {
             startScope();
 

src/org/opensolaris/opengrok/analysis/JFlexSymbolMatcher.java

@@ -103,6 +103,20 @@ protected void onSymbolMatched(String str, int start) {
         }
     }
 
+    /**
+     * Raises
+     * {@link SymbolMatchedListener#sourceCodeSeen(org.opensolaris.opengrok.analysis.SourceCodeSeenEvent)}
+     * for all subscribed listeners in turn.
+     * @param start the source code start position
+     */
+    protected void onSourceCodeSeen(int start) {
+        SymbolMatchedListener l = symbolListener;
+        if (l != null) {
+            SourceCodeSeenEvent evt = new SourceCodeSeenEvent(this, start);
+            l.sourceCodeSeen(evt);
+        }
+    }
+
     /**
      * Calls {@link #onNonSymbolMatched(java.lang.String, int)} with the
      * {@link String#valueOf(char)} {@code c} and {@code start}.

src/org/opensolaris/opengrok/analysis/JFlexTokenizer.java

@@ -108,6 +108,14 @@ public void symbolMatched(SymbolMatchedEvent evt) {
         setAttribs(evt.getStr(), evt.getStart(), evt.getEnd());
     }
 
+    /**
+     * Does nothing.
+     * @param evt ignored
+     */
+    @Override
+    public void sourceCodeSeen(SourceCodeSeenEvent evt) {
+    }
+
     /**
      * Clears, and then resets the instance's attributes per the specified
      * arguments.

src/org/opensolaris/opengrok/analysis/JFlexXref.java

@@ -60,6 +60,16 @@ public class JFlexXref implements Xrefer, SymbolMatchedListener,
     private Scope scope;
     private int scopeLevel;
 
+    /**
+     * The following field is set to {@code true} (via
+     * {@link #sourceCodeSeen(org.opensolaris.opengrok.analysis.SourceCodeSeenEvent)})
+     * when applicable during lexing before a call to {@link #startNewLine()}
+     * so that the lines-of-code count is also incremented.
+     */
+    private boolean didSeePhysicalLOC;
+
+    private int loc;
+
     /**
      * See {@link RuntimeEnvironment#getUserPage()}. Per default initialized in
      * the constructor and here to be consistent and avoid lot of unnecessary
@@ -124,7 +134,9 @@ public void setReader(Reader input) {
     @Override
     public void reset() {
         annotation = null;
+        didSeePhysicalLOC = false;
         disjointSpanClassName = null;
+        loc = 0;
         scopes = new Scopes();
         scope = null;
         scopeLevel = 0;
@@ -136,10 +148,20 @@ public void reset() {
      * of the instance's {@link ScanningSymbolMatcher}.
      * @return yyline
      */
+    @Override
     public int getLineNumber() {
         return matcher.getLineNumber();
     }
 
+    /**
+     * Gets the determined count of physical lines-of-code.
+     * @return lines-of-code count
+     */
+    @Override
+    public int getLOC() {
+        return loc;
+    }
+
     @Override
     public void setAnnotation(Annotation annotation) {
         this.annotation = annotation;
@@ -187,6 +209,17 @@ public void symbolMatched(SymbolMatchedEvent evt) {
         }
     }
 
+    /**
+     * Sets a value indicating that source code was encountered such that
+     * {@link #getLOC()} will be incremented upon the next
+     * {@link #startNewLine()} or at end-of-file.
+     * @param evt ignored
+     */
+    @Override
+    public void sourceCodeSeen(SourceCodeSeenEvent evt) {
+        didSeePhysicalLOC = true;
+    }
+
     @Override
     public void nonSymbolMatched(TextMatchedEvent evt) {
         String str = evt.getStr();
@@ -463,19 +496,33 @@ public void write(Writer out) throws IOException {
         while (!matcher.emptyStack()) {
             matcher.yypop();
         }
+
+        // Account for dangling line of code. Unlike line number, LOC is
+        // incremented post- rather than pre-.
+        if (didSeePhysicalLOC) {
+            ++loc;
+            didSeePhysicalLOC = false;
+        }
     }
 
     /**
      * Terminate the current line and insert preamble for the next line. The
-     * line count will be incremented.
-     *
+     * line count is taken from {@link ScanningSymbolMatcher#getLineNumber()}.
+     * The lines-of-code count may be incremented if {@link #didSeePhysicalLOC}
+     * has been set to {@code true} (after which it will be reset to
+     * {@code false}).
      * @throws IOException on error when writing the xref
      */
     public void startNewLine() throws IOException {
         String iconId = null;
         int line = matcher.getLineNumber();
         boolean skipNl = false;
 
+        if (didSeePhysicalLOC) {
+            ++loc;
+            didSeePhysicalLOC = false;
+        }
+
         if (scopesEnabled) {
             startScope();
 

src/org/opensolaris/opengrok/analysis/SourceCodeSeenEvent.java

@@ -0,0 +1,59 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * See LICENSE.txt included in this distribution for the specific
+ * language governing permissions and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at LICENSE.txt.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+/*
+ * Copyright (c) 2017, Chris Fraire <[email protected]>.
+ */
+
+package org.opensolaris.opengrok.analysis;
+
+/**
+ * Represents an event raised when a language lexer has seen source code.
+ */
+public class SourceCodeSeenEvent {
+
+    private final Object source;
+    private final int position;
+
+    /**
+     * Initializes an immutable instance of {@link SourceCodeSeenEvent}.
+     * @param source the event source
+     * @param position the text position
+     */
+    public SourceCodeSeenEvent(Object source, int position) {
+        this.source = source;
+        this.position = position;
+    }
+
+    /**
+     * Gets the event source.
+     * @return the initial value
+     */
+    public Object getSource() {
+        return source;
+    }
+
+    /**
+     * Gets the text position.
+     * @return the initial value
+     */
+    public int getPosition() {
+        return position;
+    }
+}

src/org/opensolaris/opengrok/analysis/SymbolMatchedListener.java

@@ -37,4 +37,10 @@ public interface SymbolMatchedListener {
      * @param evt the event
      */
     void symbolMatched(SymbolMatchedEvent evt);
+
+    /**
+     * Receives an event instance.
+     * @param evt the event
+     */
+    void sourceCodeSeen(SourceCodeSeenEvent evt);
 }