add API endpoint to refresh path descriptions (#2327)

fixes #2233

Author: Vladimir Kotal

opengrok-indexer/src/main/java/org/opengrok/indexer/configuration/Configuration.java

@@ -38,6 +38,8 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
@@ -233,7 +235,7 @@ public final class Configuration {
      * The name of the eftar file relative to the <var>DATA_ROOT</var>, which
      * contains definition tags.
      */
-    public static final String EFTAR_DTAGS_FILE = "index/dtags.eftar";
+    public static final String EFTAR_DTAGS_NAME = "dtags.eftar";
     private transient File dtagsEftar = null;
 
     /**
@@ -1221,13 +1223,20 @@ public String getForbiddenIncludeFileContent(boolean force) {
     }
 
     /**
-     * Get the eftar file, which contains definition tags.
+     * @return path to the file holding compiled path descriptions for the web application
+     */
+    public Path getDtagsEftarPath() {
+        return Paths.get(getDataRoot(), "index", EFTAR_DTAGS_NAME);
+    }
+
+    /**
+     * Get the eftar file, which contains definition tags for path descriptions.
      *
      * @return {@code null} if there is no such file, the file otherwise.
      */
     public File getDtagsEftar() {
         if (dtagsEftar == null) {
-            File tmp = new File(getDataRoot() + "/" + EFTAR_DTAGS_FILE);
+            File tmp = getDtagsEftarPath().toFile();
             if (tmp.canRead()) {
                 dtagsEftar = tmp;
             }

opengrok-indexer/src/main/java/org/opengrok/indexer/web/EftarFile.java

@@ -26,15 +26,16 @@
 import java.io.BufferedOutputStream;
 import java.io.BufferedReader;
 import java.io.DataOutputStream;
+import java.io.File;
 import java.io.FileNotFoundException;
 import java.io.FileOutputStream;
 import java.io.FileReader;
 import java.io.IOException;
 import java.io.RandomAccessFile;
+import java.io.StringReader;
 import java.util.Map;
 import java.util.StringTokenizer;
 import java.util.TreeMap;
-import java.util.logging.Level;
 import java.util.logging.Logger;
 
 import org.opengrok.indexer.logger.LoggerFactory;
@@ -200,12 +201,23 @@ private void traverse(Node n) {
     }
     private Node root;
 
-    public void readInput(String tagsPath) throws IOException {
-        try (BufferedReader r = new BufferedReader(new FileReader(tagsPath))) {
+    public void readInput(File inputFile) throws IOException {
+        try (BufferedReader r = new BufferedReader(new FileReader(inputFile))) {
             readInput(r);
         }
     }
 
+    public void readInput(String input) throws IOException {
+        try (BufferedReader r = new BufferedReader(new StringReader(input))) {
+            readInput(r);
+        }
+    }
+
+    /**
+     * Reads the input into interim representation. Can be called multiple times.
+     * @param r reader
+     * @throws IOException
+     */
     private void readInput(BufferedReader r) throws IOException {
         if (root == null) {
             root = new Node(1, null);
@@ -240,32 +252,13 @@ public void write(String outPath) throws FileNotFoundException, IOException {
         }
     }
 
-    public void create(String[] args) throws IOException, FileNotFoundException {
-        for (int i = 0; i < args.length - 1; i++) {
-            readInput(args[i]);
-        }
-        write(args[args.length - 1]);
+    public void create(File inputFile, String outputPath) throws IOException {
+        readInput(inputFile);
+        write(outputPath);
     }
 
-    /**
-     * Main method is used to generate eftar file from the path description
-     * file in the run scripts.
-     *
-     * @param args Input files and output file
-     */
-    @SuppressWarnings("PMD.SystemPrintln")
-    public static void main(String[] args) {
-        if (args.length < 2) {
-            System.err.println("Usage inputFile [inputFile ...] outputFile");
-            System.exit(1);
-        }
-
-        try {
-            EftarFile ef = new EftarFile();
-            ef.create(args);
-        } catch (IOException e) {
-            LOGGER.log(Level.WARNING, "EftarFile error", e);
-        }
+    public void create(String input, String outputPath) throws IOException, FileNotFoundException {
+        readInput(input);
+        write(outputPath);
     }
-
 }

opengrok-indexer/src/main/java/org/opengrok/indexer/web/EftarFileReader.java

@@ -142,6 +142,7 @@ public int getChildOffset() {
     public EftarFileReader(String file) throws FileNotFoundException {
         this(new File(file));
     }
+
     public EftarFileReader(File file) throws FileNotFoundException {
         f = new RandomAccessFile(file, "r");
         isOpen = true;
@@ -174,6 +175,12 @@ public String getChildTag(FNode fn, String name) throws IOException {
         return null;
     }
 
+    /**
+     * Get description for path
+     * @param path path relative to source root
+     * @return path description string
+     * @throws IOException
+     */
     public String get(String path) throws IOException {
         StringTokenizer toks = new StringTokenizer(path, "/");
         f.seek(0);

opengrok-indexer/src/test/java/org/opengrok/indexer/web/EftarFileTest.java

@@ -45,6 +45,8 @@ public class EftarFileTest {
     public EftarFileTest() {
     }
 
+    private final static String PATH_STRING = "/path";
+
     @BeforeClass
     public static void setUpClass() throws Exception {
         tsv = File.createTempFile("paths", ".tsv");
@@ -55,7 +57,7 @@ public static void setUpClass() throws Exception {
             out = new PrintWriter(new FileWriter(tsv));
             StringBuilder sb = new StringBuilder();
             for (int ii = 0; ii < 100; ii++) {
-                sb.append("/path");
+                sb.append(PATH_STRING);
                 sb.append(Integer.toString(ii));
                 out.print(sb.toString());
                 out.print("\tDescription ");
@@ -65,13 +67,13 @@ public static void setUpClass() throws Exception {
         } finally {
             try { out.close(); } catch (Exception e) { }
         }
-        //create eftar files
-        String[] args = new String[2];
-        args[0] = tsv.getAbsolutePath();
-        args[1] = eftar.getAbsolutePath();
+
+        // Create eftar files.
+        String inputFile = tsv.getAbsolutePath();
+        String outputFile = eftar.getAbsolutePath();
 
         EftarFile ef = new EftarFile();
-        ef.create(args);
+        ef.create(new File(inputFile), outputFile);
     }
 
     @AfterClass
@@ -109,7 +111,7 @@ private void searchEftarFile(EftarFileReader er) throws IOException {
         match.append("Description ");
         int offset = match.length();
         for (int ii = 0; ii < 100; ii++) {
-            sb.append("/path");
+            sb.append(PATH_STRING);
             sb.append(Integer.toString(ii));
             match.setLength(offset);
             match.append(Integer.toString(ii));

opengrok-web/src/main/java/org/opengrok/web/api/v1/controller/SystemController.java

@@ -23,13 +23,16 @@
 package org.opengrok.web.api.v1.controller;
 
 import org.opengrok.indexer.configuration.RuntimeEnvironment;
+import org.opengrok.indexer.web.EftarFile;
 import org.opengrok.web.api.v1.suggester.provider.service.SuggesterService;
 
 import javax.inject.Inject;
 import javax.ws.rs.Consumes;
+import javax.ws.rs.POST;
 import javax.ws.rs.PUT;
 import javax.ws.rs.Path;
 import javax.ws.rs.core.MediaType;
+import java.io.IOException;
 import java.util.Collections;
 
 @Path("/system")
@@ -53,4 +56,12 @@ public void refresh(final String project) {
     public void reloadIncludes() {
         env.reloadIncludeFiles(env.getConfiguration());
     }
+
+    @POST
+    @Path("/pathdesc")
+    @Consumes(MediaType.TEXT_PLAIN)
+    public void loadPathDescriptions(final String input) throws IOException {
+        EftarFile ef = new EftarFile();
+        ef.create(input, env.getConfiguration().getDtagsEftarPath().toString());
+    }
 }

opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/SystemControllerTest.java

@@ -8,20 +8,24 @@
 import org.opengrok.indexer.configuration.Configuration;
 import org.opengrok.indexer.configuration.RuntimeEnvironment;
 import org.opengrok.indexer.util.IOUtils;
+import org.opengrok.indexer.web.EftarFileReader;
 import org.opengrok.web.api.v1.RestApp;
 
 import javax.ws.rs.client.Entity;
 import javax.ws.rs.core.Application;
 import javax.ws.rs.core.Response;
 
 import java.io.File;
+import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.PrintWriter;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.nio.file.Paths;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertTrue;
 
 public class SystemControllerTest extends JerseyTest {
 
@@ -75,4 +79,44 @@ public void testIncludeReload() throws IOException {
         // Cleanup
         IOUtils.removeRecursive(includeRootPath);
     }
+
+    @Test
+    public void testDtagsEftarReload() throws IOException {
+        // The output file will be located in a directory under data root so create it first.
+        Path dataRoot = Files.createTempDirectory("api_dtags_test");
+        env.setDataRoot(dataRoot.toString());
+        Paths.get(dataRoot.toString(), "index").toFile().mkdir();
+
+        // Create path descriptions string.
+        StringBuilder sb = new StringBuilder();
+        String[][] descriptions = {
+                { "/path1", "foo foo" },
+                { "/path2", "bar bar" }
+        };
+
+        for (int i = 0; i < descriptions.length; i++) {
+            sb.append(descriptions[i][0]);
+            sb.append("\t");
+            sb.append(descriptions[i][1]);
+            sb.append("\n");
+        }
+        String input = sb.toString();
+
+        // Reload the contents via API call.
+        Response r = target("system")
+                .path("pathdesc")
+                .request().post(Entity.text(input));
+        assertEquals(Response.Status.NO_CONTENT.getStatusCode(), r.getStatus());
+
+        // Check
+        Path eftarPath = env.getConfiguration().getDtagsEftarPath();
+        assertTrue(eftarPath.toFile().exists());
+        EftarFileReader er = new EftarFileReader(eftarPath.toString());
+        for (int i = 0; i < descriptions.length; i++) {
+            assertEquals(descriptions[i][1], er.get(descriptions[i][0]));
+        }
+
+        // Cleanup
+        IOUtils.removeRecursive(dataRoot);
+    }
 }