using include and ignore names in history generation

fixes #1351
fixes #1315

Author: tulinkry

src/org/opensolaris/opengrok/history/FileHistoryCache.java

@@ -18,7 +18,7 @@
  */
 
 /*
- * Copyright (c) 2008, 2016, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2008, 2017, Oracle and/or its affiliates. All rights reserved.
  */
 
 package org.opensolaris.opengrok.history;
@@ -52,8 +52,10 @@
 import java.util.logging.Logger;
 import java.util.zip.GZIPInputStream;
 import java.util.zip.GZIPOutputStream;
+import org.opensolaris.opengrok.configuration.Project;
 import org.opensolaris.opengrok.configuration.RuntimeEnvironment;
 import org.opensolaris.opengrok.logger.LoggerFactory;
+import org.opensolaris.opengrok.util.AcceptHelper;
 import org.opensolaris.opengrok.util.IOUtils;
 
 /*
@@ -364,6 +366,29 @@ public void store(History history, Repository repository)
                     continue;
                 }
 
+                if (env.hasProjects()) {
+                    /*
+                     * Problem here is that if the original file was a symlink
+                     * it's been already dereferenced by
+                     * getPathRelativeToSourceRoot(). We have to solve the only
+                     * case which is that the symlink led to the project's root.
+                     *
+                     * @see
+                     * RuntimeEnvironment#getPathRelativeToSourceRoot(java.io.File, int)
+                     */
+                    Project project = Project.getProject(test);
+                    File parent = test.equals(new File(env.getSourceRootPath(), project.getPath()))
+                            /* this is a project's root */
+                            ? test
+                            /* this isn't a project's root */
+                            : test.getParentFile();
+                    if (!AcceptHelper.accept(project, parent, test)) {
+                        continue;
+                    }
+                } else if (!AcceptHelper.accept(null, test.getParentFile(), test)) {
+                    continue;
+                }
+
                 List<HistoryEntry> list = map.get(s);
                 if (list == null) {
                     list = new ArrayList<>();
@@ -380,7 +405,7 @@ public void store(History history, Repository repository)
                 }
             }
         }
-
+        
         /*
          * Now traverse the list of files from the hash map built above
          * and for each file store its history (saved in the value of the
@@ -492,6 +517,10 @@ public History get(File file, Repository repository, boolean withFiles)
                 return null;
         }
 
+        if (!AcceptHelper.accept(Project.getProject(file), file)) {
+            return null;
+        }
+
         final History history;
         long time;
         try {
@@ -584,6 +613,7 @@ private String getRepositoryCachedRevPath(Repository repository) {
 
     /**
      * Store latest indexed revision for the repository under data directory.
+     *
      * @param repository repository
      * @param rev latest revision which has been just indexed
      */
@@ -592,11 +622,11 @@ private void storeLatestCachedRevision(Repository repository, String rev) {
 
         try {
             writer = new BufferedWriter(new OutputStreamWriter(
-                  new FileOutputStream(getRepositoryCachedRevPath(repository))));
+                    new FileOutputStream(getRepositoryCachedRevPath(repository))));
             writer.write(rev);
         } catch (IOException ex) {
             LOGGER.log(Level.WARNING, "Cannot write latest cached revision to file for "+repository.getDirectoryName(),
-                ex);
+                    ex);
         } finally {
            try {
                if (writer != null) {

src/org/opensolaris/opengrok/index/IndexDatabase.java

@@ -18,7 +18,7 @@
  */
 
 /*
- * Copyright (c) 2008, 2016, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2008, 2017, Oracle and/or its affiliates. All rights reserved.
  */
 package org.opensolaris.opengrok.index;
 
@@ -76,6 +76,7 @@
 import org.opensolaris.opengrok.history.HistoryGuru;
 import org.opensolaris.opengrok.logger.LoggerFactory;
 import org.opensolaris.opengrok.search.QueryBuilder;
+import org.opensolaris.opengrok.util.AcceptHelper;
 import org.opensolaris.opengrok.util.IOUtils;
 import org.opensolaris.opengrok.web.Util;
 
@@ -665,150 +666,6 @@ private void cleanupResources(Document doc) {
         }
     }
 
-    /**
-     * Check if I should accept this file into the index database
-     *
-     * @param file the file to check
-     * @return true if the file should be included, false otherwise
-     */
-    private boolean accept(File file) {
-
-        if (!includedNames.isEmpty()
-                && // the filter should not affect directory names
-                (!(file.isDirectory() || includedNames.match(file)))) {
-            return false;
-        }
-
-        String absolutePath = file.getAbsolutePath();
-
-        if (ignoredNames.ignore(file)) {
-            LOGGER.log(Level.FINER, "ignoring {0}", absolutePath);
-            return false;
-        }
-
-        if (!file.canRead()) {
-            LOGGER.log(Level.WARNING, "Could not read {0}", absolutePath);
-            return false;
-        }
-
-        try {
-            String canonicalPath = file.getCanonicalPath();
-            if (!absolutePath.equals(canonicalPath)
-                && !acceptSymlink(absolutePath, canonicalPath)) {
-
-                LOGGER.log(Level.FINE, "Skipped symlink ''{0}'' -> ''{1}''",
-                    new Object[]{absolutePath, canonicalPath});
-                return false;
-            }
-            //below will only let go files and directories, anything else is considered special and is not added
-            if (!file.isFile() && !file.isDirectory()) {
-                LOGGER.log(Level.WARNING, "Ignored special file {0}",
-                    absolutePath);
-                return false;
-            }
-        } catch (IOException exp) {
-            LOGGER.log(Level.WARNING, "Failed to resolve name: {0}",
-                absolutePath);
-            LOGGER.log(Level.FINE, "Stack Trace: ", exp);
-        }
-
-        if (file.isDirectory()) {
-            // always accept directories so that their files can be examined
-            return true;
-        }
-
-        if (HistoryGuru.getInstance().hasHistory(file)) {
-            // versioned files should always be accepted
-            return true;
-        }
-
-        // this is an unversioned file, check if it should be indexed
-        return !RuntimeEnvironment.getInstance().isIndexVersionedFilesOnly();
-    }
-
-    boolean accept(File parent, File file) {
-        try {
-            File f1 = parent.getCanonicalFile();
-            File f2 = file.getCanonicalFile();
-            if (f1.equals(f2)) {
-                LOGGER.log(Level.INFO, "Skipping links to itself...: {0} {1}",
-                        new Object[]{parent.getAbsolutePath(), file.getAbsolutePath()});
-                return false;
-            }
-
-            // Now, let's verify that it's not a link back up the chain...
-            File t1 = f1;
-            while ((t1 = t1.getParentFile()) != null) {
-                if (f2.equals(t1)) {
-                    LOGGER.log(Level.INFO, "Skipping links to parent...: {0} {1}",
-                            new Object[]{parent.getAbsolutePath(), file.getAbsolutePath()});
-                    return false;
-                }
-            }
-
-            return accept(file);
-        } catch (IOException ex) {
-            LOGGER.log(Level.WARNING, "Failed to resolve name: {0} {1}",
-                    new Object[]{parent.getAbsolutePath(), file.getAbsolutePath()});
-        }
-        return false;
-    }
-
-    /**
-     * Check if I should accept the path containing a symlink
-     *
-     * @param absolutePath the path with a symlink to check
-     * @param canonicalPath the canonical path to the file
-     * @return true if the file should be accepted, false otherwise
-     */
-    private boolean acceptSymlink(String absolutePath, String canonicalPath) throws IOException {
-        // Always accept local symlinks
-        if (isLocal(canonicalPath)) {
-            return true;
-        }
-
-        for (String allowedSymlink : RuntimeEnvironment.getInstance().getAllowedSymlinks()) {
-            if (absolutePath.startsWith(allowedSymlink)) {
-                String allowedTarget = new File(allowedSymlink).getCanonicalPath();
-                if (canonicalPath.startsWith(allowedTarget)
-                        && absolutePath.substring(allowedSymlink.length()).equals(canonicalPath.substring(allowedTarget.length()))) {
-                    return true;
-                }
-            }
-        }
-        return false;
-    }
-
-    /**
-     * Check if a file is local to the current project. If we don't have
-     * projects, check if the file is in the source root.
-     *
-     * @param path the path to a file
-     * @return true if the file is local to the current repository
-     */
-    private boolean isLocal(String path) {
-        RuntimeEnvironment env = RuntimeEnvironment.getInstance();
-        String srcRoot = env.getSourceRootPath();
-
-        boolean local = false;
-
-        if (path.startsWith(srcRoot)) {
-            if (env.hasProjects()) {
-                String relPath = path.substring(srcRoot.length());
-                if (project.equals(Project.getProject(relPath))) {
-                    // File is under the current project, so it's local.
-                    local = true;
-                }
-            } else {
-                // File is under source root, and we don't have projects, so
-                // consider it local.
-                local = true;
-            }
-        }
-
-        return local;
-    }
-
     /**
      * Generate indexes recursively
      *
@@ -828,7 +685,7 @@ private int indexDown(File dir, String parent, boolean count_only,
             return lcur_count;
         }
 
-        if (!accept(dir)) {
+        if (!AcceptHelper.accept(project, dir)) {
             return lcur_count;
         }
 
@@ -841,7 +698,7 @@ private int indexDown(File dir, String parent, boolean count_only,
         Arrays.sort(files, fileComparator);
 
         for (File file : files) {
-            if (accept(dir, file)) {
+            if (AcceptHelper.accept(project, dir, file)) {
                 String path = parent + '/' + file.getName();
 
                 if (file.isDirectory()) {