Skip to content

History generation respects ignored files #1386

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 5 commits into from
Closed

History generation respects ignored files #1386

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
8ed071e
using include and ignore names in history generation
tulinkry Jan 30, 2017
8ad37b1
setting page name for error pages
tulinkry Jan 30, 2017
f0dd85b
making sure the history cache is created before the last revision file
tulinkry Jan 30, 2017
c2d1b72
adding a test case for accept files
tulinkry Feb 6, 2017
e90286a
improving performance when ignoring files
tulinkry Feb 7, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
58 changes: 52 additions & 6 deletions src/org/opensolaris/opengrok/history/FileHistoryCache.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
*/

/*
* Copyright (c) 2008, 2016, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2008, 2017, Oracle and/or its affiliates. All rights reserved.
*/

package org.opensolaris.opengrok.history;
Expand Down Expand Up @@ -52,8 +52,10 @@
import java.util.logging.Logger;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;
import org.opensolaris.opengrok.configuration.Project;
import org.opensolaris.opengrok.configuration.RuntimeEnvironment;
import org.opensolaris.opengrok.logger.LoggerFactory;
import org.opensolaris.opengrok.util.AcceptHelper;
import org.opensolaris.opengrok.util.IOUtils;

/*
Expand Down Expand Up @@ -366,9 +368,40 @@ public void store(History history, Repository repository)

List<HistoryEntry> list = map.get(s);
if (list == null) {
/*
* This means that the file has not been added yet. So we
* try to see if we should accept it.
*/
if (env.hasProjects()) {
/*
* Problem here is that if the original file was a
* symlink it's been already dereferenced by
* getPathRelativeToSourceRoot(). We have to solve the
* only case which is that the symlink led to the
* project's root.
*
* @see
* RuntimeEnvironment#getPathRelativeToSourceRoot(java.io.File,
* int)
*/
Project project = Project.getProject(test);
File parent = test.equals(new File(env.getSourceRootPath(), project.getPath()))
/* this is a project's root */
? test
/* this isn't a project's root */
: test.getParentFile();
if (!AcceptHelper.accept(project, parent, test)) {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is where the inverse map gets constructed so if there are lots of changesets that touch given file, accept() check will be performed for that file for each changeset. This could be quite taxing on large repos. I'd very much prefer to run the accept() check only after the map is constructed.

continue;
}
} else if (list == null && !AcceptHelper.accept(null, test.getParentFile(), test)) {
continue;
}

// create a new empty record in the map
list = new ArrayList<>();
map.put(s, list);
}

/*
* We need to do deep copy in order to have different tags
* per each commit.
Expand All @@ -380,7 +413,7 @@ public void store(History history, Repository repository)
}
}
}

/*
* Now traverse the list of files from the hash map built above
* and for each file store its history (saved in the value of the
Expand Down Expand Up @@ -492,6 +525,10 @@ public History get(File file, Repository repository, boolean withFiles)
return null;
}

if (!AcceptHelper.accept(Project.getProject(file), file)) {
return null;
}

final History history;
long time;
try {
Expand Down Expand Up @@ -584,19 +621,28 @@ private String getRepositoryCachedRevPath(Repository repository) {

/**
* Store latest indexed revision for the repository under data directory.
*
* @param repository repository
* @param rev latest revision which has been just indexed
*/
private void storeLatestCachedRevision(Repository repository, String rev) {
Writer writer = null;

File file = new File(getRepositoryHistDataDirname(repository));
if (!file.exists() || !file.isDirectory()) {
if (!file.mkdirs()) {
LOGGER.log(Level.WARNING,
"Cannot create the history cache directory to write the latest cached revision for {}",
repository.getDirectoryName());
}
}
try {
writer = new BufferedWriter(new OutputStreamWriter(
new FileOutputStream(getRepositoryCachedRevPath(repository))));
new FileOutputStream(getRepositoryCachedRevPath(repository))));
writer.write(rev);
} catch (IOException ex) {
LOGGER.log(Level.WARNING, "Cannot write latest cached revision to file for "+repository.getDirectoryName(),
ex);
LOGGER.log(Level.WARNING,
"Cannot write latest cached revision to file for " + repository.getDirectoryName(),
ex);
} finally {
try {
if (writer != null) {
Expand Down
151 changes: 4 additions & 147 deletions src/org/opensolaris/opengrok/index/IndexDatabase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
*/

/*
* Copyright (c) 2008, 2016, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2008, 2017, Oracle and/or its affiliates. All rights reserved.
*/
package org.opensolaris.opengrok.index;

Expand Down Expand Up @@ -76,6 +76,7 @@
import org.opensolaris.opengrok.history.HistoryGuru;
import org.opensolaris.opengrok.logger.LoggerFactory;
import org.opensolaris.opengrok.search.QueryBuilder;
import org.opensolaris.opengrok.util.AcceptHelper;
import org.opensolaris.opengrok.util.IOUtils;
import org.opensolaris.opengrok.web.Util;

Expand Down Expand Up @@ -665,150 +666,6 @@ private void cleanupResources(Document doc) {
}
}

/**
* Check if I should accept this file into the index database
*
* @param file the file to check
* @return true if the file should be included, false otherwise
*/
private boolean accept(File file) {

if (!includedNames.isEmpty()
&& // the filter should not affect directory names
(!(file.isDirectory() || includedNames.match(file)))) {
return false;
}

String absolutePath = file.getAbsolutePath();

if (ignoredNames.ignore(file)) {
LOGGER.log(Level.FINER, "ignoring {0}", absolutePath);
return false;
}

if (!file.canRead()) {
LOGGER.log(Level.WARNING, "Could not read {0}", absolutePath);
return false;
}

try {
String canonicalPath = file.getCanonicalPath();
if (!absolutePath.equals(canonicalPath)
&& !acceptSymlink(absolutePath, canonicalPath)) {

LOGGER.log(Level.FINE, "Skipped symlink ''{0}'' -> ''{1}''",
new Object[]{absolutePath, canonicalPath});
return false;
}
//below will only let go files and directories, anything else is considered special and is not added
if (!file.isFile() && !file.isDirectory()) {
LOGGER.log(Level.WARNING, "Ignored special file {0}",
absolutePath);
return false;
}
} catch (IOException exp) {
LOGGER.log(Level.WARNING, "Failed to resolve name: {0}",
absolutePath);
LOGGER.log(Level.FINE, "Stack Trace: ", exp);
}

if (file.isDirectory()) {
// always accept directories so that their files can be examined
return true;
}

if (HistoryGuru.getInstance().hasHistory(file)) {
// versioned files should always be accepted
return true;
}

// this is an unversioned file, check if it should be indexed
return !RuntimeEnvironment.getInstance().isIndexVersionedFilesOnly();
}

boolean accept(File parent, File file) {
try {
File f1 = parent.getCanonicalFile();
File f2 = file.getCanonicalFile();
if (f1.equals(f2)) {
LOGGER.log(Level.INFO, "Skipping links to itself...: {0} {1}",
new Object[]{parent.getAbsolutePath(), file.getAbsolutePath()});
return false;
}

// Now, let's verify that it's not a link back up the chain...
File t1 = f1;
while ((t1 = t1.getParentFile()) != null) {
if (f2.equals(t1)) {
LOGGER.log(Level.INFO, "Skipping links to parent...: {0} {1}",
new Object[]{parent.getAbsolutePath(), file.getAbsolutePath()});
return false;
}
}

return accept(file);
} catch (IOException ex) {
LOGGER.log(Level.WARNING, "Failed to resolve name: {0} {1}",
new Object[]{parent.getAbsolutePath(), file.getAbsolutePath()});
}
return false;
}

/**
* Check if I should accept the path containing a symlink
*
* @param absolutePath the path with a symlink to check
* @param canonicalPath the canonical path to the file
* @return true if the file should be accepted, false otherwise
*/
private boolean acceptSymlink(String absolutePath, String canonicalPath) throws IOException {
// Always accept local symlinks
if (isLocal(canonicalPath)) {
return true;
}

for (String allowedSymlink : RuntimeEnvironment.getInstance().getAllowedSymlinks()) {
if (absolutePath.startsWith(allowedSymlink)) {
String allowedTarget = new File(allowedSymlink).getCanonicalPath();
if (canonicalPath.startsWith(allowedTarget)
&& absolutePath.substring(allowedSymlink.length()).equals(canonicalPath.substring(allowedTarget.length()))) {
return true;
}
}
}
return false;
}

/**
* Check if a file is local to the current project. If we don't have
* projects, check if the file is in the source root.
*
* @param path the path to a file
* @return true if the file is local to the current repository
*/
private boolean isLocal(String path) {
RuntimeEnvironment env = RuntimeEnvironment.getInstance();
String srcRoot = env.getSourceRootPath();

boolean local = false;

if (path.startsWith(srcRoot)) {
if (env.hasProjects()) {
String relPath = path.substring(srcRoot.length());
if (project.equals(Project.getProject(relPath))) {
// File is under the current project, so it's local.
local = true;
}
} else {
// File is under source root, and we don't have projects, so
// consider it local.
local = true;
}
}

return local;
}

/**
* Generate indexes recursively
*
Expand All @@ -828,7 +685,7 @@ private int indexDown(File dir, String parent, boolean count_only,
return lcur_count;
}

if (!accept(dir)) {
if (!AcceptHelper.accept(project, dir)) {
return lcur_count;
}

Expand All @@ -841,7 +698,7 @@ private int indexDown(File dir, String parent, boolean count_only,
Arrays.sort(files, fileComparator);

for (File file : files) {
if (accept(dir, file)) {
if (AcceptHelper.accept(project, dir, file)) {
String path = parent + '/' + file.getName();

if (file.isDirectory()) {
Expand Down
Loading