Skip to content

Commit b9a6d4c

Browse files
vladakkahatlen
vladak
authored and
kahatlen
committed
generate history of files renamed in Mercurial repo separately
fixes #22
1 parent c6963a7 commit b9a6d4c

File tree

12 files changed

+300
-51
lines changed

12 files changed

+300
-51
lines changed

‎src/org/opensolaris/opengrok/configuration/RuntimeEnvironment.java

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -189,10 +189,12 @@ public String getPathRelativeToSourceRoot(File file, int stripCount) throws IOEx
189189
for (String allowedSymlink : getAllowedSymlinks()) {
190190
String allowedTarget = new File(allowedSymlink).getCanonicalPath();
191191
if (canonicalPath.startsWith(allowedTarget)) {
192-
return canonicalPath.substring(allowedTarget.length() + stripCount);
192+
return canonicalPath.substring(allowedTarget.length() +
193+
stripCount);
193194
}
194195
}
195-
throw new FileNotFoundException("Failed to resolve [" + canonicalPath + "] relative to source root [" + sourceRoot + "]");
196+
throw new FileNotFoundException("Failed to resolve [" + canonicalPath +
197+
"] relative to source root [" + sourceRoot + "]");
196198
}
197199

198200
/**

‎src/org/opensolaris/opengrok/history/FileHistoryCache.java

Lines changed: 48 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@
4141
import java.util.List;
4242
import java.util.Map;
4343
import java.util.logging.Level;
44+
import java.util.logging.Logger;
4445
import java.util.zip.GZIPInputStream;
4546
import java.util.zip.GZIPOutputStream;
4647
import org.opensolaris.opengrok.OpenGrokLogger;
@@ -100,7 +101,7 @@ private static File getCachedFile(File file) throws HistoryException {
100101
sb.append(".gz");
101102
} catch (IOException e) {
102103
throw new HistoryException("Failed to get path relative to " +
103-
"source root for " + file, e);
104+
"source root for " + file, e);
104105
}
105106

106107
return new File(sb.toString());
@@ -147,8 +148,8 @@ private void storeFile(History history, File file) throws HistoryException {
147148
output = File.createTempFile("oghist", null, dir);
148149
try (FileOutputStream out = new FileOutputStream(output);
149150
XMLEncoder e = new XMLEncoder(
150-
new BufferedOutputStream(
151-
new GZIPOutputStream(out)))) {
151+
new BufferedOutputStream(
152+
new GZIPOutputStream(out)))) {
152153
e.setPersistenceDelegate(File.class,
153154
new FilePersistenceDelegate());
154155
e.writeObject(history);
@@ -230,11 +231,49 @@ public void store(History history, Repository repository)
230231
*/
231232
File root = RuntimeEnvironment.getInstance().getSourceRootFile();
232233
for (Map.Entry<String, List<HistoryEntry>> e : map.entrySet()) {
233-
for (HistoryEntry ent : e.getValue()) {
234-
ent.strip();
234+
History hist = null;
235+
236+
/*
237+
* We do not want to generate history cache for files which
238+
* do not currently exist in the repository.
239+
*/
240+
File test = new File(env.getSourceRootPath() + e.getKey());
241+
if (!test.exists()) {
242+
continue;
243+
}
244+
245+
/*
246+
* Certain files require special handling - this is mainly for
247+
* files which have been renamed in Mercurial repository.
248+
* This ensures that their complete history (follow) will be
249+
* saved.
250+
*/
251+
String fullfile = e.getKey();
252+
try {
253+
String repodir = env.getPathRelativeToSourceRoot(
254+
new File(repository.getDirectoryName()), 0);
255+
String shortestfile = fullfile.substring(repodir.length() + 1);
256+
if (history.isIgnored(shortestfile)) {
257+
hist = repository.getHistory(test);
258+
}
259+
} catch (IOException ex) {
260+
Logger.getLogger(FileHistoryCache.class.getName()).log(
261+
Level.SEVERE, null, ex);
262+
}
263+
264+
if (hist == null) {
265+
hist = new History();
266+
267+
for (HistoryEntry ent : e.getValue()) {
268+
ent.strip();
269+
}
270+
// add all history entries
271+
hist.setHistoryEntries(e.getValue());
272+
} else {
273+
for (HistoryEntry ent : hist.getHistoryEntries()) {
274+
ent.strip();
275+
}
235276
}
236-
History hist = new History();
237-
hist.setHistoryEntries(e.getValue());
238277

239278
// Assign tags to changesets they represent
240279
if (env.isTagsEnabled() && repository.hasFileBasedTags()) {
@@ -321,10 +360,10 @@ public boolean hasCacheForDirectory(File directory, Repository repository)
321360
dir = new File(dir, "historycache");
322361
try {
323362
dir = new File(dir, env.getPathRelativeToSourceRoot(
324-
new File(repos.getDirectoryName()), 0));
363+
new File(repos.getDirectoryName()), 0));
325364
} catch (IOException e) {
326365
throw new HistoryException("Could not resolve " +
327-
repos.getDirectoryName()+" relative to source root", e);
366+
repos.getDirectoryName()+" relative to source root", e);
328367
}
329368
return dir.exists();
330369
}

‎src/org/opensolaris/opengrok/history/History.java

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,12 @@
3232
public class History {
3333
/** Entries in the log. The first entry is the most recent one. */
3434
private List<HistoryEntry> entries;
35-
35+
/**
36+
* files to ignore during cache creation.
37+
* These are relative to repository root.
38+
*/
39+
private List<String> ignoredFiles = new ArrayList<String>();
40+
3641
public History() {
3742
this(new ArrayList<HistoryEntry>());
3843
}
@@ -41,6 +46,11 @@ public History() {
4146
this.entries = entries;
4247
}
4348

49+
History(List<HistoryEntry> entries, List<String> ignored) {
50+
this.entries = entries;
51+
this.ignoredFiles = ignored;
52+
}
53+
4454
/**
4555
* Set the list of log entries for the file. The first entry is the most
4656
* recent one.
@@ -90,4 +100,12 @@ public boolean hasTags() {
90100
}
91101
return false;
92102
}
103+
104+
public boolean isIgnored(String file) {
105+
return ignoredFiles.contains(file);
106+
}
107+
108+
public List<String> getIgnoredFiles() {
109+
return ignoredFiles;
110+
}
93111
}

‎src/org/opensolaris/opengrok/history/HistoryEntry.java

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -94,21 +94,25 @@ public void dump() {
9494
log.log(Level.FINE, "HistoryEntry : tags = {0}", tags);
9595
log.log(Level.FINE, "HistoryEntry : date = {0}", date);
9696
log.log(Level.FINE, "HistoryEntry : author = {0}", author);
97-
log.log(Level.FINE, "HistoryEntry : active = {0}", (active ? "True" : "False"));
97+
log.log(Level.FINE, "HistoryEntry : active = {0}", (active ?
98+
"True" : "False"));
9899
String[] lines = message.toString().split("\n");
99100
String separator = "=";
100101
for (String line : lines) {
101-
log.log(Level.FINE, "HistoryEntry : message {0} {1}", new Object[]{separator, line});
102+
log.log(Level.FINE, "HistoryEntry : message {0} {1}",
103+
new Object[]{separator, line});
102104
separator = ">";
103105
}
104106
separator = "=";
105107
for (String cr : changeRequests) {
106-
log.log(Level.FINE, "HistoryEntry : changeRequests {0} {1}", new Object[]{separator, cr});
108+
log.log(Level.FINE, "HistoryEntry : changeRequests {0} {1}",
109+
new Object[]{separator, cr});
107110
separator = ">";
108111
}
109112
separator = "=";
110113
for (String file : files) {
111-
log.log(Level.FINE, "HistoryEntry : files {0} {1}", new Object[]{separator, file});
114+
log.log(Level.FINE, "HistoryEntry : files {0} {1}",
115+
new Object[]{separator, file});
112116
separator = ">";
113117
}
114118
}

‎src/org/opensolaris/opengrok/history/JDBCHistoryCache.java

Lines changed: 96 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -500,6 +500,10 @@ private static String truncate(String str, int length) {
500500
private static final PreparedQuery GET_CS_FILES =
501501
new PreparedQuery(getQuery("getFilesInChangeset"));
502502

503+
/** Statement for getting ID of given revision */
504+
private static final PreparedQuery GET_REV_ID =
505+
new PreparedQuery(getQuery("getChangesetIdForRevision"));
506+
503507
@Override
504508
public History get(File file, Repository repository, boolean withFiles)
505509
throws HistoryException {
@@ -614,6 +618,24 @@ public void store(History history, Repository repository)
614618
private static final PreparedQuery ADD_FILECHANGE =
615619
new PreparedQuery(getQuery("addFilechange"));
616620

621+
/**
622+
* Get ID value for revision string by querying the DB.
623+
* @param revision
624+
* @return ID
625+
*/
626+
private int getIdForRevision(String revision) throws SQLException {
627+
final ConnectionResource conn =
628+
connectionManager.getConnectionResource();
629+
try {
630+
PreparedStatement ps = conn.getStatement(GET_REV_ID);
631+
ps.setString(1, revision);
632+
ResultSet rs = ps.executeQuery();
633+
return rs.next() ? Integer.valueOf(rs.getString(1)).intValue() : -1;
634+
} finally {
635+
connectionManager.releaseConnection(conn);
636+
}
637+
}
638+
617639
private void storeHistory(ConnectionResource conn, History history,
618640
Repository repository) throws SQLException {
619641

@@ -624,6 +646,13 @@ private void storeHistory(ConnectionResource conn, History history,
624646
PreparedStatement addChangeset = null;
625647
PreparedStatement addDirchange = null;
626648
PreparedStatement addFilechange = null;
649+
RuntimeEnvironment env = RuntimeEnvironment.getInstance();
650+
651+
// return immediately when there is nothing to do
652+
List<HistoryEntry> entries = history.getHistoryEntries();
653+
if (entries.isEmpty()) {
654+
return;
655+
}
627656

628657
for (int i = 0;; i++) {
629658
try {
@@ -675,7 +704,6 @@ private void storeHistory(ConnectionResource conn, History history,
675704
// ordering column. Otherwise, incremental updates will make the
676705
// identity column unusable for chronological ordering. So therefore
677706
// we walk the list backwards.
678-
List<HistoryEntry> entries = history.getHistoryEntries();
679707
for (ListIterator<HistoryEntry> it =
680708
entries.listIterator(entries.size());
681709
it.hasPrevious();) {
@@ -704,10 +732,25 @@ private void storeHistory(ConnectionResource conn, History history,
704732
addDirchange.setInt(1, changesetId);
705733
addFilechange.setInt(1, changesetId);
706734
for (String file : entry.getFiles()) {
735+
// ignore ignored files
736+
String repodir = "";
737+
try {
738+
repodir = env.getPathRelativeToSourceRoot(
739+
new File(repository.getDirectoryName()), 0);
740+
} catch (IOException ex) {
741+
Logger.getLogger(
742+
JDBCHistoryCache.class.getName()).log(
743+
Level.SEVERE, null, ex);
744+
}
745+
707746
String fullPath = toUnixPath(file);
708-
int fileId = files.get(fullPath);
709-
addFilechange.setInt(2, fileId);
710-
addFilechange.executeUpdate();
747+
if (!history.isIgnored(
748+
file.substring(repodir.length() + 1))) {
749+
int fileId = files.get(fullPath);
750+
addFilechange.setInt(2, fileId);
751+
addFilechange.setInt(3, 0);
752+
addFilechange.executeUpdate();
753+
}
711754
String[] pathElts = splitPath(fullPath);
712755
for (int j = 0; j < pathElts.length; j++) {
713756
String dir = unsplitPath(pathElts, j);
@@ -732,6 +775,55 @@ private void storeHistory(ConnectionResource conn, History history,
732775
}
733776
}
734777
}
778+
779+
/*
780+
* Special handling for certain files - this is mainly for files which
781+
* have been renamed in Mercurial repository.
782+
* This ensures that their complete history (follow) will be saved.
783+
*/
784+
for (String filename: history.getIgnoredFiles()) {
785+
String file_path = repository.getDirectoryName() +
786+
File.separatorChar + filename;
787+
File file = new File(file_path);
788+
String repo_path = file_path.substring(env.getSourceRootPath().length());
789+
History hist;
790+
try {
791+
hist = repository.getHistory(file);
792+
} catch (HistoryException ex) {
793+
Logger.getLogger(JDBCHistoryCache.class.getName()).log(
794+
Level.SEVERE, null, ex);
795+
continue;
796+
}
797+
798+
int fileId = files.get(repo_path);
799+
for (HistoryEntry entry : hist.getHistoryEntries()) {
800+
retry:
801+
for (int i = 0;; i++) {
802+
try {
803+
int changesetId = getIdForRevision(entry.getRevision());
804+
addFilechange.setInt(1, changesetId);
805+
addFilechange.setInt(2, fileId);
806+
/*
807+
* If the file exists in the changeset, set its
808+
* moved value to 0 so it can be found when performing
809+
* historyget on directory.
810+
*/
811+
if (entry.getFiles().contains(repo_path)) {
812+
addFilechange.setInt(3, 0);
813+
} else {
814+
addFilechange.setInt(3, 1);
815+
}
816+
addFilechange.executeUpdate();
817+
818+
conn.commit();
819+
break retry;
820+
} catch (SQLException sqle) {
821+
handleSQLException(sqle, i);
822+
conn.rollback();
823+
}
824+
}
825+
}
826+
}
735827
}
736828

737829
/**

‎src/org/opensolaris/opengrok/history/JDBCHistoryCache_queries.properties

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -94,7 +94,8 @@ createTableFilechanges=\
9494
CHANGESET INT NOT NULL \
9595
CONSTRAINT FILECHANGES_FK_CS \
9696
REFERENCES OPENGROK.CHANGESETS ON DELETE CASCADE, \
97-
CONSTRAINT FILECHANGES_PK PRIMARY KEY (FILE, CHANGESET))
97+
MOVED SMALLINT NOT NULL, \
98+
CONSTRAINT FILECHANGES_PK PRIMARY KEY (FILE, CHANGESET, MOVED))
9899

99100
hasCacheForDirectory=\
100101
SELECT 1 FROM OPENGROK.REPOSITORIES R, OPENGROK.DIRECTORIES D \
@@ -127,8 +128,9 @@ getDirHistory=\
127128

128129
getFilesInChangeset=\
129130
SELECT D.PATH || '/' || F.NAME \
130-
FROM OPENGROK.DIRECTORIES D, OPENGROK.FILES F, OPENGROK.FILECHANGES FC \
131-
WHERE D.ID = F.DIRECTORY AND F.ID = FC.FILE AND FC.CHANGESET = ?
131+
FROM OPENGROK.DIRECTORIES D, OPENGROK.FILES F, OPENGROK.FILECHANGES FC \
132+
WHERE D.ID = F.DIRECTORY AND F.ID = FC.FILE AND FC.MOVED = 0 \
133+
AND FC.CHANGESET = ?
132134

133135
getRepository=SELECT ID FROM OPENGROK.REPOSITORIES WHERE PATH = ?
134136

@@ -141,7 +143,8 @@ addChangeset=\
141143

142144
addDirchange=INSERT INTO OPENGROK.DIRCHANGES(CHANGESET, DIRECTORY) VALUES (?,?)
143145

144-
addFilechange=INSERT INTO OPENGROK.FILECHANGES(CHANGESET, FILE) VALUES (?,?)
146+
addFilechange=INSERT INTO OPENGROK.FILECHANGES(CHANGESET, FILE, MOVED) \
147+
VALUES (?,?,?)
145148

146149
getAuthors=SELECT NAME, ID FROM OPENGROK.AUTHORS WHERE REPOSITORY = ?
147150

@@ -164,6 +167,9 @@ getLatestCachedRevision=\
164167
(SELECT MAX(CS.ID) FROM OPENGROK.CHANGESETS CS, OPENGROK.REPOSITORIES R \
165168
WHERE CS.REPOSITORY = R.ID AND R.PATH = ?)
166169

170+
getChangesetIdForRevision=\
171+
SELECT ID FROM OPENGROK.CHANGESETS WHERE REVISION = ?
172+
167173
getMaxFileId=SELECT MAX(ID) FROM OPENGROK.FILES
168174

169175
getMaxDirId=SELECT MAX(ID) FROM OPENGROK.DIRECTORIES

0 commit comments

Comments
 (0)