fix duplicate documents and tests for add/remove files to index

this fixes #7

Author: Lubos Kosco

src/org/opensolaris/opengrok/index/IndexDatabase.java

@@ -75,7 +75,7 @@
  * one index database per project.
  *
  * @author Trond Norbye
- * @author Lubos Kosco , update for lucene 4.0.0
+ * @author Lubos Kosco , update for lucene 4.2.0
  */
 public class IndexDatabase {
 
@@ -345,7 +345,7 @@ public void update() throws IOException, HistoryException {
                     directories.add(project.getPath());
                 }
             }
-
+            
             for (String dir : directories) {
                 File sourceRoot;
                 if ("".equals(dir)) {
@@ -367,9 +367,11 @@ public void update() throws IOException, HistoryException {
 
                 try {
                     if (numDocs > 0) {
-                        uidIter = terms.iterator(null);                        
-                        TermsEnum.SeekStatus stat = uidIter.seekCeil(new BytesRef(startuid), true); //init uid                        
-                        if (stat==TermsEnum.SeekStatus.END || stat==TermsEnum.SeekStatus.NOT_FOUND) { uidIter=null; }
+                        uidIter = terms.iterator(uidIter);                        
+                        TermsEnum.SeekStatus stat = uidIter.seekCeil(new BytesRef(startuid)); //init uid                        
+                        if (stat==TermsEnum.SeekStatus.END) { uidIter=null; 
+                         log.log(Level.WARNING, "Couldn't find a start term for {0}, empty u field?", startuid);
+                        }
                     }
                     //TODO below should be optional, since it traverses the tree once more to get total count! :(
                     int file_cnt = 0;
@@ -385,7 +387,8 @@ public void update() throws IOException, HistoryException {
 
                     while (uidIter != null && uidIter.term() != null && uidIter.term().utf8ToString().startsWith(startuid)) {
                         removeFile();
-                        uidIter.next();
+                        BytesRef next = uidIter.next();
+                        if (next==null) {uidIter=null;}
                     }
                 } finally {
                     reader.close();
@@ -802,7 +805,7 @@ private boolean isLocal(String path) {
      *
      */
     private int indexDown(File dir, String parent, boolean count_only, int cur_count, int est_total) throws IOException {
-        int lcur_count = cur_count;
+        int lcur_count = cur_count;        
         if (isInterrupted()) {
             return lcur_count;
         }
@@ -842,16 +845,18 @@ public int compare(File p1, File p2) {
                     if (uidIter != null) {
                         String uid = Util.path2uid(path, DateTools.timeToString(file.lastModified(), DateTools.Resolution.MILLISECOND)); // construct uid for doc
                         BytesRef buid = new BytesRef(uid);                        
-                        while (uidIter.term() != null 
+                        while (uidIter != null && uidIter.term() != null 
                                 && uidIter.term().compareTo(emptyBR) !=0
                                 && uidIter.term().compareTo(buid) < 0) {
                             removeFile();
-                            uidIter.next();
+                            BytesRef next = uidIter.next();
+                            if (next==null) {uidIter=null;}
                         }
 
-                        if (uidIter.term() != null
+                        if (uidIter != null && uidIter.term() != null
                                 && uidIter.term().bytesEquals(buid)) {
-                            uidIter.next(); // keep matching docs
+                            BytesRef next = uidIter.next(); // keep matching docs
+                            if (next==null) {uidIter=null;}
                             continue;
                         }
                     }
@@ -953,8 +958,8 @@ public static void listAllFiles(List<String> subFiles) throws IOException {
      */
     public void listFiles() throws IOException {
         IndexReader ireader = null;
-        TermsEnum iter;
-        Terms terms = null;
+        TermsEnum iter=null;
+        Terms terms = null;        
 
         try {
             ireader = DirectoryReader.open(indexDirectory); // open existing index
@@ -963,10 +968,11 @@ public void listFiles() throws IOException {
                 Fields uFields = MultiFields.getFields(ireader);//reader.getTermVectors(0);
                 terms = uFields.terms(QueryBuilder.U);
             }
-            iter = terms.iterator(null); // init uid iterator
-            while (iter.term() != null) {
+            iter = terms.iterator(iter); // init uid iterator
+            while (iter != null && iter.term() != null) {
                 log.fine(Util.uid2url(iter.term().utf8ToString()));
-                iter.next();
+                BytesRef next=iter.next();
+                if (next==null) {iter=null;}
             }
         } finally {
 
@@ -1014,7 +1020,7 @@ static void listFrequentTokens(List<String> subFiles) throws IOException {
     public void listTokens(int freq) throws IOException {
         IndexReader ireader = null;
         TermsEnum iter = null;
-        Terms terms = null;
+        Terms terms = null;         
 
         try {
             ireader = DirectoryReader.open(indexDirectory);
@@ -1023,13 +1029,14 @@ public void listTokens(int freq) throws IOException {
                 Fields uFields = MultiFields.getFields(ireader);//reader.getTermVectors(0);
                 terms = uFields.terms(QueryBuilder.DEFS);
             }
-            iter = terms.iterator(null); // init uid iterator            
-            while (iter.term() != null) {
+            iter = terms.iterator(iter); // init uid iterator            
+            while (iter != null && iter.term() != null) {
                 //if (iter.term().field().startsWith("f")) {
                 if (iter.docFreq() > 16 && iter.term().utf8ToString().length() > freq) {
                     log.warning(iter.term().utf8ToString());
                 }
-                iter.next();
+                BytesRef next = iter.next();
+                if (next==null) {iter=null;}
                 /*} else {
                  break;
                  }*/

test/org/opensolaris/opengrok/index/IndexerTest.java

@@ -181,7 +181,8 @@ public void testMain() throws IOException {
 
     private class MyIndexChangeListener implements org.opensolaris.opengrok.index.IndexChangedListener {
 
-        List<String> files = new ArrayList<String>();
+        List<String> files = new ArrayList<>();
+        List<String> removedFiles = new ArrayList<>();
 
         @Override
         public void fileAdd(String path, String analyzer) {
@@ -203,10 +204,11 @@ public void fileUpdate(String path) {
         @Override
         public void fileRemoved(String path) {
             files.remove(path);
+            removedFiles.add(path);
         }
 
         public void reset() {
-            this.files = new ArrayList<String>();
+            this.files = new ArrayList<>();
         }
     }
 
@@ -311,11 +313,11 @@ public void testIncrementalIndexAddRemoveFile() throws Exception {
             listener.reset();
             repository.addDummyFile(ppath);            
             idb.update();
-            assertEquals("No new file added",2, listener.files.size());            
-            listener.reset();
+            assertEquals("No new file added",1, listener.files.size());            
             repository.removeDummyFile(ppath);            
             idb.update();
-            assertEquals("Didn't remove the dummy file",1, listener.files.size());
+            assertEquals("Didn't remove the dummy file",0, listener.files.size());
+            assertEquals("Didn't remove the dummy file",1, listener.removedFiles.size());
         } else {
             System.out.println("Skipping test. Could not find a ctags I could use in path.");
         }