Fix deadlock threads while indexing (#2740)

* wait call must be inside a synchronized block. In other cases it
throws IllegalMonitorStateException.
* Use a specific lock to synchronize the modification of indexing set.
Port of PR #2548 to 3.4.x branch.

Author: juanluisrp

core/src/main/java/org/fao/geonet/kernel/DataManager.java

@@ -200,7 +200,8 @@
 public class DataManager implements ApplicationEventPublisherAware {
 
     private static final int METADATA_BATCH_PAGE_SIZE = 100000;
-    Lock indexLock = new ReentrantLock();
+    Lock waitLoopLock = new ReentrantLock();
+    Lock indexingLock = new ReentrantLock();
     Set<String> waitForIndexing = new HashSet<String>();
     Set<String> indexing = new HashSet<String>();
     Set<IndexMetadataTask> batchIndex = new ConcurrentHashSet<IndexMetadataTask>();
@@ -595,11 +596,11 @@ public void batchIndexInThreadPool(ServiceContext context, List<?> metadataIds)
     }
 
     public boolean isIndexing() {
-        indexLock.lock();
+        indexingLock.lock();
         try {
             return !indexing.isEmpty() || !batchIndex.isEmpty();
         } finally {
-            indexLock.unlock();
+            indexingLock.unlock();
         }
     }
 
@@ -622,7 +623,7 @@ public void indexMetadata(final List<String> metadataIds) throws Exception {
      * TODO javadoc.
      */
     public void indexMetadata(final String metadataId, boolean forceRefreshReaders, ISearchManager searchManager) throws Exception {
-        indexLock.lock();
+        waitLoopLock.lock();
         try {
             if (waitForIndexing.contains(metadataId)) {
                 return;
@@ -631,16 +632,23 @@ public void indexMetadata(final String metadataId, boolean forceRefreshReaders,
                 try {
                     waitForIndexing.add(metadataId);
                     // don't index the same metadata 2x
-                    wait(200);
+                    synchronized (this) {
+                    	wait(200);
+                    }
                 } catch (InterruptedException e) {
                     return;
                 } finally {
                     waitForIndexing.remove(metadataId);
                 }
             }
-            indexing.add(metadataId);
+            indexingLock.lock();
+            try {
+            	indexing.add(metadataId);
+            } finally {
+                indexingLock.unlock();
+            }
         } finally {
-            indexLock.unlock();
+            waitLoopLock.unlock();
         }
         Metadata fullMd;
 
@@ -844,11 +852,13 @@ public void indexMetadata(final String metadataId, boolean forceRefreshReaders,
             Log.error(Geonet.DATA_MANAGER, "The metadata document index with id=" + metadataId + " is corrupt/invalid - ignoring it. Error: " + x.getMessage(), x);
             fullMd = null;
         } finally {
-            indexLock.lock();
+            indexingLock.lock();
             try {
                 indexing.remove(metadataId);
+            } catch (Exception e) {
+                Log.warning(Geonet.INDEX_ENGINE, "Exception removing " + metadataId + " from indexing set", e);
             } finally {
-                indexLock.unlock();
+                indexingLock.unlock();
             }
         }
         if (fullMd != null) {