add properties for VIAF to set thread pool size and rate limit

Author: codeforkjeff

conciliator.properties

@@ -52,3 +52,10 @@ cache.size=64MB
 # datasource.anothersolr.nametype.name=Person
 # datasource.anothersolr.url.query=http://SOME_OTHER_SOLR_INSTANCE:8983/solr/SOME_COLLECTION/select?wt=xml&q={{QUERY}}&rows={{ROWS}}
 # etc.
+
+#### VIAF
+
+## override default threadpool size for VIAF
+#datasource.viaf.threadpool.size=1
+## delay after every request, in milliseconds, to self-rate limit
+#datasource.viaf.delay=100

src/main/java/com/codefork/refine/datasource/WebServiceSearchTask.java

@@ -19,11 +19,17 @@ public class WebServiceSearchTask implements SearchTask {
     private WebServiceDataSource dataSource;
     private String key;
     private SearchQuery searchQuery;
+    private int delay;
 
-    public WebServiceSearchTask(WebServiceDataSource dataSource, String key, SearchQuery searchQuery) {
+    public WebServiceSearchTask(WebServiceDataSource dataSource, String key, SearchQuery searchQuery, int delay) {
         this.key = key;
         this.searchQuery = searchQuery;
         this.dataSource = dataSource;
+        this.delay = delay;
+    }
+
+    public WebServiceSearchTask(WebServiceDataSource dataSource, String key, SearchQuery searchQuery) {
+        this(dataSource, key, searchQuery, 0);
     }
 
     @Override
@@ -51,6 +57,17 @@ public SearchResult call() {
             return new SearchResult(key, SearchResult.ErrorType.UNKNOWN);
         }
         results.sort(BY_SCORE_REVERSED);
+
+        if(delay > 0) {
+            // TODO: it would be nice to sleep only we if actually made a request (if the result wasn't cached)
+            // but we'd need to make some modifications in order to know that
+            try {
+                Thread.sleep(delay);
+            } catch (InterruptedException e) {
+                dataSource.getLog().error("sleep interrupted in WebServiceSearchTask");
+            }
+        }
+
         return new SearchResult(key, results);
     }
 

src/main/java/com/codefork/refine/viaf/VIAF.java

@@ -6,6 +6,8 @@
 import com.codefork.refine.ThreadPool;
 import com.codefork.refine.ThreadPoolFactory;
 import com.codefork.refine.datasource.ConnectionFactory;
+import com.codefork.refine.datasource.SearchTask;
+import com.codefork.refine.datasource.WebServiceSearchTask;
 import com.codefork.refine.datasource.stats.Stats;
 import com.codefork.refine.datasource.WebServiceDataSource;
 import com.codefork.refine.resources.NameType;
@@ -47,15 +49,39 @@ public class VIAF extends WebServiceDataSource {
     private VIAFSource viafSource = null;
     private Map<String, NonVIAFSource> nonViafSources = new HashMap<>();
 
+    private int delay = 0;
+
     @Autowired
     public VIAF(Config config, CacheManager cacheManager, ThreadPoolFactory threadPoolFactory, ConnectionFactory connectionFactory, Stats stats) {
         super(config, cacheManager, threadPoolFactory, connectionFactory, stats);
 
         setCacheEnabled(true);
 
+        var dataSourceProperties = config.getDataSourceProperties("viaf");
+
+        var threadPoolSize = dataSourceProperties.getProperty("threadpool.size");
+        if(threadPoolSize != null) {
+            getLog().info("Setting pool size for VIAF to " + threadPoolSize);
+            getThreadPool().setPoolSize(Integer.parseInt(threadPoolSize.strip()));
+        }
+
+        var delay = dataSourceProperties.getProperty("delay");
+        if(delay != null) {
+            getLog().info("Setting delay to " + delay);
+            setDelay(Integer.parseInt(delay.strip()));
+        }
+
         spf = SAXParserFactory.newInstance();
     }
 
+    public int getDelay() {
+        return delay;
+    }
+
+    public void setDelay(int delay) {
+        this.delay = delay;
+    }
+
     /**
      * Factory method for getting a NonVIAFSource object
      */
@@ -115,6 +141,14 @@ public static String createCqlQueryString(SearchQuery searchQuery) {
         return cql;
     }
 
+    /**
+     * Override so we can add an optional delay to the task
+     **/
+    @Override
+    public SearchTask createSearchTask(String key, SearchQuery searchQuery) {
+        return new WebServiceSearchTask(this, key, searchQuery, getDelay());
+    }
+
     /**
      * Does actual work of performing a search and parsing the XML.
      * @param query