Merge pull request #99 from ral-facilities/98_search_files

Add /search/files endpoint using icat.lucene

Author: patrick-austin

src/main/config/run.properties.example

@@ -156,6 +156,11 @@ queue.priority.investigationUser.default = 4
 queue.priority.authenticated = {"orcid": 6, "anon": 0}
 queue.priority.default = 5
 
+# Whether the API endpoint perform Lucene searches is enabled
+search.enabled = false
+# The maximum number of results to return in a single request to the Lucene component
+search.maxResults = 10000
+
 # Configurable limit for the length of the GET URL for requesting Datafiles by a list of file locations
 # The exact limit may depend on the server
 getUrlLimit=1024

src/main/java/org/icatproject/topcat/httpclient/HttpClient.java

@@ -34,46 +34,56 @@ public Integer urlLength(String offset){
 	}
 
 	public Response get(String offset, Map<String, String> headers, int readTimeout) throws Exception {
-		return send("GET", offset, headers, null, readTimeout);
+		return send("GET", offset, headers, null, readTimeout, null);
 	}
 
 	public Response get(String offset, Map<String, String> headers) throws Exception {
 		return get(offset, headers, -1);
 	}
 
+	public Response post(String offset, Map<String, String> headers, String data, int readTimeout, String contentType)
+			throws Exception {
+		return send("POST", offset, headers, data, readTimeout, contentType);
+	}
+
 	public Response post(String offset, Map<String, String> headers, String data, int readTimeout) throws Exception {
-		return send("POST", offset, headers, data, readTimeout);
+		return send("POST", offset, headers, data, readTimeout, null);
+	}
+
+	public Response post(String offset, Map<String, String> headers, String data, String contentType) throws Exception {
+		return post(offset, headers, data, -1, contentType);
 	}
 
 	public Response post(String offset, Map<String, String> headers, String data) throws Exception {
-		return post(offset, headers, data, -1);
+		return post(offset, headers, data, -1, null);
 	}
 
 	public Response delete(String offset, Map<String, String> headers, int readTimeout) throws Exception {
-		return send("DELETE", offset, headers, null, readTimeout);
+		return send("DELETE", offset, headers, null, readTimeout, null);
 	}
 
 	public Response delete(String offset, Map<String, String> headers) throws Exception {
 		return delete(offset, headers, -1);
 	}
 
 	public Response put(String offset, Map<String, String> headers, String data, int readTimeout) throws Exception {
-		return send("PUT", offset, headers, data, readTimeout);
+		return send("PUT", offset, headers, data, readTimeout, null);
 	}
 
 	public Response put(String offset, Map<String, String> headers, String data) throws Exception {
 		return put(offset, headers, data, -1);
 	}
 
 	public Response head(String offset, Map<String, String> headers, int readTimeout) throws Exception {
-		return send("HEAD", offset, headers, null, readTimeout);
+		return send("HEAD", offset, headers, null, readTimeout, null);
 	}
 
 	public Response head(String offset, Map<String, String> headers) throws Exception {
 		return head(offset, headers, -1);
 	}
 
-	private Response send(String method, String offset, Map<String, String> headers, String body, int readTimeout) throws Exception {
+	private Response send(String method, String offset, Map<String, String> headers, String body, int readTimeout,
+			String contentType) throws Exception {
 		StringBuilder url = new StringBuilder(this.url + "/" + offset);
 
 		HttpURLConnection connection = null;
@@ -95,12 +105,14 @@ private Response send(String method, String offset, Map<String, String> headers,
     		if(body != null && (method.equals("POST") || method.equals("PUT"))){
     			connection.setDoOutput(true);
     			connection.setRequestProperty("Content-Length", Integer.toString(body.toString().getBytes().length));
+				if (contentType != null) {
+					connection.setRequestProperty("Content-Type", contentType);
+				}
 
 	    		DataOutputStream request = new DataOutputStream(connection.getOutputStream());
 	    		request.writeBytes(body.toString());
 	    		request.close();
 	    	}
-
 	    	Integer responseCode = connection.getResponseCode();
 
 	    	Map<String, String> responseHeaders = new HashMap();

src/main/java/org/icatproject/topcat/web/rest/UserResource.java

@@ -33,6 +33,7 @@
 
 import org.icatproject.topcat.domain.*;
 import org.icatproject.topcat.exceptions.*;
+import org.icatproject.topcat.httpclient.HttpClient;
 import org.icatproject.topcat.repository.*;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -69,6 +70,9 @@ public class UserResource {
 
 	private String anonUserName;
 	private String defaultPlugin;
+	private boolean queryEnabled;
+	private int maxResults;
+	private int maxFileCount;
 
 	@PersistenceContext(unitName = "topcat")
 	EntityManager em;
@@ -77,6 +81,9 @@ public UserResource() {
 		Properties properties = Properties.getInstance();
 		this.anonUserName = properties.getProperty("anonUserName", "");
 		this.defaultPlugin = properties.getProperty("defaultPlugin", "simple");
+		this.queryEnabled = Boolean.valueOf(properties.getProperty("search.enabled", "false"));
+		this.maxResults = Integer.valueOf(properties.getProperty("search.maxResults", "10000"));
+		this.maxFileCount = Integer.valueOf(properties.getProperty("queue.files.maxFileCount", "10000"));
     }
 
 	/**
@@ -1030,6 +1037,74 @@ public Response queueFiles(@FormParam("facilityName") String facilityName,
 		return Response.ok(jsonObjectBuilder.build()).build();
 	}
 
+	/**
+	 * Perform a search using the icat.lucene component for Datafile.locations, so that
+	 * these can then be used with the /queue/files endpoint.
+	 * 
+	 * Must be explicitly enabled by the run.properties.
+	 * 
+	 * @param facilityName ICAT Facility.name
+	 * @param sessionId    ICAT sessionId
+	 * @param maxResults   The number of results to request in this batch. If there are
+	 *                     more matching Datafiles, these should be requested separately
+	 *                     using searchAfter.
+	 * @param query        Query using Lucene syntax and supported Datafile fields.
+	 * @param searchAfter  Optionally ignore initial results and return those after the
+	 *                     Datafile represented by this object. For use when
+	 *                     batching/paginating results.
+	 * @return The Datafile.location of matching results, plus the searchAfter object
+	 *         for subsequent searches if the final result wasn't reached.
+	 * @throws Exception If authentication fails, more than the configured maxResults
+	 *                   requested, or if the underlying search fails.
+	 */
+	@POST
+	@Path("/search/files")
+	public Response searchFiles(@QueryParam("facilityName") String facilityName,
+			@QueryParam("sessionId") String sessionId, @QueryParam("maxResults") int maxResults,
+			@FormParam("query") String query, @FormParam("searchAfter") String searchAfter) throws Exception {
+
+		if (!queryEnabled) {
+			throw new BadRequestException("Querying not enabled");
+		}
+		FacilityMap facilityMap = FacilityMap.getInstance();
+		facilityName = facilityMap.validateFacilityName(facilityName);
+		String icatUrl = facilityMap.getIcatUrl(facilityName);
+		IcatClient icatClient = new IcatClient(icatUrl, sessionId);
+		String userName = icatClient.getUserName();
+
+		if (maxResults > this.maxResults) {
+			throw new BadRequestException("maxResults cannot exceed " + this.maxResults);
+		} else if (maxResults < 1) {
+			maxResults = Math.min(this.maxResults, maxFileCount);
+		}
+		String queryParameters = "?maxResults=" + maxResults;
+		if (searchAfter != null && !searchAfter.equals("")) {
+			queryParameters += "&search_after=" + searchAfter;
+		}
+
+		JsonArrayBuilder fieldsBuilder = Json.createArrayBuilder();
+		fieldsBuilder.add("location");
+		JsonObjectBuilder queryBuilder = Json.createObjectBuilder();
+		queryBuilder.add("text", query);
+		if (!icatClient.isAdmin()) {
+			queryBuilder.add("user", userName);
+		}
+		JsonObjectBuilder bodyBuilder = Json.createObjectBuilder();
+		bodyBuilder.add("fields", fieldsBuilder);
+		bodyBuilder.add("query", queryBuilder);
+		String body = bodyBuilder.build().toString();
+
+		HttpClient httpClient = new HttpClient(icatUrl + "/icat.lucene");
+		String path = "datafile" + queryParameters;
+		String contentType = "application/json; charset=utf-8";
+		org.icatproject.topcat.httpclient.Response response = httpClient.post(path, Map.of(), body, contentType);
+
+		if (response.getCode() != 200) {
+			throw new TopcatException(response.getCode(), response.toString());
+		}
+		return Response.ok(response.toString()).build();
+	}
+
 	/**
 	 * Format the filename for a queued Download, possibly one part of many.
 	 * 

src/test/java/org/icatproject/topcat/UserResourceTest.java

@@ -88,6 +88,7 @@ public static JavaArchive createDeployment() {
 	private UserResource userResource;
 
 	private static String sessionId;
+	private static String nonAdminSessionId;
 
 	@BeforeAll
 	public static void beforeAll() {
@@ -101,6 +102,12 @@ public void setup() throws Exception {
 				"{\"plugin\":\"simple\", \"credentials\":[{\"username\":\"root\"}, {\"password\":\"pw\"}]}", "UTF8");
 		String response = httpClient.post("session", new HashMap<String, String>(), loginData).toString();
 		sessionId = Utils.parseJsonObject(response).getString("sessionId");
+
+		loginData = "json=" + URLEncoder.encode(
+				"{\"plugin\":\"simple\", \"credentials\":[{\"username\":\"icatuser\"}, {\"password\":\"icatuserpw\"}]}",
+				"UTF8");
+		response = httpClient.post("session", new HashMap<String, String>(), loginData).toString();
+		nonAdminSessionId = Utils.parseJsonObject(response).getString("sessionId");
 	}
 
 	@Test
@@ -487,6 +494,59 @@ public void testQueueFiles() throws Exception {
 		}
 	}
 
+	@Test
+	public void testSearchFiles() throws Exception {
+		System.out.println("DEBUG testSearchFiles");
+		Response response = userResource.searchFiles(null, sessionId, 100, "visitId:\"Proposal 0 - 0 0\"", null);
+		assertEquals(200, response.getStatus());
+		JsonObject responseObject = Utils.parseJsonObject(response.getEntity().toString());
+		JsonArray results = responseObject.getJsonArray("results");
+		String firstSearchAfter = responseObject.getJsonObject("search_after").toString();
+		assertEquals(100, results.size());
+		JsonObject firstSearchAfterObject = Utils.parseJsonObject(firstSearchAfter);
+		JsonArray firstFields = firstSearchAfterObject.getJsonArray("fields");
+		int firstDoc = firstSearchAfterObject.getJsonNumber("doc").intValueExact();
+		double firstScore = firstFields.getJsonNumber(0).doubleValue();
+		long firstIcatId = firstFields.getJsonNumber(1).longValueExact();
+		assertEquals(0, firstSearchAfterObject.getJsonNumber("shardIndex").intValueExact());
+		assertEquals(firstScore, firstSearchAfterObject.getJsonNumber("score").doubleValue());
+
+		// If maxResults is not provided, it will default to 0 and then should use the queue.maxFileCount value of 3
+		response = userResource.searchFiles(null, sessionId, 0, "+visitId:\"Proposal 0 - 0 0\"", firstSearchAfter);
+		assertEquals(200, response.getStatus());
+		responseObject = Utils.parseJsonObject(response.getEntity().toString());
+		results = responseObject.getJsonArray("results");
+		String secondSearchAfter = responseObject.getJsonObject("search_after").toString();
+		assertEquals(3, results.size());
+		JsonObject secondSearchAfterObject = Utils.parseJsonObject(secondSearchAfter);
+		JsonArray secondFields = secondSearchAfterObject.getJsonArray("fields");
+		assertEquals(0, secondSearchAfterObject.getJsonNumber("shardIndex").intValueExact());
+		assertTrue(secondSearchAfterObject.getJsonNumber("doc").intValueExact() >= firstDoc + 3);
+		assertEquals(firstScore, secondSearchAfterObject.getJsonNumber("score").doubleValue());
+		assertEquals(firstScore, secondFields.getJsonNumber(0).doubleValue());
+		assertEquals(firstIcatId + 3, secondFields.getJsonNumber(1).longValueExact());
+	}
+
+	@Test
+	public void testSearchFilesUnauthorized() throws Exception {
+		System.out.println("DEBUG testSearchFilesUnauthorized");
+		// This user is not on any of the investigations, so should not see any results
+		Response response = userResource.searchFiles(null, nonAdminSessionId, 100, "visitId:\"Proposal - 0 0\"", null);
+		assertEquals(200, response.getStatus());
+		JsonObject responseObject = Utils.parseJsonObject(response.getEntity().toString());
+		JsonArray results = responseObject.getJsonArray("results");
+		JsonObject searchAfter = responseObject.getJsonObject("search_after");
+		assertEquals(0, results.size());
+		assertNull(searchAfter);
+	}
+
+	@Test
+	public void testSearchFilesMaxResultsExceeded() throws Exception {
+		System.out.println("DEBUG testSearchFilesMaxResultsExceeded");
+		Executable executable = () -> userResource.searchFiles(null, sessionId, 10001, "visitId:\"Proposal - 0 0\"", null);
+		assertThrows(BadRequestException.class, executable);
+	}
+
 	@Test
 	public void testQueueFilesBadRequestEmpty() throws Exception {
 		System.out.println("DEBUG testQueueFilesBadRequestEmpty");

src/test/resources/run.properties

@@ -42,6 +42,8 @@ queue.priority.user = {"simple/test": 1}
 queue.priority.authenticated = {"anon": 0}
 queue.priority.default = 2
 
+search.enabled = true
+
 # Each get request for Datafiles has a minimum size of 132, each of 3 locations is ~25
 # A value of 200 allows us to chunk this into one chunk of 2, and a second chunk of 1, hitting both branches of the code
 getUrlLimit=200