diff --git a/src/main/config/run.properties.example b/src/main/config/run.properties.example index 00225be5..9d173126 100644 --- a/src/main/config/run.properties.example +++ b/src/main/config/run.properties.example @@ -108,7 +108,11 @@ queue.maxActiveDownloads = 10 # Limit the number files per queued Download part. Multiple Datasets will be combined into part # Downloads based on their fileCount up to this limit. If a single Dataset has a fileCount # greater than this limit, it will still be submitted in a part by itself. -queue.maxFileCount = 10000 +queue.visit.maxPartFileCount = 10000 + +# Requests to the /queue/files endpoint will be rejected if they exceed this number of files +# Any chunking should be done clientside +queue.files.maxFileCount = 10000 # When queueing Downloads a positive priority will allow a User to proceed. # Non-positive values will block that User from submitting a request to the queue. diff --git a/src/main/java/org/icatproject/topcat/IcatClient.java b/src/main/java/org/icatproject/topcat/IcatClient.java index 3ebd964b..72d8dd21 100644 --- a/src/main/java/org/icatproject/topcat/IcatClient.java +++ b/src/main/java/org/icatproject/topcat/IcatClient.java @@ -1,7 +1,9 @@ package org.icatproject.topcat; import java.util.Map; +import java.util.Set; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.ListIterator; import java.util.ArrayList; @@ -21,6 +23,30 @@ public class IcatClient { + public class DatafilesResponse { + public final List ids = new ArrayList<>(); + public final Set missing = new HashSet<>(); + public long totalSize = 0L; + + /** + * Submit a query for Datafiles, then appends the ids, increments the size, and + * records any missing file locations. + * + * @param query Query to submit + * @throws TopcatException If the query returns not authorized, or not found. + */ + public void submitDatafilesQuery(String query) + throws TopcatException { + JsonArray jsonArray = submitQuery(query); + for (JsonObject jsonObject : jsonArray.getValuesAs(JsonObject.class)) { + JsonObject datafile = jsonObject.getJsonObject("Datafile"); + ids.add(datafile.getJsonNumber("id").longValueExact()); + missing.remove(datafile.getString("location")); + totalSize += datafile.getJsonNumber("fileSize").longValueExact(); + } + } + } + private Logger logger = LoggerFactory.getLogger(IcatClient.class); private HttpClient httpClient; @@ -139,7 +165,7 @@ public String getFullName() throws TopcatException { * @throws TopcatException */ public JsonArray getDatasets(String visitId) throws TopcatException { - String query = "SELECT dataset.id, dataset.fileCount from Dataset dataset"; + String query = "SELECT dataset.id, dataset.fileCount, dataset.fileSize from Dataset dataset"; query += " WHERE dataset.investigation.visitId = '" + visitId + "' ORDER BY dataset.id"; return submitQuery(query); } @@ -153,45 +179,44 @@ public JsonArray getDatasets(String visitId) throws TopcatException { * @throws TopcatException * @throws UnsupportedEncodingException */ - public List getDatafiles(List files) throws TopcatException, UnsupportedEncodingException { - List datafileIds = new ArrayList<>(); + public DatafilesResponse getDatafiles(List files) throws TopcatException, UnsupportedEncodingException { + DatafilesResponse response = new DatafilesResponse(); if (files.size() == 0) { // Ensure that we don't error when calling .next() below by returning early - return datafileIds; + return response; } // Total limit - "entityManager?sessionId=" - `sessionId` - "?query=" - `queryPrefix` - `querySuffix - // Limit is 1024 - 24 - 36 - 7 - 51 - 17 + // Limit is 1024 - 24 - 36 - 7 - 48 - 17 int getUrlLimit = Integer.parseInt(Properties.getInstance().getProperty("getUrlLimit", "1024")); - int chunkLimit = getUrlLimit - 135; - String queryPrefix = "SELECT d.id from Datafile d WHERE d.location in ("; + int chunkLimit = getUrlLimit - 132; + String queryPrefix = "SELECT d from Datafile d WHERE d.location in ("; String querySuffix = ") ORDER BY d.id"; ListIterator iterator = files.listIterator(); - String chunkedFiles = "'" + iterator.next() + "'"; + String file = iterator.next(); + String chunkedFiles = "'" + file + "'"; + response.missing.add(file); int chunkSize = URLEncoder.encode(chunkedFiles, "UTF8").length(); while (iterator.hasNext()) { - String file = "'" + iterator.next() + "'"; - int encodedFileLength = URLEncoder.encode(file, "UTF8").length(); + file = iterator.next(); + String quotedFile = "'" + file + "'"; + int encodedFileLength = URLEncoder.encode(quotedFile, "UTF8").length(); if (chunkSize + 3 + encodedFileLength > chunkLimit) { - JsonArray jsonArray = submitQuery(queryPrefix + chunkedFiles + querySuffix); - for (JsonNumber datafileIdJsonNumber : jsonArray.getValuesAs(JsonNumber.class)) { - datafileIds.add(datafileIdJsonNumber.longValueExact()); - } + response.submitDatafilesQuery(queryPrefix + chunkedFiles + querySuffix); - chunkedFiles = file; + chunkedFiles = quotedFile; chunkSize = encodedFileLength; + response.missing.add(file); } else { - chunkedFiles += "," + file; + chunkedFiles += "," + quotedFile; chunkSize += 3 + encodedFileLength; // 3 is size of , when encoded as %2C + response.missing.add(file); } } - JsonArray jsonArray = submitQuery(queryPrefix + chunkedFiles + querySuffix); - for (JsonNumber datafileIdJsonNumber : jsonArray.getValuesAs(JsonNumber.class)) { - datafileIds.add(datafileIdJsonNumber.longValueExact()); - } + response.submitDatafilesQuery(queryPrefix + chunkedFiles + querySuffix); - return datafileIds; + return response; } /** @@ -209,6 +234,21 @@ public long getDatasetFileCount(long datasetId) throws TopcatException { return jsonArray.getJsonNumber(0).longValueExact(); } + /** + * Utility method to get the fileSize (not size) of a Dataset by SELECTing its + * child Datafiles. Ideally the fileSize field should be used, this is a + * fallback option if that field is not set. + * + * @param datasetId ICAT Dataset.id + * @return The total size of Datafiles in the specified Dataset + * @throws TopcatException + */ + public long getDatasetFileSize(long datasetId) throws TopcatException { + String query = "SELECT SUM(datafile.fileSize) FROM Datafile datafile WHERE datafile.dataset.id = " + datasetId; + JsonArray jsonArray = submitQuery(query); + return jsonArray.getJsonNumber(0).longValueExact(); + } + /** * Utility method for submitting an unencoded query to the entityManager * endpoint, and returning the resultant JsonArray. @@ -238,6 +278,11 @@ private JsonArray submitQuery(String query) throws TopcatException { /** * Gets a single Entity of the specified type, without any other conditions. * + * NOTE: This function is written and intended for getting Investigation, + * Dataset or Datafile entities as part of the tests. It does not handle casing of + * entities containing multiple words, or querying for a specific instance of an + * entity. + * * @param entityType Type of ICAT Entity to get * @return A single ICAT Entity of the specified type as a JsonObject * @throws TopcatException @@ -391,7 +436,8 @@ public int getQueuePriority(String userName) throws TopcatException { } } - if (!userName.startsWith(Properties.getInstance().getProperty("anonUserName"))) { + String anonUserName = Properties.getInstance().getProperty("anonUserName"); + if (anonUserName == null || !userName.startsWith(anonUserName)) { // The anonymous cart username will end with the user's sessionId so cannot do .equals return priorityMap.getAuthenticatedPriority(); } else { diff --git a/src/main/java/org/icatproject/topcat/PriorityMap.java b/src/main/java/org/icatproject/topcat/PriorityMap.java index e32cf820..21bf29fc 100644 --- a/src/main/java/org/icatproject/topcat/PriorityMap.java +++ b/src/main/java/org/icatproject/topcat/PriorityMap.java @@ -35,6 +35,10 @@ public PriorityMap() { Properties properties = Properties.getInstance(); anonUserName = properties.getProperty("anonUserName", ""); + if (anonUserName.equals("")) { + logger.warn("anonUserName not defined, cannot distinguish anonymous and authenticated users so " + + "authenticated priority will be used as default level"); + } anonDownloadEnabled = Boolean.parseBoolean(properties.getProperty("anonDownloadEnabled", "true")); String defaultString; if (anonDownloadEnabled) { diff --git a/src/main/java/org/icatproject/topcat/StatusCheck.java b/src/main/java/org/icatproject/topcat/StatusCheck.java index 15496c00..5d206652 100644 --- a/src/main/java/org/icatproject/topcat/StatusCheck.java +++ b/src/main/java/org/icatproject/topcat/StatusCheck.java @@ -14,6 +14,7 @@ import jakarta.ejb.EJB; import jakarta.ejb.Schedule; import jakarta.ejb.Singleton; +import jakarta.json.JsonObject; import jakarta.persistence.EntityManager; import jakarta.persistence.PersistenceContext; import jakarta.persistence.TypedQuery; @@ -114,35 +115,35 @@ public void updateStatuses(int pollDelay, int pollIntervalWait, IdsClient inject String queryString = selectString + " and " + notExpiredCondition + " and (" + isActiveCondition + ")"; TypedQuery query = em.createQuery(queryString, Download.class); - List downloads = query.getResultList(); + List downloads = query.getResultList(); for (Download download : downloads) { - Date lastCheck = lastChecks.get(download.getId()); - Date now = new Date(); - long createdSecondsAgo = (now.getTime() - download.getCreatedAt().getTime()) / 1000; + Date lastCheck = lastChecks.get(download.getId()); + Date now = new Date(); + long createdSecondsAgo = (now.getTime() - download.getCreatedAt().getTime()) / 1000; if (download.getStatus() == DownloadStatus.PREPARING) { // If prepareDownload was called previously but caught an exception (other than // TopcatException), we should not call it again immediately, but should impose // a delay. See issue #462. if (lastCheck == null) { - prepareDownload(download, injectedIdsClient); - } else { - long lastCheckSecondsAgo = (now.getTime() - lastCheck.getTime()) / 1000; + prepareDownload(download, injectedIdsClient); + } else { + long lastCheckSecondsAgo = (now.getTime() - lastCheck.getTime()) / 1000; if (lastCheckSecondsAgo >= pollIntervalWait) { - prepareDownload(download, injectedIdsClient); - } - } - } else if (createdSecondsAgo >= pollDelay) { + prepareDownload(download, injectedIdsClient); + } + } + } else if (download.getPreparedId() != null && createdSecondsAgo >= pollDelay) { if (lastCheck == null) { - performCheck(download, injectedIdsClient); - } else { - long lastCheckSecondsAgo = (now.getTime() - lastCheck.getTime()) / 1000; + performCheck(download, injectedIdsClient); + } else { + long lastCheckSecondsAgo = (now.getTime() - lastCheck.getTime()) / 1000; if (lastCheckSecondsAgo >= pollIntervalWait) { - performCheck(download, injectedIdsClient); - } + performCheck(download, injectedIdsClient); } } - } + } + } } private void performCheck(Download download, IdsClient injectedIdsClient) { @@ -250,12 +251,14 @@ private void prepareDownload(Download download, IdsClient injectedIdsClient, Str String preparedId = idsClient.prepareData(sessionId, download.getInvestigationIds(), download.getDatasetIds(), download.getDatafileIds()); download.setPreparedId(preparedId); - try { - Long size = idsClient.getSize(sessionId, download.getInvestigationIds(), download.getDatasetIds(), download.getDatafileIds()); - download.setSize(size); - } catch(Exception e) { - logger.error("prepareDownload: setting size to -1 as getSize threw exception: " + e.getMessage()); - download.setSize(-1); + if (download.getSize() <= 0) { + try { + Long size = idsClient.getSize(sessionId, download.getInvestigationIds(), download.getDatasetIds(), download.getDatafileIds()); + download.setSize(size); + } catch(Exception e) { + logger.error("prepareDownload: setting size to -1 as getSize threw exception: " + e.getMessage()); + download.setSize(-1); + } } if (download.getIsTwoLevel() || !download.getTransport().matches("https|http")) { @@ -285,11 +288,10 @@ private void prepareDownload(Download download, IdsClient injectedIdsClient, Str * @param sessionIds Map from Facility to functional sessionId * @param facilityName Name of ICAT Facility to get the sessionId for * @return Functional ICAT sessionId - * @throws InternalException If the facilityName cannot be mapped to an ICAT url - * @throws BadRequestException If the login fails + * @throws Exception If the login fails */ private String getQueueSessionId(Map sessionIds, String facilityName) - throws InternalException, BadRequestException { + throws Exception { String sessionId = sessionIds.get(facilityName); if (sessionId == null) { IcatClient icatClient = new IcatClient(FacilityMap.getInstance().getIcatUrl(facilityName)); @@ -297,7 +299,9 @@ private String getQueueSessionId(Map sessionIds, String facility String plugin = properties.getProperty("queue.account." + facilityName + ".plugin"); String username = properties.getProperty("queue.account." + facilityName + ".username"); String password = properties.getProperty("queue.account." + facilityName + ".password"); - sessionId = icatClient.login(plugin, username, password); + String jsonString = icatClient.login(plugin, username, password); + JsonObject jsonObject = Utils.parseJsonObject(jsonString); + sessionId = jsonObject.getString("sessionId"); sessionIds.put(facilityName, sessionId); } return sessionId; @@ -315,7 +319,7 @@ private String getQueueSessionId(Map sessionIds, String facility */ public void startQueuedDownloads(int maxActiveDownloads) throws Exception { if (maxActiveDownloads == 0) { - logger.debug("Preparing of queued jobs disabled by config, skipping"); + logger.trace("Preparing of queued jobs disabled by config, skipping"); return; } @@ -332,7 +336,7 @@ public void startQueuedDownloads(int maxActiveDownloads) throws Exception { int activeDownloadsSize = activeDownloads.size(); if (activeDownloadsSize >= maxActiveDownloads) { String format = "More downloads currently RESTORING {} than maxActiveDownloads {}, cannot prepare queued jobs"; - logger.info(format, activeDownloadsSize, maxActiveDownloads); + logger.trace(format, activeDownloadsSize, maxActiveDownloads); return; } availableDownloads -= activeDownloadsSize; @@ -346,13 +350,13 @@ public void startQueuedDownloads(int maxActiveDownloads) throws Exception { Map sessionIds = new HashMap<>(); if (maxActiveDownloads <= 0) { // No limits on how many to submit - logger.info("Preparing {} queued downloads", queuedDownloads.size()); + logger.trace("Preparing {} queued downloads", queuedDownloads.size()); for (Download queuedDownload : queuedDownloads) { queuedDownload.setStatus(DownloadStatus.PREPARING); prepareDownload(queuedDownload, null, getQueueSessionId(sessionIds, queuedDownload.getFacilityName())); } } else { - logger.info("Preparing up to {} queued downloads", availableDownloads); + logger.trace("Preparing up to {} queued downloads", availableDownloads); HashMap> mapping = new HashMap<>(); for (Download queuedDownload : queuedDownloads) { String sessionId = getQueueSessionId(sessionIds, queuedDownload.getFacilityName()); diff --git a/src/main/java/org/icatproject/topcat/web/rest/UserResource.java b/src/main/java/org/icatproject/topcat/web/rest/UserResource.java index 6562f732..2e019941 100644 --- a/src/main/java/org/icatproject/topcat/web/rest/UserResource.java +++ b/src/main/java/org/icatproject/topcat/web/rest/UserResource.java @@ -42,6 +42,7 @@ import org.icatproject.topcat.FacilityMap; import org.icatproject.topcat.IcatClient; import org.icatproject.topcat.Properties; +import org.icatproject.topcat.IcatClient.DatafilesResponse; @Stateless @LocalBean @@ -65,7 +66,8 @@ public class UserResource { private String anonUserName; private String defaultPlugin; - private long queueMaxFileCount; + private long queueVisitMaxPartFileCount; + private long queueFilesMaxFileCount; @PersistenceContext(unitName = "topcat") EntityManager em; @@ -74,7 +76,8 @@ public UserResource() { Properties properties = Properties.getInstance(); this.anonUserName = properties.getProperty("anonUserName", ""); this.defaultPlugin = properties.getProperty("defaultPlugin", "simple"); - this.queueMaxFileCount = Long.valueOf(properties.getProperty("queue.maxFileCount", "10000")); + this.queueVisitMaxPartFileCount = Long.valueOf(properties.getProperty("queue.visit.maxPartFileCount", "10000")); + this.queueFilesMaxFileCount = Long.valueOf(properties.getProperty("queue.files.maxFileCount", "10000")); } /** @@ -870,6 +873,9 @@ public Response queueVisitId(@FormParam("facilityName") String facilityName, @FormParam("fileName") String fileName, @FormParam("email") String email, @FormParam("visitId") String visitId) throws TopcatException { + if (visitId == null || visitId.equals("")) { + throw new BadRequestException("visitId must be provided"); + } logger.info("queueVisitId called for {}", visitId); validateTransport(transport); @@ -883,11 +889,15 @@ public Response queueVisitId(@FormParam("facilityName") String facilityName, String fullName = icatClient.getFullName(); icatClient.checkQueueAllowed(userName); JsonArray datasets = icatClient.getDatasets(visitId); + if (datasets.size() == 0) { + throw new NotFoundException("No Datasets found for " + visitId); + } long downloadId; JsonArrayBuilder jsonArrayBuilder = Json.createArrayBuilder(); long downloadFileCount = 0L; + long downloadFileSize = 0L; List downloadItems = new ArrayList(); List downloads = new ArrayList(); Download newDownload = createDownload(sessionId, facilityName, "", userName, fullName, transport, email); @@ -896,16 +906,22 @@ public Response queueVisitId(@FormParam("facilityName") String facilityName, JsonArray datasetArray = dataset.asJsonArray(); long datasetId = datasetArray.getJsonNumber(0).longValueExact(); long datasetFileCount = datasetArray.getJsonNumber(1).longValueExact(); + long datasetFileSize = datasetArray.getJsonNumber(2).longValueExact(); + // Database triggers should set these, but check explicitly anyway if (datasetFileCount < 1L) { - // Database triggers should set this, but check explicitly anyway datasetFileCount = icatClient.getDatasetFileCount(datasetId); } + if (datasetFileSize < 1L) { + datasetFileSize = icatClient.getDatasetFileSize(datasetId); + } - if (downloadFileCount > 0L && downloadFileCount + datasetFileCount > queueMaxFileCount) { + if (downloadFileCount > 0L && downloadFileCount + datasetFileCount > queueVisitMaxPartFileCount) { newDownload.setDownloadItems(downloadItems); + newDownload.setSize(downloadFileSize); downloads.add(newDownload); downloadFileCount = 0L; + downloadFileSize = 0L; downloadItems = new ArrayList(); newDownload = createDownload(sessionId, facilityName, "", userName, fullName, transport, email); } @@ -913,8 +929,10 @@ public Response queueVisitId(@FormParam("facilityName") String facilityName, DownloadItem downloadItem = createDownloadItem(newDownload, datasetId, EntityType.dataset); downloadItems.add(downloadItem); downloadFileCount += datasetFileCount; + downloadFileSize += datasetFileSize; } newDownload.setDownloadItems(downloadItems); + newDownload.setSize(downloadFileSize); downloads.add(newDownload); int part = 1; @@ -956,8 +974,7 @@ public Response queueAllowed(@QueryParam("sessionId") String sessionId, } /** - * Queue download of Datafiles by location, splitting into part Downloads if - * needed. + * Queue download of Datafiles by location, up to a configurable limit. * * @param facilityName ICAT Facility.name * @param sessionId ICAT sessionId @@ -966,7 +983,8 @@ public Response queueAllowed(@QueryParam("sessionId") String sessionId, * Download. Defaults to facilityName_visitId. * @param email Optional email to notify upon completion * @param files ICAT Datafile.locations to download - * @return Array of Download ids + * @return The resultant downloadId and an array of any locations which could not + * be found * @throws TopcatException * @throws UnsupportedEncodingException */ @@ -979,59 +997,48 @@ public Response queueFiles(@FormParam("facilityName") String facilityName, if (files == null || files.size() == 0) { throw new BadRequestException("At least one Datafile.location required"); + } else if (files.size() > queueFilesMaxFileCount) { + throw new BadRequestException("Limit of " + queueFilesMaxFileCount + " files exceeded"); } logger.info("queueFiles called for {} files", files.size()); validateTransport(transport); + if (fileName == null) { + fileName = facilityName + "_files"; + } String icatUrl = getIcatUrl(facilityName); IcatClient icatClient = new IcatClient(icatUrl, sessionId); String transportUrl = getDownloadUrl(facilityName, transport); IdsClient idsClient = new IdsClient(transportUrl); - // If we wanted to block the user, this is where we would do it String userName = icatClient.getUserName(); String fullName = icatClient.getFullName(); icatClient.checkQueueAllowed(userName); - List datafileIds = icatClient.getDatafiles(files); - long downloadId; - JsonArrayBuilder jsonArrayBuilder = Json.createArrayBuilder(); - - long downloadFileCount = 0L; - List downloadItems = new ArrayList(); - List downloads = new ArrayList(); - Download newDownload = createDownload(sessionId, facilityName, "", userName, fullName, transport, email); - - for (long datafileId : datafileIds) { - if (downloadFileCount >= queueMaxFileCount) { - newDownload.setDownloadItems(downloadItems); - downloads.add(newDownload); - - downloadFileCount = 0L; - downloadItems = new ArrayList(); - newDownload = createDownload(sessionId, facilityName, "", userName, fullName, transport, email); - } + DatafilesResponse response = icatClient.getDatafiles(files); + if (response.ids.size() == 0) { + throw new NotFoundException("No Datafiles found"); + } - DownloadItem downloadItem = createDownloadItem(newDownload, datafileId, EntityType.datafile); + List downloadItems = new ArrayList<>(); + Download download = createDownload(sessionId, facilityName, fileName, userName, fullName, transport, email); + for (long datafileId : response.ids) { + DownloadItem downloadItem = createDownloadItem(download, datafileId, EntityType.datafile); downloadItems.add(downloadItem); - downloadFileCount += 1L; } - newDownload.setDownloadItems(downloadItems); - downloads.add(newDownload); + download.setDownloadItems(downloadItems); + download.setSize(response.totalSize); - int part = 1; - if (fileName == null) { - fileName = facilityName + "_files"; - } - for (Download download : downloads) { - String partFilename = formatQueuedFilename(fileName, part, downloads.size()); - download.setFileName(partFilename); - downloadId = submitDownload(idsClient, download, DownloadStatus.PAUSED); - jsonArrayBuilder.add(downloadId); - part += 1; - } + long downloadId = submitDownload(idsClient, download, DownloadStatus.PAUSED); - return Response.ok(jsonArrayBuilder.build()).build(); + JsonObjectBuilder jsonObjectBuilder = Json.createObjectBuilder(); + JsonArrayBuilder jsonArrayBuilder = Json.createArrayBuilder(); + for (String missingFile : response.missing) { + jsonArrayBuilder.add(missingFile); + } + jsonObjectBuilder.add("downloadId", downloadId); + jsonObjectBuilder.add("notFound", jsonArrayBuilder); + return Response.ok(jsonObjectBuilder.build()).build(); } /** diff --git a/src/test/java/org/icatproject/topcat/IcatClientTest.java b/src/test/java/org/icatproject/topcat/IcatClientTest.java index f8ddf16d..4cbebec3 100644 --- a/src/test/java/org/icatproject/topcat/IcatClientTest.java +++ b/src/test/java/org/icatproject/topcat/IcatClientTest.java @@ -12,6 +12,8 @@ import org.icatproject.topcat.httpclient.HttpClient; import org.icatproject.topcat.httpclient.Response; import org.icatproject.topcat.domain.*; +import org.icatproject.topcat.exceptions.TopcatException; + import java.net.URLEncoder; import org.icatproject.topcat.repository.CacheRepository; @@ -238,6 +240,20 @@ public void testCheckUserFound() throws Exception { } } + @Test + public void testGetDatasetFileCount() throws TopcatException { + IcatClient icatClient = new IcatClient("https://localhost:8181", sessionId); + long datasetId = icatClient.getEntity("Dataset").getJsonNumber("id").longValueExact(); + assertNotEquals(0, icatClient.getDatasetFileCount(datasetId)); + } + + @Test + public void testGetDatasetFileSize() throws TopcatException { + IcatClient icatClient = new IcatClient("https://localhost:8181", sessionId); + long datasetId = icatClient.getEntity("Dataset").getJsonNumber("id").longValueExact(); + assertNotEquals(0, icatClient.getDatasetFileSize(datasetId)); + } + /* * @Test public void testGetSize() throws Exception { IcatClient icatClient = * new IcatClient("https://localhost:8181", sessionId); diff --git a/src/test/java/org/icatproject/topcat/StatusCheckTest.java b/src/test/java/org/icatproject/topcat/StatusCheckTest.java index bf51aeab..fe9e67c7 100644 --- a/src/test/java/org/icatproject/topcat/StatusCheckTest.java +++ b/src/test/java/org/icatproject/topcat/StatusCheckTest.java @@ -762,17 +762,12 @@ public void testStartQueuedDownloadsNegative() throws Exception { statusCheck.startQueuedDownloads(-1); - // All Downloads should have been prepared, but because they have dummy - // (non-UUID) sessionId when prepareData is called it will throw and then mark - // the Downloads as EXPIRED as part of the error handling. This is OK, as it - // still indicates startQueuedDownloads called prepareData - Download postDownload1 = TestHelpers.getDummyDownload(downloadId1, downloadRepository); Download postDownload2 = TestHelpers.getDummyDownload(downloadId2, downloadRepository); - assertEquals(DownloadStatus.EXPIRED, postDownload1.getStatus()); + assertEquals(DownloadStatus.PREPARING, postDownload1.getStatus()); assertNull(postDownload1.getPreparedId()); - assertEquals(DownloadStatus.EXPIRED, postDownload2.getStatus()); + assertEquals(DownloadStatus.PREPARING, postDownload2.getStatus()); assertNull(postDownload2.getPreparedId()); } finally { // clean up @@ -821,15 +816,10 @@ public void testStartQueuedDownloadsNonZero() throws Exception { statusCheck.startQueuedDownloads(1); - // Should schedule only the first download, but because it has a dummy - // (non-UUID) sessionId when prepareData is called it will throw and then mark - // the Download as EXPIRED as part of the error handling. This is OK, as it - // still indicates startQueuedDownloads called prepareData - Download postDownload1 = TestHelpers.getDummyDownload(downloadId1, downloadRepository); Download postDownload2 = TestHelpers.getDummyDownload(downloadId2, downloadRepository); - assertEquals(DownloadStatus.EXPIRED, postDownload1.getStatus()); + assertEquals(DownloadStatus.PREPARING, postDownload1.getStatus()); assertNull(postDownload1.getPreparedId()); assertEquals(DownloadStatus.PAUSED, postDownload2.getStatus()); assertNull(postDownload2.getPreparedId()); diff --git a/src/test/java/org/icatproject/topcat/UserResourceTest.java b/src/test/java/org/icatproject/topcat/UserResourceTest.java index f168eb12..4a47576d 100644 --- a/src/test/java/org/icatproject/topcat/UserResourceTest.java +++ b/src/test/java/org/icatproject/topcat/UserResourceTest.java @@ -309,6 +309,7 @@ public void testQueueVisitId() throws Exception { assertEquals("simple/root", download.getUserName()); assertEquals("simple/root", download.getFullName()); assertEquals("", download.getEmail()); + assertNotEquals(0L, download.getSize()); part += 1; } } finally { @@ -318,52 +319,113 @@ public void testQueueVisitId() throws Exception { } } + @Test + public void testQueueVisitIdBadRequest() throws Exception { + System.out.println("DEBUG testQueueVisitIdBadRequest"); + String facilityName = "LILS"; + String transport = "http"; + String email = ""; + ThrowingRunnable runnable = () -> userResource.queueVisitId(facilityName, sessionId, transport, null, email, null); + Throwable throwable = assertThrows(BadRequestException.class, runnable); + assertEquals("(400) : visitId must be provided", throwable.getMessage()); + + runnable = () -> userResource.queueVisitId(facilityName, sessionId, transport, null, email, ""); + throwable = assertThrows(BadRequestException.class, runnable); + assertEquals("(400) : visitId must be provided", throwable.getMessage()); + } + + @Test + public void testQueueVisitIdNotFound() throws Exception { + System.out.println("DEBUG testQueueVisitIdBadRequest"); + String facilityName = "LILS"; + String transport = "http"; + String email = ""; + String visitId = "test"; + ThrowingRunnable runnable = () -> userResource.queueVisitId(facilityName, sessionId, transport, null, email, visitId); + Throwable throwable = assertThrows(NotFoundException.class, runnable); + assertEquals("(404) : No Datasets found for " + visitId, throwable.getMessage()); + } + @Test public void testQueueFiles() throws Exception { System.out.println("DEBUG testQueueFiles"); - List downloadIds = new ArrayList<>(); + Long downloadId = null; try { String facilityName = "LILS"; String transport = "http"; String email = ""; - + String file = "abcdefghijklmnopqrstuvwxyz"; IcatClient icatClient = new IcatClient("https://localhost:8181", sessionId); - List datafiles = icatClient.getEntities("datafile", 3L); + List datafiles = icatClient.getEntities("datafile", 2L); List files = new ArrayList<>(); + files.add(file); for (JsonObject datafile : datafiles) { files.add(datafile.getString("location")); } Response response = userResource.queueFiles(facilityName, sessionId, transport, null, email, files); assertEquals(200, response.getStatus()); - - JsonArray downloadIdsArray = Utils.parseJsonArray(response.getEntity().toString()); - long part = 1; - for (JsonNumber downloadIdJson : downloadIdsArray.getValuesAs(JsonNumber.class)) { - long downloadId = downloadIdJson.longValueExact(); - downloadIds.add(downloadId); - } - assertEquals(3, downloadIds.size()); - for (long downloadId : downloadIds) { - Download download = downloadRepository.getDownload(downloadId); - assertNull(download.getPreparedId()); - assertEquals(DownloadStatus.PAUSED, download.getStatus()); - assertEquals(0, download.getInvestigationIds().size()); - assertEquals(0, download.getDatasetIds().size()); - assertEquals(1, download.getDatafileIds().size()); - assertEquals("LILS_files_part_" + part + "_of_3", download.getFileName()); - assertEquals(transport, download.getTransport()); - assertEquals("simple/root", download.getUserName()); - assertEquals("simple/root", download.getFullName()); - assertEquals("", download.getEmail()); - part += 1; - } + JsonObject responseObject = Utils.parseJsonObject(response.getEntity().toString()); + downloadId = responseObject.getJsonNumber("downloadId").longValueExact(); + JsonArray missingArray = responseObject.getJsonArray("notFound"); + assertEquals(1, missingArray.size()); + assertEquals(file, missingArray.getString(0)); + Download download = downloadRepository.getDownload(downloadId); + assertNull(download.getPreparedId()); + assertEquals(DownloadStatus.PAUSED, download.getStatus()); + assertEquals(0, download.getInvestigationIds().size()); + assertEquals(0, download.getDatasetIds().size()); + assertEquals(2, download.getDatafileIds().size()); + assertEquals("LILS_files", download.getFileName()); + assertEquals(transport, download.getTransport()); + assertEquals("simple/root", download.getUserName()); + assertEquals("simple/root", download.getFullName()); + assertEquals("", download.getEmail()); + assertNotEquals(0L, download.getSize()); } finally { - for (long downloadId : downloadIds) { + if (downloadId != null) { downloadRepository.removeDownload(downloadId); } } } + @Test + public void testQueueFilesBadRequestEmpty() throws Exception { + System.out.println("DEBUG testQueueFilesBadRequestEmpty"); + String facilityName = "LILS"; + String transport = "http"; + String email = ""; + ThrowingRunnable runnable = () -> userResource.queueFiles(facilityName, sessionId, transport, null, email, null); + Throwable throwable = assertThrows(BadRequestException.class, runnable); + assertEquals("(400) : At least one Datafile.location required", throwable.getMessage()); + + runnable = () -> userResource.queueFiles(facilityName, sessionId, transport, null, email, new ArrayList<>()); + throwable = assertThrows(BadRequestException.class, runnable); + assertEquals("(400) : At least one Datafile.location required", throwable.getMessage()); + } + + @Test + public void testQueueFilesBadRequestTooMany() throws Exception { + System.out.println("DEBUG testQueueFilesBadRequestTooMany"); + String facilityName = "LILS"; + String transport = "http"; + String email = ""; + List files = List.of("1", "2", "3", "4"); + ThrowingRunnable runnable = () -> userResource.queueFiles(facilityName, sessionId, transport, null, email, files); + Throwable throwable = assertThrows(BadRequestException.class, runnable); + assertEquals("(400) : Limit of 3 files exceeded", throwable.getMessage()); + } + + @Test + public void testQueueFilesNotFound() throws Exception { + System.out.println("DEBUG testQueueFilesNotFound"); + String facilityName = "LILS"; + String transport = "http"; + String email = ""; + ThrowingRunnable runnable = () -> userResource.queueFiles(facilityName, sessionId, transport, null, email, List.of("test")); + Throwable throwable = assertThrows(NotFoundException.class, runnable); + assertEquals("(404) : No Datafiles found", throwable.getMessage()); + } + @Test public void testQueueAllowed() throws Exception { System.out.println("DEBUG testQueueAllowed"); diff --git a/src/test/resources/run.properties b/src/test/resources/run.properties index 31b93702..806ca0bc 100644 --- a/src/test/resources/run.properties +++ b/src/test/resources/run.properties @@ -17,10 +17,11 @@ queue.account.LILS.username=root queue.account.LILS.password=pw # Test data has 100 files per Dataset, set this to a small number to ensure coverage of the batching logic -queue.maxFileCount = 1 +queue.visit.maxPartFileCount = 1 +queue.files.maxFileCount = 3 queue.priority.user = {"simple/test": 1} queue.priority.default = 2 -# Each get request for Datafiles has a minimum size of 135, each of 3 locations is ~25 +# Each get request for Datafiles has a minimum size of 132, each of 3 locations is ~25 # A value of 200 allows us to chunk this into one chunk of 2, and a second chunk of 1, hitting both branches of the code getUrlLimit=200 diff --git a/tools/datagateway_admin b/tools/datagateway_admin index 7cd97858..d0337012 100644 --- a/tools/datagateway_admin +++ b/tools/datagateway_admin @@ -288,6 +288,90 @@ def manage_download_types(): }, verify=verifySsl).text) download_statuses[option_number] = download_status + +def queue_visit(): + transport = input("Enter transport mechanism: ") + email = input("Enter email to notify upon completion: ") + visit_id = input("Enter visit id: ") + data = { + "facilityName": facility_name, + "sessionId": session_id, + "transport": transport, + "email": email, + "visitId": visit_id, + } + url = topcat_url + "/user/queue/visit" + print(requests.post(url=url, data=data, verify=verifySsl).text) + + +def queue_files(): + transport = input("Enter transport mechanism: ") + email = input("Enter email to notify upon completion: ") + local_file = input("Enter path to local file containing newline delimited file locations: ") + with open(local_file) as f: + files = [l.strip() for l in f.readlines()] + + data = { + "facilityName": facility_name, + "sessionId": session_id, + "transport": transport, + "email": email, + "files": files, + } + url = topcat_url + "/user/queue/files" + print(requests.post(url=url, data=data, verify=verifySsl).text) + + + +def get_all_queued_downloads(): + query_offset = "WHERE download.isDeleted != true" + query_offset += " AND download.status = org.icatproject.topcat.domain.DownloadStatus.PAUSED" + query_offset += " AND download.preparedId = null" + params = { + "facilityName": facility_name, + "sessionId": session_id, + "queryOffset": query_offset, + } + return requests.get(topcat_url + "/admin/downloads", params=params, verify=verifySsl).text + + +def prepare_download(download_id): + data = { + "facilityName": facility_name, + "sessionId": session_id, + "value": "PREPARING" + } + url = topcat_url + "/admin/download/" + download_id + "/status" + requests.put(url=url, data=data, verify=verifySsl) + + +def show_all_queued_downloads(): + print(get_all_queued_downloads()) + + +def start_queued_download(): + download_id = input("Enter download id: ") + prepare_download(download_id=download_id) + + +def start_queued_downloads(): + text = get_all_queued_downloads() + downloads = json.loads(text) + for download in downloads: + prepare_download(str(download["id"])) + + +def requeue_download(): + download_id = input("Enter download id: ") + data = { + "facilityName": facility_name, + "sessionId": session_id, + "value": "PAUSED" + } + url = topcat_url + "/admin/download/" + download_id + "/status" + requests.put(url=url, data=data, verify=verifySsl) + + while True: print("") print("What do you want to do?") @@ -297,7 +381,13 @@ while True: print(" * 4: Set a download status to 'EXPIRED'.") print(" * 5: Expire all pending downloads.") print(" * 6: Enable or disable download types.") - print(" * 7: Exit") + print(" * 7: Queue visit.") + print(" * 8: Queue files.") + print(" * 9: Show all queued downloads.") + print(" * 10: Start a queued download.") + print(" * 11: Start all queued downloads.") + print(" * 12: Re-queue expired download.") + print(" * 13: Exit") option_number = input("Enter option number: "); @@ -315,6 +405,18 @@ while True: elif option_number == "6": manage_download_types() elif option_number == "7": + queue_visit() + elif option_number == "8": + queue_files() + elif option_number == "9": + show_all_queued_downloads() + elif option_number == "10": + start_queued_download() + elif option_number == "11": + start_queued_downloads() + elif option_number == "12": + requeue_download() + elif option_number == "13": break else: print("")