Skip to content

Commit e6c79e0

Browse files
patrick-austinpatrick-austin
authored
Merge pull request #89 from ral-facilities/submit_collection_endpoint
Add /queue/dataCollection endpoint, refactor into DownloadBuilder class
2 parents f5d691f + afe911e commit e6c79e0

File tree

5 files changed

+522
-191
lines changed

5 files changed

+522
-191
lines changed

‎src/main/java/org/icatproject/topcat/DownloadBuilder.java‎

Lines changed: 333 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,333 @@
1+
package org.icatproject.topcat;
2+
3+
import java.io.UnsupportedEncodingException;
4+
import java.util.ArrayList;
5+
import java.util.List;
6+
7+
import org.icatproject.topcat.IcatClient.DatafilesResponse;
8+
import org.icatproject.topcat.domain.Download;
9+
import org.icatproject.topcat.domain.DownloadItem;
10+
import org.icatproject.topcat.domain.EntityType;
11+
import org.icatproject.topcat.exceptions.BadRequestException;
12+
import org.icatproject.topcat.exceptions.InternalException;
13+
import org.icatproject.topcat.exceptions.NotFoundException;
14+
import org.icatproject.topcat.exceptions.TopcatException;
15+
import org.slf4j.Logger;
16+
import org.slf4j.LoggerFactory;
17+
18+
import jakarta.json.Json;
19+
import jakarta.json.JsonArray;
20+
import jakarta.json.JsonArrayBuilder;
21+
import jakarta.json.JsonValue;
22+
23+
public class DownloadBuilder {
24+
25+
private static final Logger logger = LoggerFactory.getLogger(DownloadBuilder.class);
26+
27+
private long queueVisitMaxPartFileCount;
28+
private long queueFilesMaxFileCount;
29+
30+
private String sessionId;
31+
private String email;
32+
private String userName;
33+
private String fullName;
34+
private int priority = 0;
35+
private IcatClient icatClient;
36+
37+
public String transport;
38+
public String facilityName;
39+
public String fileName;
40+
public final List<Download> downloads = new ArrayList<Download>();
41+
42+
public DownloadBuilder(String sessionId, String email, String fileName, String transport, String facilityName) throws TopcatException {
43+
Properties properties = Properties.getInstance();
44+
queueVisitMaxPartFileCount = Long.valueOf(properties.getProperty("queue.visit.maxPartFileCount", "10000"));
45+
queueFilesMaxFileCount = Long.valueOf(properties.getProperty("queue.files.maxFileCount", "10000"));
46+
47+
this.sessionId = sessionId;
48+
this.fileName = fileName;
49+
this.transport = validateTransport(transport);
50+
this.facilityName = validateFacilityName(facilityName);
51+
this.email = validateEmail(transport, email);
52+
String icatUrl = getIcatUrl(this.facilityName);
53+
54+
icatClient = new IcatClient(icatUrl, sessionId);
55+
56+
userName = icatClient.getUserName();
57+
fullName = icatClient.getFullName();
58+
priority = icatClient.getQueuePriority(userName);
59+
icatClient.checkQueueAllowed(userName);
60+
}
61+
62+
/**
63+
* Validate that the submitted transport mechanism is not null or empty.
64+
*
65+
* @param transport Transport mechanism to use
66+
* @return transport
67+
* @throws BadRequestException if null or empty
68+
*/
69+
public static String validateTransport(String transport) throws BadRequestException {
70+
if (transport == null || transport.trim().isEmpty()) {
71+
throw new BadRequestException("transport is required");
72+
}
73+
return transport;
74+
}
75+
76+
/**
77+
* Validate that if the submitted facilityName is null, then a default is defined.
78+
*
79+
* @param facilityName ICAT Facility.name
80+
* @return facilityName or the default facilityName
81+
* @throws BadRequestException if null and default not defined
82+
*/
83+
public static String validateFacilityName(String facilityName) throws BadRequestException {
84+
try {
85+
return FacilityMap.getInstance().validateFacilityName(facilityName);
86+
} catch (InternalException ie) {
87+
throw new BadRequestException(ie.getMessage());
88+
}
89+
}
90+
91+
/**
92+
* Validate that the submitted email is not null or empty if mail.required is true.
93+
*
94+
* @param transport Transport mechanism to use (which may require email)
95+
* @param email Users email address, which may be null or empty
96+
* @return The original email, or null if it was an empty string
97+
* @throws BadRequestException if email null or empty and mail.required is true
98+
*/
99+
public static String validateEmail(String transport, String email) throws BadRequestException {
100+
if(email != null && email.equals("")){
101+
email = null;
102+
}
103+
104+
String emailRequired = Properties.getInstance().getProperty("mail.required." + transport, "false");
105+
if (Boolean.parseBoolean(emailRequired) && email == null) {
106+
throw new BadRequestException("email is required for " + transport);
107+
}
108+
109+
return email;
110+
}
111+
112+
/**
113+
* @param facilityName ICAT Facility.name
114+
* @return ICAT server url
115+
* @throws BadRequestException If facilityName is not recognised
116+
*/
117+
public static String getIcatUrl(String facilityName) throws BadRequestException{
118+
try {
119+
return FacilityMap.getInstance().getIcatUrl(facilityName);
120+
} catch (InternalException ie) {
121+
throw new BadRequestException(ie.getMessage());
122+
}
123+
}
124+
125+
/**
126+
* Create a new Download object and set basic fields, excluding data and status.
127+
*
128+
* @param sessionId ICAT sessionId
129+
* @param facilityName ICAT Facility.name
130+
* @param fileName Filename for the resultant Download
131+
* @param userName ICAT User.name
132+
* @param fullName ICAT User.fullName
133+
* @param transport Transport mechanism to use
134+
* @param email Optional email to send notification to on completion
135+
* @return Download object with basic fields set
136+
*/
137+
private Download createDownload() {
138+
Download download = new Download();
139+
download.setSessionId(sessionId);
140+
download.setFacilityName(facilityName);
141+
download.setFileName(fileName);
142+
download.setUserName(userName);
143+
download.setFullName(fullName);
144+
download.setTransport(transport);
145+
download.setEmail(email);
146+
download.setIsEmailSent(false);
147+
download.setSize(0);
148+
return download;
149+
}
150+
151+
/**
152+
* Create a new DownloadItem.
153+
*
154+
* @param download Parent Download
155+
* @param entityId ICAT Entity.id
156+
* @param entityType EntityType
157+
* @return DownloadItem with fields set
158+
*/
159+
private static DownloadItem createDownloadItem(Download download, long entityId, EntityType entityType) {
160+
DownloadItem downloadItem = new DownloadItem();
161+
downloadItem.setEntityId(entityId);
162+
downloadItem.setEntityType(entityType);
163+
downloadItem.setDownload(download);
164+
return downloadItem;
165+
}
166+
167+
/**
168+
* Adds Datasets and Datafiles from from a DataCollection as DownloadItems.
169+
* Investigations will be split into constituent Datasets.
170+
*
171+
* @param dataCollectionId ICAT DataCollection.id
172+
* @throws TopcatException if querying ICAT fails
173+
*/
174+
public void extractDataCollection(Long dataCollectionId) throws TopcatException {
175+
176+
logger.info("extractDataCollection called for {}", dataCollectionId);
177+
if (dataCollectionId == null || dataCollectionId < 1) {
178+
throw new BadRequestException("Valid dataCollectionId must be provided");
179+
}
180+
181+
if (fileName == null) {
182+
fileName = facilityName + "_DataCollection" + dataCollectionId;
183+
}
184+
185+
JsonArrayBuilder datasetsBuilder = Json.createArrayBuilder();
186+
JsonArray datafiles = JsonArray.EMPTY_JSON_ARRAY;
187+
for (JsonValue dataset : icatClient.getDataCollectionDatasets(dataCollectionId)) {
188+
datasetsBuilder.add(dataset);
189+
}
190+
datafiles = icatClient.getDataCollectionDatafiles(dataCollectionId);
191+
JsonArray datasets = datasetsBuilder.build();
192+
193+
if (datasets.size() == 0 && datafiles.size() == 0) {
194+
throw new NotFoundException("No data found for DataCollection " + dataCollectionId);
195+
}
196+
197+
buildDownloads(datasets, datafiles);
198+
}
199+
200+
/**
201+
* Adds Datasets from an Investigation as DownloadItems.
202+
*
203+
* @param visitId ICAT Investigation.visitId
204+
* @throws TopcatException if querying ICAT fails
205+
*/
206+
public void extractVisit(String visitId) throws TopcatException {
207+
logger.info("extractVisit called for {}", visitId);
208+
209+
if (visitId == null || visitId.equals("")) {
210+
throw new BadRequestException("visitId must be provided");
211+
}
212+
213+
if (fileName == null) {
214+
fileName = facilityName + "_" + visitId;
215+
}
216+
217+
JsonArray datasets = icatClient.getDatasets(visitId);
218+
if (datasets.size() == 0) {
219+
throw new NotFoundException("No Datasets found for " + visitId);
220+
}
221+
222+
buildDownloads(datasets, JsonArray.EMPTY_JSON_ARRAY);
223+
}
224+
225+
/**
226+
* Adds Datafiles as DownloadItems.
227+
*
228+
* @param files List of ICAT Datafile.locations
229+
* @return DatafilesResponse object representing the found and missing Datafiles
230+
* @throws TopcatException if querying ICAT fails
231+
* @throws UnsupportedEncodingException if query encoding fails
232+
*/
233+
public DatafilesResponse extractLocations(List<String> files) throws TopcatException, UnsupportedEncodingException {
234+
if (files == null || files.size() == 0) {
235+
throw new BadRequestException("At least one Datafile.location required");
236+
} else if (files.size() > queueFilesMaxFileCount) {
237+
throw new BadRequestException("Limit of " + queueFilesMaxFileCount + " files exceeded");
238+
}
239+
240+
logger.info("extractLocations called for {} files", files.size());
241+
242+
if (fileName == null) {
243+
fileName = facilityName + "_files";
244+
}
245+
246+
DatafilesResponse response = icatClient.getDatafiles(files);
247+
if (response.ids.size() == 0) {
248+
throw new NotFoundException("No Datafiles found");
249+
}
250+
251+
List<DownloadItem> downloadItems = new ArrayList<>();
252+
Download download = createDownload();
253+
for (long datafileId : response.ids) {
254+
DownloadItem downloadItem = createDownloadItem(download, datafileId, EntityType.datafile);
255+
downloadItems.add(downloadItem);
256+
}
257+
download.setDownloadItems(downloadItems);
258+
download.setSize(response.totalSize);
259+
download.setPriority(priority);
260+
downloads.add(download);
261+
262+
return response;
263+
}
264+
265+
/**
266+
* Add Downloads to this.downloads containing datasets and datafiles as
267+
* DownloadItems, so that none exceeds the configured part limit.
268+
*
269+
* @param datasets JsonArray of [dataset.id, dataset.fileCount, dataset.fileSize]
270+
* @param datafiles JsonArray of [datafile.id, datafile.fileSize]
271+
* @throws TopcatException if size calculation fails
272+
*/
273+
private void buildDownloads(JsonArray datasets, JsonArray datafiles) throws TopcatException {
274+
long downloadFileCount = 0L;
275+
long downloadFileSize = 0L;
276+
List<DownloadItem> downloadItems = new ArrayList<DownloadItem>();
277+
Download newDownload = createDownload();
278+
279+
for (JsonValue dataset : datasets) {
280+
JsonArray datasetArray = dataset.asJsonArray();
281+
long datasetId = datasetArray.getJsonNumber(0).longValueExact();
282+
long datasetFileCount = datasetArray.getJsonNumber(1).longValueExact();
283+
long datasetFileSize = datasetArray.getJsonNumber(2).longValueExact();
284+
// Database triggers should set these, but check explicitly anyway
285+
if (datasetFileCount < 1L) {
286+
datasetFileCount = icatClient.getDatasetFileCount(datasetId);
287+
}
288+
if (datasetFileSize < 1L) {
289+
datasetFileSize = icatClient.getDatasetFileSize(datasetId);
290+
}
291+
292+
if (downloadFileCount > 0L && downloadFileCount + datasetFileCount > queueVisitMaxPartFileCount) {
293+
newDownload.setDownloadItems(downloadItems);
294+
newDownload.setSize(downloadFileSize);
295+
downloads.add(newDownload);
296+
297+
downloadFileCount = 0L;
298+
downloadFileSize = 0L;
299+
downloadItems = new ArrayList<DownloadItem>();
300+
newDownload = createDownload();
301+
}
302+
303+
DownloadItem downloadItem = createDownloadItem(newDownload, datasetId, EntityType.dataset);
304+
downloadItems.add(downloadItem);
305+
downloadFileCount += datasetFileCount;
306+
downloadFileSize += datasetFileSize;
307+
}
308+
for (JsonValue datafile : datafiles) {
309+
JsonArray datafileArray = datafile.asJsonArray();
310+
long datafileId = datafileArray.getJsonNumber(0).longValueExact();
311+
long datafileSize = datafileArray.getJsonNumber(1).longValueExact();
312+
313+
if (downloadFileCount >= queueVisitMaxPartFileCount) {
314+
newDownload.setDownloadItems(downloadItems);
315+
newDownload.setSize(downloadFileSize);
316+
downloads.add(newDownload);
317+
318+
downloadFileCount = 0L;
319+
downloadFileSize = 0L;
320+
downloadItems = new ArrayList<DownloadItem>();
321+
newDownload = createDownload();
322+
}
323+
324+
DownloadItem downloadItem = createDownloadItem(newDownload, datafileId, EntityType.datafile);
325+
downloadItems.add(downloadItem);
326+
downloadFileCount += 1L;
327+
downloadFileSize += datafileSize;
328+
}
329+
newDownload.setDownloadItems(downloadItems);
330+
newDownload.setSize(downloadFileSize);
331+
downloads.add(newDownload);
332+
}
333+
}

‎src/main/java/org/icatproject/topcat/IcatClient.java‎

Lines changed: 33 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -234,6 +234,38 @@ public DatafilesResponse getDatafiles(List<String> files) throws TopcatException
234234
return response;
235235
}
236236

237+
/**
238+
* Get all Datasets in the specified DataCollection.
239+
*
240+
* @param dataCollectionId ICAT DataCollection.id
241+
* @return JsonArray of Dataset fields, where each entry is a JsonArray of
242+
* [dataset.id, dataset.fileCount].
243+
* @throws TopcatException
244+
*/
245+
public JsonArray getDataCollectionDatasets(long dataCollectionId) throws TopcatException {
246+
String query = "SELECT dataCollectionDataset.dataset.id, dataCollectionDataset.dataset.fileCount,";
247+
query += " dataCollectionDataset.dataset.fileSize FROM DataCollectionDataset dataCollectionDataset";
248+
query += " WHERE dataCollectionDataset.dataCollection.id = " + dataCollectionId;
249+
query += " ORDER BY dataCollectionDataset.dataset.id";
250+
return submitQuery(query);
251+
}
252+
253+
/**
254+
* Get all Datafiles in the specified DataCollection.
255+
*
256+
* @param dataCollectionId ICAT DataCollection.id
257+
* @return JsonArray of Datafile fields, where each entry is a JsonArray of
258+
* [datafile.id, datafile.fileSize].
259+
* @throws TopcatException
260+
*/
261+
public JsonArray getDataCollectionDatafiles(long dataCollectionId) throws TopcatException {
262+
String query = "SELECT dataCollectionDatafile.datafile.id, dataCollectionDatafile.datafile.fileSize";
263+
query += " FROM DataCollectionDatafile dataCollectionDatafile";
264+
query += " WHERE dataCollectionDatafile.dataCollection.id = " + dataCollectionId;
265+
query += " ORDER BY dataCollectionDatafile.datafile.id";
266+
return submitQuery(query);
267+
}
268+
237269
/**
238270
* Utility method to get the fileCount (not size) of a Dataset by COUNT of its
239271
* child Datafiles. Ideally the fileCount field should be used, this is a
@@ -326,7 +358,7 @@ public JsonObject getEntity(String entityType) throws TopcatException {
326358
*/
327359
public List<JsonObject> getEntities(String entityType, long limit) throws TopcatException {
328360
try {
329-
String entityCapital = StringUtils.capitalize(entityType.toLowerCase());
361+
String entityCapital = StringUtils.capitalize(entityType);
330362
String query = URLEncoder.encode("SELECT o FROM " + entityCapital + " o LIMIT 0, " + limit, "UTF8");
331363
String url = "entityManager?sessionId=" + URLEncoder.encode(sessionId, "UTF8") + "&query=" + query;
332364
Response response = httpClient.get(url, new HashMap<String, String>());

0 commit comments

Comments
 (0)