Skip to content

Commit b220d00

Browse files
patrick-austinpatrick-austin
authored
Merge pull request #94 from ral-facilities/93_cart_limits
Enforce cart size and count limits in the submitCart endpoint
2 parents e6c79e0 + 86206e9 commit b220d00

File tree

6 files changed

+310
-44
lines changed

6 files changed

+310
-44
lines changed

‎src/main/config/run.properties.example‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,12 @@ facility.YFH.downloadType.globus.displayName = Globus
3636
facility.YFH.downloadType.globus.description = Example description for Globus access method.
3737
facility.YFH.downloadType.globus.allowedGroupings = principal_beamline_scientists admins
3838

39+
# Maximum number of Datafiles that can be included in a single cart request.
40+
# Investigations and Datasets are expanded into their constituent Datafiles for this count.
41+
facility.LILS.limit.count = 250000
42+
# Maximum total volume, in bytes, that can be included in a single cart request.
43+
facility.LILS.limit.size = 10000000000000
44+
3945
# enable send email
4046
mail.enable=true
4147

‎src/main/java/org/icatproject/topcat/FacilityMap.java‎

Lines changed: 76 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,12 @@
99
import org.slf4j.LoggerFactory;
1010

1111
public class FacilityMap {
12+
public static class Facility {
13+
public String icatUrl;
14+
public String idsUrl;
15+
public Long countLimit = null;
16+
public Long sizeLimit = null;
17+
}
1218

1319
private static FacilityMap instance = null;
1420

@@ -22,8 +28,7 @@ public synchronized static FacilityMap getInstance() throws InternalException {
2228
private Logger logger = LoggerFactory.getLogger(FacilityMap.class);
2329

2430
private Properties properties;
25-
private Map<String,String> facilityIcatUrl;
26-
private Map<String,String> facilityIdsUrl;
31+
private Map<String, Facility> facilityMapping = new HashMap<>();
2732

2833
public FacilityMap() throws InternalException{
2934
// The "normal" case: use the Topcat Properties instance (that reads run.properties)
@@ -34,9 +39,6 @@ public FacilityMap(Properties injectedProperties) throws InternalException{
3439

3540
// This allows us to inject a mock Properties instance for testing
3641

37-
facilityIcatUrl = new HashMap<String,String>();
38-
facilityIdsUrl = new HashMap<String,String>();
39-
4042
properties = injectedProperties;
4143

4244
logger.info("FacilityMap: facility.list = '" + properties.getProperty("facility.list","") + "'");
@@ -49,26 +51,40 @@ public FacilityMap(Properties injectedProperties) throws InternalException{
4951
throw new InternalException("Property facility.list is not defined.");
5052
}
5153

52-
for( String facility : facilities ){
53-
logger.info("FacilityMap: looking for properties for facility '" + facility + "'...");
54-
String icatUrl = properties.getProperty("facility." + facility + ".icatUrl","");
54+
for( String facilityName : facilities ){
55+
logger.info("FacilityMap: looking for properties for facility '" + facilityName + "'...");
56+
Facility facility = new Facility();
57+
String icatUrl = properties.getProperty("facility." + facilityName + ".icatUrl","");
5558
// Complain/log if property is not set
5659
if( icatUrl.length() == 0 ){
57-
String error = "FacilityMap: property facility." + facility + ".icatUrl is not defined.";
60+
String error = "FacilityMap: property facility." + facilityName + ".icatUrl is not defined.";
5861
logger.error( error );
5962
throw new InternalException( error );
6063
}
61-
logger.info("FacilityMap: icatUrl for facility '" + facility + "' is '" + icatUrl + "'");
62-
facilityIcatUrl.put( facility, icatUrl );
63-
String idsUrl = properties.getProperty("facility." + facility + ".idsUrl","");
64+
logger.info("FacilityMap: icatUrl for facility '" + facilityName + "' is '" + icatUrl + "'");
65+
facility.icatUrl = icatUrl;
66+
67+
String idsUrl = properties.getProperty("facility." + facilityName + ".idsUrl","");
6468
// Complain/log if property is not set
6569
if( idsUrl.length() == 0 ){
66-
String error = "FacilityMap: property facility." + facility + ".idsUrl is not defined.";
70+
String error = "FacilityMap: property facility." + facilityName + ".idsUrl is not defined.";
6771
logger.error( error );
6872
throw new InternalException( error );
6973
}
70-
logger.info("FacilityMap: idsUrl for facility '" + facility + "' is '" + idsUrl + "'");
71-
facilityIdsUrl.put( facility, idsUrl );
74+
logger.info("FacilityMap: idsUrl for facility '" + facilityName + "' is '" + idsUrl + "'");
75+
facility.idsUrl = idsUrl;
76+
77+
String countString = properties.getProperty("facility." + facilityName + ".limit.count");
78+
if (countString != null) {
79+
facility.countLimit = Long.valueOf(countString);
80+
}
81+
82+
String sizeString = properties.getProperty("facility." + facilityName + ".limit.size");
83+
if (sizeString != null) {
84+
facility.sizeLimit = Long.valueOf(sizeString);
85+
}
86+
87+
facilityMapping.put(facilityName, facility);
7288
}
7389
}
7490

@@ -84,34 +100,22 @@ public String validateFacilityName(String facility) throws InternalException {
84100
}
85101
return facility;
86102
}
103+
104+
public String getIcatUrl(String facilityName) throws InternalException {
105+
Facility facility = getFacility(facilityName);
106+
return facility.icatUrl;
107+
}
87108

88109
/**
89110
* @return All the ICAT Facility.names from the config file
90111
*/
91112
public Set<String> getFacilities() {
92-
return facilityIcatUrl.keySet();
113+
return facilityMapping.keySet();
93114
}
94115

95-
public String getIcatUrl( String facility ) throws InternalException{
96-
facility = validateFacilityName(facility);
97-
String url = facilityIcatUrl.get( facility );
98-
if( url == null ){
99-
String error = "FacilityMap.getIcatUrl: unknown facility: " + facility;
100-
logger.error( error );
101-
throw new InternalException( error );
102-
}
103-
return url;
104-
}
105-
106-
public String getIdsUrl( String facility ) throws InternalException{
107-
facility = validateFacilityName(facility);
108-
String url = facilityIdsUrl.get( facility );
109-
if( url == null ){
110-
String error = "FacilityMap.getIdsUrl: unknown facility: " + facility;
111-
logger.error( error );
112-
throw new InternalException( error );
113-
}
114-
return url;
116+
public String getIdsUrl(String facilityName) throws InternalException {
117+
Facility facility = getFacility(facilityName);
118+
return facility.idsUrl;
115119
}
116120

117121
public String getDownloadUrl( String facility, String downloadType ) throws InternalException{
@@ -127,4 +131,40 @@ public String getDownloadUrl( String facility, String downloadType ) throws Inte
127131
}
128132
return url;
129133
}
134+
135+
/**
136+
* @param facilityName ICAT Facility.name
137+
* @return Limit on the number of Datafiles allowed in a cart, or null if not limit set
138+
* @throws InternalException if facilityName is not a key in facilityMapping
139+
*/
140+
public Long getCountLimit(String facilityName) throws InternalException {
141+
Facility facility = getFacility(facilityName);
142+
return facility.countLimit;
143+
}
144+
145+
/**
146+
* @param facilityName ICAT Facility.name
147+
* @return Limit on the total size of Datafiles allowed in a cart, or null if not limit set
148+
* @throws InternalException if facilityName is not a key in facilityMapping
149+
*/
150+
public Long getSizeLimit(String facilityName) throws InternalException {
151+
Facility facility = getFacility(facilityName);
152+
return facility.sizeLimit;
153+
}
154+
155+
/**
156+
* @param facilityName ICAT Facility.name
157+
* @return Facility config object with the given name
158+
* @throws InternalException if facilityName is not a key in facilityMapping
159+
*/
160+
private Facility getFacility(String facilityName) throws InternalException {
161+
facilityName = validateFacilityName(facilityName);
162+
Facility facility = facilityMapping.get(facilityName);
163+
if (facility == null) {
164+
String error = "FacilityMap.getFacility: unknown facility: " + facility;
165+
logger.error(error);
166+
throw new InternalException(error);
167+
}
168+
return facility;
169+
}
130170
}

‎src/main/java/org/icatproject/topcat/IcatClient.java‎

Lines changed: 110 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -48,11 +48,114 @@ public void submitDatafilesQuery(String query)
4848
}
4949
}
5050

51+
/**
52+
* Utility class for calculating the count and size of a cart.
53+
*/
54+
public class EntityCounter {
55+
private int getUrlLimit = Integer.parseInt(Properties.getInstance().getProperty("getUrlLimit", "1024"));
56+
public long totalSize = 0L;
57+
public long totalCount = 0L;
58+
59+
/**
60+
* Calculate the totalSize and totalCount of all ICAT Entities provided.
61+
*
62+
* @param investigationIds List of ICAT Investigation.id in the cart
63+
* @param datasetIds List of ICAT Dataset.id in the cart
64+
* @param datafileIds List of ICAT Datafile.id in the cart
65+
* @throws UnsupportedEncodingException if Entity ids cannot be URL encoded for ICAT queries
66+
* @throws TopcatException if ICAT query fails
67+
*/
68+
public EntityCounter(List<Long> investigationIds, List<Long> datasetIds, List<Long> datafileIds)
69+
throws UnsupportedEncodingException, TopcatException {
70+
71+
processIds(investigationIds, "SELECT SUM(i.fileSize), SUM(i.fileCount) FROM Investigation i WHERE i.id IN (");
72+
processIds(datasetIds, "SELECT SUM(d.fileSize), SUM(d.fileCount) FROM Dataset d WHERE d.id IN (");
73+
processIds(datafileIds, "SELECT SUM(d.fileSize) FROM Datafile d WHERE d.id IN (");
74+
totalCount += datafileIds.size();
75+
}
76+
77+
/**
78+
* Process ids for either Investigations, Datasets or Datafile. These will be
79+
* chunked into a series of IN clauses to avoid exceeding the max url length.
80+
*
81+
* @param ids List of ICAT Entity.id
82+
* @param queryPrefix SELECT query up to but not including a chunked list of ids
83+
* @throws UnsupportedEncodingException if Entity ids cannot be URL encoded for ICAT queries
84+
* @throws TopcatException if ICAT query fails
85+
*/
86+
private void processIds(List<Long> ids, String queryPrefix) throws UnsupportedEncodingException, TopcatException {
87+
if (!ids.isEmpty()) {
88+
int chunkLimit = getUrlLimit - minimumQuerySize - URLEncoder.encode(queryPrefix, "UTF8").length() - parenthesisSize;
89+
ListIterator<Long> iterator = ids.listIterator();
90+
Long id = iterator.next();
91+
String chunkedIds = id.toString();
92+
int chunkSize = URLEncoder.encode(chunkedIds, "UTF8").length();
93+
while (iterator.hasNext()) {
94+
id = iterator.next();
95+
String idString = id.toString();
96+
int encodedIdLength = URLEncoder.encode(idString, "UTF8").length();
97+
if (chunkSize + commaSize + encodedIdLength > chunkLimit) {
98+
submitIdsChunk(queryPrefix, chunkedIds);
99+
chunkedIds = idString;
100+
chunkSize = encodedIdLength;
101+
} else {
102+
chunkedIds += "," + idString;
103+
chunkSize += commaSize + encodedIdLength;
104+
}
105+
}
106+
submitIdsChunk(queryPrefix, chunkedIds);
107+
}
108+
}
109+
110+
/**
111+
* Submits and processes the result from a chunk of ids.
112+
*
113+
* @param queryPrefix SELECT query up to but not including a chunked list of ids
114+
* @param chunkedIds Comma separated list of ICAT Entity ids
115+
* @throws TopcatException if ICAT query fails
116+
*/
117+
private void submitIdsChunk(String queryPrefix, String chunkedIds) throws TopcatException {
118+
JsonArray jsonArray = submitQuery(queryPrefix + chunkedIds + ")");
119+
JsonValue jsonValue = jsonArray.get(0);
120+
if (jsonValue.getValueType().equals(ValueType.NUMBER)) {
121+
totalSize += ((JsonNumber) jsonValue).longValueExact();
122+
} else {
123+
JsonArray jsonArrayInner = jsonValue.asJsonArray();
124+
if (jsonArrayInner.get(0).getValueType().equals(ValueType.NUMBER)) {
125+
totalSize += jsonArrayInner.getJsonNumber(0).longValueExact();
126+
}
127+
if (jsonArrayInner.size() > 1 && jsonArrayInner.get(1).getValueType().equals(ValueType.NUMBER)) {
128+
totalCount += jsonArrayInner.getJsonNumber(1).longValueExact();
129+
}
130+
}
131+
}
132+
}
133+
51134
private Logger logger = LoggerFactory.getLogger(IcatClient.class);
52135

53136
private HttpClient httpClient;
54137
private String sessionId;
55138

139+
private static final int minimumQuerySize = "entityManager?sessionId=&query=".length() + 36; // sessionIds are 36 characters
140+
private static final int commaSize;
141+
private static final int parenthesisSize;
142+
143+
static {
144+
int parenthesisSizeNonFinal = 3;
145+
try {
146+
parenthesisSizeNonFinal = URLEncoder.encode(")", "UTF8").length();
147+
} catch (UnsupportedEncodingException e) {} finally{
148+
parenthesisSize = parenthesisSizeNonFinal;
149+
}
150+
151+
int commaSizeNonFinal = 3;
152+
try {
153+
commaSizeNonFinal = URLEncoder.encode(",", "UTF8").length();
154+
} catch (UnsupportedEncodingException e) {} finally{
155+
commaSize = commaSizeNonFinal;
156+
}
157+
}
158+
56159
public IcatClient(String url) {
57160
this.httpClient = new HttpClient(url + "/icat");
58161
}
@@ -201,12 +304,12 @@ public DatafilesResponse getDatafiles(List<String> files) throws TopcatException
201304
return response;
202305
}
203306

204-
// Total limit - "entityManager?sessionId=" - `sessionId` - "?query=" - `queryPrefix` - `querySuffix
205-
// Limit is 1024 - 24 - 36 - 7 - 48 - 17
206-
int getUrlLimit = Integer.parseInt(Properties.getInstance().getProperty("getUrlLimit", "1024"));
207-
int chunkLimit = getUrlLimit - 132;
208307
String queryPrefix = "SELECT d from Datafile d WHERE d.location in (";
209308
String querySuffix = ") ORDER BY d.id";
309+
int getUrlLimit = Integer.parseInt(Properties.getInstance().getProperty("getUrlLimit", "1024"));
310+
int queryPrefixSize = URLEncoder.encode(queryPrefix, "UTF8").length();
311+
int querySuffixSize = URLEncoder.encode(querySuffix, "UTF8").length();
312+
int chunkLimit = getUrlLimit - minimumQuerySize - queryPrefixSize - querySuffixSize;
210313
ListIterator<String> iterator = files.listIterator();
211314

212315
String file = iterator.next();
@@ -217,15 +320,15 @@ public DatafilesResponse getDatafiles(List<String> files) throws TopcatException
217320
file = iterator.next();
218321
String quotedFile = "'" + file + "'";
219322
int encodedFileLength = URLEncoder.encode(quotedFile, "UTF8").length();
220-
if (chunkSize + 3 + encodedFileLength > chunkLimit) {
323+
if (chunkSize + commaSize + encodedFileLength > chunkLimit) {
221324
response.submitDatafilesQuery(queryPrefix + chunkedFiles + querySuffix);
222325

223326
chunkedFiles = quotedFile;
224327
chunkSize = encodedFileLength;
225328
response.missing.add(file);
226329
} else {
227330
chunkedFiles += "," + quotedFile;
228-
chunkSize += 3 + encodedFileLength; // 3 is size of , when encoded as %2C
331+
chunkSize += commaSize + encodedFileLength;
229332
response.missing.add(file);
230333
}
231334
}
@@ -291,7 +394,7 @@ public long getDatasetFileCount(long datasetId) throws TopcatException {
291394
* @throws TopcatException
292395
*/
293396
public long getDatasetFileSize(long datasetId) throws TopcatException {
294-
String query = "SELECT SUM(datafile.fileSize) FROM Datafile datafile WHERE datafile.dataset.id = " + datasetId;
397+
String query = "SELECT SUM(datafile.fileSize) FROM Datafile datafile WHERE datafile.dataset.id = " + datasetId;
295398
JsonArray jsonArray = submitQuery(query);
296399
if (jsonArray.get(0).getValueType().equals(ValueType.NUMBER)) {
297400
return jsonArray.getJsonNumber(0).longValueExact();

‎src/main/java/org/icatproject/topcat/web/rest/UserResource.java‎

Lines changed: 33 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -711,6 +711,8 @@ public Response deleteCartItems(@PathParam("facilityName") String facilityName,
711711
*
712712
* @throws TopcatException
713713
* if anything else goes wrong.
714+
* @throws UnsupportedEncodingException
715+
* if Entity ids cannot be URL encoded for ICAT queries
714716
*/
715717
@POST
716718
@Path("/cart/{facilityName}/submit")
@@ -721,7 +723,7 @@ public Response submitCart(@PathParam("facilityName") String facilityName,
721723
@FormParam("email") String email,
722724
@FormParam("fileName") String fileName,
723725
@FormParam("zipType") String zipType)
724-
throws TopcatException, MalformedURLException, ParseException {
726+
throws TopcatException, MalformedURLException, ParseException, UnsupportedEncodingException {
725727

726728
logger.info("submitCart called");
727729

@@ -750,6 +752,36 @@ public Response submitCart(@PathParam("facilityName") String facilityName,
750752

751753
if (cart != null) {
752754
em.refresh(cart);
755+
FacilityMap facilityMap = FacilityMap.getInstance();
756+
Long countLimit = facilityMap.getCountLimit(facilityName);
757+
Long sizeLimit = facilityMap.getSizeLimit(facilityName);
758+
if (countLimit != null || sizeLimit != null) {
759+
List<Long> investigationIds = new ArrayList<>();
760+
List<Long> datasetIds = new ArrayList<>();
761+
List<Long> datafileIds = new ArrayList<>();
762+
for (CartItem cartItem : cart.getCartItems()) {
763+
switch (cartItem.getEntityType()) {
764+
case investigation:
765+
investigationIds.add(cartItem.getEntityId());
766+
continue;
767+
case dataset:
768+
datasetIds.add(cartItem.getEntityId());
769+
continue;
770+
case datafile:
771+
datafileIds.add(cartItem.getEntityId());
772+
continue;
773+
default:
774+
throw new InternalException("Unrecognised entityType: " + cartItem.getEntityType());
775+
}
776+
}
777+
IcatClient.EntityCounter entityCounter = icatClient.new EntityCounter(investigationIds, datasetIds, datafileIds);
778+
if (countLimit != null && entityCounter.totalCount > countLimit) {
779+
throw new BadRequestException("Unable to submit for cart for download, number of files exceeds limit");
780+
}
781+
if (sizeLimit != null && entityCounter.totalSize > sizeLimit) {
782+
throw new BadRequestException("Unable to submit for cart for download, size of files exceeds limit");
783+
}
784+
}
753785
Download download = createDownload(sessionId, cart.getFacilityName(), fileName, cart.getUserName(),
754786
fullName, transport, email);
755787
List<DownloadItem> downloadItems = new ArrayList<DownloadItem>();

0 commit comments

Comments
 (0)