Merge branch 'master' into submit_collection_endpoint

Author: patrick-austin

src/main/config/run.properties.example

@@ -22,8 +22,19 @@ facility.YFH.idsUrl = https://your.facility.idsUrl.here
 # If the property is not defined, Topcat will use the facility's idsUrl by default.
 
 facility.LILS.downloadType.http = http://localhost:8080
+facility.LILS.downloadType.list = http
+facility.LILS.downloadType.http.displayName = HTTP
+facility.LILS.downloadType.http.description = Example description for HTTP access method.
+
+facility.YFH.downloadType.list = https globus
 facility.YFH.downloadType.https = https://your.facility.downloadIdsUrl.here
+facility.YFH.downloadType.https.displayName = HTTPS
+facility.YFH.downloadType.https.description = Example description for HTTPS access method.
+facility.YFH.downloadType.https.disallowedAuthn = simple db
 facility.YFH.downloadType.globus = https://your.facility.globusUrl.here
+facility.YFH.downloadType.globus.displayName = Globus
+facility.YFH.downloadType.globus.description = Example description for Globus access method.
+facility.YFH.downloadType.globus.allowedGroupings = principal_beamline_scientists admins
 
 # enable send email
 mail.enable=true
@@ -36,6 +47,7 @@ mail.enable=true
 # ${fileName} - the download name
 # ${size} - the download size
 # ${downloadUrl} - the download url
+# ${customValue} - optionally set by calls to setDownloadStatus where the status is complete
 mail.subject=TopCAT Download ${fileName} Ready
 
 # The email body message for https downloads. All subject tokens as above are available.
@@ -50,6 +62,10 @@ mail.body.smartclient=Hi ${userName}, \n\nYour ${size} SmartClient download ${fi
 # The email body message for SCARF downloads. All subject tokens as above are available.
 mail.body.scarf=Hi ${userName}, \n\nYour ${size} SCARF download ${fileName} is ready. Please see https:/example.com/#/scarf-faq for more information on how to download using SCARF.\n\nThank you for using TopCAT
 
+# The email body message for Ada downloads. All subject tokens as above are available.
+mail.body.ada = Hi ${userName}, \n\nYour ${size} recall ${fileName} is ready as an Ada data share. Please visit https:/example.com to redeem access using the following token.\n\n${customValue}\n\nThank you for using DataGateway
+mail.required.ada = true
+
 # The maximum number of datafiles for a getStatus call to the IDS for two level storage
 ids.getStatus.max=100
 
@@ -131,8 +147,8 @@ queue.priority.instrumentScientist.instruments = {"ABC": 1}
 queue.priority.instrumentScientist.default = 2
 queue.priority.investigationUser.roles = {"ABC": 3}
 queue.priority.investigationUser.default = 4
-queue.priority.authenticated = 5
-queue.priority.default = 0
+queue.priority.authenticated = {"orcid": 6, "anon": 0}
+queue.priority.default = 5
 
 # Configurable limit for the length of the GET URL for requesting Datafiles by a list of file locations
 # The exact limit may depend on the server

src/main/java/org/icatproject/topcat/DownloadBuilder.java

@@ -45,10 +45,10 @@ public DownloadBuilder(String sessionId, String email, String fileName, String t
 		queueFilesMaxFileCount = Long.valueOf(properties.getProperty("queue.files.maxFileCount", "10000"));
 
         this.sessionId = sessionId;
-        this.email = email;
         this.fileName = fileName;
 		this.transport = validateTransport(transport);
 		this.facilityName = validateFacilityName(facilityName);
+		this.email = validateEmail(transport, email);
 		String icatUrl = getIcatUrl(this.facilityName);
 
 		icatClient = new IcatClient(icatUrl, sessionId);
@@ -88,6 +88,27 @@ public static String validateFacilityName(String facilityName) throws BadRequest
 		}
 	}
 
+	/**
+	 * Validate that the submitted email is not null or empty if mail.required is true.
+	 * 
+	 * @param transport Transport mechanism to use (which may require email)
+	 * @param email Users email address, which may be null or empty
+	 * @return The original email, or null if it was an empty string
+	 * @throws BadRequestException if email null or empty and mail.required is true
+	 */
+	public static String validateEmail(String transport, String email) throws BadRequestException {
+		if(email != null && email.equals("")){
+			email = null;
+		}
+
+		String emailRequired = Properties.getInstance().getProperty("mail.required." + transport, "false");
+		if (Boolean.parseBoolean(emailRequired) && email == null) {
+			throw new BadRequestException("email is required for " + transport);
+		}
+
+		return email;
+	}
+
 	/**
 	 * @param facilityName ICAT Facility.name
 	 * @return ICAT server url

src/main/java/org/icatproject/topcat/FacilityMap.java

@@ -2,6 +2,7 @@
 
 import java.util.HashMap;
 import java.util.Map;
+import java.util.Set;
 
 import org.icatproject.topcat.exceptions.InternalException;
 import org.slf4j.Logger;
@@ -83,7 +84,14 @@ public String validateFacilityName(String facility) throws InternalException {
 		}
 		return facility;
 	}
-	
+
+	/**
+	 * @return All the ICAT Facility.names from the config file
+	 */
+	public Set<String> getFacilities() {
+		return facilityIcatUrl.keySet();
+	}
+
 	public String getIcatUrl( String facility ) throws InternalException{
 		facility = validateFacilityName(facility);
 		String url = facilityIcatUrl.get( facility );

src/main/java/org/icatproject/topcat/IcatClient.java

@@ -487,7 +487,7 @@ public int getQueuePriority(String userName) throws TopcatException {
 		if (userPriority != null) {
 			return userPriority;
 		}
-		HashMap<Integer, String> mapping = priorityMap.getMapping();
+		HashMap<Integer, String> mapping = priorityMap.getQueryMapping();
 		List<Integer> keyList = new ArrayList<>(mapping.keySet());
 		Collections.sort(keyList);
 		for (Integer priority : keyList) {
@@ -496,13 +496,7 @@ public int getQueuePriority(String userName) throws TopcatException {
 			}
 		}
 
-		String anonUserName = Properties.getInstance().getProperty("anonUserName");
-		if (anonUserName == null || !userName.startsWith(anonUserName)) {
-			// The anonymous cart username will end with the user's sessionId so cannot do .equals
-			return priorityMap.getAuthenticatedPriority();
-		} else {
-			return priorityMap.getDefaultPriority();
-		}
+		return priorityMap.getAuthenticatedPriority(userName);
 	}
 
 	/**
@@ -519,9 +513,29 @@ int checkUser(String userName, String condition) throws TopcatException {
 		return results.size();
 	}
 
+	/**
+	 * @param userName  ICAT User.name
+	 * @param groupings ICAT Grouping.names
+	 * @return whether userName is in any of the named groupings
+	 * @throws TopcatException if the query fails
+	 */
+	public boolean isInGroups(String userName, Set<String> groupings) throws TopcatException {
+		String query = "SELECT userGroup FROM UserGroup userGroup WHERE userGroup.user.name = :user";
+		query += " AND userGroup.grouping.name IN ('" + String.join("','", groupings) + "')";
+		JsonArray results = submitQuery(query);
+		return results.size() > 0;
+	}
+
 	protected String[] getAdminUserNames() throws Exception {
 		return Properties.getInstance().getProperty("adminUserNames", "").split("([ ]*,[ ]*|[ ]+)");
 	}
 
+	/**
+	 * @param sessionId ICAT sessionId
+	 */
+	public void setSessionId(String sessionId) {
+		this.sessionId = sessionId;
+	}
+
 }
 

src/main/java/org/icatproject/topcat/PriorityMap.java

@@ -26,9 +26,9 @@ public synchronized static PriorityMap getInstance() throws InternalException {
     String anonUserName;
     boolean anonDownloadEnabled;
     private int defaultPriority;
-    private int authenticatedPriority;
+    private HashMap<String, Integer> authenticatedMapping = new HashMap<>();
     private HashMap<String, Integer> userMapping = new HashMap<>();
-    private HashMap<Integer, String> mapping = new HashMap<>();
+    private HashMap<Integer, String> queryMapping = new HashMap<>();
     private Logger logger = LoggerFactory.getLogger(PriorityMap.class);
 
     public PriorityMap() {
@@ -40,32 +40,22 @@ public PriorityMap() {
                     + "authenticated priority will be used as default level");
         }
         anonDownloadEnabled = Boolean.parseBoolean(properties.getProperty("anonDownloadEnabled", "true"));
-        String defaultString;
-        if (anonDownloadEnabled) {
-            defaultString = properties.getProperty("queue.priority.default", "0");
-            defaultPriority = Integer.valueOf(defaultString);
-        } else {
-            defaultString = "0";
-            defaultPriority = 0;
-        }
+        String defaultString = properties.getProperty("queue.priority.default", "0");
+        defaultPriority = Integer.valueOf(defaultString);
+
 
-        String authenticatedString = properties.getProperty("queue.priority.authenticated", defaultString);
-        setAuthenticatedPriority(authenticatedString);
+        String authenticatedString = properties.getProperty("queue.priority.authenticated", "{}");
+        parseObject(authenticatedString, authenticatedMapping);
 
         String userString = properties.getProperty("queue.priority.user", "{}");
-        JsonReader reader = Json.createReader(new ByteArrayInputStream(userString.getBytes()));
-        JsonObject object = reader.readObject();
-        for (String key : object.keySet()) {
-            int priority = object.getInt(key);
-            userMapping.put(key, priority);
-        }
+        parseObject(userString, userMapping);
 
         String property = "queue.priority.investigationUser.default";
-        String investigationUserString = properties.getProperty(property, authenticatedString);
+        String investigationUserString = properties.getProperty(property, defaultString);
         updateMapping(Integer.valueOf(investigationUserString), "user.investigationUsers IS NOT EMPTY");
 
         property = "queue.priority.instrumentScientist.default";
-        String instrumentScientistString = properties.getProperty(property, authenticatedString);
+        String instrumentScientistString = properties.getProperty(property, defaultString);
         updateMapping(Integer.valueOf(instrumentScientistString), "user.instrumentScientists IS NOT EMPTY");
 
         String investigationUserProperty = properties.getProperty("queue.priority.investigationUser.roles");
@@ -93,25 +83,18 @@ public void checkAnonDownloadEnabled(String userName) throws ForbiddenException
     }
 
     /**
-     * Set the minimum priority for all authenticated Users. This cannot be lower
-     * than the defaultPriority, which will be used instead if this is the case.
+     * Extracts a String key to priority level mapping for any criteria (user, authn).
      * 
-     * @param authenticatedString The value read from the run.properties file
+     * @param propertyString String representing a JsonObject from the run.properties
+     *                       file, or {}
+     * @param mapping        HashMap from String key to numeric priority level
      */
-    private void setAuthenticatedPriority(String authenticatedString) {
-        authenticatedPriority = Integer.valueOf(authenticatedString);
-        if (authenticatedPriority < 1 && defaultPriority >= 1) {
-            String msg = "queue.priority.authenticated disabled with value " + authenticatedString;
-            msg += " but queue.priority.default enabled with value " + defaultPriority;
-            msg += "\nAuthenticated users will use default priority if no superseding priority applies";
-            logger.warn(msg);
-            authenticatedPriority = defaultPriority;
-        } else if (authenticatedPriority >= 1 && authenticatedPriority > defaultPriority) {
-            String msg = "queue.priority.authenticated enabled with value " + authenticatedString;
-            msg += " but queue.priority.default supersedes with value " + defaultPriority;
-            msg += "\nAuthenticated users will use default priority if no superseding priority applies";
-            logger.warn(msg);
-            authenticatedPriority = defaultPriority;
+    private void parseObject(String propertyString, HashMap<String, Integer> mapping) {
+        JsonReader reader = Json.createReader(new ByteArrayInputStream(propertyString.getBytes()));
+        JsonObject object = reader.readObject();
+        for (String key : object.keySet()) {
+            int priority = object.getInt(key);
+            mapping.put(key, priority);
         }
     }
 
@@ -147,25 +130,25 @@ private void updateMapping(int priority, String newCondition) {
         if (priority < 1) {
             logger.warn("Non-positive priority found in mapping, ignoring entry");
             return;
-        } else if (authenticatedPriority >= 1 && priority >= authenticatedPriority) {
-            logger.warn("Priority set in mapping would be superseded by queue.priority.authenticated, ignoring entry");
+        } else if (defaultPriority >= 1 && priority >= defaultPriority) {
+            logger.warn("Priority set in mapping would be superseded by queue.priority.default, ignoring entry");
             return;
         }
 
-        String oldCondition = mapping.get(priority);
+        String oldCondition = queryMapping.get(priority);
         if (oldCondition != null) {
-            mapping.put(priority, oldCondition + " OR " + newCondition);
+            queryMapping.put(priority, oldCondition + " OR " + newCondition);
         } else {
-            mapping.put(priority, newCondition);
+            queryMapping.put(priority, newCondition);
         }
     }
 
     /**
      * @return Mapping of priority level to a JPQL condition which defines the Users
      *         who have this priority
      */
-    public HashMap<Integer, String> getMapping() {
-        return mapping;
+    public HashMap<Integer, String> getQueryMapping() {
+        return queryMapping;
     }
 
     /**
@@ -177,14 +160,23 @@ public Integer getUserPriority(String userName) {
     }
 
     /**
-     * @return The priority which applies to all authenticated users
+     * @param userName String in the format prefix/userName
+     * @return Relevant priority if prefix present and recognised,
+     *         otherwise defaultPriority
      */
-    public int getAuthenticatedPriority() {
-        return authenticatedPriority;
+    public Integer getAuthenticatedPriority(String userName) {
+        int index = userName.indexOf("/");
+        if (index < 0) {
+            String format = "No explicit authentication mechanism for {}, using default priority {}";
+            logger.debug(format, userName, defaultPriority);
+            return defaultPriority;
+        }
+        String prefix = userName.substring(0, index);
+        return authenticatedMapping.getOrDefault(prefix, defaultPriority);
     }
 
     /**
-     * @return The priority which applies to all users, included anonymous access
+     * @return The priority which applies to any user without a specific setting.
      */
     public int getDefaultPriority() {
         return defaultPriority;

src/main/java/org/icatproject/topcat/StatusCheck.java

@@ -172,13 +172,15 @@ private void performCheck(Download download, IdsClient injectedIdsClient) {
       if( idsClient == null ) {
     	  idsClient = new IdsClient(getDownloadUrl(download.getFacilityName(),download.getTransport()));
       }
-      if(!download.getIsEmailSent() && download.getStatus() == DownloadStatus.COMPLETE){
+      if (download.getStatus() == DownloadStatus.COMPLETE) {
     	  logger.info("Download COMPLETE for " + download.getFileName() + " " + download.getId() + "; checking whether to send email...");
-        download.setIsEmailSent(true);
-        em.persist(download);
-        em.flush();
+        if (!download.getIsEmailSent()) {
+          sendDownloadReadyEmail(download);
+          download.setIsEmailSent(true);
+          em.persist(download);
+          em.flush();
+        }
         lastChecks.remove(download.getId());
-        sendDownloadReadyEmail(download);
       } else if(download.getTransport().matches("https|http") && idsClient.isPrepared(download.getPreparedId())){
     	  logger.info("Download (http[s]) for " + download.getFileName() + " " + download.getId() + " is Prepared, so setting COMPLETE and checking email...");
         download.setStatus(DownloadStatus.COMPLETE);
@@ -203,7 +205,28 @@ private void performCheck(Download download, IdsClient injectedIdsClient) {
     }
   }
 
-  private void sendDownloadReadyEmail(Download download) throws InternalException{
+  /**
+   * Send an email for a completed Download.
+   * 
+   * @param download Download that has completed
+   * @throws InternalException If download does not have a valid facilityName
+   */
+  private void sendDownloadReadyEmail(Download download) throws InternalException {
+    String downloadUrl = getDownloadUrl(download.getFacilityName(),download.getTransport());
+    downloadUrl += "/ids/getData?preparedId=" + download.getPreparedId();
+    downloadUrl += "&outname=" + download.getFileName();
+    sendDownloadReadyEmail(mailSession, download, downloadUrl, null);
+  }
+
+  /**
+   * Send an email for a completed Download.
+   * 
+   * @param mailSession Jakarta mail session to use
+   * @param download    Download that has completed
+   * @param downloadUrl URL that provides the recipient access to their data
+   * @param customValue Custom value to substitute into the message body, set by Pollcat
+   */
+  public static void sendDownloadReadyEmail(Session mailSession, Download download, String downloadUrl, String customValue) {
     EmailValidator emailValidator = EmailValidator.getInstance();
     Properties properties = Properties.getInstance();
 
@@ -216,10 +239,6 @@ private void sendDownloadReadyEmail(Download download) throws InternalException{
           userName = fullName;
         }
 
-        String downloadUrl = getDownloadUrl(download.getFacilityName(),download.getTransport());
-        downloadUrl += "/ids/getData?preparedId=" + download.getPreparedId();
-        downloadUrl += "&outname=" + download.getFileName();
-
         Map<String, String> valuesMap = new HashMap<String, String>();
         valuesMap.put("email", download.getEmail());
         valuesMap.put("userName", userName);
@@ -228,6 +247,7 @@ private void sendDownloadReadyEmail(Download download) throws InternalException{
         valuesMap.put("downloadUrl", downloadUrl);
         valuesMap.put("fileName", download.getFileName());
         valuesMap.put("size", Utils.bytesToHumanReadable(download.getSize()));
+        valuesMap.put("customValue", customValue);
 
         StringSubstitutor sub = new StringSubstitutor(valuesMap);
         String subject = sub.replace(properties.getProperty("mail.subject", "mail.subject not set in run.properties"));