Merge pull request #11125 from IQSS/11057-globus-downloads

Globus support: download optimizations

Author: ofahimIQSS

doc/release-notes/11057-globus-improvements.md

@@ -0,0 +1,11 @@
+## Globus framework improvements
+
+The improvements and optimizations in this release build on top of the earlier work (such as PR #10781). They are based on the experience gained at IQSS as part of the production rollout of the Large Data Storage services that utilizes Globus.
+
+The changes in this PR (#11125) focus on improving Globus *downloads* (i.e., transfers from Dataverse-linked Globus volumes to users' Globus collections). Most importantly, th mechanism of "Asynchronous Task Monitoring", first introduced in #10781 for *uploads*, has been extended to handle downloads as well. This generally makes downloads more reliable (specifically, in how Dataverse manages temporary access rules granted to users, minimizing the risk of consequent downloads failing because of stale access rules left in place).
+
+See `globus-use-experimental-async-framework` under [Feature Flags](https://guides.dataverse.org/en/latest/installation/config.html#feature-flags) and [dataverse.files.globus-monitoring-server](https://guides.dataverse.org/en/latest/installation/config.html#dataverse-files-globus-monitoring-server) in the Installation Guide.
+
+Multiple other improvements have been made making the underlying Globus framework more reliable and robust.
+
+

doc/sphinx-guides/source/installation/config.rst

@@ -3501,7 +3501,7 @@ please find all known feature flags below. Any of these flags can be activated u
       - Turns off automatic selection of a dataset thumbnail from image files in that dataset. When set to ``On``, a user can still manually pick a thumbnail image or upload a dedicated thumbnail image.
       - ``Off``
     * - globus-use-experimental-async-framework
-      - Activates a new experimental implementation of Globus polling of ongoing remote data transfers that does not rely on the instance staying up continuously for the duration of the transfers and saves the state information about Globus upload requests in the database. Added in v6.4. Affects :ref:`:GlobusPollingInterval`. Note that the JVM option :ref:`dataverse.files.globus-monitoring-server` described above must also be enabled on one (and only one, in a multi-node installation) Dataverse instance. 
+      - Activates a new experimental implementation of Globus polling of ongoing remote data transfers that does not rely on the instance staying up continuously for the duration of the transfers and saves the state information about Globus upload requests in the database. Added in v6.4; extended in v6.6 to cover download transfers, in addition to uploads. Affects :ref:`:GlobusPollingInterval`. Note that the JVM option :ref:`dataverse.files.globus-monitoring-server` described above must also be enabled on one (and only one, in a multi-node installation) Dataverse instance. 
       - ``Off``
     * - index-harvested-metadata-source
       - Index the nickname or the source name (See the optional ``sourceName`` field in :ref:`create-a-harvesting-client`) of the harvesting client as the "metadata source" of harvested datasets and files. If enabled, the Metadata Source facet will show separate groupings of the content harvested from different sources (by harvesting client nickname or source name) instead of the default behavior where there is one "Harvested" grouping for all harvested content.

src/main/java/edu/harvard/iq/dataverse/DatasetPage.java

@@ -5867,13 +5867,12 @@ public List<DatasetField> getDatasetSummaryFields() {
         return DatasetUtil.getDatasetSummaryFields(workingVersion, customFields);
     }
 
-    public boolean isShowPreviewButton(Long fileId) {
-        List<ExternalTool> previewTools = getPreviewToolsForDataFile(fileId);
+    public boolean isShowPreviewButton(DataFile dataFile) {
+        List<ExternalTool> previewTools = getPreviewToolsForDataFile(dataFile);
         return previewTools.size() > 0;
     }
 
-    public boolean isShowQueryButton(Long fileId) { 
-        DataFile dataFile = datafileService.find(fileId);
+    public boolean isShowQueryButton(DataFile dataFile) { 
 
         if(dataFile.isRestricted()
                 || !dataFile.isReleased()
@@ -5882,26 +5881,28 @@ public boolean isShowQueryButton(Long fileId) {
             return false;
         }
 
-        List<ExternalTool> fileQueryTools = getQueryToolsForDataFile(fileId);
+        List<ExternalTool> fileQueryTools = getQueryToolsForDataFile(dataFile);
         return fileQueryTools.size() > 0;
     }
 
-    public List<ExternalTool> getPreviewToolsForDataFile(Long fileId) {
-        return getCachedToolsForDataFile(fileId, ExternalTool.Type.PREVIEW);
+    public List<ExternalTool> getPreviewToolsForDataFile(DataFile dataFile) {
+        return getCachedToolsForDataFile(dataFile, ExternalTool.Type.PREVIEW);
     }
 
-    public List<ExternalTool> getQueryToolsForDataFile(Long fileId) {
-        return getCachedToolsForDataFile(fileId, ExternalTool.Type.QUERY);
+    public List<ExternalTool> getQueryToolsForDataFile(DataFile dataFile) {
+        return getCachedToolsForDataFile(dataFile, ExternalTool.Type.QUERY);
     }
-    public List<ExternalTool> getConfigureToolsForDataFile(Long fileId) {
-        return getCachedToolsForDataFile(fileId, ExternalTool.Type.CONFIGURE);
+    
+    public List<ExternalTool> getConfigureToolsForDataFile(DataFile dataFile) {
+        return getCachedToolsForDataFile(dataFile, ExternalTool.Type.CONFIGURE);
     }
 
-    public List<ExternalTool> getExploreToolsForDataFile(Long fileId) {
-        return getCachedToolsForDataFile(fileId, ExternalTool.Type.EXPLORE);
+    public List<ExternalTool> getExploreToolsForDataFile(DataFile dataFile) {
+        return getCachedToolsForDataFile(dataFile, ExternalTool.Type.EXPLORE);
     }
 
-    public List<ExternalTool> getCachedToolsForDataFile(Long fileId, ExternalTool.Type type) {
+    public List<ExternalTool> getCachedToolsForDataFile(DataFile dataFile, ExternalTool.Type type) {
+        Long fileId = dataFile.getId();
         Map<Long, List<ExternalTool>> cachedToolsByFileId = new HashMap<>();
         List<ExternalTool> externalTools = new ArrayList<>();
         switch (type) {
@@ -5928,7 +5929,6 @@ public List<ExternalTool> getCachedToolsForDataFile(Long fileId, ExternalTool.Ty
         if (cachedTools != null) { //if already queried before and added to list
             return cachedTools;
         }
-        DataFile dataFile = datafileService.find(fileId);
         cachedTools = externalToolService.findExternalToolsByFile(externalTools, dataFile);
         cachedToolsByFileId.put(fileId, cachedTools); //add to map so we don't have to do the lifting again
         return cachedTools;
@@ -6728,6 +6728,7 @@ public boolean isGlobusTransferRequested() {
      *                    valid files to transfer.
      */
     public void startGlobusTransfer(boolean transferAll, boolean popupShown) {
+        logger.fine("inside startGlobusTransfer; "+(transferAll ? "transferAll" : "NOTtransferAll") + " " + (popupShown ? "popupShown" : "NOTpopupShown"));
         if (transferAll) {
             this.setSelectedFiles(workingVersion.getFileMetadatas());
         }

src/main/java/edu/harvard/iq/dataverse/FileDownloadHelper.java

@@ -72,13 +72,23 @@ public FileDownloadHelper() {
     // file downloads and multiple (batch) downloads - since both use the same
     // terms/etc. popup.
     public void writeGuestbookAndStartDownload(GuestbookResponse guestbookResponse, boolean isGlobusTransfer) {
+        logger.fine("inside FileDownloadHelper.writeGuestbookAndStartDownload() " + (isGlobusTransfer ? "Globus Transfer" : "NOT a Globus Transfer"));
         PrimeFaces.current().executeScript("PF('guestbookAndTermsPopup').hide()");
         guestbookResponse.setEventType(GuestbookResponse.DOWNLOAD);
          // Note that this method is only ever called from the file-download-popup -
          // meaning we know for the fact that we DO want to save this
          // guestbookResponse permanently in the database.
-        if(isGlobusTransfer) {
-            globusService.writeGuestbookAndStartTransfer(guestbookResponse, true);
+         // Do keep in mind that "true" in writeGuestbookAndStartTransfer() below
+         // would mean "DO SKIP writing the guestbookResponse", and "false" means 
+         // "DO write ..."
+         if(isGlobusTransfer) {
+            // Note that *single-file* Globus transfers are NOT handled here. 
+            // Instead they are coming in through this method with isGlobusTransfer=false,
+            // and then picked up by the fileDownloadService, below, which in turn 
+            // recognizes them as Globus types via guestbookResponse.getFileFormat() == "GlobusTransfer"
+            // and treats them as such... I'm not super clear as to why they can't 
+            // be handled here instead... Don't ask. 
+            globusService.writeGuestbookAndStartTransfer(guestbookResponse, false);
         } else {
             if (guestbookResponse.getSelectedFileIds() != null) {
                 // this is a batch (multiple file) download.

src/main/java/edu/harvard/iq/dataverse/api/Datasets.java

@@ -4123,7 +4123,7 @@ public Response requestGlobusUpload(@Context ContainerRequestContext crc, @PathP
                     case 400:
                         return badRequest("Unable to grant permission");
                     case 409:
-                        return conflict("Permission already exists");
+                        return conflict("Permission already exists or no more permissions allowed");
                     default:
                         return error(null, "Unexpected error when granting permission");
                     }
@@ -4494,7 +4494,7 @@ public Response requestGlobusDownload(@Context ContainerRequestContext crc, @Pat
             case 400:
                 return badRequest("Unable to grant permission");
             case 409:
-                return conflict("Permission already exists");
+                return conflict("Permission already exists or no more permissions allowed");
             default:
                 return error(null, "Unexpected error when granting permission");
             }
@@ -4548,8 +4548,17 @@ public Response monitorGlobusDownload(@Context ContainerRequestContext crc, @Pat
             return wr.getResponse();
         }
 
+        JsonObject jsonObject = null;
+        try {
+            jsonObject = JsonUtil.getJsonObject(jsonData);
+        } catch (Exception ex) {
+            logger.warning("Globus download monitoring: error parsing json: " + jsonData + " " + ex.getMessage());
+            return badRequest("Error parsing json body");
+
+        }
+        
         // Async Call
-        globusService.globusDownload(jsonData, dataset, authUser);
+        globusService.globusDownload(jsonObject, dataset, authUser);
 
         return ok("Async call to Globus Download started");
 

src/main/java/edu/harvard/iq/dataverse/globus/ExpiredTokenException.java

@@ -0,0 +1,16 @@
+package edu.harvard.iq.dataverse.globus;
+
+/**
+ *
+ * @author landreev
+ */
+public class ExpiredTokenException extends Exception {
+    public ExpiredTokenException(String message) {
+        super(message);
+    }
+    
+    public ExpiredTokenException(String message, Throwable cause) {
+        super(message, cause);
+    }
+    
+}