Merge branch 'develop' into 11323-includeDeaccesioned-compare-ds-versions

sekmiller · sekmiller · commit 0d324a9015a9 · 2025-03-25T13:18:21.000-04:00
diff --git a/doc/release-notes/11280-search-api-count-per-object.md b/doc/release-notes/11280-search-api-count-per-object.md
@@ -0,0 +1,7 @@
+## Bug fix to Search API. Now includes all type totals (Dataverses, Dataset, and Files) regardless of the list of types requested
+
+None requested types were returned with total count set to 0.
+&type=dataverse&type=dataset would result in "Files" : 0 since type=file was not requested
+
+Now all counts show the correct totals.
+Note: This is only true for the first page requested. Subsequent pages could have 0 counts and should not be used. This is due to the need for speed. Getting the totals is an additional search call in the background.
diff --git a/doc/release-notes/11299-file-restrict-api-extension.md b/doc/release-notes/11299-file-restrict-api-extension.md
@@ -0,0 +1,8 @@
+### Extend Restrict API to include new attributes
+
+Original /restrict API only allowed for a boolean to update the restricted attribute of a file.
+The extended API still allows for the single boolean for backward compatibility.
+This change also allows for a JSON object to be passed which allows for the required `restrict` flag as well as optional attributes: `enableAccessRequest` and `termsOfAccess`.
+If `enableAccessRequest` is false then the `termsOfAccess` text must also be included.
+
+See [the guides](https://dataverse-guide--11349.org.readthedocs.build/en/11349/api/native-api.html#restrict-files), #11299, and #11349.
diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst
@@ -4039,6 +4039,8 @@ Restrict Files
 ~~~~~~~~~~~~~~
 
 Restrict or unrestrict an existing file where ``id`` is the database id of the file or ``pid`` is the persistent id (DOI or Handle) of the file to restrict. Note that some Dataverse installations do not allow the ability to restrict files (see :ref:`:PublicInstall`).
+Restricting or Unrestricting a file, not in a draft version of the Dataset, will result in a new Draft version being created.
+Optionally the API can receive a JSON string with additional parameters related to the ability to request access to the file and the terms of that access.
 
 A curl example using an ``id``
 
@@ -4072,6 +4074,33 @@ The fully expanded example above (without environment variables) looks like this
 
   curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X PUT -d true "https://demo.dataverse.org/api/files/:persistentId/restrict?persistentId=doi:10.5072/FK2/AAA000"
 
+Optional JSON string with additional attributes:
+
+.. code-block:: bash
+
+  export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+  export SERVER_URL=https://demo.dataverse.org
+  export ID=24
+
+  curl -H "X-Dataverse-key:$API_TOKEN" -X PUT "$SERVER_URL/api/files/$ID/restrict" \
+  -H "Content-Type: application/json" \
+  -d '{"restrict": true, "enableAccessRequest":false, "termsOfAccess": "Reason for the restricted access"}'
+
+Note the behavior of the optional parameters:
+
+- If restrict is false then enableAccessRequest and termsOfAccess are ignored
+- If restrict is true and enableAccessRequest is false then termsOfAccess is required. A status of CONFLICT (409) will be returned if the termsOfAccess is missing
+
+The enableAccessRequest and termsOfAccess are applied to the Draft version of the Dataset and affect all of the restricted files in said Draft version.
+
+The fully expanded example above (without environment variables) looks like this:
+
+.. code-block:: bash
+
+  curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X PUT "https://demo.dataverse.org/api/files/:persistentId/restrict?persistentId=doi:10.5072/FK2/AAA000" \
+  -H "Content-Type: application/json" \
+  -d '{"restrict": true, "enableAccessRequest":false, "termsOfAccess": "Reason for the restricted access"}'
+
 .. _file-uningest:
 
 Uningest a File
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Files.java b/src/main/java/edu/harvard/iq/dataverse/api/Files.java
@@ -141,13 +141,43 @@ public Response restrictFileInDataset(@Context ContainerRequestContext crc, @Pat
             return error(BAD_REQUEST, "Could not find datafile with id " + fileToRestrictId);
         }
 
-        boolean restrict = Boolean.valueOf(restrictStr);
+        Boolean restrict = null;
+        Boolean enableAccessRequest = null;
+        String termsOfAccess = null;
+        String returnMessage = " ";
+        // Backward comparability - allow true/false in string(old) or json(new)
+        if (restrictStr != null && restrictStr.trim().startsWith("{")) {
+            // process as json
+            jakarta.json.JsonObject jsonObject;
+            try (StringReader stringReader = new StringReader(restrictStr)) {
+                jsonObject = Json.createReader(stringReader).readObject();
+                if (jsonObject.containsKey("restrict")) {
+                    restrict = Boolean.valueOf(jsonObject.getBoolean("restrict"));
+                    returnMessage += restrict ? "restricted." : "unrestricted.";
+                } else {
+                    return badRequest("Error parsing Json: 'restrict' is required.");
+                }
+                if (jsonObject.containsKey("enableAccessRequest")) {
+                    enableAccessRequest = Boolean.valueOf(jsonObject.getBoolean("enableAccessRequest"));
+                    returnMessage += " Access Request is " + (enableAccessRequest ? "enabled." : "disabled.");
+                }
+                if (jsonObject.containsKey("termsOfAccess")) {
+                    termsOfAccess = jsonObject.getString("termsOfAccess");
+                    returnMessage += " Terms of Access for restricted files: " + termsOfAccess;
+                }
+            } catch (JsonParsingException jpe) {
+                return badRequest("Error parsing Json: " + jpe.getMessage());
+            }
+        } else {
+            restrict = Boolean.valueOf(restrictStr);
+            returnMessage += restrict ? "restricted." : "unrestricted.";
+        }
 
         dataverseRequest = createDataverseRequest(getRequestUser(crc));
 
         // try to restrict the datafile
         try {
-            engineSvc.submit(new RestrictFileCommand(dataFile, dataverseRequest, restrict));
+            engineSvc.submit(new RestrictFileCommand(dataFile, dataverseRequest, restrict, enableAccessRequest, termsOfAccess));
         } catch (CommandException ex) {
             return error(BAD_REQUEST, "Problem trying to update restriction status on " + dataFile.getDisplayName() + ": " + ex.getLocalizedMessage());
         }
@@ -165,8 +195,7 @@ public Response restrictFileInDataset(@Context ContainerRequestContext crc, @Pat
             return error(BAD_REQUEST, "Problem saving datafile " + dataFile.getDisplayName() + ": " + ex.getLocalizedMessage());
         }
 
-        String text =  restrict ? "restricted." : "unrestricted.";
-        return ok("File " + dataFile.getDisplayName() + " " + text);
+        return ok("File " + dataFile.getDisplayName() + returnMessage);
     }
         
     
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Search.java b/src/main/java/edu/harvard/iq/dataverse/api/Search.java
@@ -2,6 +2,7 @@
 
 import edu.harvard.iq.dataverse.*;
 import edu.harvard.iq.dataverse.api.auth.AuthRequired;
+import edu.harvard.iq.dataverse.engine.command.DataverseRequest;
 import edu.harvard.iq.dataverse.search.SearchFields;
 import edu.harvard.iq.dataverse.search.FacetCategory;
 import edu.harvard.iq.dataverse.search.FacetLabel;
@@ -17,10 +18,7 @@
 import edu.harvard.iq.dataverse.search.SortBy;
 import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
 import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
 import java.util.logging.Logger;
 import jakarta.ejb.EJB;
 import jakarta.inject.Inject;
@@ -92,9 +90,50 @@ public Response search(
             String geoPoint;
             String geoRadius;
             List<Dataverse> dataverseSubtrees = new ArrayList<>();
+            DataverseRequest requestUser = createDataverseRequest(user);
+            String allTypes = ":(" + SearchConstants.DATAVERSES + " OR " + SearchConstants.DATASETS + " OR " + SearchConstants.FILES + ")";
+            Map<String, Long> objectTypeCountsMap = new HashMap<>(3);
+            objectTypeCountsMap.put(SearchConstants.UI_DATAVERSES, 0L);
+            objectTypeCountsMap.put(SearchConstants.UI_DATASETS, 0L);
+            objectTypeCountsMap.put(SearchConstants.UI_FILES, 0L);
+
+            // users can't change these (yet anyway)
+            boolean dataRelatedToMe = showMyData; //getDataRelatedToMe();
 
             try {
+                // we have to add "" (root) otherwise there is no permissions check
+                if (subtrees.isEmpty()) {
+                    dataverseSubtrees.add(getSubtree(""));
+                }
+                else {
+                    for (String subtree : subtrees) {
+                        dataverseSubtrees.add(getSubtree(subtree));
+                    }
+                }
+                filterQueries.add(getFilterQueryFromSubtrees(dataverseSubtrees));
+
                 if (!types.isEmpty()) {
+                    // Query to get the totals if needed.
+                    // Only needed if the list of types doesn't include all types since missing types will default to count of 0
+                    // Only get the totals for the first page (paginationStart == 0) for speed
+                    if (showTypeCounts && types.size() < objectTypeCountsMap.size() && paginationStart == 0) {
+                        List<String> totalFilterQueries = new ArrayList<>();
+                        totalFilterQueries.addAll(filterQueries);
+                        totalFilterQueries.add(SearchFields.TYPE + allTypes);
+                        try {
+                            SolrQueryResponse resp = searchService.search(requestUser, dataverseSubtrees, query, totalFilterQueries, null, null, 0,
+                                    dataRelatedToMe, 1, false, null, null, false, false);
+                            if (resp != null) {
+                                for (FacetCategory facetCategory : resp.getTypeFacetCategories()) {
+                                    for (FacetLabel facetLabel : facetCategory.getFacetLabel()) {
+                                        objectTypeCountsMap.put(facetLabel.getName(), facetLabel.getCount());
+                                    }
+                                }
+                            }
+                        } catch(Exception e) {
+                            logger.info("Search getting total counts: " + e.getMessage());
+                        }
+                    }
                     filterQueries.add(getFilterQueryFromTypes(types));
                 } else {
                     /**
@@ -103,22 +142,11 @@ public Response search(
                      * SearchServiceBean tries to get SearchFields.TYPE. The GUI
                      * always seems to add SearchFields.TYPE, even for superusers.
                      */
-                    filterQueries.add(SearchFields.TYPE + ":(" + SearchConstants.DATAVERSES + " OR " + SearchConstants.DATASETS + " OR " + SearchConstants.FILES + ")");
+                    filterQueries.add(SearchFields.TYPE + allTypes);
                 }
                 sortBy = SearchUtil.getSortBy(sortField, sortOrder);
                 numResultsPerPage = getNumberOfResultsPerPage(numResultsPerPageRequested);
                 
-                 // we have to add "" (root) otherwise there is no permissions check
-                if(subtrees.isEmpty()) {
-                    dataverseSubtrees.add(getSubtree(""));
-                }
-                else {
-                    for(String subtree : subtrees) {
-                        dataverseSubtrees.add(getSubtree(subtree));
-                    }
-                }
-                filterQueries.add(getFilterQueryFromSubtrees(dataverseSubtrees));
-                
                 if(filterQueries.isEmpty()) { //Extra sanity check just in case someone else touches this
                     throw new IOException("Filter is empty, which should never happen, as this allows unfettered searching of our index");
                 }
@@ -137,13 +165,10 @@ public Response search(
             } catch (Exception ex) {
                 return error(Response.Status.BAD_REQUEST, ex.getLocalizedMessage());
             }
-
-            // users can't change these (yet anyway)
-            boolean dataRelatedToMe = showMyData; //getDataRelatedToMe();
             
             SolrQueryResponse solrQueryResponse;
             try {
-                solrQueryResponse = searchService.search(createDataverseRequest(user),
+                solrQueryResponse = searchService.search(requestUser,
                         dataverseSubtrees,
                         query,
                         filterQueries,
@@ -211,50 +236,17 @@ public Response search(
             }
 
             value.add("count_in_response", solrSearchResults.size());
-            
-            // we want to show the missing dvobject types with count = 0
-            // per https://github.com/IQSS/dataverse/issues/11127
 
             if (showTypeCounts) {
-                JsonObjectBuilder objectTypeCounts = Json.createObjectBuilder();
-                if (!solrQueryResponse.getTypeFacetCategories().isEmpty()) {
-                    boolean filesMissing = true;
-                    boolean datasetsMissing = true;
-                    boolean dataversesMissing = true;
-                    for (FacetCategory facetCategory : solrQueryResponse.getTypeFacetCategories()) {
-                        for (FacetLabel facetLabel : facetCategory.getFacetLabel()) {
-                            objectTypeCounts.add(facetLabel.getName(), facetLabel.getCount());
-                            if (facetLabel.getName().equals((SearchConstants.UI_DATAVERSES))) {
-                                dataversesMissing = false;
-                            }
-                            if (facetLabel.getName().equals((SearchConstants.UI_DATASETS))) {
-                                datasetsMissing = false;
-                            }
-                            if (facetLabel.getName().equals((SearchConstants.UI_FILES))) {
-                                filesMissing = false;
-                            }
-                        }
-                    }
-
-                    if (solrQueryResponse.getTypeFacetCategories().size() < 3) {
-                        if (dataversesMissing) {
-                            objectTypeCounts.add(SearchConstants.UI_DATAVERSES, 0);
-                        }
-                        if (datasetsMissing) {
-                            objectTypeCounts.add(SearchConstants.UI_DATASETS, 0);
-                        }
-                        if (filesMissing) {
-                            objectTypeCounts.add(SearchConstants.UI_FILES, 0);
+                for (FacetCategory facetCategory : solrQueryResponse.getTypeFacetCategories()) {
+                    for (FacetLabel facetLabel : facetCategory.getFacetLabel()) {
+                        if (facetLabel.getCount() > 0) {
+                            objectTypeCountsMap.put(facetLabel.getName(), facetLabel.getCount());
                         }
                     }
-
-                }
-                if (showTypeCounts && solrQueryResponse.getTypeFacetCategories().isEmpty()) {
-                    objectTypeCounts.add(SearchConstants.UI_DATAVERSES, 0);
-                    objectTypeCounts.add(SearchConstants.UI_DATASETS, 0);
-                    objectTypeCounts.add(SearchConstants.UI_FILES, 0);
                 }
-
+                JsonObjectBuilder objectTypeCounts = Json.createObjectBuilder();
+                objectTypeCountsMap.forEach((k,v) -> objectTypeCounts.add(k,v));
                 value.add("total_count_per_object_type", objectTypeCounts);
             }
             /**
diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RestrictFileCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RestrictFileCommand.java
@@ -5,10 +5,7 @@
  */
 package edu.harvard.iq.dataverse.engine.command.impl;
 
-import edu.harvard.iq.dataverse.DataFile;
-import edu.harvard.iq.dataverse.Dataset;
-import edu.harvard.iq.dataverse.DatasetVersion;
-import edu.harvard.iq.dataverse.FileMetadata;
+import edu.harvard.iq.dataverse.*;
 import edu.harvard.iq.dataverse.authorization.Permission;
 import edu.harvard.iq.dataverse.engine.command.AbstractVoidCommand;
 import edu.harvard.iq.dataverse.engine.command.CommandContext;
@@ -34,13 +31,25 @@ public class RestrictFileCommand extends AbstractVoidCommand {
 
     private final DataFile file;
     private final boolean restrict;
-    
+    private final Boolean fileAccessRequest;
+    private final String termsOfAccess;
+
+    public RestrictFileCommand(DataFile file, DataverseRequest aRequest, boolean restrict, Boolean fileAccessRequest, String termsOfAccess) {
+        super(aRequest, file.getOwner());
+        this.file = file;
+        this.restrict = restrict;
+        this.fileAccessRequest = fileAccessRequest;
+        this.termsOfAccess = termsOfAccess;
+    }
+
     public RestrictFileCommand(DataFile file, DataverseRequest aRequest, boolean restrict) {
         super(aRequest, file.getOwner());
         this.file = file;
         this.restrict = restrict;
-    }    
-       
+        this.fileAccessRequest = null;
+        this.termsOfAccess = null;
+    }
+
     @Override
     protected void executeImpl(CommandContext ctxt) throws CommandException {
         // check if public install & don't allow
@@ -64,6 +73,13 @@ protected void executeImpl(CommandContext ctxt) throws CommandException {
         else {
             Dataset dataset = file.getOwner();
             DatasetVersion workingVersion = dataset.getOrCreateEditVersion();
+            if (restrict && fileAccessRequest != null) {
+                if (workingVersion.getTermsOfUseAndAccess() == null) {
+                    workingVersion.setTermsOfUseAndAccess(new TermsOfUseAndAccess());
+                }
+                workingVersion.getTermsOfUseAndAccess().setFileAccessRequest(fileAccessRequest);
+                workingVersion.getTermsOfUseAndAccess().setTermsOfAccess(termsOfAccess);
+            }
             // We need the FileMetadata for the file in the draft dataset version and the
             // file we have may still reference the fmd from the prior released version
             FileMetadata draftFmd = file.getFileMetadata();
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java
@@ -1027,14 +1027,37 @@ public void testRestrictFile() {
                 .body("message", equalTo("Problem trying to update restriction status on dataverseproject.png: File dataverseproject.png is already restricted"))
                 .statusCode(BAD_REQUEST.getStatusCode());
 
-        //unrestrict file
-        restrict = false;
-        Response unrestrictResponse = UtilIT.restrictFile(origFileId.toString(), restrict, apiToken);
+        //unrestrict file using json with missing "restrict"
+        String restrictJson = "{}";
+        Response unrestrictResponse = UtilIT.restrictFile(origFileId.toString(), restrictJson, apiToken);
+        unrestrictResponse.prettyPrint();
+        unrestrictResponse.then().assertThat()
+                .body("message", equalTo("Error parsing Json: 'restrict' is required."))
+                .statusCode(BAD_REQUEST.getStatusCode());
+
+        //unrestrict file using json
+        restrictJson = "{\"restrict\":false}";
+        unrestrictResponse = UtilIT.restrictFile(origFileId.toString(), restrictJson, apiToken);
         unrestrictResponse.prettyPrint();
         unrestrictResponse.then().assertThat()
                 .body("data.message", equalTo("File dataverseproject.png unrestricted."))
                 .statusCode(OK.getStatusCode());
 
+        //restrict file using json with enableAccessRequest false and missing TOA
+        restrictJson = "{\"restrict\":true, \"enableAccessRequest\":false}";
+        restrictResponse = UtilIT.restrictFile(origFileId.toString(), restrictJson, apiToken);
+        restrictResponse.prettyPrint();
+        restrictResponse.then().assertThat()
+                .body("message", equalTo(BundleUtil.getStringFromBundle("dataset.message.toua.invalid")))
+                .statusCode(CONFLICT.getStatusCode());
+        //restrict file using json
+        restrictJson = "{\"restrict\":true, \"enableAccessRequest\":false, \"termsOfAccess\":\"Testing terms of access\"}";
+        restrictResponse = UtilIT.restrictFile(origFileId.toString(), restrictJson, apiToken);
+        restrictResponse.prettyPrint();
+        restrictResponse.then().assertThat()
+                .body("data.message", equalTo("File dataverseproject.png restricted. Access Request is disabled. Terms of Access for restricted files: Testing terms of access"))
+                .statusCode(OK.getStatusCode());
+
         //reset public install
         UtilIT.setSetting(SettingsServiceBean.Key.PublicInstall, publicInstall);
 
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java b/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
