diff --git a/doc/release-notes/11275-add-limit-to-number-of-dataset-files.md b/doc/release-notes/11275-add-limit-to-number-of-dataset-files.md new file mode 100644 index 00000000000..37ec0d64bed --- /dev/null +++ b/doc/release-notes/11275-add-limit-to-number-of-dataset-files.md @@ -0,0 +1,5 @@ +### Files attached to a Dataset can now be limited by count + +Added the ability to set a limit on the number of files that can be uploaded to a Dataset. Limits can be set globally through a JVM setting or set per Collection or Dataset. + +See also [the guides](https://dataverse-guide--11359.org.readthedocs.build/en/11359/api/native-api.html#imposing-a-limit-to-the-number-of-files-allowed-to-be-uploaded-to-a-dataset), #11275, and #11359. diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 51a6dae2bb4..6b0dec19cf7 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -2474,7 +2474,7 @@ When adding a file to a dataset, you can optionally specify the following: - Whether or not the file is restricted. - Whether or not the file skips :doc:`tabular ingest `. If the ``tabIngest`` parameter is not specified, it defaults to ``true``. -Note that when a Dataverse installation is configured to use S3 storage with direct upload enabled, there is API support to send a file directly to S3. This is more complex and is described in the :doc:`/developers/s3-direct-upload-api` guide. +Note that when a Dataverse installation is configured to use S3 storage with direct upload enabled, there is API support to send a file directly to S3. This is more complex and is described in the :doc:`/developers/s3-direct-upload-api` guide. Also, see :ref:`set-dataset-file-limit-api`, for limitations to the number of files allowed per Dataset. In the curl example below, all of the above are specified but they are optional. @@ -2699,6 +2699,58 @@ In some circumstances, it may be useful to move or copy files into Dataverse's s Two API calls are available for this use case to add files to a dataset or to replace files that were already in the dataset. These calls were developed as part of Dataverse's direct upload mechanism and are detailed in :doc:`/developers/s3-direct-upload-api`. +Imposing a limit to the number of files allowed to be uploaded to a Dataset +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Having thousands of files in a Dataset can cause issues. Most users would be better off with the data repackaged in fewer large bundles. To help curtail these issues, a limit can be set to prevent the number of file uploads from getting out of hand. + +The limit can be set via JVM setting :ref:`dataverse.files.default-dataset-file-count-limit` to be installation wide, or, set on each Collection/Dataset. + +For Installation wide limit, the limit can be set via JVM. ./asadmin $ASADMIN_OPTS create-jvm-options "-Ddataverse.files.default-dataset-file-count-limit=" + +For Collections, the attribute can be controlled by calling the Create or Update Dataverse API and adding ``datasetFileCountLimit=500`` to the Json body. + +For Datasets, the attribute can be set using the `Update Dataset Files Limit <#setting-the-files-count-limit-on-a-dataset>`_ API and passing the qp `fileCountLimit=500`. + +Setting a value of -1 will clear the limit for that level. If no limit is found on the Dataset, the hierarchy of parent nodes will be checked until finally the JVM setting is checked. + +With this setting set a 400 error response stating that the limit has been reached, including the effective limit, will be returned. + +Please note that a superuser will be exempt from this rule. + +The check will use the value defined in the Dataset first, and if not set (value <1) the Dataverse/Collection will be checked, and finally the JVM setting. + +.. _set-dataset-file-limit-api: + +Setting the files count limit on a Dataset +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +In order to update the number of files allowed for a Dataset, without causing a Draft version of the Dataset being created, the following API can be used + +.. note:: To clear the limit simply set the limit to -1 or call the DELETE API. + +.. code-block:: bash + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export SERVER_URL=https://demo.dataverse.org + export ID=24 + export LIMIT=500 + + curl -H "X-Dataverse-key:$API_TOKEN" -X POST "$SERVER_URL/api/datasets/$ID/files/uploadlimit/$LIMIT" + +To delete the existing limit: + + curl -H "X-Dataverse-key:$API_TOKEN" -X DELETE "$SERVER_URL/api/datasets/$ID/files/uploadlimit" + +The fully expanded example above (without environment variables) looks like this: + +.. code-block:: bash + + curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST "https://demo.dataverse.org/api/datasets/24/files/uploadlimit/500" + +To delete the existing limit: + + curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X DELETE "https://demo.dataverse.org/api/datasets/24/files/uploadlimit" + Report the data (file) size of a Dataset ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index ac6b9e48347..6a3c64291af 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -2561,6 +2561,20 @@ Notes: - During startup, this directory will be checked for existence and write access. It will be created for you if missing. If it cannot be created or does not have proper write access, application deployment will fail. +.. _dataverse.files.default-dataset-file-count-limit: + +dataverse.files.default-dataset-file-count-limit +++++++++++++++++++++++++++++++++++++++++++++++++ + +Configure a limit to the maximum number of Datafiles that can be uploaded to a Dataset. + +Notes: + +- This is a default that can be overwritten in any Dataverse/Collection or Dataset. +- A value less than 1 will be treated as no limit set. +- Changing this value will not delete any existing files. It is only intended for preventing new files from being uploaded. +- Superusers will not be governed by this rule. + .. _dataverse.files.uploads: dataverse.files.uploads diff --git a/scripts/search/tests/data/dataset-finch1-fileLimit.json b/scripts/search/tests/data/dataset-finch1-fileLimit.json new file mode 100644 index 00000000000..d223ace47e5 --- /dev/null +++ b/scripts/search/tests/data/dataset-finch1-fileLimit.json @@ -0,0 +1,82 @@ +{ + "datasetVersion": { + "license": { + "name": "CC0 1.0", + "uri": "http://creativecommons.org/publicdomain/zero/1.0" + }, + "metadataBlocks": { + "citation": { + "fields": [ + { + "value": "Darwin's Finches", + "typeClass": "primitive", + "multiple": false, + "typeName": "title" + }, + { + "value": [ + { + "authorName": { + "value": "Finch, Fiona", + "typeClass": "primitive", + "multiple": false, + "typeName": "authorName" + }, + "authorAffiliation": { + "value": "Birds Inc.", + "typeClass": "primitive", + "multiple": false, + "typeName": "authorAffiliation" + } + } + ], + "typeClass": "compound", + "multiple": true, + "typeName": "author" + }, + { + "value": [ + { "datasetContactEmail" : { + "typeClass": "primitive", + "multiple": false, + "typeName": "datasetContactEmail", + "value" : "finch@mailinator.com" + }, + "datasetContactName" : { + "typeClass": "primitive", + "multiple": false, + "typeName": "datasetContactName", + "value": "Finch, Fiona" + } + }], + "typeClass": "compound", + "multiple": true, + "typeName": "datasetContact" + }, + { + "value": [ { + "dsDescriptionValue":{ + "value": "Darwin's finches (also known as the Galápagos finches) are a group of about fifteen species of passerine birds.", + "multiple":false, + "typeClass": "primitive", + "typeName": "dsDescriptionValue" + }}], + "typeClass": "compound", + "multiple": true, + "typeName": "dsDescription" + }, + { + "value": [ + "Medicine, Health and Life Sciences" + ], + "typeClass": "controlledVocabulary", + "multiple": true, + "typeName": "subject" + } + ], + "displayName": "Citation Metadata" + } + } + }, + "datasetFileCountLimit": 100 +} diff --git a/src/main/java/edu/harvard/iq/dataverse/Dataset.java b/src/main/java/edu/harvard/iq/dataverse/Dataset.java index fd3f8333768..e7e5903482c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/Dataset.java +++ b/src/main/java/edu/harvard/iq/dataverse/Dataset.java @@ -72,7 +72,9 @@ @NamedQuery(name = "Dataset.findByReleaseUserId", query = "SELECT o FROM Dataset o WHERE o.releaseUser.id=:releaseUserId"), @NamedQuery(name = "Dataset.countAll", - query = "SELECT COUNT(ds) FROM Dataset ds") + query = "SELECT COUNT(ds) FROM Dataset ds"), + @NamedQuery(name = "Dataset.countFilesByOwnerId", + query = "SELECT COUNT(dvo) FROM DvObject dvo WHERE dvo.owner.id=:ownerId AND dvo.dtype='DataFile'") }) @NamedNativeQuery( name = "Dataset.findAllOrSubsetOrderByFilesOwned", diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java index ace1a6223f1..9f0441062c2 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java @@ -4082,7 +4082,16 @@ public String save() { // have been created in the dataset. dataset = datasetService.find(dataset.getId()); - List filesAdded = ingestService.saveAndAddFilesToDataset(dataset.getOrCreateEditVersion(), newFiles, null, true); + boolean ignoreUploadFileLimits = this.session.getUser() != null ? this.session.getUser().isSuperuser() : false; + List filesAdded = ingestService.saveAndAddFilesToDataset(dataset.getOrCreateEditVersion(), newFiles, null, true, ignoreUploadFileLimits); + if (filesAdded.size() < nNewFiles) { + // Not all files were saved + Integer limit = dataset.getEffectiveDatasetFileCountLimit(); + if (limit != null) { + String msg = BundleUtil.getStringFromBundle("file.add.count_exceeds_limit", List.of(limit.toString())); + JsfHelper.addInfoMessage(msg); + } + } newFiles.clear(); // and another update command: diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java index 202800d027b..303a6d8a5ac 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java @@ -1077,4 +1077,14 @@ public long getDatasetCount() { return em.createNamedQuery("Dataset.countAll", Long.class).getSingleResult(); } + /** + * + * @param id - owner id + * @return Total number of datafiles for this dataset/owner + */ + public int getDataFileCountByOwner(long id) { + Long c = em.createNamedQuery("Dataset.countFilesByOwnerId", Long.class).setParameter("ownerId", id).getSingleResult(); + return c.intValue(); // ignoring the truncation since the number should never be too large + } + } diff --git a/src/main/java/edu/harvard/iq/dataverse/DvObject.java b/src/main/java/edu/harvard/iq/dataverse/DvObject.java index a9a92950837..68ff739a77f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DvObject.java +++ b/src/main/java/edu/harvard/iq/dataverse/DvObject.java @@ -506,7 +506,6 @@ public StorageQuota getStorageQuota() { public void setStorageQuota(StorageQuota storageQuota) { this.storageQuota = storageQuota; } - /** * * @param other diff --git a/src/main/java/edu/harvard/iq/dataverse/DvObjectContainer.java b/src/main/java/edu/harvard/iq/dataverse/DvObjectContainer.java index 56d26a7260d..0fc43d391ce 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DvObjectContainer.java +++ b/src/main/java/edu/harvard/iq/dataverse/DvObjectContainer.java @@ -9,11 +9,9 @@ import edu.harvard.iq.dataverse.util.json.JsonUtil; import jakarta.json.JsonObject; import jakarta.json.JsonObjectBuilder; -import jakarta.persistence.CascadeType; +import jakarta.persistence.*; + import java.util.Optional; -import jakarta.persistence.MappedSuperclass; -import jakarta.persistence.OneToOne; -import jakarta.persistence.Transient; import org.apache.commons.lang3.StringUtils; @@ -56,6 +54,9 @@ public boolean isEffectivelyPermissionRoot() { @OneToOne(mappedBy = "dvObjectContainer",cascade={ CascadeType.REMOVE, CascadeType.PERSIST}, orphanRemoval=true) private StorageUse storageUse; + + @Column( nullable = true ) + private Integer datasetFileCountLimit; public String getEffectiveStorageDriverId() { String id = storageDriver; @@ -260,5 +261,23 @@ public PidProvider getEffectivePidGenerator() { } return pidGenerator; } + public Integer getDatasetFileCountLimit() { + return datasetFileCountLimit; + } + public void setDatasetFileCountLimit(Integer datasetFileCountLimit) { + this.datasetFileCountLimit = datasetFileCountLimit != null && datasetFileCountLimit < 0 ? null : datasetFileCountLimit; + } + public Integer getEffectiveDatasetFileCountLimit() { + if (!isDatasetFileCountLimitSet(getDatasetFileCountLimit()) && getOwner() != null) { + return getOwner().getEffectiveDatasetFileCountLimit(); + } else if (!isDatasetFileCountLimitSet(getDatasetFileCountLimit())) { + Optional opt = JvmSettings.DEFAULT_DATASET_FILE_COUNT_LIMIT.lookupOptional(Integer.class); + return (opt.isPresent()) ? opt.get() : null; + } + return getDatasetFileCountLimit(); + } + public boolean isDatasetFileCountLimitSet(Integer datasetFileCountLimit) { + return datasetFileCountLimit != null && datasetFileCountLimit >= 0; + } } diff --git a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java index 008276422f1..62c80be37cb 100644 --- a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java @@ -201,7 +201,10 @@ public enum Referrer { private Long maxIngestSizeInBytes = null; // CSV: 4.8 MB, DTA: 976.6 KB, XLSX: 5.7 MB, etc. private String humanPerFormatTabularLimits = null; - private Integer multipleUploadFilesLimit = null; + private Integer multipleUploadFilesLimit = null; + // Maximum number of files per dataset allowed ot be uploaded + private Integer maxFileUploadCount = null; + private Integer fileUploadsAvailable = null; //MutableBoolean so it can be passed from DatasetPage, supporting DatasetPage.cancelCreate() private MutableBoolean uploadInProgress = null; @@ -393,6 +396,10 @@ public String populateHumanPerFormatTabularLimits() { return String.join(", ", formatLimits); } + public Integer getFileUploadsAvailable() { + return fileUploadsAvailable != null ? fileUploadsAvailable : -1; + } + /* The number of files the GUI user is allowed to upload in one batch, via drag-and-drop, or through the file select dialog. Now configurable @@ -543,17 +550,28 @@ public String initCreateMode(String modeToken, DatasetVersion version, MutableBo this.maxIngestSizeInBytes = systemConfig.getTabularIngestSizeLimit(); this.humanPerFormatTabularLimits = populateHumanPerFormatTabularLimits(); this.multipleUploadFilesLimit = systemConfig.getMultipleUploadFilesLimit(); - + setFileUploadCountLimits(0); logger.fine("done"); saveEnabled = true; return null; } + private void setFileUploadCountLimits(int preLoaded) { + this.maxFileUploadCount = this.maxFileUploadCount == null ? dataset.getEffectiveDatasetFileCountLimit() : this.maxFileUploadCount; + Long id = dataset.getId() != null ? dataset.getId() : dataset.getOwner() != null ? dataset.getOwner().getId() : null; + this.fileUploadsAvailable = this.maxFileUploadCount != null && id != null ? + Math.max(0, this.maxFileUploadCount - datasetService.getDataFileCountByOwner(id) - preLoaded) : + -1; + } public boolean isQuotaExceeded() { return systemConfig.isStorageQuotasEnforced() && uploadSessionQuota != null && uploadSessionQuota.getRemainingQuotaInBytes() == 0; } + public boolean isFileUploadCountExceeded() { + boolean ignoreLimit = this.session.getUser().isSuperuser(); + return !ignoreLimit && !isFileReplaceOperation() && fileUploadsAvailable != null && fileUploadsAvailable == 0; + } public String init() { // default mode should be EDIT @@ -604,8 +622,8 @@ public String init() { } this.maxIngestSizeInBytes = systemConfig.getTabularIngestSizeLimit(); this.humanPerFormatTabularLimits = populateHumanPerFormatTabularLimits(); - this.multipleUploadFilesLimit = systemConfig.getMultipleUploadFilesLimit(); - + this.multipleUploadFilesLimit = systemConfig.getMultipleUploadFilesLimit(); + setFileUploadCountLimits(0); hasValidTermsOfAccess = isHasValidTermsOfAccess(); if (!hasValidTermsOfAccess) { PrimeFaces.current().executeScript("PF('blockDatasetForm').show()"); @@ -1103,9 +1121,17 @@ public String save() { } } } - + boolean ignoreUploadFileLimits = this.session.getUser() != null ? this.session.getUser().isSuperuser() : false; // Try to save the NEW files permanently: - List filesAdded = ingestService.saveAndAddFilesToDataset(workingVersion, newFiles, null, true); + List filesAdded = ingestService.saveAndAddFilesToDataset(workingVersion, newFiles, null, true, ignoreUploadFileLimits); + if (filesAdded.size() < nNewFiles) { + // Not all files were saved + Integer limit = dataset.getEffectiveDatasetFileCountLimit(); + if (limit != null) { + String msg = BundleUtil.getStringFromBundle("file.add.count_exceeds_limit", List.of(limit.toString())); + JsfHelper.addInfoMessage(msg); + } + } // reset the working list of fileMetadatas, as to only include the ones // that have been added to the version successfully: diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 0999408a977..fd9e614eda4 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -26,8 +26,7 @@ import edu.harvard.iq.dataverse.datasetutility.OptionalFileParams; import edu.harvard.iq.dataverse.engine.command.Command; import edu.harvard.iq.dataverse.engine.command.DataverseRequest; -import edu.harvard.iq.dataverse.engine.command.exception.CommandException; -import edu.harvard.iq.dataverse.engine.command.exception.UnforcedCommandException; +import edu.harvard.iq.dataverse.engine.command.exception.*; import edu.harvard.iq.dataverse.engine.command.impl.*; import edu.harvard.iq.dataverse.export.DDIExportServiceBean; import edu.harvard.iq.dataverse.export.ExportService; @@ -97,9 +96,7 @@ import java.util.regex.Pattern; import java.util.stream.Collectors; import static edu.harvard.iq.dataverse.api.ApiConstants.*; -import edu.harvard.iq.dataverse.engine.command.exception.IllegalCommandException; -import edu.harvard.iq.dataverse.engine.command.exception.PermissionException; import edu.harvard.iq.dataverse.dataset.DatasetType; import edu.harvard.iq.dataverse.dataset.DatasetTypeServiceBean; import edu.harvard.iq.dataverse.license.License; @@ -1971,6 +1968,72 @@ public Response removeFileRetention(@Context ContainerRequestContext crc, @PathP } } + @POST + @AuthRequired + @Path("{id}/files/uploadlimit/{limit}") + public Response updateDatasetFilesLimits(@Context ContainerRequestContext crc, + @PathParam("id") String id, + @PathParam("limit") int datasetFileCountLimit) { + + // user is authenticated + AuthenticatedUser authenticatedUser = null; + try { + authenticatedUser = getRequestAuthenticatedUserOrDie(crc); + } catch (WrappedResponse ex) { + return error(Status.UNAUTHORIZED, "Authentication is required."); + } + + Dataset dataset; + try { + dataset = findDatasetOrDie(id); + } catch (WrappedResponse ex) { + return ex.getResponse(); + } + + if (authenticatedUser.isSuperuser() || permissionService.hasPermissionsFor(authenticatedUser, dataset, + EnumSet.of(Permission.EditDataset))) { + + dataset.setDatasetFileCountLimit(datasetFileCountLimit); + datasetService.merge(dataset); + + return ok("ok"); + } else { + return error(Status.FORBIDDEN, "User is not a superuser or user does not have EditDataset permissions"); + } + } + + @DELETE + @AuthRequired + @Path("{id}/files/uploadlimit") + public Response deleteDatasetFilesLimits(@Context ContainerRequestContext crc, + @PathParam("id") String id) { + // user is authenticated + AuthenticatedUser authenticatedUser = null; + try { + authenticatedUser = getRequestAuthenticatedUserOrDie(crc); + } catch (WrappedResponse ex) { + return error(Status.UNAUTHORIZED, "Authentication is required."); + } + + Dataset dataset; + try { + dataset = findDatasetOrDie(id); + } catch (WrappedResponse ex) { + return ex.getResponse(); + } + + if (authenticatedUser.isSuperuser() || permissionService.hasPermissionsFor(authenticatedUser, dataset, + EnumSet.of(Permission.EditDataset))) { + + dataset.setDatasetFileCountLimit(null); + datasetService.merge(dataset); + + return ok("ok"); + } else { + return error(Status.FORBIDDEN, "User is not a superuser or user does not have EditDataset permissions"); + } + } + @PUT @AuthRequired @Path("{linkedDatasetId}/link/{linkingDataverseAlias}") @@ -2553,7 +2616,6 @@ public Response deleteCurationStatus(@Context ContainerRequestContext crc, @Path return Response.fromResponse(wr.getResponse()).status(Response.Status.BAD_REQUEST).build(); } } - @GET @AuthRequired @Path("{id}/uploadurls") @@ -2562,7 +2624,8 @@ public Response getMPUploadUrls(@Context ContainerRequestContext crc, @PathParam Dataset dataset = findDatasetOrDie(idSupplied); boolean canUpdateDataset = false; - canUpdateDataset = permissionSvc.requestOn(createDataverseRequest(getRequestUser(crc)), dataset) + User user = getRequestUser(crc); + canUpdateDataset = permissionSvc.requestOn(createDataverseRequest(user), dataset) .canIssue(UpdateDatasetVersionCommand.class); if (!canUpdateDataset) { return error(Response.Status.FORBIDDEN, "You are not permitted to upload files to this dataset."); @@ -2572,6 +2635,17 @@ public Response getMPUploadUrls(@Context ContainerRequestContext crc, @PathParam return error(Response.Status.NOT_FOUND, "Direct upload not supported for files in this dataset: " + dataset.getId()); } + if (!user.isSuperuser()) { + Integer effectiveDatasetFileCountLimit = dataset.getEffectiveDatasetFileCountLimit(); + boolean hasFileCountLimit = dataset.isDatasetFileCountLimitSet(effectiveDatasetFileCountLimit); + if (hasFileCountLimit) { + int uploadedFileCount = datasetService.getDataFileCountByOwner(dataset.getId()); + if (uploadedFileCount >= effectiveDatasetFileCountLimit) { + return error(Response.Status.BAD_REQUEST, + BundleUtil.getStringFromBundle("file.add.count_exceeds_limit", Arrays.asList(String.valueOf(effectiveDatasetFileCountLimit)))); + } + } + } Long maxSize = systemConfig.getMaxFileUploadSizeForStore(dataset.getEffectiveStorageDriverId()); if (maxSize != null) { if(fileSize > maxSize) { @@ -2585,7 +2659,7 @@ public Response getMPUploadUrls(@Context ContainerRequestContext crc, @PathParam if(fileSize > limit.getRemainingQuotaInBytes()) { return error(Response.Status.BAD_REQUEST, "The file you are trying to upload is too large to be uploaded to this dataset. " + - "The remaing file size quota is " + limit.getRemainingQuotaInBytes() + " bytes."); + "The remaining file size quota is " + limit.getRemainingQuotaInBytes() + " bytes."); } } JsonObjectBuilder response = null; diff --git a/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/MediaResourceManagerImpl.java b/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/MediaResourceManagerImpl.java index 3f5345d8e0d..5189963ae59 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/MediaResourceManagerImpl.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/MediaResourceManagerImpl.java @@ -346,9 +346,8 @@ DepositReceipt replaceOrAddFiles(String uri, Deposit deposit, AuthCredentials au ConstraintViolation violation = constraintViolations.iterator().next(); throw new SwordError(UriRegistry.ERROR_BAD_REQUEST, "Unable to add file(s) to dataset: " + violation.getMessage() + " The invalid value was \"" + violation.getInvalidValue() + "\"."); } else { - - ingestService.saveAndAddFilesToDataset(editVersion, dataFiles, null, true); - + boolean ignoreUploadFileLimits = user != null ? user.isSuperuser() : false; + ingestService.saveAndAddFilesToDataset(editVersion, dataFiles, null, true, ignoreUploadFileLimits); } } else { throw new SwordError(UriRegistry.ERROR_BAD_REQUEST, "No files to add to dataset. Perhaps the zip file was empty."); diff --git a/src/main/java/edu/harvard/iq/dataverse/api/dto/DatasetDTO.java b/src/main/java/edu/harvard/iq/dataverse/api/dto/DatasetDTO.java index ec8adfb4eef..3b3e9d57151 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/dto/DatasetDTO.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/dto/DatasetDTO.java @@ -19,6 +19,7 @@ public class DatasetDTO implements java.io.Serializable { private String metadataLanguage; private DatasetVersionDTO datasetVersion; private List dataFiles; + private Integer datasetFileCountLimit; public String getId() { return id; @@ -114,4 +115,11 @@ public String getMetadataLanguage() { return metadataLanguage; } + public Integer getDatasetFileCountLimit() { + return datasetFileCountLimit; + } + + public void setDatasetFileCountLimit(Integer datasetFileCountLimit) { + this.datasetFileCountLimit = datasetFileCountLimit; + } } diff --git a/src/main/java/edu/harvard/iq/dataverse/api/dto/DataverseDTO.java b/src/main/java/edu/harvard/iq/dataverse/api/dto/DataverseDTO.java index 4f2f1032c07..bcb47eb5ff4 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/dto/DataverseDTO.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/dto/DataverseDTO.java @@ -12,6 +12,7 @@ public class DataverseDTO { private String affiliation; private List dataverseContacts; private Dataverse.DataverseType dataverseType; + private Integer datasetFileCountLimit; public String getAlias() { return alias; @@ -45,6 +46,14 @@ public void setAffiliation(String affiliation) { this.affiliation = affiliation; } + public Integer getDatasetFileCountLimit() { + return datasetFileCountLimit; + } + + public void setDatasetFileCountLimit(Integer datasetFileCountLimit) { + this.datasetFileCountLimit = datasetFileCountLimit; + } + public List getDataverseContacts() { return dataverseContacts; } diff --git a/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java b/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java index 6b98848021c..572b7fc5b78 100644 --- a/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java +++ b/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java @@ -1217,7 +1217,8 @@ private boolean step_030_createNewFilesViaIngest(OptionalFileParams optionalFile if (systemConfig.isStorageQuotasEnforced()) { quota = fileService.getUploadSessionQuotaLimit(dataset); } - Command cmd = new CreateNewDataFilesCommand(dvRequest, workingVersion, newFileInputStream, newFileName, newFileContentType, newStorageIdentifier, quota, newCheckSum, newCheckSumType, suppliedFileSize); + Command cmd = new CreateNewDataFilesCommand(dvRequest, workingVersion, newFileInputStream, newFileName, newFileContentType, newStorageIdentifier, + quota, newCheckSum, newCheckSumType, suppliedFileSize, isFileReplaceOperation()); CreateDataFileResult createDataFilesResult = commandEngine.submit(cmd); initialFileList = createDataFilesResult.getDataFiles(); @@ -1593,7 +1594,8 @@ private boolean step_060_addFilesViaIngestService(boolean tabIngest){ } int nFiles = finalFileList.size(); - finalFileList = ingestService.saveAndAddFilesToDataset(workingVersion, finalFileList, fileToReplace, tabIngest); + boolean ignoreUploadFileLimits = dvRequest.getAuthenticatedUser() != null ? dvRequest.getAuthenticatedUser().isSuperuser() : false; + finalFileList = ingestService.saveAndAddFilesToDataset(workingVersion, finalFileList, fileToReplace, tabIngest, ignoreUploadFileLimits); if (nFiles != finalFileList.size()) { if (nFiles == 1) { @@ -2062,6 +2064,19 @@ public Response addFiles(String jsonData, Dataset dataset, User authUser, boolea totalNumberofFiles = filesJson.getValuesAs(JsonObject.class).size(); workingVersion = dataset.getOrCreateEditVersion(); clone = workingVersion.cloneDatasetVersion(); + + if (!authUser.isSuperuser()) { + Integer effectiveDatasetFileCountLimit = dataset.getEffectiveDatasetFileCountLimit(); + boolean hasFileCountLimit = dataset.isDatasetFileCountLimitSet(effectiveDatasetFileCountLimit); + if (hasFileCountLimit) { + int uploadedFileCount = datasetService.getDataFileCountByOwner(dataset.getId()); + if (uploadedFileCount + totalNumberofFiles >= effectiveDatasetFileCountLimit) { + return error(Response.Status.BAD_REQUEST, + BundleUtil.getStringFromBundle("file.add.count_exceeds_limit", Arrays.asList(String.valueOf(effectiveDatasetFileCountLimit)))); + } + } + } + for (JsonObject fileJson : filesJson.getValuesAs(JsonObject.class)) { OptionalFileParams optionalFileParams = null; diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesCommand.java index e4130b534b3..6789b61f9dc 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesCommand.java @@ -1,8 +1,6 @@ package edu.harvard.iq.dataverse.engine.command.impl; -import edu.harvard.iq.dataverse.DataFile; -import edu.harvard.iq.dataverse.DatasetVersion; -import edu.harvard.iq.dataverse.Dataverse; +import edu.harvard.iq.dataverse.*; import edu.harvard.iq.dataverse.authorization.Permission; import edu.harvard.iq.dataverse.datasetutility.FileExceedsMaxSizeException; import edu.harvard.iq.dataverse.datasetutility.FileSizeChecker; @@ -22,6 +20,7 @@ import edu.harvard.iq.dataverse.util.file.CreateDataFileResult; import edu.harvard.iq.dataverse.util.file.FileExceedsStorageQuotaException; import jakarta.enterprise.inject.spi.CDI; +import jakarta.ws.rs.core.Response; import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; @@ -84,23 +83,33 @@ public class CreateNewDataFilesCommand extends AbstractCommand= effectiveDatasetFileCountLimit) { + throw new CommandExecutionException(BundleUtil.getStringFromBundle("file.add.count_exceeds_limit", Arrays.asList(String.valueOf(effectiveDatasetFileCountLimit))), this); + } + } + } + String warningMessage = null; // save the file, in the temporary location for now: @@ -725,4 +751,9 @@ public Map> getRequiredPermissions() { return ret; } + + // For testing + protected void setDatasetService(DatasetServiceBean datasetServiceBean) { + this.datasetServiceBean = datasetServiceBean; + } } diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseCommand.java index 55cc3708097..8d4f0102383 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseCommand.java @@ -120,6 +120,9 @@ private void updateDataverseFromDTO(Dataverse dataverse, DataverseDTO dto) { if (dto.getDataverseType() != null) { dataverse.setDataverseType(dto.getDataverseType()); } + if (dto.getDatasetFileCountLimit() != null) { + dataverse.setDatasetFileCountLimit(dto.getDatasetFileCountLimit()); + } } @Override diff --git a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java index a79c8f559a4..2917c5828a4 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java @@ -23,6 +23,7 @@ import edu.harvard.iq.dataverse.AuxiliaryFile; import edu.harvard.iq.dataverse.AuxiliaryFileServiceBean; import edu.harvard.iq.dataverse.ControlledVocabularyValue; +import edu.harvard.iq.dataverse.authorization.users.User; import edu.harvard.iq.dataverse.datavariable.VariableCategory; import edu.harvard.iq.dataverse.datavariable.VariableServiceBean; import edu.harvard.iq.dataverse.DatasetServiceBean; @@ -181,9 +182,10 @@ public class IngestServiceBean { // @todo: Is this method a good candidate for turning into a dedicated Command? public List saveAndAddFilesToDataset(DatasetVersion version, - List newFiles, - DataFile fileToReplace, - boolean tabIngest) { + List newFiles, + DataFile fileToReplace, + boolean tabIngest, + boolean ignoreUploadFileLimits) { UploadSessionQuotaLimit uploadSessionQuota = null; List ret = new ArrayList<>(); @@ -201,7 +203,14 @@ public List saveAndAddFilesToDataset(DatasetVersion version, // Check if this dataset is subject to any storage quotas: uploadSessionQuota = fileService.getUploadSessionQuotaLimit(dataset); } - + + Integer maxFiles = version.getDataset().getEffectiveDatasetFileCountLimit(); + if (!ignoreUploadFileLimits && fileToReplace == null && version.getDataset().getId() != null && version.getDataset().isDatasetFileCountLimitSet(maxFiles)) { + maxFiles = maxFiles - datasetService.getDataFileCountByOwner(version.getDataset().getId()); + } else { + maxFiles = Integer.MAX_VALUE; + } + for (DataFile dataFile : newFiles) { boolean unattached = false; boolean savedSuccess = false; @@ -212,6 +221,11 @@ public List saveAndAddFilesToDataset(DatasetVersion version, unattached = true; dataFile.setOwner(dataset); } + + if (--maxFiles < 0) { + logger.warning("Failed to save all the files due to the limit on the number of files that can be uploaded to this dataset."); + break; + } String[] storageInfo = DataAccess.getDriverIdAndStorageLocation(dataFile.getStorageIdentifier()); String driverType = DataAccess.getDriverType(storageInfo[0]); diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java b/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java index 07de576a0eb..2725c7afb6b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java @@ -56,6 +56,7 @@ public enum JvmSettings { FEATURED_ITEMS_IMAGE_MAXSIZE(SCOPE_FEATURED_ITEMS, "image-maxsize"), FEATURED_ITEMS_IMAGE_UPLOADS_DIRECTORY(SCOPE_FEATURED_ITEMS, "image-uploads"), HIDE_SCHEMA_DOT_ORG_DOWNLOAD_URLS(SCOPE_FILES, "hide-schema-dot-org-download-urls"), + DEFAULT_DATASET_FILE_COUNT_LIMIT(SCOPE_FILES, "default-dataset-file-count-limit"), //STORAGE DRIVER SETTINGS SCOPE_DRIVER(SCOPE_FILES), diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java index e5be6f9edc2..aa59a746422 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java @@ -134,6 +134,7 @@ public Dataverse parseDataverse(JsonObject jobj) throws JsonParseException { dv.setPermissionRoot(jobj.getBoolean("permissionRoot", false)); dv.setFacetRoot(jobj.getBoolean("facetRoot", false)); dv.setAffiliation(jobj.getString("affiliation", null)); + dv.setDatasetFileCountLimit(jobj.getInt("datasetFileCountLimit", -1)); if (jobj.containsKey("dataverseContacts")) { JsonArray dvContacts = jobj.getJsonArray("dataverseContacts"); @@ -227,6 +228,9 @@ public DataverseDTO parseDataverseDTO(JsonObject jsonObject) throws JsonParseExc } dataverseDTO.setDataverseContacts(contacts); } + if (jsonObject.containsKey("datasetFileCountLimit")) { + dataverseDTO.setDatasetFileCountLimit(Integer.valueOf(jsonObject.getInt("datasetFileCountLimit"))); + } return dataverseDTO; } @@ -388,6 +392,7 @@ public Dataset parseDataset(JsonObject obj) throws JsonParseException { }else { throw new JsonParseException("Specified metadatalanguage not allowed."); } + dataset.setDatasetFileCountLimit(obj.getInt("datasetFileCountLimit", -1)); String datasetTypeIn = obj.getString("datasetType", DatasetType.DEFAULT_DATASET_TYPE); logger.fine("datasetTypeIn: " + datasetTypeIn); DatasetType datasetType = datasetTypeService.getByName(datasetTypeIn); diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java index 9d3d1ceae20..182d4aa84e0 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java @@ -73,10 +73,14 @@ public class JsonPrinter { @EJB static DatasetFieldServiceBean datasetFieldService; + + @EJB + static DatasetServiceBean datasetService; - public static void injectSettingsService(SettingsServiceBean ssb, DatasetFieldServiceBean dfsb, DataverseFieldTypeInputLevelServiceBean dfils) { + public static void injectSettingsService(SettingsServiceBean ssb, DatasetFieldServiceBean dfsb, DataverseFieldTypeInputLevelServiceBean dfils, DatasetServiceBean ds) { settingsService = ssb; datasetFieldService = dfsb; + datasetService = ds; } public JsonPrinter() { @@ -306,6 +310,7 @@ public static JsonObjectBuilder json(Dataverse dv, Boolean hideEmail, Boolean re if (childCount != null) { bld.add("childCount", childCount); } + addDatasetFileCountLimit(dv, bld); return bld; } @@ -410,6 +415,8 @@ public static JsonObjectBuilder json(Dataset ds, Boolean returnOwners) { .add("publisher", BrandingUtil.getInstallationBrandName()) .add("publicationDate", ds.getPublicationDateFormattedYYYYMMDD()) .add("storageIdentifier", ds.getStorageIdentifier()); + addDatasetFileCountLimit(ds, bld); + if (DvObjectContainer.isMetadataLanguageSet(ds.getMetadataLanguage())) { bld.add("metadataLanguage", ds.getMetadataLanguage()); } @@ -420,6 +427,21 @@ public static JsonObjectBuilder json(Dataset ds, Boolean returnOwners) { return bld; } + private static void addDatasetFileCountLimit(DvObjectContainer dvo, JsonObjectBuilder bld) { + Integer effectiveDatasetFileCountLimit = dvo.getEffectiveDatasetFileCountLimit(); + if (dvo.isDatasetFileCountLimitSet(effectiveDatasetFileCountLimit)) { + bld.add("effectiveDatasetFileCountLimit", effectiveDatasetFileCountLimit); + } + Integer datasetFileCountLimit = dvo.getDatasetFileCountLimit(); + if (dvo.isDatasetFileCountLimitSet(datasetFileCountLimit)) { + bld.add("datasetFileCountLimit", datasetFileCountLimit); + } + if (dvo.isInstanceofDataset() && dvo.isDatasetFileCountLimitSet(effectiveDatasetFileCountLimit)) { + int available = effectiveDatasetFileCountLimit - datasetService.getDataFileCountByOwner(dvo.getId()); + bld.add("datasetFileUploadsAvailable", Math.max(0, available)); + } + } + public static JsonObjectBuilder json(FileDetailsHolder ds) { return Json.createObjectBuilder().add(ds.getStorageID() , Json.createObjectBuilder() @@ -462,6 +484,7 @@ public static JsonObjectBuilder json(DatasetVersion dsv, List anonymized .add("publicationDate", dataset.getPublicationDateFormattedYYYYMMDD()) .add("citationDate", dataset.getCitationDateFormattedYYYYMMDD()) .add("versionNote", dsv.getVersionNote()); + addDatasetFileCountLimit(dataset, bld); License license = DatasetUtil.getLicense(dsv); if (license != null) { diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinterHelper.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinterHelper.java index 43d553c93e7..aeba4ba797f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinterHelper.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinterHelper.java @@ -1,33 +1,37 @@ -package edu.harvard.iq.dataverse.util.json; - -import edu.harvard.iq.dataverse.DatasetFieldServiceBean; -import edu.harvard.iq.dataverse.DataverseFieldTypeInputLevelServiceBean; -import edu.harvard.iq.dataverse.settings.SettingsServiceBean; - -import jakarta.annotation.PostConstruct; -import jakarta.ejb.EJB; -import jakarta.ejb.Singleton; -import jakarta.ejb.Startup; - -/** - * This is a small helper bean - * As it is a singleton and built at application start (=deployment), it will inject the (stateless) - * settings service into the OREMap once it's ready. - */ -@Singleton -@Startup -public class JsonPrinterHelper { - @EJB - SettingsServiceBean settingsSvc; - - @EJB - DatasetFieldServiceBean datasetFieldSvc; - - @EJB - DataverseFieldTypeInputLevelServiceBean datasetFieldInpuLevelSvc; - - @PostConstruct - public void injectService() { - JsonPrinter.injectSettingsService(settingsSvc, datasetFieldSvc, datasetFieldInpuLevelSvc); - } -} +package edu.harvard.iq.dataverse.util.json; + +import edu.harvard.iq.dataverse.DatasetFieldServiceBean; +import edu.harvard.iq.dataverse.DatasetServiceBean; +import edu.harvard.iq.dataverse.DataverseFieldTypeInputLevelServiceBean; +import edu.harvard.iq.dataverse.settings.SettingsServiceBean; + +import jakarta.annotation.PostConstruct; +import jakarta.ejb.EJB; +import jakarta.ejb.Singleton; +import jakarta.ejb.Startup; + +/** + * This is a small helper bean + * As it is a singleton and built at application start (=deployment), it will inject the (stateless) + * settings service into the OREMap once it's ready. + */ +@Singleton +@Startup +public class JsonPrinterHelper { + @EJB + SettingsServiceBean settingsSvc; + + @EJB + DatasetFieldServiceBean datasetFieldSvc; + + @EJB + DataverseFieldTypeInputLevelServiceBean datasetFieldInpuLevelSvc; + + @EJB + DatasetServiceBean datasetSvc; + + @PostConstruct + public void injectService() { + JsonPrinter.injectSettingsService(settingsSvc, datasetFieldSvc, datasetFieldInpuLevelSvc, datasetSvc); + } +} diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/NullSafeJsonBuilder.java b/src/main/java/edu/harvard/iq/dataverse/util/json/NullSafeJsonBuilder.java index 21360fcd708..f033e12e532 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/NullSafeJsonBuilder.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/NullSafeJsonBuilder.java @@ -64,10 +64,14 @@ public NullSafeJsonBuilder add(String name, int value) { return this; } + public NullSafeJsonBuilder add(String name, Integer value) { + return (value != null) ? add(name, value.intValue()) : this; + } + public NullSafeJsonBuilder add(String name, Long value) { return ( value != null ) ? add(name, value.longValue()) : this; } - + @Override public NullSafeJsonBuilder add(String name, long value) { delegate.add(name, value); diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index 94cceaf7433..d11b4de9787 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -1844,6 +1844,7 @@ file.selectToAddBtn=Select Files to Add file.selectToAdd.tipLimit=File upload limit is {0} per file. file.selectToAdd.tipQuotaRemaining=Storage quota: {0} remaining. file.selectToAdd.tipMaxNumFiles=Maximum of {0} {0, choice, 0#files|1#file|2#files} per upload. +file.selectToAdd.tipMaxNumFilesUploadsAvailable=Maximum of {0} {0, choice, 0#files|1#file|2#files} available to upload. file.selectToAdd.tipTabularLimit=Tabular file ingest is limited to {2}. file.selectToAdd.tipPerFileTabularLimit=Ingest is limited to the following file sizes based on their format: {0}. file.selectToAdd.tipMoreInformation=Select files or drag and drop into the upload widget. @@ -2381,6 +2382,7 @@ file.message.deleteSuccess=The file has been deleted. file.message.replaceSuccess=The file has been replaced. # File Add/Replace operation messages +file.add.count_exceeds_limit=Number of files can not exceed the maximum number of files allowed for this Dataset ({0}). file.addreplace.file_size_ok=File size is in range. file.addreplace.error.byte_abrev=B file.addreplace.error.file_exceeds_limit=This file size ({0}) exceeds the size limit of {1}. diff --git a/src/main/resources/db/migration/V6.6.0.1.sql b/src/main/resources/db/migration/V6.6.0.1.sql new file mode 100644 index 00000000000..5861a870d93 --- /dev/null +++ b/src/main/resources/db/migration/V6.6.0.1.sql @@ -0,0 +1,2 @@ +ALTER TABLE DATASET ADD COLUMN IF NOT EXISTS datasetfilecountlimit bigint; +ALTER TABLE DATAVERSE ADD COLUMN IF NOT EXISTS datasetfilecountlimit bigint; diff --git a/src/main/webapp/editFilesFragment.xhtml b/src/main/webapp/editFilesFragment.xhtml index aec11eb1e54..1b3d29ccbf2 100644 --- a/src/main/webapp/editFilesFragment.xhtml +++ b/src/main/webapp/editFilesFragment.xhtml @@ -88,7 +88,11 @@ rendered="#{EditDatafilesPage.maxNumberOfFiles > 0}"> - + + + @@ -160,7 +164,7 @@ dragDropSupport="true" auto="#{!(systemConfig.directUploadEnabled(EditDatafilesPage.dataset))}" multiple="#{datasetPage || EditDatafilesPage.allowMultipleFileUpload()}" - disabled="#{lockedFromEdits || !(datasetPage || EditDatafilesPage.showFileUploadComponent()) || EditDatafilesPage.isQuotaExceeded()}" + disabled="#{lockedFromEdits || !(datasetPage || EditDatafilesPage.showFileUploadComponent()) || EditDatafilesPage.isQuotaExceeded() || EditDatafilesPage.isFileUploadCountExceeded()}" listener="#{EditDatafilesPage.handleFileUpload}" process="filesTable" update=":datasetForm:filesTable, @([id$=filesButtons])" @@ -169,7 +173,8 @@ onstart="javascript:uploadWidgetDropRemoveMsg();uploadStarted();" onerror="javascript:uploadFailure();" sizeLimit="#{EditDatafilesPage.getMaxFileUploadSizeInBytes()}" - fileLimit="#{EditDatafilesPage.getMaxNumberOfFiles()}" + fileLimit="#{EditDatafilesPage.getMaxNumberOfFiles()}" + fileLimitAvailable="#{EditDatafilesPage.getFileUploadsAvailable}" invalidSizeMessage="#{bundle['file.edit.error.file_exceeds_limit']}" sequential="true" previewWidth="-1" diff --git a/src/test/java/edu/harvard/iq/dataverse/DataverseTest.java b/src/test/java/edu/harvard/iq/dataverse/DataverseTest.java index 4e2bd5b3c2d..f4d6eaa6ad5 100644 --- a/src/test/java/edu/harvard/iq/dataverse/DataverseTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/DataverseTest.java @@ -1,6 +1,9 @@ package edu.harvard.iq.dataverse; import edu.harvard.iq.dataverse.mocks.MocksFactory; +import edu.harvard.iq.dataverse.settings.JvmSettings; +import edu.harvard.iq.dataverse.util.testing.JvmSetting; +import edu.harvard.iq.dataverse.util.testing.LocalJvmSettings; import org.hamcrest.MatcherAssert; import org.hamcrest.Matchers; import org.junit.jupiter.api.BeforeEach; @@ -13,6 +16,7 @@ * * @author adaybujeda */ +@LocalJvmSettings public class DataverseTest { private Dataverse OWNER; @@ -62,4 +66,14 @@ public void getMetadataBlockFacets_should_return_owner_metadatablockfacets_when_ MatcherAssert.assertThat(result, Matchers.is(OWNER_METADATABLOCKFACETS)); } -} \ No newline at end of file + @Test + @JvmSetting(key = JvmSettings.DEFAULT_DATASET_FILE_COUNT_LIMIT, value = "23") + public void testDatasetFileCountLimit() { + OWNER.setDatasetFileCountLimit(1); + MatcherAssert.assertThat(OWNER.getEffectiveDatasetFileCountLimit(), Matchers.is(1)); + OWNER.setDatasetFileCountLimit(null); + MatcherAssert.assertThat(OWNER.getEffectiveDatasetFileCountLimit(), Matchers.is(23)); + OWNER.setDatasetFileCountLimit(-1); + MatcherAssert.assertThat(OWNER.getEffectiveDatasetFileCountLimit(), Matchers.is(23)); + } +} diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java index 42637a3f882..b6484c16628 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java @@ -1,9 +1,6 @@ package edu.harvard.iq.dataverse.api; -import edu.harvard.iq.dataverse.DataFile; -import edu.harvard.iq.dataverse.DataFileTag; -import edu.harvard.iq.dataverse.DatasetVersionFilesServiceBean; -import edu.harvard.iq.dataverse.FileSearchCriteria; +import edu.harvard.iq.dataverse.*; import edu.harvard.iq.dataverse.authorization.DataverseRole; import edu.harvard.iq.dataverse.authorization.groups.impl.builtin.AuthenticatedUsers; import edu.harvard.iq.dataverse.authorization.users.PrivateUrlUser; @@ -16,6 +13,8 @@ import edu.harvard.iq.dataverse.util.BundleUtil; import edu.harvard.iq.dataverse.util.SystemConfig; import edu.harvard.iq.dataverse.util.json.JSONLDUtil; +import edu.harvard.iq.dataverse.util.json.JsonParseException; +import edu.harvard.iq.dataverse.util.json.JsonParser; import edu.harvard.iq.dataverse.util.json.JsonUtil; import io.restassured.RestAssured; import io.restassured.http.ContentType; @@ -66,7 +65,6 @@ import java.time.Year; import static org.hamcrest.CoreMatchers.*; import static org.hamcrest.Matchers.contains; -import static org.hamcrest.Matchers.hasEntry; import static org.junit.jupiter.api.Assertions.*; public class DatasetsIT { @@ -288,6 +286,7 @@ public void testCreateDataset() { Integer datasetId = UtilIT.getDatasetIdFromResponse(createDatasetResponse); Response datasetAsJson = UtilIT.nativeGet(datasetId, apiToken); + datasetAsJson.prettyPrint(); datasetAsJson.then().assertThat() .statusCode(OK.getStatusCode()); @@ -367,6 +366,109 @@ public void testCreateDataset() { } + @Test + public void testCreateUpdateDatasetFileCountLimit() throws JsonParseException { + Response createUser = UtilIT.createRandomUser(); + String apiToken = UtilIT.getApiTokenFromResponse(createUser); + Response createDataverseResponse = createFileLimitedDataverse(500, apiToken); + createDataverseResponse.prettyPrint(); + String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse); + JsonObject data = JsonUtil.getJsonObject(createDataverseResponse.getBody().asString()); + JsonParser parser = new JsonParser(); + Dataverse dv = parser.parseDataverse(data.getJsonObject("data")); + + JsonArrayBuilder metadataBlocks = Json.createArrayBuilder(); + metadataBlocks.add("citation"); + metadataBlocks.add("journal"); + metadataBlocks.add("socialscience"); + Response setMetadataBlocksResponse = UtilIT.setMetadataBlocks(dataverseAlias, metadataBlocks, apiToken); + setMetadataBlocksResponse.prettyPrint(); + setMetadataBlocksResponse.then().assertThat().statusCode(OK.getStatusCode()); + + Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken); + Integer datasetId1 = UtilIT.getDatasetIdFromResponse(createDatasetResponse); + + // Dataset defaults to owner dataverse's datasetFileCountLimit + Response datasetAsJson = UtilIT.nativeGet(datasetId1, apiToken); + datasetAsJson.prettyPrint(); + datasetAsJson.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.effectiveDatasetFileCountLimit", equalTo(500)) + .body("data.datasetFileCountLimit", equalTo(null)); + + // create dataset with datasetFileCountLimit = 100 + String pathToJsonFile = "scripts/search/tests/data/dataset-finch1-fileLimit.json"; + createDatasetResponse = UtilIT.createDatasetViaNativeApi(dataverseAlias, pathToJsonFile, apiToken); + createDatasetResponse.prettyPrint(); + Integer datasetId2 = UtilIT.getDatasetIdFromResponse(createDatasetResponse); + + datasetAsJson = UtilIT.nativeGet(datasetId2, apiToken); + datasetAsJson.prettyPrint(); + datasetAsJson.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.effectiveDatasetFileCountLimit", equalTo(100)) + .body("data.datasetFileCountLimit", equalTo(100)); + String persistentId = JsonPath.from(datasetAsJson.getBody().asString()).getString("data.latestVersion.datasetPersistentId"); + + // Update dataset with datasetFileCountLimit = 1 + Response updateDatasetResponse = UtilIT.updateDatasetFilesLimits(persistentId, 1, apiToken); + updateDatasetResponse.prettyPrint(); + updateDatasetResponse.then().assertThat() + .statusCode(OK.getStatusCode()); + + datasetAsJson = UtilIT.nativeGet(datasetId2, apiToken); + datasetAsJson.prettyPrint(); + datasetAsJson.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.effectiveDatasetFileCountLimit", equalTo(1)) + .body("data.datasetFileCountLimit", equalTo(1)); + + // Update/reset dataset with datasetFileCountLimit = -1 and expect the value from the owner dataverse + updateDatasetResponse = UtilIT.deleteDatasetFilesLimits(persistentId, apiToken); + updateDatasetResponse.prettyPrint(); + updateDatasetResponse.then().assertThat() + .statusCode(OK.getStatusCode()); + + datasetAsJson = UtilIT.nativeGet(datasetId2, apiToken); + datasetAsJson.prettyPrint(); + datasetAsJson.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.effectiveDatasetFileCountLimit", equalTo(500)) + .body("data.datasetFileCountLimit", equalTo(null)); + + // test clear limits and test that datasetFileCountLimit is not returned in json + dv.setDatasetFileCountLimit(null); + Response updateDataverseResponse = UtilIT.updateDataverse(dataverseAlias, dv, apiToken); + updateDataverseResponse.then().assertThat() + .statusCode(OK.getStatusCode()); + + datasetAsJson = UtilIT.nativeGet(datasetId2, apiToken); + datasetAsJson.prettyPrint(); + datasetAsJson.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.effectiveDatasetFileCountLimit", equalTo(null)) + .body("data.datasetFileCountLimit", equalTo(null)); + } + + @Test + public void testMultipleFileUploadOverCountLimit() { + Response createUser = UtilIT.createRandomUser(); + String apiToken = UtilIT.getApiTokenFromResponse(createUser); + Response createDataverseResponse = createFileLimitedDataverse(1, apiToken); + createDataverseResponse.prettyPrint(); + String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse); + Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken); + createDatasetResponse.prettyPrint(); + Integer datasetId = UtilIT.getDatasetIdFromResponse(createDatasetResponse); + + String json = "[{\"id\":0},{\"id\":1},{\"id\":2}]"; // simple array since we only need an array size + Response addFilesResponse = UtilIT.addFiles(datasetId.toString(), json, apiToken); + addFilesResponse.prettyPrint(); + addFilesResponse.then().assertThat() + .body("message", containsString(BundleUtil.getStringFromBundle("file.add.count_exceeds_limit", Collections.singletonList("1")))) + .statusCode(BAD_REQUEST.getStatusCode()); + } + @Test public void testAddUpdateDatasetViaNativeAPI() { @@ -6622,4 +6724,20 @@ public void testUpdateMultipleFileMetadata() { deleteSecondUserResponse.then().assertThat() .statusCode(OK.getStatusCode()); } + + private Response createFileLimitedDataverse(int datasetFileCountLimit, String apiToken) { + String dataverseAlias = UtilIT.getRandomDvAlias(); + String emailAddressOfFirstDataverseContact = dataverseAlias + "@mailinator.com"; + JsonObjectBuilder jsonToCreateDataverse = Json.createObjectBuilder() + .add("name", dataverseAlias) + .add("alias", dataverseAlias) + .add("datasetFileCountLimit", datasetFileCountLimit) + .add("dataverseContacts", Json.createArrayBuilder() + .add(Json.createObjectBuilder() + .add("contactEmail", emailAddressOfFirstDataverseContact) + ) + ); + ; + return UtilIT.createDataverse(jsonToCreateDataverse.build(), apiToken); + } } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java index 1094935182f..b5ae2a44856 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java @@ -1,5 +1,8 @@ package edu.harvard.iq.dataverse.api; +import edu.harvard.iq.dataverse.util.json.JsonParseException; +import edu.harvard.iq.dataverse.util.json.JsonParser; +import edu.harvard.iq.dataverse.util.json.JsonUtil; import io.restassured.RestAssured; import static io.restassured.RestAssured.given; @@ -1351,14 +1354,30 @@ public void testAddDataverse() { } @Test - public void testUpdateDataverse() { + public void testUpdateDataverse() throws JsonParseException { Response createUser = UtilIT.createRandomUser(); String apiToken = UtilIT.getApiTokenFromResponse(createUser); String testAliasSuffix = "-update-dataverse"; String testDataverseAlias = UtilIT.getRandomDvAlias() + testAliasSuffix; Response createSubDataverseResponse = UtilIT.createSubDataverse(testDataverseAlias, null, apiToken, "root"); - createSubDataverseResponse.then().assertThat().statusCode(CREATED.getStatusCode()); + createSubDataverseResponse.prettyPrint(); + createSubDataverseResponse.then().assertThat() + .statusCode(CREATED.getStatusCode()) + .body("data.effectiveDatasetFileCountLimit", equalTo(null)) + .body("data.datasetFileCountLimit", equalTo(null)); + + // Update the dataverse with a datasetFileCountLimit of 500 + JsonObject data = JsonUtil.getJsonObject(createSubDataverseResponse.getBody().asString()); + JsonParser parser = new JsonParser(); + Dataverse dv = parser.parseDataverse(data.getJsonObject("data")); + dv.setDatasetFileCountLimit(500); + Response updateDataverseResponse = UtilIT.updateDataverse(testDataverseAlias, dv, apiToken); + updateDataverseResponse.prettyPrint(); + updateDataverseResponse.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.effectiveDatasetFileCountLimit", equalTo(500)) + .body("data.datasetFileCountLimit", equalTo(500)); String newAlias = UtilIT.getRandomDvAlias() + testAliasSuffix; String newName = "New Test Dataverse Name"; @@ -1370,10 +1389,10 @@ public void testUpdateDataverse() { String[] newMetadataBlockNames = new String[]{"citation", "geospatial", "biomedical"}; // Assert that the error is returned for having both MetadataBlockNames and inheritMetadataBlocksFromParent - Response updateDataverseResponse = UtilIT.updateDataverse( + updateDataverseResponse = UtilIT.updateDataverse( testDataverseAlias, newAlias, newName, newAffiliation, newDataverseType, newContactEmails, newInputLevelNames, null, newMetadataBlockNames, apiToken, - Boolean.TRUE, Boolean.TRUE + Boolean.TRUE, Boolean.TRUE, null ); updateDataverseResponse.then().assertThat() .statusCode(BAD_REQUEST.getStatusCode()) @@ -1383,7 +1402,7 @@ public void testUpdateDataverse() { updateDataverseResponse = UtilIT.updateDataverse( testDataverseAlias, newAlias, newName, newAffiliation, newDataverseType, newContactEmails, newInputLevelNames, newFacetIds, null, apiToken, - Boolean.TRUE, Boolean.TRUE + Boolean.TRUE, Boolean.TRUE, null ); updateDataverseResponse.then().assertThat() .statusCode(BAD_REQUEST.getStatusCode()) @@ -1495,7 +1514,7 @@ public void testUpdateDataverse() { null, null, apiToken, - Boolean.TRUE, Boolean.TRUE + Boolean.TRUE, Boolean.TRUE, null ); updateDataverseResponse.then().assertThat().statusCode(OK.getStatusCode()); @@ -1561,6 +1580,11 @@ public void testUpdateDataverse() { rootCollectionInfoResponse.then().assertThat() .statusCode(OK.getStatusCode()) .body("data.name", equalTo("Root")); + + + updateDataverseResponse = UtilIT.updateDataverse( + testDataverseAlias, newAlias, newName, newAffiliation, newDataverseType, newContactEmails, newInputLevelNames, + newFacetIds, newMetadataBlockNames, apiToken); } @Test diff --git a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java index 8b3b6476930..e34a916c3ba 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java @@ -1,6 +1,10 @@ package edu.harvard.iq.dataverse.api; +import edu.harvard.iq.dataverse.Dataverse; import edu.harvard.iq.dataverse.authorization.DataverseRole; +import edu.harvard.iq.dataverse.util.json.JsonParseException; +import edu.harvard.iq.dataverse.util.json.JsonParser; +import edu.harvard.iq.dataverse.util.json.JsonUtil; import io.restassured.RestAssured; import io.restassured.response.Response; @@ -8,6 +12,7 @@ import java.util.logging.Logger; import edu.harvard.iq.dataverse.api.auth.ApiKeyAuthMechanism; +import jakarta.json.JsonObject; import org.assertj.core.util.Lists; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.BeforeAll; @@ -3201,4 +3206,101 @@ public void testFileCitationByVersion() throws IOException { } + @Test + public void testUploadFilesWithLimits() throws JsonParseException { + Response createUser = UtilIT.createRandomUser(); + assertEquals(200, createUser.getStatusCode()); + String username = UtilIT.getUsernameFromResponse(createUser); + String adminApiToken = UtilIT.getApiTokenFromResponse(createUser); + Response makeSuperUser = UtilIT.makeSuperUser(username); + assertEquals(200, makeSuperUser.getStatusCode()); + + createUser = UtilIT.createRandomUser(); + createUser.then().assertThat().statusCode(OK.getStatusCode()); + String apiToken = UtilIT.getApiTokenFromResponse(createUser); + + Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken); + createDataverseResponse.prettyPrint(); + String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse); + // Update the dataverse with a datasetFileCountLimit of 1 + JsonObject data = JsonUtil.getJsonObject(createDataverseResponse.getBody().asString()); + JsonParser parser = new JsonParser(); + Dataverse dv = parser.parseDataverse(data.getJsonObject("data")); + dv.setDatasetFileCountLimit(1); + Response updateDataverseResponse = UtilIT.updateDataverse(dataverseAlias, dv, apiToken); + updateDataverseResponse.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.effectiveDatasetFileCountLimit", equalTo(1)) + .body("data.datasetFileCountLimit", equalTo(1)); + + Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken); + createDatasetResponse.prettyPrint(); + Integer datasetId = JsonPath.from(createDatasetResponse.body().asString()).getInt("data.id"); + String datasetPersistenceId = JsonPath.from(createDatasetResponse.body().asString()).getString("data.persistentId"); + createDatasetResponse.then().assertThat() + .statusCode(CREATED.getStatusCode()); + + // ------------------------- + // Add initial file + // ------------------------- + String pathToFile = "scripts/search/data/tabular/50by1000.dta"; + Response uploadFileResponse = UtilIT.uploadFileViaNative(datasetId.toString(), pathToFile, apiToken); + uploadFileResponse.prettyPrint(); + uploadFileResponse.then().assertThat() + .statusCode(OK.getStatusCode()); + String fileId = String.valueOf(JsonPath.from(uploadFileResponse.body().asString()).getInt("data.files[0].dataFile.id")); + UtilIT.sleepForLock(datasetId, null, apiToken, UtilIT.MAXIMUM_INGEST_LOCK_DURATION); + + // upload a second file should fail since the limit is 1 file per dataset + pathToFile = "scripts/search/data/tabular/open-source-at-harvard118.dta"; + uploadFileResponse = UtilIT.uploadFileViaNative(datasetId.toString(), pathToFile, apiToken); + uploadFileResponse.prettyPrint(); + uploadFileResponse.then().assertThat() + .body("message", containsString(BundleUtil.getStringFromBundle("file.add.count_exceeds_limit", Collections.singletonList("1")))) + .statusCode(BAD_REQUEST.getStatusCode()); + + // Add 1 to file limit and upload a second file + dv.setDatasetFileCountLimit(2); + updateDataverseResponse = UtilIT.updateDataverse(dataverseAlias, dv, apiToken);updateDataverseResponse.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.effectiveDatasetFileCountLimit", equalTo(2)) + .body("data.datasetFileCountLimit", equalTo(2)); + uploadFileResponse = UtilIT.uploadFileViaNative(datasetId.toString(), pathToFile, apiToken); + uploadFileResponse.prettyPrint(); + uploadFileResponse.then().assertThat() + .statusCode(OK.getStatusCode()); + UtilIT.sleepForLock(datasetId, null, apiToken, UtilIT.MAXIMUM_INGEST_LOCK_DURATION); + + // Set limit back to 1 even though the number of files is 2 + dv.setDatasetFileCountLimit(1); + updateDataverseResponse = UtilIT.updateDataverse(dataverseAlias, dv, apiToken);updateDataverseResponse.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.effectiveDatasetFileCountLimit", equalTo(1)) + .body("data.datasetFileCountLimit", equalTo(1)); + + Response getDatasetResponse = UtilIT.getDatasetVersion(datasetPersistenceId, DS_VERSION_DRAFT, apiToken); + getDatasetResponse.prettyPrint(); + getDatasetResponse.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.effectiveDatasetFileCountLimit", equalTo(1)) + .body("data.datasetFileUploadsAvailable", equalTo(0)); + + // Replace a file should be allowed + pathToFile = "scripts/search/data/tabular/120745.dta"; + Response replaceFileResponse = UtilIT.replaceFile(fileId, pathToFile, apiToken); + replaceFileResponse.prettyPrint(); + replaceFileResponse.then().assertThat() + .statusCode(OK.getStatusCode()); + + // Superuser file uploads can exceed the limit! + pathToFile = "scripts/search/data/tabular/stata13-auto.dta"; + uploadFileResponse = UtilIT.uploadFileViaNative(datasetId.toString(), pathToFile, apiToken); + uploadFileResponse.prettyPrint(); + uploadFileResponse.then().assertThat() + .statusCode(BAD_REQUEST.getStatusCode()); + uploadFileResponse = UtilIT.uploadFileViaNative(datasetId.toString(), pathToFile, adminApiToken); + uploadFileResponse.prettyPrint(); + uploadFileResponse.then().assertThat() + .statusCode(OK.getStatusCode()); + } } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/S3AccessIT.java b/src/test/java/edu/harvard/iq/dataverse/api/S3AccessIT.java index c57e50e4acb..28346dfede1 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/S3AccessIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/S3AccessIT.java @@ -7,14 +7,16 @@ import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3ClientBuilder; import com.amazonaws.services.s3.model.AmazonS3Exception; -import com.amazonaws.services.s3.model.Bucket; import com.amazonaws.services.s3.model.HeadBucketRequest; +import edu.harvard.iq.dataverse.Dataverse; +import edu.harvard.iq.dataverse.util.BundleUtil; +import edu.harvard.iq.dataverse.util.json.JsonParseException; +import edu.harvard.iq.dataverse.util.json.JsonParser; +import edu.harvard.iq.dataverse.util.json.JsonUtil; import io.restassured.RestAssured; -import static io.restassured.RestAssured.given; import io.restassured.http.Header; import io.restassured.path.json.JsonPath; import io.restassured.response.Response; -import io.restassured.specification.RequestSpecification; import static jakarta.ws.rs.core.Response.Status.BAD_REQUEST; import static jakarta.ws.rs.core.Response.Status.OK; import java.io.ByteArrayInputStream; @@ -25,9 +27,13 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.Collections; import java.util.logging.Level; import java.util.logging.Logger; -import org.apache.commons.lang3.math.NumberUtils; + +import jakarta.json.JsonObject; + +import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.Matchers.startsWith; import org.junit.jupiter.api.Assertions; @@ -590,4 +596,97 @@ public void testDirectUploadDetectStataFile() { } + @Test + public void testDirectUploadWithFileCountLimit() throws JsonParseException { + String driverId = "localstack1"; + String driverLabel = "LocalStack"; + Response createSuperuser = UtilIT.createRandomUser(); + createSuperuser.then().assertThat().statusCode(200); + String superuserApiToken = UtilIT.getApiTokenFromResponse(createSuperuser); + String superusername = UtilIT.getUsernameFromResponse(createSuperuser); + UtilIT.makeSuperUser(superusername).then().assertThat().statusCode(200); + Response storageDrivers = UtilIT.listStorageDrivers(superuserApiToken); + storageDrivers.prettyPrint(); + // TODO where is "Local/local" coming from? + String drivers = """ +{ + "status": "OK", + "data": { + "LocalStack": "localstack1", + "MinIO": "minio1", + "Local": "local", + "Filesystem": "file1" + } +}"""; + + //create user who will make a dataverse/dataset + Response createUser = UtilIT.createRandomUser(); + createUser.then().assertThat().statusCode(200); + String username = UtilIT.getUsernameFromResponse(createUser); + String apiToken = UtilIT.getApiTokenFromResponse(createUser); + + Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken); + createDataverseResponse.prettyPrint(); + String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse); + // Update the dataverse with a datasetFileCountLimit of 1 + JsonObject data = JsonUtil.getJsonObject(createDataverseResponse.getBody().asString()); + JsonParser parser = new JsonParser(); + Dataverse dv = parser.parseDataverse(data.getJsonObject("data")); + dv.setDatasetFileCountLimit(1); + Response updateDataverseResponse = UtilIT.updateDataverse(dataverseAlias, dv, apiToken); + updateDataverseResponse.prettyPrint(); + updateDataverseResponse.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.effectiveDatasetFileCountLimit", equalTo(1)) + .body("data.datasetFileCountLimit", equalTo(1)); + + Response originalStorageDriver = UtilIT.getStorageDriver(dataverseAlias, superuserApiToken); + originalStorageDriver.prettyPrint(); + originalStorageDriver.then().assertThat() + .body("data.message", equalTo("undefined")) + .statusCode(200); + + Response setStorageDriverToS3 = UtilIT.setStorageDriver(dataverseAlias, driverLabel, superuserApiToken); + setStorageDriverToS3.prettyPrint(); + setStorageDriverToS3.then().assertThat() + .statusCode(200); + + Response updatedStorageDriver = UtilIT.getStorageDriver(dataverseAlias, superuserApiToken); + updatedStorageDriver.prettyPrint(); + updatedStorageDriver.then().assertThat() + .statusCode(200); + + Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken); + createDatasetResponse.prettyPrint(); + createDatasetResponse.then().assertThat().statusCode(201); + Integer datasetId = JsonPath.from(createDatasetResponse.body().asString()).getInt("data.id"); + String datasetPid = JsonPath.from(createDatasetResponse.body().asString()).getString("data.persistentId"); + + Response getDatasetMetadata = UtilIT.nativeGet(datasetId, apiToken); + getDatasetMetadata.prettyPrint(); + getDatasetMetadata.then().assertThat().statusCode(200); + + // ------------------------- + // Add initial file + // ------------------------- + String pathToFile = "scripts/search/data/tabular/50by1000.dta"; + Response uploadFileResponse = UtilIT.uploadFileViaNative(datasetId.toString(), pathToFile, apiToken); + uploadFileResponse.then().assertThat() + .statusCode(OK.getStatusCode()); + UtilIT.sleepForLock(datasetId, null, apiToken, UtilIT.MAXIMUM_INGEST_LOCK_DURATION); + + // Get upload Urls when limit has been reached + long size = 1000000000l; + Response getUploadUrls = UtilIT.getUploadUrls(datasetPid, size, apiToken); + getUploadUrls.prettyPrint(); + getUploadUrls.then().assertThat() + .body("message", containsString(BundleUtil.getStringFromBundle("file.add.count_exceeds_limit", Collections.singletonList("1")))) + .statusCode(BAD_REQUEST.getStatusCode()); + + // Get upload Urls as superuser when limit has been reached (superuser ignores limit) + getUploadUrls = UtilIT.getUploadUrls(datasetPid, size, superuserApiToken); + getUploadUrls.prettyPrint(); + getUploadUrls.then().assertThat() + .statusCode(OK.getStatusCode()); + } } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index 979a305af7a..d4328163cc4 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -1,5 +1,6 @@ package edu.harvard.iq.dataverse.api; +import edu.harvard.iq.dataverse.Dataverse; import edu.harvard.iq.dataverse.util.json.NullSafeJsonBuilder; import io.restassured.http.ContentType; import io.restassured.path.json.JsonPath; @@ -269,7 +270,7 @@ private static String getAuthenticatedUserAsJsonString(String persistentUserId, return userAsJson; } - private static String getEmailFromUserName(String username) { + protected static String getEmailFromUserName(String username) { return username + "@mailinator.com"; } @@ -437,7 +438,7 @@ static Response updateDataverse(String alias, String apiToken) { return updateDataverse(alias, newAlias, newName, newAffiliation, newDataverseType, newContactEmails, - newInputLevelNames, newFacetIds, newMetadataBlockNames, apiToken, null, null); + newInputLevelNames, newFacetIds, newMetadataBlockNames, apiToken, null, null, null); } static Response updateDataverse(String alias, @@ -451,7 +452,8 @@ static Response updateDataverse(String alias, String[] newMetadataBlockNames, String apiToken, Boolean inheritMetadataBlocksFromParent, - Boolean inheritFacetsFromParent) { + Boolean inheritFacetsFromParent, + Integer datasetFileCountLimit) { JsonArrayBuilder contactArrayBuilder = Json.createArrayBuilder(); for(String contactEmail : newContactEmails) { contactArrayBuilder.add(Json.createObjectBuilder().add("contactEmail", contactEmail)); @@ -462,17 +464,36 @@ static Response updateDataverse(String alias, .add("affiliation", newAffiliation) .add("dataverseContacts", contactArrayBuilder) .add("dataverseType", newDataverseType) - .add("affiliation", newAffiliation); + .add("affiliation", newAffiliation) + .add("datasetFileCountLimit", datasetFileCountLimit) + ; updateDataverseRequestJsonWithMetadataBlocksConfiguration(newInputLevelNames, newFacetIds, newMetadataBlockNames, inheritMetadataBlocksFromParent, inheritFacetsFromParent, jsonBuilder); JsonObject dvData = jsonBuilder.build(); + String jsonBody = dvData.toString(); return given() - .body(dvData.toString()).contentType(ContentType.JSON) + .body(jsonBody).contentType(ContentType.JSON) .when().put("/api/dataverses/" + alias + "?key=" + apiToken); } + static Response updateDataverse(String alias, Dataverse dv, String apiToken) { + return updateDataverse(alias, + dv.getAlias(), + dv.getName(), + dv.getAffiliation(), + null, + dv.getContactEmails().split(","), + null, + null, + null, + apiToken, + null, + null, + dv.isDatasetFileCountLimitSet(dv.getDatasetFileCountLimit()) ? dv.getDatasetFileCountLimit() : -1); + } + private static void updateDataverseRequestJsonWithMetadataBlocksConfiguration(String[] inputLevelNames, String[] facetIds, String[] metadataBlockNames, @@ -601,7 +622,7 @@ static Response createRandomDatasetViaNativeApi(String dataverseAlias, String ap private static String getDatasetJson() { return getDatasetJson(false); } - + private static String getDatasetJson(boolean nolicense) { File datasetVersionJson; if (nolicense) { @@ -750,8 +771,20 @@ static Response addDatasetMetadataViaNative(String persistentId, String pathToJs .put("/api/datasets/:persistentId/editMetadata/?persistentId=" + persistentId); return response; } - - + + static Response updateDatasetFilesLimits(String persistentId, int limit, String apiToken) { + Response response = given() + .header(API_TOKEN_HTTP_HEADER, apiToken) + .post("/api/datasets/:persistentId/files/uploadlimit/" + limit + "?persistentId=" + persistentId); + return response; + } + static Response deleteDatasetFilesLimits(String persistentId, String apiToken) { + Response response = given() + .header(API_TOKEN_HTTP_HEADER, apiToken) + .delete("/api/datasets/:persistentId/files/uploadlimit?persistentId=" + persistentId); + return response; + } + static Response deleteDatasetMetadataViaNative(String persistentId, String pathToJsonFile, String apiToken) { String jsonIn = getDatasetJson(pathToJsonFile); @@ -2858,6 +2891,14 @@ static Response getUploadUrls(String idOrPersistentIdOfDataset, long sizeInBytes return requestSpecification.get("/api/datasets/" + idInPath + "/uploadurls?size=" + sizeInBytes + optionalQueryParam); } + static Response addFiles(String idInPath, String jsonData, String apiToken) { + RequestSpecification requestSpecification = given() + .header(API_TOKEN_HTTP_HEADER, apiToken) + .contentType(ContentType.MULTIPART) + .multiPart("jsonData", jsonData); + return requestSpecification.post("/api/datasets/" + idInPath + "/addFiles"); + } + /** * If you set dataverse.files.localstack1.disable-tagging=true you will see * an error like below. diff --git a/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesTest.java b/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesTest.java index f49ebcea39c..f4220b44183 100644 --- a/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesTest.java @@ -2,6 +2,7 @@ import edu.harvard.iq.dataverse.DataFile; import edu.harvard.iq.dataverse.Dataset; +import edu.harvard.iq.dataverse.DatasetServiceBean; import edu.harvard.iq.dataverse.DatasetVersion; import edu.harvard.iq.dataverse.engine.command.CommandContext; import edu.harvard.iq.dataverse.engine.command.DataverseRequest; @@ -18,8 +19,14 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; import org.mockito.MockedStatic; import org.mockito.Mockito; +import org.mockito.junit.jupiter.MockitoExtension; +import org.mockito.junit.jupiter.MockitoSettings; +import org.mockito.quality.Strictness; + import java.io.FileInputStream; import java.io.FileNotFoundException; @@ -43,11 +50,24 @@ @LocalJvmSettings +@ExtendWith(MockitoExtension.class) +@MockitoSettings(strictness = Strictness.LENIENT) public class CreateNewDataFilesTest { // TODO keep constants for annotations in sync with class name Path testDir = Path.of("target/test/").resolve(getClass().getSimpleName()); PrintStream original_stderr; + @Mock + Dataset mockDataset; + @Mock + DatasetVersion mockDatasetVersion; + + @BeforeEach + public void setupMock() { + Mockito.when(mockDataset.getId()).thenReturn(2L); + Mockito.when(mockDataset.getEffectiveDatasetFileCountLimit()).thenReturn(1000); + Mockito.when(mockDatasetVersion.getDataset()).thenReturn(mockDataset); + } @BeforeEach public void cleanTmpDir() throws IOException { original_stderr = System.err; @@ -63,8 +83,8 @@ public void cleanTmpDir() throws IOException { @JvmSetting(key = JvmSettings.FILES_DIRECTORY, value = "target/test/CreateNewDataFilesTest/tmp") public void execute_fails_to_upload_when_tmp_does_not_exist() throws FileNotFoundException { - mockTmpLookup(); - var cmd = createCmd("scripts/search/data/shape/shapefile.zip", mockDatasetVersion(), 1000L, 500L); + var cmd = createCmd("scripts/search/data/shape/shapefile.zip", mockDatasetVersion, 1000L, 500L); + cmd.setDatasetService(mockDatasetServiceBean()); var ctxt = mockCommandContext(mockSysConfig(true, 0L, MD5, 10)); assertThatThrownBy(() -> cmd.execute(ctxt)) @@ -80,8 +100,8 @@ public void execute_fails_to_upload_when_tmp_does_not_exist() throws FileNotFoun public void execute_fails_on_size_limit() throws Exception { createDirectories(Path.of("target/test/CreateNewDataFilesTest/tmp/temp")); - mockTmpLookup(); - var cmd = createCmd("scripts/search/data/binary/3files.zip", mockDatasetVersion(), 1000L, 500L); + var cmd = createCmd("scripts/search/data/binary/3files.zip", mockDatasetVersion, 1000L, 500L); + cmd.setDatasetService(mockDatasetServiceBean()); var ctxt = mockCommandContext(mockSysConfig(true, 50L, MD5, 0)); try (var mockedStatic = Mockito.mockStatic(JhoveFileType.class)) { mockedStatic.when(JhoveFileType::getJhoveConfigFile).thenReturn("conf/jhove/jhove.conf"); @@ -98,8 +118,8 @@ public void execute_loads_individual_files_from_uploaded_zip() throws Exception var tempDir = testDir.resolve("tmp/temp"); createDirectories(tempDir); - mockTmpLookup(); - var cmd = createCmd("src/test/resources/own-cloud-downloads/greetings.zip", mockDatasetVersion(), 1000L, 500L); + var cmd = createCmd("src/test/resources/own-cloud-downloads/greetings.zip", mockDatasetVersion, 1000L, 500L); + cmd.setDatasetService(mockDatasetServiceBean()); var ctxt = mockCommandContext(mockSysConfig(false, 1000000L, MD5, 10)); try (MockedStatic mockedStatic = Mockito.mockStatic(JhoveFileType.class)) { mockedStatic.when(JhoveFileType::getJhoveConfigFile).thenReturn("conf/jhove/jhove.conf"); @@ -125,8 +145,8 @@ public void execute_rezips_sets_of_shape_files_from_uploaded_zip() throws Except var tempDir = testDir.resolve("tmp/temp"); createDirectories(tempDir); - mockTmpLookup(); - var cmd = createCmd("src/test/resources/own-cloud-downloads/shapes.zip", mockDatasetVersion(), 1000L, 500L); + var cmd = createCmd("src/test/resources/own-cloud-downloads/shapes.zip", mockDatasetVersion, 1000L, 500L); + cmd.setDatasetService(mockDatasetServiceBean()); var ctxt = mockCommandContext(mockSysConfig(false, 100000000L, MD5, 10)); try (var mockedJHoveFileType = Mockito.mockStatic(JhoveFileType.class)) { mockedJHoveFileType.when(JhoveFileType::getJhoveConfigFile).thenReturn("conf/jhove/jhove.conf"); @@ -205,7 +225,9 @@ public void extract_zip_performance() throws Exception { // upload the zip var before = DateTime.now(); - var result = createCmd(zip.toString(), mockDatasetVersion(), 1000L, 500L) + @NotNull CreateNewDataFilesCommand cmd = createCmd(zip.toString(), mockDatasetVersion, 1000L, 500L); + cmd.setDatasetService(mockDatasetServiceBean()); + var result = cmd .execute(ctxt); totalTime += DateTime.now().getMillis() - before.getMillis(); @@ -250,15 +272,9 @@ public void extract_zip_performance() throws Exception { return sysCfg; } - private static void mockTmpLookup() { - JvmSettings mockFilesDirectory = Mockito.mock(JvmSettings.class); - Mockito.when(mockFilesDirectory.lookup()).thenReturn("/mocked/path"); + private static @NotNull DatasetServiceBean mockDatasetServiceBean() { + var datasetService = Mockito.mock(DatasetServiceBean.class); + Mockito.when(datasetService.getDataFileCountByOwner(2L)).thenReturn(0); + return datasetService; } - - private static @NotNull DatasetVersion mockDatasetVersion() { - var dsVersion = Mockito.mock(DatasetVersion.class); - Mockito.when(dsVersion.getDataset()).thenReturn(Mockito.mock(Dataset.class)); - return dsVersion; - } - } diff --git a/src/test/java/edu/harvard/iq/dataverse/util/json/JsonPrinterTest.java b/src/test/java/edu/harvard/iq/dataverse/util/json/JsonPrinterTest.java index 9d37a554820..5d7cde86fff 100644 --- a/src/test/java/edu/harvard/iq/dataverse/util/json/JsonPrinterTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/util/json/JsonPrinterTest.java @@ -208,7 +208,8 @@ public void testDatasetContactOutOfBoxNoPrivacy() { SettingsServiceBean nullServiceBean = null; DatasetFieldServiceBean nullDFServiceBean = null; DataverseFieldTypeInputLevelServiceBean nullDFILServiceBean = null; - JsonPrinter.injectSettingsService(nullServiceBean, nullDFServiceBean, nullDFILServiceBean); + DatasetServiceBean nullDatasetServiceBean = null; + JsonPrinter.injectSettingsService(nullServiceBean, nullDFServiceBean, nullDFILServiceBean, nullDatasetServiceBean); JsonObject jsonObject = JsonPrinter.json(block, fields).build(); assertNotNull(jsonObject); @@ -251,7 +252,8 @@ public void testDatasetContactWithPrivacy() { DatasetFieldServiceBean nullDFServiceBean = null; DataverseFieldTypeInputLevelServiceBean nullDFILServiceBean = null; - JsonPrinter.injectSettingsService(new MockSettingsSvc(), nullDFServiceBean, nullDFILServiceBean); + DatasetServiceBean nullDatasetServiceBean = null; + JsonPrinter.injectSettingsService(new MockSettingsSvc(), nullDFServiceBean, nullDFILServiceBean, nullDatasetServiceBean); JsonObject jsonObject = JsonPrinter.json(block, fields).build(); assertNotNull(jsonObject); @@ -303,7 +305,8 @@ public void testDatasetFieldTypesWithChildren() { DatasetFieldServiceBean nullDFServiceBean = null; DataverseFieldTypeInputLevelServiceBean nullDFILServiceBean = null; - JsonPrinter.injectSettingsService(new MockSettingsSvc(), nullDFServiceBean, nullDFILServiceBean); + DatasetServiceBean nullDatasetServiceBean = null; + JsonPrinter.injectSettingsService(new MockSettingsSvc(), nullDFServiceBean, nullDFILServiceBean, nullDatasetServiceBean); JsonObject jsonObject = JsonPrinter.json(block).build(); assertNotNull(jsonObject);