Merge branch 'develop' of github.com:IQSS/dataverse into 11157-builtin-users-oidc-auth

Author: GPortas

doc/release-notes/11041-datatype-for-vtt.md

@@ -0,0 +1,43 @@
+### video subtitles (vtt files)
+
+The `IQSS/dataverse` PR sets the content type for new(!) files with extension `vtt` to `text/vtt`
+what is presented as "_Web Video Text Tracks_". The PR also enables full text indexing for these files,
+if [configured](https://guides.dataverse.org/en/latest/installation/config.html#solrfulltextindexing).
+
+The `gdcc/dataverse-previewer` PRs provide a new version of the video previewer. 
+The new previewer version presents `vtt` files as subtitles for videos,
+the naming convention is `<video-basename>.<language-tag>.vtt`.
+The previewer does not rely on the content type.
+A proper content type may hint users to ask permission for the subtitles together with a video.
+
+Existing files with extension `vtt` will keep content type `application/octet-stream` presented as "_Unknown_".
+The following query shows the number of files per extension with an "_Unknown_" content type:
+
+      SELECT substring(m.label from (length(label) - strpos(reverse(m.label), '.') + 2)) AS extension, COUNT(*) as count
+      FROM datafile f LEFT JOIN filemetadata m ON f.id = m.datafile_id
+      WHERE f.contenttype = 'application/octet-stream'
+      GROUP BY extension;
+
+If `vtt` does not appear in the result, you are done. 
+Otherwise, you may want to update the content type for existing files and reindex those datasets.
+
+First figure out which datasets would need [reindexing](https://guides.dataverse.org/en/latest/admin/solr-search-index.html#manual-reindexing):
+
+      select distinct
+        o.protocol, o.authority, o.identifier,
+        v.versionnumber, v.minorversionnumber, v.versionstate
+        from      datafile       f
+        left join filemetadata   m on f.id = m.datafile_id
+        left join datasetversion v on v.id = m.datasetversion_id
+        left join dvobject       o on o.id = v.dataset_id
+        WHERE contenttype = 'application/octet-stream' 
+        AND 'vtt' = substring(m.label from (length(label) - strpos(reverse(m.label), '.') + 2))
+        ;
+
+Then update the content type for the files:
+
+      UPDATE datafile SET contenttype = 'text/vtt' WHERE id IN (
+        SELECT datafile_id FROM filemetadata m
+        WHERE contenttype = 'application/octet-stream' 
+        AND 'vtt' = substring(m.label from (length(label) - strpos(reverse(m.label), '.') + 2))
+      );

doc/release-notes/11147-string-field-type.md

@@ -0,0 +1,5 @@
+The "string" type has been added as a new field type for metadata fields.
+
+In contrast to "text" fields, "string" fields are stored and indexed exactly as provided, without any text analysis or transformations.
+
+This field type is suitable for fields like IDs (e.g. ORCIDs) or enums, where exact matches are required when searching.

doc/release-notes/11287-show_my_data.md

@@ -0,0 +1,3 @@
+### Backward Incompatibilities
+
+An undocumented Search API parameter called "show_my_data" has been removed. It was never exercised by tests and is believed to be unused. API users should use the [MyData] API instead. See the [API changelog](https://dataverse-guide--11375.org.readthedocs.build/en/11375/api/changelog.html), #11287 and #11375.

doc/release-notes/11292-update-tabular-tags-with-replace.md

@@ -0,0 +1,5 @@
+### Tabular Tags can now be replaced
+
+Previously the API POST /files/{id}/metadata/tabularTags could only add new tags to the tabular tags list. Now with the query parameter ?replace=true the list of tags will be replaced.
+
+See also [the guides](https://dataverse-guide--11359.org.readthedocs.build/en/11359/api/native-api.html#updating-file-tabular-tags), #11292, and #11359.

doc/sphinx-guides/source/admin/metadatacustomization.rst

@@ -144,6 +144,7 @@ Each of the three main sections own sets of properties:
 |                           |                                                        |                                                          | \• email              |
 |                           |                                                        |                                                          | \• text               |
 |                           |                                                        |                                                          | \• textbox            |
+|                           |                                                        |                                                          | \• string             |
 |                           |                                                        |                                                          | \• url                |
 |                           |                                                        |                                                          | \• int                |
 |                           |                                                        |                                                          | \• float              |
@@ -315,6 +316,12 @@ FieldType definitions
 |               | section of the Dataset + File      |
 |               | Management page in the User Guide. |
 +---------------+------------------------------------+
+| string        | Any text may be entered into this  |
+|               | field. The value is stored and     |
+|               | indexed exactly as provided,       |
+|               | without any text analysis or       |
+|               | transformations.                   |
++---------------+------------------------------------+
 | url           | If not empty, field must contain   |
 |               | a valid URL.                       |
 +---------------+------------------------------------+

doc/sphinx-guides/source/api/changelog.rst

@@ -7,6 +7,11 @@ This API changelog is experimental and we would love feedback on its usefulness.
     :local:
     :depth: 1
 
+v6.7
+----
+
+- An undocumented :doc:`search` parameter called "show_my_data" has been removed. It was never exercised by tests and is believed to be unused. API users should use the :ref:`api-mydata` API instead.
+
 v6.6
 ----
 

doc/sphinx-guides/source/api/native-api.rst

@@ -4669,6 +4669,8 @@ Updating File Tabular Tags
 
 Updates the tabular tags for an existing tabular file where ``ID`` is the database id of the file to update or ``PERSISTENT_ID`` is the persistent id (DOI or Handle) of the file. Requires a ``jsonString`` expressing the tabular tag names.
 
+The list of "tabularTags" will be added to the existing list unless the optional ``replace=true`` query parameter is included. The inclusion of this parameter will cause the pre-existing tags to be deleted and the "tabularTags" to be added. Sending an empty list will remove all of the pre-existing tags.
+
 The JSON representation of tabular tags (``tags.json``) looks like this::
 
   {
@@ -4698,6 +4700,9 @@ The fully expanded example above (without environment variables) looks like this
   curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST \
     "http://demo.dataverse.org/api/files/24/metadata/tabularTags" \
     -H "Content-type:application/json" --upload-file tags.json
+  curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST \
+    "http://demo.dataverse.org/api/files/24/metadata/tabularTags?replace=true" \
+    -H "Content-type:application/json" --upload-file tags.json
 
 A curl example using a ``PERSISTENT_ID``
 
@@ -4711,6 +4716,9 @@ A curl example using a ``PERSISTENT_ID``
   curl -H "X-Dataverse-key:$API_TOKEN" -X POST \
     "$SERVER_URL/api/files/:persistentId/metadata/tabularTags?persistentId=$PERSISTENT_ID" \
     -H "Content-type:application/json" --upload-file $FILE_PATH
+  curl -H "X-Dataverse-key:$API_TOKEN" -X POST \
+    "$SERVER_URL/api/files/:persistentId/metadata/tabularTags?persistentId=$PERSISTENT_ID&replace=true" \
+    -H "Content-type:application/json" --upload-file $FILE_PATH
 
 The fully expanded example above (without environment variables) looks like this:
 
@@ -4719,6 +4727,9 @@ The fully expanded example above (without environment variables) looks like this
   curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST \
     "https://demo.dataverse.org/api/files/:persistentId/metadata/tabularTags?persistentId=doi:10.5072/FK2/AAA000" \
     -H "Content-type:application/json" --upload-file tags.json
+  curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST \
+    "https://demo.dataverse.org/api/files/:persistentId/metadata/tabularTags?persistentId=doi:10.5072/FK2/AAA000&replace=true" \
+    -H "Content-type:application/json" --upload-file tags.json
 
 Note that the specified tabular tags must be valid. The supported tags are:
 
@@ -7423,6 +7434,8 @@ As a superuser::
 
 Note that this API is probably only useful for testing.
 
+.. _api-mydata:
+
 MyData
 ------
 

src/main/java/edu/harvard/iq/dataverse/DatasetFieldType.java

@@ -36,8 +36,8 @@ public class DatasetFieldType implements Serializable, Comparable<DatasetFieldTy
      * The set of possible metatypes of the field. Used for validation and layout.
      */
     public enum FieldType {
-        TEXT, TEXTBOX, DATE, EMAIL, URL, FLOAT, INT, NONE
-    };    
+        TEXT, TEXTBOX, STRING, DATE, EMAIL, URL, FLOAT, INT, NONE
+    };
 
     @Id
     @GeneratedValue(strategy = GenerationType.IDENTITY)
@@ -558,6 +558,8 @@ public SolrField getSolrField() {
                 solrType = SolrField.SolrType.INTEGER;
             } else if (fieldType.equals(FieldType.FLOAT)) {
                 solrType = SolrField.SolrType.FLOAT;
+            } else if (fieldType.equals(FieldType.STRING)) {
+                solrType = SolrField.SolrType.STRING;
             }
 
             Boolean anyParentAllowsMultiplesBoolean = false;

src/main/java/edu/harvard/iq/dataverse/api/Files.java

@@ -1,5 +1,6 @@
 package edu.harvard.iq.dataverse.api;
 
+import com.google.api.client.util.Lists;
 import com.google.gson.Gson;
 import com.google.gson.JsonObject;
 import edu.harvard.iq.dataverse.*;
@@ -947,7 +948,7 @@ public Response setFileCategories(@Context ContainerRequestContext crc, @PathPar
     @AuthRequired
     @Path("{id}/metadata/tabularTags")
     @Produces(MediaType.APPLICATION_JSON)
-    public Response setFileTabularTags(@Context ContainerRequestContext crc, @PathParam("id") String dataFileId, String jsonBody) {
+    public Response setFileTabularTags(@Context ContainerRequestContext crc, @PathParam("id") String dataFileId, String jsonBody, @QueryParam("replace") boolean replaceData) {
         return response(req -> {
             DataFile dataFile = execCommand(new GetDataFileCommand(req, findDataFileOrDie(dataFileId)));
             if (!dataFile.isTabularData()) {
@@ -957,6 +958,9 @@ public Response setFileTabularTags(@Context ContainerRequestContext crc, @PathPa
             try (StringReader stringReader = new StringReader(jsonBody)) {
                 jsonObject = Json.createReader(stringReader).readObject();
                 JsonArray requestedTabularTagsJson = jsonObject.getJsonArray("tabularTags");
+                if (replaceData) {
+                    dataFile.setTags(Lists.newArrayList());
+                }
                 for (JsonValue jsonValue : requestedTabularTagsJson) {
                     JsonString jsonString = (JsonString) jsonValue;
                     try {

src/main/java/edu/harvard/iq/dataverse/api/Search.java

@@ -66,7 +66,6 @@ public Response search(
             @QueryParam("fq") final List<String> filterQueries,
             @QueryParam("show_entity_ids") boolean showEntityIds,
             @QueryParam("show_api_urls") boolean showApiUrls,
-            @QueryParam("show_my_data") boolean showMyData,
             @QueryParam("query_entities") boolean queryEntities,
             @QueryParam("metadata_fields") List<String> metadataFields,
             @QueryParam("geo_point") String geoPointRequested,
@@ -97,8 +96,8 @@ public Response search(
             objectTypeCountsMap.put(SearchConstants.UI_DATASETS, 0L);
             objectTypeCountsMap.put(SearchConstants.UI_FILES, 0L);
 
-            // users can't change these (yet anyway)
-            boolean dataRelatedToMe = showMyData; //getDataRelatedToMe();
+            // hard-coded to false since dataRelatedToMe is only used by MyData (DataRetrieverAPI)
+            boolean dataRelatedToMe = false;
 
             try {
                 // we have to add "" (root) otherwise there is no permissions check
@@ -290,15 +289,6 @@ public boolean tokenLessSearchAllowed() {
         return tokenLessSearchAllowed;
     }
 
-    private boolean getDataRelatedToMe() {
-        /**
-         * @todo support Data Related To Me:
-         * https://github.com/IQSS/dataverse/issues/1299
-         */
-        boolean dataRelatedToMe = false;
-        return dataRelatedToMe;
-    }
-
     private int getNumberOfResultsPerPage(int numResultsPerPage) {
         /**
          * @todo should maxLimit be configurable?