Merge branch 'develop' into 11752-croissant-restricted

Author: ofahimIQSS

.github/workflows/spi_release.yml

@@ -42,7 +42,7 @@ jobs:
               with:
                   java-version: '17'
                   distribution: 'adopt'
-                  server-id: ossrh
+                  server-id: central
                   server-username: MAVEN_USERNAME
                   server-password: MAVEN_PASSWORD
             - uses: actions/cache@v4
@@ -80,7 +80,7 @@ jobs:
                 with:
                     java-version: '17'
                     distribution: 'adopt'
-                    server-id: ossrh
+                    server-id: central
                     server-username: MAVEN_USERNAME
                     server-password: MAVEN_PASSWORD
                     gpg-private-key: ${{ secrets.DATAVERSEBOT_GPG_KEY }}

conf/mdc/counter_weekly.sh

@@ -0,0 +1,92 @@
+#!/bin/sh
+#counter_weekly.sh
+
+# This script iterates through all published Datasets in all Dataverses and calls the Make Data Count API to update their citations from DataCite
+# Note: Requires curl and jq for parsing JSON responses form curl
+
+# A recursive method to process each Dataverse
+processDV () {
+echo "Processing Dataverse ID#: $1"
+
+#Call the Dataverse API to get the contents of the Dataverse (without credentials, this will only list published datasets and dataverses
+DVCONTENTS=$(curl -s http://localhost:8080/api/dataverses/$1/contents)
+
+# Iterate over all datasets, pulling the value of their DOIs (as part of the persistentUrl) from the json returned
+for subds in $(echo "${DVCONTENTS}" | jq -r '.data[] | select(.type == "dataset") | .persistentUrl'); do
+
+#The authority/identifier are preceded by a protocol/host, i.e. https://doi.org/
+DOI=`expr "$subds" : '.*:\/\/\doi\.org\/\(.*\)'`
+
+# Call the Dataverse API for this dataset and capture both the response and HTTP status code
+HTTP_RESPONSE=$(curl -s -w "\n%{http_code}" -X POST "http://localhost:8080/api/admin/makeDataCount/:persistentId/updateCitationsForDataset?persistentId=doi:$DOI")
+
+# Extract the HTTP status code from the last line
+HTTP_STATUS=$(echo "$HTTP_RESPONSE" | tail -n1)
+# Extract the response body (everything except the last line)
+RESPONSE_BODY=$(echo "$HTTP_RESPONSE" | sed '$d')
+
+# Check the HTTP status code and report accordingly
+case $HTTP_STATUS in
+    200)
+        # Successfully queued
+        # Extract status from the nested data object
+        STATUS=$(echo "$RESPONSE_BODY" | jq -r '.data.status')
+        
+        # Extract message from the nested data object
+        if echo "$RESPONSE_BODY" | jq -e '.data.message' > /dev/null 2>&1 && [ "$(echo "$RESPONSE_BODY" | jq -r '.data.message')" != "null" ]; then
+            MESSAGE=$(echo "$RESPONSE_BODY" | jq -r '.data.message')
+            echo "[SUCCESS] doi:$DOI - $STATUS: $MESSAGE"
+        else
+            # If message is missing or null, just show the status
+            echo "[SUCCESS] doi:$DOI - $STATUS: Citation update queued"
+        fi
+        ;;
+    400)
+        # Bad request
+        if echo "$RESPONSE_BODY" | jq -e '.message' > /dev/null 2>&1; then
+            ERROR=$(echo "$RESPONSE_BODY" | jq -r '.message')
+            echo "[ERROR 400] doi:$DOI - Bad request: $ERROR"
+        else
+            echo "[ERROR 400] doi:$DOI - Bad request"
+        fi
+        ;;
+    404)
+        # Not found
+        if echo "$RESPONSE_BODY" | jq -e '.message' > /dev/null 2>&1; then
+            ERROR=$(echo "$RESPONSE_BODY" | jq -r '.message')
+            echo "[ERROR 404] doi:$DOI - Not found: $ERROR"
+        else
+            echo "[ERROR 404] doi:$DOI - Not found"
+        fi
+        ;;
+    503)
+        # Service unavailable (queue full)
+        if echo "$RESPONSE_BODY" | jq -e '.message' > /dev/null 2>&1; then
+            ERROR=$(echo "$RESPONSE_BODY" | jq -r '.message')
+            echo "[ERROR 503] doi:$DOI - Service unavailable: $ERROR"
+        elif echo "$RESPONSE_BODY" | jq -e '.data.message' > /dev/null 2>&1; then
+            ERROR=$(echo "$RESPONSE_BODY" | jq -r '.data.message')
+            echo "[ERROR 503] doi:$DOI - Service unavailable: $ERROR"
+        else
+            echo "[ERROR 503] doi:$DOI - Service unavailable: Queue is full"
+        fi
+        ;;
+    *)
+        # Other error
+        echo "[ERROR $HTTP_STATUS] doi:$DOI - Unexpected error"
+        echo "Response: $RESPONSE_BODY"
+        ;;
+esac
+
+done
+
+# Now iterate over any child Dataverses and recursively process them
+for subdv in $(echo "${DVCONTENTS}" | jq -r '.data[] | select(.type == "dataverse") | .id'); do
+echo $subdv
+processDV $subdv
+done
+
+}
+
+# Call the function on the root dataverse to start processing 
+processDV 1

doc/release-notes/11766-new-io.gdcc.dataverse-spi.md

@@ -0,0 +1,2 @@
+The ExportDataProvider framework in the dataverse-spi package has been extended, adding some extra options for developers of metadata exporter plugins. 
+See the [documentation](https://guides.dataverse.org/en/latest/developers/metadataexport.html#building-an-exporter) in the Metadata Export guide for details. 

doc/release-notes/11777-MDC-citation-api-improvement.md

@@ -0,0 +1,7 @@
+The /api/admin/makeDataCount/{id}/updateCitationsForDataset endpoint, which allows citations for a dataset to be retrieved from DataCite, is often called periodically for all datasets. However, allowing calls for many datasets to be processed in parallel can cause performance problems in Dataverse and/or cause calls to DataCite to fail due to rate limiting. The existing implementation was also inefficient w.r.t. memory use when used on datasets with many (>~1K) files. This release configures Dataverse to queue calls to this api, processes them serially, adds optional throttling to avoid hitting DataCite rate limits and improves memory use.
+
+New optional MPConfig setting: 
+ 
+dataverse.api.mdc.min-delay-ms - number of milliseconds to wait between calls to DataCite. A value of ~100 should conservatively address DataCite's current 3000/5 minute limit. A value of 250 may be required for their test service. 
+
+Backward compatibility: This api call is now asynchronous and will return an OK response when the call is queued or a 503 if the queue is full.

doc/sphinx-guides/source/admin/make-data-count.rst

@@ -166,6 +166,8 @@ The example :download:`counter_weekly.sh <../_static/util/counter_weekly.sh>` wi
 
 Citations will be retrieved for each published dataset and recorded in the your Dataverse installation's database.
 
+Note that the :ref:`dataverse.api.mdc.min-delay-ms` setting can be used to avoid getting rate-limit errors from DataCite.
+
 For how to get the citations out of your Dataverse installation, see "Retrieving Citations for a Dataset" under :ref:`Dataset Metrics <dataset-metrics-api>` in the :doc:`/api/native-api` section of the API Guide.
 
 Please note that while the Dataverse Software has a metadata field for "Related Dataset" this information is not currently sent as a citation to Crossref.

doc/sphinx-guides/source/api/changelog.rst

@@ -7,6 +7,10 @@ This API changelog is experimental and we would love feedback on its usefulness.
     :local:
     :depth: 1
 
+v6.9
+----
+- The POST /api/admin/makeDataCount/{id}/updateCitationsForDataset processing is now asynchronous and the response no longer includes the number of citations. The response can be OK if the request is queued or 503 if the queue is full (default queue size is 1000).
+
 v6.8
 ----
 

doc/sphinx-guides/source/developers/making-library-releases.rst

@@ -36,6 +36,32 @@ Releasing a Snapshot Version to Maven Central
 
 That is to say, to make a snapshot release, you only need to get one or more commits into the default branch.
 
+It's possible, of course, to make snapshot releases outside of GitHub Actions, from environments such as your laptop. Generally, you'll want to look at the GitHub Action and try to do the equivalent. You'll need a file set up locally at ``~/.m2/settings.xml`` with the following (contact a core developer for the redacted bits):
+
+.. code-block:: bash
+
+  <settings>
+    <servers>
+      <server>
+        <id>central</id>
+        <username>REDACTED</username>
+        <password>REDACTED</password>
+      </server>
+    </servers>
+  </settings>
+
+Then, study the GitHub Action and perform similar commands from your local environment. For example, as of this writing, for the dataverse-spi project, you can run the following commands, substituting the suffix you need:
+
+``mvn -f modules/dataverse-spi -Dproject.version.suffix="2.1.0-PR11767-SNAPSHOT" verify``
+
+``mvn -f modules/dataverse-spi -Dproject.version.suffix="2.1.0-PR11767-SNAPSHOT" deploy``
+
+This will upload the snapshot here, for example: https://central.sonatype.com/repository/maven-snapshots/io/gdcc/dataverse-spi/2.1.02.1.0-PR11767-SNAPSHOT/dataverse-spi-2.1.02.1.0-PR11767-20250827.182026-1.jar
+
+Before OSSRH was retired, you could browse through snapshot jars you published at https://s01.oss.sonatype.org/content/repositories/snapshots/io/gdcc/dataverse-spi/2.0.0-PR9685-SNAPSHOT/, for example. Now, even though you may see the URL of the jar as shown above during the "deploy" step, if you try to browse the various snapshot jars at https://central.sonatype.com/repository/maven-snapshots/io/gdcc/dataverse-spi/2.1.02.1.0-PR11767-SNAPSHOT/ you'll see "This maven2 hosted repository is not directly browseable at this URL. Please use the browse or HTML index views to inspect the contents of this repository." Sadly, the "browse" and "HTML index" links don't work, as noted in a `question <https://community.sonatype.com/t/this-maven2-group-repository-is-not-directly-browseable-at-this-url/8991>`_ on the Sonatype Community forum. Below is a suggestion for confirming that the jar was uploaded properly, which is to use Maven to copy the jar to your local directory. You could then compare checksums.
+
+``mvn dependency:copy -DrepoUrl=https://central.sonatype.com/repository/maven-snapshots/ -Dartifact=io.gdcc:dataverse-spi:2.1.02.1.0-PR11767-SNAPSHOT -DoutputDirectory=.``
+
 Releasing a Release (Non-Snapshot) Version to Maven Central
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

doc/sphinx-guides/source/installation/config.rst

@@ -3731,6 +3731,22 @@ Example:
 
 Can also be set via any `supported MicroProfile Config API source`_, e.g. the environment variable ``DATAVERSE_CORS_HEADERS_EXPOSE``.
 
+
+.. _dataverse.api.mdc.min-delay-ms:
+
+dataverse.api.mdc.min-delay-ms
+++++++++++++++++++++++++++++++
+
+Minimum delay in milliseconds between Make Data Count (MDC) API requests from the /api/admin/makeDataCount/{id}/updateCitationsForDataset api.
+This setting helps prevent overloading the MDC service by enforcing a minimum time interval between consecutive requests.
+If a request arrives before this interval has elapsed since the previous request, it will be rate-limited.
+
+Default: ``0`` (no delay enforced)
+
+Example: ``dataverse.api.mdc.min-delay-ms=100`` (enforces a minimum 100ms delay between MDC API requests)
+
+Can also be set via any `supported MicroProfile Config API source`_, e.g. the environment variable ``DATAVERSE_API_MDC_MIN_DELAY_MS``.
+
 .. _feature-flags:
 
 Feature Flags

modules/dataverse-spi/pom.xml

@@ -13,7 +13,7 @@
 
     <groupId>io.gdcc</groupId>
     <artifactId>dataverse-spi</artifactId>
-    <version>2.0.0${project.version.suffix}</version>
+    <version>2.1.0${project.version.suffix}</version>
     <packaging>jar</packaging>
 
     <name>Dataverse SPI Plugin API</name>
@@ -64,11 +64,13 @@
 
     <distributionManagement>
         <snapshotRepository>
-            <id>ossrh</id>
-            <url>https://s01.oss.sonatype.org/content/repositories/snapshots</url>
+            <id>central</id>
+            <url>https://central.sonatype.com/repository/maven-snapshots/</url>
         </snapshotRepository>
         <repository>
+            <!--TODO: change this from ossrh to central?-->
             <id>ossrh</id>
+            <!--TODO: change this url?-->
             <url>https://s01.oss.sonatype.org/service/local/staging/deploy/maven2/</url>
         </repository>
     </distributionManagement>
@@ -110,7 +112,9 @@
                 <artifactId>nexus-staging-maven-plugin</artifactId>
                 <extensions>true</extensions>
                 <configuration>
+                    <!--TODO: change this from ossrh to central?-->
                     <serverId>ossrh</serverId>
+                    <!--TODO: change this URL?-->
                     <nexusUrl>https://s01.oss.sonatype.org</nexusUrl>
                     <autoReleaseAfterClose>true</autoReleaseAfterClose>
                 </configuration>

modules/dataverse-spi/src/main/java/io/gdcc/spi/export/ExportDataContext.java

@@ -0,0 +1,61 @@
+package io.gdcc.spi.export;
+
+/**
+ *
+ * @author landreev
+ * Provides an optional mechanism for defining various data retrieval options 
+ * for the export subsystem in a way that should allow us adding support for 
+ * more options going forward with minimal or no changes to the already 
+ * implemented export plugins. 
+ */
+public class ExportDataContext {
+    private boolean datasetMetadataOnly = false; 
+    private boolean publicFilesOnly = false;
+    private Integer offset = null; 
+    private Integer length = null; 
+    
+    private ExportDataContext() {
+        
+    }
+    
+    public static ExportDataContext context() {
+        ExportDataContext context = new ExportDataContext();
+        return context; 
+    }
+    
+    public ExportDataContext withDatasetMetadataOnly() {
+        this.datasetMetadataOnly = true;
+        return this; 
+    }
+    
+    public ExportDataContext withPublicFilesOnly() {
+        this.publicFilesOnly = true;
+        return this; 
+    }
+    
+    public ExportDataContext withOffset(Integer offset) {
+        this.offset = offset; 
+        return this;
+    }
+    
+    public ExportDataContext withLength(Integer length) {
+        this.length = length; 
+        return this;
+    }
+    
+    public boolean isDatasetMetadataOnly() {
+        return datasetMetadataOnly;
+    }
+    
+    public boolean isPublicFilesOnly() {
+        return publicFilesOnly;
+    }
+    
+    public Integer getOffset() {
+        return offset; 
+    }
+    
+    public Integer getLength() {
+        return length; 
+    }    
+}