Merge pull request #11622 from IQSS/11605-existing-external-users-api-auth

ofahimIQSS · web-flow · commit 20aaf5a780eb · 2025-07-28T11:20:47.000-04:00
OIDC-Brokered Shibboleth users API auth
diff --git a/conf/keycloak/builtin-users-spi/src/main/java/edu/harvard/iq/keycloak/auth/spi/services/DataverseUserService.java b/conf/keycloak/builtin-users-spi/src/main/java/edu/harvard/iq/keycloak/auth/spi/services/DataverseUserService.java
@@ -27,11 +27,16 @@ public DataverseUser getUserById(String id) {
 
         DataverseBuiltinUser builtinUser = em.find(DataverseBuiltinUser.class, persistenceId);
         if (builtinUser == null) {
-            logger.debugf("User not found for external ID: %s", persistenceId);
+            logger.debugf("Builtin user not found for external ID: %s", persistenceId);
             return null;
         }
 
-        DataverseAuthenticatedUser authenticatedUser = getAuthenticatedUserByUsername(builtinUser.getUsername());
+        String username = builtinUser.getUsername();
+        DataverseAuthenticatedUser authenticatedUser = getAuthenticatedUserByUsername(username);
+        if (authenticatedUser == null) {
+            logger.debugf("Authenticated user not found by username: %s", username);
+            return null;
+        }
 
         return new DataverseUser(authenticatedUser, builtinUser);
     }
@@ -43,11 +48,15 @@ public DataverseUser getUserByUsername(String username) {
                 .getResultList();
 
         if (users.isEmpty()) {
-            logger.debugf("User not found by username: %s", username);
+            logger.debugf("Builtin user not found by username: %s", username);
             return null;
         }
 
         DataverseAuthenticatedUser authenticatedUser = getAuthenticatedUserByUsername(username);
+        if (authenticatedUser == null) {
+            logger.debugf("Authenticated user not found by username: %s", username);
+            return null;
+        }
 
         return new DataverseUser(authenticatedUser, users.get(0));
     }
@@ -59,7 +68,7 @@ public DataverseUser getUserByEmail(String email) {
                 .getResultList();
 
         if (authUsers.isEmpty()) {
-            logger.debugf("User not found by email: %s", email);
+            logger.debugf("Authenticated user not found by email: %s", email);
             return null;
         }
 
@@ -68,6 +77,11 @@ public DataverseUser getUserByEmail(String email) {
                 .setParameter("username", username)
                 .getResultList();
 
+        if (builtinUsers.isEmpty()) {
+            logger.debugf("Builtin user not found by username: %s", username);
+            return null;
+        }
+
         return new DataverseUser(authUsers.get(0), builtinUsers.get(0));
     }
 
diff --git a/doc/release-notes/11605-existing-shib-external-users-auth.md b/doc/release-notes/11605-existing-shib-external-users-auth.md
@@ -0,0 +1 @@
+Implemented a new feature flag ``dataverse.feature.api-bearer-auth-use-shib-user-on-id-match``, which supports the use of the new Dataverse client in instances that have historically allowed login via Shibboleth. Specifically, with this flag enabled, when an OIDC bridge is configured to allow OIDC login with validation by the bridged Shibboleth providers, users with existing Shibboleth-based accounts in Dataverse can log in to those accounts, thereby maintaining access to their existing content and retaining their roles. (For security reasons, Dataverse's current support for direct login via Shibboleth cannot be used in browser-based clients.)
diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst
@@ -3726,6 +3726,9 @@ please find all known feature flags below. Any of these flags can be activated u
     * - api-bearer-auth-use-builtin-user-on-id-match
       - Allows the use of a built-in user account when an identity match is found during API bearer authentication. This feature enables automatic association of an incoming IdP identity with an existing built-in user account, bypassing the need for additional user registration steps. This feature only works when the feature flag ``api-bearer-auth`` is also enabled. **Caution: Enabling this feature flag exposes the installation to potential user impersonation issues depending on the specifics of the IdP configured (For example, if it is configured such that an attacker can create a new account in the IdP, or configured social login account, matching a Dataverse built-in account).**
       - ``Off``
+    * - api-bearer-auth-use-shib-user-on-id-match
+      - Allows the use of a Shibboleth user account when an identity match is found during API bearer authentication. This feature enables automatic association of an incoming IdP identity with an existing Shibboleth user account, bypassing the need for additional user registration steps. This feature only works when the feature flag ``api-bearer-auth`` is also enabled. **Caution: Enabling this flag could result in impersonation risks if (and only if) used with a misconfigured IdP.**
+      - ``Off``
     * - avoid-expensive-solr-join
       - Changes the way Solr queries are constructed for public content (published Collections, Datasets and Files). It removes a very expensive Solr join on all such documents, improving overall performance, especially for large instances under heavy load. Before this feature flag is enabled, the corresponding indexing feature (see next feature flag) must be turned on and a full reindex performed (otherwise public objects are not going to be shown in search results). See :doc:`/admin/solr-search-index`. 
       - ``Off``
diff --git a/src/main/java/edu/harvard/iq/dataverse/Shib.java b/src/main/java/edu/harvard/iq/dataverse/Shib.java
@@ -76,21 +76,6 @@ public class Shib implements java.io.Serializable {
     private final String loginpage = "/loginpage.xhtml";
     private final String identityProviderProblem = "Problem with Identity Provider";
 
-    /**
-     * We only have one field in which to store a unique
-     * useridentifier/persistentuserid so we have to jam the the "entityId" for
-     * a Shibboleth Identity Provider (IdP) and the unique persistent identifier
-     * per user into the same field and a separator between these two would be
-     * nice, in case we ever want to answer questions like "How many users
-     * logged in from Harvard's Identity Provider?".
-     *
-     * A pipe ("|") is used as a separator because it's considered "unwise" to
-     * use in a URL and the "entityId" for a Shibboleth Identity Provider (IdP)
-     * looks like a URL:
-     * http://stackoverflow.com/questions/1547899/which-characters-make-a-url-invalid
-     */
-    private String persistentUserIdSeparator = "|";
-
     /**
      * The Shibboleth Identity Provider (IdP), an "entityId" which often but not
      * always looks like a URL.
@@ -248,7 +233,7 @@ else if (ShibAffiliationOrder.equals("firstAffiliation")) {
 //        emailAddress = "willFailBeanValidation"; // for testing createAuthenticatedUser exceptions
         displayInfo = new AuthenticatedUserDisplayInfo(firstName, lastName, emailAddress, affiliation, null);
 
-        userPersistentId = shibIdp + persistentUserIdSeparator + shibUserIdentifier;
+        userPersistentId = ShibUtil.createUserPersistentIdentifier(shibIdp, shibUserIdentifier);
         ShibAuthenticationProvider shibAuthProvider = new ShibAuthenticationProvider();
         AuthenticatedUser au = authSvc.lookupUser(shibAuthProvider.getId(), userPersistentId);
         if (au != null) {
diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/AuthenticationServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/authorization/AuthenticationServiceBean.java
@@ -14,6 +14,7 @@
 import edu.harvard.iq.dataverse.authorization.providers.oauth2.OAuth2UserRecord;
 import edu.harvard.iq.dataverse.authorization.providers.oauth2.impl.OrcidOAuth2AP;
 import edu.harvard.iq.dataverse.authorization.providers.oauth2.oidc.OIDCAuthProvider;
+import edu.harvard.iq.dataverse.authorization.providers.shib.ShibUtil;
 import edu.harvard.iq.dataverse.search.IndexServiceBean;
 import edu.harvard.iq.dataverse.actionlogging.ActionLogRecord;
 import edu.harvard.iq.dataverse.actionlogging.ActionLogServiceBean;
@@ -995,10 +996,23 @@ public AuthenticatedUser lookupUserByOIDCBearerToken(String bearerToken) throws
         // TODO: Get the identifier from an invalidating cache to avoid lookup bursts of the same token.
         // Tokens in the cache should be removed after some (configurable) time.
         OAuth2UserRecord oAuth2UserRecord = verifyOIDCBearerTokenAndGetOAuth2UserRecord(bearerToken);
-        if (FeatureFlags.API_BEARER_AUTH_USE_BUILTIN_USER_ON_ID_MATCH.enabled()) {
-            AuthenticatedUser builtinAuthenticatedUser = lookupUser(BuiltinAuthenticationProvider.PROVIDER_ID, oAuth2UserRecord.getUsername());
-            return (builtinAuthenticatedUser != null) ? builtinAuthenticatedUser : lookupUser(oAuth2UserRecord.getUserRecordIdentifier());
+        AuthenticatedUser authenticatedUser;
+        if (FeatureFlags.API_BEARER_AUTH_USE_SHIB_USER_ON_ID_MATCH.enabled() && oAuth2UserRecord.hasShibAttributes()) {
+            logger.log(Level.FINE, "OAuth2UserRecord has Shibboleth attributes");
+            String userPersistentId = ShibUtil.createUserPersistentIdentifier(oAuth2UserRecord.getShibIdp(), oAuth2UserRecord.getShibUniquePersistentIdentifier());
+            authenticatedUser = lookupUser(ShibAuthenticationProvider.PROVIDER_ID, userPersistentId);
+            if (authenticatedUser != null) {
+                logger.log(Level.FINE, "Shibboleth user found for the given bearer token");
+                return authenticatedUser;
+            }
+        } else if (FeatureFlags.API_BEARER_AUTH_USE_BUILTIN_USER_ON_ID_MATCH.enabled()) {
+            authenticatedUser = lookupUser(BuiltinAuthenticationProvider.PROVIDER_ID, oAuth2UserRecord.getUsername());
+            if (authenticatedUser != null) {
+                logger.log(Level.FINE, "Builtin user found for the given bearer token");
+                return authenticatedUser;
+            }
         }
+
         return lookupUser(oAuth2UserRecord.getUserRecordIdentifier());
     }
 
diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/OAuth2UserRecord.java b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/OAuth2UserRecord.java
@@ -2,39 +2,75 @@
 
 import edu.harvard.iq.dataverse.authorization.AuthenticatedUserDisplayInfo;
 import edu.harvard.iq.dataverse.authorization.UserRecordIdentifier;
+
+import java.io.Serializable;
 import java.util.List;
 
 /**
  * Describes a single user on a remote IDP that uses OAuth2.
  * Normally generated by {@link AbstractOAuth2Idp}.
- * 
+ *
  * @author michael
  */
-public class OAuth2UserRecord implements java.io.Serializable {
-    
+public class OAuth2UserRecord implements Serializable {
+
     private final String serviceId;
-    
-    /** An immutable value, probably a number. Not a username that may change. */
+
+    /**
+     * An immutable value, probably a number. Not a username that may change.
+     */
     private final String idInService;
 
-    /** A potentially mutable String that is easier on the eye than a number. */
+    /**
+     * A potentially mutable String that is easier on the eye than a number.
+     */
     private final String username;
-    
+
+    /**
+     * For users originally coming from a Shibboleth IdP
+     */
+    private final String shibUniquePersistentIdentifier;
+    private final String shibIdp;
+
     private final AuthenticatedUserDisplayInfo displayInfo;
-    
     private final List<String> availableEmailAddresses;
-    
     private final OAuth2TokenData tokenData;
-    
-    public OAuth2UserRecord(String aServiceId, String anIdInService, String aUsername,
-                            OAuth2TokenData someTokenData,  AuthenticatedUserDisplayInfo aDisplayInfo,
-                            List<String> someAvailableEmailAddresses) {
-        serviceId = aServiceId;
-        idInService = anIdInService;
-        username = aUsername;
-        tokenData = someTokenData;
-        displayInfo = aDisplayInfo;
-        availableEmailAddresses = someAvailableEmailAddresses;
+
+    /**
+     * Constructor for users without Shibboleth attributes.
+     */
+    public OAuth2UserRecord(
+            String serviceId,
+            String idInService,
+            String username,
+            OAuth2TokenData tokenData,
+            AuthenticatedUserDisplayInfo displayInfo,
+            List<String> availableEmailAddresses
+    ) {
+        this(serviceId, idInService, username, null, null, tokenData, displayInfo, availableEmailAddresses);
+    }
+
+    /**
+     * Full constructor for OAuth2 user records.
+     */
+    public OAuth2UserRecord(
+            String serviceId,
+            String idInService,
+            String username,
+            String shibUniquePersistentIdentifier,
+            String shibIdp,
+            OAuth2TokenData tokenData,
+            AuthenticatedUserDisplayInfo displayInfo,
+            List<String> availableEmailAddresses
+    ) {
+        this.serviceId = serviceId;
+        this.idInService = idInService;
+        this.username = username;
+        this.shibUniquePersistentIdentifier = shibUniquePersistentIdentifier;
+        this.shibIdp = shibIdp;
+        this.tokenData = tokenData;
+        this.displayInfo = displayInfo;
+        this.availableEmailAddresses = availableEmailAddresses;
     }
 
     public String getServiceId() {
@@ -49,10 +85,18 @@ public String getUsername() {
         return username;
     }
 
+    public String getShibUniquePersistentIdentifier() {
+        return shibUniquePersistentIdentifier;
+    }
+
+    public String getShibIdp() {
+        return shibIdp;
+    }
+
     public List<String> getAvailableEmailAddresses() {
         return availableEmailAddresses;
     }
-    
+
     public AuthenticatedUserDisplayInfo getDisplayInfo() {
         return displayInfo;
     }
@@ -61,12 +105,19 @@ public OAuth2TokenData getTokenData() {
         return tokenData;
     }
 
-    @Override
-    public String toString() {
-        return "OAuth2UserRecord{" + "serviceId=" + serviceId + ", idInService=" + idInService + '}';
-    }
-    
     public UserRecordIdentifier getUserRecordIdentifier() {
         return new UserRecordIdentifier(serviceId, idInService);
     }
+
+    public boolean hasShibAttributes() {
+        return shibIdp != null && shibUniquePersistentIdentifier != null;
+    }
+
+    @Override
+    public String toString() {
+        return "OAuth2UserRecord{" +
+                "serviceId='" + serviceId + '\'' +
+                ", idInService='" + idInService + '\'' +
+                '}';
+    }
 }
diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/oidc/OIDCAuthProvider.java b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/oidc/OIDCAuthProvider.java
@@ -33,11 +33,11 @@
 import com.nimbusds.openid.connect.sdk.op.OIDCProviderConfigurationRequest;
 import com.nimbusds.openid.connect.sdk.op.OIDCProviderMetadata;
 import edu.harvard.iq.dataverse.authorization.AuthenticatedUserDisplayInfo;
-import edu.harvard.iq.dataverse.authorization.UserRecordIdentifier;
 import edu.harvard.iq.dataverse.authorization.exceptions.AuthorizationSetupException;
 import edu.harvard.iq.dataverse.authorization.providers.oauth2.AbstractOAuth2AuthenticationProvider;
 import edu.harvard.iq.dataverse.authorization.providers.oauth2.OAuth2Exception;
 import edu.harvard.iq.dataverse.authorization.providers.oauth2.OAuth2UserRecord;
+import edu.harvard.iq.dataverse.authorization.providers.shib.ShibUtil;
 import edu.harvard.iq.dataverse.settings.JvmSettings;
 import edu.harvard.iq.dataverse.util.BundleUtil;
 
@@ -47,11 +47,8 @@
 import java.time.temporal.ChronoUnit;
 import java.util.Arrays;
 import java.util.List;
-import java.util.Map;
 import java.util.Optional;
-import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ExecutionException;
-import java.util.logging.Level;
 import java.util.logging.Logger;
 
 /**
@@ -231,16 +228,34 @@ public OAuth2UserRecord getUserRecord(String code, String state, String redirect
      * @return the usable user record for processing ing {@link edu.harvard.iq.dataverse.authorization.providers.oauth2.OAuth2LoginBackingBean}
      */
     public OAuth2UserRecord getUserRecord(UserInfo userInfo) {
+        // Extract Shibboleth attributes if present
+        Object shibUniqueIdObj = userInfo.getClaim(ShibUtil.uniquePersistentIdentifier);
+        Object shibIdpObj = userInfo.getClaim(ShibUtil.shibIdpAttribute);
+
+        String shibUniqueId = (shibUniqueIdObj != null) ? shibUniqueIdObj.toString() : null;
+        String shibIdp = (shibIdpObj != null) ? shibIdpObj.toString() : null;
+
+        // Build display info from user attributes
+        AuthenticatedUserDisplayInfo displayInfo = new AuthenticatedUserDisplayInfo(
+                userInfo.getGivenName(),
+                userInfo.getFamilyName(),
+                userInfo.getEmailAddress(),
+                "",
+                ""
+        );
+
         return new OAuth2UserRecord(
-            this.getId(),
-            userInfo.getSubject().getValue(),
-            userInfo.getPreferredUsername(),
-            null,
-            new AuthenticatedUserDisplayInfo(userInfo.getGivenName(), userInfo.getFamilyName(), userInfo.getEmailAddress(), "", ""),
-            null
+                this.getId(),
+                userInfo.getSubject().getValue(),
+                userInfo.getPreferredUsername(),
+                shibUniqueId,
+                shibIdp,
+                null,
+                displayInfo,
+                null
         );
     }
-    
+
     /**
      * Retrieve the Access Token from provider. Encapsulate for testing.
      * @param grant
diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/shib/ShibUtil.java b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/shib/ShibUtil.java
@@ -405,4 +405,23 @@ public static void printAttributes(HttpServletRequest request) {
         logger.fine("shib values: " + shibValues);
     }
 
+    /**
+     * Creates a persistent identifier for a user authenticated via a Shibboleth Identity Provider (IdP).
+     *
+     * <p>This method combines the IdP's entity ID and the user's unique identifier into a single string,
+     * using a pipe character ("|") as a separator. This is necessary because there is only one field
+     * available to store the full identifier.</p>
+     *
+     * <p>The pipe character is chosen because it's considered "unwise" to use in URLs, and the
+     * Shibboleth IdP entity ID often resembles a URL. Using this separator allows for future parsing,
+     * such as answering questions like "How many users logged in from Harvard's Identity Provider?"</p>
+     *
+     * @param shibIdp the entity ID of the Shibboleth Identity Provider
+     * @param shibUserIdentifier the unique persistent identifier for the user from the IdP
+     * @return a combined string containing both the IdP and user identifier, separated by a pipe
+     */
+    public static String createUserPersistentIdentifier(String shibIdp, String shibUserIdentifier) {
+        String persistentUserIdSeparator = "|";
+        return shibIdp + persistentUserIdSeparator + shibUserIdentifier;
+    }
 }
diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/FeatureFlags.java b/src/main/java/edu/harvard/iq/dataverse/settings/FeatureFlags.java
@@ -71,6 +71,20 @@ public enum FeatureFlags {
      * @since Dataverse @6.7:
      */
     API_BEARER_AUTH_USE_BUILTIN_USER_ON_ID_MATCH("api-bearer-auth-use-builtin-user-on-id-match"),
+
+    /**
+     * Allows the use of a Shibboleth user account when an identity match is found during API bearer authentication.
+     * This feature enables automatic association of an incoming IdP identity with an existing Shibboleth user account,
+     * bypassing the need for additional user registration steps.
+     *
+     * <p>The value of this feature flag is only considered when the feature flag
+     * {@link #API_BEARER_AUTH} is enabled.</p>
+     *
+     * @apiNote Raise flag by setting "dataverse.feature.api-bearer-auth-use-shib-user-on-id-match"
+     * @since Dataverse @TODO:
+     */
+    API_BEARER_AUTH_USE_SHIB_USER_ON_ID_MATCH("api-bearer-auth-use-shib-user-on-id-match"),
+
     /**
      * For published (public) objects, don't use a join when searching Solr. 
      * Experimental! Requires a reindex with the following feature flag enabled,
diff --git a/src/test/java/edu/harvard/iq/dataverse/authorization/AuthenticationServiceBeanTest.java b/src/test/java/edu/harvard/iq/dataverse/authorization/AuthenticationServiceBeanTest.java

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	+Implemented a new feature flag ``dataverse.feature.api-bearer-auth-use-shib-user-on-id-match``, which supports the use of the new Dataverse client in instances that have historically allowed login via Shibboleth. Specifically, with this flag enabled, when an OIDC bridge is configured to allow OIDC login with validation by the bridged Shibboleth providers, users with existing Shibboleth-based accounts in Dataverse can log in to those accounts, thereby maintaining access to their existing content and retaining their roles. (For security reasons, Dataverse's current support for direct login via Shibboleth cannot be used in browser-based clients.)