Merge pull request #11502 from IQSS/11404-shib-login-mdq

Login page and Shibboleth auth refactored to work with new InCommon services

Author: landreev

doc/release-notes/11404-shibboleth-mdq-wayfinder.md

@@ -0,0 +1,44 @@
+### For Dataverse instances that use Shibboleth as members of InCommon federation
+
+Please note that most of the known Dataverse instances that support Shibboleth logins do so without being part of InCommon, and therefore are not affected. All such instances will be able to continue using the old login workflow without needing to make any configuration changes. 
+
+For the relatively few instances using InCommon: Since InCommon discontinued their old-style federation metadata feed, a new Shibboleth implementation has been added to utilize the recommended replacements: the MDQ protocol and the WayFinder service. In order to continue using InCommon, such instances will need to modify their shibd configuration and their registration with Incommon, plus set a new feature flag. See the upgrade instructions for details.
+
+
+### New Settings
+
+- dataverse.feature.shibboleth-use-wayfinder
+- dataverse.feature.shibboleth-use-localhost
+
+### For the Upgrade Instruction:
+
+[(strip this from the real release note) this should be the very last of the optional upgrade steps; as it's been pointed out to me that there may not be any instances affected by this aside from Harvard and UNC, both of which have been active participants in the development of the underlying code and the upgrade process below. ... which kind of makes including them in the release note somewhat unnecessary (?). but I figure we should include them anyway, in case there's an instance out there we are not aware of. - L.A.]
+
+
+If your instance is offering institutional Shibboleth logins as part of the InCommon federation, you must make some changes to your service configuration:
+
+Note that if your Dataverse instance is using Shibboleth outside of InCommon, your login workflow should continue working unchanged, so please skip this section.
+
+a. Configure your Service Provider (SP) in the InCommon Federation Manager to use WayFinder following [their instructions](https://spaces.at.internet2.edu/display/federation/how-to-configure-service-to-use-wayfinder).
+
+b. Reconfigure your locally-running `shibd` service to use WayFinder and the new MDQ metadata retrieval protocol.
+Download and place the new [production signing key](https://spaces.at.internet2.edu/display/MDQ/production-mdq-signing-key) in `/etc/shibboleth` and name it `inc-md-cert-mdq.pem`.
+Change the `SSO` and `MetadataProvider` sections of the `/etc/shibboleth/shibboleth2.xml` configuration file as follows:
+
+```
+<SSO discoveryProtocol="SAMLDS" discoveryURL="https://wayf.incommonfederation.org/DS/WAYF">
+     SAML2 SAML1
+</SSO>
+```
+and
+```
+<MetadataProvider type="MDQ" id="incommon" ignoreTransport="true" cacheDirectory="inc-mdq-cache"
+  maxCacheDuration="86400" minCacheDuration="60" baseUrl="https://mdq.incommon.org/">
+    <MetadataFilter type="Signature" certificate="inc-md-cert-mdq.pem"/>
+    <MetadataFilter type="RequireValidUntil" maxValidityInterval="1209600"/>
+</MetadataProvider>
+```
+See [How to configure a Shibboleth service provider (SP) to use MDQ](https://spaces.at.internet2.edu/display/MDQ/how-to-configure-shib-sp-to-use-mdq) for more information.
+
+c. Set the feature flag `dataverse.feature.shibboleth-use-wayfinder=true`. 
+

doc/sphinx-guides/source/_static/installation/files/etc/shibboleth/shibboleth2.xml

@@ -23,6 +23,12 @@ https://wiki.shibboleth.net/confluence/display/SHIB2/NativeSPConfiguration
 	    <SSO>
 	      SAML2 SAML1
 	    </SSO>
+	    <!-- If you are planning to use Shibboleth as a member of the InCommon federation, -->
+	    <!-- comment out the section above, and un-comment out the following, in order to -->
+	    <!-- use the new WayFinder service for the login page workflow: -->
+	    <!-- SSO discoveryProtocol="SAMLDS" discoveryURL="https://wayf.incommonfederation.org/DS/WAYF">
+              SAML2 SAML1
+            </SSO -->
 
             <!-- SAML and local-only logout. -->
             <!-- https://wiki.shibboleth.net/confluence/display/SHIB2/NativeSPServiceLogout -->
@@ -58,19 +64,9 @@ https://wiki.shibboleth.net/confluence/display/SHIB2/NativeSPConfiguration
         <MetadataProvider type="XML" path="dataverse-idp-metadata.xml" backingFilePath="local-idp-metadata.xml" legacyOrgNames="true" reloadInterval="7200"/>
         <!-- Uncomment to enable all the Research & Scholarship IdPs from InCommon -->
         <!--
-        <MetadataProvider type="XML" url="http://md.incommon.org/InCommon/InCommon-metadata.xml" backingFilePath="InCommon-metadata.xml" maxRefreshDelay="3600">
-            <DiscoveryFilter type="Whitelist" matcher="EntityAttributes">
-                <saml:Attribute
-                    Name="http://macedir.org/entity-category-support"
-                    NameFormat="urn:oasis:names:tc:SAML:2.0:attrname-format:uri">
-                    <saml:AttributeValue>http://id.incommon.org/category/research-and-scholarship</saml:AttributeValue>
-                </saml:Attribute>
-                <saml:Attribute
-                    Name="http://macedir.org/entity-category-support"
-                    NameFormat="urn:oasis:names:tc:SAML:2.0:attrname-format:uri">
-                    <saml:AttributeValue>http://refeds.org/category/research-and-scholarship</saml:AttributeValue>
-                </saml:Attribute>
-            </DiscoveryFilter>
+            <MetadataProvider type="MDQ" id="incommon" ignoreTransport="true" cacheDirectory="inc-mdq-cache" maxCacheDuration="86400" minCacheDuration="60" baseUrl="https://mdq.incommon.org/">
+	    <MetadataFilter type="Signature" certificate="inc-md-cert-mdq.pem"/>
+	    <MetadataFilter type="RequireValidUntil" maxValidityInterval="1209600"/>
         </MetadataProvider>
         -->
 

doc/sphinx-guides/source/installation/config.rst

@@ -3791,6 +3791,12 @@ please find all known feature flags below. Any of these flags can be activated u
     * - enable-version-note
       - Turns on the ability to add/view/edit/delete per-dataset-version notes intended to provide :ref:`provenance` information about why the dataset/version was created.  
       - ``Off``
+    * - shibboleth-use-wayfinder
+      - This flag allows an instance to use Shibboleth with InCommon federation services. Our original Shibboleth implementation that relies on DiscoFeed can no longer be used since InCommon discontinued their old-style metadata feed. An alternative mechanism had to be implemented in order to use WayFinder service, their recommended replacements, instead.
+      - ``Off``
+    * - shibboleth-use-localhost
+      - A Shibboleth-using Dataverse instance needs to make network calls to the locally-running ``shibd`` service. The default behavior is to use the address configured via the ``siteUrl`` setting. There are however situations (firewalls, etc.) where localhost would be preferable.
+      - ``Off``
 
 **Note:** Feature flags can be set via any `supported MicroProfile Config API source`_, e.g. the environment variable
 ``DATAVERSE_FEATURE_XXX`` (e.g. ``DATAVERSE_FEATURE_API_SESSION_AUTH=1``). These environment variables can be set in your shell before starting Payara. If you are using :doc:`Docker for development </container/dev-usage>`, you can set them in the `docker compose <https://docs.docker.com/compose/environment-variables/set-environment-variables/>`_ file.

doc/sphinx-guides/source/installation/shibboleth.rst

@@ -159,6 +159,8 @@ Rather than or in addition to specifying individual Identity Providers (see :ref
 
 For example, in the United States, you would register your Dataverse installation with `InCommon <https://incommon.org>`_. For a list of federations around the world, see `REFEDS (the Research and Education FEDerations group) <https://refeds.org/federations>`_. The details of how to register with an identity federation are out of scope for this document.
 
+If you are planning to use InCommon, please note that ``shibd`` needs to be configured to use the new MDQ protocol and WayFinder `service <https://spaces.at.internet2.edu/display/federation/incommon-wayfinder-announcement>`_ `announced <https://lists.incommon.org/sympa/arc/inc-ops-notifications/2024-04/msg00000.html>`_ `by <https://incommon.org/news/incommon-federation-service-enhancements/>`_ InCommon. The sample ``shibboleth2.xml`` provided already contains commented-out sections pre-configured to work with this new InCommon framework. Please see https://spaces.at.internet2.edu/display/MDQ/how-to-configure-shib-sp-to-use-mdq and https://spaces.at.internet2.edu/display/federation/how-to-configure-service-to-use-wayfinder for more information. You will also need to set the feature flag ``dataverse.feature.shibboleth-use-wayfinder=true`` (see :ref:`feature-flags`). 
+
 For a successful login to Dataverse, certain :ref:`shibboleth-attributes` must be released by the Identity Provider (IdP). Otherwise, in the federation context, users will have the frustrating experience of selecting their IdP in the list but then getting an error like ``Problem with Identity Provider – The SAML assertion for "eppn" was null``. We definitely want to prevent this! There's even some guidance about this problem in the User Guide under the heading :ref:`fix-shib-login` that links back here.
 
 For InCommon, a decent strategy for ensuring that IdPs release the necessary attributes is to have both the SP (your Dataverse installation) and the IdP (there are many of these around the world) join the Research & Scholarship (R&S) category. The `R&S website <https://incommon.org/federation/research-and-scholarship/>`_ explains the R&S dream well:
@@ -259,14 +261,23 @@ On CentOS 6:
 
 ``chkconfig shibd on``
 
-Verify DiscoFeed and Metadata URLs
-----------------------------------
+Verify the Metadata URL
+-----------------------
 
-As a sanity check, visit the following URLs (substituting your hostname) to make sure you see JSON and XML:
+Substitute your hostname and verify that you are seeing your service provider metadata in XML format:
 
-- https://dataverse.example.edu/Shibboleth.sso/DiscoFeed
 - https://dataverse.example.edu/Shibboleth.sso/Metadata
 
+
+If Your Instance is Using Discofeed: Verify DiscoFeed URL
+---------------------------------------------------------
+
+As another sanity check, substitute your hostname and make sure you see well-formed JSON:
+
+- https://dataverse.example.edu/Shibboleth.sso/DiscoFeed
+
+(Skip this step if you'll be using Shibboleth as a registered member of InCommon federation, since the DiscoFeed will not be part of the workflow.)
+
 The JSON in ``DiscoFeed`` comes from the list of IdPs you configured in the ``MetadataProvider`` section of ``shibboleth2.xml`` and will form a dropdown list on the Login Page.
 
 Add the Shibboleth Authentication Provider to Your Dataverse Installation

src/main/java/edu/harvard/iq/dataverse/LoginPage.java

@@ -9,7 +9,9 @@
 import edu.harvard.iq.dataverse.authorization.exceptions.AuthenticationFailedException;
 import edu.harvard.iq.dataverse.authorization.providers.builtin.BuiltinAuthenticationProvider;
 import edu.harvard.iq.dataverse.authorization.providers.builtin.BuiltinUserServiceBean;
+import edu.harvard.iq.dataverse.authorization.providers.shib.ShibServiceBean;
 import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser;
+import edu.harvard.iq.dataverse.settings.FeatureFlags;
 import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
 import edu.harvard.iq.dataverse.util.BundleUtil;
 import edu.harvard.iq.dataverse.util.JsfHelper;
@@ -92,6 +94,9 @@ public enum EditMode {LOGIN, SUCCESS, FAILED};
     @EJB
     SystemConfig systemConfig;
 
+    @EJB
+    ShibServiceBean shibService;
+    
     @Inject
     DataverseRequestServiceBean dvRequestService;
 
@@ -254,6 +259,37 @@ public String getRedirectPage() {
     public void setRedirectPage(String redirectPage) {
         this.redirectPage = redirectPage;
     }
+    
+    /*
+     * Starting v6.7 the default Shibboleth login mechanism is to use the new 
+     * Wayfinder service from InCommon. We no longer use the javascript from the 
+     * idp package to generate the list of participating institutions and then 
+     * generate the redirect url to the auth. service they choose. Under the new 
+     * model the user is redirected to InCommon and the workflow of picking 
+     * their institutional auth. service provider will be handled there. 
+     */
+    public String getShibWayfinderRedirect() {
+        String wayFinderUrl = shibService.getWayfinderRedirectUrl();
+        logger.fine("wayfinder url provided by the shib service: " + wayFinderUrl);
+        // In order to produce a complete url, we need to add the final redirect
+        // parameter (this will be the FOURTH redirect in the shib. authentication
+        // loop), the redirectPage= pointing to the final destination Dataverse 
+        // page. Note the corresponding multiple-level URL encoding involved.
+        String finalRedirectUrl = wayFinderUrl 
+                + "%253FredirectPage%253D" 
+                + getRedirectPage().replaceAll("/", "%25252F").replaceAll("=", "%25253D").replaceAll("\\?", "%25253F").replaceAll("&", "%252526");
+        logger.fine("final redirect url: " + finalRedirectUrl);
+        return finalRedirectUrl;
+    }
+    
+    /* 
+     * An instance that uses Shibboleth as part of InCommon can switch to using
+     * the new login workflow that relies on WayFinder and MDQ. 
+     */ 
+    public boolean isShibbolethUseWayFinder() {
+        return FeatureFlags.SHIBBOLETH_USE_WAYFINDER.enabled();
+    }
+
 
     public AuthenticationProvider getAuthProvider() {
         return authProvider;

src/main/java/edu/harvard/iq/dataverse/authorization/AuthenticationProvidersRegistrationServiceBean.java

@@ -17,8 +17,12 @@
 import edu.harvard.iq.dataverse.authorization.providers.oauth2.OAuth2AuthenticationProviderFactory;
 import edu.harvard.iq.dataverse.authorization.providers.oauth2.impl.OrcidOAuth2AP;
 import edu.harvard.iq.dataverse.authorization.providers.oauth2.oidc.OIDCAuthenticationProviderFactory;
+import edu.harvard.iq.dataverse.authorization.providers.shib.ShibAuthenticationProvider;
 import edu.harvard.iq.dataverse.authorization.providers.shib.ShibAuthenticationProviderFactory;
+import edu.harvard.iq.dataverse.settings.FeatureFlags;
 import edu.harvard.iq.dataverse.settings.JvmSettings;
+import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
+import edu.harvard.iq.dataverse.util.SystemConfig;
 import edu.harvard.iq.dataverse.validation.PasswordValidatorServiceBean;
 import java.util.HashMap;
 import java.util.Map;
@@ -33,6 +37,16 @@
 import jakarta.inject.Named;
 import jakarta.persistence.EntityManager;
 import jakarta.persistence.PersistenceContext;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.net.HttpURLConnection;
+import java.net.MalformedURLException;
+import java.net.URL;
+import javax.xml.stream.XMLInputFactory;
+import javax.xml.stream.XMLStreamConstants;
+import javax.xml.stream.XMLStreamException;
+import javax.xml.stream.XMLStreamReader;
 
 /**
  *
@@ -63,6 +77,9 @@ public class AuthenticationProvidersRegistrationServiceBean {
     @EJB
     AuthenticationServiceBean authenticationService;
 
+    @EJB
+    SettingsServiceBean settingsService;
+      
     /**
      * The maps below (the objects themselves) are "final", but the
      * values will be populated in @PostConstruct (see below) during 
@@ -117,7 +134,32 @@ public void startup() {
                 .getResultList().forEach((row) -> {
                     if(row.isEnabled()) {
                     try {
-                        registerProvider( loadProvider(row) );
+                        AuthenticationProvider authProvider = loadProvider(row);
+                        
+                        registerProvider( authProvider );
+                        
+                        // For production Shibboleth instances that are not using 
+                        // the legacy DiscoFeed-based workflow, we need to call 
+                        // shibd to look up and cache its entityID, since it will 
+                        // be needed in order to issue WayFinder service redirects.
+                                                
+                        if ("shib".equals(authProvider.getId())
+                                && FeatureFlags.SHIBBOLETH_USE_WAYFINDER.enabled()) {
+                            // ... is this a prod. shibboleth instance?
+                            String shibTypeSetting = settingsService.getValueForKey(SettingsServiceBean.Key.DebugShibAccountType, null);
+                            boolean isProduction = shibTypeSetting == null || shibTypeSetting.equals("PRODUCTION");
+
+                            if (isProduction) {
+                                String spEntityId = lookupShibbolethEntityId();
+                                logger.info("Looked up the entityId of the shibboleth service provider (via a call to shibd): "
+                                        + spEntityId);
+                                if (spEntityId == null) {
+                                    // we'll make this educated guess - it may or may not help us later on:
+                                    spEntityId = SystemConfig.getDataverseSiteUrlStatic() + "/sp";
+                                }
+                                ((ShibAuthenticationProvider) authProvider).setServiceProviderEntityId(spEntityId);
+                            }
+                        }
 
                     } catch ( AuthenticationProviderFactoryNotFoundException e ) {
                         logger.log(Level.SEVERE, "Cannot find authentication provider factory with alias '" + e.getFactoryAlias() + "'",e);
@@ -307,5 +349,75 @@ public boolean isOrcidEnabled() {
         return oAuth2authenticationProviders.values().stream().anyMatch( s -> s.getId().toLowerCase().contains("orcid") );
     }
     */
+    
+    private String lookupShibbolethEntityId() {
+        
+        String baseUrl; 
+        if (FeatureFlags.SHIBBOLETH_USE_LOCALHOST.enabled()) {
+            baseUrl = "http://localhost";
+        } else {
+            baseUrl = SystemConfig.getDataverseSiteUrlStatic();
+        }
+        
+        String urlString = baseUrl + "/Shibboleth.sso/Metadata";
+        
+        URL url = null;
+        try {
+            url = new URL(urlString);
+        } catch (MalformedURLException ex) {
+            logger.warning(ex.toString());
+            return null;
+        }
+        
+        if (url == null) {
+            logger.warning("url object was null after parsing " + urlString);
+            return null;
+        }
+        
+        HttpURLConnection metadataRequest = null;
+        try {
+            metadataRequest = (HttpURLConnection) url.openConnection();
+        } catch (IOException ex) {
+            logger.warning(ex.toString());
+            return null;
+        }
+        if (metadataRequest == null) {
+            logger.warning("http request was null for a local /Shibboleth.sso/Metadata call");
+            return null;
+        }
+        try {
+            metadataRequest.connect();
+        } catch (IOException ex) {
+            logger.warning(ex.toString());
+            return null;
+        }
+        
+        XMLStreamReader xmlr = null;
+
+        try {
+            XMLInputFactory xmlFactory = javax.xml.stream.XMLInputFactory.newInstance();
+            xmlr =  xmlFactory.createXMLStreamReader(new InputStreamReader((InputStream) metadataRequest.getInputStream()));
+            
+            while ( xmlr.next() == XMLStreamConstants.COMMENT);
+            xmlr.require(XMLStreamConstants.START_ELEMENT, null, "EntityDescriptor");
+            
+            return xmlr.getAttributeValue(null, "entityID");
+            
+        } catch (IOException ioex) {
+            logger.warning("IOException instantiating a stream reader of the /Shibboleth.sso/Metadata output" + ioex.getMessage());
+        } catch (XMLStreamException xsex) {
+            logger.warning("Failed to parse the xml output of the /Shibboleth.sso/Metadata; " + xsex.getMessage());
+        } finally {
+            if (xmlr != null) {
+                try {
+                    logger.fine("closing xml reader");
+                    xmlr.close();
+                } catch (XMLStreamException xsex) {
+                    // we don't care
+                }
+            }
+        }
+        return null; 
+    }
 
 }

src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/DataverseUserPage.java

@@ -213,7 +213,7 @@ public String init() {
         } else {
             return permissionsWrapper.notAuthorized();
         }
-
+        
         return "";
     }