Login page and Shibboleth auth refactored to work with new InCommon services

doc/release-notes/11404-shibboleth-mdq-wayfinder.md

@@ -0,0 +1,39 @@
+### For Dataverse instances using Shibboleth
+
+Since the old-style federation metadata feed was discontinued by InCommon, the Shibboleth login components have been re-implemented to utilize the recommended replacements: the MDQ protocol and the WayFinder service. From now on, this will be the default behavior of the login page for shib. users. Dataverse instances using Shibboleth as members of the InCommon federation will need to modify their shibd configuration and, possibly, their registration with Incommon. See the upgrade instruction for details.
+
+It is also possible for a Dataverse instance to continue using the old login page mechanism (the most likely use case for this would be if you are using Shibboleth without being part of InCommon, for example, by running shibd with a static list of known metadata providers). In this case, set the feature flag `dataverse.feature.shibboleth-use-discofeed=true` to preserve the legacy workflow as is. 
+
+### New Settings
+
+- dataverse.feature.shibboleth-use-discofeed
+- dataverse.feature.shibboleth-use-localhost
+
+### For the Upgrade Instruction:
+
+If your instance is offering institutional Shibboleth logins as part of the InCommon federation, you must make some changes to your service configuration.
+
+a. Configure your Service Provider (SP) in the InCommon Federation Manager to use WayFinder following [their instructions](https://spaces.at.internet2.edu/display/federation/how-to-configure-service-to-use-wayfinder).
+
+b. Reconfigure your locally-running `shibd` service to use WayFinder and the new MDQ metadata retrieval protocol.
+Download and place the new [production signing key](https://spaces.at.internet2.edu/display/MDQ/production-mdq-signing-key) in `/etc/shibboleth` and name it `inc-md-cert-mdq.pem`.
+Change the `SSO` and `MetadataProvider` sections of the `/etc/shibboleth/shibboleth2.xml` configuration file as follows:
+
+```
+<SSO discoveryProtocol="SAMLDS" discoveryURL="https://wayf.incommonfederation.org/DS/WAYF">
+     SAML2 SAML1
+</SSO>
+```
+and
+```
+<MetadataProvider type="MDQ" id="incommon" ignoreTransport="true" cacheDirectory="inc-mdq-cache"
+  maxCacheDuration="86400" minCacheDuration="60" baseUrl="https://mdq.incommon.org/">
+    <MetadataFilter type="Signature" certificate="inc-md-cert-mdq.pem"/>
+    <MetadataFilter type="RequireValidUntil" maxValidityInterval="1209600"/>
+</MetadataProvider>
+```
+See [How to configure a Shibboleth service provider (SP) to use MDQ](https://spaces.at.internet2.edu/display/MDQ/how-to-configure-shib-sp-to-use-mdq) for more information.
+
+
+If your Dataverse instance is using Shibboleth without being a member of the InCommon federation, you can preserve your working configuration as is and configure Dataverse to continue using the old-style login workflow by setting the feature flag `dataverse.feature.shibboleth-use-discofeed=true`.
+

doc/sphinx-guides/source/_static/installation/files/etc/shibboleth/shibboleth2.xml

@@ -23,6 +23,12 @@ https://wiki.shibboleth.net/confluence/display/SHIB2/NativeSPConfiguration
 	    <SSO>
 	      SAML2 SAML1
 	    </SSO>
+	    <!-- If you are planning to use Shibboleth as a member of the InCommon federation, -->
+	    <!-- comment out the section above, and un-comment out the following, in order to -->
+	    <!-- use the new WayFinder service for the login page workflow: -->
+	    <!-- SSO discoveryProtocol="SAMLDS" discoveryURL="https://wayf.incommonfederation.org/DS/WAYF">
+              SAML2 SAML1
+            </SSO -->
 
             <!-- SAML and local-only logout. -->
             <!-- https://wiki.shibboleth.net/confluence/display/SHIB2/NativeSPServiceLogout -->
@@ -58,19 +64,9 @@ https://wiki.shibboleth.net/confluence/display/SHIB2/NativeSPConfiguration
         <MetadataProvider type="XML" path="dataverse-idp-metadata.xml" backingFilePath="local-idp-metadata.xml" legacyOrgNames="true" reloadInterval="7200"/>
         <!-- Uncomment to enable all the Research & Scholarship IdPs from InCommon -->
         <!--
-        <MetadataProvider type="XML" url="http://md.incommon.org/InCommon/InCommon-metadata.xml" backingFilePath="InCommon-metadata.xml" maxRefreshDelay="3600">
-            <DiscoveryFilter type="Whitelist" matcher="EntityAttributes">
-                <saml:Attribute
-                    Name="http://macedir.org/entity-category-support"
-                    NameFormat="urn:oasis:names:tc:SAML:2.0:attrname-format:uri">
-                    <saml:AttributeValue>http://id.incommon.org/category/research-and-scholarship</saml:AttributeValue>
-                </saml:Attribute>
-                <saml:Attribute
-                    Name="http://macedir.org/entity-category-support"
-                    NameFormat="urn:oasis:names:tc:SAML:2.0:attrname-format:uri">
-                    <saml:AttributeValue>http://refeds.org/category/research-and-scholarship</saml:AttributeValue>
-                </saml:Attribute>
-            </DiscoveryFilter>
+            <MetadataProvider type="MDQ" id="incommon" ignoreTransport="true" cacheDirectory="inc-mdq-cache" maxCacheDuration="86400" minCacheDuration="60" baseUrl="https://mdq.incommon.org/">
+	    <MetadataFilter type="Signature" certificate="inc-md-cert-mdq.pem"/>
+	    <MetadataFilter type="RequireValidUntil" maxValidityInterval="1209600"/>
         </MetadataProvider>
         -->
 

doc/sphinx-guides/source/installation/config.rst

@@ -3524,6 +3524,12 @@ please find all known feature flags below. Any of these flags can be activated u
     * - enable-version-note
       - Turns on the ability to add/view/edit/delete per-dataset-version notes intended to provide :ref:`provenance` information about why the dataset/version was created.  
       - ``Off``
+    * - shibboleth-use-discofeed
+      - This flag allows an instance to continue using the old-style Shibboleth implementation that relies on DiscoFeed. The default behavior, starting v6.7 will be to use the new implementation that utilizes the MDQ protocol and Wayfinder service.
+      - ``Off``
+    * - shibboleth-use-localhost
+      - In both the new and old Shibboleth implementations Dataverse needs to make network calls to the local shibd service running on the same server. The default behavior is use the address configured via the siteUrl setting. There are however situations (firewalls, etc.) where localhost would be preferable.
+      - ``Off``
 
 **Note:** Feature flags can be set via any `supported MicroProfile Config API source`_, e.g. the environment variable
 ``DATAVERSE_FEATURE_XXX`` (e.g. ``DATAVERSE_FEATURE_API_SESSION_AUTH=1``). These environment variables can be set in your shell before starting Payara. If you are using :doc:`Docker for development </container/dev-usage>`, you can set them in the `docker compose <https://docs.docker.com/compose/environment-variables/set-environment-variables/>`_ file.

doc/sphinx-guides/source/installation/shibboleth.rst

@@ -159,6 +159,10 @@ Rather than or in addition to specifying individual Identity Providers (see :ref
 
 For example, in the United States, you would register your Dataverse installation with `InCommon <https://incommon.org>`_. For a list of federations around the world, see `REFEDS (the Research and Education FEDerations group) <https://refeds.org/federations>`_. The details of how to register with an identity federation are out of scope for this document.
 
+If you are planning to use InCommon, please note that ``shibd`` needs to be configured to use the new MDQ protocol and WayFinder `service <https://spaces.at.internet2.edu/display/federation/incommon-wayfinder-announcement>`_ `announced <https://lists.incommon.org/sympa/arc/inc-ops-notifications/2024-04/msg00000.html>`_ `by <https://incommon.org/news/incommon-federation-service-enhancements/>`_ InCommon. The sample ``shibboleth2.xml`` provided already contains commented-out sections pre-configured to work with this new InCommon framework. Please see https://spaces.at.internet2.edu/display/MDQ/how-to-configure-shib-sp-to-use-mdq and https://spaces.at.internet2.edu/display/federation/how-to-configure-service-to-use-wayfinder for more information.
+
+If your instance is not a part of InCommon and your ``shibd`` instance will be using provider metadata in the old ``type="XML"`` format, you will need to set the feature flag ``dataverse.feature.shibboleth-use-discofeed=true`` (see :ref:`feature-flags`). 
+
 For a successful login to Dataverse, certain :ref:`shibboleth-attributes` must be released by the Identity Provider (IdP). Otherwise, in the federation context, users will have the frustrating experience of selecting their IdP in the list but then getting an error like ``Problem with Identity Provider – The SAML assertion for "eppn" was null``. We definitely want to prevent this! There's even some guidance about this problem in the User Guide under the heading :ref:`fix-shib-login` that links back here.
 
 For InCommon, a decent strategy for ensuring that IdPs release the necessary attributes is to have both the SP (your Dataverse installation) and the IdP (there are many of these around the world) join the Research & Scholarship (R&S) category. The `R&S website <https://incommon.org/federation/research-and-scholarship/>`_ explains the R&S dream well:
@@ -259,14 +263,23 @@ On CentOS 6:
 
 ``chkconfig shibd on``
 
-Verify DiscoFeed and Metadata URLs
-----------------------------------
+Verify the Metadata URL
+-----------------------
 
-As a sanity check, visit the following URLs (substituting your hostname) to make sure you see JSON and XML:
+Substitute your hostname and verify that you are seeing your service provider metadata in XML format:
 
-- https://dataverse.example.edu/Shibboleth.sso/DiscoFeed
 - https://dataverse.example.edu/Shibboleth.sso/Metadata
 
+
+If Your Instance is Using Discofeed: Verify DiscoFeed URL
+---------------------------------------------------------
+
+As another sanity check, substitute your hostname and make sure you see well-formed JSON:
+
+- https://dataverse.example.edu/Shibboleth.sso/DiscoFeed
+
+(Skip this step if you'll be using Shibboleth as a registered member of InCommon federation. This is only relevant if you are setting ``dataverse.feature.shibboleth-use-discofeed=true`` as described under :ref:`identity-federation`.)
+
 The JSON in ``DiscoFeed`` comes from the list of IdPs you configured in the ``MetadataProvider`` section of ``shibboleth2.xml`` and will form a dropdown list on the Login Page.
 
 Add the Shibboleth Authentication Provider to Your Dataverse Installation

src/main/java/edu/harvard/iq/dataverse/LoginPage.java

@@ -9,7 +9,9 @@
 import edu.harvard.iq.dataverse.authorization.exceptions.AuthenticationFailedException;
 import edu.harvard.iq.dataverse.authorization.providers.builtin.BuiltinAuthenticationProvider;
 import edu.harvard.iq.dataverse.authorization.providers.builtin.BuiltinUserServiceBean;
+import edu.harvard.iq.dataverse.authorization.providers.shib.ShibServiceBean;
 import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser;
+import edu.harvard.iq.dataverse.settings.FeatureFlags;
 import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
 import edu.harvard.iq.dataverse.util.BundleUtil;
 import edu.harvard.iq.dataverse.util.JsfHelper;
@@ -92,6 +94,9 @@ public enum EditMode {LOGIN, SUCCESS, FAILED};
     @EJB
     SystemConfig systemConfig;
 
+    @EJB
+    ShibServiceBean shibService;
+
     @Inject
     DataverseRequestServiceBean dvRequestService;
 
@@ -254,6 +259,35 @@ public String getRedirectPage() {
     public void setRedirectPage(String redirectPage) {
         this.redirectPage = redirectPage;
     }
+
+    /*
+     * Starting v6.7 the default Shibboleth login mechanism is to use the new 
+     * Wayfinder service from InCommon. We no longer use the javascript from the 
+     * idp package to generate the list of participating institutions and then 
+     * generate the redirect url to the auth. service they choose. Under the new 
+     * model the user is redirected to InCommon and the workflow of picking 
+     * their institutional auth. service provider will be handled there. 
+     */
+    public String getShibWayfinderRedirect() {
+        String wayFinderUrl = shibService.getWayfinderRedirectUrl();
+        logger.fine("wayfinder url provided by the shib service: " + wayFinderUrl);
+        // In order to produce a complete url, we need to add the final redirect
+        // parameter (this will be the FOURTH redirect in the shib. authentication
+        // loop), the redirectPage= pointing to the final destination Dataverse 
+        // page. Note the corresponding multiple-level URL encoding involved.
+        String finalRedirectUrl = wayFinderUrl + "%253FredirectPage%253D" + getRedirectPage().replaceAll("/", "%25252F");
+        logger.fine("final redirect url: " + finalRedirectUrl);
+        return finalRedirectUrl;
+    }
+
+    /* 
+     * An instance can still choose to continue using the old, DiscoFeed-based 
+     * workflow, if that is preferable for whatever reason. 
+     */ 
+    public boolean isShibbolethUseDiscoFeed() {
+        return FeatureFlags.SHIBBOLETH_USE_DISCOFEED.enabled();
+    }
+
 
     public AuthenticationProvider getAuthProvider() {
         return authProvider;

src/main/java/edu/harvard/iq/dataverse/authorization/AuthenticationProvidersRegistrationServiceBean.java

@@ -17,8 +17,12 @@
 import edu.harvard.iq.dataverse.authorization.providers.oauth2.OAuth2AuthenticationProviderFactory;
 import edu.harvard.iq.dataverse.authorization.providers.oauth2.impl.OrcidOAuth2AP;
 import edu.harvard.iq.dataverse.authorization.providers.oauth2.oidc.OIDCAuthenticationProviderFactory;
+import edu.harvard.iq.dataverse.authorization.providers.shib.ShibAuthenticationProvider;
 import edu.harvard.iq.dataverse.authorization.providers.shib.ShibAuthenticationProviderFactory;
+import edu.harvard.iq.dataverse.settings.FeatureFlags;
 import edu.harvard.iq.dataverse.settings.JvmSettings;
+import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
+import edu.harvard.iq.dataverse.util.SystemConfig;
 import edu.harvard.iq.dataverse.validation.PasswordValidatorServiceBean;
 import java.util.HashMap;
 import java.util.Map;
@@ -33,6 +37,16 @@
 import jakarta.inject.Named;
 import jakarta.persistence.EntityManager;
 import jakarta.persistence.PersistenceContext;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.net.HttpURLConnection;
+import java.net.MalformedURLException;
+import java.net.URL;
+import javax.xml.stream.XMLInputFactory;
+import javax.xml.stream.XMLStreamConstants;
+import javax.xml.stream.XMLStreamException;
+import javax.xml.stream.XMLStreamReader;
 
 /**
  *
@@ -63,6 +77,9 @@
     @EJB
     AuthenticationServiceBean authenticationService;
 
+    @EJB
+    SettingsServiceBean settingsService;
+
     /**
      * The maps below (the objects themselves) are "final", but the
      * values will be populated in @PostConstruct (see below) during 
@@ -117,7 +134,32 @@
                 .getResultList().forEach((row) -> {
                     if(row.isEnabled()) {
                     try {
-                        registerProvider( loadProvider(row) );
+                        AuthenticationProvider authProvider = loadProvider(row);
+
+                        registerProvider( authProvider );
+
+                        // For production Shibboleth instances that are not using 
+                        // the legacy DiscoFeed-based workflow, we need to call 
+                        // shibd to look up and cache its entityID, since it will 
+                        // be needed in order to issue WayFinder service redirects.
+
+                        if ("shib".equals(authProvider.getId())
+                                && !FeatureFlags.SHIBBOLETH_USE_DISCOFEED.enabled()) {
+                            // ... is this a prod. shibboleth instance?
+                            String shibTypeSetting = settingsService.getValueForKey(SettingsServiceBean.Key.DebugShibAccountType, null);
+                            boolean isProduction = shibTypeSetting == null || shibTypeSetting.equals("PRODUCTION");
+
+                            if (isProduction) {
+                                String spEntityId = lookupShibbolethEntityId();
+                                logger.info("Looked up the entityId of the shibboleth service provider (via a call to shibd): "
+                                        + spEntityId);
+                                if (spEntityId == null) {
+                                    // we'll make this educated guess - it may or may not help us later on:
+                                    spEntityId = SystemConfig.getDataverseSiteUrlStatic() + "/sp";
+                                }
+                                ((ShibAuthenticationProvider) authProvider).setServiceProviderEntityId(spEntityId);
+                            }
+                        }
 
                     } catch ( AuthenticationProviderFactoryNotFoundException e ) {
                         logger.log(Level.SEVERE, "Cannot find authentication provider factory with alias '" + e.getFactoryAlias() + "'",e);
@@ -307,5 +349,75 @@
         return oAuth2authenticationProviders.values().stream().anyMatch( s -> s.getId().toLowerCase().contains("orcid") );
     }
     */
+
+    private String lookupShibbolethEntityId() {
+
+        String baseUrl; 
+        if (FeatureFlags.SHIBBOLETH_USE_LOCALHOST.enabled()) {
+            baseUrl = "http://localhost";
+        } else {
+            baseUrl = SystemConfig.getDataverseSiteUrlStatic();
+        }
+
+        String urlString = baseUrl + "/Shibboleth.sso/Metadata";
+
+        URL url = null;
+        try {
+            url = new URL(urlString);
+        } catch (MalformedURLException ex) {
+            logger.warning(ex.toString());
+            return null;
+        }
+
+        if (url == null) {
+            logger.warning("url object was null after parsing " + urlString);
+            return null;
+        }
+
+        HttpURLConnection metadataRequest = null;
+        try {
+            metadataRequest = (HttpURLConnection) url.openConnection();
+        } catch (IOException ex) {
+            logger.warning(ex.toString());
+            return null;
+        }
+        if (metadataRequest == null) {
+            logger.warning("http request was null for a local /Shibboleth.sso/Metadata call");
+            return null;
+        }
+        try {
+            metadataRequest.connect();
+        } catch (IOException ex) {
+            logger.warning(ex.toString());
+            return null;
+        }
+
+        XMLStreamReader xmlr = null;
+
+        try {
+            XMLInputFactory xmlFactory = javax.xml.stream.XMLInputFactory.newInstance();
+            xmlr =  xmlFactory.createXMLStreamReader(new InputStreamReader((InputStream) metadataRequest.getInputStream()));
+
+            while ( xmlr.next() == XMLStreamConstants.COMMENT);
+            xmlr.require(XMLStreamConstants.START_ELEMENT, null, "EntityDescriptor");
+
+            return xmlr.getAttributeValue(null, "entityID");
+
+        } catch (IOException ioex) {
+            logger.warning("IOException instantiating a stream reader of the /Shibboleth.sso/Metadata output" + ioex.getMessage());
+        } catch (XMLStreamException xsex) {
+            logger.warning("Failed to parse the xml output of the /Shibboleth.sso/Metadata; " + xsex.getMessage());
+        } finally {
+            if (xmlr != null) {
+                try {
+                    logger.fine("closing xml reader");
+                    xmlr.close();
+                } catch (XMLStreamException xsex) {
+                    // we don't care
+                }
+            }
+        }
+        return null; 
+    }
 
 }

src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/DataverseUserPage.java

@@ -213,7 +213,7 @@ public String init() {
         } else {
             return permissionsWrapper.notAuthorized();
         }
-
+        
         return "";
     }