Merge pull request #2518 from constantine2nd/develop

Berlin Group features

Author: simonredfern

obp-api/src/main/resources/props/sample.props.template

@@ -165,8 +165,12 @@ jwt.use.ssl=false
 # Bypass TPP signature validation
 # bypass_tpp_signature_validation = false
 
-## Reject Berlin Group Consents in status "received" after defined time
-# berlin_group_outdated_consents_interval = 5
+## Reject Berlin Group consents with status "received" after a defined time (in seconds)
+# berlin_group_outdated_consents_time_in_seconds = 300
+# berlin_group_outdated_consents_interval_in_seconds =
+
+## Expire Berlin Group consents with status "valid"
+# berlin_group_expired_consents_interval_in_seconds =
 
 
 ## Enable writing API metrics (which APIs are called) to RDBMS
@@ -1124,7 +1128,7 @@ default_auth_context_update_request_key=CUSTOMER_NUMBER
 #berlin_group_error_message_show_path = true
 
 # Check presence of the mandatory headers
-#berlin_group_mandatory_headers = X-Request-ID,PSU-IP-Address,PSU-Device-ID,PSU-Device-Name
+#berlin_group_mandatory_headers = Content-Type,Date,Digest,PSU-Device-ID,PSU-Device-Name,PSU-IP-Address,Signature,TPP-Signature-Certificate,X-Request-ID
 #berlin_group_mandatory_header_consent = TPP-Redirect-URL
 
 ## Berlin Group Create Consent Frequency per Day Upper Limit

obp-api/src/main/scala/code/api/ResourceDocs1_4_0/SwaggerDefinitionsJSON.scala

@@ -5508,6 +5508,9 @@ object SwaggerDefinitionsJSON {
   val consumerLogoUrlJson = ConsumerLogoUrlJson(
     "http://localhost:8888"
   )
+  val consumerCertificateJson = ConsumerCertificateJson(
+    "QmFnIEF0dHJpYnV0ZXMNCiAgICBsb2NhbEtleUlEOiBFMSA3RiBCMyBCOCBEQiA4QyA2NCBGNiA4QyA1NSAzNCA3QSAyNiBCRSBEMCBCNCBENCBBMyBGRCA2NiANCnN1YmplY3Q9QyA9IE1ELCBPID0gTUFJQiwgQ04gPSBNQUlCIFByaXNhY2FydSBTZXJnaXUgKFRlc3QpDQoNCmlzc3Vlcj1DID0gTUQsIE8gPSBCTk0sIE9VID0gRFRJLCBDTiA9IEJOTSBDQSAodGVzdCksIGVtYWlsQWRkcmVzcyA9IGFkbWluQGJubS5tZA0KDQotLS0tLUJFR0lOIENFUlRJRklDQVRFLS0tLS0NCk1JSUdoVENDQkcyZ0F3SUJBZ0lDQkRvd0RRWUpLb1pJaHZjTkFRRUZCUUF3WGpFTE1Ba0dBMVVFQmhNQ1RVUXgNCkREQUtCZ05WQkFvTUEwSk9UVEVNTUFvR0ExVUVDd3dEUkZSSk1SWXdGQVlEVlFRRERBMUNUazBnUTBFZ0tIUmwNCmMzUXBNUnN3R1FZSktvWklodmNOQVFrQkZneGhaRzFwYmtCaWJtMHViV1F3SGhjTk1qUXdOREU0TVRFME5qUXgNCldoY05Nall3TkRFNE1URTBOalF4V2pCRE1Rc3dDUVlEVlFRR0V3Sk5SREVOTUFzR0ExVUVDZ3dFVFVGSlFqRWwNCk1DTUdBMVVFQXd3Y1RVRkpRaUJRY21sellXTmhjblVnVTJWeVoybDFJQ2hVWlhOMEtUQ0NBU0l3RFFZSktvWkkNCmh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBTFdYMzlFSmZLNEg5MDZKSVpMbHRxTU56amxDd3NyMm0rZjMNCjVYdHZ4SVY1akEvUWlZSDdDVjBQK0E1U2grKytaNldUb1NnQStQemYwdTdWYWRVbWtyWEZBV0lzOXlPemduUjQNCmZ5TVVSNXR4UWJYdmZYcXVJUS9XQ0ZnRHBIU3I4eWN0UHlsOGdsUjFidVF0UmlTdEdMT0RnalhsTmhTMlhTYTMNCmFwVGhUVHAya3o1dEoyWjBXRnlxa1ZVM1FJNkdNVGU5eWhDdnVZQkI1QWJuUUU4bXVPb2NNaEJkRFREY2ZGdW0NCk5paUozelhLMXZzKzEzNW5sZEMxOXozWnBuaVBSeER2WGthR00wc0xiNnk5T1NIOUdmYTZHcXJnendTTmpubEkNCnZCeWFlK1dtbG16TzlBZXVKNVRaUFhPdzNwcFdpTWdTOVlZOWp1UUtFQUFBQUFBQTQ4c0NBd0VBQWFPQ0FtWXcNCmdnSmlNQkVHQ1dDR1NBR0crRUlCQVFRRUF3SUZvREFwQmdOVkhTVUVJakFnQmdnckJnRUZCUWNEQWdZSUt3WUINCkJRVUhBd1FHQ2lzR0FRUUJnamNVQWdJd0hRWURWUjBPQkJZRUZGR2ptcXM4OXUyMXcvZmNHVlgrb0pNZSsvWTYNCk1JR1FCZ05WSFNNRWdZZ3dnWVdBRkh1ckdvcWhWYVFUVkJwRVlObmNnRUl5Vkd3dG9XS2tZREJlTVFzd0NRWUQNClZRUUdFd0pOUkRFTU1Bb0dBMVVFQ2d3RFFrNU5NUXd3Q2dZRFZRUUxEQU5FVkVreEZqQVVCZ05WQkFNTURVSk8NClRTQkRRU0FvZEdWemRDa3hHekFaQmdrcWhraUc5dzBCQ1FFV0RHRmtiV2x1UUdKdWJTNXRaSUlKQUpuU0UxdVoNCkU1MU5NQlFHQTFVZEVnUU5NQXVCQ1VOQlFHSnViUzV0WkRBMkJnbGdoa2dCaHZoQ0FRUUVLUlluYUhSMGNEb3YNCkwzQnJhUzVpYm0wdWJXUXZjR3RwTDNCMVlpOWpjbXd2WTJGamNtd3VZM0pzTURZR0NXQ0dTQUdHK0VJQkF3UXANCkZpZG9kSFJ3T2k4dmNHdHBMbUp1YlM1dFpDOXdhMmt2Y0hWaUwyTnliQzlqWVdOeWJDNWpjbXd3T0FZRFZSMGYNCkJERXdMekF0b0N1Z0tZWW5hSFIwY0RvdkwzQnJhUzVpYm0wdWJXUXZjR3RwTDNCMVlpOWpjbXd2WTJGamNtd3UNClkzSnNNRU1HQ0NzR0FRVUZCd0VCQkRjd05UQXpCZ2dyQmdFRkJRY3dBb1luYUhSMGNEb3ZMM2QzZHk1aWJtMHUNCmJXUXZjSFZpTDJOaFkyVnlkQzlqWVdObGNuUXVZM0owTUZBR0NXQ0dTQUdHK0VJQkRRUkRGa0ZMWlhrZ1VHRnANCmNpQkhaVzVsY21GMFpXUWdZbmtnVlc1cFEzSjVjSFFnZGk0d0xqWXVPQzQySUdadmNpQlFTME5USXpFeUxXWnANCmJHVWdjM1J2Y21GblpUQUpCZ05WSFJNRUFqQUFNQTRHQTFVZER3RUIvd1FFQXdJR1FEQU5CZ2txaGtpRzl3MEINCkFRVUZBQU9DQWdFQUpTU0ZhRWZOOWVna2wyYVFEc3QvVEtWWmxSbFdWZWkrVmZwMnM1ZXpWNG9ibnZRUXI5QkcNCmZrNklqaU8zbGZHTjQyTkVZSTV6SGh3SDl2WTRiMjM2ZkdMZWltbmZDc2lGb0FyTEtGUDR6Y0dvS0ZJR2ZBNDINCnQzSmxIcENvbmNpMmxqUzg4MzN2c1k5M2xGSzFTa2NvUjBMT0s0NzdaNlBWMjVtdjVjdmhCN1ZkNWs4SWpLU3MNCllwWkpaSi9STWZNT3dPQUtqeDFhWDNxQUhhNVhTOUNINEJaMEl4SnBYcWZpMm5GUFVNRy8yU0JmSTN4dDhsM1UNClJtVy9qZVRoRG5tL0Vsb05sb3pObzdRS3AvbysyUVBFZDBUWkFBdUljQWFiM09waUptOWlrUlh3c21mNkFmS0INCnIwQmtHcTFiTi9RQk1DMDM4RHA4S1pKZmdmaTYxYnBiVUNFdDRsVWY0R252TW9FdjZnbTh1czE2VTI1d0Y0SUwNCnd5cmFBZHJUVHhaWEVydGY2c3pWY2JBRUY0QmdFM0hCVmF2V2FxbDZac1FFRFJoTGVtWVJwMHhleUtwYXI4d3INClhqN1oycmJteWpFci9ES1hMdHF2UlFIQVVrVDBEQXRST2R4NmpsNUtGSFVvbTM2QUZmeU5UcjJ6a0p2MkZWTlENCmc0TnJMRnk0WldidE84ZDc2M2NoMEpjaWYzZUdadnFmQnVETUs3Q25jUWluamxVcTg1cFpzeGlFUW56VTJOdGgNClRFUzBqZjZ6ZS9ibHpVaUsrRXlyeWpEeWNaYlk3RHlwWWVlTlJJbk9zVUVjZmtFT3BVL3dFTG83dnpNaGY1b2MNCmdjcUFKSzdOQWlEQzVHR0Iyb296ZzNSTTJBbGdPT1ZpRFZwRzRMaUxPenpqVStqaXlyclY3OGs9DQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tDQo="
+  )
   val consumerNameJson = ConsumerNameJson(
     "App name"
   )

obp-api/src/main/scala/code/api/berlin/group/v1_3/AccountInformationServiceAISApi.scala

@@ -242,10 +242,10 @@ recurringIndicator:
            for {
              (Full(user), callContext) <- authenticatedAccess(cc)
              _ <- passesPsd2Aisp(callContext)
-             consent <- Future(Consents.consentProvider.vend.getConsentByConsentId(consentId)) map {
+             _ <- Future(Consents.consentProvider.vend.getConsentByConsentId(consentId)) map {
                unboxFullOrFail(_, callContext, ConsentNotFound)
              }
-             consent <- Future(Consents.consentProvider.vend.revoke(consentId)) map {
+             _ <- Future(Consents.consentProvider.vend.revokeBerlinGroupConsent(consentId)) map {
                i => connectorEmptyResponse(i, callContext)
              }
            } yield {
@@ -694,8 +694,10 @@ where the consent was directly managed between ASPSP and PSU e.g. in a re-direct
              consent <- Future(Consents.consentProvider.vend.getConsentByConsentId(consentId)) map {
                unboxFullOrFail(_, callContext, s"$ConsentNotFound ($consentId)")
              }
-             _ <- Helper.booleanToFuture(failMsg = s"${consent.mConsumerId.get} != ${cc.consumer.map(_.consumerId.get).getOrElse("None")}", failCode = 404, cc = cc.callContext) {
-               consent.mConsumerId.get == callContext.map(_.consumer.map(_.consumerId.get).getOrElse("None")).getOrElse("None")
+             consumerIdFromConsent = consent.mConsumerId.get
+             consumerIdFromCurrentCall = callContext.map(_.consumer.map(_.consumerId.get).getOrElse("None")).getOrElse("None")
+             _ <- Helper.booleanToFuture(failMsg = s"$ConsentNotFound $consumerIdFromConsent != $consumerIdFromCurrentCall", failCode = 403, cc = cc.callContext) {
+               consumerIdFromConsent == consumerIdFromCurrentCall
              }
            } yield {
              (createGetConsentResponseJson(consent), HttpCode.`200`(callContext))
@@ -767,8 +769,7 @@ This method returns the SCA status of a consent initiation's authorisation sub-r
                unboxFullOrFail(_, callContext, ConsentNotFound)
              }
            } yield {
-             val status = consent.status.toLowerCase()
-               .replace(ConsentStatus.REVOKED.toString.toLowerCase(), "revokedByPsu")
+             val status = consent.status
              (JSONFactory_BERLIN_GROUP_1_3.ConsentStatusJsonV13(status), HttpCode.`200`(callContext))
            }
 

obp-api/src/main/scala/code/api/berlin/group/v1_3/BgSpecValidation.scala

@@ -48,19 +48,19 @@ object BgSpecValidation {
   // Example usage
   def main(args: Array[String]): Unit = {
     val testDates = Seq(
-      "2025-05-10",  // ❌ More than 180 days ahead
-      "9999-12-31",  // ❌ Exceeds max allowed
-      "2015-01-01",  // ❌ In the past
-      "invalid-date", // ❌ Invalid format
-      LocalDate.now().plusDays(90).toString,  // ✅ Valid (within 180 days)
-      LocalDate.now().plusDays(180).toString, // ✅ Valid (exactly 180 days)
-      LocalDate.now().plusDays(181).toString  // ❌ More than 180 days
+      "2025-05-10",  // More than 180 days ahead
+      "9999-12-31",  // Exceeds max allowed
+      "2015-01-01",  // In the past
+      "invalid-date", // Invalid format
+      LocalDate.now().plusDays(90).toString,  // Valid (within 180 days)
+      LocalDate.now().plusDays(180).toString, // Valid (exactly 180 days)
+      LocalDate.now().plusDays(181).toString  // More than 180 days
     )
 
     testDates.foreach { date =>
       validateValidUntil(date) match {
-        case Right(validDate) => println(s"✅ Valid date: $validDate")
-        case Left(error)      => println(s"❌ Error: $error")
+        case Right(validDate) => println(s"Valid date: $validDate")
+        case Left(error)      => println(s"Error: $error")
       }
     }
   }

obp-api/src/main/scala/code/api/util/APIUtil.scala

@@ -254,16 +254,6 @@ object APIUtil extends MdcLoggable with CustomJsonFormats{
       case _ => None
     }
   }
-  /**
-   * Purpose of this helper function is to get the PSD2-CERT value from a Request Headers.
-   * @return the PSD2-CERT value from a Request Header as a String
-   */
-  def getTppSignatureCertificate(requestHeaders: List[HTTPParam]): Option[String] = {
-    requestHeaders.toSet.filter(_.name == RequestHeader.`TPP-Signature-Certificate`).toList match {
-      case x :: Nil => Some(x.values.mkString(", "))
-      case _ => None
-    }
-  }
 
   def getRequestHeader(name: String, requestHeaders: List[HTTPParam]): String = {
     requestHeaders.toSet.filter(_.name.toLowerCase == name.toLowerCase).toList match {
@@ -527,22 +517,31 @@ object APIUtil extends MdcLoggable with CustomJsonFormats{
    *
    */
   def getRequestHeadersToMirror(callContext: Option[CallContextLight]): CustomResponseHeaders = {
+    val mirrorByProperties = getPropsValue("mirror_request_headers_to_response", "").split(",").toList.map(_.trim)
+
     val mirrorRequestHeadersToResponse: List[String] =
-      getPropsValue("mirror_request_headers_to_response", "").split(",").toList.map(_.trim)
+      if (callContext.exists(_.url.contains(ApiVersion.berlinGroupV13.urlPrefix))) {
+        // Berlin Group Specification
+        RequestHeader.`X-Request-ID` :: mirrorByProperties
+      } else {
+        mirrorByProperties
+      }
+
     callContext match {
       case Some(cc) =>
         cc.requestHeaders match {
           case Nil => CustomResponseHeaders(Nil)
           case _   =>
             val headers = cc.requestHeaders
               .filter(item => mirrorRequestHeadersToResponse.contains(item.name))
-              .map(item => (item.name, item.values.head))
+              .map(item => (item.name, item.values.headOption.getOrElse(""))) // Safe extraction
             CustomResponseHeaders(headers)
         }
       case None =>
         CustomResponseHeaders(Nil)
     }
   }
+
   /**
    *
    */
@@ -2993,15 +2992,23 @@ object APIUtil extends MdcLoggable with CustomJsonFormats{
     val remoteIpAddress = getRemoteIpAddress()
 
     val authHeaders = AuthorisationUtil.getAuthorisationHeaders(reqHeaders)
+    val authHeadersWithEmptyValues = RequestHeadersUtil.checkEmptyRequestHeaderValues(reqHeaders)
+    val authHeadersWithEmptyNames = RequestHeadersUtil.checkEmptyRequestHeaderNames(reqHeaders)
 
     // Identify consumer via certificate
-    val consumerByCertificate = Consent.getCurrentConsumerViaMtls(callContext = cc)
+    val consumerByCertificate = Consent.getCurrentConsumerViaTppSignatureCertOrMtls(callContext = cc)
 
     val res =
-      if (authHeaders.size > 1) { // Check Authorization Headers ambiguity
+      if (authHeadersWithEmptyValues.nonEmpty) { // Check Authorization Headers Empty Values
+        val message = ErrorMessages.EmptyRequestHeaders + s"Header names: ${authHeadersWithEmptyValues.mkString(", ")}"
+        Future { (fullBoxOrException(Empty ~> APIFailureNewStyle(message, 400, Some(cc.toLight))), None) }
+      } else if (authHeadersWithEmptyNames.nonEmpty) { // Check Authorization Headers Empty Names
+        val message = ErrorMessages.EmptyRequestHeaders + s"Header values: ${authHeadersWithEmptyNames.mkString(", ")}"
+        Future { (fullBoxOrException(Empty ~> APIFailureNewStyle(message, 400, Some(cc.toLight))), None) }
+      } else if (authHeaders.size > 1) { // Check Authorization Headers ambiguity
         Future { (Failure(ErrorMessages.AuthorizationHeaderAmbiguity + s"${authHeaders}"), None) }
       } else if (APIUtil.`hasConsent-ID`(reqHeaders)) { // Berlin Group's Consent
-        Consent.applyBerlinGroupRules(APIUtil.`getConsent-ID`(reqHeaders), cc)
+        Consent.applyBerlinGroupRules(APIUtil.`getConsent-ID`(reqHeaders), cc.copy(consumer = consumerByCertificate))
       } else if (APIUtil.hasConsentJWT(reqHeaders)) { // Open Bank Project's Consent
         val consentValue = APIUtil.getConsentJWT(reqHeaders)
         Consent.getConsentJwtValueByConsentId(consentValue.getOrElse("")) match {
@@ -3011,12 +3018,12 @@ object APIUtil extends MdcLoggable with CustomJsonFormats{
               // Note: At this point we are getting the Consumer from the Consumer in the Consent.
               // This may later be cross checked via the value in consumer_validation_method_for_consent.
               // Get the source of truth for Consumer (e.g. CONSUMER_CERTIFICATE) as early as possible.
-              cc.copy(consumer = Consent.getCurrentConsumerViaMtls(callContext = cc))
+              cc.copy(consumer = consumerByCertificate)
             )
           case _ =>
             JwtUtil.checkIfStringIsJWTValue(consentValue.getOrElse("")).isDefined match {
               case true => // It's JWT obtained via "Consent-JWT" request header
-                Consent.applyRules(APIUtil.getConsentJWT(reqHeaders), cc)
+                Consent.applyRules(APIUtil.getConsentJWT(reqHeaders), cc.copy(consumer = consumerByCertificate))
               case false => // Unrecognised consent value
                 Future { (Failure(ErrorMessages.ConsentHeaderValueInvalid), None) }
             }
@@ -3115,8 +3122,7 @@ object APIUtil extends MdcLoggable with CustomJsonFormats{
       }
       else if(Option(cc).flatMap(_.user).isDefined) {
         Future{(cc.user, Some(cc))}
-      }
-      else {
+      } else {
         if(hasAuthorizationHeader(reqHeaders)) {
           // We want to throw error in case of wrong or unsupported header. For instance:
           // - Authorization: mF_9.B5f-4.1JqM

obp-api/src/main/scala/code/api/util/ApiRole.scala

@@ -243,6 +243,8 @@ object ApiRole extends MdcLoggable{
 
   case class CanUpdateConsumerLogoUrl(requiresBankId: Boolean = false) extends ApiRole
   lazy val canUpdateConsumerLogoUrl = CanUpdateConsumerLogoUrl()
+  case class CanUpdateConsumerCertificate(requiresBankId: Boolean = false) extends ApiRole
+  lazy val canUpdateConsumerCertificate = CanUpdateConsumerCertificate()
   case class CanUpdateConsumerName(requiresBankId: Boolean = false) extends ApiRole
   lazy val canUpdateConsumerName = CanUpdateConsumerName()
 

obp-api/src/main/scala/code/api/util/BerlinGroupCheck.scala

@@ -9,8 +9,10 @@ import net.liftweb.http.provider.HTTPParam
 
 object BerlinGroupCheck {
 
+
+  private val defaultMandatoryHeaders = "Content-Type,Date,Digest,PSU-Device-ID,PSU-Device-Name,PSU-IP-Address,Signature,TPP-Signature-Certificate,X-Request-ID"
   // Parse mandatory headers from a comma-separated string
-  private val berlinGroupMandatoryHeaders: List[String] = APIUtil.getPropsValue("berlin_group_mandatory_headers", defaultValue = "X-Request-ID,PSU-IP-Address,PSU-Device-ID,PSU-Device-Name")
+  private val berlinGroupMandatoryHeaders: List[String] = APIUtil.getPropsValue("berlin_group_mandatory_headers", defaultValue = defaultMandatoryHeaders)
     .split(",")
     .map(_.trim.toLowerCase)
     .toList.filterNot(_.isEmpty)

obp-api/src/main/scala/code/api/util/BerlinGroupError.scala

@@ -56,6 +56,9 @@ object BerlinGroupError {
       case "401" if message.contains("OBP-20207") => "PSU_CREDENTIALS_INVALID"
 
       case "401" if message.contains("OBP-20204") => "TOKEN_EXPIRED"
+      case "401" if message.contains("OBP-20215") => "TOKEN_INVALID"
+      case "401" if message.contains("OBP-20205") => "TOKEN_INVALID"
+      case "401" if message.contains("OBP-20204") => "TOKEN_INVALID"
 
       case "401" if message.contains("OBP-35003") => "CONSENT_EXPIRED"
 
@@ -66,6 +69,11 @@ object BerlinGroupError {
       case "401" if message.contains("OBP-35018") => "CONSENT_INVALID"
       case "401" if message.contains("OBP-35005") => "CONSENT_INVALID"
 
+      case "403" if message.contains("OBP-35001") => "CONSENT_UNKNOWN"
+
+      case "401" if message.contains("OBP-20312") => "CERTIFICATE_INVALID"
+      case "401" if message.contains("OBP-20310") => "SIGNATURE_INVALID"
+
       case "401" if message.contains("OBP-20060") => "ROLE_INVALID"
 
       case "400" if message.contains("OBP-35018") => "CONSENT_UNKNOWN"
@@ -78,6 +86,8 @@ object BerlinGroupError {
       case "400" if message.contains("OBP-10001") => "FORMAT_ERROR"
       case "400" if message.contains("OBP-20062") => "FORMAT_ERROR"
       case "400" if message.contains("OBP-20063") => "FORMAT_ERROR"
+      case "400" if message.contains("OBP-20252") => "FORMAT_ERROR"
+      case "400" if message.contains("OBP-20251") => "FORMAT_ERROR"
 
       case "429" if message.contains("OBP-10018") => "ACCESS_EXCEEDED"
       case _ => code

obp-api/src/main/scala/code/api/util/BerlinGroupSigning.scala

@@ -156,7 +156,8 @@ object BerlinGroupSigning extends MdcLoggable {
   }
 
   def getHeaderValue(name: String, requestHeaders: List[HTTPParam]): String = {
-    requestHeaders.find(_.name.toLowerCase() == name.toLowerCase()).map(_.values.mkString).getOrElse("None")
+    requestHeaders.find(_.name.toLowerCase() == name.toLowerCase()).map(_.values.mkString)
+      .getOrElse(SecureRandomUtil.csprng.nextLong().toString)
   }
   private def getPem(requestHeaders: List[HTTPParam]): String = {
     val certificate = getHeaderValue(RequestHeader.`TPP-Signature-Certificate`, requestHeaders)