PIL-2624 - Update BTN to handle raw response from ETMP (#639)

JamesMMiller · web-flow · commit c7a11fbe32cf · 2026-02-27T10:58:28.000Z
This PR implements the frontend portion of the BTN Audit refactoring
(PIL-2624).

**Key Changes:**
1. **Raw Response Handling**: The `BTNConnector` and `BTNService` now
return the raw `HttpResponse` from the `pillar2` backend, which is
acting as a transparent proxy.
2. **Enhanced Auditing**: The `BtnSubmissionService` has been updated to
capture the exact status code and response body from the downstream ETMP
service.
3. **Audit Model Updates**: Added `ApiResponseData.fromHttpResponse` to
correctly parse raw responses for audit events.
4. **Error Handling**: Logic for handling success (201) vs failure
(non-201) has been moved here from the backend.
5. **Test Updates**: Comprehensive updates to `BTNConnectorSpec`,
`BTNServiceSpec`, `BtnSubmissionServiceSpec`, and
`CheckYourAnswersControllerSpec` to mock and verify `HttpResponse`
objects and ensure compilation success.

**Motivation:**
To improve visibility into downstream errors and ensure the audit trail
accurately reflects the raw response from ETMP.
diff --git a/app/connectors/BTNConnector.scala b/app/connectors/BTNConnector.scala
@@ -16,12 +16,9 @@
 
 package connectors
 
-import cats.syntax.either.*
 import config.FrontendAppConfig
-import models.InternalIssueError
 import models.btn.*
 import play.api.Logging
-import play.api.http.Status.*
 import play.api.libs.json.*
 import play.api.libs.ws.JsonBodyWritables.writeableOf_JsValue
 import uk.gov.hmrc.http.*
@@ -31,12 +28,11 @@ import uk.gov.hmrc.http.{HeaderCarrier, HttpResponse}
 
 import javax.inject.Inject
 import scala.concurrent.{ExecutionContext, Future}
-import scala.util.Try
 
 class BTNConnector @Inject() (val config: FrontendAppConfig, val httpClientV2: HttpClientV2) extends Logging {
   def submitBTN(
     btnRequest: BTNRequest
-  )(using hc: HeaderCarrier, pillar2Id: String, ec: ExecutionContext): Future[BtnResponse] = {
+  )(using hc: HeaderCarrier, pillar2Id: String, ec: ExecutionContext): Future[HttpResponse] = {
     val urlBTN = s"${config.pillar2BaseUrl}/report-pillar2-top-up-taxes/below-threshold-notification/submit"
 
     logger.info(s"Calling pillar-2 backend url = $urlBTN with pillar2Id: $pillar2Id.")
@@ -45,22 +41,5 @@ class BTNConnector @Inject() (val config: FrontendAppConfig, val httpClientV2: H
       .withBody(Json.toJson(btnRequest))
       .setHeader("X-Pillar2-Id" -> pillar2Id)
       .execute[HttpResponse]
-      .flatMap { (response: HttpResponse) =>
-        response.status match {
-          case CREATED =>
-            logger.info(s"submitBTN request successful with status = ${response.status}. HttpResponse = ${response.body}. ")
-            Future.fromTry(Try(BtnResponse(response.json.as[BtnSuccess].asRight, httpStatusCode = response.status)))
-          case BAD_REQUEST | UNPROCESSABLE_ENTITY | INTERNAL_SERVER_ERROR =>
-            logger.warn(
-              s"submitBTN failed with handled status = ${response.status} for pillar2Id $pillar2Id and (accountingPeriodFrom, To) = $btnRequest."
-            )
-            Future.fromTry(Try(BtnResponse(response.json.as[BtnError].asLeft, httpStatusCode = response.status)))
-          case _ =>
-            logger.error(
-              s"submitBTN failed with unexpected status = ${response.status} for pillar2Id $pillar2Id and (accountingPeriodFrom, To) = $btnRequest."
-            )
-            Future.failed(InternalIssueError)
-        }
-      }
   }
 }
diff --git a/app/models/audit/AuditBtn.scala b/app/models/audit/AuditBtn.scala
@@ -16,12 +16,16 @@
 
 package models.audit
 
+import models.btn.BTNSuccessResponse
 import models.btn.BtnResponse
+import models.hip.ApiFailureResponse
 import models.subscription.AccountingPeriod
 import play.api.libs.functional.syntax.*
 import play.api.libs.json.*
+import uk.gov.hmrc.http.HttpResponse
 
 import java.time.*
+import scala.util.Try
 
 case class CreateBtnAuditEvent(
   pillarReference:            String,
@@ -74,6 +78,47 @@ object ApiResponseData {
       )
   }
 
+  def fromHttpResponse(response: HttpResponse)(using clock: Clock): ApiResponseData = {
+
+    val jsonOpt = Try(response.json).toOption
+
+    response.status match {
+      case 201 =>
+        val processingDate = jsonOpt
+          .flatMap(_.validate[BTNSuccessResponse].asOpt)
+          .map(_.success.processingDate)
+          .getOrElse(ZonedDateTime.now(clock))
+        ApiResponseSuccess(response.status, processingDate)
+
+      case _ =>
+        val (errorCode, message) = jsonOpt.flatMap(_.validate[ApiFailureResponse].asOpt) match {
+          case Some(etmpError) =>
+            (etmpError.errors.code, etmpError.errors.text)
+          case None =>
+            // Fallback for non-422 errors (like 400/500 from API platform/ETMP)
+            val jsonFallbackCode = jsonOpt
+              .flatMap(j =>
+                (j \ "code")
+                  .asOpt[String]
+                  .orElse((j \ "failures" \ 0 \ "code").asOpt[String])
+              )
+              .getOrElse("UNKNOWN")
+
+            val jsonFallbackMessage = jsonOpt
+              .flatMap(j =>
+                (j \ "message")
+                  .asOpt[String]
+                  .orElse((j \ "failures" \ 0 \ "reason").asOpt[String])
+              )
+              .getOrElse(response.body)
+
+            (jsonFallbackCode, jsonFallbackMessage)
+        }
+
+        ApiResponseFailure(response.status, ZonedDateTime.now(clock), errorCode, message)
+    }
+  }
+
   given writes: Writes[ApiResponseData] = {
     case success @ ApiResponseSuccess(_, _)       => Json.toJson(success)
     case failure @ ApiResponseFailure(_, _, _, _) => Json.toJson(failure)
diff --git a/app/models/btn/BTNSuccessResponse.scala b/app/models/btn/BTNSuccessResponse.scala
@@ -0,0 +1,31 @@
+/*
+ * Copyright 2024 HM Revenue & Customs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package models.btn
+
+import play.api.libs.json.{Json, OFormat}
+
+import java.time.ZonedDateTime
+
+case class BTNSuccess(processingDate: ZonedDateTime)
+object BTNSuccess {
+  given format: OFormat[BTNSuccess] = Json.format[BTNSuccess]
+}
+
+case class BTNSuccessResponse(success: BTNSuccess)
+object BTNSuccessResponse {
+  given format: OFormat[BTNSuccessResponse] = Json.format[BTNSuccessResponse]
+}
diff --git a/app/models/hip/ApiFailureResponse.scala b/app/models/hip/ApiFailureResponse.scala
@@ -0,0 +1,31 @@
+/*
+ * Copyright 2024 HM Revenue & Customs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package models.hip
+
+import play.api.libs.json.{Json, OFormat}
+
+import java.time.ZonedDateTime
+
+case class ApiFailure(processingDate: ZonedDateTime, code: String, text: String)
+object ApiFailure {
+  given format: OFormat[ApiFailure] = Json.format[ApiFailure]
+}
+
+case class ApiFailureResponse(errors: ApiFailure)
+object ApiFailureResponse {
+  given format: OFormat[ApiFailureResponse] = Json.format[ApiFailureResponse]
+}
diff --git a/app/services/BTNService.scala b/app/services/BTNService.scala
@@ -17,10 +17,9 @@
 package services
 
 import connectors.*
-import models.InternalIssueError
-import models.btn.{BTNRequest, BtnResponse}
+import models.btn.BTNRequest
 import play.api.Logging
-import uk.gov.hmrc.http.HeaderCarrier
+import uk.gov.hmrc.http.{HeaderCarrier, HttpResponse}
 
 import javax.inject.{Inject, Singleton}
 import scala.concurrent.{ExecutionContext, Future}
@@ -31,24 +30,6 @@ class BTNService @Inject() (
 )(using ec: ExecutionContext)
     extends Logging {
 
-  def submitBTN(btnRequest: BTNRequest)(using headerCarrier: HeaderCarrier, pillar2Id: String): Future[BtnResponse] =
-    btnConnector
-      .submitBTN(btnRequest)
-      .map { btnResponse =>
-        btnResponse.result match {
-          case Left(failure) =>
-            logger.info(
-              s"BTN Request Submission failed with ${failure.errorCode}: ${failure.message}"
-            )
-          case Right(success) =>
-            logger.info(
-              s"BTN Request Submission was successful. Processed ${success.processingDate}"
-            )
-        }
-        btnResponse
-      }
-      .recoverWith { case ex: Throwable =>
-        logger.warn(s"BTNService Request failed with an exception: " + ex)
-        Future.failed(InternalIssueError)
-      }
+  def submitBTN(btnRequest: BTNRequest)(using headerCarrier: HeaderCarrier, pillar2Id: String): Future[HttpResponse] =
+    btnConnector.submitBTN(btnRequest)
 }
diff --git a/app/services/BtnSubmissionService.scala b/app/services/BtnSubmissionService.scala
@@ -77,8 +77,8 @@ class BtnSubmissionService @Inject() (
       .flatMap { resp =>
         sessionRepository.get(userId).flatMap {
           case Some(latest) =>
-            resp.result match {
-              case Right(_) =>
+            resp.status match {
+              case 201 =>
                 for {
                   submittedAnswers <- Future.fromTry {
                                         latest
@@ -90,18 +90,18 @@ class BtnSubmissionService @Inject() (
                          pillarReference = pillar2Id,
                          accountingPeriod = accountingPeriod,
                          entitiesInsideAndOutsideUK = originalAnswers.get(EntitiesInsideOutsideUKPage).getOrElse(false),
-                         response = ApiResponseData.fromBtnResponse(resp)(using clock)
+                         response = ApiResponseData.fromHttpResponse(resp)(using clock)
                        )
                 } yield ()
-              case Left(_) =>
+              case _ =>
                 for {
                   errorAnswers <- Future.fromTry(latest.set(BTNStatus, BTNStatus.error))
                   _            <- sessionRepository.set(errorAnswers)
                   _            <- auditService.auditBTNSubmission(
                          pillarReference = pillar2Id,
                          accountingPeriod = accountingPeriod,
                          entitiesInsideAndOutsideUK = originalAnswers.get(EntitiesInsideOutsideUKPage).getOrElse(false),
-                         response = ApiResponseData.fromBtnResponse(resp)(using clock)
+                         response = ApiResponseData.fromHttpResponse(resp)(using clock)
                        )
                 } yield ()
             }
diff --git a/test/connectors/BTNConnectorSpec.scala b/test/connectors/BTNConnectorSpec.scala
@@ -17,8 +17,6 @@
 package connectors
 
 import base.{SpecBase, WireMockServerHandler}
-import cats.syntax.either.*
-import models.InternalIssueError
 import models.btn.*
 import org.scalacheck.Arbitrary.arbitrary
 import org.scalacheck.Gen
@@ -29,7 +27,7 @@ import play.api.inject.guice.GuiceApplicationBuilder
 import play.api.libs.json.*
 import uk.gov.hmrc.http.test.WireMockSupport
 
-import java.time.{LocalDate, ZonedDateTime}
+import java.time.LocalDate
 
 class BTNConnectorSpec extends SpecBase with WireMockSupport with WireMockServerHandler with EitherValues with ScalaCheckDrivenPropertyChecks {
   override lazy val app: Application = new GuiceApplicationBuilder()
@@ -41,23 +39,22 @@ class BTNConnectorSpec extends SpecBase with WireMockSupport with WireMockServer
     accountingPeriodFrom = LocalDate.now.minusYears(1),
     accountingPeriodTo = LocalDate.now
   )
-  val rawProcessedDateTime:               String        = "2022-01-31T09:26:17Z"
-  val stubProcessedDateTime:              ZonedDateTime = ZonedDateTime.parse(rawProcessedDateTime)
-  val successfulBTNResponseBody:          JsObject      = Json.obj("processingDate" -> rawProcessedDateTime)
-  val successfulBtnResponse:              BtnResponse   = BtnResponse(BtnSuccess(stubProcessedDateTime).asRight, CREATED)
-  val accountingPeriodFromDateMinus1Year: LocalDate     = LocalDate.now.minusYears(1)
-  val accountingPeriodToDateNow:          LocalDate     = LocalDate.now
-  val btnRequestDatesMinus1YearAndNow:    BTNRequest    = BTNRequest(accountingPeriodFromDateMinus1Year, accountingPeriodToDateNow)
+  val rawProcessedDateTime:               String     = "2022-01-31T09:26:17Z"
+  val successfulBTNResponseBody:          JsObject   = Json.obj("processingDate" -> rawProcessedDateTime)
+  val accountingPeriodFromDateMinus1Year: LocalDate  = LocalDate.now.minusYears(1)
+  val accountingPeriodToDateNow:          LocalDate  = LocalDate.now
+  val btnRequestDatesMinus1YearAndNow:    BTNRequest = BTNRequest(accountingPeriodFromDateMinus1Year, accountingPeriodToDateNow)
 
   "submit BTN connector" should {
-    "return a BtnSuccess when the pillar-2 backend has returned status 201." in {
+    "return the response when the pillar-2 backend has returned status 201." in {
       given pillar2Id: String = "XEPLR0000000000"
       stubResponse(submitBTNPath, CREATED, successfulBTNResponseBody.toString())
       val result = connector.submitBTN(btnRequestDatesMinus1YearAndNow).futureValue
-      result mustBe successfulBtnResponse
+      result.status mustBe CREATED
+      result.body mustBe successfulBTNResponseBody.toString()
     }
 
-    "surface the failure details when dealing with a modelled 400 or 500 error" in forAll(
+    "return the response when dealing with a 400 or 500 error" in forAll(
       Gen.oneOf(BAD_REQUEST, INTERNAL_SERVER_ERROR),
       arbitrary[String],
       arbitrary[String]
@@ -69,13 +66,11 @@ class BTNConnectorSpec extends SpecBase with WireMockSupport with WireMockServer
       given pillar2Id: String = "XEPLR0000000000"
       stubResponse(submitBTNPath, httpStatusCode, jsonResponse.toString())
       val result = connector.submitBTN(btnRequestDatesMinus1YearAndNow).futureValue
-      result mustBe BtnResponse(
-        BtnError(errorCode, errorMessage).asLeft,
-        httpStatusCode
-      )
+      result.status mustBe httpStatusCode
+      result.body mustBe jsonResponse.toString()
     }
 
-    "surface the failure details when dealing with a modelled 422 error" in forAll { (errorCode: String, errorMessage: String) =>
+    "return the response when dealing with a 422 error" in forAll { (errorCode: String, errorMessage: String) =>
       val jsonResponse = Json.obj(
         "processingDate" -> rawProcessedDateTime,
         "code"           -> errorCode,
@@ -84,26 +79,17 @@ class BTNConnectorSpec extends SpecBase with WireMockSupport with WireMockServer
       given pillar2Id: String = "XEPLR0000000000"
       stubResponse(submitBTNPath, UNPROCESSABLE_ENTITY, jsonResponse.toString())
       val result = connector.submitBTN(btnRequestDatesMinus1YearAndNow).futureValue
-      result mustBe BtnResponse(
-        BtnError(errorCode, errorMessage).asLeft,
-        UNPROCESSABLE_ENTITY
-      )
+      result.status mustBe UNPROCESSABLE_ENTITY
+      result.body mustBe jsonResponse.toString()
     }
 
-    "raise an Exception when the expected response field-value-checking fails." in {
-      given pillar2Id: String = "XEPLR9999999999"
-      stubResponse(submitBTNPath, CREATED, Json.obj().toString())
-      val result = connector.submitBTN(btnRequestDatesMinus1YearAndNow).failed.futureValue
-      result mustBe JsResultException(Seq(((__ \ "processingDate"), Seq(JsonValidationError(Seq("error.path.missing"))))))
-    }
-
-    "return InternalIssueError when the pillar-2 backend returns any unsupported status." in forAll(
+    "return the response even when the pillar-2 backend returns any unsupported status." in forAll(
       Gen.posNum[Int].retryUntil(code => !Seq(CREATED, BAD_REQUEST, UNPROCESSABLE_ENTITY, INTERNAL_SERVER_ERROR).contains(code))
     ) { (httpStatus: Int) =>
       given pillar2Id: String = "XEPLR4000000000"
       stubResponse(submitBTNPath, httpStatus, successfulBTNResponseBody.toString())
-      val result = connector.submitBTN(btnRequestDatesMinus1YearAndNow)
-      result.failed.futureValue mustBe InternalIssueError
+      val result = connector.submitBTN(btnRequestDatesMinus1YearAndNow).futureValue
+      result.status mustBe httpStatus
     }
   }
 }
diff --git a/test/controllers/btn/CheckYourAnswersControllerSpec.scala b/test/controllers/btn/CheckYourAnswersControllerSpec.scala
diff --git a/test/services/BTNServiceSpec.scala b/test/services/BTNServiceSpec.scala
diff --git a/test/services/BtnSubmissionServiceSpec.scala b/test/services/BtnSubmissionServiceSpec.scala
