DTR-3713: CIS MRF Business Functions F18d,18f,18g

app/uk/gov/hmrc/constructionindustryscheme/config/AppConfig.scala

@@ -19,11 +19,16 @@ package uk.gov.hmrc.constructionindustryscheme.config
 import javax.inject.{Inject, Singleton}
 import play.api.{Configuration, Environment}
 import uk.gov.hmrc.constructionindustryscheme.utils.SchemaLoader
+import uk.gov.hmrc.play.bootstrap.config.ServicesConfig
 
 import javax.xml.validation.Schema
 
 @Singleton
-class AppConfig @Inject() (val config: Configuration, val environment: Environment) {
+class AppConfig @Inject() (
+  val config: Configuration,
+  val environment: Environment,
+  val servicesConfig: ServicesConfig
+) {
   val appName: String = config.get[String]("appName")
 
   val chrisHost: Seq[String] = config.get[Seq[String]]("submissionPollUrlKnownHosts")
@@ -51,4 +56,6 @@ class AppConfig @Inject() (val config: Configuration, val environment: Environme
   val agentClientCryptoKey: String = config.get[String]("agentClientCrypto.key")
   val cryptoToggle: Boolean        = config.get[Boolean]("encryptionToggle")
 
+  lazy val chrisGatewayUrl: String =
+    servicesConfig.baseUrl("chris") + servicesConfig.getString("microservice.services.chris.submit-url")
 }

app/uk/gov/hmrc/constructionindustryscheme/connectors/ChrisConnector.scala

@@ -25,7 +25,7 @@ import uk.gov.hmrc.constructionindustryscheme.models.{FATAL_ERROR, GovTalkError,
 import uk.gov.hmrc.constructionindustryscheme.services.chris.{ChrisPollXmlMapper, ChrisSubmissionXmlMapper}
 import uk.gov.hmrc.http.client.HttpClientV2
 import uk.gov.hmrc.http.HttpReads.Implicits.*
-import uk.gov.hmrc.http.{HeaderCarrier, HttpResponse, StringContextOps}
+import uk.gov.hmrc.http.{HeaderCarrier, HttpResponse, StringContextOps, UpstreamErrorResponse}
 import uk.gov.hmrc.play.bootstrap.config.ServicesConfig
 
 import scala.concurrent.{ExecutionContext, Future}
@@ -85,70 +85,50 @@ class ChrisConnector @Inject() (
       )
       .withBody(envelope.toString)
       .execute[HttpResponse]
-      .map { resp =>
+      .flatMap { resp =>
         if (is2xx(resp.status)) {
           logger.info(s"[ChrisConnector] corrId=$correlationId status=${resp.status} full-response-body:\n${resp.body}")
+          Future.successful(handle2xxResponse(resp, correlationId))
+        } else if (resp.status >= 500) {
+          Future.failed(UpstreamErrorResponse(resp.body, resp.status, resp.status))
+        } else {
+          Future.successful(httpError(correlationId, resp.body, resp.status))
         }
-        handleResponse(resp, correlationId)
-      }
-      .recover { case NonFatal(e) =>
-        logger.error(
-          s"[ChrisConnector] Transport exception calling $chrisCisReturnUrl corrId=$correlationId: ${e.getClass.getSimpleName}: ${Option(e.getMessage)
-              .getOrElse("")}"
-        )
-        connectionError(correlationId, e)
       }
 
-  private def handleResponse(resp: HttpResponse, correlationId: String): SubmissionResult = {
+  private def handle2xxResponse(resp: HttpResponse, correlationId: String): SubmissionResult = {
     val body = resp.body
-    if (is2xx(resp.status)) {
-      ChrisSubmissionXmlMapper
-        .parse(body)
-        .fold(
-          err => parseError(correlationId, body, err),
-          ok => ok
-        )
-    } else {
-      httpError(correlationId, body, resp.status)
-    }
+    ChrisSubmissionXmlMapper
+      .parse(body)
+      .fold(
+        err => parseError(correlationId, body, err),
+        ok => ok
+      )
   }
 
   private def is2xx(status: Int): Boolean =
     status >= 200 && status < 300
 
+  private def truncate(s: String, maxCharacters: Int = 254): String =
+    if (s.length <= maxCharacters) s else s.take(maxCharacters) + "…"
+
   private def parseError(correlationId: String, rawXml: String, err: String): SubmissionResult =
-    errorResult(
-      correlationId = correlationId,
-      rawXml = rawXml,
-      errorNumber = "parse",
-      errorType = "fatal",
-      errorText = err
-    )
+    errorResult(correlationId = correlationId, rawXml = rawXml, errorNumber = "parse", errorText = err)
 
   private def httpError(correlationId: String, rawXml: String, status: Int): SubmissionResult =
     errorResult(
       correlationId = correlationId,
       rawXml = rawXml,
-      errorNumber = s"http-$status",
-      errorType = "fatal",
+      errorNumber = s"http$status",
       errorText = truncate(rawXml)
     )
 
-  private def connectionError(correlationId: String, e: Throwable): SubmissionResult =
-    errorResult(
-      correlationId = correlationId,
-      rawXml = "<connection-error/>",
-      errorNumber = "conn",
-      errorType = "fatal",
-      errorText = s"Connection error: ${e.getClass.getSimpleName}"
-    )
-
   private def errorResult(
     correlationId: String,
     rawXml: String,
     errorNumber: String,
-    errorType: String,
-    errorText: String
+    errorText: String,
+    errorType: String = "fatal"
   ): SubmissionResult =
     SubmissionResult(
       status = FATAL_ERROR,
@@ -163,7 +143,4 @@ class ChrisConnector @Inject() (
         error = Some(GovTalkError(errorNumber, errorType, errorText))
       )
     )
-
-  private def truncate(s: String, maxCharacters: Int = 254): String =
-    if (s.length <= maxCharacters) s else s.take(maxCharacters) + "…"
 }

app/uk/gov/hmrc/constructionindustryscheme/connectors/FormpProxyConnector.scala

@@ -223,4 +223,36 @@ class FormpProxyConnector @Inject() (
         else Future.failed(UpstreamErrorResponse(response.body, response.status, response.status))
       }
 
+  def getGovTalkStatus(
+    request: GetGovTalkStatusRequest
+  )(implicit hc: HeaderCarrier): Future[Option[GetGovTalkStatusResponse]] =
+    http
+      .post(url"$base/cis/govtalkstatus/get")
+      .withBody(Json.toJson(request))
+      .execute[GetGovTalkStatusResponse]
+      .map(Some(_))
+      .recover {
+        case e: UpstreamErrorResponse if e.statusCode == NOT_FOUND =>
+          None
+      }
+
+  def createGovTalkStatusRecord(request: CreateGovTalkStatusRecordRequest)(implicit hc: HeaderCarrier): Future[Unit] =
+    http
+      .post(url"$base/cis/govtalkstatus/create")
+      .withBody(Json.toJson(request))
+      .execute[HttpResponse]
+      .flatMap { response =>
+        if (response.status == 201) Future.unit
+        else Future.failed(UpstreamErrorResponse(response.body, response.status, response.status))
+      }
+
+  def updateGovTalkStatus(request: UpdateGovTalkStatusRequest)(implicit hc: HeaderCarrier): Future[Unit] =
+    http
+      .post(url"$base/cis/govtalkstatus/update-status")
+      .withBody(Json.toJson(request))
+      .execute[HttpResponse]
+      .flatMap { response =>
+        if (response.status == 204) Future.unit
+        else Future.failed(UpstreamErrorResponse(response.body, response.status, response.status))
+      }
 }

app/uk/gov/hmrc/constructionindustryscheme/controllers/SubmissionController.scala

@@ -20,14 +20,14 @@ import javax.inject.Inject
 import play.api.mvc.*
 import play.api.libs.json.*
 import play.api.mvc.Results.*
+
 import scala.concurrent.{ExecutionContext, Future}
 import play.api.Logging
-import play.api.http.Status.BAD_GATEWAY
 import uk.gov.hmrc.constructionindustryscheme.actions.AuthAction
-import uk.gov.hmrc.constructionindustryscheme.models.{ACCEPTED as AcceptedStatus, BuiltSubmissionPayload, DEPARTMENTAL_ERROR as DepartmentalErrorStatus, FATAL_ERROR as FatalErrorStatus, SUBMITTED as SubmittedStatus, SUBMITTED_NO_RECEIPT as SubmittedNoReceiptStatus, SubmissionResult}
+import uk.gov.hmrc.constructionindustryscheme.models.{ACCEPTED as AcceptedStatus, BuiltSubmissionPayload, DEPARTMENTAL_ERROR as DepartmentalErrorStatus, EmployerReference, FATAL_ERROR as FatalErrorStatus, SUBMITTED as SubmittedStatus, SUBMITTED_NO_RECEIPT as SubmittedNoReceiptStatus, SubmissionResult}
 import uk.gov.hmrc.constructionindustryscheme.config.AppConfig
 import uk.gov.hmrc.play.bootstrap.backend.controller.BackendController
-import uk.gov.hmrc.constructionindustryscheme.models.requests.{ChrisSubmissionRequest, CreateSubmissionRequest, SendSuccessEmailRequest, UpdateSubmissionRequest}
+import uk.gov.hmrc.constructionindustryscheme.models.requests.*
 import uk.gov.hmrc.constructionindustryscheme.services.{AuditService, SubmissionService}
 import uk.gov.hmrc.constructionindustryscheme.services.chris.ChrisSubmissionEnvelopeBuilder
 import uk.gov.hmrc.http.HeaderCarrier
@@ -40,6 +40,7 @@ import uk.gov.hmrc.play.bootstrap.binders.RedirectUrl.*
 import java.time.{Clock, Instant}
 import java.util.UUID
 import scala.util.{Failure, Success}
+import scala.util.control.NonFatal
 
 class SubmissionController @Inject() (
   authorise: AuthAction,
@@ -78,40 +79,7 @@ class SubmissionController @Inject() (
         .validate[ChrisSubmissionRequest]
         .fold(
           errs => Future.successful(BadRequest(Json.obj("message" -> JsError.toJson(errs)))),
-          csr => {
-            logger.info(s"Submitting Nil Monthly Return to ChRIS for UTR=${csr.utr}")
-
-            val correlationId = UUID.randomUUID().toString.replace("-", "").toUpperCase
-            val payload       = ChrisSubmissionEnvelopeBuilder.buildPayload(csr, req, correlationId)
-
-            val monthlyNilReturnRequestJson: JsValue = createMonthlyNilReturnRequestJson(payload)
-            auditService.monthlyNilReturnRequestEvent(monthlyNilReturnRequestJson)
-
-            xmlValidator.validate(payload.irEnvelope) match {
-              case Success(_) =>
-                logger.info(
-                  s"ChRIS XML validation successful. Sending ChRIS submission for a correlationId = $correlationId."
-                )
-                submissionService
-                  .submitToChris(payload)
-                  .map(renderSubmissionResponse(submissionId, payload))
-                  .recover { case ex =>
-                    logger.error("[submitToChris] upstream failure", ex)
-                    val fatalErrorJson           = Json.obj(
-                      "submissionId"      -> submissionId,
-                      "status"            -> "FATAL_ERROR",
-                      "hmrcMarkGenerated" -> payload.irMark,
-                      "error"             -> "upstream-failure"
-                    )
-                    val monthlyNilReturnResponse = AuditResponseReceivedModel(BAD_GATEWAY.toString, fatalErrorJson)
-                    auditService.monthlyNilReturnResponseEvent(monthlyNilReturnResponse)
-                    BadGateway(fatalErrorJson)
-                  }
-              case Failure(e) =>
-                logger.error(s"ChRIS XML validation failed: ${e.getMessage}", e)
-                Future.failed(new RuntimeException(s"XML validation failed: ${e.getMessage}", e))
-            }
-          }
+          csr => handleSubmitToChris(submissionId, csr)
         )
     }
 
@@ -185,6 +153,19 @@ class SubmissionController @Inject() (
         )
     }
 
+  def createMonthlyNilReturnRequestJson(payload: BuiltSubmissionPayload): JsValue =
+    XmlToJsonConvertor.convertXmlToJson(payload.envelope.toString) match {
+      case XmlConversionResult(true, Some(json), _)   => json
+      case XmlConversionResult(false, _, Some(error)) => Json.obj("error" -> error)
+      case _                                          => Json.obj("error" -> "unexpected conversion failure")
+    }
+
+  def createMonthlyNilReturnResponseJson(res: SubmissionResult): JsValue =
+    XmlToJsonConvertor.convertXmlToJson(res.rawXml) match {
+      case XmlConversionResult(true, Some(json), _) => json
+      case _                                        => Json.toJson(res.rawXml)
+    }
+
   private def renderSubmissionResponse(submissionId: String, payload: BuiltSubmissionPayload)(
     res: SubmissionResult
   ): Result = {
@@ -237,17 +218,121 @@ class SubmissionController @Inject() (
     }
   }
 
-  def createMonthlyNilReturnRequestJson(payload: BuiltSubmissionPayload): JsValue =
-    XmlToJsonConvertor.convertXmlToJson(payload.envelope.toString) match {
-      case XmlConversionResult(true, Some(json), _)   => json
-      case XmlConversionResult(false, _, Some(error)) => Json.obj("error" -> error)
-      case _                                          => Json.obj("error" -> "unexpected conversion failure")
+  private def handleSubmitToChris(submissionId: String, csr: ChrisSubmissionRequest)(implicit
+    req: AuthenticatedRequest[JsValue]
+  ): Future[Result] = {
+    val correlationId = UUID.randomUUID().toString.replace("-", "").toUpperCase
+    val payload       = ChrisSubmissionEnvelopeBuilder.buildPayload(csr, req, correlationId)
+
+    auditService.monthlyNilReturnRequestEvent(createMonthlyNilReturnRequestJson(payload))
+
+    xmlValidator.validate(payload.irEnvelope) match {
+      case Failure(e) =>
+        logger.error(s"ChRIS XML validation failed: ${e.getMessage}", e)
+        Future.failed(new RuntimeException(s"XML validation failed: ${e.getMessage}", e))
+
+      case Success(_) =>
+        logger.info(s"ChRIS XML validation successful. Sending ChRIS submission for a correlationId = $correlationId.")
+        submissionService
+          .submitToChris(payload)
+          .flatMap(res => handleChrisResponse(submissionId, csr, correlationId, payload, res))
+          .recoverWith { case NonFatal(ex) =>
+            handleChrisFailure(submissionId, csr, correlationId, payload, ex)
+          }
     }
+  }
 
-  def createMonthlyNilReturnResponseJson(res: SubmissionResult): JsValue =
-    XmlToJsonConvertor.convertXmlToJson(res.rawXml) match {
-      case XmlConversionResult(true, Some(json), _) => json
-      case _                                        => Json.toJson(res.rawXml)
+  private def handleChrisResponse(
+    submissionId: String,
+    csr: ChrisSubmissionRequest,
+    correlationId: String,
+    payload: BuiltSubmissionPayload,
+    res: SubmissionResult
+  )(implicit hc: HeaderCarrier): Future[Result] =
+    validateCorrelationId(correlationId, res.meta.correlationId) match {
+      case Left(reason) =>
+        logger.error(s"Correlation ID validation failed: $reason")
+        Future.successful(
+          BadGateway(withError(baseSubmissionResponseJson(submissionId, payload, correlationId, "FATAL_ERROR"), reason))
+        )
+
+      case Right(_) =>
+        submissionService
+          .initialiseGovTalkStatus(
+            EmployerReference(csr.clientTaxOfficeNumber, csr.clientTaxOfficeRef),
+            submissionId,
+            correlationId,
+            appConfig.chrisGatewayUrl
+          )
+          .map(_ => renderSubmissionResponse(submissionId, payload)(res))
     }
 
+  private def handleChrisFailure(
+    submissionId: String,
+    csr: ChrisSubmissionRequest,
+    correlationId: String,
+    payload: BuiltSubmissionPayload,
+    ex: Throwable
+  )(implicit hc: HeaderCarrier): Future[Result] = {
+    logger.error(s"Received 5xx/Exception from ChRIS, treating as RESUBMIT for submissionId=$submissionId", ex)
+
+    submissionService
+      .initialiseGovTalkStatus(
+        EmployerReference(csr.clientTaxOfficeNumber, csr.clientTaxOfficeRef),
+        submissionId,
+        correlationId,
+        appConfig.chrisGatewayUrl
+      )
+      .flatMap { instanceId =>
+        submissionService
+          .updateGovTalkStatus(
+            UpdateGovTalkStatusRequest(
+              userIdentifier = instanceId,
+              formResultID = submissionId,
+              protocolStatus = "dataRequest"
+            )
+          )
+          .map { _ =>
+            Ok(
+              withError(baseSubmissionResponseJson(submissionId, payload, correlationId, "STARTED"), "Chris failure")
+            )
+          }
+      }
+      .recover { case ex =>
+        logger.error(s"Failed to initialise/update GovTalk status after 5xx", ex)
+        InternalServerError(
+          withError(
+            baseSubmissionResponseJson(submissionId, payload, correlationId, "FATAL_ERROR"),
+            "GovTalk status already exists"
+          )
+        )
+      }
+  }
+
+  private def baseSubmissionResponseJson(
+    submissionId: String,
+    payload: BuiltSubmissionPayload,
+    correlationId: String,
+    status: String,
+    gatewayTimestamp: String = Instant.now(clock).toString
+  ): JsObject =
+    Json.obj(
+      "submissionId"      -> submissionId,
+      "hmrcMarkGenerated" -> payload.irMark,
+      "correlationId"     -> correlationId,
+      "gatewayTimestamp"  -> gatewayTimestamp,
+      "status"            -> status
+    )
+
+  private def withError(json: JsObject, text: String): JsObject =
+    json ++ Json.obj("error" -> Json.obj("text" -> text))
+
+  private def validateCorrelationId(sentRaw: String, ackRaw: String): Either[String, Unit] = {
+    val sent = sentRaw.trim
+    val ack  = ackRaw.trim
+
+    if (sent.isEmpty || ack.isEmpty) Left("empty correlationId")
+    else if (sent != ack) Left(s"correlationId mismatch: sent '$sent' but got '$ack'")
+    else Right(())
+  }
 }