CSCEXAM-1551 Support alternative format of importing course language

Author: lupari

app/impl/ExternalCourseHandlerImpl.scala

@@ -14,7 +14,7 @@ import org.apache.pekko.util.ByteString
 import org.joda.time.DateTime
 import org.springframework.beans.BeanUtils
 import play.api.Logging
-import play.api.libs.json.{JsValue, Json}
+import play.api.libs.json.{JsResultException, JsValue, Json}
 import play.api.libs.ws.{WSClient, WSResponse}
 import play.mvc.Http
 import schema.ExternalCourseValidator.{CourseUnitInfo, GradeScale as ExtGradeScale}
@@ -26,6 +26,7 @@ import javax.inject.Inject
 import scala.collection.immutable.TreeSet
 import scala.concurrent.{ExecutionContext, Future}
 import scala.jdk.CollectionConverters.*
+import scala.util.Try
 
 class ExternalCourseHandlerImpl @Inject (
     private val wsClient: WSClient,
@@ -93,34 +94,49 @@ class ExternalCourseHandlerImpl @Inject (
         external.setId(local.getId)
       case _ => external.save()
 
-  private def stripBom(response: WSResponse) =
-    val bomCandidate = response.bodyAsBytes.splitAt(3)
-    if bomCandidate._1 == BOM then
-      logger.warn("BOM character detected in the beginning of response body")
-      Json.parse(bomCandidate._2.toArray)
-    else response.json
+  private def parseResponseBody(response: WSResponse): Option[JsValue] =
+    val bytes = response.bodyAsBytes
+    val bodyBytes =
+      if bytes.length >= 3 && bytes.splitAt(3)._1 == BOM then
+        logger.warn("BOM character detected in the beginning of response body")
+        bytes.drop(3)
+      else bytes
+    val body = bodyBytes.utf8String.trim
+    if body.startsWith("<") then
+      logger.warn("Response is not JSON (e.g. HTML error page). Body starts with '<'.")
+      None
+    else
+      Try(Json.parse(bodyBytes.toArray)).toOption match
+        case None =>
+          logger.warn("Response was not valid JSON (e.g. HTML error page).")
+          None
+        case some => some
 
   private def downloadCourses(url: URL) =
     queryRequest(url)
       .get()
       .map(response =>
         val status = response.status
         if status == Http.Status.OK then
-          val root = stripBom(response)
-          parseCourses(root).flatMap(parseCourse)
+          parseResponseBody(response) match
+            case Some(root) => parseCourses(root).flatMap(parseCourse)
+            case None       => Seq.empty
         else
           logger.info(s"Non-OK response received for URL: %url. Status: $status")
           Seq.empty
       )
-      .recover { case e: Exception =>
+      .recover { case e: JsResultException =>
+        logger.error("Unable to parse course data: JSON structure did not match expected format", e)
+        Seq.empty
+      case e: Exception =>
         logger.error("Unable to download course data due to exception in network connection", e)
         Seq.empty
       }
 
   private def parseCourses(root: JsValue): Seq[CourseUnitInfo] =
-    val single = (root \\ "CourseUnitInfo").map(_.asOpt[CourseUnitInfo])
-    if single.head.nonEmpty then single.flatten.toSeq
-    else (root \\ "CourseUnitInfo").flatMap(_.as[Seq[CourseUnitInfo]]).toSeq
+    (root \\ "CourseUnitInfo").flatMap { node =>
+      node.asOpt[CourseUnitInfo].toSeq ++ node.asOpt[Seq[CourseUnitInfo]].getOrElse(Seq.empty)
+    }.toSeq
 
   private def parseCourse(cui: CourseUnitInfo): Option[Course] =
     (validateStart(cui.startDate), validateEnd(cui.endDate)) match

app/schema/ExternalCourseValidator.scala

@@ -64,8 +64,13 @@ object ExternalCourseValidator:
   case class CreditLanguage(name: String)
 
   object CourseUnitInfo:
-    val asScales: Reads[Seq[GradeScale]]  = implicitly[Reads[GradeScale]].map(Seq(_))
-    val readScale: Reads[Seq[GradeScale]] = implicitly[Reads[Seq[GradeScale]]].orElse(asScales)
+    private val asScales: Reads[Seq[GradeScale]] = implicitly[Reads[GradeScale]].map(Seq(_))
+    private val readScale: Reads[Seq[GradeScale]] =
+      implicitly[Reads[Seq[GradeScale]]].orElse(asScales)
+    private val creditsLanguageAsString: Reads[Seq[CreditLanguage]] =
+      implicitly[Reads[String]].map(lang => Seq(CreditLanguage(lang)))
+    private val readCreditsLanguage: Reads[Seq[CreditLanguage]] =
+      creditsLanguageAsString.orElse(implicitly[Reads[Seq[CreditLanguage]]])
     implicit val cuiReads: Reads[CourseUnitInfo] = (
       (JsPath \ "identifier").read[String](using readInt) and
         (JsPath \ "courseUnitCode").read[String] and
@@ -84,7 +89,7 @@ object ExternalCourseValidator:
         (JsPath \ "department").readNullable[Seq[Department]] and
         (JsPath \ "lecturerResponsible").readNullable[Seq[LecturerResponsible]] and
         (JsPath \ "lecturer").readNullable[Seq[Lecturer]] and
-        (JsPath \ "creditsLanguage").readNullable[Seq[CreditLanguage]] and
+        (JsPath \ "creditsLanguage").readNullable[Seq[CreditLanguage]](using readCreditsLanguage) and
         (JsPath \ "gradeScale").readNullable[Seq[GradeScale]](using readScale)
     )(CourseUnitInfo.apply)
   case class CourseUnitInfo(

conf/integrationtest.conf

@@ -10,6 +10,10 @@ play.mailer.host = localhost
 play.mailer.port = 3025
 play.mailer.ssl = no
 play.mailer.tls = no
+# Pin credentials so env (e.g. act/GitHub Actions SMTP_USER) does not override;
+# avoids GreenMail "Mailbox X already exists" when many concurrent SMTP connections AUTH.
+play.mailer.user = "user"
+play.mailer.password = "password"
 
 play.evolutions.db.default.autoApply = false
 play.evolutions.db.default.enabled = false

test/resources/courseUnitInfo.json

@@ -7,11 +7,7 @@
     "courseUnitType" : 1,
     "courseImplementation": "abcdefghijklmnop",
     "credits" : 3,
-    "creditsLanguage" : [
-      {
-        "name" : "fi"
-      }
-    ],
+    "creditsLanguage" : "fi",
     "gradeScale": [
       {
         "name" : "0-5",