json api

Author: Karasiq

build.sbt

@@ -35,13 +35,13 @@ lazy val commonSettings = Seq(
 lazy val backendSettings = Seq(
   name := "webm-tv",
   libraryDependencies ++= {
-    val akkaV = "2.4.6"
+    val akkaV = "2.4.10"
     Seq(
       "org.jsoup" % "jsoup" % "1.9.2",
       "com.typesafe.akka" %% "akka-actor" % akkaV,
       "com.typesafe.akka" %% "akka-http-experimental" % akkaV,
       "com.lihaoyi" %% "scalatags" % "0.5.4",
-      "com.lihaoyi" %% "upickle" % "0.3.8",
+      "com.lihaoyi" %% "upickle" % "0.4.1",
       "net.databinder.dispatch" %% "dispatch-core" % "0.11.2",
       "org.scala-lang.modules" %% "scala-async" % "0.9.6-RC2",
       "org.scalatest" %% "scalatest" % "2.2.4" % "test",

src/main/resources/reference.conf

@@ -6,6 +6,7 @@ webm-tv {
   thread-ttl = 1h
 
   sosach {
+    host = "2ch.hk"
     boards = [
       b
     ]

src/main/scala/com/karasiq/webmtv/app/Main.scala

@@ -5,9 +5,9 @@ import java.util.concurrent.TimeUnit
 import akka.actor._
 import akka.http.scaladsl.Http
 import akka.http.scaladsl.Http.ServerBinding
-import akka.stream.{ActorMaterializer, ActorMaterializerSettings}
+import akka.stream.ActorMaterializer
 import akka.util.Timeout
-import com.karasiq.webmtv.sosach.M2chBoardApi
+import com.karasiq.webmtv.sosach.{BoardApi, Json2chBoardApi, M2chBoardApi}
 import com.typesafe.config.ConfigFactory
 
 import scala.concurrent.Await
@@ -21,12 +21,23 @@ object Main extends App {
 
     implicit val actorSystem = ActorSystem("webm-tv")
     implicit val executionContext = actorSystem.dispatcher
-    implicit val actorMaterializer = ActorMaterializer(ActorMaterializerSettings(actorSystem))
+    implicit val actorMaterializer = ActorMaterializer()
 
-    val boardApi = new M2chBoardApi()
+    val config = ConfigFactory.load().getConfig("webm-tv")
+    val boardApi = new BoardApi {
+      val jsonApi = new Json2chBoardApi(config.getString("sosach.host"))
+      val htmlApi = new M2chBoardApi() // Fallback
+
+      def board(name: String) = {
+        jsonApi.board(name).recoverWithRetries(1, { case _ ⇒ htmlApi.board(name) })
+      }
+
+      def thread(board: String, id: Long) = {
+        jsonApi.thread(board, id).recoverWithRetries(1, { case _ ⇒ htmlApi.thread(board, id) })
+      }
+    }
     val store = WebmFileStore
     val storeDispatcher = actorSystem.actorOf(Props(classOf[WebmStoreDispatcher], boardApi, store), "storeDispatcher")
-    val config = ConfigFactory.load().getConfig("webm-tv")
     val server = new Server(storeDispatcher)
 
     Http().bindAndHandle(server.route, config.getString("host"), config.getInt("port")).onComplete {

src/main/scala/com/karasiq/webmtv/sosach/Json2chBoardApi.scala

@@ -0,0 +1,76 @@
+package com.karasiq.webmtv.sosach
+
+import akka.actor.ActorSystem
+import akka.http.scaladsl.Http
+import akka.http.scaladsl.model.headers.Accept
+import akka.http.scaladsl.model.{HttpRequest, MediaRange, MediaTypes}
+import akka.stream.ActorMaterializer
+import akka.stream.scaladsl.Source
+import akka.util.ByteString
+import derive.key
+import upickle.Js
+import upickle.default._
+
+private object JsonApiObjects {
+  case class PostId(value: Long) extends AnyVal
+  object PostId {
+    implicit def postIdToLong(postId: PostId): Long = postId.value
+  }
+
+  case class File(path: String)
+  case class Post(@key("num") id: PostId, subject: String, comment: String, files: Seq[File])
+  case class Thread(posts: Seq[Post])
+  case class Board(@key("Board") name: String, pages: Seq[Int], threads: Seq[Thread])
+  case class ThreadWrapped(@key("Board") board: String, @key("threads") thread: Option[Thread])
+
+  object Implicits {
+    implicit val postIdReader = Reader[PostId] {
+      case Js.Str(str) ⇒
+        PostId(str.toLong)
+
+      case Js.Num(num) ⇒
+        PostId(num.toLong)
+    }
+  }
+}
+
+class Json2chBoardApi(host: String = "2ch.hk")(implicit as: ActorSystem, am: ActorMaterializer) extends BoardApi {
+  import JsonApiObjects.Implicits._
+
+  private val http = Http()
+
+  private def retrieveJson[T: Reader](url: String) = {
+    Source
+      .fromFuture(http.singleRequest(HttpRequest(uri = url, headers = List(Accept(MediaRange(MediaTypes.`application/json`))))))
+      .flatMapConcat(_.entity.dataBytes)
+      .fold(ByteString.empty)(_ ++ _)
+      .map(bs ⇒ read[T](bs.utf8String))
+  }
+
+  private def jsonToAppPost(board: String, postObj: JsonApiObjects.Post): Board.Post = {
+    Board.Post(postObj.id, postObj.subject, postObj.comment, postObj.files.map(file ⇒ s"https://$host/$board/${file.path}"))
+  }
+
+  def board(name: String) = {
+    val url = s"https://$host/$name/index.json"
+    retrieveJson[JsonApiObjects.Board](url)
+      .flatMapConcat { page ⇒
+        val pages = page.pages.filter(_ > 0).map(page ⇒ s"https://$host/$name/$page.json")
+        Source.single(page).concat(
+          Source(pages.toVector)
+            .flatMapConcat(url ⇒ retrieveJson[JsonApiObjects.Board](url).recoverWithRetries(1, { case _ ⇒ Source.empty }))
+        )
+      }
+      .mapConcat { board ⇒
+        val threads = for (threadObj ← board.threads)
+          yield Board.Thread(board.name, threadObj.posts.map(jsonToAppPost(board.name, _)))
+        threads.toVector
+      }
+  }
+
+  def thread(board: String, id: Long) = {
+    val url = s"https://$host/$board/res/$id.json"
+    retrieveJson[JsonApiObjects.ThreadWrapped](url)
+      .map(thread ⇒ Board.Thread(thread.board, thread.thread.toSeq.flatMap(_.posts.map(jsonToAppPost(thread.board, _)))))
+  }
+}