switch to URL

Author: Nick Grippin

Documents/panel.md

@@ -4,22 +4,29 @@
 The panel loader is designed to read a CSV file and load the data onto the HMDA-Platform.  The CSV file should use the `|` (pipe) delimiter, and should include a header row as the first line.
 
 ## Environment Variables
-There are two environment variables used by the panel loader.  Both must be set correctly in order for the data to be sent to the admin API.
+There is only one environment variable used by the panel loader.  It must be set correctly in order for the data to be sent to the admin API.
 
-For testing locally, no changes need to be made.  The defaults for both of these variables will point to the correct local admin API.
+For testing on an API running in SBT, no changes need to be made.  The default for this variable will point to the correct local admin API.
 
-For loading panel data into a remote system, you'll need to set the following environment variables:
+For loading panel data into a remote system or into a local Docker container, you'll need to set the following environment variable:
 ```shell
-> export HMDA_HTTP_ADMIN_HOST={ip address}
-> export HMDA_HTTP_ADMIN_PORT={port #}
+> export HMDA_HTTP_ADMIN_URL={base URL}
+```
+
+**IMPORTANT NOTE:** The base URL should *include* `http://` or `https://`, but *exclude* any trailing backslash `/`.  For example:
+
+```shell
+> export HMDA_HTTP_ADMIN_URL=http://192.168.99.100:8081
 ```
 
 ## Running the parser
 A small example file is located at `panel/src/main/resources/inst_data_2017_dummy.csv`
 
 The real panel file is located at `panel/src/main/resources/inst_data_2017.csv`
 
-In order for the panel data to be loaded locally, the API project must be up and running, along with Docker containers running Cassandra and Zookeper.  Otherwise, no other running services are needed (but make sure your environment variables are set).  In a terminal, execute the following commands:
+In order for the panel data to be loaded locally, the API project must be up and running, along with Docker containers running Cassandra and Zookeper, or run the full `docker-compose` setup.  To load panel data into the cluster, simply find the URL of the admin api (for the release branch: `https://hmda-ops-api.demo.cfpb.gov/admin`).  No other running services are necessary.
+
+In a terminal, execute the following commands:
 
 ```shell
 > sbt

panel/src/main/resources/application.conf

@@ -4,9 +4,7 @@ akka {
 }
 
 hmda {
-  adminHost = "0.0.0.0"
-  adminHost = ${?HMDA_HTTP_ADMIN_HOST}
-  adminPort = 80
-  adminPort = ${?HMDA_HTTP_ADMIN_PORT}
+  adminUrl = "http://0.0.0.0:8081"
+  adminUrl = ${?HMDA_HTTP_ADMIN_URL}
 }
 

panel/src/main/scala/hmda/panel/PanelCsvLoader.scala

@@ -26,8 +26,7 @@ object PanelCsvLoader extends WriteInstitutionProtocol {
   val log = LoggerFactory.getLogger("hmda")
 
   val config = ConfigFactory.load()
-  val host = config.getString("hmda.adminHost")
-  val port = config.getInt("hmda.adminPort")
+  val url = config.getString("hmda.adminUrl")
 
   def main(args: Array[String]): Unit = {
 
@@ -44,7 +43,6 @@ object PanelCsvLoader extends WriteInstitutionProtocol {
       val response = sendRequest("create")
 
       val source = FileIO.fromPath(file.toPath)
-      val connectionFlow = Http().outgoingConnection(host, port)
 
       response.onComplete(s =>
         if (s.getOrElse("").equals("InstitutionSchemaCreated()")) {
@@ -54,7 +52,7 @@ object PanelCsvLoader extends WriteInstitutionProtocol {
             .drop(1)
             .via(byte2StringFlow)
             .via(stringToHttpFlow)
-            .via(connectionFlow)
+            .via(singleConnectionFlow)
             .runWith(Sink.foreach[HttpResponse](elem => log.info(elem.entity.toString)))
 
           completedF.onComplete(result => {
@@ -72,16 +70,19 @@ object PanelCsvLoader extends WriteInstitutionProtocol {
 
   private def singleConnectionFlow: Flow[HttpRequest, HttpResponse, NotUsed] =
     Flow[HttpRequest]
-    .map(r => {
-      Http().singleRequest(r).
-    })
+      .mapAsync[HttpResponse](5)(request => {
+        for {
+          response <- Http().singleRequest(request)
+        } yield response
+      })
 
   private def stringToHttpFlow: Flow[String, HttpRequest, NotUsed] =
     Flow[String]
       .map(x => {
         val payload = ByteString(InstitutionParser(x).toJson.toString)
         HttpRequest(
           HttpMethods.POST,
+          uri = s"$url/institutions",
           entity = HttpEntity(MediaTypes.`application/json`, payload)
         )
       })
@@ -90,7 +91,7 @@ object PanelCsvLoader extends WriteInstitutionProtocol {
     Flow[ByteString].map(bs => bs.utf8String)
 
   private def sendRequest(req: String) = {
-    val request = HttpRequest(HttpMethods.GET, uri = s"http://$host:$port/institutions/$req")
+    val request = HttpRequest(HttpMethods.GET, uri = s"$url/institutions/$req")
     for {
       response <- Http().singleRequest(request)
       content <- Unmarshal(response.entity).to[String]