Awsify

.dockerignore

@@ -0,0 +1,6 @@
+*
+!_targets.R
+!in/
+!src/
+!docker/
+!aws/

.gitignore

@@ -6,3 +6,6 @@
 _targets/
 *.Rproj
 *.svg
+# vim swap files
+.*.swp
+.swp

README.md

@@ -4,7 +4,7 @@ This repository uses the [`targets` package](https://docs.ropensci.org/targets/)
 
 Install all packages needed to run: 
 ```
-install.packages(c('targets', 'tidyverse', 'lubridate', 'geofacet', 'cowplot','ggfx', 'showtext', 'magick', 'maps', 'svglite', 'xml2'))
+install.packages(c('targets', 'tidyverse', 'lubridate', 'geofacet', 'cowplot','ggfx', 'showtext', 'magick', 'maps', 'svglite', 'xml2', 'dataRetrieval', 'rsvg'))
 ```
 
 Running `tar_make()` will produce: 

_targets.R

@@ -1,7 +1,17 @@
 library(targets)
 
 options(tidyverse.quiet = TRUE)
-tar_option_set(packages = c('tidyverse', 'lubridate', 'geofacet', 'cowplot','ggfx', 'showtext', 'xml2'))
+tar_option_set(
+  packages = c(
+    'cowplot',
+    'geofacet',
+    'ggfx',
+    'lubridate',
+    'showtext',
+    'tidyverse',
+    'xml2'
+  )
+)
 
 source("src/prep_data.R")
 source("src/plot_cartogram.R")
@@ -18,7 +28,8 @@ list(
   # Read in data from gage-flow-conditions pipeline output
   tar_target(
     dv,
-    read_csv("https://labs.waterdata.usgs.gov/visualizations/data/flow_conditions_202211.csv", col_types = "cTnnnn")
+    #read_csv("https://labs.waterdata.usgs.gov/visualizations/data/flow_conditions_202209_full.csv", col_types = "cTnnnn")
+    readRDS("/ephemeral/dv_stats.rds")
   ),
   tar_target(
     date_start,
@@ -84,22 +95,34 @@ list(
     plot_nat,
     plot_national_area(national_data = flow_national, pal = pal_wetdry, date_start, date_end, color_bknd)
   ),
+  # Build a string for the prior month, e.g. 2022-11
+  tar_target(
+    prior_month,
+    strftime(Sys.Date() - lubridate::days(lubridate::day(Sys.Date())),
+             format = "%Y-%m")
+  ),
   # Combine charts and assemble final plot
   tar_target(
     flow_cartogram_svg,
-    combine_plots(file_svg = "flow_cartogram.svg", 
-                  plot_left = plot_nat, 
-                  plot_right = plot_cart, 
-                  date_start,
-                  width = 16, height = 9, color_bknd),
+    combine_plots(
+      file_svg = paste("flow_cartogram_", prior_month, ".svg", sep = ""), 
+      plot_left = plot_nat, 
+      plot_right = plot_cart, 
+      date_start,
+      width = 16,
+      height = 9,
+      color_bknd
+    ),
     format = "file"
   ),
   # Remove facet clipping and save as png
   tar_target(
     flow_cartogram_png,
-    rm_facet_clip(svg_in = flow_cartogram_svg, 
-                  file_out = "flow_cartogram.png",
-                  width = 16),
+    rm_facet_clip(
+      svg_in = flow_cartogram_svg, 
+      file_out = paste("flow_cartogram_", prior_month, ".png", sep = ""),
+      width = 16
+    ),
     format = "file"
   )
 )

aws/README.md

@@ -0,0 +1,22 @@
+## Docker Image
+### Building the Image
+```bash
+cd docker
+docker-compose build
+```
+
+This builds an image with the name `807615458658.dkr.ecr.us-west-2.amazonaws.com/vizlab/flow-tiles:latest`. It also will tag this with the name `flow-tiles`, allowing for convenient local runs a la `docker run -it flow-tiles bash`.
+
+### Pushing the Image to ECR
+```bash
+aws ecr get-login-password --region us-west-2 | docker login --username AWS --password-stdin 807615458658.dkr.ecr.us-west-2.amazonaws.com
+cd docker
+docker-compose push 
+# line above will throw an error when it tries to push the short name
+```
+
+Image will now be available as `807615458658.dkr.ecr.us-west-2.amazonaws.com/vizlab/flow-tiles`.
+
+## ECS Setup
+* Created a "network-only" cluster named `vizlab-cluster` on ECS.
+* Copied the task definition for fetch-daily-obs-task

aws/flow-tiles-task.json

@@ -0,0 +1,136 @@
+{
+    "executionRoleArn": "arn:aws:iam::807615458658:role/ecsTaskExecutionRole",
+    "containerDefinitions": [
+        {
+            "logConfiguration": {
+                "logDriver": "awslogs",
+                "options": {
+                    "awslogs-group": "/ecs/flow-tiles-task",
+                    "awslogs-region": "us-west-2",
+                    "awslogs-stream-prefix": "ecs"
+                }
+            },
+            "portMappings": [],
+            "command": [
+                "s3",
+                "cp",
+                "s3://vizlab-flow-tiles/in/dv_stats.rds",
+                "/ephemeral/"
+            ],
+            "cpu": 0,
+            "environment": [],
+            "mountPoints": [
+                {
+                    "containerPath": "/ephemeral",
+                    "sourceVolume": "bind-mount"
+                }
+            ],
+            "volumesFrom": [],
+            "image": "amazon/aws-cli:2.4.6",
+            "essential": false,
+            "name": "get-gage-conditions"
+        },
+        {
+            "logConfiguration": {
+                "logDriver": "awslogs",
+                "options": {
+                    "awslogs-group": "/ecs/flow-tiles-task",
+                    "awslogs-region": "us-west-2",
+                    "awslogs-stream-prefix": "ecs"
+                }
+            },
+            "portMappings": [],
+            "command": [
+                "bash",
+                "aws/run-pipeline.sh"
+            ],
+            "cpu": 0,
+            "environment": [],
+            "mountPoints": [
+                {
+                    "containerPath": "/ephemeral",
+                    "sourceVolume": "bind-mount"
+                }
+            ],
+            "workingDirectory": "/flow-tiles",
+            "volumesFrom": [],
+            "image": "807615458658.dkr.ecr.us-west-2.amazonaws.com/vizlab/flow-tiles:latest",
+            "dependsOn": [
+                {
+                    "containerName": "get-gage-conditions",
+                    "condition": "SUCCESS"
+                }
+            ],
+            "essential": false,
+            "name": "build-viz"
+        },
+        {
+            "logConfiguration": {
+                "logDriver": "awslogs",
+                "options": {
+                    "awslogs-group": "/ecs/flow-tiles-task",
+                    "awslogs-region": "us-west-2",
+                    "awslogs-stream-prefix": "ecs"
+                }
+            },
+            "portMappings": [],
+            "command": [
+                "s3",
+                "cp",
+                "--recursive",
+                "/ephemeral/",
+                "s3://vizlab-flow-tiles/out/",
+                "--exclude='*'",
+                "--include='flow_cartogram*'"
+            ],
+            "cpu": 0,
+            "environment": [],
+            "mountPoints": [
+                {
+                    "containerPath": "/ephemeral",
+                    "sourceVolume": "bind-mount"
+                }
+            ],
+            "volumesFrom": [],
+            "image": "amazon/aws-cli:2.4.6",
+            "dependsOn": [
+                {
+                    "containerName": "build-viz",
+                    "condition": "SUCCESS"
+                }
+            ],
+            "essential": true,
+            "name": "upload-result"
+        }
+    ],
+    "memory": "2048",
+    "taskRoleArn": "arn:aws:iam::807615458658:role/ecsTaskExecutionRole",
+    "family": "vizlab-flow-tiles-task",
+    "requiresCompatibilities": [
+        "FARGATE"
+    ],
+    "networkMode": "awsvpc",
+    "runtimePlatform": {
+        "operatingSystemFamily": "LINUX"
+    },
+    "cpu": "256",
+    "inferenceAccelerators": [],
+    "volumes": [
+        {
+            "name": "bind-mount",
+            "host": {
+                "sourcePath": null
+            }
+        }
+    ],
+    "tags": [
+        {
+            "key": "wma:project_id",
+            "value": "vizlab"
+        },
+        {
+            "key": "wma:contact",
+            "value": "Jesse Ross [email protected]"
+        }
+    ]
+}

aws/run-pipeline.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+Rscript -e 'targets::tar_make()'
+cp flow_cartogram* /ephemeral/

aws/state_machine.asm.json

@@ -0,0 +1,82 @@
+{
+  "Comment": "Run gage-conditions and create the flow-tiles PNG",
+  "StartAt": "Run gage-conditions",
+  "TimeoutSeconds": 7200,
+  "States": {
+    "Run gage-conditions": {
+      "Type": "Task",
+      "Resource": "arn:aws:states:::ecs:runTask.sync",
+      "Parameters": {
+        "LaunchType": "FARGATE",
+        "Cluster": "arn:aws:ecs:us-west-2:807615458658:cluster/vizlab-cluster",
+        "TaskDefinition": "arn:aws:ecs:us-west-2:807615458658:task-definition/vizlab-gage-conditions-task:12",
+        "NetworkConfiguration": {
+          "AwsvpcConfiguration": {
+            "Subnets": [
+              "subnet-0b250cbaebb683b8d"
+            ],
+            "AssignPublicIp": "ENABLED"
+          }
+        }
+      },
+      "Next": "Create flow-tiles PNG",
+      "Catch": [
+        {
+          "ErrorEquals": [ "States.ALL" ],
+          "Next": "Notify gage-conditions Failure"
+        }
+      ]
+    },
+    "Create flow-tiles PNG": {
+      "Type": "Task",
+      "Resource": "arn:aws:states:::ecs:runTask.sync",
+      "Parameters": {
+        "LaunchType": "FARGATE",
+        "Cluster": "arn:aws:ecs:us-west-2:807615458658:cluster/vizlab-cluster",
+        "TaskDefinition": "arn:aws:ecs:us-west-2:807615458658:task-definition/vizlab-flow-tiles-task:14",
+        "NetworkConfiguration": {
+          "AwsvpcConfiguration": {
+            "Subnets": [
+              "subnet-0b250cbaebb683b8d"
+            ],
+            "AssignPublicIp": "ENABLED"
+          }
+        }
+      },
+      "Next": "Notify Success",
+      "Catch": [
+        {
+          "ErrorEquals": [ "States.ALL" ],
+          "Next": "Notify flow-tiles Failure"
+        }
+      ]
+    },
+    "Notify Success": {
+      "Type": "Task",
+      "Resource": "arn:aws:states:::sns:publish",
+      "Parameters": {
+        "Message": "The pipeline to build the flow-tiles visualization has run successfully. See the result: https://s3.console.aws.amazon.com/s3/buckets/vizlab-flow-tiles?region=us-west-2&prefix=out/&showversions=false",
+        "TopicArn": "arn:aws:sns:us-west-2:807615458658:vizlab-flow-tiles-pipeline"
+      },
+      "End": true
+    },
+    "Notify gage-conditions Failure": {
+      "Type": "Task",
+      "Resource": "arn:aws:states:::sns:publish",
+      "Parameters": {
+        "Message": "The pipeline to build the flow-tiles visualization has FAILED in the gage-conditions task. See logs in the developer account: https://us-west-2.console.aws.amazon.com/cloudwatch/home?region=us-west-2#logsV2:log-groups/log-group/$252Fecs$252Fgage-conditions-task",
+        "TopicArn": "arn:aws:sns:us-west-2:807615458658:vizlab-flow-tiles-pipeline"
+      },
+      "End": true
+    },
+    "Notify flow-tiles Failure": {
+      "Type": "Task",
+      "Resource": "arn:aws:states:::sns:publish",
+      "Parameters": {
+        "Message": "The pipeline to build the flow-tiles visualization has FAILED in the flow-tiles task. See logs in the developer account: https://us-west-2.console.aws.amazon.com/cloudwatch/home?region=us-west-2#logsV2:log-groups/log-group/$252Fecs$252Fflow-tiles-task",
+        "TopicArn": "arn:aws:sns:us-west-2:807615458658:vizlab-flow-tiles-pipeline"
+      },
+      "End": true
+    }
+  }
+}

docker/DOIRootCA2.crt

@@ -0,0 +1,56 @@
+-----BEGIN CERTIFICATE-----
+MIIJ+jCCB+KgAwIBAgIQSeV7u0WVHrFBqkjcKiEnEzANBgkqhkiG9w0BAQsFADAV
+MRMwEQYDVQQDEwpET0lSb290Q0EyMB4XDTE2MDQyNjE4MjE1MFoXDTM2MDQyNjE4
+MjE1MFowFTETMBEGA1UEAxMKRE9JUm9vdENBMjCCAiIwDQYJKoZIhvcNAQEBBQAD
+ggIPADCCAgoCggIBANTHy0AvCFT0CZOsktj3kpFfFJgrhiS5haBK7DvYYpbZoaWa
+jOpldnvoqnd1bOJUJ9jUwxZERo27FJHZ8HSdU8ac63hdK1N6t1OrlQOjgs/Kn0LY
+b7xOf6iDBfhdhXcjpq2KZBzVi0tWuyBPhDmZyTLqZ5UREy1mV3/p1pJ3Hx5lPziH
+Fid+wILxe2fk+N1ExY/GA+cgdsSCP4kp4aBxXLNTq+oMu/NahyF+NmcbqUk3xh+v
+U1UA7h35b8kjd/3Kx2Bv1EDveWbzaS7sn8T3OVnU6n9UObUcqaoJOXF3PDdQqPIh
+YVLT8/s15YaUznTe7jc46YjnqZPaJJGbDir/m03QR8qWi0qMhLjTTfPYe4DFYa27
+4e8sTeK0DTSpUMBhQqagRQEFcYRd6QaZ4wfj+8zwX7EUha9jOKrMALvxRurkEqsa
+m9NZntaHlSkRjQAZ562TUYowBpb841O0v0c9+i8SM6D9kRVV+NIj3StPNkQG7qlc
++PhF5YA4jYAifZ7AtWraLeopTPonfX0avWbIt5ryy4Y+sISwsg4HZ+rdNrJq9MUu
+YWDbdO/lRclnFJ64VmD0rH7Fuef7CDiQvwn0NPJHQoU6h/zHnfOEIlh44h+0uy+R
+lEp41vrb9mA/a7ZZEohcJroQ6JL1Z8b+KLY47ryuqneLklCVTGbMNGZxusOtAgMB
+AAGjggVEMIIFQDALBgNVHQ8EBAMCAYYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4E
+FgQUv4YryvNsbT5fHDtOTtiN52rHak8wEAYJKwYBBAGCNxUBBAMCAQAwggTtBgNV
+HSAEggTkMIIE4DCCAg8GCWCGSAFlAwIBEzCCAgAwMAYIKwYBBQUHAgEWJGh0dHA6
+Ly9wa2kyLmRvaS5uZXQvbGVnYWxwb2xpY3kuYXNwADCCAcoGCCsGAQUFBwICMIIB
+vB6CAbgAQwBlAHIAdABpAGYAaQBjAGEAdABlACAAaQBzAHMAdQBlAGQAIABiAHkA
+IAB0AGgAZQAgAEQAZQBwAGEAcgB0AG0AZQBuAHQAIABvAGYAIAB0AGgAZQAgAEkA
+bgB0AGUAcgBpAG8AcgAgAGEAcgBlACAAbwBuAGwAeQAgAGYAbwByACAAaQBuAHQA
+ZQByAG4AYQBsACAAdQBuAGMAbABhAHMAcwBpAGYAaQBlAGQAIABVAFMAIABHAG8A
+dgBlAHIAbgBtAGUAbgB0ACAAdQBzAGUAIABhAGwAbAAgAG8AdABoAGUAcgAgAHUA
+cwBlACAAaQBzACAAcAByAG8AaABpAGIAaQB0AGUAZAAuACAAVQBuAGEAdQB0AGgA
+bwByAGkAegBlAGQAIAB1AHMAZQAgAG0AYQB5ACAAcwB1AGIAagBlAGMAdAAgAHYA
+aQBvAGwAYQB0AG8AcgBzACAAdABvACAAYwByAGkAbQBpAG4AYQBsACwAIABjAGkA
+dgBpAGwAIABhAG4AZAAvAG8AcgAgAGQAaQBzAGMAaQBwAGwAaQBuAGEAcgB5ACAA
+YQBjAHQAaQBvAG4ALjCCAskGCmCGSAFlAwIBEwEwggK5MDUGCCsGAQUFBwIBFilo
+dHRwOi8vcGtpMi5kb2kubmV0L2xpbWl0ZWR1c2Vwb2xpY3kuYXNwADCCAn4GCCsG
+AQUFBwICMIICcB6CAmwAVQBzAGUAIABvAGYAIAB0AGgAaQBzACAAQwBlAHIAdABp
+AGYAaQBjAGEAdABlACAAaQBzACAAbABpAG0AaQB0AGUAZAAgAHQAbwAgAEkAbgB0
+AGUAcgBuAGEAbAAgAEcAbwB2AGUAcgBuAG0AZQBuAHQAIAB1AHMAZQAgAGIAeQAg
+AC8AIABmAG8AcgAgAHQAaABlACAARABlAHAAYQByAHQAbQBlAG4AdAAgAG8AZgAg
+AHQAaABlACAASQBuAHQAZQByAGkAbwByACAAbwBuAGwAeQAuACAARQB4AHQAZQBy
+AG4AYQBsACAAdQBzAGUAIABvAHIAIAByAGUAYwBlAGkAcAB0ACAAbwBmACAAdABo
+AGkAcwAgAEMAZQByAHQAaQBmAGkAYwBhAHQAZQAgAHMAaABvAHUAbABkACAAbgBv
+AHQAIABiAGUAIAB0AHIAdQBzAHQAZQBkAC4AIABBAGwAbAAgAHMAdQBzAHAAZQBj
+AHQAZQBkACAAbQBpAHMAdQBzAGUAIABvAHIAIABjAG8AbQBwAHIAbwBtAGkAcwBl
+ACAAbwBmACAAdABoAGkAcwAgAGMAZQByAHQAaQBmAGkAYwBhAHQAZQAgAHMAaABv
+AHUAbABkACAAYgBlACAAcgBlAHAAbwByAHQAZQBkACAAaQBtAG0AZQBkAGkAYQB0
+AGUAbAB5ACAAdABvACAAYQAgAEQAZQBwAGEAcgB0AG0AZQBuAHQAIABvAGYAIAB0
+AGgAZQAgAEkAbgB0AGUAcgBpAG8AcgAgAFMAZQBjAHUAcgBpAHQAeQAgAE8AZgBm
+AGkAYwBlAHIALjANBgkqhkiG9w0BAQsFAAOCAgEAF/q4Z2mRTIYJMu5mzlWsbV4o
+gGQJ9YcSdUZRq2vzINJCpGDXstAIE81Pfz/Fna98KOkjEB8XGXVUGQf07c9ylGJS
+XFoBwcN8GgOuys5iiP9/yd2yLHB8rBb8pu9RForl9RoTsYY8nFuOOtl9o2EfB/1O
+PbRYkfHhhqrfvvHdvDKWPmT+ZhaliWJrg2my432yqBqPePjqMZSl4sxiPYi9WicU
+UWYdJpxQlys3igICD4GXOcSh316jfaqfN8+9jps+lgO7rqOA41B8fU9Gwi4B8jjx
+Tw0pgvbuebwwL5IQwrsGcA8rFfRPR6CaSY5v3XXqTMbCXyYjNK1/44I9MoFFaFPc
+e3cqZ5cQ+lCoW3UE0SLNZb3YKh28ES/Gi5CO0Bq5P8QVLRJQL5xOaSzV9blszHv5
+okR+lkSsVo2QzR/mzFD7lXtwznkd/uak0hripTB7MtZenBzoQ8zAgjgw5TXjRSAZ
+goWiJTAg+YTKclhJ7Cfg/m4XeCxzNgz/pU1XEdBF2Ngvp3C9M5CSBcqzb234uiFF
+SyvJl/6erDTkQ5dLrnSnsJIw1ZS/XG/Fi41u8il0piLc5depTLn9qiWf29BRBEtG
+xwFKSmqlRWsClj/zADirBTjcctw7ajPMkRpebgn+Bzv1eWDx4+OolQuR/a45644Q
+GHVtIa/kVEl2DE0WcUw=
+-----END CERTIFICATE-----