Airflow integration (#39)

* Airflow eks template for terraform.
* rename `eks` folder to `eks_argo`

Author: valayDave

examples/eks_airflow/README.md

@@ -0,0 +1,51 @@
+# An example of deploying Metaflow with Airflow on an EKS cluster
+
+This example will create Metaflow infrastructure from scratch, with a Kubernetes cluster using Amazon EKS. It uses [`datastore`](../../modules/datastore/) and [`metadata-service`](../../modules/metadata-service/) submodules to provision S3 bucket, RDS database and Metaflow Metadata service running on AWS Fargate.
+
+To run Metaflow jobs, it provisions a EKS cluster using [this popular open source terraform module](https://registry.terraform.io/modules/terraform-aws-modules/eks/aws/latest). In that cluster, it also installs [Cluster Autoscaler](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler) and [Airflow](https://airflow.apache.org/) using Helm.
+
+Specifically, it'll create following resources in your AWS account:
+* General networking infra:
+    * AWS VPC
+    * NAT gateway for private subnets in the VPC
+* For storing data artifacts:
+    * S3 bucket
+* For Metaflow metadata:
+    * RDS Database instance (on-demand, Multi-AZ, db.t2.small)
+    * ECS service for Metaflow Metadata service
+    * Network load balancer
+    * API Gateway
+* For executing Metaflow tasks:
+    * Autoscaling EKS cluster with at least one instance running
+
+Note that all this infrastructure costs a non-trivial amount at rest, up to $400/month and more if being actively used.
+
+## Instructions
+
+0. Run `terraform init`
+1. Run `terraform apply` to create infrastructure. This command will typically take ~20 minutes to execute.
+2. Make note of the EKS cluster name (it is a short string that starts with `mf-`). Use AWS CLI to generate cluster configuration:
+    ```bash
+    aws eks update-kubeconfig --name <CLUSTER NAME>
+    ```
+2. Copy `config.json` to `~/.metaflowconfig/`
+3. You should be ready to run Metaflow flows using `@kubernetes`
+and be able to deploy them to Airflow.
+
+Airflow UI is not accessible from outside the cluster, but you can use port forwarding to see it. Run
+```bash
+kubectl port-forward -n airflow deployment/airflow-deployment-webserver 8080:8080
+```
+..and you should be able to access it at `localhost:8080`.
+
+## Destroying the infrastructure
+
+Note that this will destroy everything including the S3 bucket with artifacts!
+
+Run `terraform destroy`
+
+# What's missing
+
+⚠️ This is meant as a reference example, with many things omitted for simplicity, such as proper RBAC setup, production-grade autoscaling and UI. For example, all workloads running in the cluster use the same AWS IAM role. We do not recommend using this as a production deployment of Metaflow on Kubernetes.
+
+For learn more about production-grade deployments, you can talk to us on [the Outerbounds slack](http://slack.outerbounds.co). We are happy to help you there!

examples/eks_airflow/airflow.tf

@@ -0,0 +1,104 @@
+provider "helm" {
+  kubernetes {
+    host                   = data.aws_eks_cluster.cluster.endpoint
+    cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority.0.data)
+    token                  = data.aws_eks_cluster_auth.cluster.token
+  }
+}
+
+resource "helm_release" "cluster_autoscaler" {
+  name = "autoscaler"
+
+  depends_on = [module.eks]
+
+  repository = "https://kubernetes.github.io/autoscaler"
+  chart      = "cluster-autoscaler"
+  namespace  = "kube-system"
+
+  set {
+    name  = "autoDiscovery.clusterName"
+    value = local.cluster_name
+  }
+
+  set {
+    name  = "awsRegion"
+    value = data.aws_region.current.name
+  }
+}
+
+
+resource "kubernetes_namespace" "airflow" {
+  metadata {
+    name = "airflow"
+  }
+}
+
+data "aws_region" "current" {}
+
+variable "airflow_webserver_secret" {
+  type    = string
+  default = "mysupersecr3tv0lue"
+}
+
+# This secret is that the airflow webserver used to sign session cookies.
+# https://airflow.apache.org/docs/helm-chart/stable/production-guide.html#webserver-secret-key
+resource "kubernetes_secret" "airflow-webserver-secret" {
+  metadata {
+    name      = "airflow-webserver-secret"
+    namespace = kubernetes_namespace.airflow.metadata[0].name
+  }
+  type = "Opaque"
+  data = {
+    webserver-secret-key = var.airflow_webserver_secret
+  }
+}
+
+
+locals {
+  airflow_values = {
+    "executor"                     = "LocalExecutor"
+    "defaultAirflowTag"            = "2.3.3"
+    "airflowVersion"               = "2.3.3"
+    "webserverSecretKeySecretName" = kubernetes_secret.airflow-webserver-secret.metadata[0].name
+    "env" = [
+      {
+        "name"  = "AIRFLOW_CONN_AWS_DEFAULT"
+        "value" = "aws://"
+      },
+      {
+        "name"  = "AIRFLOW__LOGGING__REMOTE_LOGGING"
+        "value" = "True"
+      },
+      {
+        "name"  = "AIRFLOW__LOGGING__REMOTE_BASE_LOG_FOLDER"
+        "value" = "s3://${module.metaflow-datastore.s3_bucket_name}/airflow-logs"
+      },
+      {
+        "name"  = "AIRFLOW__LOGGING__REMOTE_LOG_CONN_ID"
+        "value" = "aws_default"
+      }
+    ]
+  }
+}
+
+resource "helm_release" "airflow" {
+
+  depends_on = [module.eks]
+
+  name = "airflow-deployment"
+
+  repository = "https://airflow.apache.org"
+  chart      = "airflow"
+
+  namespace = kubernetes_namespace.airflow.metadata[0].name
+
+  timeout = 1200
+
+  wait = false # Why set `wait=false`
+  #: Read this (https://github.com/hashicorp/terraform-provider-helm/issues/683#issuecomment-830872443)
+  # Short summary : If this is not set then airflow doesn't end up running migrations on the database. That makes the scheduler and other containers to keep waiting for migrations.
+
+  values = [
+    yamlencode(local.airflow_values)
+  ]
+}

examples/eks/eks.tf renamed to examples/eks_airflow/eks.tf

@@ -0,0 +1,16 @@
+data "aws_api_gateway_api_key" "metadata_api_key" {
+  id = module.metaflow-metadata-service.api_gateway_rest_api_id_key_id
+}
+
+resource "local_file" "foo" {
+  content = jsonencode({
+    "METAFLOW_SERVICE_AUTH_KEY"     = data.aws_api_gateway_api_key.metadata_api_key.value
+    "METAFLOW_DATASTORE_SYSROOT_S3" = module.metaflow-datastore.METAFLOW_DATASTORE_SYSROOT_S3,
+    "METAFLOW_DATATOOLS_S3ROOT"     = module.metaflow-datastore.METAFLOW_DATATOOLS_S3ROOT,
+    "METAFLOW_SERVICE_URL"          = module.metaflow-metadata-service.METAFLOW_SERVICE_URL,
+    "METAFLOW_KUBERNETES_NAMESPACE" = "airflow",
+    "METAFLOW_DEFAULT_DATASTORE"    = "s3",
+    "METAFLOW_DEFAULT_METADATA"     = "service"
+  })
+  filename = "${path.module}/config.json"
+}

examples/eks/iam-ecs-execution.tf renamed to examples/eks_airflow/iam-ecs-execution.tf

@@ -1,4 +1,4 @@
-# An example of deploying Metaflow with a EKS cluster
+# An example of deploying Metaflow with Argo on an EKS cluster
 
 This example will create Metaflow infrastructure from scratch, with a Kubernetes cluster using Amazon EKS. It uses [`datastore`](../../modules/datastore/) and [`metadata-service`](../../modules/metadata-service/) submodules to provision S3 bucket, RDS database and Metaflow Metadata service running on AWS Fargate.