fixes to get things working remotely too

Author: hobu

.gitignore

@@ -1,2 +1,3 @@
-lambda-deploy.zip
-
+*.tfstate*
+*.hcl
+.terraform/

docker/test-remote.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+eventfilename=$1
+
+FUNCTION_NAME=$(cat ../terraform/terraform.tfstate | jq '.outputs.info_lambda_name.value // empty' -r)
+
+
+aws lambda invoke \
+    --function-name "$FUNCTION_NAME" \
+    --invocation-type RequestResponse \
+    --payload fileb://$eventfilename \
+   response.json
+
+
+

handlers/python/ecr/info.py

@@ -23,16 +23,18 @@ def handler(event, context):
         logger.debug(f'processing {path}')
 
 
+
         command = f'pdal info --debug {path.as_uri()}'
         env = os.environ.copy()
 
         # FIXME This hardcodes our s3 region to east to read our test file
         env['AWS_REGION'] = "us-east-1"
+        # env['VERBOSE'] = "1"
 
         response = util.run(command, env = env)
         logger.debug(f'response {response}')
 
-        j = json.loads(response[0])
+        j = json.loads(response)
         infos.append(j)
 
     return infos

handlers/python/ecr/util.py

@@ -26,19 +26,19 @@ def run(cargs, working_dir=None, env = None):
         stdout=subprocess.PIPE,
         stderr=subprocess.PIPE,
         encoding="utf8",
-        cwd=working_dir,
+        cwd = working_dir,
         env = local_env,
     )
     ret = p.communicate()
 
+    body, error = ret
     if p.returncode != 0:
-        error = ret[1]
         logger.error(cargs)
         logger.error(error)
-        error = {"args": cargs, "error": error}
+        error = {"args": cargs, "error": error, "body": body}
         raise AttributeError(error)
 
-    return ret
+    return body
 
 
 

terraform/main.tf

@@ -40,3 +40,10 @@ output "stage" {
     description = "Deployment stage"
     value = var.stage
 }
+
+
+
+output "info_lambda_name" {
+    description = "Lambda Name"
+    value = module.resources.info_lambda_name
+}

terraform/resources/data.tf

@@ -1,3 +1,5 @@
 data "aws_region" "current" { }
 
-data "aws_caller_identity" "current" { }
+data "aws_caller_identity" "current" { }
+
+

terraform/resources/lambda.tf

@@ -1,47 +1,19 @@
-resource "aws_lambda_function" "lambda-pdal-pipeline" {
-    function_name = "${var.prefix}-${var.stage}-pipeline"
-    description   = "Runs PDAL pipelines"
 
-    image_uri = "${data.aws_caller_identity.current.account_id}.dkr.ecr.${data.aws_region.current.name}.amazonaws.com/${var.prefix}-${var.stage}-pdal_runner:${var.arch}"
-
-    package_type  = "Image"
-    architectures = ["${var.arch == "amd64" ? "x86_64" : "arm64"}"]
-    role          = aws_iam_role.lambda_role.arn
-    depends_on = [ null_resource.ecr_image ]
-    timeout  = var.function_timeout
-    memory_size = 1024
-
-    image_config {
-      command = ["lambda.ecr.pipeline.handler"]
-    }
-    tags = {
-       name = var.prefix
-       Name = "${var.prefix}:lambda.${var.stage}.pipeline"
-       stage = var.stage
-    }
-
-    environment {
-      variables = {
-        HOME = "/var/task"
-      }
-    }
-}
-
-resource "aws_lambda_function" "lambda-pdal-info" {
+resource "aws_lambda_function" "lambda_pdal_info" {
     function_name = "${var.prefix}-${var.stage}-info"
     description   = "Extracts metadata info from point clouds and raster files"
 
     image_uri = "${data.aws_caller_identity.current.account_id}.dkr.ecr.${data.aws_region.current.name}.amazonaws.com/${var.prefix}-${var.stage}-pdal_runner:${var.arch}"
 
     package_type  = "Image"
     architectures = ["${var.arch == "amd64" ? "x86_64" : "arm64"}"]
-    role          = aws_iam_role.lambda_role.arn
+    role          = aws_iam_role.iam_role_for_lambda.arn
     depends_on = [ null_resource.ecr_image ]
     timeout  = var.function_timeout
     memory_size = 1024
 
     image_config {
-      command = ["pdal_handler.info_handler"]
+      command = ["pdal_lambda.ecr.info.handler"]
     }
     tags = {
        name = var.prefix
@@ -54,4 +26,15 @@ resource "aws_lambda_function" "lambda-pdal-info" {
         HOME = "/var/task"
       }
     }
-}
+}
+
+resource "aws_lambda_function_event_invoke_config" "lambda_event_invoke_config" {
+  function_name                = aws_lambda_function.lambda_pdal_info.function_name
+  maximum_event_age_in_seconds = 60
+  maximum_retry_attempts       = 0
+}
+
+output "info_lambda_name" {
+  value = aws_lambda_function.lambda_pdal_info.function_name
+}
+

terraform/resources/roles.tf

@@ -1,46 +1,67 @@
-resource "aws_iam_role" "lambda_role" {
+data "aws_iam_policy_document" "lambda_trust_policy" {
+  statement {
+    actions    = ["sts:AssumeRole"]
+    effect     = "Allow"
+    principals {
+      type        = "Service"
+      identifiers = ["lambda.amazonaws.com"]
+    }
+  }
+}
+
+
+resource "aws_iam_role" "iam_role_for_lambda" {
   name   = "${var.prefix}_${var.stage}_Lambda_Function_Role"
+  assume_role_policy = data.aws_iam_policy_document.lambda_trust_policy.json
+}
+
+
+data "aws_iam_policy_document" "iam_for_lambda_policy_document" {
+  statement {
+    actions = [
+      "logs:CreateLogGroup",
+      "logs:CreateLogStream",
+      "logs:PutLogEvents"
+    ]
+    effect    = "Allow"
+    resources = ["*"]
+    sid       = "CreateCloudWatchLogs"
+  }
+
+  statement {
+    actions = [
+        "s3:GetObject",
+    ]
+    effect    = "Allow"
+    resources = ["*"]
+    sid       = "ReadS3"
+  }
+
+
+
 
-assume_role_policy = <<EOF
-{
- "Version": "2012-10-17",
- "Statement": [
-   {
-     "Action": "sts:AssumeRole",
-     "Principal": {
-       "Service": "lambda.amazonaws.com"
-     },
-     "Effect": "Allow",
-     "Sid": ""
-   }
- ]
 }
-EOF
+
+resource "aws_iam_policy" "lambda_logging_policy" {
+  name = "${var.prefix}-${var.stage}-lambda-logging-policy"
+  path        = "/"
+  description = "IAM policy for logging from a lambda"
+  policy      = data.aws_iam_policy_document.iam_for_lambda_policy_document.json
+}
+
+resource "aws_iam_role_policy_attachment" "lambda_logging_policy_attachment" {
+  role       = aws_iam_role.iam_role_for_lambda.name
+  policy_arn = aws_iam_policy.lambda_logging_policy.arn
 }
 
 resource "aws_iam_policy" "iam_policy_for_lambda" {
   name         = "${var.prefix}_${var.stage}_aws_iam_policy_for_terraform_aws_lambda_role"
   path         = "/"
   description  = "AWS IAM Policy for managing aws lambda role"
-  policy = <<EOF
-{
- "Version": "2012-10-17",
- "Statement": [
-   {
-     "Action": [
-       "logs:CreateLogGroup",
-       "logs:CreateLogStream",
-       "logs:PutLogEvents"
-     ],
-     "Resource": "arn:aws:logs:*:*:*",
-     "Effect": "Allow"
-   }
- ]
-}
-EOF
+  policy = data.aws_iam_policy_document.iam_for_lambda_policy_document.json
 }
 
 resource "aws_iam_role_policy_attachment" "basic" {
   policy_arn = "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole"
-  role       = aws_iam_role.lambda_role.name
+  role       = aws_iam_role.iam_role_for_lambda.name
 }