Merge pull request #5151 from nulib/deploy/staging

Deploy v10.3.0 to production

Author: mbklein

.github/workflows/test.yml

@@ -43,9 +43,6 @@ jobs:
       MIX_ENV: test
     steps:
       - uses: actions/checkout@v2
-      - name: Start service containers in background
-        run: docker compose up -d &
-        working-directory: ./infrastructure/localstack
       - uses: actions/setup-node@v3
         with:
           node-version-file: ./.tool-versions
@@ -54,27 +51,13 @@ jobs:
             app/assets/package-lock.json
             app/priv/nodejs/*/package-lock.json
             lambdas/*/package-lock.json
+      - name: Start service containers in background
+        run: make localstack-start &
       - uses: erlef/setup-beam@v1
         with:
-          otp-version: "27"
-          elixir-version: "1.18.2"
+          version-file: ./.tool-versions
+          version-type: strict
       - uses: hashicorp/setup-terraform@v3
-      - name: Install FFMPEG
-        run: |
-          mkdir -p ${{ runner.temp }}/ffmpeg && \
-          cd ${{ runner.temp }}/ffmpeg && \
-          curl https://s3.amazonaws.com/nul-repo-deploy/ffmpeg-release-64bit-static.tar.xz | tar xJ && \
-          sudo cp $(find . -type f -executable) /usr/local/bin/ && \
-          echo "FFMPEG VERSION: $(ffmpeg -version | sed -n "s/ffmpeg version \([-0-9.]*\).*/\1/p;")"
-      - name: Install ExifTool
-        run: |
-          EXIFTOOL_VERSION=$(curl -s https://exiftool.org/ver.txt)
-          mkdir -p ${{ runner.temp }}/exiftool && \
-          cd ${{ runner.temp }}/exiftool && \
-          curl -L -s https://exiftool.org/Image-ExifTool-${EXIFTOOL_VERSION}.tar.gz | tar xz && \
-          cd Image-ExifTool-${EXIFTOOL_VERSION} && \
-          perl Makefile.PL && \
-          sudo make install
       - name: Cache Elixir dependencies and build
         uses: actions/cache@v4
         with:
@@ -90,33 +73,23 @@ jobs:
         working-directory: app
         env:
           MIX_ENV: test
-      - name: Install Lambda dependencies
-        run: |
-          for pkg in $(find app/priv/nodejs -maxdepth 2 -name package-lock.json) $(find lambdas -maxdepth 2 -name package-lock.json)
-          do
-            cd $(dirname $pkg)
-            npm ci --no-fund
-            cd -
-          done
-      - name: Wait for services to be ready
-        run: timeout 60 bash -c 'while ! ./all-healthy.sh; do sleep 2; done'
-        working-directory: ./infrastructure/localstack
-      - name: Set SECRET_KEY_BASE
-        run: echo "SECRET_KEY_BASE=$(openssl rand -hex 32)" >> $GITHUB_ENV
-      - name: Provision Localstack using Terraform
-        run: |
-          terraform init
-          terraform apply -auto-approve -var-file test.tfvars -var localstack_endpoint=https://localhost.localstack.cloud:4566
-        working-directory: ./infrastructure/localstack
       - name: Install Claude Code
         run: npm install -g @anthropic-ai/claude-code
       - name: Elixir Static Analysis
         run: mix credo
         working-directory: app
+      - name: Wait for services to be ready
+        run: timeout 120 bash -c 'while ! ./all-healthy.sh; do sleep 2; done'
+        working-directory: ./infrastructure/localstack
+      - name: Set SECRET_KEY_BASE
+        run: echo "SECRET_KEY_BASE=$(openssl rand -hex 32)" >> $GITHUB_ENV
+      - name: Provision Localstack using Terraform
+        run: make localstack-provision
       - name: Elixir Tests
         run: mix test || mix test --seed $(cat ${MEADOW_TEST_SAVE_SEED}) --failed
         env:
           AWS_LOCALSTACK: true
+          USE_SAM_LAMBDAS: true
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           MEADOW_TEST_SAVE_SEED: ${{ runner.temp }}/MEADOW_TEST_SEED
         working-directory: app

.gitignore

@@ -44,17 +44,22 @@ npm-debug.log
 /app/priv/cert/
 /app/priv/seed_data/
 
+/lambdas/**/*.log
+
 .elixir_ls
 
 *.tfvars
 
+/infrastructure/**/.aws-sam
 /infrastructure/**/.terraform/
 /infrastructure/**/*.tfstate*
 /infrastructure/**/_build/
-/infrastructure/**/build/
+/infrastructure/**/build/*
+!/infrastructure/pipeline/layers/build/*
 /infrastructure/**/builds/
 /infrastructure/**/package/
 /infrastructure/**/*.plan
+/infrastructure/pipeline/layers/*.zip
 
 /app/tmp/
 
@@ -71,6 +76,8 @@ yarn.lock
 .DS_Store
 **/*/.DS_Store
 
+**/*.pid
+
 lambdas/stream-authorizer/config
 .env
 AGENT.md
@@ -82,4 +89,7 @@ AGENT.md
 /app/priv/python/agent/**/__pycache__/
 /app/priv/python/agent/**/*.pyc
 /app/priv/python/agent/**/*.pyo
-/app/priv/python/agent/**/*.pyd
+/app/priv/python/agent/**/*.pyd
+
+.aws-sam/
+samconfig.*

.tool-versions

@@ -1,6 +1,6 @@
-nodejs 22.14.0
-erlang 27.3
-elixir 1.18.2-otp-27
-terraform 1.7.4
-tflint 0.34.1
+nodejs      22.14.0
+erlang      27.3
+elixir      1.18.2-otp-27
+terraform   1.7.4
+aws-sam-cli 1.152.0
 #npm 11.6.0

Makefile

@@ -0,0 +1,24 @@
+ifndef VERBOSE
+.SILENT:
+endif
+
+MAKEFLAGS += --no-print-directory
+SHELL := /bin/bash
+LOCALSTACK_DIR ?= ./infrastructure/localstack
+PIPELINE_DIR ?= ./infrastructure/pipeline
+TERRAFORM_DIR ?= ./infrastructure/deploy
+
+.PHONY: help
+
+help:
+	@make localstack-help | sed 's/^make /make localstack-/'
+	@make pipeline-help   | sed 's/^make /make pipeline-/'
+
+localstack-%: 
+	cd $(LOCALSTACK_DIR) && make $*
+
+pipeline-%:
+	cd $(PIPELINE_DIR) && make $*
+
+api-%:
+	cd ../dc-api-v2 && make $*

README.md

@@ -15,7 +15,7 @@
 - Run `mix meadow.setup`. This creates the Sequins pipeline, S3 buckets, and database.
 - Install Node.js dependencies with `mix assets.install`
   - `assets.install` looks for all `package-lock.json` files project-wide and runs `npm install` in each directory found, so you don't need to run `npm install` in individual directories.
-- run `sg open all 3001`
+- run `sgport open all 3001`
 - Start the Phoenix server with `mix phx.server` (or `iex -S mix phx.server` if you want to an interactive shell).
 
 Now you can visit [`https://[YOURENV].dev.rdc.library.northwestern.edu:3001/`](https://[YOURENV].dev.rdc.library.northwestern.edu:3001/) from your browser.
@@ -28,13 +28,14 @@ You can stop the Phoenix server with `Ctrl + C` twice
 
 If you need to clear your data and reset the entire development environment, from `meadow/app` run:
 
-```
+```bash
 mix ecto.reset
 mix meadow.search.clear
 mix meadow.pipeline.purge
 clean-s3 dev -y
-
+```
 ...then
+```bash
 mix deps.get
 mix meadow.setup
 mix phx.server
@@ -56,36 +57,30 @@ If you would like to interact directly with the database
 
 ### Run the Elixir test suite
 
-#### Start Test Services
+#### Start/Provision Test Environment
 
 In one terminal:
+```bash
+make localstack-provision
 ```
-cd infrastructure/localstack
-docker compose up
-```
-
-#### Provision Test Environment
-
-Watch the logs until the services seem to stabilize. Then, in another terminal:
-```
-cd infrastructure/localstack
-terraform init
-terraform apply -auto-approve \
-  -var-file test.tfvars \
-  -var localstack_endpoint=https://localhost.localstack.cloud:4566
-```
-
-You will probably see `Warning: AWS account ID not found for provider`, but this can be safely ignored.
 
 #### Run Tests
 
-```
+```bash
 cd app
-export AWS_LOCALSTACK=true 
+export AWS_LOCALSTACK=true
 mix test [test args...]
 ```
 
-**Note:** `mix test` can be run repeatedly without re-provisioning as long as the Docker services are running. If you stop the services, you will need to run Terraform again. Also, do not try to run Meadow with `export AWS_LOCALSTACK=true` set.
+**Note:** Do not try to run Meadow with `export AWS_LOCALSTACK=true` set.
+
+#### Stop/Deprovision Test Environment
+
+Back in the meadow root directory:
+
+```bash
+make localstack-stop
+```
 
 ### GraphQL API
 
@@ -113,13 +108,13 @@ To force an Elasticsearch re-index, and not wait for the 2-minute cycle to kick
 
 Run the interactive shell in a terminal tab
 
-```
+```bash
 iex -S mix
 ```
 
 And force a re-index:
 
-```
+```elixir
 Meadow.Data.Indexer.reindex_all()
 ```
 
@@ -254,18 +249,28 @@ change_b = Enum.at(changes, 1)
 
 ### Doing development on the Meadow Pipeline lambdas
 
-In the AWS developer environment, the lambdas associated with the pipeline are shared amongst developers. In order to do development and see whether it's working you can override the configuration to use your local files instead of the deployed lambdas. Example below (you don't have to override them all. Just the ones you need).
+In the AWS developer environment, the lambdas associated with the pipeline are shared amongst developers. In order to do development and see whether it's working you can override the configuration to use the SAM pipeline the deployed lambdas.
 
-Edit `config/dev.local.exs` to get the lambdas to use the local copy through the port:`
+In one terminal:
+```bash
+make pipeline-start
+```
 
-```elixir
-  config :meadow, :lambda,
-    digester: {:local, {Path.expand("../lambdas/digester/index.js"), "handler"}},
-    exif: {:local, {Path.expand("../lambdas/exif/index.js"), "handler"}},
-    frame_extractor: {:local, {Path.expand("../lambdas/frame-extractor/index.js"), "handler"}},
-    mediainfo: {:local, {Path.expand("../lambdas/mediainfo/index.js"), "handler"}},
-    mime_type: {:local, {Path.expand("../lambdas/mime-type/index.js"), "handler"}},
-    tiff: {:local, {Path.expand("../lambdas/pyramid-tiff/index.js"), "handler"}}
+In another terminal:
+```bash
+cd app
+USE_SAM_LAMBDAS=true iex -S mix phx.server
+```
+
+
+#### Deploying lambdas with SAM
+
+The pipeline infrastructure is defined in `infrastructure/pipeline/template.yaml` and can be deployed with 
+the AWS SAM CLI. There are `make` tasks to assist. Make sure `AWS_PROFILE` is set to the correct admin 
+profile and logged in, and then:
+
+```bash
+make pipeline-deploy ENV=staging
 ```
 
 ### TypeScript/GraphQL Types

app/Dockerfile

@@ -31,9 +31,7 @@ RUN npm ci --force
 COPY ./priv/nodejs /app/priv/nodejs
 WORKDIR /app/priv/nodejs
 RUN for flag in $(find . -name .docker-npm); do \
-      cd $(dirname ${flag}); \
-      npm ci; \
-      cd -; \
+      (cd $(dirname ${flag}) && npm ci); \
     done
 WORKDIR /app
 ARG HONEYBADGER_API_KEY=