Merge pull request #5167 from nulib/deploy/staging

Deploy v10.3.1 to production

Author: kdid

.dockerignore

@@ -3,7 +3,9 @@
 !livebook/**/*
 app/.dockerignore
 app/.elixir_ls
+app/.expert
 app/.git
+app/.lexical
 app/_build
 app/assets/coverage
 app/**/node_modules

.github/PULL_REQUEST_TEMPLATE.md

@@ -4,6 +4,7 @@ Brief high level description of this feature or fix
 # Specific Changes in this PR
 - list changes
 - list changes
+
 # Version bump required by the PR
 
 See [Semantic Versioning 2.0.0](https://semver.org/) for help discerning which is required.
@@ -38,11 +39,10 @@ Also please let developers know if there are any special instructions to test th
 - [ ] Requires data migration
 - [ ] Requires database triggers disabled during deployment/migration
 - [ ] Requires reindex
-- [ ] Terraform changes
-  - [ ] Adds/requires new or changed Terraform variables
-- [ ] Pipeline configuration changes (requires `mix meadow.pipeline.setup` run)
-- [ ] Requires new variable added to `miscellany`
-- [ ] Specific deployment synchronization instructions with other apps/API's
+- [ ] Requires `terraform apply`
+  - [ ] Adds/requires new or changed `ENVIRONMENT.tfvars` variables
+- [ ] Requires `sam deploy`
+  - [ ] Adds/requires new or changed `samconfig.yaml` variables
 - [ ] Other specific instructions/tasks
 
 

.github/scripts/upload_source_maps.sh

@@ -4,7 +4,7 @@ docker pull $MEADOW_IMAGE
 container_id=$(docker create ${MEADOW_IMAGE})
 docker cp ${container_id}:/app/lib/meadow-${MEADOW_VERSION}/priv/static/js ${container_id}
 cd $container_id
-for minified_file in app*.js app*.css; do
+for minified_file in app*.js; do
   source_map=$(tail -1 $minified_file | sed -E 's/^.+sourceMappingURL=\W*(\S+).*$/\1/')
   for ext in "" ".gz"; do
     curl https://api.honeybadger.io/v1/source_maps \

.github/workflows/deploy.yml

@@ -17,7 +17,7 @@ jobs:
     runs-on: ubuntu-latest
     outputs:
       deploy_env: ${{ steps.set-deploy-env.outputs.result }}
-      container_tag: ${{ steps.set-container-tag.outputs.result }}
+      image_tag: ${{ steps.set-image-tag.outputs.result }}
       meadow_tenant: ${{ steps.set-meadow-prefix.outputs.result }}
     steps:
       - name: Set DEPLOY_ENV from Branch Name
@@ -30,8 +30,8 @@ jobs:
           fi
         env:
           BRANCH: ${{ github.ref }}
-      - name: Set container tag from DEPLOY_ENV
-        id: set-container-tag
+      - name: Set image tag from DEPLOY_ENV
+        id: set-image-tag
         run: |
           if [[ $DEPLOY_ENV == 'production' || $DEPLOY_ENV == 'staging' ]]; then
             echo "result=latest" >> $GITHUB_OUTPUT
@@ -53,15 +53,15 @@ jobs:
       - name: Output Environment Info
         run: |
           echo "Deploying to environment: ${{ steps.set-deploy-env.outputs.result }}"
-          echo "Using container tag: ${{ steps.set-container-tag.outputs.result }}"
+          echo "Using image tag: ${{ steps.set-image-tag.outputs.result }}"
           echo "Using meadow tenant prefix: ${{ steps.set-meadow-prefix.outputs.result }}"
   build:
     if: ${{ !github.event.pull_request && !contains(github.event.head_commit.message, '[no-deploy]') }}
     runs-on: ubuntu-latest
     needs: get-environment
     env:
       DEPLOY_ENV: ${{ needs.get-environment.outputs.deploy_env }}
-      CONTAINER_TAG: ${{ needs.get-environment.outputs.container_tag }}
+      IMAGE_TAG: ${{ needs.get-environment.outputs.image_tag }}
       MEADOW_TENANT: ${{ needs.get-environment.outputs.meadow_tenant }}
     steps:
       - uses: actions/checkout@v2
@@ -72,9 +72,8 @@ jobs:
           GITHUB_ENV: ${{ env.GITHUB_ENV }}
           SECRETS: ${{ toJSON(secrets) }}
       - name: Extract MEADOW_VERSION from mix.exs
-        run: echo "MEADOW_VERSION=$(grep '@app_version "' mix.exs | sed -n 's/^.*"\(.*\)".*/\1/p')" >> $GITHUB_ENV
-        working-directory: app
-      - run: echo "Building Meadow v${MEADOW_VERSION} as nulib/meadow:${CONTAINER_TAG}"
+        run: echo "MEADOW_VERSION=$(make app-version)" >> $GITHUB_ENV
+      - run: echo "Building Meadow v${MEADOW_VERSION} as nulib/meadow:${IMAGE_TAG}"
       - name: Tag Meadow Release
         if: ${{ github.ref == 'refs/heads/main' }}
         run: |
@@ -86,42 +85,45 @@ jobs:
       - name: Login to Amazon ECR
         id: login-ecr
         uses: aws-actions/amazon-ecr-login@v1
-      - name: Determine container tags
-        id: determine-container-tags
+      - name: Determine image tags
+        id: determine-image-tags
         run: |
           {
             echo "tags<<EOF"
-            echo "${REGISTRY_NAME}/meadow:${CONTAINER_TAG}"
-            if [[ ${CONTAINER_TAG} == "latest" ]]; then
+            echo "${REGISTRY_NAME}/meadow:${IMAGE_TAG}"
+            if [[ ${IMAGE_TAG} == "latest" ]]; then
               echo "${REGISTRY_NAME}/meadow:${MEADOW_VERSION}"
             fi
             echo "EOF"
           } >> $GITHUB_OUTPUT
         env:
           REGISTRY_NAME: ${{ steps.login-ecr.outputs.registry }}
-      - name: Set Meadow Release Name
-        id: meadow-release
-        run: echo "name=meadow-$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+      - name: Set Build Options
+        id: build-options
+        run: |
+          echo "release_name=meadow-$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+          echo "build_image=$(make build-image)" >> $GITHUB_OUTPUT
+          echo "runtime_image=$(make runtime-image)" >> $GITHUB_OUTPUT
       - uses: docker/build-push-action@v2
         with:
           context: ./app
           push: true
-          tags: ${{ steps.determine-container-tags.outputs.tags }}
+          tags: ${{ steps.determine-image-tags.outputs.tags }}
           build-args: |
-            BUILD_IMAGE=hexpm/elixir:1.18.2-erlang-27.3-debian-bookworm-20250224
-            RUNTIME_IMAGE=node:22-bookworm-slim
+            BUILD_IMAGE=${{ steps.build-options.outputs.build_image }}
+            RUNTIME_IMAGE=${{ steps.build-options.outputs.runtime_image }}
             HONEYBADGER_API_KEY=${{ secrets.HONEYBADGER_API_KEY }}
             HONEYBADGER_API_KEY_FRONTEND=${{ secrets.HONEYBADGER_API_KEY_FRONTEND }}
             HONEYBADGER_ENVIRONMENT=${{ env.DEPLOY_ENV }}
             HONEYBADGER_REVISION=${{ github.sha }}
-            RELEASE_NAME=${{ steps.meadow-release.outputs.name }}
+            RELEASE_NAME=${{ steps.build-options.outputs.release_name }}
             MEADOW_TENANT=${{ env.MEADOW_TENANT }}
             MEADOW_VERSION=${{ env.MEADOW_VERSION }}
       - name: Upload Source Maps to Honeybadger
         run: .github/scripts/upload_source_maps.sh
         env:
           DEPLOY_ENV: ${{ env.DEPLOY_ENV }}
-          MEADOW_IMAGE: ${{ steps.login-ecr.outputs.registry }}/meadow:${{ env.CONTAINER_TAG }}
+          MEADOW_IMAGE: ${{ steps.login-ecr.outputs.registry }}/meadow:${{ env.IMAGE_TAG }}
           MEADOW_VERSION: ${{ env.MEADOW_VERSION }}
           HONEYBADGER_API_KEY_FRONTEND: ${{ secrets.HONEYBADGER_API_KEY_FRONTEND }}
           HONEYBADGER_REVISION: ${{ github.sha }}
@@ -150,13 +152,12 @@ jobs:
     needs: get-environment
     env:
       DEPLOY_ENV: ${{ needs.get-environment.outputs.deploy_env }}
-      CONTAINER_TAG: ${{ needs.get-environment.outputs.container_tag }}
+      IMAGE_TAG: ${{ needs.get-environment.outputs.image_tag }}
       MEADOW_TENANT: ${{ needs.get-environment.outputs.meadow_tenant }}
     steps:
       - uses: actions/checkout@v2
       - name: Extract MEADOW_VERSION from mix.exs
-        run: echo "MEADOW_VERSION=$(grep '@app_version "' mix.exs | sed -n 's/^.*"\(.*\)".*/\1/p')" >> $GITHUB_ENV
-        working-directory: app
+        run: echo "MEADOW_VERSION=$(make app-version)" >> $GITHUB_ENV
       - name: Configure AWS
         run: .github/scripts/configure_aws.sh
         env:
@@ -168,13 +169,13 @@ jobs:
       - name: Login to Amazon ECR
         id: login-ecr
         uses: aws-actions/amazon-ecr-login@v1
-      - name: Determine container tags
-        id: determine-container-tags
+      - name: Determine image tags
+        id: determine-image-tags
         run: |
           {
             echo "tags<<EOF"
-            echo "${REGISTRY_NAME}/meadow:livebook-${CONTAINER_TAG}"
-            if [[ ${CONTAINER_TAG} == "latest" ]]; then
+            echo "${REGISTRY_NAME}/meadow:livebook-${IMAGE_TAG}"
+            if [[ ${IMAGE_TAG} == "latest" ]]; then
               echo "${REGISTRY_NAME}/meadow:livebook-${MEADOW_VERSION}"
             fi
             echo "EOF"
@@ -186,7 +187,7 @@ jobs:
           context: .
           file: ./Dockerfile.livebook
           push: true
-          tags: ${{ steps.determine-container-tags.outputs.tags }}
+          tags: ${{ steps.determine-image-tags.outputs.tags }}
   deploy:
     needs: 
       - get-environment
@@ -196,7 +197,7 @@ jobs:
     runs-on: ubuntu-latest
     env:
       DEPLOY_ENV: ${{ needs.get-environment.outputs.deploy_env }}
-      CONTAINER_TAG: ${{ needs.get-environment.outputs.container_tag }}
+      IMAGE_TAG: ${{ needs.get-environment.outputs.image_tag }}
       MEADOW_TENANT: ${{ needs.get-environment.outputs.meadow_tenant }}
     steps:
       - uses: actions/checkout@v2

.github/workflows/test.yml

@@ -23,20 +23,8 @@ jobs:
         run: npm install -g npm@${{ env.NPM_VERSION }}
         env:
           NPM_VERSION: 11.6.0
-      - name: Install JS dependencies
-        run: |
-          npm ci --force --no-fund
-          npm list
-        working-directory: app/assets
-      - name: JS Static Analysis
-        run: npm run-script prettier
-        working-directory: app/assets
-      - name: JS Tests
-        run: npm run-script ci:silent -- -w 1
-        working-directory: app/assets
-      - name: Test ESBuild
-        run: npm run-script deploy
-        working-directory: app/assets
+      - name: Frontend CI
+        run: make app-frontend-ci
   elixir-test:
     runs-on: ubuntu-latest
     env:
@@ -69,15 +57,11 @@ jobs:
             ${{ runner.os }}-hex-v6-${{ hashFiles('app/mix.lock') }}
             ${{ runner.os }}-hex-v6-
       - name: Install Elixir dependencies
-        run: mix do deps.get, compile
-        working-directory: app
+        run: make app-compile
         env:
           MIX_ENV: test
-      - name: Install Claude Code
-        run: npm install -g @anthropic-ai/claude-code
       - name: Elixir Static Analysis
-        run: mix credo
-        working-directory: app
+        run: make app-lint
       - name: Wait for services to be ready
         run: timeout 120 bash -c 'while ! ./all-healthy.sh; do sleep 2; done'
         working-directory: ./infrastructure/localstack
@@ -86,13 +70,12 @@ jobs:
       - name: Provision Localstack using Terraform
         run: make localstack-provision
       - name: Elixir Tests
-        run: mix test || mix test --seed $(cat ${MEADOW_TEST_SAVE_SEED}) --failed
+        run: make app-test
         env:
           AWS_LOCALSTACK: true
           USE_SAM_LAMBDAS: true
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           MEADOW_TEST_SAVE_SEED: ${{ runner.temp }}/MEADOW_TEST_SEED
-        working-directory: app
       # - name: Test DB Rollback
       #   run: mix ecto.rollback --all
       #   working-directory: app

.gitignore

@@ -51,6 +51,7 @@ npm-debug.log
 *.tfvars
 
 /infrastructure/**/.aws-sam
+/infrastructure/**/.localstack/
 /infrastructure/**/.terraform/
 /infrastructure/**/*.tfstate*
 /infrastructure/**/_build/
@@ -78,18 +79,18 @@ yarn.lock
 
 **/*.pid
 
-lambdas/stream-authorizer/config
 .env
 AGENT.md
 
 # Python build artifacts
-/app/priv/python/agent/build/
-/app/priv/python/agent/dist/
-/app/priv/python/agent/src/*.egg-info/
-/app/priv/python/agent/**/__pycache__/
-/app/priv/python/agent/**/*.pyc
-/app/priv/python/agent/**/*.pyo
-/app/priv/python/agent/**/*.pyd
+/lambdas/metadata-agent/requirements.txt
+/lambdas/metadata-agent/.venv/
+/lambdas/metadata-agent/**/*.egg-info/
+/lambdas/metadata-agent/**/__pycache__/
+/lambdas/metadata-agent/**/*.pyc
+/lambdas/metadata-agent/**/*.pyo
+/lambdas/metadata-agent/**/*.pyd
 
 .aws-sam/
 samconfig.*
+livebook/versions

.tool-versions

@@ -1,6 +1,6 @@
 nodejs      22.14.0
-erlang      27.3
-elixir      1.18.2-otp-27
+erlang      28.1.1
+elixir      1.19.3-otp-28
 terraform   1.7.4
 aws-sam-cli 1.152.0
 #npm 11.6.0

Makefile

@@ -4,21 +4,85 @@ endif
 
 MAKEFLAGS += --no-print-directory
 SHELL := /bin/bash
+
+APP_DIR ?= ./app
 LOCALSTACK_DIR ?= ./infrastructure/localstack
 PIPELINE_DIR ?= ./infrastructure/pipeline
-TERRAFORM_DIR ?= ./infrastructure/deploy
-
-.PHONY: help
+MEADOW_VERSION ?= $(shell $(MAKE) --no-print-directory app-version)
+IMAGE_TAG ?= latest
+VERSIONS_FILE ?= livebook/versions
+ELIXIR_VERSION ?= $(shell . $(VERSIONS_FILE) && echo $${elixir})
+OTP_VERSION ?= $(shell . $(VERSIONS_FILE) && echo $${otp})
+UBUNTU_VERSION ?= $(shell . $(VERSIONS_FILE) && echo $${ubuntu})
+BUILD_IMAGE ?= hexpm/elixir:$(ELIXIR_VERSION)-erlang-$(OTP_VERSION)-ubuntu-$(UBUNTU_VERSION)
+RUNTIME_IMAGE ?= ubuntu:$(UBUNTU_VERSION)
 
 help:
+	@make app-help | sed 's/^make /make app-/'
+	@echo "make ci                    | install all dependencies, start test environment, and run all tests"
 	@make localstack-help | sed 's/^make /make localstack-/'
 	@make pipeline-help   | sed 's/^make /make pipeline-/'
 
-localstack-%: 
+$(VERSIONS_FILE):
+	livebook_version=$$(curl -s https://api.github.com/repos/livebook-dev/livebook/releases | jq -r '[.[] | select(.tag_name != "nightly")][0].tag_name') && \
+	curl -sL -o $@ "https://raw.githubusercontent.com/livebook-dev/livebook/refs/tags/$${livebook_version}/versions"
+
+.tool-versions: $(VERSIONS_FILE)
+	. $(VERSIONS_FILE) && \
+	otp_major="$${otp%%.*}" && \
+	sed -i "s/^erlang .*/erlang      $${otp}/" .tool-versions	&& \
+	sed -i "s/^elixir .*/elixir      $${elixir}-otp-$${otp_major}/" .tool-versions
+
+build-image: $(VERSIONS_FILE)
+	@echo $(BUILD_IMAGE)
+
+runtime-image: $(VERSIONS_FILE)
+	@echo $(RUNTIME_IMAGE)
+
+app-image: $(VERSIONS_FILE)
+app-image: AWS_ACCOUNT_ID ?= $(shell aws sts get-caller-identity --query Account --output text)
+app-image: AWS_REGION ?= us-east-1
+app-image: ECR_REPO ?= $(AWS_ACCOUNT_ID).dkr.ecr.$(AWS_REGION).amazonaws.com
+app-image: RELEASE_NAME ?= meadow-$(shell git rev-parse --short HEAD)
+app-image: NODE_VERSION ?= 22
+app-image: MEADOW_TENANT ?= meadow
+app-image:
+	cd app && \
+	docker build \
+		--build-arg HONEYBADGER_API_KEY=$(HONEYBADGER_API_KEY) \
+		--build-arg HONEYBADGER_API_KEY_FRONTEND=$(HONEYBADGER_API_KEY_FRONTEND) \
+		--build-arg HONEYBADGER_ENVIRONMENT=$(HONEYBADGER_ENVIRONMENT) \
+		--build-arg HONEYBADGER_REVISION=$(HONEYBADGER_REVISION) \
+		--build-arg BUILD_IMAGE=$(BUILD_IMAGE) \
+		--build-arg RUNTIME_IMAGE=$(RUNTIME_IMAGE) \
+		--build-arg NODE_VERSION=$(NODE_VERSION) \
+		--build-arg MEADOW_TENANT=$(MEADOW_TENANT) \
+		--build-arg MEADOW_VERSION=$(MEADOW_VERSION) \
+		--tag $(ECR_REPO)/meadow:$(IMAGE_TAG) \
+		--tag $(ECR_REPO)/meadow:$(MEADOW_VERSION) \
+		.
+
+livebook-image:
+	docker build \
+		--tag nulib/meadow:livebook-$(MEADOW_VERSION) \
+		--tag nulib/meadow:livebook-$(IMAGE_TAG) \
+		-f Dockerfile.livebook \
+		.
+
+localstack-%:
 	cd $(LOCALSTACK_DIR) && make $*
 
 pipeline-%:
 	cd $(PIPELINE_DIR) && make $*
 
-api-%:
-	cd ../dc-api-v2 && make $*
+app-test: localstack-provision
+app-all-test: localstack-provision
+
+app-%:
+	cd $(APP_DIR) && make $*
+
+ci: localstack-provision app-all-deps app-all-test localstack-stop
+
+clean: localstack-clean pipeline-clean
+
+distclean: clean app-distclean