Merge pull request #5025 from nulib/deploy/staging

Deploy v10.0.0 to production

Author: mbklein

.github/workflows/deploy.yml

@@ -12,20 +12,59 @@ concurrency:
 permissions:
   actions: write
 jobs:
-  build:
+  get-environment:
     if: ${{ !github.event.pull_request && !contains(github.event.head_commit.message, '[no-deploy]') }}
     runs-on: ubuntu-latest
+    outputs:
+      deploy_env: ${{ steps.set-deploy-env.outputs.result }}
+      container_tag: ${{ steps.set-container-tag.outputs.result }}
+      meadow_tenant: ${{ steps.set-meadow-prefix.outputs.result }}
     steps:
-      - uses: actions/checkout@v2
       - name: Set DEPLOY_ENV from Branch Name
+        id: set-deploy-env
         run: |
           if [[ $BRANCH == 'refs/heads/main' ]]; then
-            echo "DEPLOY_ENV=production" >> $GITHUB_ENV
+            echo "result=production" >> $GITHUB_OUTPUT
           else
-            echo "DEPLOY_ENV=$(echo $BRANCH | awk -F/ '{print $NF}')" >> $GITHUB_ENV
+            echo "result=$(echo $BRANCH | awk -F/ '{print $NF}')" >> $GITHUB_OUTPUT
           fi
         env:
           BRANCH: ${{ github.ref }}
+      - name: Set container tag from DEPLOY_ENV
+        id: set-container-tag
+        run: |
+          if [[ $DEPLOY_ENV == 'production' || $DEPLOY_ENV == 'staging' ]]; then
+            echo "result=latest" >> $GITHUB_OUTPUT
+          else
+            echo "result=$DEPLOY_ENV" >> $GITHUB_OUTPUT
+          fi
+        env:
+          DEPLOY_ENV: ${{ steps.set-deploy-env.outputs.result }}
+      - name: Set MEADOW_TENANT from DEPLOY_ENV
+        id: set-meadow-prefix
+        run: |
+          if [[ $DEPLOY_ENV == 'production' || $DEPLOY_ENV == 'staging' ]]; then
+            echo "result=meadow" >> $GITHUB_OUTPUT
+          else
+            echo "result=$DEPLOY_ENV" >> $GITHUB_OUTPUT
+          fi
+        env:
+          DEPLOY_ENV: ${{ steps.set-deploy-env.outputs.result }}
+      - name: Output Environment Info
+        run: |
+          echo "Deploying to environment: ${{ steps.set-deploy-env.outputs.result }}"
+          echo "Using container tag: ${{ steps.set-container-tag.outputs.result }}"
+          echo "Using meadow tenant prefix: ${{ steps.set-meadow-prefix.outputs.result }}"
+  build:
+    if: ${{ !github.event.pull_request && !contains(github.event.head_commit.message, '[no-deploy]') }}
+    runs-on: ubuntu-latest
+    needs: get-environment
+    env:
+      DEPLOY_ENV: ${{ needs.get-environment.outputs.deploy_env }}
+      CONTAINER_TAG: ${{ needs.get-environment.outputs.container_tag }}
+      MEADOW_TENANT: ${{ needs.get-environment.outputs.meadow_tenant }}
+    steps:
+      - uses: actions/checkout@v2
       - name: Configure AWS
         run: .github/scripts/configure_aws.sh
         env:
@@ -35,7 +74,7 @@ jobs:
       - name: Extract MEADOW_VERSION from mix.exs
         run: echo "MEADOW_VERSION=$(grep '@app_version "' mix.exs | sed -n 's/^.*"\(.*\)".*/\1/p')" >> $GITHUB_ENV
         working-directory: app
-      - run: echo "Building Meadow v${MEADOW_VERSION} as nulib/meadow:latest"
+      - run: echo "Building Meadow v${MEADOW_VERSION} as nulib/meadow:${CONTAINER_TAG}"
       - name: Tag Meadow Release
         if: ${{ github.ref == 'refs/heads/main' }}
         run: |
@@ -47,26 +86,38 @@ jobs:
       - name: Login to Amazon ECR
         id: login-ecr
         uses: aws-actions/amazon-ecr-login@v1
+      - name: Determine container tags
+        id: determine-container-tags
+        run: |
+          {
+            echo "tags<<EOF"
+            echo "${REGISTRY_NAME}/meadow:${CONTAINER_TAG}"
+            if [[ ${CONTAINER_TAG} == "latest" ]]; then
+              echo "${REGISTRY_NAME}/meadow:${MEADOW_VERSION}"
+            fi
+            echo "EOF"
+          } >> $GITHUB_OUTPUT
+        env:
+          REGISTRY_NAME: ${{ steps.login-ecr.outputs.registry }}
       - uses: docker/build-push-action@v2
         with:
           context: ./app
           push: true
-          tags: |
-            ${{ steps.login-ecr.outputs.registry }}/meadow:latest
-            ${{ steps.login-ecr.outputs.registry }}/meadow:${{ env.MEADOW_VERSION }}
+          tags: ${{ steps.determine-container-tags.outputs.tags }}
           build-args: |
             BUILD_IMAGE=hexpm/elixir:1.18.2-erlang-27.3-debian-bookworm-20250224
             RUNTIME_IMAGE=node:22-bookworm-slim
             HONEYBADGER_API_KEY=${{ secrets.HONEYBADGER_API_KEY }}
             HONEYBADGER_API_KEY_FRONTEND=${{ secrets.HONEYBADGER_API_KEY_FRONTEND }}
             HONEYBADGER_ENVIRONMENT=${{ env.DEPLOY_ENV }}
             HONEYBADGER_REVISION=${{ github.sha }}
+            MEADOW_TENANT=${{ env.MEADOW_TENANT }}
             MEADOW_VERSION=${{ env.MEADOW_VERSION }}
       - name: Upload Source Maps to Honeybadger
         run: .github/scripts/upload_source_maps.sh
         env:
           DEPLOY_ENV: ${{ env.DEPLOY_ENV }}
-          MEADOW_IMAGE: ${{ steps.login-ecr.outputs.registry }}/meadow:latest
+          MEADOW_IMAGE: ${{ steps.login-ecr.outputs.registry }}/meadow:${{ env.CONTAINER_TAG }}
           MEADOW_VERSION: ${{ env.MEADOW_VERSION }}
           HONEYBADGER_API_KEY_FRONTEND: ${{ secrets.HONEYBADGER_API_KEY_FRONTEND }}
           HONEYBADGER_REVISION: ${{ github.sha }}
@@ -92,17 +143,13 @@ jobs:
   build-livebook:
     if: ${{ !github.event.pull_request && !contains(github.event.head_commit.message, '[no-deploy]') }}
     runs-on: ubuntu-latest
+    needs: get-environment
+    env:
+      DEPLOY_ENV: ${{ needs.get-environment.outputs.deploy_env }}
+      CONTAINER_TAG: ${{ needs.get-environment.outputs.container_tag }}
+      MEADOW_TENANT: ${{ needs.get-environment.outputs.meadow_tenant }}
     steps:
       - uses: actions/checkout@v2
-      - name: Set DEPLOY_ENV from Branch Name
-        run: |
-          if [[ $BRANCH == 'refs/heads/main' ]]; then
-            echo "DEPLOY_ENV=production" >> $GITHUB_ENV
-          else
-            echo "DEPLOY_ENV=$(echo $BRANCH | awk -F/ '{print $NF}')" >> $GITHUB_ENV
-          fi
-        env:
-          BRANCH: ${{ github.ref }}
       - name: Extract MEADOW_VERSION from mix.exs
         run: echo "MEADOW_VERSION=$(grep '@app_version "' mix.exs | sed -n 's/^.*"\(.*\)".*/\1/p')" >> $GITHUB_ENV
         working-directory: app
@@ -117,46 +164,63 @@ jobs:
       - name: Login to Amazon ECR
         id: login-ecr
         uses: aws-actions/amazon-ecr-login@v1
+      - name: Determine container tags
+        id: determine-container-tags
+        run: |
+          {
+            echo "tags<<EOF"
+            echo "${REGISTRY_NAME}/meadow:livebook-${CONTAINER_TAG}"
+            if [[ ${CONTAINER_TAG} == "latest" ]]; then
+              echo "${REGISTRY_NAME}/meadow:livebook-${MEADOW_VERSION}"
+            fi
+            echo "EOF"
+          } >> $GITHUB_OUTPUT
+        env:
+          REGISTRY_NAME: ${{ steps.login-ecr.outputs.registry }}
       - uses: docker/build-push-action@v2
         with:
           context: .
           file: ./Dockerfile.livebook
           push: true
-          tags: |
-            ${{ steps.login-ecr.outputs.registry }}/meadow:livebook-latest
-            ${{ steps.login-ecr.outputs.registry }}/meadow:livebook-${{ env.MEADOW_VERSION }}
+          tags: ${{ steps.determine-container-tags.outputs.tags }}
   deploy:
     needs: 
+      - get-environment
       - build
       - build-livebook
     if: ${{ ! startsWith(github.ref, 'refs/heads/build/') }}
     runs-on: ubuntu-latest
+    env:
+      DEPLOY_ENV: ${{ needs.get-environment.outputs.deploy_env }}
+      CONTAINER_TAG: ${{ needs.get-environment.outputs.container_tag }}
+      MEADOW_TENANT: ${{ needs.get-environment.outputs.meadow_tenant }}
     steps:
       - uses: actions/checkout@v2
         with:
           fetch-depth: 2
-      - name: Set DEPLOY_ENV from Branch Name
-        run: |
-          if [[ $BRANCH == 'refs/heads/main' ]]; then
-            echo "DEPLOY_ENV=production" >> $GITHUB_ENV
-          else
-            echo "DEPLOY_ENV=$(echo $BRANCH | awk -F/ '{print $NF}')" >> $GITHUB_ENV
-          fi
-        env:
-          BRANCH: ${{ github.ref }}
       - name: Configure AWS
         run: .github/scripts/configure_aws.sh
         env:
           DEPLOY_ENV: ${{ env.DEPLOY_ENV }}
           GITHUB_ENV: ${{ env.GITHUB_ENV }}
           SECRETS: ${{ toJSON(secrets) }}
+      - name: Set Service and Task Names
+        id: set-service-task
+        run: |
+          if [[ "$DEPLOY_ENV" == "production" || "$DEPLOY_ENV" == "staging" ]]; then
+            echo "cluster=meadow" >> $GITHUB_OUTPUT
+            echo "task=meadow-all" >> $GITHUB_OUTPUT
+          else
+            echo "cluster=${MEADOW_TENANT}" >> $GITHUB_OUTPUT
+            echo "task=${MEADOW_TENANT}-all" >> $GITHUB_OUTPUT
+          fi
       - name: Update ECS Service
         run: .github/scripts/update_ecs_service.sh
         env:
-          ECS_CLUSTER: meadow
+          ECS_CLUSTER: ${{ steps.set-service-task.outputs.cluster }}
           ECS_CONTAINER: meadow
           ECS_SERVICE: meadow
-          ECS_TASK: meadow-all
+          ECS_TASK: ${{ steps.set-service-task.outputs.task }}
           PRIOR_HEAD: ${{ github.event.before }}
       - name: Notify Honeybadger
         run: .github/scripts/honeybadger_deploy_notification.sh

.github/workflows/test.yml

@@ -108,6 +108,8 @@ jobs:
           terraform init
           terraform apply -auto-approve -var-file test.tfvars -var localstack_endpoint=https://localhost.localstack.cloud:4566
         working-directory: ./infrastructure/localstack
+      - name: Install Claude Code
+        run: npm install -g @anthropic-ai/claude-code
       - name: Elixir Static Analysis
         run: mix credo
         working-directory: app

.gitignore

@@ -72,3 +72,14 @@ yarn.lock
 **/*/.DS_Store
 
 lambdas/stream-authorizer/config
+.env
+AGENT.md
+
+# Python build artifacts
+/app/priv/python/agent/build/
+/app/priv/python/agent/dist/
+/app/priv/python/agent/src/*.egg-info/
+/app/priv/python/agent/**/__pycache__/
+/app/priv/python/agent/**/*.pyc
+/app/priv/python/agent/**/*.pyo
+/app/priv/python/agent/**/*.pyd

README.md

@@ -123,6 +123,135 @@ And force a re-index:
 Meadow.Data.Indexer.reindex_all()
 ```
 
+### AI Agent Plans
+
+Meadow supports AI agent-generated plans for batch modifications to works. The system uses a two-table structure that allows agents to propose work-specific changes based on high-level prompts.
+
+#### Data Model
+
+**Plans** - High-level task definitions
+- `prompt`: Natural language instruction (e.g., "Add a date_created EDTF string for the work based on the work's existing description, creator, and temporal subjects")
+- `query`: OpenSearch query string identifying target works
+  - Collection query: `"collection.id:abc-123"`
+  - Specific works: `"id:(work-id-1 OR work-id-2 OR work-id-3)"`
+- `status`: `:pending, `:proposed`, `:approved`, `:rejected`, `:completed`, or `:error`
+
+**PlanChanges** - Work-specific modifications
+- `plan_id`: Foreign key to parent plan
+- `work_id`: Specific work being modified
+- `add`: Map of values to append to existing work data
+- `delete`: Map of values to remove from existing work data
+- `replace`: Map of values to fully replace in work data
+- `status`: Individual approval/rejection tracking
+
+Each PlanChange must specify at least one operation (`add`, `delete`, or `replace`).
+
+#### PlanChange payloads
+
+- `add` merges values into existing metadata. For lists (like subjects or notes) the values are appended when they are not already present. Scalar fields (e.g., `title`) are merged according to the context (`:append` for `add`, `:replace` for `replace`).
+- `delete` removes the provided values verbatim. For controlled vocabularies this means the JSON structure must match what is stored in the database (role/term maps). The planner normalizes structs and string-keyed maps automatically when applying changes.
+- `replace` overwrites existing values for the provided keys. Use this when the existing content should be replaced entirely instead of appended or removed.
+
+Controlled metadata entries (subjects, creators, contributors, etc.) follow the shape below. For subjects you must supply both the `role` (with at least `id`/`scheme`) and the `term.id`; extra fields such as `label` or `variants` are ignored when applying but can be included when working with structs in IEx:
+
+```elixir
+%{
+  descriptive_metadata: %{
+    subject: [
+      %{
+        role: %{id: "TOPICAL", scheme: "subject_role"},
+        term: %{
+          id: "http://id.loc.gov/authorities/subjects/sh85141086",
+          label: "Universities and colleges",
+          variants: ["Colleges", "Higher education institutions"]
+        }
+      }
+    ]
+  }
+}
+```
+
+When constructing PlanChanges you can mix-and-match operations as needed. For example, to remove an outdated subject and add a new one in a single change:
+
+```elixir
+delete: %{
+  descriptive_metadata: %{
+    subject: [
+      %{role: %{id: "TOPICAL", scheme: "subject_role"}, term: %{id: "mock1:result2"}}
+    ]
+  }
+},
+add: %{
+  descriptive_metadata: %{
+    subject: [
+      %{role: %{id: "TOPICAL", scheme: "subject_role"}, term: %{id: "mock1:result5"}}
+    ]
+  }
+}
+```
+
+#### Example Workflows
+
+**Adding new metadata:**
+```elixir
+# 1. Create a plan with a query - PlanChanges are auto-generated for matching works
+{:ok, plan} = Meadow.Data.Planner.create_plan(%{
+  prompt: "Add a date_created EDTF string for the work based on the work's existing description, creator, and temporal subjects",
+  query: "collection.id:abc-123"
+})
+
+# 2. Agent updates each auto-generated PlanChange with work-specific values
+changes = Meadow.Data.Planner.list_plan_changes(plan.id)
+
+change_a = Enum.at(changes, 0)
+{:ok, updated_change_a} = Meadow.Data.Planner.update_plan_change(change_a, %{
+  add: %{descriptive_metadata: %{date_created: ["1896-11-10"]}}
+})
+
+change_b = Enum.at(changes, 1)
+{:ok, updated_change_b} = Meadow.Data.Planner.update_plan_change(change_b, %{
+  add: %{descriptive_metadata: %{date_created: ["1923-05"]}}
+})
+```
+
+**Removing unwanted values:**
+```elixir
+# Remove extraneous subject headings
+{:ok, change} = Meadow.Data.Planner.create_plan_change(%{
+  plan_id: plan.id,
+  work_id: "work-id",
+  delete: %{
+    descriptive_metadata: %{
+      subject: [
+        %{role: %{id: "TOPICAL", scheme: "subject_role"}, term: %{id: "http://example.org/photograph"}},
+        %{role: %{id: "TOPICAL", scheme: "subject_role"}, term: %{id: "http://example.org/image"}}
+      ]
+    }
+  }
+})
+```
+
+**Replacing existing values:**
+```elixir
+# Replace the title
+{:ok, change} = Meadow.Data.Planner.create_plan_change(%{
+  plan_id: plan.id,
+  work_id: "work-id",
+  replace: %{descriptive_metadata: %{title: "New Title"}}
+})
+```
+
+**Reviewing and applying:**
+```elixir
+# 3. User reviews and approves
+{:ok, _} = Meadow.Data.Planner.approve_plan(plan, "user@example.com")
+{:ok, _} = Meadow.Data.Planner.approve_plan_change(change_a, "user@example.com")
+{:ok, _} = Meadow.Data.Planner.approve_plan_change(change_b, "user@example.com")
+
+# 4. Apply approved changes
+{:ok, completed_plan} = Meadow.Data.Planner.apply_plan(plan)
+```
+
 ### Doing development on the Meadow Pipeline lambdas
 
 In the AWS developer environment, the lambdas associated with the pipeline are shared amongst developers. In order to do development and see whether it's working you can override the configuration to use your local files instead of the deployed lambdas. Example below (you don't have to override them all. Just the ones you need).

app/Dockerfile

@@ -10,6 +10,7 @@ ARG HONEYBADGER_API_KEY=
 ARG HONEYBADGER_API_KEY_FRONTEND=
 ARG HONEYBADGER_ENVIRONMENT=
 ARG HONEYBADGER_REVISION=
+ARG MEADOW_TENANT=
 ARG MEADOW_VERSION=
 ENV MIX_ENV=prod
 RUN  mix local.hex --force \
@@ -47,10 +48,12 @@ RUN mix release --overwrite
 FROM ${RUNTIME_IMAGE}
 LABEL edu.northwestern.library.app=meadow \
   edu.northwestern.library.stage=runtime
-RUN apt update -qq && apt install -y curl jq libssl-dev libncurses5-dev
+RUN apt update -qq && apt install -y curl git jq libssl-dev libncurses5-dev
 ENV LANG=C.UTF-8
 EXPOSE 4000 4369
 COPY --from=build /app/_build/prod/rel/meadow /app
 WORKDIR /app
+RUN npm install -g @anthropic-ai/claude-code \
+ && bin/meadow eval "MeadowAI.Verification.verify_claude_and_exit()"
 ENTRYPOINT ["./bin/meadow"]
 CMD ["start"]