Increment arthur-engine version to 2.1.135 (#732)

Author: Kacpersoftberry

.github/workflows/arthur-engine-workflow.yml

@@ -9,6 +9,13 @@ on:
     branches:
       - main
       - dev
+  workflow_dispatch:
+    inputs:
+      commit_sha:
+        description: "Commit SHA for testing"
+        required: false
+        default: ""
+        type: string
 
 jobs:
   run-genai-engine-linter:
@@ -32,12 +39,33 @@ jobs:
       - uses: ./.github/workflows/composite-actions/setup-poetry
         with:
           working-directory: genai-engine
-      - name: Run pre-commit
+          with-groups: dev
+      - name: Run isort
+        id: isort
+        continue-on-error: true
         run: |
-          poetry -C genai-engine run pre-commit install
-          CHANGED_FILES=$(git diff-tree --no-commit-id --name-only -r ${{ github.sha }})
-          echo "$CHANGED_FILES" | xargs poetry -C genai-engine run pre-commit run --color always --verbose --files
-
+          poetry -C genai-engine run isort src --profile black --check
+      - name: Run autoflake
+        id: autoflake
+        continue-on-error: true
+        run: |
+          poetry -C genai-engine run autoflake --remove-all-unused-imports --in-place --check --recursive --quiet src
+      - name: Run black
+        id: black
+        continue-on-error: true
+        run: |
+          poetry -C genai-engine run black --check src
+      # TODO: Uncomment this when mypy is fixed
+      # - name: Run mypy
+      #   id: mypy
+      #   continue-on-error: true
+      #   run: |
+      #     poetry -C genai-engine run mypy src
+      - name: Check linter results
+        if: steps.isort.outcome == 'failure' || steps.autoflake.outcome == 'failure' || steps.black.outcome == 'failure'
+        run: |
+          echo "One or more linters failed"
+          exit 1
 
   run-ml-engine-linter:
     if: |
@@ -60,20 +88,39 @@ jobs:
       - uses: ./.github/workflows/composite-actions/setup-poetry
         with:
           working-directory: ml-engine
+          with-groups: linters
       - name: Generate GenAI Client
         run: |
           pip install openapi-generator-cli[jdk4py]
           openapi-generator-cli generate -g python \
             -i ./genai-engine/staging.openapi.json \
             -o ./ml-engine/src/genai_client \
             --skip-validate-spec --package-name genai_client
-      - name: Run linters
+      - name: Run isort
+        id: isort
+        continue-on-error: true
         run: |
-          poetry -C ml-engine install --with linters
-          poetry -C ml-engine run autoflake --remove-all-unused-imports --in-place --recursive src/ml_engine
-          poetry -C ml-engine run isort src/ml_engine --profile black
-          poetry -C ml-engine run black src/ml_engine
-
+          poetry -C ml-engine run isort src/ml_engine --profile black --check
+      - name: Run autoflake
+        id: autoflake
+        continue-on-error: true
+        run: |
+          poetry -C ml-engine run autoflake --remove-all-unused-imports --in-place --check --recursive --quiet src/ml_engine
+      - name: Run black
+        id: black
+        continue-on-error: true
+        run: |
+          poetry -C ml-engine run black --check src/ml_engine
+      - name: Run mypy
+        id: mypy
+        continue-on-error: true
+        run: |
+          poetry -C ml-engine run mypy src/ml_engine
+      - name: Check linter results
+        if: steps.isort.outcome == 'failure' || steps.autoflake.outcome == 'failure' || steps.black.outcome == 'failure' || steps.mypy.outcome == 'failure'
+        run: |
+          echo "One or more linters failed"
+          exit 1
 
   run-genai-engine-changelog-cop:
     if: |
@@ -98,6 +145,8 @@ jobs:
           curl -L -o "oasdiff.deb" https://github.com/Tufin/oasdiff/releases/download/v1.11.7/oasdiff_1.11.7_linux_amd64.deb
           dpkg -i oasdiff.deb
       - name: Generate changelog
+        env:
+          GENAI_ENGINE_SECRET_STORE_KEY: changeme_secret_store_key
         run: |
           export PYTHONPATH=src
           poetry -C genai-engine run generate_changelog
@@ -107,9 +156,9 @@ jobs:
         with:
           name: changelog-artifacts
           path: |
-            new.openapi.json
-            staging.openapi.json
-            src/api_changelog.md
+            genai-engine/new.openapi.json
+            genai-engine/staging.openapi.json
+            genai-engine/src/api_changelog.md
 
   run-genai-engine-unit-tests:
     if: |
@@ -129,6 +178,8 @@ jobs:
           working-directory: genai-engine
       - name: Run unit tests
         shell: bash
+        env:
+          GENAI_ENGINE_SECRET_STORE_KEY: changeme_secret_store_key
         run: |
           set -o pipefail
           poetry -C genai-engine run pytest --cov --cov=genai-engine/src \
@@ -151,6 +202,48 @@ jobs:
             genai-engine/report.xml
             pytest-coverage.txt
 
+  run-genai-engine-alembic-migration-check:
+    if: |
+      (
+        github.event_name == 'pull_request' ||
+        (github.event_name == 'push' && github.ref_type == 'branch')
+      )
+    runs-on: ubuntu-latest
+    container: python:3.12.9-bullseye
+    services:
+      postgres:
+        image: postgres:15
+        ports:
+          - "5432:5432"
+        env:
+          POSTGRES_PASSWORD: changeme_pg_password
+          POSTGRES_DB: arthur_genai_engine
+        volumes:
+          - postgres_data:/var/lib/postgresql/data
+    steps:
+      - uses: actions/checkout@v5
+        with:
+          fetch-depth: ${{ env.GIT_DEPTH }}
+      - uses: ./.github/workflows/composite-actions/setup-git
+        with:
+          safe-directory: ${{ runner.workspace }}
+      - uses: ./.github/workflows/composite-actions/setup-poetry
+        with:
+          working-directory: genai-engine
+      - name: Run alembic migration check
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: changeme_pg_password
+          POSTGRES_URL: postgres
+          POSTGRES_PORT: 5432
+          POSTGRES_DB: arthur_genai_engine
+          POSTGRES_USE_SSL: false
+          GENAI_ENGINE_SECRET_STORE_KEY: changeme_secret_store_key
+          PYTHONPATH: "src:$PYTHONPATH"
+        run: |
+          cd genai-engine
+          poetry run alembic upgrade head
+
   run-ml-engine-unit-tests:
     if: |
       (
@@ -183,9 +276,13 @@ jobs:
         run: |
           poetry -C ml-engine run pytest
 
-
   build-genai-engine-docker-images:
-    needs: [run-genai-engine-linter, run-genai-engine-changelog-cop, run-genai-engine-unit-tests]
+    needs:
+      [
+        run-genai-engine-linter,
+        run-genai-engine-changelog-cop,
+        run-genai-engine-unit-tests,
+      ]
     environment: shared-protected-branch-secrets
     if: |
       contains(github.event.head_commit.message, 'Increment arthur-engine version to') &&
@@ -209,7 +306,7 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3.11.1
       - name: Log in to Docker Hub
-        uses: docker/login-action@v3.5.0
+        uses: docker/login-action@v3.6.0
         with:
           username: ${{ secrets.DOCKER_HUB_USERNAME }}
           password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
@@ -228,7 +325,6 @@ jobs:
           push: true
           platforms: linux/amd64
 
-
   build-ml-engine-docker-images:
     needs: [run-ml-engine-linter, run-ml-engine-unit-tests]
     environment: shared-protected-branch-secrets
@@ -255,7 +351,7 @@ jobs:
             --skip-validate-spec --package-name genai_client
           sed -i 's/license = "NoLicense"/license = "Unlicense"/g' ./ml-engine/src/genai_client/pyproject.toml
       - name: Log in to Docker Hub
-        uses: docker/login-action@v3.5.0
+        uses: docker/login-action@v3.6.0
         with:
           username: ${{ secrets.DOCKER_HUB_USERNAME }}
           password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
@@ -289,7 +385,6 @@ jobs:
             docker push docker.arthur.ai/arthur/scope/ml-engine:latest-dev
           fi
 
-
   push-all-cfts:
     needs: [build-genai-engine-docker-images, build-ml-engine-docker-images]
     environment: shared-protected-branch-secrets
@@ -372,3 +467,20 @@ jobs:
         run: |
           git tag -a "${{ env.VERSION }}" -m "Create tag for Arthur Engine ${{ env.VERSION }}"
           git push origin "${{ env.VERSION }}"
+
+  deploy-genai-engine-cloudformation:
+    needs: [push-tag]
+    if: |
+      contains(github.event.head_commit.message, 'Increment arthur-engine version to') &&
+      (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev')
+    runs-on: ubuntu-latest
+    container: node:20-alpine
+    environment: shared-protected-branch-secrets
+    steps:
+      - name: Trigger GitLab Pipeline
+        uses: fjogeleit/http-request-action@v1.16.5
+        with:
+          url: "https://gitlab.com/api/v4/projects/${{ secrets.GITLAB_PROJECT_ID }}/trigger/pipeline"
+          method: POST
+          contentType: application/x-www-form-urlencoded
+          data: "token=${{ secrets.DEPLOY_TRIGGER_TOKEN }}&ref=main&variables[TRIGGER_JOB]=deploy_arthur_engine&variables[ARTHUR_ENGINE_COMMIT_SHA]=${{ inputs.commit_sha || github.sha }}"

.github/workflows/composite-actions/setup-poetry/action.yml

@@ -1,16 +1,24 @@
-name: 'Setup Poetry'
-description: 'Installs and configures Poetry'
+name: "Setup Poetry"
+description: "Installs and configures Poetry"
 
 inputs:
   working-directory:
-    description: 'Directory containing the pyproject.toml file'
+    description: "Directory containing the pyproject.toml file"
     required: true
+  with-groups:
+    description: "Additional dependency groups to install (comma-separated)"
+    required: false
+    default: ""
 
 runs:
-  using: 'composite'
+  using: "composite"
   steps:
-    - name: 'Setup Poetry'
+    - name: "Setup Poetry"
       shell: bash
       run: |
         pip install poetry
-        poetry -C ${{ inputs.working-directory }} install
+        GROUPS_FLAG=""
+        if [ -n "${{ inputs.with-groups }}" ]; then
+          GROUPS_FLAG="--with ${{ inputs.with-groups }}"
+        fi
+        poetry -C ${{ inputs.working-directory }} install $GROUPS_FLAG

.github/workflows/version-workflow.yml

@@ -1,17 +1,20 @@
 name: Version Management
 
 on:
+  push:
+    branches:
+      - dev
   workflow_dispatch:
     inputs:
       intent:
-        description: 'Trigger version bump'
+        description: "Trigger version bump"
         required: true
-        default: 'bump_version'
+        default: "bump_version"
         type: string
       bump_type:
-        description: 'Version bump type (patch, minor, major)'
+        description: "Version bump type (patch, minor, major)"
         required: true
-        default: 'patch'
+        default: "patch"
         type: choice
         options:
           - patch
@@ -21,7 +24,15 @@ on:
 jobs:
   bump-arthur-engine-version:
     environment: shared-protected-branch-secrets
-    if: github.event_name == 'workflow_dispatch' && github.event.inputs.intent == 'bump_version' && github.ref != 'refs/heads/main'
+    if: |
+      (github.event_name == 'push' && github.ref == 'refs/heads/dev' &&
+       !startsWith(github.event.head_commit.message, 'fix(deps)') &&
+       !startsWith(github.event.head_commit.message, 'Increment arthur-engine') &&
+       !startsWith(github.event.head_commit.message, 'Update dependency') &&
+       !startsWith(github.event.head_commit.message, 'chore(deps)')) ||
+      (github.event_name == 'workflow_dispatch' &&
+       github.event.inputs.intent == 'bump_version' &&
+       github.ref != 'refs/heads/main')
     runs-on: ubuntu-latest
     permissions:
       contents: write
@@ -56,7 +67,12 @@ jobs:
         run: |
           set -e
 
-          export BUMP_COMMAND="${{ github.event.inputs.bump_type }}"
+          # For automatic triggers (push to dev), use patch. For manual triggers, use the user's choice
+          if [ "${{ github.event_name }}" == "push" ]; then
+            export BUMP_COMMAND="patch"
+          else
+            export BUMP_COMMAND="${{ github.event.inputs.bump_type }}"
+          fi
           export BRANCH_NAME="arthur-engine-version-bump/$(date +%Y%m%d-%H%M%S)"
           # git checkout -b $BRANCH_NAME
 
@@ -70,7 +86,7 @@ jobs:
           MAJOR=${VERSION_PARTS[0]}
           MINOR=${VERSION_PARTS[1]}
           PATCH=${VERSION_PARTS[2]}
-          
+
           # Bump version based on command
           case "$BUMP_COMMAND" in
             "major")
@@ -118,12 +134,12 @@ jobs:
           sed -i "s|version:.*|version: $NEW_ARTHUR_ENGINE_VERSION|" "$GENAI_ENGINE_HELM_CHART_YAML_PATH"
           sed -i "s|appVersion:.*|appVersion: $NEW_ARTHUR_ENGINE_VERSION|" "$GENAI_ENGINE_HELM_CHART_YAML_PATH"
           sed -i "s|genaiEngineVersion:.*|genaiEngineVersion: '$NEW_ARTHUR_ENGINE_VERSION'|" "$GENAI_ENGINE_HELM_VALUES_YAML_PATH"
-          
+
           # Include the ml-engine Chart.yaml and values.yaml
           sed -i "s|version:.*|version: $NEW_ARTHUR_ENGINE_VERSION|" "$ML_ENGINE_CHART_YAML_PATH"
           sed -i "s|appVersion:.*|appVersion: $NEW_ARTHUR_ENGINE_VERSION|" "$ML_ENGINE_CHART_YAML_PATH"
           sed -i "s|containerImageVersion:.*|containerImageVersion: \"$NEW_ARTHUR_ENGINE_VERSION\"|" "$ML_ENGINE_VALUES_YAML_PATH"
-    
+
           # Update the genai and ml engine pyproject.toml files and the openapi json
           sed -i "s/^version = \".*\"/version = \"$NEW_ARTHUR_ENGINE_VERSION\"/" genai-engine/pyproject.toml
           sed -i "s/\"version\": \".*\"/\"version\": \"$NEW_ARTHUR_ENGINE_VERSION\"/" genai-engine/staging.openapi.json

.gitignore

@@ -13,6 +13,10 @@ __pycache__
 *.egg-info
 build/
 
+# Node.js
+node_modules/
+package-lock.json
+
 .html_cov*
 .coverage*
 coverage.xml