Remove built-in GitHub and Jira issue tools (#586) #1687
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: ci | |
| on: | |
| push: | |
| branches: | |
| - master | |
| tags: | |
| - "v[0-9]+.[0-9]+.[0-9]+" | |
| pull_request: | |
| permissions: | |
| contents: write | |
| # Set the CI_RUNNER repository or organization variable to a larger runner label | |
| # for the heavier e2e path jobs. Other CI jobs intentionally stay on the | |
| # standard GitHub-hosted Ubuntu runner. | |
| jobs: | |
| e2e-build-artifact: | |
| timeout-minutes: 30 | |
| runs-on: ${{ vars.CI_RUNNER || 'ubuntu-latest' }} | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - uses: actions/setup-node@v6 | |
| with: | |
| node-version: 24.11.0 | |
| - name: Build plugin bundle for e2e jobs | |
| run: make dist | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: e2e-plugin-dist | |
| path: ./dist/*.tar.gz | |
| retention-days: 1 | |
| e2e: | |
| needs: e2e-build-artifact | |
| timeout-minutes: 60 | |
| runs-on: ${{ vars.CI_RUNNER || 'ubuntu-latest' }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| shard: | |
| - name: shard-1 | |
| group: e2e-shard-1 | |
| - name: shard-2 | |
| group: e2e-shard-2 | |
| - name: shard-3 | |
| group: e2e-shard-3 | |
| - name: shard-4 | |
| group: e2e-shard-4 | |
| name: e2e-${{ matrix.shard.name }} | |
| env: | |
| MM_IMAGE: ${{ vars.MM_IMAGE || 'mattermost/mattermost-enterprise-edition:11.5.1' }} | |
| EXCLUDE_REAL_API_TESTS: 'true' | |
| defaults: | |
| run: | |
| working-directory: ./e2e | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - uses: actions/setup-node@v6 | |
| with: | |
| node-version: 24.11.0 | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: e2e-plugin-dist | |
| path: ./dist/ | |
| - name: Install dependencies | |
| run: npm ci | |
| - name: Install Playwright Browsers | |
| run: npx playwright install --with-deps | |
| - name: Validate CI test groups | |
| run: node ./scripts/ci-test-groups.mjs validate | |
| - name: Run Playwright tests for ${{ matrix.shard.name }} | |
| shell: bash | |
| run: | | |
| SPEC_OUTPUT="$(node ./scripts/ci-test-groups.mjs list "${{ matrix.shard.group }}")" | |
| [[ -n "${SPEC_OUTPUT}" ]] | |
| mapfile -t SPEC_FILES <<<"${SPEC_OUTPUT}" | |
| npx playwright test --project=chromium "${SPEC_FILES[@]}" | |
| - uses: actions/upload-artifact@v4 | |
| if: always() | |
| with: | |
| name: playwright-report-${{ matrix.shard.name }} | |
| path: ./e2e/playwright-report/ | |
| retention-days: 30 | |
| - uses: actions/upload-artifact@v4 | |
| if: always() | |
| with: | |
| name: server-logs-${{ matrix.shard.name }} | |
| path: ./e2e/logs/server-logs.log | |
| retention-days: 30 | |
| e2e-llmbot-real-apis: | |
| needs: e2e-build-artifact | |
| timeout-minutes: 180 | |
| runs-on: ${{ vars.CI_RUNNER || 'ubuntu-latest' }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| shard: | |
| - name: citations | |
| group: llmbot-real-citations | |
| - name: reasoning | |
| group: llmbot-real-reasoning | |
| - name: edge-cases | |
| group: llmbot-real-edge-cases | |
| name: e2e-llmbot-real-apis-${{ matrix.shard.name }} | |
| env: | |
| MM_IMAGE: ${{ vars.MM_IMAGE || 'mattermost/mattermost-enterprise-edition:11.5.1' }} | |
| defaults: | |
| run: | |
| working-directory: ./e2e | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - uses: actions/setup-node@v6 | |
| with: | |
| node-version: 24.11.0 | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: e2e-plugin-dist | |
| path: ./dist/ | |
| - name: Install dependencies | |
| run: npm ci | |
| - name: Install Playwright Browsers | |
| run: npx playwright install --with-deps | |
| - name: Validate CI test groups | |
| run: node ./scripts/ci-test-groups.mjs validate | |
| - name: Run LLMBot tests with real APIs for ${{ matrix.shard.name }} | |
| env: | |
| ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| shell: bash | |
| run: | | |
| SPEC_OUTPUT="$(node ./scripts/ci-test-groups.mjs list "${{ matrix.shard.group }}")" | |
| [[ -n "${SPEC_OUTPUT}" ]] | |
| mapfile -t SPEC_FILES <<<"${SPEC_OUTPUT}" | |
| npx playwright test --project=chromium "${SPEC_FILES[@]}" | |
| - uses: actions/upload-artifact@v4 | |
| if: always() | |
| with: | |
| name: playwright-report-llmbot-real-apis-${{ matrix.shard.name }} | |
| path: ./e2e/playwright-report/ | |
| retention-days: 30 | |
| - uses: actions/upload-artifact@v4 | |
| if: always() | |
| with: | |
| name: server-logs-llmbot-real-apis-${{ matrix.shard.name }} | |
| path: ./e2e/logs/server-logs.log | |
| retention-days: 30 | |
| e2e-channel-analysis-real-apis: | |
| needs: e2e-build-artifact | |
| timeout-minutes: 180 | |
| runs-on: ${{ vars.CI_RUNNER || 'ubuntu-latest' }} | |
| env: | |
| MM_IMAGE: ${{ vars.MM_IMAGE || 'mattermost/mattermost-enterprise-edition:11.5.1' }} | |
| defaults: | |
| run: | |
| working-directory: ./e2e | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - uses: actions/setup-node@v6 | |
| with: | |
| node-version: 24.11.0 | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: e2e-plugin-dist | |
| path: ./dist/ | |
| - name: Install dependencies | |
| run: npm ci | |
| - name: Install Playwright Browsers | |
| run: npx playwright install --with-deps | |
| - name: Validate CI test groups | |
| run: node ./scripts/ci-test-groups.mjs validate | |
| - name: Run Channel Analysis tests with real APIs | |
| env: | |
| ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| shell: bash | |
| run: | | |
| SPEC_OUTPUT="$(node ./scripts/ci-test-groups.mjs list channel-analysis-real)" | |
| [[ -n "${SPEC_OUTPUT}" ]] | |
| mapfile -t SPEC_FILES <<<"${SPEC_OUTPUT}" | |
| npx playwright test --project=chromium "${SPEC_FILES[@]}" | |
| - uses: actions/upload-artifact@v4 | |
| if: always() | |
| with: | |
| name: playwright-report-channel-analysis-real-apis | |
| path: ./e2e/playwright-report/ | |
| retention-days: 30 | |
| - uses: actions/upload-artifact@v4 | |
| if: always() | |
| with: | |
| name: server-logs-channel-analysis-real-apis | |
| path: ./e2e/logs/server-logs.log | |
| retention-days: 30 | |
| e2e-system-console-real-apis: | |
| needs: e2e-build-artifact | |
| timeout-minutes: 180 | |
| runs-on: ${{ vars.CI_RUNNER || 'ubuntu-latest' }} | |
| env: | |
| MM_IMAGE: ${{ vars.MM_IMAGE || 'mattermost/mattermost-enterprise-edition:11.5.1' }} | |
| defaults: | |
| run: | |
| working-directory: ./e2e | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - uses: actions/setup-node@v6 | |
| with: | |
| node-version: 24.11.0 | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: e2e-plugin-dist | |
| path: ./dist/ | |
| - name: Install dependencies | |
| run: npm ci | |
| - name: Install Playwright Browsers | |
| run: npx playwright install --with-deps | |
| - name: Validate CI test groups | |
| run: node ./scripts/ci-test-groups.mjs validate | |
| - name: Run System Console tests with real APIs | |
| env: | |
| ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| shell: bash | |
| run: | | |
| SPEC_OUTPUT="$(node ./scripts/ci-test-groups.mjs list system-console-real)" | |
| [[ -n "${SPEC_OUTPUT}" ]] | |
| mapfile -t SPEC_FILES <<<"${SPEC_OUTPUT}" | |
| npx playwright test --project=chromium "${SPEC_FILES[@]}" | |
| - uses: actions/upload-artifact@v4 | |
| if: always() | |
| with: | |
| name: playwright-report-system-console-real-apis | |
| path: ./e2e/playwright-report/ | |
| retention-days: 30 | |
| - uses: actions/upload-artifact@v4 | |
| if: always() | |
| with: | |
| name: server-logs-system-console-real-apis | |
| path: ./e2e/logs/server-logs.log | |
| retention-days: 30 | |
| e2e-tool-config-real-apis: | |
| needs: e2e-build-artifact | |
| timeout-minutes: 180 | |
| runs-on: ${{ vars.CI_RUNNER || 'ubuntu-latest' }} | |
| env: | |
| MM_IMAGE: ${{ vars.MM_IMAGE || 'mattermost/mattermost-enterprise-edition:11.5.1' }} | |
| defaults: | |
| run: | |
| working-directory: ./e2e | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version: 18 | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: e2e-plugin-dist | |
| path: ./dist/ | |
| - name: Install dependencies | |
| run: npm ci | |
| - name: Install Playwright Browsers | |
| run: npx playwright install --with-deps | |
| - name: Validate CI test groups | |
| run: node ./scripts/ci-test-groups.mjs validate | |
| - name: Run Tool Config tests with real APIs | |
| env: | |
| ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| shell: bash | |
| run: | | |
| SPEC_OUTPUT="$(node ./scripts/ci-test-groups.mjs list tool-config-real)" | |
| [[ -n "${SPEC_OUTPUT}" ]] | |
| mapfile -t SPEC_FILES <<<"${SPEC_OUTPUT}" | |
| npx playwright test --project=chromium "${SPEC_FILES[@]}" | |
| - uses: actions/upload-artifact@v4 | |
| if: always() | |
| with: | |
| name: playwright-report-tool-config-real-apis | |
| path: ./e2e/playwright-report/ | |
| retention-days: 30 | |
| - uses: actions/upload-artifact@v4 | |
| if: always() | |
| with: | |
| name: server-logs-tool-config-real-apis | |
| path: ./e2e/logs/server-logs.log | |
| retention-days: 30 | |
| e2e-tool-calling: | |
| needs: e2e-build-artifact | |
| timeout-minutes: 180 | |
| runs-on: ${{ vars.CI_RUNNER || 'ubuntu-latest' }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| provider: | |
| - name: anthropic | |
| display_name: Anthropic | |
| - name: openai | |
| display_name: OpenAI | |
| name: e2e-tool-calling-${{ matrix.provider.name }} | |
| env: | |
| MM_IMAGE: ${{ vars.MM_IMAGE || 'mattermost/mattermost-enterprise-edition:11.5.1' }} | |
| defaults: | |
| run: | |
| working-directory: ./e2e | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - uses: actions/setup-node@v6 | |
| with: | |
| node-version: 24.11.0 | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: e2e-plugin-dist | |
| path: ./dist/ | |
| - name: Install dependencies | |
| run: npm ci | |
| - name: Install Playwright Browsers | |
| run: npx playwright install --with-deps | |
| - name: Validate CI test groups | |
| run: node ./scripts/ci-test-groups.mjs validate | |
| - name: Run Tool Calling tests with real APIs for ${{ matrix.provider.display_name }} | |
| env: | |
| ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| E2E_PROVIDER: ${{ matrix.provider.name }} | |
| shell: bash | |
| run: | | |
| SPEC_OUTPUT="$(node ./scripts/ci-test-groups.mjs list tool-calling-real)" | |
| [[ -n "${SPEC_OUTPUT}" ]] | |
| mapfile -t SPEC_FILES <<<"${SPEC_OUTPUT}" | |
| npx playwright test --project=chromium "${SPEC_FILES[@]}" | |
| - uses: actions/upload-artifact@v4 | |
| if: always() | |
| with: | |
| name: playwright-report-tool-calling-${{ matrix.provider.name }} | |
| path: ./e2e/playwright-report/ | |
| retention-days: 30 | |
| - uses: actions/upload-artifact@v4 | |
| if: always() | |
| with: | |
| name: server-logs-tool-calling-${{ matrix.provider.name }} | |
| path: ./e2e/logs/server-logs.log | |
| retention-days: 30 | |
| plugin-tests: | |
| runs-on: ubuntu-latest | |
| services: | |
| postgres: | |
| image: pgvector/pgvector:pg15 | |
| env: | |
| POSTGRES_USER: mmuser | |
| POSTGRES_PASSWORD: mostest | |
| POSTGRES_DB: postgres | |
| ports: | |
| - 5432:5432 | |
| options: >- | |
| --health-cmd pg_isready | |
| --health-interval 10s | |
| --health-timeout 5s | |
| --health-retries 5 | |
| steps: | |
| - uses: actions/checkout@v5 | |
| with: | |
| fetch-depth: 0 | |
| - uses: actions/setup-go@v5 | |
| with: | |
| go-version-file: 'go.mod' | |
| - name: Run tests | |
| env: | |
| PG_ROOT_DSN: "postgres://mmuser:mostest@localhost:5432/postgres?sslmode=disable" | |
| MM_SERVICESETTINGS_ENABLEDEVELOPER: true | |
| run: | | |
| make test | |
| evals: | |
| runs-on: ubuntu-latest | |
| if: github.event_name == 'pull_request' | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| provider: | |
| - name: openai | |
| display_name: OpenAI | |
| - name: anthropic | |
| display_name: Anthropic | |
| - name: azure | |
| display_name: Azure OpenAI | |
| - name: mistral | |
| display_name: Mistral | |
| - name: bedrock | |
| display_name: AWS Bedrock | |
| name: evals-${{ matrix.provider.name }} | |
| steps: | |
| - uses: actions/checkout@v5 | |
| with: | |
| fetch-depth: 0 | |
| - uses: actions/setup-go@v5 | |
| with: | |
| go-version-file: 'go.mod' | |
| - name: Run ${{ matrix.provider.display_name }} evaluations | |
| id: run-evals | |
| env: | |
| LLM_PROVIDER: ${{ matrix.provider.name }} | |
| # OpenAI | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| OPENAI_MODEL: ${{ vars.OPENAI_MODEL || 'gpt-5.2' }} | |
| # Anthropic | |
| ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} | |
| ANTHROPIC_MODEL: ${{ vars.ANTHROPIC_MODEL || 'claude-sonnet-4-6' }} | |
| # Azure | |
| AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }} | |
| AZURE_OPENAI_ENDPOINT: ${{ secrets.AZURE_OPENAI_ENDPOINT }} | |
| AZURE_OPENAI_MODEL: ${{ vars.AZURE_OPENAI_MODEL || 'gpt-5.2' }} | |
| # Mistral | |
| MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }} | |
| MISTRAL_MODEL: ${{ vars.MISTRAL_MODEL || 'mistral-large-latest' }} | |
| # AWS Bedrock | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_BEDROCK_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_BEDROCK_ACCESS_KEY_SECRET }} | |
| AWS_BEDROCK_REGION: ${{ vars.AWS_BEDROCK_REGION || 'us-east-1' }} | |
| AWS_BEDROCK_MODEL: ${{ vars.AWS_BEDROCK_MODEL || 'global.anthropic.claude-sonnet-4-6-v1:0' }} | |
| run: | | |
| # Run evals - evalviewer writes directly to comment.md | |
| make evals-comment || true | |
| # Save provider name for aggregation | |
| echo "${{ matrix.provider.display_name }}" > provider_name.txt | |
| - name: Upload evaluation results | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: eval-results-${{ matrix.provider.name }} | |
| path: | | |
| evals.jsonl | |
| comment.md | |
| provider_name.txt | |
| retention-days: 30 | |
| evals-comment: | |
| runs-on: ubuntu-latest | |
| needs: evals | |
| if: always() && github.event_name == 'pull_request' | |
| permissions: | |
| pull-requests: write | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - name: Download all evaluation results | |
| uses: actions/download-artifact@v4 | |
| with: | |
| path: eval-results | |
| - name: Aggregate results and post comment | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const script = require('./.github/scripts/aggregate-eval-results.js'); | |
| await script.aggregateAndPostResults(github, context); | |
| lint: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v5 | |
| with: | |
| fetch-depth: 0 | |
| - uses: actions/setup-go@v5 | |
| with: | |
| go-version-file: 'go.mod' | |
| - uses: mattermost/actions/plugin-ci/lint@main | |
| build: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v5 | |
| with: | |
| fetch-depth: 0 | |
| - uses: mattermost/actions/plugin-ci/setup@main | |
| with: | |
| golangci-lint-version: "v1.54.2" | |
| - uses: mattermost/actions/plugin-ci/build@main | |
| upload-s3-pr: | |
| needs: [lint] | |
| if: github.event_name == 'pull_request' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v5 | |
| with: | |
| fetch-depth: 0 | |
| - name: ci/get-short-sha | |
| id: short-sha | |
| run: echo "sha=$(echo ${{ github.event.pull_request.head.sha }} | cut -c1-7)" >> $GITHUB_OUTPUT | |
| - name: Build plugin | |
| run: make dist-ci | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@a03048d87541d1d9fcf2ecf528a4a65ba9bd7838 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| aws-region: us-east-1 | |
| - name: Upload dist to S3 | |
| run: | | |
| aws s3 cp dist/mattermost-ai-*.tar.gz s3://${{ secrets.AWS_S3_BUCKET }}/mattermost-plugin-agents/mattermost-plugin-agents-${{ steps.short-sha.outputs.sha }}.tar.gz | |
| deploy: | |
| needs: [lint, build, plugin-tests, e2e, e2e-llmbot-real-apis, e2e-channel-analysis-real-apis, e2e-system-console-real-apis, e2e-tool-config-real-apis, e2e-tool-calling] | |
| if: github.ref == 'refs/heads/master' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v5 | |
| with: | |
| fetch-depth: 0 | |
| - name: Build plugin | |
| run: make dist | |
| - name: Update latest-master release | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| PLUGIN_FILE=$(ls dist/*.tar.gz) | |
| # Get the commit message and hash | |
| COMMIT_MSG=$(git log -1 --pretty=%B) | |
| COMMIT_HASH=$(git rev-parse --short HEAD) | |
| # Delete the old release if it exists | |
| gh release delete latest-master --yes || true | |
| # Delete the old tag if it exists | |
| git tag -d latest-master || true | |
| git push origin :refs/tags/latest-master || true | |
| # Create new tag and release | |
| git tag latest-master | |
| git push origin latest-master | |
| gh release create latest-master \ | |
| --title "Latest Master Build" \ | |
| --notes "Latest build from master branch - ${COMMIT_HASH}" \ | |
| --prerelease \ | |
| $PLUGIN_FILE | |
| - name: Deploy to Mattermost instances | |
| env: | |
| MM_SERVERS_CONFIG: ${{ secrets.MM_SERVERS_CONFIG }} | |
| run: | | |
| PLUGIN_FILE=$(ls dist/*.tar.gz) | |
| echo "Found plugin bundle: $PLUGIN_FILE" | |
| echo "$MM_SERVERS_CONFIG" | jq -c '.[]' | while read -r server; do | |
| url=$(echo "$server" | jq -r '.url') | |
| token=$(echo "$server" | jq -r '.token') | |
| echo "Deploying to $url" | |
| curl -H "Authorization: Bearer $token" \ | |
| -X POST \ | |
| -F "plugin=@$PLUGIN_FILE" \ | |
| -F "force=true" \ | |
| "$url/api/v4/plugins" | |
| done |