Skip to content

Commit ae81588

Browse files
alhendricksonalhendrickson
committed
Merge branch 'main' into medcat-v2-testing
2 parents e49aa32 + f713469 commit ae81588

File tree

1,062 files changed

+715983
-111
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

1,062 files changed

+715983
-111
lines changed

‎.devcontainer/devcontainer.json‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
// For format details, see https://aka.ms/devcontainer.json. For config options, see the
2+
// README at: https://github.com/devcontainers/templates/tree/main/src/python
3+
{
4+
"name": "Python 3",
5+
// Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile
6+
"image": "mcr.microsoft.com/devcontainers/python:1-3.12-bullseye",
7+
"features": {
8+
"ghcr.io/devcontainers/features/docker-outside-of-docker:1": {},
9+
"ghcr.io/va-h/devcontainers-features/uv:1": {
10+
"shellautocompletion": true
11+
}
12+
},
13+
14+
// Features to add to the dev container. More info: https://containers.dev/features.
15+
// "features": {},
16+
17+
// Use 'forwardPorts' to make a list of ports inside the container available locally.
18+
// "forwardPorts": [],
19+
20+
// Use 'postCreateCommand' to run commands after the container is created.
21+
// "postCreateCommand": "pip3 install --user -r requirements.txt",
22+
23+
// Configure tool-specific properties.
24+
// "customizations": {},
25+
26+
// Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root.
27+
// "remoteUser": "root"
28+
"workspaceFolder": "${localWorkspaceFolder}",
29+
"workspaceMount": "source=${localWorkspaceFolder},target=${localWorkspaceFolder},type=bind",
30+
"mounts": [
31+
"source=${localEnv:HOME}/.cache/pip,target=/home/vscode/.cache/pip,type=bind",
32+
"source=${localEnv:HOME}/.cache/uv,target=/home/vscode/.cache/uv,type=bind",
33+
"source=${localEnv:HOME}/.cache/huggingface,target=/home/vscode/.cache/huggingface,type=bind",
34+
],
35+
"customizations": {
36+
"vscode": {
37+
"extensions": [
38+
"charliermarsh.ruff"
39+
]
40+
}
41+
}
42+
}

‎.github/dependabot.yml‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
# To get started with Dependabot version updates, you'll need to specify which
2+
# package ecosystems to update and where the package manifests are located.
3+
# Please see the documentation for more information:
4+
# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
5+
# https://containers.dev/guide/dependabot
6+
7+
version: 2
8+
updates:
9+
- package-ecosystem: "devcontainers"
10+
directory: "/"
11+
commit-message:
12+
prefix: "chore: "
13+
schedule:
14+
interval: weekly
15+
- package-ecosystem: "github-actions"
16+
directory: "/"
17+
schedule:
18+
interval: "weekly"
19+
commit-message:
20+
prefix: "build: "
21+
groups:
22+
actions-deps:
23+
patterns:
24+
- "*"

‎.github/workflows/codeql.yml‎

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
# For most projects, this workflow file will not need changing; you simply need
2+
# to commit it to your repository.
3+
#
4+
# You may wish to alter this file to override the set of languages analyzed,
5+
# or to provide custom queries or build logic.
6+
#
7+
# ******** NOTE ********
8+
# We have attempted to detect the languages in your repository. Please check
9+
# the `language` matrix defined below to confirm you have the correct set of
10+
# supported CodeQL languages.
11+
#
12+
name: "CodeQL Advanced"
13+
14+
on:
15+
push:
16+
branches: [ "main" ]
17+
pull_request:
18+
branches: [ "main" ]
19+
schedule:
20+
- cron: '19 7 * * 0'
21+
22+
jobs:
23+
analyze:
24+
name: Analyze (${{ matrix.language }})
25+
# Runner size impacts CodeQL analysis time. To learn more, please see:
26+
# - https://gh.io/recommended-hardware-resources-for-running-codeql
27+
# - https://gh.io/supported-runners-and-hardware-resources
28+
# - https://gh.io/using-larger-runners (GitHub.com only)
29+
# Consider using larger runners or machines with greater resources for possible analysis time improvements.
30+
runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }}
31+
permissions:
32+
# required for all workflows
33+
security-events: write
34+
35+
# required to fetch internal or private CodeQL packs
36+
packages: read
37+
38+
# only required for workflows in private repositories
39+
actions: read
40+
contents: read
41+
42+
strategy:
43+
fail-fast: false
44+
matrix:
45+
include:
46+
- language: actions
47+
build-mode: none
48+
- language: javascript-typescript
49+
build-mode: none
50+
- language: python
51+
build-mode: none
52+
# CodeQL supports the following values keywords for 'language': 'actions', 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'rust', 'swift'
53+
# Use `c-cpp` to analyze code written in C, C++ or both
54+
# Use 'java-kotlin' to analyze code written in Java, Kotlin or both
55+
# Use 'javascript-typescript' to analyze code written in JavaScript, TypeScript or both
56+
# To learn more about changing the languages that are analyzed or customizing the build mode for your analysis,
57+
# see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/customizing-your-advanced-setup-for-code-scanning.
58+
# If you are analyzing a compiled language, you can modify the 'build-mode' for that language to customize how
59+
# your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages
60+
steps:
61+
- name: Checkout repository
62+
uses: actions/checkout@v4
63+
64+
# Add any setup steps before running the `github/codeql-action/init` action.
65+
# This includes steps like installing compilers or runtimes (`actions/setup-node`
66+
# or others). This is typically only required for manual builds.
67+
# - name: Setup runtime (example)
68+
# uses: actions/setup-example@v1
69+
70+
# Initializes the CodeQL tools for scanning.
71+
- name: Initialize CodeQL
72+
uses: github/codeql-action/init@v3
73+
with:
74+
languages: ${{ matrix.language }}
75+
build-mode: ${{ matrix.build-mode }}
76+
# If you wish to specify custom queries, you can do so here or in a config file.
77+
# By default, queries listed here will override any specified in a config file.
78+
# Prefix the list here with "+" to use these queries and those in the config file.
79+
80+
# For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
81+
# queries: security-extended,security-and-quality
82+
83+
# If the analyze step fails for one of the languages you are analyzing with
84+
# "We were unable to automatically build your code", modify the matrix above
85+
# to set the build mode to "manual" for that language. Then modify this step
86+
# to build your code.
87+
# ℹ️ Command-line programs to run using the OS shell.
88+
# 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
89+
- if: matrix.build-mode == 'manual'
90+
shell: bash
91+
run: |
92+
echo 'If you are using a "manual" build mode for one or more of the' \
93+
'languages you are analyzing, replace this with the commands to build' \
94+
'your code, for example:'
95+
echo ' make bootstrap'
96+
echo ' make release'
97+
exit 1
98+
99+
- name: Perform CodeQL Analysis
100+
uses: github/codeql-action/analyze@v3
101+
with:
102+
category: "/language:${{matrix.language}}"

‎.github/workflows/medcat-demo-app_build.yml‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
name: medcat-demo-app - Test
2+
3+
on:
4+
push:
5+
branches: [ main ]
6+
pull_request:
7+
paths:
8+
- 'medcat-demo-app/**'
9+
- 'medcat-v2/**'
10+
- '.github/workflows/medcat-demo-app**'
11+
defaults:
12+
run:
13+
working-directory: ./medcat-demo-app
14+
15+
jobs:
16+
integration-test:
17+
runs-on: ubuntu-latest
18+
env:
19+
MEDCAT_BRANCH: ${{ github.head_ref || github.ref_name }}
20+
21+
steps:
22+
- uses: actions/checkout@v3
23+
24+
- name: Set up Docker Compose
25+
run: sudo apt-get update && sudo apt-get install -y docker-compose
26+
27+
- name: Make medact-v2 available within webapp
28+
run: cp -r ../medcat-v2 webapp/medcat-v2
29+
30+
- name: Build and start containers
31+
run: docker-compose -f docker-compose-test.yml up -d --build
32+
33+
- name: Run integration test
34+
run: ./tests/test_integration.sh
35+
36+
- name: Check container logs for errors
37+
run: |
38+
docker-compose logs medcatweb
39+
docker-compose logs medcatweb | grep -i 'error' && exit 1 || true
40+
41+
- name: Tear down
42+
run: docker-compose -f docker-compose-test.yml down

‎.github/workflows/medcat-service_docker.yml‎

Lines changed: 136 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,136 @@
1+
name: medcat-service - Build Images
2+
3+
on:
4+
push:
5+
branches: [ main ]
6+
tags:
7+
- 'medcat-service/v*.*.*' # e.g., medcat-serice/v0.1.1
8+
pull_request:
9+
paths:
10+
- 'medcat-v2/**'
11+
- 'medcat-service/**'
12+
- '.github/workflows/medcat-service**'
13+
14+
# Allows you to run this workflow manually from the Actions tab
15+
workflow_dispatch:
16+
defaults:
17+
run:
18+
working-directory: ./medcat-service
19+
jobs:
20+
build:
21+
if: github.event.pull_request.user.login != 'dependabot[bot]' && github.repository == 'CogStack/cogstack-nlp'
22+
runs-on: ubuntu-latest
23+
outputs:
24+
image_version: ${{ steps.meta.outputs.version }}
25+
steps:
26+
- name: Checkout
27+
uses: actions/checkout@v4
28+
29+
- name: Log in to Docker Hub
30+
uses: docker/login-action@v1
31+
with:
32+
username: ${{ secrets.DOCKERHUB_USERNAME }}
33+
password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
34+
35+
- name: Set up Docker Buildx
36+
uses: docker/setup-buildx-action@v3
37+
38+
- name: Extract metadata (tags, labels) for Docker MedCATservice
39+
id: meta
40+
uses: docker/metadata-action@v5
41+
with:
42+
images: cogstacksystems/medcat-service
43+
tags: |
44+
# set latest tag for default branch
45+
type=raw,value=latest,enable={{is_default_branch}}
46+
# Include all default tags
47+
type=schedule
48+
type=ref,event=branch
49+
type=ref,event=tag
50+
type=ref,event=pr
51+
type=sha
52+
# Create version tag based on tag prefix
53+
type=match,pattern=medcat-service/v(\d+\.\d+\.\d+),group=1
54+
flavor: latest=false
55+
56+
- name: Make medact-v2 available within build
57+
run: cp -r ../medcat-v2 medcat-v2
58+
59+
- name: Build and push Docker MedCATservice image
60+
id: docker_build
61+
uses: docker/build-push-action@v6
62+
with:
63+
context: ./medcat-service/
64+
push: true
65+
allow: network.host
66+
tags: ${{ steps.meta.outputs.tags }}
67+
labels: ${{ steps.meta.outputs.labels }}
68+
cache-from: type=registry,ref=cogstacksystems/medcat-service:buildcache
69+
cache-to: type=registry,ref=cogstacksystems/medcat-service:buildcache,mode=max
70+
build-args: |
71+
REINSTALL_CORE_FROM_LOCAL=true
72+
73+
- name: Extract metadata (tags, labels) for Docker MedCATservice-gpu
74+
id: meta-gpu
75+
uses: docker/metadata-action@v5
76+
with:
77+
images: cogstacksystems/medcat-service-gpu
78+
tags: |
79+
# set latest tag for default branch
80+
type=raw,value=latest,enable={{is_default_branch}}
81+
# Include all default tags
82+
type=schedule
83+
type=ref,event=branch
84+
type=ref,event=tag
85+
type=ref,event=pr
86+
type=sha
87+
# Create version tag based on tag prefix
88+
type=match,pattern=medcat-service/v(\d+\.\d+\.\d+),group=1
89+
flavor: latest=false
90+
build-args: |
91+
REINSTALL_CORE_FROM_LOCAL=true
92+
93+
- name: Build and push Docker Jupyter singleuser image with GPU support
94+
id: docker_build_gpu
95+
uses: docker/build-push-action@v6
96+
with:
97+
context: ./medcat-service/
98+
file : "./medcat-service/Dockerfile_gpu"
99+
allow: network.host
100+
tags: ${{ steps.meta-gpu.outputs.tags }}
101+
labels: ${{ steps.meta-gpu.outputs.labels }}
102+
push: true
103+
cache-from: type=registry,ref=cogstacksystems/medcat-service-gpu:buildcache
104+
cache-to: type=registry,ref=cogstacksystems/medcat-service-gpu:buildcache,mode=max
105+
build-args: |
106+
REINSTALL_CORE_FROM_LOCAL=true
107+
108+
- name: Image digest
109+
run: echo ${{ steps.docker_build.outputs.digest }}
110+
working-directory: "./"
111+
integration_test:
112+
runs-on: ubuntu-latest
113+
needs: build
114+
strategy:
115+
matrix:
116+
variant: [v1, v2]
117+
steps:
118+
- name: Check out code
119+
uses: actions/checkout@v4
120+
121+
- name: Run Test with example model pack
122+
run: |
123+
echo "🧪 Running Examples test..."
124+
export IMAGE_TAG=${{ needs.build.outputs.image_version }} # Improve by running exact digest instead
125+
echo "Running with image tag ${IMAGE_TAG} and variant ${{ matrix.variant }}"
126+
bash scripts/test_examples.sh ${{ matrix.variant }}
127+
128+
- name: Report test result
129+
if: always()
130+
run: |
131+
if [ $? -eq 0 ]; then
132+
echo "✅ Integration test PASSED"
133+
else
134+
echo "❌ Integration test FAILED"
135+
exit 1
136+
fi

0 commit comments

Comments
 (0)