Skip to content

Commit e2e9faa

Browse files
EstelleDaEstelleDa
committed
Merge remote-tracking branch 'origin/release-2025.2.2' into estelle/geneOntologyControlledKeywords
2 parents b138694 + 422f408 commit e2e9faa

File tree

146 files changed

+14087
-5485
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

146 files changed

+14087
-5485
lines changed

‎.github/workflows/run-tests-on-push.yml‎

Lines changed: 54 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -6,51 +6,99 @@ env:
66
LOG_CONFIG: test
77

88
jobs:
9+
run-tests-3_9-core-dependencies:
10+
runs-on: ubuntu-latest
11+
name: Pytest on Core Dependencies-- Python 3.9
12+
steps:
13+
- uses: actions/checkout@v4
14+
- uses: actions/setup-python@v5
15+
with:
16+
python-version: "3.9"
17+
cache: 'pip'
18+
- run: pip install --upgrade pip
19+
- run: pip install poetry
20+
- run: poetry install --with dev
21+
- run: poetry run pytest tests/
22+
923
run-tests-3_9:
10-
runs-on: ubuntu-20.04
11-
name: Pytest on Python 3.9 / Ubuntu 20.04
24+
runs-on: ubuntu-latest
25+
name: Pytest on Optional Dependencies-- Python 3.9
1226
steps:
1327
- uses: actions/checkout@v4
1428
- uses: actions/setup-python@v5
1529
with:
1630
python-version: "3.9"
1731
cache: 'pip'
32+
- run: sudo apt-get update
33+
- run: sudo apt-get install -y tabix
1834
- run: pip install --upgrade pip
1935
- run: pip install poetry
2036
- run: poetry install --with dev --extras server
2137
- run: poetry run pytest tests/ --show-capture=stdout --cov=src
2238

39+
run-tests-3_10-core-dependencies:
40+
runs-on: ubuntu-latest
41+
name: Pytest on Core Dependencies-- Python 3.10
42+
steps:
43+
- uses: actions/checkout@v4
44+
- uses: actions/setup-python@v5
45+
with:
46+
python-version: "3.10"
47+
cache: 'pip'
48+
- run: pip install --upgrade pip
49+
- run: pip install poetry
50+
- run: poetry install --with dev
51+
- run: poetry run pytest tests/
52+
2353
run-tests-3_10:
2454
runs-on: ubuntu-latest
25-
name: Pytest on Python 3.10
55+
name: Pytest on Optional Dependencies-- Python 3.10
2656
steps:
2757
- uses: actions/checkout@v4
2858
- uses: actions/setup-python@v5
2959
with:
3060
python-version: "3.10"
3161
cache: 'pip'
62+
- run: sudo apt-get update
63+
- run: sudo apt-get install -y tabix
3264
- run: pip install --upgrade pip
3365
- run: pip install poetry
3466
- run: poetry install --with dev --extras server
3567
- run: poetry run pytest tests/ --show-capture=stdout --cov=src
3668

69+
run-tests-3_11-core-dependencies:
70+
runs-on: ubuntu-latest
71+
name: Pytest on Core Dependencies-- Python 3.11
72+
steps:
73+
- uses: actions/checkout@v4
74+
- uses: actions/setup-python@v5
75+
with:
76+
python-version: "3.11"
77+
cache: 'pip'
78+
- run: pip install --upgrade pip
79+
- run: pip install poetry
80+
- run: poetry install --with dev
81+
- run: poetry run pytest tests/
82+
3783
run-tests-3_11:
3884
runs-on: ubuntu-latest
39-
name: Pytest on Python 3.11
85+
name: Pytest on Optional Dependencies-- Python 3.11
4086
steps:
4187
- uses: actions/checkout@v4
4288
- uses: actions/setup-python@v5
4389
with:
4490
python-version: "3.11"
4591
cache: 'pip'
92+
- run: sudo apt-get update
93+
- run: sudo apt-get install -y tabix
4694
- run: pip install --upgrade pip
4795
- run: pip install poetry
4896
- run: poetry install --with dev --extras server
4997
- run: poetry run pytest tests/ --show-capture=stdout --cov=src
5098

5199
run-mypy-3_10:
52100
runs-on: ubuntu-latest
53-
name: MyPy checks on Python 3.10
101+
name: MyPy on Full Codebase-- Python 3.10
54102
steps:
55103
- uses: actions/checkout@v4
56104
- uses: actions/setup-python@v5
@@ -64,7 +112,7 @@ jobs:
64112

65113
run-ruff-lint:
66114
runs-on: ubuntu-latest
67-
name: Ruff linting on Python 3.10
115+
name: Ruff on Full Codebase-- Python 3.10
68116
steps:
69117
- uses: actions/checkout@v4
70118
- uses: actions/setup-python@v5

‎Dockerfile‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
# python-base
33
# Set up shared environment variables
44
################################
5-
FROM python:3.9 as python-base
5+
FROM python:3.9 AS python-base
66

77
# Poetry
88
# https://python-poetry.org/docs/configuration/#using-environment-variables
@@ -69,7 +69,7 @@ RUN samtools faidx GCF_000001405.39_GRCh38.p13_genomic.fna.gz
6969
# builder
7070
# Builds application dependencies and creates venv
7171
################################
72-
FROM python-base as builder
72+
FROM python-base AS builder
7373

7474
WORKDIR /code
7575

@@ -90,7 +90,7 @@ COPY src/mavedb/server_main.py /code/main.py
9090
# worker
9191
# Worker image
9292
################################
93-
FROM builder as worker
93+
FROM builder AS worker
9494
COPY --from=downloader /data /data
9595

9696
# copy pre-built poetry + venv
@@ -103,7 +103,7 @@ CMD ["arq", "mavedb.worker.WorkerSettings"]
103103
# application
104104
# Application image
105105
################################
106-
FROM builder as application
106+
FROM builder AS application
107107
COPY --from=downloader /data /data
108108

109109
# copy pre-built poetry + venv

‎alembic/manual_migrations/migrate_consolidated_score_ranges_and_calibrations.py‎

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
"""
2+
To be run as part of revision af87c9953d2d
3+
"""
4+
5+
import sqlalchemy as sa
6+
from sqlalchemy.orm import Session, configure_mappers
7+
8+
from mavedb.models import *
9+
from sqlalchemy.orm import Session
10+
11+
from mavedb.models.score_set import ScoreSet
12+
from mavedb.view_models.score_range import ScoreSetRangesCreate, InvestigatorScoreRangesCreate, PillarProjectScoreRangesCreate, PillarProjectScoreRangeCreate
13+
14+
15+
from mavedb.db.session import SessionLocal
16+
17+
configure_mappers()
18+
19+
20+
evidence_strength_to_label = {
21+
-8: "BS3_VERY_STRONG",
22+
-4: "BS3_STRONG",
23+
-3: "BS3_MODERATE+",
24+
-2: "BS3_MODERATE",
25+
-1: "BS3_SUPPORTING",
26+
1: "PS3_SUPPORTING",
27+
2: "PS3_MODERATE",
28+
3: "PS3_MODERATE+",
29+
4: "PS3_STRONG",
30+
8: "PS3_VERY_STRONG",
31+
}
32+
33+
34+
def do_migration(db: Session):
35+
score_sets_with_ranges_or_calibrations = db.scalars(sa.select(ScoreSet).where(ScoreSet.score_ranges.isnot(None) | ScoreSet.score_calibrations.isnot(None))).all()
36+
37+
for score_set in score_sets_with_ranges_or_calibrations:
38+
if score_set.score_ranges is not None:
39+
investigator_ranges = InvestigatorScoreRangesCreate(**score_set.score_ranges)
40+
else:
41+
investigator_ranges = None
42+
43+
if score_set.score_calibrations is not None:
44+
thresholds = score_set.score_calibrations.get("pillar_project", {}).get("thresholds", [])
45+
evidence_strengths = score_set.score_calibrations.get("pillar_project", {}).get("evidence_strengths", [])
46+
positive_likelihood_ratios = score_set.score_calibrations.get("pillar_project", {}).get("positive_likelihood_ratios", [])
47+
prior_probability_pathogenicity = score_set.score_calibrations.get("pillar_project", {}).get("prior_probability_pathogenicity", None)
48+
parameter_sets = score_set.score_calibrations.get("pillar_project", {}).get("parameter_sets", [])
49+
50+
ranges = []
51+
boundary_direction = -1 # Start with a negative sign to indicate the first range has the lower boundary appearing prior to the threshold
52+
for idx, vals in enumerate(zip(thresholds, evidence_strengths, positive_likelihood_ratios)):
53+
threshold, evidence_strength, positive_likelihood_ratio = vals
54+
55+
if idx == 0:
56+
calculated_range = (None, threshold)
57+
ranges.append(PillarProjectScoreRangeCreate(
58+
range=(None, threshold),
59+
classification="normal" if evidence_strength < 0 else "abnormal",
60+
label=str(evidence_strength),
61+
evidence_strength=evidence_strength,
62+
positive_likelihood_ratio=positive_likelihood_ratio,
63+
))
64+
elif idx == len(thresholds) - 1:
65+
calculated_range = (threshold, None)
66+
ranges.append(PillarProjectScoreRangeCreate(
67+
range=(threshold, None),
68+
classification="normal" if evidence_strength < 0 else "abnormal",
69+
evidence_strength=evidence_strength,
70+
label=str(evidence_strength),
71+
positive_likelihood_ratio=positive_likelihood_ratio,
72+
))
73+
else:
74+
if boundary_direction < 0:
75+
calculated_range = (thresholds[idx - 1], threshold)
76+
else:
77+
calculated_range = (threshold, thresholds[idx + 1])
78+
79+
ranges.append(PillarProjectScoreRangeCreate(
80+
range=calculated_range,
81+
classification="normal" if evidence_strength < 0 else "abnormal",
82+
label=str(evidence_strength),
83+
evidence_strength=evidence_strength,
84+
positive_likelihood_ratio=positive_likelihood_ratio,
85+
))
86+
87+
# Set boundary_direction if the sign of evidence_strength flips compared to the next one
88+
if idx != len(evidence_strengths) - 1 and (evidence_strengths[idx + 1] * evidence_strength < 0):
89+
boundary_direction = -boundary_direction
90+
91+
pillar_project_ranges = PillarProjectScoreRangesCreate(
92+
prior_probability_pathogenicity=prior_probability_pathogenicity,
93+
parameter_sets=parameter_sets,
94+
ranges=ranges,
95+
)
96+
else:
97+
pillar_project_ranges = None
98+
99+
score_set.score_ranges = ScoreSetRangesCreate(
100+
investigator_provided=investigator_ranges if investigator_ranges else None,
101+
pillar_project=pillar_project_ranges if pillar_project_ranges else None,
102+
).dict()
103+
db.add(score_set)
104+
105+
106+
if __name__ == "__main__":
107+
db = SessionLocal()
108+
db.current_user = None # type: ignore
109+
110+
do_migration(db)
111+
112+
db.commit()
113+
db.close()

‎alembic/versions/34026092c7f8_clinvar_variant_table.py‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
"""clinvar variant table
2+
3+
Revision ID: 34026092c7f8
4+
Revises: e8a3b5d8f885
5+
Create Date: 2025-01-28 21:48:42.532346
6+
7+
"""
8+
9+
from alembic import op
10+
import sqlalchemy as sa
11+
12+
13+
# revision identifiers, used by Alembic.
14+
revision = "34026092c7f8"
15+
down_revision = "e8a3b5d8f885"
16+
branch_labels = None
17+
depends_on = None
18+
19+
20+
def upgrade():
21+
# ### commands auto generated by Alembic - please adjust! ###
22+
op.create_table(
23+
"clinvar_variants",
24+
sa.Column("id", sa.Integer(), nullable=False),
25+
sa.Column("allele_id", sa.Integer(), nullable=False),
26+
sa.Column("gene_symbol", sa.String(), nullable=False),
27+
sa.Column("clinical_significance", sa.String(), nullable=False),
28+
sa.Column("clinical_review_status", sa.String(), nullable=False),
29+
sa.Column("clinvar_db_version", sa.String(), nullable=False),
30+
sa.Column("creation_date", sa.Date(), nullable=False),
31+
sa.Column("modification_date", sa.Date(), nullable=False),
32+
sa.PrimaryKeyConstraint("id"),
33+
)
34+
op.create_index(op.f("ix_clinvar_variants_allele_id"), "clinvar_variants", ["allele_id"], unique=False)
35+
op.add_column("mapped_variants", sa.Column("clinvar_variant_id", sa.Integer(), nullable=True))
36+
op.create_index(
37+
op.f("ix_mapped_variants_clinvar_variant_id"), "mapped_variants", ["clinvar_variant_id"], unique=False
38+
)
39+
op.create_foreign_key(
40+
"mapped_variant_clinvar_variant_id_foreign_key_constraint",
41+
"mapped_variants",
42+
"clinvar_variants",
43+
["clinvar_variant_id"],
44+
["id"],
45+
)
46+
# ### end Alembic commands ###
47+
48+
49+
def downgrade():
50+
# ### commands auto generated by Alembic - please adjust! ###
51+
op.drop_constraint(
52+
"mapped_variant_clinvar_variant_id_foreign_key_constraint", "mapped_variants", type_="foreignkey"
53+
)
54+
op.drop_index(op.f("ix_mapped_variants_clinvar_variant_id"), table_name="mapped_variants")
55+
op.drop_column("mapped_variants", "clinvar_variant_id")
56+
op.drop_index(op.f("ix_clinvar_variants_allele_id"), table_name="clinvar_variants")
57+
op.drop_table("clinvar_variants")
58+
# ### end Alembic commands ###

‎alembic/versions/4726e4dddde8_simplify_published_variants_mv_indexes.py‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
"""simplify published variants mv indexes
2+
3+
Revision ID: 4726e4dddde8
4+
Revises: b85bc7b1bec7
5+
Create Date: 2025-03-25 10:46:46.641777
6+
7+
"""
8+
9+
from alembic import op
10+
11+
from mavedb.models.published_variant import signature
12+
13+
14+
# revision identifiers, used by Alembic.
15+
revision = "4726e4dddde8"
16+
down_revision = "b85bc7b1bec7"
17+
branch_labels = None
18+
depends_on = None
19+
20+
21+
def upgrade():
22+
# ### commands auto generated by Alembic - please adjust! ###
23+
op.drop_index(f"idx_{signature}_variant_id", signature)
24+
op.drop_index(f"idx_{signature}_variant_urn", signature)
25+
op.drop_index(f"idx_{signature}_mapped_variant_id", signature)
26+
op.drop_index(f"idx_{signature}_score_set_id", signature)
27+
op.drop_index(f"idx_{signature}_score_set_urn", signature)
28+
29+
op.create_index(
30+
f"idx_{signature}_ids",
31+
signature,
32+
["mapped_variant_id", "variant_id", "score_set_id"],
33+
unique=True,
34+
)
35+
# ### end Alembic commands ###
36+
37+
38+
def downgrade():
39+
# ### commands auto generated by Alembic - please adjust! ###
40+
op.drop_index(f"idx_{signature}_ids", signature)
41+
42+
op.create_index(
43+
f"idx_{signature}_variant_id",
44+
signature,
45+
["variant_id"],
46+
unique=False,
47+
)
48+
op.create_index(
49+
f"idx_{signature}_variant_urn",
50+
signature,
51+
["variant_urn"],
52+
unique=False,
53+
)
54+
op.create_index(
55+
f"idx_{signature}_score_set_id",
56+
signature,
57+
["score_set_id"],
58+
unique=False,
59+
)
60+
op.create_index(
61+
f"idx_{signature}_score_set_urn",
62+
signature,
63+
["score_set_urn"],
64+
unique=False,
65+
)
66+
op.create_index(
67+
f"idx_{signature}_mapped_variant_id",
68+
signature,
69+
["mapped_variant_id"],
70+
unique=True,
71+
)
72+
# ### end Alembic commands ###

0 commit comments

Comments
 (0)