diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index b7d2e7b9..4f5280d5 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -3,39 +3,24 @@ name: Tests on: [push] -env: - CONDA_PREFIX: /home/runner/work/microSALT/microSALT/tests - COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }} - jobs: build-linux: runs-on: ubuntu-latest - container: - image: python:3.6-slim - + steps: - uses: actions/checkout@v4 - - - name: Create conda/mamba environment using micromamba - uses: mamba-org/setup-micromamba@v1 - with: - environment-file: environment.yml - cache-downloads: true - - name: Install microSALT - run: | - mkdir ~/.microSALT && cp configExample.json ~/.microSALT/config.json - GITHUB_BRANCH=${GITHUB_REF##*/} - pip install -r requirements.txt -r requirements-dev.txt - pip install -U . + - name: Install uv + uses: astral-sh/setup-uv@v5 + - name: Set up Python + run: uv python install + - name: Install microSALT and dev dependencies + run: uv sync --dev - name: Lint with flake8 run: | # stop the build if there are Python syntax errors or undefined names - flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics + uv run flake8 . --exclude=.venv --count --select=E9,F63,F7,F82 --show-source --statistics # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + uv run flake8 . --exclude=.venv --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - name: Test with pytest run: | - pytest --cov=microSALT tests - - name: Run Coveralls - run: | - coveralls + uv run pytest tests \ No newline at end of file diff --git a/.gitignore b/.gitignore index 9ef8fead..cfb0274f 100644 --- a/.gitignore +++ b/.gitignore @@ -4,7 +4,7 @@ microSALT/__pycache__ microSALT.log instance/sqlalchemy_config.py instance/paths.yml -unique_references/*.n* +microSALT/unique_references/*.n* .pytest_cache/ *.egg-info *pyc @@ -16,3 +16,4 @@ tmp.json .venv/ .vscode* .markdownlint.jsonc +build diff --git a/.python-version b/.python-version new file mode 100644 index 00000000..e4fba218 --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +3.12 diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index 972b31c4..00000000 --- a/MANIFEST.in +++ /dev/null @@ -1,8 +0,0 @@ -include LICENSE -include README.md -include requirements.txt -recursive-include microSALT/server/templates * -recursive-include artwork/ * -recursive-include unique_references/ * -recursive-include tests/testdata/ * -include configExample.json diff --git a/README.md b/README.md index bb774637..30ad9158 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,9 @@ [![Build status](https://github.com/clinical-genomics/microsalt/actions/workflows/run_tests.yml/badge.svg)](https://github.com/clinical-genomics/microsalt/actions/workflows/run_tests.yml) -[![Coverage Status](https://coveralls.io/repos/github/Clinical-Genomics/microSALT/badge.svg?branch=master)](https://coveralls.io/github/Clinical-Genomics/microSALT?branch=master) [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.4026043.svg)](https://doi.org/10.5281/zenodo.4026043)

- - + +

@@ -16,16 +15,20 @@ sample, determines a sample's organism specific sequence type, and its resistance pattern. microSALT also provides a database storage solution and report generation of these results._ -_microSALT uses a combination of python, sqLite and flask. Python is used for -the majority of functionality, the database is handled through sqLite and the -front-end is handled through flask. All analysis activity by microSALT requires -a SLURM cluster._ +_microSALT uses a combination of Python, MySQL and Jinja2. Python is used for +the majority of functionality, the database is handled through MySQL via +SQLAlchemy and reports are rendered through Jinja2. All analysis activity by +microSALT requires a SLURM cluster._ ## Quick installation -1. `yes | bash <(curl https://raw.githubusercontent.com/Clinical-Genomics/microSALT/master/install.sh)` -2. `cp configExample.json $HOME/.microSALT/config.json` -3. `vim $HOME/.microSALT/config.json` +> [!IMPORTANT] +> This install requires `uv` to be installed on the system. For installation instructions, see [https://docs.astral.sh/uv/getting-started/installation/](https://docs.astral.sh/uv/getting-started/installation/). + +1. `bash <(curl https://raw.githubusercontent.com/Clinical-Genomics/microSALT/master/install.sh)` +2. `source microSALT/.venv/bin/activate` +3. `cp microSALT/configExample.json $HOME/.microSALT/config.json` +4. `vim $HOME/.microSALT/config.json` ## Configuration @@ -46,11 +49,11 @@ export MICROSALT_CONFIG=/MY/FAV/FOLDER/config.json ## Usage -- `microSALT analyse` contains functions to start sbatch job(s) & produce +- `microsalt analyse` contains functions to start sbatch job(s) & produce output to `folders['results']`. Afterwards the parsed results are uploaded to the SQL back-end and produce reports (HTML), which are then automatically e-mailed to the user. -- `microSALT utils` contains various functionality, including generating the +- `microsalt utils` contains various functionality, including generating the sample description json, manually adding new reference organisms and re-generating reports. @@ -72,13 +75,12 @@ Any definitions will work, as long as they retain the same formatting. ### Hardware - A [SLURM](https://slurm.schedmd.com) enabled HPC system -- A (clarity) LIMS server ### Software -- [Conda](https://conda.io) >= 3.6 -- Python 3.6 -- [SQLite](https://www.sqlite.org) +- [uv](https://docs.astral.sh/uv) >= 0.4 +- Python >= 3.10 +- [MySQL](https://www.mysql.com) server ## Contributing to this repo diff --git a/configExample.json b/configExample.json index cc2b7499..6451985f 100644 --- a/configExample.json +++ b/configExample.json @@ -21,8 +21,6 @@ "results": "/tmp/MLST/results/", "_comment": "Report collection folder", "reports": "/tmp/MLST/reports/", - "_comment": "Log file position and name", - "log_file": "/tmp/microsalt.log", "_comment": "Root folder for input fasta sequencing data", "seqdata": "/tmp/projects/", "_comment": "ST profiles. Each ST profile file under 'profiles' have an identicial folder under references", @@ -69,12 +67,6 @@ "bp_50x_warn": 50, "bp_100x_warn": 20 }, - "_comment": "Genologics temporary configuration file", - "genologics": { - "baseuri": "https://lims.facility.se/", - "username": "limsuser", - "password": "mypassword" - }, "_comment": "Credentials", "pubmlst": { "client_id": "", @@ -83,5 +75,24 @@ "pasteur": { "client_id": "", "client_secret": "" + }, + "_comment": "Singularity runtime configuration", + "singularity": { + "binary": "/usr/bin/singularity", + "_comment": "Paths that must be visible inside every container (passed as --bind)", + "bind_paths": ["/data"], + "_comment": "Adapter path inside the trimmomatic container (bioconda default)", + "trimmomatic_adapters": "/opt/conda/share/trimmomatic/adapters/" + }, + "_comment": "Absolute paths to pre-built Singularity SIF images on the cluster", + "containers": { + "skesa": "/fs1/resources/containers/skesa_2.5.1.sif", + "blast": "/fs1/resources/containers/blast_2.12.0.sif", + "bwa": "/fs1/resources/containers/bwa_0.7.17.sif", + "samtools": "/fs1/resources/containers/samtools_1.13.sif", + "picard": "/fs1/resources/containers/picard_2.20.3.sif", + "trimmomatic": "/fs1/resources/containers/trimmomatic_0.39.sif", + "quast": "/fs1/resources/containers/quast_5.3.0.sif" } -} \ No newline at end of file +} + diff --git a/containers/Dockerfile.blast b/containers/Dockerfile.blast new file mode 100644 index 00000000..a0f69ac3 --- /dev/null +++ b/containers/Dockerfile.blast @@ -0,0 +1,19 @@ +# NCBI BLAST 2.12.0 — nucleotide BLAST search and database indexing +# Build: docker build -f Dockerfile.blast -t blast:2.12.0 . +# Convert: singularity pull --name blast_2.12.0.sif docker://your-repo/blast:2.12.0 +FROM mambaorg/micromamba:1-bookworm-slim + +ARG MAMBA_DOCKERFILE_ACTIVATE=1 + +USER root +RUN apt-get update \ + && apt-get upgrade -y \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* +USER $MAMBA_USER + +RUN micromamba install -y -n base -c conda-forge -c bioconda \ + blast=2.12.0 \ + && micromamba clean -afy + +ENV PATH="/opt/conda/bin:$PATH" diff --git a/containers/Dockerfile.bwa b/containers/Dockerfile.bwa new file mode 100644 index 00000000..7898185c --- /dev/null +++ b/containers/Dockerfile.bwa @@ -0,0 +1,19 @@ +# BWA 0.7.17 — short read aligner (mem mode) and reference indexing +# Build: docker build -f Dockerfile.bwa -t bwa:0.7.17 . +# Convert: singularity pull --name bwa_0.7.17.sif docker://your-repo/bwa:0.7.17 +FROM mambaorg/micromamba:1-bookworm-slim + +ARG MAMBA_DOCKERFILE_ACTIVATE=1 + +USER root +RUN apt-get update \ + && apt-get upgrade -y \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* +USER $MAMBA_USER + +RUN micromamba install -y -n base -c conda-forge -c bioconda \ + bwa=0.7.17 \ + && micromamba clean -afy + +ENV PATH="/opt/conda/bin:$PATH" diff --git a/containers/Dockerfile.picard b/containers/Dockerfile.picard new file mode 100644 index 00000000..7ad48934 --- /dev/null +++ b/containers/Dockerfile.picard @@ -0,0 +1,23 @@ +# Picard 2.20.3 — duplicate marking and insert size metrics +# Build: docker build -f Dockerfile.picard -t picard:2.20.3 . +# Convert: singularity pull --name picard_2.20.3.sif docker://your-repo/picard:2.20.3 +# +# The bioconda picard package installs a wrapper script that calls the JAR via +# the bundled Java runtime; no separate openjdk install is needed. +FROM mambaorg/micromamba:1-bookworm-slim + +ARG MAMBA_DOCKERFILE_ACTIVATE=1 + +USER root +RUN apt-get update \ + && apt-get upgrade -y \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* +USER $MAMBA_USER + +RUN micromamba install -y -n base -c conda-forge -c bioconda \ + picard=2.20.3 \ + r-base=4.1.1 \ + && micromamba clean -afy + +ENV PATH="/opt/conda/bin:$PATH" diff --git a/containers/Dockerfile.quast b/containers/Dockerfile.quast new file mode 100644 index 00000000..58acb1b2 --- /dev/null +++ b/containers/Dockerfile.quast @@ -0,0 +1,21 @@ +# QUAST 5.3.0 — assembly quality assessment +# Build: docker build -f Dockerfile.quast -t quast:5.3.0 . +# Convert: singularity pull --name quast_5.3.0.sif docker://your-repo/quast:5.3.0 +# +# The entry point is quast.py (as installed by bioconda). +FROM mambaorg/micromamba:1-bookworm-slim + +ARG MAMBA_DOCKERFILE_ACTIVATE=1 + +USER root +RUN apt-get update \ + && apt-get upgrade -y \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* +USER $MAMBA_USER + +RUN micromamba install -y -n base -c conda-forge -c bioconda \ + quast=5.3.0 \ + && micromamba clean -afy + +ENV PATH="/opt/conda/bin:$PATH" diff --git a/containers/Dockerfile.samtools b/containers/Dockerfile.samtools new file mode 100644 index 00000000..e5e48a41 --- /dev/null +++ b/containers/Dockerfile.samtools @@ -0,0 +1,19 @@ +# SAMtools 1.13 — SAM/BAM processing, indexing, and statistics +# Build: docker build -f Dockerfile.samtools -t samtools:1.13 . +# Convert: singularity pull --name samtools_1.13.sif docker://your-repo/samtools:1.13 +FROM mambaorg/micromamba:1-bookworm-slim + +ARG MAMBA_DOCKERFILE_ACTIVATE=1 + +USER root +RUN apt-get update \ + && apt-get upgrade -y \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* +USER $MAMBA_USER + +RUN micromamba install -y -n base -c conda-forge -c bioconda \ + samtools=1.13 \ + && micromamba clean -afy + +ENV PATH="/opt/conda/bin:$PATH" diff --git a/containers/Dockerfile.skesa b/containers/Dockerfile.skesa new file mode 100644 index 00000000..ffc738d9 --- /dev/null +++ b/containers/Dockerfile.skesa @@ -0,0 +1,19 @@ +# SKESA 2.5.1 — de novo assembler +# Build: docker build -f Dockerfile.skesa -t skesa:2.5.1 . +# Convert: singularity pull --name skesa_2.5.1.sif docker://your-repo/skesa:2.5.1 +FROM mambaorg/micromamba:1-bookworm-slim + +ARG MAMBA_DOCKERFILE_ACTIVATE=1 + +USER root +RUN apt-get update \ + && apt-get upgrade -y \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* +USER $MAMBA_USER + +RUN micromamba install -y -n base -c conda-forge -c bioconda \ + skesa=2.5.1 \ + && micromamba clean -afy + +ENV PATH="/opt/conda/bin:$PATH" diff --git a/containers/Dockerfile.trimmomatic b/containers/Dockerfile.trimmomatic new file mode 100644 index 00000000..9aad05f2 --- /dev/null +++ b/containers/Dockerfile.trimmomatic @@ -0,0 +1,23 @@ +# Trimmomatic 0.39 — quality trimming for paired-end Illumina reads +# Build: docker build -f Dockerfile.trimmomatic -t trimmomatic:0.39 . +# Convert: singularity pull --name trimmomatic_0.39.sif docker://your-repo/trimmomatic:0.39 +# +# Adapter files are located inside the container at: +# /opt/conda/share/trimmomatic/adapters/ +# Set singularity.trimmomatic_adapters in your config to this path. +FROM mambaorg/micromamba:1-bookworm-slim + +ARG MAMBA_DOCKERFILE_ACTIVATE=1 + +USER root +RUN apt-get update \ + && apt-get upgrade -y \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* +USER $MAMBA_USER + +RUN micromamba install -y -n base -c conda-forge -c bioconda \ + trimmomatic=0.39 \ + && micromamba clean -afy + +ENV PATH="/opt/conda/bin:$PATH" diff --git a/environment.yml b/environment.yml index 4366a44d..f0c49037 100644 --- a/environment.yml +++ b/environment.yml @@ -1,19 +1,3 @@ name: P_microSALT -channels: - - conda-forge - - bioconda - - defaults dependencies: - - python=3.6 - - blast=2.12.0 - - bwa=0.7.17 - - picard=2.20.3 - - pigz>=2.4 - - quast=5.0.2 - - samtools=1.13 - - trimmomatic=0.39 - - r-base=4.1.1 - - openjdk=11.0.9.1 - - pytest=6.2.5 - - pytest-cov=2.12.1 - - skesa=2.5.1 + - python=3.12 diff --git a/install.sh b/install.sh index dc5dc696..0246efd4 100644 --- a/install.sh +++ b/install.sh @@ -6,31 +6,17 @@ shopt -s nullglob #Suggests provided branch. Else suggests master default_branch=${1-master} -default_name=${2-microSALT} echo "Welcome to the microSALT installation script. Q to exit" -while true; do - echo "Name your microSALT environment ['microSALT']:" - read input - if [[ $input = "q" ]] || [[ $input = "Q" ]]; then - break - elif [[ $input = "y" ]] || [[ $input = "yes" ]] || [[ $input = "" ]]; then - cname=$default_name - break - else - cname=$input - break - fi -done while true; do echo "Would you like a 'release' or 'source' (development) environment ['release']?" read input if [[ $input = "q" ]] || [[ $input = "Q" ]]; then - break + exit 0 elif [[ $input = "y" ]] || [[ $input = "yes" ]] || [[ $input = "" ]]; then type="release" break - elif [[ $input == "source" ]] || [[ $input == "release" ]]; then + elif [[ $input == "source" ]] || [[ $input == "release" ]]; then type=$input break fi @@ -46,44 +32,24 @@ while true; do else branch=$input fi - curl https://raw.githubusercontent.com/Clinical-Genomics/microSALT/$branch/LICENSE | tac | tac | grep -q 'License' && validbranch=true||echo "Invalid branch name" + curl https://raw.githubusercontent.com/Clinical-Genomics/microSALT/$branch/LICENSE | tac | tac | grep -q 'License' && validbranch=true || echo "Invalid branch name" done break done -echo "Thank you, setting up environment $cname!" +echo "Thank you, installing branch $branch!" -#Only use micromamba if inside GitHub Actions -if [[ -z "${GITHUB_ACTIONS}" ]]; then - conda_cmd="micromamba" -else - conda_cmd="conda" +if [ -d microSALT ]; then + rm -rf microSALT fi -#Unload environment -$conda_cmd info | tac | tac | grep -q $cname && source deactivate || : -#Remove environment if already present -$conda_cmd remove -y -n $cname --all || : -$conda_cmd env create -n $cname -f https://raw.githubusercontent.com/Clinical-Genomics/microSALT/$branch/environment.yml -source activate $cname +git clone https://github.com/Clinical-Genomics/microSALT +cd microSALT && git checkout $branch if [[ $type == "release" ]]; then - pip install -r https://raw.githubusercontent.com/Clinical-Genomics/microSALT/$branch/requirements.txt -r https://raw.githubusercontent.com/Clinical-Genomics/microSALT/$branch/requirements-dev.txt - pip install -U git+https://github.com/Clinical-Genomics/microSALT@$branch + uv sync elif [[ $type == "source" ]]; then - HERE=$PWD - if [ -d ${HERE}/microSALT ]; then - rm -rf microSALT - fi - git clone https://github.com/Clinical-Genomics/microSALT - cd microSALT && git checkout $branch - pip install -r requirements.txt -r requirements-dev.txt && pip install -e . && cd ${HERE} - echo "Source installed under ${HERE}/microSALT" -fi -echo "Installation Complete!" -while true; do - echo "Configuration requires manual set-up as described in README.md ['yes']:" - read input - if [[ $input = "y" ]] || [[ $input = "yes" ]]; then - break - fi -done + uv sync --group dev +fi + +echo "Installation Complete! Activate the environment with: source microSALT/.venv/bin/activate" +echo "Configuration requires manual set-up as described in README.md" diff --git a/microSALT/__init__.py b/microSALT/__init__.py index 13343482..19d80286 100644 --- a/microSALT/__init__.py +++ b/microSALT/__init__.py @@ -1,29 +1,6 @@ -import collections -import json import logging -import os -import pathlib -import re -import subprocess -import sys -from distutils.sysconfig import get_python_lib -from enum import Enum -from logging import Logger -from flask import Flask - -__version__ = "4.2.8" - -app = Flask(__name__, template_folder="server/templates") -app.config.setdefault("SQLALCHEMY_DATABASE_URI", "sqlite:///:memory:") -app.config.setdefault("SQLALCHEMY_BINDS", None) -app.config.setdefault("SQLALCHEMY_TRACK_MODIFICATIONS", False) - -# Keep track of microSALT installation -wd = os.path.dirname(os.path.realpath(__file__)) - -# Load configuration -preset_config = "" +__version__ = "4.3.0" logger = None @@ -46,128 +23,8 @@ def setup_logger(logging_level: str) -> None: logger.setLevel(logging_levels[logging_level]) ch = logging.StreamHandler() ch.setLevel(logging_levels[logging_level]) - formatter = logging.Formatter("%(asctime)s\t%(levelname)s\t%(message)s", "%Y-%m-%d %H:%M:%S") + formatter = logging.Formatter( + "%(asctime)s - %(name)s - %(levelname)s - %(message)s", "%Y-%m-%d %H:%M:%S" + ) ch.setFormatter(formatter) logger.addHandler(ch) - - -default = os.path.join(os.environ["HOME"], ".microSALT/config.json") - -if "MICROSALT_CONFIG" in os.environ: - try: - envvar = os.environ["MICROSALT_CONFIG"] - with open(envvar, "r") as conf: - preset_config = json.load(conf) - except Exception as e: - print("Config error: {}".format(str(e))) - pass -elif os.path.exists(default): - try: - with open(os.path.abspath(default), "r") as conf: - preset_config = json.load(conf) - except Exception as e: - print("Config error: {}".format(str(e))) - pass - -# Config dependent section: -if preset_config != "": - try: - # Load flask info - app.config.update(preset_config["database"]) - - # Add `folders` configuration - app.config["folders"] = preset_config.get("folders", {}) - - # Ensure PubMLST configuration is included - - app.config["pubmlst"] = preset_config.get("pubmlst", {"client_id": "", "client_secret": ""}) - - app.config["pasteur"] = preset_config.get("pasteur", {"client_id": "", "client_secret": ""}) - - # Add extrapaths to config - preset_config["folders"]["expec"] = os.path.abspath( - os.path.join(pathlib.Path(__file__).parent.parent, "unique_references/ExPEC.fsa") - ) - # Check if release install exists - for entry in os.listdir(get_python_lib()): - if "microSALT-" in entry: - preset_config["folders"]["expec"] = os.path.abspath( - os.path.join(os.path.expandvars("$CONDA_PREFIX"), "expec/ExPEC.fsa") - ) - break - preset_config["folders"]["adapters"] = os.path.abspath( - os.path.join( - os.path.expandvars("$CONDA_PREFIX"), - "share/trimmomatic/adapters/", - ) - ) - - # Initialize logger - setup_logger(logging_level="INFO") - - # Create paths mentioned in config - db_file = re.search( - "sqlite:///(.+)", - preset_config["database"]["SQLALCHEMY_DATABASE_URI"], - ).group(1) - for entry in preset_config.keys(): - if entry != "_comment": - if ( - isinstance(preset_config[entry], str) - and "/" in preset_config[entry] - and entry not in ["genologics"] - ): - if not preset_config[entry].startswith("/"): - sys.exit(-1) - unmade_fldr = os.path.abspath(preset_config[entry]) - if not pathlib.Path(unmade_fldr).exists(): - os.makedirs(unmade_fldr) - logger.info("Created path {}".format(unmade_fldr)) - - # level two - elif isinstance(preset_config[entry], collections.Mapping): - for thing in preset_config[entry].keys(): - if ( - isinstance(preset_config[entry][thing], str) - and "/" in preset_config[entry][thing] - and entry not in ["genologics"] - ): - # Special string, mangling - if thing == "log_file": - unmade_fldr = os.path.dirname(preset_config[entry][thing]) - bash_cmd = "touch {}".format(preset_config[entry][thing]) - proc = subprocess.Popen(bash_cmd.split(), stdout=subprocess.PIPE) - output, error = proc.communicate() - elif thing == "SQLALCHEMY_DATABASE_URI": - unmade_fldr = os.path.dirname(db_file) - bash_cmd = "touch {}".format(db_file) - proc = subprocess.Popen(bash_cmd.split(), stdout=subprocess.PIPE) - output, error = proc.communicate() - if proc.returncode != 0: - logger.error( - "Database writing failed! Invalid user access detected!" - ) - sys.exit(-1) - else: - unmade_fldr = preset_config[entry][thing] - if not pathlib.Path(unmade_fldr).exists(): - os.makedirs(unmade_fldr) - logger.info("Created path {}".format(unmade_fldr)) - - fh = logging.FileHandler(os.path.expanduser(preset_config["folders"]["log_file"])) - fh.setFormatter(logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")) - logger.addHandler(fh) - - # Integrity check database - cmd = "sqlite3 {0}".format(db_file) - cmd = cmd.split() - cmd.append("pragma integrity_check;") - proc = subprocess.Popen(cmd, stdout=subprocess.PIPE) - output, error = proc.communicate() - if "ok" not in str(output): - logger.error("Database integrity failed! Lock-state detected!") - sys.exit(-1) - - except Exception as e: - print("Config error: {}".format(str(e))) - pass diff --git a/artwork/microsalt.emf b/microSALT/artwork/microsalt.emf similarity index 100% rename from artwork/microsalt.emf rename to microSALT/artwork/microsalt.emf diff --git a/artwork/microsalt.jpg b/microSALT/artwork/microsalt.jpg similarity index 100% rename from artwork/microsalt.jpg rename to microSALT/artwork/microsalt.jpg diff --git a/artwork/swedac-old.jpg b/microSALT/artwork/swedac-old.jpg similarity index 100% rename from artwork/swedac-old.jpg rename to microSALT/artwork/swedac-old.jpg diff --git a/artwork/swedac.jpg b/microSALT/artwork/swedac.jpg similarity index 100% rename from artwork/swedac.jpg rename to microSALT/artwork/swedac.jpg diff --git a/microSALT/cli.py b/microSALT/cli.py index bfaf1c76..b87a2e69 100644 --- a/microSALT/cli.py +++ b/microSALT/cli.py @@ -6,12 +6,23 @@ import json import logging import os +import pathlib +import re +import subprocess import sys import click -from microSALT import __version__, logging_levels, preset_config +from microSALT import __version__, logging_levels, setup_logger +from microSALT.config import MicroSALTConfig, load_config +from microSALT.exc.exceptions import RefUpdateLockError +from microSALT.store.database import ( + create_tables, + get_scoped_session_registry, + initialize_database, +) from microSALT.utils.job_creator import Job_Creator +from microSALT.utils.pubmlst.get_credentials import main as get_bigsdb_credentials_main from microSALT.utils.referencer import Referencer from microSALT.utils.reporter import Reporter from microSALT.utils.scraper import Scraper @@ -35,27 +46,6 @@ logger = logging.getLogger("main_logger") -if preset_config == "": - click.echo( - "ERROR - No properly set-up config under neither envvar MICROSALT_CONFIG nor ~/.microSALT/config.json. Exiting." - ) - sys.exit(-1) - - -@click.pass_context -def set_cli_config(ctx, config): - if config != "": - if os.path.exists(config): - try: - t = ctx.obj["config"] - with open(os.path.abspath(config), "r") as conf: - ctx.obj["config"] = json.load(conf) - ctx.obj["config"]["folders"]["expec"] = t["folders"]["expec"] - ctx.obj["config"]["folders"]["adapters"] = t["folders"]["adapters"] - ctx.obj["config"]["config_path"] = os.path.abspath(config) - except Exception: - pass - def done(): click.echo("INFO - Execution finished!") @@ -77,23 +67,66 @@ def review_sampleinfo(pfile): for k, v in default_sampleinfo.items(): if k not in entry: click.echo( - "WARNING - Parameter {} needs to be provided in sample json. Formatting example: ({})".format( - k, v - ) + f"WARNING - Parameter {k} needs to be provided in sample json. Formatting example: ({v})" ) else: for k, v in default_sampleinfo.items(): if k not in data: click.echo( - "WARNING - Parameter {} needs to be provided in sample json. Formatting example: ({})".format( - k, v - ) + f"WARNING - Parameter {k} needs to be provided in sample json. Formatting example: ({v})" ) return data +def teardown_session(): + """Ensure that the session is closed and all resources are released to the connection pool.""" + registry = get_scoped_session_registry() + if registry: + registry.remove() + + +def _ensure_directories(config: MicroSALTConfig) -> None: + """Create any configured directory paths that do not yet exist.""" + db_uri = config.database.SQLALCHEMY_DATABASE_URI + db_match = re.search("sqlite:///(.+)", db_uri) + if db_match: + db_file = db_match.group(1) + db_dir = os.path.dirname(db_file) + if db_dir and not pathlib.Path(db_dir).exists(): + os.makedirs(db_dir) + proc = subprocess.Popen(f"touch {db_file}".split(), stdout=subprocess.PIPE) + _, _ = proc.communicate() + if proc.returncode != 0: + click.echo("ERROR - Database writing failed! Invalid user access detected!") + sys.exit(-1) + + folder_paths = [ + config.folders.results, + config.folders.reports, + config.folders.seqdata, + config.folders.profiles, + config.folders.references, + config.folders.resistances, + config.folders.genomes, + config.folders.credentials, + ] + for path in folder_paths: + p = pathlib.Path(os.path.expandvars(os.path.expanduser(path))) + if not p.exists(): + os.makedirs(p) + + +pass_config = click.make_pass_decorator(MicroSALTConfig, ensure=True) + + @click.group() @click.version_option(__version__) +@click.option( + "--config", + required=True, + help="Path to microSALT config JSON file", + type=click.Path(exists=True, dir_okay=False), +) @click.option( "--logging-level", default="INFO", @@ -101,31 +134,39 @@ def review_sampleinfo(pfile): help="Set the logging level for the CLI", ) @click.pass_context -def root(ctx, logging_level): +def root(ctx, config, logging_level): """microbial Sequence Analysis and Loci-based Typing (microSALT) pipeline""" - ctx.obj = {} - ctx.obj["config"] = preset_config + cfg = load_config(config) + initialize_database(cfg.database.SQLALCHEMY_DATABASE_URI) + setup_logger(logging_level=logging_level) logger.setLevel(logging_levels[logging_level]) for handler in logger.handlers: handler.setLevel(logging_levels[logging_level]) - logger.debug(f"Setting logging level to {logging_levels[logging_level]}") + ctx.obj = cfg + ctx.call_on_close(teardown_session) + + +@root.command() +@pass_config +def setup(config: MicroSALTConfig): + """Create all configured directories and verify database access. Run once after installation.""" + _ensure_directories(config) + click.echo("INFO - Directory setup complete.") + create_tables() + click.echo("INFO - Database tables created (or already exist).") + done() @root.command() @click.argument("sampleinfo_file") @click.option("--input", help="Full path to input folder", default="") -@click.option("--config", help="microSALT config to override default", default="") @click.option( "--dry", help="Builds instance without posting to SLURM", default=False, is_flag=True, ) -@click.option( - "--email", - default=preset_config["regex"]["mail_recipient"], - help="Forced e-mail recipient", -) +@click.option("--email", default="", help="Forced e-mail recipient") @click.option("--skip_update", default=False, help="Skips downloading of references", is_flag=True) @click.option( "--force_update", @@ -134,12 +175,11 @@ def root(ctx, logging_level): is_flag=True, ) @click.option("--untrimmed", help="Use untrimmed input data", default=False, is_flag=True) -@click.pass_context +@pass_config def analyse( - ctx, + config: MicroSALTConfig, sampleinfo_file, input, - config, dry, email, skip_update, @@ -147,43 +187,59 @@ def analyse( untrimmed, ): """Sequence analysis, typing and resistance identification""" - # Run section pool = [] - trimmed = not untrimmed - set_cli_config(config) - ctx.obj["config"]["regex"]["mail_recipient"] = email - ctx.obj["config"]["dry"] = dry + if email: + config.regex.mail_recipient = email + config.dry = dry if not os.path.isdir(input): - click.echo("ERROR - Sequence data folder {} does not exist.".format(input)) - ctx.abort() + click.echo(f"ERROR - Sequence data folder {input} does not exist.") + click.Abort() for subfolder in os.listdir(input): - if os.path.isdir("{}/{}".format(input, subfolder)): + if os.path.isdir(f"{input}/{subfolder}"): pool.append(subfolder) run_settings = { "input": input, "dry": dry, - "email": email, + "email": config.regex.mail_recipient, "skip_update": skip_update, "trimmed": not untrimmed, "pool": pool, } - # Samples section sampleinfo = review_sampleinfo(sampleinfo_file) run_creator = Job_Creator( - config=ctx.obj["config"], log=logger, + folders=config.folders, + slurm_header=config.slurm_header, + regex=config.regex, + dry=config.dry, + config_path=config.config_path, + threshold=config.threshold, + pubmlst=config.pubmlst, + pasteur=config.pasteur, + singularity=config.singularity, + containers=config.containers, sampleinfo=sampleinfo, run_settings=run_settings, ) ext_refs = Referencer( - config=ctx.obj["config"], log=logger, + folders=config.folders, + threshold=config.threshold, + pubmlst=config.pubmlst, + pasteur=config.pasteur, + singularity=config.singularity, + containers=config.containers, sampleinfo=sampleinfo, force=force_update, ) + try: + ext_refs.db_access.check_ref_lock() + except RefUpdateLockError as e: + click.echo("ERROR - {}".format(e)) + click.Abort() click.echo("INFO - Checking versions of references..") try: if not skip_update: @@ -193,13 +249,13 @@ def analyse( else: click.echo("INFO - Skipping version check.") except Exception as e: - click.echo("{}".format(e)) + click.echo(f"{e}") if len(sampleinfo) > 1: run_creator.project_job() elif len(sampleinfo) == 1: run_creator.project_job(single_sample=True) else: - ctx.abort() + click.Abort() done() @@ -227,18 +283,13 @@ def refer(ctx): default="default", type=click.Choice(["default", "typing", "qc", "cgmlst"]), ) -@click.option("--config", help="microSALT config to override default", default="") @click.option( "--dry", help="Builds instance without posting to SLURM", default=False, is_flag=True, ) -@click.option( - "--email", - default=preset_config["regex"]["mail_recipient"], - help="Forced e-mail recipient", -) +@click.option("--email", default="", help="Forced e-mail recipient") @click.option("--skip_update", default=False, help="Skips downloading of references", is_flag=True) @click.option( "--report", @@ -246,34 +297,40 @@ def refer(ctx): type=click.Choice(["default", "typing", "motif_overview", "qc", "json_dump", "st_update"]), ) @click.option("--output", help="Report output folder", default="") -@click.pass_context -def finish(ctx, sampleinfo_file, input, track, config, dry, email, skip_update, report, output): +@pass_config +def finish( + config: MicroSALTConfig, sampleinfo_file, input, track, dry, email, skip_update, report, output +): """Sequence analysis, typing and resistance identification""" - # Run section pool = [] - set_cli_config(config) - ctx.obj["config"]["regex"]["mail_recipient"] = email - ctx.obj["config"]["dry"] = dry + if email: + config.regex.mail_recipient = email + config.dry = dry if not os.path.isdir(input): - click.echo("ERROR - Sequence data folder {} does not exist.".format(input)) - ctx.abort() + click.echo(f"ERROR - Sequence data folder {input} does not exist.") + click.Abort() if output == "": output = input for subfolder in os.listdir(input): - if os.path.isdir("{}/{}".format(input, subfolder)): + if os.path.isdir(f"{input}/{subfolder}"): pool.append(subfolder) - run_settings = { - "input": input, - "track": track, - "dry": dry, - "email": email, - "skip_update": skip_update, - } - - # Samples section sampleinfo = review_sampleinfo(sampleinfo_file) - ext_refs = Referencer(config=ctx.obj["config"], log=logger, sampleinfo=sampleinfo) + ext_refs = Referencer( + log=logger, + folders=config.folders, + threshold=config.threshold, + pubmlst=config.pubmlst, + pasteur=config.pasteur, + singularity=config.singularity, + containers=config.containers, + sampleinfo=sampleinfo, + ) + try: + ext_refs.db_access.check_ref_lock() + except RefUpdateLockError as e: + click.echo("ERROR - {}".format(e)) + click.Abort() click.echo("INFO - Checking versions of references..") try: if not skip_update: @@ -283,19 +340,33 @@ def finish(ctx, sampleinfo_file, input, track, config, dry, email, skip_update, else: click.echo("INFO - Skipping version check.") except Exception as e: - click.echo("{}".format(e)) + click.echo(f"{e}") - res_scraper = Scraper(config=ctx.obj["config"], log=logger, sampleinfo=sampleinfo, input=input) + res_scraper = Scraper( + log=logger, + folders=config.folders, + threshold=config.threshold, + slurm_header=config.slurm_header, + regex=config.regex, + dry=config.dry, + config_path=config.config_path, + pubmlst=config.pubmlst, + pasteur=config.pasteur, + singularity=config.singularity, + containers=config.containers, + sampleinfo=sampleinfo, + input=input, + ) if isinstance(sampleinfo, list) and len(sampleinfo) > 1: res_scraper.scrape_project() - # for subfolder in pool: - # res_scraper.scrape_sample() else: res_scraper.scrape_sample() codemonkey = Reporter( - config=ctx.obj["config"], log=logger, + folders=config.folders, + threshold=config.threshold, + regex=config.regex, sampleinfo=sampleinfo, output=output, collection=True, @@ -307,25 +378,51 @@ def finish(ctx, sampleinfo_file, input, track, config, dry, email, skip_update, @refer.command() @click.argument("organism") @click.option("--force", help="Redownloads existing organism", default=False, is_flag=True) -@click.pass_context -def add(ctx, organism, force): +@pass_config +def add(config: MicroSALTConfig, organism, force): """Adds a new internal organism from pubMLST""" - referee = Referencer(config=ctx.obj["config"], log=logger, force=force) + referee = Referencer( + log=logger, + folders=config.folders, + threshold=config.threshold, + pubmlst=config.pubmlst, + pasteur=config.pasteur, + singularity=config.singularity, + containers=config.containers, + force=force, + ) try: referee.add_pubmlst(organism) except Exception as e: click.echo(e.args[0]) - ctx.abort() + click.Abort() click.echo("INFO - Checking versions of all references..") - referee = Referencer(config=ctx.obj["config"], log=logger, force=force) + referee = Referencer( + log=logger, + folders=config.folders, + threshold=config.threshold, + pubmlst=config.pubmlst, + pasteur=config.pasteur, + singularity=config.singularity, + containers=config.containers, + force=force, + ) referee.update_refs() @refer.command() -@click.pass_context -def observe(ctx): +@pass_config +def observe(config: MicroSALTConfig): """Lists all stored organisms""" - refe = Referencer(config=ctx.obj["config"], log=logger) + refe = Referencer( + log=logger, + folders=config.folders, + threshold=config.threshold, + pubmlst=config.pubmlst, + pasteur=config.pasteur, + singularity=config.singularity, + containers=config.containers, + ) click.echo("INFO - Currently stored organisms:") for org in sorted(refe.existing_organisms()): click.echo(org.replace("_", " ").capitalize()) @@ -333,11 +430,7 @@ def observe(ctx): @utils.command() @click.argument("sampleinfo_file") -@click.option( - "--email", - default=preset_config["regex"]["mail_recipient"], - help="Forced e-mail recipient", -) +@click.option("--email", default="", help="Forced e-mail recipient") @click.option( "--type", default="default", @@ -345,14 +438,17 @@ def observe(ctx): ) @click.option("--output", help="Full path to output folder", default="") @click.option("--collection", default=False, is_flag=True) -@click.pass_context -def report(ctx, sampleinfo_file, email, type, output, collection): +@pass_config +def report(config: MicroSALTConfig, sampleinfo_file, email, type, output, collection): """Re-generates report for a project""" - ctx.obj["config"]["regex"]["mail_recipient"] = email + if email: + config.regex.mail_recipient = email sampleinfo = review_sampleinfo(sampleinfo_file) codemonkey = Reporter( - config=ctx.obj["config"], log=logger, + folders=config.folders, + threshold=config.threshold, + regex=config.regex, sampleinfo=sampleinfo, output=output, collection=collection, @@ -361,12 +457,13 @@ def report(ctx, sampleinfo_file, email, type, output, collection): done() -@utils.command() -@click.pass_context -def view(ctx): - """Starts an interactive webserver for viewing""" - codemonkey = Reporter(config=ctx.obj["config"], log=logger) - codemonkey.start_web() +@utils.command("get-bigsdb-credentials") +@click.argument("service", type=click.Choice(["pubmlst", "pasteur"])) +@click.option("--species", default=None, help="Species name (required for the 'pasteur' service)") +@pass_config +def get_bigsdb_credentials(config: MicroSALTConfig, service, species): + """Obtain and store BIGSdb OAuth credentials for SERVICE (pubmlst or pasteur)""" + get_bigsdb_credentials_main(service, config, species) @utils.command() @@ -381,11 +478,11 @@ def generate(ctx, input): pool = [] if not os.path.isdir(input): - click.echo("ERROR - Sequence data folder {} does not exist.".format(project_name)) + click.echo(f"ERROR - Sequence data folder {project_name} does not exist.") ctx.abort() elif input != os.getcwd(): for subfolder in os.listdir(input): - if os.path.isdir("{}/{}".format(input, subfolder)): + if os.path.isdir(f"{input}/{subfolder}"): pool.append(defaults.copy()) pool[-1]["CG_ID_project"] = project_name pool[-1]["CG_ID_sample"] = subfolder @@ -393,9 +490,9 @@ def generate(ctx, input): project_name = "default_sample_info" pool.append(defaults.copy()) - with open("{}/{}.json".format(os.getcwd(), project_name), "w") as output: + with open(f"{os.getcwd()}/{project_name}.json", "w") as output: json.dump(pool, output, indent=2) - click.echo("INFO - Created {}.json in current folder".format(project_name)) + click.echo(f"INFO - Created {project_name}.json in current folder") done() @@ -414,24 +511,34 @@ def resync(ctx): ) @click.option("--customer", default="all", help="Customer id filter") @click.option("--skip_update", default=False, help="Skips downloading of references", is_flag=True) -@click.option( - "--email", - default=preset_config["regex"]["mail_recipient"], - help="Forced e-mail recipient", -) +@click.option("--email", default="", help="Forced e-mail recipient") @click.option("--output", help="Full path to output folder", default="") -@click.pass_context -def review(ctx, type, customer, skip_update, email, output): +@pass_config +def review(config: MicroSALTConfig, type, customer, skip_update, email, output): """Generates information about novel ST""" - # Trace exists by some samples having pubMLST_ST filled in. Make trace function later - ctx.obj["config"]["regex"]["mail_recipient"] = email - ext_refs = Referencer(config=ctx.obj["config"], log=logger) + if email: + config.regex.mail_recipient = email + ext_refs = Referencer( + log=logger, + folders=config.folders, + threshold=config.threshold, + pubmlst=config.pubmlst, + pasteur=config.pasteur, + singularity=config.singularity, + containers=config.containers, + ) if not skip_update: ext_refs.update_refs() ext_refs.resync() click.echo("INFO - Version check done. Generating output") if type == "report": - codemonkey = Reporter(config=ctx.obj["config"], log=logger, output=output) + codemonkey = Reporter( + log=logger, + folders=config.folders, + threshold=config.threshold, + regex=config.regex, + output=output, + ) codemonkey.report(type="st_update", customer=customer) elif type == "list": ext_refs.resync(type=type) @@ -440,20 +547,38 @@ def review(ctx, type, customer, skip_update, email, output): @resync.command() @click.option("--force-update", default=False, is_flag=True, help="Forces update") -@click.pass_context -def update_refs(ctx, force_update: bool): +@pass_config +def update_refs(config: MicroSALTConfig, force_update: bool): """Updates all references""" - ext_refs = Referencer(config=ctx.obj["config"], log=logger, force=force_update) + ext_refs = Referencer( + log=logger, + folders=config.folders, + threshold=config.threshold, + pubmlst=config.pubmlst, + pasteur=config.pasteur, + singularity=config.singularity, + containers=config.containers, + force=force_update, + ) ext_refs.update_refs() done() @resync.command() @click.option("--force-update", default=False, is_flag=True, help="Forces update") -@click.pass_context -def update_from_static(ctx, force_update: bool): +@pass_config +def update_from_static(config: MicroSALTConfig, force_update: bool): """Updates a specific organism""" - ext_refs = Referencer(config=ctx.obj["config"], force=force_update, log=logger) + ext_refs = Referencer( + log=logger, + folders=config.folders, + threshold=config.threshold, + pubmlst=config.pubmlst, + pasteur=config.pasteur, + singularity=config.singularity, + containers=config.containers, + force=force_update, + ) ext_refs.fetch_external() done() @@ -462,10 +587,19 @@ def update_from_static(ctx, force_update: bool): @click.argument("organism") @click.option("--force-update", default=False, is_flag=True, help="Forces update") @click.option("--external", is_flag=True, default=False, help="Updates from external sources") -@click.pass_context -def update_organism(ctx, external: bool, force_update: bool, organism: str): +@pass_config +def update_organism(config: MicroSALTConfig, external: bool, force_update: bool, organism: str): """Updates a specific organism""" - ext_refs = Referencer(config=ctx.obj["config"], log=logger, force=force_update) + ext_refs = Referencer( + log=logger, + folders=config.folders, + threshold=config.threshold, + pubmlst=config.pubmlst, + pasteur=config.pasteur, + singularity=config.singularity, + containers=config.containers, + force=force_update, + ) ext_refs.update_organism(external=external, organism=organism) done() @@ -478,9 +612,17 @@ def update_organism(ctx, external: bool, force_update: bool, organism: str): is_flag=True, help="Resolves sample without checking for pubMLST match", ) -@click.pass_context -def overwrite(ctx, sample_name, force): +@pass_config +def overwrite(config: MicroSALTConfig, sample_name, force): """Flags sample as resolved""" - ext_refs = Referencer(config=ctx.obj["config"], log=logger) + ext_refs = Referencer( + log=logger, + folders=config.folders, + threshold=config.threshold, + pubmlst=config.pubmlst, + pasteur=config.pasteur, + singularity=config.singularity, + containers=config.containers, + ) ext_refs.resync(type="overwrite", sample=sample_name, ignore=force) done() diff --git a/microSALT/config.py b/microSALT/config.py new file mode 100644 index 00000000..a7418cd8 --- /dev/null +++ b/microSALT/config.py @@ -0,0 +1,130 @@ +import json +from importlib.resources import files as resource_files +from pathlib import Path + +from pydantic import BaseModel + + +class SlurmHeader(BaseModel): + time: str + threads: str + qos: str + job_prefix: str + project: str + type: str + + +class Regex(BaseModel): + mail_recipient: str + file_pattern: str + verified_organisms: list[str] = [] + + +class Folders(BaseModel): + results: str + reports: str + seqdata: str + profiles: str + references: str + resistances: str + genomes: str + credentials: str + expec: str = "" # filled in after construction + + +class Database(BaseModel): + SQLALCHEMY_DATABASE_URI: str + SQLALCHEMY_TRACK_MODIFICATIONS: str = "False" + DEBUG: str = "True" + + +class Threshold(BaseModel): + mlst_id: float = 100 + mlst_novel_id: float = 99.5 + mlst_span: float = 90 + motif_id: float = 97 + motif_span: float = 90 + total_reads_warn: float = 75 + total_reads_fail: float = 70 + NTC_total_reads_warn: float = 10 + NTC_total_reads_fail: float = 20 + mapped_rate_warn: float = 50 + mapped_rate_fail: float = 30 + duplication_rate_warn: float = 20 + duplication_rate_fail: float = 80 + insert_size_warn: float = 140 + insert_size_fail: float = 100 + average_coverage_warn: float = 100 + average_coverage_fail: float = 10 + bp_10x_warn: float = 85 + bp_10x_fail: float = 75 + bp_30x_warn: float = 70 + bp_50x_warn: float = 50 + bp_100x_warn: float = 20 + + +class BIGSdbCredentials(BaseModel): + client_id: str = "" + client_secret: str = "" + + +class PubMLSTCredentials(BIGSdbCredentials): + pass + + +class PasteurCredentials(BIGSdbCredentials): + pass + + +class Singularity(BaseModel): + binary: str = "/usr/bin/singularity" + bind_paths: list[str] = [] + trimmomatic_adapters: str = "/opt/conda/share/trimmomatic/adapters/" + + +class Containers(BaseModel): + skesa: str = "" + blast: str = "" + bwa: str = "" + samtools: str = "" + picard: str = "" + trimmomatic: str = "" + quast: str = "" + + +class MicroSALTConfig(BaseModel): + slurm_header: SlurmHeader + regex: Regex + folders: Folders + database: Database + threshold: Threshold + pubmlst: PubMLSTCredentials = PubMLSTCredentials() + pasteur: PasteurCredentials = PasteurCredentials() + singularity: Singularity = Singularity() + containers: Containers = Containers() + # Runtime fields set by the CLI, not from the JSON file + dry: bool = False + config_path: str = "" + + +def _strip_comments(obj): + if isinstance(obj, dict): + return {k: _strip_comments(v) for k, v in obj.items() if k != "_comment"} + return obj + + +def load_config(path: str) -> MicroSALTConfig: + """Parse and validate a microSALT JSON config file. + + The expec reference path is derived from the package data and injected + after parsing, so it does not need to be present in the JSON file. + """ + with open(path) as f: + data = json.load(f) + data = _strip_comments(data) + config = MicroSALTConfig(**data) + config.folders.expec = str( + resource_files("microSALT").joinpath("unique_references", "ExPEC.fsa") + ) + config.config_path = str(Path(path).resolve()) + return config diff --git a/microSALT/exc/exceptions.py b/microSALT/exc/exceptions.py new file mode 100644 index 00000000..64823662 --- /dev/null +++ b/microSALT/exc/exceptions.py @@ -0,0 +1,17 @@ +class MicroSALTError(Exception): + """Base class for exceptions in MicroSALT.""" + + def __init__(self, message: str = ""): + super().__init__(message) + + +class RefUpdateLockError(MicroSALTError): + """Raised when an operation is attempted while a reference update is in progress.""" + + pass + + +class JobCreationError(MicroSALTError): + """Raised when there is an error creating a job.""" + + pass diff --git a/microSALT/server/app.py b/microSALT/server/app.py deleted file mode 100644 index 23f18c87..00000000 --- a/microSALT/server/app.py +++ /dev/null @@ -1,4 +0,0 @@ -from microSALT.server.views import app - -if __name__ == "__main__": - app.run() diff --git a/microSALT/server/templates/typing_page.html b/microSALT/server/templates/typing_page.html index e7318cd5..7bb8944e 100644 --- a/microSALT/server/templates/typing_page.html +++ b/microSALT/server/templates/typing_page.html @@ -10,7 +10,7 @@ MicroSALT Logo - {% if 'MW' in topsample.application_tag %} + {% if topsample.application_tag and 'MW' in topsample.application_tag %} Swedac Logo {% endif %}

@@ -282,7 +282,7 @@

översikt

Analys datum - {{sample.date_analysis.date()}} + {% if sample.date_analysis is not none %}{{sample.date_analysis.date()}}{% endif %} Organism diff --git a/microSALT/server/views.py b/microSALT/server/views.py index 09b44ee8..f903ea56 100644 --- a/microSALT/server/views.py +++ b/microSALT/server/views.py @@ -1,107 +1,122 @@ -import math import logging import subprocess - from datetime import date -from flask import Flask, render_template -from io import StringIO, BytesIO +from pathlib import Path +from typing import Optional -from sqlalchemy import * -from sqlalchemy.orm import sessionmaker -from sqlalchemy.sql import * -from sqlalchemy.sql.expression import case, func +from jinja2 import Environment, FileSystemLoader -from microSALT import preset_config, __version__ -from microSALT.store.db_manipulator import app +from microSALT import __version__ +from microSALT.config import Threshold from microSALT.store.orm_models import ( Collections, - Projects, Reports, Samples, - Seq_types, Versions, ) -engine = create_engine( - app.config["SQLALCHEMY_DATABASE_URI"], connect_args={"check_same_thread": False,'timeout':15} -) -Session = sessionmaker(bind=engine) -session = Session() -app.debug = 0 +from microSALT.store.database import get_session + # Removes server start messages log = logging.getLogger("werkzeug") log.setLevel(logging.CRITICAL) +TEMPLATE_FOLDER = Path(__file__).parent / "templates" + -@app.route("/") -def start_page(): - projects = session.query(Projects).all() - session.close() - return render_template("start_page.html", projects=projects) +def _make_url_for(project=None): + """Creates a url_for stub for use in static Jinja2 templates.""" + def url_for(endpoint, **kwargs): + if endpoint == "static": + return kwargs.get("filename", "") + elif endpoint == "start_page": + return "#" + elif endpoint == "project_page": + p = kwargs.get("project", project or "") + return "#project-{}".format(p) + elif endpoint == "typing_page": + p = kwargs.get("project", project or "") + og = kwargs.get("organism_group", "all") + return "#typing-{}-{}".format(p, og) + return "#" -@app.route("/microSALT/") -def reroute_page(): - projects = session.query(Projects).all() - session.close() - return render_template("start_page.html", projects=projects) + return url_for -@app.route("/microSALT/") -def project_page(project): +def render_template(template_folder, template_name, **context): + """Renders a template using Jinja2 directly to avoid Flask overhead""" + template_loader = FileSystemLoader(searchpath=str(template_folder)) + jinja_env = Environment(loader=template_loader) + template = jinja_env.get_template(template_name) + if "url_for" not in context: + context["url_for"] = _make_url_for() + return template.render(**context) + + +def project_page(project, template_folder: Path = TEMPLATE_FOLDER): + session = get_session() organism_groups = list() organism_groups.append("all") - distinct_organisms = ( - session.query(Samples).filter_by(CG_ID_project=project).distinct() - ) - session.close() + distinct_organisms = session.query(Samples).filter_by(CG_ID_project=project).distinct() for one_guy in distinct_organisms: if one_guy.organism not in organism_groups and one_guy.organism is not None: organism_groups.append(one_guy.organism) organism_groups.sort() return render_template( - "project_page.html", organisms=organism_groups, project=project + template_folder=template_folder, + template_name="project_page.html", + organisms=organism_groups, + project=project, + url_for=_make_url_for(project), ) -@app.route("/microSALT//qc") -def alignment_page(project): - sample_info = gen_reportdata(project) +def alignment_page(project, threshold: Threshold, template_folder: Path = TEMPLATE_FOLDER): + sample_info = gen_reportdata(project, threshold=threshold) return render_template( - "alignment_page.html", + template_folder=template_folder, + template_name="alignment_page.html", samples=sample_info["samples"], topsample=sample_info["single_sample"], date=date.today().isoformat(), version=sample_info["versions"], user=sample_info["user"], - threshold=preset_config["threshold"], + threshold=threshold, reports=sample_info["reports"], build=__version__, ) -@app.route("/microSALT//typing/") -def typing_page(project, organism_group): - sample_info = gen_reportdata(project, organism_group) +def render_alignment_page(project, threshold: Threshold, template_folder: Path = TEMPLATE_FOLDER): + return alignment_page(project, threshold=threshold, template_folder=template_folder) + + +def typing_page(project, organism_group, threshold: Threshold, verified_organisms: list, template_folder: Path = TEMPLATE_FOLDER): + sample_info = gen_reportdata(project, organism_group, threshold=threshold) return render_template( - "typing_page.html", + template_folder=template_folder, + template_name="typing_page.html", samples=sample_info["samples"], topsample=sample_info["single_sample"], date=date.today().isoformat(), version=sample_info["versions"], user=sample_info["user"], - threshold=preset_config["threshold"], - verified_organisms=preset_config["regex"]["verified_organisms"], + threshold=threshold, + verified_organisms=verified_organisms, reports=sample_info["reports"], build=__version__, ) -@app.route("/microSALT/STtracker/") -def STtracker_page(customer): - sample_info = gen_reportdata(pid="all", organism_group="all") +def render_typing_page(project, organism_group, threshold: Threshold, verified_organisms: list, template_folder: Path = TEMPLATE_FOLDER): + return typing_page(project, organism_group, threshold=threshold, verified_organisms=verified_organisms, template_folder=template_folder) + + +def STtracker_page(customer, threshold: Threshold, template_folder: Path = TEMPLATE_FOLDER): + sample_info = gen_reportdata(project_id="all", organism_group="all", threshold=threshold) final_samples = list() for s in sample_info["samples"]: if customer == "all" or s.projects.Customer_ID == customer: @@ -111,51 +126,48 @@ def STtracker_page(customer): final_samples = sorted(final_samples, key=lambda sample: (sample.CG_ID_sample)) return render_template( - "STtracker_page.html", date=date.today().isoformat(), internal=final_samples + template_folder=template_folder, + template_name="STtracker_page.html", + date=date.today().isoformat(), + internal=final_samples, ) -def gen_collectiondata(collect_id=[]): - """ Queries database using a set of samples""" - arglist = [] - samples = ( - session.query(Collections).filter(Collections.ID_collection == collect_id).all() - ) - for sample in samples: - arglist.append("Samples.CG_ID_sample=='{}'".format(sample.CG_ID_sample)) - sample_info = session.query(Samples).filter( - eval("or_({})".format(",".join(arglist))) - ) - sample_info = gen_add_info(sample_info) +def gen_collectiondata(collect_id=[], threshold: Optional[Threshold] = None): + """Queries database using a set of samples""" + session = get_session() + samples = session.query(Collections).filter(Collections.ID_collection == collect_id).all() + sample_ids = [s.CG_ID_sample for s in samples] + sample_info = session.query(Samples).filter(Samples.CG_ID_sample.in_(sample_ids)) + sample_info = gen_add_info(sample_info, threshold=threshold) return sample_info -def gen_reportdata(pid="all", organism_group="all"): - """ Queries database for all necessary information for the reports """ - if pid == "all" and organism_group == "all": +def gen_reportdata(project_id="all", organism_group="all", threshold: Optional[Threshold] = None): + """Queries database for all necessary information for the reports""" + session = get_session() + if project_id == "all" and organism_group == "all": sample_info = session.query(Samples) - elif pid == "all": + elif project_id == "all": sample_info = session.query(Samples).filter(Samples.organism == organism_group) elif organism_group == "all": - sample_info = session.query(Samples).filter(Samples.CG_ID_project == pid) + sample_info = session.query(Samples).filter(Samples.CG_ID_project == project_id) else: sample_info = session.query(Samples).filter( - Samples.CG_ID_project == pid, Samples.organism == organism_group + Samples.CG_ID_project == project_id, Samples.organism == organism_group ) - sample_info = gen_add_info(sample_info) + sample_info = gen_add_info(sample_info, threshold=threshold) - reports = session.query(Reports).filter(Reports.CG_ID_project == pid).all() - session.close() - sample_info["reports"] = reports = sorted( - reports, key=lambda x: x.version, reverse=True - ) + reports = session.query(Reports).filter(Reports.CG_ID_project == project_id).all() + sample_info["reports"] = reports = sorted(reports, key=lambda x: x.version, reverse=True) return sample_info -def gen_add_info(sample_info=dict()): - """ Enhances a sample info struct by adding ST_status, threshold info, versioning and sorting """ +def gen_add_info(sample_info=dict(), threshold: Optional[Threshold] = None): + """Enhances a sample info struct by adding ST_status, threshold info, versioning and sorting""" + session = get_session() # Set ST status output = dict() output["samples"] = list() @@ -172,11 +184,9 @@ def gen_add_info(sample_info=dict()): try: sample_info = sorted( sample_info, - key=lambda sample: int( - sample.CG_ID_sample.replace(sample.CG_ID_project, "")[1:] - ), + key=lambda sample: int(sample.CG_ID_sample.replace(sample.CG_ID_project, "")[1:]), ) - except ValueError as e: + except ValueError: pass for s in sample_info: @@ -202,24 +212,25 @@ def gen_add_info(sample_info=dict()): elif hasattr(s, "seq_types") and s.seq_types != [] or s.ST == -2: near_hits = 0 s.threshold = "Passed" - for seq_type in s.seq_types: - # Identify single deviating allele - if ( - seq_type.st_predictor - and seq_type.identity >= preset_config["threshold"]["mlst_novel_id"] - and preset_config["threshold"]["mlst_id"] > seq_type.identity - and 1 - abs(1 - seq_type.span) - >= (preset_config["threshold"]["mlst_span"] / 100.0) - ): - near_hits = near_hits + 1 - elif ( - seq_type.identity < preset_config["threshold"]["mlst_novel_id"] - or seq_type.span < (preset_config["threshold"]["mlst_span"] / 100.0) - ) and seq_type.st_predictor: - s.threshold = "Failed" + if threshold is not None: + for seq_type in s.seq_types: + # Identify single deviating allele + if ( + seq_type.st_predictor + and seq_type.identity >= threshold.mlst_novel_id + and threshold.mlst_id > seq_type.identity + and 1 - abs(1 - seq_type.span) + >= (threshold.mlst_span / 100.0) + ): + near_hits = near_hits + 1 + elif ( + seq_type.identity < threshold.mlst_novel_id + or seq_type.span < (threshold.mlst_span / 100.0) + ) and seq_type.st_predictor: + s.threshold = "Failed" if near_hits > 0 and s.threshold == "Passed": - s.ST_status = "Okänd ({} allele[r])".format(near_hits) + s.ST_status = f"Okänd ({near_hits} allele[r])" else: s.threshold = "Failed" @@ -232,22 +243,23 @@ def gen_add_info(sample_info=dict()): s.ST_status = "None" # Resistence filter - for r in s.resistances: - if ( - r.identity >= preset_config["threshold"]["motif_id"] - and r.span >= preset_config["threshold"]["motif_span"] / 100.0 - ): - r.threshold = "Passed" - else: - r.threshold = "Failed" - for v in s.expacs: - if ( - v.identity >= preset_config["threshold"]["motif_id"] - and v.span >= preset_config["threshold"]["motif_span"] / 100.0 - ): - v.threshold = "Passed" - else: - v.threshold = "Failed" + if threshold is not None: + for r in s.resistances: + if ( + r.identity >= threshold.motif_id + and r.span >= threshold.motif_span / 100.0 + ): + r.threshold = "Passed" + else: + r.threshold = "Failed" + for v in s.expacs: + if ( + v.identity >= threshold.motif_id + and v.span >= threshold.motif_span / 100.0 + ): + v.threshold = "Passed" + else: + v.threshold = "Failed" # Seq_type and resistance sorting s.seq_types = sorted(s.seq_types, key=lambda x: x.loci) @@ -257,7 +269,6 @@ def gen_add_info(sample_info=dict()): output["single_sample"] = s versions = session.query(Versions).all() - session.close() for version in versions: name = version.name[8:] output["versions"][name] = version.version diff --git a/microSALT/store/database.py b/microSALT/store/database.py new file mode 100644 index 00000000..45e84cf8 --- /dev/null +++ b/microSALT/store/database.py @@ -0,0 +1,44 @@ +from sqlalchemy import create_engine +from sqlalchemy.engine import Engine +from sqlalchemy.orm import Session, scoped_session, sessionmaker + +from microSALT.exc.exceptions import MicroSALTError +from microSALT.store.orm_models import Base + +session: scoped_session | None = None +engine: Engine | None = None + + +def initialize_database(db_uri: str) -> None: + """Initialize the SQLAlchemy engine and session for status db.""" + global engine, session + + engine = create_engine(db_uri, pool_pre_ping=True) + session_factory = sessionmaker(engine) + session = scoped_session(session_factory) + + +def get_session() -> Session: + """Get a SQLAlchemy session with a connection to status db.""" + if not session: + raise MicroSALTError("Database not initialised") + return session + + +def get_scoped_session_registry() -> scoped_session | None: + """Get the scoped session registry for status db.""" + return session + + +def get_engine() -> Engine: + """Get the SQLAlchemy engine with a connection to status db.""" + if not engine: + raise MicroSALTError("Database not initialised") + return engine + + +def create_tables() -> None: + """Create all ORM-defined tables that do not yet exist in the database.""" + if not engine: + raise MicroSALTError("Database not initialised") + Base.metadata.create_all(engine) diff --git a/microSALT/store/db_manipulator.py b/microSALT/store/db_manipulator.py index f8eb3d70..f0b8bee5 100644 --- a/microSALT/store/db_manipulator.py +++ b/microSALT/store/db_manipulator.py @@ -1,25 +1,25 @@ -""" Delivers and fetches data from the database - By: Isak Sylvin, @sylvinite""" +"""Delivers and fetches data from the database +By: Isak Sylvin, @sylvinite""" #!/usr/bin/env python import hashlib import sys import warnings - from collections import OrderedDict from datetime import datetime, timezone -from sqlalchemy import * -from sqlalchemy.orm import sessionmaker -from dateutil.parser import parse -# maintain the same connection per thread -from sqlalchemy.pool import SingletonThreadPool -from typing import Dict, List +from dateutil.parser import parse +from sqlalchemy import DateTime as SADateTime +from sqlalchemy import MetaData, Table, and_, desc, or_, text +from sqlalchemy import inspect as sa_inspect from microSALT import __version__ +from microSALT.config import Folders, Threshold +from microSALT.exc.exceptions import RefUpdateLockError +from microSALT.store.database import get_engine, get_session +from microSALT.store.models import ProfileTable from microSALT.store.orm_models import ( - app, Collections, Expacs, Projects, @@ -27,23 +27,44 @@ Resistances, Samples, Seq_types, + SystemLock, Versions, ) -from microSALT.store.models import Profiles, Novel + +# Maps string table names (as passed by callers) to ORM classes. +_ORM_TABLES = { + "Collections": Collections, + "Expacs": Expacs, + "Projects": Projects, + "Reports": Reports, + "Resistances": Resistances, + "Samples": Samples, + "Seq_types": Seq_types, + "Versions": Versions, +} + + +def _resolve_orm_table(tablename: str): + """Return the ORM class for *tablename*, raising KeyError on unknown names.""" + if tablename not in _ORM_TABLES: + raise KeyError(f"Unknown ORM table: {tablename!r}") + return _ORM_TABLES[tablename] class DB_Manipulator: - def __init__(self, config, log): - self.config = config + def __init__(self, log, folders: Folders, threshold: Threshold): + self.folders = folders + self.threshold = threshold self.logger = log - self.engine = create_engine( - app.config["SQLALCHEMY_DATABASE_URI"], poolclass=SingletonThreadPool - ) - Session = sessionmaker(bind=self.engine) - self.session = Session() - self.metadata = MetaData(self.engine) - self.profiles = Profiles(self.metadata, self.config, self.logger).tables - self.novel = Novel(self.metadata, self.config, self.logger).tables + self.session = get_session() + self.engine = get_engine() + self.metadata = MetaData() + self.profiles = ProfileTable( + "profile_", self.metadata, self.folders.profiles, self.logger + ).tables + self.novel = ProfileTable( + "novel_", self.metadata, self.folders.profiles, self.logger + ).tables # Turns off pymysql deprecation warnings until they can update their code with warnings.catch_warnings(): warnings.simplefilter("ignore") @@ -51,302 +72,390 @@ def __init__(self, config, log): def create_tables(self): """Creates all tables individually. A bit more control than usual""" - if not self.engine.dialect.has_table(self.engine, "projects"): + inspector = sa_inspect(self.engine) + if not inspector.has_table("projects"): Projects.__table__.create(self.engine) self.logger.info("Created projects table") - if not self.engine.dialect.has_table(self.engine, "samples"): + if not inspector.has_table("samples"): Samples.__table__.create(self.engine) self.logger.info("Created samples table") - if not self.engine.dialect.has_table(self.engine, "versions"): + if not inspector.has_table("versions"): Versions.__table__.create(self.engine) self.logger.info("Created versions table") - if not self.engine.dialect.has_table(self.engine, "seq_types"): + if not inspector.has_table("seq_types"): Seq_types.__table__.create(self.engine) self.logger.info("Created sequencing types table") - if not self.engine.dialect.has_table(self.engine, "resistances"): + if not inspector.has_table("resistances"): Resistances.__table__.create(self.engine) self.logger.info("Created resistance table") - if not self.engine.dialect.has_table(self.engine, "reports"): + if not inspector.has_table("reports"): Reports.__table__.create(self.engine) self.logger.info("Created reports table") - if not self.engine.dialect.has_table(self.engine, "collections"): + if not inspector.has_table("collections"): Collections.__table__.create(self.engine) self.logger.info("Created collections table") - if not self.engine.dialect.has_table(self.engine, "expacs"): + if not inspector.has_table("expacs"): Expacs.__table__.create(self.engine) self.logger.info("Created ExPEC table") + if not inspector.has_table("system_locks"): + SystemLock.__table__.create(self.engine) + self.logger.info("Created system_locks table") for k, v in self.profiles.items(): - if not self.engine.dialect.has_table(self.engine, "profile_{}".format(k)): - self.profiles[k].create() - self.init_profiletable(k, v) - self.add_rec( - {"name": "profile_{}".format(k), "version": "0"}, - "Versions", - force=True, - ) - self.logger.info("Profile table profile_{} initialized".format(k)) + if not inspector.has_table(f"profile_{k}"): + self.profiles[k].create(self.engine) + self.populate_profiletable(k, v) + self.add_to_session(self.add_version(name=f"profile_{k}", version="0")) + self.commit_session() + self.logger.info(f"Profile table profile_{k} created and populated") for k, v in self.novel.items(): - if not self.engine.dialect.has_table(self.engine, "novel_{}".format(k)): - self.novel[k].create() - self.add_rec( - {"name": "novel_{}".format(k), "version": "0"}, - "Versions", - force=True, - ) - self.logger.info("Profile table novel_{} initialized".format(k)) + if not inspector.has_table(f"novel_{k}"): + self.novel[k].create(self.engine) + self.add_to_session(self.add_version(name=f"novel_{k}", version="0")) + self.commit_session() + self.logger.info(f"Profile table novel_{k} initialized") - def add_rec(self, data_dict: Dict[str, str], tablename: str, force=False): - """Adds a record to the specified table through a dict with columns as keys.""" - pk_list = list() - # Non-orm - if not isinstance(tablename, str): - # check for existence - table = tablename - pk_list = table.primary_key.columns.keys() - args = list() - for pk in pk_list: - args.append("table.c.{}=={}".format(pk, data_dict[pk])) - args = "or_(" + ",".join(args) + ")" - exist = self.session.query(table).filter(eval(args)).all() - # Add record - if len(exist) == 0: - data = table.insert() - # Loads any dates as datetime objects - for k, v in data_dict.items(): - if isinstance(v, str): - try: - parse(v, fuzzy=False) - data_dict[k] = datetime.strptime(v, "%Y-%m-%d %H:%M:%S") - except ValueError as ve: - if len(ve.args) > 0 and ve.args[0].startswith( - "unconverted data remains: " - ): - data_dict[k] = datetime.strptime(v, "%Y-%m-%d %H:%M:%S.%f") - else: - pass - data.execute(data_dict) - self.logger.info("Added entry to table {}".format(tablename.fullname)) - # ORM - else: - try: - table = eval(tablename) - # Check for existing entry - pk_list = table.__table__.primary_key.columns.keys() - except Exception as e: - self.logger.error( - "Attempted to access table {} which has not been created".format(tablename) - ) - pk_values = list() - for item in pk_list: - pk_values.append(data_dict[item]) - existing = self.session.query(table).get(pk_values) - # Add record - if not existing or force: - newobj = table() - # Loads any dates as datetime objects - for k, v in data_dict.items(): - if isinstance(v, str): - try: - parse(v, fuzzy=False) - data_dict[k] = datetime.strptime(v, "%Y-%m-%d %H:%M:%S") - except ValueError as ve: - if len(ve.args) > 0 and ve.args[0].startswith( - "unconverted data remains: " - ): - data_dict[k] = datetime.strptime(v, "%Y-%m-%d %H:%M:%S.%f") - else: - pass - for k, v in data_dict.items(): - setattr(newobj, k, v) - self.session.add(newobj) - self.session.commit() - else: - self.logger.warning( - "Record [{}]=[{}] in table {} already exists".format( - ", ".join(pk_list), ", ".join(pk_values), tablename - ) - ) + def acquire_ref_lock(self): + """Acquire the reference-update exclusive lock. - def upd_rec(self, req_dict: Dict[str, str], tablename: str, upd_dict: Dict[str, str]): - """Updates a record to the specified table through a dict with columns as keys.""" - table = eval(tablename) - self.logger.debug(f"Updating table {tablename} with {upd_dict}") - argy = list() - for k, v in req_dict.items(): - if v != None: - argy.append(".filter(table.{}=='{}')".format(k, v)) - filter = "".join(argy) - megastring = "self.session.query(table){}".format(filter) - if len(eval(megastring + ".all()")) > 1: - self.logger.error("More than 1 record found when orm updating. Exited.") - sys.exit() - else: - eval(megastring + ".update(upd_dict)") - self.session.commit() - self.logger.debug(f"Updated table {tablename} with {upd_dict} for {req_dict}") - - def purge_rec(self, name: str, type: str): - """Removes seq_data, resistances, sample(s) and possibly project""" - entries = list() - if type == "Projects": - entries.append( - self.session.query(Expacs) - .filter(Expacs.CG_ID_sample.like("{}%".format(name))) - .all() - ) - entries.append( - self.session.query(Seq_types) - .filter(Seq_types.CG_ID_sample.like("{}%".format(name))) - .all() - ) - entries.append( - self.session.query(Resistances) - .filter(Resistances.CG_ID_sample.like("{}%".format(name))) - .all() - ) - entries.append( - self.session.query(Samples) - .filter(Samples.CG_ID_sample.like("{}%".format(name))) - .all() - ) - # entries.append(self.session.query(Projects).filter(Projects.CG_ID_project==name).all()) - elif type == "Samples": - entries.append(self.session.query(Expacs).filter(Expacs.CG_ID_sample == name).all()) - entries.append( - self.session.query(Seq_types).filter(Seq_types.CG_ID_sample == name).all() - ) - entries.append( - self.session.query(Resistances).filter(Resistances.CG_ID_sample == name).all() - ) - entries.append(self.session.query(Samples).filter(Samples.CG_ID_sample == name).all()) - elif type == "Collections": - entries.append( - self.session.query(Collections).filter(Collections.ID_collection == name).all() + Raises RefUpdateLockError if the lock is already held by another process. + """ + existing = self.session.query(SystemLock).filter_by(lock_name="ref_update").scalar() + if existing: + raise RefUpdateLockError( + "A reference update is already in progress (lock acquired at {}). " + "Please try again later.".format(existing.acquired_at) ) - else: - self.logger.error( - "Incorrect type {} specified for removal of {}. Check code".format(type, name) + self.session.add(SystemLock(lock_name="ref_update", acquired_at=datetime.now(timezone.utc))) + self.session.commit() + self.logger.info("Reference update lock acquired") + + def release_ref_lock(self): + """Release the reference-update exclusive lock.""" + self.session.query(SystemLock).filter_by(lock_name="ref_update").delete() + self.session.commit() + self.logger.info("Reference update lock released") + + def check_ref_lock(self): + """Raise RefUpdateLockError if a reference update is currently in progress.""" + lock = self.session.query(SystemLock).filter_by(lock_name="ref_update").scalar() + if lock: + raise RefUpdateLockError( + "The reference database is currently being updated (started at {}). " + "Please try again later.".format(lock.acquired_at) ) + + def add_rec(self, data_dict: dict, tablename) -> None: + """Adds a record to a non-ORM (ProfileTable) table via a raw Table object.""" + table = tablename + pk_list = table.primary_key.columns.keys() + filter_clauses = [table.c[pk] == data_dict[pk] for pk in pk_list] + exist = self.session.query(table).filter(or_(*filter_clauses)).all() + if len(exist) == 0: + data = table.insert() + for k, v in data_dict.items(): + if isinstance(v, str): + try: + parse(v, fuzzy=False) + data_dict[k] = datetime.strptime(v, "%Y-%m-%d %H:%M:%S") + except ValueError as ve: + if len(ve.args) > 0 and ve.args[0].startswith("unconverted data remains: "): + data_dict[k] = datetime.strptime(v, "%Y-%m-%d %H:%M:%S.%f") + else: + pass + self.session.execute(data, data_dict) + self.session.commit() + self.logger.info(f"Added entry to table {tablename.fullname}") + + # ------------------------------------------------------------------ + # Per-model factory methods — construct and return an ORM object. + # Callers are responsible for add_to_session() and commit_session(). + # ------------------------------------------------------------------ + + def add_sample(self, **kwargs) -> Samples: + return Samples(**kwargs) + + def add_project(self, **kwargs) -> Projects: + return Projects(**kwargs) + + def add_seq_type(self, **kwargs) -> Seq_types: + return Seq_types(**kwargs) + + def add_resistance(self, **kwargs) -> Resistances: + return Resistances(**kwargs) + + def add_expac(self, **kwargs) -> Expacs: + return Expacs(**kwargs) + + def add_report(self, **kwargs) -> Reports: + return Reports(**kwargs) + + def add_collection(self, **kwargs) -> Collections: + return Collections(**kwargs) + + def add_version(self, **kwargs) -> Versions: + return Versions(**kwargs) + + def add_to_session(self, obj) -> None: + """Coerce string DateTime fields, then stage the object. + + If an object with the same primary key already exists in the database + the insert is silently skipped (same semantics as the previous + _add_orm_record behaviour). + """ + for col in obj.__table__.columns: + if isinstance(col.type, SADateTime): + val = getattr(obj, col.name) + if isinstance(val, str): + try: + setattr(obj, col.name, datetime.strptime(val, "%Y-%m-%d %H:%M:%S")) + except ValueError: + setattr(obj, col.name, datetime.strptime(val, "%Y-%m-%d %H:%M:%S.%f")) + pk_cols = list(obj.__table__.primary_key.columns.keys()) + pk_vals = [getattr(obj, c) for c in pk_cols] + if None not in pk_vals: + existing = self.session.get(type(obj), pk_vals if len(pk_vals) > 1 else pk_vals[0]) + if existing is not None: + return + self.session.add(obj) + + def commit_session(self) -> None: + self.session.commit() + + # ------------------------------------------------------------------ + # Per-model update methods + # ------------------------------------------------------------------ + + def update_sample(self, req_dict: dict, upd_dict: dict) -> None: + """Update a Samples row. Cascades CG_ID_sample renames to child tables.""" + filter_clauses = [getattr(Samples, k) == v for k, v in req_dict.items() if v is not None] + query = self.session.query(Samples).filter(and_(*filter_clauses)) + if len(query.all()) > 1: + self.logger.error("More than 1 Samples record found when updating. Exited.") + sys.exit() + if "CG_ID_sample" in upd_dict: + old_id = req_dict.get("CG_ID_sample") + new_id = upd_dict["CG_ID_sample"] + if old_id and old_id != new_id: + for child_table in (Seq_types, Resistances, Expacs, Collections): + self.session.query(child_table).filter( + child_table.CG_ID_sample == old_id + ).update({"CG_ID_sample": new_id}) + query.update(upd_dict) + self.session.commit() + self.logger.debug(f"Updated Samples for {req_dict} with {upd_dict}") + + def update_project(self, req_dict: dict, upd_dict: dict) -> None: + """Update a Projects row.""" + filter_clauses = [getattr(Projects, k) == v for k, v in req_dict.items() if v is not None] + query = self.session.query(Projects).filter(and_(*filter_clauses)) + if len(query.all()) > 1: + self.logger.error("More than 1 Projects record found when updating. Exited.") sys.exit() - for entry in entries: - for instance in entry: - self.session.delete(instance) - self.session.commit() - self.logger.info("Removed information for {}".format(name)) + query.update(upd_dict) + self.session.commit() + self.logger.debug(f"Updated Projects for {req_dict} with {upd_dict}") - def query_rec(self, tablename: str, filters: Dict[str, str]): + def update_version(self, req_dict: dict, upd_dict: dict) -> None: + """Update a Versions row.""" + filter_clauses = [getattr(Versions, k) == v for k, v in req_dict.items() if v is not None] + self.session.query(Versions).filter(and_(*filter_clauses)).update(upd_dict) + self.session.commit() + self.logger.debug(f"Updated Versions for {req_dict} with {upd_dict}") + + # ------------------------------------------------------------------ + # Per-model delete methods + # ------------------------------------------------------------------ + + def delete_sample(self, cg_id: str) -> None: + """Delete a sample and all its child rows (seq_types, resistances, expacs).""" + for obj in self.session.query(Expacs).filter(Expacs.CG_ID_sample == cg_id).all(): + self.session.delete(obj) + for obj in self.session.query(Seq_types).filter(Seq_types.CG_ID_sample == cg_id).all(): + self.session.delete(obj) + for obj in self.session.query(Resistances).filter(Resistances.CG_ID_sample == cg_id).all(): + self.session.delete(obj) + for obj in self.session.query(Samples).filter(Samples.CG_ID_sample == cg_id).all(): + self.session.delete(obj) + self.session.commit() + self.logger.info(f"Removed sample {cg_id} and its child rows") + + def delete_sample_results(self, cg_id: str) -> None: + """Delete only the analysis result rows for a sample (seq_types, resistances, expacs) + without removing the Samples row itself.""" + for obj in self.session.query(Expacs).filter(Expacs.CG_ID_sample == cg_id).all(): + self.session.delete(obj) + for obj in self.session.query(Seq_types).filter(Seq_types.CG_ID_sample == cg_id).all(): + self.session.delete(obj) + for obj in self.session.query(Resistances).filter(Resistances.CG_ID_sample == cg_id).all(): + self.session.delete(obj) + self.session.commit() + self.logger.info(f"Cleared analysis results for sample {cg_id}") + + def delete_project(self, name: str) -> None: + """Delete all samples (and their child rows) belonging to a project.""" + for obj in self.session.query(Expacs).filter(Expacs.CG_ID_sample.like(f"{name}%")).all(): + self.session.delete(obj) + for obj in ( + self.session.query(Seq_types).filter(Seq_types.CG_ID_sample.like(f"{name}%")).all() + ): + self.session.delete(obj) + for obj in ( + self.session.query(Resistances).filter(Resistances.CG_ID_sample.like(f"{name}%")).all() + ): + self.session.delete(obj) + for obj in self.session.query(Samples).filter(Samples.CG_ID_sample.like(f"{name}%")).all(): + self.session.delete(obj) + self.session.commit() + self.logger.info(f"Removed all samples for project {name}") + + def delete_collection(self, name: str) -> None: + """Delete all entries for the given collection ID.""" + for obj in self.session.query(Collections).filter(Collections.ID_collection == name).all(): + self.session.delete(obj) + self.session.commit() + self.logger.info(f"Removed collection {name}") + + def read_records(self, tablename: str, filters: dict[str, str]): """Fetches records table, using a primary-key dict with columns as keys. Non-PK are ignored""" # Non-orm if not isinstance(tablename, str): - # check for existence table = tablename - pk_list = table.primary_key.columns.keys() - args = list() - for k, v in filters.items(): - args.append("table.c.{}=={}".format(k, v)) - args = "or_(" + ",".join(args) + ")" - exist = self.session.query(table).filter(eval(args)).all() - return exist + filter_clauses = [table.c[k] == v for k, v in filters.items()] + return self.session.query(table).filter(or_(*filter_clauses)).all() # ORM else: - table = eval(tablename) - args = list() - for k, v in filters.items(): - if v != None: - args.append("table.{}=='{}'".format(k, v)) - filter = " and ".join(args) - entries = self.session.query(table).filter(eval(filter)).all() - return entries - - def top_index(self, table_str: str, filters: Dict[str, str], column: str): + table = _resolve_orm_table(tablename) + filter_clauses = [getattr(table, k) == v for k, v in filters.items() if v is not None] + return self.session.query(table).filter(and_(*filter_clauses)).all() + + def read_top_index(self, table_str: str, filters: dict[str, str], column: str): """Fetches the top index from column of table, by applying a dict with columns as keys.""" - table = eval(table_str) - args = list() - for k, v in filters.items(): - if v != None: - args.append("table.{}=='{}'".format(k, v)) - filter = " and ".join(args) + table = _resolve_orm_table(table_str) + filter_clauses = [getattr(table, k) == v for k, v in filters.items() if v is not None] entry = ( self.session.query(table) - .filter(eval(filter)) - .order_by(desc(eval("{}.{}".format(table_str, column)))) + .filter(and_(*filter_clauses)) + .order_by(desc(getattr(table, column))) .limit(1) .all() ) if entry == []: return int(-1) else: - return eval("entry[0].{}".format(column)) + return getattr(entry[0], column) def reload_profiletable(self, organism: str): - """Drop the named non-orm table, then load it with fresh data""" - table = self.profiles[organism] + """Drop the named profile table, rebuild schema from disk, and reload with fresh data. + + The Python Table object is rebuilt from the current file on disk before the DB + table is recreated, so schema changes (new or renamed loci columns) are picked up. + """ self.logger.debug(f"Reloading profile table for {organism}") - self.profiles[organism].drop() + self.profiles[organism].drop(self.engine) self.logger.debug(f"Dropped profile table for {organism}") - self.profiles[organism].create() + # Rebuild the Table object from the file currently on disk (schema may have changed). + fresh_metadata = MetaData() + fresh = ProfileTable("profile_", fresh_metadata, self.folders.profiles, self.logger).tables + if organism in fresh: + self.profiles[organism] = fresh[organism] + self.profiles[organism].create(self.engine) self.logger.debug(f"Recreated profile table for {organism}") - self.init_profiletable(organism, table) - self.logger.debug(f"Initialized profile table for {organism}") - - def init_profiletable(self, filename: str, table): - """Creates profile tables by looping, since a lot of infiles exist""" - data = table.insert() - linedict = dict.fromkeys(table.c.keys()) - file_path = f"{self.config['folders']['profiles']}/{filename}" + self.populate_profiletable(organism, self.profiles[organism]) + self.logger.debug(f"Populated profile table for {organism}") + + def refresh_profiletable(self, organism: str): + """Reload profile table content without dropping the table when possible. + + Reads the downloaded CSV header and compares it against the current + table's columns (first 8). If the schema is unchanged, the table is + truncated and reloaded in place. If the loci scheme has changed (new + or renamed columns) the method falls back to a full drop/recreate via + reload_profiletable() so the schema stays in sync with the CSV. + """ + table = self.profiles[organism] + file_path = f"{self.folders.profiles}/{organism}" + + with open(file_path, "r") as fh: + csv_cols = fh.readline().rstrip().split("\t")[:8] + + current_cols = list(table.c.keys()) + + if csv_cols == current_cols: + self.logger.info( + f"Schema unchanged for {organism}, truncating and reloading profile table" + ) + self.session.execute(table.delete()) + self.session.commit() + self.populate_profiletable(organism, table) + else: + self.logger.info( + f"Schema changed for {organism} ({current_cols} -> {csv_cols}), " + f"dropping and recreating profile table" + ) + self.reload_profiletable(organism) + + def populate_profiletable(self, filename: str, table) -> None: + """Bulk-inserts all data rows from a profile file into an already-created *table*.""" + file_path = f"{self.folders.profiles}/{filename}" self.logger.debug(f"Opening profile file: {file_path}") + keys = list(table.c.keys()) + rows = [] with open(file_path, "r") as fh: - # Skips header - head = fh.readline() - head = head.rstrip().split("\t") - self.logger.debug(f"Header columns: {head}") - line_num = 0 - for line in fh: - line_num += 1 - line = line.rstrip().split("\t") - self.logger.debug(f"Processing line {line_num}: {line}") - index = 0 - while index < len(line): - linedict[head[index]] = line[index] - index = index + 1 - self.logger.debug(f"Linedict before insert: {linedict}") - try: - data.execute(linedict) - self.logger.debug(f"Inserted line {line_num} into table") - except Exception as e: - self.logger.error(f"Failed to insert line {line_num}: {e}") - self.logger.debug(f"Initialized profile table for {filename} with {line_num} entries") - - def get_columns(self, tablename: str): + head = fh.readline().rstrip().split("\t") + for raw_line in fh: + values = raw_line.rstrip().split("\t") + row = {col: None for col in keys} + for i, val in enumerate(values[: len(keys)]): + row[head[i]] = val + rows.append(row) + if not rows: + self.logger.warning(f"No data rows found in profile file {filename}") + return + try: + self.session.execute(table.insert(), rows) + self.session.commit() + self.logger.debug(f"Inserted {len(rows)} rows into profile table for {filename}") + except Exception as e: + self.session.rollback() + self.logger.error(f"Failed to bulk-insert profile data for {filename}: {e}") + + def read_columns(self, tablename: str): """Returns all records for a given ORM table""" - table = eval(tablename) + table = _resolve_orm_table(tablename) return dict.fromkeys(table.__table__.columns.keys()) - def exists(self, table, item: Dict[str, str]): - """Takes a k-v pair and checks for the entrys existence in the given table""" - filterstring = "" - for k, v in item.items(): - filterstring += "{}.{}=='{}',".format(table, k, v) - filterstring = filterstring[:-1] - table = eval(table) - entry = self.session.query(table).filter(eval(filterstring)).scalar() - if entry is None: - return False - else: - return True + def get_projects_by_cg_id_project(self, cg_id_project_name: str) -> Projects | None: + """Fetch a Projects record by CG_ID_project.""" + return ( + self.session.query(Projects) + .filter(Projects.CG_ID_project == cg_id_project_name) + .scalar() + ) + + def get_collection_by_id(self, collection_id: str) -> Collections | None: + return ( + self.session.query(Collections) + .filter(Collections.ID_collection == collection_id) + .scalar() + ) - def get_version(self, name: str): + def get_sample_by_cg_id_sample(self, cg_id_sample: str) -> Samples | None: + return self.session.query(Samples).filter(Samples.CG_ID_sample == cg_id_sample).scalar() + + def read_version(self, name: str): """Gets the version from a given name. Should be generalized to return any value for any input""" - version = self.session.query(Versions).filter(Versions.name == name).scalar() + version: Versions | None = ( + self.session.query(Versions).filter(Versions.name == name).scalar() + ) if version is None: return "0" else: return version.version - def get_report(self, name: str): + def read_report(self, name: str) -> Reports | None: # Sort based on version - prev_report = [] + prev_report: Reports | None = None prev_reports = ( self.session.query(Reports) .filter(Reports.CG_ID_project == name) @@ -359,7 +468,7 @@ def get_report(self, name: str): def set_report(self, name: str): # Generate string - totalstring = list() + totalstring: list[str] = [] dt = datetime.now() default_method = "Not in LIMS" samples = ( @@ -401,31 +510,29 @@ def set_report(self, name: str): totalstring = "".join(totalstring).encode() hashstring = hashlib.md5(totalstring).hexdigest() - prev_report = self.get_report(name) - # Compare - if prev_report: + if prev_report := self.read_report(name): if "steps_aggregate" in dir(prev_report) and prev_report.steps_aggregate != hashstring: - self.add_rec( - { - "CG_ID_project": name, - "steps_aggregate": hashstring, - "date": dt, - "version": prev_report.version + 1, - }, - "Reports", + self.add_to_session( + self.add_report( + CG_ID_project=name, + steps_aggregate=hashstring, + date=dt, + version=prev_report.version + 1, + ) ) + self.commit_session() else: - self.add_rec( - { - "CG_ID_project": name, - "steps_aggregate": hashstring, - "date": dt, - "version": 1, - }, - "Reports", + self.add_to_session( + self.add_report( + CG_ID_project=name, + steps_aggregate=hashstring, + date=dt, + version=1, + ) ) + self.commit_session() - def sync_novel(self, overwrite=False, sample=""): + def set_novel_st(self, overwrite=False, sample=""): """Looks at each novel table. See if any record has a profile match in the profile table. Updates these based on parameters""" prequery = self.session.query(Samples) @@ -436,14 +543,12 @@ def sync_novel(self, overwrite=False, sample=""): profile_list = self.session.query(self.profiles[org]).all() # Filter for novel in novel_list: - args = list() - for key in org_keys: - if key != "ST" and key != "clonal_complex" and key != "species": - args.append( - "self.profiles[org].c.{}=={}".format(key, eval("novel.{}".format(key))) - ) - args = "and_(" + ",".join(args) + ")" - exist = self.session.query(self.profiles[org]).filter(eval(args)).all() + filter_clauses = [ + self.profiles[org].c[key] == getattr(novel, key) + for key in org_keys + if key not in ("ST", "clonal_complex", "species") + ] + exist = self.session.query(self.profiles[org]).filter(and_(*filter_clauses)).all() if exist: exist = exist[0] @@ -468,42 +573,34 @@ def sync_novel(self, overwrite=False, sample=""): # review if entry.pubmlst_ST == -1 and not overwrite: self.logger.info( - "Update: Sample {} of organism {}; Internal ST {} is now linked to {} '{}'".format( - entry.CG_ID_sample, org, novel.ST, exist.ST, exist - ) + f"Update: Sample {entry.CG_ID_sample} of organism {org}; Internal ST {novel.ST} is now linked to {exist.ST} '{exist}'" ) - self.upd_rec( + self.update_sample( {"CG_ID_sample": entry.CG_ID_sample}, - "Samples", {"pubmlst_ST": exist.ST}, ) # overwrite elif overwrite: self.logger.info( - "Replacement: Sample {} of organism {}; Internal ST {} is now {} '{}'".format( - entry.CG_ID_sample, org, novel.ST, exist.ST, exist - ) + f"Replacement: Sample {entry.CG_ID_sample} of organism {org}; Internal ST {novel.ST} is now {exist.ST} '{exist}'" ) - self.upd_rec( + self.update_sample( {"CG_ID_sample": entry.CG_ID_sample}, - "Samples", {"ST": exist.ST, "pubmlst_ST": exist.ST}, ) - def rm_novel(self, sample=""): + def set_novel_ignored(self, sample=""): """Flags a sample as pubMLST resolved by merit of ignoring it""" query = self.session.query(Samples).filter(Samples.CG_ID_sample == sample).all() if len(query) > 0: self.logger.info( - "Ignore: Sample {} from organism {} with ST {}; is now flagged as resolved.".format( - query[0].CG_ID_sample, query[0].organism, query[0].ST - ) + f"Ignore: Sample {query[0].CG_ID_sample} from organism {query[0].organism} with ST {query[0].ST}; is now flagged as resolved." ) - self.upd_rec({"CG_ID_sample": query[0].CG_ID_sample}, "Samples", {"pubmlst_ST": 0}) + self.update_sample({"CG_ID_sample": query[0].CG_ID_sample}, {"pubmlst_ST": 0}) else: - self.logger.error("Sample {} not found in database. Verify name".format(sample)) + self.logger.error(f"Sample {sample} not found in database. Verify name") - def list_unresolved(self): + def read_unresolved(self): """Lists all novel samples that current havent been flagged as resolved""" # ST currently not updated at all novelbkt = OrderedDict() @@ -513,9 +610,9 @@ def list_unresolved(self): .all() ) for entry in prequery: - if not entry.organism in novelbkt: + if entry.organism not in novelbkt: novelbkt[entry.organism] = dict() - if not entry.ST in novelbkt[entry.organism]: + if entry.ST not in novelbkt[entry.organism]: novelbkt[entry.organism][entry.ST] = list() novelbkt[entry.organism][entry.ST].append(entry.CG_ID_sample) novelbkt = OrderedDict(sorted(novelbkt.items(), key=lambda t: t[0])) @@ -528,9 +625,9 @@ def list_unresolved(self): .all() ) for entry in postquery: - if not entry.organism in novelbkt2: + if entry.organism not in novelbkt2: novelbkt2[entry.organism] = dict() - if not entry.ST in novelbkt2[entry.organism]: + if entry.ST not in novelbkt2[entry.organism]: novelbkt2[entry.organism][entry.ST] = list() novelbkt2[entry.organism][entry.ST].append(entry.CG_ID_sample) @@ -542,9 +639,9 @@ def list_unresolved(self): .all() ) for entry in naquery: - if not entry.ST in novelbkt3: + if entry.ST not in novelbkt3: novelbkt3[entry.ST] = dict() - if not entry.organism in novelbkt3[entry.ST]: + if entry.organism not in novelbkt3[entry.ST]: novelbkt3[entry.ST][entry.organism] = list() novelbkt3[entry.ST][entry.organism].append(entry.CG_ID_sample) novelbkt3 = OrderedDict(sorted(novelbkt3.items(), key=lambda t: t[0], reverse=True)) @@ -558,17 +655,11 @@ def list_unresolved(self): print("\n####Unresolved samples and their respective error flags:####\n") for k, v in novelbkt3.items(): - print("\n##Code {} - {}##".format(k, codetrans[k])) + print(f"\n##Code {k} - {codetrans[k]}##") for x, y in v.items(): if x is not None: x = x.replace("_", " ").capitalize() - print( - "{} ({} samples):\n{}".format( - x, - len(y), - sorted(y), - ) - ) + print(f"{x} ({len(y)} samples):\n{sorted(y)}") if len(novelbkt3) == 0: print("None!") @@ -576,9 +667,9 @@ def list_unresolved(self): for k, v in novelbkt2.items(): if k is not None: k = k.replace("_", " ").capitalize() - print("Organism {} ({}):".format(k, len(v))) + print(f"Organism {k} ({len(v)}):") for x, y in v.items(): - print("{}:{} ({} ST)".format(x, sorted(y), len(y))) + print(f"{x}:{sorted(y)} ({len(y)} ST)") if len(novelbkt2) == 0: print("None!") @@ -586,13 +677,13 @@ def list_unresolved(self): for k, v in novelbkt.items(): if k is not None: k = k.replace("_", " ").capitalize() - print("Organism {} ({}):".format(k, len(v))) + print(f"Organism {k} ({len(v)}):") for x, y in v.items(): - print("{}:{} ({} novel ST)".format(x, sorted(y), len(y))) + print(f"{x}:{sorted(y)} ({len(y)} novel ST)") if len(novelbkt) == 0: print("None!") - def setPredictor(self, cg_sid: str, pks=dict()): + def set_predictor(self, cg_sid: str, pks=dict()): """Helper function. Flags a set of seq_types as part of the final prediction. Uses optional pks[PK_NAME] = VALUE dictionary to distinguish in scenarios where an allele number has multiple hits """ @@ -605,241 +696,186 @@ def setPredictor(self, cg_sid: str, pks=dict()): sample.update({Seq_types.st_predictor: None}) # Set subset for loci, columns in pks.items(): - arglist = list() - for key, val in columns.items(): - arglist.append("Seq_types.{}=='{}'".format(key, val)) - args = "and_(" + ", ".join(arglist) + ")" - sample.filter(eval(args)).update({Seq_types.st_predictor: 1}) + filter_clauses = [getattr(Seq_types, key) == val for key, val in columns.items()] + sample.filter(and_(*filter_clauses)).update({Seq_types.st_predictor: 1}) self.session.commit() - def alleles2st(self, cg_sid: str): - """Takes a CG_ID_sample and predicts the correct ST""" - threshold = True - organism = ( + def _build_allele_filter_clauses(self, alleles: dict, table) -> list: + """Builds SQLAlchemy filter clauses for allele matching against a profile or novel table.""" + filter_clauses = [] + for key, val in alleles.items(): + col = table.c[key] + if len(val) > 1: + filter_clauses.append(or_(*[col == num for num in val])) + else: + filter_clauses.append(col == val[0]) + return filter_clauses + + def _query_st_profiles(self, alleles: dict, table) -> list: + """Queries a profile or novel table with allele filter clauses and returns all matching rows.""" + filter_clauses = self._build_allele_filter_clauses(alleles, table) + return self.session.query(table).filter(and_(*filter_clauses)).all() + + def _next_novel_st(self, organism: str) -> int: + """Returns the next available negative novel ST: one below the current minimum, at most -10.""" + st = -9 + for entry in self.session.query(self.novel[organism]).all(): + if entry.ST < st: + st = entry.ST + return st - 1 + + def _create_novel_st_entry(self, cg_sid: str, organism: str) -> int: + """Creates a new novel ST row built from the sample's best alleles. Returns the new ST number.""" + st = self._next_novel_st(organism) + best_alleles = self.read_best_alleles(cg_sid) + new_entry: dict = {allele: columns["allele"] for allele, columns in best_alleles.items()} + new_entry["ST"] = st + self.add_rec(new_entry, self.novel[organism]) + return st + + def _allele_hit_score(self, allele) -> tuple: + """Returns a comparable score tuple (span*identity, -evalue, contig_coverage) for one allele hit.""" + return ( + float(allele.span) * float(allele.identity), + -float(allele.evalue), + float(allele.contig_coverage), + ) + + def _score_profile(self, cg_sid: str, prof) -> tuple[dict, dict]: + """For one profile row, fetches matching Seq_type alleles, keeps the best hit per locus, + and returns (contig_names, score) where score sums spanid/eval/cc across all loci.""" + non_locus = {"ST", "clonal_complex", "species"} + prof_keys = list(prof._fields) + alleleconditions: list = [] + alleledict: dict = {} + for index, allele_num in enumerate(prof): + col_name = prof_keys[index] + if col_name in non_locus: + continue + alleledict[col_name] = None + alleleconditions.append( + and_(Seq_types.loci == col_name, Seq_types.allele == allele_num) + ) + all_alleles = ( + self.session.query(Seq_types) + .filter(and_(Seq_types.CG_ID_sample == cg_sid, or_(*alleleconditions))) + .all() + ) + for allele in all_alleles: + existing = alleledict[allele.loci] + if existing is None or self._allele_hit_score(allele) > self._allele_hit_score( + existing + ): + alleledict[allele.loci] = allele + score: dict = {"spanid": 0.0, "eval": 0.0, "cc": 0.0} + contig_names: dict = {} + for locus, allele in alleledict.items(): + if allele is None: + continue + score["spanid"] += float(allele.span) * float(allele.identity) + score["eval"] += float(allele.evalue) + score["cc"] += float(allele.contig_coverage) + contig_names[locus] = {"contig_name": str(allele.contig_name)} + return contig_names, score + + def _pick_top_st(self, scores: dict) -> int | str: + """Selects the ST with the highest composite score (spanid → eval → contig_coverage).""" + top_st: int | str | None = None + top_spanid = -1.0 + top_eval = float("inf") + top_cc = -1.0 + for st, val in scores.items(): + if ( + val["spanid"] > top_spanid + or (val["spanid"] == top_spanid and val["eval"] < top_eval) + or (val["spanid"] == top_spanid and val["eval"] == top_eval and val["cc"] > top_cc) + ): + top_spanid = val["spanid"] + top_eval = val["eval"] + top_cc = val["cc"] + top_st = st + return top_st + + def read_st(self, cg_sid: str) -> int | str: + """Takes a CG_ID_sample and predicts the correct ST.""" + organism: str | None = ( self.session.query(Samples.organism).filter(Samples.CG_ID_sample == cg_sid).scalar() ) if organism is None: self.logger.warning( - "No organism set for {}. Most likely control sample. Setting ST to -1".format( - cg_sid - ) + f"No organism set for {cg_sid}. Most likely control sample. Setting ST to -1" ) return -1 - [alleles, allelediff] = self.get_unique_alleles(cg_sid, organism, threshold) + + threshold = True + alleles, allelediff = self.read_unique_alleles(cg_sid, organism, threshold) if allelediff < 0: threshold = False - [alleles, allelediff] = self.get_unique_alleles(cg_sid, organism, threshold) + alleles, allelediff = self.read_unique_alleles(cg_sid, organism, threshold) if allelediff < 0: self.logger.warning( - "Insufficient allele hits to establish ST for sample {}, even without thresholds. Setting ST to -3".format( - cg_sid, organism - ) + f"Insufficient allele hits to establish ST for sample {cg_sid}, even without thresholds. Setting ST to -3" ) - self.setPredictor(cg_sid) + self.set_predictor(cg_sid) return -3 - # Tests all allele combinations found to see if any of them result in ST - filter = list() - for key, val in alleles.items(): - subfilter = list() - for num in val: - subfilter.append(" self.profiles[organism].c.{}=={} ".format(key, num)) - subfilter = ",".join(subfilter) - if len(val) > 1: - subfilter = "or_({})".format(subfilter) - filter.append(subfilter) - filter = ",".join(filter) - filter = "and_({})".format(filter) - output = self.session.query(self.profiles[organism]).filter(eval(filter)).all() - - # Check for existence in profile database - if len(output) > 1: - STlist = list() - for st in output: - STlist.append(st.ST) - best = self.bestST(cg_sid, STlist, "profile") - if threshold: + # Try matching against the curated profile table + output = self._query_st_profiles(alleles, self.profiles[organism]) + if output: + st_list = [row.ST for row in output] + best = self.read_best_st(cg_sid, st_list, "profile") + if threshold and len(st_list) > 1: self.logger.warning( - "Multiple ST within threshold found for sample {}, list: {}. Established ST{} as best hit.".format( - cg_sid, STlist, best - ) + f"Multiple ST within threshold found for sample {cg_sid}, list: {st_list}. Established ST{best} as best hit." ) return best - elif len(output) == 1: - # Arbitary call - return self.bestST(cg_sid, [output[0].ST], "profile") - # Check for existence in novel database - elif threshold: + + # Try matching against the novel ST table (only when hits are above threshold) + if threshold: self.logger.info( - "Sample {} on {} has novel ST reliably established. Searching for prior novel definition...".format( - cg_sid, organism - ) + f"Sample {cg_sid} on {organism} has novel ST reliably established. Searching for prior novel definition..." ) - filter = list() - for key, val in alleles.items(): - subfilter = list() - for num in val: - subfilter.append(" self.novel[organism].c.{}=={} ".format(key, num)) - subfilter = ",".join(subfilter) - if len(val) > 1: - subfilter = "or_({})".format(subfilter) - filter.append(subfilter) - filter = ",".join(filter) - filter = "and_({})".format(filter) - output = self.session.query(self.novel[organism]).filter(eval(filter)).all() - - if len(output) > 1: - STlist = list() - for st in output: - STlist.append(st.ST) - best = self.bestST(cg_sid, STlist, "novel") - if threshold: + output = self._query_st_profiles(alleles, self.novel[organism]) + if output: + st_list = [row.ST for row in output] + best = self.read_best_st(cg_sid, st_list, "novel") + if len(st_list) > 1: self.logger.warning( - "Multiple ST within novel threshold found for sample {}, list: {}. Established ST{} as best hit.".format( - cg_sid, STlist, best - ) + f"Multiple ST within novel threshold found for sample {cg_sid}, list: {st_list}. Established ST{best} as best hit." ) return best - elif len(output) == 1: - return self.bestST(cg_sid, [output[0].ST], "novel") - else: - # Create new novel ST - # Set ST -10 per default, or one below the current min, whichever is smaller. - st = -9 - query = self.session.query(self.novel[organism]).all() - for entry in query: - if entry.ST < st: - st = entry.ST - st = st - 1 - - bestSet = self.bestAlleles(cg_sid) - newEntry = dict() - for allele, columns in bestSet.items(): - newEntry[allele] = columns["allele"] - newEntry["ST"] = st - self.add_rec(newEntry, self.novel[organism]) - return self.bestST(cg_sid, [st], "novel") - else: - self.logger.warning( - "Sample {} on {} has an allele set but hits are low-quality and\ - do not resolve to an ST. Setting ST to -2".format( - cg_sid, organism - ) - ) - bestSet = self.bestAlleles(cg_sid) - self.setPredictor(cg_sid, bestSet) - return -2 - - def bestST(self, cg_sid: str, st_list: List, type="profile"): - """Takes in a list of ST and a sample. - Establishes which ST is most likely by criteria id*span -> eval -> contig coverage - & flags involved alleles""" - profiles = list() - scores = dict() - bestalleles = dict() - organism = ( + # No prior novel match — create a new novel ST + new_st = self._create_novel_st_entry(cg_sid, organism) + return self.read_best_st(cg_sid, [new_st], "novel") + + self.logger.warning( + f"Sample {cg_sid} on {organism} has an allele set but hits are low-quality and do not resolve to an ST. Setting ST to -2" + ) + best_set = self.read_best_alleles(cg_sid) + self.set_predictor(cg_sid, best_set) + return -2 + + def read_best_st(self, cg_sid: str, st_list: list, type: str = "profile") -> int | str: + """Establishes which ST is most likely by criteria id*span → eval → contig coverage + and flags the involved alleles as predictors.""" + organism: str = ( self.session.query(Samples.organism).filter(Samples.CG_ID_sample == cg_sid).scalar() ) - for st in st_list: - scores[st] = dict() - bestalleles[st] = dict() - scores[st]["spanid"] = 0 - scores[st]["eval"] = 0 - scores[st]["cc"] = 0 - scores[st]["span"] = 0 - if type == "profile": - profiles.append( - self.session.query(self.profiles[organism]) - .filter(text("ST={}".format(st))) - .first() - ) - elif type == "novel": - profiles.append( - self.session.query(self.novel[organism]) - .filter(text("ST={}".format(st))) - .first() - ) - - # Get values for each allele set that resolves an ST + table: Table = self.profiles[organism] if type == "profile" else self.novel[organism] + profiles = [self.session.query(table).filter(text(f"ST={st}")).first() for st in st_list] + scores: dict = {} + best_alleles: dict = {} for prof in profiles: - alleleconditions = list() - alleledict = dict() - allconditions = ["Seq_types.CG_ID_sample=='{}'".format(cg_sid)] - - for index, allele in enumerate(prof): - if ( - "ST" not in prof.keys()[index] - and "clonal_complex" not in prof.keys()[index] - and "species" not in prof.keys()[index] - ): - condition = "Seq_types.loci=='{}' , Seq_types.allele=='{}'".format( - prof.keys()[index], allele - ) - alleledict[prof.keys()[index]] = "" - alleleconditions.append("and_({})".format(condition)) - - alleleconditions = "or_({})".format(",".join(alleleconditions)) - allconditions.append(alleleconditions) - allconditions = "and_({})".format(",".join(allconditions)) - all_alleles = self.session.query(Seq_types).filter(eval(allconditions)).all() - - # Keep only best hit each loci - for allele in all_alleles: - if alleledict[allele.loci] == "": - alleledict[allele.loci] = allele - else: - old_al = alleledict[allele.loci] - - if allele.span * allele.identity >= old_al.span * old_al.identity: - if allele.span * allele.identity > old_al.span * old_al.identity: - alleledict[allele.loci] = allele - elif float(allele.evalue) <= float(old_al.evalue): - if float(allele.evalue) < float(old_al.evalue): - alleledict[allele.loci] = allele - elif allele.contig_coverage > old_al.contig_coverage: - alleledict[allele.loci] = allele - - # Create score dict for the ST - for key, allele in alleledict.items(): - scores[prof.ST]["spanid"] += allele.span * allele.identity - scores[prof.ST]["eval"] += float(allele.evalue) - scores[prof.ST]["cc"] += allele.contig_coverage - if not allele.loci in bestalleles[prof.ST].keys(): - bestalleles[prof.ST][allele.loci] = dict() - if not "contig_name" in bestalleles[prof.ST][allele.loci].keys(): - bestalleles[prof.ST][allele.loci]["contig_name"] = str(allele.contig_name) - - # Establish best ST - topST = "" - topID = 0 - topEval = 100 - topCC = 0 - for key, val in scores.items(): - if scores[key]["spanid"] > topID: - topID = scores[key]["spanid"] - topEval = scores[key]["eval"] - topCC = scores[key]["cc"] - topST = key - elif scores[key]["spanid"] == topID and scores[key]["eval"] < topEval: - topID = scores[key]["spanid"] - topEval = scores[key]["eval"] - topCC = scores[key]["cc"] - topST = key - elif ( - scores[key]["spanid"] == topID - and scores[key]["eval"] == topEval - and scores[key]["cc"] > topCC - ): - topID = scores[key]["spanid"] - topEval = scores[key]["eval"] - topCC = scores[key]["cc"] - topST = key - self.setPredictor(cg_sid, bestalleles[topST]) - return topST - - def bestAlleles(self, cg_sid: str): + contig_names, score = self._score_profile(cg_sid, prof) + scores[prof.ST] = score + best_alleles[prof.ST] = contig_names + top_st = self._pick_top_st(scores) + self.set_predictor(cg_sid, best_alleles[top_st]) + return top_st + + def read_best_alleles(self, cg_sid: str): """Establishes which allele set (for bad samples) is most likely by criteria span* id -> eval -> contig coverage""" - hits = ( + hits: list[tuple[str, str, float, float, float, float, str]] = ( self.session.query( Seq_types.contig_name, Seq_types.loci, @@ -852,20 +888,10 @@ def bestAlleles(self, cg_sid: str): .filter(Seq_types.CG_ID_sample == cg_sid) .all() ) - bestHits = dict() - alleledict = dict() + bestHits: dict[str, dict[str, str]] = {} + alleledict: dict[str, list[float]] = {} for allele in hits: - if allele.loci not in bestHits.keys(): - bestHits[allele.loci] = dict() - bestHits[allele.loci]["contig_name"] = allele.contig_name - bestHits[allele.loci]["allele"] = allele.allele - alleledict[allele.loci] = [ - allele.identity, - allele.evalue, - allele.contig_coverage, - allele.span, - ] - else: + if allele.loci in bestHits: if ( ( allele.identity * allele.span @@ -890,14 +916,25 @@ def bestAlleles(self, cg_sid: str): allele.contig_coverage, allele.span, ] + else: + bestHits[allele.loci] = { + "contig_name": allele.contig_name, + "allele": allele.allele, + } + alleledict[allele.loci] = [ + allele.identity, + allele.evalue, + allele.contig_coverage, + allele.span, + ] return bestHits - def get_unique_alleles(self, cg_sid: str, organism: str, threshold=True): + def read_unique_alleles(self, cg_sid: str, organism: str, threshold=True): """Returns a dict containing all unique alleles at every loci, and allele difference from expected""" - tid = float(self.config["threshold"]["mlst_id"]) - tspan = (self.config["threshold"]["mlst_span"]) / 100.0 + tspan = (self.threshold.mlst_span) / 100.0 if threshold: - hits = ( + tid = float(self.threshold.mlst_id) + hits: list[tuple[str, str]] = ( self.session.query(Seq_types.loci, Seq_types.allele) .filter( Seq_types.CG_ID_sample == cg_sid, @@ -907,18 +944,17 @@ def get_unique_alleles(self, cg_sid: str, organism: str, threshold=True): .all() ) else: - hits = ( + hits: list[tuple[str, str]] = ( self.session.query(Seq_types.loci, Seq_types.allele) .filter(Seq_types.CG_ID_sample == cg_sid) .all() ) # Establish number of unique hits - uniqueDict = dict() + uniqueDict: dict[str, list[str]] = {} for hit in hits: if hit.loci not in uniqueDict.keys(): - uniqueDict[hit.loci] = list() - uniqueDict[hit.loci].append(hit.allele) + uniqueDict[hit.loci] = [hit.allele] elif hit.allele not in uniqueDict[hit.loci]: uniqueDict[hit.loci].append(hit.allele) non_allele_columns = 1 diff --git a/microSALT/store/models.py b/microSALT/store/models.py index a1b1871a..812a47bf 100644 --- a/microSALT/store/models.py +++ b/microSALT/store/models.py @@ -1,84 +1,55 @@ -"""Table definitions for profiles databases. Bit special since it spawns multiple tables. - By: Isak Sylvin, @sylvinite""" +"""Table definitions for profile databases. + +Profile tables (profile_* and novel_*) cannot use the declarative ORM because +their names and column sets are determined at runtime by scanning the profiles +folder on disk. SQLAlchemy Core Table objects are used instead. + +By: Isak Sylvin, @sylvinite""" #!/usr/bin/env python import os -from sqlalchemy import * - +from sqlalchemy import Column, SmallInteger, Table -class Profiles: - def __init__(self, metadata, config, log): - self.tables = dict() - self.metadata = metadata - self.config = config - self.logger = log - try: - indata = os.listdir(self.config["folders"]["profiles"]) - for file in indata: - self.add_table(file) - except Exception as e: - self.logger.error( - "Unable to open profile folder {}".format(self.config["folders"]["profiles"]) - ) - def add_table(self, file): - try: - with open("{}/{}".format(self.config["folders"]["profiles"], file), "r") as fh: - # Sets profile_* headers - head = fh.readline() - head = head.rstrip().split("\t")[:8] # Only consider the first 8 elements - index = 0 +class ProfileTable: + """Builds a dict of SQLAlchemy Core Table objects from the profiles folder. - header = "Table('profile_{}'.format(file), self.metadata,".format(file) - while index < len(head): - # Set ST as PK - if head[index] == "ST": - header += "Column(head[{}], SmallInteger, primary_key=True),".format(index) - else: - header += "Column(head[{}], SmallInteger),".format(index) - index = index + 1 - header += ")" - p = eval(header) - self.tables[file] = p - except Exception as e: - self.logger.error("Unable to open profile file {}".format(file)) + Each file in the folder produces one table whose name is + ``{prefix}{filename}``. The first eight tab-separated fields of the + file header become the column names; the column named ``ST`` is used as + the primary key. + Args: + prefix: Table name prefix, e.g. ``"profile_"`` or ``"novel_"``. + metadata: The shared SQLAlchemy MetaData instance. + profiles_path: Path to the folder containing ST profile files. + log: Logger instance. + """ -class Novel: - def __init__(self, metadata, config, log): - self.tables = dict() + def __init__(self, prefix: str, metadata, profiles_path: str, log): + self.tables: dict[str, Table] = {} + self.prefix = prefix self.metadata = metadata - self.config = config + self.profiles_path = profiles_path self.logger = log try: - indata = os.listdir(self.config["folders"]["profiles"]) - for file in indata: - self.add_table(file) - except Exception as e: + for filename in os.listdir(self.profiles_path): + self._add_table(filename) + except Exception: self.logger.error( - "Unable to open profile folder {}".format(self.config["folders"]["profiles"]) + f"Unable to open profile folder {self.profiles_path}" ) - def add_table(self, file): + def _add_table(self, filename: str) -> None: try: - with open("{}/{}".format(self.config["folders"]["profiles"], file), "r") as fh: - # Sets profile_* headers - head = fh.readline() - head = head.rstrip().split("\t")[:8] # Only consider the first 8 elements - index = 0 - - header = "Table('novel_{}'.format(file), self.metadata,".format(file) - while index < len(head): - # Set ST as PK - if head[index] == "ST": - header += "Column(head[{}], SmallInteger, primary_key=True),".format(index) - # Set Clonal complex as string - else: - header += "Column(head[{}], SmallInteger),".format(index) - index = index + 1 - header += ")" - p = eval(header) - self.tables[file] = p - except Exception as e: - self.logger.error("Unable to open profile file {}".format(file)) + with open(f"{self.profiles_path}/{filename}", "r") as fh: + head = fh.readline().rstrip().split("\t")[:8] + columns = [ + Column(col, SmallInteger, primary_key=(col == "ST")) + for col in head + ] + table = Table(f"{self.prefix}{filename}", self.metadata, *columns) + self.tables[filename] = table + except Exception: + self.logger.error(f"Unable to open profile file {filename}") diff --git a/microSALT/store/orm_models.py b/microSALT/store/orm_models.py index a8493b83..13b7fe3f 100644 --- a/microSALT/store/orm_models.py +++ b/microSALT/store/orm_models.py @@ -1,18 +1,22 @@ """Samples table definition - By: Isak Sylvin, @sylvinite""" +By: Isak Sylvin, @sylvinite""" -#!/usr/bin/env python +from sqlalchemy import ( + Boolean, + Column, + DateTime, + Float, + ForeignKey, + Integer, + SmallInteger, + String, +) +from sqlalchemy.orm import declarative_base, relationship -from flask_sqlalchemy import SQLAlchemy -from sqlalchemy import * -from sqlalchemy.orm import relationship +Base = declarative_base() -from microSALT import app -db = SQLAlchemy(app) - - -class Samples(db.Model): +class Samples(Base): __tablename__ = "samples" seq_types = relationship("Seq_types", back_populates="samples") projects = relationship("Projects", back_populates="samples") @@ -20,152 +24,146 @@ class Samples(db.Model): # steps = relationship("Steps", back_populates="samples") expacs = relationship("Expacs", back_populates="samples") - CG_ID_sample = db.Column(db.String(15), primary_key=True, nullable=False) - CG_ID_project = db.Column(db.String(15), ForeignKey("projects.CG_ID_project")) - Customer_ID_sample = db.Column(db.String(40)) - organism = db.Column(db.String(30)) - ST = db.Column(db.SmallInteger, default=-1) - pubmlst_ST = db.Column(db.SmallInteger, default=-1) - date_analysis = db.Column(db.DateTime) - genome_length = db.Column(db.Integer, default=-1) - gc_percentage = db.Column(db.Float(3, 2), default=0.0) - n50 = db.Column(db.Integer, default=-1) - contigs = db.Column(db.Integer, default=-1) - priority = db.Column(db.String(20)) - - total_reads = db.Column(db.Integer) # Fetch from bcl2fastq - insert_size = db.Column(db.Integer) - duplication_rate = db.Column(db.Float) - mapped_rate = db.Column(db.Float) - coverage_10x = db.Column(db.Float) - coverage_30x = db.Column(db.Float) - coverage_50x = db.Column(db.Float) - coverage_100x = db.Column(db.Float) - average_coverage = db.Column(db.Float) - reference_genome = db.Column(db.String(32)) - reference_length = db.Column(db.Integer) - - application_tag = db.Column(db.String(15)) - date_arrival = db.Column(db.DateTime) - date_analysis = db.Column(db.DateTime) - date_sequencing = db.Column(db.DateTime) - date_libprep = db.Column(db.DateTime) - method_sequencing = db.Column(db.String(15)) - method_libprep = db.Column(db.String(15)) - - -class Seq_types(db.Model): + CG_ID_sample = Column(String(32), primary_key=True, nullable=False) + CG_ID_project = Column(String(32), ForeignKey("projects.CG_ID_project")) + Customer_ID_sample = Column(String(128)) + organism = Column(String(100)) + ST = Column(SmallInteger, default=-1) + pubmlst_ST = Column(SmallInteger, default=-1) + date_analysis = Column(DateTime) + genome_length = Column(Integer, default=-1) + gc_percentage = Column(Float(3, 2), default=0.0) + n50 = Column(Integer, default=-1) + contigs = Column(Integer, default=-1) + priority = Column(String(20)) + + total_reads = Column(Integer) # Fetch from bcl2fastq + insert_size = Column(Integer) + duplication_rate = Column(Float) + mapped_rate = Column(Float) + coverage_10x = Column(Float) + coverage_30x = Column(Float) + coverage_50x = Column(Float) + coverage_100x = Column(Float) + average_coverage = Column(Float) + reference_genome = Column(String(32)) + reference_length = Column(Integer) + + application_tag = Column(String(15)) + date_arrival = Column(DateTime) + date_analysis = Column(DateTime) + date_sequencing = Column(DateTime) + date_libprep = Column(DateTime) + method_sequencing = Column(String(128)) + method_libprep = Column(String(128)) + + +class Seq_types(Base): __tablename__ = "seq_types" samples = relationship("Samples", back_populates="seq_types") - CG_ID_sample = db.Column( - db.String(15), ForeignKey("samples.CG_ID_sample"), primary_key=True - ) - loci = db.Column(db.String(10), primary_key=True) - allele = db.Column(db.SmallInteger) - contig_name = db.Column(db.String(20), primary_key=True) - contig_length = db.Column(db.Integer) - contig_coverage = db.Column(db.Float(6, 2)) - identity = db.Column(db.Float(3, 2), default=0.0) - span = db.Column(db.Float(3, 2), default=0.0) - evalue = db.Column(db.String(10)) - bitscore = db.Column(db.SmallInteger) - subject_length = db.Column(db.Integer) - st_predictor = db.Column(db.Boolean, default=0) - contig_start = db.Column(db.Integer) - contig_end = db.Column(db.Integer) - - -class Resistances(db.Model): + CG_ID_sample = Column(String(32), ForeignKey("samples.CG_ID_sample"), primary_key=True) + loci = Column(String(10), primary_key=True) + allele = Column(SmallInteger) + contig_name = Column(String(20), primary_key=True) + contig_length = Column(Integer) + contig_coverage = Column(Float(6, 2)) + identity = Column(Float(3, 2), default=0.0) + span = Column(Float(3, 2), default=0.0) + evalue = Column(String(10)) + bitscore = Column(SmallInteger) + subject_length = Column(Integer) + st_predictor = Column(Boolean, default=0) + contig_start = Column(Integer) + contig_end = Column(Integer) + + +class Resistances(Base): __tablename__ = "resistances" samples = relationship("Samples", back_populates="resistances") - CG_ID_sample = db.Column( - db.String(15), ForeignKey("samples.CG_ID_sample"), primary_key=True - ) - gene = db.Column(db.String(50), primary_key=True) - instance = db.Column(db.String(30), primary_key=True) - contig_name = db.Column(db.String(20), primary_key=True) - contig_length = db.Column(db.Integer) - contig_coverage = db.Column(db.Float(6, 2)) - identity = db.Column(db.Float(3, 2), default=0.0) - span = db.Column(db.Float(3, 2), default=0.0) - evalue = db.Column(db.String(10)) - bitscore = db.Column(db.SmallInteger) - subject_length = db.Column(db.Integer) - reference = db.Column(db.String(40)) - resistance = db.Column(db.String(120)) - contig_start = db.Column(db.Integer) - contig_end = db.Column(db.Integer) - - -class Expacs(db.Model): + CG_ID_sample = Column(String(32), ForeignKey("samples.CG_ID_sample"), primary_key=True) + gene = Column(String(50), primary_key=True) + instance = Column(String(120), primary_key=True) + contig_name = Column(String(20), primary_key=True) + contig_length = Column(Integer) + contig_coverage = Column(Float(6, 2)) + identity = Column(Float(3, 2), default=0.0) + span = Column(Float(3, 2), default=0.0) + evalue = Column(String(10)) + bitscore = Column(SmallInteger) + subject_length = Column(Integer) + reference = Column(String(40)) + resistance = Column(String(120)) + contig_start = Column(Integer) + contig_end = Column(Integer) + + +class Expacs(Base): __tablename__ = "expacs" samples = relationship("Samples", back_populates="expacs") - CG_ID_sample = db.Column( - db.String(15), ForeignKey("samples.CG_ID_sample"), primary_key=True - ) - gene = db.Column(db.String(50), primary_key=True) - instance = db.Column(db.String(30), primary_key=True) - contig_name = db.Column(db.String(20), primary_key=True) - contig_length = db.Column(db.Integer) - contig_coverage = db.Column(db.Float(6, 2)) - identity = db.Column(db.Float(3, 2), default=0.0) - span = db.Column(db.Float(3, 2), default=0.0) - evalue = db.Column(db.String(10)) - bitscore = db.Column(db.SmallInteger) - subject_length = db.Column(db.Integer) - reference = db.Column(db.String(40)) - virulence = db.Column(db.String(120)) - contig_start = db.Column(db.Integer) - contig_end = db.Column(db.Integer) - - -class Projects(db.Model): + CG_ID_sample = Column(String(32), ForeignKey("samples.CG_ID_sample"), primary_key=True) + gene = Column(String(50), primary_key=True) + instance = Column(String(120), primary_key=True) + contig_name = Column(String(20), primary_key=True) + contig_length = Column(Integer) + contig_coverage = Column(Float(6, 2)) + identity = Column(Float(3, 2), default=0.0) + span = Column(Float(3, 2), default=0.0) + evalue = Column(String(10)) + bitscore = Column(SmallInteger) + subject_length = Column(Integer) + reference = Column(String(40)) + virulence = Column(String(120)) + contig_start = Column(Integer) + contig_end = Column(Integer) + + +class Projects(Base): __tablename__ = "projects" samples = relationship("Samples", back_populates="projects") reports = relationship("Reports", back_populates="projects") - CG_ID_project = db.Column(db.String(15), primary_key=True, nullable=False) - Customer_ID_project = db.Column(db.String(15)) - date_ordered = db.Column(db.DateTime) - Customer_ID = db.Column(db.String(15)) + CG_ID_project = Column(String(32), primary_key=True, nullable=False) + Customer_ID_project = Column(String(32)) + Customer_ID = Column(String(32)) -class Versions(db.Model): +class Versions(Base): __tablename__ = "versions" - name = db.Column(db.String(45), primary_key=True, nullable=False) - version = db.Column(db.String(10)) + name = Column(String(45), primary_key=True, nullable=False) + version = Column(String(10)) # Keeps and aggregate step string, makes a new version whenever one is not found -class Reports(db.Model): +class Reports(Base): __tablename__ = "reports" projects = relationship("Projects", back_populates="reports") - CG_ID_project = db.Column( - db.String(15), ForeignKey("projects.CG_ID_project"), primary_key=True - ) - steps_aggregate = db.Column(db.String(100)) - date = db.Column(db.DateTime) - version = db.Column(db.Integer, default=1, primary_key=True) + CG_ID_project = Column(String(32), ForeignKey("projects.CG_ID_project"), primary_key=True) + steps_aggregate = Column(String(100)) + date = Column(DateTime) + version = Column(Integer, default=1, primary_key=True) -class Collections(db.Model): +class Collections(Base): __tablename__ = "collections" - ID_collection = db.Column(db.String(15), primary_key=True) - CG_ID_sample = db.Column(db.String(15), primary_key=True) + ID_collection = Column(String(32), primary_key=True) + CG_ID_sample = Column(String(32), primary_key=True) + + +class SystemLock(Base): + """Rows in this table act as advisory locks for long-running operations. + + A row with lock_name='ref_update' signals that a reference update is in + progress. No other processes should modify or read profile tables while + this lock is held. + """ + __tablename__ = "system_locks" -# Multi-date support for libprep/sequencing/analysis -# class Steps(db.Model): -# __tablename__ = 'steps' -# samples = relationship("Samples", back_populates="steps") -# -# CG_ID_sample = db.Column(db.String(15), ForeignKey('samples.CG_ID_sample'), primary_key=True) -# step = db.Column(db.String(40), primary_key=True) -# method = db.Column(db.String(40), primary_key=True) -# date = db.Column(db.DateTime) + lock_name = Column(String(60), primary_key=True, nullable=False) + acquired_at = Column(DateTime, nullable=False) diff --git a/unique_references/ExPEC.fsa b/microSALT/unique_references/ExPEC.fsa similarity index 100% rename from unique_references/ExPEC.fsa rename to microSALT/unique_references/ExPEC.fsa diff --git a/microSALT/utils/job_creator.py b/microSALT/utils/job_creator.py index 352b8b37..613d8fd2 100644 --- a/microSALT/utils/job_creator.py +++ b/microSALT/utils/job_creator.py @@ -2,7 +2,6 @@ By: Isak Sylvin, @sylvinite""" #!/usr/bin/env python - import glob import gzip import json @@ -10,20 +9,58 @@ import re import shutil import subprocess +import sys import time from datetime import datetime +from importlib.metadata import entry_points from pathlib import Path import yaml from microSALT import __version__ +from microSALT.config import ( + Containers, + Folders, + PasteurCredentials, + PubMLSTCredentials, + Regex, + Singularity, + SlurmHeader, + Threshold, +) +from microSALT.exc.exceptions import JobCreationError from microSALT.store.db_manipulator import DB_Manipulator +from microSALT.store.orm_models import Projects from microSALT.utils.referencer import Referencer class Job_Creator: - def __init__(self, config, log, sampleinfo={}, run_settings={}): - self.config = config + def __init__( + self, + log, + folders: Folders, + slurm_header: SlurmHeader, + regex: Regex, + dry: bool, + config_path: str, + threshold: Threshold, + pubmlst: PubMLSTCredentials, + pasteur: PasteurCredentials, + singularity: Singularity, + containers: Containers, + sampleinfo={}, + run_settings={}, + ): + self.folders = folders + self.slurm_header = slurm_header + self.regex = regex + self.dry = dry + self.config_path = config_path + self.threshold = threshold + self.pubmlst = pubmlst + self.pasteur = pasteur + self.singularity = singularity + self.containers = containers self.logger = log self.batchfile = "/tmp/batchfile.sbatch" @@ -70,57 +107,56 @@ def __init__(self, config, log, sampleinfo={}, run_settings={}): else: self.dt = datetime.now() self.now = time.strftime( - "{}.{}.{}_{}.{}.{}".format( - self.dt.year, - self.dt.month, - self.dt.day, - self.dt.hour, - self.dt.minute, - self.dt.second, - ) + f"{self.dt.year}.{self.dt.month}.{self.dt.day}_{self.dt.hour}.{self.dt.minute}.{self.dt.second}" ) if run_settings.get("finishdir") is None: - self.finishdir = "{}/{}_{}".format(config["folders"]["results"], self.name, self.now) - self.db_pusher = DB_Manipulator(config, log) + self.finishdir = f"{folders.results}/{self.name}_{self.now}" + self.db_pusher = DB_Manipulator(log=log, folders=folders, threshold=threshold) self.concat_files = dict() - self.ref_resolver = Referencer(config, log) + self.ref_resolver = Referencer( + log=log, + folders=folders, + threshold=threshold, + pubmlst=pubmlst, + pasteur=pasteur, + singularity=singularity, + containers=containers, + ) def get_sbatch(self): """Returns sbatchfile, slightly superflous""" return self.batchfile def get_headerargs(self): - headerline = "-A {} -p {} -n {} -t {} -J {}_{} --qos {} --output {}/slurm_{}.log".format( - self.config["slurm_header"]["project"], - self.config["slurm_header"]["type"], - self.config["slurm_header"]["threads"], - self.config["slurm_header"]["time"], - self.config["slurm_header"]["job_prefix"], - self.name, - self.config["slurm_header"]["qos"], - self.finishdir, - self.name, - ) + headerline = f"-A {self.slurm_header.project} -p {self.slurm_header.type} -n {self.slurm_header.threads} -t {self.slurm_header.time} -J {self.slurm_header.job_prefix}_{self.name} --qos {self.slurm_header.qos} --output {self.finishdir}/slurm_{self.name}.log" return headerline + def _singularity_exec(self, tool: str, command: str) -> str: + """Return command wrapped with singularity exec for the given tool container.""" + sif = getattr(self.containers, tool) + binary = self.singularity.binary + bind_list = list(self.singularity.bind_paths) + if self.finishdir and self.finishdir not in bind_list: + bind_list.append(self.finishdir) + bind = f"--bind {','.join(bind_list)}" if bind_list else "" + return f"{binary} exec {bind} {sif} {command}" + def verify_fastq(self): """Uses arg indir to return a dict of PE fastq tuples fulfilling naming convention""" verified_files = list() files = os.listdir(self.indir) if files == []: - raise Exception("Directory {} lacks fastq files.".format(self.indir)) + raise Exception(f"Directory {self.indir} lacks fastq files.") for file in files: - file_match = re.match(self.config["regex"]["file_pattern"], file) + file_match = re.match(self.regex.file_pattern, file) if file_match: # Check that symlinks resolve - path = "{}/{}".format(self.indir, file) + path = f"{self.indir}/{file}" if os.path.islink(path): if not os.path.exists(os.readlink(path)): raise Exception( - "Some fastq files are unresolved symlinks in directory {}.".format( - self.indir - ) + f"Some fastq files are unresolved symlinks in directory {self.indir}." ) # Make sure both mates exist @@ -138,44 +174,34 @@ def verify_fastq(self): else: pairno = 2 - 1 % int(file_match[1]) # 1->2, 2->1 # Construct mate name - pairname = "{}{}{}".format( - file_match.string[: file_match.end(1) - 1], - pairno, - file_match.string[file_match.end(1) : file_match.end()], - ) + pairname = f"{file_match.string[: file_match.end(1) - 1]}{pairno}{file_match.string[file_match.end(1) : file_match.end()]}" if pairname in files: files.pop(files.index(pairname)) verified_files.append(file_match[0]) verified_files.append(pairname) else: - raise Exception( - "Some fastq files have no mate in directory {}.".format(self.indir) - ) + raise Exception(f"Some fastq files have no mate in directory {self.indir}.") if verified_files == []: raise Exception( - "No files in directory {} match file_pattern '{}'.".format( - self.indir, self.config["regex"]["file_pattern"] - ) + f"No files in directory {self.indir} match file_pattern '{self.regex.file_pattern}'." ) # Warn about file sizes for vfile in verified_files: try: - bsize = os.stat("{}/{}".format(self.indir, vfile)).st_size + bsize = os.stat(f"{self.indir}/{vfile}").st_size bsize = bsize >> 20 if bsize > 1000: - self.logger.warning("Input fastq {} exceeds 1000MB".format(vfile)) + self.logger.warning(f"Input fastq {vfile} exceeds 1000MB") except Exception: - self.logger.warning( - "Unable to verify size of input file {}/{}".format(self.indir, vfile) - ) + self.logger.warning(f"Unable to verify size of input file {self.indir}/{vfile}") # Warn about invalid fastq files for vfile in verified_files: - f = gzip.open("{}/{}".format(self.indir, vfile), "r") + f = gzip.open(f"{self.indir}/{vfile}", "r") lines = f.read().splitlines() if len(lines) < 2 or "+" not in str(lines[-2]): - self.logger.warning("Input fastq {} does not seem to end properly".format(vfile)) + self.logger.warning(f"Input fastq {vfile} does not seem to end properly") return sorted(verified_files) @staticmethod @@ -194,14 +220,14 @@ def create_assemblysection(self): batchfile = open(self.batchfile, "a+") # memory is actually 128 per node regardless of cores. batchfile.write("# SKESA assembly\n") - batchfile.write( - f"mkdir -p {assembly_dir} &" + skesa_cmd = ( f"skesa " - f"--cores {self.config['slurm_header']['threads']} " - f"--memory {8 * int(self.config['slurm_header']['threads'])} " + f"--cores {self.slurm_header.threads} " + f"--memory {8 * int(self.slurm_header.threads)} " f"--contigs_out {contigs_file_raw} " - f"--reads {self.concat_files['f']},{self.concat_files['r']}\n" + f"--reads {self.concat_files['f']},{self.concat_files['r']}" ) + batchfile.write(f"mkdir -p {assembly_dir} &{self._singularity_exec('skesa', skesa_cmd)}\n") # Convert sequence naming in Skesa output into Spades format in the contigs fasta file: # ---------------------------------------------- @@ -217,7 +243,7 @@ def create_assemblysection(self): # Note: The match function requires GNU awk (gawk) to be able to capture groups in regexes. batchfile.write( "gawk " - + "'/^>/ { match($0, /Contig_([0-9]+)_([0-9\.]+)/, m) } " + + r"'/^>/ { match($0, /Contig_([0-9]+)_([0-9\.]+)/, m) } " + '!/^>/ { seqlen=length($0); print ">NODE_" m[1] "_length_" seqlen "_cov_" m[2]; print $0; }\' ' + f"{contigs_file_raw} > {contigs_file}\n" ) @@ -233,7 +259,7 @@ def blast_subset(self, name, search_string): # Create run file_list = glob.glob(search_string) batchfile = open(self.batchfile, "a+") - batchfile.write("mkdir {}/blast_search/{}\n".format(self.finishdir, name)) + batchfile.write(f"mkdir {self.finishdir}/blast_search/{name}\n") blast_format = '"7 stitle sstrand qaccver saccver pident evalue bitscore qstart qend sstart send length"' if len(file_list) > 1: @@ -245,121 +271,115 @@ def blast_subset(self, name, search_string): else: ref_nosuf = re.search(r"(\w+(?:\-\w+)*)\.\w+", os.path.basename(ref)).group(1) batchfile.write( - "# BLAST {} search for {}, {}\n".format( - name, self.sample.get("organism"), ref_nosuf - ) + f"# BLAST {name} search for {self.sample.get('organism')}, {ref_nosuf}\n" ) if name == "mlst": batchfile.write( - "blastn -db {}/{} -query {}/assembly/{}_contigs.fasta -out {}/blast_search/{}/loci_query_{}.txt -task megablast -num_threads {} -outfmt {}\n".format( - os.path.dirname(ref), - ref_nosuf, - self.finishdir, - self.name, - self.finishdir, - name, - ref_nosuf, - self.config["slurm_header"]["threads"], - blast_format, + self._singularity_exec( + "blast", + f"blastn -db {os.path.dirname(ref)}/{ref_nosuf} -query {self.finishdir}/assembly/{self.name}_contigs.fasta -out {self.finishdir}/blast_search/{name}/loci_query_{ref_nosuf}.txt -task megablast -num_threads {self.slurm_header.threads} -outfmt {blast_format}", ) + + "\n" ) else: batchfile.write( - "blastn -db {}/{} -query {}/assembly/{}_contigs.fasta -out {}/blast_search/{}/{}.txt -task megablast -num_threads {} -outfmt {}\n".format( - os.path.dirname(ref), - ref_nosuf, - self.finishdir, - self.name, - self.finishdir, - name, - ref_nosuf, - self.config["slurm_header"]["threads"], - blast_format, + self._singularity_exec( + "blast", + f"blastn -db {os.path.dirname(ref)}/{ref_nosuf} -query {self.finishdir}/assembly/{self.name}_contigs.fasta -out {self.finishdir}/blast_search/{name}/{ref_nosuf}.txt -task megablast -num_threads {self.slurm_header.threads} -outfmt {blast_format}", ) + + "\n" ) elif len(file_list) == 1: ref_nosuf = re.search(r"(\w+(?:\-\w+)*)\.\w+", os.path.basename(file_list[0])).group(1) batchfile.write( - "## BLAST {} search in {}\n".format( - name, self.sample.get("organism").replace("_", " ").capitalize() - ) + f"## BLAST {name} search in {self.sample.get('organism').replace('_', ' ').capitalize()}\n" ) batchfile.write( - "blastn -db {}/{} -query {}/assembly/{}_contigs.fasta -out {}/blast_search/{}/{}.txt -task megablast -num_threads {} -outfmt {}\n".format( - os.path.dirname(search_string), - ref_nosuf, - self.finishdir, - self.name, - self.finishdir, - name, - ref_nosuf, - self.config["slurm_header"]["threads"], - blast_format, + self._singularity_exec( + "blast", + f"blastn -db {os.path.dirname(search_string)}/{ref_nosuf} -query {self.finishdir}/assembly/{self.name}_contigs.fasta -out {self.finishdir}/blast_search/{name}/{ref_nosuf}.txt -task megablast -num_threads {self.slurm_header.threads} -outfmt {blast_format}", ) + + "\n" ) batchfile.write("\n") batchfile.close() def create_variantsection(self): """Creates a job for variant calling based on local alignment""" - ref = "{}/{}.fasta".format(self.config["folders"]["genomes"], self.sample.get("reference")) - localdir = "{}/alignment".format(self.finishdir) - outbase = "{}/{}_{}".format(localdir, self.name, self.sample.get("reference")) + ref = f"{self.folders.genomes}/{self.sample.get('reference')}.fasta" + localdir = f"{self.finishdir}/alignment" + outbase = f"{localdir}/{self.name}_{self.sample.get('reference')}" # Create run batchfile = open(self.batchfile, "a+") batchfile.write("# Variant calling based on local alignment\n") - batchfile.write("mkdir {}\n".format(localdir)) + batchfile.write(f"mkdir {localdir}\n") batchfile.write("## Alignment & Deduplication\n") batchfile.write( - "bwa mem -M -t {} {} {} {} > {}.sam\n".format( - self.config["slurm_header"]["threads"], - ref, - self.concat_files["f"], - self.concat_files["r"], - outbase, + self._singularity_exec( + "bwa", + f"bwa mem -M -t {self.slurm_header.threads} {ref} {self.concat_files['f']} {self.concat_files['r']} > {outbase}.sam", ) + + "\n" ) batchfile.write( - "samtools view --threads {} -b -o {}.bam -T {} {}.sam\n".format( - self.config["slurm_header"]["threads"], outbase, ref, outbase + self._singularity_exec( + "samtools", + f"samtools view --threads {self.slurm_header.threads} -b -o {outbase}.bam -T {ref} {outbase}.sam", ) + + "\n" ) batchfile.write( - "samtools sort --threads {} -o {}.bam_sort {}.bam\n".format( - self.config["slurm_header"]["threads"], outbase, outbase + self._singularity_exec( + "samtools", + f"samtools sort --threads {self.slurm_header.threads} -o {outbase}.bam_sort {outbase}.bam", ) + + "\n" ) batchfile.write( - "picard MarkDuplicates I={}.bam_sort O={}.bam_sort_rmdup M={}.stats.dup REMOVE_DUPLICATES=true\n".format( - outbase, outbase, outbase + self._singularity_exec( + "picard", + f"picard MarkDuplicates I={outbase}.bam_sort O={outbase}.bam_sort_rmdup M={outbase}.stats.dup REMOVE_DUPLICATES=true", ) + + "\n" ) - batchfile.write("samtools index {}.bam_sort_rmdup\n".format(outbase)) batchfile.write( - "samtools idxstats {}.bam_sort_rmdup &> {}.stats.ref\n".format(outbase, outbase) + self._singularity_exec("samtools", f"samtools index {outbase}.bam_sort_rmdup") + "\n" + ) + batchfile.write( + self._singularity_exec("samtools", f"samtools idxstats {outbase}.bam_sort_rmdup") + + f" &> {outbase}.stats.ref\n" ) # Removal of temp aligment files - batchfile.write("rm {}.bam {}.sam\n".format(outbase, outbase)) + batchfile.write(f"rm {outbase}.bam {outbase}.sam\n") batchfile.write("## Primary stats generation\n") # Insert stats, dedupped batchfile.write( - "picard CollectInsertSizeMetrics I={}.bam_sort_rmdup O={}.stats.ins H={}.hist.ins\n".format( - outbase, outbase, outbase + self._singularity_exec( + "picard", + f"picard CollectInsertSizeMetrics I={outbase}.bam_sort_rmdup O={outbase}.stats.ins H={outbase}.hist.ins", ) + + "\n" ) # Coverage batchfile.write( - "samtools stats --coverage 1,10000,1 {}.bam_sort_rmdup |grep ^COV | cut -f 2- &> {}.stats.cov\n".format( - outbase, outbase + self._singularity_exec( + "samtools", f"samtools stats --coverage 1,10000,1 {outbase}.bam_sort_rmdup" ) + + f" |grep ^COV | cut -f 2- &> {outbase}.stats.cov\n" ) # Mapped rate, no dedup,dedup in MWGS (trimming has no effect)! - batchfile.write("samtools flagstat {}.bam_sort &> {}.stats.map\n".format(outbase, outbase)) + batchfile.write( + self._singularity_exec("samtools", f"samtools flagstat {outbase}.bam_sort") + + f" &> {outbase}.stats.map\n" + ) # Total reads, no dedup,dedup in MWGS (trimming has no effect)! - batchfile.write("samtools view -c {}.bam_sort &> {}.stats.raw\n".format(outbase, outbase)) + batchfile.write( + self._singularity_exec("samtools", f"samtools view -c {outbase}.bam_sort") + + f" &> {outbase}.stats.raw\n" + ) batchfile.write("\n\n") batchfile.close() @@ -368,22 +388,16 @@ def create_preprocsection(self): """Concatinates data, possibly trims it, then makes the unstranded reads usable""" forward = list() reverse = list() - for root, dirs, files in os.walk(self.config["folders"]["adapters"]): - if "NexteraPE-PE.fa" not in files: - self.logger.error( - "Adapters folder at {} does not contain NexteraPE-PE.fa. Review paths.yml" - ) - else: - break - trimdir = "{}/trimmed".format(self.finishdir) + + trimdir = f"{self.finishdir}/trimmed" files = self.verify_fastq() batchfile = open(self.batchfile, "a+") batchfile.write("#Trimmomatic section\n") - batchfile.write("mkdir {}\n".format(trimdir)) + batchfile.write(f"mkdir {trimdir}\n") batchfile.write("##Pre-concatination\n") for file in files: - fullfile = "{}/{}".format(self.indir, file) + fullfile = f"{self.indir}/{file}" # Even indexes = Forward if not files.index(file) % 2: forward.append(fullfile) @@ -391,57 +405,53 @@ def create_preprocsection(self): reverse.append(fullfile) outfile = files[0].split("_")[0] - self.concat_files["f"] = "{}/trimmed/{}_forward_reads.fastq.gz".format( - self.finishdir, self.name - ) - self.concat_files["r"] = "{}/trimmed/{}_reverse_reads.fastq.gz".format( - self.finishdir, self.name - ) - batchfile.write("cat {} > {}\n".format(" ".join(forward), self.concat_files.get("f"))) - batchfile.write("cat {} > {}\n".format(" ".join(reverse), self.concat_files.get("r"))) + self.concat_files["f"] = f"{self.finishdir}/trimmed/{self.name}_forward_reads.fastq.gz" + self.concat_files["r"] = f"{self.finishdir}/trimmed/{self.name}_reverse_reads.fastq.gz" + batchfile.write(f"cat {' '.join(forward)} > {self.concat_files.get('f')}\n") + batchfile.write(f"cat {' '.join(reverse)} > {self.concat_files.get('r')}\n") if self.trimmed: - fp = "{}/{}_trim_front_pair.fastq.gz".format(trimdir, outfile) - fu = "{}/{}_trim_front_unpair.fastq.gz".format(trimdir, outfile) - rp = "{}/{}_trim_rev_pair.fastq.gz".format(trimdir, outfile) - ru = "{}/{}_trim_rev_unpair.fastq.gz".format(trimdir, outfile) + fp = f"{trimdir}/{outfile}_trim_front_pair.fastq.gz" + fu = f"{trimdir}/{outfile}_trim_front_unpair.fastq.gz" + rp = f"{trimdir}/{outfile}_trim_rev_pair.fastq.gz" + ru = f"{trimdir}/{outfile}_trim_rev_unpair.fastq.gz" batchfile.write("##Trimming section\n") batchfile.write( - "trimmomatic PE -threads {} -phred33 {} {} {} {} {} {}\ - ILLUMINACLIP:{}/NexteraPE-PE.fa:2:30:10 LEADING:3 TRAILING:3 SLIDINGWINDOW:4:15 MINLEN:36\n".format( - self.config["slurm_header"]["threads"], - self.concat_files.get("f"), - self.concat_files.get("r"), - fp, - fu, - rp, - ru, - self.config["folders"]["adapters"], + self._singularity_exec( + "trimmomatic", + ( + f"trimmomatic PE -threads {self.slurm_header.threads}" + f" -phred33 {self.concat_files.get('f')} {self.concat_files.get('r')}" + f" {fp} {fu} {rp} {ru}" + f" ILLUMINACLIP:{self.singularity.trimmomatic_adapters}/NexteraPE-PE.fa:2:30:10" + " LEADING:3 TRAILING:3 SLIDINGWINDOW:4:15 MINLEN:36" + ), ) + + "\n" ) batchfile.write("## Interlaced trimmed files\n") self.concat_files["f"] = fp self.concat_files["r"] = rp - self.concat_files["i"] = "{}/{}_trim_unpair.fastq.gz".format(trimdir, outfile) + self.concat_files["i"] = f"{trimdir}/{outfile}_trim_unpair.fastq.gz" - batchfile.write("cat {} >> {}\n".format(" ".join([fu, ru]), self.concat_files.get("i"))) + batchfile.write(f"cat {' '.join([fu, ru])} >> {self.concat_files.get('i')}\n") batchfile.write("\n") batchfile.close() def create_assemblystats_section(self): batchfile = open(self.batchfile, "a+") batchfile.write("# QUAST QC metrics\n") - batchfile.write("mkdir {}/assembly/quast\n".format(self.finishdir)) + batchfile.write(f"mkdir {self.finishdir}/assembly/quast\n") batchfile.write( - "quast.py {}/assembly/{}_contigs.fasta -o {}/assembly/quast\n".format( - self.finishdir, self.name, self.finishdir + self._singularity_exec( + "quast", + f"quast.py {self.finishdir}/assembly/{self.name}_contigs.fasta -o {self.finishdir}/assembly/quast", ) + + "\n" ) batchfile.write( - "mv {}/assembly/quast/report.tsv {}/assembly/quast/{}_report.tsv\n\n".format( - self.finishdir, self.finishdir, self.name - ) + f"mv {self.finishdir}/assembly/quast/report.tsv {self.finishdir}/assembly/quast/{self.name}_report.tsv\n\n" ) batchfile.close() @@ -460,43 +470,31 @@ def create_snpsection(self): name = item.split("/")[-2] if "_" in name: name = name.split("_")[0] - batchfile.write("# Basecalling for sample {}\n".format(name)) - ref = "{}/{}.fasta".format( - self.config["folders"]["genomes"], self.sample.get("reference") - ) - outbase = "{}/{}_{}".format(item, name, self.sample.get("reference")) + batchfile.write(f"# Basecalling for sample {name}\n") + ref = f"{self.folders.genomes}/{self.sample.get('reference')}.fasta" + outbase = f"{item}/{name}_{self.sample.get('reference')}" batchfile.write( - "samtools view -h -q 1 -F 4 -F 256 {}.bam_sort_rmdup | grep -v XA:Z | grep -v SA:Z| samtools view -b - > {}/{}.unique\n".format( - outbase, self.finishdir, name - ) + f"samtools view -h -q 1 -F 4 -F 256 {outbase}.bam_sort_rmdup | grep -v XA:Z | grep -v SA:Z| samtools view -b - > {self.finishdir}/{name}.unique\n" ) batchfile.write( - "freebayes -= --pvar 0.7 -j -J --standard-filters -C 6 --min-coverage 30 --ploidy 1 -f {} -b {}/{}.unique -v {}/{}.vcf\n".format( - ref, self.finishdir, name, self.finishdir, name - ) + f"freebayes -= --pvar 0.7 -j -J --standard-filters -C 6 --min-coverage 30 --ploidy 1 -f {ref} -b {self.finishdir}/{name}.unique -v {self.finishdir}/{name}.vcf\n" ) batchfile.write( - "bcftools view {}/{}.vcf -o {}/{}.bcf.gz -O b --exclude-uncalled --types snps\n".format( - self.finishdir, name, self.finishdir, name - ) + f"bcftools view {self.finishdir}/{name}.vcf -o {self.finishdir}/{name}.bcf.gz -O b --exclude-uncalled --types snps\n" ) - batchfile.write("bcftools index {}/{}.bcf.gz\n".format(self.finishdir, name)) + batchfile.write(f"bcftools index {self.finishdir}/{name}.bcf.gz\n") batchfile.write("\n") batchfile.write( - "vcftools --bcf {}/{}.bcf.gz {} --remove-filtered-all --recode-INFO-all --recode-bcf --out {}/{}\n".format( - self.finishdir, name, vcffilter, self.finishdir, name - ) + f"vcftools --bcf {self.finishdir}/{name}.bcf.gz {vcffilter} --remove-filtered-all --recode-INFO-all --recode-bcf --out {self.finishdir}/{name}\n" ) batchfile.write( - 'bcftools view {}/{}.recode.bcf -i "{}" -o {}/{}.recode.bcf.gz -O b --exclude-uncalled --types snps\n'.format( - self.finishdir, name, bcffilter, self.finishdir, name - ) + f'bcftools view {self.finishdir}/{name}.recode.bcf -i "{bcffilter}" -o {self.finishdir}/{name}.recode.bcf.gz -O b --exclude-uncalled --types snps\n' ) - batchfile.write("bcftools index {}/{}.recode.bcf.gz\n\n".format(self.finishdir, name)) + batchfile.write(f"bcftools index {self.finishdir}/{name}.recode.bcf.gz\n\n") batchfile.write("# SNP pair-wise distance\n") - batchfile.write("touch {}/stats.out\n".format(self.finishdir)) + batchfile.write(f"touch {self.finishdir}/stats.out\n") while len(snplist) > 1: nameOne = "" nameTwo = "" @@ -511,83 +509,94 @@ def create_snpsection(self): if "_" in nameTwo: nameTwo = nameTwo.split("_")[0] - pair = "{}_{}".format(nameOne, nameTwo) + pair = f"{nameOne}_{nameTwo}" batchfile.write( - "bcftools isec {}/{}.recode.bcf.gz {}/{}.recode.bcf.gz -n=1 -c all -p {}/tmp -O b\n".format( - self.finishdir, nameOne, self.finishdir, nameTwo, self.finishdir - ) + f"bcftools isec {self.finishdir}/{nameOne}.recode.bcf.gz {self.finishdir}/{nameTwo}.recode.bcf.gz -n=1 -c all -p {self.finishdir}/tmp -O b\n" ) batchfile.write( - "bcftools merge -O b -o {}/{}.bcf.gz --force-samples {}/tmp/0000.bcf {}/tmp/0001.bcf\n".format( - self.finishdir, pair, self.finishdir, self.finishdir - ) + f"bcftools merge -O b -o {self.finishdir}/{pair}.bcf.gz --force-samples {self.finishdir}/tmp/0000.bcf {self.finishdir}/tmp/0001.bcf\n" ) - batchfile.write("bcftools index {}/{}.bcf.gz\n".format(self.finishdir, pair)) + batchfile.write(f"bcftools index {self.finishdir}/{pair}.bcf.gz\n") batchfile.write( - "echo {} $( bcftools stats {}/{}.bcf.gz |grep SNPs: | cut -d $'\\t' -f4 ) >> {}/stats.out\n".format( - pair, self.finishdir, pair, self.finishdir - ) + f"echo {pair} $( bcftools stats {self.finishdir}/{pair}.bcf.gz |grep SNPs: | cut -d $'\\t' -f4 ) >> {self.finishdir}/stats.out\n" ) batchfile.write("\n") batchfile.close() def create_collection(self): """Creates collection entry in database""" - if self.db_pusher.exists("Collections", {"ID_collection": self.name}): - self.db_pusher.purge_rec(name=self.name, type="Collections") + if self.db_pusher.get_collection_by_id(self.name): + self.db_pusher.delete_collection(self.name) for sample in self.pool: - self.db_pusher.add_rec( - {"ID_collection": self.name, "CG_ID_sample": sample}, "Collections" - ) + self.db_pusher.add_collection(ID_collectiuon=self.name, CG_ID_sample=sample) - addedprojs = list() + addedprojs = [] for sample in self.pool: - proj = re.search(r"(\w+)A(?:\w+)", sample).group(1) - if proj not in addedprojs: - self.create_project(proj) - addedprojs.append(proj) - - def create_project(self, name): - """Creates project in database""" - proj_col = dict() - proj_col["CG_ID_project"] = name - proj_col["Customer_ID_project"] = self.sample.get("Customer_ID_project") - proj_col["Customer_ID"] = self.sample.get("Customer_ID") - self.db_pusher.add_rec(proj_col, "Projects") - self.db_pusher.upd_rec({"CG_ID_project": name}, "Projects", proj_col) - - def create_sample(self, name): - """Creates sample in database""" - try: - sample_col = self.db_pusher.get_columns("Samples") - sample_col["CG_ID_sample"] = self.sample.get("CG_ID_sample") - sample_col["CG_ID_project"] = self.sample.get("CG_ID_project") - sample_col["Customer_ID_sample"] = self.sample.get("Customer_ID_sample") - sample_col["reference_genome"] = self.sample.get("reference") - sample_col["reference_length"] = self.sample.get("reference_length") - sample_col["date_analysis"] = self.dt - sample_col["organism"] = self.sample.get("organism") - sample_col["application_tag"] = self.sample.get("application_tag") - sample_col["priority"] = self.sample.get("priority") - sample_col["date_arrival"] = datetime.strptime( - self.sample.get("date_arrival"), "%Y-%m-%d %H:%M:%S" - ) - sample_col["date_sequencing"] = datetime.strptime( - self.sample.get("date_sequencing"), "%Y-%m-%d %H:%M:%S" + lims_project = re.search(r"(\w+)A(?:\w+)", sample).group(1) + projects: Projects | None = self.db_pusher.read_report(lims_project) + if not projects and lims_project not in addedprojs: + self.create_project(lims_project) + addedprojs.append(lims_project) + + def create_project(self, name: str) -> None: + """Creates or updates a project in the database.""" + if not self.sample: + raise JobCreationError( + "No sample information provided. Cannot create project in database." ) - sample_col["date_libprep"] = datetime.strptime( - self.sample.get("date_libprep"), "%Y-%m-%d %H:%M:%S" - ) - sample_col["method_libprep"] = self.sample.get("method_libprep") - sample_col["method_sequencing"] = self.sample.get("method_sequencing") - # self.db_pusher.purge_rec(sample_col['CG_ID_sample'], 'sample') - self.db_pusher.add_rec(sample_col, "Samples") - except Exception: - self.logger.error("Unable to add sample {} to database".format(self.name)) + project_data: dict[str, str] = { + "CG_ID_project": name, + "Customer_ID_project": self.sample["Customer_ID_project"], + "Customer_ID": self.sample["Customer_ID"], + } + if self.db_pusher.get_projects_by_cg_id_project(name): + update_data = {k: v for k, v in project_data.items() if k != "CG_ID_project"} + self.db_pusher.update_project({"CG_ID_project": name}, update_data) + else: + self.db_pusher.add_to_session(self.db_pusher.add_project(**project_data)) + self.db_pusher.commit_session() + + def create_sample(self) -> None: + """Creates or updates a sample in the database.""" + try: + if not self.sample: + raise JobCreationError( + "No sample information provided. Cannot create sample in database." + ) + sample_data: dict[str, str | datetime] = { + "CG_ID_sample": self.sample["CG_ID_sample"], + "CG_ID_project": self.sample["CG_ID_project"], + "Customer_ID_sample": self.sample["Customer_ID_sample"], + "reference_genome": self.sample["reference"], + "reference_length": self.sample["reference_length"], + "date_analysis": self.dt, + "organism": self.sample["organism"], + "application_tag": self.sample["application_tag"], + "priority": self.sample["priority"], + "date_arrival": datetime.strptime(self.sample["date_arrival"], "%Y-%m-%d %H:%M:%S"), + "date_sequencing": datetime.strptime( + self.sample["date_sequencing"], "%Y-%m-%d %H:%M:%S" + ), + "date_libprep": datetime.strptime(self.sample["date_libprep"], "%Y-%m-%d %H:%M:%S"), + "method_libprep": self.sample["method_libprep"], + "method_sequencing": self.sample["method_sequencing"], + } + cg_id = self.sample["CG_ID_sample"] + if self.db_pusher.get_sample_by_cg_id_sample(cg_id): + update_data = {k: v for k, v in sample_data.items() if k != "CG_ID_sample"} + self.db_pusher.update_sample({"CG_ID_sample": cg_id}, update_data) + else: + self.db_pusher.add_to_session(self.db_pusher.add_sample(**sample_data)) + self.db_pusher.commit_session() + except JobCreationError as e: + self.logger.error(f"Unable to add sample {self.name} to database: {e}") + except KeyError as e: + self.logger.error(f"Missing key {e} in sample information for sample {self.name}") + raise JobCreationError(f"Missing key {e} in sample information for sample {self.name}") def project_job(self, single_sample=False): - if "dry" in self.config and self.config["dry"] == True: + if self.dry: dry = True else: dry = False @@ -604,34 +613,32 @@ def project_job(self, single_sample=False): else: self.create_project(self.name) except Exception: - self.logger.error( - "LIMS interaction failed. Unable to read/write project {}".format(self.name) - ) + self.logger.error(f"LIMS interaction failed. Unable to read/write project {self.name}") # Writes the job creation sbatch if single_sample: try: self.sample_job() headerargs = self.get_headerargs() outfile = self.get_sbatch() - bash_cmd = "sbatch {} {}".format(headerargs, outfile) + bash_cmd = f"sbatch {headerargs} {outfile}" if not dry and outfile != "": samproc = subprocess.Popen(bash_cmd.split(), stdout=subprocess.PIPE) output, error = samproc.communicate() jobno = re.search(r"(\d+)", str(output)).group(0) jobarray.append(jobno) else: - self.logger.info("Suppressed command: {}".format(bash_cmd)) + self.logger.info(f"Suppressed command: {bash_cmd}") except Exception: - self.logger.error("Unable to analyze single sample {}".format(self.name)) + self.logger.error(f"Unable to analyze single sample {self.name}") else: - for ldir in glob.glob("{}/*/".format(self.indir)): + for ldir in glob.glob(f"{self.indir}/*/"): ldir = os.path.basename(os.path.normpath(ldir)) try: - sample_in = "{}/{}".format(self.indir, ldir) - sample_out = "{}/{}".format(self.finishdir, ldir) + sample_in = f"{self.indir}/{ldir}" + sample_out = f"{self.finishdir}/{ldir}" local_sampleinfo = [p for p in self.sampleinfo if p["CG_ID_sample"] == ldir] if local_sampleinfo == []: - raise Exception("Sample {} has no counterpart in json file".format(ldir)) + raise Exception(f"Sample {ldir} has no counterpart in json file") else: local_sampleinfo = local_sampleinfo[0] sample_settings = dict(self.run_settings) @@ -639,8 +646,17 @@ def project_job(self, single_sample=False): sample_settings["finishdir"] = sample_out sample_settings["timestamp"] = self.now sample_instance = Job_Creator( - config=self.config, log=self.logger, + folders=self.folders, + slurm_header=self.slurm_header, + regex=self.regex, + dry=self.dry, + config_path=self.config_path, + threshold=self.threshold, + pubmlst=self.pubmlst, + pasteur=self.pasteur, + singularity=self.singularity, + containers=self.containers, sampleinfo=local_sampleinfo, run_settings=sample_settings, ) @@ -649,27 +665,46 @@ def project_job(self, single_sample=False): outfile = "" if os.path.isfile(sample_instance.get_sbatch()): outfile = sample_instance.get_sbatch() - bash_cmd = "sbatch {} {}".format(headerargs, outfile) + bash_cmd = f"sbatch {headerargs} {outfile}" if not dry and outfile != "": projproc = subprocess.Popen(bash_cmd.split(), stdout=subprocess.PIPE) output, error = projproc.communicate() jobno = re.search(r"(\d+)", str(output)).group(0) jobarray.append(jobno) else: - self.logger.info("Suppressed command: {}".format(bash_cmd)) + self.logger.info(f"Suppressed command: {bash_cmd}") except Exception: pass if not dry: self.finish_job(jobarray, single_sample) + def _write_mailjob(self, mailfile: str, report: str, custom_conf: str) -> None: + """Write the mailjob.sh script that runs `microsalt utils finish` after all jobs complete.""" + _ep = next( + ep for ep in entry_points(group="console_scripts") if ep.value == "microSALT.cli:root" + ) + microsalt_bin = Path(sys.executable).parent / _ep.name + with open(mailfile, "w+") as mb: + mb.write("#!/usr/bin/env bash\n\n") + mb.write("#Uploading of results to database and production of report\n") + finish_cmd = ( + f"{microsalt_bin} --config {self.config_path} utils finish {self.finishdir}/sampleinfo.json " + f"--input {self.finishdir} " + f"--email {self.regex.mail_recipient} " + f"--report {report} " + f"{custom_conf}\n" + ) + mb.write(finish_cmd) + mb.write(f"touch {self.finishdir}/run_complete.out\n") + def finish_job(self, joblist, single_sample=False): """Uploads data and sends an email once all analysis jobs are complete.""" report = "default" if self.qc_only: report = "qc" custom_conf = "" - if "config_path" in self.config: - custom_conf = "--config {}".format(self.config["config_path"]) + if self.config_path: + custom_conf = f"--config {self.config_path}" process = subprocess.Popen("id -un".split(), stdout=subprocess.PIPE) user, error = process.communicate() @@ -677,36 +712,25 @@ def finish_job(self, joblist, single_sample=False): # if not os.path.exists(self.finishdir): # os.makedirs(self.finishdir) - startfile = "{}/run_started.out".format(self.finishdir) - configfile = "{}/config.log".format(self.finishdir) - mailfile = "{}/mailjob.sh".format(self.finishdir) - samplefile = "{}/sampleinfo.json".format(self.finishdir) + startfile = f"{self.finishdir}/run_started.out" + configfile = f"{self.finishdir}/config.log" + mailfile = f"{self.finishdir}/mailjob.sh" + samplefile = f"{self.finishdir}/sampleinfo.json" with open(samplefile, "w+") as outfile: json.dump(self.sampleinfo, outfile) with open(startfile, "w+") as sb: sb.write("#!/usr/bin/env bash\n") with open(configfile, "w+") as cb: - configout = self.config.copy() - if "genologics" in configout: - del configout["genologics"] - cb.write("ANALYSIS STARTED BY: {}\n".format(user)) + configout = { + "folders": self.folders.model_dump(), + "slurm_header": self.slurm_header.model_dump(), + "regex": self.regex.model_dump(), + "threshold": self.threshold.model_dump(), + } + cb.write(f"ANALYSIS STARTED BY: {user}\n") cb.write(json.dumps(configout, indent=2, separators=(",", ":"))) - with open(mailfile, "w+") as mb: - mb.write("#!/usr/bin/env bash\n\n") - mb.write("#Uploading of results to database and production of report\n") - if "MICROSALT_CONFIG" in os.environ: - mb.write(f"export MICROSALT_CONFIG={os.environ['MICROSALT_CONFIG']}\n") - conda_cmd = ( - f"conda run -p {os.environ['CONDA_PREFIX']} " - f"microSALT utils finish {self.finishdir}/sampleinfo.json " - f"--input {self.finishdir} " - f"--email {self.config['regex']['mail_recipient']} " - f"--report {report} " - f"{custom_conf}\n" - ) - mb.write(conda_cmd) - mb.write(f"touch {self.finishdir}/run_complete.out\n") + self._write_mailjob(mailfile, report, custom_conf) massagedJobs = list() final = ":".join(joblist) @@ -722,29 +746,23 @@ def finish_job(self, joblist, single_sample=False): i += maxlen for entry in massagedJobs: if massagedJobs.index(entry) < len(massagedJobs) - 1: - head = "-A {} -p core -n 1 -t 00:00:10 -J {}_{}_SUBTRACKER --qos {} --dependency=afterany:{}".format( - self.config["slurm_header"]["project"], - self.config["slurm_header"]["job_prefix"], - self.name, - self.config["slurm_header"]["qos"], - entry, - ) - bash_cmd = "sbatch {} {}".format(head, startfile) + head = f"-A {self.slurm_header.project} -p core -n 1 -t 00:00:10 -J {self.slurm_header.job_prefix}_{self.name}_SUBTRACKER --qos {self.slurm_header.qos} --dependency=afterany:{entry}" + bash_cmd = f"sbatch {head} {startfile}" mailproc = subprocess.Popen(bash_cmd.split(), stdout=subprocess.PIPE) output, error = mailproc.communicate() jobno = re.search(r"(\d+)", str(output)).group(0) - massagedJobs[massagedJobs.index(entry) + 1] += ":{}".format(jobno) + massagedJobs[massagedJobs.index(entry) + 1] += f":{jobno}" else: final = entry break head = ( - f"-A {self.config['slurm_header']['project']} -p core -n 1 -t 6:00:00 " - f"-J {self.config['slurm_header']['job_prefix']}_{self.name}_MAILJOB " - f"--qos {self.config['slurm_header']['qos']} --open-mode append " - f"--dependency=afterany:{final} --output {self.config['folders']['log_file']}" + f"-A {self.slurm_header.project} -p core -n 1 -t 6:00:00 " + f"-J {self.slurm_header.job_prefix}_{self.name}_MAILJOB " + f"--qos {self.slurm_header.qos} --open-mode append " + f"--dependency=afterany:{final} --output {self.finishdir}/mailjob.out" ) - bash_cmd = "sbatch {} {}".format(head, mailfile) + bash_cmd = f"sbatch {head} {mailfile}" mailproc = subprocess.Popen(bash_cmd.split(), stdout=subprocess.PIPE) output, error = mailproc.communicate() @@ -752,12 +770,12 @@ def finish_job(self, joblist, single_sample=False): jobno = str(re.search(r"(\d+)", str(output)).group(0)) joblist.append(jobno) except Exception: - self.logger.info("Unable to grab SLURMID for {0}".format(self.name)) + self.logger.info(f"Unable to grab SLURMID for {self.name}") try: # Generates file with all slurm ids - slurmname = "{}_slurm_ids.yaml".format(self.name) - slurmreport_storedir = Path(self.config["folders"]["reports"], "trailblazer", slurmname) + slurmname = f"{self.name}_slurm_ids.yaml" + slurmreport_storedir = Path(self.folders.reports, "trailblazer", slurmname) slurmreport_workdir = Path(self.finishdir, slurmname) with slurmreport_workdir.open("w") as slurmreport_file: yaml.safe_dump( @@ -780,10 +798,10 @@ def sample_job(self): os.makedirs(self.finishdir) try: # This is one job - self.batchfile = "{}/runfile.sbatch".format(self.finishdir) + self.batchfile = f"{self.finishdir}/runfile.sbatch" batchfile = open(self.batchfile, "w+") batchfile.write("#!/bin/sh\n\n") - batchfile.write("mkdir -p {}\n".format(self.finishdir)) + batchfile.write(f"mkdir -p {self.finishdir}\n") batchfile.close() self.create_preprocsection() self.create_variantsection() @@ -795,37 +813,32 @@ def sample_job(self): batchfile.close() self.logger.info( - "Created runfile for sample {} in folder {}".format(self.name, self.finishdir) + f"Created runfile for sample {self.name} in folder {self.finishdir}" ) except Exception: raise try: - self.create_sample(self.name) + self.create_sample() except Exception: - self.logger.error("Unable to access LIMS info for sample {}".format(self.name)) + self.logger.error(f"Unable to access LIMS info for sample {self.name}") except Exception as e: - self.logger.error( - "Unable to create job for sample {}\nSource: {}".format(self.name, str(e)) - ) + self.logger.error(f"Unable to create job for sample {self.name}\nSource: {e!s}") shutil.rmtree(self.finishdir, ignore_errors=True) raise def create_blast_search(self): reforganism = self.ref_resolver.organism2reference(self.sample.get("organism")) - self.batchfile = "{}/runfile.sbatch".format(self.finishdir) + self.batchfile = f"{self.finishdir}/runfile.sbatch" batchfile = open(self.batchfile, "a+") - batchfile.write("mkdir -p {}/blast_search\n".format(self.finishdir)) + batchfile.write(f"mkdir -p {self.finishdir}/blast_search\n") batchfile.close() self.blast_subset( "mlst", - "{}/{}/*.tfa".format(self.config["folders"]["references"], reforganism), + f"{self.folders.references}/{reforganism}/*.tfa", ) - self.blast_subset("resistance", "{}/*.fsa".format(self.config["folders"]["resistances"])) + self.blast_subset("resistance", f"{self.folders.resistances}/*.fsa") if reforganism == "escherichia_coli": - ss = "{}/*{}".format( - os.path.dirname(self.config["folders"]["expec"]), - os.path.splitext(self.config["folders"]["expec"])[1], - ) + ss = f"{os.path.dirname(self.folders.expec)}/*{os.path.splitext(self.folders.expec)[1]}" self.blast_subset("expec", ss) def snp_job(self): @@ -833,28 +846,18 @@ def snp_job(self): if not os.path.exists(self.finishdir): os.makedirs(self.finishdir) - self.batchfile = "{}/runfile.sbatch".format(self.finishdir) + self.batchfile = f"{self.finishdir}/runfile.sbatch" batchfile = open(self.batchfile, "w+") batchfile.write("#!/usr/bin/env bash\n") - batchfile.write("mkdir -p {}\n\n".format(self.finishdir)) + batchfile.write(f"mkdir -p {self.finishdir}\n\n") batchfile.close() self.create_snpsection() batchfile = open(self.batchfile, "a+") batchfile.close() - headerline = ( - "-A {} -p {} -n 1 -t 24:00:00 -J {}_{} --qos {} --output {}/slurm_{}.log".format( - self.config["slurm_header"]["project"], - self.config["slurm_header"]["type"], - self.config["slurm_header"]["job_prefix"], - self.name, - self.config["slurm_header"]["qos"], - self.finishdir, - self.name, - ) - ) + headerline = f"-A {self.slurm_header.project} -p {self.slurm_header.type} -n 1 -t 24:00:00 -J {self.slurm_header.job_prefix}_{self.name} --qos {self.slurm_header.qos} --output {self.finishdir}/slurm_{self.name}.log" outfile = self.get_sbatch() - bash_cmd = "sbatch {} {}".format(headerline, outfile) + bash_cmd = f"sbatch {headerline} {outfile}" samproc = subprocess.Popen(bash_cmd.split(), stdout=subprocess.PIPE) output, error = samproc.communicate() diff --git a/microSALT/utils/pubmlst/authentication.py b/microSALT/utils/pubmlst/authentication.py index 794221df..69ba7c77 100644 --- a/microSALT/utils/pubmlst/authentication.py +++ b/microSALT/utils/pubmlst/authentication.py @@ -1,35 +1,45 @@ import json +import logging import os from datetime import datetime, timedelta from dateutil import parser -from rauth import OAuth1Session +from requests_oauthlib import OAuth1Session -from microSALT import app, logger +from microSALT.config import Folders, PubMLSTCredentials, PasteurCredentials +from microSALT.utils.pubmlst.constants import CREDENTIALS_KEY from microSALT.utils.pubmlst.exceptions import ( PubMLSTError, SessionTokenRequestError, SessionTokenResponseError, ) from microSALT.utils.pubmlst.helpers import ( - folders_config, get_path, get_service_config, load_auth_credentials, save_session_token, ) -from microSALT.utils.pubmlst.constants import CREDENTIALS_KEY + +logger = logging.getLogger("main_logger") session_token_validity = 12 # 12-hour validity session_expiration_buffer = 60 # 60-second buffer class ClientAuthentication: - - def __init__(self, service: str): + def __init__( + self, + service: str, + folders: Folders, + pubmlst: PubMLSTCredentials, + pasteur: PasteurCredentials, + ): """Initialize the client with the specified service.""" self.service: str = service - self.service_config: dict = get_service_config(service) + self.folders = folders + self.pubmlst = pubmlst + self.pasteur = pasteur + self.service_config: dict = get_service_config(service, pubmlst=pubmlst, pasteur=pasteur) self.base_api: str = self.service_config["base_api"] def get_new_session_token(self, db: str): @@ -37,7 +47,7 @@ def get_new_session_token(self, db: str): try: consumer_key, consumer_secret, access_token, access_secret = load_auth_credentials( - self.service + self.service, self.folders, self.pubmlst, self.pasteur ) logger.debug(f"Consumer Key: {consumer_key[:5]}***") logger.debug(f"Consumer Secret: {consumer_secret[:5]}***") @@ -49,10 +59,11 @@ def get_new_session_token(self, db: str): logger.debug(f"Requesting session token from URL: {url}") session = OAuth1Session( - consumer_key=consumer_key, - consumer_secret=consumer_secret, - access_token=access_token, - access_token_secret=access_secret, + client_key=consumer_key, + client_secret=consumer_secret, + resource_owner_key=access_token, + resource_owner_secret=access_secret, + signature_type="query", ) response = session.get(url, headers={"User-Agent": "BIGSdb API downloader"}) @@ -78,6 +89,9 @@ def get_new_session_token(self, db: str): token=session_token, secret=session_secret, expiration_date=expiration_time, + folders=self.folders, + pubmlst=self.pubmlst, + pasteur=self.pasteur, ) return session_token, session_secret @@ -102,7 +116,7 @@ def load_session_credentials( """Load session token from file for a specific database.""" try: credentials_file = os.path.join( - get_path(folders_config, CREDENTIALS_KEY), + get_path(self.folders, CREDENTIALS_KEY), self.service_config["session_credentials_file_name"], ) diff --git a/microSALT/utils/pubmlst/client.py b/microSALT/utils/pubmlst/client.py index 8ee5e617..10dac147 100644 --- a/microSALT/utils/pubmlst/client.py +++ b/microSALT/utils/pubmlst/client.py @@ -1,9 +1,10 @@ from urllib.parse import urlencode +import logging import requests from werkzeug.exceptions import NotFound -from rauth import OAuth1Session +from requests_oauthlib import OAuth1Session -from microSALT import logger +from microSALT.config import Folders, PubMLSTCredentials, PasteurCredentials from microSALT.utils.pubmlst.authentication import ClientAuthentication from microSALT.utils.pubmlst.constants import HTTPMethod, RequestType, ResponseHandler from microSALT.utils.pubmlst.exceptions import ( @@ -20,19 +21,31 @@ should_skip_endpoint, ) +logger = logging.getLogger("main_logger") + class BaseClient: """Base client for interacting with authenticated APIs.""" - def __init__(self, service: str, database: str = None): + def __init__( + self, + service: str, + database: str = None, + folders: Folders = None, + pubmlst: PubMLSTCredentials = None, + pasteur: PasteurCredentials = None, + ): """Initialize the client with the specified service.""" try: self.service = service + self.folders = folders + self.pubmlst = pubmlst + self.pasteur = pasteur self.consumer_key, self.consumer_secret, self.access_token, self.access_secret = ( - load_auth_credentials(service) + load_auth_credentials(service, folders, pubmlst, pasteur) ) - self.client_auth = ClientAuthentication(service) - service_config = get_service_config(service) + self.client_auth = ClientAuthentication(service, folders, pubmlst, pasteur) + service_config = get_service_config(service, pubmlst=pubmlst, pasteur=pasteur) self.base_api = service_config["base_api"] self.database = database or service_config["database"] self.base_api_host = service_config["base_api_host"] @@ -85,7 +98,8 @@ def _make_request( if request_type == RequestType.DB: token, secret = ( self.client_auth.load_session_credentials(db) - if db else (self.session_token, self.session_secret) + if db + else (self.session_token, self.session_secret) ) elif request_type == RequestType.AUTH: token, secret = self.access_token, self.access_secret @@ -94,10 +108,11 @@ def _make_request( # Create session with OAuth1 session = OAuth1Session( - self.consumer_key, - self.consumer_secret, - access_token=token, - access_token_secret=secret, + client_key=self.consumer_key, + client_secret=self.consumer_secret, + resource_owner_key=token, + resource_owner_secret=secret, + signature_type="query", ) response = session.request(method.value, url) @@ -197,24 +212,43 @@ def list_schemes(self, db: str): class PubMLSTClient(BaseClient): """Client for interacting with the PubMLST authenticated API.""" - def __init__(self): + def __init__( + self, + folders: Folders = None, + pubmlst: PubMLSTCredentials = None, + pasteur: PasteurCredentials = None, + ): """Initialize the PubMLST client.""" - super().__init__(service="pubmlst") + super().__init__(service="pubmlst", folders=folders, pubmlst=pubmlst, pasteur=pasteur) class PasteurClient(BaseClient): """Client for interacting with the Pasteur authenticated API.""" - def __init__(self, database: str): + def __init__( + self, + database: str, + folders: Folders = None, + pubmlst: PubMLSTCredentials = None, + pasteur: PasteurCredentials = None, + ): """Initialize the Pasteur client.""" - super().__init__(service="pasteur", database=database) + super().__init__( + service="pasteur", database=database, folders=folders, pubmlst=pubmlst, pasteur=pasteur + ) -def get_client(service: str, database: str = None): +def get_client( + service: str, + database: str = None, + folders: Folders = None, + pubmlst: PubMLSTCredentials = None, + pasteur: PasteurCredentials = None, +): """Get the appropriate client for the specified service.""" if service == "pasteur": - return PasteurClient(database=database) + return PasteurClient(database=database, folders=folders, pubmlst=pubmlst, pasteur=pasteur) elif service == "pubmlst": - return PubMLSTClient() + return PubMLSTClient(folders=folders, pubmlst=pubmlst, pasteur=pasteur) else: raise ValueError(f"Unknown service: {service}") diff --git a/microSALT/utils/pubmlst/get_credentials.py b/microSALT/utils/pubmlst/get_credentials.py index 6fbf05e3..9515fd44 100644 --- a/microSALT/utils/pubmlst/get_credentials.py +++ b/microSALT/utils/pubmlst/get_credentials.py @@ -1,11 +1,13 @@ -import sys import os +import sys -from argparse import ArgumentParser -from rauth import OAuth1Service -from microSALT import app -from microSALT.utils.pubmlst.helpers import get_path, get_service_config, folders_config +from requests_oauthlib import OAuth1Session + +from microSALT.config import ( + MicroSALTConfig, +) from microSALT.utils.pubmlst.constants import CREDENTIALS_KEY +from microSALT.utils.pubmlst.helpers import get_path, get_service_config def validate_credentials(client_id, client_secret): @@ -16,27 +18,27 @@ def validate_credentials(client_id, client_secret): raise ValueError("Invalid CLIENT_SECRET: It must not be empty.") -def get_request_token(service): - """Handle JSON response from the request token endpoint.""" - response = service.get_raw_request_token(params={"oauth_callback": "oob"}) +def get_new_access_token( + client_id, client_secret, db: str, base_api: str, base_web: str +) -> tuple[str, str]: + """Obtain a new access token and secret. + + BIGSdb OAuth endpoints require GET (not POST), and expect OAuth parameters + as query string parameters (not in the Authorization header), so we use + signature_type='query' and oauth.get() directly. + """ + # Step 1: GET request token + oauth = OAuth1Session( + client_id, client_secret=client_secret, callback_uri="oob", signature_type="query" + ) + response = oauth.get(f"{base_api}/db/{db}/oauth/get_request_token") if not response.ok: print(f"Error obtaining request token: {response.text}") sys.exit(1) - data = response.json() - return data["oauth_token"], data["oauth_token_secret"] - - -def get_new_access_token(client_id, client_secret, db: str, base_api: str, base_web: str): - """Obtain a new access token and secret.""" - service = OAuth1Service( - name="BIGSdb_downloader", - consumer_key=client_id, - consumer_secret=client_secret, - request_token_url=f"{base_api}/db/{db}/oauth/get_request_token", - access_token_url=f"{base_api}/db/{db}/oauth/get_access_token", - base_url=base_api, - ) - request_token, request_secret = get_request_token(service) + token_data = response.json() + request_token = token_data["oauth_token"] + request_secret = token_data["oauth_token_secret"] + print( "Please log in using your user account at " f"{base_web}?db={db}&page=authorizeClient&oauth_token={request_token} " @@ -44,20 +46,26 @@ def get_new_access_token(client_id, client_secret, db: str, base_api: str, base_ ) verifier = input("Please enter verification code: ") - raw_access = service.get_raw_access_token( - request_token, request_secret, params={"oauth_verifier": verifier} + # Step 2: GET access token + oauth = OAuth1Session( + client_id, + client_secret=client_secret, + resource_owner_key=request_token, + resource_owner_secret=request_secret, + verifier=verifier, + signature_type="query", ) - if not raw_access.ok: - print(f"Error obtaining access token: {raw_access.text}") + response = oauth.get(f"{base_api}/db/{db}/oauth/get_access_token") + if not response.ok: + print(f"Error obtaining access token: {response.text}") sys.exit(1) - - access_data = raw_access.json() + access_data = response.json() return access_data["oauth_token"], access_data["oauth_token_secret"] def save_to_credentials_py( client_id, client_secret, access_token, access_secret, credentials_path, credentials_file -): +) -> None: """Save tokens in the credentials.py file.""" credentials_path.mkdir(parents=True, exist_ok=True) @@ -69,12 +77,12 @@ def save_to_credentials_py( print(f"Tokens saved to {credentials_file}") -def main(service, species=None): +def main(service: str, config: MicroSALTConfig, species: str | None = None): try: - service_config = get_service_config(service) + service_config = get_service_config(service, pubmlst=config.pubmlst, pasteur=config.pasteur) bigsd_config = service_config["config"] - client_id = bigsd_config["client_id"] - client_secret = bigsd_config["client_secret"] + client_id = bigsd_config.client_id + client_secret = bigsd_config.client_secret validate_credentials(client_id, client_secret) # Determine the database @@ -87,7 +95,7 @@ def main(service, species=None): else: raise ValueError(f"Unknown service: {service}") - credentials_path = get_path(folders_config, CREDENTIALS_KEY) + credentials_path = get_path(folders=config.folders, config_key=CREDENTIALS_KEY) credentials_file = os.path.join( credentials_path, service_config.get("auth_credentials_file_name") ) @@ -115,21 +123,3 @@ def main(service, species=None): except Exception as e: print(f"Error: {e}") sys.exit(1) - - -if __name__ == "__main__": - parser = ArgumentParser(description="Get PubMLST or Pasteur credentials.") - parser.add_argument( - "-s", - "--service", - type=str, - default="pubmlst", - help="Service name (default: pubmlst)", - ) - parser.add_argument( - "--species", - type=str, - help="Species name (required for the 'pasteur' service)", - ) - args = parser.parse_args() - main(args.service, args.species) diff --git a/microSALT/utils/pubmlst/helpers.py b/microSALT/utils/pubmlst/helpers.py index 94b62874..f8c92539 100644 --- a/microSALT/utils/pubmlst/helpers.py +++ b/microSALT/utils/pubmlst/helpers.py @@ -1,22 +1,27 @@ -import os -import requests -import base64 -import hashlib import json -import hmac -import time +import logging +import os from pathlib import Path -from urllib.parse import quote_plus, urlencode -from microSALT import app, logger -from microSALT.utils.pubmlst.exceptions import PathResolutionError, CredentialsFileNotFound, InvalidCredentials, PubMLSTError, SaveSessionError, InvalidURLError -from microSALT.utils.pubmlst.constants import CREDENTIALS_KEY, Encoding, URL_MAPS -folders_config = app.config["folders"] +import requests + +from microSALT.config import Folders, PubMLSTCredentials, PasteurCredentials +from microSALT.utils.pubmlst.constants import CREDENTIALS_KEY, URL_MAPS +from microSALT.utils.pubmlst.exceptions import ( + CredentialsFileNotFound, + InvalidCredentials, + PathResolutionError, + PubMLSTError, + SaveSessionError, +) + +logger = logging.getLogger("main_logger") -def get_path(config, config_key: str): + +def get_path(folders: Folders, config_key: str) -> Path: """Get and expand the file path from the configuration.""" try: - path = config.get(config_key) + path = getattr(folders, config_key, None) if not path: raise PathResolutionError(config_key) @@ -29,11 +34,13 @@ def get_path(config, config_key: str): raise PathResolutionError(config_key) from e -def get_service_config(service: str): +def get_service_config(service: str, pubmlst: PubMLSTCredentials | None = None, pasteur: PasteurCredentials | None= None): """ Get the configuration for the specified service (e.g., 'pubmlst' or 'pasteur'). :param service: The name of the service ('pubmlst' or 'pasteur'). + :param pubmlst: PubMLST sub-config (optional). + :param pasteur: Pasteur sub-config (optional). :return: A dictionary containing the configuration for the service. """ services = { @@ -44,7 +51,7 @@ def get_service_config(service: str): "database": "pubmlst_test_seqdef", "auth_credentials_file_name": "pubmlst_credentials.env", "session_credentials_file_name": "pubmlst_session_credentials.json", - "config": app.config["pubmlst"], + "config": pubmlst, }, "pasteur": { "base_web": "https://bigsdb.pasteur.fr/cgi-bin/bigsdb/bigsdb.pl", @@ -52,7 +59,7 @@ def get_service_config(service: str): "base_api_host": "bigsdb.pasteur.fr", "auth_credentials_file_name": "pasteur_credentials.env", "session_credentials_file_name": "pasteur_session_credentials.json", - "config": app.config["pasteur"], + "config": pasteur, }, } @@ -88,17 +95,18 @@ def get_url_map(service: str): return url_map -def load_auth_credentials(service: str): +def load_auth_credentials(service: str, folders: Folders, pubmlst: PubMLSTCredentials, pasteur: PasteurCredentials): """ Load client ID, client secret, access token, and access secret from the credentials file for the specified service. :param service: The name of the service ('pubmlst' or 'pasteur'). + :param config: The MicroSALTConfig object. :return: A tuple containing the credentials (consumer_key, consumer_secret, access_token, access_secret). """ try: - service_config = get_service_config(service) + service_config = get_service_config(service, pubmlst=pubmlst, pasteur=pasteur) credentials_file = os.path.join( - get_path(folders_config, CREDENTIALS_KEY), + get_path(folders, CREDENTIALS_KEY), service_config["auth_credentials_file_name"], ) @@ -138,9 +146,9 @@ def load_auth_credentials(service: str): raise except Exception as e: raise PubMLSTError(f"An unexpected error occurred while loading {service} credentials: {e}") - -def save_session_token(service: str, db: str, token: str, secret: str, expiration_date: str): + +def save_session_token(service: str, db: str, token: str, secret: str, expiration_date: str, folders: Folders, pubmlst: PubMLSTCredentials, pasteur: PasteurCredentials): """ Save session token, secret, and expiration to a JSON file for the specified service and database. @@ -149,11 +157,12 @@ def save_session_token(service: str, db: str, token: str, secret: str, expiratio :param token: The session token. :param secret: The session secret. :param expiration_date: The expiration date of the session token. + :param config: The MicroSALTConfig object. """ try: - service_config = get_service_config(service) + service_config = get_service_config(service, pubmlst=pubmlst, pasteur=pasteur) session_file = os.path.join( - get_path(folders_config, CREDENTIALS_KEY), + get_path(folders, CREDENTIALS_KEY), service_config["session_credentials_file_name"], ) diff --git a/microSALT/utils/referencer.py b/microSALT/utils/referencer.py index b6cb012e..bcaff0eb 100644 --- a/microSALT/utils/referencer.py +++ b/microSALT/utils/referencer.py @@ -9,10 +9,10 @@ import subprocess import urllib.request import xml.etree.ElementTree as ET -from typing import Optional, Tuple, Union from Bio import Entrez +from microSALT.config import Folders, Threshold, PubMLSTCredentials, PasteurCredentials, Singularity, Containers from microSALT.store.db_manipulator import DB_Manipulator from microSALT.utils.pubmlst.client import BaseClient, get_client from microSALT.utils.pubmlst.exceptions import InvalidURLError, PubMLSTError @@ -20,10 +20,15 @@ class Referencer: - def __init__(self, config, log, sampleinfo={}, force=False): - self.config = config + def __init__(self, log, folders: Folders, threshold: Threshold, pubmlst: PubMLSTCredentials, pasteur: PasteurCredentials, singularity: Singularity = None, containers: Containers = None, sampleinfo={}, force=False): + self.folders = folders + self.threshold = threshold + self.pubmlst = pubmlst + self.pasteur = pasteur + self.singularity = singularity or Singularity() + self.containers = containers or Containers() self.logger = log - self.db_access = DB_Manipulator(config, log) + self.db_access = DB_Manipulator(log=log, folders=folders, threshold=threshold) self.updated = list() # Fetch names of existing refs self.refs = self.db_access.profiles @@ -50,7 +55,15 @@ def __init__(self, config, log, sampleinfo={}, force=False): def set_client(self, service: str, database: str = None): """Set the client for PubMLST API interactions.""" - self.client: BaseClient = get_client(service, database) + self.client: BaseClient = get_client(service, database, self.folders, self.pubmlst, self.pasteur) + + def _singularity_exec(self, tool: str, command: str) -> str: + """Return command wrapped with singularity exec for the given tool container.""" + sif = getattr(self.containers, tool) + binary = self.singularity.binary + bind_list = list(self.singularity.bind_paths) + bind = f"--bind {','.join(bind_list)}" if bind_list else "" + return f"{binary} exec {bind} {sif} {command}" def identify_new(self, cg_id="", project=False): """Automatically downloads pubMLST & NCBI organisms not already downloaded""" @@ -68,8 +81,8 @@ def identify_new(self, cg_id="", project=False): if ref not in self.organisms and org not in neworgs: neworgs.append(org) if ( - "{}.fasta".format(entry.get("reference")) - not in os.listdir(self.config["folders"]["genomes"]) + f"{entry.get('reference')}.fasta" + not in os.listdir(self.folders.genomes) and entry.get("reference") not in newrefs ): newrefs.append(entry.get("reference")) @@ -84,21 +97,25 @@ def identify_new(self, cg_id="", project=False): def update_refs(self): """Updates all references. Order is important, since no object is updated twice""" - # Updates - self.fetch_pubmlst(self.force) - self.fetch_external() - self.fetch_resistances(self.force) + self.db_access.acquire_ref_lock() + try: + # Updates + self.fetch_pubmlst(self.force) + self.fetch_external() + self.fetch_resistances(self.force) - # Reindexes - self.index_db(os.path.dirname(self.config["folders"]["expec"]), ".fsa") + # Reindexes + self.index_db(os.path.dirname(self.folders.expec), ".fsa") + finally: + self.db_access.release_ref_lock() def index_db(self, full_dir, suffix): """Check for indexation, makeblastdb job if not enough of them.""" reindexation = False files = os.listdir(full_dir) - sufx_files = glob.glob("{}/*{}".format(full_dir, suffix)) # List of source files + sufx_files = glob.glob(f"{full_dir}/*{suffix}") # List of source files for file in sufx_files: - subsuf = "\{}$".format(suffix) + subsuf = f"\\{suffix}$" base = re.sub(subsuf, "", file) bases = 0 @@ -108,7 +125,7 @@ def index_db(self, full_dir, suffix): if os.path.basename(base) == elem[: elem.rfind(".")]: bases = bases + 1 # Number of index files fresher than source (6) - if os.stat(file).st_mtime < os.stat("{}/{}".format(full_dir, elem)).st_mtime: + if os.stat(file).st_mtime < os.stat(f"{full_dir}/{elem}").st_mtime: newer = newer + 1 # 7 for parse_seqids, 4 for not. if not (bases == 7 or newer == 6) and not (bases == 4 and newer == 3): @@ -116,28 +133,26 @@ def index_db(self, full_dir, suffix): try: # Resistence files if ".fsa" in suffix: - bash_cmd = "makeblastdb -in {}/{} -dbtype nucl -out {}".format( - full_dir, os.path.basename(file), os.path.basename(base) + bash_cmd = self._singularity_exec('blast', + f"makeblastdb -in {full_dir}/{os.path.basename(file)} -dbtype nucl -out {os.path.basename(base)}" ) # MLST locis else: - bash_cmd = ( - "makeblastdb -in {}/{} -dbtype nucl -parse_seqids -out {}".format( - full_dir, os.path.basename(file), os.path.basename(base) - ) + bash_cmd = self._singularity_exec('blast', + f"makeblastdb -in {full_dir}/{os.path.basename(file)} -dbtype nucl -parse_seqids -out {os.path.basename(base)}" ) proc = subprocess.Popen(bash_cmd.split(), cwd=full_dir, stdout=subprocess.PIPE) proc.communicate() except Exception: self.logger.error( - "Unable to index requested target {} in {}".format(file, full_dir) + f"Unable to index requested target {file} in {full_dir}" ) if reindexation: - self.logger.info("Re-indexed contents of {}".format(full_dir)) + self.logger.info(f"Re-indexed contents of {full_dir}") def _parse_external_xml( self, url: str = "https://pubmlst.org/static/data/dbases.xml" - ) -> Optional[ET.Element]: + ) -> ET.Element | None: """Fetch and parse the external XML, returning the root element.""" try: query = urllib.request.urlopen(url).read() @@ -148,7 +163,7 @@ def _parse_external_xml( def _find_entry_for_organism( self, root: ET.Element, organism_name: str - ) -> Tuple[Optional[ET.Element], Optional[str]]: + ) -> tuple[ET.Element | None, str | None]: """Find the XML entry for a given organism name.""" organism_name = organism_name.lower().replace(" ", "_") for entry in root: @@ -162,9 +177,9 @@ def _find_entry_for_organism( return entry, organ return None, None - def _should_update_external(self, organ: str, entry: ET.Element) -> Union[dict, bool]: + def _should_update_external(self, organ: str, entry: ET.Element) -> dict | bool: """Determine if the external data for an organism should be updated.""" - currver = self.db_access.get_version(f"profile_{organ}") + currver = self.db_access.read_version(f"profile_{organ}") st_link = entry.find("./mlst/database/profiles/url").text service = get_service_by_url(st_link) if service == "pasteur": @@ -224,7 +239,7 @@ def _update_external_organism(self, update_info: dict) -> None: ) # Step 1: Download the profiles CSV - st_target = f"{self.config['folders']['profiles']}/{organ}" + st_target = f"{self.folders.profiles}/{organ}" profiles_csv = self.client.download_profiles_csv(db, scheme_id) profiles_csv = profiles_csv.split("\n") trimmed_profiles = [] @@ -238,7 +253,7 @@ def _update_external_organism(self, update_info: dict) -> None: loci_list = scheme_info.get("loci", []) # Step 3: Download loci FASTA files - output = f"{self.config['folders']['references']}/{organ}" + output = f"{self.folders.references}/{organ}" if os.path.isdir(output): shutil.rmtree(output) os.makedirs(output) @@ -305,40 +320,40 @@ def resync(self, type="", sample="", ignore=False): """Manipulates samples that have an internal ST that differs from pubMLST ST""" if type == "list": # Add single sample support later - self.db_access.list_unresolved() + self.db_access.read_unresolved() elif type == "overwrite": if ignore: - self.db_access.rm_novel(sample=sample) + self.db_access.set_novel_ignored(sample=sample) else: - self.db_access.sync_novel(overwrite=True, sample=sample) + self.db_access.set_novel_st(overwrite=True, sample=sample) else: - self.db_access.sync_novel(overwrite=False, sample=sample) + self.db_access.set_novel_st(overwrite=False, sample=sample) def fetch_resistances(self, force=False): cwd = os.getcwd() url = "https://bitbucket.org/genomicepidemiology/resfinder_db.git" - hiddensrc = "{}/.resfinder_db".format(self.config["folders"]["resistances"]) + hiddensrc = f"{self.folders.resistances}/.resfinder_db" wipeIndex = False if not os.path.exists(hiddensrc) or len(os.listdir(hiddensrc)) == 0: self.logger.info("resFinder database not found. Caching..") if not os.path.exists(hiddensrc): os.makedirs(hiddensrc) - cmd = "git clone {} --quiet".format(url) + cmd = f"git clone {url} --quiet" process = subprocess.Popen( cmd.split(), - cwd=self.config["folders"]["resistances"], + cwd=self.folders.resistances, stdout=subprocess.PIPE, ) output, error = process.communicate() os.rename( - "{}/resfinder_db".format(self.config["folders"]["resistances"]), + f"{self.folders.resistances}/resfinder_db", hiddensrc, ) wipeIndex = True else: if not wipeIndex: - actual = os.listdir(self.config["folders"]["resistances"]) + actual = os.listdir(self.folders.resistances) for file in os.listdir(hiddensrc): if file not in actual and (".fsa" in file): @@ -363,15 +378,15 @@ def fetch_resistances(self, force=False): # Actual update of resistance folder if wipeIndex: for file in os.listdir(hiddensrc): - if os.path.isfile("{}/{}".format(hiddensrc, file)): + if os.path.isfile(f"{hiddensrc}/{file}"): # Copy fresh shutil.copy( - "{}/{}".format(hiddensrc, file), - self.config["folders"]["resistances"], + f"{hiddensrc}/{file}", + self.folders.resistances, ) # Double checks indexation is current. - self.index_db(self.config["folders"]["resistances"], ".fsa") + self.index_db(self.folders.resistances, ".fsa") def existing_organisms(self): """Returns list of all organisms currently added""" @@ -380,7 +395,7 @@ def existing_organisms(self): def organism2reference(self, normal_organism_name): """Finds which reference contains the same words as the organism and returns it in a format for database calls. Returns empty string if none found""" - orgs = os.listdir(self.config["folders"]["references"]) + orgs = os.listdir(self.folders.references) organism = re.split(r"\W+", normal_organism_name.lower()) try: for target in orgs: @@ -397,9 +412,7 @@ def organism2reference(self, normal_organism_name): return target except Exception as e: self.logger.warning( - "Unable to find existing reference for {}, strain {} has no reference match\nSource: {}".format( - organism, normal_organism_name, e - ) + f"Unable to find existing reference for {organism}, strain {normal_organism_name} has no reference match\nSource: {e}" ) def download_ncbi(self, reference): @@ -409,28 +422,28 @@ def download_ncbi(self, reference): Entrez.email = "2@2.com" record = Entrez.efetch(db="nucleotide", id=reference, rettype="fasta", retmod="text") sequence = record.read() - output = "{}/{}.fasta".format(self.config["folders"]["genomes"], reference) + output = f"{self.folders.genomes}/{reference}.fasta" with open(output, "w") as f: f.write(sequence) - bwaindex = "bwa index {}".format(output) + bwaindex = self._singularity_exec('bwa', f"bwa index {output}") proc = subprocess.Popen( bwaindex.split(), - cwd=self.config["folders"]["genomes"], + cwd=self.folders.genomes, stdout=DEVNULL, stderr=DEVNULL, ) out, err = proc.communicate() - samindex = "samtools faidx {}".format(output) + samindex = self._singularity_exec('samtools', f"samtools faidx {output}") proc = subprocess.Popen( samindex.split(), - cwd=self.config["folders"]["genomes"], + cwd=self.folders.genomes, stdout=DEVNULL, stderr=DEVNULL, ) out, err = proc.communicate() - self.logger.info("Downloaded reference {}".format(reference)) + self.logger.info(f"Downloaded reference {reference}") except Exception: - self.logger.warning("Unable to download genome '{}' from NCBI".format(reference)) + self.logger.warning(f"Unable to download genome '{reference}' from NCBI") def add_pubmlst(self, organism: str): """Checks pubmlst for references of given organism and downloads them""" @@ -439,7 +452,7 @@ def add_pubmlst(self, organism: str): try: organism = organism.lower().replace(".", " ") if organism.replace(" ", "_") in self.organisms and not self.force: - self.logger.info("Organism {} already stored in microSALT".format(organism)) + self.logger.info(f"Organism {organism} already stored in microSALT") return db_query = self.query_pubmlst() @@ -461,25 +474,23 @@ def add_pubmlst(self, organism: str): seqdef_url = subtype["href"] desc = subtype["description"] counter += 1.0 - self.logger.info("Located pubMLST hit {} for sample".format(desc)) + self.logger.info(f"Located pubMLST hit {desc} for sample") if counter > 2.0: raise Exception( - "Reference '{}' resolved to {} organisms. Please be more stringent".format( - errorg, int(counter / 2) - ) + f"Reference '{errorg}' resolved to {int(counter / 2)} organisms. Please be more stringent" ) elif counter < 1.0: # add external raise Exception( - "Unable to find requested organism '{}' in pubMLST database".format(errorg) + f"Unable to find requested organism '{errorg}' in pubMLST database" ) else: truename = desc.lower().split(" ") - truename = "{}_{}".format(truename[0], truename[1]) + truename = f"{truename[0]}_{truename[1]}" self.download_pubmlst(truename, seqdef_url, force=self.force) # Update organism list self.refs = self.db_access.profiles - self.logger.info("Created table profile_{}".format(truename)) + self.logger.info(f"Created table profile_{truename}") except Exception as e: self.logger.warning(e.args[0]) @@ -560,7 +571,7 @@ def download_pubmlst(self, organism, subtype_href, force=False): try: # Pull version extver = self.external_version(organism, subtype_href) - currver = self.db_access.get_version(f"profile_{organism}") + currver = self.db_access.read_version(f"profile_{organism}") if int(extver.replace("-", "")) <= int(currver.replace("-", "")) and not force: self.logger.info( f"Profile for {organism.replace('_', ' ').capitalize()} already at the latest version." @@ -584,7 +595,7 @@ def download_pubmlst(self, organism, subtype_href, force=False): return None # Step 1: Download the profiles CSV - st_target = f"{self.config['folders']['profiles']}/{organism}" + st_target = f"{self.folders.profiles}/{organism}" profiles_csv = self.client.download_profiles_csv(db, scheme_id) # Only write the first 8 columns, this avoids adding information such as "clonal_complex" and "species" profiles_csv = profiles_csv.split("\n") @@ -603,7 +614,7 @@ def download_pubmlst(self, organism, subtype_href, force=False): loci_list = scheme_info.get("loci", []) # Step 3: Download loci FASTA files - output = f"{self.config['folders']['references']}/{organism}" + output = f"{self.folders.references}/{organism}" if os.path.isdir(output): shutil.rmtree(output) os.makedirs(output) @@ -638,7 +649,7 @@ def fetch_pubmlst(self, force=False): seqdef_url[name] = subtype["href"] for key, val in seqdef_url.items(): - internal_ver = self.db_access.get_version(f"profile_{key}") + internal_ver = self.db_access.read_version(f"profile_{key}") external_ver = self.external_version(key, val) if (internal_ver < external_ver) or force: @@ -647,8 +658,8 @@ def fetch_pubmlst(self, force=False): ) self.download_pubmlst(key, val, force) self.db_access.upd_rec( - {"name": "profile_{}".format(key)}, + {"name": f"profile_{key}"}, "Versions", {"version": external_ver}, ) - self.db_access.reload_profiletable(key) + self.db_access.refresh_profiletable(key) diff --git a/microSALT/utils/reporter.py b/microSALT/utils/reporter.py index 9595c1fe..e14dda86 100644 --- a/microSALT/utils/reporter.py +++ b/microSALT/utils/reporter.py @@ -1,56 +1,50 @@ """Generates various reports by tapping into Flask and mySQL - By: Isak Sylvin, @sylvinite""" +By: Isak Sylvin, @sylvinite""" #!/usr/bin/env python import json -import requests import os +import smtplib import socket import sys -import smtplib import time -import yaml - from datetime import datetime -from shutil import copyfile - -from email.mime.multipart import MIMEMultipart from email.mime.application import MIMEApplication +from email.mime.multipart import MIMEMultipart +from shutil import copyfile -from multiprocessing import Process +import yaml -from microSALT import __version__ -from microSALT.server.views import app, gen_reportdata, gen_collectiondata +from microSALT.server.views import ( + alignment_page, + gen_collectiondata, + gen_reportdata, + STtracker_page, + typing_page, +) +from microSALT.config import Folders, Threshold, Regex from microSALT.store.db_manipulator import DB_Manipulator class Reporter: - def __init__(self, config, log, sampleinfo={}, name="", output="", collection=False): - self.db_pusher = DB_Manipulator(config, log) + def __init__(self, log, folders: Folders, threshold: Threshold, regex: Regex, sampleinfo={}, name="", output="", collection=False): + self.folders = folders + self.threshold = threshold + self.regex = regex + self.db_pusher = DB_Manipulator(log=log, folders=folders, threshold=threshold) self.name = name self.collection = collection if output == "": self.output = os.getcwd() else: self.output = output + "/" - self.config = config self.logger = log - for k, v in config.items(): - app.config[k] = v - self.server = Process(target=app.run) self.attachments = list() self.filedict = dict() self.error = False self.dt = datetime.now() self.now = time.strftime( - "{}.{}.{}_{}.{}.{}".format( - self.dt.year, - self.dt.month, - self.dt.day, - self.dt.hour, - self.dt.minute, - self.dt.second, - ) + f"{self.dt.year}.{self.dt.month}.{self.dt.day}_{self.dt.hour}.{self.dt.minute}.{self.dt.second}" ) self.sampleinfo = sampleinfo @@ -70,9 +64,9 @@ def __init__(self, config, log, sampleinfo={}, name="", output="", collection=Fa self.sample = self.sampleinfo def create_subfolders(self): - os.makedirs("{0}/deliverables".format(self.config["folders"]["reports"]), exist_ok=True) - os.makedirs("{0}/json".format(self.config["folders"]["reports"]), exist_ok=True) - os.makedirs("{0}/analysis".format(self.config["folders"]["reports"]), exist_ok=True) + os.makedirs(f"{self.folders.reports}/deliverables", exist_ok=True) + os.makedirs(f"{self.folders.reports}/json", exist_ok=True) + os.makedirs(f"{self.folders.reports}/analysis", exist_ok=True) def report(self, type="default", customer="all"): self.create_subfolders() @@ -80,7 +74,6 @@ def report(self, type="default", customer="all"): # Only typing and qc reports are version controlled self.gen_version(self.name) if type in ["default", "typing", "qc", "st_update"]: - self.restart_web() if type == "default": self.gen_typing() self.gen_qc() @@ -92,7 +85,6 @@ def report(self, type="default", customer="all"): self.gen_qc() elif type == "st_update": self.gen_STtracker(customer) - self.kill_flask() elif type in ["json_dump", "motif_overview"]: if type == "json_dump": self.gen_json() @@ -112,127 +104,104 @@ def report(self, type="default", customer="all"): copyfile(k, v) def gen_version(self, name): - self.db_pusher.get_report(name) + self.db_pusher.read_report(name) self.db_pusher.set_report(name) def gen_STtracker(self, customer="all", silent=False): self.name = "Sequence Type Update" try: - r = requests.get( - "http://127.0.0.1:5000/microSALT/STtracker/{}".format(customer), - allow_redirects=True, - ) - outname = "{}/ST_updates_{}.html".format(self.output, self.now) + content = STtracker_page(customer, threshold=self.threshold) + outname = f"{self.output}/ST_updates_{self.now}.html" outfile = open(outname, "wb") - outfile.write(r.content.decode("iso-8859-1").encode("utf8")) + outfile.write(content.encode("utf8")) outfile.close() self.filedict[outname] = "" if not silent: self.attachments.append(outname) - except Exception as e: - self.logger.error( - "Flask instance currently occupied. Possible rogue process. Retry command" - ) + except Exception: + self.logger.error("Failed to generate ST tracker report") self.error = True def gen_qc(self, silent=False): try: - last_version = self.db_pusher.get_report(self.name).version - except Exception as e: - self.logger.error("Project {} does not exist".format(self.name)) - self.kill_flask() + last_version = self.db_pusher.read_report(self.name).version + except Exception: + self.logger.error(f"Project {self.name} does not exist") sys.exit(-1) try: - q = requests.get( - "http://127.0.0.1:5000/microSALT/{}/qc".format(self.name), - allow_redirects=True, - ) - outfile = "{}_QC_{}.html".format(self.sample.get("Customer_ID_project"), last_version) - local = "{}/{}".format(self.output, outfile) - output = "{}/analysis/{}".format(self.config["folders"]["reports"], outfile) + content = alignment_page(self.name, threshold=self.threshold) + outfile = f"{self.sample.get('Customer_ID_project')}_QC_{last_version}.html" + local = f"{self.output}/{outfile}" + output = f"{self.folders.reports}/analysis/{outfile}" - outfile = open(output, "wb") - outfile.write(q.content.decode("iso-8859-1").encode("utf8")) - outfile.close() + with open(output, "wb") as f: + f.write(content.encode("utf8")) if os.path.isfile(output): self.filedict[output] = local if not silent: self.attachments.append(output) - except Exception as e: - self.logger.error( - "Flask instance currently occupied. Possible rogue process. Retry command" - ) + except Exception: + self.logger.error("Failed to generate QC report") self.error = True def gen_typing(self, silent=False): try: - last_version = self.db_pusher.get_report(self.name).version - except Exception as e: - self.logger.error("Project {} does not exist".format(self.name)) - self.kill_flask() + last_version = self.db_pusher.read_report(self.name).version + except Exception: + self.logger.error(f"Project {self.name} does not exist") sys.exit(-1) try: - r = requests.get( - "http://127.0.0.1:5000/microSALT/{}/typing/all".format(self.name), - allow_redirects=True, - ) - outfile = "{}_Typing_{}.html".format( - self.sample.get("Customer_ID_project"), last_version - ) - local = "{}/{}".format(self.output, outfile) - output = "{}/analysis/{}".format(self.config["folders"]["reports"], outfile) + content = typing_page(self.name, "all", threshold=self.threshold, verified_organisms=self.regex.verified_organisms) + outfile = f"{self.sample.get('Customer_ID_project')}_Typing_{last_version}.html" + local = f"{self.output}/{outfile}" + output = f"{self.folders.reports}/analysis/{outfile}" - outfile = open(output, "wb") - outfile.write(r.content.decode("iso-8859-1").encode("utf8")) - outfile.close() + with open(output, "wb") as f: + f.write(content.encode("utf8")) if os.path.isfile(output): self.filedict[output] = local if not silent: self.attachments.append(output) - except Exception as e: - self.logger.error( - "Flask instance currently occupied. Possible rogue process. Retry command" - ) + except Exception: + self.logger.error("Failed to generate typing report") self.error = True def gen_motif(self, motif="resistance", silent=False): if motif not in ["resistance", "expec"]: self.logger.error("Invalid motif type specified for gen_motif function") if self.collection: - sample_info = gen_collectiondata(self.name) + sample_info = gen_collectiondata(self.name, threshold=self.threshold) else: - sample_info = gen_reportdata(self.name) - output = "{}/{}_{}_{}.csv".format(self.output, self.name, motif, self.now) + sample_info = gen_reportdata(self.name, threshold=self.threshold) + output = f"{self.output}/{self.name}_{motif}_{self.now}.csv" # Load motif & gene names into dict motifdict = dict() for s in sample_info["samples"]: if motif == "resistance": for r in s.resistances: - if not (r.resistance in motifdict.keys()) and r.threshold == "Passed": + if r.resistance not in motifdict.keys() and r.threshold == "Passed": if r.resistance is None: r.resistance = "None" motifdict[r.resistance] = list() - if r.threshold == "Passed" and not r.gene in motifdict[r.resistance]: + if r.threshold == "Passed" and r.gene not in motifdict[r.resistance]: motifdict[r.resistance].append(r.gene) elif motif == "expec": for e in s.expacs: - if not (e.virulence in motifdict.keys()) and e.threshold == "Passed": + if e.virulence not in motifdict.keys() and e.threshold == "Passed": if e.virulence is None: e.virulence = "None" motifdict[e.virulence] = list() - if e.threshold == "Passed" and not e.gene in motifdict[e.virulence]: + if e.threshold == "Passed" and e.gene not in motifdict[e.virulence]: motifdict[e.virulence].append(e.gene) for k, v in motifdict.items(): motifdict[k] = sorted(v) # Top 2 Header sepfix = "sep=," - topline = "Identity {}% & Span {}%,,,".format( - self.config["threshold"]["motif_id"], self.config["threshold"]["motif_span"] - ) + topline = f"Identity {self.threshold.motif_id}% & Span {self.threshold.motif_span}%,,," botline = "CG Sample ID,Sample ID,Organism,Sequence Type,Thresholds" for k in sorted(motifdict.keys()): genes = [""] * len(motifdict[k]) @@ -240,38 +209,32 @@ def gen_motif(self, motif="resistance", silent=False): if active_gene == "": active_gene = "Uncategorized hits" geneholder = ",".join(genes) - topline += ",,{}{}".format(active_gene, geneholder) + topline += f",,{active_gene}{geneholder}" resnames = ",".join(sorted(motifdict[k])) - botline += ",,{}".format(resnames) + botline += f",,{resnames}" try: excel = open(output, "w+") - excel.write("{}\n".format(sepfix)) - excel.write("{}\n".format(topline)) - excel.write("{}\n".format(botline)) + excel.write(f"{sepfix}\n") + excel.write(f"{topline}\n") + excel.write(f"{botline}\n") # Create each individual row past the 2nd, per iteration for s in sample_info["samples"]: rowdict = dict() - pref = "{},{},{},{},{}".format( - s.CG_ID_sample, - s.Customer_ID_sample, - s.organism, - s.ST_status.replace(",", ";"), - s.threshold, - ) + pref = f"{s.CG_ID_sample},{s.Customer_ID_sample},{s.organism},{s.ST_status.replace(',', ';')},{s.threshold}" # Load single sample if motif == "resistance": for r in s.resistances: - if not (r.resistance in rowdict.keys()) and r.threshold == "Passed": + if r.resistance not in rowdict.keys() and r.threshold == "Passed": rowdict[r.resistance] = dict() - if r.threshold == "Passed" and not r.gene in rowdict[r.resistance]: + if r.threshold == "Passed" and r.gene not in rowdict[r.resistance]: rowdict[r.resistance][r.gene] = r.identity elif motif == "expec": for e in s.expacs: - if not (e.virulence in rowdict.keys()) and e.threshold == "Passed": + if e.virulence not in rowdict.keys() and e.threshold == "Passed": rowdict[e.virulence] = dict() - if e.threshold == "Passed" and not e.gene in rowdict[e.virulence]: + if e.threshold == "Passed" and e.gene not in rowdict[e.virulence]: rowdict[e.virulence][e.gene] = e.identity # Compare single sample to all hits = "" @@ -282,7 +245,7 @@ def gen_motif(self, motif="resistance", silent=False): hits += "," if gen in rowdict[res].keys(): # UPD: Change this to identity of hit - hits += "{}".format(rowdict[res][gen]) + hits += f"{rowdict[res][gen]}" else: hits += "0" else: @@ -291,7 +254,7 @@ def gen_motif(self, motif="resistance", silent=False): pad = ["0"] * len(motifdict[res]) hits += ",".join(pad) - excel.write("{}{}\n".format(pref, hits)) + excel.write(f"{pref}{hits}\n") excel.close() if os.path.isfile(output): @@ -300,21 +263,15 @@ def gen_motif(self, motif="resistance", silent=False): self.attachments.append(output) except FileNotFoundError as e: self.logger.error( - "Gen_motif unable to produce excel file. Path {} does not exist".format( - os.path.basename(output) - ) + f"Gen_motif unable to produce excel file. Path {os.path.basename(output)} does not exist" ) def gen_delivery(self): deliv = dict() deliv["files"] = list() - last_version = self.db_pusher.get_report(self.name).version - output = "{}/deliverables/{}_deliverables.yaml".format( - self.config["folders"]["reports"], self.sample.get("Customer_ID_project") - ) - local = "{}/{}_deliverables.yaml".format( - self.output, self.sample.get("Customer_ID_project") - ) + last_version = self.db_pusher.read_report(self.name).version + output = f"{self.folders.reports}/deliverables/{self.sample.get('Customer_ID_project')}_deliverables.yaml" + local = f"{self.output}/{self.sample.get('Customer_ID_project')}_deliverables.yaml" # Project-wide # Sampleinfo @@ -322,7 +279,7 @@ def gen_delivery(self): { "format": "json", "id": str(self.sample.get("Customer_ID_project")), - "path": "{}/sampleinfo.json".format(self.output), + "path": f"{self.output}/sampleinfo.json", "path_index": "~", "step": "analysis", "tag": "sampleinfo", @@ -333,9 +290,7 @@ def gen_delivery(self): { "format": "html", "id": str(self.sample.get("Customer_ID_project")), - "path": "{}/{}_QC_{}.html".format( - self.output, self.sample.get("Customer_ID_project"), last_version - ), + "path": f"{self.output}/{self.sample.get('Customer_ID_project')}_QC_{last_version}.html", "path_index": "~", "step": "result_aggregation", "tag": "microsalt-qc", @@ -346,9 +301,7 @@ def gen_delivery(self): { "format": "html", "id": str(self.sample.get("Customer_ID_project")), - "path": "{}/{}_Typing_{}.html".format( - self.output, self.sample.get("Customer_ID_project"), last_version - ), + "path": f"{self.output}/{self.sample.get('Customer_ID_project')}_Typing_{last_version}.html", "path_index": "~", "step": "result_aggregation", "tag": "microsalt-type", @@ -359,7 +312,7 @@ def gen_delivery(self): { "format": "json", "id": str(self.sample.get("Customer_ID_project")), - "path": "{}/{}.json".format(self.output, self.sample.get("CG_ID_project")), + "path": f"{self.output}/{self.sample.get('CG_ID_project')}.json", "path_index": "~", "step": "result_aggregation", "tag": "microsalt-json", @@ -370,7 +323,7 @@ def gen_delivery(self): { "format": "txt", "id": str(self.sample.get("Customer_ID_project")), - "path": "{}/config.log".format(self.output), + "path": f"{self.output}/config.log", "path_index": "~", "step": "analysis", "tag": "runtime-settings", @@ -382,7 +335,7 @@ def gen_delivery(self): { "format": "txt", "id": str(self.sample.get("Customer_ID_project")), - "path": "{}/version.txt".format(self.output), + "path": f"{self.output}/version.txt", "path_index": "~", "step": "result_aggregation", "tag": "microsalt-version", @@ -407,9 +360,7 @@ def gen_delivery(self): { "format": "fasta", "id": s["CG_ID_sample"], - "path": "{0}/assembly/{1}_trimmed_contigs.fasta".format( - resultsdir, s["CG_ID_sample"] - ), + "path": f"{resultsdir}/assembly/{s['CG_ID_sample']}_trimmed_contigs.fasta", "path_index": "~", "step": "assembly", "tag": "assembly", @@ -420,9 +371,7 @@ def gen_delivery(self): { "format": "fastq", "id": s["CG_ID_sample"], - "path": "{0}/trimmed/{1}_trim_front_pair.fastq.gz".format( - resultsdir, s["CG_ID_sample"] - ), + "path": f"{resultsdir}/trimmed/{s['CG_ID_sample']}_trim_front_pair.fastq.gz", "path_index": "~", "step": "concatination", "tag": "trimmed-forward-reads", @@ -433,9 +382,7 @@ def gen_delivery(self): { "format": "fastq", "id": s["CG_ID_sample"], - "path": "{0}/trimmed/{1}_trim_rev_pair.fastq.gz".format( - resultsdir, s["CG_ID_sample"] - ), + "path": f"{resultsdir}/trimmed/{s['CG_ID_sample']}_trim_rev_pair.fastq.gz", "path_index": "~", "step": "concatination", "tag": "trimmed-reverse-reads", @@ -446,9 +393,7 @@ def gen_delivery(self): { "format": "fastq", "id": s["CG_ID_sample"], - "path": "{0}/trimmed/{1}_trim_unpair.fastq.gz".format( - resultsdir, s["CG_ID_sample"] - ), + "path": f"{resultsdir}/trimmed/{s['CG_ID_sample']}_trim_unpair.fastq.gz", "path_index": "~", "step": "concatination", "tag": "trimmed-unpaired-reads", @@ -459,7 +404,7 @@ def gen_delivery(self): { "format": "txt", "id": s["CG_ID_sample"], - "path": "{0}/slurm_{1}.log".format(resultsdir, s["CG_ID_sample"]), + "path": f"{resultsdir}/slurm_{s['CG_ID_sample']}.log", "path_index": "~", "step": "analysis", "tag": "logfile", @@ -470,9 +415,7 @@ def gen_delivery(self): { "format": "tsv", "id": s["CG_ID_sample"], - "path": "{0}/assembly/quast/{1}_report.tsv".format( - resultsdir, s["CG_ID_sample"] - ), + "path": f"{resultsdir}/assembly/quast/{s['CG_ID_sample']}_report.tsv", "path_index": "~", "step": "assembly", "tag": "quast-results", @@ -483,9 +426,7 @@ def gen_delivery(self): { "format": "bam", "id": s["CG_ID_sample"], - "path": "{0}/alignment/{1}_{2}.bam_sort".format( - resultsdir, s["CG_ID_sample"], s["reference"] - ), + "path": f"{resultsdir}/alignment/{s['CG_ID_sample']}_{s['reference']}.bam_sort", "path_index": "~", "step": "alignment", "tag": "reference-alignment-sorted", @@ -496,9 +437,7 @@ def gen_delivery(self): { "format": "bam", "id": s["CG_ID_sample"], - "path": "{0}/alignment/{1}_{2}.bam_sort_rmdup".format( - resultsdir, s["CG_ID_sample"], s["reference"] - ), + "path": f"{resultsdir}/alignment/{s['CG_ID_sample']}_{s['reference']}.bam_sort_rmdup", "path_index": "~", "step": "alignment", "tag": "reference-alignment-deduplicated", @@ -509,9 +448,7 @@ def gen_delivery(self): { "format": "meta", "id": s["CG_ID_sample"], - "path": "{0}/alignment/{1}_{2}.stats.ins".format( - resultsdir, s["CG_ID_sample"], s["reference"] - ), + "path": f"{resultsdir}/alignment/{s['CG_ID_sample']}_{s['reference']}.stats.ins", "path_index": "~", "step": "insertsize_calc", "tag": "picard-insertsize", @@ -533,10 +470,10 @@ def gen_delivery(self): def gen_json(self, silent=False): report = dict() - local = "{}/{}.json".format(self.output, self.name) - output = "{}/json/{}.json".format(self.config["folders"]["reports"], self.name) + local = f"{self.output}/{self.name}.json" + output = f"{self.folders.reports}/json/{self.name}.json" - sample_info = gen_reportdata(self.name) + sample_info = gen_reportdata(self.name, threshold=self.threshold) analyses = [ "blast_pubmlst", "quast_assembly", @@ -618,7 +555,7 @@ def gen_json(self, silent=False): for r in s.resistances: if ( - not (r.gene in report[s.CG_ID_sample]["blast_resfinder_resistence"]) + r.gene not in report[s.CG_ID_sample]["blast_resfinder_resistence"] and r.threshold == "Passed" ): report[s.CG_ID_sample]["blast_resfinder_resistence"].append(r.gene) @@ -632,32 +569,30 @@ def gen_json(self, silent=False): self.filedict[output] = local if not silent: self.attachments.append(output) - except FileNotFoundError as e: + except FileNotFoundError: self.logger.error( - "Gen_json unable to produce json file. Path {} does not exist".format( - os.path.basename(output) - ) + f"Gen_json unable to produce json file. Path {os.path.basename(output)} does not exist" ) def mail(self): msg = MIMEMultipart() if not self.error and self.attachments: - msg["Subject"] = "{} ({}) Reports".format(self.name, self.attachments[0].split("_")[0]) + msg["Subject"] = f"{self.name} ({self.attachments[0].split('_')[0]}) Reports" else: - msg["Subject"] = "{} Failed Generating Report".format(self.name) + msg["Subject"] = f"{self.name} Failed Generating Report" sender = socket.gethostname() - sender_fixed = "{}.com".format(os.path.splitext(sender)[0]) + sender_fixed = f"{os.path.splitext(sender)[0]}.com" msg["From"] = sender_fixed - msg["To"] = self.config["regex"]["mail_recipient"] + msg["To"] = self.regex.mail_recipient if not self.error: for file in self.attachments: part = MIMEApplication(open(file).read()) part.add_header( "Content-Disposition", - 'attachment; filename="%s"' % os.path.basename(file), + f'attachment; filename="{os.path.basename(file)}"', ) msg.attach(part) @@ -665,22 +600,4 @@ def mail(self): s.connect() s.sendmail(msg["From"], msg["To"], msg.as_string()) s.quit() - self.logger.info("Mail containing report sent to {} from {}".format(msg["To"], msg["From"])) - - def start_web(self): - self.server.start() - self.logger.info("Started webserver on http://127.0.0.1:5000/") - # Hinders requests before server goes up - time.sleep(0.15) - - def kill_flask(self): - self.server.terminate() - self.server.join() - self.logger.info("Closed webserver on http://127.0.0.1:5000/") - - def restart_web(self): - try: - self.kill_flask() - except Exception as e: - pass - self.start_web() + self.logger.info(f"Mail containing report sent to {msg['To']} from {msg['From']}") diff --git a/microSALT/utils/scraper.py b/microSALT/utils/scraper.py index 3ac3f84f..bdd4b8f9 100644 --- a/microSALT/utils/scraper.py +++ b/microSALT/utils/scraper.py @@ -1,5 +1,5 @@ """Scrapes output files for data and adds them to the database - By: Isak Sylvin, @sylvinite""" +By: Isak Sylvin, @sylvinite""" #!/usr/bin/env python @@ -10,18 +10,74 @@ import sys import time +from microSALT.config import ( + Folders, + Threshold, + SlurmHeader, + Regex, + PubMLSTCredentials, + PasteurCredentials, + Singularity, + Containers, +) from microSALT.store.db_manipulator import DB_Manipulator from microSALT.utils.referencer import Referencer from microSALT.utils.job_creator import Job_Creator + # TODO: Rewrite so samples use seperate objects class Scraper: - def __init__(self, config, log, sampleinfo={}, input=""): - self.config = config + def __init__( + self, + log, + folders: Folders, + threshold: Threshold, + slurm_header: SlurmHeader, + regex: Regex, + dry: bool, + config_path: str, + pubmlst: PubMLSTCredentials, + pasteur: PasteurCredentials, + singularity: Singularity = None, + containers: Containers = None, + sampleinfo={}, + input="", + ): + self.folders = folders + self.threshold = threshold + self.slurm_header = slurm_header + self.regex = regex + self.dry = dry + self.config_path = config_path + self.pubmlst = pubmlst + self.pasteur = pasteur + self.singularity = singularity or Singularity() + self.containers = containers or Containers() self.logger = log - self.db_pusher = DB_Manipulator(config, log) - self.referencer = Referencer(config, log) - self.job_fallback = Job_Creator(config=config, log=log, sampleinfo=sampleinfo) + self.db_pusher = DB_Manipulator(log=log, folders=folders, threshold=threshold) + self.referencer = Referencer( + log=log, + folders=folders, + threshold=threshold, + pubmlst=pubmlst, + pasteur=pasteur, + singularity=self.singularity, + containers=self.containers, + ) + self.job_fallback = Job_Creator( + log=log, + folders=folders, + slurm_header=slurm_header, + regex=regex, + dry=dry, + config_path=config_path, + threshold=threshold, + pubmlst=pubmlst, + pasteur=pasteur, + singularity=self.singularity, + containers=self.containers, + sampleinfo=sampleinfo, + ) self.infolder = os.path.abspath(input) self.sampledir = "" @@ -51,56 +107,46 @@ def scrape_project(self, project=None): """Scrapes a project folder for information""" if project is None: project = self.name - self.db_pusher.purge_rec(project, "Projects") - if not self.db_pusher.exists("Projects", {"CG_ID_project": project}): - self.logger.warning("Replacing project {}".format(project)) - self.job_fallback.create_project(project) + self.db_pusher.delete_project(project) + self.job_fallback.create_project(project) # Scrape order matters a lot! for item in os.listdir(self.infolder): - sampledir = "{}/{}".format(self.infolder, item) + sampledir = f"{self.infolder}/{item}" if os.path.isdir(sampledir): local_param = [p for p in self.sampleinfo if p["CG_ID_sample"] == item] if local_param != []: local_param = local_param[0] sample_scraper = Scraper( - config=self.config, log=self.logger, + folders=self.folders, + threshold=self.threshold, + slurm_header=self.slurm_header, + regex=self.regex, + dry=self.dry, + config_path=self.config_path, + pubmlst=self.pubmlst, + pasteur=self.pasteur, sampleinfo=local_param, input=sampledir, ) sample_scraper.scrape_sample() else: - self.logger.warning( - "Skipping {} due to lacking info in sample_json file".format(dir) - ) + self.logger.warning(f"Skipping {dir} due to lacking info in sample_json file") def scrape_sample(self, sample=None): """Scrapes a sample folder for information""" if sample is None: sample = self.name - self.db_pusher.purge_rec(sample, "Samples") - - if not self.db_pusher.exists( - "Projects", {"CG_ID_project": self.sample.get("CG_ID_project")} - ): - self.logger.warning( - "Replacing project {}".format(self.sample.get("CG_ID_project")) - ) - self.job_fallback.create_project(self.sample.get("CG_ID_project")) - - if not self.db_pusher.exists("Samples", {"CG_ID_sample": sample}): - self.logger.info("Replacing sample {}".format(sample)) - self.job_fallback.create_sample(sample) + self.db_pusher.delete_sample_results(sample) + self.job_fallback.create_project(self.sample.get("CG_ID_project")) + self.job_fallback.create_sample() # Scrape order matters a lot! self.sampledir = self.infolder self.scrape_blast(type="seq_type") self.scrape_blast(type="resistance") - if ( - self.referencer.organism2reference(self.sample.get("organism")) - == "escherichia_coli" - ): + if self.referencer.organism2reference(self.sample.get("organism")) == "escherichia_coli": self.scrape_blast(type="expec") self.scrape_alignment() self.scrape_quast() @@ -109,9 +155,9 @@ def scrape_sample(self, sample=None): def scrape_quast(self, filename=""): """Scrapes a quast report for assembly information""" if filename == "": - filename = "{}/assembly/quast/{}_report.tsv".format(self.sampledir, self.name) + filename = f"{self.sampledir}/assembly/quast/{self.name}_report.tsv" if not os.path.isfile(filename): - filename = "{}/assembly/quast/report.tsv".format(self.sampledir) + filename = f"{self.sampledir}/assembly/quast/report.tsv" quast = dict() try: @@ -127,20 +173,20 @@ def scrape_quast(self, filename=""): elif lsplit[0] == "N50": quast["n50"] = int(lsplit[1]) - self.db_pusher.upd_rec({"CG_ID_sample": self.name}, "Samples", quast) - self.logger.debug( - "Project {} recieved quast stats: {}".format(self.name, quast) - ) + self.db_pusher.update_sample({"CG_ID_sample": self.name}, quast) + self.logger.debug(f"Project {self.name} recieved quast stats: {quast}") except Exception as e: - self.logger.warning( - "Cannot generate quast statistics for {}".format(self.name) - ) + self.logger.warning(f"Cannot generate quast statistics for {self.name}") def scrape_reference(self) -> None: """Scrapes a reference assembly to calculate the size""" - assembly = "{}/../../../references/genomes/{}.fasta".format(self.sampledir, self.sample.get("reference")) + assembly = ( + f"{self.sampledir}/../../../references/genomes/{self.sample.get('reference')}.fasta" + ) if not os.path.exists(assembly): - assembly = "{}/../../references/genomes/{}.fasta".format(self.sampledir, self.sample.get("reference")) + assembly = ( + f"{self.sampledir}/../../references/genomes/{self.sample.get('reference')}.fasta" + ) reference_data = dict() try: with open(assembly, "r") as infile: @@ -150,24 +196,20 @@ def scrape_reference(self) -> None: curated_line = line.strip() assembly_length += len(curated_line) reference_data["reference_length"] = assembly_length - self.db_pusher.upd_rec({"CG_ID_sample": self.name}, "Samples", reference_data) - self.logger.debug( - "Project {} recieved quast stats: {}".format(self.name, reference_data) - ) + self.db_pusher.update_sample({"CG_ID_sample": self.name}, reference_data) + self.logger.debug(f"Project {self.name} recieved quast stats: {reference_data}") except Exception as e: - self.logger.warning( - "Cannot find assembly size for reference {}".format(self.name) - ) + self.logger.warning(f"Cannot find assembly size for reference {self.name}") def get_locilengths(self, foldername, suffix): - """ Generate a dict of length for any given loci """ + """Generate a dict of length for any given loci""" # Create dict with full name as key, associated nucleotides as value. alleles = dict() finalalleles = dict() for file in os.listdir(foldername): if file.endswith(suffix): lastallele = "" - f = open("{}/{}".format(foldername, file), "r") + f = open(f"{foldername}/{file}", "r") for row in f: if ">" in row: lastallele = row.strip() @@ -181,271 +223,239 @@ def get_locilengths(self, foldername, suffix): return finalalleles def scrape_blast(self, type="", file_list=[]): - hypo = list() - type2db = type.capitalize() + "s" - if type == "expec": - type2db = "Expacs" - - if file_list == []: - if type == "seq_type": - file_list = glob.glob("{}/blast_search/mlst/*".format(self.sampledir)) - else: - file_list = glob.glob( - "{}/blast_search/{}/*".format(self.sampledir, type) - ) - organism = self.referencer.organism2reference(self.sample.get("organism")) if organism: - self.db_pusher.upd_rec( - {"CG_ID_sample": self.name}, "Samples", {"organism": organism} + self.db_pusher.update_sample({"CG_ID_sample": self.name}, {"organism": organism}) + type2db = "Expacs" if type == "expec" else type.capitalize() + "s" + if not file_list: + file_list = self._resolve_blast_files(type, organism) + hits = self._parse_blast_files(type, file_list, organism) + identifier = "loci" if type == "seq_type" else "gene" + hits = self._deduplicate_hits(hits, identifier) + self._persist_blast_hits(type, type2db, hits) + return hits + + def _resolve_blast_files(self, type: str, organism: str) -> list: + """Resolves which blast output files to parse for a given type.""" + if type == "seq_type": + return glob.glob(f"{self.sampledir}/blast_search/mlst/*") + return glob.glob(f"{self.sampledir}/blast_search/{type}/*") + + def _resolve_ref_folder_and_suffix(self, type: str, organism: str) -> tuple: + """Returns the reference folder and file suffix for locilength lookup.""" + if type == "resistance": + return self.folders.resistances, "fsa" + elif type == "expec": + return ( + os.path.dirname(self.folders.expec), + os.path.basename(self.folders.expec).rsplit(".", 1)[1], ) - res_cols = self.db_pusher.get_columns("{}".format(type2db)) + else: # seq_type + return f"{self.folders.references}/{organism}", "tfa" + def _parse_common_fields(self, elem_list: list) -> dict: + """Parses the fields shared across all BLAST hit types from one TSV line.""" + if int(elem_list[7]) < int(elem_list[8]): + contig_start, contig_end = int(elem_list[7]), int(elem_list[8]) + else: + contig_start, contig_end = int(elem_list[8]), int(elem_list[7]) + nodeinfo = elem_list[2].split("_") + return { + "CG_ID_sample": self.name, + "identity": elem_list[4], + "evalue": elem_list[5], + "bitscore": elem_list[6], + "subject_length": int(elem_list[11]), + "contig_start": contig_start, + "contig_end": contig_end, + "contig_name": f"{nodeinfo[0]}_{nodeinfo[1]}", + "contig_length": int(nodeinfo[3]), + "contig_coverage": nodeinfo[5], + } + + def _parse_resistance_hit( + self, hit: dict, filename: str, elem_list: list, locilengths: dict + ) -> dict: + """Enriches a hit dict with resistance-specific fields.""" + hit["instance"] = filename + partials = re.search(r"(?:>)*(.+)_(\d+){1,3}_(.+)", elem_list[3]) + hit["reference"] = partials.group(3) + hit["gene"] = partials.group(1) + hit["resistance"] = self.gene2resistance.get(hit["gene"], hit["instance"].capitalize()) + padder = [x for x in locilengths if x.startswith(f">{partials.group(1)}_")] + if not padder: + padder = [x for x in locilengths if x.startswith(f">{partials.group(1)[:-1]}_")] + try: + padder = padder[0] + except IndexError: + self.logger.warning( + f"In {self.name} gene {partials.group(1)} can't be resolved. Wrong resistance?" + ) + hit["span"] = float(hit["subject_length"]) / locilengths[padder] + return hit + + def _parse_expec_hit( + self, hit: dict, filename: str, elem_list: list, locilengths: dict + ) -> dict: + """Enriches a hit dict with ExPEC-specific fields.""" + hit["instance"] = filename + # Thanks, precompiled list standards + if ">" in elem_list[3]: + partials = re.search( + r">*(\w+_\w+\.*\w+).+\((\w+)\).+\((\w+)\)_(\w+)_\[.+\]", + elem_list[3], + ) + else: + partials = re.search( + r"(\w+)\(gb\|\w+\)_\((\S+)\)_(.+)_\[(\S+)_.+\]_\[\S+\]", + elem_list[3], + ) + if not partials: + partials = re.search( + r"(\w+\.*\w+)\:*\w*_*(?:\(\w+\-\w+\))*_\((\w+)\)_([^[]+)\[\S+\]", + elem_list[3], + ) + hit["reference"] = partials.group(1) + hit["gene"] = partials.group(2) + hit["instance"] = partials.group(3).strip("_") + hit["virulence"] = ( + partials.group(4).replace("_", " ").capitalize() if len(partials.groups()) >= 4 else "" + ) + hit["span"] = float(hit["subject_length"]) / locilengths[f">{elem_list[3]}"] + return hit + + def _parse_seq_type_hit(self, hit: dict, elem_list: list, locilengths: dict) -> dict: + """Enriches a hit dict with seq_type (MLST) specific fields.""" + partials = re.search(r"(.+)_(\d+){1,3}(?:_(\w+))*", elem_list[3]) + hit["loci"] = partials.group(1) + hit["allele"] = int(partials.group(2)) + padder = [x for x in locilengths if x.startswith(f">{partials[0]}")] + if not padder: + padder = [x for x in locilengths if x.startswith(f">{partials[0][:-1]}")] + try: + padder = padder[0] + except IndexError: + self.logger.warning( + f"In {self.name} allele {partials[0]} can't be resolved. Wrong organism?" + ) + hit["span"] = float(hit["subject_length"]) / locilengths[padder] + return hit + + def _parse_blast_files(self, type: str, file_list: list, organism: str) -> list: + """Parses all blast output files and returns a list of candidate hit dicts.""" + type2db = "Expacs" if type == "expec" else type.capitalize() + "s" + ref_folder, suffix = self._resolve_ref_folder_and_suffix(type, organism) + locilengths = self.get_locilengths(ref_folder, suffix) + hits = [] try: for file in file_list: filename = os.path.basename(file).rsplit(".", 1)[0] # Removes suffix if filename == "lactam": filename = "beta-lactam" - if type == "resistance": - ref_folder = self.config["folders"]["resistances"] - suffix = "fsa" - elif type == "expec": - ref_folder = os.path.dirname(self.config["folders"]["expec"]) - suffix = os.path.basename(self.config["folders"]["expec"]).rsplit( - ".", 1 - )[1] - elif type == "seq_type": - ref_folder = "{}/{}".format( - self.config["folders"]["references"], organism - ) - suffix = "tfa" - locilengths = self.get_locilengths(ref_folder, suffix) - - with open("{}".format(file), "r") as sample: - for line in sample: - # Ignore commented fields - if not line[0] == "#": - - elem_list = line.rstrip().split("\t") - if not elem_list[1] == "N/A": - hypo.append(dict()) - hypo[-1]["CG_ID_sample"] = self.name - hypo[-1]["identity"] = elem_list[4] - hypo[-1]["evalue"] = elem_list[5] - hypo[-1]["bitscore"] = elem_list[6] - if int(elem_list[7]) < int(elem_list[8]): - hypo[-1]["contig_start"] = int(elem_list[7]) - hypo[-1]["contig_end"] = int(elem_list[8]) - else: - hypo[-1]["contig_start"] = int(elem_list[8]) - hypo[-1]["contig_end"] = int(elem_list[7]) - hypo[-1]["subject_length"] = int(elem_list[11]) - - if type == "resistance": - hypo[-1]["instance"] = filename - partials = re.search("(?:\>)*(.+)_(\d+){1,3}(?:_(.+))",elem_list[3]) - hypo[-1]["reference"] = partials.group(3) - hypo[-1]["gene"] = partials.group(1) - if hypo[-1]["gene"] in self.gene2resistance.keys(): - hypo[-1]["resistance"] = self.gene2resistance[ - hypo[-1]["gene"] - ] - else: - hypo[-1]["{}".format(type)] = hypo[-1][ - "instance" - ].capitalize() - #Ignores reference name and finds relevant resFinder entry - - padder = [x for x in locilengths.keys() if x.startswith('>{}_'.format(partials[1]))] - if len(padder) == 0: - padder = [x for x in locilengths.keys() if x.startswith('>{}_'.format(partials[1][:-1]))] - try: - padder = padder[0] - except IndexError as e: - self.logger.warning("In {} gene {} can't be resolved. Wrong resistance?".format(self.name, partials[1])) - - hypo[-1]["span"] = ( - float(hypo[-1]["subject_length"]) - / locilengths[padder] - ) - - elif type == "expec": - hypo[-1]["instance"] = filename - # Thanks, precompiled list standards - if ">" in elem_list[3]: - partials = re.search( - r">*(\w+_\w+\.*\w+).+\((\w+)\).+\((\w+)\)_(\w+)_\[.+\]", - elem_list[3], - ) - else: - partials = re.search( - r"(\w+)\(gb\|\w+\)_\((\S+)\)_(.+)_\[(\S+)_.+\]_\[\S+\]", - elem_list[3], - ) - if not partials: - partials = re.search( - r"(\w+\.*\w+)\:*\w*_*(?:\(\w+\-\w+\))*_\((\w+)\)_([^[]+)\[\S+\]", - elem_list[3], - ) - # NC/Protein reference - hypo[-1]["reference"] = partials.group(1) - # Full gene name - hypo[-1]["gene"] = partials.group(2) - # More generic group - hypo[-1]["instance"] = partials.group(3).strip("_") - # Description - if len(partials.groups()) >= 4: - hypo[-1]["virulence"] = ( - partials.group(4) - .replace("_", " ") - .capitalize() - ) - else: - hypo[-1]["virulence"] = "" - #padder = [x for x in locilengths.keys() if x.startswith('>{}'.format(partials[1]))][0] - hypo[-1]["span"] = ( - float(hypo[-1]["subject_length"]) - / locilengths[">{}".format(elem_list[3])] - ) - - elif type == "seq_type": - partials = re.search( - r"(.+)_(\d+){1,3}(?:_(\w+))*", elem_list[3] - ) - hypo[-1]["loci"] = partials.group(1) - hypo[-1]["allele"] = int(partials.group(2)) - #Ignores reference name and finds relevant resFinder entry - - padder = [x for x in locilengths.keys() if x.startswith('>{}'.format(partials[0]))] - if len(padder) == 0: - padder = [x for x in locilengths.keys() if x.startswith('>{}'.format(partials[0][:-1]))] - try: - padder = padder[0] - except IndexError as e: - self.logger.warning("In {} allele {} can't be resolved. Wrong organism?".format(self.name, partials[0])) - hypo[-1]["span"] = ( - float(hypo[-1]["subject_length"]) - / locilengths[padder] - ) - - # split elem 2 into contig node_NO, length, cov - nodeinfo = elem_list[2].split("_") - hypo[-1]["contig_name"] = "{}_{}".format( - nodeinfo[0], nodeinfo[1] - ) - hypo[-1]["contig_length"] = int(nodeinfo[3]) - hypo[-1]["contig_coverage"] = nodeinfo[5] - self.logger.debug("scrape_blast scrape loop hit '{}'".format(elem_list[3])) - self.logger.info("{} candidate {} hits found".format(len(hypo), type2db)) + with open(file, "r") as fh: + for line in fh: + if line[0] == "#": + continue + elem_list = line.rstrip().split("\t") + if elem_list[1] == "N/A": + continue + hit = self._parse_common_fields(elem_list) + if type == "resistance": + hit = self._parse_resistance_hit(hit, filename, elem_list, locilengths) + elif type == "expec": + hit = self._parse_expec_hit(hit, filename, elem_list, locilengths) + elif type == "seq_type": + hit = self._parse_seq_type_hit(hit, elem_list, locilengths) + self.logger.debug(f"scrape_blast scrape loop hit '{elem_list[3]}'") + hits.append(hit) + self.logger.info(f"{len(hits)} candidate {type2db} hits found") except Exception as e: - self.logger.error("Unable to process the pattern of {}".format(str(e))) + self.logger.error(f"Unable to process the pattern of {e!s}") + return hits - # Cleanup of overlapping hits - if type == "seq_type": - identifier = "loci" - elif type == "resistance" or type == "expec": - identifier = "gene" + def _deduplicate_hits(self, hits: list, identifier: str) -> list: + """Removes overlapping or duplicate BLAST hits, keeping the highest-scoring one.""" ind = 0 - while ind < len(hypo) - 1: + while ind < len(hits) - 1: targ = ind + 1 - while targ < len(hypo): + while targ < len(hits): ignore = False if ( - hypo[ind]["contig_name"] == hypo[targ]["contig_name"] - or hypo[ind][identifier] == hypo[targ][identifier] + hits[ind]["contig_name"] == hits[targ]["contig_name"] + or hits[ind][identifier] == hits[targ][identifier] ): - # Overlapping or shared gene + # Overlapping or shared gene/loci if ( ( - hypo[ind].get("contig_start") - >= hypo[targ].get("contig_start") - and hypo[ind].get("contig_start") - <= hypo[targ].get("contig_end") + hits[ind].get("contig_start") >= hits[targ].get("contig_start") + and hits[ind].get("contig_start") <= hits[targ].get("contig_end") ) or ( - hypo[ind].get("contig_end") - >= hypo[targ].get("contig_start") - and hypo[ind].get("contig_end") - <= hypo[targ].get("contig_end") + hits[ind].get("contig_end") >= hits[targ].get("contig_start") + and hits[ind].get("contig_end") <= hits[targ].get("contig_end") ) - or (hypo[ind].get(identifier) == hypo[targ].get(identifier)) + or (hits[ind].get(identifier) == hits[targ].get(identifier)) ): + score_ind = float(hits[ind]["identity"]) * (1 - abs(1 - hits[ind]["span"])) + score_targ = float(hits[targ]["identity"]) * ( + 1 - abs(1 - hits[targ]["span"]) + ) # Rightmost is worse - if float(hypo[ind].get("identity")) * ( - 1 - abs(1 - hypo[ind].get("span")) - ) > float(hypo[targ].get("identity")) * ( - 1 - abs(1 - hypo[targ].get("span")) - ): - del hypo[targ] + if score_ind > score_targ: + del hits[targ] ignore = True # Leftmost is worse - elif float(hypo[ind].get("identity")) * ( - 1 - abs(1 - hypo[ind].get("span")) - ) < float(hypo[targ].get("identity")) * ( - 1 - abs(1 - hypo[targ].get("span")) - ): - del hypo[ind] + elif score_ind < score_targ: + del hits[ind] targ = ind + 1 ignore = True - # Identical identity and span, seperating based on contig coverage + # Identical score — break tie by contig coverage else: - # Rightmost is worse - if float(hypo[ind].get("contig_coverage")) >= float( - hypo[targ].get("contig_coverage") + if float(hits[ind]["contig_coverage"]) >= float( + hits[targ]["contig_coverage"] ): - del hypo[targ] + del hits[targ] ignore = True - # Leftmost is worse - elif float(hypo[ind].get("contig_coverage")) < float( - hypo[targ].get("contig_coverage") + elif float(hits[ind]["contig_coverage"]) < float( + hits[targ]["contig_coverage"] ): - del hypo[ind] + del hits[ind] targ = ind + 1 ignore = True if not ignore: targ += 1 - else: - pass ind += 1 - + return hits + + def _persist_blast_hits(self, type: str, type2db: str, hits: list) -> None: + """Writes deduplicated blast hits to the database and updates sample ST if applicable.""" + _ADDERS = { + "Seq_types": self.db_pusher.add_seq_type, + "Resistances": self.db_pusher.add_resistance, + "Expacs": self.db_pusher.add_expac, + } self.logger.info( - "{} {} hits were added after removing overlaps and duplicate hits".format( - len(hypo), type - ) + f"{len(hits)} {type} hits were added after removing overlaps and duplicate hits" ) - for hit in hypo: + for hit in hits: self.logger.debug( - "Kept {}:{} with span {} and id {}".format( - hit.get("loci"), - hit.get("allele"), - hit.get("span"), - hit.get("identity"), - ) + f"Kept {hit.get('loci')}:{hit.get('allele')} with span {hit.get('span')} and id {hit.get('identity')}" ) - self.db_pusher.add_rec(hit, "{}".format(type2db)) - + self.db_pusher.add_to_session(_ADDERS[type2db](**hit)) + self.db_pusher.commit_session() if type == "seq_type": try: - ST = self.db_pusher.alleles2st(self.name) - self.db_pusher.upd_rec( - {"CG_ID_sample": self.name}, "Samples", {"ST": ST} - ) - self.logger.info("Sample {} received ST {}".format(self.name, ST)) + ST = self.db_pusher.read_st(self.name) + self.db_pusher.update_sample({"CG_ID_sample": self.name}, {"ST": ST}) + self.logger.info(f"Sample {self.name} received ST {ST}") except Exception as e: - self.logger.warning( - "Unable to type sample {} due to data value '{}'".format( - self.name, str(e) - ) - ) - return hypo + self.logger.warning(f"Unable to type sample {self.name} due to data value '{e!s}'") def load_resistances(self): """Legacy function, loads common resistance names for genes from notes file""" conversions = dict() try: - with open( - "{}/notes.txt".format(self.config["folders"]["resistances"]) - ) as fh: + with open(f"{self.folders.resistances}/notes.txt") as fh: for line in fh: if "#" not in line: line = line.split(":") @@ -454,15 +464,13 @@ def load_resistances(self): # Workaround for case issues conversions[line[0].lower()] = cropped except Exception as e: - self.logger.error( - "Unable to initialize trivial names for resistances ({})".format(e) - ) + self.logger.error(f"Unable to initialize trivial names for resistances ({e})") return conversions def scrape_alignment(self, file_list=[]): """Scrapes a single alignment result""" if file_list == []: - file_list = glob.glob("{}/alignment/*.stats.*".format(self.sampledir)) + file_list = glob.glob(f"{self.sampledir}/alignment/*.stats.*") ins_list = list() cov_dict = dict() align_dict = dict() @@ -543,4 +551,4 @@ def scrape_alignment(self, file_list=[]): align_dict["duplication_rate"] = 0.0 align_dict["average_coverage"] = 0.0 align_dict["total_reads"] = tot_reads - self.db_pusher.upd_rec({"CG_ID_sample": self.name}, "Samples", align_dict) + self.db_pusher.update_sample({"CG_ID_sample": self.name}, align_dict) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..fe47f49d --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,52 @@ +[build-system] +requires = ["setuptools>=61", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "microSALT" +dynamic = ["version"] +description = "Microbial Sequence Analysis and Loci-based Typing" +authors = [ + {name = "Isak Sylvin", email = "isak.sylvin@scilifelab.se"}, +] +requires-python = ">=3.12" +dependencies = [ + "biopython>=1.81", + "bs4==0.0.1", + "click>=8.0", + "Jinja2>=3.1", + "pydantic>=2.0", + "python-dateutil>=2.8", + "pymysql>=1.0", + "pyyaml>=6.0", + "sqlalchemy>=2.0", + "requests-oauthlib>=1.3", + "werkzeug>=3.1.6", +] + +[project.scripts] +microsalt = "microSALT.cli:root" + +[project.urls] +Homepage = "https://github.com/Clinical-Genomics/microSALT" + +[tool.setuptools.dynamic] +version = {attr = "microSALT.__version__"} + +[tool.setuptools.packages.find] +where = ["."] +include = ["microSALT*"] + +[tool.setuptools.package-data] +"microSALT" = ["unique_references/*", "artwork/*"] +"microSALT.server" = ["templates/*"] + +[dependency-groups] +dev = [ + "black>=23.0", + "flake8>=6.0", + "gitlint>=0.19.1", + "pytest>=7.4", + "pylint>=3.0", + "yamllint>=1.32", +] diff --git a/requirements-dev.txt b/requirements-dev.txt deleted file mode 100644 index d5448f31..00000000 --- a/requirements-dev.txt +++ /dev/null @@ -1,10 +0,0 @@ -black==21.9b0 -coverage<5 -flake8>=4.0.1,<4.1 -gitlint==0.15.1 -pytest==6.2.5 -pytest-cov -python-coveralls -pylint==2.10.2 -mock==4.0.3 -yamllint==1.26.3 diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 5cdd9804..00000000 --- a/requirements.txt +++ /dev/null @@ -1,11 +0,0 @@ -biopython==1.78 -bs4==0.0.1 -click==7.1.2 -flask==1.1.2 -flask_sqlalchemy==2.4.4 -pymysql==0.10.1 -pyyaml==5.4.1 -sqlalchemy==1.3.19 -genologics==0.4.6 -rauth==0.7.3 - diff --git a/scripts/compare_typing_reports.py b/scripts/compare_typing_reports.py new file mode 100644 index 00000000..d9b26e2e --- /dev/null +++ b/scripts/compare_typing_reports.py @@ -0,0 +1,333 @@ +#!/usr/bin/env python +"""Compare typing results between two microSALT rendered HTML reports. + +Usage: + python scripts/compare_typing_reports.py report_v1.html report_v2.html + +Compares: + - Per-sample sequence types (ST calls) + - Per-sample MLST allele assignments + - Per-sample resistance gene calls + - QC threshold status + +Outputs a human-readable diff to stdout. Exit code 0 when identical, 1 when +differences are found, 2 on parse errors. +""" + +import argparse +import re +import sys +from dataclasses import dataclass, field +from pathlib import Path + +from bs4 import BeautifulSoup, Tag + +# --------------------------------------------------------------------------- +# Data containers +# --------------------------------------------------------------------------- + + +@dataclass +class SampleSummary: + customer_id: str + cg_id: str + organism: str + sequence_type: str + threshold: str + + +@dataclass +class MLSTRow: + loci: str + allele: str + identity: str + span: str + + +@dataclass +class ResistanceRow: + gene: str + group: str + reference: str + identity: str + span: str + + +@dataclass +class SampleDetail: + cg_id: str + mlst: list[MLSTRow] = field(default_factory=list) + resistances: list[ResistanceRow] = field(default_factory=list) + + +@dataclass +class Report: + path: Path + project_id: str + report_version: str + summaries: dict[str, SampleSummary] = field(default_factory=dict) # keyed by CG Prov ID + details: dict[str, SampleDetail] = field(default_factory=dict) + + +# --------------------------------------------------------------------------- +# Parsing helpers +# --------------------------------------------------------------------------- + + +def _clean(text: str) -> str: + """Strip whitespace and normalise unicode spaces.""" + return text.replace("\xa0", " ").strip() + + +def _find_table_by_header(soup: BeautifulSoup, *header_snippets: str) -> Tag | None: + """Return the first whose
text contains all given snippets.""" + for table in soup.find_all("table"): + headers = [_clean(th.get_text()) for th in table.find_all("th")] + if all(any(s in h for h in headers) for s in header_snippets): + return table + return None + + +def _rows(table: Tag) -> list[list[str]]: + """Return non-header rows as lists of cell text.""" + result = [] + for tr in table.find_all("tr"): + cells = tr.find_all("td") + if cells: + result.append([_clean(c.get_text()) for c in cells]) + return result + + +def parse_report(path: Path) -> Report: + html = path.read_text(encoding="utf-8") + soup = BeautifulSoup(html, "html.parser") + + # --- project summary metadata ---------------------------------------- + project_id = "" + report_version = "" + + # Look for the "Projektsammanställning" table (key/value pairs via ) + for table in soup.find_all("table"): + td_texts = [_clean(td.get_text()) for td in table.find_all("td")] + for i, txt in enumerate(td_texts): + if "CG Projekt ID" in txt and i + 1 < len(td_texts): + # e.g. "ACC1234 (CG1234)" — extract the CG id inside parens + val = td_texts[i + 1] + m = re.search(r"\(([^)]+)\)", val) + project_id = m.group(1) if m else val + if "Rapport version" in txt and i + 1 < len(td_texts): + report_version = td_texts[i + 1] + + # --- summary table --------------------------------------------------- + summaries: dict[str, SampleSummary] = {} + summary_table = _find_table_by_header(soup, "Sekvenstyp", "Tröskelv") + if summary_table: + for row in _rows(summary_table): + if len(row) >= 5: + s = SampleSummary( + customer_id=row[0], + cg_id=row[1], + organism=row[2], + sequence_type=row[3], + threshold=row[4], + ) + summaries[s.cg_id] = s + + # --- per-sample detail tables ---------------------------------------- + details: dict[str, SampleDetail] = {} + + # Each sample detail section has an overview table containing "CG Prov ID". + # We walk all tables and use context to pair MLST/resistance tables with + # the sample they belong to. + current_detail: SampleDetail | None = None + + for table in soup.find_all("table"): + td_texts = [_clean(td.get_text()) for td in table.find_all("td")] + th_texts = [_clean(th.get_text()) for th in table.find_all("th")] + + # Detect sample overview table (contains "Prov ID (CG Prov ID)" header row) + if any("CG Prov ID" in t for t in td_texts): + # Reset context before extraction so a failed parse doesn't leak + # the previous sample's detail into the tables that follow. + current_detail = None + for i, t in enumerate(td_texts): + if "CG Prov ID" in t and i + 1 < len(td_texts): + # value cell may look like "ACC5551 (CG5551)" + m = re.search(r"\(([^)]+)\)", td_texts[i + 1]) + cg_id = m.group(1) if m else td_texts[i + 1].split()[-1] + current_detail = SampleDetail(cg_id=cg_id) + details[cg_id] = current_detail + break + continue + + # MLST table + if current_detail and "Loci" in th_texts and "Allel" in th_texts: + for row in _rows(table): + if len(row) >= 4: + current_detail.mlst.append( + MLSTRow( + loci=row[1], + allele=row[2], + identity=row[3], + span=row[4] if len(row) > 4 else "", + ) + ) + continue + + # Resistance table + if current_detail and "Gen" in th_texts and "Grupp" in th_texts: + for row in _rows(table): + if len(row) >= 5: + current_detail.resistances.append( + ResistanceRow( + gene=row[1], + group=row[2], + reference=row[3], + identity=row[4], + span=row[5] if len(row) > 5 else "", + ) + ) + + return Report( + path=path, + project_id=project_id, + report_version=report_version, + summaries=summaries, + details=details, + ) + + +# --------------------------------------------------------------------------- +# Comparison logic +# --------------------------------------------------------------------------- + +_RESET = "\033[0m" +_RED = "\033[31m" +_GREEN = "\033[32m" +_YELLOW = "\033[33m" +_BOLD = "\033[1m" + + +def _diff_label(a: str, b: str) -> str: + if a == b: + return a + return f"{_RED}{a}{_RESET} → {_GREEN}{b}{_RESET}" + + +def compare_reports(r1: Report, r2: Report) -> int: + """Print differences between two reports. Returns number of differences.""" + diffs = 0 + + print( + f"\n{_BOLD}Report A:{_RESET} {r1.path} (version {r1.report_version}, project {r1.project_id})" + ) + print( + f"{_BOLD}Report B:{_RESET} {r2.path} (version {r2.report_version}, project {r2.project_id})\n" + ) + + all_cg_ids = sorted(set(r1.summaries) | set(r2.summaries)) + if not all_cg_ids: + print( + f"{_YELLOW}WARNING: No samples found — check that the HTML files are valid microSALT typing reports.{_RESET}" + ) + return 0 + + for cg_id in all_cg_ids: + s1 = r1.summaries.get(cg_id) + s2 = r2.summaries.get(cg_id) + + if s1 is None: + print(f"{_GREEN}+ {cg_id}: present only in report B{_RESET}") + diffs += 1 + continue + if s2 is None: + print(f"{_RED}- {cg_id}: present only in report A{_RESET}") + diffs += 1 + continue + + sample_diffs = [] + + if s1.sequence_type != s2.sequence_type: + sample_diffs.append(f" ST: {_diff_label(s1.sequence_type, s2.sequence_type)}") + if s1.organism != s2.organism: + sample_diffs.append(f" Organism: {_diff_label(s1.organism, s2.organism)}") + if s1.threshold != s2.threshold: + sample_diffs.append(f" Threshold: {_diff_label(s1.threshold, s2.threshold)}") + + # MLST allele comparison + d1 = r1.details.get(cg_id) + d2 = r2.details.get(cg_id) + if d1 is None or d2 is None: + missing = "A" if d1 is None else "B" + sample_diffs.append( + f" {_YELLOW}WARNING: per-sample detail section missing in report {missing}{_RESET}" + ) + if d1 and d2: + loci1 = {m.loci: m.allele for m in d1.mlst} + loci2 = {m.loci: m.allele for m in d2.mlst} + all_loci = sorted(set(loci1) | set(loci2)) + for locus in all_loci: + a1 = loci1.get(locus, "—") + a2 = loci2.get(locus, "—") + if a1 != a2: + sample_diffs.append(f" MLST [{locus}]: {_diff_label(a1, a2)}") + + # Resistance comparison (gene-level set diff) + genes1 = {r.gene for r in d1.resistances} + genes2 = {r.gene for r in d2.resistances} + for g in sorted(genes1 - genes2): + sample_diffs.append(f" {_RED}- resistance: {g}{_RESET}") + for g in sorted(genes2 - genes1): + sample_diffs.append(f" {_GREEN}+ resistance: {g}{_RESET}") + + if sample_diffs: + diffs += len(sample_diffs) + print(f"{_BOLD}{cg_id}{_RESET} ({s1.customer_id}):") + for line in sample_diffs: + print(line) + print() + + if diffs == 0: + print(f"{_GREEN}No differences found.{_RESET}") + else: + print(f"{_YELLOW}{diffs} difference(s) found.{_RESET}") + + return diffs + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Compare typing results between two microSALT rendered HTML reports." + ) + parser.add_argument("report_a", type=Path, help="Path to first (older) HTML report") + parser.add_argument("report_b", type=Path, help="Path to second (newer) HTML report") + parser.add_argument("--no-color", action="store_true", help="Disable ANSI color output") + args = parser.parse_args() + + if args.no_color: + global _RESET, _RED, _GREEN, _YELLOW, _BOLD + _RESET = _RED = _GREEN = _YELLOW = _BOLD = "" + + for p in (args.report_a, args.report_b): + if not p.exists(): + print(f"ERROR: File not found: {p}", file=sys.stderr) + return 2 + + try: + r1 = parse_report(args.report_a) + r2 = parse_report(args.report_b) + except Exception as exc: + print(f"ERROR: Failed to parse report — {exc}", file=sys.stderr) + return 2 + + diffs = compare_reports(r1, r2) + return 0 if diffs == 0 else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/migrate_sqlite_to_mysql.py b/scripts/migrate_sqlite_to_mysql.py new file mode 100644 index 00000000..59e9ca6e --- /dev/null +++ b/scripts/migrate_sqlite_to_mysql.py @@ -0,0 +1,190 @@ +#!/usr/bin/env python +"""Migrate ORM table data from an old SQLite database to a new MySQL database. + +Tables migrated (insertion order respects FK constraints): + projects → samples → seq_types → resistances → expacs + → reports → collections → versions → system_locks + +Profile/Novel tables (managed outside of the ORM declarative base) are +intentionally excluded. + +Usage: + python scripts/migrate_sqlite_to_mysql.py \\ + --sqlite sqlite:////path/to/old/microsalt.db \\ + --mysql mysql+pymysql://user:pass@host/dbname + +The script is idempotent: rows already present in the target (matched by +primary key) are skipped rather than duplicated or overwritten. +""" + +import argparse +import sys + +from sqlalchemy import create_engine, inspect +from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm import Session + +from microSALT.store.orm_models import ( + Base, + Collections, + Expacs, + Projects, + Reports, + Resistances, + Samples, + Seq_types, + SystemLock, + Versions, +) + +# Tables in the order they must be inserted to satisfy FK constraints. +TABLES: list[type] = [ + Projects, + Samples, + Seq_types, + Resistances, + Expacs, + Reports, + Collections, + Versions, + SystemLock, +] + + +def _columns(model: type) -> list[str]: + """Return the list of column attribute names for an ORM model.""" + return [c.key for c in inspect(model).mapper.column_attrs] + + +def _pk_columns(model: type) -> list[str]: + """Return the primary key column names for an ORM model.""" + return [col.name for col in inspect(model).mapper.primary_key] + + +def migrate_table(src: Session, dst: Session, model: type) -> tuple[int, int]: + """Copy rows from src to dst for the given model. + + Returns (inserted, skipped) counts. + """ + table_name = model.__tablename__ + cols = _columns(model) + pk_cols = _pk_columns(model) + + inserted = 0 + skipped = 0 + orphaned = 0 + + rows = src.query(model).all() + for row in rows: + # Check if the row already exists in the destination by PK lookup + pk_filter = {col: getattr(row, col) for col in pk_cols} + exists = dst.get( + model, + tuple(pk_filter[c] for c in pk_cols) + if len(pk_cols) > 1 + else next(iter(pk_filter.values())), + ) + if exists is not None: + skipped += 1 + continue + + # Build a fresh detached copy so we don't accidentally modify the + # source session's identity map. + new_obj = model(**{col: getattr(row, col) for col in cols}) + dst.add(new_obj) + try: + dst.flush() + inserted += 1 + except IntegrityError: + dst.rollback() + orphaned += 1 + + return inserted, skipped, orphaned + + +def main() -> int: + parser = argparse.ArgumentParser(description="Migrate microSALT ORM data from SQLite to MySQL.") + parser.add_argument( + "--sqlite", + required=True, + metavar="PATH_OR_URI", + help="Path to the source SQLite file, or a full SQLAlchemy URI (e.g. sqlite:////path/to/microsalt.db)", + ) + parser.add_argument( + "--mysql", + required=True, + metavar="URI", + help="SQLAlchemy URI for the target MySQL DB (e.g. mysql+pymysql://user:pass@host/db)", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Read source data and report counts without writing to MySQL.", + ) + args = parser.parse_args() + + # Accept a bare file path as well as a full SQLAlchemy URI + sqlite_uri = args.sqlite + if not sqlite_uri.startswith("sqlite:"): + import os + + sqlite_uri = "sqlite:///" + os.path.abspath(sqlite_uri) + + print(f"Source : {sqlite_uri}") + print(f"Target : {args.mysql}") + if args.dry_run: + print("DRY RUN — no data will be written.\n") + + src_engine = create_engine(sqlite_uri, pool_pre_ping=True) + dst_engine = create_engine(args.mysql, pool_pre_ping=True) + + # Ensure all ORM tables exist in the destination + if not args.dry_run: + Base.metadata.create_all(dst_engine) + + total_inserted = 0 + total_skipped = 0 + + with Session(src_engine) as src_session, Session(dst_engine) as dst_session: + src_inspector = inspect(src_engine) + for model in TABLES: + table_name = model.__tablename__ + + # Check whether the table exists in the source at all + if table_name not in src_inspector.get_table_names(): + print(f" {table_name:<20} — not present in source, skipping") + continue + + row_count = src_session.query(model).count() + if row_count == 0: + print(f" {table_name:<20} — 0 rows in source, skipping") + continue + + if args.dry_run: + print(f" {table_name:<20} — {row_count} rows would be processed") + total_inserted += row_count + continue + + try: + inserted, skipped, orphaned = migrate_table(src_session, dst_session, model) + summary = f"inserted {inserted}, skipped {skipped} (already present)" + if orphaned: + summary += f", skipped {orphaned} (orphaned, no parent row)" + print(f" {table_name:<20} — {summary}") + total_inserted += inserted + total_skipped += skipped + orphaned + except Exception as exc: + dst_session.rollback() + print(f" {table_name:<20} — ERROR: {exc}", file=sys.stderr) + print("Rolling back entire transaction.", file=sys.stderr) + return 1 + + if not args.dry_run: + dst_session.commit() + + print(f"\nDone. Inserted {total_inserted} rows, skipped {total_skipped} duplicates.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/setup.py b/setup.py deleted file mode 100644 index 7de35ffb..00000000 --- a/setup.py +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env python -from microSALT import __version__ -from setuptools import setup, find_packages - -version = __version__ - -try: - with open("requirements.txt", "r") as f: - install_requires = [x.strip() for x in f.readlines()] -except IOError: - install_requires = [] - -setup( - name="microSALT", - version=version, - long_description=__doc__, - url="https://github.com/Clinical-Genomics/microSALT", - author="Isak Sylvin", - author_email='isak.sylvin@scilifelab.se', - install_requires=install_requires, - packages=find_packages(), - include_package_data=True, - zip_safe=False, - data_files=[('expec', ['unique_references/ExPEC.fsa']), - ('logos', ['artwork/microsalt.jpg', 'artwork/swedac.jpg']), - ('testdata', ['tests/testdata/sampleinfo_samples.json','tests/testdata/sampleinfo_mlst.json','tests/testdata/sampleinfo_projects.json','tests/testdata/sampleinfo_reports.json','tests/testdata/sampleinfo_expec.json','tests/testdata/sampleinfo_resistance.json','tests/testdata/quast_results.tsv', 'tests/testdata/blast_single_resistance.txt','tests/testdata/blast_single_loci.txt','tests/testdata/alignment.stats.ref','tests/testdata/alignment.stats.raw','tests/testdata/alignment.stats.map','tests/testdata/alignment.stats.ins','tests/testdata/alignment.stats.dup','tests/testdata/alignment.stats.cov','configExample.json']), - ('tests/testdata/AAA1234_2000.1.2_3.4.5/AAA1234A1/', ['tests/testdata/AAA1234_2000.1.2_3.4.5/AAA1234A1/dummysequences_1.fastq.gz', 'tests/testdata/AAA1234_2000.1.2_3.4.5/AAA1234A1/dummysequences_2.fastq.gz']), - ('tests/testdata/AAA1234_2000.1.2_3.4.5/AAA1234A2/', ['tests/testdata/AAA1234_2000.1.2_3.4.5/AAA1234A2/dummysequences_1.fastq.gz', 'tests/testdata/AAA1234_2000.1.2_3.4.5/AAA1234A2/dummysequences_2.fastq.gz']), - ('tests/testdata/AAA1234_2000.1.2_3.4.5/AAA1234A3/', ['tests/testdata/AAA1234_2000.1.2_3.4.5/AAA1234A3/dummysequences_1.fastq.gz', 'tests/testdata/AAA1234_2000.1.2_3.4.5/AAA1234A3/dummysequences_2.fastq.gz']), - ('tests/testdata/AAA1234_2000.1.2_3.4.5/AAA1234A4/', ['tests/testdata/AAA1234_2000.1.2_3.4.5/AAA1234A4/dummysequences_1.fastq.gz', 'tests/testdata/AAA1234_2000.1.2_3.4.5/AAA1234A4/dummysequences_2.fastq.gz'])], - entry_points={ - 'console_scripts': ['microSALT=microSALT.cli:root'], - }, -) - diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 00000000..5197aba0 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,173 @@ +import json +import logging +import pathlib +from importlib.resources import files as resource_files + +import pytest + +from microSALT import setup_logger +from microSALT.config import ( + Containers, + Database, + Folders, + MicroSALTConfig, + PasteurCredentials, + PubMLSTCredentials, + Regex, + Singularity, + SlurmHeader, + Threshold, +) +from microSALT.store.database import initialize_database +from microSALT.store.db_manipulator import DB_Manipulator + + +@pytest.fixture(scope="session") +def config(tmp_path_factory: pytest.TempPathFactory) -> MicroSALTConfig: + """Session-scoped config built from tmp_path_factory so all paths are isolated.""" + base = tmp_path_factory.mktemp("microsalt") + + results = base / "results" + reports = base / "reports" + seqdata = base / "projects" + profiles = base / "references" / "ST_profiles" + references = base / "references" / "ST_loci" + resistances = base / "references" / "resistances" + genomes = base / "references" / "genomes" + credentials = base / "credentials" + + for d in (results, reports, seqdata, profiles, references, resistances, genomes, credentials): + d.mkdir(parents=True, exist_ok=True) + + db_path = base / "microsalt.db" + + cfg = MicroSALTConfig( + slurm_header=SlurmHeader( + time="12:00:00", + threads="8", + qos="normal", + job_prefix="MLST", + project="production", + type="core", + ), + regex=Regex( + mail_recipient="username@suffix.com", + file_pattern=r"\w{8,12}_\w{8,10}(?:-\d+)*_L\d_(?:R)*(\d{1}).fastq.gz", + verified_organisms=[], + ), + folders=Folders( + results=str(results), + reports=str(reports), + seqdata=str(seqdata), + profiles=str(profiles), + references=str(references), + resistances=str(resistances), + genomes=str(genomes), + credentials=str(credentials), + ), + database=Database( + SQLALCHEMY_DATABASE_URI=f"sqlite:///{db_path}", + SQLALCHEMY_TRACK_MODIFICATIONS="False", + DEBUG="True", + ), + threshold=Threshold(), + pubmlst=PubMLSTCredentials(), + pasteur=PasteurCredentials(), + singularity=Singularity(), + containers=Containers(), + ) + cfg.folders.expec = str(resource_files("microSALT").joinpath("unique_references", "ExPEC.fsa")) + cfg.config_path = str(base / "config.json") + + setup_logger(logging_level="INFO") + initialize_database(cfg.database.SQLALCHEMY_DATABASE_URI) + return cfg + + +@pytest.fixture(scope="session") +def logger(): + return logging.getLogger("main_logger") + + +@pytest.fixture +def unpack_db_json(): + """Factory fixture: returns a callable that loads JSON files from tests/testdata/.""" + + def _load(filename: str) -> list: + path = pathlib.Path(__file__).parent / "testdata" / filename + return json.loads(path.read_text()) + + return _load + + +@pytest.fixture +def dbm(config: MicroSALTConfig, logger: logging.Logger, unpack_db_json): + """DB_Manipulator populated with the standard set of test data.""" + dbm = DB_Manipulator(log=logger, folders=config.folders, threshold=config.threshold) + dbm.create_tables() + + for entry in unpack_db_json("sampleinfo_projects.json"): + dbm.add_to_session(dbm.add_project(**entry)) + dbm.commit_session() + for entry in unpack_db_json("sampleinfo_mlst.json"): + dbm.add_to_session(dbm.add_seq_type(**entry)) + dbm.commit_session() + for entry in unpack_db_json("sampleinfo_resistance.json"): + dbm.add_to_session(dbm.add_resistance(**entry)) + dbm.commit_session() + for entry in unpack_db_json("sampleinfo_expec.json"): + dbm.add_to_session(dbm.add_expac(**entry)) + dbm.commit_session() + for entry in unpack_db_json("sampleinfo_reports.json"): + dbm.add_to_session(dbm.add_report(**entry)) + dbm.commit_session() + return dbm + + +@pytest.fixture +def exp_config(): + """Expected configuration structure for config validation tests.""" + return { + "slurm_header": {"time", "threads", "qos", "job_prefix", "project", "type"}, + "regex": {"file_pattern", "mail_recipient", "verified_organisms"}, + "folders": { + "results", + "reports", + "seqdata", + "profiles", + "references", + "resistances", + "genomes", + "expec", + "credentials", + }, + "threshold": { + "mlst_id", + "mlst_novel_id", + "mlst_span", + "motif_id", + "motif_span", + "total_reads_warn", + "total_reads_fail", + "NTC_total_reads_warn", + "NTC_total_reads_fail", + "mapped_rate_warn", + "mapped_rate_fail", + "duplication_rate_warn", + "duplication_rate_fail", + "insert_size_warn", + "insert_size_fail", + "average_coverage_warn", + "average_coverage_fail", + "bp_10x_warn", + "bp_10x_fail", + "bp_30x_warn", + "bp_50x_warn", + "bp_100x_warn", + }, + "database": {"SQLALCHEMY_DATABASE_URI", "SQLALCHEMY_TRACK_MODIFICATIONS", "DEBUG"}, + "pubmlst": {"client_id", "client_secret"}, + "pasteur": {"client_id", "client_secret"}, + "singularity": {"binary", "bind_paths", "trimmomatic_adapters"}, + "containers": {"skesa", "blast", "bwa", "samtools", "picard", "trimmomatic", "quast"}, + } diff --git a/tests/server/__init__.py b/tests/server/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/server/conftest.py b/tests/server/conftest.py new file mode 100644 index 00000000..ab0d8d5b --- /dev/null +++ b/tests/server/conftest.py @@ -0,0 +1,140 @@ +import pytest +from datetime import datetime + +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker, scoped_session + +import microSALT.store.database as db_module +from microSALT.store.orm_models import ( + Base, + Projects, + Reports, + Samples, + Seq_types, + Versions, +) + + +@pytest.fixture(scope="module") +def db_engine(): + """In-memory SQLite engine shared for the whole module.""" + engine = create_engine("sqlite:///:memory:") + Base.metadata.create_all(engine) + return engine + + +@pytest.fixture(scope="module") +def db_session(config, db_engine): + """Scoped session bound to the in-memory engine. + + Depends on ``config`` to ensure initialize_database() (called by that + fixture) runs *before* we wire db_module.session to the in-memory engine, + so the in-memory session is not overwritten afterwards. + + Also wires up the module-level session used by get_session() in views.py. + """ + session_factory = sessionmaker(db_engine) + session = scoped_session(session_factory) + + # Wire up the database module so get_session() returns this session + db_module.engine = db_engine + db_module.session = session + + yield session + + session.remove() + + +@pytest.fixture(scope="module") +def populated_db(db_session): + """Populate the in-memory database with a minimal set of test data.""" + # Project + project = Projects( + CG_ID_project="AAA1234", + Customer_ID_project="999999", + Customer_ID="cust000", + ) + db_session.add(project) + + # Samples + sample1 = Samples( + CG_ID_sample="AAA1234A1", + CG_ID_project="AAA1234", + Customer_ID_sample="XXX0000Y1", + organism="staphylococcus_aureus", + ST=8, + pubmlst_ST=-1, + total_reads=500000, + insert_size=200, + duplication_rate=0.05, + mapped_rate=0.95, + average_coverage=120.0, + coverage_10x=0.99, + coverage_30x=0.97, + coverage_50x=0.90, + coverage_100x=0.50, + genome_length=2800000, + gc_percentage=32.5, + n50=250000, + contigs=12, + ) + sample2 = Samples( + CG_ID_sample="AAA1234A2", + CG_ID_project="AAA1234", + Customer_ID_sample="XXX0000Y2", + organism="escherichia_coli", + ST=131, + pubmlst_ST=-1, + total_reads=600000, + insert_size=180, + duplication_rate=0.04, + mapped_rate=0.97, + average_coverage=150.0, + coverage_10x=0.99, + coverage_30x=0.98, + coverage_50x=0.95, + coverage_100x=0.70, + genome_length=5000000, + gc_percentage=50.0, + n50=400000, + contigs=10, + ) + db_session.add(sample1) + db_session.add(sample2) + + # Seq types for sample1 + for loci, allele in [("arcC", 6), ("aroE", 57), ("glpF", 45)]: + st = Seq_types( + CG_ID_sample="AAA1234A1", + loci=loci, + allele=allele, + contig_name="NODE_1", + contig_length=592262, + contig_coverage=150.0, + identity=100.0, + span=1.0, + evalue="0.0", + bitscore=900, + subject_length=456, + st_predictor=True, + contig_start=100, + contig_end=556, + ) + db_session.add(st) + + # Report + report = Reports( + CG_ID_project="AAA1234", + steps_aggregate="abc123", + date=datetime(2020, 7, 14, 15, 3, 51), + version=1, + ) + db_session.add(report) + + # Versions + v = Versions(name="software_microSALT", version="4.3.0") + db_session.add(v) + + db_session.commit() + + yield db_session diff --git a/tests/server/test_jinja2_rendering.py b/tests/server/test_jinja2_rendering.py new file mode 100644 index 00000000..dd9b89a5 --- /dev/null +++ b/tests/server/test_jinja2_rendering.py @@ -0,0 +1,321 @@ +"""Tests for Jinja2 rendering and ORM model creation. + +These tests verify: +1. ORM model instantiation with correct attributes +2. render_template produces valid HTML output +3. Each page rendering function (project_page, alignment_page, typing_page, + STtracker_page) generates HTML that contains expected content +""" + +import pytest + +from microSALT.store.orm_models import ( + Collections, + Expacs, + Projects, + Reports, + Resistances, + Samples, + Seq_types, + Versions, +) +from microSALT.server.views import ( + TEMPLATE_FOLDER, + STtracker_page, + alignment_page, + project_page, + render_template, + typing_page, +) + + +# --------------------------------------------------------------------------- +# ORM model creation tests +# --------------------------------------------------------------------------- + + +class TestOrmModels: + """Verify ORM models can be instantiated and have correct attributes.""" + + def test_projects_model_attributes(self): + p = Projects( + CG_ID_project="TST0001", + Customer_ID_project="CUST001", + Customer_ID="cust001", + ) + assert p.CG_ID_project == "TST0001" + assert p.Customer_ID_project == "CUST001" + assert p.Customer_ID == "cust001" + + def test_samples_model_attributes(self): + s = Samples( + CG_ID_sample="TST0001A1", + CG_ID_project="TST0001", + organism="staphylococcus_aureus", + ST=8, + pubmlst_ST=-1, + ) + assert s.CG_ID_sample == "TST0001A1" + assert s.CG_ID_project == "TST0001" + assert s.organism == "staphylococcus_aureus" + assert s.ST == 8 + + def test_seq_types_model_attributes(self): + st = Seq_types( + CG_ID_sample="TST0001A1", + loci="arcC", + allele=6, + contig_name="NODE_1", + identity=100.0, + span=1.0, + st_predictor=True, + ) + assert st.loci == "arcC" + assert st.allele == 6 + assert st.identity == 100.0 + assert st.st_predictor is True + + def test_resistances_model_attributes(self): + r = Resistances( + CG_ID_sample="TST0001A1", + gene="blaZ", + instance="blaZ_1", + contig_name="NODE_2", + identity=99.5, + span=0.98, + resistance="beta-lactam", + ) + assert r.gene == "blaZ" + assert r.resistance == "beta-lactam" + assert r.identity == 99.5 + + def test_expacs_model_attributes(self): + e = Expacs( + CG_ID_sample="TST0001A1", + gene="kpsMII", + instance="kpsMII_1", + contig_name="NODE_3", + identity=98.0, + span=0.95, + virulence="capsule", + ) + assert e.gene == "kpsMII" + assert e.virulence == "capsule" + + def test_reports_model_attributes(self): + r = Reports( + CG_ID_project="TST0001", + steps_aggregate="abc", + version=1, + ) + assert r.CG_ID_project == "TST0001" + assert r.version == 1 + + def test_versions_model_attributes(self): + v = Versions(name="software_microSALT", version="4.3.0") + assert v.name == "software_microSALT" + assert v.version == "4.3.0" + + def test_collections_model_attributes(self): + c = Collections(ID_collection="COL001", CG_ID_sample="TST0001A1") + assert c.ID_collection == "COL001" + assert c.CG_ID_sample == "TST0001A1" + + def test_samples_model_can_be_persisted(self, db_session): + """Verify a model can be added to the session and queried.""" + project = Projects( + CG_ID_project="MODEL_TEST", + Customer_ID_project="MT001", + Customer_ID="cust_mt", + ) + sample = Samples( + CG_ID_sample="MODEL_TESTA1", + CG_ID_project="MODEL_TEST", + organism="test_organism", + ST=42, + pubmlst_ST=-1, + ) + db_session.add(project) + db_session.add(sample) + db_session.commit() + + result = db_session.query(Samples).filter_by(CG_ID_sample="MODEL_TESTA1").one() + assert result.organism == "test_organism" + assert result.ST == 42 + + # Cleanup + db_session.delete(sample) + db_session.delete(project) + db_session.commit() + + +# --------------------------------------------------------------------------- +# render_template tests +# --------------------------------------------------------------------------- + + +class TestRenderTemplate: + """Verify the raw render_template function produces correct HTML.""" + + def test_renders_html_string(self): + """render_template returns a non-empty string.""" + html = render_template( + template_folder=TEMPLATE_FOLDER, + template_name="project_page.html", + organisms=["all", "staphylococcus_aureus"], + project="TST0001", + ) + assert isinstance(html, str) + assert len(html) > 0 + + def test_renders_project_in_output(self): + """Project ID appears in rendered project_page output.""" + html = render_template( + template_folder=TEMPLATE_FOLDER, + template_name="project_page.html", + organisms=["all"], + project="MYPROJECT", + ) + assert "MYPROJECT" in html + + def test_renders_organisms_in_output(self): + """Organism names appear in rendered project_page output.""" + html = render_template( + template_folder=TEMPLATE_FOLDER, + template_name="project_page.html", + organisms=["all", "staphylococcus_aureus"], + project="TST0001", + ) + assert "staphylococcus_aureus" in html or "Staphylococcus aureus" in html + + def test_url_for_stub_injected_automatically(self): + """url_for stub is injected when not provided, preventing TemplateError.""" + # If url_for were missing, Jinja2 would raise UndefinedError + html = render_template( + template_folder=TEMPLATE_FOLDER, + template_name="project_page.html", + organisms=["all"], + project="TST0001", + ) + # Should still render without error and contain HTML + assert " 100 + + def test_project_page_contains_project_id(self, populated_db): + """project_page HTML contains the project identifier.""" + html = project_page("AAA1234") + assert "AAA1234" in html + + def test_project_page_contains_organisms(self, populated_db): + """project_page lists organisms found for the project.""" + html = project_page("AAA1234") + # Both organisms should appear (in some form) + assert "staphylococcus_aureus" in html or "Staphylococcus aureus" in html + assert "escherichia_coli" in html or "Escherichia coli" in html + + def test_project_page_contains_all_organism(self, populated_db): + """project_page always includes the 'all' organism group.""" + html = project_page("AAA1234") + assert "all" in html.lower() + + def test_alignment_page_renders_html(self, populated_db, config): + """alignment_page returns non-empty HTML string.""" + html = alignment_page("AAA1234", threshold=config.threshold) + assert isinstance(html, str) + assert len(html) > 100 + + def test_alignment_page_is_valid_html(self, populated_db, config): + """alignment_page output contains basic HTML structure.""" + html = alignment_page("AAA1234", threshold=config.threshold) + assert "" in html.lower() or "" in html.lower() + + def test_alignment_page_contains_sample_ids(self, populated_db, config): + """alignment_page HTML contains sample identifiers.""" + html = alignment_page("AAA1234", threshold=config.threshold) + assert "AAA1234A1" in html or "AAA1234A2" in html + + def test_typing_page_renders_html(self, populated_db, config): + """typing_page returns non-empty HTML string.""" + html = typing_page("AAA1234", "all", threshold=config.threshold, verified_organisms=config.regex.verified_organisms) + assert isinstance(html, str) + assert len(html) > 100 + + def test_typing_page_is_valid_html(self, populated_db, config): + """typing_page output contains basic HTML structure.""" + html = typing_page("AAA1234", "all", threshold=config.threshold, verified_organisms=config.regex.verified_organisms) + assert "" in html.lower() or "" in html.lower() + + def test_typing_page_contains_sample_ids(self, populated_db, config): + """typing_page HTML contains sample identifiers.""" + html = typing_page("AAA1234", "all", threshold=config.threshold, verified_organisms=config.regex.verified_organisms) + assert "AAA1234A1" in html or "AAA1234A2" in html + + def test_typing_page_organism_filter(self, populated_db, config): + """typing_page filters by organism correctly.""" + html_staph = typing_page("AAA1234", "staphylococcus_aureus", threshold=config.threshold, verified_organisms=config.regex.verified_organisms) + assert "AAA1234A1" in html_staph + + def test_sttracker_page_renders_html(self, populated_db, config): + """STtracker_page returns non-empty HTML string.""" + html = STtracker_page("all", threshold=config.threshold) + assert isinstance(html, str) + assert len(html) > 100 + + def test_sttracker_page_is_valid_html(self, populated_db, config): + """STtracker_page output contains basic HTML structure.""" + html = STtracker_page("all", threshold=config.threshold) + assert "" in html.lower() or "" in html.lower() + + def test_sttracker_page_customer_filter(self, populated_db, config): + """STtracker_page customer='all' does not raise errors.""" + html = STtracker_page("cust000", threshold=config.threshold) + assert isinstance(html, str) + + def test_project_page_empty_project(self, populated_db): + """project_page for unknown project renders without error.""" + html = project_page("UNKNOWN99") + assert isinstance(html, str) + assert "UNKNOWN99" in html + + def test_render_alignment_page_delegates(self, populated_db, config): + """render_alignment_page produces same output as alignment_page.""" + from microSALT.server.views import render_alignment_page + + html_a = alignment_page("AAA1234", threshold=config.threshold) + html_b = render_alignment_page("AAA1234", threshold=config.threshold) + assert html_a == html_b + + def test_render_typing_page_delegates(self, populated_db, config): + """render_typing_page produces same output as typing_page.""" + from microSALT.server.views import render_typing_page + + html_a = typing_page("AAA1234", "all", threshold=config.threshold, verified_organisms=config.regex.verified_organisms) + html_b = render_typing_page("AAA1234", "all", threshold=config.threshold, verified_organisms=config.regex.verified_organisms) + assert html_a == html_b diff --git a/tests/store/__init__.py b/tests/store/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/store/conftest.py b/tests/store/conftest.py new file mode 100644 index 00000000..66855351 --- /dev/null +++ b/tests/store/conftest.py @@ -0,0 +1,74 @@ +import copy + +import pytest +from sqlalchemy import inspect as sa_inspect + +from microSALT.store.db_manipulator import DB_Manipulator +from microSALT.store.orm_models import SystemLock + + +@pytest.fixture +def tmp_profiles_dir(tmp_path): + """Creates a temporary profiles directory with a staphylococcus_aureus profile file. + + The file contains two rows: ST=130 (matching sampleinfo_mlst.json alleles) and ST=1. + """ + content = ( + "ST\tarcC\taroE\tglpF\tgmk\tpta\ttpi\tyqiL\n" + "1\t1\t1\t1\t1\t1\t1\t1\n" + "130\t6\t57\t45\t2\t7\t58\t52\n" + ) + (tmp_path / "staphylococcus_aureus").write_text(content) + return tmp_path + + +@pytest.fixture +def profile_dbm(config, logger, tmp_profiles_dir, unpack_db_json): + """DB_Manipulator with profile/novel tables freshly built from tmp_profiles_dir. + + Uses the shared SQLite database but drops and recreates all profile/novel tables + on each invocation so tests start with known, clean data. + """ + cfg = copy.deepcopy(config) + cfg.folders.profiles = str(tmp_profiles_dir) + + dbm = DB_Manipulator(log=logger, folders=cfg.folders, threshold=cfg.threshold) + dbm.create_tables() + + inspector = sa_inspect(dbm.engine) + + # Drop and recreate profile tables with data from tmp_profiles_dir. + for name, table in list(dbm.profiles.items()): + if inspector.has_table(table.name): + table.drop(dbm.engine) + table.create(dbm.engine) + dbm.populate_profiletable(name, table) + + # Drop and recreate novel tables (empty — entries are written by application logic). + for name, table in list(dbm.novel.items()): + if inspector.has_table(table.name): + table.drop(dbm.engine) + table.create(dbm.engine) + + for entry in unpack_db_json("sampleinfo_projects.json"): + dbm.add_to_session(dbm.add_project(**entry)) + dbm.commit_session() + for entry in unpack_db_json("sampleinfo_mlst.json"): + dbm.add_to_session(dbm.add_seq_type(**entry)) + dbm.commit_session() + + return dbm + + +@pytest.fixture +def clean_lock_dbm(dbm): + """Wraps dbm and guarantees the ref_update lock is absent before and after each test. + + Without this, a test that fails mid-way while holding the lock would leave dirty + state in the shared SQLite DB and cause unrelated tests to fail. + """ + dbm.session.query(SystemLock).filter_by(lock_name="ref_update").delete() + dbm.session.commit() + yield dbm + dbm.session.query(SystemLock).filter_by(lock_name="ref_update").delete() + dbm.session.commit() diff --git a/tests/store/test_database.py b/tests/store/test_database.py new file mode 100644 index 00000000..eeb8b8bf --- /dev/null +++ b/tests/store/test_database.py @@ -0,0 +1,458 @@ +from unittest.mock import patch + +import pytest +from sqlalchemy import inspect as sa_inspect + +from microSALT.exc.exceptions import RefUpdateLockError +from microSALT.store.db_manipulator import DB_Manipulator, _resolve_orm_table +from microSALT.store.orm_models import Reports, Samples, SystemLock + + +def test_create_every_table(dbm): + inspector = sa_inspect(dbm.engine) + assert inspector.has_table("samples") + assert inspector.has_table("seq_types") + assert inspector.has_table("resistances") + assert inspector.has_table("expacs") + assert inspector.has_table("projects") + assert inspector.has_table("reports") + assert inspector.has_table("collections") + + +def test_add_rec(caplog, profile_dbm): + dbm = profile_dbm + # Profile table + dbm.add_rec( + { + "ST": "130", + "arcC": "6", + "aroE": "57", + "glpF": "45", + "gmk": "2", + "pta": "7", + "tpi": "58", + "yqiL": "52", + }, + dbm.profiles["staphylococcus_aureus"], + ) + assert len(dbm.read_records(dbm.profiles["staphylococcus_aureus"], {"ST": "130"})) == 1 + assert len(dbm.read_records(dbm.profiles["staphylococcus_aureus"], {"ST": "-1"})) == 0 + + # Novel table + dbm.add_rec( + { + "ST": "130", + "arcC": "6", + "aroE": "57", + "glpF": "45", + "gmk": "2", + "pta": "7", + "tpi": "58", + "yqiL": "52", + }, + dbm.novel["staphylococcus_aureus"], + ) + assert len(dbm.read_records(dbm.novel["staphylococcus_aureus"], {"ST": "130"})) == 1 + assert len(dbm.read_records(dbm.novel["staphylococcus_aureus"], {"ST": "-1"})) == 0 + + # ORM tables + dbm.add_to_session(dbm.add_sample(CG_ID_sample="ADD1234A1")) + dbm.commit_session() + assert len(dbm.read_records("Samples", {"CG_ID_sample": "ADD1234A1"})) > 0 + assert len(dbm.read_records("Samples", {"CG_ID_sample": "XXX1234A10"})) == 0 + + dbm.add_to_session(dbm.add_seq_type(CG_ID_sample="ADD1234A1", loci="mdh", contig_name="NODE_1")) + dbm.commit_session() + assert ( + len( + dbm.read_records( + "Seq_types", {"CG_ID_sample": "ADD1234A1", "loci": "mdh", "contig_name": "NODE_1"} + ) + ) + > 0 + ) + assert ( + len( + dbm.read_records( + "Seq_types", {"CG_ID_sample": "XXX1234A10", "loci": "mdh", "contig_name": "NODE_1"} + ) + ) + == 0 + ) + + dbm.add_to_session( + dbm.add_resistance( + CG_ID_sample="ADD1234A1", + gene="Type 1", + instance="Type 1", + contig_name="NODE_1", + ) + ) + dbm.commit_session() + assert ( + len( + dbm.read_records( + "Resistances", + { + "CG_ID_sample": "ADD1234A1", + "gene": "Type 1", + "instance": "Type 1", + "contig_name": "NODE_1", + }, + ) + ) + > 0 + ) + assert ( + len( + dbm.read_records( + "Resistances", + { + "CG_ID_sample": "XXX1234A10", + "gene": "Type 1", + "instance": "Type 1", + "contig_name": "NODE_1", + }, + ) + ) + == 0 + ) + + dbm.add_to_session( + dbm.add_expac( + CG_ID_sample="ADD1234A1", + gene="Type 1", + instance="Type 1", + contig_name="NODE_1", + ) + ) + dbm.commit_session() + assert ( + len( + dbm.read_records( + "Expacs", + { + "CG_ID_sample": "ADD1234A1", + "gene": "Type 1", + "instance": "Type 1", + "contig_name": "NODE_1", + }, + ) + ) + > 0 + ) + assert ( + len( + dbm.read_records( + "Expacs", + { + "CG_ID_sample": "XXX1234A10", + "gene": "Type 1", + "instance": "Type 1", + "contig_name": "NODE_1", + }, + ) + ) + == 0 + ) + + dbm.add_to_session(dbm.add_project(CG_ID_project="ADD1234")) + dbm.commit_session() + assert len(dbm.read_records("Projects", {"CG_ID_project": "ADD1234"})) > 0 + assert len(dbm.read_records("Projects", {"CG_ID_project": "XXX1234"})) == 0 + + dbm.add_to_session(dbm.add_report(CG_ID_project="ADD1234", version="1")) + dbm.commit_session() + assert len(dbm.read_records("Reports", {"CG_ID_project": "ADD1234", "version": "1"})) > 0 + assert len(dbm.read_records("Reports", {"CG_ID_project": "XXX1234", "version": "1"})) == 0 + + dbm.add_to_session( + dbm.add_collection(CG_ID_sample="ADD1234", ID_collection="MyCollectionFolder") + ) + dbm.commit_session() + assert ( + len( + dbm.read_records( + "Collections", {"CG_ID_sample": "ADD1234", "ID_collection": "MyCollectionFolder"} + ) + ) + > 0 + ) + assert ( + len( + dbm.read_records( + "Collections", {"CG_ID_sample": "XXX1234", "ID_collection": "MyCollectionFolder"} + ) + ) + == 0 + ) + + +@patch("sys.exit") +def test_upd_rec(sysexit, caplog, dbm): + dbm.add_to_session(dbm.add_sample(CG_ID_sample="UPD1234A1")) + dbm.commit_session() + assert len(dbm.read_records("Samples", {"CG_ID_sample": "UPD1234A1"})) == 1 + assert len(dbm.read_records("Samples", {"CG_ID_sample": "UPD1234A2"})) == 0 + + dbm.update_sample({"CG_ID_sample": "UPD1234A1"}, {"CG_ID_sample": "UPD1234A2"}) + assert len(dbm.read_records("Samples", {"CG_ID_sample": "UPD1234A1"})) == 0 + assert len(dbm.read_records("Samples", {"CG_ID_sample": "UPD1234A2"})) == 1 + + dbm.update_sample({"CG_ID_sample": "UPD1234A2"}, {"CG_ID_sample": "UPD1234A1"}) + + caplog.clear() + dbm.add_to_session(dbm.add_sample(CG_ID_sample="UPD1234A1_uniq", Customer_ID_sample="cust000")) + dbm.add_to_session(dbm.add_sample(CG_ID_sample="UPD1234A2_uniq", Customer_ID_sample="cust000")) + dbm.commit_session() + dbm.update_sample({"Customer_ID_sample": "cust000"}, {"Customer_ID_sample": "cust030"}) + dbm.update_sample({"Customer_ID_sample": "cust000"}, {"Customer_ID_sample": "cust030"}) + assert "More than 1 Samples record found" in caplog.text + + +def test_allele_ranker(profile_dbm, unpack_db_json): + dbm = profile_dbm + dbm.add_to_session( + dbm.add_sample( + CG_ID_sample="MLS1234A1", + CG_ID_project="MLS1234", + organism="staphylococcus_aureus", + ) + ) + dbm.commit_session() + assert dbm.read_st("MLS1234A1") == 130 + best_alleles = { + "arcC": {"contig_name": "NODE_1", "allele": 6}, + "aroE": {"contig_name": "NODE_1", "allele": 57}, + "glpF": {"contig_name": "NODE_1", "allele": 45}, + "gmk": {"contig_name": "NODE_1", "allele": 2}, + "pta": {"contig_name": "NODE_1", "allele": 7}, + "tpi": {"contig_name": "NODE_1", "allele": 58}, + "yqiL": {"contig_name": "NODE_1", "allele": 52}, + } + assert dbm.read_best_alleles("MLS1234A1") == best_alleles + + for entry in unpack_db_json("sampleinfo_mlst.json"): + entry["allele"] = 0 + entry["CG_ID_sample"] = "MLS1234A2" + dbm.add_to_session(dbm.add_seq_type(**entry)) + dbm.commit_session() + assert dbm.read_st("MLS1234A2") == -1 + + +def test_get_and_set_report(dbm): + # Clean up any leftover data from prior runs to keep the test idempotent. + dbm.session.query(Reports).filter(Reports.CG_ID_project == "ADD1234").delete() + dbm.session.query(Samples).filter(Samples.CG_ID_sample == "ADD1234A1").delete() + dbm.session.commit() + + dbm.add_to_session(dbm.add_sample(CG_ID_sample="ADD1234A1", method_sequencing="1000:1")) + dbm.commit_session() + dbm.add_to_session(dbm.add_report(CG_ID_project="ADD1234", version="1")) + dbm.commit_session() + assert dbm.read_report("ADD1234").version == 1 + + dbm.update_sample( + {"CG_ID_sample": "ADD1234A1", "method_sequencing": "1000:1"}, + {"CG_ID_sample": "ADD1234A1", "method_sequencing": "1000:2"}, + ) + dbm.set_report("ADD1234") + assert dbm.read_report("ADD1234").version != 1 + + +@patch("sys.exit") +def test_purge_rec(sysexit, caplog, dbm): + dbm.add_to_session(dbm.add_sample(CG_ID_sample="UPD1234A1")) + dbm.commit_session() + dbm.delete_collection("UPD1234A1") + + +def test_query_rec(dbm): + dbm.add_to_session(dbm.add_sample(CG_ID_sample="QRY_001")) + dbm.add_to_session(dbm.add_sample(CG_ID_sample="QRY_002")) + dbm.commit_session() + + hits = dbm.read_records("Samples", {"CG_ID_sample": "QRY_001"}) + assert len(hits) == 1 + assert hits[0].CG_ID_sample == "QRY_001" + + no_hits = dbm.read_records("Samples", {"CG_ID_sample": "DOES_NOT_EXIST"}) + assert len(no_hits) == 0 + + multi_filter = dbm.read_records("Samples", {"CG_ID_sample": "QRY_001", "ST": None}) + assert len(multi_filter) == 1 + + +def test_get_columns(dbm): + cols = dbm.read_columns("Samples") + assert isinstance(cols, dict) + assert "CG_ID_sample" in cols + assert "organism" in cols + + +def test_get_sample_by_cg_id(dbm: DB_Manipulator): + dbm.add_to_session(dbm.add_sample(CG_ID_sample="EXS_001")) + dbm.commit_session() + + assert dbm.get_sample_by_cg_id_sample("EXS_001") + assert dbm.get_sample_by_cg_id_sample("DOES_NOT_EXIST") is None + + +def test_get_collection_by_id(dbm: DB_Manipulator): + dbm.add_to_session(dbm.add_collection(CG_ID_sample="EXS_001", ID_collection="COLL_001")) + dbm.commit_session() + + assert dbm.get_collection_by_id("COLL_001") + assert dbm.get_collection_by_id("DOES_NOT_EXIST") is None + + +def test_resolve_orm_table_unknown(): + with pytest.raises(KeyError): + _resolve_orm_table("NonExistentTable") + + +def test_resolve_orm_table_known(): + assert _resolve_orm_table("Samples") is Samples + + +def test_populate_profiletable(profile_dbm): + """populate_profiletable bulk-inserts all data rows from the profile file.""" + # Given: a profile table created from a file with two STs (ST=1 and ST=130). + dbm = profile_dbm + table = dbm.profiles["staphylococcus_aureus"] + + # When: populate_profiletable has been called by the profile_dbm fixture. + + # Then: the table contains exactly the two rows from the file. + rows = dbm.session.query(table).all() + assert len(rows) == 2 + + sts = {row.ST for row in rows} + assert 130 in sts + assert 1 in sts + + # Then: allele values for ST=130 match the source file exactly. + st130 = next(r for r in rows if r.ST == 130) + assert st130.arcC == 6 + assert st130.aroE == 57 + assert st130.glpF == 45 + assert st130.gmk == 2 + assert st130.pta == 7 + assert st130.tpi == 58 + assert st130.yqiL == 52 + + +def test_refresh_profiletable_same_schema(tmp_profiles_dir, profile_dbm): + """refresh_profiletable with unchanged columns truncates and reloads data.""" + # Given: a populated profile table (ST=1, ST=130) and a new file with the + # same column layout but different data (ST=99 only). + dbm = profile_dbm + new_content = "ST\tarcC\taroE\tglpF\tgmk\tpta\ttpi\tyqiL\n99\t3\t3\t3\t3\t3\t3\t3\n" + (tmp_profiles_dir / "staphylococcus_aureus").write_text(new_content) + + # When: refresh_profiletable is called with the updated file. + dbm.refresh_profiletable("staphylococcus_aureus") + + # Then: only the new row is present. + table = dbm.profiles["staphylococcus_aureus"] + rows = dbm.session.query(table).all() + assert len(rows) == 1 + assert rows[0].ST == 99 + + # Then: old data has been cleared. + old_rows = dbm.session.query(table).filter(table.c.ST == 130).all() + assert len(old_rows) == 0 + + +def test_refresh_profiletable_schema_change(tmp_profiles_dir, profile_dbm): + """refresh_profiletable drops and rebuilds the table when column layout changes.""" + # Given: a populated profile table with columns ST+7 loci (including yqiL) + # and a new file that renames yqiL to renamedLocus within the 8-column window. + dbm = profile_dbm + new_content = "ST\tarcC\taroE\tglpF\tgmk\tpta\ttpi\trenamedLocus\n200\t1\t2\t3\t4\t5\t6\t7\n" + (tmp_profiles_dir / "staphylococcus_aureus").write_text(new_content) + + # When: refresh_profiletable detects the schema change and does a full drop/recreate. + dbm.refresh_profiletable("staphylococcus_aureus") + + # Then: the table schema reflects the new column layout. + table = dbm.profiles["staphylococcus_aureus"] + cols = list(table.c.keys()) + assert "renamedLocus" in cols + assert "yqiL" not in cols + assert len(cols) == 8 # ST + 7 loci + + # Then: the table is populated with data from the new file. + rows = dbm.session.query(table).all() + assert len(rows) == 1 + assert rows[0].ST == 200 + + +def test_acquire_ref_lock(clean_lock_dbm): + """acquire_ref_lock inserts a lock row into the database.""" + dbm = clean_lock_dbm + + # Given: no lock exists. + assert dbm.session.query(SystemLock).filter_by(lock_name="ref_update").scalar() is None + + # When: the lock is acquired. + dbm.acquire_ref_lock() + + # Then: a SystemLock row with lock_name='ref_update' is present. + lock = dbm.session.query(SystemLock).filter_by(lock_name="ref_update").scalar() + assert lock is not None + assert lock.lock_name == "ref_update" + assert lock.acquired_at is not None + + +def test_acquire_ref_lock_raises_when_already_held(clean_lock_dbm): + """acquire_ref_lock raises RefUpdateLockError if the lock is already held.""" + dbm = clean_lock_dbm + + # Given: the lock is already held. + dbm.acquire_ref_lock() + + # When / Then: a second acquire attempt raises immediately. + with pytest.raises(RefUpdateLockError): + dbm.acquire_ref_lock() + + +def test_release_ref_lock(clean_lock_dbm): + """release_ref_lock removes the lock row so the lock can be re-acquired.""" + dbm = clean_lock_dbm + + # Given: the lock is held. + dbm.acquire_ref_lock() + assert dbm.session.query(SystemLock).filter_by(lock_name="ref_update").scalar() is not None + + # When: the lock is released. + dbm.release_ref_lock() + + # Then: no lock row exists. + assert dbm.session.query(SystemLock).filter_by(lock_name="ref_update").scalar() is None + + # Then: the lock can be acquired again without raising. + dbm.acquire_ref_lock() + assert dbm.session.query(SystemLock).filter_by(lock_name="ref_update").scalar() is not None + + +def test_check_ref_lock(clean_lock_dbm): + """check_ref_lock raises when the lock is held and is silent when it is not.""" + dbm = clean_lock_dbm + + # Given: no lock exists. + # When / Then: check passes silently. + dbm.check_ref_lock() # must not raise + + # Given: the lock is acquired. + dbm.acquire_ref_lock() + + # When / Then: check raises RefUpdateLockError. + with pytest.raises(RefUpdateLockError): + dbm.check_ref_lock() + + # Given: the lock is released. + dbm.release_ref_lock() + + # When / Then: check passes silently again. + dbm.check_ref_lock() # must not raise diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 00000000..6d1e2b32 --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,129 @@ +"""Tests for the CLI entry point — verifies config loading and propagation.""" + +import json +from pathlib import Path + +import pytest +from click.testing import CliRunner +from sqlalchemy import create_engine, inspect + +from microSALT.cli import root +from microSALT.config import ( + Containers, + Database, + Folders, + MicroSALTConfig, + PasteurCredentials, + PubMLSTCredentials, + Regex, + Singularity, + SlurmHeader, + Threshold, +) +from microSALT.store.orm_models import Base + + +@pytest.fixture +def config_file(config: MicroSALTConfig, tmp_path: Path) -> Path: + """Write the session config to a temporary JSON file for CLI consumption.""" + path = tmp_path / "config.json" + path.write_text(config.model_dump_json()) + return path + + +@pytest.fixture +def setup_config(tmp_path: Path) -> MicroSALTConfig: + """Fresh MicroSALTConfig whose directories do not yet exist — used to test `setup`.""" + from importlib.resources import files as resource_files + + base = tmp_path / "microsalt" + base.mkdir() + cfg = MicroSALTConfig( + slurm_header=SlurmHeader( + time="1:00:00", + threads="4", + qos="normal", + job_prefix="MLST", + project="test", + type="core", + ), + regex=Regex(mail_recipient="test@test.com", file_pattern=".*", verified_organisms=[]), + folders=Folders( + results=str(base / "results"), + reports=str(base / "reports"), + seqdata=str(base / "seqdata"), + profiles=str(base / "profiles"), + references=str(base / "references"), + resistances=str(base / "resistances"), + genomes=str(base / "genomes"), + credentials=str(base / "credentials"), + ), + database=Database(SQLALCHEMY_DATABASE_URI=f"sqlite:///{base / 'microsalt.db'}"), + threshold=Threshold(), + pubmlst=PubMLSTCredentials(), + pasteur=PasteurCredentials(), + singularity=Singularity(), + containers=Containers(), + ) + cfg.folders.expec = str(resource_files("microSALT").joinpath("unique_references", "ExPEC.fsa")) + return cfg + + +@pytest.fixture +def setup_config_file(setup_config: MicroSALTConfig, tmp_path: Path) -> Path: + """Write the setup_config to a JSON file and return its path.""" + path = tmp_path / "config.json" + path.write_text(setup_config.model_dump_json()) + return path + + +def invoke_root(config_file: Path, *args: str): + """Invoke the root CLI group with the test config and additional arguments.""" + runner = CliRunner() + return runner.invoke(root, ["--config", str(config_file), *args]) + + +def test_root_loads_config(config_file): + """root command loads without error when given a valid config file.""" + result = invoke_root(config_file, "--help") + assert result.exit_code == 0 + + +def test_root_version(config_file): + """--version flag returns the package version.""" + from microSALT import __version__ + + result = invoke_root(config_file, "--version") + assert result.exit_code == 0 + assert __version__ in result.output + + +def test_config_fields_propagated(config: MicroSALTConfig, config_file: Path): + """Config loaded from the JSON file matches the original MicroSALTConfig fields.""" + loaded = MicroSALTConfig.model_validate(json.loads(config_file.read_text())) + assert loaded.folders.results == config.folders.results + assert loaded.folders.reports == config.folders.reports + assert loaded.database.SQLALCHEMY_DATABASE_URI == config.database.SQLALCHEMY_DATABASE_URI + assert loaded.regex.mail_recipient == config.regex.mail_recipient + assert loaded.slurm_header.project == config.slurm_header.project + + +def test_setup_command(setup_config: MicroSALTConfig, setup_config_file: Path): + """setup command creates the configured directories and reports success.""" + result = invoke_root(setup_config_file, "setup") + assert result.exit_code == 0, result.output + assert "Directory setup complete" in result.output + assert "Database tables created" in result.output + assert Path(setup_config.folders.results).exists() + assert Path(setup_config.folders.reports).exists() + + +def test_setup_command_creates_tables(setup_config: MicroSALTConfig, setup_config_file: Path): + """setup command creates every ORM-defined table in the database.""" + result = invoke_root(setup_config_file, "setup") + assert result.exit_code == 0, result.output + + engine = create_engine(setup_config.database.SQLALCHEMY_DATABASE_URI) + created = inspect(engine).get_table_names() + expected = set(Base.metadata.tables.keys()) + assert expected == set(created), f"Missing tables: {expected - set(created)}" diff --git a/tests/test_config.py b/tests/test_config.py index b07ec467..527c65a1 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1,80 +1,92 @@ -#!/usr/bin/env python - -import collections -import os import pathlib + import pytest +from pydantic import ValidationError + +from microSALT.config import ( + Database, + Folders, + MicroSALTConfig, + Regex, + SlurmHeader, + Threshold, + load_config, +) + +CONFIGEXAMPLE = str(pathlib.Path(__file__).parent.parent / "configExample.json") + + +def test_load_config_parses_example(): + """load_config() successfully parses configExample.json into a MicroSALTConfig.""" + cfg = load_config(CONFIGEXAMPLE) + assert isinstance(cfg, MicroSALTConfig) + + +def test_config_sections_present(): + """All top-level config sections are populated after parsing.""" + cfg = load_config(CONFIGEXAMPLE) + assert isinstance(cfg.slurm_header, SlurmHeader) + assert isinstance(cfg.regex, Regex) + assert isinstance(cfg.folders, Folders) + assert isinstance(cfg.database, Database) + assert isinstance(cfg.threshold, Threshold) + + +def test_slurm_header_fields(): + cfg = load_config(CONFIGEXAMPLE) + assert cfg.slurm_header.time + assert cfg.slurm_header.threads + assert cfg.slurm_header.qos + assert cfg.slurm_header.job_prefix + assert cfg.slurm_header.project + assert cfg.slurm_header.type + + +def test_folders_fields(): + cfg = load_config(CONFIGEXAMPLE) + assert cfg.folders.results + assert cfg.folders.reports + assert cfg.folders.seqdata + assert cfg.folders.profiles + assert cfg.folders.references + assert cfg.folders.resistances + assert cfg.folders.genomes + assert cfg.folders.credentials + + +def test_expec_path_injected(): + """The expec path is derived from package data and injected by load_config.""" + cfg = load_config(CONFIGEXAMPLE) + assert cfg.folders.expec + assert "ExPEC.fsa" in cfg.folders.expec + + +def test_config_path_injected(): + """config_path is set to the resolved path of the config file.""" + cfg = load_config(CONFIGEXAMPLE) + assert cfg.config_path + assert cfg.config_path.endswith("configExample.json") + + +def test_threshold_defaults(): + cfg = load_config(CONFIGEXAMPLE) + assert cfg.threshold.mlst_id == 100 + assert cfg.threshold.mlst_novel_id == 99.5 + assert cfg.threshold.mlst_span == 90 + + +def test_runtime_defaults(): + """Runtime fields default to safe values before CLI sets them.""" + cfg = load_config(CONFIGEXAMPLE) + assert cfg.dry is False + assert cfg.config_path != "" + + +def test_missing_required_field_raises(tmp_path): + """A config missing a required section raises a Pydantic ValidationError.""" + import json -from microSALT import preset_config - -@pytest.fixture -def exp_config(): - precon = { - 'slurm_header': {'time', 'threads', 'qos', 'job_prefix', 'project', 'type'}, - 'regex': {'file_pattern', 'mail_recipient', 'verified_organisms'}, - 'folders': {'results', 'reports', 'log_file', 'seqdata', 'profiles', 'references', 'resistances', 'genomes', 'expec', 'adapters', 'credentials'}, - 'threshold': {'mlst_id', 'mlst_novel_id', 'mlst_span', 'motif_id', 'motif_span', 'total_reads_warn', 'total_reads_fail', - 'NTC_total_reads_warn', 'NTC_total_reads_fail', 'mapped_rate_warn', 'mapped_rate_fail', 'duplication_rate_warn', - 'duplication_rate_fail', 'insert_size_warn', 'insert_size_fail', 'average_coverage_warn', 'average_coverage_fail', - 'bp_10x_warn', 'bp_10x_fail', 'bp_30x_warn', 'bp_50x_warn', 'bp_100x_warn'}, - 'database': {'SQLALCHEMY_DATABASE_URI', 'SQLALCHEMY_TRACK_MODIFICATIONS', 'DEBUG'}, - 'genologics': {'baseuri', 'username', 'password'}, - 'pubmlst': {'client_id', 'client_secret'}, - 'pasteur': {'client_id', 'client_secret'}, - 'dry': True, - } - return precon - -def test_existence(exp_config): - """Checks that the configuration contains certain key variables""" - # level one - config_level_one = preset_config.keys() - for entry in exp_config.keys(): - if entry != 'dry': - assert entry in config_level_one - - # level two - if isinstance(preset_config[entry], collections.Mapping): - config_level_two = preset_config[entry].keys() - for thing in exp_config[entry]: - assert thing in config_level_two - -def test_reverse_existence(exp_config): - """Check that the configuration doesn't contain outdated variables""" - - # level one - config_level_one = exp_config.keys() - for entry in preset_config.keys(): - if entry not in ['_comment']: - assert entry in config_level_one - - # level two - config_level_two = exp_config[entry] - if isinstance(preset_config[entry], collections.Mapping): - for thing in preset_config[entry].keys(): - if thing != '_comment': - assert thing in config_level_two - -def test_paths(exp_config): - """Tests existence for all paths mentioned in variables""" - # level one - for entry in preset_config.keys(): - if entry != '_comment': - if isinstance(preset_config[entry], str) and '/' in preset_config[entry] and entry not in ['database', 'genologics']: - unmade_fldr = preset_config[entry] - # Embed logic to expand vars and user here - unmade_fldr = os.path.expandvars(unmade_fldr) - unmade_fldr = os.path.expanduser(unmade_fldr) - unmade_fldr = os.path.abspath(unmade_fldr) - assert (pathlib.Path(unmade_fldr).exists()) - - # level two - elif isinstance(preset_config[entry], collections.Mapping): - for thing in preset_config[entry].keys(): - if isinstance(preset_config[entry][thing], str) and '/' in preset_config[entry][thing] and entry not in ['database', 'genologics']: - unmade_fldr = preset_config[entry][thing] - # Embed logic to expand vars and user here - unmade_fldr = os.path.expandvars(unmade_fldr) - unmade_fldr = os.path.expanduser(unmade_fldr) - unmade_fldr = os.path.abspath(unmade_fldr) - assert (pathlib.Path(unmade_fldr).exists()) + bad_config = tmp_path / "bad.json" + bad_config.write_text(json.dumps({"slurm_header": {"time": "1:00:00"}})) + with pytest.raises(ValidationError): + load_config(str(bad_config)) diff --git a/tests/test_database.py b/tests/test_database.py deleted file mode 100644 index e9ca73d8..00000000 --- a/tests/test_database.py +++ /dev/null @@ -1,180 +0,0 @@ -#!/usr/bin/env python - -import json -import os -import pathlib -import pdb -import pytest -import re -import requests -import sys -import time - -from distutils.sysconfig import get_python_lib -from unittest.mock import patch - -from microSALT.store.db_manipulator import DB_Manipulator -from microSALT import preset_config, logger -from microSALT.cli import root - - -def unpack_db_json(filename): - testdata = os.path.abspath(os.path.join(pathlib.Path(__file__).parent.parent, 'tests/testdata/{}'.format(filename))) - #Check if release install exists - for entry in os.listdir(get_python_lib()): - if 'microSALT-' in entry: - testdata = os.path.abspath( - os.path.join(os.path.expandvars('$CONDA_PREFIX'), 'testdata/{}'.format(filename))) - with open(testdata) as json_file: - data = json.load(json_file) - return data - - -@pytest.fixture -def dbm(): - db_file = re.search('sqlite:///(.+)', preset_config['database']['SQLALCHEMY_DATABASE_URI']).group(1) - dbm = DB_Manipulator(config=preset_config, log=logger) - dbm.create_tables() - - for antry in unpack_db_json('sampleinfo_projects.json'): - dbm.add_rec(antry, 'Projects') - for entry in unpack_db_json('sampleinfo_mlst.json'): - dbm.add_rec(entry, 'Seq_types') - for bentry in unpack_db_json('sampleinfo_resistance.json'): - dbm.add_rec(bentry, 'Resistances') - for centry in unpack_db_json('sampleinfo_expec.json'): - dbm.add_rec(centry, 'Expacs') - for dentry in unpack_db_json('sampleinfo_reports.json'): - dbm.add_rec(dentry, 'Reports') - return dbm - - -def test_create_every_table(dbm): - assert dbm.engine.dialect.has_table(dbm.engine, 'samples') - assert dbm.engine.dialect.has_table(dbm.engine, 'seq_types') - assert dbm.engine.dialect.has_table(dbm.engine, 'resistances') - assert dbm.engine.dialect.has_table(dbm.engine, 'expacs') - assert dbm.engine.dialect.has_table(dbm.engine, 'projects') - assert dbm.engine.dialect.has_table(dbm.engine, 'reports') - assert dbm.engine.dialect.has_table(dbm.engine, 'collections') - - -@pytest.mark.xfail(reason="Can no longer fetch from databases without authenticating") -def test_add_rec(caplog, dbm): - #Adds records to all databases - dbm.add_rec( - {'ST': '130', 'arcC': '6', 'aroE': '57', 'glpF': '45', 'gmk': '2', 'pta': '7', 'tpi': '58', 'yqiL': '52', - 'clonal_complex': 'CC1'}, dbm.profiles['staphylococcus_aureus']) - assert len(dbm.query_rec(dbm.profiles['staphylococcus_aureus'], {'ST': '130'})) == 1 - assert len(dbm.query_rec(dbm.profiles['staphylococcus_aureus'], {'ST': '-1'})) == 0 - - dbm.add_rec( - {'ST': '130', 'arcC': '6', 'aroE': '57', 'glpF': '45', 'gmk': '2', 'pta': '7', 'tpi': '58', 'yqiL': '52', - 'clonal_complex': 'CC1'}, dbm.novel['staphylococcus_aureus']) - assert len(dbm.query_rec(dbm.novel['staphylococcus_aureus'], {'ST': '130'})) == 1 - assert len(dbm.query_rec(dbm.novel['staphylococcus_aureus'], {'ST': '-1'})) == 0 - - dbm.add_rec({'CG_ID_sample': 'ADD1234A1'}, 'Samples') - assert len(dbm.query_rec('Samples', {'CG_ID_sample': 'ADD1234A1'})) > 0 - assert len(dbm.query_rec('Samples', {'CG_ID_sample': 'XXX1234A10'})) == 0 - - dbm.add_rec({'CG_ID_sample': 'ADD1234A1', 'loci': 'mdh', 'contig_name': 'NODE_1'}, 'Seq_types') - assert len(dbm.query_rec('Seq_types', {'CG_ID_sample': 'ADD1234A1', 'loci': 'mdh', 'contig_name': 'NODE_1'})) > 0 - assert len(dbm.query_rec('Seq_types', {'CG_ID_sample': 'XXX1234A10', 'loci': 'mdh', 'contig_name': 'NODE_1'})) == 0 - - dbm.add_rec({'CG_ID_sample': 'ADD1234A1', 'gene': 'Type 1', 'instance': 'Type 1', 'contig_name': 'NODE_1'}, - 'Resistances') - assert len(dbm.query_rec('Resistances', {'CG_ID_sample': 'ADD1234A1', 'gene': 'Type 1', 'instance': 'Type 1', - 'contig_name': 'NODE_1'})) > 0 - assert len(dbm.query_rec('Resistances', {'CG_ID_sample': 'XXX1234A10', 'gene': 'Type 1', 'instance': 'Type 1', - 'contig_name': 'NODE_1'})) == 0 - - dbm.add_rec({'CG_ID_sample': 'ADD1234A1', 'gene': 'Type 1', 'instance': 'Type 1', 'contig_name': 'NODE_1'}, - 'Expacs') - assert len(dbm.query_rec('Expacs', {'CG_ID_sample': 'ADD1234A1', 'gene': 'Type 1', 'instance': 'Type 1', - 'contig_name': 'NODE_1'})) > 0 - assert len(dbm.query_rec('Expacs', {'CG_ID_sample': 'XXX1234A10', 'gene': 'Type 1', 'instance': 'Type 1', - 'contig_name': 'NODE_1'})) == 0 - - dbm.add_rec({'CG_ID_project': 'ADD1234'}, 'Projects') - assert len(dbm.query_rec('Projects', {'CG_ID_project': 'ADD1234'})) > 0 - assert len(dbm.query_rec('Projects', {'CG_ID_project': 'XXX1234'})) == 0 - - dbm.add_rec({'CG_ID_project': 'ADD1234', 'version': '1'}, 'Reports') - assert len(dbm.query_rec('Reports', {'CG_ID_project': 'ADD1234', 'version': '1'})) > 0 - assert len(dbm.query_rec('Reports', {'CG_ID_project': 'XXX1234', 'version': '1'})) == 0 - - dbm.add_rec({'CG_ID_sample': 'ADD1234', 'ID_collection': 'MyCollectionFolder'}, 'Collections') - assert len(dbm.query_rec('Collections', {'CG_ID_sample': 'ADD1234', 'ID_collection': 'MyCollectionFolder'})) > 0 - assert len(dbm.query_rec('Collections', {'CG_ID_sample': 'XXX1234', 'ID_collection': 'MyCollectionFolder'})) == 0 - - caplog.clear() - with pytest.raises(Exception): - dbm.add_rec({'CG_ID_sample': 'ADD1234A1'}, 'An_entry_that_does_not_exist') - assert "Attempted to access table" in caplog.text - - -@patch('sys.exit') -def test_upd_rec(sysexit, caplog, dbm): - dbm.add_rec({'CG_ID_sample': 'UPD1234A1'}, 'Samples') - assert len(dbm.query_rec('Samples', {'CG_ID_sample': 'UPD1234A1'})) == 1 - assert len(dbm.query_rec('Samples', {'CG_ID_sample': 'UPD1234A2'})) == 0 - - dbm.upd_rec({'CG_ID_sample': 'UPD1234A1'}, 'Samples', {'CG_ID_sample': 'UPD1234A2'}) - assert len(dbm.query_rec('Samples', {'CG_ID_sample': 'UPD1234A1'})) == 0 - assert len(dbm.query_rec('Samples', {'CG_ID_sample': 'UPD1234A2'})) == 1 - - dbm.upd_rec({'CG_ID_sample': 'UPD1234A2'}, 'Samples', {'CG_ID_sample': 'UPD1234A1'}) - - caplog.clear() - dbm.add_rec({'CG_ID_sample': 'UPD1234A1_uniq', 'Customer_ID_sample': 'cust000'}, 'Samples') - dbm.add_rec({'CG_ID_sample': 'UPD1234A2_uniq', 'Customer_ID_sample': 'cust000'}, 'Samples') - dbm.upd_rec({'Customer_ID_sample': 'cust000'}, 'Samples', {'Customer_ID_sample': 'cust030'}) - dbm.upd_rec({'Customer_ID_sample': 'cust000'}, 'Samples', {'Customer_ID_sample': 'cust030'}) - assert "More than 1 record found" in caplog.text - - -@pytest.mark.xfail(reason="Can no longer fetch from databases without authenticating") -def test_allele_ranker(dbm): - dbm.add_rec({'CG_ID_sample': 'MLS1234A1', 'CG_ID_project': 'MLS1234', 'organism': 'staphylococcus_aureus'}, - 'Samples') - assert dbm.alleles2st('MLS1234A1') == 130 - best_alleles = {'arcC': {'contig_name': 'NODE_1', 'allele': 6}, 'aroE': {'contig_name': 'NODE_1', 'allele': 57}, - 'glpF': {'contig_name': 'NODE_1', 'allele': 45}, 'gmk': {'contig_name': 'NODE_1', 'allele': 2}, - 'pta': {'contig_name': 'NODE_1', 'allele': 7}, 'tpi': {'contig_name': 'NODE_1', 'allele': 58}, - 'yqiL': {'contig_name': 'NODE_1', 'allele': 52}} - assert dbm.bestAlleles('MLS1234A1') == best_alleles - - for entry in unpack_db_json('sampleinfo_mlst.json'): - entry['allele'] = 0 - entry['CG_ID_sample'] = 'MLS1234A2' - dbm.add_rec(entry, 'Seq_types') - dbm.alleles2st('MLS1234A2') == -1 - - -@pytest.mark.xfail(reason="Can no longer fetch from databases without authenticating") -def test_get_and_set_report(dbm): - dbm.add_rec({'CG_ID_sample': 'ADD1234A1', 'method_sequencing': '1000:1'}, 'Samples') - dbm.add_rec({'CG_ID_project': 'ADD1234', 'version': '1'}, 'Reports') - assert dbm.get_report('ADD1234').version == 1 - - dbm.upd_rec({'CG_ID_sample': 'ADD1234A1', 'method_sequencing': '1000:1'}, 'Samples', - {'CG_ID_sample': 'ADD1234A1', 'method_sequencing': '1000:2'}) - dbm.set_report('ADD1234') - assert dbm.get_report('ADD1234').version != 1 - - -@patch('sys.exit') -def test_purge_rec(sysexit, caplog, dbm): - dbm.add_rec({'CG_ID_sample': 'UPD1234A1'}, 'Samples') - dbm.purge_rec('UPD1234A1', 'Collections') - - caplog.clear() - dbm.purge_rec('UPD1234A1', 'Not_Samples_nor_Collections') - assert "Incorrect type" in caplog.text - - -def test_top_index(dbm): - dbm.add_rec({'CG_ID_sample': 'Uniq_ID_123', 'total_reads': 100}, 'Samples') - dbm.add_rec({'CG_ID_sample': 'Uniq_ID_321', 'total_reads': 100}, 'Samples') - ti_returned = dbm.top_index('Samples', {'total_reads': '100'}, 'total_reads') diff --git a/tests/test_jobcreator.py b/tests/test_jobcreator.py deleted file mode 100644 index c3ad7c51..00000000 --- a/tests/test_jobcreator.py +++ /dev/null @@ -1,111 +0,0 @@ -#!/usr/bin/env python - -import json -import mock -import os -import pathlib -import pdb -import pytest -import re - -from distutils.sysconfig import get_python_lib -from unittest.mock import patch - -from microSALT.store.db_manipulator import DB_Manipulator -from microSALT.utils.job_creator import Job_Creator -from microSALT import preset_config, logger -from microSALT.cli import root - - -@pytest.fixture -def testdata(): - testdata = os.path.abspath( - os.path.join(pathlib.Path(__file__).parent.parent, 'tests/testdata/sampleinfo_samples.json')) - #Check if release install exists - for entry in os.listdir(get_python_lib()): - if 'microSALT-' in entry: - testdata = os.path.abspath( - os.path.join(os.path.expandvars('$CONDA_PREFIX'), 'testdata/sampleinfo_samples.json')) - with open(testdata) as json_file: - data = json.load(json_file) - return data - - -def fake_search(int): - return "fake" - - - -@patch('os.listdir') -@patch('os.stat') -@patch('gzip.open') -@pytest.mark.xfail(reason="Can no longer fetch from databases without authenticating") -def test_verify_fastq(gopen, stat, listdir, testdata): - listdir.return_value = ["ACC6438A3_HVMHWDSXX_L1_1.fastq.gz", "ACC6438A3_HVMHWDSXX_L1_2.fastq.gz", - "ACC6438A3_HVMHWDSXX_L2_2.fastq.gz", "ACC6438A3_HVMHWDSXX_L2_2.fastq.gz"] - stata = mock.MagicMock() - stata.st_size = 2000 - stat.return_value = stata - - jc = Job_Creator(run_settings={'input': '/tmp/'}, config=preset_config, log=logger, sampleinfo=testdata) - t = jc.verify_fastq() - assert len(t) > 0 - - - -@patch('re.search') -@patch('microSALT.utils.job_creator.glob.glob') -@pytest.mark.xfail(reason="Can no longer fetch from databases without authenticating") -def test_blast_subset(glob_search, research, testdata): - jc = Job_Creator(run_settings={'input': '/tmp/'}, config=preset_config, log=logger, sampleinfo=testdata) - researcha = mock.MagicMock() - researcha.group = fake_search - research.return_value = researcha - glob_search.return_value = ["/a/a/a", "/a/a/b", "/a/a/c"] - - jc.blast_subset('mlst', '/tmp/*') - jc.blast_subset('other', '/tmp/*') - outfile = open(jc.get_sbatch(), 'r') - count = 0 - for x in outfile.readlines(): - if "blastn -db" in x: - count = count + 1 - assert count > 0 - - -@pytest.mark.xfail(reason="Can no longer fetch from databases without authenticating") -def test_create_snpsection(subproc, testdata): - #Sets up subprocess mocking - process_mock = mock.Mock() - attrs = {'communicate.return_value': ('output 123456789', 'error')} - process_mock.configure_mock(**attrs) - subproc.return_value = process_mock - - testdata = [testdata[0]] - jc = Job_Creator(run_settings={'input': ['AAA1234A1', 'AAA1234A2']}, config=preset_config, log=logger, - sampleinfo=testdata) - jc.snp_job() - outfile = open(jc.get_sbatch(), 'r') - count = 0 - for x in outfile.readlines(): - if "# SNP pair-wise distance" in x: - count = count + 1 - assert count > 0 - - -@pytest.mark.xfail(reason="Can no longer fetch from databases without authenticating") -@patch('subprocess.Popen') -def test_project_job(subproc, testdata): - #Sets up subprocess mocking - process_mock = mock.Mock() - attrs = {'communicate.return_value': ('output 123456789', 'error')} - process_mock.configure_mock(**attrs) - subproc.return_value = process_mock - - jc = Job_Creator(config=preset_config, log=logger, sampleinfo=testdata, - run_settings={'pool': ["AAA1234A1", "AAA1234A2"], 'input': '/tmp/AAA1234'}) - jc.project_job() - - -def test_create_collection(): - pass diff --git a/tests/test_reporter.py b/tests/test_reporter.py deleted file mode 100644 index 113272cb..00000000 --- a/tests/test_reporter.py +++ /dev/null @@ -1,126 +0,0 @@ -#!/usr/bin/env python - -import datetime -import glob -import json -import logging -import os -import pathlib -import pdb -import pytest -import re -import sys - -from distutils.sysconfig import get_python_lib -from unittest.mock import patch - -from microSALT import preset_config, logger -from microSALT.utils.reporter import Reporter -from microSALT.store.db_manipulator import DB_Manipulator - -def unpack_db_json(filename): - testdata = os.path.abspath(os.path.join(pathlib.Path(__file__).parent.parent, 'tests/testdata/{}'.format(filename))) - #Check if release install exists - for entry in os.listdir(get_python_lib()): - if 'microSALT-' in entry: - testdata = os.path.abspath(os.path.join(os.path.expandvars('$CONDA_PREFIX'), 'testdata/{}'.format(filename))) - with open(testdata) as json_file: - data = json.load(json_file) - return data - -@pytest.fixture -def mock_db(): - db_file = re.search('sqlite:///(.+)', preset_config['database']['SQLALCHEMY_DATABASE_URI']).group(1) - dbm = DB_Manipulator(config=preset_config,log=logger) - dbm.create_tables() - - for antry in unpack_db_json('sampleinfo_projects.json'): - dbm.add_rec(antry, 'Projects') - for entry in unpack_db_json('sampleinfo_mlst.json'): - dbm.add_rec(entry, 'Seq_types') - for bentry in unpack_db_json('sampleinfo_resistance.json'): - dbm.add_rec(bentry, 'Resistances') - for centry in unpack_db_json('sampleinfo_expec.json'): - dbm.add_rec(centry, 'Expacs') - for dentry in unpack_db_json('sampleinfo_reports.json'): - dbm.add_rec(dentry, 'Reports') - return dbm - -@pytest.fixture -def reporter(): - reporter_obj = Reporter(config=preset_config, log=logger, sampleinfo=unpack_db_json('sampleinfo_samples.json')[0], name="MIC1234A1", output="/tmp/MLST") - return reporter_obj - -def test_motif(mock_db, reporter): - reporter.create_subfolders() - reporter.gen_motif(motif="resistance") - assert len( glob.glob("{}/AAA1234_resistance*".format(reporter.output))) > 0 - - reporter.gen_motif(motif="expec") - assert len( glob.glob("{}/AAA1234_expec*".format(reporter.output))) > 0 - -def test_deliveryreport(mock_db, reporter): - reporter.create_subfolders() - reporter.gen_delivery() - assert len( glob.glob("{}/deliverables/999999_deliverables.yaml".format(preset_config['folders']['reports']))) > 0 - -def test_jsonreport(mock_db, reporter): - reporter.create_subfolders() - reporter.gen_json() - assert len( glob.glob("{}/json/AAA1234.json".format(preset_config['folders']['reports']))) > 0 - -def test_gen_qc(mock_db, reporter): - reporter.name = "name_that_do_not_exist" - with pytest.raises(Exception): - reporter.gen_qc() - -def test_gen_typing(mock_db, reporter): - reporter.name = "name_that_do_not_exist" - with pytest.raises(Exception): - reporter.gen_typing() - -def test_gen_motif(caplog, reporter): - caplog.clear() - reporter.gen_motif(motif="unrecognized") - assert "Invalid motif type" in caplog.text - caplog.clear() - reporter.output = "/path/that/do/not/exists/" - reporter.gen_motif() - assert "Gen_motif unable to produce" in caplog.text - -def test_gen_json(caplog, reporter): - caplog.clear() - reporter.output = "/path/that/do/not/exists/" - preset_config["folders"]["reports"] = "/path/that/do/not/exists/" - reporter.config = preset_config - reporter.gen_json() - assert "Gen_json unable to produce" in caplog.text - -def test_report(caplog, reporter): - caplog.clear() - reporter.type = "type_not_mentioned_in_list" - with pytest.raises(Exception): - reporter.report() - assert "Report function recieved invalid format" in caplog.text - -@patch('microSALT.utils.reporter.Reporter.start_web') -def test_restart_web(sw, reporter): - reporter.restart_web() - -def test_constructor(): - sample_info = [ - {"CG_ID_project" : "AAA1234","CG_ID_sample" : "AAA1234A1","Customer_ID_project" : - "999999","Customer_ID_sample" : "XXX0000Y1","Customer_ID" : "cust000","application_tag" : - "NONE","date_arrival" : "0001-01-01 00:00:00","date_libprep" : "0001-01-01 00:00:00", - "date_sequencing" : "0001-01-01 00:00:00","method_libprep" : "Not in LIMS", - "method_sequencing" : "Not in LIMS","organism" : "Staphylococcus aureus","priority" : - "standard","reference" : "AP017922.1"}, - {"CG_ID_project" : "Something_else_than_AAA1234","CG_ID_sample" : "AAA1234A2", - "Customer_ID_project" : "999999","Customer_ID_sample" : "XXX0000Y1","Customer_ID" : - "cust000","application_tag" : "NONE","date_arrival" : "0001-01-01 00:00:00","date_libprep" : - "0001-01-01 00:00:00","date_sequencing" : "0001-01-01 00:00:00","method_libprep" : - "Not in LIMS","method_sequencing" : "Not in LIMS","organism" : "Escherichia coli", - "priority" : "standard","reference" : "NC_011751.1"} - ] - reporter_obj = Reporter(config=preset_config, log=logger, sampleinfo=sample_info, - name="MIC1234A1", output="/tmp/MLST") diff --git a/tests/test_scraper.py b/tests/test_scraper.py deleted file mode 100644 index 8046bce3..00000000 --- a/tests/test_scraper.py +++ /dev/null @@ -1,76 +0,0 @@ -#!/usr/bin/env python - -import glob -import json -import logging -import os -import pathlib -import pdb -import pytest - -from distutils.sysconfig import get_python_lib - -from microSALT import preset_config, logger -from microSALT.utils.scraper import Scraper -from microSALT.utils.referencer import Referencer - - -@pytest.fixture -def testdata_prefix(): - test_path = os.path.abspath(os.path.join(pathlib.Path(__file__).parent.parent, 'tests/testdata/')) - #Check if release install exists - for entry in os.listdir(get_python_lib()): - if 'microSALT-' in entry: - test_path = os.path.abspath(os.path.join(os.path.expandvars('$CONDA_PREFIX'), 'testdata/')) - return test_path - - -@pytest.fixture -def testdata(): - testdata = os.path.abspath( - os.path.join(pathlib.Path(__file__).parent.parent, 'tests/testdata/sampleinfo_samples.json')) - #Check if release install exists - for entry in os.listdir(get_python_lib()): - if 'microSALT-' in entry: - testdata = os.path.abspath( - os.path.join(os.path.expandvars('$CONDA_PREFIX'), 'testdata/sampleinfo_samples.json')) - with open(testdata) as json_file: - data = json.load(json_file) - return data - - -@pytest.fixture -def scraper(testdata): - scrape_obj = Scraper(config=preset_config, log=logger, sampleinfo=testdata[0]) - return scrape_obj - - -@pytest.fixture -def init_references(testdata): - ref_obj = Referencer(config=preset_config, log=logger, sampleinfo=testdata) - ref_obj.identify_new(testdata[0].get('CG_ID_project'), project=True) - ref_obj.update_refs() - - -@pytest.mark.xfail(reason="Can no longer fetch from databases without authenticating") -def test_quast_scraping(scraper, testdata_prefix, caplog): - scraper.scrape_quast(filename="{}/quast_results.tsv".format(testdata_prefix)) - - -@pytest.mark.xfail(reason="Can no longer fetch from databases without authenticating") -def test_blast_scraping(scraper, testdata_prefix, caplog): - caplog.set_level(logging.DEBUG) - scraper.scrape_blast(type='seq_type', file_list=["{}/blast_single_loci.txt".format(testdata_prefix)]) - assert "candidate" in caplog.text - - caplog.clear() - hits = scraper.scrape_blast(type='resistance', file_list=["{}/blast_single_resistance.txt".format(testdata_prefix)]) - genes = [h["gene"] for h in hits] - - assert "blaOXA-48" in genes - assert "blaVIM-4" in genes - - -@pytest.mark.xfail(reason="Can no longer fetch from databases without authenticating") -def test_alignment_scraping(scraper, init_references, testdata_prefix): - scraper.scrape_alignment(file_list=glob.glob("{}/*.stats.*".format(testdata_prefix))) diff --git a/tests/test_webserver.py b/tests/test_webserver.py deleted file mode 100644 index 0a48a128..00000000 --- a/tests/test_webserver.py +++ /dev/null @@ -1,140 +0,0 @@ -#!/usr/bin/env python - -import json -import os -import pathlib -import pdb -import pytest -import requests -import re -import runpy -import time - -from distutils.sysconfig import get_python_lib -from unittest.mock import patch - -from microSALT.utils.reporter import Reporter -from microSALT import preset_config, logger -from microSALT.cli import root -from microSALT.server.views import * -from microSALT.store.db_manipulator import DB_Manipulator - -def unpack_db_json(filename): - testdata = os.path.abspath(os.path.join(pathlib.Path(__file__).parent.parent, 'tests/testdata/{}'.format(filename))) - #Check if release install exists - for entry in os.listdir(get_python_lib()): - if 'microSALT-' in entry: - testdata = os.path.abspath(os.path.join(os.path.expandvars('$CONDA_PREFIX'), 'testdata/{}'.format(filename))) - with open(testdata) as json_file: - data = json.load(json_file) - return data - -@pytest.fixture -def mock_db(): - db_file = re.search('sqlite:///(.+)', preset_config['database']['SQLALCHEMY_DATABASE_URI']).group(1) - dbm = DB_Manipulator(config=preset_config,log=logger) - dbm.create_tables() - - for antry in unpack_db_json('sampleinfo_projects.json'): - dbm.add_rec(antry, 'Projects') - for entry in unpack_db_json('sampleinfo_mlst.json'): - dbm.add_rec(entry, 'Seq_types') - for bentry in unpack_db_json('sampleinfo_resistance.json'): - dbm.add_rec(bentry, 'Resistances') - for centry in unpack_db_json('sampleinfo_expec.json'): - dbm.add_rec(centry, 'Expacs') - for dentry in unpack_db_json('sampleinfo_reports.json'): - dbm.add_rec(dentry, 'Reports') - return dbm - -@pytest.fixture -def testdata(): - testdata = os.path.abspath(os.path.join(pathlib.Path(__file__).parent.parent, 'tests/testdata/sampleinfo_samples.json')) - #Check if release install exists - for entry in os.listdir(get_python_lib()): - if 'microSALT-' in entry: - testdata = os.path.abspath(os.path.join(os.path.expandvars('$CONDA_PREFIX'), 'testdata/sampleinfo_samples.json')) - with open(testdata) as json_file: - data = json.load(json_file) - return data - -@pytest.fixture -def report_obj(testdata): - report = Reporter(config=preset_config, log=logger, sampleinfo=testdata) - return report - -@pytest.fixture -def appscript(): - script = os.path.abspath(os.path.join(pathlib.Path(__file__).parent.parent, 'microSALT/server/app.py')) - return script - -def test_webserver(report_obj): - report_obj.start_web() - report_obj.kill_flask() - -def test_appobject(appscript): - runpy.run_path(appscript) - -def test_pages(report_obj, mock_db): - report_obj.start_web() - #Valid pages with available data - time.sleep(3) - a = requests.get("http://127.0.0.1:5000/", allow_redirects=True) - assert a.status_code == 200 - - time.sleep(0.15) - b = requests.get("http://127.0.0.1:5000/microSALT/", allow_redirects=True) - assert b.status_code == 200 - - time.sleep(0.15) - c = requests.get("http://127.0.0.1:5000/microSALT/AAA1234", allow_redirects=True) - assert c.status_code == 200 - - time.sleep(0.15) - e = requests.get("http://127.0.0.1:5000/microSALT/AAA1234/typing/all", allow_redirects=True) - assert e.status_code in [200, 500] - - time.sleep(0.15) - d = requests.get("http://127.0.0.1:5000/microSALT/AAA1234/qc", allow_redirects=True) - assert d.status_code in [200, 500] - - #Valid pages with unavailable data - time.sleep(0.15) - f = requests.get("http://127.0.0.1:5000/microSALT/AAA1234/typing/escherichia_coli", allow_redirects=True) - assert f.status_code in [200, 500] - - time.sleep(0.15) - g = requests.get("http://127.0.0.1:5000/microSALT/STtracker/all", allow_redirects=True) - assert g.status_code in [200, 500] - - time.sleep(0.15) - h = requests.get("http://127.0.0.1:5000/microSALT/STtracker/cust000", allow_redirects=True) - assert h.status_code in [200, 500] - - report_obj.kill_flask() - -@patch('microSALT.server.views.render_template') -def test_index_views(renderpatch, mock_db): - renderpatch.return_value = "ok" - start = start_page() - assert start == "ok" - reroute = reroute_page() - assert reroute == "ok" - -@patch('microSALT.server.views.render_template') -def test_project_views(renderpatch, mock_db): - renderpatch.return_value = "ok" - a = project_page("AAA1234") - assert a == "ok" - b = alignment_page("AAA1234") - assert b == "ok" - c = typing_page("AAA1234","all") - assert c == "ok" - -@patch('microSALT.server.views.gen_add_info') -@patch('microSALT.server.views.render_template') -def test_tracker_view(renderpatch, addinfo, mock_db): - renderpatch.return_value = "ok" - a = STtracker_page("cust000") - assert a == "ok" - diff --git a/tests/testdata/sampleinfo_projects.json b/tests/testdata/sampleinfo_projects.json index a25fb75a..67a53e35 100644 --- a/tests/testdata/sampleinfo_projects.json +++ b/tests/testdata/sampleinfo_projects.json @@ -2,19 +2,16 @@ { "CG_ID_project":"AAA1234", "Customer_ID_project":"Trams", -"date_ordered":"2020-07-14 15:03:51.111", "Customer_ID":"1" }, { "CG_ID_project":"MLS1234", "Customer_ID_project":"Trams", -"date_ordered":"2020-07-14 15:03:51.111", "Customer_ID":"1" }, { "CG_ID_project":"MIC1234", "Customer_ID_project":"Trams", -"date_ordered":"2020-07-14 15:03:51.111", "Customer_ID":"1" } ] diff --git a/tests/utils/__init__.py b/tests/utils/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/utils/conftest.py b/tests/utils/conftest.py new file mode 100644 index 00000000..56a94cc8 --- /dev/null +++ b/tests/utils/conftest.py @@ -0,0 +1,66 @@ +import logging +from pathlib import Path + +import pytest + +from microSALT.config import MicroSALTConfig +from microSALT.utils.referencer import Referencer +from microSALT.utils.reporter import Reporter +from microSALT.utils.scraper import Scraper + + +@pytest.fixture +def testdata_prefix(): + return str(Path(__file__).parent.parent / "testdata") + + +@pytest.fixture +def testdata(unpack_db_json) -> list[dict]: + return unpack_db_json("sampleinfo_samples.json") + + +@pytest.fixture +def scraper(config: MicroSALTConfig, logger: logging.Logger, testdata: list[dict]) -> Scraper: + return Scraper( + log=logger, + folders=config.folders, + threshold=config.threshold, + slurm_header=config.slurm_header, + regex=config.regex, + dry=False, + config_path=config.config_path, + pubmlst=config.pubmlst, + pasteur=config.pasteur, + singularity=config.singularity, + containers=config.containers, + sampleinfo=testdata[0], + ) + + +@pytest.fixture +def init_references(config: MicroSALTConfig, logger: logging.Logger, testdata: list[dict]) -> None: + ref_obj = Referencer( + log=logger, + folders=config.folders, + threshold=config.threshold, + pubmlst=config.pubmlst, + pasteur=config.pasteur, + singularity=config.singularity, + containers=config.containers, + sampleinfo=testdata, + ) + ref_obj.identify_new(testdata[0].get("CG_ID_project"), project=True) + ref_obj.update_refs() + + +@pytest.fixture +def reporter(config: MicroSALTConfig, logger: logging.Logger, unpack_db_json): + return Reporter( + log=logger, + folders=config.folders, + threshold=config.threshold, + regex=config.regex, + sampleinfo=unpack_db_json("sampleinfo_samples.json")[0], + name="MIC1234A1", + output=config.folders.reports, + ) diff --git a/tests/utils/test_jobcreator.py b/tests/utils/test_jobcreator.py new file mode 100644 index 00000000..71de20e3 --- /dev/null +++ b/tests/utils/test_jobcreator.py @@ -0,0 +1,230 @@ +#!/usr/bin/env python + +from unittest import mock +from unittest.mock import patch + +from microSALT.utils.job_creator import Job_Creator + + +def fake_search(int): + return "fake" + + +@patch("os.listdir") +@patch("os.stat") +@patch("gzip.open") +def test_verify_fastq(gopen, stat, listdir, config, logger, testdata): + listdir.return_value = [ + "ACC6438A3_HVMHWDSXX_L1_1.fastq.gz", + "ACC6438A3_HVMHWDSXX_L1_2.fastq.gz", + "ACC6438A3_HVMHWDSXX_L2_2.fastq.gz", + "ACC6438A3_HVMHWDSXX_L2_2.fastq.gz", + ] + stata = mock.MagicMock() + stata.st_size = 2000 + stat.return_value = stata + + jc = Job_Creator( + log=logger, + folders=config.folders, + slurm_header=config.slurm_header, + regex=config.regex, + dry=False, + config_path=config.config_path, + threshold=config.threshold, + pubmlst=config.pubmlst, + pasteur=config.pasteur, + singularity=config.singularity, + containers=config.containers, + sampleinfo=testdata, + run_settings={"input": "/tmp/"}, + ) + t = jc.verify_fastq() + assert len(t) > 0 + + +@patch("re.search") +@patch("microSALT.utils.job_creator.glob.glob") +def test_blast_subset(glob_search, research, config, logger, testdata): + jc = Job_Creator( + log=logger, + folders=config.folders, + slurm_header=config.slurm_header, + regex=config.regex, + dry=False, + config_path=config.config_path, + threshold=config.threshold, + pubmlst=config.pubmlst, + pasteur=config.pasteur, + singularity=config.singularity, + containers=config.containers, + sampleinfo=testdata, + run_settings={"input": "/tmp/"}, + ) + researcha = mock.MagicMock() + researcha.group = fake_search + research.return_value = researcha + glob_search.return_value = ["/a/a/a", "/a/a/b", "/a/a/c"] + + jc.blast_subset("mlst", "/tmp/*") + jc.blast_subset("other", "/tmp/*") + outfile = open(jc.get_sbatch(), "r") + count = 0 + for x in outfile.readlines(): + if "blastn -db" in x: + count = count + 1 + assert count > 0 + + +@patch("subprocess.Popen") +def test_create_snpsection(subproc, config, logger, testdata): + # Sets up subprocess mocking + process_mock = mock.Mock() + attrs = {"communicate.return_value": ("output 123456789", "error")} + process_mock.configure_mock(**attrs) + subproc.return_value = process_mock + + testdata = [testdata[0]] + jc = Job_Creator( + log=logger, + folders=config.folders, + slurm_header=config.slurm_header, + regex=config.regex, + dry=False, + config_path=config.config_path, + threshold=config.threshold, + pubmlst=config.pubmlst, + pasteur=config.pasteur, + singularity=config.singularity, + containers=config.containers, + sampleinfo=testdata, + run_settings={"input": ["AAA1234A1", "AAA1234A2"]}, + ) + jc.snp_job() + outfile = open(jc.get_sbatch(), "r") + count = 0 + for x in outfile.readlines(): + if "# SNP pair-wise distance" in x: + count = count + 1 + assert count > 0 + + +@patch("subprocess.Popen") +def test_project_job(subproc, config, logger, testdata): + # Sets up subprocess mocking + process_mock = mock.Mock() + attrs = {"communicate.return_value": ("output 123456789", "error")} + process_mock.configure_mock(**attrs) + subproc.return_value = process_mock + + with patch.dict("os.environ", {"CONDA_PREFIX": "/tmp/mock_conda"}): + jc = Job_Creator( + log=logger, + folders=config.folders, + slurm_header=config.slurm_header, + regex=config.regex, + dry=False, + config_path=config.config_path, + threshold=config.threshold, + pubmlst=config.pubmlst, + pasteur=config.pasteur, + singularity=config.singularity, + containers=config.containers, + sampleinfo=testdata, + run_settings={"pool": ["AAA1234A1", "AAA1234A2"], "input": "/tmp/AAA1234"}, + ) + jc.project_job() + +def test_singularity_exec_binds_finishdir(config, logger, testdata): + """finishdir is automatically added to the --bind list of every singularity exec call.""" + jc = Job_Creator( + log=logger, + folders=config.folders, + slurm_header=config.slurm_header, + regex=config.regex, + dry=False, + config_path=config.config_path, + threshold=config.threshold, + pubmlst=config.pubmlst, + pasteur=config.pasteur, + singularity=config.singularity, + containers=config.containers, + sampleinfo=testdata, + run_settings={"input": "/tmp/", "finishdir": "/tmp/test_runfolder"}, + ) + cmd = jc._singularity_exec("blast", "blastn -help") + assert "/tmp/test_runfolder" in cmd + + +def test_singularity_exec_does_not_duplicate_finishdir(config, logger, testdata): + """finishdir is not listed twice when it already appears in singularity.bind_paths.""" + from microSALT.config import Singularity + singularity = Singularity(bind_paths=["/tmp/test_runfolder", "/data"]) + jc = Job_Creator( + log=logger, + folders=config.folders, + slurm_header=config.slurm_header, + regex=config.regex, + dry=False, + config_path=config.config_path, + threshold=config.threshold, + pubmlst=config.pubmlst, + pasteur=config.pasteur, + singularity=singularity, + containers=config.containers, + sampleinfo=testdata, + run_settings={"input": "/tmp/", "finishdir": "/tmp/test_runfolder"}, + ) + cmd = jc._singularity_exec("blast", "blastn -help") + assert cmd.count("/tmp/test_runfolder") == 1 + + +def _make_jc(config, logger, testdata, tmp_path) -> Job_Creator: + return Job_Creator( + log=logger, + folders=config.folders, + slurm_header=config.slurm_header, + regex=config.regex, + dry=False, + config_path=config.config_path, + threshold=config.threshold, + pubmlst=config.pubmlst, + pasteur=config.pasteur, + singularity=config.singularity, + containers=config.containers, + sampleinfo=testdata, + run_settings={"input": "/tmp/", "finishdir": str(tmp_path)}, + ) + + +def test_write_mailjob_uses_existing_executable(config, logger, testdata, tmp_path): + """The binary embedded in mailjob.sh must exist on the filesystem.""" + import pathlib + jc = _make_jc(config, logger, testdata, tmp_path) + mailfile = str(tmp_path / "mailjob.sh") + + jc._write_mailjob(mailfile, "default", "") + + content = pathlib.Path(mailfile).read_text() + # Extract the first token of the finish command (the binary path) + bin_path = next( + line.split()[0] + for line in content.splitlines() + if "utils finish" in line + ) + assert pathlib.Path(bin_path).exists(), f"Binary not found on disk: {bin_path}" + + +def test_write_mailjob_contains_finish_command(config, logger, testdata, tmp_path): + """mailjob.sh must contain the expected microsalt utils finish invocation.""" + import pathlib + jc = _make_jc(config, logger, testdata, tmp_path) + mailfile = str(tmp_path / "mailjob.sh") + + jc._write_mailjob(mailfile, "qc", "") + + content = pathlib.Path(mailfile).read_text() + assert "utils finish" in content + assert "--report qc" in content + assert str(tmp_path) in content + diff --git a/tests/utils/test_reporter.py b/tests/utils/test_reporter.py new file mode 100644 index 00000000..9f8c55a5 --- /dev/null +++ b/tests/utils/test_reporter.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python +import glob + +import pytest + +from microSALT.store.db_manipulator import DB_Manipulator +from microSALT.utils.reporter import Reporter + + +def test_motif(dbm: DB_Manipulator, reporter: Reporter): + reporter.create_subfolders() + reporter.gen_motif(motif="resistance") + assert len(glob.glob(f"{reporter.output}/AAA1234_resistance*")) > 0 + + reporter.gen_motif(motif="expec") + assert len(glob.glob(f"{reporter.output}/AAA1234_expec*")) > 0 + + +def test_deliveryreport(config, dbm, reporter): + reporter.create_subfolders() + reporter.gen_delivery() + assert ( + len( + glob.glob( + f"{config.folders.reports}/deliverables/999999_deliverables.yaml" + ) + ) + > 0 + ) + + +def test_jsonreport(config, dbm, reporter): + reporter.create_subfolders() + reporter.gen_json() + assert len(glob.glob(f"{config.folders.reports}/json/AAA1234.json")) > 0 + + +def test_gen_qc_name_does_not_exist(dbm, reporter): + reporter.name = "name_that_do_not_exist" + with pytest.raises(SystemExit): + reporter.gen_qc() + + +def test_gen_typing_name_does_not_exist(dbm, reporter): + reporter.name = "name_that_do_not_exist" + with pytest.raises(SystemExit): + reporter.gen_typing() + + +def test_gen_motif(caplog, reporter): + caplog.clear() + reporter.gen_motif(motif="unrecognized") + assert "Invalid motif type" in caplog.text + caplog.clear() + reporter.output = "/path/that/do/not/exists/" + reporter.gen_motif() + assert "Gen_motif unable to produce" in caplog.text + + +def test_gen_json(caplog, reporter): + caplog.clear() + reporter.output = "/path/that/do/not/exists/" + reporter.folders.reports = "/path/that/do/not/exists/" + reporter.gen_json() + assert "Gen_json unable to produce" in caplog.text + + +def test_report(caplog, reporter): + caplog.clear() + reporter.type = "type_not_mentioned_in_list" + with pytest.raises(Exception): + reporter.report() + assert "Report function recieved invalid format" in caplog.text + + +def test_constructor(config, logger, unpack_db_json): + sample_info = unpack_db_json("sampleinfo_samples.json") + reporter_obj = Reporter( + log=logger, + folders=config.folders, + threshold=config.threshold, + regex=config.regex, + sampleinfo=sample_info, + name="MIC1234A1", + output="/tmp/MLST", + ) diff --git a/tests/utils/test_scraper.py b/tests/utils/test_scraper.py new file mode 100644 index 00000000..8e4497a2 --- /dev/null +++ b/tests/utils/test_scraper.py @@ -0,0 +1,196 @@ +#!/usr/bin/env python +"""Tests for microSALT.utils.scraper (Scraper).""" + +import glob +import logging +import shutil +from pathlib import Path +from typing import Generator + +import pytest + +from microSALT.config import MicroSALTConfig +from microSALT.store.db_manipulator import DB_Manipulator +from microSALT.utils.scraper import Scraper + + +class BlastScraperContext: + """Filesystem and DB context for BLAST scraping tests. + + Attributes: + refs_dir: Directory containing the organism-specific '.tfa' FASTA files. + profile_path: Path to the 'staphylococcus_aureus' MLST profile file. + combined_fsa: Path to the resistance 'combined.fsa' file. + scraper: The configured 'Scraper' instance. + """ + + refs_dir: Path + profile_path: Path + combined_fsa: Path + scraper: Scraper + + def setup_loci_fasta(self, testdata_dir: Path, references_dir: Path) -> None: + """Write ``{references}/staphylococcus_aureus/arcC.tfa`` from blast_single_loci.txt.""" + alleles: dict[str, int] = {} + with open(testdata_dir / "blast_single_loci.txt") as fh: + for line in fh: + if line.startswith("#"): + continue + parts = line.rstrip("\n").split("\t") + if len(parts) >= 12 and parts[1] != "N/A": + alleles[parts[3]] = int(parts[11]) + + self.refs_dir = references_dir / "staphylococcus_aureus" + self.refs_dir.mkdir(parents=True, exist_ok=True) + (self.refs_dir / "arcC.tfa").write_text( + "\n".join( + line for name, length in alleles.items() for line in (f">{name}", "A" * length) + ) + + "\n" + ) + + def setup_profile(self, profiles_dir: Path) -> None: + """Write a minimal ``staphylococcus_aureus`` MLST profile (ST 130 → arcC allele 3).""" + profiles_dir.mkdir(parents=True, exist_ok=True) + self.profile_path = profiles_dir / "staphylococcus_aureus" + self.profile_path.write_text("ST\tarcC\n130\t3\n") + + def setup_resistance_fasta(self, testdata_dir: Path, resistances_dir: Path) -> None: + """Write ``{resistances}/combined.fsa`` from blast_single_resistance.txt.""" + alleles: dict[str, int] = {} + with open(testdata_dir / "blast_single_resistance.txt") as fh: + for line in fh: + if line.startswith("#"): + continue + parts = line.rstrip("\n").split("\t") + if len(parts) >= 12 and parts[1] != "N/A": + alleles[parts[3]] = int(parts[11]) + + resistances_dir.mkdir(parents=True, exist_ok=True) + self.combined_fsa = resistances_dir / "combined.fsa" + self.combined_fsa.write_text( + "\n".join( + line for name, length in alleles.items() for line in (f">{name}", "A" * length) + ) + + "\n" + ) + + def teardown(self) -> None: + """Remove generated filesystem artifacts created by the setup methods.""" + shutil.rmtree(self.refs_dir, ignore_errors=True) + self.profile_path.unlink(missing_ok=True) + self.combined_fsa.unlink(missing_ok=True) + + def _get_blast_scaper_context( + self, + config: MicroSALTConfig, + logger: logging.Logger, + testdata: list[dict], + dbm: DB_Manipulator, + ) -> Generator["BlastScraperContext", None, None]: + """Return a setup BlastScraperContext instance configured for BLAST scraping tests.""" + self.setup_loci_fasta( + Path(__file__).parent.parent / "testdata", Path(config.folders.references) + ) + self.setup_profile(Path(config.folders.profiles)) + self.setup_resistance_fasta( + Path(__file__).parent.parent / "testdata", Path(config.folders.resistances) + ) + + dbm.add_to_session(dbm.add_sample(CG_ID_sample="AAA1234A1", CG_ID_project="AAA1234")) + dbm.commit_session() + + self.scraper = Scraper( + log=logger, + folders=config.folders, + threshold=config.threshold, + slurm_header=config.slurm_header, + regex=config.regex, + dry=False, + config_path=config.config_path, + pubmlst=config.pubmlst, + pasteur=config.pasteur, + singularity=config.singularity, + containers=config.containers, + sampleinfo=testdata[0], + ) + + yield self + + self.teardown() + + +@pytest.fixture +def blast_scraper_context( + config: MicroSALTConfig, + logger: logging.Logger, + testdata: list[dict], + dbm: DB_Manipulator, +) -> Generator[BlastScraperContext, None, None]: + """Yield a fully set-up :class:`BlastScraperContext`, then tear it down.""" + testdata_dir = Path(__file__).parent.parent / "testdata" + + ctx = BlastScraperContext() + ctx.setup_loci_fasta(testdata_dir, Path(config.folders.references)) + ctx.setup_profile(Path(config.folders.profiles)) + ctx.setup_resistance_fasta(testdata_dir, Path(config.folders.resistances)) + + dbm.add_to_session(dbm.add_sample(CG_ID_sample="AAA1234A1", CG_ID_project="AAA1234")) + dbm.commit_session() + + ctx.scraper = Scraper( + log=logger, + folders=config.folders, + threshold=config.threshold, + slurm_header=config.slurm_header, + regex=config.regex, + dry=False, + config_path=config.config_path, + pubmlst=config.pubmlst, + pasteur=config.pasteur, + singularity=config.singularity, + containers=config.containers, + sampleinfo=testdata[0], + ) + + yield ctx + + ctx.teardown() + + +def test_quast_scraping( + scraper: Scraper, testdata_prefix: str, caplog: pytest.LogCaptureFixture +) -> None: + """Scraping a QUAST results file should not raise.""" + scraper.scrape_quast(filename=f"{testdata_prefix}/quast_results.tsv") + + +def test_blast_scraping( + blast_scraper_context: BlastScraperContext, + testdata_prefix: str, + caplog: pytest.LogCaptureFixture, +) -> None: + """BLAST scraping should find sequence-type candidates and resistance genes.""" + caplog.set_level(logging.DEBUG) + + blast_scraper_context.scraper.scrape_blast( + type="seq_type", file_list=[f"{testdata_prefix}/blast_single_loci.txt"] + ) + assert "candidate" in caplog.text + + caplog.clear() + hits = blast_scraper_context.scraper.scrape_blast( + type="resistance", + file_list=[f"{testdata_prefix}/blast_single_resistance.txt"], + ) + genes = [h["gene"] for h in hits] + assert "blaOXA-48" in genes + assert "blaVIM-4" in genes + + +def test_alignment_scraping(scraper: Scraper, testdata_prefix: str) -> None: + """Scraping alignment stats files should not raise. + + init_references is NOT needed: scrape_alignment only reads local .stats.* files. + """ + scraper.scrape_alignment(file_list=glob.glob(f"{testdata_prefix}/*.stats.*")) diff --git a/uv.lock b/uv.lock new file mode 100644 index 00000000..db33bc48 --- /dev/null +++ b/uv.lock @@ -0,0 +1,997 @@ +version = 1 +revision = 2 +requires-python = ">=3.12" + +[[package]] +name = "annotated-types" +version = "0.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ee/67/531ea369ba64dcff5ec9c3402f9f51bf748cec26dde048a2f973a4eea7f5/annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89", size = 16081, upload-time = "2024-05-20T21:33:25.928Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53", size = 13643, upload-time = "2024-05-20T21:33:24.1Z" }, +] + +[[package]] +name = "arrow" +version = "1.2.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "python-dateutil" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7f/c0/c601ea7811f422700ef809f167683899cdfddec5aa3f83597edf97349962/arrow-1.2.3.tar.gz", hash = "sha256:3934b30ca1b9f292376d9db15b19446088d12ec58629bc3f0da28fd55fb633a1", size = 127552, upload-time = "2022-09-03T19:35:32.65Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/67/67/4bca5a595e2f89bff271724ddb1098e6c9e16f7f3d018d120255e3c30313/arrow-1.2.3-py3-none-any.whl", hash = "sha256:5a49ab92e3b7b71d96cd6bfcc4df14efefc9dfa96ea19045815914a6ab6b1fe2", size = 66391, upload-time = "2022-09-03T19:35:29.66Z" }, +] + +[[package]] +name = "astroid" +version = "4.0.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/07/63/0adf26577da5eff6eb7a177876c1cfa213856be9926a000f65c4add9692b/astroid-4.0.4.tar.gz", hash = "sha256:986fed8bcf79fb82c78b18a53352a0b287a73817d6dbcfba3162da36667c49a0", size = 406358, upload-time = "2026-02-07T23:35:07.509Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b0/cf/1c5f42b110e57bc5502eb80dbc3b03d256926062519224835ef08134f1f9/astroid-4.0.4-py3-none-any.whl", hash = "sha256:52f39653876c7dec3e3afd4c2696920e05c83832b9737afc21928f2d2eb7a753", size = 276445, upload-time = "2026-02-07T23:35:05.344Z" }, +] + +[[package]] +name = "beautifulsoup4" +version = "4.14.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "soupsieve" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c3/b0/1c6a16426d389813b48d95e26898aff79abbde42ad353958ad95cc8c9b21/beautifulsoup4-4.14.3.tar.gz", hash = "sha256:6292b1c5186d356bba669ef9f7f051757099565ad9ada5dd630bd9de5fa7fb86", size = 627737, upload-time = "2025-11-30T15:08:26.084Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1a/39/47f9197bdd44df24d67ac8893641e16f386c984a0619ef2ee4c51fbbc019/beautifulsoup4-4.14.3-py3-none-any.whl", hash = "sha256:0918bfe44902e6ad8d57732ba310582e98da931428d231a5ecb9e7c703a735bb", size = 107721, upload-time = "2025-11-30T15:08:24.087Z" }, +] + +[[package]] +name = "biopython" +version = "1.86" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9d/61/c59a849bd457c8a1b408ae828dbcc15e674962b5a29705e869e15b32bf25/biopython-1.86.tar.gz", hash = "sha256:93a50b586a4d2cec68ab2f99d03ef583c5761d8fba5535cb8e81da781d0d92ff", size = 19835323, upload-time = "2025-10-28T21:18:31.041Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/98/e2/199b8ccbd4b9bf234157db0668177b5b7784d62f29d9096fd0d3a70e3b86/biopython-1.86-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:f8d372aae21d79b11613751c6ae23c88db0e94d25b7567b1f67aa0304fb61667", size = 2693171, upload-time = "2025-10-29T00:26:59.028Z" }, + { url = "https://files.pythonhosted.org/packages/d8/2f/1a7da2a55212b3d0a03866d22213f91273fee3722b5364575419fbe574a5/biopython-1.86-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:baf19d9237aaaa387a68f8f055f978af5c80338d7e037ab028e8d768928f1250", size = 2692543, upload-time = "2025-10-28T21:27:31.855Z" }, + { url = "https://files.pythonhosted.org/packages/5b/e9/4057d4c2aa22ca25c180ecbed2ce9e7d65bf787999778bc63b41df0d03b5/biopython-1.86-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:04f9abdf6cbf0087850de5f8148da0d420c4cb87905bf4de3145ad24a8d55dcd", size = 2669975, upload-time = "2025-10-28T21:26:54.181Z" }, + { url = "https://files.pythonhosted.org/packages/a7/b2/3e6862720d7c51f0fbe7d6d25be72a95486779d9d98122283b4e8032fb40/biopython-1.86-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:187c3c24dd2255e7328f3e0523ab5d6350b73ff562517de0c1922385617101d2", size = 3209367, upload-time = "2025-10-29T00:36:06.522Z" }, + { url = "https://files.pythonhosted.org/packages/d7/cb/61877367bf08670573d62513b239dc65cf2b7488dc74322cc6051da2e55e/biopython-1.86-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1859830b8262785c6b59dfe0c82cddb643974f63b9d2779bb9f3e2c47c0a95da", size = 3235466, upload-time = "2025-10-29T00:36:11.516Z" }, + { url = "https://files.pythonhosted.org/packages/84/1a/3182a77776b76f3f5c64825ee1acf9355f665bed72ee9e8ff49e48f25d98/biopython-1.86-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dfd906c47b6fb38e3abb9f52e0c06822e6e82a043d38c2000773692c29db1ed8", size = 3178776, upload-time = "2025-10-28T23:53:41.487Z" }, + { url = "https://files.pythonhosted.org/packages/1a/22/828b08fac8dbc8c1dbc1ad03815137cebc9c78303ec7d21b568544028119/biopython-1.86-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4a6ab2c60742f1c8494cfbbe3b7a8b45f0400c8f2b36b686b895d5e4d625f04e", size = 3197586, upload-time = "2025-10-28T23:53:47.136Z" }, + { url = "https://files.pythonhosted.org/packages/36/7a/122aea7653fa93d7eb72978928e80759082efffa70afe0c25a17e18521da/biopython-1.86-cp312-cp312-win32.whl", hash = "sha256:192c61bc3d782c171b7d50bb7d8189d84790d6e3c4b24fd41d1d7ffc7d303efe", size = 2698043, upload-time = "2025-10-28T21:32:39.452Z" }, + { url = "https://files.pythonhosted.org/packages/a9/13/00db03b01e54070d5b0ec9c71eef86e61afa733d9af76e5b9b09f5dc9165/biopython-1.86-cp312-cp312-win_amd64.whl", hash = "sha256:35a6b9c5dcdfb5c2631a313a007f3f41a7d72573ba2b68c962e10ea92096ff3b", size = 2733610, upload-time = "2025-10-28T21:32:34.99Z" }, + { url = "https://files.pythonhosted.org/packages/fd/6e/84d6c66ab93095aa7adb998a8eef045328470eafd36b9237c4db213e587c/biopython-1.86-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:fb3a11a98e49428720dca227e2a5bdd57c973ee7c4df3cf6734c0aa13fd134c7", size = 2693185, upload-time = "2025-10-28T21:27:39.709Z" }, + { url = "https://files.pythonhosted.org/packages/12/75/60386f2640f13765b1651f2f26d8b4f893c46ee663df3ca76eda966d4f6a/biopython-1.86-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e161f3d3b6e65fbfd1ce22a01c3e9fa9da789adde4972fd0cc2370795ea5357b", size = 2669980, upload-time = "2025-10-28T21:26:58.839Z" }, + { url = "https://files.pythonhosted.org/packages/dd/de/a39adb98a0552a257219503c236ef17f007598af55326c0d143db52e5a92/biopython-1.86-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5aa8c9e92ee6fe59dfe0d2c2daf9a9eec6b812c78328caad038f79163c500218", size = 3209657, upload-time = "2025-10-29T00:36:28.842Z" }, + { url = "https://files.pythonhosted.org/packages/0b/c7/b2e7aca3de8981f4ecb6ab1e0334c3c4a512e5e9898b57b3d8734b086da7/biopython-1.86-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:593ec6a2a4fedec08ddcee1a8a0e0b0ed56835b2714904b352ec4a93d5b9d973", size = 3235774, upload-time = "2025-10-29T00:36:34.07Z" }, + { url = "https://files.pythonhosted.org/packages/52/ed/e6647b0b9cf2bb67347612e8e443b84378c44768a8d8439276e4ba881178/biopython-1.86-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dd2f9ebf9b14d67ca92f48779c4f0ba404c35dba3e8b9d6c34d1a3591c3b746d", size = 3178415, upload-time = "2025-10-28T23:54:05.475Z" }, + { url = "https://files.pythonhosted.org/packages/ff/37/f6a14b835842c66a52f212136a99416265f5ce76813d668ceac1cb306357/biopython-1.86-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:137fe9aafd93baa5127d17534b473f6646f92a883f52b34f7c306b800ac50038", size = 3197201, upload-time = "2025-10-28T23:54:10.462Z" }, + { url = "https://files.pythonhosted.org/packages/f2/73/0eac930016c509763c174a0e25e92e6d7a711f6f5de1f7001e54fd5c49f7/biopython-1.86-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e784dc8382430c9893aa084ca18fe8a8815b5811f1c324492ef3f4b54e664fff", size = 3145106, upload-time = "2025-10-28T23:54:15.235Z" }, + { url = "https://files.pythonhosted.org/packages/00/aa/26e836274d03402e8011b04a1714d4ac2f704add303a493e54d2d5646973/biopython-1.86-cp313-cp313-win32.whl", hash = "sha256:5329a777ba90ea624447173046e77c4df2862acc46eea4e94fe2211fe041750f", size = 2698051, upload-time = "2025-10-28T21:32:55.225Z" }, + { url = "https://files.pythonhosted.org/packages/ae/27/fa1f8fa57f2ac8fdc41d14ab36001b8ba0fce5eac01585227b99a4da0e9d/biopython-1.86-cp313-cp313-win_amd64.whl", hash = "sha256:f6f2f1dc75423b15d8a22b8eceae32785736612b6740688526401b8c2d821270", size = 2733649, upload-time = "2025-10-28T21:32:51.052Z" }, + { url = "https://files.pythonhosted.org/packages/a4/2d/5b87ab859d38f2c7d7d1f9df375b4734737c2ef62cf8506983e882419a30/biopython-1.86-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:236ca61aa996f12cbc65a8d6a15abfac70b9ee800656629b784c6a240e7d8dc0", size = 2694733, upload-time = "2025-10-29T00:27:49.142Z" }, + { url = "https://files.pythonhosted.org/packages/24/7e/a80fad6dbfa1335c506b1565d2b3fdd78cda705408a839c5583a9cfca8b6/biopython-1.86-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:f96b7441f456c7eecad5c6e61e75b0db1435c489be7cc5e4f97dd4e60921747c", size = 2670131, upload-time = "2025-10-29T00:27:53.758Z" }, + { url = "https://files.pythonhosted.org/packages/2d/0a/6c12e9262b99f395bd66535c4a4203bd70833c11f47ac0730fca6ba2b5f8/biopython-1.86-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d53a78bf960397826219f08f87b061ad7f227527d19986e830eeab60d370b597", size = 3209810, upload-time = "2025-10-29T00:36:45.88Z" }, + { url = "https://files.pythonhosted.org/packages/3a/f9/265211154d2bb4cffe78a57b8e57cfbb165cf41cf3d1b68e2a6b073b3b8a/biopython-1.86-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bb86e4383c02fdb2571a38947153346e6f5cd38e22de1df40f54d2a3c51d02a8", size = 3235347, upload-time = "2025-10-29T00:36:51.164Z" }, + { url = "https://files.pythonhosted.org/packages/64/e5/58d8e48d3b4100a7fd8bae97f0dd7179c30f19861841d1a0bb7827e0033e/biopython-1.86-cp314-cp314-win32.whl", hash = "sha256:ffeba620c4786ea836efee235a9c6333b94e922b89de1449a4782dcc15246ff1", size = 2698198, upload-time = "2025-10-29T00:28:02.812Z" }, + { url = "https://files.pythonhosted.org/packages/e2/ca/aa166eb588a2d4eea381c92e5a2a3d09b4b4887b0f0e8f3acf999fb88157/biopython-1.86-cp314-cp314-win_amd64.whl", hash = "sha256:efbb9bc4415a1e2c1c986ba261b02857bc0c9eed098b15493f1cc5c4a1e02409", size = 2734693, upload-time = "2025-10-29T00:27:58.312Z" }, + { url = "https://files.pythonhosted.org/packages/50/da/8c227d701ec9c94d9870b1879982e3dd114da130b0816d3f9b937318d31a/biopython-1.86-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:caa70c1639b3306549605f9273753bdbf8cd6d6d352cecf23afbda3c911694f3", size = 2697389, upload-time = "2025-10-29T00:28:07.037Z" }, + { url = "https://files.pythonhosted.org/packages/8c/1e/66b0b5622ef6a3a14c449d1c8d69749480b37518e4c1e3a8a86fc668dad7/biopython-1.86-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:d077f01d1f69f77a26cac46163d4ea45eb4e6509a68feb7f15e665b7e1de0a99", size = 2673857, upload-time = "2025-10-29T00:28:11.488Z" }, + { url = "https://files.pythonhosted.org/packages/76/05/7c8f9800e6960da2007eb75128c8ec0b22e1a0064e8802e8acfad53cdca8/biopython-1.86-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4506ce7dbdf885cb24d1f5439362c3c07f1b6f90761a0d20fe16a2a9ea5702a5", size = 3253007, upload-time = "2025-10-29T00:36:56.066Z" }, + { url = "https://files.pythonhosted.org/packages/14/dd/a2177328d841fda0a12e67c65d06279691e25363a2805f561b3665cae114/biopython-1.86-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:dcd94717e83ba891ebd9acaecbf05ad38313095ca5706caf6c38fa3f2aa17528", size = 3272883, upload-time = "2025-10-29T00:37:01.189Z" }, + { url = "https://files.pythonhosted.org/packages/ce/04/1aa91f64db5e0728d596fcf7302e2ae2035800c0676e94ea09645a948b91/biopython-1.86-cp314-cp314t-win32.whl", hash = "sha256:2f6b205dcb4101cefa5c615114bd35a19f656abb9d340eb3cf190f829e43800a", size = 2701649, upload-time = "2025-10-29T00:28:20.527Z" }, + { url = "https://files.pythonhosted.org/packages/63/7c/4acaca39102d667175bb3d6502dea91c346f8674c06d5df0dbb678971596/biopython-1.86-cp314-cp314t-win_amd64.whl", hash = "sha256:efeee7c37f2331d2c55704df39e122189cc237ffd7511f34158418ad728131b8", size = 2741364, upload-time = "2025-10-29T00:28:15.752Z" }, +] + +[[package]] +name = "black" +version = "26.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "mypy-extensions" }, + { name = "packaging" }, + { name = "pathspec" }, + { name = "platformdirs" }, + { name = "pytokens" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/13/88/560b11e521c522440af991d46848a2bde64b5f7202ec14e1f46f9509d328/black-26.1.0.tar.gz", hash = "sha256:d294ac3340eef9c9eb5d29288e96dc719ff269a88e27b396340459dd85da4c58", size = 658785, upload-time = "2026-01-18T04:50:11.993Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f5/13/710298938a61f0f54cdb4d1c0baeb672c01ff0358712eddaf29f76d32a0b/black-26.1.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:6eeca41e70b5f5c84f2f913af857cf2ce17410847e1d54642e658e078da6544f", size = 1878189, upload-time = "2026-01-18T04:59:30.682Z" }, + { url = "https://files.pythonhosted.org/packages/79/a6/5179beaa57e5dbd2ec9f1c64016214057b4265647c62125aa6aeffb05392/black-26.1.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:dd39eef053e58e60204f2cdf059e2442e2eb08f15989eefe259870f89614c8b6", size = 1700178, upload-time = "2026-01-18T04:59:32.387Z" }, + { url = "https://files.pythonhosted.org/packages/8c/04/c96f79d7b93e8f09d9298b333ca0d31cd9b2ee6c46c274fd0f531de9dc61/black-26.1.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9459ad0d6cd483eacad4c6566b0f8e42af5e8b583cee917d90ffaa3778420a0a", size = 1777029, upload-time = "2026-01-18T04:59:33.767Z" }, + { url = "https://files.pythonhosted.org/packages/49/f9/71c161c4c7aa18bdda3776b66ac2dc07aed62053c7c0ff8bbda8c2624fe2/black-26.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:a19915ec61f3a8746e8b10adbac4a577c6ba9851fa4a9e9fbfbcf319887a5791", size = 1406466, upload-time = "2026-01-18T04:59:35.177Z" }, + { url = "https://files.pythonhosted.org/packages/4a/8b/a7b0f974e473b159d0ac1b6bcefffeb6bec465898a516ee5cc989503cbc7/black-26.1.0-cp312-cp312-win_arm64.whl", hash = "sha256:643d27fb5facc167c0b1b59d0315f2674a6e950341aed0fc05cf307d22bf4954", size = 1216393, upload-time = "2026-01-18T04:59:37.18Z" }, + { url = "https://files.pythonhosted.org/packages/79/04/fa2f4784f7237279332aa735cdfd5ae2e7730db0072fb2041dadda9ae551/black-26.1.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:ba1d768fbfb6930fc93b0ecc32a43d8861ded16f47a40f14afa9bb04ab93d304", size = 1877781, upload-time = "2026-01-18T04:59:39.054Z" }, + { url = "https://files.pythonhosted.org/packages/cf/ad/5a131b01acc0e5336740a039628c0ab69d60cf09a2c87a4ec49f5826acda/black-26.1.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2b807c240b64609cb0e80d2200a35b23c7df82259f80bef1b2c96eb422b4aac9", size = 1699670, upload-time = "2026-01-18T04:59:41.005Z" }, + { url = "https://files.pythonhosted.org/packages/da/7c/b05f22964316a52ab6b4265bcd52c0ad2c30d7ca6bd3d0637e438fc32d6e/black-26.1.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1de0f7d01cc894066a1153b738145b194414cc6eeaad8ef4397ac9abacf40f6b", size = 1775212, upload-time = "2026-01-18T04:59:42.545Z" }, + { url = "https://files.pythonhosted.org/packages/a6/a3/e8d1526bea0446e040193185353920a9506eab60a7d8beb062029129c7d2/black-26.1.0-cp313-cp313-win_amd64.whl", hash = "sha256:91a68ae46bf07868963671e4d05611b179c2313301bd756a89ad4e3b3db2325b", size = 1409953, upload-time = "2026-01-18T04:59:44.357Z" }, + { url = "https://files.pythonhosted.org/packages/c7/5a/d62ebf4d8f5e3a1daa54adaab94c107b57be1b1a2f115a0249b41931e188/black-26.1.0-cp313-cp313-win_arm64.whl", hash = "sha256:be5e2fe860b9bd9edbf676d5b60a9282994c03fbbd40fe8f5e75d194f96064ca", size = 1217707, upload-time = "2026-01-18T04:59:45.719Z" }, + { url = "https://files.pythonhosted.org/packages/6a/83/be35a175aacfce4b05584ac415fd317dd6c24e93a0af2dcedce0f686f5d8/black-26.1.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:9dc8c71656a79ca49b8d3e2ce8103210c9481c57798b48deeb3a8bb02db5f115", size = 1871864, upload-time = "2026-01-18T04:59:47.586Z" }, + { url = "https://files.pythonhosted.org/packages/a5/f5/d33696c099450b1274d925a42b7a030cd3ea1f56d72e5ca8bbed5f52759c/black-26.1.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:b22b3810451abe359a964cc88121d57f7bce482b53a066de0f1584988ca36e79", size = 1701009, upload-time = "2026-01-18T04:59:49.443Z" }, + { url = "https://files.pythonhosted.org/packages/1b/87/670dd888c537acb53a863bc15abbd85b22b429237d9de1b77c0ed6b79c42/black-26.1.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:53c62883b3f999f14e5d30b5a79bd437236658ad45b2f853906c7cbe79de00af", size = 1767806, upload-time = "2026-01-18T04:59:50.769Z" }, + { url = "https://files.pythonhosted.org/packages/fe/9c/cd3deb79bfec5bcf30f9d2100ffeec63eecce826eb63e3961708b9431ff1/black-26.1.0-cp314-cp314-win_amd64.whl", hash = "sha256:f016baaadc423dc960cdddf9acae679e71ee02c4c341f78f3179d7e4819c095f", size = 1433217, upload-time = "2026-01-18T04:59:52.218Z" }, + { url = "https://files.pythonhosted.org/packages/4e/29/f3be41a1cf502a283506f40f5d27203249d181f7a1a2abce1c6ce188035a/black-26.1.0-cp314-cp314-win_arm64.whl", hash = "sha256:66912475200b67ef5a0ab665011964bf924745103f51977a78b4fb92a9fc1bf0", size = 1245773, upload-time = "2026-01-18T04:59:54.457Z" }, + { url = "https://files.pythonhosted.org/packages/e4/3d/51bdb3ecbfadfaf825ec0c75e1de6077422b4afa2091c6c9ba34fbfc0c2d/black-26.1.0-py3-none-any.whl", hash = "sha256:1054e8e47ebd686e078c0bb0eaf31e6ce69c966058d122f2c0c950311f9f3ede", size = 204010, upload-time = "2026-01-18T04:50:09.978Z" }, +] + +[[package]] +name = "bs4" +version = "0.0.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "beautifulsoup4" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/10/ed/7e8b97591f6f456174139ec089c769f89a94a1a4025fe967691de971f314/bs4-0.0.1.tar.gz", hash = "sha256:36ecea1fd7cc5c0c6e4a1ff075df26d50da647b75376626cc186e2212886dd3a", size = 1121, upload-time = "2016-03-03T13:25:12.284Z" } + +[[package]] +name = "certifi" +version = "2026.1.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e0/2d/a891ca51311197f6ad14a7ef42e2399f36cf2f9bd44752b3dc4eab60fdc5/certifi-2026.1.4.tar.gz", hash = "sha256:ac726dd470482006e014ad384921ed6438c457018f4b3d204aea4281258b2120", size = 154268, upload-time = "2026-01-04T02:42:41.825Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e6/ad/3cc14f097111b4de0040c83a525973216457bbeeb63739ef1ed275c1c021/certifi-2026.1.4-py3-none-any.whl", hash = "sha256:9943707519e4add1115f44c2bc244f782c0249876bf51b6599fee1ffbedd685c", size = 152900, upload-time = "2026-01-04T02:42:40.15Z" }, +] + +[[package]] +name = "charset-normalizer" +version = "3.4.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/13/69/33ddede1939fdd074bce5434295f38fae7136463422fe4fd3e0e89b98062/charset_normalizer-3.4.4.tar.gz", hash = "sha256:94537985111c35f28720e43603b8e7b43a6ecfb2ce1d3058bbe955b73404e21a", size = 129418, upload-time = "2025-10-14T04:42:32.879Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f3/85/1637cd4af66fa687396e757dec650f28025f2a2f5a5531a3208dc0ec43f2/charset_normalizer-3.4.4-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:0a98e6759f854bd25a58a73fa88833fba3b7c491169f86ce1180c948ab3fd394", size = 208425, upload-time = "2025-10-14T04:40:53.353Z" }, + { url = "https://files.pythonhosted.org/packages/9d/6a/04130023fef2a0d9c62d0bae2649b69f7b7d8d24ea5536feef50551029df/charset_normalizer-3.4.4-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b5b290ccc2a263e8d185130284f8501e3e36c5e02750fc6b6bdeb2e9e96f1e25", size = 148162, upload-time = "2025-10-14T04:40:54.558Z" }, + { url = "https://files.pythonhosted.org/packages/78/29/62328d79aa60da22c9e0b9a66539feae06ca0f5a4171ac4f7dc285b83688/charset_normalizer-3.4.4-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:74bb723680f9f7a6234dcf67aea57e708ec1fbdf5699fb91dfd6f511b0a320ef", size = 144558, upload-time = "2025-10-14T04:40:55.677Z" }, + { url = "https://files.pythonhosted.org/packages/86/bb/b32194a4bf15b88403537c2e120b817c61cd4ecffa9b6876e941c3ee38fe/charset_normalizer-3.4.4-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f1e34719c6ed0b92f418c7c780480b26b5d9c50349e9a9af7d76bf757530350d", size = 161497, upload-time = "2025-10-14T04:40:57.217Z" }, + { url = "https://files.pythonhosted.org/packages/19/89/a54c82b253d5b9b111dc74aca196ba5ccfcca8242d0fb64146d4d3183ff1/charset_normalizer-3.4.4-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2437418e20515acec67d86e12bf70056a33abdacb5cb1655042f6538d6b085a8", size = 159240, upload-time = "2025-10-14T04:40:58.358Z" }, + { url = "https://files.pythonhosted.org/packages/c0/10/d20b513afe03acc89ec33948320a5544d31f21b05368436d580dec4e234d/charset_normalizer-3.4.4-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:11d694519d7f29d6cd09f6ac70028dba10f92f6cdd059096db198c283794ac86", size = 153471, upload-time = "2025-10-14T04:40:59.468Z" }, + { url = "https://files.pythonhosted.org/packages/61/fa/fbf177b55bdd727010f9c0a3c49eefa1d10f960e5f09d1d887bf93c2e698/charset_normalizer-3.4.4-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ac1c4a689edcc530fc9d9aa11f5774b9e2f33f9a0c6a57864e90908f5208d30a", size = 150864, upload-time = "2025-10-14T04:41:00.623Z" }, + { url = "https://files.pythonhosted.org/packages/05/12/9fbc6a4d39c0198adeebbde20b619790e9236557ca59fc40e0e3cebe6f40/charset_normalizer-3.4.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:21d142cc6c0ec30d2efee5068ca36c128a30b0f2c53c1c07bd78cb6bc1d3be5f", size = 150647, upload-time = "2025-10-14T04:41:01.754Z" }, + { url = "https://files.pythonhosted.org/packages/ad/1f/6a9a593d52e3e8c5d2b167daf8c6b968808efb57ef4c210acb907c365bc4/charset_normalizer-3.4.4-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:5dbe56a36425d26d6cfb40ce79c314a2e4dd6211d51d6d2191c00bed34f354cc", size = 145110, upload-time = "2025-10-14T04:41:03.231Z" }, + { url = "https://files.pythonhosted.org/packages/30/42/9a52c609e72471b0fc54386dc63c3781a387bb4fe61c20231a4ebcd58bdd/charset_normalizer-3.4.4-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:5bfbb1b9acf3334612667b61bd3002196fe2a1eb4dd74d247e0f2a4d50ec9bbf", size = 162839, upload-time = "2025-10-14T04:41:04.715Z" }, + { url = "https://files.pythonhosted.org/packages/c4/5b/c0682bbf9f11597073052628ddd38344a3d673fda35a36773f7d19344b23/charset_normalizer-3.4.4-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:d055ec1e26e441f6187acf818b73564e6e6282709e9bcb5b63f5b23068356a15", size = 150667, upload-time = "2025-10-14T04:41:05.827Z" }, + { url = "https://files.pythonhosted.org/packages/e4/24/a41afeab6f990cf2daf6cb8c67419b63b48cf518e4f56022230840c9bfb2/charset_normalizer-3.4.4-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:af2d8c67d8e573d6de5bc30cdb27e9b95e49115cd9baad5ddbd1a6207aaa82a9", size = 160535, upload-time = "2025-10-14T04:41:06.938Z" }, + { url = "https://files.pythonhosted.org/packages/2a/e5/6a4ce77ed243c4a50a1fecca6aaaab419628c818a49434be428fe24c9957/charset_normalizer-3.4.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:780236ac706e66881f3b7f2f32dfe90507a09e67d1d454c762cf642e6e1586e0", size = 154816, upload-time = "2025-10-14T04:41:08.101Z" }, + { url = "https://files.pythonhosted.org/packages/a8/ef/89297262b8092b312d29cdb2517cb1237e51db8ecef2e9af5edbe7b683b1/charset_normalizer-3.4.4-cp312-cp312-win32.whl", hash = "sha256:5833d2c39d8896e4e19b689ffc198f08ea58116bee26dea51e362ecc7cd3ed26", size = 99694, upload-time = "2025-10-14T04:41:09.23Z" }, + { url = "https://files.pythonhosted.org/packages/3d/2d/1e5ed9dd3b3803994c155cd9aacb60c82c331bad84daf75bcb9c91b3295e/charset_normalizer-3.4.4-cp312-cp312-win_amd64.whl", hash = "sha256:a79cfe37875f822425b89a82333404539ae63dbdddf97f84dcbc3d339aae9525", size = 107131, upload-time = "2025-10-14T04:41:10.467Z" }, + { url = "https://files.pythonhosted.org/packages/d0/d9/0ed4c7098a861482a7b6a95603edce4c0d9db2311af23da1fb2b75ec26fc/charset_normalizer-3.4.4-cp312-cp312-win_arm64.whl", hash = "sha256:376bec83a63b8021bb5c8ea75e21c4ccb86e7e45ca4eb81146091b56599b80c3", size = 100390, upload-time = "2025-10-14T04:41:11.915Z" }, + { url = "https://files.pythonhosted.org/packages/97/45/4b3a1239bbacd321068ea6e7ac28875b03ab8bc0aa0966452db17cd36714/charset_normalizer-3.4.4-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:e1f185f86a6f3403aa2420e815904c67b2f9ebc443f045edd0de921108345794", size = 208091, upload-time = "2025-10-14T04:41:13.346Z" }, + { url = "https://files.pythonhosted.org/packages/7d/62/73a6d7450829655a35bb88a88fca7d736f9882a27eacdca2c6d505b57e2e/charset_normalizer-3.4.4-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6b39f987ae8ccdf0d2642338faf2abb1862340facc796048b604ef14919e55ed", size = 147936, upload-time = "2025-10-14T04:41:14.461Z" }, + { url = "https://files.pythonhosted.org/packages/89/c5/adb8c8b3d6625bef6d88b251bbb0d95f8205831b987631ab0c8bb5d937c2/charset_normalizer-3.4.4-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3162d5d8ce1bb98dd51af660f2121c55d0fa541b46dff7bb9b9f86ea1d87de72", size = 144180, upload-time = "2025-10-14T04:41:15.588Z" }, + { url = "https://files.pythonhosted.org/packages/91/ed/9706e4070682d1cc219050b6048bfd293ccf67b3d4f5a4f39207453d4b99/charset_normalizer-3.4.4-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:81d5eb2a312700f4ecaa977a8235b634ce853200e828fbadf3a9c50bab278328", size = 161346, upload-time = "2025-10-14T04:41:16.738Z" }, + { url = "https://files.pythonhosted.org/packages/d5/0d/031f0d95e4972901a2f6f09ef055751805ff541511dc1252ba3ca1f80cf5/charset_normalizer-3.4.4-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5bd2293095d766545ec1a8f612559f6b40abc0eb18bb2f5d1171872d34036ede", size = 158874, upload-time = "2025-10-14T04:41:17.923Z" }, + { url = "https://files.pythonhosted.org/packages/f5/83/6ab5883f57c9c801ce5e5677242328aa45592be8a00644310a008d04f922/charset_normalizer-3.4.4-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a8a8b89589086a25749f471e6a900d3f662d1d3b6e2e59dcecf787b1cc3a1894", size = 153076, upload-time = "2025-10-14T04:41:19.106Z" }, + { url = "https://files.pythonhosted.org/packages/75/1e/5ff781ddf5260e387d6419959ee89ef13878229732732ee73cdae01800f2/charset_normalizer-3.4.4-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:bc7637e2f80d8530ee4a78e878bce464f70087ce73cf7c1caf142416923b98f1", size = 150601, upload-time = "2025-10-14T04:41:20.245Z" }, + { url = "https://files.pythonhosted.org/packages/d7/57/71be810965493d3510a6ca79b90c19e48696fb1ff964da319334b12677f0/charset_normalizer-3.4.4-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f8bf04158c6b607d747e93949aa60618b61312fe647a6369f88ce2ff16043490", size = 150376, upload-time = "2025-10-14T04:41:21.398Z" }, + { url = "https://files.pythonhosted.org/packages/e5/d5/c3d057a78c181d007014feb7e9f2e65905a6c4ef182c0ddf0de2924edd65/charset_normalizer-3.4.4-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:554af85e960429cf30784dd47447d5125aaa3b99a6f0683589dbd27e2f45da44", size = 144825, upload-time = "2025-10-14T04:41:22.583Z" }, + { url = "https://files.pythonhosted.org/packages/e6/8c/d0406294828d4976f275ffbe66f00266c4b3136b7506941d87c00cab5272/charset_normalizer-3.4.4-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:74018750915ee7ad843a774364e13a3db91682f26142baddf775342c3f5b1133", size = 162583, upload-time = "2025-10-14T04:41:23.754Z" }, + { url = "https://files.pythonhosted.org/packages/d7/24/e2aa1f18c8f15c4c0e932d9287b8609dd30ad56dbe41d926bd846e22fb8d/charset_normalizer-3.4.4-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:c0463276121fdee9c49b98908b3a89c39be45d86d1dbaa22957e38f6321d4ce3", size = 150366, upload-time = "2025-10-14T04:41:25.27Z" }, + { url = "https://files.pythonhosted.org/packages/e4/5b/1e6160c7739aad1e2df054300cc618b06bf784a7a164b0f238360721ab86/charset_normalizer-3.4.4-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:362d61fd13843997c1c446760ef36f240cf81d3ebf74ac62652aebaf7838561e", size = 160300, upload-time = "2025-10-14T04:41:26.725Z" }, + { url = "https://files.pythonhosted.org/packages/7a/10/f882167cd207fbdd743e55534d5d9620e095089d176d55cb22d5322f2afd/charset_normalizer-3.4.4-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9a26f18905b8dd5d685d6d07b0cdf98a79f3c7a918906af7cc143ea2e164c8bc", size = 154465, upload-time = "2025-10-14T04:41:28.322Z" }, + { url = "https://files.pythonhosted.org/packages/89/66/c7a9e1b7429be72123441bfdbaf2bc13faab3f90b933f664db506dea5915/charset_normalizer-3.4.4-cp313-cp313-win32.whl", hash = "sha256:9b35f4c90079ff2e2edc5b26c0c77925e5d2d255c42c74fdb70fb49b172726ac", size = 99404, upload-time = "2025-10-14T04:41:29.95Z" }, + { url = "https://files.pythonhosted.org/packages/c4/26/b9924fa27db384bdcd97ab83b4f0a8058d96ad9626ead570674d5e737d90/charset_normalizer-3.4.4-cp313-cp313-win_amd64.whl", hash = "sha256:b435cba5f4f750aa6c0a0d92c541fb79f69a387c91e61f1795227e4ed9cece14", size = 107092, upload-time = "2025-10-14T04:41:31.188Z" }, + { url = "https://files.pythonhosted.org/packages/af/8f/3ed4bfa0c0c72a7ca17f0380cd9e4dd842b09f664e780c13cff1dcf2ef1b/charset_normalizer-3.4.4-cp313-cp313-win_arm64.whl", hash = "sha256:542d2cee80be6f80247095cc36c418f7bddd14f4a6de45af91dfad36d817bba2", size = 100408, upload-time = "2025-10-14T04:41:32.624Z" }, + { url = "https://files.pythonhosted.org/packages/2a/35/7051599bd493e62411d6ede36fd5af83a38f37c4767b92884df7301db25d/charset_normalizer-3.4.4-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:da3326d9e65ef63a817ecbcc0df6e94463713b754fe293eaa03da99befb9a5bd", size = 207746, upload-time = "2025-10-14T04:41:33.773Z" }, + { url = "https://files.pythonhosted.org/packages/10/9a/97c8d48ef10d6cd4fcead2415523221624bf58bcf68a802721a6bc807c8f/charset_normalizer-3.4.4-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8af65f14dc14a79b924524b1e7fffe304517b2bff5a58bf64f30b98bbc5079eb", size = 147889, upload-time = "2025-10-14T04:41:34.897Z" }, + { url = "https://files.pythonhosted.org/packages/10/bf/979224a919a1b606c82bd2c5fa49b5c6d5727aa47b4312bb27b1734f53cd/charset_normalizer-3.4.4-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:74664978bb272435107de04e36db5a9735e78232b85b77d45cfb38f758efd33e", size = 143641, upload-time = "2025-10-14T04:41:36.116Z" }, + { url = "https://files.pythonhosted.org/packages/ba/33/0ad65587441fc730dc7bd90e9716b30b4702dc7b617e6ba4997dc8651495/charset_normalizer-3.4.4-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:752944c7ffbfdd10c074dc58ec2d5a8a4cd9493b314d367c14d24c17684ddd14", size = 160779, upload-time = "2025-10-14T04:41:37.229Z" }, + { url = "https://files.pythonhosted.org/packages/67/ed/331d6b249259ee71ddea93f6f2f0a56cfebd46938bde6fcc6f7b9a3d0e09/charset_normalizer-3.4.4-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d1f13550535ad8cff21b8d757a3257963e951d96e20ec82ab44bc64aeb62a191", size = 159035, upload-time = "2025-10-14T04:41:38.368Z" }, + { url = "https://files.pythonhosted.org/packages/67/ff/f6b948ca32e4f2a4576aa129d8bed61f2e0543bf9f5f2b7fc3758ed005c9/charset_normalizer-3.4.4-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ecaae4149d99b1c9e7b88bb03e3221956f68fd6d50be2ef061b2381b61d20838", size = 152542, upload-time = "2025-10-14T04:41:39.862Z" }, + { url = "https://files.pythonhosted.org/packages/16/85/276033dcbcc369eb176594de22728541a925b2632f9716428c851b149e83/charset_normalizer-3.4.4-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cb6254dc36b47a990e59e1068afacdcd02958bdcce30bb50cc1700a8b9d624a6", size = 149524, upload-time = "2025-10-14T04:41:41.319Z" }, + { url = "https://files.pythonhosted.org/packages/9e/f2/6a2a1f722b6aba37050e626530a46a68f74e63683947a8acff92569f979a/charset_normalizer-3.4.4-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c8ae8a0f02f57a6e61203a31428fa1d677cbe50c93622b4149d5c0f319c1d19e", size = 150395, upload-time = "2025-10-14T04:41:42.539Z" }, + { url = "https://files.pythonhosted.org/packages/60/bb/2186cb2f2bbaea6338cad15ce23a67f9b0672929744381e28b0592676824/charset_normalizer-3.4.4-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:47cc91b2f4dd2833fddaedd2893006b0106129d4b94fdb6af1f4ce5a9965577c", size = 143680, upload-time = "2025-10-14T04:41:43.661Z" }, + { url = "https://files.pythonhosted.org/packages/7d/a5/bf6f13b772fbb2a90360eb620d52ed8f796f3c5caee8398c3b2eb7b1c60d/charset_normalizer-3.4.4-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:82004af6c302b5d3ab2cfc4cc5f29db16123b1a8417f2e25f9066f91d4411090", size = 162045, upload-time = "2025-10-14T04:41:44.821Z" }, + { url = "https://files.pythonhosted.org/packages/df/c5/d1be898bf0dc3ef9030c3825e5d3b83f2c528d207d246cbabe245966808d/charset_normalizer-3.4.4-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:2b7d8f6c26245217bd2ad053761201e9f9680f8ce52f0fcd8d0755aeae5b2152", size = 149687, upload-time = "2025-10-14T04:41:46.442Z" }, + { url = "https://files.pythonhosted.org/packages/a5/42/90c1f7b9341eef50c8a1cb3f098ac43b0508413f33affd762855f67a410e/charset_normalizer-3.4.4-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:799a7a5e4fb2d5898c60b640fd4981d6a25f1c11790935a44ce38c54e985f828", size = 160014, upload-time = "2025-10-14T04:41:47.631Z" }, + { url = "https://files.pythonhosted.org/packages/76/be/4d3ee471e8145d12795ab655ece37baed0929462a86e72372fd25859047c/charset_normalizer-3.4.4-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:99ae2cffebb06e6c22bdc25801d7b30f503cc87dbd283479e7b606f70aff57ec", size = 154044, upload-time = "2025-10-14T04:41:48.81Z" }, + { url = "https://files.pythonhosted.org/packages/b0/6f/8f7af07237c34a1defe7defc565a9bc1807762f672c0fde711a4b22bf9c0/charset_normalizer-3.4.4-cp314-cp314-win32.whl", hash = "sha256:f9d332f8c2a2fcbffe1378594431458ddbef721c1769d78e2cbc06280d8155f9", size = 99940, upload-time = "2025-10-14T04:41:49.946Z" }, + { url = "https://files.pythonhosted.org/packages/4b/51/8ade005e5ca5b0d80fb4aff72a3775b325bdc3d27408c8113811a7cbe640/charset_normalizer-3.4.4-cp314-cp314-win_amd64.whl", hash = "sha256:8a6562c3700cce886c5be75ade4a5db4214fda19fede41d9792d100288d8f94c", size = 107104, upload-time = "2025-10-14T04:41:51.051Z" }, + { url = "https://files.pythonhosted.org/packages/da/5f/6b8f83a55bb8278772c5ae54a577f3099025f9ade59d0136ac24a0df4bde/charset_normalizer-3.4.4-cp314-cp314-win_arm64.whl", hash = "sha256:de00632ca48df9daf77a2c65a484531649261ec9f25489917f09e455cb09ddb2", size = 100743, upload-time = "2025-10-14T04:41:52.122Z" }, + { url = "https://files.pythonhosted.org/packages/0a/4c/925909008ed5a988ccbb72dcc897407e5d6d3bd72410d69e051fc0c14647/charset_normalizer-3.4.4-py3-none-any.whl", hash = "sha256:7a32c560861a02ff789ad905a2fe94e3f840803362c84fecf1851cb4cf3dc37f", size = 53402, upload-time = "2025-10-14T04:42:31.76Z" }, +] + +[[package]] +name = "click" +version = "8.1.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/59/87/84326af34517fca8c58418d148f2403df25303e02736832403587318e9e8/click-8.1.3.tar.gz", hash = "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e", size = 331147, upload-time = "2022-04-28T17:36:09.097Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c2/f1/df59e28c642d583f7dacffb1e0965d0e00b218e0186d7858ac5233dce840/click-8.1.3-py3-none-any.whl", hash = "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48", size = 96588, upload-time = "2022-04-28T17:36:06.952Z" }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "dill" +version = "0.4.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/81/e1/56027a71e31b02ddc53c7d65b01e68edf64dea2932122fe7746a516f75d5/dill-0.4.1.tar.gz", hash = "sha256:423092df4182177d4d8ba8290c8a5b640c66ab35ec7da59ccfa00f6fa3eea5fa", size = 187315, upload-time = "2026-01-19T02:36:56.85Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/77/dc8c558f7593132cf8fefec57c4f60c83b16941c574ac5f619abb3ae7933/dill-0.4.1-py3-none-any.whl", hash = "sha256:1e1ce33e978ae97fcfcff5638477032b801c46c7c65cf717f95fbc2248f79a9d", size = 120019, upload-time = "2026-01-19T02:36:55.663Z" }, +] + +[[package]] +name = "flake8" +version = "7.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mccabe" }, + { name = "pycodestyle" }, + { name = "pyflakes" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9b/af/fbfe3c4b5a657d79e5c47a2827a362f9e1b763336a52f926126aa6dc7123/flake8-7.3.0.tar.gz", hash = "sha256:fe044858146b9fc69b551a4b490d69cf960fcb78ad1edcb84e7fbb1b4a8e3872", size = 48326, upload-time = "2025-06-20T19:31:35.838Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9f/56/13ab06b4f93ca7cac71078fbe37fcea175d3216f31f85c3168a6bbd0bb9a/flake8-7.3.0-py2.py3-none-any.whl", hash = "sha256:b9696257b9ce8beb888cdbe31cf885c90d31928fe202be0889a7cdafad32f01e", size = 57922, upload-time = "2025-06-20T19:31:34.425Z" }, +] + +[[package]] +name = "gitlint" +version = "0.19.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "gitlint-core", extra = ["trusted-deps"] }, +] +sdist = { url = "https://files.pythonhosted.org/packages/95/d2/67e308b3f26394b2e98d2b0df594aa8c116e165d09d15094d45ecade7272/gitlint-0.19.1.tar.gz", hash = "sha256:b5b70fb894e80849b69abbb65ee7dbb3520fc3511f202a6e6b6ddf1a71ee8f61", size = 4799, upload-time = "2023-03-10T12:51:45.875Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/84/90/8b6a29ddceb51272a054399d4864eefd28083530349f5735e0bb979c5f0f/gitlint-0.19.1-py3-none-any.whl", hash = "sha256:26bb085959148d99fbbc178b4e56fda6c3edd7646b7c2a24d8ee1f8e036ed85d", size = 2649, upload-time = "2023-03-10T12:51:44.191Z" }, +] + +[[package]] +name = "gitlint-core" +version = "0.19.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "arrow" }, + { name = "click" }, + { name = "sh", marker = "sys_platform != 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/73/51/b59270264aabcab5b933f3eb9bfb022464ca9205b04feef1bdc1635fd9b4/gitlint_core-0.19.1.tar.gz", hash = "sha256:7bf977b03ff581624a9e03f65ebb8502cc12dfaa3e92d23e8b2b54bbdaa29992", size = 36141, upload-time = "2023-03-10T12:51:42.713Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b0/57/dea471da24ceac6de8c3dc5d37e4ddde57a5c340d6bac90010898734de34/gitlint_core-0.19.1-py3-none-any.whl", hash = "sha256:f41effd1dcbc06ffbfc56b6888cce72241796f517b46bd9fd4ab1b145056988c", size = 44207, upload-time = "2023-03-10T12:51:41.054Z" }, +] + +[package.optional-dependencies] +trusted-deps = [ + { name = "arrow" }, + { name = "click" }, + { name = "sh", marker = "sys_platform != 'win32'" }, +] + +[[package]] +name = "greenlet" +version = "3.3.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a3/51/1664f6b78fc6ebbd98019a1fd730e83fa78f2db7058f72b1463d3612b8db/greenlet-3.3.2.tar.gz", hash = "sha256:2eaf067fc6d886931c7962e8c6bede15d2f01965560f3359b27c80bde2d151f2", size = 188267, upload-time = "2026-02-20T20:54:15.531Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ea/ab/1608e5a7578e62113506740b88066bf09888322a311cff602105e619bd87/greenlet-3.3.2-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:ac8d61d4343b799d1e526db579833d72f23759c71e07181c2d2944e429eb09cd", size = 280358, upload-time = "2026-02-20T20:17:43.971Z" }, + { url = "https://files.pythonhosted.org/packages/a5/23/0eae412a4ade4e6623ff7626e38998cb9b11e9ff1ebacaa021e4e108ec15/greenlet-3.3.2-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3ceec72030dae6ac0c8ed7591b96b70410a8be370b6a477b1dbc072856ad02bd", size = 601217, upload-time = "2026-02-20T20:47:31.462Z" }, + { url = "https://files.pythonhosted.org/packages/f8/16/5b1678a9c07098ecb9ab2dd159fafaf12e963293e61ee8d10ecb55273e5e/greenlet-3.3.2-cp312-cp312-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a2a5be83a45ce6188c045bcc44b0ee037d6a518978de9a5d97438548b953a1ac", size = 611792, upload-time = "2026-02-20T20:55:58.423Z" }, + { url = "https://files.pythonhosted.org/packages/5c/c5/cc09412a29e43406eba18d61c70baa936e299bc27e074e2be3806ed29098/greenlet-3.3.2-cp312-cp312-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ae9e21c84035c490506c17002f5c8ab25f980205c3e61ddb3a2a2a2e6c411fcb", size = 626250, upload-time = "2026-02-20T21:02:46.596Z" }, + { url = "https://files.pythonhosted.org/packages/50/1f/5155f55bd71cabd03765a4aac9ac446be129895271f73872c36ebd4b04b6/greenlet-3.3.2-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:43e99d1749147ac21dde49b99c9abffcbc1e2d55c67501465ef0930d6e78e070", size = 613875, upload-time = "2026-02-20T20:21:01.102Z" }, + { url = "https://files.pythonhosted.org/packages/fc/dd/845f249c3fcd69e32df80cdab059b4be8b766ef5830a3d0aa9d6cad55beb/greenlet-3.3.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4c956a19350e2c37f2c48b336a3afb4bff120b36076d9d7fb68cb44e05d95b79", size = 1571467, upload-time = "2026-02-20T20:49:33.495Z" }, + { url = "https://files.pythonhosted.org/packages/2a/50/2649fe21fcc2b56659a452868e695634722a6655ba245d9f77f5656010bf/greenlet-3.3.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6c6f8ba97d17a1e7d664151284cb3315fc5f8353e75221ed4324f84eb162b395", size = 1640001, upload-time = "2026-02-20T20:21:09.154Z" }, + { url = "https://files.pythonhosted.org/packages/9b/40/cc802e067d02af8b60b6771cea7d57e21ef5e6659912814babb42b864713/greenlet-3.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:34308836d8370bddadb41f5a7ce96879b72e2fdfb4e87729330c6ab52376409f", size = 231081, upload-time = "2026-02-20T20:17:28.121Z" }, + { url = "https://files.pythonhosted.org/packages/58/2e/fe7f36ff1982d6b10a60d5e0740c759259a7d6d2e1dc41da6d96de32fff6/greenlet-3.3.2-cp312-cp312-win_arm64.whl", hash = "sha256:d3a62fa76a32b462a97198e4c9e99afb9ab375115e74e9a83ce180e7a496f643", size = 230331, upload-time = "2026-02-20T20:17:23.34Z" }, + { url = "https://files.pythonhosted.org/packages/ac/48/f8b875fa7dea7dd9b33245e37f065af59df6a25af2f9561efa8d822fde51/greenlet-3.3.2-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:aa6ac98bdfd716a749b84d4034486863fd81c3abde9aa3cf8eff9127981a4ae4", size = 279120, upload-time = "2026-02-20T20:19:01.9Z" }, + { url = "https://files.pythonhosted.org/packages/49/8d/9771d03e7a8b1ee456511961e1b97a6d77ae1dea4a34a5b98eee706689d3/greenlet-3.3.2-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ab0c7e7901a00bc0a7284907273dc165b32e0d109a6713babd04471327ff7986", size = 603238, upload-time = "2026-02-20T20:47:32.873Z" }, + { url = "https://files.pythonhosted.org/packages/59/0e/4223c2bbb63cd5c97f28ffb2a8aee71bdfb30b323c35d409450f51b91e3e/greenlet-3.3.2-cp313-cp313-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d248d8c23c67d2291ffd47af766e2a3aa9fa1c6703155c099feb11f526c63a92", size = 614219, upload-time = "2026-02-20T20:55:59.817Z" }, + { url = "https://files.pythonhosted.org/packages/94/2b/4d012a69759ac9d77210b8bfb128bc621125f5b20fc398bce3940d036b1c/greenlet-3.3.2-cp313-cp313-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ccd21bb86944ca9be6d967cf7691e658e43417782bce90b5d2faeda0ff78a7dd", size = 628268, upload-time = "2026-02-20T21:02:48.024Z" }, + { url = "https://files.pythonhosted.org/packages/7a/34/259b28ea7a2a0c904b11cd36c79b8cef8019b26ee5dbe24e73b469dea347/greenlet-3.3.2-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b6997d360a4e6a4e936c0f9625b1c20416b8a0ea18a8e19cabbefc712e7397ab", size = 616774, upload-time = "2026-02-20T20:21:02.454Z" }, + { url = "https://files.pythonhosted.org/packages/0a/03/996c2d1689d486a6e199cb0f1cf9e4aa940c500e01bdf201299d7d61fa69/greenlet-3.3.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:64970c33a50551c7c50491671265d8954046cb6e8e2999aacdd60e439b70418a", size = 1571277, upload-time = "2026-02-20T20:49:34.795Z" }, + { url = "https://files.pythonhosted.org/packages/d9/c4/2570fc07f34a39f2caf0bf9f24b0a1a0a47bc2e8e465b2c2424821389dfc/greenlet-3.3.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:1a9172f5bf6bd88e6ba5a84e0a68afeac9dc7b6b412b245dd64f52d83c81e55b", size = 1640455, upload-time = "2026-02-20T20:21:10.261Z" }, + { url = "https://files.pythonhosted.org/packages/91/39/5ef5aa23bc545aa0d31e1b9b55822b32c8da93ba657295840b6b34124009/greenlet-3.3.2-cp313-cp313-win_amd64.whl", hash = "sha256:a7945dd0eab63ded0a48e4dcade82939783c172290a7903ebde9e184333ca124", size = 230961, upload-time = "2026-02-20T20:16:58.461Z" }, + { url = "https://files.pythonhosted.org/packages/62/6b/a89f8456dcb06becff288f563618e9f20deed8dd29beea14f9a168aef64b/greenlet-3.3.2-cp313-cp313-win_arm64.whl", hash = "sha256:394ead29063ee3515b4e775216cb756b2e3b4a7e55ae8fd884f17fa579e6b327", size = 230221, upload-time = "2026-02-20T20:17:37.152Z" }, + { url = "https://files.pythonhosted.org/packages/3f/ae/8bffcbd373b57a5992cd077cbe8858fff39110480a9d50697091faea6f39/greenlet-3.3.2-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:8d1658d7291f9859beed69a776c10822a0a799bc4bfe1bd4272bb60e62507dab", size = 279650, upload-time = "2026-02-20T20:18:00.783Z" }, + { url = "https://files.pythonhosted.org/packages/d1/c0/45f93f348fa49abf32ac8439938726c480bd96b2a3c6f4d949ec0124b69f/greenlet-3.3.2-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:18cb1b7337bca281915b3c5d5ae19f4e76d35e1df80f4ad3c1a7be91fadf1082", size = 650295, upload-time = "2026-02-20T20:47:34.036Z" }, + { url = "https://files.pythonhosted.org/packages/b3/de/dd7589b3f2b8372069ab3e4763ea5329940fc7ad9dcd3e272a37516d7c9b/greenlet-3.3.2-cp314-cp314-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c2e47408e8ce1c6f1ceea0dffcdf6ebb85cc09e55c7af407c99f1112016e45e9", size = 662163, upload-time = "2026-02-20T20:56:01.295Z" }, + { url = "https://files.pythonhosted.org/packages/cd/ac/85804f74f1ccea31ba518dcc8ee6f14c79f73fe36fa1beba38930806df09/greenlet-3.3.2-cp314-cp314-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e3cb43ce200f59483eb82949bf1835a99cf43d7571e900d7c8d5c62cdf25d2f9", size = 675371, upload-time = "2026-02-20T21:02:49.664Z" }, + { url = "https://files.pythonhosted.org/packages/d2/d8/09bfa816572a4d83bccd6750df1926f79158b1c36c5f73786e26dbe4ee38/greenlet-3.3.2-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:63d10328839d1973e5ba35e98cccbca71b232b14051fd957b6f8b6e8e80d0506", size = 664160, upload-time = "2026-02-20T20:21:04.015Z" }, + { url = "https://files.pythonhosted.org/packages/48/cf/56832f0c8255d27f6c35d41b5ec91168d74ec721d85f01a12131eec6b93c/greenlet-3.3.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:8e4ab3cfb02993c8cc248ea73d7dae6cec0253e9afa311c9b37e603ca9fad2ce", size = 1619181, upload-time = "2026-02-20T20:49:36.052Z" }, + { url = "https://files.pythonhosted.org/packages/0a/23/b90b60a4aabb4cec0796e55f25ffbfb579a907c3898cd2905c8918acaa16/greenlet-3.3.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:94ad81f0fd3c0c0681a018a976e5c2bd2ca2d9d94895f23e7bb1af4e8af4e2d5", size = 1687713, upload-time = "2026-02-20T20:21:11.684Z" }, + { url = "https://files.pythonhosted.org/packages/f3/ca/2101ca3d9223a1dc125140dbc063644dca76df6ff356531eb27bc267b446/greenlet-3.3.2-cp314-cp314-win_amd64.whl", hash = "sha256:8c4dd0f3997cf2512f7601563cc90dfb8957c0cff1e3a1b23991d4ea1776c492", size = 232034, upload-time = "2026-02-20T20:20:08.186Z" }, + { url = "https://files.pythonhosted.org/packages/f6/4a/ecf894e962a59dea60f04877eea0fd5724618da89f1867b28ee8b91e811f/greenlet-3.3.2-cp314-cp314-win_arm64.whl", hash = "sha256:cd6f9e2bbd46321ba3bbb4c8a15794d32960e3b0ae2cc4d49a1a53d314805d71", size = 231437, upload-time = "2026-02-20T20:18:59.722Z" }, + { url = "https://files.pythonhosted.org/packages/98/6d/8f2ef704e614bcf58ed43cfb8d87afa1c285e98194ab2cfad351bf04f81e/greenlet-3.3.2-cp314-cp314t-macosx_11_0_universal2.whl", hash = "sha256:e26e72bec7ab387ac80caa7496e0f908ff954f31065b0ffc1f8ecb1338b11b54", size = 286617, upload-time = "2026-02-20T20:19:29.856Z" }, + { url = "https://files.pythonhosted.org/packages/5e/0d/93894161d307c6ea237a43988f27eba0947b360b99ac5239ad3fe09f0b47/greenlet-3.3.2-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8b466dff7a4ffda6ca975979bab80bdadde979e29fc947ac3be4451428d8b0e4", size = 655189, upload-time = "2026-02-20T20:47:35.742Z" }, + { url = "https://files.pythonhosted.org/packages/f5/2c/d2d506ebd8abcb57386ec4f7ba20f4030cbe56eae541bc6fd6ef399c0b41/greenlet-3.3.2-cp314-cp314t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b8bddc5b73c9720bea487b3bffdb1840fe4e3656fba3bd40aa1489e9f37877ff", size = 658225, upload-time = "2026-02-20T20:56:02.527Z" }, + { url = "https://files.pythonhosted.org/packages/d1/67/8197b7e7e602150938049d8e7f30de1660cfb87e4c8ee349b42b67bdb2e1/greenlet-3.3.2-cp314-cp314t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:59b3e2c40f6706b05a9cd299c836c6aa2378cabe25d021acd80f13abf81181cf", size = 666581, upload-time = "2026-02-20T21:02:51.526Z" }, + { url = "https://files.pythonhosted.org/packages/8e/30/3a09155fbf728673a1dea713572d2d31159f824a37c22da82127056c44e4/greenlet-3.3.2-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b26b0f4428b871a751968285a1ac9648944cea09807177ac639b030bddebcea4", size = 657907, upload-time = "2026-02-20T20:21:05.259Z" }, + { url = "https://files.pythonhosted.org/packages/f3/fd/d05a4b7acd0154ed758797f0a43b4c0962a843bedfe980115e842c5b2d08/greenlet-3.3.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:1fb39a11ee2e4d94be9a76671482be9398560955c9e568550de0224e41104727", size = 1618857, upload-time = "2026-02-20T20:49:37.309Z" }, + { url = "https://files.pythonhosted.org/packages/6f/e1/50ee92a5db521de8f35075b5eff060dd43d39ebd46c2181a2042f7070385/greenlet-3.3.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:20154044d9085151bc309e7689d6f7ba10027f8f5a8c0676ad398b951913d89e", size = 1680010, upload-time = "2026-02-20T20:21:13.427Z" }, + { url = "https://files.pythonhosted.org/packages/29/4b/45d90626aef8e65336bed690106d1382f7a43665e2249017e9527df8823b/greenlet-3.3.2-cp314-cp314t-win_amd64.whl", hash = "sha256:c04c5e06ec3e022cbfe2cd4a846e1d4e50087444f875ff6d2c2ad8445495cf1a", size = 237086, upload-time = "2026-02-20T20:20:45.786Z" }, +] + +[[package]] +name = "idna" +version = "3.11" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6f/6d/0703ccc57f3a7233505399edb88de3cbd678da106337b9fcde432b65ed60/idna-3.11.tar.gz", hash = "sha256:795dafcc9c04ed0c1fb032c2aa73654d8e8c5023a7df64a53f39190ada629902", size = 194582, upload-time = "2025-10-12T14:55:20.501Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0e/61/66938bbb5fc52dbdf84594873d5b51fb1f7c7794e9c0f5bd885f30bc507b/idna-3.11-py3-none-any.whl", hash = "sha256:771a87f49d9defaf64091e6e6fe9c18d4833f140bd19464795bc32d966ca37ea", size = 71008, upload-time = "2025-10-12T14:55:18.883Z" }, +] + +[[package]] +name = "iniconfig" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, +] + +[[package]] +name = "isort" +version = "8.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bf/e3/e72b0b3a85f24cf5fc2cd8e92b996592798f896024c5cdf3709232e6e377/isort-8.0.0.tar.gz", hash = "sha256:fddea59202f231e170e52e71e3510b99c373b6e571b55d9c7b31b679c0fed47c", size = 769482, upload-time = "2026-02-19T16:31:59.716Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/74/ea/cf3aad99dd12c026e2d6835d559efb6fc50ccfd5b46d42d5fec2608b116a/isort-8.0.0-py3-none-any.whl", hash = "sha256:184916a933041c7cf718787f7e52064f3c06272aff69a5cb4dc46497bd8911d9", size = 89715, upload-time = "2026-02-19T16:31:57.745Z" }, +] + +[[package]] +name = "jinja2" +version = "3.1.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markupsafe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/df/bf/f7da0350254c0ed7c72f3e33cef02e048281fec7ecec5f032d4aac52226b/jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d", size = 245115, upload-time = "2025-03-05T20:05:02.478Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899, upload-time = "2025-03-05T20:05:00.369Z" }, +] + +[[package]] +name = "markupsafe" +version = "3.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7e/99/7690b6d4034fffd95959cbe0c02de8deb3098cc577c67bb6a24fe5d7caa7/markupsafe-3.0.3.tar.gz", hash = "sha256:722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698", size = 80313, upload-time = "2025-09-27T18:37:40.426Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5a/72/147da192e38635ada20e0a2e1a51cf8823d2119ce8883f7053879c2199b5/markupsafe-3.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d53197da72cc091b024dd97249dfc7794d6a56530370992a5e1a08983ad9230e", size = 11615, upload-time = "2025-09-27T18:36:30.854Z" }, + { url = "https://files.pythonhosted.org/packages/9a/81/7e4e08678a1f98521201c3079f77db69fb552acd56067661f8c2f534a718/markupsafe-3.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1872df69a4de6aead3491198eaf13810b565bdbeec3ae2dc8780f14458ec73ce", size = 12020, upload-time = "2025-09-27T18:36:31.971Z" }, + { url = "https://files.pythonhosted.org/packages/1e/2c/799f4742efc39633a1b54a92eec4082e4f815314869865d876824c257c1e/markupsafe-3.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3a7e8ae81ae39e62a41ec302f972ba6ae23a5c5396c8e60113e9066ef893da0d", size = 24332, upload-time = "2025-09-27T18:36:32.813Z" }, + { url = "https://files.pythonhosted.org/packages/3c/2e/8d0c2ab90a8c1d9a24f0399058ab8519a3279d1bd4289511d74e909f060e/markupsafe-3.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d6dd0be5b5b189d31db7cda48b91d7e0a9795f31430b7f271219ab30f1d3ac9d", size = 22947, upload-time = "2025-09-27T18:36:33.86Z" }, + { url = "https://files.pythonhosted.org/packages/2c/54/887f3092a85238093a0b2154bd629c89444f395618842e8b0c41783898ea/markupsafe-3.0.3-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:94c6f0bb423f739146aec64595853541634bde58b2135f27f61c1ffd1cd4d16a", size = 21962, upload-time = "2025-09-27T18:36:35.099Z" }, + { url = "https://files.pythonhosted.org/packages/c9/2f/336b8c7b6f4a4d95e91119dc8521402461b74a485558d8f238a68312f11c/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:be8813b57049a7dc738189df53d69395eba14fb99345e0a5994914a3864c8a4b", size = 23760, upload-time = "2025-09-27T18:36:36.001Z" }, + { url = "https://files.pythonhosted.org/packages/32/43/67935f2b7e4982ffb50a4d169b724d74b62a3964bc1a9a527f5ac4f1ee2b/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:83891d0e9fb81a825d9a6d61e3f07550ca70a076484292a70fde82c4b807286f", size = 21529, upload-time = "2025-09-27T18:36:36.906Z" }, + { url = "https://files.pythonhosted.org/packages/89/e0/4486f11e51bbba8b0c041098859e869e304d1c261e59244baa3d295d47b7/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:77f0643abe7495da77fb436f50f8dab76dbc6e5fd25d39589a0f1fe6548bfa2b", size = 23015, upload-time = "2025-09-27T18:36:37.868Z" }, + { url = "https://files.pythonhosted.org/packages/2f/e1/78ee7a023dac597a5825441ebd17170785a9dab23de95d2c7508ade94e0e/markupsafe-3.0.3-cp312-cp312-win32.whl", hash = "sha256:d88b440e37a16e651bda4c7c2b930eb586fd15ca7406cb39e211fcff3bf3017d", size = 14540, upload-time = "2025-09-27T18:36:38.761Z" }, + { url = "https://files.pythonhosted.org/packages/aa/5b/bec5aa9bbbb2c946ca2733ef9c4ca91c91b6a24580193e891b5f7dbe8e1e/markupsafe-3.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:26a5784ded40c9e318cfc2bdb30fe164bdb8665ded9cd64d500a34fb42067b1c", size = 15105, upload-time = "2025-09-27T18:36:39.701Z" }, + { url = "https://files.pythonhosted.org/packages/e5/f1/216fc1bbfd74011693a4fd837e7026152e89c4bcf3e77b6692fba9923123/markupsafe-3.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:35add3b638a5d900e807944a078b51922212fb3dedb01633a8defc4b01a3c85f", size = 13906, upload-time = "2025-09-27T18:36:40.689Z" }, + { url = "https://files.pythonhosted.org/packages/38/2f/907b9c7bbba283e68f20259574b13d005c121a0fa4c175f9bed27c4597ff/markupsafe-3.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e1cf1972137e83c5d4c136c43ced9ac51d0e124706ee1c8aa8532c1287fa8795", size = 11622, upload-time = "2025-09-27T18:36:41.777Z" }, + { url = "https://files.pythonhosted.org/packages/9c/d9/5f7756922cdd676869eca1c4e3c0cd0df60ed30199ffd775e319089cb3ed/markupsafe-3.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:116bb52f642a37c115f517494ea5feb03889e04df47eeff5b130b1808ce7c219", size = 12029, upload-time = "2025-09-27T18:36:43.257Z" }, + { url = "https://files.pythonhosted.org/packages/00/07/575a68c754943058c78f30db02ee03a64b3c638586fba6a6dd56830b30a3/markupsafe-3.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:133a43e73a802c5562be9bbcd03d090aa5a1fe899db609c29e8c8d815c5f6de6", size = 24374, upload-time = "2025-09-27T18:36:44.508Z" }, + { url = "https://files.pythonhosted.org/packages/a9/21/9b05698b46f218fc0e118e1f8168395c65c8a2c750ae2bab54fc4bd4e0e8/markupsafe-3.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ccfcd093f13f0f0b7fdd0f198b90053bf7b2f02a3927a30e63f3ccc9df56b676", size = 22980, upload-time = "2025-09-27T18:36:45.385Z" }, + { url = "https://files.pythonhosted.org/packages/7f/71/544260864f893f18b6827315b988c146b559391e6e7e8f7252839b1b846a/markupsafe-3.0.3-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:509fa21c6deb7a7a273d629cf5ec029bc209d1a51178615ddf718f5918992ab9", size = 21990, upload-time = "2025-09-27T18:36:46.916Z" }, + { url = "https://files.pythonhosted.org/packages/c2/28/b50fc2f74d1ad761af2f5dcce7492648b983d00a65b8c0e0cb457c82ebbe/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a4afe79fb3de0b7097d81da19090f4df4f8d3a2b3adaa8764138aac2e44f3af1", size = 23784, upload-time = "2025-09-27T18:36:47.884Z" }, + { url = "https://files.pythonhosted.org/packages/ed/76/104b2aa106a208da8b17a2fb72e033a5a9d7073c68f7e508b94916ed47a9/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:795e7751525cae078558e679d646ae45574b47ed6e7771863fcc079a6171a0fc", size = 21588, upload-time = "2025-09-27T18:36:48.82Z" }, + { url = "https://files.pythonhosted.org/packages/b5/99/16a5eb2d140087ebd97180d95249b00a03aa87e29cc224056274f2e45fd6/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:8485f406a96febb5140bfeca44a73e3ce5116b2501ac54fe953e488fb1d03b12", size = 23041, upload-time = "2025-09-27T18:36:49.797Z" }, + { url = "https://files.pythonhosted.org/packages/19/bc/e7140ed90c5d61d77cea142eed9f9c303f4c4806f60a1044c13e3f1471d0/markupsafe-3.0.3-cp313-cp313-win32.whl", hash = "sha256:bdd37121970bfd8be76c5fb069c7751683bdf373db1ed6c010162b2a130248ed", size = 14543, upload-time = "2025-09-27T18:36:51.584Z" }, + { url = "https://files.pythonhosted.org/packages/05/73/c4abe620b841b6b791f2edc248f556900667a5a1cf023a6646967ae98335/markupsafe-3.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:9a1abfdc021a164803f4d485104931fb8f8c1efd55bc6b748d2f5774e78b62c5", size = 15113, upload-time = "2025-09-27T18:36:52.537Z" }, + { url = "https://files.pythonhosted.org/packages/f0/3a/fa34a0f7cfef23cf9500d68cb7c32dd64ffd58a12b09225fb03dd37d5b80/markupsafe-3.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:7e68f88e5b8799aa49c85cd116c932a1ac15caaa3f5db09087854d218359e485", size = 13911, upload-time = "2025-09-27T18:36:53.513Z" }, + { url = "https://files.pythonhosted.org/packages/e4/d7/e05cd7efe43a88a17a37b3ae96e79a19e846f3f456fe79c57ca61356ef01/markupsafe-3.0.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:218551f6df4868a8d527e3062d0fb968682fe92054e89978594c28e642c43a73", size = 11658, upload-time = "2025-09-27T18:36:54.819Z" }, + { url = "https://files.pythonhosted.org/packages/99/9e/e412117548182ce2148bdeacdda3bb494260c0b0184360fe0d56389b523b/markupsafe-3.0.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:3524b778fe5cfb3452a09d31e7b5adefeea8c5be1d43c4f810ba09f2ceb29d37", size = 12066, upload-time = "2025-09-27T18:36:55.714Z" }, + { url = "https://files.pythonhosted.org/packages/bc/e6/fa0ffcda717ef64a5108eaa7b4f5ed28d56122c9a6d70ab8b72f9f715c80/markupsafe-3.0.3-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4e885a3d1efa2eadc93c894a21770e4bc67899e3543680313b09f139e149ab19", size = 25639, upload-time = "2025-09-27T18:36:56.908Z" }, + { url = "https://files.pythonhosted.org/packages/96/ec/2102e881fe9d25fc16cb4b25d5f5cde50970967ffa5dddafdb771237062d/markupsafe-3.0.3-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8709b08f4a89aa7586de0aadc8da56180242ee0ada3999749b183aa23df95025", size = 23569, upload-time = "2025-09-27T18:36:57.913Z" }, + { url = "https://files.pythonhosted.org/packages/4b/30/6f2fce1f1f205fc9323255b216ca8a235b15860c34b6798f810f05828e32/markupsafe-3.0.3-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b8512a91625c9b3da6f127803b166b629725e68af71f8184ae7e7d54686a56d6", size = 23284, upload-time = "2025-09-27T18:36:58.833Z" }, + { url = "https://files.pythonhosted.org/packages/58/47/4a0ccea4ab9f5dcb6f79c0236d954acb382202721e704223a8aafa38b5c8/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:9b79b7a16f7fedff2495d684f2b59b0457c3b493778c9eed31111be64d58279f", size = 24801, upload-time = "2025-09-27T18:36:59.739Z" }, + { url = "https://files.pythonhosted.org/packages/6a/70/3780e9b72180b6fecb83a4814d84c3bf4b4ae4bf0b19c27196104149734c/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:12c63dfb4a98206f045aa9563db46507995f7ef6d83b2f68eda65c307c6829eb", size = 22769, upload-time = "2025-09-27T18:37:00.719Z" }, + { url = "https://files.pythonhosted.org/packages/98/c5/c03c7f4125180fc215220c035beac6b9cb684bc7a067c84fc69414d315f5/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:8f71bc33915be5186016f675cd83a1e08523649b0e33efdb898db577ef5bb009", size = 23642, upload-time = "2025-09-27T18:37:01.673Z" }, + { url = "https://files.pythonhosted.org/packages/80/d6/2d1b89f6ca4bff1036499b1e29a1d02d282259f3681540e16563f27ebc23/markupsafe-3.0.3-cp313-cp313t-win32.whl", hash = "sha256:69c0b73548bc525c8cb9a251cddf1931d1db4d2258e9599c28c07ef3580ef354", size = 14612, upload-time = "2025-09-27T18:37:02.639Z" }, + { url = "https://files.pythonhosted.org/packages/2b/98/e48a4bfba0a0ffcf9925fe2d69240bfaa19c6f7507b8cd09c70684a53c1e/markupsafe-3.0.3-cp313-cp313t-win_amd64.whl", hash = "sha256:1b4b79e8ebf6b55351f0d91fe80f893b4743f104bff22e90697db1590e47a218", size = 15200, upload-time = "2025-09-27T18:37:03.582Z" }, + { url = "https://files.pythonhosted.org/packages/0e/72/e3cc540f351f316e9ed0f092757459afbc595824ca724cbc5a5d4263713f/markupsafe-3.0.3-cp313-cp313t-win_arm64.whl", hash = "sha256:ad2cf8aa28b8c020ab2fc8287b0f823d0a7d8630784c31e9ee5edea20f406287", size = 13973, upload-time = "2025-09-27T18:37:04.929Z" }, + { url = "https://files.pythonhosted.org/packages/33/8a/8e42d4838cd89b7dde187011e97fe6c3af66d8c044997d2183fbd6d31352/markupsafe-3.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:eaa9599de571d72e2daf60164784109f19978b327a3910d3e9de8c97b5b70cfe", size = 11619, upload-time = "2025-09-27T18:37:06.342Z" }, + { url = "https://files.pythonhosted.org/packages/b5/64/7660f8a4a8e53c924d0fa05dc3a55c9cee10bbd82b11c5afb27d44b096ce/markupsafe-3.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c47a551199eb8eb2121d4f0f15ae0f923d31350ab9280078d1e5f12b249e0026", size = 12029, upload-time = "2025-09-27T18:37:07.213Z" }, + { url = "https://files.pythonhosted.org/packages/da/ef/e648bfd021127bef5fa12e1720ffed0c6cbb8310c8d9bea7266337ff06de/markupsafe-3.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f34c41761022dd093b4b6896d4810782ffbabe30f2d443ff5f083e0cbbb8c737", size = 24408, upload-time = "2025-09-27T18:37:09.572Z" }, + { url = "https://files.pythonhosted.org/packages/41/3c/a36c2450754618e62008bf7435ccb0f88053e07592e6028a34776213d877/markupsafe-3.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:457a69a9577064c05a97c41f4e65148652db078a3a509039e64d3467b9e7ef97", size = 23005, upload-time = "2025-09-27T18:37:10.58Z" }, + { url = "https://files.pythonhosted.org/packages/bc/20/b7fdf89a8456b099837cd1dc21974632a02a999ec9bf7ca3e490aacd98e7/markupsafe-3.0.3-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e8afc3f2ccfa24215f8cb28dcf43f0113ac3c37c2f0f0806d8c70e4228c5cf4d", size = 22048, upload-time = "2025-09-27T18:37:11.547Z" }, + { url = "https://files.pythonhosted.org/packages/9a/a7/591f592afdc734f47db08a75793a55d7fbcc6902a723ae4cfbab61010cc5/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ec15a59cf5af7be74194f7ab02d0f59a62bdcf1a537677ce67a2537c9b87fcda", size = 23821, upload-time = "2025-09-27T18:37:12.48Z" }, + { url = "https://files.pythonhosted.org/packages/7d/33/45b24e4f44195b26521bc6f1a82197118f74df348556594bd2262bda1038/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:0eb9ff8191e8498cca014656ae6b8d61f39da5f95b488805da4bb029cccbfbaf", size = 21606, upload-time = "2025-09-27T18:37:13.485Z" }, + { url = "https://files.pythonhosted.org/packages/ff/0e/53dfaca23a69fbfbbf17a4b64072090e70717344c52eaaaa9c5ddff1e5f0/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2713baf880df847f2bece4230d4d094280f4e67b1e813eec43b4c0e144a34ffe", size = 23043, upload-time = "2025-09-27T18:37:14.408Z" }, + { url = "https://files.pythonhosted.org/packages/46/11/f333a06fc16236d5238bfe74daccbca41459dcd8d1fa952e8fbd5dccfb70/markupsafe-3.0.3-cp314-cp314-win32.whl", hash = "sha256:729586769a26dbceff69f7a7dbbf59ab6572b99d94576a5592625d5b411576b9", size = 14747, upload-time = "2025-09-27T18:37:15.36Z" }, + { url = "https://files.pythonhosted.org/packages/28/52/182836104b33b444e400b14f797212f720cbc9ed6ba34c800639d154e821/markupsafe-3.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:bdc919ead48f234740ad807933cdf545180bfbe9342c2bb451556db2ed958581", size = 15341, upload-time = "2025-09-27T18:37:16.496Z" }, + { url = "https://files.pythonhosted.org/packages/6f/18/acf23e91bd94fd7b3031558b1f013adfa21a8e407a3fdb32745538730382/markupsafe-3.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:5a7d5dc5140555cf21a6fefbdbf8723f06fcd2f63ef108f2854de715e4422cb4", size = 14073, upload-time = "2025-09-27T18:37:17.476Z" }, + { url = "https://files.pythonhosted.org/packages/3c/f0/57689aa4076e1b43b15fdfa646b04653969d50cf30c32a102762be2485da/markupsafe-3.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:1353ef0c1b138e1907ae78e2f6c63ff67501122006b0f9abad68fda5f4ffc6ab", size = 11661, upload-time = "2025-09-27T18:37:18.453Z" }, + { url = "https://files.pythonhosted.org/packages/89/c3/2e67a7ca217c6912985ec766c6393b636fb0c2344443ff9d91404dc4c79f/markupsafe-3.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1085e7fbddd3be5f89cc898938f42c0b3c711fdcb37d75221de2666af647c175", size = 12069, upload-time = "2025-09-27T18:37:19.332Z" }, + { url = "https://files.pythonhosted.org/packages/f0/00/be561dce4e6ca66b15276e184ce4b8aec61fe83662cce2f7d72bd3249d28/markupsafe-3.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1b52b4fb9df4eb9ae465f8d0c228a00624de2334f216f178a995ccdcf82c4634", size = 25670, upload-time = "2025-09-27T18:37:20.245Z" }, + { url = "https://files.pythonhosted.org/packages/50/09/c419f6f5a92e5fadde27efd190eca90f05e1261b10dbd8cbcb39cd8ea1dc/markupsafe-3.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fed51ac40f757d41b7c48425901843666a6677e3e8eb0abcff09e4ba6e664f50", size = 23598, upload-time = "2025-09-27T18:37:21.177Z" }, + { url = "https://files.pythonhosted.org/packages/22/44/a0681611106e0b2921b3033fc19bc53323e0b50bc70cffdd19f7d679bb66/markupsafe-3.0.3-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f190daf01f13c72eac4efd5c430a8de82489d9cff23c364c3ea822545032993e", size = 23261, upload-time = "2025-09-27T18:37:22.167Z" }, + { url = "https://files.pythonhosted.org/packages/5f/57/1b0b3f100259dc9fffe780cfb60d4be71375510e435efec3d116b6436d43/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e56b7d45a839a697b5eb268c82a71bd8c7f6c94d6fd50c3d577fa39a9f1409f5", size = 24835, upload-time = "2025-09-27T18:37:23.296Z" }, + { url = "https://files.pythonhosted.org/packages/26/6a/4bf6d0c97c4920f1597cc14dd720705eca0bf7c787aebc6bb4d1bead5388/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:f3e98bb3798ead92273dc0e5fd0f31ade220f59a266ffd8a4f6065e0a3ce0523", size = 22733, upload-time = "2025-09-27T18:37:24.237Z" }, + { url = "https://files.pythonhosted.org/packages/14/c7/ca723101509b518797fedc2fdf79ba57f886b4aca8a7d31857ba3ee8281f/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5678211cb9333a6468fb8d8be0305520aa073f50d17f089b5b4b477ea6e67fdc", size = 23672, upload-time = "2025-09-27T18:37:25.271Z" }, + { url = "https://files.pythonhosted.org/packages/fb/df/5bd7a48c256faecd1d36edc13133e51397e41b73bb77e1a69deab746ebac/markupsafe-3.0.3-cp314-cp314t-win32.whl", hash = "sha256:915c04ba3851909ce68ccc2b8e2cd691618c4dc4c4232fb7982bca3f41fd8c3d", size = 14819, upload-time = "2025-09-27T18:37:26.285Z" }, + { url = "https://files.pythonhosted.org/packages/1a/8a/0402ba61a2f16038b48b39bccca271134be00c5c9f0f623208399333c448/markupsafe-3.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4faffd047e07c38848ce017e8725090413cd80cbc23d86e55c587bf979e579c9", size = 15426, upload-time = "2025-09-27T18:37:27.316Z" }, + { url = "https://files.pythonhosted.org/packages/70/bc/6f1c2f612465f5fa89b95bead1f44dcb607670fd42891d8fdcd5d039f4f4/markupsafe-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:32001d6a8fc98c8cb5c947787c5d08b0a50663d139f1305bac5885d98d9b40fa", size = 14146, upload-time = "2025-09-27T18:37:28.327Z" }, +] + +[[package]] +name = "mccabe" +version = "0.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e7/ff/0ffefdcac38932a54d2b5eed4e0ba8a408f215002cd178ad1df0f2806ff8/mccabe-0.7.0.tar.gz", hash = "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325", size = 9658, upload-time = "2022-01-24T01:14:51.113Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/27/1a/1f68f9ba0c207934b35b86a8ca3aad8395a3d6dd7921c0686e23853ff5a9/mccabe-0.7.0-py2.py3-none-any.whl", hash = "sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e", size = 7350, upload-time = "2022-01-24T01:14:49.62Z" }, +] + +[[package]] +name = "microsalt" +source = { editable = "." } +dependencies = [ + { name = "biopython" }, + { name = "bs4" }, + { name = "click" }, + { name = "jinja2" }, + { name = "pydantic" }, + { name = "pymysql" }, + { name = "python-dateutil" }, + { name = "pyyaml" }, + { name = "requests-oauthlib" }, + { name = "sqlalchemy" }, + { name = "werkzeug" }, +] + +[package.dev-dependencies] +dev = [ + { name = "black" }, + { name = "flake8" }, + { name = "gitlint" }, + { name = "pylint" }, + { name = "pytest" }, + { name = "yamllint" }, +] + +[package.metadata] +requires-dist = [ + { name = "biopython", specifier = ">=1.81" }, + { name = "bs4", specifier = "==0.0.1" }, + { name = "click", specifier = ">=8.0" }, + { name = "jinja2", specifier = ">=3.1" }, + { name = "pydantic", specifier = ">=2.0" }, + { name = "pymysql", specifier = ">=1.0" }, + { name = "python-dateutil", specifier = ">=2.8" }, + { name = "pyyaml", specifier = ">=6.0" }, + { name = "requests-oauthlib", specifier = ">=1.3" }, + { name = "sqlalchemy", specifier = ">=2.0" }, + { name = "werkzeug", specifier = ">=3.1.6" }, +] + +[package.metadata.requires-dev] +dev = [ + { name = "black", specifier = ">=23.0" }, + { name = "flake8", specifier = ">=6.0" }, + { name = "gitlint", specifier = ">=0.19.1" }, + { name = "pylint", specifier = ">=3.0" }, + { name = "pytest", specifier = ">=7.4" }, + { name = "yamllint", specifier = ">=1.32" }, +] + +[[package]] +name = "mypy-extensions" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a2/6e/371856a3fb9d31ca8dac321cda606860fa4548858c0cc45d9d1d4ca2628b/mypy_extensions-1.1.0.tar.gz", hash = "sha256:52e68efc3284861e772bbcd66823fde5ae21fd2fdb51c62a211403730b916558", size = 6343, upload-time = "2025-04-22T14:54:24.164Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", size = 4963, upload-time = "2025-04-22T14:54:22.983Z" }, +] + +[[package]] +name = "numpy" +version = "2.4.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/57/fd/0005efbd0af48e55eb3c7208af93f2862d4b1a56cd78e84309a2d959208d/numpy-2.4.2.tar.gz", hash = "sha256:659a6107e31a83c4e33f763942275fd278b21d095094044eb35569e86a21ddae", size = 20723651, upload-time = "2026-01-31T23:13:10.135Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/51/6e/6f394c9c77668153e14d4da83bcc247beb5952f6ead7699a1a2992613bea/numpy-2.4.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:21982668592194c609de53ba4933a7471880ccbaadcc52352694a59ecc860b3a", size = 16667963, upload-time = "2026-01-31T23:10:52.147Z" }, + { url = "https://files.pythonhosted.org/packages/1f/f8/55483431f2b2fd015ae6ed4fe62288823ce908437ed49db5a03d15151678/numpy-2.4.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40397bda92382fcec844066efb11f13e1c9a3e2a8e8f318fb72ed8b6db9f60f1", size = 14693571, upload-time = "2026-01-31T23:10:54.789Z" }, + { url = "https://files.pythonhosted.org/packages/2f/20/18026832b1845cdc82248208dd929ca14c9d8f2bac391f67440707fff27c/numpy-2.4.2-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:b3a24467af63c67829bfaa61eecf18d5432d4f11992688537be59ecd6ad32f5e", size = 5203469, upload-time = "2026-01-31T23:10:57.343Z" }, + { url = "https://files.pythonhosted.org/packages/7d/33/2eb97c8a77daaba34eaa3fa7241a14ac5f51c46a6bd5911361b644c4a1e2/numpy-2.4.2-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:805cc8de9fd6e7a22da5aed858e0ab16be5a4db6c873dde1d7451c541553aa27", size = 6550820, upload-time = "2026-01-31T23:10:59.429Z" }, + { url = "https://files.pythonhosted.org/packages/b1/91/b97fdfd12dc75b02c44e26c6638241cc004d4079a0321a69c62f51470c4c/numpy-2.4.2-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6d82351358ffbcdcd7b686b90742a9b86632d6c1c051016484fa0b326a0a1548", size = 15663067, upload-time = "2026-01-31T23:11:01.291Z" }, + { url = "https://files.pythonhosted.org/packages/f5/c6/a18e59f3f0b8071cc85cbc8d80cd02d68aa9710170b2553a117203d46936/numpy-2.4.2-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9e35d3e0144137d9fdae62912e869136164534d64a169f86438bc9561b6ad49f", size = 16619782, upload-time = "2026-01-31T23:11:03.669Z" }, + { url = "https://files.pythonhosted.org/packages/b7/83/9751502164601a79e18847309f5ceec0b1446d7b6aa12305759b72cf98b2/numpy-2.4.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:adb6ed2ad29b9e15321d167d152ee909ec73395901b70936f029c3bc6d7f4460", size = 17013128, upload-time = "2026-01-31T23:11:05.913Z" }, + { url = "https://files.pythonhosted.org/packages/61/c4/c4066322256ec740acc1c8923a10047818691d2f8aec254798f3dd90f5f2/numpy-2.4.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:8906e71fd8afcb76580404e2a950caef2685df3d2a57fe82a86ac8d33cc007ba", size = 18345324, upload-time = "2026-01-31T23:11:08.248Z" }, + { url = "https://files.pythonhosted.org/packages/ab/af/6157aa6da728fa4525a755bfad486ae7e3f76d4c1864138003eb84328497/numpy-2.4.2-cp312-cp312-win32.whl", hash = "sha256:ec055f6dae239a6299cace477b479cca2fc125c5675482daf1dd886933a1076f", size = 5960282, upload-time = "2026-01-31T23:11:10.497Z" }, + { url = "https://files.pythonhosted.org/packages/92/0f/7ceaaeaacb40567071e94dbf2c9480c0ae453d5bb4f52bea3892c39dc83c/numpy-2.4.2-cp312-cp312-win_amd64.whl", hash = "sha256:209fae046e62d0ce6435fcfe3b1a10537e858249b3d9b05829e2a05218296a85", size = 12314210, upload-time = "2026-01-31T23:11:12.176Z" }, + { url = "https://files.pythonhosted.org/packages/2f/a3/56c5c604fae6dd40fa2ed3040d005fca97e91bd320d232ac9931d77ba13c/numpy-2.4.2-cp312-cp312-win_arm64.whl", hash = "sha256:fbde1b0c6e81d56f5dccd95dd4a711d9b95df1ae4009a60887e56b27e8d903fa", size = 10220171, upload-time = "2026-01-31T23:11:14.684Z" }, + { url = "https://files.pythonhosted.org/packages/a1/22/815b9fe25d1d7ae7d492152adbc7226d3eff731dffc38fe970589fcaaa38/numpy-2.4.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:25f2059807faea4b077a2b6837391b5d830864b3543627f381821c646f31a63c", size = 16663696, upload-time = "2026-01-31T23:11:17.516Z" }, + { url = "https://files.pythonhosted.org/packages/09/f0/817d03a03f93ba9c6c8993de509277d84e69f9453601915e4a69554102a1/numpy-2.4.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:bd3a7a9f5847d2fb8c2c6d1c862fa109c31a9abeca1a3c2bd5a64572955b2979", size = 14688322, upload-time = "2026-01-31T23:11:19.883Z" }, + { url = "https://files.pythonhosted.org/packages/da/b4/f805ab79293c728b9a99438775ce51885fd4f31b76178767cfc718701a39/numpy-2.4.2-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:8e4549f8a3c6d13d55041925e912bfd834285ef1dd64d6bc7d542583355e2e98", size = 5198157, upload-time = "2026-01-31T23:11:22.375Z" }, + { url = "https://files.pythonhosted.org/packages/74/09/826e4289844eccdcd64aac27d13b0fd3f32039915dd5b9ba01baae1f436c/numpy-2.4.2-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:aea4f66ff44dfddf8c2cffd66ba6538c5ec67d389285292fe428cb2c738c8aef", size = 6546330, upload-time = "2026-01-31T23:11:23.958Z" }, + { url = "https://files.pythonhosted.org/packages/19/fb/cbfdbfa3057a10aea5422c558ac57538e6acc87ec1669e666d32ac198da7/numpy-2.4.2-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c3cd545784805de05aafe1dde61752ea49a359ccba9760c1e5d1c88a93bbf2b7", size = 15660968, upload-time = "2026-01-31T23:11:25.713Z" }, + { url = "https://files.pythonhosted.org/packages/04/dc/46066ce18d01645541f0186877377b9371b8fa8017fa8262002b4ef22612/numpy-2.4.2-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d0d9b7c93578baafcbc5f0b83eaf17b79d345c6f36917ba0c67f45226911d499", size = 16607311, upload-time = "2026-01-31T23:11:28.117Z" }, + { url = "https://files.pythonhosted.org/packages/14/d9/4b5adfc39a43fa6bf918c6d544bc60c05236cc2f6339847fc5b35e6cb5b0/numpy-2.4.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f74f0f7779cc7ae07d1810aab8ac6b1464c3eafb9e283a40da7309d5e6e48fbb", size = 17012850, upload-time = "2026-01-31T23:11:30.888Z" }, + { url = "https://files.pythonhosted.org/packages/b7/20/adb6e6adde6d0130046e6fdfb7675cc62bc2f6b7b02239a09eb58435753d/numpy-2.4.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:c7ac672d699bf36275c035e16b65539931347d68b70667d28984c9fb34e07fa7", size = 18334210, upload-time = "2026-01-31T23:11:33.214Z" }, + { url = "https://files.pythonhosted.org/packages/78/0e/0a73b3dff26803a8c02baa76398015ea2a5434d9b8265a7898a6028c1591/numpy-2.4.2-cp313-cp313-win32.whl", hash = "sha256:8e9afaeb0beff068b4d9cd20d322ba0ee1cecfb0b08db145e4ab4dd44a6b5110", size = 5958199, upload-time = "2026-01-31T23:11:35.385Z" }, + { url = "https://files.pythonhosted.org/packages/43/bc/6352f343522fcb2c04dbaf94cb30cca6fd32c1a750c06ad6231b4293708c/numpy-2.4.2-cp313-cp313-win_amd64.whl", hash = "sha256:7df2de1e4fba69a51c06c28f5a3de36731eb9639feb8e1cf7e4a7b0daf4cf622", size = 12310848, upload-time = "2026-01-31T23:11:38.001Z" }, + { url = "https://files.pythonhosted.org/packages/6e/8d/6da186483e308da5da1cc6918ce913dcfe14ffde98e710bfeff2a6158d4e/numpy-2.4.2-cp313-cp313-win_arm64.whl", hash = "sha256:0fece1d1f0a89c16b03442eae5c56dc0be0c7883b5d388e0c03f53019a4bfd71", size = 10221082, upload-time = "2026-01-31T23:11:40.392Z" }, + { url = "https://files.pythonhosted.org/packages/25/a1/9510aa43555b44781968935c7548a8926274f815de42ad3997e9e83680dd/numpy-2.4.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:5633c0da313330fd20c484c78cdd3f9b175b55e1a766c4a174230c6b70ad8262", size = 14815866, upload-time = "2026-01-31T23:11:42.495Z" }, + { url = "https://files.pythonhosted.org/packages/36/30/6bbb5e76631a5ae46e7923dd16ca9d3f1c93cfa8d4ed79a129814a9d8db3/numpy-2.4.2-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:d9f64d786b3b1dd742c946c42d15b07497ed14af1a1f3ce840cce27daa0ce913", size = 5325631, upload-time = "2026-01-31T23:11:44.7Z" }, + { url = "https://files.pythonhosted.org/packages/46/00/3a490938800c1923b567b3a15cd17896e68052e2145d8662aaf3e1ffc58f/numpy-2.4.2-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:b21041e8cb6a1eb5312dd1d2f80a94d91efffb7a06b70597d44f1bd2dfc315ab", size = 6646254, upload-time = "2026-01-31T23:11:46.341Z" }, + { url = "https://files.pythonhosted.org/packages/d3/e9/fac0890149898a9b609caa5af7455a948b544746e4b8fe7c212c8edd71f8/numpy-2.4.2-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:00ab83c56211a1d7c07c25e3217ea6695e50a3e2f255053686b081dc0b091a82", size = 15720138, upload-time = "2026-01-31T23:11:48.082Z" }, + { url = "https://files.pythonhosted.org/packages/ea/5c/08887c54e68e1e28df53709f1893ce92932cc6f01f7c3d4dc952f61ffd4e/numpy-2.4.2-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2fb882da679409066b4603579619341c6d6898fc83a8995199d5249f986e8e8f", size = 16655398, upload-time = "2026-01-31T23:11:50.293Z" }, + { url = "https://files.pythonhosted.org/packages/4d/89/253db0fa0e66e9129c745e4ef25631dc37d5f1314dad2b53e907b8538e6d/numpy-2.4.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:66cb9422236317f9d44b67b4d18f44efe6e9c7f8794ac0462978513359461554", size = 17079064, upload-time = "2026-01-31T23:11:52.927Z" }, + { url = "https://files.pythonhosted.org/packages/2a/d5/cbade46ce97c59c6c3da525e8d95b7abe8a42974a1dc5c1d489c10433e88/numpy-2.4.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:0f01dcf33e73d80bd8dc0f20a71303abbafa26a19e23f6b68d1aa9990af90257", size = 18379680, upload-time = "2026-01-31T23:11:55.22Z" }, + { url = "https://files.pythonhosted.org/packages/40/62/48f99ae172a4b63d981babe683685030e8a3df4f246c893ea5c6ef99f018/numpy-2.4.2-cp313-cp313t-win32.whl", hash = "sha256:52b913ec40ff7ae845687b0b34d8d93b60cb66dcee06996dd5c99f2fc9328657", size = 6082433, upload-time = "2026-01-31T23:11:58.096Z" }, + { url = "https://files.pythonhosted.org/packages/07/38/e054a61cfe48ad9f1ed0d188e78b7e26859d0b60ef21cd9de4897cdb5326/numpy-2.4.2-cp313-cp313t-win_amd64.whl", hash = "sha256:5eea80d908b2c1f91486eb95b3fb6fab187e569ec9752ab7d9333d2e66bf2d6b", size = 12451181, upload-time = "2026-01-31T23:11:59.782Z" }, + { url = "https://files.pythonhosted.org/packages/6e/a4/a05c3a6418575e185dd84d0b9680b6bb2e2dc3e4202f036b7b4e22d6e9dc/numpy-2.4.2-cp313-cp313t-win_arm64.whl", hash = "sha256:fd49860271d52127d61197bb50b64f58454e9f578cb4b2c001a6de8b1f50b0b1", size = 10290756, upload-time = "2026-01-31T23:12:02.438Z" }, + { url = "https://files.pythonhosted.org/packages/18/88/b7df6050bf18fdcfb7046286c6535cabbdd2064a3440fca3f069d319c16e/numpy-2.4.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:444be170853f1f9d528428eceb55f12918e4fda5d8805480f36a002f1415e09b", size = 16663092, upload-time = "2026-01-31T23:12:04.521Z" }, + { url = "https://files.pythonhosted.org/packages/25/7a/1fee4329abc705a469a4afe6e69b1ef7e915117747886327104a8493a955/numpy-2.4.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:d1240d50adff70c2a88217698ca844723068533f3f5c5fa6ee2e3220e3bdb000", size = 14698770, upload-time = "2026-01-31T23:12:06.96Z" }, + { url = "https://files.pythonhosted.org/packages/fb/0b/f9e49ba6c923678ad5bc38181c08ac5e53b7a5754dbca8e581aa1a56b1ff/numpy-2.4.2-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:7cdde6de52fb6664b00b056341265441192d1291c130e99183ec0d4b110ff8b1", size = 5208562, upload-time = "2026-01-31T23:12:09.632Z" }, + { url = "https://files.pythonhosted.org/packages/7d/12/d7de8f6f53f9bb76997e5e4c069eda2051e3fe134e9181671c4391677bb2/numpy-2.4.2-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:cda077c2e5b780200b6b3e09d0b42205a3d1c68f30c6dceb90401c13bff8fe74", size = 6543710, upload-time = "2026-01-31T23:12:11.969Z" }, + { url = "https://files.pythonhosted.org/packages/09/63/c66418c2e0268a31a4cf8a8b512685748200f8e8e8ec6c507ce14e773529/numpy-2.4.2-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d30291931c915b2ab5717c2974bb95ee891a1cf22ebc16a8006bd59cd210d40a", size = 15677205, upload-time = "2026-01-31T23:12:14.33Z" }, + { url = "https://files.pythonhosted.org/packages/5d/6c/7f237821c9642fb2a04d2f1e88b4295677144ca93285fd76eff3bcba858d/numpy-2.4.2-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bba37bc29d4d85761deed3954a1bc62be7cf462b9510b51d367b769a8c8df325", size = 16611738, upload-time = "2026-01-31T23:12:16.525Z" }, + { url = "https://files.pythonhosted.org/packages/c2/a7/39c4cdda9f019b609b5c473899d87abff092fc908cfe4d1ecb2fcff453b0/numpy-2.4.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:b2f0073ed0868db1dcd86e052d37279eef185b9c8db5bf61f30f46adac63c909", size = 17028888, upload-time = "2026-01-31T23:12:19.306Z" }, + { url = "https://files.pythonhosted.org/packages/da/b3/e84bb64bdfea967cc10950d71090ec2d84b49bc691df0025dddb7c26e8e3/numpy-2.4.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:7f54844851cdb630ceb623dcec4db3240d1ac13d4990532446761baede94996a", size = 18339556, upload-time = "2026-01-31T23:12:21.816Z" }, + { url = "https://files.pythonhosted.org/packages/88/f5/954a291bc1192a27081706862ac62bb5920fbecfbaa302f64682aa90beed/numpy-2.4.2-cp314-cp314-win32.whl", hash = "sha256:12e26134a0331d8dbd9351620f037ec470b7c75929cb8a1537f6bfe411152a1a", size = 6006899, upload-time = "2026-01-31T23:12:24.14Z" }, + { url = "https://files.pythonhosted.org/packages/05/cb/eff72a91b2efdd1bc98b3b8759f6a1654aa87612fc86e3d87d6fe4f948c4/numpy-2.4.2-cp314-cp314-win_amd64.whl", hash = "sha256:068cdb2d0d644cdb45670810894f6a0600797a69c05f1ac478e8d31670b8ee75", size = 12443072, upload-time = "2026-01-31T23:12:26.33Z" }, + { url = "https://files.pythonhosted.org/packages/37/75/62726948db36a56428fce4ba80a115716dc4fad6a3a4352487f8bb950966/numpy-2.4.2-cp314-cp314-win_arm64.whl", hash = "sha256:6ed0be1ee58eef41231a5c943d7d1375f093142702d5723ca2eb07db9b934b05", size = 10494886, upload-time = "2026-01-31T23:12:28.488Z" }, + { url = "https://files.pythonhosted.org/packages/36/2f/ee93744f1e0661dc267e4b21940870cabfae187c092e1433b77b09b50ac4/numpy-2.4.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:98f16a80e917003a12c0580f97b5f875853ebc33e2eaa4bccfc8201ac6869308", size = 14818567, upload-time = "2026-01-31T23:12:30.709Z" }, + { url = "https://files.pythonhosted.org/packages/a7/24/6535212add7d76ff938d8bdc654f53f88d35cddedf807a599e180dcb8e66/numpy-2.4.2-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:20abd069b9cda45874498b245c8015b18ace6de8546bf50dfa8cea1696ed06ef", size = 5328372, upload-time = "2026-01-31T23:12:32.962Z" }, + { url = "https://files.pythonhosted.org/packages/5e/9d/c48f0a035725f925634bf6b8994253b43f2047f6778a54147d7e213bc5a7/numpy-2.4.2-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:e98c97502435b53741540a5717a6749ac2ada901056c7db951d33e11c885cc7d", size = 6649306, upload-time = "2026-01-31T23:12:34.797Z" }, + { url = "https://files.pythonhosted.org/packages/81/05/7c73a9574cd4a53a25907bad38b59ac83919c0ddc8234ec157f344d57d9a/numpy-2.4.2-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:da6cad4e82cb893db4b69105c604d805e0c3ce11501a55b5e9f9083b47d2ffe8", size = 15722394, upload-time = "2026-01-31T23:12:36.565Z" }, + { url = "https://files.pythonhosted.org/packages/35/fa/4de10089f21fc7d18442c4a767ab156b25c2a6eaf187c0db6d9ecdaeb43f/numpy-2.4.2-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9e4424677ce4b47fe73c8b5556d876571f7c6945d264201180db2dc34f676ab5", size = 16653343, upload-time = "2026-01-31T23:12:39.188Z" }, + { url = "https://files.pythonhosted.org/packages/b8/f9/d33e4ffc857f3763a57aa85650f2e82486832d7492280ac21ba9efda80da/numpy-2.4.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:2b8f157c8a6f20eb657e240f8985cc135598b2b46985c5bccbde7616dc9c6b1e", size = 17078045, upload-time = "2026-01-31T23:12:42.041Z" }, + { url = "https://files.pythonhosted.org/packages/c8/b8/54bdb43b6225badbea6389fa038c4ef868c44f5890f95dd530a218706da3/numpy-2.4.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5daf6f3914a733336dab21a05cdec343144600e964d2fcdabaac0c0269874b2a", size = 18380024, upload-time = "2026-01-31T23:12:44.331Z" }, + { url = "https://files.pythonhosted.org/packages/a5/55/6e1a61ded7af8df04016d81b5b02daa59f2ea9252ee0397cb9f631efe9e5/numpy-2.4.2-cp314-cp314t-win32.whl", hash = "sha256:8c50dd1fc8826f5b26a5ee4d77ca55d88a895f4e4819c7ecc2a9f5905047a443", size = 6153937, upload-time = "2026-01-31T23:12:47.229Z" }, + { url = "https://files.pythonhosted.org/packages/45/aa/fa6118d1ed6d776b0983f3ceac9b1a5558e80df9365b1c3aa6d42bf9eee4/numpy-2.4.2-cp314-cp314t-win_amd64.whl", hash = "sha256:fcf92bee92742edd401ba41135185866f7026c502617f422eb432cfeca4fe236", size = 12631844, upload-time = "2026-01-31T23:12:48.997Z" }, + { url = "https://files.pythonhosted.org/packages/32/0a/2ec5deea6dcd158f254a7b372fb09cfba5719419c8d66343bab35237b3fb/numpy-2.4.2-cp314-cp314t-win_arm64.whl", hash = "sha256:1f92f53998a17265194018d1cc321b2e96e900ca52d54c7c77837b71b9465181", size = 10565379, upload-time = "2026-01-31T23:12:51.345Z" }, +] + +[[package]] +name = "oauthlib" +version = "3.3.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0b/5f/19930f824ffeb0ad4372da4812c50edbd1434f678c90c2733e1188edfc63/oauthlib-3.3.1.tar.gz", hash = "sha256:0f0f8aa759826a193cf66c12ea1af1637f87b9b4622d46e866952bb022e538c9", size = 185918, upload-time = "2025-06-19T22:48:08.269Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/be/9c/92789c596b8df838baa98fa71844d84283302f7604ed565dafe5a6b5041a/oauthlib-3.3.1-py3-none-any.whl", hash = "sha256:88119c938d2b8fb88561af5f6ee0eec8cc8d552b7bb1f712743136eb7523b7a1", size = 160065, upload-time = "2025-06-19T22:48:06.508Z" }, +] + +[[package]] +name = "packaging" +version = "26.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/65/ee/299d360cdc32edc7d2cf530f3accf79c4fca01e96ffc950d8a52213bd8e4/packaging-26.0.tar.gz", hash = "sha256:00243ae351a257117b6a241061796684b084ed1c516a08c48a3f7e147a9d80b4", size = 143416, upload-time = "2026-01-21T20:50:39.064Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/b9/c538f279a4e237a006a2c98387d081e9eb060d203d8ed34467cc0f0b9b53/packaging-26.0-py3-none-any.whl", hash = "sha256:b36f1fef9334a5588b4166f8bcd26a14e521f2b55e6b9de3aaa80d3ff7a37529", size = 74366, upload-time = "2026-01-21T20:50:37.788Z" }, +] + +[[package]] +name = "pathspec" +version = "1.0.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fa/36/e27608899f9b8d4dff0617b2d9ab17ca5608956ca44461ac14ac48b44015/pathspec-1.0.4.tar.gz", hash = "sha256:0210e2ae8a21a9137c0d470578cb0e595af87edaa6ebf12ff176f14a02e0e645", size = 131200, upload-time = "2026-01-27T03:59:46.938Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ef/3c/2c197d226f9ea224a9ab8d197933f9da0ae0aac5b6e0f884e2b8d9c8e9f7/pathspec-1.0.4-py3-none-any.whl", hash = "sha256:fb6ae2fd4e7c921a165808a552060e722767cfa526f99ca5156ed2ce45a5c723", size = 55206, upload-time = "2026-01-27T03:59:45.137Z" }, +] + +[[package]] +name = "platformdirs" +version = "4.9.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1b/04/fea538adf7dbbd6d186f551d595961e564a3b6715bdf276b477460858672/platformdirs-4.9.2.tar.gz", hash = "sha256:9a33809944b9db043ad67ca0db94b14bf452cc6aeaac46a88ea55b26e2e9d291", size = 28394, upload-time = "2026-02-16T03:56:10.574Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/48/31/05e764397056194206169869b50cf2fee4dbbbc71b344705b9c0d878d4d8/platformdirs-4.9.2-py3-none-any.whl", hash = "sha256:9170634f126f8efdae22fb58ae8a0eaa86f38365bc57897a6c4f781d1f5875bd", size = 21168, upload-time = "2026-02-16T03:56:08.891Z" }, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + +[[package]] +name = "pycodestyle" +version = "2.14.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/11/e0/abfd2a0d2efe47670df87f3e3a0e2edda42f055053c85361f19c0e2c1ca8/pycodestyle-2.14.0.tar.gz", hash = "sha256:c4b5b517d278089ff9d0abdec919cd97262a3367449ea1c8b49b91529167b783", size = 39472, upload-time = "2025-06-20T18:49:48.75Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d7/27/a58ddaf8c588a3ef080db9d0b7e0b97215cee3a45df74f3a94dbbf5c893a/pycodestyle-2.14.0-py2.py3-none-any.whl", hash = "sha256:dd6bf7cb4ee77f8e016f9c8e74a35ddd9f67e1d5fd4184d86c3b98e07099f42d", size = 31594, upload-time = "2025-06-20T18:49:47.491Z" }, +] + +[[package]] +name = "pydantic" +version = "2.12.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-types" }, + { name = "pydantic-core" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/69/44/36f1a6e523abc58ae5f928898e4aca2e0ea509b5aa6f6f392a5d882be928/pydantic-2.12.5.tar.gz", hash = "sha256:4d351024c75c0f085a9febbb665ce8c0c6ec5d30e903bdb6394b7ede26aebb49", size = 821591, upload-time = "2025-11-26T15:11:46.471Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5a/87/b70ad306ebb6f9b585f114d0ac2137d792b48be34d732d60e597c2f8465a/pydantic-2.12.5-py3-none-any.whl", hash = "sha256:e561593fccf61e8a20fc46dfc2dfe075b8be7d0188df33f221ad1f0139180f9d", size = 463580, upload-time = "2025-11-26T15:11:44.605Z" }, +] + +[[package]] +name = "pydantic-core" +version = "2.41.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/71/70/23b021c950c2addd24ec408e9ab05d59b035b39d97cdc1130e1bce647bb6/pydantic_core-2.41.5.tar.gz", hash = "sha256:08daa51ea16ad373ffd5e7606252cc32f07bc72b28284b6bc9c6df804816476e", size = 460952, upload-time = "2025-11-04T13:43:49.098Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5f/5d/5f6c63eebb5afee93bcaae4ce9a898f3373ca23df3ccaef086d0233a35a7/pydantic_core-2.41.5-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:f41a7489d32336dbf2199c8c0a215390a751c5b014c2c1c5366e817202e9cdf7", size = 2110990, upload-time = "2025-11-04T13:39:58.079Z" }, + { url = "https://files.pythonhosted.org/packages/aa/32/9c2e8ccb57c01111e0fd091f236c7b371c1bccea0fa85247ac55b1e2b6b6/pydantic_core-2.41.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:070259a8818988b9a84a449a2a7337c7f430a22acc0859c6b110aa7212a6d9c0", size = 1896003, upload-time = "2025-11-04T13:39:59.956Z" }, + { url = "https://files.pythonhosted.org/packages/68/b8/a01b53cb0e59139fbc9e4fda3e9724ede8de279097179be4ff31f1abb65a/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e96cea19e34778f8d59fe40775a7a574d95816eb150850a85a7a4c8f4b94ac69", size = 1919200, upload-time = "2025-11-04T13:40:02.241Z" }, + { url = "https://files.pythonhosted.org/packages/38/de/8c36b5198a29bdaade07b5985e80a233a5ac27137846f3bc2d3b40a47360/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ed2e99c456e3fadd05c991f8f437ef902e00eedf34320ba2b0842bd1c3ca3a75", size = 2052578, upload-time = "2025-11-04T13:40:04.401Z" }, + { url = "https://files.pythonhosted.org/packages/00/b5/0e8e4b5b081eac6cb3dbb7e60a65907549a1ce035a724368c330112adfdd/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:65840751b72fbfd82c3c640cff9284545342a4f1eb1586ad0636955b261b0b05", size = 2208504, upload-time = "2025-11-04T13:40:06.072Z" }, + { url = "https://files.pythonhosted.org/packages/77/56/87a61aad59c7c5b9dc8caad5a41a5545cba3810c3e828708b3d7404f6cef/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e536c98a7626a98feb2d3eaf75944ef6f3dbee447e1f841eae16f2f0a72d8ddc", size = 2335816, upload-time = "2025-11-04T13:40:07.835Z" }, + { url = "https://files.pythonhosted.org/packages/0d/76/941cc9f73529988688a665a5c0ecff1112b3d95ab48f81db5f7606f522d3/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eceb81a8d74f9267ef4081e246ffd6d129da5d87e37a77c9bde550cb04870c1c", size = 2075366, upload-time = "2025-11-04T13:40:09.804Z" }, + { url = "https://files.pythonhosted.org/packages/d3/43/ebef01f69baa07a482844faaa0a591bad1ef129253ffd0cdaa9d8a7f72d3/pydantic_core-2.41.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d38548150c39b74aeeb0ce8ee1d8e82696f4a4e16ddc6de7b1d8823f7de4b9b5", size = 2171698, upload-time = "2025-11-04T13:40:12.004Z" }, + { url = "https://files.pythonhosted.org/packages/b1/87/41f3202e4193e3bacfc2c065fab7706ebe81af46a83d3e27605029c1f5a6/pydantic_core-2.41.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:c23e27686783f60290e36827f9c626e63154b82b116d7fe9adba1fda36da706c", size = 2132603, upload-time = "2025-11-04T13:40:13.868Z" }, + { url = "https://files.pythonhosted.org/packages/49/7d/4c00df99cb12070b6bccdef4a195255e6020a550d572768d92cc54dba91a/pydantic_core-2.41.5-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:482c982f814460eabe1d3bb0adfdc583387bd4691ef00b90575ca0d2b6fe2294", size = 2329591, upload-time = "2025-11-04T13:40:15.672Z" }, + { url = "https://files.pythonhosted.org/packages/cc/6a/ebf4b1d65d458f3cda6a7335d141305dfa19bdc61140a884d165a8a1bbc7/pydantic_core-2.41.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:bfea2a5f0b4d8d43adf9d7b8bf019fb46fdd10a2e5cde477fbcb9d1fa08c68e1", size = 2319068, upload-time = "2025-11-04T13:40:17.532Z" }, + { url = "https://files.pythonhosted.org/packages/49/3b/774f2b5cd4192d5ab75870ce4381fd89cf218af999515baf07e7206753f0/pydantic_core-2.41.5-cp312-cp312-win32.whl", hash = "sha256:b74557b16e390ec12dca509bce9264c3bbd128f8a2c376eaa68003d7f327276d", size = 1985908, upload-time = "2025-11-04T13:40:19.309Z" }, + { url = "https://files.pythonhosted.org/packages/86/45/00173a033c801cacf67c190fef088789394feaf88a98a7035b0e40d53dc9/pydantic_core-2.41.5-cp312-cp312-win_amd64.whl", hash = "sha256:1962293292865bca8e54702b08a4f26da73adc83dd1fcf26fbc875b35d81c815", size = 2020145, upload-time = "2025-11-04T13:40:21.548Z" }, + { url = "https://files.pythonhosted.org/packages/f9/22/91fbc821fa6d261b376a3f73809f907cec5ca6025642c463d3488aad22fb/pydantic_core-2.41.5-cp312-cp312-win_arm64.whl", hash = "sha256:1746d4a3d9a794cacae06a5eaaccb4b8643a131d45fbc9af23e353dc0a5ba5c3", size = 1976179, upload-time = "2025-11-04T13:40:23.393Z" }, + { url = "https://files.pythonhosted.org/packages/87/06/8806241ff1f70d9939f9af039c6c35f2360cf16e93c2ca76f184e76b1564/pydantic_core-2.41.5-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:941103c9be18ac8daf7b7adca8228f8ed6bb7a1849020f643b3a14d15b1924d9", size = 2120403, upload-time = "2025-11-04T13:40:25.248Z" }, + { url = "https://files.pythonhosted.org/packages/94/02/abfa0e0bda67faa65fef1c84971c7e45928e108fe24333c81f3bfe35d5f5/pydantic_core-2.41.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:112e305c3314f40c93998e567879e887a3160bb8689ef3d2c04b6cc62c33ac34", size = 1896206, upload-time = "2025-11-04T13:40:27.099Z" }, + { url = "https://files.pythonhosted.org/packages/15/df/a4c740c0943e93e6500f9eb23f4ca7ec9bf71b19e608ae5b579678c8d02f/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0cbaad15cb0c90aa221d43c00e77bb33c93e8d36e0bf74760cd00e732d10a6a0", size = 1919307, upload-time = "2025-11-04T13:40:29.806Z" }, + { url = "https://files.pythonhosted.org/packages/9a/e3/6324802931ae1d123528988e0e86587c2072ac2e5394b4bc2bc34b61ff6e/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:03ca43e12fab6023fc79d28ca6b39b05f794ad08ec2feccc59a339b02f2b3d33", size = 2063258, upload-time = "2025-11-04T13:40:33.544Z" }, + { url = "https://files.pythonhosted.org/packages/c9/d4/2230d7151d4957dd79c3044ea26346c148c98fbf0ee6ebd41056f2d62ab5/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:dc799088c08fa04e43144b164feb0c13f9a0bc40503f8df3e9fde58a3c0c101e", size = 2214917, upload-time = "2025-11-04T13:40:35.479Z" }, + { url = "https://files.pythonhosted.org/packages/e6/9f/eaac5df17a3672fef0081b6c1bb0b82b33ee89aa5cec0d7b05f52fd4a1fa/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:97aeba56665b4c3235a0e52b2c2f5ae9cd071b8a8310ad27bddb3f7fb30e9aa2", size = 2332186, upload-time = "2025-11-04T13:40:37.436Z" }, + { url = "https://files.pythonhosted.org/packages/cf/4e/35a80cae583a37cf15604b44240e45c05e04e86f9cfd766623149297e971/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:406bf18d345822d6c21366031003612b9c77b3e29ffdb0f612367352aab7d586", size = 2073164, upload-time = "2025-11-04T13:40:40.289Z" }, + { url = "https://files.pythonhosted.org/packages/bf/e3/f6e262673c6140dd3305d144d032f7bd5f7497d3871c1428521f19f9efa2/pydantic_core-2.41.5-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b93590ae81f7010dbe380cdeab6f515902ebcbefe0b9327cc4804d74e93ae69d", size = 2179146, upload-time = "2025-11-04T13:40:42.809Z" }, + { url = "https://files.pythonhosted.org/packages/75/c7/20bd7fc05f0c6ea2056a4565c6f36f8968c0924f19b7d97bbfea55780e73/pydantic_core-2.41.5-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:01a3d0ab748ee531f4ea6c3e48ad9dac84ddba4b0d82291f87248f2f9de8d740", size = 2137788, upload-time = "2025-11-04T13:40:44.752Z" }, + { url = "https://files.pythonhosted.org/packages/3a/8d/34318ef985c45196e004bc46c6eab2eda437e744c124ef0dbe1ff2c9d06b/pydantic_core-2.41.5-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:6561e94ba9dacc9c61bce40e2d6bdc3bfaa0259d3ff36ace3b1e6901936d2e3e", size = 2340133, upload-time = "2025-11-04T13:40:46.66Z" }, + { url = "https://files.pythonhosted.org/packages/9c/59/013626bf8c78a5a5d9350d12e7697d3d4de951a75565496abd40ccd46bee/pydantic_core-2.41.5-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:915c3d10f81bec3a74fbd4faebe8391013ba61e5a1a8d48c4455b923bdda7858", size = 2324852, upload-time = "2025-11-04T13:40:48.575Z" }, + { url = "https://files.pythonhosted.org/packages/1a/d9/c248c103856f807ef70c18a4f986693a46a8ffe1602e5d361485da502d20/pydantic_core-2.41.5-cp313-cp313-win32.whl", hash = "sha256:650ae77860b45cfa6e2cdafc42618ceafab3a2d9a3811fcfbd3bbf8ac3c40d36", size = 1994679, upload-time = "2025-11-04T13:40:50.619Z" }, + { url = "https://files.pythonhosted.org/packages/9e/8b/341991b158ddab181cff136acd2552c9f35bd30380422a639c0671e99a91/pydantic_core-2.41.5-cp313-cp313-win_amd64.whl", hash = "sha256:79ec52ec461e99e13791ec6508c722742ad745571f234ea6255bed38c6480f11", size = 2019766, upload-time = "2025-11-04T13:40:52.631Z" }, + { url = "https://files.pythonhosted.org/packages/73/7d/f2f9db34af103bea3e09735bb40b021788a5e834c81eedb541991badf8f5/pydantic_core-2.41.5-cp313-cp313-win_arm64.whl", hash = "sha256:3f84d5c1b4ab906093bdc1ff10484838aca54ef08de4afa9de0f5f14d69639cd", size = 1981005, upload-time = "2025-11-04T13:40:54.734Z" }, + { url = "https://files.pythonhosted.org/packages/ea/28/46b7c5c9635ae96ea0fbb779e271a38129df2550f763937659ee6c5dbc65/pydantic_core-2.41.5-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:3f37a19d7ebcdd20b96485056ba9e8b304e27d9904d233d7b1015db320e51f0a", size = 2119622, upload-time = "2025-11-04T13:40:56.68Z" }, + { url = "https://files.pythonhosted.org/packages/74/1a/145646e5687e8d9a1e8d09acb278c8535ebe9e972e1f162ed338a622f193/pydantic_core-2.41.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1d1d9764366c73f996edd17abb6d9d7649a7eb690006ab6adbda117717099b14", size = 1891725, upload-time = "2025-11-04T13:40:58.807Z" }, + { url = "https://files.pythonhosted.org/packages/23/04/e89c29e267b8060b40dca97bfc64a19b2a3cf99018167ea1677d96368273/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:25e1c2af0fce638d5f1988b686f3b3ea8cd7de5f244ca147c777769e798a9cd1", size = 1915040, upload-time = "2025-11-04T13:41:00.853Z" }, + { url = "https://files.pythonhosted.org/packages/84/a3/15a82ac7bd97992a82257f777b3583d3e84bdb06ba6858f745daa2ec8a85/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:506d766a8727beef16b7adaeb8ee6217c64fc813646b424d0804d67c16eddb66", size = 2063691, upload-time = "2025-11-04T13:41:03.504Z" }, + { url = "https://files.pythonhosted.org/packages/74/9b/0046701313c6ef08c0c1cf0e028c67c770a4e1275ca73131563c5f2a310a/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4819fa52133c9aa3c387b3328f25c1facc356491e6135b459f1de698ff64d869", size = 2213897, upload-time = "2025-11-04T13:41:05.804Z" }, + { url = "https://files.pythonhosted.org/packages/8a/cd/6bac76ecd1b27e75a95ca3a9a559c643b3afcd2dd62086d4b7a32a18b169/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2b761d210c9ea91feda40d25b4efe82a1707da2ef62901466a42492c028553a2", size = 2333302, upload-time = "2025-11-04T13:41:07.809Z" }, + { url = "https://files.pythonhosted.org/packages/4c/d2/ef2074dc020dd6e109611a8be4449b98cd25e1b9b8a303c2f0fca2f2bcf7/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:22f0fb8c1c583a3b6f24df2470833b40207e907b90c928cc8d3594b76f874375", size = 2064877, upload-time = "2025-11-04T13:41:09.827Z" }, + { url = "https://files.pythonhosted.org/packages/18/66/e9db17a9a763d72f03de903883c057b2592c09509ccfe468187f2a2eef29/pydantic_core-2.41.5-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2782c870e99878c634505236d81e5443092fba820f0373997ff75f90f68cd553", size = 2180680, upload-time = "2025-11-04T13:41:12.379Z" }, + { url = "https://files.pythonhosted.org/packages/d3/9e/3ce66cebb929f3ced22be85d4c2399b8e85b622db77dad36b73c5387f8f8/pydantic_core-2.41.5-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:0177272f88ab8312479336e1d777f6b124537d47f2123f89cb37e0accea97f90", size = 2138960, upload-time = "2025-11-04T13:41:14.627Z" }, + { url = "https://files.pythonhosted.org/packages/a6/62/205a998f4327d2079326b01abee48e502ea739d174f0a89295c481a2272e/pydantic_core-2.41.5-cp314-cp314-musllinux_1_1_armv7l.whl", hash = "sha256:63510af5e38f8955b8ee5687740d6ebf7c2a0886d15a6d65c32814613681bc07", size = 2339102, upload-time = "2025-11-04T13:41:16.868Z" }, + { url = "https://files.pythonhosted.org/packages/3c/0d/f05e79471e889d74d3d88f5bd20d0ed189ad94c2423d81ff8d0000aab4ff/pydantic_core-2.41.5-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:e56ba91f47764cc14f1daacd723e3e82d1a89d783f0f5afe9c364b8bb491ccdb", size = 2326039, upload-time = "2025-11-04T13:41:18.934Z" }, + { url = "https://files.pythonhosted.org/packages/ec/e1/e08a6208bb100da7e0c4b288eed624a703f4d129bde2da475721a80cab32/pydantic_core-2.41.5-cp314-cp314-win32.whl", hash = "sha256:aec5cf2fd867b4ff45b9959f8b20ea3993fc93e63c7363fe6851424c8a7e7c23", size = 1995126, upload-time = "2025-11-04T13:41:21.418Z" }, + { url = "https://files.pythonhosted.org/packages/48/5d/56ba7b24e9557f99c9237e29f5c09913c81eeb2f3217e40e922353668092/pydantic_core-2.41.5-cp314-cp314-win_amd64.whl", hash = "sha256:8e7c86f27c585ef37c35e56a96363ab8de4e549a95512445b85c96d3e2f7c1bf", size = 2015489, upload-time = "2025-11-04T13:41:24.076Z" }, + { url = "https://files.pythonhosted.org/packages/4e/bb/f7a190991ec9e3e0ba22e4993d8755bbc4a32925c0b5b42775c03e8148f9/pydantic_core-2.41.5-cp314-cp314-win_arm64.whl", hash = "sha256:e672ba74fbc2dc8eea59fb6d4aed6845e6905fc2a8afe93175d94a83ba2a01a0", size = 1977288, upload-time = "2025-11-04T13:41:26.33Z" }, + { url = "https://files.pythonhosted.org/packages/92/ed/77542d0c51538e32e15afe7899d79efce4b81eee631d99850edc2f5e9349/pydantic_core-2.41.5-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:8566def80554c3faa0e65ac30ab0932b9e3a5cd7f8323764303d468e5c37595a", size = 2120255, upload-time = "2025-11-04T13:41:28.569Z" }, + { url = "https://files.pythonhosted.org/packages/bb/3d/6913dde84d5be21e284439676168b28d8bbba5600d838b9dca99de0fad71/pydantic_core-2.41.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:b80aa5095cd3109962a298ce14110ae16b8c1aece8b72f9dafe81cf597ad80b3", size = 1863760, upload-time = "2025-11-04T13:41:31.055Z" }, + { url = "https://files.pythonhosted.org/packages/5a/f0/e5e6b99d4191da102f2b0eb9687aaa7f5bea5d9964071a84effc3e40f997/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3006c3dd9ba34b0c094c544c6006cc79e87d8612999f1a5d43b769b89181f23c", size = 1878092, upload-time = "2025-11-04T13:41:33.21Z" }, + { url = "https://files.pythonhosted.org/packages/71/48/36fb760642d568925953bcc8116455513d6e34c4beaa37544118c36aba6d/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:72f6c8b11857a856bcfa48c86f5368439f74453563f951e473514579d44aa612", size = 2053385, upload-time = "2025-11-04T13:41:35.508Z" }, + { url = "https://files.pythonhosted.org/packages/20/25/92dc684dd8eb75a234bc1c764b4210cf2646479d54b47bf46061657292a8/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5cb1b2f9742240e4bb26b652a5aeb840aa4b417c7748b6f8387927bc6e45e40d", size = 2218832, upload-time = "2025-11-04T13:41:37.732Z" }, + { url = "https://files.pythonhosted.org/packages/e2/09/f53e0b05023d3e30357d82eb35835d0f6340ca344720a4599cd663dca599/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bd3d54f38609ff308209bd43acea66061494157703364ae40c951f83ba99a1a9", size = 2327585, upload-time = "2025-11-04T13:41:40Z" }, + { url = "https://files.pythonhosted.org/packages/aa/4e/2ae1aa85d6af35a39b236b1b1641de73f5a6ac4d5a7509f77b814885760c/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2ff4321e56e879ee8d2a879501c8e469414d948f4aba74a2d4593184eb326660", size = 2041078, upload-time = "2025-11-04T13:41:42.323Z" }, + { url = "https://files.pythonhosted.org/packages/cd/13/2e215f17f0ef326fc72afe94776edb77525142c693767fc347ed6288728d/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d0d2568a8c11bf8225044aa94409e21da0cb09dcdafe9ecd10250b2baad531a9", size = 2173914, upload-time = "2025-11-04T13:41:45.221Z" }, + { url = "https://files.pythonhosted.org/packages/02/7a/f999a6dcbcd0e5660bc348a3991c8915ce6599f4f2c6ac22f01d7a10816c/pydantic_core-2.41.5-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:a39455728aabd58ceabb03c90e12f71fd30fa69615760a075b9fec596456ccc3", size = 2129560, upload-time = "2025-11-04T13:41:47.474Z" }, + { url = "https://files.pythonhosted.org/packages/3a/b1/6c990ac65e3b4c079a4fb9f5b05f5b013afa0f4ed6780a3dd236d2cbdc64/pydantic_core-2.41.5-cp314-cp314t-musllinux_1_1_armv7l.whl", hash = "sha256:239edca560d05757817c13dc17c50766136d21f7cd0fac50295499ae24f90fdf", size = 2329244, upload-time = "2025-11-04T13:41:49.992Z" }, + { url = "https://files.pythonhosted.org/packages/d9/02/3c562f3a51afd4d88fff8dffb1771b30cfdfd79befd9883ee094f5b6c0d8/pydantic_core-2.41.5-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:2a5e06546e19f24c6a96a129142a75cee553cc018ffee48a460059b1185f4470", size = 2331955, upload-time = "2025-11-04T13:41:54.079Z" }, + { url = "https://files.pythonhosted.org/packages/5c/96/5fb7d8c3c17bc8c62fdb031c47d77a1af698f1d7a406b0f79aaa1338f9ad/pydantic_core-2.41.5-cp314-cp314t-win32.whl", hash = "sha256:b4ececa40ac28afa90871c2cc2b9ffd2ff0bf749380fbdf57d165fd23da353aa", size = 1988906, upload-time = "2025-11-04T13:41:56.606Z" }, + { url = "https://files.pythonhosted.org/packages/22/ed/182129d83032702912c2e2d8bbe33c036f342cc735737064668585dac28f/pydantic_core-2.41.5-cp314-cp314t-win_amd64.whl", hash = "sha256:80aa89cad80b32a912a65332f64a4450ed00966111b6615ca6816153d3585a8c", size = 1981607, upload-time = "2025-11-04T13:41:58.889Z" }, + { url = "https://files.pythonhosted.org/packages/9f/ed/068e41660b832bb0b1aa5b58011dea2a3fe0ba7861ff38c4d4904c1c1a99/pydantic_core-2.41.5-cp314-cp314t-win_arm64.whl", hash = "sha256:35b44f37a3199f771c3eaa53051bc8a70cd7b54f333531c59e29fd4db5d15008", size = 1974769, upload-time = "2025-11-04T13:42:01.186Z" }, + { url = "https://files.pythonhosted.org/packages/09/32/59b0c7e63e277fa7911c2fc70ccfb45ce4b98991e7ef37110663437005af/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:7da7087d756b19037bc2c06edc6c170eeef3c3bafcb8f532ff17d64dc427adfd", size = 2110495, upload-time = "2025-11-04T13:42:49.689Z" }, + { url = "https://files.pythonhosted.org/packages/aa/81/05e400037eaf55ad400bcd318c05bb345b57e708887f07ddb2d20e3f0e98/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:aabf5777b5c8ca26f7824cb4a120a740c9588ed58df9b2d196ce92fba42ff8dc", size = 1915388, upload-time = "2025-11-04T13:42:52.215Z" }, + { url = "https://files.pythonhosted.org/packages/6e/0d/e3549b2399f71d56476b77dbf3cf8937cec5cd70536bdc0e374a421d0599/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c007fe8a43d43b3969e8469004e9845944f1a80e6acd47c150856bb87f230c56", size = 1942879, upload-time = "2025-11-04T13:42:56.483Z" }, + { url = "https://files.pythonhosted.org/packages/f7/07/34573da085946b6a313d7c42f82f16e8920bfd730665de2d11c0c37a74b5/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:76d0819de158cd855d1cbb8fcafdf6f5cf1eb8e470abe056d5d161106e38062b", size = 2139017, upload-time = "2025-11-04T13:42:59.471Z" }, +] + +[[package]] +name = "pyflakes" +version = "3.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/45/dc/fd034dc20b4b264b3d015808458391acbf9df40b1e54750ef175d39180b1/pyflakes-3.4.0.tar.gz", hash = "sha256:b24f96fafb7d2ab0ec5075b7350b3d2d2218eab42003821c06344973d3ea2f58", size = 64669, upload-time = "2025-06-20T18:45:27.834Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c2/2f/81d580a0fb83baeb066698975cb14a618bdbed7720678566f1b046a95fe8/pyflakes-3.4.0-py2.py3-none-any.whl", hash = "sha256:f742a7dbd0d9cb9ea41e9a24a918996e8170c799fa528688d40dd582c8265f4f", size = 63551, upload-time = "2025-06-20T18:45:26.937Z" }, +] + +[[package]] +name = "pygments" +version = "2.19.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b0/77/a5b8c569bf593b0140bde72ea885a803b82086995367bf2037de0159d924/pygments-2.19.2.tar.gz", hash = "sha256:636cb2477cec7f8952536970bc533bc43743542f70392ae026374600add5b887", size = 4968631, upload-time = "2025-06-21T13:39:12.283Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c7/21/705964c7812476f378728bdf590ca4b771ec72385c533964653c68e86bdc/pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b", size = 1225217, upload-time = "2025-06-21T13:39:07.939Z" }, +] + +[[package]] +name = "pylint" +version = "4.0.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "astroid" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "dill" }, + { name = "isort" }, + { name = "mccabe" }, + { name = "platformdirs" }, + { name = "tomlkit" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e4/b6/74d9a8a68b8067efce8d07707fe6a236324ee1e7808d2eb3646ec8517c7d/pylint-4.0.5.tar.gz", hash = "sha256:8cd6a618df75deb013bd7eb98327a95f02a6fb839205a6bbf5456ef96afb317c", size = 1572474, upload-time = "2026-02-20T09:07:33.621Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d5/6f/9ac2548e290764781f9e7e2aaf0685b086379dabfb29ca38536985471eaf/pylint-4.0.5-py3-none-any.whl", hash = "sha256:00f51c9b14a3b3ae08cff6b2cdd43f28165c78b165b628692e428fb1f8dc2cf2", size = 536694, upload-time = "2026-02-20T09:07:31.028Z" }, +] + +[[package]] +name = "pymysql" +version = "1.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f5/ae/1fe3fcd9f959efa0ebe200b8de88b5a5ce3e767e38c7ac32fb179f16a388/pymysql-1.1.2.tar.gz", hash = "sha256:4961d3e165614ae65014e361811a724e2044ad3ea3739de9903ae7c21f539f03", size = 48258, upload-time = "2025-08-24T12:55:55.146Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7c/4c/ad33b92b9864cbde84f259d5df035a6447f91891f5be77788e2a3892bce3/pymysql-1.1.2-py3-none-any.whl", hash = "sha256:e6b1d89711dd51f8f74b1631fe08f039e7d76cf67a42a323d3178f0f25762ed9", size = 45300, upload-time = "2025-08-24T12:55:53.394Z" }, +] + +[[package]] +name = "pytest" +version = "9.0.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d1/db/7ef3487e0fb0049ddb5ce41d3a49c235bf9ad299b6a25d5780a89f19230f/pytest-9.0.2.tar.gz", hash = "sha256:75186651a92bd89611d1d9fc20f0b4345fd827c41ccd5c299a868a05d70edf11", size = 1568901, upload-time = "2025-12-06T21:30:51.014Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3b/ab/b3226f0bd7cdcf710fbede2b3548584366da3b19b5021e74f5bde2a8fa3f/pytest-9.0.2-py3-none-any.whl", hash = "sha256:711ffd45bf766d5264d487b917733b453d917afd2b0ad65223959f59089f875b", size = 374801, upload-time = "2025-12-06T21:30:49.154Z" }, +] + +[[package]] +name = "python-dateutil" +version = "2.9.0.post0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "six" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432, upload-time = "2024-03-01T18:36:20.211Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" }, +] + +[[package]] +name = "pytokens" +version = "0.4.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b6/34/b4e015b99031667a7b960f888889c5bd34ef585c85e1cb56a594b92836ac/pytokens-0.4.1.tar.gz", hash = "sha256:292052fe80923aae2260c073f822ceba21f3872ced9a68bb7953b348e561179a", size = 23015, upload-time = "2026-01-30T01:03:45.924Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/41/5d/e44573011401fb82e9d51e97f1290ceb377800fb4eed650b96f4753b499c/pytokens-0.4.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:140709331e846b728475786df8aeb27d24f48cbcf7bcd449f8de75cae7a45083", size = 160663, upload-time = "2026-01-30T01:03:06.473Z" }, + { url = "https://files.pythonhosted.org/packages/f0/e6/5bbc3019f8e6f21d09c41f8b8654536117e5e211a85d89212d59cbdab381/pytokens-0.4.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6d6c4268598f762bc8e91f5dbf2ab2f61f7b95bdc07953b602db879b3c8c18e1", size = 255626, upload-time = "2026-01-30T01:03:08.177Z" }, + { url = "https://files.pythonhosted.org/packages/bf/3c/2d5297d82286f6f3d92770289fd439956b201c0a4fc7e72efb9b2293758e/pytokens-0.4.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:24afde1f53d95348b5a0eb19488661147285ca4dd7ed752bbc3e1c6242a304d1", size = 269779, upload-time = "2026-01-30T01:03:09.756Z" }, + { url = "https://files.pythonhosted.org/packages/20/01/7436e9ad693cebda0551203e0bf28f7669976c60ad07d6402098208476de/pytokens-0.4.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5ad948d085ed6c16413eb5fec6b3e02fa00dc29a2534f088d3302c47eb59adf9", size = 268076, upload-time = "2026-01-30T01:03:10.957Z" }, + { url = "https://files.pythonhosted.org/packages/2e/df/533c82a3c752ba13ae7ef238b7f8cdd272cf1475f03c63ac6cf3fcfb00b6/pytokens-0.4.1-cp312-cp312-win_amd64.whl", hash = "sha256:3f901fe783e06e48e8cbdc82d631fca8f118333798193e026a50ce1b3757ea68", size = 103552, upload-time = "2026-01-30T01:03:12.066Z" }, + { url = "https://files.pythonhosted.org/packages/cb/dc/08b1a080372afda3cceb4f3c0a7ba2bde9d6a5241f1edb02a22a019ee147/pytokens-0.4.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:8bdb9d0ce90cbf99c525e75a2fa415144fd570a1ba987380190e8b786bc6ef9b", size = 160720, upload-time = "2026-01-30T01:03:13.843Z" }, + { url = "https://files.pythonhosted.org/packages/64/0c/41ea22205da480837a700e395507e6a24425151dfb7ead73343d6e2d7ffe/pytokens-0.4.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5502408cab1cb18e128570f8d598981c68a50d0cbd7c61312a90507cd3a1276f", size = 254204, upload-time = "2026-01-30T01:03:14.886Z" }, + { url = "https://files.pythonhosted.org/packages/e0/d2/afe5c7f8607018beb99971489dbb846508f1b8f351fcefc225fcf4b2adc0/pytokens-0.4.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:29d1d8fb1030af4d231789959f21821ab6325e463f0503a61d204343c9b355d1", size = 268423, upload-time = "2026-01-30T01:03:15.936Z" }, + { url = "https://files.pythonhosted.org/packages/68/d4/00ffdbd370410c04e9591da9220a68dc1693ef7499173eb3e30d06e05ed1/pytokens-0.4.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:970b08dd6b86058b6dc07efe9e98414f5102974716232d10f32ff39701e841c4", size = 266859, upload-time = "2026-01-30T01:03:17.458Z" }, + { url = "https://files.pythonhosted.org/packages/a7/c9/c3161313b4ca0c601eeefabd3d3b576edaa9afdefd32da97210700e47652/pytokens-0.4.1-cp313-cp313-win_amd64.whl", hash = "sha256:9bd7d7f544d362576be74f9d5901a22f317efc20046efe2034dced238cbbfe78", size = 103520, upload-time = "2026-01-30T01:03:18.652Z" }, + { url = "https://files.pythonhosted.org/packages/8f/a7/b470f672e6fc5fee0a01d9e75005a0e617e162381974213a945fcd274843/pytokens-0.4.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:4a14d5f5fc78ce85e426aa159489e2d5961acf0e47575e08f35584009178e321", size = 160821, upload-time = "2026-01-30T01:03:19.684Z" }, + { url = "https://files.pythonhosted.org/packages/80/98/e83a36fe8d170c911f864bfded690d2542bfcfacb9c649d11a9e6eb9dc41/pytokens-0.4.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:97f50fd18543be72da51dd505e2ed20d2228c74e0464e4262e4899797803d7fa", size = 254263, upload-time = "2026-01-30T01:03:20.834Z" }, + { url = "https://files.pythonhosted.org/packages/0f/95/70d7041273890f9f97a24234c00b746e8da86df462620194cef1d411ddeb/pytokens-0.4.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:dc74c035f9bfca0255c1af77ddd2d6ae8419012805453e4b0e7513e17904545d", size = 268071, upload-time = "2026-01-30T01:03:21.888Z" }, + { url = "https://files.pythonhosted.org/packages/da/79/76e6d09ae19c99404656d7db9c35dfd20f2086f3eb6ecb496b5b31163bad/pytokens-0.4.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f66a6bbe741bd431f6d741e617e0f39ec7257ca1f89089593479347cc4d13324", size = 271716, upload-time = "2026-01-30T01:03:23.633Z" }, + { url = "https://files.pythonhosted.org/packages/79/37/482e55fa1602e0a7ff012661d8c946bafdc05e480ea5a32f4f7e336d4aa9/pytokens-0.4.1-cp314-cp314-win_amd64.whl", hash = "sha256:b35d7e5ad269804f6697727702da3c517bb8a5228afa450ab0fa787732055fc9", size = 104539, upload-time = "2026-01-30T01:03:24.788Z" }, + { url = "https://files.pythonhosted.org/packages/30/e8/20e7db907c23f3d63b0be3b8a4fd1927f6da2395f5bcc7f72242bb963dfe/pytokens-0.4.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:8fcb9ba3709ff77e77f1c7022ff11d13553f3c30299a9fe246a166903e9091eb", size = 168474, upload-time = "2026-01-30T01:03:26.428Z" }, + { url = "https://files.pythonhosted.org/packages/d6/81/88a95ee9fafdd8f5f3452107748fd04c24930d500b9aba9738f3ade642cc/pytokens-0.4.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:79fc6b8699564e1f9b521582c35435f1bd32dd06822322ec44afdeba666d8cb3", size = 290473, upload-time = "2026-01-30T01:03:27.415Z" }, + { url = "https://files.pythonhosted.org/packages/cf/35/3aa899645e29b6375b4aed9f8d21df219e7c958c4c186b465e42ee0a06bf/pytokens-0.4.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d31b97b3de0f61571a124a00ffe9a81fb9939146c122c11060725bd5aea79975", size = 303485, upload-time = "2026-01-30T01:03:28.558Z" }, + { url = "https://files.pythonhosted.org/packages/52/a0/07907b6ff512674d9b201859f7d212298c44933633c946703a20c25e9d81/pytokens-0.4.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:967cf6e3fd4adf7de8fc73cd3043754ae79c36475c1c11d514fc72cf5490094a", size = 306698, upload-time = "2026-01-30T01:03:29.653Z" }, + { url = "https://files.pythonhosted.org/packages/39/2a/cbbf9250020a4a8dd53ba83a46c097b69e5eb49dd14e708f496f548c6612/pytokens-0.4.1-cp314-cp314t-win_amd64.whl", hash = "sha256:584c80c24b078eec1e227079d56dc22ff755e0ba8654d8383b2c549107528918", size = 116287, upload-time = "2026-01-30T01:03:30.912Z" }, + { url = "https://files.pythonhosted.org/packages/c6/78/397db326746f0a342855b81216ae1f0a32965deccfd7c830a2dbc66d2483/pytokens-0.4.1-py3-none-any.whl", hash = "sha256:26cef14744a8385f35d0e095dc8b3a7583f6c953c2e3d269c7f82484bf5ad2de", size = 13729, upload-time = "2026-01-30T01:03:45.029Z" }, +] + +[[package]] +name = "pyyaml" +version = "6.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/33/422b98d2195232ca1826284a76852ad5a86fe23e31b009c9886b2d0fb8b2/pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196", size = 182063, upload-time = "2025-09-25T21:32:11.445Z" }, + { url = "https://files.pythonhosted.org/packages/89/a0/6cf41a19a1f2f3feab0e9c0b74134aa2ce6849093d5517a0c550fe37a648/pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0", size = 173973, upload-time = "2025-09-25T21:32:12.492Z" }, + { url = "https://files.pythonhosted.org/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28", size = 775116, upload-time = "2025-09-25T21:32:13.652Z" }, + { url = "https://files.pythonhosted.org/packages/65/30/d7353c338e12baef4ecc1b09e877c1970bd3382789c159b4f89d6a70dc09/pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c", size = 844011, upload-time = "2025-09-25T21:32:15.21Z" }, + { url = "https://files.pythonhosted.org/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", size = 807870, upload-time = "2025-09-25T21:32:16.431Z" }, + { url = "https://files.pythonhosted.org/packages/05/c0/b3be26a015601b822b97d9149ff8cb5ead58c66f981e04fedf4e762f4bd4/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e", size = 761089, upload-time = "2025-09-25T21:32:17.56Z" }, + { url = "https://files.pythonhosted.org/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", size = 790181, upload-time = "2025-09-25T21:32:18.834Z" }, + { url = "https://files.pythonhosted.org/packages/74/93/7baea19427dcfbe1e5a372d81473250b379f04b1bd3c4c5ff825e2327202/pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5", size = 137658, upload-time = "2025-09-25T21:32:20.209Z" }, + { url = "https://files.pythonhosted.org/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b", size = 154003, upload-time = "2025-09-25T21:32:21.167Z" }, + { url = "https://files.pythonhosted.org/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", size = 140344, upload-time = "2025-09-25T21:32:22.617Z" }, + { url = "https://files.pythonhosted.org/packages/d1/11/0fd08f8192109f7169db964b5707a2f1e8b745d4e239b784a5a1dd80d1db/pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8", size = 181669, upload-time = "2025-09-25T21:32:23.673Z" }, + { url = "https://files.pythonhosted.org/packages/b1/16/95309993f1d3748cd644e02e38b75d50cbc0d9561d21f390a76242ce073f/pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1", size = 173252, upload-time = "2025-09-25T21:32:25.149Z" }, + { url = "https://files.pythonhosted.org/packages/50/31/b20f376d3f810b9b2371e72ef5adb33879b25edb7a6d072cb7ca0c486398/pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c", size = 767081, upload-time = "2025-09-25T21:32:26.575Z" }, + { url = "https://files.pythonhosted.org/packages/49/1e/a55ca81e949270d5d4432fbbd19dfea5321eda7c41a849d443dc92fd1ff7/pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5", size = 841159, upload-time = "2025-09-25T21:32:27.727Z" }, + { url = "https://files.pythonhosted.org/packages/74/27/e5b8f34d02d9995b80abcef563ea1f8b56d20134d8f4e5e81733b1feceb2/pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6", size = 801626, upload-time = "2025-09-25T21:32:28.878Z" }, + { url = "https://files.pythonhosted.org/packages/f9/11/ba845c23988798f40e52ba45f34849aa8a1f2d4af4b798588010792ebad6/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6", size = 753613, upload-time = "2025-09-25T21:32:30.178Z" }, + { url = "https://files.pythonhosted.org/packages/3d/e0/7966e1a7bfc0a45bf0a7fb6b98ea03fc9b8d84fa7f2229e9659680b69ee3/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be", size = 794115, upload-time = "2025-09-25T21:32:31.353Z" }, + { url = "https://files.pythonhosted.org/packages/de/94/980b50a6531b3019e45ddeada0626d45fa85cbe22300844a7983285bed3b/pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26", size = 137427, upload-time = "2025-09-25T21:32:32.58Z" }, + { url = "https://files.pythonhosted.org/packages/97/c9/39d5b874e8b28845e4ec2202b5da735d0199dbe5b8fb85f91398814a9a46/pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c", size = 154090, upload-time = "2025-09-25T21:32:33.659Z" }, + { url = "https://files.pythonhosted.org/packages/73/e8/2bdf3ca2090f68bb3d75b44da7bbc71843b19c9f2b9cb9b0f4ab7a5a4329/pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb", size = 140246, upload-time = "2025-09-25T21:32:34.663Z" }, + { url = "https://files.pythonhosted.org/packages/9d/8c/f4bd7f6465179953d3ac9bc44ac1a8a3e6122cf8ada906b4f96c60172d43/pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac", size = 181814, upload-time = "2025-09-25T21:32:35.712Z" }, + { url = "https://files.pythonhosted.org/packages/bd/9c/4d95bb87eb2063d20db7b60faa3840c1b18025517ae857371c4dd55a6b3a/pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310", size = 173809, upload-time = "2025-09-25T21:32:36.789Z" }, + { url = "https://files.pythonhosted.org/packages/92/b5/47e807c2623074914e29dabd16cbbdd4bf5e9b2db9f8090fa64411fc5382/pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7", size = 766454, upload-time = "2025-09-25T21:32:37.966Z" }, + { url = "https://files.pythonhosted.org/packages/02/9e/e5e9b168be58564121efb3de6859c452fccde0ab093d8438905899a3a483/pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788", size = 836355, upload-time = "2025-09-25T21:32:39.178Z" }, + { url = "https://files.pythonhosted.org/packages/88/f9/16491d7ed2a919954993e48aa941b200f38040928474c9e85ea9e64222c3/pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5", size = 794175, upload-time = "2025-09-25T21:32:40.865Z" }, + { url = "https://files.pythonhosted.org/packages/dd/3f/5989debef34dc6397317802b527dbbafb2b4760878a53d4166579111411e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764", size = 755228, upload-time = "2025-09-25T21:32:42.084Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ce/af88a49043cd2e265be63d083fc75b27b6ed062f5f9fd6cdc223ad62f03e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35", size = 789194, upload-time = "2025-09-25T21:32:43.362Z" }, + { url = "https://files.pythonhosted.org/packages/23/20/bb6982b26a40bb43951265ba29d4c246ef0ff59c9fdcdf0ed04e0687de4d/pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac", size = 156429, upload-time = "2025-09-25T21:32:57.844Z" }, + { url = "https://files.pythonhosted.org/packages/f4/f4/a4541072bb9422c8a883ab55255f918fa378ecf083f5b85e87fc2b4eda1b/pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3", size = 143912, upload-time = "2025-09-25T21:32:59.247Z" }, + { url = "https://files.pythonhosted.org/packages/7c/f9/07dd09ae774e4616edf6cda684ee78f97777bdd15847253637a6f052a62f/pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3", size = 189108, upload-time = "2025-09-25T21:32:44.377Z" }, + { url = "https://files.pythonhosted.org/packages/4e/78/8d08c9fb7ce09ad8c38ad533c1191cf27f7ae1effe5bb9400a46d9437fcf/pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba", size = 183641, upload-time = "2025-09-25T21:32:45.407Z" }, + { url = "https://files.pythonhosted.org/packages/7b/5b/3babb19104a46945cf816d047db2788bcaf8c94527a805610b0289a01c6b/pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c", size = 831901, upload-time = "2025-09-25T21:32:48.83Z" }, + { url = "https://files.pythonhosted.org/packages/8b/cc/dff0684d8dc44da4d22a13f35f073d558c268780ce3c6ba1b87055bb0b87/pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702", size = 861132, upload-time = "2025-09-25T21:32:50.149Z" }, + { url = "https://files.pythonhosted.org/packages/b1/5e/f77dc6b9036943e285ba76b49e118d9ea929885becb0a29ba8a7c75e29fe/pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c", size = 839261, upload-time = "2025-09-25T21:32:51.808Z" }, + { url = "https://files.pythonhosted.org/packages/ce/88/a9db1376aa2a228197c58b37302f284b5617f56a5d959fd1763fb1675ce6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065", size = 805272, upload-time = "2025-09-25T21:32:52.941Z" }, + { url = "https://files.pythonhosted.org/packages/da/92/1446574745d74df0c92e6aa4a7b0b3130706a4142b2d1a5869f2eaa423c6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65", size = 829923, upload-time = "2025-09-25T21:32:54.537Z" }, + { url = "https://files.pythonhosted.org/packages/f0/7a/1c7270340330e575b92f397352af856a8c06f230aa3e76f86b39d01b416a/pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9", size = 174062, upload-time = "2025-09-25T21:32:55.767Z" }, + { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, +] + +[[package]] +name = "requests" +version = "2.32.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "charset-normalizer" }, + { name = "idna" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c9/74/b3ff8e6c8446842c3f5c837e9c3dfcfe2018ea6ecef224c710c85ef728f4/requests-2.32.5.tar.gz", hash = "sha256:dbba0bac56e100853db0ea71b82b4dfd5fe2bf6d3754a8893c3af500cec7d7cf", size = 134517, upload-time = "2025-08-18T20:46:02.573Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/db/4254e3eabe8020b458f1a747140d32277ec7a271daf1d235b70dc0b4e6e3/requests-2.32.5-py3-none-any.whl", hash = "sha256:2462f94637a34fd532264295e186976db0f5d453d1cdd31473c85a6a161affb6", size = 64738, upload-time = "2025-08-18T20:46:00.542Z" }, +] + +[[package]] +name = "requests-oauthlib" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "oauthlib" }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/42/f2/05f29bc3913aea15eb670be136045bf5c5bbf4b99ecb839da9b422bb2c85/requests-oauthlib-2.0.0.tar.gz", hash = "sha256:b3dffaebd884d8cd778494369603a9e7b58d29111bf6b41bdc2dcd87203af4e9", size = 55650, upload-time = "2024-03-22T20:32:29.939Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3b/5d/63d4ae3b9daea098d5d6f5da83984853c1bbacd5dc826764b249fe119d24/requests_oauthlib-2.0.0-py2.py3-none-any.whl", hash = "sha256:7dd8a5c40426b779b0868c404bdef9768deccf22749cde15852df527e6269b36", size = 24179, upload-time = "2024-03-22T20:32:28.055Z" }, +] + +[[package]] +name = "sh" +version = "1.14.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b7/09/89c28aaf2a49f226fef8587c90c6386bd2cc03a0295bc4ff7fc6ee43c01d/sh-1.14.3.tar.gz", hash = "sha256:e4045b6c732d9ce75d571c79f5ac2234edd9ae4f5fa9d59b09705082bdca18c7", size = 62851, upload-time = "2022-07-18T07:17:50.947Z" } + +[[package]] +name = "six" +version = "1.17.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81", size = 34031, upload-time = "2024-12-04T17:35:28.174Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, +] + +[[package]] +name = "soupsieve" +version = "2.8.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7b/ae/2d9c981590ed9999a0d91755b47fc74f74de286b0f5cee14c9269041e6c4/soupsieve-2.8.3.tar.gz", hash = "sha256:3267f1eeea4251fb42728b6dfb746edc9acaffc4a45b27e19450b676586e8349", size = 118627, upload-time = "2026-01-20T04:27:02.457Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/46/2c/1462b1d0a634697ae9e55b3cecdcb64788e8b7d63f54d923fcd0bb140aed/soupsieve-2.8.3-py3-none-any.whl", hash = "sha256:ed64f2ba4eebeab06cc4962affce381647455978ffc1e36bb79a545b91f45a95", size = 37016, upload-time = "2026-01-20T04:27:01.012Z" }, +] + +[[package]] +name = "sqlalchemy" +version = "2.0.47" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "greenlet", marker = "platform_machine == 'AMD64' or platform_machine == 'WIN32' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'ppc64le' or platform_machine == 'win32' or platform_machine == 'x86_64'" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/cd/4b/1e00561093fe2cd8eef09d406da003c8a118ff02d6548498c1ae677d68d9/sqlalchemy-2.0.47.tar.gz", hash = "sha256:e3e7feb57b267fe897e492b9721ae46d5c7de6f9e8dee58aacf105dc4e154f3d", size = 9886323, upload-time = "2026-02-24T16:34:27.947Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/80/88/74eb470223ff88ea6572a132c0b8de8c1d8ed7b843d3b44a8a3c77f31d39/sqlalchemy-2.0.47-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:4fa91b19d6b9821c04cc8f7aa2476429cc8887b9687c762815aa629f5c0edec1", size = 2155687, upload-time = "2026-02-24T17:05:46.451Z" }, + { url = "https://files.pythonhosted.org/packages/ef/ba/1447d3d558971b036cb93b557595cb5dcdfe728f1c7ac4dec16505ef5756/sqlalchemy-2.0.47-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7c5bbbd14eff577c8c79cbfe39a0771eecd20f430f3678533476f0087138f356", size = 3336978, upload-time = "2026-02-24T17:18:04.597Z" }, + { url = "https://files.pythonhosted.org/packages/8a/07/b47472d2ffd0776826f17ccf0b4d01b224c99fbd1904aeb103dffbb4b1cc/sqlalchemy-2.0.47-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a5a6c555da8d4280a3c4c78c5b7a3f990cee2b2884e5f934f87a226191682ff7", size = 3349939, upload-time = "2026-02-24T17:27:18.937Z" }, + { url = "https://files.pythonhosted.org/packages/bb/c6/95fa32b79b57769da3e16f054cf658d90940317b5ca0ec20eac84aa19c4f/sqlalchemy-2.0.47-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ed48a1701d24dff3bb49a5bce94d6bc84cbe33d98af2aa2d3cdcce3dea1709ec", size = 3279648, upload-time = "2026-02-24T17:18:07.038Z" }, + { url = "https://files.pythonhosted.org/packages/bb/c8/3d07e7c73928dc59a0bed40961ca4e313e797bce650b088e8d5fdd3ad939/sqlalchemy-2.0.47-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:4f3178c920ad98158f0b6309382194df04b14808fa6052ae07099fdde29d5602", size = 3314695, upload-time = "2026-02-24T17:27:20.93Z" }, + { url = "https://files.pythonhosted.org/packages/6b/d2/ed32b1611c1e19fdb028eee1adc5a9aa138c2952d09ae11f1670170f80ae/sqlalchemy-2.0.47-cp312-cp312-win32.whl", hash = "sha256:b9c11ac9934dd59ece9619fe42780a08abe2faab7b0543bb00d5eabea4f421b9", size = 2115502, upload-time = "2026-02-24T17:22:52.546Z" }, + { url = "https://files.pythonhosted.org/packages/fd/52/9de590356a4dd8e9ef5a881dbba64b2bbc4cbc71bf02bc68e775fb9b1899/sqlalchemy-2.0.47-cp312-cp312-win_amd64.whl", hash = "sha256:db43b72cf8274a99e089755c9c1e0b947159b71adbc2c83c3de2e38d5d607acb", size = 2142435, upload-time = "2026-02-24T17:22:54.268Z" }, + { url = "https://files.pythonhosted.org/packages/4a/e5/0af64ce7d8f60ec5328c10084e2f449e7912a9b8bdbefdcfb44454a25f49/sqlalchemy-2.0.47-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:456a135b790da5d3c6b53d0ef71ac7b7d280b7f41eb0c438986352bf03ca7143", size = 2152551, upload-time = "2026-02-24T17:05:47.675Z" }, + { url = "https://files.pythonhosted.org/packages/63/79/746b8d15f6940e2ac469ce22d7aa5b1124b1ab820bad9b046eb3000c88a6/sqlalchemy-2.0.47-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:09a2f7698e44b3135433387da5d8846cf7cc7c10e5425af7c05fee609df978b6", size = 3278782, upload-time = "2026-02-24T17:18:10.012Z" }, + { url = "https://files.pythonhosted.org/packages/91/b1/bd793ddb34345d1ed43b13ab2d88c95d7d4eb2e28f5b5a99128b9cc2bca2/sqlalchemy-2.0.47-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a0bbc72e6a177c78d724f9106aaddc0d26a2ada89c6332b5935414eccf04cbd5", size = 3295155, upload-time = "2026-02-24T17:27:22.827Z" }, + { url = "https://files.pythonhosted.org/packages/97/84/7213def33f94e5ca6f5718d259bc9f29de0363134648425aa218d4356b23/sqlalchemy-2.0.47-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:75460456b043b78b6006e41bdf5b86747ee42eafaf7fffa3b24a6e9a456a2092", size = 3226834, upload-time = "2026-02-24T17:18:11.465Z" }, + { url = "https://files.pythonhosted.org/packages/ef/06/456810204f4dc29b5f025b1b0a03b4bd6b600ebf3c1040aebd90a257fa33/sqlalchemy-2.0.47-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:5d9adaa616c3bc7d80f9ded57cd84b51d6617cad6a5456621d858c9f23aaee01", size = 3265001, upload-time = "2026-02-24T17:27:24.813Z" }, + { url = "https://files.pythonhosted.org/packages/fb/20/df3920a4b2217dbd7390a5bd277c1902e0393f42baaf49f49b3c935e7328/sqlalchemy-2.0.47-cp313-cp313-win32.whl", hash = "sha256:76e09f974382a496a5ed985db9343628b1cb1ac911f27342e4cc46a8bac10476", size = 2113647, upload-time = "2026-02-24T17:22:55.747Z" }, + { url = "https://files.pythonhosted.org/packages/46/06/7873ddf69918efbfabd7211829f4bd8019739d0a719253112d305d3ba51d/sqlalchemy-2.0.47-cp313-cp313-win_amd64.whl", hash = "sha256:0664089b0bf6724a0bfb49a0cf4d4da24868a0a5c8e937cd7db356d5dcdf2c66", size = 2139425, upload-time = "2026-02-24T17:22:57.033Z" }, + { url = "https://files.pythonhosted.org/packages/54/fa/61ad9731370c90ac7ea5bf8f5eaa12c48bb4beec41c0fa0360becf4ac10d/sqlalchemy-2.0.47-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ed0c967c701ae13da98eb220f9ddab3044ab63504c1ba24ad6a59b26826ad003", size = 3558809, upload-time = "2026-02-24T17:12:15.232Z" }, + { url = "https://files.pythonhosted.org/packages/33/d5/221fac96f0529391fe374875633804c866f2b21a9c6d3a6ca57d9c12cfd7/sqlalchemy-2.0.47-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d3537943a61fd25b241e976426a0c6814434b93cf9b09d39e8e78f3c9eb9a487", size = 3525480, upload-time = "2026-02-24T17:27:59.602Z" }, + { url = "https://files.pythonhosted.org/packages/ec/55/8247d53998c3673e4a8d1958eba75c6f5cc3b39082029d400bb1f2a911ae/sqlalchemy-2.0.47-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:57f7e336a64a0dba686c66392d46b9bc7af2c57d55ce6dc1697b4ef32b043ceb", size = 3466569, upload-time = "2026-02-24T17:12:16.94Z" }, + { url = "https://files.pythonhosted.org/packages/6b/b5/c1f0eea1bac6790845f71420a7fe2f2a0566203aa57543117d4af3b77d1c/sqlalchemy-2.0.47-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:dff735a621858680217cb5142b779bad40ef7322ddbb7c12062190db6879772e", size = 3475770, upload-time = "2026-02-24T17:28:02.034Z" }, + { url = "https://files.pythonhosted.org/packages/c5/ed/2f43f92474ea0c43c204657dc47d9d002cd738b96ca2af8e6d29a9b5e42d/sqlalchemy-2.0.47-cp313-cp313t-win32.whl", hash = "sha256:3893dc096bb3cca9608ea3487372ffcea3ae9b162f40e4d3c51dd49db1d1b2dc", size = 2141300, upload-time = "2026-02-24T17:14:37.024Z" }, + { url = "https://files.pythonhosted.org/packages/cc/a9/8b73f9f1695b6e92f7aaf1711135a1e3bbeb78bca9eded35cb79180d3c6d/sqlalchemy-2.0.47-cp313-cp313t-win_amd64.whl", hash = "sha256:b5103427466f4b3e61f04833ae01f9a914b1280a2a8bcde3a9d7ab11f3755b42", size = 2173053, upload-time = "2026-02-24T17:14:38.688Z" }, + { url = "https://files.pythonhosted.org/packages/c1/30/98243209aae58ed80e090ea988d5182244ca7ab3ff59e6d850c3dfc7651e/sqlalchemy-2.0.47-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:b03010a5a5dfe71676bc83f2473ebe082478e32d77e6f082c8fe15a31c3b42a6", size = 2154355, upload-time = "2026-02-24T17:05:48.959Z" }, + { url = "https://files.pythonhosted.org/packages/ab/62/12ca6ea92055fe486d6558a2a4efe93e194ff597463849c01f88e5adb99d/sqlalchemy-2.0.47-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f8e3371aa9024520883a415a09cc20c33cfd3eeccf9e0f4f4c367f940b9cbd44", size = 3274486, upload-time = "2026-02-24T17:18:13.659Z" }, + { url = "https://files.pythonhosted.org/packages/97/88/7dfbdeaa8d42b1584e65d6cc713e9d33b6fa563e0d546d5cb87e545bb0e5/sqlalchemy-2.0.47-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c9449f747e50d518c6e1b40cc379e48bfc796453c47b15e627ea901c201e48a6", size = 3279481, upload-time = "2026-02-24T17:27:26.491Z" }, + { url = "https://files.pythonhosted.org/packages/d0/b7/75e1c1970616a9dd64a8a6fd788248da2ddaf81c95f4875f2a1e8aee4128/sqlalchemy-2.0.47-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:21410f60d5cac1d6bfe360e05bd91b179be4fa0aa6eea6be46054971d277608f", size = 3224269, upload-time = "2026-02-24T17:18:15.078Z" }, + { url = "https://files.pythonhosted.org/packages/31/ac/eec1a13b891df9a8bc203334caf6e6aac60b02f61b018ef3b4124b8c4120/sqlalchemy-2.0.47-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:819841dd5bb4324c284c09e2874cf96fe6338bfb57a64548d9b81a4e39c9871f", size = 3246262, upload-time = "2026-02-24T17:27:27.986Z" }, + { url = "https://files.pythonhosted.org/packages/c9/b0/661b0245b06421058610da39f8ceb34abcc90b49f90f256380968d761dbe/sqlalchemy-2.0.47-cp314-cp314-win32.whl", hash = "sha256:e255ee44821a7ef45649c43064cf94e74f81f61b4df70547304b97a351e9b7db", size = 2116528, upload-time = "2026-02-24T17:22:59.363Z" }, + { url = "https://files.pythonhosted.org/packages/aa/ef/1035a90d899e61810791c052004958be622a2cf3eb3df71c3fe20778c5d0/sqlalchemy-2.0.47-cp314-cp314-win_amd64.whl", hash = "sha256:209467ff73ea1518fe1a5aaed9ba75bb9e33b2666e2553af9ccd13387bf192cb", size = 2142181, upload-time = "2026-02-24T17:23:01.001Z" }, + { url = "https://files.pythonhosted.org/packages/76/bb/17a1dd09cbba91258218ceb582225f14b5364d2683f9f5a274f72f2d764f/sqlalchemy-2.0.47-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e78fd9186946afaa287f8a1fe147ead06e5d566b08c0afcb601226e9c7322a64", size = 3563477, upload-time = "2026-02-24T17:12:18.46Z" }, + { url = "https://files.pythonhosted.org/packages/66/8f/1a03d24c40cc321ef2f2231f05420d140bb06a84f7047eaa7eaa21d230ba/sqlalchemy-2.0.47-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5740e2f31b5987ed9619d6912ae5b750c03637f2078850da3002934c9532f172", size = 3528568, upload-time = "2026-02-24T17:28:03.732Z" }, + { url = "https://files.pythonhosted.org/packages/fd/53/d56a213055d6b038a5384f0db5ece7343334aca230ff3f0fa1561106f22c/sqlalchemy-2.0.47-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:fb9ac00d03de93acb210e8ec7243fefe3e012515bf5fd2f0898c8dff38bc77a4", size = 3472284, upload-time = "2026-02-24T17:12:20.319Z" }, + { url = "https://files.pythonhosted.org/packages/ff/19/c235d81b9cfdd6130bf63143b7bade0dc4afa46c4b634d5d6b2a96bea233/sqlalchemy-2.0.47-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:c72a0b9eb2672d70d112cb149fbaf172d466bc691014c496aaac594f1988e706", size = 3478410, upload-time = "2026-02-24T17:28:05.892Z" }, + { url = "https://files.pythonhosted.org/packages/0e/db/cafdeca5ecdaa3bb0811ba5449501da677ce0d83be8d05c5822da72d2e86/sqlalchemy-2.0.47-cp314-cp314t-win32.whl", hash = "sha256:c200db1128d72a71dc3c31c24b42eb9fd85b2b3e5a3c9ba1e751c11ac31250ff", size = 2147164, upload-time = "2026-02-24T17:14:40.783Z" }, + { url = "https://files.pythonhosted.org/packages/fc/5e/ff41a010e9e0f76418b02ad352060a4341bb15f0af66cedc924ab376c7c6/sqlalchemy-2.0.47-cp314-cp314t-win_amd64.whl", hash = "sha256:669837759b84e575407355dcff912835892058aea9b80bd1cb76d6a151cf37f7", size = 2182154, upload-time = "2026-02-24T17:14:43.205Z" }, + { url = "https://files.pythonhosted.org/packages/15/9f/7c378406b592fcf1fc157248607b495a40e3202ba4a6f1372a2ba6447717/sqlalchemy-2.0.47-py3-none-any.whl", hash = "sha256:e2647043599297a1ef10e720cf310846b7f31b6c841fee093d2b09d81215eb93", size = 1940159, upload-time = "2026-02-24T17:15:07.158Z" }, +] + +[[package]] +name = "tomlkit" +version = "0.14.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c3/af/14b24e41977adb296d6bd1fb59402cf7d60ce364f90c890bd2ec65c43b5a/tomlkit-0.14.0.tar.gz", hash = "sha256:cf00efca415dbd57575befb1f6634c4f42d2d87dbba376128adb42c121b87064", size = 187167, upload-time = "2026-01-13T01:14:53.304Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b5/11/87d6d29fb5d237229d67973a6c9e06e048f01cf4994dee194ab0ea841814/tomlkit-0.14.0-py3-none-any.whl", hash = "sha256:592064ed85b40fa213469f81ac584f67a4f2992509a7c3ea2d632208623a3680", size = 39310, upload-time = "2026-01-13T01:14:51.965Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.15.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/94/1a15dd82efb362ac84269196e94cf00f187f7ed21c242792a923cdb1c61f/typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466", size = 109391, upload-time = "2025-08-25T13:49:26.313Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548", size = 44614, upload-time = "2025-08-25T13:49:24.86Z" }, +] + +[[package]] +name = "typing-inspection" +version = "0.4.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/55/e3/70399cb7dd41c10ac53367ae42139cf4b1ca5f36bb3dc6c9d33acdb43655/typing_inspection-0.4.2.tar.gz", hash = "sha256:ba561c48a67c5958007083d386c3295464928b01faa735ab8547c5692e87f464", size = 75949, upload-time = "2025-10-01T02:14:41.687Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl", hash = "sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7", size = 14611, upload-time = "2025-10-01T02:14:40.154Z" }, +] + +[[package]] +name = "urllib3" +version = "2.6.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c7/24/5f1b3bdffd70275f6661c76461e25f024d5a38a46f04aaca912426a2b1d3/urllib3-2.6.3.tar.gz", hash = "sha256:1b62b6884944a57dbe321509ab94fd4d3b307075e0c2eae991ac71ee15ad38ed", size = 435556, upload-time = "2026-01-07T16:24:43.925Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/39/08/aaaad47bc4e9dc8c725e68f9d04865dbcb2052843ff09c97b08904852d84/urllib3-2.6.3-py3-none-any.whl", hash = "sha256:bf272323e553dfb2e87d9bfd225ca7b0f467b919d7bbd355436d3fd37cb0acd4", size = 131584, upload-time = "2026-01-07T16:24:42.685Z" }, +] + +[[package]] +name = "werkzeug" +version = "3.1.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markupsafe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/61/f1/ee81806690a87dab5f5653c1f146c92bc066d7f4cebc603ef88eb9e13957/werkzeug-3.1.6.tar.gz", hash = "sha256:210c6bede5a420a913956b4791a7f4d6843a43b6fcee4dfa08a65e93007d0d25", size = 864736, upload-time = "2026-02-19T15:17:18.884Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4d/ec/d58832f89ede95652fd01f4f24236af7d32b70cab2196dfcc2d2fd13c5c2/werkzeug-3.1.6-py3-none-any.whl", hash = "sha256:7ddf3357bb9564e407607f988f683d72038551200c704012bb9a4c523d42f131", size = 225166, upload-time = "2026-02-19T15:17:17.475Z" }, +] + +[[package]] +name = "yamllint" +version = "1.38.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pathspec" }, + { name = "pyyaml" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/28/a0/8fc2d68e132cf918f18273fdc8a1b8432b60d75ac12fdae4b0ef5c9d2e8d/yamllint-1.38.0.tar.gz", hash = "sha256:09e5f29531daab93366bb061e76019d5e91691ef0a40328f04c927387d1d364d", size = 142446, upload-time = "2026-01-13T07:47:53.276Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/05/92/aed08e68de6e6a3d7c2328ce7388072cd6affc26e2917197430b646aed02/yamllint-1.38.0-py3-none-any.whl", hash = "sha256:fc394a5b3be980a4062607b8fdddc0843f4fa394152b6da21722f5d59013c220", size = 68940, upload-time = "2026-01-13T07:47:51.343Z" }, +]