diff --git a/.codespellrc b/.codespellrc deleted file mode 100644 index a56ec23f4..000000000 --- a/.codespellrc +++ /dev/null @@ -1,5 +0,0 @@ -[codespell] -skip = .git,*.pdf,*.svg,*.csv,*.ipynb,*.drawio -# Rever -- nobody knows -# numer -- numerator variable -ignore-words-list = rever,numer diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 89a8ad868..776a32e99 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -8,6 +8,7 @@ RUN \ pip uninstall datajoint -y USER root -ENV DJ_HOST db -ENV DJ_USER root -ENV DJ_PASS password +ENV DJ_HOST=db +ENV DJ_USER=root +ENV DJ_PASS=password +ENV S3_ENDPOINT=minio:9000 diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 6ed3c52c4..51ca1e64c 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,56 +1,6 @@ -// For format details, see https://aka.ms/devcontainer.json. For config options, see the -// README at: https://github.com/devcontainers/templates/tree/main/src/docker-existing-docker-compose { - "name": "Existing Docker Compose (Extend)", - // Update the 'dockerComposeFile' list if you have more compose files or use different names. - // The .devcontainer/docker-compose.yml file contains any overrides you need/want to make. - "dockerComposeFile": [ - "../docker-compose.yaml", - "docker-compose.yml" - ], - // The 'service' property is the name of the service for the container that VS Code should - // use. Update this value and .devcontainer/docker-compose.yml to the real service name. - "service": "app", - // The optional 'workspaceFolder' property is the path VS Code should open by default when - // connected. This is typically a file mount in .devcontainer/docker-compose.yml - "workspaceFolder": "/workspaces/${localWorkspaceFolderBasename}", - // Features to add to the dev container. More info: https://containers.dev/features. - // "features": {}, - // Use 'forwardPorts' to make a list of ports inside the container available locally. - "forwardPorts": [ - 80, - 443, - 3306, - 8080, - 9000 - ], - "mounts": [ - "type=bind,source=${env:SSH_AUTH_SOCK},target=/ssh-agent" - ], - "containerEnv": { - "SSH_AUTH_SOCK": "/ssh-agent" - }, - // Uncomment the next line if you want start specific services in your Docker Compose config. - // "runServices": [], - // Uncomment the next line if you want to keep your containers running after VS Code shuts down. - "shutdownAction": "stopCompose", - "onCreateCommand": "python3 -m pip install -q -e .[dev]", - "features": { - "ghcr.io/devcontainers/features/git:1": {}, - "ghcr.io/devcontainers/features/docker-in-docker:2": {}, - "ghcr.io/devcontainers/features/github-cli:1": {}, - }, - // Configure tool-specific properties. - "customizations": { - "vscode": { - "extensions": [ - "ms-python.python" - ] - } - }, - "remoteEnv": { - "LOCAL_WORKSPACE_FOLDER": "${localWorkspaceFolder}" - } - // Uncomment to connect as an existing user other than the container default. More info: https://aka.ms/dev-containers-non-root. - // "remoteUser": "devcontainer" + "dockerComposeFile": ["../docker-compose.yaml", "docker-compose.yml"], + "service": "app", + "workspaceFolder": "/src", + "postCreateCommand": "curl -fsSL https://pixi.sh/install.sh | bash && echo 'export PATH=\"$HOME/.pixi/bin:$PATH\"' >> ~/.bashrc" } diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml index 5c22aaf14..c876f69f4 100644 --- a/.devcontainer/docker-compose.yml +++ b/.devcontainer/docker-compose.yml @@ -1,30 +1,14 @@ +# Devcontainer overrides for the app service from ../docker-compose.yaml +# Inherits db and minio services automatically services: - # Update this to the name of the service you want to work with in your docker-compose.yml file app: - # Uncomment if you want to override the service's Dockerfile to one in the .devcontainer - # folder. Note that the path of the Dockerfile and context is relative to the *primary* - # docker-compose.yml file (the first in the devcontainer.json "dockerComposeFile" - # array). The sample below assumes your primary file is in the root of your project. container_name: datajoint-python-devcontainer - image: datajoint/datajoint-python-devcontainer:${PY_VER:-3.11}-${DISTRO:-bookworm} build: - context: . + context: .. dockerfile: .devcontainer/Dockerfile args: - PY_VER=${PY_VER:-3.11} - DISTRO=${DISTRO:-bookworm} - - volumes: - # Update this to wherever you want VS Code to mount the folder of your project - - ..:/workspaces:cached - - # Uncomment the next four lines if you will use a ptrace-based debugger like C++, Go, and Rust. - # cap_add: - # - SYS_PTRACE - # security_opt: - # - seccomp:unconfined - user: root - - # Overrides default command so things don't shut down after the process ends. + # Keep container running for devcontainer command: /bin/sh -c "while sleep 1000; do :; done" diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000..887a2c18f --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +# SCM syntax highlighting & preventing 3-way merges +pixi.lock merge=binary linguist-language=YAML linguist-generated=true diff --git a/.github/pr_labeler.yaml b/.github/pr_labeler.yaml index ab722839f..51ce9afee 100644 --- a/.github/pr_labeler.yaml +++ b/.github/pr_labeler.yaml @@ -1,8 +1,8 @@ # https://github.com/actions/labeler breaking: -- head-branch: ['breaking', 'BREAKING'] +- head-branch: ['breaking', 'BREAKING', 'pre/v2.0'] bug: -- head-branch: ['fix', 'FIX', 'bug', 'BUG'] +- head-branch: ['fix', 'FIX', 'bug', 'BUG', 'pre/v2.0'] feature: - head-branch: ['feat', 'FEAT'] documentation: diff --git a/.github/workflows/label_prs.yaml b/.github/workflows/label_prs.yaml index 9797a956f..8f3fcec95 100644 --- a/.github/workflows/label_prs.yaml +++ b/.github/workflows/label_prs.yaml @@ -14,5 +14,5 @@ jobs: with: repo-token: ${{ secrets.GITHUB_TOKEN }} configuration-path: .github/pr_labeler.yaml - sync-labels: true + sync-labels: false # Don't remove manually added labels dot: true \ No newline at end of file diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 62468a983..e7e6dc2ae 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -23,7 +23,7 @@ jobs: extra_args: codespell --all-files - uses: pre-commit/action@v3.0.1 with: - extra_args: black --all-files + extra_args: ruff --all-files - uses: pre-commit/action@v3.0.1 with: - extra_args: flake8 --all-files + extra_args: ruff-format --all-files diff --git a/.github/workflows/post_draft_release_published.yaml b/.github/workflows/post_draft_release_published.yaml index 20160e62b..f9c3ee62d 100644 --- a/.github/workflows/post_draft_release_published.yaml +++ b/.github/workflows/post_draft_release_published.yaml @@ -23,7 +23,7 @@ jobs: strategy: matrix: include: - - py_ver: "3.9" + - py_ver: "3.10" runs-on: ubuntu-latest env: PY_VER: ${{matrix.py_ver}} @@ -40,14 +40,14 @@ jobs: - name: Update version.py run: | VERSION=$(echo "${{ github.event.release.name }}" | grep -oP '\d+\.\d+\.\d+') - sed -i "s/^__version__ = .*/__version__ = \"$VERSION\"/" datajoint/version.py - cat datajoint/version.py + sed -i "s/^__version__ = .*/__version__ = \"$VERSION\"/" src/datajoint/version.py + cat src/datajoint/version.py # Commit the changes BRANCH_NAME="update-version-$VERSION" git switch -c $BRANCH_NAME git config --global user.name "github-actions" git config --global user.email "github-actions@github.com" - git add datajoint/version.py + git add src/datajoint/version.py git commit -m "Update version.py to $VERSION" echo "BRANCH_NAME=$BRANCH_NAME" >> $GITHUB_ENV - name: Update README.md badge diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 196ddec22..a4a91448f 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -1,42 +1,55 @@ name: Test + on: push: branches: - - "**" # every branch - - "!gh-pages" # exclude gh-pages branch - - "!stage*" # exclude branches beginning with stage + - "**" + - "!gh-pages" + - "!stage*" paths: - - "datajoint" - - "tests" + - "src/datajoint/**" + - "tests/**" + - "pyproject.toml" + - "pixi.lock" + - ".github/workflows/test.yaml" pull_request: branches: - - "**" # every branch - - "!gh-pages" # exclude gh-pages branch - - "!stage*" # exclude branches beginning with stage + - "**" + - "!gh-pages" + - "!stage*" paths: - - "datajoint" - - "tests" + - "src/datajoint/**" + - "tests/**" + - "pyproject.toml" + - "pixi.lock" + - ".github/workflows/test.yaml" + jobs: test: runs-on: ubuntu-latest - strategy: - matrix: - py_ver: ["3.9", "3.10", "3.11", "3.12", "3.13"] - mysql_ver: ["8.0"] - include: - - py_ver: "3.9" - mysql_ver: "5.7" steps: - uses: actions/checkout@v4 - - name: Set up Python ${{matrix.py_ver}} - uses: actions/setup-python@v5 + + - name: Set up pixi + uses: prefix-dev/setup-pixi@v0.9.3 with: - python-version: ${{matrix.py_ver}} - - name: Integration test - env: - PY_VER: ${{matrix.py_ver}} - MYSQL_VER: ${{matrix.mysql_ver}} - # taking default variables set in docker-compose.yaml to sync with local test - run: | - export HOST_UID=$(id -u) - docker compose --profile test up --quiet-pull --build --exit-code-from djtest djtest + cache: true + locked: false + + - name: Run tests + run: pixi run -e test test-cov + + # Unit tests run without containers (faster feedback) + unit-tests: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up pixi + uses: prefix-dev/setup-pixi@v0.9.3 + with: + cache: true + locked: false + + - name: Run unit tests + run: pixi run -e test pytest tests/unit -v diff --git a/.gitignore b/.gitignore index f506fcb59..3c88c420c 100644 --- a/.gitignore +++ b/.gitignore @@ -185,3 +185,10 @@ cython_debug/ dj_local_conf.json *.env !.vscode/launch.json +# pixi environments +.pixi +_content/ + +# Local config +.secrets/ +datajoint.json diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4a58e0483..218134d62 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,8 +1,7 @@ -# pip install datajoint[test] # pre-commit install # pre-commit run --all-files # pre-commit autoupdate -# SKIP=flake8 git commit -m "foo" +# SKIP=ruff git commit -m "foo" # See https://pre-commit.com for more information # See https://pre-commit.com/hooks.html for more hooks @@ -20,43 +19,40 @@ repos: rev: v2.4.1 hooks: - id: codespell -- repo: https://github.com/pycqa/isort - rev: 6.0.1 # Use the latest stable version + args: [--toml, pyproject.toml] +- repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.8.4 hooks: - - id: isort - args: - - --profile=black # Optional, makes isort compatible with Black -- repo: https://github.com/psf/black - rev: 25.1.0 # matching versions in pyproject.toml and github actions - hooks: - - id: black - args: ["--check", "-v", "datajoint", "tests", "--diff"] # --required-version is conflicting with pre-commit -- repo: https://github.com/PyCQA/flake8 - rev: 7.3.0 - hooks: - # syntax tests - - id: flake8 - args: - - --select=E9,F63,F7,F82 - - --count - - --show-source - - --statistics - files: datajoint # a lot of files in tests are not compliant - # style tests - - id: flake8 - args: - - --ignore=E203,E722,W503 - - --count - - --max-complexity=62 - - --max-line-length=127 - - --statistics - - --per-file-ignores=datajoint/diagram.py:C901 - files: datajoint # a lot of files in tests are not compliant + # Run the linter + - id: ruff + args: [--fix] + files: ^(src/|tests/) + # Run the formatter + - id: ruff-format + files: ^(src/|tests/) - repo: https://github.com/rhysd/actionlint rev: v1.7.7 hooks: # lint github actions workflow yaml - id: actionlint - -## Suggest to add pytest hook that runs unit test | Prerequisite: split unit/integration test -## https://github.com/datajoint/datajoint-python/issues/1211 +- repo: https://github.com/pre-commit/mirrors-mypy + rev: v1.14.1 + hooks: + - id: mypy + files: ^src/datajoint/ + additional_dependencies: + - pydantic + - pydantic-settings + - types-PyMySQL + - types-tqdm + - pandas-stubs + - numpy +- repo: local + hooks: + - id: unit-tests + name: unit tests + entry: pytest tests/unit/ -v --tb=short + language: system + pass_filenames: false + always_run: true + stages: [pre-commit] diff --git a/DOCSTRING_STYLE.md b/DOCSTRING_STYLE.md new file mode 100644 index 000000000..77b6dc90a --- /dev/null +++ b/DOCSTRING_STYLE.md @@ -0,0 +1,499 @@ +# DataJoint Python Docstring Style Guide + +This document defines the canonical docstring format for datajoint-python. +All public APIs must follow this NumPy-style format for consistency and +automated documentation generation via mkdocstrings. + +## Quick Reference + +```python +def function(param1, param2, *, keyword_only=None): + """ + Short one-line summary (imperative mood, no period). + + Extended description providing context and details. May span + multiple lines. Explain what the function does, not how. + + Parameters + ---------- + param1 : type + Description of param1. + param2 : type + Description of param2. + keyword_only : type, optional + Description. Default is None. + + Returns + ------- + type + Description of return value. + + Raises + ------ + ExceptionType + When and why this exception is raised. + + Examples + -------- + >>> result = function("value", 42) + >>> print(result) + expected_output + + See Also + -------- + related_function : Brief description. + + Notes + ----- + Additional technical notes, algorithms, or implementation details. + """ +``` + +--- + +## Module Docstrings + +Every module must begin with a docstring explaining its purpose. + +```python +""" +Connection management for DataJoint. + +This module provides the Connection class that manages database connections, +transaction handling, and query execution. It also provides the ``conn()`` +function for accessing a persistent shared connection. + +Key Components +-------------- +Connection : class + Manages a single database connection with transaction support. +conn : function + Returns a persistent connection object shared across modules. + +Example +------- +>>> import datajoint as dj +>>> connection = dj.conn() +>>> connection.query("SHOW DATABASES") +""" +``` + +--- + +## Class Docstrings + +```python +class Table(QueryExpression): + """ + Base class for all DataJoint tables. + + Table implements data manipulation (insert, delete, update) and inherits + query functionality from QueryExpression. Concrete table classes must + define the ``definition`` property specifying the table structure. + + Parameters + ---------- + None + Tables are typically instantiated via schema decoration, not directly. + + Attributes + ---------- + definition : str + DataJoint table definition string (DDL). + primary_key : list of str + Names of primary key attributes. + heading : Heading + Table heading with attribute metadata. + + Examples + -------- + Define a table using the schema decorator: + + >>> @schema + ... class Mouse(dj.Manual): + ... definition = ''' + ... mouse_id : int + ... --- + ... dob : date + ... sex : enum("M", "F", "U") + ... ''' + + Insert data: + + >>> Mouse.insert1({"mouse_id": 1, "dob": "2024-01-15", "sex": "M"}) + + See Also + -------- + Manual : Table for manually entered data. + Computed : Table for computed results. + QueryExpression : Query operator base class. + """ +``` + +--- + +## Method Docstrings + +### Standard Method + +```python +def insert(self, rows, *, replace=False, skip_duplicates=False, ignore_extra_fields=False): + """ + Insert one or more rows into the table. + + Parameters + ---------- + rows : iterable + Rows to insert. Each row can be: + - dict: ``{"attr": value, ...}`` + - numpy.void: Record array element + - sequence: Values in heading order + - QueryExpression: Results of a query + - pathlib.Path: Path to CSV file + replace : bool, optional + If True, replace existing rows with matching primary keys. + Default is False. + skip_duplicates : bool, optional + If True, silently skip rows that would cause duplicate key errors. + Default is False. + ignore_extra_fields : bool, optional + If True, ignore fields not in the table heading. + Default is False. + + Returns + ------- + None + + Raises + ------ + DuplicateError + When inserting a row with an existing primary key and neither + ``replace`` nor ``skip_duplicates`` is True. + DataJointError + When required attributes are missing or types are incompatible. + + Examples + -------- + Insert a single row: + + >>> Mouse.insert1({"mouse_id": 1, "dob": "2024-01-15", "sex": "M"}) + + Insert multiple rows: + + >>> Mouse.insert([ + ... {"mouse_id": 2, "dob": "2024-02-01", "sex": "F"}, + ... {"mouse_id": 3, "dob": "2024-02-15", "sex": "M"}, + ... ]) + + Insert from a query: + + >>> TargetTable.insert(SourceTable & "condition > 5") + + See Also + -------- + insert1 : Insert exactly one row. + """ +``` + +### Method with Complex Return + +```python +def fetch(self, *attrs, offset=None, limit=None, order_by=None, format=None, as_dict=False): + """ + Retrieve data from the table. + + Parameters + ---------- + *attrs : str + Attribute names to fetch. If empty, fetches all attributes. + Use "KEY" to fetch primary key as dict. + offset : int, optional + Number of rows to skip. Default is None (no offset). + limit : int, optional + Maximum number of rows to return. Default is None (no limit). + order_by : str or list of str, optional + Attribute(s) to sort by. Use "KEY" for primary key order, + append " DESC" for descending. Default is None (unordered). + format : {"array", "frame"}, optional + Output format when fetching all attributes: + - "array": numpy structured array (default) + - "frame": pandas DataFrame + as_dict : bool, optional + If True, return list of dicts instead of structured array. + Default is False. + + Returns + ------- + numpy.ndarray or list of dict or pandas.DataFrame + Query results in the requested format: + - Single attribute: 1D array of values + - Multiple attributes: tuple of 1D arrays + - No attributes specified: structured array, DataFrame, or list of dicts + + Examples + -------- + Fetch all data as structured array: + + >>> data = Mouse.fetch() + + Fetch specific attributes: + + >>> ids, dobs = Mouse.fetch("mouse_id", "dob") + + Fetch as list of dicts: + + >>> rows = Mouse.fetch(as_dict=True) + >>> for row in rows: + ... print(row["mouse_id"]) + + Fetch with ordering and limit: + + >>> recent = Mouse.fetch(order_by="dob DESC", limit=10) + + See Also + -------- + fetch1 : Fetch exactly one row. + head : Fetch first N rows ordered by key. + tail : Fetch last N rows ordered by key. + """ +``` + +### Generator Method + +```python +def make(self, key): + """ + Compute and insert results for one key. + + This method must be implemented by subclasses of Computed or Imported + tables. It is called by ``populate()`` for each key in ``key_source`` + that is not yet in the table. + + The method can be implemented in two ways: + + **Simple mode** (regular method): + Fetch, compute, and insert within a single transaction. + + **Tripartite mode** (generator method): + Split into ``make_fetch``, ``make_compute``, ``make_insert`` for + long-running computations with deferred transactions. + + Parameters + ---------- + key : dict + Primary key values identifying the entity to compute. + + Yields + ------ + tuple + In tripartite mode, yields fetched data and computed results. + + Raises + ------ + NotImplementedError + If neither ``make`` nor the tripartite methods are implemented. + + Examples + -------- + Simple implementation: + + >>> class ProcessedData(dj.Computed): + ... definition = ''' + ... -> RawData + ... --- + ... result : float + ... ''' + ... + ... def make(self, key): + ... raw = (RawData & key).fetch1("data") + ... result = expensive_computation(raw) + ... self.insert1({**key, "result": result}) + + See Also + -------- + populate : Execute make for all pending keys. + key_source : Query defining keys to populate. + """ +``` + +--- + +## Property Docstrings + +```python +@property +def primary_key(self): + """ + list of str : Names of primary key attributes. + + The primary key uniquely identifies each row in the table. + Derived from the table definition. + + Examples + -------- + >>> Mouse.primary_key + ['mouse_id'] + """ + return self.heading.primary_key +``` + +--- + +## Parameter Types + +Use these type annotations in docstrings: + +| Python Type | Docstring Format | +|-------------|------------------| +| `str` | `str` | +| `int` | `int` | +| `float` | `float` | +| `bool` | `bool` | +| `None` | `None` | +| `list` | `list` or `list of str` | +| `dict` | `dict` or `dict[str, int]` | +| `tuple` | `tuple` or `tuple of (str, int)` | +| Optional | `str or None` or `str, optional` | +| Union | `str or int` | +| Literal | `{"option1", "option2"}` | +| Callable | `callable` | +| Class | `ClassName` | +| Any | `object` | + +--- + +## Section Order + +Sections must appear in this order (include only relevant sections): + +1. **Short Summary** (required) - One line, imperative mood +2. **Deprecation Warning** - If applicable +3. **Extended Summary** - Additional context +4. **Parameters** - Input arguments +5. **Returns** / **Yields** - Output values +6. **Raises** - Exceptions +7. **Warns** - Warnings issued +8. **See Also** - Related functions/classes +9. **Notes** - Technical details +10. **References** - Citations +11. **Examples** (strongly encouraged) - Usage demonstrations + +--- + +## Style Rules + +### Do + +- Use imperative mood: "Insert rows" not "Inserts rows" +- Start with capital letter, no period at end of summary +- Document all public methods +- Include at least one example for public APIs +- Use backticks for code: ``parameter``, ``ClassName`` +- Reference related items in See Also + +### Don't + +- Don't document private methods extensively (brief is fine) +- Don't repeat the function signature in the description +- Don't use "This function..." or "This method..." +- Don't include implementation details in Parameters +- Don't use first person ("I", "we") + +--- + +## Examples Section Best Practices + +```python +""" +Examples +-------- +Basic usage: + +>>> table.insert1({"id": 1, "value": 42}) + +With options: + +>>> table.insert(rows, skip_duplicates=True) + +Error handling: + +>>> try: +... table.insert1({"id": 1}) # duplicate +... except dj.errors.DuplicateError: +... print("Already exists") +Already exists +""" +``` + +--- + +## Converting from Sphinx Style + +Replace Sphinx-style docstrings: + +```python +# Before (Sphinx style) +def method(self, param1, param2): + """ + Brief description. + + :param param1: Description of param1. + :type param1: str + :param param2: Description of param2. + :type param2: int + :returns: Description of return value. + :rtype: bool + :raises ValueError: When param1 is empty. + """ + +# After (NumPy style) +def method(self, param1, param2): + """ + Brief description. + + Parameters + ---------- + param1 : str + Description of param1. + param2 : int + Description of param2. + + Returns + ------- + bool + Description of return value. + + Raises + ------ + ValueError + When param1 is empty. + """ +``` + +--- + +## Validation + +Docstrings are validated by: + +1. **mkdocstrings** - Parses for API documentation +2. **ruff** - Linting (D100-D417 rules when enabled) +3. **pytest --doctest-modules** - Executes examples + +Run locally: + +```bash +# Build docs to check parsing +mkdocs build --config-file docs/mkdocs.yaml + +# Check docstring examples +pytest --doctest-modules src/datajoint/ +``` + +--- + +## References + +- [NumPy Docstring Guide](https://numpydoc.readthedocs.io/en/latest/format.html) +- [mkdocstrings Python Handler](https://mkdocstrings.github.io/python/) +- [PEP 257 - Docstring Conventions](https://peps.python.org/pep-0257/) diff --git a/Dockerfile b/Dockerfile index 0d727f6b4..780e1c540 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,7 +11,7 @@ RUN ${CONDA_BIN} install --no-pin -qq -y -n base -c conda-forge \ ENV PATH="$PATH:/home/mambauser/.local/bin" COPY --chown=${HOST_UID:-1000}:mambauser ./pyproject.toml ./README.md ./LICENSE.txt /main/ -COPY --chown=${HOST_UID:-1000}:mambauser ./datajoint /main/datajoint +COPY --chown=${HOST_UID:-1000}:mambauser ./src/datajoint /main/src/datajoint VOLUME /src WORKDIR /src diff --git a/LICENSE b/LICENSE new file mode 100644 index 000000000..3f8b99424 --- /dev/null +++ b/LICENSE @@ -0,0 +1,190 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to the Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + Copyright 2014-2026 DataJoint Inc. and contributors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/LICENSE.txt b/LICENSE.txt deleted file mode 100644 index 90f4edaaa..000000000 --- a/LICENSE.txt +++ /dev/null @@ -1,504 +0,0 @@ - GNU LESSER GENERAL PUBLIC LICENSE - Version 2.1, February 1999 - - Copyright (C) 1991, 1999 Free Software Foundation, Inc. - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - Everyone is permitted to copy and distribute verbatim copies - of this license document, but changing it is not allowed. - -(This is the first released version of the Lesser GPL. It also counts - as the successor of the GNU Library Public License, version 2, hence - the version number 2.1.) - - Preamble - - The licenses for most software are designed to take away your -freedom to share and change it. By contrast, the GNU General Public -Licenses are intended to guarantee your freedom to share and change -free software--to make sure the software is free for all its users. - - This license, the Lesser General Public License, applies to some -specially designated software packages--typically libraries--of the -Free Software Foundation and other authors who decide to use it. You -can use it too, but we suggest you first think carefully about whether -this license or the ordinary General Public License is the better -strategy to use in any particular case, based on the explanations below. - - When we speak of free software, we are referring to freedom of use, -not price. Our General Public Licenses are designed to make sure that -you have the freedom to distribute copies of free software (and charge -for this service if you wish); that you receive source code or can get -it if you want it; that you can change the software and use pieces of -it in new free programs; and that you are informed that you can do -these things. - - To protect your rights, we need to make restrictions that forbid -distributors to deny you these rights or to ask you to surrender these -rights. These restrictions translate to certain responsibilities for -you if you distribute copies of the library or if you modify it. - - For example, if you distribute copies of the library, whether gratis -or for a fee, you must give the recipients all the rights that we gave -you. You must make sure that they, too, receive or can get the source -code. If you link other code with the library, you must provide -complete object files to the recipients, so that they can relink them -with the library after making changes to the library and recompiling -it. And you must show them these terms so they know their rights. - - We protect your rights with a two-step method: (1) we copyright the -library, and (2) we offer you this license, which gives you legal -permission to copy, distribute and/or modify the library. - - To protect each distributor, we want to make it very clear that -there is no warranty for the free library. Also, if the library is -modified by someone else and passed on, the recipients should know -that what they have is not the original version, so that the original -author's reputation will not be affected by problems that might be -introduced by others. - - Finally, software patents pose a constant threat to the existence of -any free program. We wish to make sure that a company cannot -effectively restrict the users of a free program by obtaining a -restrictive license from a patent holder. Therefore, we insist that -any patent license obtained for a version of the library must be -consistent with the full freedom of use specified in this license. - - Most GNU software, including some libraries, is covered by the -ordinary GNU General Public License. This license, the GNU Lesser -General Public License, applies to certain designated libraries, and -is quite different from the ordinary General Public License. We use -this license for certain libraries in order to permit linking those -libraries into non-free programs. - - When a program is linked with a library, whether statically or using -a shared library, the combination of the two is legally speaking a -combined work, a derivative of the original library. The ordinary -General Public License therefore permits such linking only if the -entire combination fits its criteria of freedom. The Lesser General -Public License permits more lax criteria for linking other code with -the library. - - We call this license the "Lesser" General Public License because it -does Less to protect the user's freedom than the ordinary General -Public License. It also provides other free software developers Less -of an advantage over competing non-free programs. These disadvantages -are the reason we use the ordinary General Public License for many -libraries. However, the Lesser license provides advantages in certain -special circumstances. - - For example, on rare occasions, there may be a special need to -encourage the widest possible use of a certain library, so that it becomes -a de-facto standard. To achieve this, non-free programs must be -allowed to use the library. A more frequent case is that a free -library does the same job as widely used non-free libraries. In this -case, there is little to gain by limiting the free library to free -software only, so we use the Lesser General Public License. - - In other cases, permission to use a particular library in non-free -programs enables a greater number of people to use a large body of -free software. For example, permission to use the GNU C Library in -non-free programs enables many more people to use the whole GNU -operating system, as well as its variant, the GNU/Linux operating -system. - - Although the Lesser General Public License is Less protective of the -users' freedom, it does ensure that the user of a program that is -linked with the Library has the freedom and the wherewithal to run -that program using a modified version of the Library. - - The precise terms and conditions for copying, distribution and -modification follow. Pay close attention to the difference between a -"work based on the library" and a "work that uses the library". The -former contains code derived from the library, whereas the latter must -be combined with the library in order to run. - - GNU LESSER GENERAL PUBLIC LICENSE - TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION - - 0. This License Agreement applies to any software library or other -program which contains a notice placed by the copyright holder or -other authorized party saying it may be distributed under the terms of -this Lesser General Public License (also called "this License"). -Each licensee is addressed as "you". - - A "library" means a collection of software functions and/or data -prepared so as to be conveniently linked with application programs -(which use some of those functions and data) to form executables. - - The "Library", below, refers to any such software library or work -which has been distributed under these terms. A "work based on the -Library" means either the Library or any derivative work under -copyright law: that is to say, a work containing the Library or a -portion of it, either verbatim or with modifications and/or translated -straightforwardly into another language. (Hereinafter, translation is -included without limitation in the term "modification".) - - "Source code" for a work means the preferred form of the work for -making modifications to it. For a library, complete source code means -all the source code for all modules it contains, plus any associated -interface definition files, plus the scripts used to control compilation -and installation of the library. - - Activities other than copying, distribution and modification are not -covered by this License; they are outside its scope. The act of -running a program using the Library is not restricted, and output from -such a program is covered only if its contents constitute a work based -on the Library (independent of the use of the Library in a tool for -writing it). Whether that is true depends on what the Library does -and what the program that uses the Library does. - - 1. You may copy and distribute verbatim copies of the Library's -complete source code as you receive it, in any medium, provided that -you conspicuously and appropriately publish on each copy an -appropriate copyright notice and disclaimer of warranty; keep intact -all the notices that refer to this License and to the absence of any -warranty; and distribute a copy of this License along with the -Library. - - You may charge a fee for the physical act of transferring a copy, -and you may at your option offer warranty protection in exchange for a -fee. - - 2. You may modify your copy or copies of the Library or any portion -of it, thus forming a work based on the Library, and copy and -distribute such modifications or work under the terms of Section 1 -above, provided that you also meet all of these conditions: - - a) The modified work must itself be a software library. - - b) You must cause the files modified to carry prominent notices - stating that you changed the files and the date of any change. - - c) You must cause the whole of the work to be licensed at no - charge to all third parties under the terms of this License. - - d) If a facility in the modified Library refers to a function or a - table of data to be supplied by an application program that uses - the facility, other than as an argument passed when the facility - is invoked, then you must make a good faith effort to ensure that, - in the event an application does not supply such function or - table, the facility still operates, and performs whatever part of - its purpose remains meaningful. - - (For example, a function in a library to compute square roots has - a purpose that is entirely well-defined independent of the - application. Therefore, Subsection 2d requires that any - application-supplied function or table used by this function must - be optional: if the application does not supply it, the square - root function must still compute square roots.) - -These requirements apply to the modified work as a whole. If -identifiable sections of that work are not derived from the Library, -and can be reasonably considered independent and separate works in -themselves, then this License, and its terms, do not apply to those -sections when you distribute them as separate works. But when you -distribute the same sections as part of a whole which is a work based -on the Library, the distribution of the whole must be on the terms of -this License, whose permissions for other licensees extend to the -entire whole, and thus to each and every part regardless of who wrote -it. - -Thus, it is not the intent of this section to claim rights or contest -your rights to work written entirely by you; rather, the intent is to -exercise the right to control the distribution of derivative or -collective works based on the Library. - -In addition, mere aggregation of another work not based on the Library -with the Library (or with a work based on the Library) on a volume of -a storage or distribution medium does not bring the other work under -the scope of this License. - - 3. You may opt to apply the terms of the ordinary GNU General Public -License instead of this License to a given copy of the Library. To do -this, you must alter all the notices that refer to this License, so -that they refer to the ordinary GNU General Public License, version 2, -instead of to this License. (If a newer version than version 2 of the -ordinary GNU General Public License has appeared, then you can specify -that version instead if you wish.) Do not make any other change in -these notices. - - Once this change is made in a given copy, it is irreversible for -that copy, so the ordinary GNU General Public License applies to all -subsequent copies and derivative works made from that copy. - - This option is useful when you wish to copy part of the code of -the Library into a program that is not a library. - - 4. You may copy and distribute the Library (or a portion or -derivative of it, under Section 2) in object code or executable form -under the terms of Sections 1 and 2 above provided that you accompany -it with the complete corresponding machine-readable source code, which -must be distributed under the terms of Sections 1 and 2 above on a -medium customarily used for software interchange. - - If distribution of object code is made by offering access to copy -from a designated place, then offering equivalent access to copy the -source code from the same place satisfies the requirement to -distribute the source code, even though third parties are not -compelled to copy the source along with the object code. - - 5. A program that contains no derivative of any portion of the -Library, but is designed to work with the Library by being compiled or -linked with it, is called a "work that uses the Library". Such a -work, in isolation, is not a derivative work of the Library, and -therefore falls outside the scope of this License. - - However, linking a "work that uses the Library" with the Library -creates an executable that is a derivative of the Library (because it -contains portions of the Library), rather than a "work that uses the -library". The executable is therefore covered by this License. -Section 6 states terms for distribution of such executables. - - When a "work that uses the Library" uses material from a header file -that is part of the Library, the object code for the work may be a -derivative work of the Library even though the source code is not. -Whether this is true is especially significant if the work can be -linked without the Library, or if the work is itself a library. The -threshold for this to be true is not precisely defined by law. - - If such an object file uses only numerical parameters, data -structure layouts and accessors, and small macros and small inline -functions (ten lines or less in length), then the use of the object -file is unrestricted, regardless of whether it is legally a derivative -work. (Executables containing this object code plus portions of the -Library will still fall under Section 6.) - - Otherwise, if the work is a derivative of the Library, you may -distribute the object code for the work under the terms of Section 6. -Any executables containing that work also fall under Section 6, -whether or not they are linked directly with the Library itself. - - 6. As an exception to the Sections above, you may also combine or -link a "work that uses the Library" with the Library to produce a -work containing portions of the Library, and distribute that work -under terms of your choice, provided that the terms permit -modification of the work for the customer's own use and reverse -engineering for debugging such modifications. - - You must give prominent notice with each copy of the work that the -Library is used in it and that the Library and its use are covered by -this License. You must supply a copy of this License. If the work -during execution displays copyright notices, you must include the -copyright notice for the Library among them, as well as a reference -directing the user to the copy of this License. Also, you must do one -of these things: - - a) Accompany the work with the complete corresponding - machine-readable source code for the Library including whatever - changes were used in the work (which must be distributed under - Sections 1 and 2 above); and, if the work is an executable linked - with the Library, with the complete machine-readable "work that - uses the Library", as object code and/or source code, so that the - user can modify the Library and then relink to produce a modified - executable containing the modified Library. (It is understood - that the user who changes the contents of definitions files in the - Library will not necessarily be able to recompile the application - to use the modified definitions.) - - b) Use a suitable shared library mechanism for linking with the - Library. A suitable mechanism is one that (1) uses at run time a - copy of the library already present on the user's computer system, - rather than copying library functions into the executable, and (2) - will operate properly with a modified version of the library, if - the user installs one, as long as the modified version is - interface-compatible with the version that the work was made with. - - c) Accompany the work with a written offer, valid for at - least three years, to give the same user the materials - specified in Subsection 6a, above, for a charge no more - than the cost of performing this distribution. - - d) If distribution of the work is made by offering access to copy - from a designated place, offer equivalent access to copy the above - specified materials from the same place. - - e) Verify that the user has already received a copy of these - materials or that you have already sent this user a copy. - - For an executable, the required form of the "work that uses the -Library" must include any data and utility programs needed for -reproducing the executable from it. However, as a special exception, -the materials to be distributed need not include anything that is -normally distributed (in either source or binary form) with the major -components (compiler, kernel, and so on) of the operating system on -which the executable runs, unless that component itself accompanies -the executable. - - It may happen that this requirement contradicts the license -restrictions of other proprietary libraries that do not normally -accompany the operating system. Such a contradiction means you cannot -use both them and the Library together in an executable that you -distribute. - - 7. You may place library facilities that are a work based on the -Library side-by-side in a single library together with other library -facilities not covered by this License, and distribute such a combined -library, provided that the separate distribution of the work based on -the Library and of the other library facilities is otherwise -permitted, and provided that you do these two things: - - a) Accompany the combined library with a copy of the same work - based on the Library, uncombined with any other library - facilities. This must be distributed under the terms of the - Sections above. - - b) Give prominent notice with the combined library of the fact - that part of it is a work based on the Library, and explaining - where to find the accompanying uncombined form of the same work. - - 8. You may not copy, modify, sublicense, link with, or distribute -the Library except as expressly provided under this License. Any -attempt otherwise to copy, modify, sublicense, link with, or -distribute the Library is void, and will automatically terminate your -rights under this License. However, parties who have received copies, -or rights, from you under this License will not have their licenses -terminated so long as such parties remain in full compliance. - - 9. You are not required to accept this License, since you have not -signed it. However, nothing else grants you permission to modify or -distribute the Library or its derivative works. These actions are -prohibited by law if you do not accept this License. Therefore, by -modifying or distributing the Library (or any work based on the -Library), you indicate your acceptance of this License to do so, and -all its terms and conditions for copying, distributing or modifying -the Library or works based on it. - - 10. Each time you redistribute the Library (or any work based on the -Library), the recipient automatically receives a license from the -original licensor to copy, distribute, link with or modify the Library -subject to these terms and conditions. You may not impose any further -restrictions on the recipients' exercise of the rights granted herein. -You are not responsible for enforcing compliance by third parties with -this License. - - 11. If, as a consequence of a court judgment or allegation of patent -infringement or for any other reason (not limited to patent issues), -conditions are imposed on you (whether by court order, agreement or -otherwise) that contradict the conditions of this License, they do not -excuse you from the conditions of this License. If you cannot -distribute so as to satisfy simultaneously your obligations under this -License and any other pertinent obligations, then as a consequence you -may not distribute the Library at all. For example, if a patent -license would not permit royalty-free redistribution of the Library by -all those who receive copies directly or indirectly through you, then -the only way you could satisfy both it and this License would be to -refrain entirely from distribution of the Library. - -If any portion of this section is held invalid or unenforceable under any -particular circumstance, the balance of the section is intended to apply, -and the section as a whole is intended to apply in other circumstances. - -It is not the purpose of this section to induce you to infringe any -patents or other property right claims or to contest validity of any -such claims; this section has the sole purpose of protecting the -integrity of the free software distribution system which is -implemented by public license practices. Many people have made -generous contributions to the wide range of software distributed -through that system in reliance on consistent application of that -system; it is up to the author/donor to decide if he or she is willing -to distribute software through any other system and a licensee cannot -impose that choice. - -This section is intended to make thoroughly clear what is believed to -be a consequence of the rest of this License. - - 12. If the distribution and/or use of the Library is restricted in -certain countries either by patents or by copyrighted interfaces, the -original copyright holder who places the Library under this License may add -an explicit geographical distribution limitation excluding those countries, -so that distribution is permitted only in or among countries not thus -excluded. In such case, this License incorporates the limitation as if -written in the body of this License. - - 13. The Free Software Foundation may publish revised and/or new -versions of the Lesser General Public License from time to time. -Such new versions will be similar in spirit to the present version, -but may differ in detail to address new problems or concerns. - -Each version is given a distinguishing version number. If the Library -specifies a version number of this License which applies to it and -"any later version", you have the option of following the terms and -conditions either of that version or of any later version published by -the Free Software Foundation. If the Library does not specify a -license version number, you may choose any version ever published by -the Free Software Foundation. - - 14. If you wish to incorporate parts of the Library into other free -programs whose distribution conditions are incompatible with these, -write to the author to ask for permission. For software which is -copyrighted by the Free Software Foundation, write to the Free -Software Foundation; we sometimes make exceptions for this. Our -decision will be guided by the two goals of preserving the free status -of all derivatives of our free software and of promoting the sharing -and reuse of software generally. - - NO WARRANTY - - 15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO -WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW. -EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR -OTHER PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY -KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE -LIBRARY IS WITH YOU. SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME -THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. - - 16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN -WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY -AND/OR REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU -FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR -CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE -LIBRARY (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING -RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A -FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF -SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH -DAMAGES. - - END OF TERMS AND CONDITIONS - - How to Apply These Terms to Your New Libraries - - If you develop a new library, and you want it to be of the greatest -possible use to the public, we recommend making it free software that -everyone can redistribute and change. You can do so by permitting -redistribution under these terms (or, alternatively, under the terms of the -ordinary General Public License). - - To apply these terms, attach the following notices to the library. It is -safest to attach them to the start of each source file to most effectively -convey the exclusion of warranty; and each file should have at least the -"copyright" line and a pointer to where the full notice is found. - - {description} - Copyright (C) {year} {fullname} - - This library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - This library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with this library; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 - USA - -Also add information on how to contact you by electronic and paper mail. - -You should also get your employer (if you work as a programmer) or your -school, if any, to sign a "copyright disclaimer" for the library, if -necessary. Here is a sample; alter the names: - - Yoyodyne, Inc., hereby disclaims all copyright interest in the - library `Frob' (a library for tweaking knobs) written by James Random - Hacker. - - {signature of Ty Coon}, 1 April 1990 - Ty Coon, President of Vice - -That's all there is to it! diff --git a/README.md b/README.md index e582c8ec5..40a1a2c7c 100644 --- a/README.md +++ b/README.md @@ -1,116 +1,69 @@ -# Welcome to DataJoint for Python! +# DataJoint for Python + +DataJoint is a framework for scientific data pipelines that introduces the **Relational Workflow Model**—a paradigm where your database schema is an executable specification of your workflow. + +Traditional databases store data but don't understand how it was computed. DataJoint extends relational databases with native workflow semantics: + +- **Tables represent workflow steps** — Each table is a step in your pipeline where entities are created +- **Foreign keys encode dependencies** — Parent tables must be populated before child tables +- **Computations are declarative** — Define *what* to compute; DataJoint determines *when* and tracks *what's done* +- **Results are immutable** — Computed results preserve full provenance and reproducibility + +### Object-Augmented Schemas + +Scientific data includes both structured metadata and large data objects (time series, images, movies, neural recordings, gene sequences). DataJoint solves this with **Object-Augmented Schemas (OAS)**—a unified architecture where relational tables and object storage are managed as one system with identical guarantees for integrity, transactions, and lifecycle. + +### DataJoint 2.0 + +**DataJoint 2.0** solidifies these core concepts with a modernized API, improved type system, and enhanced object storage integration. Existing users can refer to the [Migration Guide](https://docs.datajoint.com/migration/) for upgrading from earlier versions. + +**Documentation:** https://docs.datajoint.com - - - - - + - - - - - + - - - - - - - - - - - - - - + + - - - - -
PyPI pypi release -
- - pypi downloads -
Conda ForgeConda conda-forge release -
- - conda-forge downloads - -
Since Release - - commit since last release -
Test StatusTests test status
Release Status - - release status - -
Doc Status - - doc status - -
Coverage - coverage + coverage
Developer Chat - - datajoint slack - -
License - - LGPL-2.1 + + Apache-2.0 + + Citation + + DOI
Citation - - bioRxiv - -
- - zenodo - -
-DataJoint for Python is a framework for scientific workflow management based on -relational principles. DataJoint is built on the foundation of the relational data -model and prescribes a consistent method for organizing, populating, computing, and -querying data. - -DataJoint was initially developed in 2009 by Dimitri Yatsenko in Andreas Tolias' Lab at -Baylor College of Medicine for the distributed processing and management of large -volumes of data streaming from regular experiments. Starting in 2011, DataJoint has -been available as an open-source project adopted by other labs and improved through -contributions from several developers. -Presently, the primary developer of DataJoint open-source software is the company -DataJoint (). - ## Data Pipeline Example ![pipeline](https://raw.githubusercontent.com/datajoint/datajoint-python/master/images/pipeline.png) @@ -141,3 +94,129 @@ DataJoint (). - [Contribution Guidelines](https://docs.datajoint.com/about/contribute/) - [Developer Guide](https://docs.datajoint.com/core/datajoint-python/latest/develop/) + +## Developer Guide + +### Prerequisites + +- [Docker](https://docs.docker.com/get-docker/) (Docker daemon must be running) +- [pixi](https://pixi.sh) (recommended) or Python 3.10+ + +### Quick Start with pixi (Recommended) + +[pixi](https://pixi.sh) manages all dependencies including Python, graphviz, and test tools: + +```bash +# Clone the repo +git clone https://github.com/datajoint/datajoint-python.git +cd datajoint-python + +# Install dependencies and run tests (containers managed by testcontainers) +pixi run test + +# Run with coverage +pixi run test-cov + +# Run pre-commit hooks +pixi run pre-commit run --all-files +``` + +### Running Tests + +Tests use [testcontainers](https://testcontainers.com/) to automatically manage MySQL and MinIO containers. +**No manual `docker-compose up` required** - containers start when tests run and stop afterward. + +```bash +# Run all tests (recommended) +pixi run test + +# Run with coverage report +pixi run test-cov + +# Run only unit tests (no containers needed) +pixi run -e test pytest tests/unit/ + +# Run specific test file +pixi run -e test pytest tests/integration/test_blob.py -v +``` + +**macOS Docker Desktop users:** If tests fail to connect to Docker, set `DOCKER_HOST`: +```bash +export DOCKER_HOST=unix://$HOME/.docker/run/docker.sock +``` + +### Alternative: Using pip + +If you prefer pip over pixi: + +```bash +pip install -e ".[test]" +pytest tests/ +``` + +### Alternative: External Containers + +For development/debugging, you may prefer persistent containers that survive test runs: + +```bash +# Start containers manually +docker compose up -d db minio + +# Run tests using external containers +DJ_USE_EXTERNAL_CONTAINERS=1 pixi run test +# Or with pip: DJ_USE_EXTERNAL_CONTAINERS=1 pytest tests/ + +# Stop containers when done +docker compose down +``` + +### Alternative: Full Docker + +Run tests entirely in Docker (no local Python needed): + +```bash +docker compose --profile test up djtest --build +``` + +### Pre-commit Hooks + +Pre-commit hooks run automatically on `git commit` to check code quality. +**All hooks must pass before committing.** + +```bash +# Install hooks (first time only) +pixi run pre-commit install +# Or with pip: pip install pre-commit && pre-commit install + +# Run all checks manually +pixi run pre-commit run --all-files + +# Run specific hook +pixi run pre-commit run ruff --all-files +``` + +Hooks include: +- **ruff**: Python linting and formatting +- **codespell**: Spell checking +- **YAML/JSON/TOML validation** +- **Large file detection** + +### Before Submitting a PR + +1. **Run all tests**: `pixi run test` +2. **Run pre-commit**: `pixi run pre-commit run --all-files` +3. **Check coverage**: `pixi run test-cov` + +### Environment Variables + +For external container mode (`DJ_USE_EXTERNAL_CONTAINERS=1`): + +| Variable | Default | Description | +|----------|---------|-------------| +| `DJ_HOST` | `localhost` | MySQL hostname | +| `DJ_PORT` | `3306` | MySQL port | +| `DJ_USER` | `root` | MySQL username | +| `DJ_PASS` | `password` | MySQL password | +| `S3_ENDPOINT` | `localhost:9000` | MinIO endpoint | + +For Docker-based testing (devcontainer, djtest), set `DJ_HOST=db` and `S3_ENDPOINT=minio:9000`. diff --git a/RELEASE_MEMO.md b/RELEASE_MEMO.md new file mode 100644 index 000000000..25fdc6ca0 --- /dev/null +++ b/RELEASE_MEMO.md @@ -0,0 +1,117 @@ +# DataJoint 2.0 Release Memo + +## PyPI Release Process + +### Steps + +1. **Run "Manual Draft Release" workflow** on GitHub Actions +2. **Edit the draft release**: + - Change release name to `Release 2.0.0` + - Change tag to `v2.0.0` +3. **Publish the release** +4. Automation will: + - Update `version.py` to `2.0.0` + - Build and publish to PyPI + - Create PR to merge version update back to master + +### Version Note + +The release drafter computes version from the previous tag (`v0.14.6`), so it would generate `0.14.7` or `0.15.0`. You must **manually edit** the release name to include `2.0.0`. + +The regex on line 42 of `post_draft_release_published.yaml` extracts version from the release name: +```bash +VERSION=$(echo "${{ github.event.release.name }}" | grep -oP '\d+\.\d+\.\d+') +``` + +--- + +## Conda-Forge Release Process + +DataJoint has a [conda-forge feedstock](https://github.com/conda-forge/datajoint-feedstock). + +### How Conda-Forge Updates Work + +Conda-forge has **automated bots** that detect new PyPI releases and create PRs automatically: + +1. **You publish to PyPI** (via the GitHub release workflow) +2. **regro-cf-autotick-bot** detects the new version within ~24 hours +3. **Bot creates a PR** to the feedstock with updated version and hash +4. **Maintainers review and merge** (you're listed as a maintainer) +5. **Package builds automatically** for all platforms + +### Manual Update (if bot doesn't trigger) + +If the bot doesn't create a PR, manually update the feedstock: + +1. **Fork** [conda-forge/datajoint-feedstock](https://github.com/conda-forge/datajoint-feedstock) + +2. **Edit `recipe/meta.yaml`**: + ```yaml + {% set version = "2.0.0" %} + + package: + name: datajoint + version: {{ version }} + + source: + url: https://pypi.io/packages/source/d/datajoint/datajoint-{{ version }}.tar.gz + sha256: + + build: + number: 0 # Reset to 0 for new version + ``` + +3. **Get the SHA256 hash**: + ```bash + curl -sL https://pypi.org/pypi/datajoint/2.0.0/json | jq -r '.urls[] | select(.packagetype=="sdist") | .digests.sha256' + ``` + +4. **Update license** (important for 2.0!): + ```yaml + about: + license: Apache-2.0 # Changed from LGPL-2.1-only + license_file: LICENSE + ``` + +5. **Submit PR** to the feedstock + +### Action Items for 2.0 Release + +1. **First**: Publish to PyPI via GitHub release (name it "Release 2.0.0") +2. **Wait**: ~24 hours for conda-forge bot to detect +3. **Check**: [datajoint-feedstock PRs](https://github.com/conda-forge/datajoint-feedstock/pulls) for auto-PR +4. **Review**: Ensure license changed from LGPL to Apache-2.0 +5. **Merge**: As maintainer, approve and merge the PR + +### Timeline + +| Step | When | +|------|------| +| PyPI release | Day 0 | +| Bot detects & creates PR | Day 0-1 | +| Review & merge PR | Day 1-2 | +| Conda-forge package available | Day 1-2 | + +### Verification + +After release: +```bash +conda search datajoint -c conda-forge +# Should show 2.0.0 +``` + +--- + +## Maintainers + +- @datajointbot +- @dimitri-yatsenko +- @drewyangdev +- @guzman-raphael +- @ttngu207 + +## Links + +- [datajoint-feedstock on GitHub](https://github.com/conda-forge/datajoint-feedstock) +- [datajoint on Anaconda.org](https://anaconda.org/conda-forge/datajoint) +- [datajoint on PyPI](https://pypi.org/project/datajoint/) diff --git a/activate.sh b/activate.sh new file mode 100644 index 000000000..1632accc8 --- /dev/null +++ b/activate.sh @@ -0,0 +1,4 @@ +#! /usr/bin/bash +# This script registers dot plugins so that we can use graphviz +# to write png images +dot -c \ No newline at end of file diff --git a/datajoint/__init__.py b/datajoint/__init__.py deleted file mode 100644 index a7c5e7b2f..000000000 --- a/datajoint/__init__.py +++ /dev/null @@ -1,78 +0,0 @@ -""" -DataJoint for Python is a framework for building data pipelines using MySQL databases -to represent pipeline structure and bulk storage systems for large objects. -DataJoint is built on the foundation of the relational data model and prescribes a -consistent method for organizing, populating, and querying data. - -The DataJoint data model is described in https://arxiv.org/abs/1807.11104 - -DataJoint is free software under the LGPL License. In addition, we request -that any use of DataJoint leading to a publication be acknowledged in the publication. - -Please cite: - - - http://biorxiv.org/content/early/2015/11/14/031658 - - http://dx.doi.org/10.1101/031658 -""" - -__author__ = "DataJoint Contributors" -__date__ = "November 7, 2020" -__all__ = [ - "__author__", - "__version__", - "config", - "conn", - "Connection", - "Schema", - "schema", - "VirtualModule", - "create_virtual_module", - "list_schemas", - "Table", - "FreeTable", - "Manual", - "Lookup", - "Imported", - "Computed", - "Part", - "Not", - "AndList", - "Top", - "U", - "Diagram", - "Di", - "ERD", - "set_password", - "kill", - "MatCell", - "MatStruct", - "AttributeAdapter", - "errors", - "DataJointError", - "key", - "key_hash", - "logger", - "cli", -] - -from . import errors -from .admin import kill, set_password -from .attribute_adapter import AttributeAdapter -from .blob import MatCell, MatStruct -from .cli import cli -from .connection import Connection, conn -from .diagram import Diagram -from .errors import DataJointError -from .expression import AndList, Not, Top, U -from .fetch import key -from .hash import key_hash -from .logging import logger -from .schemas import Schema, VirtualModule, list_schemas -from .settings import config -from .table import FreeTable, Table -from .user_tables import Computed, Imported, Lookup, Manual, Part -from .version import __version__ - -ERD = Di = Diagram # Aliases for Diagram -schema = Schema # Aliases for Schema -create_virtual_module = VirtualModule # Aliases for VirtualModule diff --git a/datajoint/admin.py b/datajoint/admin.py deleted file mode 100644 index e1eb803ec..000000000 --- a/datajoint/admin.py +++ /dev/null @@ -1,127 +0,0 @@ -import logging -from getpass import getpass - -import pymysql -from packaging import version - -from .connection import conn -from .settings import config -from .utils import user_choice - -logger = logging.getLogger(__name__.split(".")[0]) - - -def set_password(new_password=None, connection=None, update_config=None): - connection = conn() if connection is None else connection - if new_password is None: - new_password = getpass("New password: ") - confirm_password = getpass("Confirm password: ") - if new_password != confirm_password: - logger.warning("Failed to confirm the password! Aborting password change.") - return - - if version.parse( - connection.query("select @@version;").fetchone()[0] - ) >= version.parse("5.7"): - # SET PASSWORD is deprecated as of MySQL 5.7 and removed in 8+ - connection.query("ALTER USER user() IDENTIFIED BY '%s';" % new_password) - else: - connection.query("SET PASSWORD = PASSWORD('%s')" % new_password) - logger.info("Password updated.") - - if update_config or ( - update_config is None and user_choice("Update local setting?") == "yes" - ): - config["database.password"] = new_password - config.save_local(verbose=True) - - -def kill(restriction=None, connection=None, order_by=None): - """ - view and kill database connections. - - :param restriction: restriction to be applied to processlist - :param connection: a datajoint.Connection object. Default calls datajoint.conn() - :param order_by: order by a single attribute or the list of attributes. defaults to 'id'. - - Restrictions are specified as strings and can involve any of the attributes of - information_schema.processlist: ID, USER, HOST, DB, COMMAND, TIME, STATE, INFO. - - Examples: - dj.kill('HOST LIKE "%compute%"') lists only connections from hosts containing "compute". - dj.kill('TIME > 600') lists only connections in their current state for more than 10 minutes - """ - - if connection is None: - connection = conn() - - if order_by is not None and not isinstance(order_by, str): - order_by = ",".join(order_by) - - query = ( - "SELECT * FROM information_schema.processlist WHERE id <> CONNECTION_ID()" - + ("" if restriction is None else " AND (%s)" % restriction) - + (" ORDER BY %s" % (order_by or "id")) - ) - - while True: - print(" ID USER HOST STATE TIME INFO") - print("+--+ +----------+ +-----------+ +-----------+ +-----+") - cur = ( - {k.lower(): v for k, v in elem.items()} - for elem in connection.query(query, as_dict=True) - ) - for process in cur: - try: - print( - "{id:>4d} {user:<12s} {host:<12s} {state:<12s} {time:>7d} {info}".format( - **process - ) - ) - except TypeError: - print(process) - response = input('process to kill or "q" to quit > ') - if response == "q": - break - if response: - try: - pid = int(response) - except ValueError: - pass # ignore non-numeric input - else: - try: - connection.query("kill %d" % pid) - except pymysql.err.InternalError: - logger.warn("Process not found") - - -def kill_quick(restriction=None, connection=None): - """ - Kill database connections without prompting. Returns number of terminated connections. - - :param restriction: restriction to be applied to processlist - :param connection: a datajoint.Connection object. Default calls datajoint.conn() - - Restrictions are specified as strings and can involve any of the attributes of - information_schema.processlist: ID, USER, HOST, DB, COMMAND, TIME, STATE, INFO. - - Examples: - dj.kill('HOST LIKE "%compute%"') terminates connections from hosts containing "compute". - """ - if connection is None: - connection = conn() - - query = ( - "SELECT * FROM information_schema.processlist WHERE id <> CONNECTION_ID()" - + ("" if restriction is None else " AND (%s)" % restriction) - ) - - cur = ( - {k.lower(): v for k, v in elem.items()} - for elem in connection.query(query, as_dict=True) - ) - nkill = 0 - for process in cur: - connection.query("kill %d" % process["id"]) - nkill += 1 - return nkill diff --git a/datajoint/attribute_adapter.py b/datajoint/attribute_adapter.py deleted file mode 100644 index 2a8e59a51..000000000 --- a/datajoint/attribute_adapter.py +++ /dev/null @@ -1,67 +0,0 @@ -import re - -from .errors import DataJointError, _support_adapted_types - - -class AttributeAdapter: - """ - Base class for adapter objects for user-defined attribute types. - """ - - @property - def attribute_type(self): - """ - :return: a supported DataJoint attribute type to use; e.g. "longblob", "blob@store" - """ - raise NotImplementedError("Undefined attribute adapter") - - def get(self, value): - """ - convert value retrieved from the the attribute in a table into the adapted type - - :param value: value from the database - - :return: object of the adapted type - """ - raise NotImplementedError("Undefined attribute adapter") - - def put(self, obj): - """ - convert an object of the adapted type into a value that DataJoint can store in a table attribute - - :param obj: an object of the adapted type - :return: value to store in the database - """ - raise NotImplementedError("Undefined attribute adapter") - - -def get_adapter(context, adapter_name): - """ - Extract the AttributeAdapter object by its name from the context and validate. - """ - if not _support_adapted_types(): - raise DataJointError("Support for Adapted Attribute types is disabled.") - adapter_name = adapter_name.lstrip("<").rstrip(">") - try: - adapter = context[adapter_name] - except KeyError: - raise DataJointError( - "Attribute adapter '{adapter_name}' is not defined.".format( - adapter_name=adapter_name - ) - ) - if not isinstance(adapter, AttributeAdapter): - raise DataJointError( - "Attribute adapter '{adapter_name}' must be an instance of datajoint.AttributeAdapter".format( - adapter_name=adapter_name - ) - ) - if not isinstance(adapter.attribute_type, str) or not re.match( - r"^\w", adapter.attribute_type - ): - raise DataJointError( - "Invalid attribute type {type} in attribute adapter '{adapter_name}'".format( - type=adapter.attribute_type, adapter_name=adapter_name - ) - ) - return adapter diff --git a/datajoint/autopopulate.py b/datajoint/autopopulate.py deleted file mode 100644 index 226e64dda..000000000 --- a/datajoint/autopopulate.py +++ /dev/null @@ -1,476 +0,0 @@ -"""This module defines class dj.AutoPopulate""" - -import contextlib -import datetime -import inspect -import logging -import multiprocessing as mp -import random -import signal -import traceback - -import deepdiff -from tqdm import tqdm - -from .errors import DataJointError, LostConnectionError -from .expression import AndList, QueryExpression -from .hash import key_hash - -# noinspection PyExceptionInherit,PyCallingNonCallable - -logger = logging.getLogger(__name__.split(".")[0]) - - -# --- helper functions for multiprocessing -- - - -def _initialize_populate(table, jobs, populate_kwargs): - """ - Initialize the process for multiprocessing. - Saves the unpickled copy of the table to the current process and reconnects. - """ - process = mp.current_process() - process.table = table - process.jobs = jobs - process.populate_kwargs = populate_kwargs - table.connection.connect() # reconnect - - -def _call_populate1(key): - """ - Call current process' table._populate1() - :key - a dict specifying job to compute - :return: key, error if error, otherwise None - """ - process = mp.current_process() - return process.table._populate1(key, process.jobs, **process.populate_kwargs) - - -class AutoPopulate: - """ - AutoPopulate is a mixin class that adds the method populate() to a Table class. - Auto-populated tables must inherit from both Table and AutoPopulate, - must define the property `key_source`, and must define the callback method `make`. - """ - - _key_source = None - _allow_insert = False - - @property - def key_source(self): - """ - :return: the query expression that yields primary key values to be passed, - sequentially, to the ``make`` method when populate() is called. - The default value is the join of the parent tables references from the primary key. - Subclasses may override they key_source to change the scope or the granularity - of the make calls. - """ - - def _rename_attributes(table, props): - return ( - table.proj( - **{ - attr: ref - for attr, ref in props["attr_map"].items() - if attr != ref - } - ) - if props["aliased"] - else table.proj() - ) - - if self._key_source is None: - parents = self.target.parents( - primary=True, as_objects=True, foreign_key_info=True - ) - if not parents: - raise DataJointError( - "A table must have dependencies " - "from its primary key for auto-populate to work" - ) - self._key_source = _rename_attributes(*parents[0]) - for q in parents[1:]: - self._key_source *= _rename_attributes(*q) - return self._key_source - - def make(self, key): - """ - This method must be implemented by derived classes to perform automated computation. - The method must implement the following three steps: - - 1. Fetch data from tables above in the dependency hierarchy, restricted by the given key. - 2. Compute secondary attributes based on the fetched data. - 3. Insert the new tuple(s) into the current table. - - The method can be implemented either as: - (a) Regular method: All three steps are performed in a single database transaction. - The method must return None. - (b) Generator method: - The make method is split into three functions: - - `make_fetch`: Fetches data from the parent tables. - - `make_compute`: Computes secondary attributes based on the fetched data. - - `make_insert`: Inserts the computed data into the current table. - - Then populate logic is executes as follows: - - - fetched_data1 = self.make_fetch(key) - computed_result = self.make_compute(key, *fetched_data1) - begin transaction: - fetched_data2 = self.make_fetch(key) - if fetched_data1 != fetched_data2: - cancel transaction - else: - self.make_insert(key, *computed_result) - commit_transaction - - - Importantly, the output of make_fetch is a tuple that serves as the input into `make_compute`. - The output of `make_compute` is a tuple that serves as the input into `make_insert`. - - The functionality must be strictly divided between these three methods: - - All database queries must be completed in `make_fetch`. - - All computation must be completed in `make_compute`. - - All database inserts must be completed in `make_insert`. - - DataJoint may programmatically enforce this separation in the future. - - :param key: The primary key value used to restrict the data fetching. - :raises NotImplementedError: If the derived class does not implement the required methods. - """ - - if not ( - hasattr(self, "make_fetch") - and hasattr(self, "make_insert") - and hasattr(self, "make_compute") - ): - # user must implement `make` - raise NotImplementedError( - "Subclasses of AutoPopulate must implement the method `make` " - "or (`make_fetch` + `make_compute` + `make_insert`)" - ) - - # User has implemented `_fetch`, `_compute`, and `_insert` methods instead - - # Step 1: Fetch data from parent tables - fetched_data = self.make_fetch(key) # fetched_data is a tuple - computed_result = yield fetched_data # passed as input into make_compute - - # Step 2: If computed result is not passed in, compute the result - if computed_result is None: - # this is only executed in the first invocation - computed_result = self.make_compute(key, *fetched_data) - yield computed_result # this is passed to the second invocation of make - - # Step 3: Insert the computed result into the current table. - self.make_insert(key, *computed_result) - yield - - @property - def target(self): - """ - :return: table to be populated. - In the typical case, dj.AutoPopulate is mixed into a dj.Table class by - inheritance and the target is self. - """ - return self - - def _job_key(self, key): - """ - :param key: they key returned for the job from the key source - :return: the dict to use to generate the job reservation hash - This method allows subclasses to control the job reservation granularity. - """ - return key - - def _jobs_to_do(self, restrictions): - """ - :return: the query yielding the keys to be computed (derived from self.key_source) - """ - if self.restriction: - raise DataJointError( - "Cannot call populate on a restricted table. " - "Instead, pass conditions to populate() as arguments." - ) - todo = self.key_source - - # key_source is a QueryExpression subclass -- trigger instantiation - if inspect.isclass(todo) and issubclass(todo, QueryExpression): - todo = todo() - - if not isinstance(todo, QueryExpression): - raise DataJointError("Invalid key_source value") - - try: - # check if target lacks any attributes from the primary key of key_source - raise DataJointError( - "The populate target lacks attribute %s " - "from the primary key of key_source" - % next( - name - for name in todo.heading.primary_key - if name not in self.target.heading - ) - ) - except StopIteration: - pass - return (todo & AndList(restrictions)).proj() - - def populate( - self, - *restrictions, - keys=None, - suppress_errors=False, - return_exception_objects=False, - reserve_jobs=False, - order="original", - limit=None, - max_calls=None, - display_progress=False, - processes=1, - make_kwargs=None, - ): - """ - ``table.populate()`` calls ``table.make(key)`` for every primary key in - ``self.key_source`` for which there is not already a tuple in table. - - :param restrictions: a list of restrictions each restrict - (table.key_source - target.proj()) - :param keys: The list of keys (dicts) to send to self.make(). - If None (default), then use self.key_source to query they keys. - :param suppress_errors: if True, do not terminate execution. - :param return_exception_objects: return error objects instead of just error messages - :param reserve_jobs: if True, reserve jobs to populate in asynchronous fashion - :param order: "original"|"reverse"|"random" - the order of execution - :param limit: if not None, check at most this many keys - :param max_calls: if not None, populate at most this many keys - :param display_progress: if True, report progress_bar - :param processes: number of processes to use. Set to None to use all cores - :param make_kwargs: Keyword arguments which do not affect the result of computation - to be passed down to each ``make()`` call. Computation arguments should be - specified within the pipeline e.g. using a `dj.Lookup` table. - :type make_kwargs: dict, optional - :return: a dict with two keys - "success_count": the count of successful ``make()`` calls in this ``populate()`` call - "error_list": the error list that is filled if `suppress_errors` is True - """ - if self.connection.in_transaction: - raise DataJointError("Populate cannot be called during a transaction.") - - valid_order = ["original", "reverse", "random"] - if order not in valid_order: - raise DataJointError( - "The order argument must be one of %s" % str(valid_order) - ) - jobs = ( - self.connection.schemas[self.target.database].jobs if reserve_jobs else None - ) - - if reserve_jobs: - # Define a signal handler for SIGTERM - def handler(signum, frame): - logger.info("Populate terminated by SIGTERM") - raise SystemExit("SIGTERM received") - - old_handler = signal.signal(signal.SIGTERM, handler) - - if keys is None: - keys = (self._jobs_to_do(restrictions) - self.target).fetch( - "KEY", limit=limit - ) - - # exclude "error", "ignore" or "reserved" jobs - if reserve_jobs: - exclude_key_hashes = ( - jobs - & {"table_name": self.target.table_name} - & 'status in ("error", "ignore", "reserved")' - ).fetch("key_hash") - keys = [key for key in keys if key_hash(key) not in exclude_key_hashes] - - if order == "reverse": - keys.reverse() - elif order == "random": - random.shuffle(keys) - - logger.debug("Found %d keys to populate" % len(keys)) - - keys = keys[:max_calls] - nkeys = len(keys) - - error_list = [] - success_list = [] - - if nkeys: - processes = min(_ for _ in (processes, nkeys, mp.cpu_count()) if _) - - populate_kwargs = dict( - suppress_errors=suppress_errors, - return_exception_objects=return_exception_objects, - make_kwargs=make_kwargs, - ) - - if processes == 1: - for key in ( - tqdm(keys, desc=self.__class__.__name__) - if display_progress - else keys - ): - status = self._populate1(key, jobs, **populate_kwargs) - if status is True: - success_list.append(1) - elif isinstance(status, tuple): - error_list.append(status) - else: - assert status is False - else: - # spawn multiple processes - self.connection.close() # disconnect parent process from MySQL server - del self.connection._conn.ctx # SSLContext is not pickleable - with ( - mp.Pool( - processes, _initialize_populate, (self, jobs, populate_kwargs) - ) as pool, - ( - tqdm(desc="Processes: ", total=nkeys) - if display_progress - else contextlib.nullcontext() - ) as progress_bar, - ): - for status in pool.imap(_call_populate1, keys, chunksize=1): - if status is True: - success_list.append(1) - elif isinstance(status, tuple): - error_list.append(status) - else: - assert status is False - if display_progress: - progress_bar.update() - self.connection.connect() # reconnect parent process to MySQL server - - # restore original signal handler: - if reserve_jobs: - signal.signal(signal.SIGTERM, old_handler) - - return { - "success_count": sum(success_list), - "error_list": error_list, - } - - def _populate1( - self, key, jobs, suppress_errors, return_exception_objects, make_kwargs=None - ): - """ - populates table for one source key, calling self.make inside a transaction. - :param jobs: the jobs table or None if not reserve_jobs - :param key: dict specifying job to populate - :param suppress_errors: bool if errors should be suppressed and returned - :param return_exception_objects: if True, errors must be returned as objects - :return: (key, error) when suppress_errors=True, - True if successfully invoke one `make()` call, otherwise False - """ - # use the legacy `_make_tuples` callback. - make = self._make_tuples if hasattr(self, "_make_tuples") else self.make - - if jobs is not None and not jobs.reserve( - self.target.table_name, self._job_key(key) - ): - return False - - # if make is a generator, it transaction can be delayed until the final stage - is_generator = inspect.isgeneratorfunction(make) - if not is_generator: - self.connection.start_transaction() - - if key in self.target: # already populated - if not is_generator: - self.connection.cancel_transaction() - if jobs is not None: - jobs.complete(self.target.table_name, self._job_key(key)) - return False - - logger.debug(f"Making {key} -> {self.target.full_table_name}") - self.__class__._allow_insert = True - - try: - if not is_generator: - make(dict(key), **(make_kwargs or {})) - else: - # tripartite make - transaction is delayed until the final stage - gen = make(dict(key), **(make_kwargs or {})) - fetched_data = next(gen) - fetch_hash = deepdiff.DeepHash( - fetched_data, ignore_iterable_order=False - )[fetched_data] - computed_result = next(gen) # perform the computation - # fetch and insert inside a transaction - self.connection.start_transaction() - gen = make(dict(key), **(make_kwargs or {})) # restart make - fetched_data = next(gen) - if ( - fetch_hash - != deepdiff.DeepHash(fetched_data, ignore_iterable_order=False)[ - fetched_data - ] - ): # raise error if fetched data has changed - raise DataJointError( - "Referential integrity failed! The `make_fetch` data has changed" - ) - gen.send(computed_result) # insert - - except (KeyboardInterrupt, SystemExit, Exception) as error: - try: - self.connection.cancel_transaction() - except LostConnectionError: - pass - error_message = "{exception}{msg}".format( - exception=error.__class__.__name__, - msg=": " + str(error) if str(error) else "", - ) - logger.debug( - f"Error making {key} -> {self.target.full_table_name} - {error_message}" - ) - if jobs is not None: - # show error name and error message (if any) - jobs.error( - self.target.table_name, - self._job_key(key), - error_message=error_message, - error_stack=traceback.format_exc(), - ) - if not suppress_errors or isinstance(error, SystemExit): - raise - else: - logger.error(error) - return key, error if return_exception_objects else error_message - else: - self.connection.commit_transaction() - logger.debug(f"Success making {key} -> {self.target.full_table_name}") - if jobs is not None: - jobs.complete(self.target.table_name, self._job_key(key)) - return True - finally: - self.__class__._allow_insert = False - - def progress(self, *restrictions, display=False): - """ - Report the progress of populating the table. - :return: (remaining, total) -- numbers of tuples to be populated - """ - todo = self._jobs_to_do(restrictions) - total = len(todo) - remaining = len(todo - self.target) - if display: - logger.info( - "%-20s" % self.__class__.__name__ - + " Completed %d of %d (%2.1f%%) %s" - % ( - total - remaining, - total, - 100 - 100 * remaining / (total + 1e-12), - datetime.datetime.strftime( - datetime.datetime.now(), "%Y-%m-%d %H:%M:%S" - ), - ), - ) - return remaining, total diff --git a/datajoint/cli.py b/datajoint/cli.py deleted file mode 100644 index 3b7e72c25..000000000 --- a/datajoint/cli.py +++ /dev/null @@ -1,78 +0,0 @@ -import argparse -from code import interact -from collections import ChainMap - -import datajoint as dj - - -def cli(args: list = None): - """ - Console interface for DataJoint Python - - :param args: List of arguments to be passed in, defaults to reading stdin - :type args: list, optional - """ - parser = argparse.ArgumentParser( - prog="datajoint", - description="DataJoint console interface.", - conflict_handler="resolve", - ) - parser.add_argument( - "-V", "--version", action="version", version=f"{dj.__name__} {dj.__version__}" - ) - parser.add_argument( - "-u", - "--user", - type=str, - default=dj.config["database.user"], - required=False, - help="Datajoint username", - ) - parser.add_argument( - "-p", - "--password", - type=str, - default=dj.config["database.password"], - required=False, - help="Datajoint password", - ) - parser.add_argument( - "-h", - "--host", - type=str, - default=dj.config["database.host"], - required=False, - help="Datajoint host", - ) - parser.add_argument( - "-s", - "--schemas", - nargs="+", - type=str, - required=False, - help="A list of virtual module mappings in `db:schema ...` format", - ) - kwargs = vars(parser.parse_args(args)) - mods = {} - if kwargs["user"]: - dj.config["database.user"] = kwargs["user"] - if kwargs["password"]: - dj.config["database.password"] = kwargs["password"] - if kwargs["host"]: - dj.config["database.host"] = kwargs["host"] - if kwargs["schemas"]: - for vm in kwargs["schemas"]: - d, m = vm.split(":") - mods[m] = dj.create_virtual_module(m, d) - - banner = "dj repl\n" - if mods: - modstr = "\n".join(" - {}".format(m) for m in mods) - banner += "\nschema modules:\n\n" + modstr + "\n" - interact(banner, local=dict(ChainMap(mods, locals(), globals()))) - - raise SystemExit - - -if __name__ == "__main__": - cli() diff --git a/datajoint/condition.py b/datajoint/condition.py deleted file mode 100644 index 96cfbb6ef..000000000 --- a/datajoint/condition.py +++ /dev/null @@ -1,356 +0,0 @@ -"""methods for generating SQL WHERE clauses from datajoint restriction conditions""" - -import collections -import datetime -import decimal -import inspect -import json -import re -import uuid -from dataclasses import dataclass -from typing import List, Union - -import numpy -import pandas - -from .errors import DataJointError - -JSON_PATTERN = re.compile( - r"^(?P\w+)(\.(?P[\w.*\[\]]+))?(:(?P[\w(,\s)]+))?$" -) - - -def translate_attribute(key): - match = JSON_PATTERN.match(key) - if match is None: - return match, key - match = match.groupdict() - if match["path"] is None: - return match, match["attr"] - else: - return match, "json_value(`{}`, _utf8mb4'$.{}'{})".format( - *[ - ((f" returning {v}" if k == "type" else v) if v else "") - for k, v in match.items() - ] - ) - - -class PromiscuousOperand: - """ - A container for an operand to ignore join compatibility - """ - - def __init__(self, operand): - self.operand = operand - - -class AndList(list): - """ - A list of conditions to by applied to a query expression by logical conjunction: the - conditions are AND-ed. All other collections (lists, sets, other entity sets, etc) are - applied by logical disjunction (OR). - - Example: - expr2 = expr & dj.AndList((cond1, cond2, cond3)) - is equivalent to - expr2 = expr & cond1 & cond2 & cond3 - """ - - def append(self, restriction): - if isinstance(restriction, AndList): - # extend to reduce nesting - self.extend(restriction) - else: - super().append(restriction) - - -@dataclass -class Top: - """ - A restriction to the top entities of a query. - In SQL, this corresponds to ORDER BY ... LIMIT ... OFFSET - """ - - limit: Union[int, None] = 1 - order_by: Union[str, List[str]] = "KEY" - offset: int = 0 - - def __post_init__(self): - self.order_by = self.order_by or ["KEY"] - self.offset = self.offset or 0 - - if self.limit is not None and not isinstance(self.limit, int): - raise TypeError("Top limit must be an integer") - if not isinstance(self.order_by, (str, collections.abc.Sequence)) or not all( - isinstance(r, str) for r in self.order_by - ): - raise TypeError("Top order_by attributes must all be strings") - if not isinstance(self.offset, int): - raise TypeError("The offset argument must be an integer") - if self.offset and self.limit is None: - self.limit = 999999999999 # arbitrary large number to allow query - if isinstance(self.order_by, str): - self.order_by = [self.order_by] - - -class Not: - """invert restriction""" - - def __init__(self, restriction): - self.restriction = restriction - - -def assert_join_compatibility(expr1, expr2): - """ - Determine if expressions expr1 and expr2 are join-compatible. To be join-compatible, - the matching attributes in the two expressions must be in the primary key of one or the - other expression. - Raises an exception if not compatible. - - :param expr1: A QueryExpression object - :param expr2: A QueryExpression object - """ - from .expression import QueryExpression, U - - for rel in (expr1, expr2): - if not isinstance(rel, (U, QueryExpression)): - raise DataJointError( - "Object %r is not a QueryExpression and cannot be joined." % rel - ) - if not isinstance(expr1, U) and not isinstance( - expr2, U - ): # dj.U is always compatible - try: - raise DataJointError( - "Cannot join query expressions on dependent attribute `%s`" - % next( - r - for r in set(expr1.heading.secondary_attributes).intersection( - expr2.heading.secondary_attributes - ) - ) - ) - except StopIteration: - pass # all ok - - -def make_condition(query_expression, condition, columns): - """ - Translate the input condition into the equivalent SQL condition (a string) - - :param query_expression: a dj.QueryExpression object to apply condition - :param condition: any valid restriction object. - :param columns: a set passed by reference to collect all column names used in the - condition. - :return: an SQL condition string or a boolean value. - """ - from .expression import Aggregation, QueryExpression, U - - def prep_value(k, v): - """prepare SQL condition""" - key_match, k = translate_attribute(k) - if key_match["path"] is None: - k = f"`{k}`" - if ( - query_expression.heading[key_match["attr"]].json - and key_match["path"] is not None - and isinstance(v, dict) - ): - return f"{k}='{json.dumps(v)}'" - if v is None: - return f"{k} IS NULL" - if query_expression.heading[key_match["attr"]].uuid: - if not isinstance(v, uuid.UUID): - try: - v = uuid.UUID(v) - except (AttributeError, ValueError): - raise DataJointError( - "Badly formed UUID {v} in restriction by `{k}`".format(k=k, v=v) - ) - return f"{k}=X'{v.bytes.hex()}'" - if isinstance( - v, - ( - datetime.date, - datetime.datetime, - datetime.time, - decimal.Decimal, - list, - ), - ): - return f'{k}="{v}"' - if isinstance(v, str): - v = v.replace("%", "%%").replace("\\", "\\\\") - return f'{k}="{v}"' - return f"{k}={v}" - - def combine_conditions(negate, conditions): - return f"{'NOT ' if negate else ''} ({')AND('.join(conditions)})" - - negate = False - while isinstance(condition, Not): - negate = not negate - condition = condition.restriction - - # restrict by string - if isinstance(condition, str): - columns.update(extract_column_names(condition)) - return combine_conditions( - negate, conditions=[condition.strip().replace("%", "%%")] - ) # escape %, see issue #376 - - # restrict by AndList - if isinstance(condition, AndList): - # omit all conditions that evaluate to True - items = [ - item - for item in ( - make_condition(query_expression, cond, columns) for cond in condition - ) - if item is not True - ] - if any(item is False for item in items): - return negate # if any item is False, the whole thing is False - if not items: - return not negate # and empty AndList is True - return combine_conditions(negate, conditions=items) - - # restriction by dj.U evaluates to True - if isinstance(condition, U): - return not negate - - # restrict by boolean - if isinstance(condition, bool): - return negate != condition - - # restrict by a mapping/dict -- convert to an AndList of string equality conditions - if isinstance(condition, collections.abc.Mapping): - common_attributes = set(c.split(".", 1)[0] for c in condition).intersection( - query_expression.heading.names - ) - if not common_attributes: - return not negate # no matching attributes -> evaluates to True - columns.update(common_attributes) - return combine_conditions( - negate, - conditions=[ - prep_value(k, v) - for k, v in condition.items() - if k.split(".", 1)[0] in common_attributes # handle json indexing - ], - ) - - # restrict by a numpy record -- convert to an AndList of string equality conditions - if isinstance(condition, numpy.void): - common_attributes = set(condition.dtype.fields).intersection( - query_expression.heading.names - ) - if not common_attributes: - return not negate # no matching attributes -> evaluate to True - columns.update(common_attributes) - return combine_conditions( - negate, - conditions=[prep_value(k, condition[k]) for k in common_attributes], - ) - - # restrict by a QueryExpression subclass -- trigger instantiation and move on - if inspect.isclass(condition) and issubclass(condition, QueryExpression): - condition = condition() - - # restrict by another expression (aka semijoin and antijoin) - check_compatibility = True - if isinstance(condition, PromiscuousOperand): - condition = condition.operand - check_compatibility = False - - if isinstance(condition, QueryExpression): - if check_compatibility: - assert_join_compatibility(query_expression, condition) - common_attributes = [ - q for q in condition.heading.names if q in query_expression.heading.names - ] - columns.update(common_attributes) - if isinstance(condition, Aggregation): - condition = condition.make_subquery() - return ( - # without common attributes, any non-empty set matches everything - (not negate if condition else negate) - if not common_attributes - else "({fields}) {not_}in ({subquery})".format( - fields="`" + "`,`".join(common_attributes) + "`", - not_="not " if negate else "", - subquery=condition.make_sql(common_attributes), - ) - ) - - # restrict by pandas.DataFrames - if isinstance(condition, pandas.DataFrame): - condition = condition.to_records() # convert to numpy.recarray and move on - - # if iterable (but not a string, a QueryExpression, or an AndList), treat as an OrList - try: - or_list = [make_condition(query_expression, q, columns) for q in condition] - except TypeError: - raise DataJointError("Invalid restriction type %r" % condition) - else: - or_list = [ - item for item in or_list if item is not False - ] # ignore False conditions - if any(item is True for item in or_list): # if any item is True, entirely True - return not negate - return ( - f"{'NOT ' if negate else ''} ({' OR '.join(or_list)})" - if or_list - else negate - ) - - -def extract_column_names(sql_expression): - """ - extract all presumed column names from an sql expression such as the WHERE clause, - for example. - - :param sql_expression: a string containing an SQL expression - :return: set of extracted column names - This may be MySQL-specific for now. - """ - assert isinstance(sql_expression, str) - result = set() - s = sql_expression # for terseness - # remove escaped quotes - s = re.sub(r"(\\\")|(\\\')", "", s) - # remove quoted text - s = re.sub(r"'[^']*'", "", s) - s = re.sub(r'"[^"]*"', "", s) - # find all tokens in back quotes and remove them - result.update(re.findall(r"`([a-z][a-z_0-9]*)`", s)) - s = re.sub(r"`[a-z][a-z_0-9]*`", "", s) - # remove space before parentheses - s = re.sub(r"\s*\(", "(", s) - # remove tokens followed by ( since they must be functions - s = re.sub(r"(\b[a-z][a-z_0-9]*)\(", "(", s) - remaining_tokens = set(re.findall(r"\b[a-z][a-z_0-9]*\b", s)) - # update result removing reserved words - result.update( - remaining_tokens - - { - "is", - "in", - "between", - "like", - "and", - "or", - "null", - "not", - "interval", - "second", - "minute", - "hour", - "day", - "month", - "week", - "year", - } - ) - return result diff --git a/datajoint/connection.py b/datajoint/connection.py deleted file mode 100644 index 21b1c97a4..000000000 --- a/datajoint/connection.py +++ /dev/null @@ -1,404 +0,0 @@ -""" -This module contains the Connection class that manages the connection to the database, and -the ``conn`` function that provides access to a persistent connection in datajoint. -""" - -import logging -import pathlib -import re -import warnings -from contextlib import contextmanager -from getpass import getpass - -import pymysql as client - -from . import errors -from .blob import pack, unpack -from .dependencies import Dependencies -from .hash import uuid_from_buffer -from .settings import config -from .version import __version__ - -logger = logging.getLogger(__name__.split(".")[0]) -query_log_max_length = 300 - - -cache_key = "query_cache" # the key to lookup the query_cache folder in dj.config - - -def translate_query_error(client_error, query): - """ - Take client error and original query and return the corresponding DataJoint exception. - - :param client_error: the exception raised by the client interface - :param query: sql query with placeholders - :return: an instance of the corresponding subclass of datajoint.errors.DataJointError - """ - logger.debug("type: {}, args: {}".format(type(client_error), client_error.args)) - - err, *args = client_error.args - - # Loss of connection errors - if err in (0, "(0, '')"): - return errors.LostConnectionError( - "Server connection lost due to an interface error.", *args - ) - if err == 2006: - return errors.LostConnectionError("Connection timed out", *args) - if err == 2013: - return errors.LostConnectionError("Server connection lost", *args) - # Access errors - if err in (1044, 1142): - return errors.AccessError("Insufficient privileges.", args[0], query) - # Integrity errors - if err == 1062: - return errors.DuplicateError(*args) - if err == 1217: # MySQL 8 error code - return errors.IntegrityError(*args) - if err == 1451: - return errors.IntegrityError(*args) - if err == 1452: - return errors.IntegrityError(*args) - # Syntax errors - if err == 1064: - return errors.QuerySyntaxError(args[0], query) - # Existence errors - if err == 1146: - return errors.MissingTableError(args[0], query) - if err == 1364: - return errors.MissingAttributeError(*args) - if err == 1054: - return errors.UnknownAttributeError(*args) - # all the other errors are re-raised in original form - return client_error - - -def conn( - host=None, user=None, password=None, *, init_fun=None, reset=False, use_tls=None -): - """ - Returns a persistent connection object to be shared by multiple modules. - If the connection is not yet established or reset=True, a new connection is set up. - If connection information is not provided, it is taken from config which takes the - information from dj_local_conf.json. If the password is not specified in that file - datajoint prompts for the password. - - :param host: hostname - :param user: mysql user - :param password: mysql password - :param init_fun: initialization function - :param reset: whether the connection should be reset or not - :param use_tls: TLS encryption option. Valid options are: True (required), False - (required no TLS), None (TLS preferred, default), dict (Manually specify values per - https://dev.mysql.com/doc/refman/5.7/en/connection-options.html#encrypted-connection-options). - """ - if not hasattr(conn, "connection") or reset: - host = host if host is not None else config["database.host"] - user = user if user is not None else config["database.user"] - password = password if password is not None else config["database.password"] - if user is None: - user = input("Please enter DataJoint username: ") - if password is None: - password = getpass(prompt="Please enter DataJoint password: ") - init_fun = ( - init_fun if init_fun is not None else config["connection.init_function"] - ) - use_tls = use_tls if use_tls is not None else config["database.use_tls"] - conn.connection = Connection(host, user, password, None, init_fun, use_tls) - return conn.connection - - -class EmulatedCursor: - """acts like a cursor""" - - def __init__(self, data): - self._data = data - self._iter = iter(self._data) - - def __iter__(self): - return self - - def __next__(self): - return next(self._iter) - - def fetchall(self): - return self._data - - def fetchone(self): - return next(self._iter) - - @property - def rowcount(self): - return len(self._data) - - -class Connection: - """ - A dj.Connection object manages a connection to a database server. - It also catalogues modules, schemas, tables, and their dependencies (foreign keys). - - Most of the parameters below should be set in the local configuration file. - - :param host: host name, may include port number as hostname:port, in which case it overrides the value in port - :param user: user name - :param password: password - :param port: port number - :param init_fun: connection initialization function (SQL) - :param use_tls: TLS encryption option - """ - - def __init__(self, host, user, password, port=None, init_fun=None, use_tls=None): - if ":" in host: - # the port in the hostname overrides the port argument - host, port = host.split(":") - port = int(port) - elif port is None: - port = config["database.port"] - self.conn_info = dict(host=host, port=port, user=user, passwd=password) - if use_tls is not False: - self.conn_info["ssl"] = ( - use_tls if isinstance(use_tls, dict) else {"ssl": {}} - ) - self.conn_info["ssl_input"] = use_tls - self.init_fun = init_fun - self._conn = None - self._query_cache = None - self.connect() - if self.is_connected: - logger.info( - "DataJoint {version} connected to {user}@{host}:{port}".format( - version=__version__, **self.conn_info - ) - ) - self.connection_id = self.query("SELECT connection_id()").fetchone()[0] - else: - raise errors.LostConnectionError( - "Connection failed {user}@{host}:{port}".format(**self.conn_info) - ) - self._in_transaction = False - self.schemas = dict() - self.dependencies = Dependencies(self) - - def __eq__(self, other): - return self.conn_info == other.conn_info - - def __repr__(self): - connected = "connected" if self.is_connected else "disconnected" - return "DataJoint connection ({connected}) {user}@{host}:{port}".format( - connected=connected, **self.conn_info - ) - - def connect(self): - """Connect to the database server.""" - with warnings.catch_warnings(): - warnings.filterwarnings("ignore", ".*deprecated.*") - try: - self._conn = client.connect( - init_command=self.init_fun, - sql_mode="NO_ZERO_DATE,NO_ZERO_IN_DATE,ERROR_FOR_DIVISION_BY_ZERO," - "STRICT_ALL_TABLES,NO_ENGINE_SUBSTITUTION,ONLY_FULL_GROUP_BY", - charset=config["connection.charset"], - **{ - k: v - for k, v in self.conn_info.items() - if k not in ["ssl_input"] - }, - ) - except client.err.InternalError: - self._conn = client.connect( - init_command=self.init_fun, - sql_mode="NO_ZERO_DATE,NO_ZERO_IN_DATE,ERROR_FOR_DIVISION_BY_ZERO," - "STRICT_ALL_TABLES,NO_ENGINE_SUBSTITUTION,ONLY_FULL_GROUP_BY", - charset=config["connection.charset"], - **{ - k: v - for k, v in self.conn_info.items() - if not ( - k == "ssl_input" - or k == "ssl" - and self.conn_info["ssl_input"] is None - ) - }, - ) - self._conn.autocommit(True) - - def set_query_cache(self, query_cache=None): - """ - When query_cache is not None, the connection switches into the query caching mode, which entails: - 1. Only SELECT queries are allowed. - 2. The results of queries are cached under the path indicated by dj.config['query_cache'] - 3. query_cache is a string that differentiates different cache states. - - :param query_cache: a string to initialize the hash for query results - """ - self._query_cache = query_cache - - def purge_query_cache(self): - """Purges all query cache.""" - if ( - isinstance(config.get(cache_key), str) - and pathlib.Path(config[cache_key]).is_dir() - ): - for path in pathlib.Path(config[cache_key]).iterdir(): - if not path.is_dir(): - path.unlink() - - def close(self): - self._conn.close() - - def register(self, schema): - self.schemas[schema.database] = schema - self.dependencies.clear() - - def ping(self): - """Ping the connection or raises an exception if the connection is closed.""" - self._conn.ping(reconnect=False) - - @property - def is_connected(self): - """Return true if the object is connected to the database server.""" - try: - self.ping() - except: - return False - return True - - @staticmethod - def _execute_query(cursor, query, args, suppress_warnings): - try: - with warnings.catch_warnings(): - if suppress_warnings: - # suppress all warnings arising from underlying SQL library - warnings.simplefilter("ignore") - cursor.execute(query, args) - except client.err.Error as err: - raise translate_query_error(err, query) - - def query( - self, query, args=(), *, as_dict=False, suppress_warnings=True, reconnect=None - ): - """ - Execute the specified query and return the tuple generator (cursor). - - :param query: SQL query - :param args: additional arguments for the client.cursor - :param as_dict: If as_dict is set to True, the returned cursor objects returns - query results as dictionary. - :param suppress_warnings: If True, suppress all warnings arising from underlying query library - :param reconnect: when None, get from config, when True, attempt to reconnect if disconnected - """ - # check cache first: - use_query_cache = bool(self._query_cache) - if use_query_cache and not re.match(r"\s*(SELECT|SHOW)", query): - raise errors.DataJointError( - "Only SELECT queries are allowed when query caching is on." - ) - if use_query_cache: - if not config[cache_key]: - raise errors.DataJointError( - f"Provide filepath dj.config['{cache_key}'] when using query caching." - ) - hash_ = uuid_from_buffer( - (str(self._query_cache) + re.sub(r"`\$\w+`", "", query)).encode() - + pack(args) - ) - cache_path = pathlib.Path(config[cache_key]) / str(hash_) - try: - buffer = cache_path.read_bytes() - except FileNotFoundError: - pass # proceed to query the database - else: - return EmulatedCursor(unpack(buffer)) - - if reconnect is None: - reconnect = config["database.reconnect"] - logger.debug("Executing SQL:" + query[:query_log_max_length]) - cursor_class = client.cursors.DictCursor if as_dict else client.cursors.Cursor - cursor = self._conn.cursor(cursor=cursor_class) - try: - self._execute_query(cursor, query, args, suppress_warnings) - except errors.LostConnectionError: - if not reconnect: - raise - logger.warning("Reconnecting to MySQL server.") - self.connect() - if self._in_transaction: - self.cancel_transaction() - raise errors.LostConnectionError( - "Connection was lost during a transaction." - ) - logger.debug("Re-executing") - cursor = self._conn.cursor(cursor=cursor_class) - self._execute_query(cursor, query, args, suppress_warnings) - - if use_query_cache: - data = cursor.fetchall() - cache_path.write_bytes(pack(data)) - return EmulatedCursor(data) - - return cursor - - def get_user(self): - """ - :return: the user name and host name provided by the client to the server. - """ - return self.query("SELECT user()").fetchone()[0] - - # ---------- transaction processing - @property - def in_transaction(self): - """ - :return: True if there is an open transaction. - """ - self._in_transaction = self._in_transaction and self.is_connected - return self._in_transaction - - def start_transaction(self): - """ - Starts a transaction error. - """ - if self.in_transaction: - raise errors.DataJointError("Nested connections are not supported.") - self.query("START TRANSACTION WITH CONSISTENT SNAPSHOT") - self._in_transaction = True - logger.debug("Transaction started") - - def cancel_transaction(self): - """ - Cancels the current transaction and rolls back all changes made during the transaction. - """ - self.query("ROLLBACK") - self._in_transaction = False - logger.debug("Transaction cancelled. Rolling back ...") - - def commit_transaction(self): - """ - Commit all changes made during the transaction and close it. - - """ - self.query("COMMIT") - self._in_transaction = False - logger.debug("Transaction committed and closed.") - - # -------- context manager for transactions - @property - @contextmanager - def transaction(self): - """ - Context manager for transactions. Opens an transaction and closes it after the with statement. - If an error is caught during the transaction, the commits are automatically rolled back. - All errors are raised again. - - Example: - >>> import datajoint as dj - >>> with dj.conn().transaction as conn: - >>> # transaction is open here - """ - try: - self.start_transaction() - yield self - except: - self.cancel_transaction() - raise - else: - self.commit_transaction() diff --git a/datajoint/declare.py b/datajoint/declare.py deleted file mode 100644 index 304476798..000000000 --- a/datajoint/declare.py +++ /dev/null @@ -1,591 +0,0 @@ -""" -This module hosts functions to convert DataJoint table definitions into mysql table definitions, and to -declare the corresponding mysql tables. -""" - -import logging -import re -from hashlib import sha1 - -import pyparsing as pp - -from .attribute_adapter import get_adapter -from .condition import translate_attribute -from .errors import FILEPATH_FEATURE_SWITCH, DataJointError, _support_filepath_types -from .settings import config - -UUID_DATA_TYPE = "binary(16)" -MAX_TABLE_NAME_LENGTH = 64 -CONSTANT_LITERALS = { - "CURRENT_TIMESTAMP", - "NULL", -} # SQL literals to be used without quotes (case insensitive) -EXTERNAL_TABLE_ROOT = "~external" - -TYPE_PATTERN = { - k: re.compile(v, re.I) - for k, v in dict( - INTEGER=r"((tiny|small|medium|big|)int|integer)(\s*\(.+\))?(\s+unsigned)?(\s+auto_increment)?|serial$", - DECIMAL=r"(decimal|numeric)(\s*\(.+\))?(\s+unsigned)?$", - FLOAT=r"(double|float|real)(\s*\(.+\))?(\s+unsigned)?$", - STRING=r"(var)?char\s*\(.+\)$", - JSON=r"json$", - ENUM=r"enum\s*\(.+\)$", - BOOL=r"bool(ean)?$", # aliased to tinyint(1) - TEMPORAL=r"(date|datetime|time|timestamp|year)(\s*\(.+\))?$", - INTERNAL_BLOB=r"(tiny|small|medium|long|)blob$", - EXTERNAL_BLOB=r"blob@(?P[a-z][\-\w]*)$", - INTERNAL_ATTACH=r"attach$", - EXTERNAL_ATTACH=r"attach@(?P[a-z][\-\w]*)$", - FILEPATH=r"filepath@(?P[a-z][\-\w]*)$", - UUID=r"uuid$", - ADAPTED=r"<.+>$", - ).items() -} - -# custom types are stored in attribute comment -SPECIAL_TYPES = { - "UUID", - "INTERNAL_ATTACH", - "EXTERNAL_ATTACH", - "EXTERNAL_BLOB", - "FILEPATH", - "ADAPTED", -} -NATIVE_TYPES = set(TYPE_PATTERN) - SPECIAL_TYPES -EXTERNAL_TYPES = { - "EXTERNAL_ATTACH", - "EXTERNAL_BLOB", - "FILEPATH", -} # data referenced by a UUID in external tables -SERIALIZED_TYPES = { - "EXTERNAL_ATTACH", - "INTERNAL_ATTACH", - "EXTERNAL_BLOB", - "INTERNAL_BLOB", -} # requires packing data - -assert set().union(SPECIAL_TYPES, EXTERNAL_TYPES, SERIALIZED_TYPES) <= set(TYPE_PATTERN) - - -def match_type(attribute_type): - try: - return next( - category - for category, pattern in TYPE_PATTERN.items() - if pattern.match(attribute_type) - ) - except StopIteration: - raise DataJointError( - "Unsupported attribute type {type}".format(type=attribute_type) - ) - - -logger = logging.getLogger(__name__.split(".")[0]) - - -def build_foreign_key_parser_old(): - # old-style foreign key parser. Superseded by expression-based syntax. See issue #436 - # This will be deprecated in a future release. - left = pp.Literal("(").suppress() - right = pp.Literal(")").suppress() - attribute_name = pp.Word(pp.srange("[a-z]"), pp.srange("[a-z0-9_]")) - new_attrs = pp.Optional( - left + pp.delimitedList(attribute_name) + right - ).setResultsName("new_attrs") - arrow = pp.Literal("->").suppress() - lbracket = pp.Literal("[").suppress() - rbracket = pp.Literal("]").suppress() - option = pp.Word(pp.srange("[a-zA-Z]")) - options = pp.Optional( - lbracket + pp.delimitedList(option) + rbracket - ).setResultsName("options") - ref_table = pp.Word(pp.alphas, pp.alphanums + "._").setResultsName("ref_table") - ref_attrs = pp.Optional( - left + pp.delimitedList(attribute_name) + right - ).setResultsName("ref_attrs") - return new_attrs + arrow + options + ref_table + ref_attrs - - -def build_foreign_key_parser(): - arrow = pp.Literal("->").suppress() - lbracket = pp.Literal("[").suppress() - rbracket = pp.Literal("]").suppress() - option = pp.Word(pp.srange("[a-zA-Z]")) - options = pp.Optional( - lbracket + pp.delimitedList(option) + rbracket - ).setResultsName("options") - ref_table = pp.restOfLine.setResultsName("ref_table") - return arrow + options + ref_table - - -def build_attribute_parser(): - quoted = pp.QuotedString('"') ^ pp.QuotedString("'") - colon = pp.Literal(":").suppress() - attribute_name = pp.Word(pp.srange("[a-z]"), pp.srange("[a-z0-9_]")).setResultsName( - "name" - ) - data_type = ( - pp.Combine(pp.Word(pp.alphas) + pp.SkipTo("#", ignore=quoted)) - ^ pp.QuotedString("<", endQuoteChar=">", unquoteResults=False) - ).setResultsName("type") - default = pp.Literal("=").suppress() + pp.SkipTo( - colon, ignore=quoted - ).setResultsName("default") - comment = pp.Literal("#").suppress() + pp.restOfLine.setResultsName("comment") - return attribute_name + pp.Optional(default) + colon + data_type + comment - - -foreign_key_parser_old = build_foreign_key_parser_old() -foreign_key_parser = build_foreign_key_parser() -attribute_parser = build_attribute_parser() - - -def is_foreign_key(line): - """ - - :param line: a line from the table definition - :return: true if the line appears to be a foreign key definition - """ - arrow_position = line.find("->") - return arrow_position >= 0 and not any(c in line[:arrow_position] for c in "\"#'") - - -def compile_foreign_key( - line, context, attributes, primary_key, attr_sql, foreign_key_sql, index_sql -): - """ - :param line: a line from a table definition - :param context: namespace containing referenced objects - :param attributes: list of attribute names already in the declaration -- to be updated by this function - :param primary_key: None if the current foreign key is made from the dependent section. Otherwise it is the list - of primary key attributes thus far -- to be updated by the function - :param attr_sql: list of sql statements defining attributes -- to be updated by this function. - :param foreign_key_sql: list of sql statements specifying foreign key constraints -- to be updated by this function. - :param index_sql: list of INDEX declaration statements, duplicate or redundant indexes are ok. - """ - # Parse and validate - from .expression import QueryExpression - from .table import Table - - try: - result = foreign_key_parser.parseString(line) - except pp.ParseException as err: - raise DataJointError('Parsing error in line "%s". %s.' % (line, err)) - - try: - ref = eval(result.ref_table, context) - except Exception: - raise DataJointError( - "Foreign key reference %s could not be resolved" % result.ref_table - ) - - options = [opt.upper() for opt in result.options] - for opt in options: # check for invalid options - if opt not in {"NULLABLE", "UNIQUE"}: - raise DataJointError('Invalid foreign key option "{opt}"'.format(opt=opt)) - is_nullable = "NULLABLE" in options - is_unique = "UNIQUE" in options - if is_nullable and primary_key is not None: - raise DataJointError( - 'Primary dependencies cannot be nullable in line "{line}"'.format(line=line) - ) - - if isinstance(ref, type) and issubclass(ref, Table): - ref = ref() - - # check that dependency is of a supported type - if ( - not isinstance(ref, QueryExpression) - or len(ref.restriction) - or len(ref.support) != 1 - or not isinstance(ref.support[0], str) - ): - raise DataJointError( - 'Dependency "%s" is not supported (yet). Use a base table or its projection.' - % result.ref_table - ) - - # declare new foreign key attributes - for attr in ref.primary_key: - if attr not in attributes: - attributes.append(attr) - if primary_key is not None: - primary_key.append(attr) - attr_sql.append( - ref.heading[attr].sql.replace("NOT NULL ", "", int(is_nullable)) - ) - - # declare the foreign key - foreign_key_sql.append( - "FOREIGN KEY (`{fk}`) REFERENCES {ref} (`{pk}`) ON UPDATE CASCADE ON DELETE RESTRICT".format( - fk="`,`".join(ref.primary_key), - pk="`,`".join(ref.heading[name].original_name for name in ref.primary_key), - ref=ref.support[0], - ) - ) - - # declare unique index - if is_unique: - index_sql.append( - "UNIQUE INDEX ({attrs})".format( - attrs=",".join("`%s`" % attr for attr in ref.primary_key) - ) - ) - - -def prepare_declare(definition, context): - # split definition into lines - definition = re.split(r"\s*\n\s*", definition.strip()) - # check for optional table comment - table_comment = ( - definition.pop(0)[1:].strip() if definition[0].startswith("#") else "" - ) - if table_comment.startswith(":"): - raise DataJointError('Table comment must not start with a colon ":"') - in_key = True # parse primary keys - primary_key = [] - attributes = [] - attribute_sql = [] - foreign_key_sql = [] - index_sql = [] - external_stores = [] - - for line in definition: - if not line or line.startswith("#"): # ignore additional comments - pass - elif line.startswith("---") or line.startswith("___"): - in_key = False # start parsing dependent attributes - elif is_foreign_key(line): - compile_foreign_key( - line, - context, - attributes, - primary_key if in_key else None, - attribute_sql, - foreign_key_sql, - index_sql, - ) - elif re.match(r"^(unique\s+)?index\s*.*$", line, re.I): # index - compile_index(line, index_sql) - else: - name, sql, store = compile_attribute(line, in_key, foreign_key_sql, context) - if store: - external_stores.append(store) - if in_key and name not in primary_key: - primary_key.append(name) - if name not in attributes: - attributes.append(name) - attribute_sql.append(sql) - - return ( - table_comment, - primary_key, - attribute_sql, - foreign_key_sql, - index_sql, - external_stores, - ) - - -def declare(full_table_name, definition, context): - """ - Parse declaration and generate the SQL CREATE TABLE code - - :param full_table_name: full name of the table - :param definition: DataJoint table definition - :param context: dictionary of objects that might be referred to in the table - :return: SQL CREATE TABLE statement, list of external stores used - """ - table_name = full_table_name.strip("`").split(".")[1] - if len(table_name) > MAX_TABLE_NAME_LENGTH: - raise DataJointError( - "Table name `{name}` exceeds the max length of {max_length}".format( - name=table_name, max_length=MAX_TABLE_NAME_LENGTH - ) - ) - - ( - table_comment, - primary_key, - attribute_sql, - foreign_key_sql, - index_sql, - external_stores, - ) = prepare_declare(definition, context) - - if config.get("add_hidden_timestamp", False): - metadata_attr_sql = [ - "`_{full_table_name}_timestamp` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP" - ] - attribute_sql.extend( - attr.format( - full_table_name=sha1( - full_table_name.replace("`", "").encode("utf-8") - ).hexdigest() - ) - for attr in metadata_attr_sql - ) - - if not primary_key: - raise DataJointError("Table must have a primary key") - - return ( - "CREATE TABLE IF NOT EXISTS %s (\n" % full_table_name - + ",\n".join( - attribute_sql - + ["PRIMARY KEY (`" + "`,`".join(primary_key) + "`)"] - + foreign_key_sql - + index_sql - ) - + '\n) ENGINE=InnoDB, COMMENT "%s"' % table_comment - ), external_stores - - -def _make_attribute_alter(new, old, primary_key): - """ - :param new: new attribute declarations - :param old: old attribute declarations - :param primary_key: primary key attributes - :return: list of SQL ALTER commands - """ - # parse attribute names - name_regexp = re.compile(r"^`(?P\w+)`") - original_regexp = re.compile(r'COMMENT "{\s*(?P\w+)\s*}') - matched = ((name_regexp.match(d), original_regexp.search(d)) for d in new) - new_names = dict((d.group("name"), n and n.group("name")) for d, n in matched) - old_names = [name_regexp.search(d).group("name") for d in old] - - # verify that original names are only used once - renamed = set() - for v in new_names.values(): - if v: - if v in renamed: - raise DataJointError( - "Alter attempted to rename attribute {%s} twice." % v - ) - renamed.add(v) - - # verify that all renamed attributes existed in the old definition - try: - raise DataJointError( - "Attribute {} does not exist in the original definition".format( - next(attr for attr in renamed if attr not in old_names) - ) - ) - except StopIteration: - pass - - # dropping attributes - to_drop = [n for n in old_names if n not in renamed and n not in new_names] - sql = ["DROP `%s`" % n for n in to_drop] - old_names = [name for name in old_names if name not in to_drop] - - # add or change attributes in order - prev = None - for new_def, (new_name, old_name) in zip(new, new_names.items()): - if new_name not in primary_key: - after = None # if None, then must include the AFTER clause - if prev: - try: - idx = old_names.index(old_name or new_name) - except ValueError: - after = prev[0] - else: - if idx >= 1 and old_names[idx - 1] != (prev[1] or prev[0]): - after = prev[0] - if new_def not in old or after: - sql.append( - "{command} {new_def} {after}".format( - command=( - "ADD" - if (old_name or new_name) not in old_names - else "MODIFY" if not old_name else "CHANGE `%s`" % old_name - ), - new_def=new_def, - after="" if after is None else "AFTER `%s`" % after, - ) - ) - prev = new_name, old_name - - return sql - - -def alter(definition, old_definition, context): - """ - :param definition: new table definition - :param old_definition: current table definition - :param context: the context in which to evaluate foreign key definitions - :return: string SQL ALTER command, list of new stores used for external storage - """ - ( - table_comment, - primary_key, - attribute_sql, - foreign_key_sql, - index_sql, - external_stores, - ) = prepare_declare(definition, context) - ( - table_comment_, - primary_key_, - attribute_sql_, - foreign_key_sql_, - index_sql_, - external_stores_, - ) = prepare_declare(old_definition, context) - - # analyze differences between declarations - sql = list() - if primary_key != primary_key_: - raise NotImplementedError("table.alter cannot alter the primary key (yet).") - if foreign_key_sql != foreign_key_sql_: - raise NotImplementedError("table.alter cannot alter foreign keys (yet).") - if index_sql != index_sql_: - raise NotImplementedError("table.alter cannot alter indexes (yet)") - if attribute_sql != attribute_sql_: - sql.extend(_make_attribute_alter(attribute_sql, attribute_sql_, primary_key)) - if table_comment != table_comment_: - sql.append('COMMENT="%s"' % table_comment) - return sql, [e for e in external_stores if e not in external_stores_] - - -def compile_index(line, index_sql): - def format_attribute(attr): - match, attr = translate_attribute(attr) - if match is None: - return attr - if match["path"] is None: - return f"`{attr}`" - return f"({attr})" - - match = re.match(r"(?Punique\s+)?index\s*\(\s*(?P.*)\)", line, re.I) - if match is None: - raise DataJointError(f'Table definition syntax error in line "{line}"') - match = match.groupdict() - - attr_list = re.findall(r"(?:[^,(]|\([^)]*\))+", match["args"]) - index_sql.append( - "{unique}index ({attrs})".format( - unique="unique " if match["unique"] else "", - attrs=",".join(format_attribute(a.strip()) for a in attr_list), - ) - ) - - -def substitute_special_type(match, category, foreign_key_sql, context): - """ - :param match: dict containing with keys "type" and "comment" -- will be modified in place - :param category: attribute type category from TYPE_PATTERN - :param foreign_key_sql: list of foreign key declarations to add to - :param context: context for looking up user-defined attribute_type adapters - """ - if category == "UUID": - match["type"] = UUID_DATA_TYPE - elif category == "INTERNAL_ATTACH": - match["type"] = "LONGBLOB" - elif category in EXTERNAL_TYPES: - if category == "FILEPATH" and not _support_filepath_types(): - raise DataJointError( - """ - The filepath data type is disabled until complete validation. - To turn it on as experimental feature, set the environment variable - {env} = TRUE or upgrade datajoint. - """.format( - env=FILEPATH_FEATURE_SWITCH - ) - ) - match["store"] = match["type"].split("@", 1)[1] - match["type"] = UUID_DATA_TYPE - foreign_key_sql.append( - "FOREIGN KEY (`{name}`) REFERENCES `{{database}}`.`{external_table_root}_{store}` (`hash`) " - "ON UPDATE RESTRICT ON DELETE RESTRICT".format( - external_table_root=EXTERNAL_TABLE_ROOT, **match - ) - ) - elif category == "ADAPTED": - adapter = get_adapter(context, match["type"]) - match["type"] = adapter.attribute_type - category = match_type(match["type"]) - if category in SPECIAL_TYPES: - # recursive redefinition from user-defined datatypes. - substitute_special_type(match, category, foreign_key_sql, context) - else: - assert False, "Unknown special type" - - -def compile_attribute(line, in_key, foreign_key_sql, context): - """ - Convert attribute definition from DataJoint format to SQL - - :param line: attribution line - :param in_key: set to True if attribute is in primary key set - :param foreign_key_sql: the list of foreign key declarations to add to - :param context: context in which to look up user-defined attribute type adapterss - :returns: (name, sql, is_external) -- attribute name and sql code for its declaration - """ - try: - match = attribute_parser.parseString(line + "#", parseAll=True) - except pp.ParseException as err: - raise DataJointError( - "Declaration error in position {pos} in line:\n {line}\n{msg}".format( - line=err.args[0], pos=err.args[1], msg=err.args[2] - ) - ) - match["comment"] = match["comment"].rstrip("#") - if "default" not in match: - match["default"] = "" - match = {k: v.strip() for k, v in match.items()} - match["nullable"] = match["default"].lower() == "null" - - if match["nullable"]: - if in_key: - raise DataJointError( - 'Primary key attributes cannot be nullable in line "%s"' % line - ) - match["default"] = "DEFAULT NULL" # nullable attributes default to null - else: - if match["default"]: - quote = ( - match["default"].split("(")[0].upper() not in CONSTANT_LITERALS - and match["default"][0] not in "\"'" - ) - match["default"] = ( - "NOT NULL DEFAULT " + ('"%s"' if quote else "%s") % match["default"] - ) - else: - match["default"] = "NOT NULL" - - match["comment"] = match["comment"].replace( - '"', '\\"' - ) # escape double quotes in comment - - if match["comment"].startswith(":"): - raise DataJointError( - 'An attribute comment must not start with a colon in comment "{comment}"'.format( - **match - ) - ) - - category = match_type(match["type"]) - if category in SPECIAL_TYPES: - match["comment"] = ":{type}:{comment}".format( - **match - ) # insert custom type into comment - substitute_special_type(match, category, foreign_key_sql, context) - - if category in SERIALIZED_TYPES and match["default"] not in { - "DEFAULT NULL", - "NOT NULL", - }: - raise DataJointError( - "The default value for a blob or attachment attributes can only be NULL in:\n{line}".format( - line=line - ) - ) - - sql = ( - "`{name}` {type} {default}" - + (' COMMENT "{comment}"' if match["comment"] else "") - ).format(**match) - return match["name"], sql, match.get("store") diff --git a/datajoint/errors.py b/datajoint/errors.py deleted file mode 100644 index 03555bf13..000000000 --- a/datajoint/errors.py +++ /dev/null @@ -1,129 +0,0 @@ -""" -Exception classes for the DataJoint library -""" - -import os - - -# --- Top Level --- -class DataJointError(Exception): - """ - Base class for errors specific to DataJoint internal operation. - """ - - def suggest(self, *args): - """ - regenerate the exception with additional arguments - - :param args: addition arguments - :return: a new exception of the same type with the additional arguments - """ - return self.__class__(*(self.args + args)) - - -# --- Second Level --- -class LostConnectionError(DataJointError): - """ - Loss of server connection - """ - - -class QueryError(DataJointError): - """ - Errors arising from queries to the database - """ - - -# --- Third Level: QueryErrors --- -class QuerySyntaxError(QueryError): - """ - Errors arising from incorrect query syntax - """ - - -class AccessError(QueryError): - """ - User access error: insufficient privileges. - """ - - -class MissingTableError(DataJointError): - """ - Query on a table that has not been declared - """ - - -class DuplicateError(QueryError): - """ - An integrity error caused by a duplicate entry into a unique key - """ - - -class IntegrityError(QueryError): - """ - An integrity error triggered by foreign key constraints - """ - - -class UnknownAttributeError(QueryError): - """ - User requests an attribute name not found in query heading - """ - - -class MissingAttributeError(QueryError): - """ - An error arising when a required attribute value is not provided in INSERT - """ - - -class MissingExternalFile(DataJointError): - """ - Error raised when an external file managed by DataJoint is no longer accessible - """ - - -class BucketInaccessible(DataJointError): - """ - Error raised when a S3 bucket is inaccessible - """ - - -# environment variables to control availability of experimental features - -ADAPTED_TYPE_SWITCH = "DJ_SUPPORT_ADAPTED_TYPES" -FILEPATH_FEATURE_SWITCH = "DJ_SUPPORT_FILEPATH_MANAGEMENT" - - -def _switch_adapted_types(on): - """ - Enable (on=True) or disable (on=False) support for AttributeAdapter - """ - if on: - os.environ[ADAPTED_TYPE_SWITCH] = "TRUE" - else: - del os.environ[ADAPTED_TYPE_SWITCH] - - -def _support_adapted_types(): - """ - check if support for AttributeAdapter is enabled - """ - return os.getenv(ADAPTED_TYPE_SWITCH, "FALSE").upper() == "TRUE" - - -def _switch_filepath_types(on): - """ - Enable (on=True) or disable (on=False) support for AttributeAdapter - """ - if on: - os.environ[FILEPATH_FEATURE_SWITCH] = "TRUE" - else: - del os.environ[FILEPATH_FEATURE_SWITCH] - - -def _support_filepath_types(): - """ - check if support for AttributeAdapter is enabled - """ - return os.getenv(FILEPATH_FEATURE_SWITCH, "FALSE").upper() == "TRUE" diff --git a/datajoint/expression.py b/datajoint/expression.py deleted file mode 100644 index dd90087b8..000000000 --- a/datajoint/expression.py +++ /dev/null @@ -1,991 +0,0 @@ -import copy -import inspect -import logging -import re -from itertools import count - -from .condition import ( - AndList, - Not, - PromiscuousOperand, - Top, - assert_join_compatibility, - extract_column_names, - make_condition, - translate_attribute, -) -from .declare import CONSTANT_LITERALS -from .errors import DataJointError -from .fetch import Fetch, Fetch1 -from .preview import preview, repr_html -from .settings import config - -logger = logging.getLogger(__name__.split(".")[0]) - - -class QueryExpression: - """ - QueryExpression implements query operators to derive new entity set from its input. - A QueryExpression object generates a SELECT statement in SQL. - QueryExpression operators are restrict, join, proj, aggr, and union. - - A QueryExpression object has a support, a restriction (an AndList), and heading. - Property `heading` (type dj.Heading) contains information about the attributes. - It is loaded from the database and updated by proj. - - Property `support` is the list of table names or other QueryExpressions to be joined. - - The restriction is applied first without having access to the attributes generated by the projection. - Then projection is applied by selecting modifying the heading attribute. - - Application of operators does not always lead to the creation of a subquery. - A subquery is generated when: - 1. A restriction is applied on any computed or renamed attributes - 2. A projection is applied remapping remapped attributes - 3. Subclasses: Join, Aggregation, and Union have additional specific rules. - """ - - _restriction = None - _restriction_attributes = None - _left = [] # list of booleans True for left joins, False for inner joins - _original_heading = None # heading before projections - - # subclasses or instantiators must provide values - _connection = None - _heading = None - _support = None - _top = None - - # If the query will be using distinct - _distinct = False - - @property - def connection(self): - """a dj.Connection object""" - assert self._connection is not None - return self._connection - - @property - def support(self): - """A list of table names or subqueries to from the FROM clause""" - assert self._support is not None - return self._support - - @property - def heading(self): - """a dj.Heading object, reflects the effects of the projection operator .proj""" - return self._heading - - @property - def original_heading(self): - """a dj.Heading object reflecting the attributes before projection""" - return self._original_heading or self.heading - - @property - def restriction(self): - """a AndList object of restrictions applied to input to produce the result""" - if self._restriction is None: - self._restriction = AndList() - return self._restriction - - @property - def restriction_attributes(self): - """the set of attribute names invoked in the WHERE clause""" - if self._restriction_attributes is None: - self._restriction_attributes = set() - return self._restriction_attributes - - @property - def primary_key(self): - return self.heading.primary_key - - _subquery_alias_count = count() # count for alias names used in the FROM clause - - def from_clause(self): - support = ( - ( - "(" + src.make_sql() + ") as `$%x`" % next(self._subquery_alias_count) - if isinstance(src, QueryExpression) - else src - ) - for src in self.support - ) - clause = next(support) - for s, left in zip(support, self._left): - clause += " NATURAL{left} JOIN {clause}".format( - left=" LEFT" if left else "", clause=s - ) - return clause - - def where_clause(self): - return ( - "" - if not self.restriction - else " WHERE (%s)" % ")AND(".join(str(s) for s in self.restriction) - ) - - def sorting_clauses(self): - if not self._top: - return "" - clause = ", ".join( - _wrap_attributes( - _flatten_attribute_list(self.primary_key, self._top.order_by) - ) - ) - if clause: - clause = f" ORDER BY {clause}" - if self._top.limit is not None: - clause += f" LIMIT {self._top.limit}{f' OFFSET {self._top.offset}' if self._top.offset else ''}" - - return clause - - def make_sql(self, fields=None): - """ - Make the SQL SELECT statement. - - :param fields: used to explicitly set the select attributes - """ - return "SELECT {distinct}{fields} FROM {from_}{where}{sorting}".format( - distinct="DISTINCT " if self._distinct else "", - fields=self.heading.as_sql(fields or self.heading.names), - from_=self.from_clause(), - where=self.where_clause(), - sorting=self.sorting_clauses(), - ) - - # --------- query operators ----------- - def make_subquery(self): - """create a new SELECT statement where self is the FROM clause""" - result = QueryExpression() - result._connection = self.connection - result._support = [self] - result._heading = self.heading.make_subquery_heading() - return result - - def restrict(self, restriction): - """ - Produces a new expression with the new restriction applied. - rel.restrict(restriction) is equivalent to rel & restriction. - rel.restrict(Not(restriction)) is equivalent to rel - restriction - The primary key of the result is unaffected. - Successive restrictions are combined as logical AND: r & a & b is equivalent to r & AndList((a, b)) - Any QueryExpression, collection, or sequence other than an AndList are treated as OrLists - (logical disjunction of conditions) - Inverse restriction is accomplished by either using the subtraction operator or the Not class. - - The expressions in each row equivalent: - - rel & True rel - rel & False the empty entity set - rel & 'TRUE' rel - rel & 'FALSE' the empty entity set - rel - cond rel & Not(cond) - rel - 'TRUE' rel & False - rel - 'FALSE' rel - rel & AndList((cond1,cond2)) rel & cond1 & cond2 - rel & AndList() rel - rel & [cond1, cond2] rel & OrList((cond1, cond2)) - rel & [] rel & False - rel & None rel & False - rel & any_empty_entity_set rel & False - rel - AndList((cond1,cond2)) rel & [Not(cond1), Not(cond2)] - rel - [cond1, cond2] rel & Not(cond1) & Not(cond2) - rel - AndList() rel & False - rel - [] rel - rel - None rel - rel - any_empty_entity_set rel - - When arg is another QueryExpression, the restriction rel & arg restricts rel to elements that match at least - one element in arg (hence arg is treated as an OrList). - Conversely, rel - arg restricts rel to elements that do not match any elements in arg. - Two elements match when their common attributes have equal values or when they have no common attributes. - All shared attributes must be in the primary key of either rel or arg or both or an error will be raised. - - QueryExpression.restrict is the only access point that modifies restrictions. All other operators must - ultimately call restrict() - - :param restriction: a sequence or an array (treated as OR list), another QueryExpression, an SQL condition - string, or an AndList. - """ - attributes = set() - if isinstance(restriction, Top): - result = ( - self.make_subquery() - if self._top and not self._top.__eq__(restriction) - else copy.copy(self) - ) # make subquery to avoid overwriting existing Top - result._top = restriction - return result - new_condition = make_condition(self, restriction, attributes) - if new_condition is True: - return self # restriction has no effect, return the same object - # check that all attributes in condition are present in the query - try: - raise DataJointError( - "Attribute `%s` is not found in query." - % next(attr for attr in attributes if attr not in self.heading.names) - ) - except StopIteration: - pass # all ok - # If the new condition uses any new attributes, a subquery is required. - # However, Aggregation's HAVING statement works fine with aliased attributes. - need_subquery = ( - isinstance(self, Union) - or (not isinstance(self, Aggregation) and self.heading.new_attributes) - or self._top - ) - if need_subquery: - result = self.make_subquery() - else: - result = copy.copy(self) - result._restriction = AndList( - self.restriction - ) # copy to preserve the original - result.restriction.append(new_condition) - result.restriction_attributes.update(attributes) - return result - - def restrict_in_place(self, restriction): - self.__dict__.update(self.restrict(restriction).__dict__) - - def __and__(self, restriction): - """ - Restriction operator e.g. ``q1 & q2``. - :return: a restricted copy of the input argument - See QueryExpression.restrict for more detail. - """ - return self.restrict(restriction) - - def __xor__(self, restriction): - """ - Permissive restriction operator ignoring compatibility check e.g. ``q1 ^ q2``. - """ - if inspect.isclass(restriction) and issubclass(restriction, QueryExpression): - restriction = restriction() - if isinstance(restriction, Not): - return self.restrict(Not(PromiscuousOperand(restriction.restriction))) - return self.restrict(PromiscuousOperand(restriction)) - - def __sub__(self, restriction): - """ - Inverted restriction e.g. ``q1 - q2``. - :return: a restricted copy of the input argument - See QueryExpression.restrict for more detail. - """ - return self.restrict(Not(restriction)) - - def __neg__(self): - """ - Convert between restriction and inverted restriction e.g. ``-q1``. - :return: target restriction - See QueryExpression.restrict for more detail. - """ - if isinstance(self, Not): - return self.restriction - return Not(self) - - def __mul__(self, other): - """ - join of query expressions `self` and `other` e.g. ``q1 * q2``. - """ - return self.join(other) - - def __matmul__(self, other): - """ - Permissive join of query expressions `self` and `other` ignoring compatibility check - e.g. ``q1 @ q2``. - """ - if inspect.isclass(other) and issubclass(other, QueryExpression): - other = other() # instantiate - return self.join(other, semantic_check=False) - - def join(self, other, semantic_check=True, left=False): - """ - create the joined QueryExpression. - a * b is short for A.join(B) - a @ b is short for A.join(B, semantic_check=False) - Additionally, left=True will retain the rows of self, effectively performing a left join. - """ - # trigger subqueries if joining on renamed attributes - if isinstance(other, U): - return other * self - if inspect.isclass(other) and issubclass(other, QueryExpression): - other = other() # instantiate - if not isinstance(other, QueryExpression): - raise DataJointError("The argument of join must be a QueryExpression") - if semantic_check: - assert_join_compatibility(self, other) - join_attributes = set(n for n in self.heading.names if n in other.heading.names) - # needs subquery if self's FROM clause has common attributes with other's FROM clause - need_subquery1 = need_subquery2 = bool( - (set(self.original_heading.names) & set(other.original_heading.names)) - - join_attributes - ) - # need subquery if any of the join attributes are derived - need_subquery1 = ( - need_subquery1 - or isinstance(self, Aggregation) - or any(n in self.heading.new_attributes for n in join_attributes) - or isinstance(self, Union) - ) - need_subquery2 = ( - need_subquery2 - or isinstance(other, Aggregation) - or any(n in other.heading.new_attributes for n in join_attributes) - or isinstance(self, Union) - ) - if need_subquery1: - self = self.make_subquery() - if need_subquery2: - other = other.make_subquery() - result = QueryExpression() - result._connection = self.connection - result._support = self.support + other.support - result._left = self._left + [left] + other._left - result._heading = self.heading.join(other.heading) - result._restriction = AndList(self.restriction) - result._restriction.append(other.restriction) - result._original_heading = self.original_heading.join(other.original_heading) - assert len(result.support) == len(result._left) + 1 - return result - - def __add__(self, other): - """union e.g. ``q1 + q2``.""" - return Union.create(self, other) - - def proj(self, *attributes, **named_attributes): - """ - Projection operator. - - :param attributes: attributes to be included in the result. (The primary key is already included). - :param named_attributes: new attributes computed or renamed from existing attributes. - :return: the projected expression. - Primary key attributes cannot be excluded but may be renamed. - If the attribute list contains an Ellipsis ..., then all secondary attributes are included too - Prefixing an attribute name with a dash '-attr' removes the attribute from the list if present. - Keyword arguments can be used to rename attributes as in name='attr', duplicate them as in name='(attr)', or - self.proj(...) or self.proj(Ellipsis) -- include all attributes (return self) - self.proj() -- include only primary key - self.proj('attr1', 'attr2') -- include primary key and attributes attr1 and attr2 - self.proj(..., '-attr1', '-attr2') -- include all attributes except attr1 and attr2 - self.proj(name1='attr1') -- include primary key and 'attr1' renamed as name1 - self.proj('attr1', dup='(attr1)') -- include primary key and attribute attr1 twice, with the duplicate 'dup' - self.proj(k='abs(attr1)') adds the new attribute k with the value computed as an expression (SQL syntax) - from other attributes available before the projection. - Each attribute name can only be used once. - """ - named_attributes = { - k: translate_attribute(v)[1] for k, v in named_attributes.items() - } - # new attributes in parentheses are included again with the new name without removing original - duplication_pattern = re.compile( - rf'^\s*\(\s*(?!{"|".join(CONSTANT_LITERALS)})(?P[a-zA-Z_]\w*)\s*\)\s*$' - ) - # attributes without parentheses renamed - rename_pattern = re.compile( - rf'^\s*(?!{"|".join(CONSTANT_LITERALS)})(?P[a-zA-Z_]\w*)\s*$' - ) - replicate_map = { - k: m.group("name") - for k, m in ( - (k, duplication_pattern.match(v)) for k, v in named_attributes.items() - ) - if m - } - rename_map = { - k: m.group("name") - for k, m in ( - (k, rename_pattern.match(v)) for k, v in named_attributes.items() - ) - if m - } - compute_map = { - k: v - for k, v in named_attributes.items() - if not duplication_pattern.match(v) and not rename_pattern.match(v) - } - attributes = set(attributes) - # include primary key - attributes.update((k for k in self.primary_key if k not in rename_map.values())) - # include all secondary attributes with Ellipsis - if Ellipsis in attributes: - attributes.discard(Ellipsis) - attributes.update( - ( - a - for a in self.heading.secondary_attributes - if a not in attributes and a not in rename_map.values() - ) - ) - try: - raise DataJointError( - "%s is not a valid data type for an attribute in .proj" - % next(a for a in attributes if not isinstance(a, str)) - ) - except StopIteration: - pass # normal case - # remove excluded attributes, specified as `-attr' - excluded = set(a for a in attributes if a.strip().startswith("-")) - attributes.difference_update(excluded) - excluded = set(a.lstrip("-").strip() for a in excluded) - attributes.difference_update(excluded) - try: - raise DataJointError( - "Cannot exclude primary key attribute %s", - next(a for a in excluded if a in self.primary_key), - ) - except StopIteration: - pass # all ok - # check that all attributes exist in heading - try: - raise DataJointError( - "Attribute `%s` not found." - % next(a for a in attributes if a not in self.heading.names) - ) - except StopIteration: - pass # all ok - - # check that all mentioned names are present in heading - mentions = attributes.union(replicate_map.values()).union(rename_map.values()) - try: - raise DataJointError( - "Attribute '%s' not found." - % next(a for a in mentions if not self.heading.names) - ) - except StopIteration: - pass # all ok - - # check that newly created attributes do not clash with any other selected attributes - try: - raise DataJointError( - "Attribute `%s` already exists" - % next( - a - for a in rename_map - if a in attributes.union(compute_map).union(replicate_map) - ) - ) - except StopIteration: - pass # all ok - try: - raise DataJointError( - "Attribute `%s` already exists" - % next( - a - for a in compute_map - if a in attributes.union(rename_map).union(replicate_map) - ) - ) - except StopIteration: - pass # all ok - try: - raise DataJointError( - "Attribute `%s` already exists" - % next( - a - for a in replicate_map - if a in attributes.union(rename_map).union(compute_map) - ) - ) - except StopIteration: - pass # all ok - - # need a subquery if the projection remaps any remapped attributes - used = set(q for v in compute_map.values() for q in extract_column_names(v)) - used.update(rename_map.values()) - used.update(replicate_map.values()) - used.intersection_update(self.heading.names) - need_subquery = isinstance(self, Union) or any( - self.heading[name].attribute_expression is not None for name in used - ) - if not need_subquery and self.restriction: - # need a subquery if the restriction applies to attributes that have been renamed - need_subquery = any( - name in self.restriction_attributes - for name in self.heading.new_attributes - ) - - result = self.make_subquery() if need_subquery else copy.copy(self) - result._original_heading = result.original_heading - result._heading = result.heading.select( - attributes, - rename_map=dict(**rename_map, **replicate_map), - compute_map=compute_map, - ) - return result - - def aggr(self, group, *attributes, keep_all_rows=False, **named_attributes): - """ - Aggregation of the type U('attr1','attr2').aggr(group, computation="QueryExpression") - has the primary key ('attr1','attr2') and performs aggregation computations for all matching elements of `group`. - - :param group: The query expression to be aggregated. - :param keep_all_rows: True=keep all the rows from self. False=keep only rows that match entries in group. - :param named_attributes: computations of the form new_attribute="sql expression on attributes of group" - :return: The derived query expression - """ - if Ellipsis in attributes: - # expand ellipsis to include only attributes from the left table - attributes = set(attributes) - attributes.discard(Ellipsis) - attributes.update(self.heading.secondary_attributes) - return Aggregation.create(self, group=group, keep_all_rows=keep_all_rows).proj( - *attributes, **named_attributes - ) - - aggregate = aggr # alias for aggr - - # ---------- Fetch operators -------------------- - @property - def fetch1(self): - return Fetch1(self) - - @property - def fetch(self): - return Fetch(self) - - def head(self, limit=25, **fetch_kwargs): - """ - shortcut to fetch the first few entries from query expression. - Equivalent to fetch(order_by="KEY", limit=25) - - :param limit: number of entries - :param fetch_kwargs: kwargs for fetch - :return: query result - """ - return self.fetch(order_by="KEY", limit=limit, **fetch_kwargs) - - def tail(self, limit=25, **fetch_kwargs): - """ - shortcut to fetch the last few entries from query expression. - Equivalent to fetch(order_by="KEY DESC", limit=25)[::-1] - - :param limit: number of entries - :param fetch_kwargs: kwargs for fetch - :return: query result - """ - return self.fetch(order_by="KEY DESC", limit=limit, **fetch_kwargs)[::-1] - - def __len__(self): - """:return: number of elements in the result set e.g. ``len(q1)``.""" - result = self.make_subquery() if self._top else copy.copy(self) - return result.connection.query( - "SELECT {select_} FROM {from_}{where}".format( - select_=( - "count(*)" - if any(result._left) - else "count(DISTINCT {fields})".format( - fields=result.heading.as_sql( - result.primary_key, include_aliases=False - ) - ) - ), - from_=result.from_clause(), - where=result.where_clause(), - ) - ).fetchone()[0] - - def __bool__(self): - """ - :return: True if the result is not empty. Equivalent to len(self) > 0 but often - faster e.g. ``bool(q1)``. - """ - return bool( - self.connection.query( - "SELECT EXISTS(SELECT 1 FROM {from_}{where})".format( - from_=self.from_clause(), where=self.where_clause() - ) - ).fetchone()[0] - ) - - def __contains__(self, item): - """ - returns True if the restriction in item matches any entries in self - e.g. ``restriction in q1``. - - :param item: any restriction - (item in query_expression) is equivalent to bool(query_expression & item) but may be - executed more efficiently. - """ - return bool(self & item) # May be optimized e.g. using an EXISTS query - - def __iter__(self): - """ - returns an iterator-compatible QueryExpression object e.g. ``iter(q1)``. - - :param self: iterator-compatible QueryExpression object - """ - self._iter_only_key = all(v.in_key for v in self.heading.attributes.values()) - self._iter_keys = self.fetch("KEY") - return self - - def __next__(self): - """ - returns the next record on an iterator-compatible QueryExpression object - e.g. ``next(q1)``. - - :param self: A query expression - :type self: :class:`QueryExpression` - :rtype: dict - """ - try: - key = self._iter_keys.pop(0) - except AttributeError: - # self._iter_keys is missing because __iter__ has not been called. - raise TypeError( - "A QueryExpression object is not an iterator. " - "Use iter(obj) to create an iterator." - ) - except IndexError: - raise StopIteration - else: - if self._iter_only_key: - return key - else: - try: - return (self & key).fetch1() - except DataJointError: - # The data may have been deleted since the moment the keys were fetched - # -- move on to next entry. - return next(self) - - def cursor(self, as_dict=False): - """ - See expression.fetch() for input description. - :return: query cursor - """ - sql = self.make_sql() - logger.debug(sql) - return self.connection.query(sql, as_dict=as_dict) - - def __repr__(self): - """ - returns the string representation of a QueryExpression object e.g. ``str(q1)``. - - :param self: A query expression - :type self: :class:`QueryExpression` - :rtype: str - """ - return ( - super().__repr__() - if config["loglevel"].lower() == "debug" - else self.preview() - ) - - def preview(self, limit=None, width=None): - """:return: a string of preview of the contents of the query.""" - return preview(self, limit, width) - - def _repr_html_(self): - """:return: HTML to display table in Jupyter notebook.""" - return repr_html(self) - - -class Aggregation(QueryExpression): - """ - Aggregation.create(arg, group, comp1='calc1', ..., compn='calcn') yields an entity set - with primary key from arg. - The computed arguments comp1, ..., compn use aggregation calculations on the attributes of - group or simple projections and calculations on the attributes of arg. - Aggregation is used QueryExpression.aggr and U.aggr. - Aggregation is a private class in DataJoint, not exposed to users. - """ - - _left_restrict = None # the pre-GROUP BY conditions for the WHERE clause - _subquery_alias_count = count() - - @classmethod - def create(cls, arg, group, keep_all_rows=False): - if inspect.isclass(group) and issubclass(group, QueryExpression): - group = group() # instantiate if a class - assert isinstance(group, QueryExpression) - if keep_all_rows and len(group.support) > 1 or group.heading.new_attributes: - group = group.make_subquery() # subquery if left joining a join - join = arg.join(group, left=keep_all_rows) # reuse the join logic - result = cls() - result._connection = join.connection - result._heading = join.heading.set_primary_key( - arg.primary_key - ) # use left operand's primary key - result._support = join.support - result._left = join._left - result._left_restrict = join.restriction # WHERE clause applied before GROUP BY - result._grouping_attributes = result.primary_key - - return result - - def where_clause(self): - return ( - "" - if not self._left_restrict - else " WHERE (%s)" % ")AND(".join(str(s) for s in self._left_restrict) - ) - - def make_sql(self, fields=None): - fields = self.heading.as_sql(fields or self.heading.names) - assert self._grouping_attributes or not self.restriction - distinct = set(self.heading.names) == set(self.primary_key) - return ( - "SELECT {distinct}{fields} FROM {from_}{where}{group_by}{sorting}".format( - distinct="DISTINCT " if distinct else "", - fields=fields, - from_=self.from_clause(), - where=self.where_clause(), - group_by=( - "" - if not self.primary_key - else ( - " GROUP BY `%s`" % "`,`".join(self._grouping_attributes) - + ( - "" - if not self.restriction - else " HAVING (%s)" % ")AND(".join(self.restriction) - ) - ) - ), - sorting=self.sorting_clauses(), - ) - ) - - def __len__(self): - return self.connection.query( - "SELECT count(1) FROM ({subquery}) `${alias:x}`".format( - subquery=self.make_sql(), alias=next(self._subquery_alias_count) - ) - ).fetchone()[0] - - def __bool__(self): - return bool( - self.connection.query("SELECT EXISTS({sql})".format(sql=self.make_sql())) - ) - - -class Union(QueryExpression): - """ - Union is the private DataJoint class that implements the union operator. - """ - - __count = count() - - @classmethod - def create(cls, arg1, arg2): - if inspect.isclass(arg2) and issubclass(arg2, QueryExpression): - arg2 = arg2() # instantiate if a class - if not isinstance(arg2, QueryExpression): - raise DataJointError( - "A QueryExpression can only be unioned with another QueryExpression" - ) - if arg1.connection != arg2.connection: - raise DataJointError( - "Cannot operate on QueryExpressions originating from different connections." - ) - if set(arg1.primary_key) != set(arg2.primary_key): - raise DataJointError( - "The operands of a union must share the same primary key." - ) - if set(arg1.heading.secondary_attributes) & set( - arg2.heading.secondary_attributes - ): - raise DataJointError( - "The operands of a union must not share any secondary attributes." - ) - result = cls() - result._connection = arg1.connection - result._heading = arg1.heading.join(arg2.heading) - result._support = [arg1, arg2] - return result - - def make_sql(self): - arg1, arg2 = self._support - if ( - not arg1.heading.secondary_attributes - and not arg2.heading.secondary_attributes - ): - # no secondary attributes: use UNION DISTINCT - fields = arg1.primary_key - return "SELECT * FROM (({sql1}) UNION ({sql2})) as `_u{alias}{sorting}`".format( - sql1=( - arg1.make_sql() - if isinstance(arg1, Union) - else arg1.make_sql(fields) - ), - sql2=( - arg2.make_sql() - if isinstance(arg2, Union) - else arg2.make_sql(fields) - ), - alias=next(self.__count), - sorting=self.sorting_clauses(), - ) - # with secondary attributes, use union of left join with antijoin - fields = self.heading.names - sql1 = arg1.join(arg2, left=True).make_sql(fields) - sql2 = ( - (arg2 - arg1) - .proj(..., **{k: "NULL" for k in arg1.heading.secondary_attributes}) - .make_sql(fields) - ) - return "({sql1}) UNION ({sql2})".format(sql1=sql1, sql2=sql2) - - def from_clause(self): - """The union does not use a FROM clause""" - assert False - - def where_clause(self): - """The union does not use a WHERE clause""" - assert False - - def __len__(self): - return self.connection.query( - "SELECT count(1) FROM ({subquery}) `${alias:x}`".format( - subquery=self.make_sql(), - alias=next(QueryExpression._subquery_alias_count), - ) - ).fetchone()[0] - - def __bool__(self): - return bool( - self.connection.query("SELECT EXISTS({sql})".format(sql=self.make_sql())) - ) - - -class U: - """ - dj.U objects are the universal sets representing all possible values of their attributes. - dj.U objects cannot be queried on their own but are useful for forming some queries. - dj.U('attr1', ..., 'attrn') represents the universal set with the primary key attributes attr1 ... attrn. - The universal set is the set of all possible combinations of values of the attributes. - Without any attributes, dj.U() represents the set with one element that has no attributes. - - Restriction: - - dj.U can be used to enumerate unique combinations of values of attributes from other expressions. - - The following expression yields all unique combinations of contrast and brightness found in the `stimulus` set: - - >>> dj.U('contrast', 'brightness') & stimulus - - Aggregation: - - In aggregation, dj.U is used for summary calculation over an entire set: - - The following expression yields one element with one attribute `s` containing the total number of elements in - query expression `expr`: - - >>> dj.U().aggr(expr, n='count(*)') - - The following expressions both yield one element containing the number `n` of distinct values of attribute `attr` in - query expression `expr`. - - >>> dj.U().aggr(expr, n='count(distinct attr)') - >>> dj.U().aggr(dj.U('attr').aggr(expr), 'n=count(*)') - - The following expression yields one element and one attribute `s` containing the sum of values of attribute `attr` - over entire result set of expression `expr`: - - >>> dj.U().aggr(expr, s='sum(attr)') - - The following expression yields the set of all unique combinations of attributes `attr1`, `attr2` and the number of - their occurrences in the result set of query expression `expr`. - - >>> dj.U(attr1,attr2).aggr(expr, n='count(*)') - - Joins: - - If expression `expr` has attributes 'attr1' and 'attr2', then expr * dj.U('attr1','attr2') yields the same result - as `expr` but `attr1` and `attr2` are promoted to the the primary key. This is useful for producing a join on - non-primary key attributes. - For example, if `attr` is in both expr1 and expr2 but not in their primary keys, then expr1 * expr2 will throw - an error because in most cases, it does not make sense to join on non-primary key attributes and users must first - rename `attr` in one of the operands. The expression dj.U('attr') * rel1 * rel2 overrides this constraint. - """ - - def __init__(self, *primary_key): - self._primary_key = primary_key - - @property - def primary_key(self): - return self._primary_key - - def __and__(self, other): - if inspect.isclass(other) and issubclass(other, QueryExpression): - other = other() # instantiate if a class - if not isinstance(other, QueryExpression): - raise DataJointError("Set U can only be restricted with a QueryExpression.") - result = copy.copy(other) - result._distinct = True - result._heading = result.heading.set_primary_key(self.primary_key) - result = result.proj() - return result - - def join(self, other, left=False): - """ - Joining U with a query expression has the effect of promoting the attributes of U to - the primary key of the other query expression. - - :param other: the other query expression to join with. - :param left: ignored. dj.U always acts as if left=False - :return: a copy of the other query expression with the primary key extended. - """ - if inspect.isclass(other) and issubclass(other, QueryExpression): - other = other() # instantiate if a class - if not isinstance(other, QueryExpression): - raise DataJointError("Set U can only be joined with a QueryExpression.") - try: - raise DataJointError( - "Attribute `%s` not found" - % next(k for k in self.primary_key if k not in other.heading.names) - ) - except StopIteration: - pass # all ok - result = copy.copy(other) - result._heading = result.heading.set_primary_key( - other.primary_key - + [k for k in self.primary_key if k not in other.primary_key] - ) - return result - - def __mul__(self, other): - """shorthand for join""" - return self.join(other) - - def aggr(self, group, **named_attributes): - """ - Aggregation of the type U('attr1','attr2').aggr(group, computation="QueryExpression") - has the primary key ('attr1','attr2') and performs aggregation computations for all matching elements of `group`. - - :param group: The query expression to be aggregated. - :param named_attributes: computations of the form new_attribute="sql expression on attributes of group" - :return: The derived query expression - """ - if named_attributes.get("keep_all_rows", False): - raise DataJointError( - "Cannot set keep_all_rows=True when aggregating on a universal set." - ) - return Aggregation.create(self, group=group, keep_all_rows=False).proj( - **named_attributes - ) - - aggregate = aggr # alias for aggr - - -def _flatten_attribute_list(primary_key, attrs): - """ - :param primary_key: list of attributes in primary key - :param attrs: list of attribute names, which may include "KEY", "KEY DESC" or "KEY ASC" - :return: generator of attributes where "KEY" is replaced with its component attributes - """ - for a in attrs: - if re.match(r"^\s*KEY(\s+[aA][Ss][Cc])?\s*$", a): - if primary_key: - yield from primary_key - elif re.match(r"^\s*KEY\s+[Dd][Ee][Ss][Cc]\s*$", a): - if primary_key: - yield from (q + " DESC" for q in primary_key) - else: - yield a - - -def _wrap_attributes(attr): - for entry in attr: # wrap attribute names in backquotes - yield re.sub(r"\b((?!asc|desc)\w+)\b", r"`\1`", entry, flags=re.IGNORECASE) diff --git a/datajoint/external.py b/datajoint/external.py deleted file mode 100644 index b3de2ff5d..000000000 --- a/datajoint/external.py +++ /dev/null @@ -1,515 +0,0 @@ -import logging -from collections.abc import Mapping -from pathlib import Path, PurePosixPath, PureWindowsPath - -from tqdm import tqdm - -from . import errors, s3 -from .declare import EXTERNAL_TABLE_ROOT -from .errors import DataJointError, MissingExternalFile -from .hash import uuid_from_buffer, uuid_from_file -from .heading import Heading -from .settings import config -from .table import FreeTable, Table -from .utils import safe_copy, safe_write - -logger = logging.getLogger(__name__.split(".")[0]) - -CACHE_SUBFOLDING = ( - 2, - 2, -) # (2, 2) means "0123456789abcd" will be saved as "01/23/0123456789abcd" -SUPPORT_MIGRATED_BLOBS = True # support blobs migrated from datajoint 0.11.* - - -def subfold(name, folds): - """ - subfolding for external storage: e.g. subfold('aBCdefg', (2, 3)) --> ['ab','cde'] - """ - return ( - (name[: folds[0]].lower(),) + subfold(name[folds[0] :], folds[1:]) - if folds - else () - ) - - -class ExternalTable(Table): - """ - The table tracking externally stored objects. - Declare as ExternalTable(connection, database) - """ - - def __init__(self, connection, store, database): - self.store = store - self.spec = config.get_store_spec(store) - self._s3 = None - self.database = database - self._connection = connection - self._heading = Heading( - table_info=dict( - conn=connection, - database=database, - table_name=self.table_name, - context=None, - ) - ) - self._support = [self.full_table_name] - if not self.is_declared: - self.declare() - self._s3 = None - if self.spec["protocol"] == "file" and not Path(self.spec["location"]).is_dir(): - raise FileNotFoundError( - "Inaccessible local directory %s" % self.spec["location"] - ) from None - - @property - def definition(self): - return """ - # external storage tracking - hash : uuid # hash of contents (blob), of filename + contents (attach), or relative filepath (filepath) - --- - size :bigint unsigned # size of object in bytes - attachment_name=null : varchar(255) # the filename of an attachment - filepath=null : varchar(1000) # relative filepath or attachment filename - contents_hash=null : uuid # used for the filepath datatype - timestamp=CURRENT_TIMESTAMP :timestamp # automatic timestamp - """ - - @property - def table_name(self): - return f"{EXTERNAL_TABLE_ROOT}_{self.store}" - - @property - def s3(self): - if self._s3 is None: - self._s3 = s3.Folder(**self.spec) - return self._s3 - - # - low-level operations - private - - def _make_external_filepath(self, relative_filepath): - """resolve the complete external path based on the relative path""" - # Strip root - if self.spec["protocol"] == "s3": - posix_path = PurePosixPath(PureWindowsPath(self.spec["location"])) - location_path = ( - Path(*posix_path.parts[1:]) - if len(self.spec["location"]) > 0 - and any(case in posix_path.parts[0] for case in ("\\", ":")) - else Path(posix_path) - ) - return PurePosixPath(location_path, relative_filepath) - # Preserve root - elif self.spec["protocol"] == "file": - return PurePosixPath(Path(self.spec["location"]), relative_filepath) - else: - assert False - - def _make_uuid_path(self, uuid, suffix=""): - """create external path based on the uuid hash""" - return self._make_external_filepath( - PurePosixPath( - self.database, - "/".join(subfold(uuid.hex, self.spec["subfolding"])), - uuid.hex, - ).with_suffix(suffix) - ) - - def _upload_file(self, local_path, external_path, metadata=None): - if self.spec["protocol"] == "s3": - self.s3.fput(local_path, external_path, metadata) - elif self.spec["protocol"] == "file": - safe_copy(local_path, external_path, overwrite=True) - else: - assert False - - def _download_file(self, external_path, download_path): - if self.spec["protocol"] == "s3": - self.s3.fget(external_path, download_path) - elif self.spec["protocol"] == "file": - safe_copy(external_path, download_path) - else: - assert False - - def _upload_buffer(self, buffer, external_path): - if self.spec["protocol"] == "s3": - self.s3.put(external_path, buffer) - elif self.spec["protocol"] == "file": - safe_write(external_path, buffer) - else: - assert False - - def _download_buffer(self, external_path): - if self.spec["protocol"] == "s3": - return self.s3.get(external_path) - if self.spec["protocol"] == "file": - try: - return Path(external_path).read_bytes() - except FileNotFoundError: - raise errors.MissingExternalFile( - f"Missing external file {external_path}" - ) from None - assert False - - def _remove_external_file(self, external_path): - if self.spec["protocol"] == "s3": - self.s3.remove_object(external_path) - elif self.spec["protocol"] == "file": - try: - Path(external_path).unlink() - except FileNotFoundError: - pass - - def exists(self, external_filepath): - """ - :return: True if the external file is accessible - """ - if self.spec["protocol"] == "s3": - return self.s3.exists(external_filepath) - if self.spec["protocol"] == "file": - return Path(external_filepath).is_file() - assert False - - # --- BLOBS ---- - - def put(self, blob): - """ - put a binary string (blob) in external store - """ - uuid = uuid_from_buffer(blob) - self._upload_buffer(blob, self._make_uuid_path(uuid)) - # insert tracking info - self.connection.query( - "INSERT INTO {tab} (hash, size) VALUES (%s, {size}) ON DUPLICATE KEY " - "UPDATE timestamp=CURRENT_TIMESTAMP".format( - tab=self.full_table_name, size=len(blob) - ), - args=(uuid.bytes,), - ) - return uuid - - def get(self, uuid): - """ - get an object from external store. - """ - if uuid is None: - return None - # attempt to get object from cache - blob = None - cache_folder = config.get("cache", None) - if cache_folder: - try: - cache_path = Path(cache_folder, *subfold(uuid.hex, CACHE_SUBFOLDING)) - cache_file = Path(cache_path, uuid.hex) - blob = cache_file.read_bytes() - except FileNotFoundError: - pass # not cached - # download blob from external store - if blob is None: - try: - blob = self._download_buffer(self._make_uuid_path(uuid)) - except MissingExternalFile: - if not SUPPORT_MIGRATED_BLOBS: - raise - # blobs migrated from datajoint 0.11 are stored at explicitly defined filepaths - relative_filepath, contents_hash = (self & {"hash": uuid}).fetch1( - "filepath", "contents_hash" - ) - if relative_filepath is None: - raise - blob = self._download_buffer( - self._make_external_filepath(relative_filepath) - ) - if cache_folder: - cache_path.mkdir(parents=True, exist_ok=True) - safe_write(cache_path / uuid.hex, blob) - return blob - - # --- ATTACHMENTS --- - - def upload_attachment(self, local_path): - attachment_name = Path(local_path).name - uuid = uuid_from_file(local_path, init_string=attachment_name + "\0") - external_path = self._make_uuid_path(uuid, "." + attachment_name) - self._upload_file(local_path, external_path) - # insert tracking info - self.connection.query( - """ - INSERT INTO {tab} (hash, size, attachment_name) - VALUES (%s, {size}, "{attachment_name}") - ON DUPLICATE KEY UPDATE timestamp=CURRENT_TIMESTAMP""".format( - tab=self.full_table_name, - size=Path(local_path).stat().st_size, - attachment_name=attachment_name, - ), - args=[uuid.bytes], - ) - return uuid - - def get_attachment_name(self, uuid): - return (self & {"hash": uuid}).fetch1("attachment_name") - - def download_attachment(self, uuid, attachment_name, download_path): - """save attachment from memory buffer into the save_path""" - external_path = self._make_uuid_path(uuid, "." + attachment_name) - self._download_file(external_path, download_path) - - # --- FILEPATH --- - - def upload_filepath(self, local_filepath): - """ - Raise exception if an external entry already exists with a different contents checksum. - Otherwise, copy (with overwrite) file to remote and - If an external entry exists with the same checksum, then no copying should occur - """ - local_filepath = Path(local_filepath) - try: - relative_filepath = str( - local_filepath.relative_to(self.spec["stage"]).as_posix() - ) - except ValueError: - raise DataJointError( - "The path {path} is not in stage {stage}".format( - path=local_filepath.parent, **self.spec - ) - ) - uuid = uuid_from_buffer( - init_string=relative_filepath - ) # hash relative path, not contents - contents_hash = uuid_from_file(local_filepath) - - # check if the remote file already exists and verify that it matches - check_hash = (self & {"hash": uuid}).fetch("contents_hash") - if check_hash.size: - # the tracking entry exists, check that it's the same file as before - if contents_hash != check_hash[0]: - raise DataJointError( - f"A different version of '{relative_filepath}' has already been placed." - ) - else: - # upload the file and create its tracking entry - self._upload_file( - local_filepath, - self._make_external_filepath(relative_filepath), - metadata={"contents_hash": str(contents_hash)}, - ) - self.connection.query( - "INSERT INTO {tab} (hash, size, filepath, contents_hash) VALUES (%s, {size}, '{filepath}', %s)".format( - tab=self.full_table_name, - size=Path(local_filepath).stat().st_size, - filepath=relative_filepath, - ), - args=(uuid.bytes, contents_hash.bytes), - ) - return uuid - - def download_filepath(self, filepath_hash): - """ - sync a file from external store to the local stage - - :param filepath_hash: The hash (UUID) of the relative_path - :return: hash (UUID) of the contents of the downloaded file or Nones - """ - - def _need_checksum(local_filepath, expected_size): - limit = config.get("filepath_checksum_size_limit") - actual_size = Path(local_filepath).stat().st_size - if expected_size != actual_size: - # this should never happen without outside interference - raise DataJointError( - f"'{local_filepath}' downloaded but size did not match." - ) - return limit is None or actual_size < limit - - if filepath_hash is not None: - relative_filepath, contents_hash, size = ( - self & {"hash": filepath_hash} - ).fetch1("filepath", "contents_hash", "size") - external_path = self._make_external_filepath(relative_filepath) - local_filepath = Path(self.spec["stage"]).absolute() / relative_filepath - - file_exists = Path(local_filepath).is_file() and ( - not _need_checksum(local_filepath, size) - or uuid_from_file(local_filepath) == contents_hash - ) - - if not file_exists: - self._download_file(external_path, local_filepath) - if ( - _need_checksum(local_filepath, size) - and uuid_from_file(local_filepath) != contents_hash - ): - # this should never happen without outside interference - raise DataJointError( - f"'{local_filepath}' downloaded but did not pass checksum." - ) - if not _need_checksum(local_filepath, size): - logger.warning( - f"Skipped checksum for file with hash: {contents_hash}, and path: {local_filepath}" - ) - return str(local_filepath), contents_hash - - # --- UTILITIES --- - - @property - def references(self): - """ - :return: generator of referencing table names and their referencing columns - """ - return ( - {k.lower(): v for k, v in elem.items()} - for elem in self.connection.query( - """ - SELECT concat('`', table_schema, '`.`', table_name, '`') as referencing_table, column_name - FROM information_schema.key_column_usage - WHERE referenced_table_name="{tab}" and referenced_table_schema="{db}" - """.format( - tab=self.table_name, db=self.database - ), - as_dict=True, - ) - ) - - def fetch_external_paths(self, **fetch_kwargs): - """ - generate complete external filepaths from the query. - Each element is a tuple: (uuid, path) - - :param fetch_kwargs: keyword arguments to pass to fetch - """ - fetch_kwargs.update(as_dict=True) - paths = [] - for item in self.fetch("hash", "attachment_name", "filepath", **fetch_kwargs): - if item["attachment_name"]: - # attachments - path = self._make_uuid_path(item["hash"], "." + item["attachment_name"]) - elif item["filepath"]: - # external filepaths - path = self._make_external_filepath(item["filepath"]) - else: - # blobs - path = self._make_uuid_path(item["hash"]) - paths.append((item["hash"], path)) - return paths - - def unused(self): - """ - query expression for unused hashes - - :return: self restricted to elements that are not in use by any tables in the schema - """ - return self - [ - FreeTable(self.connection, ref["referencing_table"]).proj( - hash=ref["column_name"] - ) - for ref in self.references - ] - - def used(self): - """ - query expression for used hashes - - :return: self restricted to elements that in use by tables in the schema - """ - return self & [ - FreeTable(self.connection, ref["referencing_table"]).proj( - hash=ref["column_name"] - ) - for ref in self.references - ] - - def delete( - self, - *, - delete_external_files=None, - limit=None, - display_progress=True, - errors_as_string=True, - ): - """ - - :param delete_external_files: True or False. If False, only the tracking info is removed from the external - store table but the external files remain intact. If True, then the external files themselves are deleted too. - :param errors_as_string: If True any errors returned when deleting from external files will be strings - :param limit: (integer) limit the number of items to delete - :param display_progress: if True, display progress as files are cleaned up - :return: if deleting external files, returns errors - """ - if delete_external_files not in (True, False): - raise DataJointError( - "The delete_external_files argument must be set to either " - "True or False in delete()" - ) - - if not delete_external_files: - self.unused().delete_quick() - else: - items = self.unused().fetch_external_paths(limit=limit) - if display_progress: - items = tqdm(items) - # delete items one by one, close to transaction-safe - error_list = [] - for uuid, external_path in items: - row = (self & {"hash": uuid}).fetch() - if row.size: - try: - (self & {"hash": uuid}).delete_quick() - except Exception: - pass # if delete failed, do not remove the external file - else: - try: - self._remove_external_file(external_path) - except Exception as error: - # adding row back into table after failed delete - self.insert1(row[0], skip_duplicates=True) - error_list.append( - ( - uuid, - external_path, - str(error) if errors_as_string else error, - ) - ) - return error_list - - -class ExternalMapping(Mapping): - """ - The external manager contains all the tables for all external stores for a given schema - :Example: - e = ExternalMapping(schema) - external_table = e[store] - """ - - def __init__(self, schema): - self.schema = schema - self._tables = {} - - def __repr__(self): - return "External file tables for schema `{schema}`:\n ".format( - schema=self.schema.database - ) + "\n ".join( - '"{store}" {protocol}:{location}'.format(store=k, **v.spec) - for k, v in self.items() - ) - - def __getitem__(self, store): - """ - Triggers the creation of an external table. - Should only be used when ready to save or read from external storage. - - :param store: the name of the store - :return: the ExternalTable object for the store - """ - if store not in self._tables: - self._tables[store] = ExternalTable( - connection=self.schema.connection, - store=store, - database=self.schema.database, - ) - return self._tables[store] - - def __len__(self): - return len(self._tables) - - def __iter__(self): - return iter(self._tables) diff --git a/datajoint/fetch.py b/datajoint/fetch.py deleted file mode 100644 index 1c9b811f1..000000000 --- a/datajoint/fetch.py +++ /dev/null @@ -1,344 +0,0 @@ -import itertools -import json -import numbers -import uuid -from functools import partial -from pathlib import Path - -import numpy as np -import pandas - -from datajoint.condition import Top - -from . import blob, hash -from .errors import DataJointError -from .settings import config -from .utils import safe_write - - -class key: - """ - object that allows requesting the primary key as an argument in expression.fetch() - The string "KEY" can be used instead of the class key - """ - - pass - - -def is_key(attr): - return attr is key or attr == "KEY" - - -def to_dicts(recarray): - """convert record array to a dictionaries""" - for rec in recarray: - yield dict(zip(recarray.dtype.names, rec.tolist())) - - -def _get(connection, attr, data, squeeze, download_path): - """ - This function is called for every attribute - - :param connection: a dj.Connection object - :param attr: attribute name from the table's heading - :param data: literal value fetched from the table - :param squeeze: if True squeeze blobs - :param download_path: for fetches that download data, e.g. attachments - :return: unpacked data - """ - if data is None: - return - if attr.json: - return json.loads(data) - - extern = ( - connection.schemas[attr.database].external[attr.store] - if attr.is_external - else None - ) - - # apply attribute adapter if present - adapt = attr.adapter.get if attr.adapter else lambda x: x - - if attr.is_filepath: - return adapt(extern.download_filepath(uuid.UUID(bytes=data))[0]) - if attr.is_attachment: - # Steps: - # 1. get the attachment filename - # 2. check if the file already exists at download_path, verify checksum - # 3. if exists and checksum passes then return the local filepath - # 4. Otherwise, download the remote file and return the new filepath - _uuid = uuid.UUID(bytes=data) if attr.is_external else None - attachment_name = ( - extern.get_attachment_name(_uuid) - if attr.is_external - else data.split(b"\0", 1)[0].decode() - ) - local_filepath = Path(download_path) / attachment_name - if local_filepath.is_file(): - attachment_checksum = ( - _uuid if attr.is_external else hash.uuid_from_buffer(data) - ) - if attachment_checksum == hash.uuid_from_file( - local_filepath, init_string=attachment_name + "\0" - ): - return adapt( - str(local_filepath) - ) # checksum passed, no need to download again - # generate the next available alias filename - for n in itertools.count(): - f = local_filepath.parent / ( - local_filepath.stem + "_%04x" % n + local_filepath.suffix - ) - if not f.is_file(): - local_filepath = f - break - if attachment_checksum == hash.uuid_from_file( - f, init_string=attachment_name + "\0" - ): - return adapt(str(f)) # checksum passed, no need to download again - # Save attachment - if attr.is_external: - extern.download_attachment(_uuid, attachment_name, local_filepath) - else: - # write from buffer - safe_write(local_filepath, data.split(b"\0", 1)[1]) - return adapt(str(local_filepath)) # download file from remote store - - return adapt( - uuid.UUID(bytes=data) - if attr.uuid - else ( - blob.unpack( - extern.get(uuid.UUID(bytes=data)) if attr.is_external else data, - squeeze=squeeze, - ) - if attr.is_blob - else data - ) - ) - - -class Fetch: - """ - A fetch object that handles retrieving elements from the table expression. - - :param expression: the QueryExpression object to fetch from. - """ - - def __init__(self, expression): - self._expression = expression - - def __call__( - self, - *attrs, - offset=None, - limit=None, - order_by=None, - format=None, - as_dict=None, - squeeze=False, - download_path=".", - ): - """ - Fetches the expression results from the database into an np.array or list of dictionaries and - unpacks blob attributes. - - :param attrs: zero or more attributes to fetch. If not provided, the call will return all attributes of this - table. If provided, returns tuples with an entry for each attribute. - :param offset: the number of tuples to skip in the returned result - :param limit: the maximum number of tuples to return - :param order_by: a single attribute or the list of attributes to order the results. No ordering should be assumed - if order_by=None. To reverse the order, add DESC to the attribute name or names: e.g. ("age DESC", - "frequency") To order by primary key, use "KEY" or "KEY DESC" - :param format: Effective when as_dict=None and when attrs is empty None: default from config['fetch_format'] or - 'array' if not configured "array": use numpy.key_array "frame": output pandas.DataFrame. . - :param as_dict: returns a list of dictionaries instead of a record array. Defaults to False for .fetch() and to - True for .fetch('KEY') - :param squeeze: if True, remove extra dimensions from arrays - :param download_path: for fetches that download data, e.g. attachments - :return: the contents of the table in the form of a structured numpy.array or a dict list - """ - if offset or order_by or limit: - self._expression = self._expression.restrict( - Top( - limit, - order_by, - offset, - ) - ) - - attrs_as_dict = as_dict and attrs - if attrs_as_dict: - # absorb KEY into attrs and prepare to return attributes as dict (issue #595) - if any(is_key(k) for k in attrs): - attrs = list(self._expression.primary_key) + [ - a for a in attrs if a not in self._expression.primary_key - ] - if as_dict is None: - as_dict = bool(attrs) # default to True for "KEY" and False otherwise - # format should not be specified with attrs or is_dict=True - if format is not None and (as_dict or attrs): - raise DataJointError( - "Cannot specify output format when as_dict=True or " - "when attributes are selected to be fetched separately." - ) - if format not in {None, "array", "frame"}: - raise DataJointError( - "Fetch output format must be in " - '{{"array", "frame"}} but "{}" was given'.format(format) - ) - - if not (attrs or as_dict) and format is None: - format = config["fetch_format"] # default to array - if format not in {"array", "frame"}: - raise DataJointError( - 'Invalid entry "{}" in datajoint.config["fetch_format"]: ' - 'use "array" or "frame"'.format(format) - ) - - get = partial( - _get, - self._expression.connection, - squeeze=squeeze, - download_path=download_path, - ) - if attrs: # a list of attributes provided - attributes = [a for a in attrs if not is_key(a)] - ret = self._expression.proj(*attributes) - ret = ret.fetch( - offset=offset, - limit=limit, - order_by=order_by, - as_dict=False, - squeeze=squeeze, - download_path=download_path, - format="array", - ) - if attrs_as_dict: - ret = [ - {k: v for k, v in zip(ret.dtype.names, x) if k in attrs} - for x in ret - ] - else: - return_values = [ - ( - list( - (to_dicts if as_dict else lambda x: x)( - ret[self._expression.primary_key] - ) - ) - if is_key(attribute) - else ret[attribute] - ) - for attribute in attrs - ] - ret = return_values[0] if len(attrs) == 1 else return_values - else: # fetch all attributes as a numpy.record_array or pandas.DataFrame - cur = self._expression.cursor(as_dict=as_dict) - heading = self._expression.heading - if as_dict: - ret = [ - dict((name, get(heading[name], d[name])) for name in heading.names) - for d in cur - ] - else: - ret = list(cur.fetchall()) - record_type = ( - heading.as_dtype - if not ret - else np.dtype( - [ - ( - ( - name, - type(value), - ) # use the first element to determine blob type - if heading[name].is_blob - and isinstance(value, numbers.Number) - else (name, heading.as_dtype[name]) - ) - for value, name in zip(ret[0], heading.as_dtype.names) - ] - ) - ) - try: - ret = np.array(ret, dtype=record_type) - except Exception as e: - raise e - for name in heading: - # unpack blobs and externals - ret[name] = list(map(partial(get, heading[name]), ret[name])) - if format == "frame": - ret = pandas.DataFrame(ret).set_index(heading.primary_key) - return ret - - -class Fetch1: - """ - Fetch object for fetching the result of a query yielding one row. - - :param expression: a query expression to fetch from. - """ - - def __init__(self, expression): - self._expression = expression - - def __call__(self, *attrs, squeeze=False, download_path="."): - """ - Fetches the result of a query expression that yields one entry. - - If no attributes are specified, returns the result as a dict. - If attributes are specified returns the corresponding results as a tuple. - - Examples: - d = rel.fetch1() # as a dictionary - a, b = rel.fetch1('a', 'b') # as a tuple - - :params *attrs: attributes to return when expanding into a tuple. - If attrs is empty, the return result is a dict - :param squeeze: When true, remove extra dimensions from arrays in attributes - :param download_path: for fetches that download data, e.g. attachments - :return: the one tuple in the table in the form of a dict - """ - heading = self._expression.heading - - if not attrs: # fetch all attributes, return as ordered dict - cur = self._expression.cursor(as_dict=True) - ret = cur.fetchone() - if not ret or cur.fetchone(): - raise DataJointError( - "fetch1 requires exactly one tuple in the input set." - ) - ret = dict( - ( - name, - _get( - self._expression.connection, - heading[name], - ret[name], - squeeze=squeeze, - download_path=download_path, - ), - ) - for name in heading.names - ) - else: # fetch some attributes, return as tuple - attributes = [a for a in attrs if not is_key(a)] - result = self._expression.proj(*attributes).fetch( - squeeze=squeeze, download_path=download_path, format="array" - ) - if len(result) != 1: - raise DataJointError( - "fetch1 should only return one tuple. %d tuples found" % len(result) - ) - return_values = tuple( - ( - next(to_dicts(result[self._expression.primary_key])) - if is_key(attribute) - else result[attribute][0] - ) - for attribute in attrs - ) - ret = return_values[0] if len(attrs) == 1 else return_values - return ret diff --git a/datajoint/hash.py b/datajoint/hash.py deleted file mode 100644 index f58c65732..000000000 --- a/datajoint/hash.py +++ /dev/null @@ -1,39 +0,0 @@ -import hashlib -import io -import uuid -from pathlib import Path - - -def key_hash(mapping): - """ - 32-byte hash of the mapping's key values sorted by the key name. - This is often used to convert a long primary key value into a shorter hash. - For example, the JobTable in datajoint.jobs uses this function to hash the primary key of autopopulated tables. - """ - hashed = hashlib.md5() - for k, v in sorted(mapping.items()): - hashed.update(str(v).encode()) - return hashed.hexdigest() - - -def uuid_from_stream(stream, *, init_string=""): - """ - :return: 16-byte digest of stream data - :stream: stream object or open file handle - :init_string: string to initialize the checksum - """ - hashed = hashlib.md5(init_string.encode()) - chunk = True - chunk_size = 1 << 14 - while chunk: - chunk = stream.read(chunk_size) - hashed.update(chunk) - return uuid.UUID(bytes=hashed.digest()) - - -def uuid_from_buffer(buffer=b"", *, init_string=""): - return uuid_from_stream(io.BytesIO(buffer), init_string=init_string) - - -def uuid_from_file(filepath, *, init_string=""): - return uuid_from_stream(Path(filepath).open("rb"), init_string=init_string) diff --git a/datajoint/heading.py b/datajoint/heading.py deleted file mode 100644 index c81b5a61a..000000000 --- a/datajoint/heading.py +++ /dev/null @@ -1,533 +0,0 @@ -import logging -import re -from collections import defaultdict, namedtuple -from itertools import chain - -import numpy as np - -from .attribute_adapter import AttributeAdapter, get_adapter -from .declare import ( - EXTERNAL_TYPES, - NATIVE_TYPES, - SPECIAL_TYPES, - TYPE_PATTERN, - UUID_DATA_TYPE, -) -from .errors import FILEPATH_FEATURE_SWITCH, DataJointError, _support_filepath_types - -logger = logging.getLogger(__name__.split(".")[0]) - -default_attribute_properties = ( - dict( # these default values are set in computed attributes - name=None, - type="expression", - in_key=False, - nullable=False, - default=None, - comment="calculated attribute", - autoincrement=False, - numeric=None, - string=None, - uuid=False, - json=None, - is_blob=False, - is_attachment=False, - is_filepath=False, - is_external=False, - is_hidden=False, - adapter=None, - store=None, - unsupported=False, - attribute_expression=None, - database=None, - dtype=object, - ) -) - - -class Attribute(namedtuple("_Attribute", default_attribute_properties)): - """ - Properties of a table column (attribute) - """ - - def todict(self): - """Convert namedtuple to dict.""" - return dict((name, self[i]) for i, name in enumerate(self._fields)) - - @property - def sql_type(self): - """:return: datatype (as string) in database. In most cases, it is the same as self.type""" - return UUID_DATA_TYPE if self.uuid else self.type - - @property - def sql_comment(self): - """:return: full comment for the SQL declaration. Includes custom type specification""" - return (":uuid:" if self.uuid else "") + self.comment - - @property - def sql(self): - """ - Convert primary key attribute tuple into its SQL CREATE TABLE clause. - Default values are not reflected. - This is used for declaring foreign keys in referencing tables - - :return: SQL code for attribute declaration - """ - return '`{name}` {type} NOT NULL COMMENT "{comment}"'.format( - name=self.name, type=self.sql_type, comment=self.sql_comment - ) - - @property - def original_name(self): - if self.attribute_expression is None: - return self.name - assert self.attribute_expression.startswith("`") - return self.attribute_expression.strip("`") - - -class Heading: - """ - Local class for table headings. - Heading contains the property attributes, which is an dict in which the keys are - the attribute names and the values are Attributes. - """ - - def __init__(self, attribute_specs=None, table_info=None): - """ - - :param attribute_specs: a list of dicts with the same keys as Attribute - :param table_info: a dict with information to load the heading from the database - """ - self.indexes = None - self.table_info = table_info - self._table_status = None - self._attributes = ( - None - if attribute_specs is None - else dict((q["name"], Attribute(**q)) for q in attribute_specs) - ) - - def __len__(self): - return 0 if self.attributes is None else len(self.attributes) - - @property - def table_status(self): - if self.table_info is None: - return None - if self._table_status is None: - self._init_from_database() - return self._table_status - - @property - def attributes(self): - if self._attributes is None: - self._init_from_database() # lazy loading from database - return {k: v for k, v in self._attributes.items() if not v.is_hidden} - - @property - def names(self): - return [k for k in self.attributes] - - @property - def primary_key(self): - return [k for k, v in self.attributes.items() if v.in_key] - - @property - def secondary_attributes(self): - return [k for k, v in self.attributes.items() if not v.in_key] - - @property - def blobs(self): - return [k for k, v in self.attributes.items() if v.is_blob] - - @property - def non_blobs(self): - return [ - k - for k, v in self.attributes.items() - if not (v.is_blob or v.is_attachment or v.is_filepath or v.json) - ] - - @property - def new_attributes(self): - return [ - k for k, v in self.attributes.items() if v.attribute_expression is not None - ] - - def __getitem__(self, name): - """shortcut to the attribute""" - return self.attributes[name] - - def __repr__(self): - """ - :return: heading representation in DataJoint declaration format but without foreign key expansion - """ - in_key = True - ret = "" - if self._table_status is not None: - ret += "# " + self.table_status["comment"] + "\n" - for v in self.attributes.values(): - if in_key and not v.in_key: - ret += "---\n" - in_key = False - ret += "%-20s : %-28s # %s\n" % ( - v.name if v.default is None else "%s=%s" % (v.name, v.default), - "%s%s" % (v.type, "auto_increment" if v.autoincrement else ""), - v.comment, - ) - return ret - - @property - def has_autoincrement(self): - return any(e.autoincrement for e in self.attributes.values()) - - @property - def as_dtype(self): - """ - represent the heading as a numpy dtype - """ - return np.dtype( - dict(names=self.names, formats=[v.dtype for v in self.attributes.values()]) - ) - - def as_sql(self, fields, include_aliases=True): - """ - represent heading as the SQL SELECT clause. - """ - return ",".join( - ( - "`%s`" % name - if self.attributes[name].attribute_expression is None - else self.attributes[name].attribute_expression - + (" as `%s`" % name if include_aliases else "") - ) - for name in fields - ) - - def __iter__(self): - return iter(self.attributes) - - def _init_from_database(self): - """initialize heading from an existing database table.""" - conn, database, table_name, context = ( - self.table_info[k] for k in ("conn", "database", "table_name", "context") - ) - info = conn.query( - 'SHOW TABLE STATUS FROM `{database}` WHERE name="{table_name}"'.format( - table_name=table_name, database=database - ), - as_dict=True, - ).fetchone() - if info is None: - if table_name == "~log": - logger.warning("Could not create the ~log table") - return - raise DataJointError( - "The table `{database}`.`{table_name}` is not defined.".format( - table_name=table_name, database=database - ) - ) - self._table_status = {k.lower(): v for k, v in info.items()} - cur = conn.query( - "SHOW FULL COLUMNS FROM `{table_name}` IN `{database}`".format( - table_name=table_name, database=database - ), - as_dict=True, - ) - - attributes = cur.fetchall() - - rename_map = { - "Field": "name", - "Type": "type", - "Null": "nullable", - "Default": "default", - "Key": "in_key", - "Comment": "comment", - } - - fields_to_drop = ("Privileges", "Collation") - - # rename and drop attributes - attributes = [ - { - rename_map[k] if k in rename_map else k: v - for k, v in x.items() - if k not in fields_to_drop - } - for x in attributes - ] - numeric_types = { - ("float", False): np.float64, - ("float", True): np.float64, - ("double", False): np.float64, - ("double", True): np.float64, - ("tinyint", False): np.int64, - ("tinyint", True): np.int64, - ("smallint", False): np.int64, - ("smallint", True): np.int64, - ("mediumint", False): np.int64, - ("mediumint", True): np.int64, - ("int", False): np.int64, - ("int", True): np.int64, - ("bigint", False): np.int64, - ("bigint", True): np.uint64, - } - - sql_literals = ["CURRENT_TIMESTAMP"] - - # additional attribute properties - for attr in attributes: - attr.update( - in_key=(attr["in_key"] == "PRI"), - database=database, - nullable=attr["nullable"] == "YES", - autoincrement=bool( - re.search(r"auto_increment", attr["Extra"], flags=re.I) - ), - numeric=any( - TYPE_PATTERN[t].match(attr["type"]) - for t in ("DECIMAL", "INTEGER", "FLOAT") - ), - string=any( - TYPE_PATTERN[t].match(attr["type"]) - for t in ("ENUM", "TEMPORAL", "STRING") - ), - is_blob=bool(TYPE_PATTERN["INTERNAL_BLOB"].match(attr["type"])), - uuid=False, - json=bool(TYPE_PATTERN["JSON"].match(attr["type"])), - is_attachment=False, - is_filepath=False, - adapter=None, - store=None, - is_external=False, - attribute_expression=None, - is_hidden=attr["name"].startswith("_"), - ) - - if any(TYPE_PATTERN[t].match(attr["type"]) for t in ("INTEGER", "FLOAT")): - attr["type"] = re.sub( - r"\(\d+\)", "", attr["type"], count=1 - ) # strip size off integers and floats - attr["unsupported"] = not any( - (attr["is_blob"], attr["numeric"], attr["numeric"]) - ) - attr.pop("Extra") - - # process custom DataJoint types - special = re.match(r":(?P[^:]+):(?P.*)", attr["comment"]) - if special: - special = special.groupdict() - attr.update(special) - # process adapted attribute types - if special and TYPE_PATTERN["ADAPTED"].match(attr["type"]): - assert context is not None, "Declaration context is not set" - adapter_name = special["type"] - try: - attr.update(adapter=get_adapter(context, adapter_name)) - except DataJointError: - # if no adapter, then delay the error until the first invocation - attr.update(adapter=AttributeAdapter()) - else: - attr.update(type=attr["adapter"].attribute_type) - if not any(r.match(attr["type"]) for r in TYPE_PATTERN.values()): - raise DataJointError( - "Invalid attribute type '{type}' in adapter object <{adapter_name}>.".format( - adapter_name=adapter_name, **attr - ) - ) - special = not any( - TYPE_PATTERN[c].match(attr["type"]) for c in NATIVE_TYPES - ) - - if special: - try: - category = next( - c for c in SPECIAL_TYPES if TYPE_PATTERN[c].match(attr["type"]) - ) - except StopIteration: - if attr["type"].startswith("external"): - url = ( - "https://docs.datajoint.io/python/admin/5-blob-config.html" - "#migration-between-datajoint-v0-11-and-v0-12" - ) - raise DataJointError( - "Legacy datatype `{type}`. Migrate your external stores to " - "datajoint 0.12: {url}".format(url=url, **attr) - ) - raise DataJointError( - "Unknown attribute type `{type}`".format(**attr) - ) - if category == "FILEPATH" and not _support_filepath_types(): - raise DataJointError( - """ - The filepath data type is disabled until complete validation. - To turn it on as experimental feature, set the environment variable - {env} = TRUE or upgrade datajoint. - """.format( - env=FILEPATH_FEATURE_SWITCH - ) - ) - attr.update( - unsupported=False, - is_attachment=category in ("INTERNAL_ATTACH", "EXTERNAL_ATTACH"), - is_filepath=category == "FILEPATH", - # INTERNAL_BLOB is not a custom type but is included for completeness - is_blob=category in ("INTERNAL_BLOB", "EXTERNAL_BLOB"), - uuid=category == "UUID", - is_external=category in EXTERNAL_TYPES, - store=( - attr["type"].split("@")[1] - if category in EXTERNAL_TYPES - else None - ), - ) - - if attr["in_key"] and any( - ( - attr["is_blob"], - attr["is_attachment"], - attr["is_filepath"], - attr["json"], - ) - ): - raise DataJointError( - "Json, Blob, attachment, or filepath attributes are not allowed in the primary key" - ) - - if ( - attr["string"] - and attr["default"] is not None - and attr["default"] not in sql_literals - ): - attr["default"] = '"%s"' % attr["default"] - - if attr["nullable"]: # nullable fields always default to null - attr["default"] = "null" - - # fill out dtype. All floats and non-nullable integers are turned into specific dtypes - attr["dtype"] = object - if attr["numeric"] and not attr["adapter"]: - is_integer = TYPE_PATTERN["INTEGER"].match(attr["type"]) - is_float = TYPE_PATTERN["FLOAT"].match(attr["type"]) - if is_integer and not attr["nullable"] or is_float: - is_unsigned = bool(re.match("sunsigned", attr["type"], flags=re.I)) - t = re.sub(r"\(.*\)", "", attr["type"]) # remove parentheses - t = re.sub(r" unsigned$", "", t) # remove unsigned - assert (t, is_unsigned) in numeric_types, ( - "dtype not found for type %s" % t - ) - attr["dtype"] = numeric_types[(t, is_unsigned)] - - if attr["adapter"]: - # restore adapted type name - attr["type"] = adapter_name - - self._attributes = dict(((q["name"], Attribute(**q)) for q in attributes)) - - # Read and tabulate secondary indexes - keys = defaultdict(dict) - for item in conn.query( - "SHOW KEYS FROM `{db}`.`{tab}`".format(db=database, tab=table_name), - as_dict=True, - ): - if item["Key_name"] != "PRIMARY": - keys[item["Key_name"]][item["Seq_in_index"]] = dict( - column=item["Column_name"] - or f"({item['Expression']})".replace(r"\'", "'"), - unique=(item["Non_unique"] == 0), - nullable=item["Null"].lower() == "yes", - ) - self.indexes = { - tuple(item[k]["column"] for k in sorted(item.keys())): dict( - unique=item[1]["unique"], - nullable=any(v["nullable"] for v in item.values()), - ) - for item in keys.values() - } - - def select(self, select_list, rename_map=None, compute_map=None): - """ - derive a new heading by selecting, renaming, or computing attributes. - In relational algebra these operators are known as project, rename, and extend. - - :param select_list: the full list of existing attributes to include - :param rename_map: dictionary of renamed attributes: keys=new names, values=old names - :param compute_map: a direction of computed attributes - This low-level method performs no error checking. - """ - rename_map = rename_map or {} - compute_map = compute_map or {} - copy_attrs = list() - for name in self.attributes: - if name in select_list: - copy_attrs.append(self.attributes[name].todict()) - copy_attrs.extend( - ( - dict( - self.attributes[old_name].todict(), - name=new_name, - attribute_expression="`%s`" % old_name, - ) - for new_name, old_name in rename_map.items() - if old_name == name - ) - ) - compute_attrs = ( - dict(default_attribute_properties, name=new_name, attribute_expression=expr) - for new_name, expr in compute_map.items() - ) - return Heading(chain(copy_attrs, compute_attrs)) - - def join(self, other): - """ - Join two headings into a new one. - It assumes that self and other are headings that share no common dependent attributes. - """ - return Heading( - [self.attributes[name].todict() for name in self.primary_key] - + [ - other.attributes[name].todict() - for name in other.primary_key - if name not in self.primary_key - ] - + [ - self.attributes[name].todict() - for name in self.secondary_attributes - if name not in other.primary_key - ] - + [ - other.attributes[name].todict() - for name in other.secondary_attributes - if name not in self.primary_key - ] - ) - - def set_primary_key(self, primary_key): - """ - Create a new heading with the specified primary key. - This low-level method performs no error checking. - """ - return Heading( - chain( - ( - dict(self.attributes[name].todict(), in_key=True) - for name in primary_key - ), - ( - dict(self.attributes[name].todict(), in_key=False) - for name in self.names - if name not in primary_key - ), - ) - ) - - def make_subquery_heading(self): - """ - Create a new heading with removed attribute sql_expressions. - Used by subqueries, which resolve the sql_expressions. - """ - return Heading( - dict(v.todict(), attribute_expression=None) - for v in self.attributes.values() - ) diff --git a/datajoint/jobs.py b/datajoint/jobs.py deleted file mode 100644 index d6b31e13e..000000000 --- a/datajoint/jobs.py +++ /dev/null @@ -1,163 +0,0 @@ -import os -import platform - -from .errors import DuplicateError -from .hash import key_hash -from .heading import Heading -from .settings import config -from .table import Table - -ERROR_MESSAGE_LENGTH = 2047 -TRUNCATION_APPENDIX = "...truncated" - - -class JobTable(Table): - """ - A base table with no definition. Allows reserving jobs - """ - - def __init__(self, conn, database): - self.database = database - self._connection = conn - self._heading = Heading( - table_info=dict( - conn=conn, database=database, table_name=self.table_name, context=None - ) - ) - self._support = [self.full_table_name] - - self._definition = """ # job reservation table for `{database}` - table_name :varchar(255) # className of the table - key_hash :char(32) # key hash - --- - status :enum('reserved','error','ignore') # if tuple is missing, the job is available - key=null :blob # structure containing the key - error_message="" :varchar({error_message_length}) # error message returned if failed - error_stack=null :mediumblob # error stack if failed - user="" :varchar(255) # database user - host="" :varchar(255) # system hostname - pid=0 :int unsigned # system process id - connection_id = 0 : bigint unsigned # connection_id() - timestamp=CURRENT_TIMESTAMP :timestamp # automatic timestamp - """.format( - database=database, error_message_length=ERROR_MESSAGE_LENGTH - ) - if not self.is_declared: - self.declare() - self._user = self.connection.get_user() - - @property - def definition(self): - return self._definition - - @property - def table_name(self): - return "~jobs" - - def delete(self): - """bypass interactive prompts and dependencies""" - self.delete_quick() - - def drop(self): - """bypass interactive prompts and dependencies""" - self.drop_quick() - - def reserve(self, table_name, key): - """ - Reserve a job for computation. When a job is reserved, the job table contains an entry for the - job key, identified by its hash. When jobs are completed, the entry is removed. - - :param table_name: `database`.`table_name` - :param key: the dict of the job's primary key - :return: True if reserved job successfully. False = the jobs is already taken - """ - job = dict( - table_name=table_name, - key_hash=key_hash(key), - status="reserved", - host=platform.node(), - pid=os.getpid(), - connection_id=self.connection.connection_id, - key=key, - user=self._user, - ) - try: - with config(enable_python_native_blobs=True): - self.insert1(job, ignore_extra_fields=True) - except DuplicateError: - return False - return True - - def ignore(self, table_name, key): - """ - Set a job to be ignored for computation. When a job is ignored, the job table contains an entry for the - job key, identified by its hash, with status "ignore". - - Args: - table_name: - Table name (str) - `database`.`table_name` - key: - The dict of the job's primary key - - Returns: - True if ignore job successfully. False = the jobs is already taken - """ - job = dict( - table_name=table_name, - key_hash=key_hash(key), - status="ignore", - host=platform.node(), - pid=os.getpid(), - connection_id=self.connection.connection_id, - key=key, - user=self._user, - ) - try: - with config(enable_python_native_blobs=True): - self.insert1(job, ignore_extra_fields=True) - except DuplicateError: - return False - return True - - def complete(self, table_name, key): - """ - Log a completed job. When a job is completed, its reservation entry is deleted. - - :param table_name: `database`.`table_name` - :param key: the dict of the job's primary key - """ - job_key = dict(table_name=table_name, key_hash=key_hash(key)) - (self & job_key).delete_quick() - - def error(self, table_name, key, error_message, error_stack=None): - """ - Log an error message. The job reservation is replaced with an error entry. - if an error occurs, leave an entry describing the problem - - :param table_name: `database`.`table_name` - :param key: the dict of the job's primary key - :param error_message: string error message - :param error_stack: stack trace - """ - if len(error_message) > ERROR_MESSAGE_LENGTH: - error_message = ( - error_message[: ERROR_MESSAGE_LENGTH - len(TRUNCATION_APPENDIX)] - + TRUNCATION_APPENDIX - ) - with config(enable_python_native_blobs=True): - self.insert1( - dict( - table_name=table_name, - key_hash=key_hash(key), - status="error", - host=platform.node(), - pid=os.getpid(), - connection_id=self.connection.connection_id, - user=self._user, - key=key, - error_message=error_message, - error_stack=error_stack, - ), - replace=True, - ignore_extra_fields=True, - ) diff --git a/datajoint/preview.py b/datajoint/preview.py deleted file mode 100644 index 564c92a0a..000000000 --- a/datajoint/preview.py +++ /dev/null @@ -1,153 +0,0 @@ -"""methods for generating previews of query expression results in python command line and Jupyter""" - -from .settings import config - - -def preview(query_expression, limit, width): - heading = query_expression.heading - rel = query_expression.proj(*heading.non_blobs) - if limit is None: - limit = config["display.limit"] - if width is None: - width = config["display.width"] - tuples = rel.fetch(limit=limit + 1, format="array") - has_more = len(tuples) > limit - tuples = tuples[:limit] - columns = heading.names - widths = { - f: min( - max( - [len(f)] + [len(str(e)) for e in tuples[f]] - if f in tuples.dtype.names - else [len("=BLOB=")] - ) - + 4, - width, - ) - for f in columns - } - templates = {f: "%%-%d.%ds" % (widths[f], widths[f]) for f in columns} - return ( - " ".join( - [templates[f] % ("*" + f if f in rel.primary_key else f) for f in columns] - ) - + "\n" - + " ".join(["+" + "-" * (widths[column] - 2) + "+" for column in columns]) - + "\n" - + "\n".join( - " ".join( - templates[f] % (tup[f] if f in tup.dtype.names else "=BLOB=") - for f in columns - ) - for tup in tuples - ) - + ("\n ...\n" if has_more else "\n") - + (" (Total: %d)\n" % len(rel) if config["display.show_tuple_count"] else "") - ) - - -def repr_html(query_expression): - heading = query_expression.heading - rel = query_expression.proj(*heading.non_blobs) - info = heading.table_status - tuples = rel.fetch(limit=config["display.limit"] + 1, format="array") - has_more = len(tuples) > config["display.limit"] - tuples = tuples[0 : config["display.limit"]] - - css = """ - - """ - head_template = """
-

{column}

- {comment} -
""" - return """ - {css} - {title} -
- - - {body} -
{head}
- {ellipsis} - {count}
- """.format( - css=css, - title="" if info is None else "%s" % info["comment"], - head="".join( - head_template.format( - column=c, - comment=heading.attributes[c].comment, - primary=( - "primary" if c in query_expression.primary_key else "nonprimary" - ), - ) - for c in heading.names - ), - ellipsis="

...

" if has_more else "", - body="".join( - [ - "\n".join( - [ - "%s" - % (tup[name] if name in tup.dtype.names else "=BLOB=") - for name in heading.names - ] - ) - for tup in tuples - ] - ), - count=( - ("

Total: %d

" % len(rel)) - if config["display.show_tuple_count"] - else "" - ), - ) diff --git a/datajoint/s3.py b/datajoint/s3.py deleted file mode 100644 index 98dc75708..000000000 --- a/datajoint/s3.py +++ /dev/null @@ -1,123 +0,0 @@ -""" -AWS S3 operations -""" - -import logging -import uuid -from io import BytesIO -from pathlib import Path - -import minio # https://docs.minio.io/docs/python-client-api-reference -import urllib3 - -from . import errors - -logger = logging.getLogger(__name__.split(".")[0]) - - -class Folder: - """ - A Folder instance manipulates a flat folder of objects within an S3-compatible object store - """ - - def __init__( - self, - endpoint, - bucket, - access_key, - secret_key, - *, - secure=False, - proxy_server=None, - **_, - ): - # from https://docs.min.io/docs/python-client-api-reference - self.client = minio.Minio( - endpoint, - access_key=access_key, - secret_key=secret_key, - secure=secure, - http_client=( - urllib3.ProxyManager( - proxy_server, - timeout=urllib3.Timeout.DEFAULT_TIMEOUT, - cert_reqs="CERT_REQUIRED", - retries=urllib3.Retry( - total=5, - backoff_factor=0.2, - status_forcelist=[500, 502, 503, 504], - ), - ) - if proxy_server - else None - ), - ) - self.bucket = bucket - if not self.client.bucket_exists(bucket): - raise errors.BucketInaccessible("Inaccessible s3 bucket %s" % bucket) - - def put(self, name, buffer): - logger.debug("put: {}:{}".format(self.bucket, name)) - return self.client.put_object( - self.bucket, str(name), BytesIO(buffer), length=len(buffer) - ) - - def fput(self, local_file, name, metadata=None): - logger.debug("fput: {} -> {}:{}".format(self.bucket, local_file, name)) - return self.client.fput_object( - self.bucket, str(name), str(local_file), metadata=metadata - ) - - def get(self, name): - logger.debug("get: {}:{}".format(self.bucket, name)) - try: - with self.client.get_object(self.bucket, str(name)) as result: - data = [d for d in result.stream()] - return b"".join(data) - except minio.error.S3Error as e: - if e.code == "NoSuchKey": - raise errors.MissingExternalFile("Missing s3 key %s" % name) - else: - raise e - - def fget(self, name, local_filepath): - """get file from object name to local filepath""" - logger.debug("fget: {}:{}".format(self.bucket, name)) - name = str(name) - stat = self.client.stat_object(self.bucket, name) - meta = {k.lower().lstrip("x-amz-meta"): v for k, v in stat.metadata.items()} - data = self.client.get_object(self.bucket, name) - local_filepath = Path(local_filepath) - local_filepath.parent.mkdir(parents=True, exist_ok=True) - with local_filepath.open("wb") as f: - for d in data.stream(1 << 16): - f.write(d) - if "contents_hash" in meta: - return uuid.UUID(meta["contents_hash"]) - - def exists(self, name): - logger.debug("exists: {}:{}".format(self.bucket, name)) - try: - self.client.stat_object(self.bucket, str(name)) - except minio.error.S3Error as e: - if e.code == "NoSuchKey": - return False - else: - raise e - return True - - def get_size(self, name): - logger.debug("get_size: {}:{}".format(self.bucket, name)) - try: - return self.client.stat_object(self.bucket, str(name)).size - except minio.error.S3Error as e: - if e.code == "NoSuchKey": - raise errors.MissingExternalFile - raise e - - def remove_object(self, name): - logger.debug("remove_object: {}:{}".format(self.bucket, name)) - try: - self.client.remove_object(self.bucket, str(name)) - except minio.error.MinioException: - raise errors.DataJointError("Failed to delete %s from s3 storage" % name) diff --git a/datajoint/schemas.py b/datajoint/schemas.py deleted file mode 100644 index 8cb7a3668..000000000 --- a/datajoint/schemas.py +++ /dev/null @@ -1,546 +0,0 @@ -import collections -import inspect -import itertools -import logging -import re -import types -import warnings - -from .connection import conn -from .errors import AccessError, DataJointError -from .external import ExternalMapping -from .heading import Heading -from .jobs import JobTable -from .settings import config -from .table import FreeTable, Log, lookup_class_name -from .user_tables import Computed, Imported, Lookup, Manual, Part, _get_tier -from .utils import to_camel_case, user_choice - -logger = logging.getLogger(__name__.split(".")[0]) - - -def ordered_dir(class_): - """ - List (most) attributes of the class including inherited ones, similar to `dir` built-in function, - but respects order of attribute declaration as much as possible. - - :param class_: class to list members for - :return: a list of attributes declared in class_ and its superclasses - """ - attr_list = list() - for c in reversed(class_.mro()): - attr_list.extend(e for e in c.__dict__ if e not in attr_list) - return attr_list - - -class Schema: - """ - A schema object is a decorator for UserTable classes that binds them to their database. - It also specifies the namespace `context` in which other UserTable classes are defined. - """ - - def __init__( - self, - schema_name=None, - context=None, - *, - connection=None, - create_schema=True, - create_tables=True, - add_objects=None, - ): - """ - Associate database schema `schema_name`. If the schema does not exist, attempt to - create it on the server. - - If the schema_name is omitted, then schema.activate(..) must be called later - to associate with the database. - - :param schema_name: the database schema to associate. - :param context: dictionary for looking up foreign key references, leave None to use local context. - :param connection: Connection object. Defaults to datajoint.conn(). - :param create_schema: When False, do not create the schema and raise an error if missing. - :param create_tables: When False, do not create tables and raise errors when accessing missing tables. - :param add_objects: a mapping with additional objects to make available to the context in which table classes - are declared. - """ - self._log = None - self.connection = connection - self.database = None - self.context = context - self.create_schema = create_schema - self.create_tables = create_tables - self._jobs = None - self.external = ExternalMapping(self) - self.add_objects = add_objects - self.declare_list = [] - if schema_name: - self.activate(schema_name) - - def is_activated(self): - return self.database is not None - - def activate( - self, - schema_name=None, - *, - connection=None, - create_schema=None, - create_tables=None, - add_objects=None, - ): - """ - Associate database schema `schema_name`. If the schema does not exist, attempt to - create it on the server. - - :param schema_name: the database schema to associate. - schema_name=None is used to assert that the schema has already been activated. - :param connection: Connection object. Defaults to datajoint.conn(). - :param create_schema: If False, do not create the schema and raise an error if missing. - :param create_tables: If False, do not create tables and raise errors when attempting - to access missing tables. - :param add_objects: a mapping with additional objects to make available to the context - in which table classes are declared. - """ - if schema_name is None: - if self.exists: - return - raise DataJointError("Please provide a schema_name to activate the schema.") - if self.database is not None and self.exists: - if self.database == schema_name: # already activated - return - raise DataJointError( - "The schema is already activated for schema {db}.".format( - db=self.database - ) - ) - if connection is not None: - self.connection = connection - if self.connection is None: - self.connection = conn() - self.database = schema_name - if create_schema is not None: - self.create_schema = create_schema - if create_tables is not None: - self.create_tables = create_tables - if add_objects: - self.add_objects = add_objects - if not self.exists: - if not self.create_schema or not self.database: - raise DataJointError( - "Database `{name}` has not yet been declared. " - "Set argument create_schema=True to create it.".format( - name=schema_name - ) - ) - # create database - logger.debug("Creating schema `{name}`.".format(name=schema_name)) - try: - self.connection.query( - "CREATE DATABASE `{name}`".format(name=schema_name) - ) - except AccessError: - raise DataJointError( - "Schema `{name}` does not exist and could not be created. " - "Check permissions.".format(name=schema_name) - ) - else: - self.log("created") - self.connection.register(self) - - # decorate all tables already decorated - for cls, context in self.declare_list: - if self.add_objects: - context = dict(context, **self.add_objects) - self._decorate_master(cls, context) - - def _assert_exists(self, message=None): - if not self.exists: - raise DataJointError( - message - or "Schema `{db}` has not been created.".format(db=self.database) - ) - - def __call__(self, cls, *, context=None): - """ - Binds the supplied class to a schema. This is intended to be used as a decorator. - - :param cls: class to decorate. - :param context: supplied when called from spawn_missing_classes - """ - context = context or self.context or inspect.currentframe().f_back.f_locals - if issubclass(cls, Part): - raise DataJointError( - "The schema decorator should not be applied to Part tables." - ) - if self.is_activated(): - self._decorate_master(cls, context) - else: - self.declare_list.append((cls, context)) - return cls - - def _decorate_master(self, cls, context): - """ - - :param cls: the master class to process - :param context: the class' declaration context - """ - self._decorate_table( - cls, context=dict(context, self=cls, **{cls.__name__: cls}) - ) - # Process part tables - for part in ordered_dir(cls): - if part[0].isupper(): - part = getattr(cls, part) - if inspect.isclass(part) and issubclass(part, Part): - part._master = cls - # allow addressing master by name or keyword 'master' - self._decorate_table( - part, - context=dict( - context, master=cls, self=part, **{cls.__name__: cls} - ), - ) - - def _decorate_table(self, table_class, context, assert_declared=False): - """ - assign schema properties to the table class and declare the table - """ - table_class.database = self.database - table_class._connection = self.connection - table_class._heading = Heading( - table_info=dict( - conn=self.connection, - database=self.database, - table_name=table_class.table_name, - context=context, - ) - ) - table_class._support = [table_class.full_table_name] - table_class.declaration_context = context - - # instantiate the class, declare the table if not already - instance = table_class() - is_declared = instance.is_declared - if not is_declared and not assert_declared and self.create_tables: - instance.declare(context) - self.connection.dependencies.clear() - is_declared = is_declared or instance.is_declared - - # add table definition to the doc string - if isinstance(table_class.definition, str): - table_class.__doc__ = ( - (table_class.__doc__ or "") - + "\nTable definition:\n\n" - + table_class.definition - ) - - # fill values in Lookup tables from their contents property - if ( - isinstance(instance, Lookup) - and hasattr(instance, "contents") - and is_declared - ): - contents = list(instance.contents) - if len(contents) > len(instance): - if instance.heading.has_autoincrement: - warnings.warn( - ( - "Contents has changed but cannot be inserted because " - "{table} has autoincrement." - ).format(table=instance.__class__.__name__) - ) - else: - instance.insert(contents, skip_duplicates=True) - - @property - def log(self): - self._assert_exists() - if self._log is None: - self._log = Log(self.connection, self.database) - return self._log - - def __repr__(self): - return "Schema `{name}`\n".format(name=self.database) - - @property - def size_on_disk(self): - """ - :return: size of the entire schema in bytes - """ - self._assert_exists() - return int( - self.connection.query( - """ - SELECT SUM(data_length + index_length) - FROM information_schema.tables WHERE table_schema='{db}' - """.format( - db=self.database - ) - ).fetchone()[0] - ) - - def spawn_missing_classes(self, context=None): - """ - Creates the appropriate python user table classes from tables in the schema and places them - in the context. - - :param context: alternative context to place the missing classes into, e.g. locals() - """ - self._assert_exists() - if context is None: - if self.context is not None: - context = self.context - else: - # if context is missing, use the calling namespace - frame = inspect.currentframe().f_back - context = frame.f_locals - del frame - tables = [ - row[0] - for row in self.connection.query("SHOW TABLES in `%s`" % self.database) - if lookup_class_name( - "`{db}`.`{tab}`".format(db=self.database, tab=row[0]), context, 0 - ) - is None - ] - master_classes = (Lookup, Manual, Imported, Computed) - part_tables = [] - for table_name in tables: - class_name = to_camel_case(table_name) - if class_name not in context: - try: - cls = next( - cls - for cls in master_classes - if re.fullmatch(cls.tier_regexp, table_name) - ) - except StopIteration: - if re.fullmatch(Part.tier_regexp, table_name): - part_tables.append(table_name) - else: - # declare and decorate master table classes - context[class_name] = self( - type(class_name, (cls,), dict()), context=context - ) - - # attach parts to masters - for table_name in part_tables: - groups = re.fullmatch(Part.tier_regexp, table_name).groupdict() - class_name = to_camel_case(groups["part"]) - try: - master_class = context[to_camel_case(groups["master"])] - except KeyError: - raise DataJointError( - "The table %s does not follow DataJoint naming conventions" - % table_name - ) - part_class = type(class_name, (Part,), dict(definition=...)) - part_class._master = master_class - self._decorate_table(part_class, context=context, assert_declared=True) - setattr(master_class, class_name, part_class) - - def drop(self, force=False): - """ - Drop the associated schema if it exists - """ - if not self.exists: - logger.info( - "Schema named `{database}` does not exist. Doing nothing.".format( - database=self.database - ) - ) - elif ( - not config["safemode"] - or force - or user_choice( - "Proceed to delete entire schema `%s`?" % self.database, default="no" - ) - == "yes" - ): - logger.debug("Dropping `{database}`.".format(database=self.database)) - try: - self.connection.query( - "DROP DATABASE `{database}`".format(database=self.database) - ) - logger.debug( - "Schema `{database}` was dropped successfully.".format( - database=self.database - ) - ) - except AccessError: - raise AccessError( - "An attempt to drop schema `{database}` " - "has failed. Check permissions.".format(database=self.database) - ) - - @property - def exists(self): - """ - :return: true if the associated schema exists on the server - """ - if self.database is None: - raise DataJointError("Schema must be activated first.") - return bool( - self.connection.query( - "SELECT schema_name " - "FROM information_schema.schemata " - "WHERE schema_name = '{database}'".format(database=self.database) - ).rowcount - ) - - @property - def jobs(self): - """ - schema.jobs provides a view of the job reservation table for the schema - - :return: jobs table - """ - self._assert_exists() - if self._jobs is None: - self._jobs = JobTable(self.connection, self.database) - return self._jobs - - @property - def code(self): - self._assert_exists() - return self.save() - - def save(self, python_filename=None): - """ - Generate the code for a module that recreates the schema. - This method is in preparation for a future release and is not officially supported. - - :return: a string containing the body of a complete Python module defining this schema. - """ - self.connection.dependencies.load() - self._assert_exists() - module_count = itertools.count() - # add virtual modules for referenced modules with names vmod0, vmod1, ... - module_lookup = collections.defaultdict( - lambda: "vmod" + str(next(module_count)) - ) - db = self.database - - def make_class_definition(table): - tier = _get_tier(table).__name__ - class_name = table.split(".")[1].strip("`") - indent = "" - if tier == "Part": - class_name = class_name.split("__")[-1] - indent += " " - class_name = to_camel_case(class_name) - - def replace(s): - d, tabs = s.group(1), s.group(2) - return ("" if d == db else (module_lookup[d] + ".")) + ".".join( - to_camel_case(tab) for tab in tabs.lstrip("__").split("__") - ) - - return ("" if tier == "Part" else "\n@schema\n") + ( - "{indent}class {class_name}(dj.{tier}):\n" - '{indent} definition = """\n' - '{indent} {defi}"""' - ).format( - class_name=class_name, - indent=indent, - tier=tier, - defi=re.sub( - r"`([^`]+)`.`([^`]+)`", - replace, - FreeTable(self.connection, table).describe(), - ).replace("\n", "\n " + indent), - ) - - tables = self.connection.dependencies.topo_sort() - body = "\n\n".join(make_class_definition(table) for table in tables) - python_code = "\n\n".join( - ( - '"""This module was auto-generated by datajoint from an existing schema"""', - "import datajoint as dj\n\nschema = dj.Schema('{db}')".format(db=db), - "\n".join( - "{module} = dj.VirtualModule('{module}', '{schema_name}')".format( - module=v, schema_name=k - ) - for k, v in module_lookup.items() - ), - body, - ) - ) - if python_filename is None: - return python_code - with open(python_filename, "wt") as f: - f.write(python_code) - - def list_tables(self): - """ - Return a list of all tables in the schema except tables with ~ in first character such - as ~logs and ~job - - :return: A list of table names from the database schema. - """ - self.connection.dependencies.load() - return [ - t - for d, t in ( - table_name.replace("`", "").split(".") - for table_name in self.connection.dependencies.topo_sort() - ) - if d == self.database - ] - - -class VirtualModule(types.ModuleType): - """ - A virtual module imitates a Python module representing a DataJoint schema from table definitions in the database. - It declares the schema objects and a class for each table. - """ - - def __init__( - self, - module_name, - schema_name, - *, - create_schema=False, - create_tables=False, - connection=None, - add_objects=None, - ): - """ - Creates a python module with the given name from the name of a schema on the server and - automatically adds classes to it corresponding to the tables in the schema. - - :param module_name: displayed module name - :param schema_name: name of the database in mysql - :param create_schema: if True, create the schema on the database server - :param create_tables: if True, module.schema can be used as the decorator for declaring new - :param connection: a dj.Connection object to pass into the schema - :param add_objects: additional objects to add to the module - :return: the python module containing classes from the schema object and the table classes - """ - super(VirtualModule, self).__init__(name=module_name) - _schema = Schema( - schema_name, - create_schema=create_schema, - create_tables=create_tables, - connection=connection, - ) - if add_objects: - self.__dict__.update(add_objects) - self.__dict__["schema"] = _schema - _schema.spawn_missing_classes(context=self.__dict__) - - -def list_schemas(connection=None): - """ - :param connection: a dj.Connection object - :return: list of all accessible schemas on the server - """ - return [ - r[0] - for r in (connection or conn()).query( - "SELECT schema_name " - "FROM information_schema.schemata " - 'WHERE schema_name <> "information_schema"' - ) - ] diff --git a/datajoint/settings.py b/datajoint/settings.py deleted file mode 100644 index 30b206f99..000000000 --- a/datajoint/settings.py +++ /dev/null @@ -1,303 +0,0 @@ -""" -Settings for DataJoint -""" - -import collections -import json -import logging -import os -import pprint -from contextlib import contextmanager -from enum import Enum - -from .errors import DataJointError - -LOCALCONFIG = "dj_local_conf.json" -GLOBALCONFIG = ".datajoint_config.json" -# subfolding for external storage in filesystem. -# 2, 2 means that file abcdef is stored as /ab/cd/abcdef -DEFAULT_SUBFOLDING = (2, 2) - -validators = collections.defaultdict(lambda: lambda value: True) -validators["database.port"] = lambda a: isinstance(a, int) - -Role = Enum("Role", "manual lookup imported computed job") -role_to_prefix = { - Role.manual: "", - Role.lookup: "#", - Role.imported: "_", - Role.computed: "__", - Role.job: "~", -} -prefix_to_role = dict(zip(role_to_prefix.values(), role_to_prefix)) - -default = dict( - { - "database.host": "localhost", - "database.password": None, - "database.user": None, - "database.port": 3306, - "database.reconnect": True, - "connection.init_function": None, - "connection.charset": "", # pymysql uses '' as default - "loglevel": "INFO", - "safemode": True, - "fetch_format": "array", - "display.limit": 12, - "display.width": 14, - "display.show_tuple_count": True, - "database.use_tls": None, - "enable_python_native_blobs": True, # python-native/dj0 encoding support - "add_hidden_timestamp": False, - # file size limit for when to disable checksums - "filepath_checksum_size_limit": None, - } -) - -logger = logging.getLogger(__name__.split(".")[0]) -log_levels = { - "INFO": logging.INFO, - "WARNING": logging.WARNING, - "CRITICAL": logging.CRITICAL, - "DEBUG": logging.DEBUG, - "ERROR": logging.ERROR, - None: logging.NOTSET, -} - - -class Config(collections.abc.MutableMapping): - instance = None - - def __init__(self, *args, **kwargs): - if not Config.instance: - Config.instance = Config.__Config(*args, **kwargs) - else: - Config.instance._conf.update(dict(*args, **kwargs)) - - def __getattr__(self, name): - return getattr(self.instance, name) - - def __getitem__(self, item): - return self.instance.__getitem__(item) - - def __setitem__(self, item, value): - self.instance.__setitem__(item, value) - - def __str__(self): - return pprint.pformat(self.instance._conf, indent=4) - - def __repr__(self): - return self.__str__() - - def __delitem__(self, key): - del self.instance._conf[key] - - def __iter__(self): - return iter(self.instance._conf) - - def __len__(self): - return len(self.instance._conf) - - def save(self, filename, verbose=False): - """ - Saves the settings in JSON format to the given file path. - - :param filename: filename of the local JSON settings file. - :param verbose: report having saved the settings file - """ - with open(filename, "w") as fid: - json.dump(self._conf, fid, indent=4) - if verbose: - logger.info("Saved settings in " + filename) - - def load(self, filename): - """ - Updates the setting from config file in JSON format. - - :param filename: filename of the local JSON settings file. If None, the local config file is used. - """ - if filename is None: - filename = LOCALCONFIG - with open(filename, "r") as fid: - logger.info(f"DataJoint is configured from {os.path.abspath(filename)}") - self._conf.update(json.load(fid)) - - def save_local(self, verbose=False): - """ - saves the settings in the local config file - """ - self.save(LOCALCONFIG, verbose) - - def save_global(self, verbose=False): - """ - saves the settings in the global config file - """ - self.save(os.path.expanduser(os.path.join("~", GLOBALCONFIG)), verbose) - - def get_store_spec(self, store): - """ - find configuration of external stores for blobs and attachments - """ - try: - spec = self["stores"][store] - except KeyError: - raise DataJointError( - "Storage {store} is requested but not configured".format(store=store) - ) - - spec["subfolding"] = spec.get("subfolding", DEFAULT_SUBFOLDING) - spec_keys = { # REQUIRED in uppercase and allowed in lowercase - "file": ("PROTOCOL", "LOCATION", "subfolding", "stage"), - "s3": ( - "PROTOCOL", - "ENDPOINT", - "BUCKET", - "ACCESS_KEY", - "SECRET_KEY", - "LOCATION", - "secure", - "subfolding", - "stage", - "proxy_server", - ), - } - - try: - spec_keys = spec_keys[spec.get("protocol", "").lower()] - except KeyError: - raise DataJointError( - 'Missing or invalid protocol in dj.config["stores"]["{store}"]'.format( - store=store - ) - ) - - # check that all required keys are present in spec - try: - raise DataJointError( - 'dj.config["stores"]["{store}"] is missing "{k}"'.format( - store=store, - k=next( - k.lower() - for k in spec_keys - if k.isupper() and k.lower() not in spec - ), - ) - ) - except StopIteration: - pass - - # check that only allowed keys are present in spec - try: - raise DataJointError( - 'Invalid key "{k}" in dj.config["stores"]["{store}"]'.format( - store=store, - k=next( - k - for k in spec - if k.upper() not in spec_keys and k.lower() not in spec_keys - ), - ) - ) - except StopIteration: - pass # no invalid keys - - return spec - - @contextmanager - def __call__(self, **kwargs): - """ - The config object can also be used in a with statement to change the state of the configuration - temporarily. kwargs to the context manager are the keys into config, where '.' is replaced by a - double underscore '__'. The context manager yields the changed config object. - - Example: - >>> import datajoint as dj - >>> with dj.config(safemode=False, database__host="localhost") as cfg: - >>> # do dangerous stuff here - """ - - try: - backup = self.instance - self.instance = Config.__Config(self.instance._conf) - new = {k.replace("__", "."): v for k, v in kwargs.items()} - self.instance._conf.update(new) - yield self - except: - self.instance = backup - raise - else: - self.instance = backup - - class __Config: - """ - Stores datajoint settings. Behaves like a dictionary, but applies validator functions - when certain keys are set. - - The default parameters are stored in datajoint.settings.default . If a local config file - exists, the settings specified in this file override the default settings. - """ - - def __init__(self, *args, **kwargs): - self._conf = dict(default) - # use the free update to set keys - self._conf.update(dict(*args, **kwargs)) - - def __getitem__(self, key): - return self._conf[key] - - def __setitem__(self, key, value): - logger.debug("Setting {0:s} to {1:s}".format(str(key), str(value))) - if validators[key](value): - self._conf[key] = value - else: - raise DataJointError("Validator for {0:s} did not pass".format(key)) - valid_logging_levels = {"DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"} - if key == "loglevel": - if value not in valid_logging_levels: - raise ValueError( - f"'{value}' is not a valid logging value {tuple(valid_logging_levels)}" - ) - logger.setLevel(value) - - -# Load configuration from file -config = Config() -config_files = ( - os.path.expanduser(n) for n in (LOCALCONFIG, os.path.join("~", GLOBALCONFIG)) -) -try: - config.load(next(n for n in config_files if os.path.exists(n))) -except StopIteration: - logger.info("No config file was found.") - -# override login credentials with environment variables -mapping = { - k: v - for k, v in zip( - ( - "database.host", - "database.user", - "database.password", - "external.aws_access_key_id", - "external.aws_secret_access_key", - "loglevel", - ), - map( - os.getenv, - ( - "DJ_HOST", - "DJ_USER", - "DJ_PASS", - "DJ_AWS_ACCESS_KEY_ID", - "DJ_AWS_SECRET_ACCESS_KEY", - "DJ_LOG_LEVEL", - ), - ), - ) - if v is not None -} -if mapping: - logger.info(f"Overloaded settings {tuple(mapping)} from environment variables.") - config.update(mapping) - -logger.setLevel(log_levels[config["loglevel"]]) diff --git a/datajoint/table.py b/datajoint/table.py deleted file mode 100644 index 7e3e0c3a1..000000000 --- a/datajoint/table.py +++ /dev/null @@ -1,1118 +0,0 @@ -import collections -import csv -import inspect -import itertools -import json -import logging -import platform -import re -import uuid -from pathlib import Path -from typing import Union - -import numpy as np -import pandas - -from . import blob -from .condition import make_condition -from .declare import alter, declare -from .errors import ( - AccessError, - DataJointError, - DuplicateError, - IntegrityError, - UnknownAttributeError, -) -from .expression import QueryExpression -from .heading import Heading -from .settings import config -from .utils import get_master, is_camel_case, user_choice -from .version import __version__ as version - -logger = logging.getLogger(__name__.split(".")[0]) - -foreign_key_error_regexp = re.compile( - r"[\w\s:]*\((?P`[^`]+`.`[^`]+`), " - r"CONSTRAINT (?P`[^`]+`) " - r"(FOREIGN KEY \((?P[^)]+)\) " - r"REFERENCES (?P`[^`]+`(\.`[^`]+`)?) \((?P[^)]+)\)[\s\w]+\))?" -) - -constraint_info_query = " ".join( - """ - SELECT - COLUMN_NAME as fk_attrs, - CONCAT('`', REFERENCED_TABLE_SCHEMA, '`.`', REFERENCED_TABLE_NAME, '`') as parent, - REFERENCED_COLUMN_NAME as pk_attrs - FROM INFORMATION_SCHEMA.KEY_COLUMN_USAGE - WHERE - CONSTRAINT_NAME = %s AND TABLE_SCHEMA = %s AND TABLE_NAME = %s; - """.split() -) - - -class _RenameMap(tuple): - """for internal use""" - - pass - - -class Table(QueryExpression): - """ - Table is an abstract class that represents a table in the schema. - It implements insert and delete methods and inherits query functionality. - To make it a concrete class, override the abstract properties specifying the connection, - table name, database, and definition. - """ - - _table_name = None # must be defined in subclass - _log_ = None # placeholder for the Log table object - - # These properties must be set by the schema decorator (schemas.py) at class level - # or by FreeTable at instance level - database = None - declaration_context = None - - @property - def table_name(self): - return self._table_name - - @property - def class_name(self): - return self.__class__.__name__ - - @property - def definition(self): - raise NotImplementedError( - "Subclasses of Table must implement the `definition` property" - ) - - def declare(self, context=None): - """ - Declare the table in the schema based on self.definition. - - :param context: the context for foreign key resolution. If None, foreign keys are - not allowed. - """ - if self.connection.in_transaction: - raise DataJointError( - "Cannot declare new tables inside a transaction, " - "e.g. from inside a populate/make call" - ) - # Enforce strict CamelCase #1150 - if not is_camel_case(self.class_name): - raise DataJointError( - "Table class name `{name}` is invalid. Please use CamelCase. ".format( - name=self.class_name - ) - + "Classes defining tables should be formatted in strict CamelCase." - ) - sql, external_stores = declare(self.full_table_name, self.definition, context) - sql = sql.format(database=self.database) - try: - # declare all external tables before declaring main table - for store in external_stores: - self.connection.schemas[self.database].external[store] - self.connection.query(sql) - except AccessError: - # skip if no create privilege - pass - else: - self._log("Declared " + self.full_table_name) - - def alter(self, prompt=True, context=None): - """ - Alter the table definition from self.definition - """ - if self.connection.in_transaction: - raise DataJointError( - "Cannot update table declaration inside a transaction, " - "e.g. from inside a populate/make call" - ) - if context is None: - frame = inspect.currentframe().f_back - context = dict(frame.f_globals, **frame.f_locals) - del frame - old_definition = self.describe(context=context) - sql, external_stores = alter(self.definition, old_definition, context) - if not sql: - if prompt: - logger.warning("Nothing to alter.") - else: - sql = "ALTER TABLE {tab}\n\t".format( - tab=self.full_table_name - ) + ",\n\t".join(sql) - if not prompt or user_choice(sql + "\n\nExecute?") == "yes": - try: - # declare all external tables before declaring main table - for store in external_stores: - self.connection.schemas[self.database].external[store] - self.connection.query(sql) - except AccessError: - # skip if no create privilege - pass - else: - # reset heading - self.__class__._heading = Heading( - table_info=self.heading.table_info - ) - if prompt: - logger.info("Table altered") - self._log("Altered " + self.full_table_name) - - def from_clause(self): - """ - :return: the FROM clause of SQL SELECT statements. - """ - return self.full_table_name - - def get_select_fields(self, select_fields=None): - """ - :return: the selected attributes from the SQL SELECT statement. - """ - return ( - "*" if select_fields is None else self.heading.project(select_fields).as_sql - ) - - def parents(self, primary=None, as_objects=False, foreign_key_info=False): - """ - - :param primary: if None, then all parents are returned. If True, then only foreign keys composed of - primary key attributes are considered. If False, return foreign keys including at least one - secondary attribute. - :param as_objects: if False, return table names. If True, return table objects. - :param foreign_key_info: if True, each element in result also includes foreign key info. - :return: list of parents as table names or table objects - with (optional) foreign key information. - """ - get_edge = self.connection.dependencies.parents - nodes = [ - next(iter(get_edge(name).items())) if name.isdigit() else (name, props) - for name, props in get_edge(self.full_table_name, primary).items() - ] - if as_objects: - nodes = [(FreeTable(self.connection, name), props) for name, props in nodes] - if not foreign_key_info: - nodes = [name for name, props in nodes] - return nodes - - def children(self, primary=None, as_objects=False, foreign_key_info=False): - """ - :param primary: if None, then all children are returned. If True, then only foreign keys composed of - primary key attributes are considered. If False, return foreign keys including at least one - secondary attribute. - :param as_objects: if False, return table names. If True, return table objects. - :param foreign_key_info: if True, each element in result also includes foreign key info. - :return: list of children as table names or table objects - with (optional) foreign key information. - """ - get_edge = self.connection.dependencies.children - nodes = [ - next(iter(get_edge(name).items())) if name.isdigit() else (name, props) - for name, props in get_edge(self.full_table_name, primary).items() - ] - if as_objects: - nodes = [(FreeTable(self.connection, name), props) for name, props in nodes] - if not foreign_key_info: - nodes = [name for name, props in nodes] - return nodes - - def descendants(self, as_objects=False): - """ - :param as_objects: False - a list of table names; True - a list of table objects. - :return: list of tables descendants in topological order. - """ - return [ - FreeTable(self.connection, node) if as_objects else node - for node in self.connection.dependencies.descendants(self.full_table_name) - if not node.isdigit() - ] - - def ancestors(self, as_objects=False): - """ - :param as_objects: False - a list of table names; True - a list of table objects. - :return: list of tables ancestors in topological order. - """ - return [ - FreeTable(self.connection, node) if as_objects else node - for node in self.connection.dependencies.ancestors(self.full_table_name) - if not node.isdigit() - ] - - def parts(self, as_objects=False): - """ - return part tables either as entries in a dict with foreign key information or a list of objects - - :param as_objects: if False (default), the output is a dict describing the foreign keys. If True, return table objects. - """ - self.connection.dependencies.load(force=False) - nodes = [ - node - for node in self.connection.dependencies.nodes - if not node.isdigit() and node.startswith(self.full_table_name[:-1] + "__") - ] - return [FreeTable(self.connection, c) for c in nodes] if as_objects else nodes - - @property - def is_declared(self): - """ - :return: True is the table is declared in the schema. - """ - return ( - self.connection.query( - 'SHOW TABLES in `{database}` LIKE "{table_name}"'.format( - database=self.database, table_name=self.table_name - ) - ).rowcount - > 0 - ) - - @property - def full_table_name(self): - """ - :return: full table name in the schema - """ - return r"`{0:s}`.`{1:s}`".format(self.database, self.table_name) - - @property - def _log(self): - if self._log_ is None: - self._log_ = Log( - self.connection, - database=self.database, - skip_logging=self.table_name.startswith("~"), - ) - return self._log_ - - @property - def external(self): - return self.connection.schemas[self.database].external - - def update1(self, row): - """ - ``update1`` updates one existing entry in the table. - Caution: In DataJoint the primary modes for data manipulation is to ``insert`` and - ``delete`` entire records since referential integrity works on the level of records, - not fields. Therefore, updates are reserved for corrective operations outside of main - workflow. Use UPDATE methods sparingly with full awareness of potential violations of - assumptions. - - :param row: a ``dict`` containing the primary key values and the attributes to update. - Setting an attribute value to None will reset it to the default value (if any). - - The primary key attributes must always be provided. - - Examples: - - >>> table.update1({'id': 1, 'value': 3}) # update value in record with id=1 - >>> table.update1({'id': 1, 'value': None}) # reset value to default - """ - # argument validations - if not isinstance(row, collections.abc.Mapping): - raise DataJointError("The argument of update1 must be dict-like.") - if not set(row).issuperset(self.primary_key): - raise DataJointError( - "The argument of update1 must supply all primary key values." - ) - try: - raise DataJointError( - "Attribute `%s` not found." - % next(k for k in row if k not in self.heading.names) - ) - except StopIteration: - pass # ok - if len(self.restriction): - raise DataJointError("Update cannot be applied to a restricted table.") - key = {k: row[k] for k in self.primary_key} - if len(self & key) != 1: - raise DataJointError("Update can only be applied to one existing entry.") - # UPDATE query - row = [ - self.__make_placeholder(k, v) - for k, v in row.items() - if k not in self.primary_key - ] - query = "UPDATE {table} SET {assignments} WHERE {where}".format( - table=self.full_table_name, - assignments=",".join("`%s`=%s" % r[:2] for r in row), - where=make_condition(self, key, set()), - ) - self.connection.query(query, args=list(r[2] for r in row if r[2] is not None)) - - def insert1(self, row, **kwargs): - """ - Insert one data record into the table. For ``kwargs``, see ``insert()``. - - :param row: a numpy record, a dict-like object, or an ordered sequence to be inserted - as one row. - """ - self.insert((row,), **kwargs) - - def insert( - self, - rows, - replace=False, - skip_duplicates=False, - ignore_extra_fields=False, - allow_direct_insert=None, - ): - """ - Insert a collection of rows. - - :param rows: Either (a) an iterable where an element is a numpy record, a - dict-like object, a pandas.DataFrame, a sequence, or a query expression with - the same heading as self, or (b) a pathlib.Path object specifying a path - relative to the current directory with a CSV file, the contents of which - will be inserted. - :param replace: If True, replaces the existing tuple. - :param skip_duplicates: If True, silently skip duplicate inserts. - :param ignore_extra_fields: If False, fields that are not in the heading raise error. - :param allow_direct_insert: Only applies in auto-populated tables. If False (default), - insert may only be called from inside the make callback. - - Example: - - >>> Table.insert([ - >>> dict(subject_id=7, species="mouse", date_of_birth="2014-09-01"), - >>> dict(subject_id=8, species="mouse", date_of_birth="2014-09-02")]) - """ - if isinstance(rows, pandas.DataFrame): - # drop 'extra' synthetic index for 1-field index case - - # frames with more advanced indices should be prepared by user. - rows = rows.reset_index( - drop=len(rows.index.names) == 1 and not rows.index.names[0] - ).to_records(index=False) - - if isinstance(rows, Path): - with open(rows, newline="") as data_file: - rows = list(csv.DictReader(data_file, delimiter=",")) - - # prohibit direct inserts into auto-populated tables - if not allow_direct_insert and not getattr(self, "_allow_insert", True): - raise DataJointError( - "Inserts into an auto-populated table can only be done inside " - "its make method during a populate call." - " To override, set keyword argument allow_direct_insert=True." - ) - - if inspect.isclass(rows) and issubclass(rows, QueryExpression): - rows = rows() # instantiate if a class - if isinstance(rows, QueryExpression): - # insert from select - if not ignore_extra_fields: - try: - raise DataJointError( - "Attribute %s not found. To ignore extra attributes in insert, " - "set ignore_extra_fields=True." - % next( - name for name in rows.heading if name not in self.heading - ) - ) - except StopIteration: - pass - fields = list(name for name in rows.heading if name in self.heading) - query = "{command} INTO {table} ({fields}) {select}{duplicate}".format( - command="REPLACE" if replace else "INSERT", - fields="`" + "`,`".join(fields) + "`", - table=self.full_table_name, - select=rows.make_sql(fields), - duplicate=( - " ON DUPLICATE KEY UPDATE `{pk}`={table}.`{pk}`".format( - table=self.full_table_name, pk=self.primary_key[0] - ) - if skip_duplicates - else "" - ), - ) - self.connection.query(query) - return - - # collects the field list from first row (passed by reference) - field_list = [] - rows = list( - self.__make_row_to_insert(row, field_list, ignore_extra_fields) - for row in rows - ) - if rows: - try: - query = "{command} INTO {destination}(`{fields}`) VALUES {placeholders}{duplicate}".format( - command="REPLACE" if replace else "INSERT", - destination=self.from_clause(), - fields="`,`".join(field_list), - placeholders=",".join( - "(" + ",".join(row["placeholders"]) + ")" for row in rows - ), - duplicate=( - " ON DUPLICATE KEY UPDATE `{pk}`=`{pk}`".format( - pk=self.primary_key[0] - ) - if skip_duplicates - else "" - ), - ) - self.connection.query( - query, - args=list( - itertools.chain.from_iterable( - (v for v in r["values"] if v is not None) for r in rows - ) - ), - ) - except UnknownAttributeError as err: - raise err.suggest( - "To ignore extra fields in insert, set ignore_extra_fields=True" - ) - except DuplicateError as err: - raise err.suggest( - "To ignore duplicate entries in insert, set skip_duplicates=True" - ) - - def delete_quick(self, get_count=False): - """ - Deletes the table without cascading and without user prompt. - If this table has populated dependent tables, this will fail. - """ - query = "DELETE FROM " + self.full_table_name + self.where_clause() - self.connection.query(query) - count = ( - self.connection.query("SELECT ROW_COUNT()").fetchone()[0] - if get_count - else None - ) - self._log(query[:255]) - return count - - def delete( - self, - transaction: bool = True, - safemode: Union[bool, None] = None, - force_parts: bool = False, - force_masters: bool = False, - ) -> int: - """ - Deletes the contents of the table and its dependent tables, recursively. - - Args: - transaction: If `True`, use of the entire delete becomes an atomic transaction. - This is the default and recommended behavior. Set to `False` if this delete is - nested within another transaction. - safemode: If `True`, prohibit nested transactions and prompt to confirm. Default - is `dj.config['safemode']`. - force_parts: Delete from parts even when not deleting from their masters. - force_masters: If `True`, include part/master pairs in the cascade. - Default is `False`. - - Returns: - Number of deleted rows (excluding those from dependent tables). - - Raises: - DataJointError: Delete exceeds maximum number of delete attempts. - DataJointError: When deleting within an existing transaction. - DataJointError: Deleting a part table before its master. - """ - deleted = set() - visited_masters = set() - - def cascade(table): - """service function to perform cascading deletes recursively.""" - max_attempts = 50 - for _ in range(max_attempts): - try: - delete_count = table.delete_quick(get_count=True) - except IntegrityError as error: - match = foreign_key_error_regexp.match(error.args[0]) - if match is None: - raise DataJointError( - "Cascading deletes failed because the error message is missing foreign key information." - "Make sure you have REFERENCES privilege to all dependent tables." - ) from None - match = match.groupdict() - # if schema name missing, use table - if "`.`" not in match["child"]: - match["child"] = "{}.{}".format( - table.full_table_name.split(".")[0], match["child"] - ) - if ( - match["pk_attrs"] is not None - ): # fully matched, adjusting the keys - match["fk_attrs"] = [ - k.strip("`") for k in match["fk_attrs"].split(",") - ] - match["pk_attrs"] = [ - k.strip("`") for k in match["pk_attrs"].split(",") - ] - else: # only partially matched, querying with constraint to determine keys - match["fk_attrs"], match["parent"], match["pk_attrs"] = list( - map( - list, - zip( - *table.connection.query( - constraint_info_query, - args=( - match["name"].strip("`"), - *[ - _.strip("`") - for _ in match["child"].split("`.`") - ], - ), - ).fetchall() - ), - ) - ) - match["parent"] = match["parent"][0] - - # Restrict child by table if - # 1. if table's restriction attributes are not in child's primary key - # 2. if child renames any attributes - # Otherwise restrict child by table's restriction. - child = FreeTable(table.connection, match["child"]) - if ( - set(table.restriction_attributes) <= set(child.primary_key) - and match["fk_attrs"] == match["pk_attrs"] - ): - child._restriction = table._restriction - child._restriction_attributes = table.restriction_attributes - elif match["fk_attrs"] != match["pk_attrs"]: - child &= table.proj( - **dict(zip(match["fk_attrs"], match["pk_attrs"])) - ) - else: - child &= table.proj() - - master_name = get_master(child.full_table_name) - if ( - force_masters - and master_name - and master_name != table.full_table_name - and master_name not in visited_masters - ): - master = FreeTable(table.connection, master_name) - master._restriction_attributes = set() - master._restriction = [ - make_condition( # &= may cause in target tables in subquery - master, - (master.proj() & child.proj()).fetch(), - master._restriction_attributes, - ) - ] - visited_masters.add(master_name) - cascade(master) - else: - cascade(child) - else: - deleted.add(table.full_table_name) - logger.info( - "Deleting {count} rows from {table}".format( - count=delete_count, table=table.full_table_name - ) - ) - break - else: - raise DataJointError("Exceeded maximum number of delete attempts.") - return delete_count - - safemode = config["safemode"] if safemode is None else safemode - - # Start transaction - if transaction: - if not self.connection.in_transaction: - self.connection.start_transaction() - else: - if not safemode: - transaction = False - else: - raise DataJointError( - "Delete cannot use a transaction within an ongoing transaction. " - "Set transaction=False or safemode=False)." - ) - - # Cascading delete - try: - delete_count = cascade(self) - except: - if transaction: - self.connection.cancel_transaction() - raise - - if not force_parts: - # Avoid deleting from child before master (See issue #151) - for part in deleted: - master = get_master(part) - if master and master not in deleted: - if transaction: - self.connection.cancel_transaction() - raise DataJointError( - "Attempt to delete part table {part} before deleting from " - "its master {master} first.".format(part=part, master=master) - ) - - # Confirm and commit - if delete_count == 0: - if safemode: - logger.warning("Nothing to delete.") - if transaction: - self.connection.cancel_transaction() - elif not transaction: - logger.info("Delete completed") - else: - if not safemode or user_choice("Commit deletes?", default="no") == "yes": - if transaction: - self.connection.commit_transaction() - if safemode: - logger.info("Delete committed.") - else: - if transaction: - self.connection.cancel_transaction() - if safemode: - logger.warning("Delete cancelled") - return delete_count - - def drop_quick(self): - """ - Drops the table without cascading to dependent tables and without user prompt. - """ - if self.is_declared: - query = "DROP TABLE %s" % self.full_table_name - self.connection.query(query) - logger.info("Dropped table %s" % self.full_table_name) - self._log(query[:255]) - else: - logger.info( - "Nothing to drop: table %s is not declared" % self.full_table_name - ) - - def drop(self): - """ - Drop the table and all tables that reference it, recursively. - User is prompted for confirmation if config['safemode'] is set to True. - """ - if self.restriction: - raise DataJointError( - "A table with an applied restriction cannot be dropped." - " Call drop() on the unrestricted Table." - ) - self.connection.dependencies.load() - do_drop = True - tables = [ - table - for table in self.connection.dependencies.descendants(self.full_table_name) - if not table.isdigit() - ] - - # avoid dropping part tables without their masters: See issue #374 - for part in tables: - master = get_master(part) - if master and master not in tables: - raise DataJointError( - "Attempt to drop part table {part} before dropping " - "its master. Drop {master} first.".format(part=part, master=master) - ) - - if config["safemode"]: - for table in tables: - logger.info( - table + " (%d tuples)" % len(FreeTable(self.connection, table)) - ) - do_drop = user_choice("Proceed?", default="no") == "yes" - if do_drop: - for table in reversed(tables): - FreeTable(self.connection, table).drop_quick() - logger.info("Tables dropped. Restart kernel.") - - @property - def size_on_disk(self): - """ - :return: size of data and indices in bytes on the storage device - """ - ret = self.connection.query( - 'SHOW TABLE STATUS FROM `{database}` WHERE NAME="{table}"'.format( - database=self.database, table=self.table_name - ), - as_dict=True, - ).fetchone() - return ret["Data_length"] + ret["Index_length"] - - def describe(self, context=None, printout=False): - """ - :return: the definition string for the query using DataJoint DDL. - """ - if context is None: - frame = inspect.currentframe().f_back - context = dict(frame.f_globals, **frame.f_locals) - del frame - if self.full_table_name not in self.connection.dependencies: - self.connection.dependencies.load() - parents = self.parents(foreign_key_info=True) - in_key = True - definition = ( - "# " + self.heading.table_status["comment"] + "\n" - if self.heading.table_status["comment"] - else "" - ) - attributes_thus_far = set() - attributes_declared = set() - indexes = self.heading.indexes.copy() - for attr in self.heading.attributes.values(): - if in_key and not attr.in_key: - definition += "---\n" - in_key = False - attributes_thus_far.add(attr.name) - do_include = True - for parent_name, fk_props in parents: - if attr.name in fk_props["attr_map"]: - do_include = False - if attributes_thus_far.issuperset(fk_props["attr_map"]): - # foreign key properties - try: - index_props = indexes.pop(tuple(fk_props["attr_map"])) - except KeyError: - index_props = "" - else: - index_props = [k for k, v in index_props.items() if v] - index_props = ( - " [{}]".format(", ".join(index_props)) - if index_props - else "" - ) - - if not fk_props["aliased"]: - # simple foreign key - definition += "->{props} {class_name}\n".format( - props=index_props, - class_name=lookup_class_name(parent_name, context) - or parent_name, - ) - else: - # projected foreign key - definition += ( - "->{props} {class_name}.proj({proj_list})\n".format( - props=index_props, - class_name=lookup_class_name(parent_name, context) - or parent_name, - proj_list=",".join( - '{}="{}"'.format(attr, ref) - for attr, ref in fk_props["attr_map"].items() - if ref != attr - ), - ) - ) - attributes_declared.update(fk_props["attr_map"]) - if do_include: - attributes_declared.add(attr.name) - definition += "%-20s : %-28s %s\n" % ( - ( - attr.name - if attr.default is None - else "%s=%s" % (attr.name, attr.default) - ), - "%s%s" - % (attr.type, " auto_increment" if attr.autoincrement else ""), - "# " + attr.comment if attr.comment else "", - ) - # add remaining indexes - for k, v in indexes.items(): - definition += "{unique}INDEX ({attrs})\n".format( - unique="UNIQUE " if v["unique"] else "", attrs=", ".join(k) - ) - if printout: - logger.info("\n" + definition) - return definition - - # --- private helper functions ---- - def __make_placeholder(self, name, value, ignore_extra_fields=False): - """ - For a given attribute `name` with `value`, return its processed value or value placeholder - as a string to be included in the query and the value, if any, to be submitted for - processing by mysql API. - - :param name: name of attribute to be inserted - :param value: value of attribute to be inserted - """ - if ignore_extra_fields and name not in self.heading: - return None - attr = self.heading[name] - if attr.adapter: - value = attr.adapter.put(value) - if value is None or (attr.numeric and (value == "" or np.isnan(float(value)))): - # set default value - placeholder, value = "DEFAULT", None - else: # not NULL - placeholder = "%s" - if attr.uuid: - if not isinstance(value, uuid.UUID): - try: - value = uuid.UUID(value) - except (AttributeError, ValueError): - raise DataJointError( - "badly formed UUID value {v} for attribute `{n}`".format( - v=value, n=name - ) - ) - value = value.bytes - elif attr.is_blob: - value = blob.pack(value) - value = ( - self.external[attr.store].put(value).bytes - if attr.is_external - else value - ) - elif attr.is_attachment: - attachment_path = Path(value) - if attr.is_external: - # value is hash of contents - value = ( - self.external[attr.store] - .upload_attachment(attachment_path) - .bytes - ) - else: - # value is filename + contents - value = ( - str.encode(attachment_path.name) - + b"\0" - + attachment_path.read_bytes() - ) - elif attr.is_filepath: - value = self.external[attr.store].upload_filepath(value).bytes - elif attr.numeric: - value = str(int(value) if isinstance(value, bool) else value) - elif attr.json: - value = json.dumps(value) - return name, placeholder, value - - def __make_row_to_insert(self, row, field_list, ignore_extra_fields): - """ - Helper function for insert and update - - :param row: A tuple to insert - :return: a dict with fields 'names', 'placeholders', 'values' - """ - - def check_fields(fields): - """ - Validates that all items in `fields` are valid attributes in the heading - - :param fields: field names of a tuple - """ - if not field_list: - if not ignore_extra_fields: - for field in fields: - if field not in self.heading: - raise KeyError( - "`{0:s}` is not in the table heading".format(field) - ) - elif set(field_list) != set(fields).intersection(self.heading.names): - raise DataJointError("Attempt to insert rows with different fields.") - - if isinstance(row, np.void): # np.array - check_fields(row.dtype.fields) - attributes = [ - self.__make_placeholder(name, row[name], ignore_extra_fields) - for name in self.heading - if name in row.dtype.fields - ] - elif isinstance(row, collections.abc.Mapping): # dict-based - check_fields(row) - attributes = [ - self.__make_placeholder(name, row[name], ignore_extra_fields) - for name in self.heading - if name in row - ] - else: # positional - try: - if len(row) != len(self.heading): - raise DataJointError( - "Invalid insert argument. Incorrect number of attributes: " - "{given} given; {expected} expected".format( - given=len(row), expected=len(self.heading) - ) - ) - except TypeError: - raise DataJointError("Datatype %s cannot be inserted" % type(row)) - else: - attributes = [ - self.__make_placeholder(name, value, ignore_extra_fields) - for name, value in zip(self.heading, row) - ] - if ignore_extra_fields: - attributes = [a for a in attributes if a is not None] - - assert len(attributes), "Empty tuple" - row_to_insert = dict(zip(("names", "placeholders", "values"), zip(*attributes))) - if not field_list: - # first row sets the composition of the field list - field_list.extend(row_to_insert["names"]) - else: - # reorder attributes in row_to_insert to match field_list - order = list(row_to_insert["names"].index(field) for field in field_list) - row_to_insert["names"] = list(row_to_insert["names"][i] for i in order) - row_to_insert["placeholders"] = list( - row_to_insert["placeholders"][i] for i in order - ) - row_to_insert["values"] = list(row_to_insert["values"][i] for i in order) - return row_to_insert - - -def lookup_class_name(name, context, depth=3): - """ - given a table name in the form `schema_name`.`table_name`, find its class in the context. - - :param name: `schema_name`.`table_name` - :param context: dictionary representing the namespace - :param depth: search depth into imported modules, helps avoid infinite recursion. - :return: class name found in the context or None if not found - """ - # breadth-first search - nodes = [dict(context=context, context_name="", depth=depth)] - while nodes: - node = nodes.pop(0) - for member_name, member in node["context"].items(): - # skip IPython's implicit variables - if not member_name.startswith("_"): - if inspect.isclass(member) and issubclass(member, Table): - if member.full_table_name == name: # found it! - return ".".join([node["context_name"], member_name]).lstrip(".") - try: # look for part tables - parts = member.__dict__ - except AttributeError: - pass # not a UserTable -- cannot have part tables. - else: - for part in ( - getattr(member, p) - for p in parts - if p[0].isupper() and hasattr(member, p) - ): - if ( - inspect.isclass(part) - and issubclass(part, Table) - and part.full_table_name == name - ): - return ".".join( - [node["context_name"], member_name, part.__name__] - ).lstrip(".") - elif ( - node["depth"] > 0 - and inspect.ismodule(member) - and member.__name__ != "datajoint" - ): - try: - nodes.append( - dict( - context=dict(inspect.getmembers(member)), - context_name=node["context_name"] + "." + member_name, - depth=node["depth"] - 1, - ) - ) - except ImportError: - pass # could not import, so do not attempt - return None - - -class FreeTable(Table): - """ - A base table without a dedicated class. Each instance is associated with a table - specified by full_table_name. - - :param conn: a dj.Connection object - :param full_table_name: in format `database`.`table_name` - """ - - def __init__(self, conn, full_table_name): - self.database, self._table_name = ( - s.strip("`") for s in full_table_name.split(".") - ) - self._connection = conn - self._support = [full_table_name] - self._heading = Heading( - table_info=dict( - conn=conn, - database=self.database, - table_name=self.table_name, - context=None, - ) - ) - - def __repr__(self): - return ( - "FreeTable(`%s`.`%s`)\n" % (self.database, self._table_name) - + super().__repr__() - ) - - -class Log(Table): - """ - The log table for each schema. - Instances are callable. Calls log the time and identifying information along with the event. - - :param skip_logging: if True, then log entry is skipped by default. See __call__ - """ - - _table_name = "~log" - - def __init__(self, conn, database, skip_logging=False): - self.database = database - self.skip_logging = skip_logging - self._connection = conn - self._heading = Heading( - table_info=dict( - conn=conn, database=database, table_name=self.table_name, context=None - ) - ) - self._support = [self.full_table_name] - - self._definition = """ # event logging table for `{database}` - id :int unsigned auto_increment # event order id - --- - timestamp = CURRENT_TIMESTAMP : timestamp # event timestamp - version :varchar(12) # datajoint version - user :varchar(255) # user@host - host="" :varchar(255) # system hostname - event="" :varchar(255) # event message - """.format( - database=database - ) - - super().__init__() - - if not self.is_declared: - self.declare() - self.connection.dependencies.clear() - self._user = self.connection.get_user() - - @property - def definition(self): - return self._definition - - def __call__(self, event, skip_logging=None): - """ - - :param event: string to write into the log table - :param skip_logging: If True then do not log. If None, then use self.skip_logging - """ - skip_logging = self.skip_logging if skip_logging is None else skip_logging - if not skip_logging: - try: - self.insert1( - dict( - user=self._user, - version=version + "py", - host=platform.uname().node, - event=event, - ), - skip_duplicates=True, - ignore_extra_fields=True, - ) - except DataJointError: - logger.info("could not log event in table ~log") - - def delete(self): - """ - bypass interactive prompts and cascading dependencies - - :return: number of deleted items - """ - return self.delete_quick(get_count=True) - - def drop(self): - """bypass interactive prompts and cascading dependencies""" - self.drop_quick() diff --git a/docker-compose.yaml b/docker-compose.yaml index 40b211756..2c48ffd10 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,14 +1,24 @@ -# HOST_UID=$(id -u) PY_VER=3.11 DJ_VERSION=$(grep -oP '\d+\.\d+\.\d+' datajoint/version.py) docker compose --profile test up --build --exit-code-from djtest djtest +# Development environment with MySQL and MinIO services +# +# NOTE: docker-compose is OPTIONAL for running tests. +# Tests use testcontainers to automatically manage containers. +# Just run: pytest tests/ +# +# Use docker-compose for development/debugging when you want +# persistent containers that survive test runs: +# docker compose up -d db minio # Start services manually +# pytest tests/ # Tests will use these containers +# +# Full Docker testing (CI): +# docker compose --profile test up djtest --build services: db: image: datajoint/mysql:${MYSQL_VER:-8.0} environment: - MYSQL_ROOT_PASSWORD=${DJ_PASS:-password} command: mysqld --default-authentication-plugin=mysql_native_password - # ports: - # - "3306:3306" - # volumes: - # - ./mysql/data:/var/lib/mysql + ports: + - "3306:3306" healthcheck: test: [ "CMD", "mysqladmin", "ping", "-h", "localhost" ] timeout: 30s @@ -19,18 +29,15 @@ services: environment: - MINIO_ACCESS_KEY=datajoint - MINIO_SECRET_KEY=datajoint - # ports: - # - "9000:9000" - # volumes: - # - ./minio/config:/root/.minio - # - ./minio/data:/data + ports: + - "9000:9000" command: server --address ":9000" /data healthcheck: test: - "CMD" - "curl" - "--fail" - - "http://minio:9000/minio/health/live" + - "http://localhost:9000/minio/health/live" timeout: 30s retries: 5 interval: 15s @@ -40,7 +47,7 @@ services: context: . dockerfile: Dockerfile args: - PY_VER: ${PY_VER:-3.9} + PY_VER: ${PY_VER:-3.10} HOST_UID: ${HOST_UID:-1000} depends_on: db: diff --git a/docs/mkdocs.yaml b/docs/mkdocs.yaml index 4de4f58e1..db2ea16f9 100644 --- a/docs/mkdocs.yaml +++ b/docs/mkdocs.yaml @@ -1,82 +1,16 @@ # ---------------------- PROJECT SPECIFIC --------------------------- -site_name: DataJoint Documentation +site_name: DataJoint Python - Developer Documentation +site_description: Developer documentation for DataJoint Python contributors repo_url: https://github.com/datajoint/datajoint-python repo_name: datajoint/datajoint-python nav: - - DataJoint Python: index.md - - Quick Start Guide: quick-start.md - - Concepts: - - Principles: concepts/principles.md - - Data Model: concepts/data-model.md - - Data Pipelines: concepts/data-pipelines.md - - Teamwork: concepts/teamwork.md - - Terminology: concepts/terminology.md - - System Administration: - - Database Administration: sysadmin/database-admin.md - - Bulk Storage Systems: sysadmin/bulk-storage.md - - External Store: sysadmin/external-store.md - - Client Configuration: - - Install: client/install.md - - Credentials: client/credentials.md - - Settings: client/settings.md - - File Stores: client/stores.md - - Schema Design: - - Schema Creation: design/schema.md - - Table Definition: - - Table Tiers: design/tables/tiers.md - - Declaration Syntax: design/tables/declare.md - - Primary Key: design/tables/primary.md - - Attributes: design/tables/attributes.md - - Lookup Tables: design/tables/lookup.md - - Manual Tables: design/tables/manual.md - - Blobs: design/tables/blobs.md - - Attachments: design/tables/attach.md - - Filepaths: design/tables/filepath.md - - Custom Datatypes: design/tables/customtype.md - - Dependencies: design/tables/dependencies.md - - Indexes: design/tables/indexes.md - - Master-Part Relationships: design/tables/master-part.md - - Schema Diagrams: design/diagrams.md - - Entity Normalization: design/normalization.md - - Data Integrity: design/integrity.md - - Schema Recall: design/recall.md - - Schema Drop: design/drop.md - - Schema Modification: design/alter.md - - Data Manipulations: - - manipulation/index.md - - Insert: manipulation/insert.md - - Delete: manipulation/delete.md - - Update: manipulation/update.md - - Transactions: manipulation/transactions.md - - Data Queries: - - Principles: query/principles.md - - Example Schema: query/example-schema.md - - Fetch: query/fetch.md - - Iteration: query/iteration.md - - Operators: query/operators.md - - Restrict: query/restrict.md - - Projection: query/project.md - - Join: query/join.md - - Aggregation: query/aggregation.md - - Union: query/union.md - - Universal Sets: query/universals.md - - Query Caching: query/query-caching.md - - Computations: - - Make Method: compute/make.md - - Populate: compute/populate.md - - Key Source: compute/key-source.md - - Distributed Computing: compute/distributed.md - - Publish Data: publish-data.md - - Internals: - - SQL Transpilation: internal/transpilation.md - - Tutorials: - - JSON Datatype: tutorials/json.ipynb - - FAQ: faq.md - - Developer Guide: develop.md - - Citation: citation.md - - Changelog: changelog.md - - API: api/ # defer to gen-files + literate-nav + - Home: index.md + - Contributing: develop.md + - Architecture: + - architecture/index.md + - SQL Transpilation: architecture/transpilation.md + - API Reference: api/ # defer to gen-files + literate-nav # ---------------------------- STANDARD ----------------------------- @@ -93,7 +27,7 @@ theme: favicon: assets/images/company-logo-blue.png features: - toc.integrate - - content.code.annotate # Add codeblock annotations + - content.code.annotate palette: - media: "(prefers-color-scheme: light)" scheme: datajoint @@ -113,26 +47,18 @@ plugins: handlers: python: paths: - - "." - - /main/ + - "../src" options: - filters: - - "!^_" - docstring_style: sphinx # Replaces google default pending docstring updates + docstring_style: numpy members_order: source group_by_category: false line_length: 88 + show_source: false - gen-files: scripts: - ./src/api/make_pages.py - literate-nav: nav_file: navigation.md - - exclude-search: - exclude: - - "*/navigation.md" - - "*/archive/*md" - - mkdocs-jupyter: - include: ["*.ipynb"] - section-index markdown_extensions: - attr_list @@ -154,41 +80,23 @@ markdown_extensions: - name: mermaid class: mermaid format: !!python/name:pymdownx.superfences.fence_code_format - - pymdownx.magiclink # Displays bare URLs as links - - pymdownx.tasklist: # Renders check boxes in tasks lists + - pymdownx.magiclink + - pymdownx.tasklist: custom_checkbox: true - md_in_html extra: - generator: false # Disable watermark + generator: false version: provider: mike social: - icon: main/company-logo link: https://www.datajoint.com name: DataJoint - - icon: fontawesome/brands/slack - link: https://datajoint.slack.com - name: Slack - - icon: fontawesome/brands/linkedin - link: https://www.linkedin.com/company/datajoint - name: LinkedIn - - icon: fontawesome/brands/twitter - link: https://twitter.com/datajoint - name: Twitter - icon: fontawesome/brands/github link: https://github.com/datajoint name: GitHub - - icon: fontawesome/brands/docker - link: https://hub.docker.com/u/datajoint - name: DockerHub - - icon: fontawesome/brands/python - link: https://pypi.org/user/datajointbot - name: PyPI - - icon: fontawesome/brands/stack-overflow - link: https://stackoverflow.com/questions/tagged/datajoint - name: StackOverflow - - icon: fontawesome/brands/youtube - link: https://www.youtube.com/channel/UCdeCuFOTCXlVMRzh6Wk-lGg - name: YouTube + - icon: fontawesome/brands/slack + link: https://datajoint.slack.com + name: Slack extra_css: - assets/stylesheets/extra.css diff --git a/docs/src/api/make_pages.py b/docs/src/api/make_pages.py index 3072cb46a..25dc29943 100644 --- a/docs/src/api/make_pages.py +++ b/docs/src/api/make_pages.py @@ -9,9 +9,7 @@ nav = mkdocs_gen_files.Nav() for path in sorted(Path(package).glob("**/*.py")): with mkdocs_gen_files.open(f"api/{path.with_suffix('')}.md", "w") as f: - module_path = ".".join( - [p for p in path.with_suffix("").parts if p != "__init__"] - ) + module_path = ".".join([p for p in path.with_suffix("").parts if p != "__init__"]) print(f"::: {module_path}", file=f) nav[path.parts] = f"{path.with_suffix('')}.md" diff --git a/docs/src/architecture/index.md b/docs/src/architecture/index.md new file mode 100644 index 000000000..953fd7962 --- /dev/null +++ b/docs/src/architecture/index.md @@ -0,0 +1,34 @@ +# Architecture + +Internal design documentation for DataJoint developers. + +## Query System + +- [SQL Transpilation](transpilation.md) — How DataJoint translates query expressions to SQL + +## Design Principles + +DataJoint's architecture follows several key principles: + +1. **Immutable Query Expressions** — Query expressions are immutable; operators create new objects +2. **Lazy Evaluation** — Queries are not executed until data is fetched +3. **Query Optimization** — Unnecessary attributes are projected out before execution +4. **Semantic Matching** — Joins use lineage-based attribute matching + +## Module Overview + +| Module | Purpose | +|--------|---------| +| `expression.py` | QueryExpression base class and operators | +| `table.py` | Table class with data manipulation | +| `fetch.py` | Data retrieval implementation | +| `declare.py` | Table definition parsing | +| `heading.py` | Attribute and heading management | +| `blob.py` | Blob serialization | +| `codecs.py` | Type codec system | +| `connection.py` | Database connection management | +| `schemas.py` | Schema binding and activation | + +## Contributing + +See the [Contributing Guide](../develop.md) for development setup instructions. diff --git a/docs/src/internal/transpilation.md b/docs/src/architecture/transpilation.md similarity index 100% rename from docs/src/internal/transpilation.md rename to docs/src/architecture/transpilation.md diff --git a/docs/src/changelog.md b/docs/src/changelog.md deleted file mode 120000 index 699cc9e7b..000000000 --- a/docs/src/changelog.md +++ /dev/null @@ -1 +0,0 @@ -../../CHANGELOG.md \ No newline at end of file diff --git a/docs/src/citation.md b/docs/src/citation.md deleted file mode 100644 index b5eb2d88b..000000000 --- a/docs/src/citation.md +++ /dev/null @@ -1,7 +0,0 @@ -# Citation - -If your work uses the DataJoint for Python, please cite the following manuscript and Research Resource Identifier (RRID): - -- Yatsenko D, Reimer J, Ecker AS, Walker EY, Sinz F, Berens P, Hoenselaar A, Cotton RJ, Siapas AS, Tolias AS. DataJoint: managing big scientific data using MATLAB or Python. bioRxiv. 2015 Jan 1:031658. doi: https://doi.org/10.1101/031658 - -- DataJoint for Python - [RRID:SCR_014543](https://scicrunch.org/resolver/SCR_014543) - Version `Enter datajoint-python version you are using here` diff --git a/docs/src/client/credentials.md b/docs/src/client/credentials.md deleted file mode 100644 index bac54a6cf..000000000 --- a/docs/src/client/credentials.md +++ /dev/null @@ -1,46 +0,0 @@ -# Credentials - -Configure the connection through DataJoint's `config` object: - -```python -> import datajoint as dj -DataJoint 0.4.9 (February 1, 2017) -No configuration found. Use `dj.config` to configure and save the configuration. -``` - -You may now set the database credentials: - -```python -dj.config['database.host'] = "alicelab.datajoint.io" -dj.config['database.user'] = "alice" -dj.config['database.password'] = "haha not my real password" -``` - -Skip setting the password to make DataJoint prompt to enter the password every time. - -You may save the configuration in the local work directory with -`dj.config.save_local()` or for all your projects in `dj.config.save_global()`. -Configuration changes should be made through the `dj.config` interface; the config file -should not be modified directly by the user. - -You may leave the user or the password as `None`, in which case you will be prompted to -enter them manually for every session. -Setting the password as an empty string allows access without a password. - -Note that the system environment variables `DJ_HOST`, `DJ_USER`, and `DJ_PASS` will -overwrite the settings in the config file. -You can use them to set the connection credentials instead of config files. - -To change the password, the `dj.set_password` function will walk you through the -process: - -```python -dj.set_password() -``` - -After that, update the password in the configuration and save it as described above: - -```python -dj.config['database.password'] = 'my#cool!new*psswrd' -dj.config.save_local() # or dj.config.save_global() -``` diff --git a/docs/src/client/install.md b/docs/src/client/install.md deleted file mode 100644 index d9684f302..000000000 --- a/docs/src/client/install.md +++ /dev/null @@ -1,209 +0,0 @@ -# Install and Connect - -DataJoint is implemented for Python 3.4+. -You may install it from [PyPI](https://pypi.python.org/pypi/datajoint): - -```bash -pip3 install datajoint -``` - -or upgrade - -```bash -pip3 install --upgrade datajoint -``` - -## DataJoint Python Windows Install Guide - -This document outlines the steps necessary to install DataJoint on Windows for use in -connecting to a remote server hosting a DataJoint database. -Some limited discussion of installing MySQL is discussed in `MySQL for Windows`, but is -not covered in-depth since this is an uncommon usage scenario and not strictly required -to connect to DataJoint pipelines. - -### Quick steps - -Quick install steps for advanced users are as follows: - -- Install latest Python 3.x and ensure it is in `PATH` (3.6.3 current at time of writing) - ```bash - pip install datajoint - ``` - -For ERD drawing support: - -- Install Graphviz for Windows and ensure it is in `PATH` (64 bit builds currently -tested; URL below.) - ```bash - pip install pydotplus matplotlib - ``` - -Detailed instructions follow. - -### Step 1: install Python - -Python for Windows is available from: - -https://www.python.org/downloads/windows - -The latest 64 bit 3.x version, currently 3.6.3, is available from the [Python site](https://www.python.org/ftp/python/3.6.3/python-3.6.3-amd64.exe). - -From here run the installer to install Python. - -For a single-user machine, the regular installation process is sufficient - be sure to -select the `Add Python to PATH` option: - -![install-python-simple](../images/install-python-simple.png){: style="align:left"} - -For a shared machine, run the installer as administrator (right-click, run as -administrator) and select the advanced installation. -Be sure to select options as follows: - -![install-python-advanced-1](../images/install-python-advanced-1.png){: style="align:left"} -![install-python-advanced-2](../images/install-python-advanced-2.png){: style="align:left"} - -### Step 2: verify installation - -To verify the Python installation and make sure that your system is ready to install -DataJoint, open a command window by entering `cmd` into the Windows search bar: - -![install-cmd-prompt](../images/install-cmd-prompt.png){: style="align:left"} - -From here `python` and the Python package manager `pip` can be verified by running -`python -V` and `pip -V`, respectively: - -![install-verify-python](../images/install-verify-python.png){: style="align:left"} - -If you receive the error message that either `pip` or `python` is not a recognized -command, please uninstall Python and ensure that the option to add Python to the `PATH` -variable was properly configured. - -### Step 3: install DataJoint - -DataJoint (and other Python modules) can be easily installed using the `pip` Python -package manager which is installed as a part of Python and was verified in the previous -step. - -To install DataJoint simply run `pip install datajoint`: - -![install-datajoint-1](../images/install-datajoint-1.png){: style="align:left"} - -This will proceed to install DataJoint, along with several other required packages from -the PIP repository. -When finished, a summary of the activity should be presented: - -![install-datajoint-2](../images/install-datajoint-2.png){: style="align:left"} - -Note: You can find out more about the packages installed here and many other freely -available open source packages via [pypi](https://pypi.python.org/pypi), the Python -package index site. - -### (Optional) step 4: install packages for ERD support - -To draw diagrams of your DataJoint schema, the following additional steps should be -followed. - -#### Install Graphviz - -DataJoint currently utilizes [Graphviz](http://graphviz.org) to generate the ERD -visualizations. -Although a Windows version of Graphviz is available from the main site, it is an older -and out of date 32-bit version. -The recommended pre-release builds of the 64 bit version are available here: - -https://ci.appveyor.com/project/ellson/graphviz-pl238 - -More specifically, the build artifacts from the `Win64; Configuration: Release` are -recommended, available -[here](https://ci.appveyor.com/api/buildjobs/hlkclpfhf6gnakjq/artifacts/build%2FGraphviz-install.exe). - -This is a regular Windows installer executable, and will present a dialog when starting: - -![install-graphviz-1](../images/install-graphviz-1.png){: style="align:left"} - -It is important that an option to place Graphviz in the `PATH` be selected. - -For a personal installation: - -![install-graphviz-2a](../images/install-graphviz-2a.png){: style="align:left"} - -To install system wide: - -![install-graphviz-2b](../images/install-graphviz-2b.png){: style="align:left"} - -Once installed, Graphviz can be verified from a fresh command window as follows: - -![install-verify-graphviz](../images/install-verify-graphviz.png){: style="align:left"} - -If you receive the error message that the `dot` program is not a recognized command, -please uninstall Graphviz and ensure that the -option to add Python to the PATH variable was properly configured. - -Important: in some cases, running the `dot -c` command in a command prompt is required -to properly initialize the Graphviz installation. - -#### Install PyDotPlus - -The PyDotPlus library links the Graphviz installation to DataJoint and is easily -installed via `pip`: - -![install-pydotplus](../images/install-pydotplus.png){: style="align:left"} - -#### Install Matplotlib - -The Matplotlib library provides useful plotting utilities which are also used by -DataJoint's `Diagram` drawing facility. -The package is easily installed via `pip`: - -![install-matplotlib](../images/install-matplotlib.png){: style="align:left"} - -### (Optional) step 5: install Jupyter Notebook - -As described on the www.jupyter.org website: - -''' -The Jupyter Notebook is an open-source web application that allows -you to create and share documents that contain live code, equations, -visualizations and narrative text. -''' - -Although not a part of DataJoint, Jupyter Notebook can be a very useful tool for -building and interacting with DataJoint pipelines. -It is easily installed from `pip` as well: - -![install-jupyter-1](../images/install-jupyter-1.png){: style="align:left"} -![install-jupyter-2](../images/install-jupyter-2.png){: style="align:left"} - -Once installed, Jupyter Notebook can be started via the `jupyter notebook` command, -which should now be on your path: - -![install-verify-jupyter](../images/install-verify-jupyter.png){: style="align:left"} - -By default Jupyter Notebook will start a local private web server session from the -directory where it was started and start a web browser session connected to the session. - -![install-run-jupyter-1](../images/install-run-jupyter-1.png){: style="align:left"} -![install-run-jupyter-2](../images/install-run-jupyter-2.png){: style="align:left"} - -You now should be able to use the notebook viewer to navigate the filesystem and to -create new project folders and interactive Jupyter/Python/DataJoint notebooks. - -### Git for Windows - -The [Git](https://git-scm.com/) version control system is not a part of DataJoint but -is recommended for interacting with the broader Python/Git/GitHub sharing ecosystem. - -The Git for Windows installer is available from https://git-scm.com/download/win. - -![install-git-1](../images/install-git-1.png){: style="align:left"} - -The default settings should be sufficient and correct in most cases. - -### MySQL for Windows - -For hosting pipelines locally, the MySQL server package is required. - -MySQL for windows can be installed via the installers available from the -[MySQL website](https://dev.mysql.com/downloads/windows/). -Please note that although DataJoint should be fully compatible with a Windows MySQL -server installation, this mode of operation is not tested by the DataJoint team. diff --git a/docs/src/client/settings.md b/docs/src/client/settings.md deleted file mode 100644 index cb9a69fff..000000000 --- a/docs/src/client/settings.md +++ /dev/null @@ -1,11 +0,0 @@ -# Configuration Settings - -If you are not using DataJoint on your own, or are setting up a DataJoint -system for other users, some additional configuration options may be required -to support [TLS](#tls-configuration) or -[external storage](../sysadmin/external-store.md). - -## TLS Configuration - -Starting with v0.12, DataJoint will by default use TLS if it is available. TLS can be -forced on or off with the boolean `dj.config['database.use_tls']`. diff --git a/docs/src/compute/distributed.md b/docs/src/compute/distributed.md deleted file mode 100644 index 68c31f093..000000000 --- a/docs/src/compute/distributed.md +++ /dev/null @@ -1,166 +0,0 @@ -# Distributed Computing - -## Job reservations - -Running `populate` on the same table on multiple computers will causes them to attempt -to compute the same data all at once. -This will not corrupt the data since DataJoint will reject any duplication. -One solution could be to cause the different computing nodes to populate the tables in -random order. -This would reduce some collisions but not completely prevent them. - -To allow efficient distributed computing, DataJoint provides a built-in job reservation -process. -When `dj.Computed` tables are auto-populated using job reservation, a record of each -ongoing computation is kept in a schema-wide `jobs` table, which is used internally by -DataJoint to coordinate the auto-population effort among multiple computing processes. - -Job reservations are activated by setting the keyword argument `reserve_jobs=True` in -`populate` calls. - -With job management enabled, the `make` method of each table class will also consult -the `jobs` table for reserved jobs as part of determining the next record to compute -and will create an entry in the `jobs` table as part of the attempt to compute the -resulting record for that key. -If the operation is a success, the record is removed. -In the event of failure, the job reservation entry is updated to indicate the details -of failure. -Using this simple mechanism, multiple processes can participate in the auto-population -effort without duplicating computational effort, and any errors encountered during the -course of the computation can be individually inspected to determine the cause of the -issue. - -As part of DataJoint, the jobs table can be queried using native DataJoint syntax. For -example, to list the jobs currently being run: - -```python -In [1]: schema.jobs -Out[1]: -*table_name *key_hash status error_message user host pid connection_id timestamp key error_stack -+------------+ +------------+ +----------+ +------------+ +------------+ +------------+ +-------+ +------------+ +------------+ +--------+ +------------+ -__job_results e4da3b7fbbce23 reserved datajoint@localhos localhost 15571 59 2017-09-04 14: -(2 tuples) -``` - -The above output shows that a record for the `JobResults` table is currently reserved -for computation, along with various related details of the reservation, such as the -MySQL connection ID, client user and host, process ID on the remote system, timestamp, -and the key for the record that the job is using for its computation. -Since DataJoint table keys can be of varying types, the key is stored in a binary -format to allow the table to store arbitrary types of record key data. -The subsequent sections will discuss querying the jobs table for key data. - -As mentioned above, jobs encountering errors during computation will leave their record -reservations in place, and update the reservation record with details of the error. - -For example, if a Python process is interrupted via the keyboard, a KeyboardError will -be logged to the database as follows: - -```python -In [2]: schema.jobs -Out[2]: -*table_name *key_hash status error_message user host pid connection_id timestamp key error_stack -+------------+ +------------+ +--------+ +------------+ +------------+ +------------+ +-------+ +------------+ +------------+ +--------+ +------------+ -__job_results 3416a75f4cea91 error KeyboardInterr datajoint@localhos localhost 15571 59 2017-09-04 14: -(1 tuples) -``` - -By leaving the job reservation record in place, the error can be inspected, and if -necessary the corresponding `dj.Computed` update logic can be corrected. -From there the jobs entry can be cleared, and the computation can then be resumed. -In the meantime, the presence of the job reservation will prevent this particular -record from being processed during subsequent auto-population calls. -Inspecting the job record for failure details can proceed much like any other DataJoint -query. - -For example, given the above table, errors can be inspected as follows: - -```python -In [3]: (schema.jobs & 'status="error"' ).fetch(as_dict=True) -Out[3]: -[OrderedDict([('table_name', '__job_results'), - ('key_hash', 'c81e728d9d4c2f636f067f89cc14862c'), - ('status', 'error'), - ('key', rec.array([(2,)], - dtype=[('id', 'O')])), - ('error_message', 'KeyboardInterrupt'), - ('error_stack', None), - ('user', 'datajoint@localhost'), - ('host', 'localhost'), - ('pid', 15571), - ('connection_id', 59), - ('timestamp', datetime.datetime(2017, 9, 4, 15, 3, 53))])] -``` - -This particular error occurred when processing the record with ID `2`, resulted from a -`KeyboardInterrupt`, and has no additional -error trace. - -After any system or code errors have been resolved, the table can simply be cleaned of -errors and the computation rerun. - -For example: - -```python -In [4]: (schema.jobs & 'status="error"' ).delete() -``` - -In some cases, it may be preferable to inspect the jobs table records using populate -keys. -Since job keys are hashed and stored as a blob in the jobs table to support the varying -types of keys, we need to query using the key hash instead of simply using the raw key -data. - -This can be done by using `dj.key_hash` to convert the key as follows: - -```python -In [4]: jk = {'table_name': JobResults.table_name, 'key_hash' : dj.key_hash({'id': 2})} - -In [5]: schema.jobs & jk -Out[5]: -*table_name *key_hash status key error_message error_stac user host pid connection_id timestamp -+------------+ +------------+ +--------+ +--------+ +------------+ +--------+ +------------+ +-------+ +--------+ +------------+ +------------+ -__job_results c81e728d9d4c2f error =BLOB= KeyboardInterr =BLOB= datajoint@localhost localhost 15571 59 2017-09-04 14: -(Total: 1) - -In [6]: (schema.jobs & jk).delete() - -In [7]: schema.jobs & jk -Out[7]: -*table_name *key_hash status key error_message error_stac user host pid connection_id timestamp -+------------+ +----------+ +--------+ +--------+ +------------+ +--------+ +------+ +------+ +-----+ +------------+ +-----------+ - -(Total: 0) -``` - -## Managing connections - -The DataJoint method `dj.kill` allows for viewing and termination of database -connections. -Restrictive conditions can be used to identify specific connections. -Restrictions are specified as strings and can involve any of the attributes of -`information_schema.processlist`: `ID`, `USER`, `HOST`, `DB`, `COMMAND`, `TIME`, -`STATE`, and `INFO`. - -Examples: - - `dj.kill('HOST LIKE "%compute%"')` lists only connections from hosts containing "compute". - `dj.kill('TIME > 600')` lists only connections older than 10 minutes. - -A list of connections meeting the restriction conditions (if present) are presented to -the user, along with the option to kill processes. By default, output is ordered by -ascending connection ID. To change the output order of dj.kill(), an additional -order_by argument can be provided. - -For example, to sort the output by hostname in descending order: - -```python -In [3]: dj.kill(None, None, 'host desc') -Out[3]: - ID USER HOST STATE TIME INFO -+--+ +----------+ +-----------+ +-----------+ +-----+ - 33 chris localhost:54840 1261 None - 17 chris localhost:54587 3246 None - 4 event_scheduler localhost Waiting on empty queue 187180 None -process to kill or "q" to quit > q -``` diff --git a/docs/src/compute/key-source.md b/docs/src/compute/key-source.md deleted file mode 100644 index 76796ec0c..000000000 --- a/docs/src/compute/key-source.md +++ /dev/null @@ -1,51 +0,0 @@ -# Key Source - -## Default key source - -**Key source** refers to the set of primary key values over which -[autopopulate](./populate.md) iterates, calling the `make` method at each iteration. -Each `key` from the key source is passed to the table's `make` call. -By default, the key source for a table is the [join](../query/join.md) of its primary -[dependencies](../design/tables/dependencies.md). - -For example, consider a schema with three tables. -The `Stimulus` table contains one attribute `stimulus_type` with one of two values, -"Visual" or "Auditory". -The `Modality` table contains one attribute `modality` with one of three values, "EEG", -"fMRI", and "PET". -The `Protocol` table has primary dependencies on both the `Stimulus` and `Modality` tables. - -The key source for `Protocol` will then be all six combinations of `stimulus_type` and -`modality` as shown in the figure below. - -![Combination of stimulus_type and modality](../images/key_source_combination.png){: style="align:center"} - -## Custom key source - -A custom key source can be configured by setting the `key_source` property within a -table class, after the `definition` string. - -Any [query object](../query/fetch.md) can be used as the key source. -In most cases the new key source will be some alteration of the default key source. -Custom key sources often involve restriction to limit the key source to only relevant -entities. -Other designs may involve using only one of a table's primary dependencies. - -In the example below, the `EEG` table depends on the `Recording` table that lists all -recording sessions. -However, the `populate` method of `EEG` should only ingest recordings where the -`recording_type` is `EEG`. -Setting a custom key source prevents the `populate` call from iterating over recordings -of the wrong type. - -```python -@schema -class EEG(dj.Imported): -definition = """ --> Recording ---- -sample_rate : float -eeg_data : longblob -""" -key_source = Recording & 'recording_type = "EEG"' -``` diff --git a/docs/src/compute/make.md b/docs/src/compute/make.md deleted file mode 100644 index 1b5569b65..000000000 --- a/docs/src/compute/make.md +++ /dev/null @@ -1,215 +0,0 @@ -# Transactions in Make - -Each call of the [make](../compute/make.md) method is enclosed in a transaction. -DataJoint users do not need to explicitly manage transactions but must be aware of -their use. - -Transactions produce two effects: - -First, the state of the database appears stable within the `make` call throughout the -transaction: -two executions of the same query will yield identical results within the same `make` -call. - -Second, any changes to the database (inserts) produced by the `make` method will not -become visible to other processes until the `make` call completes execution. -If the `make` method raises an exception, all changes made so far will be discarded and -will never become visible to other processes. - -Transactions are particularly important in maintaining -[group integrity](../design/integrity.md#group-integrity) with -[master-part relationships](../design/tables/master-part.md). -The `make` call of a master table first inserts the master entity and then inserts all -the matching part entities in the part tables. -None of the entities become visible to other processes until the entire `make` call -completes, at which point they all become visible. - -### Three-Part Make Pattern for Long Computations - -For long-running computations, DataJoint provides an advanced pattern called the -**three-part make** that separates the `make` method into three distinct phases. -This pattern is essential for maintaining database performance and data integrity -during expensive computations. - -#### The Problem: Long Transactions - -Traditional `make` methods perform all operations within a single database transaction: - -```python -def make(self, key): - # All within one transaction - data = (ParentTable & key).fetch1() # Fetch - result = expensive_computation(data) # Compute (could take hours) - self.insert1(dict(key, result=result)) # Insert -``` - -This approach has significant limitations: -- **Database locks**: Long transactions hold locks on tables, blocking other operations -- **Connection timeouts**: Database connections may timeout during long computations -- **Memory pressure**: All fetched data must remain in memory throughout the computation -- **Failure recovery**: If computation fails, the entire transaction is rolled back - -#### The Solution: Three-Part Make Pattern - -The three-part make pattern splits the `make` method into three distinct phases, -allowing the expensive computation to occur outside of database transactions: - -```python -def make_fetch(self, key): - """Phase 1: Fetch all required data from parent tables""" - fetched_data = ((ParentTable1 & key).fetch1(), (ParentTable2 & key).fetch1()) - return fetched_data # must be a sequence, eg tuple or list - -def make_compute(self, key, *fetched_data): - """Phase 2: Perform expensive computation (outside transaction)""" - computed_result = expensive_computation(*fetched_data) - return computed_result # must be a sequence, eg tuple or list - -def make_insert(self, key, *computed_result): - """Phase 3: Insert results into the current table""" - self.insert1(dict(key, result=computed_result)) -``` - -#### Execution Flow - -To achieve data intensity without long transactions, the three-part make pattern follows this sophisticated execution sequence: - -```python -# Step 1: Fetch data outside transaction -fetched_data1 = self.make_fetch(key) -computed_result = self.make_compute(key, *fetched_data1) - -# Step 2: Begin transaction and verify data consistency -begin transaction: - fetched_data2 = self.make_fetch(key) - if fetched_data1 != fetched_data2: # deep comparison - cancel transaction # Data changed during computation - else: - self.make_insert(key, *computed_result) - commit_transaction -``` - -#### Key Benefits - -1. **Reduced Database Lock Time**: Only the fetch and insert operations occur within transactions, minimizing lock duration -2. **Connection Efficiency**: Database connections are only used briefly for data transfer -3. **Memory Management**: Fetched data can be processed and released during computation -4. **Fault Tolerance**: Computation failures don't affect database state -5. **Scalability**: Multiple computations can run concurrently without database contention - -#### Referential Integrity Protection - -The pattern includes a critical safety mechanism: **referential integrity verification**. -Before inserting results, the system: - -1. Re-fetches the source data within the transaction -2. Compares it with the originally fetched data using deep hashing -3. Only proceeds with insertion if the data hasn't changed - -This prevents the "phantom read" problem where source data changes during long computations, -ensuring that results remain consistent with their inputs. - -#### Implementation Details - -The pattern is implemented using Python generators in the `AutoPopulate` class: - -```python -def make(self, key): - # Step 1: Fetch data from parent tables - fetched_data = self.make_fetch(key) - computed_result = yield fetched_data - - # Step 2: Compute if not provided - if computed_result is None: - computed_result = self.make_compute(key, *fetched_data) - yield computed_result - - # Step 3: Insert the computed result - self.make_insert(key, *computed_result) - yield -``` -Therefore, it is possible to override the `make` method to implement the three-part make pattern by using the `yield` statement to return the fetched data and computed result as above. - -#### Use Cases - -This pattern is particularly valuable for: - -- **Machine learning model training**: Hours-long training sessions -- **Image processing pipelines**: Large-scale image analysis -- **Statistical computations**: Complex statistical analyses -- **Data transformations**: ETL processes with heavy computation -- **Simulation runs**: Time-consuming simulations - -#### Example: Long-Running Image Analysis - -Here's an example of how to implement the three-part make pattern for a -long-running image analysis task: - -```python -@schema -class ImageAnalysis(dj.Computed): - definition = """ - # Complex image analysis results - -> Image - --- - analysis_result : longblob - processing_time : float - """ - - def make_fetch(self, key): - """Fetch the image data needed for analysis""" - image_data = (Image & key).fetch1('image') - params = (Params & key).fetch1('params') - return (image_data, params) # pack fetched_data - - def make_compute(self, key, image_data, params): - """Perform expensive image analysis outside transaction""" - import time - start_time = time.time() - - # Expensive computation that could take hours - result = complex_image_analysis(image_data, params) - processing_time = time.time() - start_time - return result, processing_time - - def make_insert(self, key, analysis_result, processing_time): - """Insert the analysis results""" - self.insert1(dict(key, - analysis_result=analysis_result, - processing_time=processing_time)) -``` - -The exact same effect may be achieved by overriding the `make` method as a generator function using the `yield` statement to return the fetched data and computed result as above: - -```python -@schema -class ImageAnalysis(dj.Computed): - definition = """ - # Complex image analysis results - -> Image - --- - analysis_result : longblob - processing_time : float - """ - - def make(self, key): - image_data = (Image & key).fetch1('image') - params = (Params & key).fetch1('params') - computed_result = yield (image, params) # pack fetched_data - - if computed_result is None: - # Expensive computation that could take hours - import time - start_time = time.time() - result = complex_image_analysis(image_data, params) - processing_time = time.time() - start_time - computed_result = result, processing_time #pack - yield computed_result - - result, processing_time = computed_result # unpack - self.insert1(dict(key, - analysis_result=result, - processing_time=processing_time)) - yield # yield control back to the caller -``` -We expect that most users will prefer to use the three-part implementation over the generator function implementation due to its conceptual complexity. \ No newline at end of file diff --git a/docs/src/compute/populate.md b/docs/src/compute/populate.md deleted file mode 100644 index 45c863f17..000000000 --- a/docs/src/compute/populate.md +++ /dev/null @@ -1,317 +0,0 @@ -# Auto-populate - -Auto-populated tables are used to define, execute, and coordinate computations in a -DataJoint pipeline. - -Tables in the initial portions of the pipeline are populated from outside the pipeline. -In subsequent steps, computations are performed automatically by the DataJoint pipeline -in auto-populated tables. - -Computed tables belong to one of the two auto-populated -[data tiers](../design/tables/tiers.md): `dj.Imported` and `dj.Computed`. -DataJoint does not enforce the distinction between imported and computed tables: the -difference is purely semantic, a convention for developers to follow. -If populating a table requires access to external files such as raw storage that is not -part of the database, the table is designated as **imported**. -Otherwise it is **computed**. - -Auto-populated tables are defined and queried exactly as other tables. -(See [Manual Tables](../design/tables/manual.md).) -Their data definition follows the same [definition syntax](../design/tables/declare.md). - -## Make - -For auto-populated tables, data should never be entered using -[insert](../manipulation/insert.md) directly. -Instead these tables must define the callback method `make(self, key)`. -The `insert` method then can only be called on `self` inside this callback method. - -Imagine that there is a table `test.Image` that contains 2D grayscale images in its -`image` attribute. -Let us define the computed table, `test.FilteredImage` that filters the image in some -way and saves the result in its `filtered_image` attribute. - -The class will be defined as follows. - -```python -@schema -class FilteredImage(dj.Computed): - definition = """ - # Filtered image - -> Image - --- - filtered_image : longblob - """ - - def make(self, key): - img = (test.Image & key).fetch1('image') - key['filtered_image'] = myfilter(img) - self.insert1(key) -``` - -The `make` method receives one argument: the dict `key` containing the primary key -value of an element of [key source](key-source.md) to be worked on. - -The key represents the partially filled entity, usually already containing the -[primary key](../design/tables/primary.md) attributes of the key source. - -The `make` callback does three things: - -1. [Fetches](../query/fetch.md) data from tables upstream in the pipeline using the -`key` for [restriction](../query/restrict.md). -2. Computes and adds any missing attributes to the fields already in `key`. -3. Inserts the entire entity into `self`. - -A single `make` call may populate multiple entities when `key` does not specify the -entire primary key of the populated table, when the definition adds new attributes to the primary key. -This design is uncommon and not recommended. -The standard practice for autopopulated tables is to have its primary key composed of -foreign keys pointing to parent tables. - -### Three-Part Make Pattern for Long Computations - -For long-running computations, DataJoint provides an advanced pattern called the -**three-part make** that separates the `make` method into three distinct phases. -This pattern is essential for maintaining database performance and data integrity -during expensive computations. - -#### The Problem: Long Transactions - -Traditional `make` methods perform all operations within a single database transaction: - -```python -def make(self, key): - # All within one transaction - data = (ParentTable & key).fetch1() # Fetch - result = expensive_computation(data) # Compute (could take hours) - self.insert1(dict(key, result=result)) # Insert -``` - -This approach has significant limitations: -- **Database locks**: Long transactions hold locks on tables, blocking other operations -- **Connection timeouts**: Database connections may timeout during long computations -- **Memory pressure**: All fetched data must remain in memory throughout the computation -- **Failure recovery**: If computation fails, the entire transaction is rolled back - -#### The Solution: Three-Part Make Pattern - -The three-part make pattern splits the `make` method into three distinct phases, -allowing the expensive computation to occur outside of database transactions: - -```python -def make_fetch(self, key): - """Phase 1: Fetch all required data from parent tables""" - fetched_data = ((ParentTable & key).fetch1(),) - return fetched_data # must be a sequence, eg tuple or list - -def make_compute(self, key, *fetched_data): - """Phase 2: Perform expensive computation (outside transaction)""" - computed_result = expensive_computation(*fetched_data) - return computed_result # must be a sequence, eg tuple or list - -def make_insert(self, key, *computed_result): - """Phase 3: Insert results into the current table""" - self.insert1(dict(key, result=computed_result)) -``` - -#### Execution Flow - -To achieve data intensity without long transactions, the three-part make pattern follows this sophisticated execution sequence: - -```python -# Step 1: Fetch data outside transaction -fetched_data1 = self.make_fetch(key) -computed_result = self.make_compute(key, *fetched_data1) - -# Step 2: Begin transaction and verify data consistency -begin transaction: - fetched_data2 = self.make_fetch(key) - if fetched_data1 != fetched_data2: # deep comparison - cancel transaction # Data changed during computation - else: - self.make_insert(key, *computed_result) - commit_transaction -``` - -#### Key Benefits - -1. **Reduced Database Lock Time**: Only the fetch and insert operations occur within transactions, minimizing lock duration -2. **Connection Efficiency**: Database connections are only used briefly for data transfer -3. **Memory Management**: Fetched data can be processed and released during computation -4. **Fault Tolerance**: Computation failures don't affect database state -5. **Scalability**: Multiple computations can run concurrently without database contention - -#### Referential Integrity Protection - -The pattern includes a critical safety mechanism: **referential integrity verification**. -Before inserting results, the system: - -1. Re-fetches the source data within the transaction -2. Compares it with the originally fetched data using deep hashing -3. Only proceeds with insertion if the data hasn't changed - -This prevents the "phantom read" problem where source data changes during long computations, -ensuring that results remain consistent with their inputs. - -#### Implementation Details - -The pattern is implemented using Python generators in the `AutoPopulate` class: - -```python -def make(self, key): - # Step 1: Fetch data from parent tables - fetched_data = self.make_fetch(key) - computed_result = yield fetched_data - - # Step 2: Compute if not provided - if computed_result is None: - computed_result = self.make_compute(key, *fetched_data) - yield computed_result - - # Step 3: Insert the computed result - self.make_insert(key, *computed_result) - yield -``` -Therefore, it is possible to override the `make` method to implement the three-part make pattern by using the `yield` statement to return the fetched data and computed result as above. - -#### Use Cases - -This pattern is particularly valuable for: - -- **Machine learning model training**: Hours-long training sessions -- **Image processing pipelines**: Large-scale image analysis -- **Statistical computations**: Complex statistical analyses -- **Data transformations**: ETL processes with heavy computation -- **Simulation runs**: Time-consuming simulations - -#### Example: Long-Running Image Analysis - -Here's an example of how to implement the three-part make pattern for a -long-running image analysis task: - -```python -@schema -class ImageAnalysis(dj.Computed): - definition = """ - # Complex image analysis results - -> Image - --- - analysis_result : longblob - processing_time : float - """ - - def make_fetch(self, key): - """Fetch the image data needed for analysis""" - return (Image & key).fetch1('image'), - - def make_compute(self, key, image_data): - """Perform expensive image analysis outside transaction""" - import time - start_time = time.time() - - # Expensive computation that could take hours - result = complex_image_analysis(image_data) - processing_time = time.time() - start_time - return result, processing_time - - def make_insert(self, key, analysis_result, processing_time): - """Insert the analysis results""" - self.insert1(dict(key, - analysis_result=analysis_result, - processing_time=processing_time)) -``` - -The exact same effect may be achieved by overriding the `make` method as a generator function using the `yield` statement to return the fetched data and computed result as above: - -```python -@schema -class ImageAnalysis(dj.Computed): - definition = """ - # Complex image analysis results - -> Image - --- - analysis_result : longblob - processing_time : float - """ - - def make(self, key): - image_data = (Image & key).fetch1('image') - computed_result = yield (image_data, ) # pack fetched_data - - if computed_result is None: - # Expensive computation that could take hours - import time - start_time = time.time() - result = complex_image_analysis(image_data) - processing_time = time.time() - start_time - computed_result = result, processing_time #pack - yield computed_result - - result, processing_time = computed_result # unpack - self.insert1(dict(key, - analysis_result=result, - processing_time=processing_time)) - yield # yield control back to the caller -``` -We expect that most users will prefer to use the three-part implementation over the generator function implementation due to its conceptual complexity. - -## Populate - -The inherited `populate` method of `dj.Imported` and `dj.Computed` automatically calls -`make` for every key for which the auto-populated table is missing data. - -The `FilteredImage` table can be populated as - -```python -FilteredImage.populate() -``` - -The progress of long-running calls to `populate()` in datajoint-python can be -visualized by adding the `display_progress=True` argument to the populate call. - -Note that it is not necessary to specify which data needs to be computed. -DataJoint will call `make`, one-by-one, for every key in `Image` for which -`FilteredImage` has not yet been computed. - -Chains of auto-populated tables form computational pipelines in DataJoint. - -## Populate options - -The `populate` method accepts a number of optional arguments that provide more features -and allow greater control over the method's behavior. - -- `restrictions` - A list of restrictions, restricting as -`(tab.key_source & AndList(restrictions)) - tab.proj()`. - Here `target` is the table to be populated, usually `tab` itself. -- `suppress_errors` - If `True`, encountering an error will cancel the current `make` -call, log the error, and continue to the next `make` call. - Error messages will be logged in the job reservation table (if `reserve_jobs` is - `True`) and returned as a list. - See also `return_exception_objects` and `reserve_jobs`. - Defaults to `False`. -- `return_exception_objects` - If `True`, error objects are returned instead of error - messages. - This applies only when `suppress_errors` is `True`. - Defaults to `False`. -- `reserve_jobs` - If `True`, reserves job to indicate to other distributed processes. - The job reservation table may be access as `schema.jobs`. - Errors are logged in the jobs table. - Defaults to `False`. -- `order` - The order of execution, either `"original"`, `"reverse"`, or `"random"`. - Defaults to `"original"`. -- `display_progress` - If `True`, displays a progress bar. - Defaults to `False`. -- `limit` - If not `None`, checks at most this number of keys. - Defaults to `None`. -- `max_calls` - If not `None`, populates at most this many keys. - Defaults to `None`, which means no limit. - -## Progress - -The method `table.progress` reports how many `key_source` entries have been populated -and how many remain. -Two optional parameters allow more advanced use of the method. -A parameter of restriction conditions can be provided, specifying which entities to -consider. -A Boolean parameter `display` (default is `True`) allows disabling the output, such -that the numbers of remaining and total entities are returned but not printed. diff --git a/docs/src/concepts/data-model.md b/docs/src/concepts/data-model.md deleted file mode 100644 index 90460361a..000000000 --- a/docs/src/concepts/data-model.md +++ /dev/null @@ -1,172 +0,0 @@ -# Data Model - -## What is a data model? - -A **data model** is a conceptual framework that defines how data is organized, -represented, and transformed. It gives us the components for creating blueprints for the -structure and operations of data management systems, ensuring consistency and efficiency -in data handling. - -Data management systems are built to accommodate these models, allowing us to manage -data according to the principles laid out by the model. If you’re studying data science -or engineering, you’ve likely encountered different data models, each providing a unique -approach to organizing and manipulating data. - -A data model is defined by considering the following key aspects: - -+ What are the fundamental elements used to structure the data? -+ What operations are available for defining, creating, and manipulating the data? -+ What mechanisms exist to enforce the structure and rules governing valid data interactions? - -## Types of data models - -Among the most familiar data models are those based on files and folders: data of any -kind are lumped together into binary strings called **files**, files are collected into -folders, and folders can be nested within other folders to create a folder hierarchy. - -Another family of data models are various **tabular models**. -For example, items in CSV files are listed in rows, and the attributes of each item are -stored in columns. -Various **spreadsheet** models allow forming dependencies between cells and groups of -cells, including complex calculations. - -The **object data model** is common in programming, where data are represented as -objects in memory with properties and methods for transformations of such data. - -## Relational data model - -The **relational model** is a way of thinking about data as sets and operations on sets. -Formalized almost a half-century ago ([Codd, -1969](https://dl.acm.org/citation.cfm?doid=362384.362685)). The relational data model is -one of the most powerful and precise ways to store and manage structured data. At its -core, this model organizes all data into tables--representing mathematical -relations---where each table consists of rows (representing mathematical tuples) and -columns (often called attributes). - -### Core principles of the relational data model - -**Data representation:** - Data are represented and manipulated in the form of relations. - A relation is a set (i.e. an unordered collection) of entities of values for each of - the respective named attributes of the relation. - Base relations represent stored data while derived relations are formed from base - relations through query expressions. - A collection of base relations with their attributes, domain constraints, uniqueness - constraints, and referential constraints is called a schema. - -**Domain constraints:** - Each attribute (column) in a table is associated with a specific attribute domain (or - datatype, a set of possible values), ensuring that the data entered is valid. - Attribute domains may not include relations, which keeps the data model - flat, i.e. free of nested structures. - -**Uniqueness constraints:** - Entities within relations are addressed by values of their attributes. - To identify and relate data elements, uniqueness constraints are imposed on subsets - of attributes. - Such subsets are then referred to as keys. - One key in a relation is designated as the primary key used for referencing its elements. - -**Referential constraints:** - Associations among data are established by means of referential constraints with the - help of foreign keys. - A referential constraint on relation A referencing relation B allows only those - entities in A whose foreign key attributes match the key attributes of an entity in B. - -**Declarative queries:** - Data queries are formulated through declarative, as opposed to imperative, - specifications of sought results. - This means that query expressions convey the logic for the result rather than the - procedure for obtaining it. - Formal languages for query expressions include relational algebra, relational - calculus, and SQL. - -The relational model has many advantages over both hierarchical file systems and -tabular models for maintaining data integrity and providing flexible access to -interesting subsets of the data. - -Popular implementations of the relational data model rely on the Structured Query -Language (SQL). -SQL comprises distinct sublanguages for schema definition, data manipulation, and data -queries. -SQL thoroughly dominates in the space of relational databases and is often conflated -with the relational data model in casual discourse. -Various terminologies are used to describe related concepts from the relational data -model. -Similar to spreadsheets, relations are often visualized as tables with *attributes* -corresponding to *columns* and *entities* corresponding to *rows*. -In particular, SQL uses the terms *table*, *column*, and *row*. - -## The DataJoint Model - -DataJoint is a conceptual refinement of the relational data model offering a more -expressive and rigorous framework for database programming ([Yatsenko et al., -2018](https://arxiv.org/abs/1807.11104)). The DataJoint model facilitates conceptual -clarity, efficiency, workflow management, and precise and flexible data -queries. By enforcing entity normalization, -simplifying dependency declarations, offering a rich query algebra, and visualizing -relationships through schema diagrams, DataJoint makes relational database programming -more intuitive and robust for complex data pipelines. - -The model has emerged over a decade of continuous development of complex data -pipelines for neuroscience experiments ([Yatsenko et al., -2015](https://www.biorxiv.org/content/early/2015/11/14/031658)). DataJoint has allowed -researchers with no prior knowledge of databases to collaborate effectively on common -data pipelines sustaining data integrity and supporting flexible access. DataJoint is -currently implemented as client libraries in MATLAB and Python. These libraries work by -transpiling DataJoint queries into SQL before passing them on to conventional relational -database systems that serve as the backend, in combination with bulk storage systems for -storing large contiguous data objects. - -DataJoint comprises: - -+ a schema [definition](../design/tables/declare.md) language -+ a data [manipulation](../manipulation/index.md) language -+ a data [query](../query/principles.md) language -+ a [diagramming](../design/diagrams.md) notation for visualizing relationships between -modeled entities - -The key refinement of DataJoint over other relational data models and their -implementations is DataJoint's support of -[entity normalization](../design/normalization.md). - -### Core principles of the DataJoint model - -**Entity Normalization** - DataJoint enforces entity normalization, ensuring that every entity set (table) is - well-defined, with each element belonging to the same type, sharing the same - attributes, and distinguished by the same primary key. This principle reduces - redundancy and avoids data anomalies, similar to Boyce-Codd Normal Form, but with a - more intuitive structure than traditional SQL. - -**Simplified Schema Definition and Dependency Management** - DataJoint introduces a schema definition language that is more expressive and less - error-prone than SQL. Dependencies are explicitly declared using arrow notation - (->), making referential constraints easier to understand and visualize. The - dependency structure is enforced as an acyclic directed graph, which simplifies - workflows by preventing circular dependencies. - -**Integrated Query Operators producing a Relational Algebra** - DataJoint introduces five query operators (restrict, join, project, aggregate, and - union) with algebraic closure, allowing them to be combined seamlessly. These - operators are designed to maintain operational entity normalization, ensuring query - outputs remain valid entity sets. - -**Diagramming Notation for Conceptual Clarity** - DataJoint’s schema diagrams simplify the representation of relationships between - entity sets compared to ERM diagrams. Relationships are expressed as dependencies - between entity sets, which are visualized using solid or dashed lines for primary - and secondary dependencies, respectively. - -**Unified Logic for Binary Operators** - DataJoint simplifies binary operations by requiring attributes involved in joins or - comparisons to be homologous (i.e., sharing the same origin). This avoids the - ambiguity and pitfalls of natural joins in SQL, ensuring more predictable query - results. - -**Optimized Data Pipelines for Scientific Workflows** - DataJoint treats the database as a data pipeline where each entity set defines a - step in the workflow. This makes it ideal for scientific experiments and complex - data processing, such as in neuroscience. Its MATLAB and Python libraries transpile - DataJoint queries into SQL, bridging the gap between scientific programming and - relational databases. diff --git a/docs/src/concepts/data-pipelines.md b/docs/src/concepts/data-pipelines.md deleted file mode 100644 index cf20b075b..000000000 --- a/docs/src/concepts/data-pipelines.md +++ /dev/null @@ -1,166 +0,0 @@ -# Data Pipelines - -## What is a data pipeline? - -A scientific **data pipeline** is a collection of processes and systems for organizing -the data, computations, and workflows used by a research group as they jointly perform -complex sequences of data acquisition, processing, and analysis. - -A variety of tools can be used for supporting shared data pipelines: - -Data repositories - Research teams set up a shared **data repository**. - This minimal data management tool allows depositing and retrieving data and managing - user access. - For example, this may include a collection of files with standard naming conventions - organized into folders and sub-folders. - Or a data repository might reside on the cloud, for example in a collection of S3 - buckets. - This image of data management -- where files are warehoused and retrieved from a - hierarchically-organized system of folders -- is an approach that is likely familiar - to most scientists. - -Database systems - **Databases** are a form of data repository providing additional capabilities: - - 1. Defining, communicating, and enforcing structure in the stored data. - 2. Maintaining data integrity: correct identification of data and consistent cross-references, dependencies, and groupings among the data. - 3. Supporting queries that retrieve various cross-sections and transformation of the deposited data. - - Most scientists have some familiarity with these concepts, for example the notion of maintaining consistency between data and the metadata that describes it, or applying a filter to an Excel spreadsheet to retrieve specific subsets of information. - However, usually the more advanced concepts involved in building and using relational databases fall under the specific expertise of data scientists. - -Data pipelines - **Data pipeline** frameworks may include all the features of a database system along - with additional functionality: - - 1. Integrating computations to perform analyses and manage intermediate results in a principled way. - 2. Supporting distributed computations without conflict. - 3. Defining, communicating, and enforcing **workflow**, making clear the sequence of steps that must be performed for data entry, acquisition, and processing. - - Again, the informal notion of an analysis "workflow" will be familiar to most scientists, along with the logistical difficulties associated with managing a workflow that is shared by multiple scientists within or across labs. - - Therefore, a full-featured data pipeline framework may also be described as a [scientific workflow system](https://en.wikipedia.org/wiki/Scientific_workflow_system). - -Major features of data management frameworks: data repositories, databases, and data pipelines. - -![data pipelines vs databases vs data repositories](../images/pipeline-database.png){: style="align:center"} - -## What is DataJoint? - -DataJoint is a free open-source framework for creating scientific data pipelines -directly from MATLAB or Python (or any mixture of the two). -The data are stored in a language-independent way that allows interoperability between -MATLAB and Python, with additional languages in the works. -DataJoint pipelines become the central tool in the operations of data-intensive labs or -consortia as they organize participants with different roles and skills around a common -framework. - -In DataJoint, a data pipeline is a sequence of steps (more generally, a directed -acyclic graph) with integrated data storage at each step. -The pipeline may have some nodes requiring manual data entry or import from external -sources, some that read from raw data files, and some that perform computations on data -stored in other database nodes. -In a typical scenario, experimenters and acquisition instruments feed data into nodes -at the head of the pipeline, while downstream nodes perform automated computations for -data processing and analysis. - -For example, this is the pipeline for a simple mouse experiment involving calcium -imaging in mice. - -![A data pipeline](../images/pipeline.png){: style="width:250px; align:center"} - -In this example, the experimenter first enters information about a mouse, then enters -information about each imaging session in that mouse, and then each scan performed in -each imaging session. -Next the automated portion of the pipeline takes over to import the raw imaging data, -perform image alignment to compensate for motion, image segmentation to identify cells -in the images, and extraction of calcium traces. -Finally, the receptive field (RF) computation is performed by relating the calcium -signals to the visual stimulus information. - -## How DataJoint works - -DataJoint enables data scientists to build and operate scientific data pipelines. - -Conceptual overview of DataJoint operation. - -![DataJoint operation](../images/how-it-works.png){: style="align:center"} - -DataJoint provides a simple and powerful data model, which is detailed more formally in [Yatsenko D, Walker EY, Tolias AS (2018). DataJoint: A Simpler Relational Data Model.](https://arxiv.org/abs/1807.11104). -Put most generally, a "data model" defines how to think about data and the operations -that can be performed on them. -DataJoint's model is a refinement of the relational data model: all nodes in the -pipeline are simple tables storing data, tables are related by their shared attributes, -and query operations can combine the contents of multiple tables. -DataJoint enforces specific constraints on the relationships between tables that help -maintain data integrity and enable flexible access. -DataJoint uses a succinct data definition language, a powerful data query language, and -expressive visualizations of the pipeline. -A well-defined and principled approach to data organization and computation enables -teams of scientists to work together efficiently. -The data become immediately available to all participants with appropriate access privileges. -Some of the "participants" may be computational agents that perform processing and -analysis, and so DataJoint features a built-in distributed job management process to -allow distributing analysis between any number of computers. - -From a practical point of view, the back-end data architecture may vary depending on -project requirements. -Typically, the data architecture includes a relational database server (e.g. MySQL) and -a bulk data storage system (e.g. [AWS S3](https://aws.amazon.com/s3/) or a filesystem). -However, users need not interact with the database directly, but via MATLAB or Python -objects that are each associated with an individual table in the database. -One of the main advantages of this approach is that DataJoint clearly separates the -data model facing the user from the data architecture implementing data management and -computing. DataJoint works well in combination with good code sharing (e.g. with -[git](https://git-scm.com/)) and environment sharing (e.g. with -[Docker](https://www.docker.com/)). - -DataJoint is designed for quick prototyping and continuous exploration as experimental -designs change or evolve. -New analysis methods can be added or removed at any time, and the structure of the -workflow itself can change over time, for example as new data acquisition methods are -developed. - -With DataJoint, data sharing and publishing is no longer a separate step at the end of -the project. -Instead data sharing is an inherent feature of the process: to share data with other -collaborators or to publish the data to the world, one only needs to set the access -privileges. - -## Real-life example - -The [Mesoscale Activity Project](https://www.simonsfoundation.org/funded-project/%20multi-regional-neuronal-dynamics-of-memory-guided-flexible-behavior/) -(MAP) is a collaborative project between four neuroscience labs. -MAP uses DataJoint for data acquisition, processing, analysis, interfaces, and external sharing. - -The DataJoint pipeline for the MAP project. - -![A data pipeline for the MAP project](../images/map-dataflow.png){: style="align:center"} - -The pipeline is hosted in the cloud through [Amazon Web Services](https://aws.amazon.com/) (AWS). -MAP data scientists at the Janelia Research Campus and Baylor College of Medicine -defined the data pipeline. -Experimental scientists enter manual data directly into the pipeline using the -[Helium web interface](https://github.com/mattbdean/Helium). -The raw data are preprocessed using the DataJoint client libraries in MATLAB and Python; -the preprocessed data are ingested into the pipeline while the bulky and raw data are -shared using [Globus](https://globus.org) transfer through the -[PETREL](https://www.alcf.anl.gov/petrel) storage servers provided by the Argonne -National Lab. -Data are made immediately available for exploration and analysis to collaborating labs, -and the analysis results are also immediately shared. -Analysis data may be visualized through web interfaces. -Intermediate results may be exported into the [NWB](https://nwb.org) format for sharing -with external groups. - -## Summary of DataJoint features - -1. A free, open-source framework for scientific data pipelines and workflow management -2. Data hosting in cloud or in-house -3. MySQL, filesystems, S3, and Globus for data management -4. Define, visualize, and query data pipelines from MATLAB or Python -5. Enter and view data through GUIs -6. Concurrent access by multiple users and computational agents -7. Data integrity: identification, dependencies, groupings -8. Automated distributed computation diff --git a/docs/src/concepts/principles.md b/docs/src/concepts/principles.md deleted file mode 100644 index 2bf491590..000000000 --- a/docs/src/concepts/principles.md +++ /dev/null @@ -1,136 +0,0 @@ -# Principles - -## Theoretical Foundations - -*DataJoint Core* implements a systematic framework for the joint management of -structured scientific data and its associated computations. -The framework builds on the theoretical foundations of the -[Relational Model](https://en.wikipedia.org/wiki/Relational_model) and -the [Entity-Relationship Model](https://en.wikipedia.org/wiki/Entity%E2%80%93relationship_model), -introducing a number of critical clarifications for the effective use of databases as -scientific data pipelines. -Notably, DataJoint introduces the concept of *computational dependencies* as a native -first-class citizen of the data model. -This integration of data structure and computation into a single model, defines a new -class of *computational scientific databases*. - -This page defines the key principles of this model without attachment to a specific -implementation while a more complete description of the model can be found in -[Yatsenko et al, 2018](https://doi.org/10.48550/arXiv.1807.11104). - -DataJoint developers are developing these principles into an -[open standard](https://en.wikipedia.org/wiki/Open_standard) to allow multiple -alternative implementations. - -## Data Representation - -### Tables = Entity Sets - -DataJoint uses only one data structure in all its operations—the *entity set*. - -1. All data are represented in the form of *entity sets*, i.e. an ordered collection of -*entities*. -2. All entities of an entity set belong to the same well-defined entity class and have -the same set of named attributes. -3. Attributes in an entity set has a *data type* (or *domain*), representing the set of -its valid values. -4. Each entity in an entity set provides the *attribute values* for all of the -attributes of its entity class. -5. Each entity set has a *primary key*, *i.e.* a subset of attributes that, jointly, -uniquely identify any entity in the set. - -These formal terms have more common (even if less precise) variants: - -| formal | common | -|:-:|:--:| -| entity set | *table* | -| attribute | *column* | -| attribute value | *field* | - -A collection of *stored tables* make up a *database*. -*Derived tables* are formed through *query expressions*. - -### Table Definition - -DataJoint introduces a streamlined syntax for defining a stored table. - -Each line in the definition defines an attribute with its name, data type, an optional -default value, and an optional comment in the format: - -```python -name [=default] : type [# comment] -``` - -Primary attributes come first and are separated from the rest of the attributes with -the divider `---`. - -For example, the following code defines the entity set for entities of class `Employee`: - -```python -employee_id : int ---- -ssn = null : int # optional social security number -date_of_birth : date -gender : enum('male', 'female', 'other') -home_address="" : varchar(1000) -primary_phone="" : varchar(12) -``` - -### Data Tiers - -Stored tables are designated into one of four *tiers* indicating how their data -originates. - -| table tier | data origin | -| --- | --- | -| lookup | contents are part of the table definition, defined *a priori* rather than entered externally. Typical stores general facts, parameters, options, *etc.* | -| manual | contents are populated by external mechanisms such as manual entry through web apps or by data ingest scripts | -| imported | contents are populated automatically by pipeline computations accessing data from upstream in the pipeline **and** from external data sources such as raw data stores.| -| computed | contents are populated automatically by pipeline computations accessing data from upstream in the pipeline. | - -### Object Serialization - -### Data Normalization - -A collection of data is considered normalized when organized into a collection of -entity sets, where each entity set represents a well-defined entity class with all its -attributes applicable to each entity in the set and the same primary key identifying - -The normalization procedure often includes splitting data from one table into several -tables, one for each proper entity set. - -### Databases and Schemas - -Stored tables are named and grouped into namespaces called *schemas*. -A collection of schemas make up a *database*. -A *database* has a globally unique address or name. -A *schema* has a unique name within its database. -Within a *connection* to a particular database, a stored table is identified as -`schema.Table`. -A schema typically groups tables that are logically related. - -## Dependencies - -Entity sets can form referential dependencies that express and - -### Diagramming - -## Data integrity - -### Entity integrity - -*Entity integrity* is the guarantee made by the data management process of the 1:1 -mapping between real-world entities and their digital representations. -In practice, entity integrity is ensured when it is made clear - -### Referential integrity - -### Group integrity - -## Data manipulations - -## Data queries - -### Query Operators - -## Pipeline computations diff --git a/docs/src/concepts/teamwork.md b/docs/src/concepts/teamwork.md deleted file mode 100644 index a0a782dde..000000000 --- a/docs/src/concepts/teamwork.md +++ /dev/null @@ -1,97 +0,0 @@ -# Teamwork - -## Data management in a science project - -Science labs organize their projects as a sequence of activities of experiment design, -data acquisition, and processing and analysis. - -![data science in a science lab](../images/data-science-before.png){: style="width:510px; display:block; margin: 0 auto;"} - -
Workflow and dataflow in a common findings-centered approach to data science in a science lab.
- -Many labs lack a uniform data management strategy that would span longitudinally across -the entire project lifecycle as well as laterally across different projects. - -Prior to publishing their findings, the research team may need to publish the data to -support their findings. -Without a data management system, this requires custom repackaging of the data to -conform to the [FAIR principles](https://www.nature.com/articles/sdata201618) for -scientific data management. - -## Data-centric project organization - -DataJoint is designed to support a data-centric approach to large science projects in -which data are viewed as a principal output of the research project and are managed -systematically throughout in a single framework through the entire process. - -This approach requires formulating a general data science plan and upfront investment -for setting up resources and processes and training the teams. -The team uses DataJoint to build data pipelines to support multiple projects. - -![data science in a science lab](../images/data-science-after.png){: style="width:510px; display:block; margin: 0 auto;"} - -
Workflow and dataflow in a data pipeline-centered approach.
- -Data pipelines support project data across their entire lifecycle, including the -following functions - -- experiment design -- animal colony management -- electronic lab book: manual data entry during experiments through graphical user interfaces. -- acquisition from instrumentation in the course of experiments -- ingest from raw acquired data -- computations for data analysis -- visualization of analysis results -- export for sharing and publishing - -Through all these activities, all these data are made accessible to all authorized -participants and distributed computations can be done in parallel without compromising -data integrity. - -## Team roles - -The adoption of a uniform data management framework allows separation of roles and -division of labor among team members, leading to greater efficiency and better scaling. - -![data science in a science lab](../images/data-engineering.png){: style="width:510px; display:block; margin: 0 auto;"} - -
Distinct responsibilities of data science and data engineering.
- -### Scientists - -Design and conduct experiments, collecting data. -They interact with the data pipeline through graphical user interfaces designed by -others. -They understand what analysis is used to test their hypotheses. - -### Data scientists - -Have the domain expertise and select and implement the processing and analysis -methods for experimental data. -Data scientists are in charge of defining and managing the data pipeline using -DataJoint's data model, but they may not know the details of the underlying -architecture. -They interact with the pipeline using client programming interfaces directly from -languages such as MATLAB and Python. - -The bulk of this manual is written for working data scientists, except for System -Administration. - -### Data engineers - -Work with the data scientists to support the data pipeline. -They rely on their understanding of the DataJoint data model to configure and -administer the required IT resources such as database servers, data storage -servers, networks, cloud instances, [Globus](https://globus.org) endpoints, etc. -Data engineers can provide general solutions such as web hosting, data publishing, -interfaces, exports and imports. - -The System Administration section of this tutorial contains materials helpful in -accomplishing these tasks. - -DataJoint is designed to delineate a clean boundary between **data science** and **data -engineering**. -This allows data scientists to use the same uniform data model for data pipelines -backed by a variety of information technologies. -This delineation also enables economies of scale as a single data engineering team can -support a wide spectrum of science projects. diff --git a/docs/src/concepts/terminology.md b/docs/src/concepts/terminology.md deleted file mode 100644 index 0fdc41e96..000000000 --- a/docs/src/concepts/terminology.md +++ /dev/null @@ -1,127 +0,0 @@ - - -# Terminology - -DataJoint introduces a principled data model, which is described in detail in -[Yatsenko et al., 2018](https://arxiv.org/abs/1807.11104). -This data model is a conceptual refinement of the Relational Data Model and also draws -on the Entity-Relationship Model (ERM). - -The Relational Data Model was inspired by the concepts of relations in Set Theory. -When the formal relational data model was formulated, it introduced additional -terminology (e.g. *relation*, *attribute*, *tuple*, *domain*). -Practical programming languages such as SQL do not precisely follow the relational data -model and introduce other terms to approximate relational concepts (e.g. *table*, -*column*, *row*, *datatype*). -Subsequent data models (e.g. ERM) refined the relational data model and introduced -their own terminology to describe analogous concepts (e.g. *entity set*, -*relationship set*, *attribute set*). -As a result, similar concepts may be described using different sets of terminologies, -depending on the context and the speaker's background. - -For example, what is known as a **relation** in the formal relational model is called a -**table** in SQL; the analogous concept in ERM and DataJoint is called an **entity -set**. - -The DataJoint documentation follows the terminology defined in -[Yatsenko et al, 2018](https://arxiv.org/abs/1807.11104), except *entity set* is -replaced with the more colloquial *table* or *query result* in most cases. - -The table below summarizes the terms used for similar concepts across the related data -models. - -Data model terminology -| Relational | ERM | SQL | DataJoint (formal) | This manual | -| -- | -- | -- | -- | -- | -| relation | entity set | table | entity set | table | -| tuple | entity | row | entity | entity | -| domain | value set | datatype | datatype | datatype | -| attribute | attribute | column | attribute | attribute | -| attribute value | attribute value | field value | attribute value | attribute value | -| primary key | primary key | primary key | primary key | primary key | -| foreign key | foreign key | foreign key | foreign key | foreign key | -| schema | schema | schema or database | schema | schema | -| relational expression | data query | `SELECT` statement | query expression | query expression | - -## DataJoint: databases, schemas, packages, and modules - -A **database** is collection of tables on the database server. -DataJoint users do not interact with it directly. - -A **DataJoint schema** is - - - a database on the database server containing tables with data *and* - - a collection of classes (in MATLAB or Python) associated with the database, one - class for each table. - -In MATLAB, the collection of classes is organized as a **package**, i.e. a file folder -starting with a `+`. - -In Python, the collection of classes is any set of classes decorated with the -appropriate `schema` object. -Very commonly classes for tables in one database are organized as a distinct Python -module. -Thus, typical DataJoint projects have one module per database. -However, this organization is up to the user's discretion. - -## Base tables - -**Base tables** are tables stored in the database, and are often referred to simply as -*tables* in DataJoint. -Base tables are distinguished from **derived tables**, which result from relational -[operators](../query/operators.md). - -## Relvars and relation values - -Early versions of the DataJoint documentation referred to the relation objects as -[relvars](https://en.wikipedia.org/wiki/Relvar). -This term emphasizes the fact that relational variables and expressions do not contain -actual data but are rather symbolic representations of data to be retrieved from the -database. -The specific value of a relvar would then be referred to as the **relation value**. -The value of a relvar can change with changes in the state of the database. - -The more recent iteration of the documentation has grown less pedantic and more often -uses the term *table* instead. - -## Metadata - -The vocabulary of DataJoint does not include this term. - -In data science, the term **metadata** commonly means "data about the data" rather than -the data themselves. -For example, metadata could include data sizes, timestamps, data types, indexes, -keywords. - -In contrast, neuroscientists often use the term to refer to conditions and annotations -about experiments. -This distinction arose when such information was stored separately from experimental -recordings, such as in physical notebooks. -Such "metadata" are used to search and to classify the data and are in fact an integral -part of the *actual* data. - -In DataJoint, all data other than blobs can be used in searches and categorization. -These fields may originate from manual annotations, preprocessing, or analyses just as -easily as from recordings or behavioral performance. -Since "metadata" in the neuroscience sense are not distinguished from any other data in -a pipeline, DataJoint avoids the term entirely. -Instead, DataJoint differentiates data into [data tiers](../design/tables/tiers.md). - -## Glossary - -We've taken careful consideration to use consistent terminology. - - - -| Term | Definition | -| --- | --- | -| DAG | directed acyclic graph (DAG) is a set of nodes and connected with a set of directed edges that form no cycles. This means that there is never a path back to a node after passing through it by following the directed edges. Formal workflow management systems represent workflows in the form of DAGs. | -| data pipeline | A sequence of data transformation steps from data sources through multiple intermediate structures. More generally, a data pipeline is a directed acyclic graph. In DataJoint, each step is represented by a table in a relational database. | -| DataJoint | a software framework for database programming directly from matlab and python. Thanks to its support of automated computational dependencies, DataJoint serves as a workflow management system. | -| DataJoint Elements | software modules implementing portions of experiment workflows designed for ease of integration into diverse custom workflows. | -| DataJoint pipeline | the data schemas and transformations underlying a DataJoint workflow. DataJoint allows defining code that specifies both the workflow and the data pipeline, and we have used the words "pipeline" and "workflow" almost interchangeably. | -| DataJoint schema | a software module implementing a portion of an experiment workflow. Includes database table definitions, dependencies, and associated computations. | -| foreign key | a field that is linked to another table's primary key. | -| primary key | the subset of table attributes that uniquely identify each entity in the table. | -| secondray attribute | any field in a table not in the primary key. | -| workflow | a formal representation of the steps for executing an experiment from data collection to analysis. Also the software configured for performing these steps. A typical workflow is composed of tables with inter-dependencies and processes to compute and insert data into the tables. | diff --git a/docs/src/design/alter.md b/docs/src/design/alter.md deleted file mode 100644 index 70ed39341..000000000 --- a/docs/src/design/alter.md +++ /dev/null @@ -1,53 +0,0 @@ -# Altering Populated Pipelines - -Tables can be altered after they have been declared and populated. This is useful when -you want to add new secondary attributes or change the data type of existing attributes. -Users can use the `definition` property to update a table's attributes and then use -`alter` to apply the changes in the database. Currently, `alter` does not support -changes to primary key attributes. - -Let's say we have a table `Student` with the following attributes: - -```python -@schema -class Student(dj.Manual): - definition = """ - student_id: int - --- - first_name: varchar(40) - last_name: varchar(40) - home_address: varchar(100) - """ -``` - -We can modify the table to include a new attribute `email`: - -```python -Student.definition = """ -student_id: int ---- -first_name: varchar(40) -last_name: varchar(40) -home_address: varchar(100) -email: varchar(100) -""" -Student.alter() -``` - -The `alter` method will update the table in the database to include the new attribute -`email` added by the user in the table's `definition` property. - -Similarly, you can modify the data type or length of an existing attribute. For example, -to alter the `home_address` attribute to have a length of 200 characters: - -```python -Student.definition = """ -student_id: int ---- -first_name: varchar(40) -last_name: varchar(40) -home_address: varchar(200) -email: varchar(100) -""" -Student.alter() -``` diff --git a/docs/src/design/diagrams.md b/docs/src/design/diagrams.md deleted file mode 100644 index 826f78926..000000000 --- a/docs/src/design/diagrams.md +++ /dev/null @@ -1,110 +0,0 @@ -# Diagrams - -Diagrams are a great way to visualize the pipeline and understand the flow -of data. DataJoint diagrams are based on **entity relationship diagram** (ERD). -Objects of type `dj.Diagram` allow visualizing portions of the data pipeline in -graphical form. -Tables are depicted as nodes and [dependencies](./tables/dependencies.md) as directed -edges between them. -The `draw` method plots the graph. - -## Diagram notation - -Consider the following diagram - -![mp-diagram](../images/mp-diagram.png){: style="align:center"} - -DataJoint uses the following conventions: - -- Tables are indicated as nodes in the graph. - The corresponding class name is indicated by each node. -- [Data tiers](./tables/tiers.md) are indicated as colors and symbols: - - Lookup=gray rectangle - - Manual=green rectangle - - Imported=blue oval - - Computed=red circle - - Part=black text - The names of [part tables](./tables/master-part.md) are indicated in a smaller font. -- [Dependencies](./tables/dependencies.md) are indicated as edges in the graph and -always directed downward, forming a **directed acyclic graph**. -- Foreign keys contained within the primary key are indicated as solid lines. - This means that the referenced table becomes part of the primary key of the dependent table. -- Foreign keys that are outside the primary key are indicated by dashed lines. -- If the primary key of the dependent table has no other attributes besides the foreign -key, the foreign key is a thick solid line, indicating a 1:{0,1} relationship. -- Foreign keys made without renaming the foreign key attributes are in black whereas -foreign keys that rename the attributes are indicated in red. - -## Diagramming an entire schema - -To plot the Diagram for an entire schema, an Diagram object can be initialized with the -schema object (which is normally used to decorate table objects) - -```python -import datajoint as dj -schema = dj.Schema('my_database') -dj.Diagram(schema).draw() -``` - -or alternatively an object that has the schema object as an attribute, such as the -module defining a schema: - -```python -import datajoint as dj -import seq # import the sequence module defining the seq database -dj.Diagram(seq).draw() # draw the Diagram -``` - -Note that calling the `.draw()` method is not necessary when working in a Jupyter -notebook. -You can simply let the object display itself, for example by entering `dj.Diagram(seq)` -in a notebook cell. -The Diagram will automatically render in the notebook by calling its `_repr_html_` -method. -A Diagram displayed without `.draw()` will be rendered as an SVG, and hovering the -mouse over a table will reveal a compact version of the output of the `.describe()` -method. - -### Initializing with a single table - -A `dj.Diagram` object can be initialized with a single table. - -```python -dj.Diagram(seq.Genome).draw() -``` - -A single node makes a rather boring graph but ERDs can be added together or subtracted -from each other using graph algebra. - -### Adding diagrams together - -However two graphs can be added, resulting in new graph containing the union of the -sets of nodes from the two original graphs. -The corresponding foreign keys will be automatically - -```python -# plot the Diagram with tables Genome and Species from module seq. -(dj.Diagram(seq.Genome) + dj.Diagram(seq.Species)).draw() -``` - -### Expanding diagrams upstream and downstream - -Adding a number to an Diagram object adds nodes downstream in the pipeline while -subtracting a number from Diagram object adds nodes upstream in the pipeline. - -Examples: - -```python -# Plot all the tables directly downstream from `seq.Genome` -(dj.Diagram(seq.Genome)+1).draw() -``` - -```python -# Plot all the tables directly upstream from `seq.Genome` -(dj.Diagram(seq.Genome)-1).draw() -``` - -```python -# Plot the local neighborhood of `seq.Genome` -(dj.Diagram(seq.Genome)+1-1+1-1).draw() -``` diff --git a/docs/src/design/drop.md b/docs/src/design/drop.md deleted file mode 100644 index 35a9ac513..000000000 --- a/docs/src/design/drop.md +++ /dev/null @@ -1,23 +0,0 @@ -# Drop - -The `drop` method completely removes a table from the database, including its -definition. -It also removes all dependent tables, recursively. -DataJoint will first display the tables being dropped and the number of entities in -each before prompting the user for confirmation to proceed. - -The `drop` method is often used during initial design to allow altered table -definitions to take effect. - -```python -# drop the Person table from its schema -Person.drop() -``` - -## Dropping part tables - -A [part table](../design/tables/master-part.md) is usually removed as a consequence of -calling `drop` on its master table. -To enforce this workflow, calling `drop` directly on a part table produces an error. -In some cases, it may be necessary to override this behavior. -To remove a part table without removing its master, use the argument `force=True`. diff --git a/docs/src/design/integrity.md b/docs/src/design/integrity.md deleted file mode 100644 index cb7122755..000000000 --- a/docs/src/design/integrity.md +++ /dev/null @@ -1,218 +0,0 @@ -# Data Integrity - -The term **data integrity** describes guarantees made by the data management process -that prevent errors and corruption in data due to technical failures and human errors -arising in the course of continuous use by multiple agents. -DataJoint pipelines respect the following forms of data integrity: **entity -integrity**, **referential integrity**, and **group integrity** as described in more -detail below. - -## Entity integrity - -In a proper relational design, each table represents a collection of discrete -real-world entities of some kind. -**Entity integrity** is the guarantee made by the data management process that entities -from the real world are reliably and uniquely represented in the database system. -Entity integrity states that the data management process must prevent duplicate -representations or misidentification of entities. -DataJoint enforces entity integrity through the use of -[primary keys](./tables/primary.md). - -Entity integrity breaks down when a process allows data pertaining to the same -real-world entity to be entered into the database system multiple times. -For example, a school database system may use unique ID numbers to distinguish students. -Suppose the system automatically generates an ID number each time a student record is -entered into the database without checking whether a record already exists for that -student. -Such a system violates entity integrity, because the same student may be assigned -multiple ID numbers. -The ID numbers succeed in uniquely identifying each student record but fail to do so -for the actual students. - -Note that a database cannot guarantee or enforce entity integrity by itself. -Entity integrity is a property of the entire data management process as a whole, -including institutional practices and user actions in addition to database -configurations. - -## Referential integrity - -**Referential integrity** is the guarantee made by the data management process that -related data across the database remain present, correctly associated, and mutually -consistent. -Guaranteeing referential integrity means enforcing the constraint that no entity can -exist in the database without all the other entities on which it depends. -Referential integrity cannot exist without entity integrity: references to entity -cannot be validated if the identity of the entity itself is not guaranteed. - -Referential integrity fails when a data management process allows new data to be -entered that refers to other data missing from the database. -For example, assume that each electrophysiology recording must refer to the mouse -subject used during data collection. -Perhaps an experimenter attempts to insert ephys data into the database that refers to -a nonexistent mouse, due to a misspelling. -A system guaranteeing referential integrity, such as DataJoint, will refuse the -erroneous data. - -Enforcement of referential integrity does not stop with data ingest. -[Deleting](../manipulation/delete.md) data in DataJoint also deletes any dependent -downstream data. -Such cascading deletions are necessary to maintain referential integrity. -Consider the deletion of a mouse subject without the deletion of the experimental -sessions involving that mouse. -A database that allows such deletion will break referential integrity, as the -experimental sessions for the removed mouse depend on missing data. -Any data management process that allows data to be deleted with no consideration of -dependent data cannot maintain referential integrity. - -[Updating](../manipulation/update.md) data already present in a database system also -jeopardizes referential integrity. -For this reason, the DataJoint workflow does not include updates to entities once they -have been ingested into a pipeline. -Allowing updates to upstream entities would break the referential integrity of any -dependent data downstream. -For example, permitting a user to change the name of a mouse subject would invalidate -any experimental sessions that used that mouse, presuming the mouse name was part of -the primary key. -The proper way to change data in DataJoint is to delete the existing entities and to -insert corrected ones, preserving referential integrity. - -## Group integrity - -**Group integrity** denotes the guarantee made by the data management process that -entities composed of multiple parts always appear in their complete form. -Group integrity in DataJoint is formalized through -[master-part](./tables/master-part.md) relationships. -The master-part relationship has important implications for dependencies, because a -downstream entity depending on a master entity set may be considered to depend on the -parts as well. - -## Relationships - -In DataJoint, the term **relationship** is used rather generally to describe the -effects of particular configurations of [dependencies](./tables/dependencies.md) -between multiple entity sets. -It is often useful to classify relationships as one-to-one, many-to-one, one-to-many, -and many-to-many. - -In a **one-to-one relationship**, each entity in a downstream table has exactly one -corresponding entity in the upstream table. -A dependency of an entity set containing the death dates of mice on an entity set -describing the mice themselves would obviously be a one-to-one relationship, as in the -example below. - -```python -@schema -class Mouse(dj.Manual): -definition = """ -mouse_name : varchar(64) ---- -mouse_dob : datetime -""" - -@schema -class MouseDeath(dj.Manual): -definition = """ --> Mouse ---- -death_date : datetime -""" -``` - -![doc_1-1](../images/doc_1-1.png){: style="align:center"} - -In a **one-to-many relationship**, multiple entities in a downstream table may depend -on the same entity in the upstream table. -The example below shows a table containing individual channel data from multi-channel -recordings, representing a one-to-many relationship. - -```python -@schema -class EEGRecording(dj.Manual): -definition = """ --> Session -eeg_recording_id : int ---- -eeg_system : varchar(64) -num_channels : int -""" - -@schema -class ChannelData(dj.Imported): -definition = """ --> EEGRecording -channel_idx : int ---- -channel_data : longblob -""" -``` -![doc_1-many](../images/doc_1-many.png){: style="align:center"} - -In a **many-to-one relationship**, each entity in a table is associated with multiple -entities from another table. -Many-to-one relationships between two tables are usually established using a separate -membership table. -The example below includes a table of mouse subjects, a table of subject groups, and a -membership [part table](./tables/master-part.md) listing the subjects in each group. -A many-to-one relationship exists between the `Mouse` table and the `SubjectGroup` -table, with is expressed through entities in `GroupMember`. - -```python -@schema -class Mouse(dj.Manual): -definition = """ -mouse_name : varchar(64) ---- -mouse_dob : datetime -""" - -@schema -class SubjectGroup(dj.Manual): -definition = """ -group_number : int ---- -group_name : varchar(64) -""" - -class GroupMember(dj.Part): - definition = """ - -> master - -> Mouse - """ -``` - -![doc_many-1](../images/doc_many-1.png){: style="align:center"} - -In a **many-to-many relationship**, multiple entities in one table may each relate to -multiple entities in another upstream table. -Many-to-many relationships between two tables are usually established using a separate -association table. -Each entity in the association table links one entity from each of the two upstream -tables it depends on. -The below example of a many-to-many relationship contains a table of recording -modalities and a table of multimodal recording sessions. -Entities in a third table represent the modes used for each session. - -```python -@schema -class RecordingModality(dj.Lookup): -definition = """ -modality : varchar(64) -""" - -@schema -class MultimodalSession(dj.Manual): -definition = """ --> Session -modes : int -""" -class SessionMode(dj.Part): - definition = """ - -> master - -> RecordingModality - """ -``` - -![doc_many-many](../images/doc_many-many.png){: style="align:center"} - -The types of relationships between entity sets are expressed in the -[Diagram](diagrams.md) of a schema. diff --git a/docs/src/design/normalization.md b/docs/src/design/normalization.md deleted file mode 100644 index 000028396..000000000 --- a/docs/src/design/normalization.md +++ /dev/null @@ -1,117 +0,0 @@ -# Entity Normalization - -DataJoint uses a uniform way of representing any data. -It does so in the form of **entity sets**, unordered collections of entities of the -same type. -The term **entity normalization** describes the commitment to represent all data as -well-formed entity sets. -Entity normalization is a conceptual refinement of the -[relational data model](../concepts/data-model.md) and is the central principle of the -DataJoint model ([Yatsenko et al., 2018](https://arxiv.org/abs/1807.11104)). -Entity normalization leads to clear and logical database designs and to easily -comprehensible data queries. - -Entity sets are a type of **relation** -(from the [relational data model](../concepts/data-model.md)) and are often visualized -as **tables**. -Hence the terms **relation**, **entity set**, and **table** can be used interchangeably -when entity normalization is assumed. - -## Criteria of a well-formed entity set - -1. All elements of an entity set belong to the same well-defined and readily identified -**entity type** from the model world. -2. All attributes of an entity set are applicable directly to each of its elements, -although some attribute values may be missing (set to null). -3. All elements of an entity set must be distinguishable form each other by the same -primary key. -4. Primary key attribute values cannot be missing, i.e. set to null. -5. All elements of an entity set participate in the same types of relationships with -other entity sets. - -## Entity normalization in schema design - -Entity normalization applies to schema design in that the designer is responsible for -the identification of the essential entity types in their model world and of the -dependencies among the entity types. - -The term entity normalization may also apply to a procedure for refactoring a schema -design that does not meet the above criteria into one that does. -In some cases, this may require breaking up some entity sets into multiple entity sets, -which may cause some entities to be represented across multiple entity sets. -In other cases, this may require converting attributes into their own entity sets. -Technically speaking, entity normalization entails compliance with the -[Boyce-Codd normal form](https://en.wikipedia.org/wiki/Boyce%E2%80%93Codd_normal_form) -while lacking the representational power for the applicability of more complex normal -forms ([Kent, 1983](https://dl.acm.org/citation.cfm?id=358054)). -Adherence to entity normalization prevents redundancies in storage and data -manipulation anomalies. -The same criteria originally motivated the formulation of the classical relational -normal forms. - -## Entity normalization in data queries - -Entity normalization applies to data queries as well. -DataJoint's [query operators](../query/operators.md) are designed to preserve the -entity normalization of their inputs. -For example, the outputs of operators [restriction](../query/restrict.md), -[proj](../query/project.md), and [aggr](../query/aggregation.md) retain the same entity -type as the (first) input. -The [join](../query/join.md) operator produces a new entity type comprising the pairing -of the entity types of its inputs. -[Universal sets](../query/universals.md) explicitly introduce virtual entity sets when -necessary to accomplish a query. - -## Examples of poor normalization - -Design choices lacking entity normalization may lead to data inconsistencies or -anomalies. -Below are several examples of poorly normalized designs and their normalized -alternatives. - -### Indirect attributes - -All attributes should apply to the entity itself. -Avoid attributes that actually apply to one of the entity's other attributes. -For example, consider the table `Author` with attributes `author_name`, `institution`, -and `institution_address`. -The attribute `institution_address` should really be held in a separate `Institution` -table that `Author` depends on. - -### Repeated attributes - -Avoid tables with repeated attributes of the same category. -A better solution is to create a separate table that depends on the first (often a -[part table](../design/tables/master-part.md)), with multiple individual entities -rather than repeated attributes. -For example, consider the table `Protocol` that includes the attributes `equipment1`, -`equipment2`, and `equipment3`. -A better design would be to create a `ProtocolEquipment` table that links each entity -in `Protocol` with multiple entities in `Equipment` through -[dependencies](../design/tables/dependencies.md). - -### Attributes that do not apply to all entities - -All attributes should be relevant to every entity in a table. -Attributes that apply only to a subset of entities in a table likely belong in a -separate table containing only that subset of entities. -For example, a table `Protocol` should include the attribute `stimulus` only if all -experiment protocols include stimulation. -If the not all entities in `Protocol` involve stimulation, then the `stimulus` -attribute should be moved to a part table that has `Protocol` as its master. -Only protocols using stimulation will have an entry in this part table. - -### Transient attributes - -Attributes should be relevant to all entities in a table at all times. -Attributes that do not apply to all entities should be moved to another dependent table -containing only the appropriate entities. -This principle also applies to attributes that have not yet become meaningful for some -entities or that will not remain meaningful indefinitely. -For example, consider the table `Mouse` with attributes `birth_date` and `death_date`, -where `death_date` is set to `NULL` for living mice. -Since the `death_date` attribute is not meaningful for mice that are still living, -the proper design would include a separate table `DeceasedMouse` that depends on -`Mouse`. -`DeceasedMouse` would only contain entities for dead mice, which improves integrity and -averts the need for [updates](../manipulation/update.md). diff --git a/docs/src/design/recall.md b/docs/src/design/recall.md deleted file mode 100644 index 56226cabd..000000000 --- a/docs/src/design/recall.md +++ /dev/null @@ -1,207 +0,0 @@ -# Work with Existing Pipelines - -## Loading Classes - -This section describes how to work with database schemas without access to the -original code that generated the schema. These situations often arise when the -database is created by another user who has not shared the generating code yet -or when the database schema is created from a programming language other than -Python. - -```python -import datajoint as dj -``` - -### Working with schemas and their modules - -Typically a DataJoint schema is created as a dedicated Python module. This -module defines a schema object that is used to link classes declared in the -module to tables in the database schema. As an example, examine the university -module: [university.py](https://github.com/datajoint-company/db-programming-with-datajoint/blob/master/notebooks/university.py). - -You may then import the module to interact with its tables: - -```python -import university as uni -dj.Diagram(uni) -``` - -![query object preview](../images/virtual-module-ERD.svg){: style="align:center"} - -Note that dj.Diagram can extract the diagram from a schema object or from a -Python module containing its schema object, lending further support to the -convention of one-to-one correspondence between database schemas and Python -modules in a DataJoint project: - -`dj.Diagram(uni)` - -is equivalent to - -`dj.Diagram(uni.schema)` - -```python -# students without majors -uni.Student - uni.StudentMajor -``` - -![query object preview](../images/StudentTable.png){: style="align:center"} - -### Spawning missing classes - -Now imagine that you do not have access to `university.py` or you do not have -its latest version. You can still connect to the database schema but you will -not have classes declared to interact with it. - -So let's start over in this scenario. - -You may use the `dj.list_schemas` function (new in DataJoint 0.12.0) to -list the names of database schemas available to you. - -```python -import datajoint as dj -dj.list_schemas() -``` - -```text -*['dimitri_alter','dimitri_attach','dimitri_blob','dimitri_blobs', -'dimitri_nphoton','dimitri_schema','dimitri_university','dimitri_uuid', -'university']* -``` - -Just as with a new schema, we start by creating a schema object to connect to -the chosen database schema: - -```python -schema = dj.Schema('dimitri_university') -``` - -If the schema already exists, `dj.Schema` is initialized as usual and you may plot -the schema diagram. But instead of seeing class names, you will see the raw -table names as they appear in the database. - -```python -# let's plot its diagram -dj.Diagram(schema) -``` - -![query object preview](../images/dimitri-ERD.svg){: style="align:center"} - -You may view the diagram but, at this point, there is no way to interact with -these tables. A similar situation arises when another developer has added new -tables to the schema but has not yet shared the updated module code with you. -Then the diagram will show a mixture of class names and database table names. - -Now you may use the `spawn_missing_classes` method to spawn classes into -the local namespace for any tables missing their classes: - -```python -schema.spawn_missing_classes() -dj.Diagram(schema) -``` - -![query object preview](../images/spawned-classes-ERD.svg){: style="align:center"} - -Now you may interact with these tables as if they were declared right here in -this namespace: - -```python -# students without majors -Student - StudentMajor -``` - -![query object preview](../images/StudentTable.png){: style="align:center"} - -### Creating a virtual module - -Virtual modules provide a way to access the classes corresponding to tables in a -DataJoint schema without having to create local files. - -`spawn_missing_classes` creates the new classes in the local namespace. -However, it is often more convenient to import a schema with its Python module, -equivalent to the Python command: - -```python -import university as uni -``` - -We can mimic this import without having access to `university.py` using the -`VirtualModule` class object: - -```python -import datajoint as dj - -uni = dj.VirtualModule(module_name='university.py', schema_name='dimitri_university') -``` - -Now `uni` behaves as an imported module complete with the schema object and all -the table classes. - -```python -dj.Diagram(uni) -``` - -![query object preview](../images/added-example-ERD.svg){: style="align:center"} - -```python -uni.Student - uni.StudentMajor -``` - -![query object preview](../images/StudentTable.png){: style="align:center"} - -`dj.VirtualModule` takes required arguments - -- `module_name`: displayed module name. - -- `schema_name`: name of the database in MySQL. - -And `dj.VirtualModule` takes optional arguments. - -First, `create_schema=False` assures that an error is raised when the schema -does not already exist. Set it to `True` if you want to create an empty schema. - -```python -dj.VirtualModule('what', 'nonexistent') -``` - -Returns - -```python ---------------------------------------------------------------------------- -DataJointError Traceback (most recent call last) -. -. -. -DataJointError: Database named `nonexistent` was not defined. Set argument create_schema=True to create it. -``` - -The other optional argument, `create_tables=False` is passed to the schema -object. It prevents the use of the schema object of the virtual module for -creating new tables in the existing schema. This is a precautionary measure -since virtual modules are often used for completed schemas. You may set this -argument to `True` if you wish to add new tables to the existing schema. A -more common approach in this scenario would be to create a new schema object and -to use the `spawn_missing_classes` function to make the classes available. - -However, you if do decide to create new tables in an existing tables using the -virtual module, you may do so by using the schema object from the module as the -decorator for declaring new tables: - -```python -uni = dj.VirtualModule('university.py', 'dimitri_university', create_tables=True) -``` - -```python -@uni.schema -class Example(dj.Manual): - definition = """ - -> uni.Student - --- - example : varchar(255) - """ -``` - -```python -dj.Diagram(uni) -``` - -![query object preview](../images/added-example-ERD.svg){: style="align:center"} diff --git a/docs/src/design/schema.md b/docs/src/design/schema.md deleted file mode 100644 index 94bf6cdcc..000000000 --- a/docs/src/design/schema.md +++ /dev/null @@ -1,49 +0,0 @@ -# Schema Creation - -## Schemas - -On the database server, related tables are grouped into a named collection called a **schema**. -This grouping organizes the data and allows control of user access. -A database server may contain multiple schemas each containing a subset of the tables. -A single pipeline may comprise multiple schemas. -Tables are defined within a schema, so a schema must be created before the creation of -any tables. - -By convention, the `datajoint` package is imported as `dj`. - The documentation refers to the package as `dj` throughout. - -Create a new schema using the `dj.Schema` class object: - -```python -import datajoint as dj -schema = dj.Schema('alice_experiment') -``` - -This statement creates the database schema `alice_experiment` on the server. - -The returned object `schema` will then serve as a decorator for DataJoint classes, as -described in [table declaration syntax](./tables/declare.md). - -It is a common practice to have a separate Python module for each schema. -Therefore, each such module has only one `dj.Schema` object defined and is usually -named `schema`. - -The `dj.Schema` constructor can take a number of optional parameters after the schema -name. - -- `context` - Dictionary for looking up foreign key references. - Defaults to `None` to use local context. -- `connection` - Specifies the DataJoint connection object. - Defaults to `dj.conn()`. -- `create_schema` - When `False`, the schema object will not create a schema on the -database and will raise an error if one does not already exist. - Defaults to `True`. -- `create_tables` - When `False`, the schema object will not create tables on the -database and will raise errors when accessing missing tables. - Defaults to `True`. - -## Working with existing data - -See the chapter [recall](recall.md) for how to work with data in -existing pipelines, including accessing a pipeline from one language when the pipeline -was developed using another. diff --git a/docs/src/design/tables/attach.md b/docs/src/design/tables/attach.md deleted file mode 100644 index c4950ffdf..000000000 --- a/docs/src/design/tables/attach.md +++ /dev/null @@ -1,67 +0,0 @@ -# External Data - -## File Attachment Datatype - -### Configuration & Usage - -Corresponding to issue -[#480](https://github.com/datajoint/datajoint-python/issues/480), -the `attach` attribute type allows users to `attach` files into DataJoint -schemas as DataJoint-managed files. This is in contrast to traditional `blobs` -which are encodings of programming language data structures such as arrays. - -The functionality is modeled after email attachments, where users `attach` -a file along with a message and message recipients have access to a -copy of that file upon retrieval of the message. - -For DataJoint `attach` attributes, DataJoint will copy the input -file into a DataJoint store, hash the file contents, and track -the input file name. Subsequent `fetch` operations will transfer a -copy of the file to the local directory of the Python process and -return a pointer to it's location for subsequent client usage. This -allows arbitrary files to be `uploaded` or `attached` to a DataJoint -schema for later use in processing. File integrity is preserved by -checksum comparison against the attachment data and verifying the contents -during retrieval. - -For example, given a `localattach` store: - -```python -dj.config['stores'] = { - 'localattach': { - 'protocol': 'file', - 'location': '/data/attach' - } -} -``` - -A `ScanAttachment` table can be created: - -```python -@schema -class ScanAttachment(dj.Manual): - definition = """ - -> Session - --- - scan_image: attach@localattach # attached image scans - """ -``` - -Files can be added using an insert pointing to the source file: - -```python ->>> ScanAttachment.insert1((0, '/input/image0.tif')) -``` - -And then retrieved to the current directory using `fetch`: - -```python ->>> s0 = (ScanAttachment & {'session_id': 0}).fetch1() ->>> s0 -{'session_id': 0, 'scan_image': './image0.tif'} ->>> fh = open(s0['scan_image'], 'rb') ->>> fh -<_io.BufferedReader name='./image0.tif') -``` - - diff --git a/docs/src/design/tables/attributes.md b/docs/src/design/tables/attributes.md deleted file mode 100644 index 0c2e7a8f9..000000000 --- a/docs/src/design/tables/attributes.md +++ /dev/null @@ -1,88 +0,0 @@ -# Datatypes - -DataJoint supports the following datatypes. -To conserve database resources, use the smallest and most restrictive datatype -sufficient for your data. -This also ensures that only valid data are entered into the pipeline. - -## Most common datatypes - -- `tinyint`: an 8-bit integer number, ranging from -128 to 127. -- `tinyint unsigned`: an 8-bit positive integer number, ranging from 0 to 255. -- `smallint`: a 16-bit integer number, ranging from -32,768 to 32,767. -- `smallint unsigned`: a 16-bit positive integer, ranging from 0 to 65,535. -- `int`: a 32-bit integer number, ranging from -2,147,483,648 to 2,147,483,647. -- `int unsigned`: a 32-bit positive integer, ranging from 0 to 4,294,967,295. -- `enum`: one of several explicitly enumerated values specified as strings. - Use this datatype instead of text strings to avoid spelling variations and to save - storage space. - For example, the datatype for an anesthesia attribute could be - `enum("urethane", "isoflurane", "fentanyl")`. - Do not use enums in primary keys due to the difficulty of changing their definitions - consistently in multiple tables. - -- `date`: date as `'YYYY-MM-DD'`. -- `time`: time as `'HH:MM:SS'`. -- `datetime`: Date and time to the second as `'YYYY-MM-DD HH:MM:SS'` -- `timestamp`: Date and time to the second as `'YYYY-MM-DD HH:MM:SS'`. - The default value may be set to `CURRENT_TIMESTAMP`. - Unlike `datetime`, a `timestamp` value will be adjusted to the local time zone. - -- `char(N)`: a character string up to *N* characters (but always takes the entire *N* -bytes to store). -- `varchar(N)`: a text string of arbitrary length up to *N* characters that takes -*M+1* or *M+2* bytes of storage, where *M* is the actual length of each stored string. -- `float`: a single-precision floating-point number. - Takes 4 bytes. - Single precision is sufficient for many measurements. - -- `double`: a double-precision floating-point number. - Takes 8 bytes. - Because equality comparisons are error-prone, neither `float` nor `double` should be - used in primary keys. -- `decimal(N,F)`: a fixed-point number with *N* total decimal digits and *F* -fractional digits. - This datatype is well suited to represent numbers whose magnitude is well defined - and does not warrant the use of floating-point representation or requires precise - decimal representations (e.g. dollars and cents). - Because of its well-defined precision, `decimal` values can be used in equality - comparison and be included in primary keys. - -- `longblob`: arbitrary numeric array (e.g. matrix, image, structure), up to 4 -[GiB](http://en.wikipedia.org/wiki/Gibibyte) in size. - Numeric arrays are compatible between MATLAB and Python (NumPy). - The `longblob` and other `blob` datatypes can be configured to store data - [externally](../../sysadmin/external-store.md) by using the `blob@store` syntax. - -## Less common (but supported) datatypes - -- `decimal(N,F) unsigned`: same as `decimal`, but limited to nonnegative values. -- `mediumint` a 24-bit integer number, ranging from -8,388,608 to 8,388,607. -- `mediumint unsigned`: a 24-bit positive integer, ranging from 0 to 16,777,216. -- `mediumblob`: arbitrary numeric array, up to 16 -[MiB](http://en.wikipedia.org/wiki/Mibibyte) -- `blob`: arbitrary numeric array, up to 64 -[KiB](http://en.wikipedia.org/wiki/Kibibyte) -- `tinyblob`: arbitrary numeric array, up to 256 bytes (actually smaller due to header -info). - -## Special DataJoint-only datatypes - -These types abstract certain kinds of non-database data to facilitate use -together with DataJoint. - -- `attach`: a [file attachment](attach.md) similar to email attachments facillitating -sending/receiving an opaque data file to/from a DataJoint pipeline. - -- `filepath@store`: a [filepath](filepath.md) used to link non-DataJoint managed files -into a DataJoint pipeline. - -## Datatypes not (yet) supported - -- `binary` -- `text` -- `longtext` -- `bit` - -For additional information about these datatypes, see -http://dev.mysql.com/doc/refman/5.6/en/data-types.html diff --git a/docs/src/design/tables/blobs.md b/docs/src/design/tables/blobs.md deleted file mode 100644 index 9f73d54d4..000000000 --- a/docs/src/design/tables/blobs.md +++ /dev/null @@ -1,26 +0,0 @@ -# Blobs - -DataJoint provides functionality for serializing and deserializing complex data types -into binary blobs for efficient storage and compatibility with MATLAB's mYm -serialization. This includes support for: - -+ Basic Python data types (e.g., integers, floats, strings, dictionaries). -+ NumPy arrays and scalars. -+ Specialized data types like UUIDs, decimals, and datetime objects. - -## Serialization and Deserialization Process - -Serialization converts Python objects into a binary representation for efficient storage -within the database. Deserialization converts the binary representation back into the -original Python object. - -Blobs over 1 KiB are compressed using the zlib library to reduce storage requirements. - -## Supported Data Types - -DataJoint supports the following data types for serialization: - -+ Scalars: Integers, floats, booleans, strings. -+ Collections: Lists, tuples, sets, dictionaries. -+ NumPy: Arrays, structured arrays, and scalars. -+ Custom Types: UUIDs, decimals, datetime objects, MATLAB cell and struct arrays. diff --git a/docs/src/design/tables/customtype.md b/docs/src/design/tables/customtype.md deleted file mode 100644 index aad194ff5..000000000 --- a/docs/src/design/tables/customtype.md +++ /dev/null @@ -1,80 +0,0 @@ -# Custom Types - -In modern scientific research, data pipelines often involve complex workflows that -generate diverse data types. From high-dimensional imaging data to machine learning -models, these data types frequently exceed the basic representations supported by -traditional relational databases. For example: - -+ A lab working on neural connectivity might use graph objects to represent brain - networks. -+ Researchers processing raw imaging data might store custom objects for pre-processing - configurations. -+ Computational biologists might store fitted machine learning models or parameter - objects for downstream predictions. - -To handle these diverse needs, DataJoint provides the `dj.AttributeAdapter` method. It -enables researchers to store and retrieve complex, non-standard data types—like Python -objects or data structures—in a relational database while maintaining the -reproducibility, modularity, and query capabilities required for scientific workflows. - -## Uses in Scientific Research - -Imagine a neuroscience lab studying neural connectivity. Researchers might generate -graphs (e.g., networkx.Graph) to represent connections between brain regions, where: - -+ Nodes are brain regions. -+ Edges represent connections weighted by signal strength or another metric. - -Storing these graph objects in a database alongside other experimental data (e.g., -subject metadata, imaging parameters) ensures: - -1. Centralized Data Management: All experimental data and analysis results are stored - together for easy access and querying. -2. Reproducibility: The exact graph objects used in analysis can be retrieved later for - validation or further exploration. -3. Scalability: Graph data can be integrated into workflows for larger datasets or - across experiments. - -However, since graphs are not natively supported by relational databases, here’s where -`dj.AttributeAdapter` becomes essential. It allows researchers to define custom logic for -serializing graphs (e.g., as edge lists) and deserializing them back into Python -objects, bridging the gap between advanced data types and the database. - -### Example: Storing Graphs in DataJoint - -To store a networkx.Graph object in a DataJoint table, researchers can define a custom -attribute type in a datajoint table class: - -```python -import datajoint as dj - -class GraphAdapter(dj.AttributeAdapter): - - attribute_type = 'longblob' # this is how the attribute will be declared - - def put(self, obj): - # convert the nx.Graph object into an edge list - assert isinstance(obj, nx.Graph) - return list(obj.edges) - - def get(self, value): - # convert edge list back into an nx.Graph - return nx.Graph(value) - - -# instantiate for use as a datajoint type -graph = GraphAdapter() - - -# define a table with a graph attribute -schema = dj.schema('test_graphs') - - -@schema -class Connectivity(dj.Manual): - definition = """ - conn_id : int - --- - conn_graph = null : # a networkx.Graph object - """ -``` diff --git a/docs/src/design/tables/declare.md b/docs/src/design/tables/declare.md deleted file mode 100644 index d4fb070a2..000000000 --- a/docs/src/design/tables/declare.md +++ /dev/null @@ -1,242 +0,0 @@ -# Declaration Syntax - -## Creating Tables - -### Classes represent tables - -To make it easy to work with tables in MATLAB and Python, DataJoint programs create a -separate class for each table. -Computer programmers refer to this concept as -[object-relational mapping](https://en.wikipedia.org/wiki/Object-relational_mapping). -For example, the class `experiment.Subject` in the DataJoint client language may -correspond to the table called `subject` on the database server. -Users never need to see the database directly; they only interact with data in the -database by creating and interacting with DataJoint classes. - -#### Data tiers - -The table class must inherit from one of the following superclasses to indicate its -data tier: `dj.Lookup`, `dj.Manual`, `dj.Imported`, `dj.Computed`, or `dj.Part`. -See [tiers](tiers.md) and [master-part](./master-part.md). - -### Defining a table - -To define a DataJoint table in Python: - -1. Define a class inheriting from the appropriate DataJoint class: `dj.Lookup`, -`dj.Manual`, `dj.Imported` or `dj.Computed`. - -2. Decorate the class with the schema object (see [schema](../schema.md)) - -3. Define the class property `definition` to define the table heading. - -For example, the following code defines the table `Person`: - -```python -import datajoint as dj -schema = dj.Schema('alice_experiment') - -@schema -class Person(dj.Manual): - definition = ''' - username : varchar(20) # unique user name - --- - first_name : varchar(30) - last_name : varchar(30) - ''' -``` - -The `@schema` decorator uses the class name and the data tier to check whether an -appropriate table exists on the database. -If a table does not already exist, the decorator creates one on the database using the -definition property. -The decorator attaches the information about the table to the class, and then returns -the class. - -The class will become usable after you define the `definition` property as described in -[Table definition](#table-definition). - -#### DataJoint classes in Python - -DataJoint for Python is implemented through the use of classes providing access to the -actual tables stored on the database. -Since only a single table exists on the database for any class, interactions with all -instances of the class are equivalent. -As such, most methods can be called on the classes themselves rather than on an object, -for convenience. -Whether calling a DataJoint method on a class or on an instance, the result will only -depend on or apply to the corresponding table. -All of the basic functionality of DataJoint is built to operate on the classes -themselves, even when called on an instance. -For example, calling `Person.insert(...)` (on the class) and `Person.insert(...)` (on -an instance) both have the identical effect of inserting data into the table on the -database server. -DataJoint does not prevent a user from working with instances, but the workflow is -complete without the need for instantiation. -It is up to the user whether to implement additional functionality as class methods or -methods called on instances. - -### Valid class names - -Note that in both MATLAB and Python, the class names must follow the CamelCase compound -word notation: - -- start with a capital letter and -- contain only alphanumerical characters (no underscores). - -Examples of valid class names: - -`TwoPhotonScan`, `Scan2P`, `Ephys`, `MembraneVoltage` - -Invalid class names: - -`Two_photon_Scan`, `twoPhotonScan`, `2PhotonScan`, `membranePotential`, `membrane_potential` - -## Table Definition - -DataJoint models data as sets of **entities** with shared **attributes**, often -visualized as tables with rows and columns. -Each row represents a single entity and the values of all of its attributes. -Each column represents a single attribute with a name and a datatype, applicable to -entity in the table. -Unlike rows in a spreadsheet, entities in DataJoint don't have names or numbers: they -can only be identified by the values of their attributes. -Defining a table means defining the names and datatypes of the attributes as well as -the constraints to be applied to those attributes. -Both MATLAB and Python use the same syntax define tables. - -For example, the following code in defines the table `User`, that contains users of the -database: - -The table definition is contained in the `definition` property of the class. - -```python -@schema -class User(dj.Manual): - definition = """ - # database users - username : varchar(20) # unique user name - --- - first_name : varchar(30) - last_name : varchar(30) - role : enum('admin', 'contributor', 'viewer') - """ -``` - -This defines the class `User` that creates the table in the database and provides all -its data manipulation functionality. - -### Table creation on the database server - -Users do not need to do anything special to have a table created in the database. -Tables are created at the time of class definition. -In fact, table creation on the database is one of the jobs performed by the decorator -`@schema` of the class. - -### Changing the definition of an existing table - -Once the table is created in the database, the definition string has no further effect. -In other words, changing the definition string in the class of an existing table will -not actually update the table definition. -To change the table definition, one must first [drop](../drop.md) the existing table. -This means that all the data will be lost, and the new definition will be applied to -create the new empty table. - -Therefore, in the initial phases of designing a DataJoint pipeline, it is common to -experiment with variations of the design before populating it with substantial amounts -of data. - -It is possible to modify a table without dropping it. -This topic is covered separately. - -### Reverse-engineering the table definition - -DataJoint objects provide the `describe` method, which displays the table definition -used to define the table when it was created in the database. -This definition may differ from the definition string of the class if the definition -string has been edited after creation of the table. - -Examples - -```python -s = lab.User.describe() -``` - -## Definition Syntax - -The table definition consists of one or more lines. -Each line can be one of the following: - -- The optional first line starting with a `#` provides a description of the table's purpose. - It may also be thought of as the table's long title. -- A new attribute definition in any of the following forms (see -[Attributes](./attributes.md) for valid datatypes): - ``name : datatype`` - ``name : datatype # comment`` - ``name = default : datatype`` - ``name = default : datatype # comment`` -- The divider `---` (at least three hyphens) separating primary key attributes above -from secondary attributes below. -- A foreign key in the format `-> ReferencedTable`. - (See [Dependencies](dependencies.md).) - -For example, the table for Persons may have the following definition: - -```python -# Persons in the lab -username : varchar(16) # username in the database ---- -full_name : varchar(255) -start_date : date # date when joined the lab -``` - -This will define the table with attributes `username`, `full_name`, and `start_date`, -in which `username` is the [primary key](primary.md). - -### Attribute names - -Attribute names must be in lowercase and must start with a letter. -They can only contain alphanumerical characters and underscores. -The attribute name cannot exceed 64 characters. - -Valid attribute names - `first_name`, `two_photon_scan`, `scan_2p`, `two_photon_scan` - -Invalid attribute names - `firstName`, `first name`, `2photon_scan`, `two-photon_scan`, `TwoPhotonScan` - -Ideally, attribute names should be unique across all tables that are likely to be used -in queries together. -For example, tables often have attributes representing the start times of sessions, -recordings, etc. -Such attributes must be uniquely named in each table, such as `session_start_time` or -`recording_start_time`. - -### Default values - -Secondary attributes can be given default values. -A default value will be used for an attribute if no other value is given at the time -the entity is [inserted](../../manipulation/insert.md) into the table. -Generally, default values are numerical values or character strings. -Default values for dates must be given as strings as well, contained within quotes -(with the exception of `CURRENT_TIMESTAMP`). -Note that default values can only be used when inserting as a mapping. -Primary key attributes cannot have default values (with the exceptions of -`auto_increment` and `CURRENT_TIMESTAMP` attributes; see [primary-key](primary.md)). - -An attribute with a default value of `NULL` is called a **nullable attribute**. -A nullable attribute can be thought of as applying to all entities in a table but -having an optional *value* that may be absent in some entities. -Nullable attributes should *not* be used to indicate that an attribute is inapplicable -to some entities in a table (see [normalization](../normalization.md)). -Nullable attributes should be used sparingly to indicate optional rather than -inapplicable attributes that still apply to all entities in the table. -`NULL` is a special literal value and does not need to be enclosed in quotes. - -Here are some examples of attributes with default values: - -```python -failures = 0 : int -due_date = "2020-05-31" : date -additional_comments = NULL : varchar(256) -``` diff --git a/docs/src/design/tables/dependencies.md b/docs/src/design/tables/dependencies.md deleted file mode 100644 index e06278ee8..000000000 --- a/docs/src/design/tables/dependencies.md +++ /dev/null @@ -1,241 +0,0 @@ -# Dependencies - -## Understanding dependencies - -A schema contains collections of tables of related data. -Accordingly, entities in one table often derive some of their meaning or context from -entities in other tables. -A **foreign key** defines a **dependency** of entities in one table on entities in -another within a schema. -In more complex designs, dependencies can even exist between entities in tables from -different schemas. -Dependencies play a functional role in DataJoint and do not simply label the structure -of a pipeline. -Dependencies provide entities in one table with access to data in another table and -establish certain constraints on entities containing a foreign key. - -A DataJoint pipeline, including the dependency relationships established by foreign -keys, can be visualized as a graph with nodes and edges. -The diagram of such a graph is called the **entity relationship diagram** or -[Diagram](../diagrams.md). -The nodes of the graph are tables and the edges connecting them are foreign keys. -The edges are directed and the overall graph is a **directed acyclic graph**, a graph -with no loops. - -For example, the Diagram below is the pipeline for multipatching experiments - -![mp-diagram](../../images/mp-diagram.png){: style="align:center"} - -The graph defines the direction of the workflow. -The tables at the top of the flow need to be populated first, followed by those tables -one step below and so forth until the last table is populated at the bottom of the -pipeline. -The top of the pipeline tends to be dominated by lookup tables (gray stars) and manual -tables (green squares). -The middle has many imported tables (blue triangles), and the bottom has computed -tables (red stars). - -## Defining a dependency - -Foreign keys are defined with arrows `->` in the [table definition](declare.md), -pointing to another table. - -A foreign key may be defined as part of the [primary-key](primary.md). - -In the Diagram, foreign keys from the primary key are shown as solid lines. -This means that the primary key of the referenced table becomes part of the primary key -of the new table. -A foreign key outside the primary key is indicated by dashed line in the ERD. - -For example, the following definition for the table `mp.Slice` has three foreign keys, -including one within the primary key. - -```python -# brain slice --> mp.Subject -slice_id : smallint # slice number within subject ---- --> mp.BrainRegion --> mp.Plane -slice_date : date # date of the slicing (not patching) -thickness : smallint unsigned # slice thickness in microns -experimenter : varchar(20) # person who performed this experiment -``` - -You can examine the resulting table heading with - -```python -mp.BrainSlice.heading -``` - -The heading of `mp.Slice` may look something like - -```python -subject_id : char(8) # experiment subject id -slice_id : smallint # slice number within subject ---- -brain_region : varchar(12) # abbreviated name for brain region -plane : varchar(12) # plane of section -slice_date : date # date of the slicing (not patching) -thickness : smallint unsigned # slice thickness in microns -experimenter : varchar(20) # person who performed this experiment -``` - -This displayed heading reflects the actual attributes in the table. -The foreign keys have been replaced by the primary key attributes of the referenced -tables, including their data types and comments. - -## How dependencies work - -The foreign key `-> A` in the definition of table `B` has the following effects: - -1. The primary key attributes of `A` are made part of `B`'s definition. -2. A referential constraint is created in `B` with reference to `A`. -3. If one does not already exist, an index is created to speed up searches in `B` for -matches to `A`. - (The reverse search is already fast because it uses the primary key of `A`.) - -A referential constraint means that an entity in `B` cannot exist without a matching -entity in `A`. -**Matching** means attributes in `B` that correspond to the primary key of `A` must -have the same values. -An attempt to insert an entity into `B` that does not have a matching counterpart in -`A` will fail. -Conversely, deleting an entity from `A` that has matching entities in `B` will result -in the deletion of those matching entities and so forth, recursively, downstream in the -pipeline. - -When `B` references `A` with a foreign key, one can say that `B` **depends** on `A`. -In DataJoint terms, `B` is the **dependent table** and `A` is the **referenced table** -with respect to the foreign key from `B` to `A`. - -Note to those already familiar with the theory of relational databases: The usage of -the words "depends" and "dependency" here should not be confused with the unrelated -concept of *functional dependencies* that is used to define normal forms. - -## Referential integrity - -Dependencies enforce the desired property of databases known as -**referential integrity**. -Referential integrity is the guarantee made by the data management process that related -data across the database remain present, correctly associated, and mutually consistent. -Guaranteeing referential integrity means enforcing the constraint that no entity can -exist in the database without all the other entities on which it depends. -An entity in table `B` depends on an entity in table `A` when they belong to them or -are computed from them. - -## Dependencies with renamed attributes - -In most cases, a dependency includes the primary key attributes of the referenced table -as they appear in its table definition. -Sometimes it can be helpful to choose a new name for a foreign key attribute that -better fits the context of the dependent table. -DataJoint provides the following [projection](../../query/project.md) syntax to rename -the primary key attributes when they are included in the new table. - -The dependency - -```python --> Table.project(new_attr='old_attr') -``` - -renames the primary key attribute `old_attr` of `Table` as `new_attr` before -integrating it into the table definition. -Any additional primary key attributes will retain their original names. -For example, the table `Experiment` may depend on table `User` but rename the `user` -attribute into `operator` as follows: - -```python --> User.proj(operator='user') -``` - -In the above example, an entity in the dependent table depends on exactly one entity in -the referenced table. -Sometimes entities may depend on multiple entities from the same table. -Such a design requires a way to distinguish between dependent attributes having the -same name in the reference table. -For example, a table for `Synapse` may reference the table `Cell` twice as -`presynaptic` and `postsynaptic`. -The table definition may appear as - -```python -# synapse between two cells --> Cell.proj(presynaptic='cell_id') --> Cell.proj(postsynaptic='cell_id') ---- -connection_strength : double # (pA) peak synaptic current -``` - -If the primary key of `Cell` is (`animal_id`, `slice_id`, `cell_id`), then the primary -key of `Synapse` resulting from the above definition will be (`animal_id`, `slice_id`, -`presynaptic`, `postsynaptic`). -Projection always returns all of the primary key attributes of a table, so `animal_id` -and `slice_id` are included, with their original names. - -Note that the design of the `Synapse` table above imposes the constraint that the -synapse can only be found between cells in the same animal and in the same slice. - -Allowing representation of synapses between cells from different slices requires the -renamimg of `slice_id` as well: - -```python -# synapse between two cells --> Cell(presynaptic_slice='slice_id', presynaptic_cell='cell_id') --> Cell(postsynaptic_slice='slice_id', postsynaptic_cell='cell_id') ---- -connection_strength : double # (pA) peak synaptic current -``` - -In this case, the primary key of `Synapse` will be (`animal_id`, `presynaptic_slice`, -`presynaptic_cell`, `postsynaptic_slice`, `postsynaptic_cell`). -This primary key still imposes the constraint that synapses can only form between cells -within the same animal but now allows connecting cells across different slices. - -In the Diagram, renamed foreign keys are shown as red lines with an additional dot node -in the middle to indicate that a renaming took place. - -## Foreign key options - -Note: Foreign key options are currently in development. - -Foreign keys allow the additional options `nullable` and `unique`, which can be -inserted in square brackets following the arrow. - -For example, in the following table definition - -```python -rig_id : char(4) # experimental rig ---- --> Person -``` - -each rig belongs to a person, but the table definition does not prevent one person -owning multiple rigs. -With the `unique` option, a person may only appear once in the entire table, which -means that no one person can own more than one rig. - -```python -rig_id : char(4) # experimental rig ---- --> [unique] Person -``` - -With the `nullable` option, a rig may not belong to anyone, in which case the foreign -key attributes for `Person` are set to `NULL`: - -```python -rig_id : char(4) # experimental rig ---- --> [nullable] Person -``` - -Finally with both `unique` and `nullable`, a rig may or may not be owned by anyone and -each person may own up to one rig. - -```python -rig_id : char(4) # experimental rig ---- --> [unique, nullable] Person -``` - -Foreign keys made from the primary key cannot be nullable but may be unique. diff --git a/docs/src/design/tables/filepath.md b/docs/src/design/tables/filepath.md deleted file mode 100644 index 05e9ca744..000000000 --- a/docs/src/design/tables/filepath.md +++ /dev/null @@ -1,96 +0,0 @@ -# Filepath Datatype - -Note: Filepath Datatype is available as a preview feature in DataJoint Python v0.12. -This means that the feature is required to be explicitly enabled. To do so, make sure -to set the environment variable `FILEPATH_FEATURE_SWITCH=TRUE` prior to use. - -## Configuration & Usage - -Corresponding to issue -[#481](https://github.com/datajoint/datajoint-python/issues/481), -the `filepath` attribute type links DataJoint records to files already -managed outside of DataJoint. This can aid in sharing data with -other systems such as allowing an image viewer application to -directly use files from a DataJoint pipeline, or to allow downstream -tables to reference data which reside outside of DataJoint -pipelines. - -To define a table using the `filepath` datatype, an existing DataJoint -[store](../../sysadmin/external-store.md) should be created and then referenced in the -new table definition. For example, given a simple store: - -```python - dj.config['stores'] = { - 'data': { - 'protocol': 'file', - 'location': '/data', - 'stage': '/data' - } - } -``` - -we can define an `ScanImages` table as follows: - -```python -@schema -class ScanImages(dj.Manual): - definition = """ - -> Session - image_id: int - --- - image_path: filepath@data - """ -``` - -This table can now be used for tracking paths within the `/data` local directory. -For example: - -```python ->>> ScanImages.insert1((0, 0, '/data/images/image_0.tif')) ->>> (ScanImages() & {'session_id': 0}).fetch1(as_dict=True) -{'session_id': 0, 'image_id': 0, 'image_path': '/data/images/image_0.tif'} -``` - -As can be seen from the example, unlike [blob](blobs.md) records, file -paths are managed as path locations to the underlying file. - -## Integrity Notes - -Unlike other data in DataJoint, data in `filepath` records are -deliberately intended for shared use outside of DataJoint. To help -ensure integrity of `filepath` records, DataJoint will record a -checksum of the file data on `insert`, and will verify this checksum -on `fetch`. However, since the underlying file data may be shared -with other applications, special care should be taken to ensure -records stored in `filepath` attributes are not modified outside -of the pipeline, or, if they are, that records in the pipeline are -updated accordingly. A safe method of changing `filepath` data is -as follows: - -1. Delete the `filepath` database record. - This will ensure that any downstream records in the pipeline depending - on the `filepath` record are purged from the database. -2. Modify `filepath` data. -3. Re-insert corresponding the `filepath` record. - This will add the record back to DataJoint with an updated file checksum. -4. Compute any downstream dependencies, if needed. - This will ensure that downstream results dependent on the `filepath` - record are updated to reflect the newer `filepath` contents. - -### Disable Fetch Verification - -Note: Skipping the checksum is not recommended as it ensures file integrity i.e. -downloaded files are not corrupted. With S3 stores, most of the time to complete a -`.fetch()` is from the file download itself as opposed to evaluating the checksum. This -option will primarily benefit `filepath` usage connected to a local `file` store. - -To disable checksums you can set a threshold in bytes -for when to stop evaluating checksums like in the example below: - -```python -dj.config["filepath_checksum_size_limit"] = 5 * 1024**3 # Skip for all files greater than 5GiB -``` - -The default is `None` which means it will always verify checksums. - - diff --git a/docs/src/design/tables/indexes.md b/docs/src/design/tables/indexes.md deleted file mode 100644 index 9d8148c36..000000000 --- a/docs/src/design/tables/indexes.md +++ /dev/null @@ -1,97 +0,0 @@ -# Indexes - -Table indexes are data structures that allow fast lookups by an indexed attribute or -combination of attributes. - -In DataJoint, indexes are created by one of the three mechanisms: - -1. Primary key -2. Foreign key -3. Explicitly defined indexes - -The first two mechanisms are obligatory. Every table has a primary key, which serves as -an unique index. Therefore, restrictions by a primary key are very fast. Foreign keys -create additional indexes unless a suitable index already exists. - -## Indexes for single primary key tables - -Let’s say a mouse in the lab has a lab-specific ID but it also has a separate id issued -by the animal facility. - -```python -@schema -class Mouse(dj.Manual): - definition = """ - mouse_id : int # lab-specific ID - --- - tag_id : int # animal facility ID - """ -``` - -In this case, searching for a mouse by `mouse_id` is much faster than by `tag_id` -because `mouse_id` is a primary key, and is therefore indexed. - -To make searches faster on fields other than the primary key or a foreign key, you can -add a secondary index explicitly. - -Regular indexes are declared as `index(attr1, ..., attrN)` on a separate line anywhere in -the table declaration (below the primary key divide). - -Indexes can be declared with unique constraint as `unique index (attr1, ..., attrN)`. - -Let’s redeclare the table with a unique index on `tag_id`. - -```python -@schema -class Mouse(dj.Manual): - definition = """ - mouse_id : int # lab-specific ID - --- - tag_id : int # animal facility ID - unique index (tag_id) - """ -``` -Now, searches with `mouse_id` and `tag_id` are similarly fast. - -## Indexes for tables with multiple primary keys - -Let’s now imagine that rats in a lab are identified by the combination of `lab_name` and -`rat_id` in a table `Rat`. - -```python -@schema -class Rat(dj.Manual): - definition = """ - lab_name : char(16) - rat_id : int unsigned # lab-specific ID - --- - date_of_birth = null : date - """ -``` -Note that despite the fact that `rat_id` is in the index, searches by `rat_id` alone are not -helped by the index because it is not first in the index. This is similar to searching for -a word in a dictionary that orders words alphabetically. Searching by the first letters -of a word is easy but searching by the last few letters of a word requires scanning the -whole dictionary. - -In this table, the primary key is a unique index on the combination `(lab_name, rat_id)`. -Therefore searches on these attributes or on `lab_name` alone are fast. But this index -cannot help searches on `rat_id` alone. Similarly, searing by `date_of_birth` requires a -full-table scan and is inefficient. - -To speed up searches by the `rat_id` and `date_of_birth`, we can explicit indexes to -`Rat`: - -```python -@schema -class Rat2(dj.Manual): - definition = """ - lab_name : char(16) - rat_id : int unsigned # lab-specific ID - --- - date_of_birth = null : date - - index(rat_id) - index(date_of_birth) - """ -``` diff --git a/docs/src/design/tables/lookup.md b/docs/src/design/tables/lookup.md deleted file mode 100644 index 79b2c67ba..000000000 --- a/docs/src/design/tables/lookup.md +++ /dev/null @@ -1,31 +0,0 @@ -# Lookup Tables - -Lookup tables contain basic facts that are not specific to an experiment and are fairly -persistent. -Their contents are typically small. -In GUIs, lookup tables are often used for drop-down menus or radio buttons. -In computed tables, they are often used to specify alternative methods for computations. -Lookup tables are commonly populated from their `contents` property. -In a [diagram](../diagrams.md) they are shown in gray. -The decision of which tables are lookup tables and which are manual can be somewhat -arbitrary. - -The table below is declared as a lookup table with its contents property provided to -generate entities. - -```python -@schema -class User(dj.Lookup): - definition = """ - # users in the lab - username : varchar(20) # user in the lab - --- - first_name : varchar(20) # user first name - last_name : varchar(20) # user last name - """ - contents = [ - ['cajal', 'Santiago', 'Cajal'], - ['hubel', 'David', 'Hubel'], - ['wiesel', 'Torsten', 'Wiesel'] - ] -``` diff --git a/docs/src/design/tables/manual.md b/docs/src/design/tables/manual.md deleted file mode 100644 index d97b6ce52..000000000 --- a/docs/src/design/tables/manual.md +++ /dev/null @@ -1,47 +0,0 @@ -# Manual Tables - -Manual tables are populated during experiments through a variety of interfaces. -Not all manual information is entered by typing. -Automated software can enter it directly into the database. -What makes a manual table manual is that it does not perform any computations within -the DataJoint pipeline. - -The following code defines three manual tables `Animal`, `Session`, and `Scan`: - -```python -@schema -class Animal(dj.Manual): - definition = """ - # information about animal - animal_id : int # animal id assigned by the lab - --- - -> Species - date_of_birth=null : date # YYYY-MM-DD optional - sex='' : enum('M', 'F', '') # leave empty if unspecified - """ - -@schema -class Session(dj.Manual): - definition = """ - # Experiment Session - -> Animal - session : smallint # session number for the animal - --- - session_date : date # YYYY-MM-DD - -> User - -> Anesthesia - -> Rig - """ - -@schema -class Scan(dj.Manual): - definition = """ - # Two-photon imaging scan - -> Session - scan : smallint # scan number within the session - --- - -> Lens - laser_wavelength : decimal(5,1) # um - laser_power : decimal(4,1) # mW - """ -``` diff --git a/docs/src/design/tables/master-part.md b/docs/src/design/tables/master-part.md deleted file mode 100644 index 629bfb8ab..000000000 --- a/docs/src/design/tables/master-part.md +++ /dev/null @@ -1,112 +0,0 @@ -# Master-Part Relationship - -Often an entity in one table is inseparably associated with a group of entities in -another, forming a **master-part** relationship. -The master-part relationship ensures that all parts of a complex representation appear -together or not at all. -This has become one of the most powerful data integrity principles in DataJoint. - -As an example, imagine segmenting an image to identify regions of interest. -The resulting segmentation is inseparable from the ROIs that it produces. -In this case, the two tables might be called `Segmentation` and `Segmentation.ROI`. - -In Python, the master-part relationship is expressed by making the part a nested class -of the master. -The part is subclassed from `dj.Part` and does not need the `@schema` decorator. - -```python -@schema -class Segmentation(dj.Computed): - definition = """ # image segmentation - -> Image - """ - - class ROI(dj.Part): - definition = """ # Region of interest resulting from segmentation - -> Segmentation - roi : smallint # roi number - --- - roi_pixels : longblob # indices of pixels - roi_weights : longblob # weights of pixels - """ - - def make(self, key): - image = (Image & key).fetch1('image') - self.insert1(key) - count = itertools.count() - Segmentation.ROI.insert( - dict(key, roi=next(count), roi_pixel=roi_pixels, roi_weights=roi_weights) - for roi_pixels, roi_weights in mylib.segment(image)) -``` - -## Populating - -Master-part relationships can form in any data tier, but DataJoint observes them more -strictly for auto-populated tables. -To populate both the master `Segmentation` and the part `Segmentation.ROI`, it is -sufficient to call the `populate` method of the master: - -```python -Segmentation.populate() -``` - -Note that the entities in the master and the matching entities in the part are inserted -within a single `make` call of the master, which means that they are a processed inside -a single transactions: either all are inserted and committed or the entire transaction -is rolled back. -This ensures that partial results never appear in the database. - -For example, imagine that a segmentation is performed, but an error occurs halfway -through inserting the results. -If this situation were allowed to persist, then it might appear that 20 ROIs were -detected where 45 had actually been found. - -## Deleting - -To delete from a master-part pair, one should never delete from the part tables -directly. -The only valid method to delete from a part table is to delete the master. -This has been an unenforced rule, but upcoming versions of DataJoint will prohibit -direct deletes from the master table. -DataJoint's [delete](../../manipulation/delete.md) operation is also enclosed in a -transaction. - -Together, the rules of master-part relationships ensure a key aspect of data integrity: -results of computations involving multiple components and steps appear in their -entirety or not at all. - -## Multiple parts - -The master-part relationship cannot be chained or nested. -DataJoint does not allow part tables of other part tables per se. -However, it is common to have a master table with multiple part tables that depend on -each other. -For example: - -```python -@schema -class ArrayResponse(dj.Computed): -definition = """ -array: int -""" - -class ElectrodeResponse(dj.Part): -definition = """ --> master -electrode: int # electrode number on the probe -""" - -class ChannelResponse(dj.Part): -definition = """ --> ElectrodeResponse -channel: int ---- -response: longblob # response of a channel -""" -``` - -Conceptually, one or more channels belongs to an electrode, and one or more electrodes -belong to an array. -This example assumes that information about an array's response (which consists -ultimately of the responses of multiple electrodes each consisting of multiple channel -responses) including it's electrodes and channels are entered together. diff --git a/docs/src/design/tables/primary.md b/docs/src/design/tables/primary.md deleted file mode 100644 index fc4f5b8e0..000000000 --- a/docs/src/design/tables/primary.md +++ /dev/null @@ -1,178 +0,0 @@ -# Primary Key - -## Primary keys in DataJoint - -Entities in tables are neither named nor numbered. -DataJoint does not answer questions of the type "What is the 10th element of this table?" -Instead, entities are distinguished by the values of their attributes. -Furthermore, the entire entity is not required for identification. -In each table, a subset of its attributes are designated to be the **primary key**. -Attributes in the primary key alone are sufficient to differentiate any entity from any -other within the table. - -Each table must have exactly one -[primary key](http://en.wikipedia.org/wiki/Primary_key): a subset of its attributes -that uniquely identify each entity in the table. -The database uses the primary key to prevent duplicate entries, to relate data across -tables, and to accelerate data queries. -The choice of the primary key will determine how you identify entities. -Therefore, make the primary key **short**, **expressive**, and **persistent**. - -For example, mice in our lab are assigned unique IDs. -The mouse ID number `animal_id` of type `smallint` can serve as the primary key for the -table `Mice`. -An experiment performed on a mouse may be identified in the table `Experiments` by two -attributes: `animal_id` and `experiment_number`. - -DataJoint takes the concept of primary keys somewhat more seriously than other models -and query languages. -Even **table expressions**, i.e. those tables produced through operations on other -tables, have a well-defined primary key. -All operators on tables are designed in such a way that the results always have a -well-defined primary key. - -In all representations of tables in DataJoint, the primary key attributes are always -listed before other attributes and highlighted for emphasis (e.g. in a **bold** font or -marked with an asterisk \*) - -## Defining a primary key - -In table declarations, the primary key attributes always come first and are separated -from the other attributes with a line containing at least three hyphens. -For example, the following is the definition of a table containing database users where -`username` is the primary key. - -```python -# database users -username : varchar(20) # unique user name ---- -first_name : varchar(30) -last_name : varchar(30) -role : enum('admin', 'contributor', 'viewer') -``` - -## Entity integrity - -The primary key defines and enforces the desired property of databases known as -[entity integrity](../integrity.md). -**Entity integrity** ensures that there is a one-to-one and unambiguous mapping between -real-world entities and their representations in the database system. -The data management process must prevent any duplication or misidentification of -entities. - -To enforce entity integrity, DataJoint implements several rules: - -- Every table must have a primary key. -- Primary key attributes cannot have default values (with the exception of -`auto_increment` and `CURRENT_TIMESTAMP`; see below). -- Operators on tables are defined with respect to the primary key and preserve a -primary key in their results. - -## Datatypes in primary keys - -All integer types, dates, timestamps, and short character strings make good primary key -attributes. -Character strings are somewhat less suitable because they can be long and because they -may have invisible trailing spaces. -Floating-point numbers should be avoided because rounding errors may lead to -misidentification of entities. -Enums are okay as long as they do not need to be modified after -[dependencies](dependencies.md) are already created referencing the table. -Finally, DataJoint does not support blob types in primary keys. - -The primary key may be composite, i.e. comprising several attributes. -In DataJoint, hierarchical designs often produce tables whose primary keys comprise -many attributes. - -## Choosing primary key attributes - -A primary key comprising real-world attributes is a good choice when such real-world -attributes are already properly and permanently assigned. -Whatever characteristics are used to uniquely identify the actual entities can be used -to identify their representations in the database. - -If there are no attributes that could readily serve as a primary key, an artificial -attribute may be created solely for the purpose of distinguishing entities. -In such cases, the primary key created for management in the database must also be used -to uniquely identify the entities themselves. -If the primary key resides only in the database while entities remain indistinguishable -in the real world, then the process cannot ensure entity integrity. -When a primary key is created as part of data management rather than based on -real-world attributes, an institutional process must ensure the uniqueness and -permanence of such an identifier. - -For example, the U.S. government assigns every worker an identifying attribute, the -social security number. -However, the government must go to great lengths to ensure that this primary key is -assigned exactly once, by checking against other less convenient candidate keys (i.e. -the combination of name, parents' names, date of birth, place of birth, etc.). -Just like the SSN, well managed primary keys tend to get institutionalized and find -multiple uses. - -Your lab must maintain a system for uniquely identifying important entities. -For example, experiment subjects and experiment protocols must have unique IDs. -Use these as the primary keys in the corresponding tables in your DataJoint databases. - -### Using hashes as primary keys - -Some tables include too many attributes in their primary keys. -For example, the stimulus condition in a psychophysics experiment may have a dozen -parameters such that a change in any one of them makes a different valid stimulus -condition. -In such a case, all the attributes would need to be included in the primary key to -ensure entity integrity. -However, long primary keys make it difficult to reference individual entities. -To be most useful, primary keys need to be relatively short. - -This problem is effectively solved through the use of a hash of all the identifying -attributes as the primary key. -For example, MD5 or SHA-1 hash algorithms can be used for this purpose. -To keep their representations human-readable, they may be encoded in base-64 ASCII. -For example, the 128-bit MD5 hash can be represented by 21 base-64 ASCII characters, -but for many applications, taking the first 8 to 12 characters is sufficient to avoid -collisions. - -### `auto_increment` - -Some entities are created by the very action of being entered into the database. -The action of entering them into the database gives them their identity. -It is impossible to duplicate them since entering the same thing twice still means -creating two distinct entities. - -In such cases, the use of an auto-incremented primary key is warranted. -These are declared by adding the word `auto_increment` after the data type in the -declaration. -The datatype must be an integer. -Then the database will assign incrementing numbers at each insert. - -The example definition below defines an auto-incremented primary key - -```python -# log entries -entry_id : smallint auto_increment ---- -entry_text : varchar(4000) -entry_time = CURRENT_TIMESTAMP : timestamp(3) # automatic timestamp with millisecond precision -``` - -DataJoint passes `auto_increment` behavior to the underlying MySQL and therefore it has -the same limitation: it can only be used for tables with a single attribute in the -primary key. - -If you need to auto-increment an attribute in a composite primary key, you will need to -do so programmatically within a transaction to avoid collisions. - -For example, let’s say that you want to auto-increment `scan_idx` in a table called -`Scan` whose primary key is `(animal_id, session, scan_idx)`. -You must already have the values for `animal_id` and `session` in the dictionary `key`. -Then you can do the following: - -```python -U().aggr(Scan & key, next='max(scan_idx)+1') - -# or - -Session.aggr(Scan, next='max(scan_idx)+1') & key -``` - -Note that the first option uses a [universal set](../../query/universals.md). diff --git a/docs/src/design/tables/tiers.md b/docs/src/design/tables/tiers.md deleted file mode 100644 index 2cf1f9428..000000000 --- a/docs/src/design/tables/tiers.md +++ /dev/null @@ -1,68 +0,0 @@ -# Data Tiers - -DataJoint assigns all tables to one of the following data tiers that differentiate how -the data originate. - -## Table tiers - -| Tier | Superclass | Description | -| -- | -- | -- | -| Lookup | `dj.Lookup` | Small tables containing general facts and settings of the data pipeline; not specific to any experiment or dataset. | -| Manual | `dj.Manual` | Data entered from outside the pipeline, either by hand or with external helper scripts. | -| Imported | `dj.Imported` | Data ingested automatically inside the pipeline but requiring access to data outside the pipeline. | -| Computed | `dj.Computed` | Data computed automatically entirely inside the pipeline. | - -Table data tiers indicate to database administrators how valuable the data are. -Manual data are the most valuable, as re-entry may be tedious or impossible. -Computed data are safe to delete, as the data can always be recomputed from within DataJoint. -Imported data are safer than manual data but less safe than computed data because of -dependency on external data sources. -With these considerations, database administrators may opt not to back up computed -data, for example, or to back up imported data less frequently than manual data. - -The data tier of a table is specified by the superclass of its class. -For example, the User class in [definitions](declare.md) uses the `dj.Manual` -superclass. -Therefore, the corresponding User table on the database would be of the Manual tier. -Furthermore, the classes for **imported** and **computed** tables have additional -capabilities for automated processing as described in -[Auto-populate](../../compute/populate.md). - -## Internal conventions for naming tables - -On the server side, DataJoint uses a naming scheme to generate a table name -corresponding to a given class. -The naming scheme includes prefixes specifying each table's data tier. - -First, the name of the class is converted from `CamelCase` to `snake_case` -([separation by underscores](https://en.wikipedia.org/wiki/Snake_case)). -Then the name is prefixed according to the data tier. - -- `Manual` tables have no prefix. -- `Lookup` tables are prefixed with `#`. -- `Imported` tables are prefixed with `_`, a single underscore. -- `Computed` tables are prefixed with `__`, two underscores. - -For example: - -The table for the class `StructuralScan` subclassing `dj.Manual` will be named -`structural_scan`. - -The table for the class `SpatialFilter` subclassing `dj.Lookup` will be named -`#spatial_filter`. - -Again, the internal table names including prefixes are used only on the server side. -These are never visible to the user, and DataJoint users do not need to know these -conventions -However, database administrators may use these naming patterns to set backup policies -or to restrict access based on data tiers. - -## Part tables - -[Part tables](master-part.md) do not have their own tier. -Instead, they share the same tier as their master table. -The prefix for part tables also differs from the other tiers. -They are prefixed by the name of their master table, separated by two underscores. - -For example, the table for the class `Channel(dj.Part)` with the master -`Ephys(dj.Imported)` will be named `_ephys__channel`. diff --git a/docs/src/develop.md b/docs/src/develop.md index bc636cc20..4643683b6 100644 --- a/docs/src/develop.md +++ b/docs/src/develop.md @@ -1,202 +1,101 @@ -# Developer Guide +# Contributing Guide -## Table of Contents - -- [Contribute to DataJoint Python Documentation](#contribute-to-datajoint-python-documentation) -- [Setup Development Environment](#setup-development-environment) - - [Prerequisites](#prerequisites) - - [With Virtual Environment](#with-virtual-environment) - - [With DevContainer](#with-devcontainer) - - [Extra Efficiency, Optional But Recommended](#extra-efficiency-optional-but-recommended) - - [Pre-commit Hooks](#pre-commit-hooks) - - [Integration Tests](#integration-tests) - - [VSCode](#vscode) - - [Jupyter Extension](#jupyter-extension) - - [Debugger](#debugger) - - [MySQL CLI](#mysql-cli) - -## Contribute to DataJoint Python Documentation - -> Contributions to documentations are equivalently important to any code for the community, please help us to resolve any confusions in documentations. - -[Here](https://github.com/datajoint/datajoint-python/blob/master/docs/README.md) is the instructions for contributing documentations, or you can find the same instructions at `$PROJECT_DIR/docs/README.md` in the repository. - -[Back to top](#table-of-contents) - -## Setup Development Environment - -> We have [DevContainer](https://containers.dev/) ready for contributors to develop without setting up their environment. If you are familiar with DevContainer, Docker or Github Codespace, this is the recommended development environment for you. -> If you have never used Docker, it might be easier for you to use a virtual environment through `conda/mamba/venv`, it is also very straightforward to set up. - -### Prerequisites - -- Clone datajoint-python repository +## Quick Start ```bash -# If you have your SSH key set up with GitHub, you can clone using SSH -git clone git@github.com:datajoint/datajoint-python.git -# Otherwise, you can clone using HTTPS +# Clone the repository git clone https://github.com/datajoint/datajoint-python.git -``` -- If you don't use DevContainer, then either install Anaconda/[Miniconda](https://www.anaconda.com/docs/getting-started/miniconda/install)/Mamba, or just use Python's built-in `venv` module without install anything else. - -### With Virtual Environment +cd datajoint-python -```bash -# Check if you have Python 3.9 or higher, if not please upgrade -python --version -# Create a virtual environment with venv +# Create virtual environment (Python 3.10+) python -m venv .venv -source .venv/bin/activate -pip install -e .[dev] +source .venv/bin/activate # On Windows: .venv\Scripts\activate -# Or create a virtual environment with conda -conda create -n dj python=3.13 # any 3.9+ is fine -conda activate dj -pip install -e .[dev] -``` +# Install with development dependencies +pip install -e ".[dev]" -[Back to top](#table-of-contents) +# Install pre-commit hooks +pre-commit install -### With DevContainer +# Run tests +pytest tests +``` -#### Launch Environment +## Development Environment -Here are some options that provide a great developer experience: +### Local Setup -- **Cloud-based IDE**: (*recommended*) - - Launch using [GitHub Codespaces](https://github.com/features/codespaces) using the option `Create codespace on master` in the codebase repository on your fork. - - Build time for a 2-Core codespace is **~6m**. This is done infrequently and cached for convenience. - - Start time for a 2-Core codespace is **~2m**. This will pull the built codespace from cache when you need it. - - *Tip*: GitHub auto names the codespace but you can rename the codespace so that it is easier to identify later. -- **Local IDE (VSCode - Dev Containers)**: - - Ensure you have [Git](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git) - - Ensure you have [Docker](https://docs.docker.com/get-docker/) - - Ensure you have [VSCode](https://code.visualstudio.com/) - - Install the [Dev Containers extension](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers) - - `git clone` the codebase repository and open it in VSCode - - Use the `Dev Containers extension` to `Reopen in Container` (More info in the `Getting started` included with the extension) - - You will know your environment has finished loading once you see a terminal open related to `Running postStartCommand` with a final message: `Done. Press any key to close the terminal.`. -- **Local IDE (Docker Compose)**: - - Ensure you have [Git](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git) - - Ensure you have [Docker](https://docs.docker.com/get-docker/) - - `git clone` the codebase repository and open it in VSCode - - Issue the following command in the terminal to build and run the Docker container: `HOST_UID=$(id -u) PY_VER=3.11 DJ_VERSION=$(grep -oP '\d+\.\d+\.\d+' datajoint/version.py) docker compose --profile test run --rm -it djtest -- sh -c 'pip install -qe ".[dev]" && bash'` - - Issue the following command in the terminal to stop the Docker compose stack: `docker compose --profile test down` +Requirements: -[Back to top](#table-of-contents) +- Python 3.10 or higher +- MySQL 8.0+ or Docker (for running tests) -## Extra Efficiency, Optional But Recommended +The `[dev]` extras install all development tools: pytest, pre-commit, black, ruff, and documentation builders. -### Pre-commit Hooks +### Using Docker for Database -We recommend using [pre-commit](https://pre-commit.com/) to automatically run linters and formatters on your code before committing. -To set up pre-commit, run the following command in your terminal: +Tests require a MySQL database. Start one with Docker: ```bash -pip install pre-commit -pre-commit install +docker compose up -d db ``` -You can manually run pre-commit on all files with the following command: +Configure connection (or set environment variables): ```bash -pre-commit run --all-files +export DJ_HOST=localhost +export DJ_USER=root +export DJ_PASS=password ``` -This will run all the linters and formatters specified in the `.pre-commit-config.yaml` file. If all check passed, you can commit your code. Otherwise, you need to fix the failed checks and run the command again. -> Pre-commit will automatically run the linters and formatters on all staged files before committing. However, if your code doesn't follow the linters and formatters, the commit will fail. -> Some hooks will automatically fix your problem, and add the fixed files as git's `unstaged` files, you just need to add them(`git add .`) to git's `staged` files and commit again. -> Some hooks will not automatically fix your problem, so you need to check the pre-commit failed log to fix them manually and include the update to your `staged` files and commit again. +### Alternative: GitHub Codespaces -If you really don't want to use pre-commit, or if you don't like it, you can uninstall it with the following command: +For a pre-configured environment, use [GitHub Codespaces](https://github.com/features/codespaces): -```bash -pre-commit uninstall -``` +1. Fork the repository +2. Click "Create codespace on master" +3. Wait for environment to build (~6 minutes first time, ~2 minutes from cache) -But when you issue a pull request, the same linter and formatter check will run against your contribution, you are going to have the same failure as well. So without pre-commit, you need to **manually run these linters and formatters before committing your code**: +## Code Quality -- Syntax tests +### Pre-commit Hooks -The following will verify that there are no syntax errors. +Pre-commit runs automatically on `git commit`. To run manually: -``` -flake8 datajoint --count --select=E9,F63,F7,F82 --show-source --statistics +```bash +pre-commit run --all-files ``` -- Style tests +Hooks include: -The following will verify that there are no code styling errors. +- **ruff** — Linting and import sorting +- **black** — Code formatting +- **mypy** — Type checking (optional) -``` -flake8 --ignore=E203,E722,W503 datajoint --count --max-complexity=62 --max-line-length=127 --statistics -``` - -The following will ensure the codebase has been formatted with [black](https://black.readthedocs.io/en/stable/). +### Running Tests -``` -black datajoint --check -v --diff -``` +```bash +# Full test suite with coverage +pytest -sv --cov=datajoint tests -The following will ensure the test suite has been formatted with [black](https://black.readthedocs.io/en/stable/). +# Single test file +pytest tests/test_connection.py +# Single test function +pytest tests/test_connection.py::test_dj_conn -v ``` -black tests --check -v --diff -``` - -[Back to top](#table-of-contents) - -### Integration Tests - -The following will verify there are no regression errors by running our test suite of unit and integration tests. - -- Entire test suite: - ``` - pytest -sv --cov-report term-missing --cov=datajoint tests - ``` - -- A single functional test: - ``` - pytest -sv tests/test_connection.py::test_dj_conn - ``` -- A single class test: - ``` - pytest -sv tests/test_aggr_regressions.py::TestIssue558 - ``` -[Back to top](#table-of-contents) +## Submitting Changes -### VSCode +1. Create a feature branch from `master` +2. Make your changes +3. Ensure tests pass and pre-commit is clean +4. Submit a pull request -#### Jupyter Extension +PRs trigger CI checks automatically. All checks must pass before merge. -Be sure to go through this documentation if you are new to [Running Jupyter Notebooks with VSCode](https://code.visualstudio.com/docs/datascience/jupyter-notebooks#_create-or-open-a-jupyter-notebook). +## Documentation -#### Debugger - -[VSCode Debugger](https://code.visualstudio.com/docs/editor/debugging) is a powerful tool that can really accelerate fixes. - -Try it as follows: - -- Create a python script of your choice -- `import datajoint` (This will use the current state of the source) -- Add breakpoints by adding red dots next to line numbers -- Select the `Run and Debug` tab -- Start by clicking the button `Run and Debug` - -[Back to top](#table-of-contents) - -### MySQL CLI - -> Installation instruction is in [here](https://dev.mysql.com/doc/mysql-shell/8.0/en/mysql-shell-install.html) - -It is often useful in development to connect to DataJoint's relational database backend directly using the MySQL CLI. - -Connect as follows to the database running within your developer environment: - -``` -mysql -hdb -uroot -ppassword -``` +Docstrings use NumPy style. See [DOCSTRING_STYLE.md](https://github.com/datajoint/datajoint-python/blob/master/DOCSTRING_STYLE.md) for guidelines. -[Back to top](#table-of-contents) \ No newline at end of file +User documentation is maintained at [docs.datajoint.com](https://docs.datajoint.com). diff --git a/docs/src/faq.md b/docs/src/faq.md deleted file mode 100644 index c4c82d014..000000000 --- a/docs/src/faq.md +++ /dev/null @@ -1,192 +0,0 @@ -# Frequently Asked Questions - -## How do I use DataJoint with a GUI? - -It is common to enter data during experiments using a graphical user interface. - -1. The [DataJoint platform](https://works.datajoint.com) platform is a web-based, - end-to-end platform to host and execute data pipelines. - -2. [DataJoint LabBook](https://github.com/datajoint/datajoint-labbook) is an open -source project for data entry but is no longer actively maintained. - -## Does DataJoint support other programming languages? - -DataJoint [Python](https://docs.datajoint.com/core/datajoint-python/) is the most -up-to-date version and all future development will focus on the Python API. The -[Matlab](https://datajoint.com/docs/core/datajoint-matlab/) API was actively developed -through 2023. Previous projects implemented some DataJoint features in -[Julia](https://github.com/BrainCOGS/neuronex_workshop_2018/tree/julia/julia) and -[Rust](https://github.com/datajoint/datajoint-core). DataJoint's data model and data -representation are largely language independent, which means that any language with a -DataJoint client can work with a data pipeline defined in any other language. DataJoint -clients for other programming languages will be implemented based on demand. All -languages must comply to the same data model and computation approach as defined in -[DataJoint: a simpler relational data model](https://arxiv.org/abs/1807.11104). - -## Can I use DataJoint with my current database? - -Researchers use many different tools to keep records, from simple formalized file -hierarchies to complete software packages for colony management and standard file types -like NWB. Existing projects have built interfaces with many such tools, such as -[PyRAT](https://github.com/SFB1089/adamacs/blob/main/notebooks/03_pyrat_insert.ipynb). -The only requirement for interface is that tool has an open API. Contact -[support@datajoint.com](mailto:Support@DataJoint.com) with inquiries. The DataJoint -team will consider development requests based on community demand. - -## Is DataJoint an ORM? - -Programmers are familiar with object-relational mappings (ORM) in various programming -languages. Python in particular has several popular ORMs such as -[SQLAlchemy](https://www.sqlalchemy.org/) and [Django ORM](https://tutorial.djangogirls.org/en/django_orm/). -The purpose of ORMs is to allow representations and manipulations of objects from the -host programming language as data in a relational database. ORMs allow making objects -persistent between program executions by creating a bridge (i.e., mapping) between the -object model used by the host language and the relational model allowed by the database. -The result is always a compromise, usually toward the object model. ORMs usually forgo -key concepts, features, and capabilities of the relational model for the sake of -convenient programming constructs in the language. - -In contrast, DataJoint implements a data model that is a refinement of the relational -data model without compromising its core principles of data representation and queries. -DataJoint supports data integrity (entity integrity, referential integrity, and group -integrity) and provides a fully capable relational query language. DataJoint remains -absolutely data-centric, with the primary focus on the structure and integrity of the -data pipeline. Other ORMs are more application-centric, primarily focusing on the -application design while the database plays a secondary role supporting the application -with object persistence and sharing. - -## What is the difference between DataJoint and Alyx? - -[Alyx](https://github.com/cortex-lab/alyx) is an experiment management database -application developed in Kenneth Harris' lab at UCL. - -Alyx is an application with a fixed pipeline design with a nice graphical user -interface. In contrast, DataJoint is a general-purpose library for designing and -building data processing pipelines. - -Alyx is geared towards ease of data entry and tracking for a specific workflow -(e.g. mouse colony information and some pre-specified experiments) and data types. -DataJoint could be used as a more general purposes tool to design, implement, and -execute processing on such workflows/pipelines from scratch, and DataJoint focuses on -flexibility, data integrity, and ease of data analysis. The purposes are partly -overlapping and complementary. The -[International Brain Lab project](https://internationalbrainlab.com) is developing a -bridge from Alyx to DataJoint, hosted as an -[open-source project](https://github.com/datajoint-company/ibl-pipeline). It -implements a DataJoint schema that replicates the major features of the Alyx -application and a synchronization script from an existing Alyx database to its -DataJoint counterpart. - -## Where is my data? - -New users often ask this question thinking of passive **data repositories** -- -collections of files and folders and a separate collection of metadata -- information -about how the files were collected and what they contain. -Let's address metadata first, since the answer there is easy: Everything goes in the -database! -Any information about the experiment that would normally be stored in a lab notebook, -in an Excel spreadsheet, or in a Word document is entered into tables in the database. -These tables can accommodate numbers, strings, dates, or numerical arrays. -The entry of metadata can be manual, or it can be an automated part of data acquisition -(in this case the acquisition software itself is modified to enter information directly -into the database). - -Depending on their size and contents, raw data files can be stored in a number of ways. -In the simplest and most common scenario, raw data continues to be stored in either a -local filesystem or in the cloud as collections of files and folders. -The paths to these files are entered in the database (again, either manually or by -automated processes). -This is the point at which the notion of a **data pipeline** begins. -Below these "manual tables" that contain metadata and file paths are a series of tables -that load raw data from these files, process it in some way, and insert derived or -summarized data directly into the database. -For example, in an imaging application, the very large raw `.TIFF` stacks would reside on -the filesystem, but the extracted fluorescent trace timeseries for each cell in the -image would be stored as a numerical array directly in the database. -Or the raw video used for animal tracking might be stored in a standard video format on -the filesystem, but the computed X/Y positions of the animal would be stored in the -database. -Storing these intermediate computations in the database makes them easily available for -downstream analyses and queries. - -## Do I have to manually enter all my data into the database? - -No! While some of the data will be manually entered (the same way that it would be -manually recorded in a lab notebook), the advantage of DataJoint is that standard -downstream processing steps can be run automatically on all new data with a single -command. -This is where the notion of a **data pipeline** comes into play. -When the workflow of cleaning and processing the data, extracting important features, -and performing basic analyses is all implemented in a DataJoint pipeline, minimal -effort is required to analyze newly-collected data. -Depending on the size of the raw files and the complexity of analysis, useful results -may be available in a matter of minutes or hours. -Because these results are stored in the database, they can be made available to anyone -who is given access credentials for additional downstream analyses. - -## Won't the database get too big if all my data are there? - -Typically, this is not a problem. -If you find that your database is getting larger than a few dozen TB, DataJoint -provides transparent solutions for storing very large chunks of data (larger than the 4 -GB that can be natively stored as a LONGBLOB in MySQL). -However, in many scenarios even long time series or images can be stored directly in -the database with little effect on performance. - -## Why not just process the data and save them back to a file? - -There are two main advantages to storing results in the database. -The first is data integrity. -Because the relationships between data are enforced by the structure of the database, -DataJoint ensures that the metadata in the upstream nodes always correctly describes -the computed results downstream in the pipeline. -If a specific experimental session is deleted, for example, all the data extracted from -that session are automatically removed as well, so there is no chance of "orphaned" -data. -Likewise, the database ensures that computations are atomic. -This means that any computation performed on a dataset is performed in an all-or-none -fashion. -Either all of the data are processed and inserted, or none at all. -This ensures that there are no incomplete data. -Neither of these important features of data integrity can be guaranteed by a file -system. - -The second advantage of storing intermediate results in a data pipeline is flexible -access. -Accessing arbitrarily complex subsets of the data can be achieved with DataJoint's -flexible query language. -When data are stored in files, collecting the desired data requires trawling through -the file hierarchy, finding and loading the files of interest, and selecting the -interesting parts of the data. - -This brings us to the final important question: - -## How do I get my data out? - -This is the fun part. See [queries](query/operators.md) for details of the DataJoint -query language directly from Python. - -## Interfaces - -Multiple interfaces may be used to get the data into and out of the pipeline. - -Some labs use third-party GUI applications such as -[HeidiSQL](https://www.heidisql.com/) and -[Navicat](https://www.navicat.com/), for example. These applications allow entering -and editing data in tables similarly to spreadsheets. - -The Helium Application (https://mattbdean.github.io/Helium/ and -https://github.com/mattbdean/Helium) is web application for browsing DataJoint -pipelines and entering new data. -Matt Dean develops and maintains Helium under the direction of members of Karel -Svoboda's lab at Janelia Research Campus and Vathes LLC. - -Data may also be imported or synchronized into a DataJoint pipeline from existing LIMS -(laboratory information management systems). -For example, the [International Brain Lab](https://internationalbrainlab.com) -synchronizes data from an [Alyx database](https://github.com/cortex-lab/alyx). -For implementation details, see https://github.com/int-brain-lab/IBL-pipeline. - -Other labs (e.g. Sinz Lab) have developed GUI interfaces using the Flask web framework -in Python. diff --git a/docs/src/images/StudentTable.png b/docs/src/images/StudentTable.png deleted file mode 100644 index c8623f2ab..000000000 Binary files a/docs/src/images/StudentTable.png and /dev/null differ diff --git a/docs/src/images/added-example-ERD.svg b/docs/src/images/added-example-ERD.svg deleted file mode 100644 index 0884853f4..000000000 --- a/docs/src/images/added-example-ERD.svg +++ /dev/null @@ -1,207 +0,0 @@ - - - -%3 - - - -uni.Term - - -uni.Term - - - - - -uni.Section - - -uni.Section - - - - - -uni.Term->uni.Section - - - - -uni.CurrentTerm - - -uni.CurrentTerm - - - - - -uni.Term->uni.CurrentTerm - - - - -uni.Student - - -uni.Student - - - - - -uni.Enroll - - -uni.Enroll - - - - - -uni.Student->uni.Enroll - - - - -uni.StudentMajor - - -uni.StudentMajor - - - - - -uni.Student->uni.StudentMajor - - - - -uni.Example - - -uni.Example - - - - - -uni.Student->uni.Example - - - - -uni.Grade - - -uni.Grade - - - - - -uni.Enroll->uni.Grade - - - - -uni.Section->uni.Enroll - - - - -uni.Course - - -uni.Course - - - - - -uni.Course->uni.Section - - - - -uni.Department - - -uni.Department - - - - - -uni.Department->uni.StudentMajor - - - - -uni.Department->uni.Course - - - - -uni.LetterGrade - - -uni.LetterGrade - - - - - -uni.LetterGrade->uni.Grade - - - - diff --git a/docs/src/images/data-engineering.png b/docs/src/images/data-engineering.png deleted file mode 100644 index e038ac299..000000000 Binary files a/docs/src/images/data-engineering.png and /dev/null differ diff --git a/docs/src/images/data-science-after.png b/docs/src/images/data-science-after.png deleted file mode 100644 index e4f824cab..000000000 Binary files a/docs/src/images/data-science-after.png and /dev/null differ diff --git a/docs/src/images/data-science-before.png b/docs/src/images/data-science-before.png deleted file mode 100644 index eb8ee311d..000000000 Binary files a/docs/src/images/data-science-before.png and /dev/null differ diff --git a/docs/src/images/diff-example1.png b/docs/src/images/diff-example1.png deleted file mode 100644 index 2c8844b81..000000000 Binary files a/docs/src/images/diff-example1.png and /dev/null differ diff --git a/docs/src/images/diff-example2.png b/docs/src/images/diff-example2.png deleted file mode 100644 index ab7465c7b..000000000 Binary files a/docs/src/images/diff-example2.png and /dev/null differ diff --git a/docs/src/images/diff-example3.png b/docs/src/images/diff-example3.png deleted file mode 100644 index b4f511fec..000000000 Binary files a/docs/src/images/diff-example3.png and /dev/null differ diff --git a/docs/src/images/dimitri-ERD.svg b/docs/src/images/dimitri-ERD.svg deleted file mode 100644 index 590b30887..000000000 --- a/docs/src/images/dimitri-ERD.svg +++ /dev/null @@ -1,117 +0,0 @@ - - - -%3 - - - -`dimitri_university`.`course` - -`dimitri_university`.`course` - - - -`dimitri_university`.`section` - -`dimitri_university`.`section` - - - -`dimitri_university`.`course`->`dimitri_university`.`section` - - - - -`dimitri_university`.`current_term` - -`dimitri_university`.`current_term` - - - -`dimitri_university`.`department` - -`dimitri_university`.`department` - - - -`dimitri_university`.`department`->`dimitri_university`.`course` - - - - -`dimitri_university`.`student_major` - -`dimitri_university`.`student_major` - - - -`dimitri_university`.`department`->`dimitri_university`.`student_major` - - - - -`dimitri_university`.`enroll` - -`dimitri_university`.`enroll` - - - -`dimitri_university`.`grade` - -`dimitri_university`.`grade` - - - -`dimitri_university`.`enroll`->`dimitri_university`.`grade` - - - - -`dimitri_university`.`letter_grade` - -`dimitri_university`.`letter_grade` - - - -`dimitri_university`.`letter_grade`->`dimitri_university`.`grade` - - - - -`dimitri_university`.`section`->`dimitri_university`.`enroll` - - - - -`dimitri_university`.`student` - -`dimitri_university`.`student` - - - -`dimitri_university`.`student`->`dimitri_university`.`enroll` - - - - -`dimitri_university`.`student`->`dimitri_university`.`student_major` - - - - -`dimitri_university`.`term` - -`dimitri_university`.`term` - - - -`dimitri_university`.`term`->`dimitri_university`.`current_term` - - - - -`dimitri_university`.`term`->`dimitri_university`.`section` - - - - diff --git a/docs/src/images/doc_1-1.png b/docs/src/images/doc_1-1.png deleted file mode 100644 index 4f6f0fa0b..000000000 Binary files a/docs/src/images/doc_1-1.png and /dev/null differ diff --git a/docs/src/images/doc_1-many.png b/docs/src/images/doc_1-many.png deleted file mode 100644 index 32fbbf15b..000000000 Binary files a/docs/src/images/doc_1-many.png and /dev/null differ diff --git a/docs/src/images/doc_many-1.png b/docs/src/images/doc_many-1.png deleted file mode 100644 index 961a306dc..000000000 Binary files a/docs/src/images/doc_many-1.png and /dev/null differ diff --git a/docs/src/images/doc_many-many.png b/docs/src/images/doc_many-many.png deleted file mode 100644 index 3aa484dd6..000000000 Binary files a/docs/src/images/doc_many-many.png and /dev/null differ diff --git a/docs/src/images/how-it-works.png b/docs/src/images/how-it-works.png deleted file mode 100644 index 10c611f3d..000000000 Binary files a/docs/src/images/how-it-works.png and /dev/null differ diff --git a/docs/src/images/install-cmd-prompt.png b/docs/src/images/install-cmd-prompt.png deleted file mode 100644 index 58c9fa964..000000000 Binary files a/docs/src/images/install-cmd-prompt.png and /dev/null differ diff --git a/docs/src/images/install-datajoint-1.png b/docs/src/images/install-datajoint-1.png deleted file mode 100644 index 7aa0a7133..000000000 Binary files a/docs/src/images/install-datajoint-1.png and /dev/null differ diff --git a/docs/src/images/install-datajoint-2.png b/docs/src/images/install-datajoint-2.png deleted file mode 100644 index 970e8c6d4..000000000 Binary files a/docs/src/images/install-datajoint-2.png and /dev/null differ diff --git a/docs/src/images/install-git-1.png b/docs/src/images/install-git-1.png deleted file mode 100644 index 7503dbb61..000000000 Binary files a/docs/src/images/install-git-1.png and /dev/null differ diff --git a/docs/src/images/install-graphviz-1.png b/docs/src/images/install-graphviz-1.png deleted file mode 100644 index dc79e58f1..000000000 Binary files a/docs/src/images/install-graphviz-1.png and /dev/null differ diff --git a/docs/src/images/install-graphviz-2a.png b/docs/src/images/install-graphviz-2a.png deleted file mode 100644 index 394598db7..000000000 Binary files a/docs/src/images/install-graphviz-2a.png and /dev/null differ diff --git a/docs/src/images/install-graphviz-2b.png b/docs/src/images/install-graphviz-2b.png deleted file mode 100644 index 790f88d40..000000000 Binary files a/docs/src/images/install-graphviz-2b.png and /dev/null differ diff --git a/docs/src/images/install-jupyter-1.png b/docs/src/images/install-jupyter-1.png deleted file mode 100644 index 14d697942..000000000 Binary files a/docs/src/images/install-jupyter-1.png and /dev/null differ diff --git a/docs/src/images/install-jupyter-2.png b/docs/src/images/install-jupyter-2.png deleted file mode 100644 index 0d69e6667..000000000 Binary files a/docs/src/images/install-jupyter-2.png and /dev/null differ diff --git a/docs/src/images/install-matplotlib.png b/docs/src/images/install-matplotlib.png deleted file mode 100644 index d092376bb..000000000 Binary files a/docs/src/images/install-matplotlib.png and /dev/null differ diff --git a/docs/src/images/install-pydotplus.png b/docs/src/images/install-pydotplus.png deleted file mode 100644 index 4a0b33f91..000000000 Binary files a/docs/src/images/install-pydotplus.png and /dev/null differ diff --git a/docs/src/images/install-python-advanced-1.png b/docs/src/images/install-python-advanced-1.png deleted file mode 100644 index b07c70e94..000000000 Binary files a/docs/src/images/install-python-advanced-1.png and /dev/null differ diff --git a/docs/src/images/install-python-advanced-2.png b/docs/src/images/install-python-advanced-2.png deleted file mode 100644 index b10be09cc..000000000 Binary files a/docs/src/images/install-python-advanced-2.png and /dev/null differ diff --git a/docs/src/images/install-python-simple.png b/docs/src/images/install-python-simple.png deleted file mode 100644 index ec28cf8cc..000000000 Binary files a/docs/src/images/install-python-simple.png and /dev/null differ diff --git a/docs/src/images/install-run-jupyter-1.png b/docs/src/images/install-run-jupyter-1.png deleted file mode 100644 index cd1e9cfb5..000000000 Binary files a/docs/src/images/install-run-jupyter-1.png and /dev/null differ diff --git a/docs/src/images/install-run-jupyter-2.png b/docs/src/images/install-run-jupyter-2.png deleted file mode 100644 index 7fcee8ee7..000000000 Binary files a/docs/src/images/install-run-jupyter-2.png and /dev/null differ diff --git a/docs/src/images/install-verify-graphviz.png b/docs/src/images/install-verify-graphviz.png deleted file mode 100644 index 6468a98c3..000000000 Binary files a/docs/src/images/install-verify-graphviz.png and /dev/null differ diff --git a/docs/src/images/install-verify-jupyter.png b/docs/src/images/install-verify-jupyter.png deleted file mode 100644 index 73defac5d..000000000 Binary files a/docs/src/images/install-verify-jupyter.png and /dev/null differ diff --git a/docs/src/images/install-verify-python.png b/docs/src/images/install-verify-python.png deleted file mode 100644 index 54ad47290..000000000 Binary files a/docs/src/images/install-verify-python.png and /dev/null differ diff --git a/docs/src/images/join-example1.png b/docs/src/images/join-example1.png deleted file mode 100644 index a518896ef..000000000 Binary files a/docs/src/images/join-example1.png and /dev/null differ diff --git a/docs/src/images/join-example2.png b/docs/src/images/join-example2.png deleted file mode 100644 index c219a6a02..000000000 Binary files a/docs/src/images/join-example2.png and /dev/null differ diff --git a/docs/src/images/join-example3.png b/docs/src/images/join-example3.png deleted file mode 100644 index b2782469e..000000000 Binary files a/docs/src/images/join-example3.png and /dev/null differ diff --git a/docs/src/images/key_source_combination.png b/docs/src/images/key_source_combination.png deleted file mode 100644 index 3db45de37..000000000 Binary files a/docs/src/images/key_source_combination.png and /dev/null differ diff --git a/docs/src/images/map-dataflow.png b/docs/src/images/map-dataflow.png deleted file mode 100644 index 5a3bb34ce..000000000 Binary files a/docs/src/images/map-dataflow.png and /dev/null differ diff --git a/docs/src/images/matched_tuples1.png b/docs/src/images/matched_tuples1.png deleted file mode 100644 index c27593e14..000000000 Binary files a/docs/src/images/matched_tuples1.png and /dev/null differ diff --git a/docs/src/images/matched_tuples2.png b/docs/src/images/matched_tuples2.png deleted file mode 100644 index 673fa5865..000000000 Binary files a/docs/src/images/matched_tuples2.png and /dev/null differ diff --git a/docs/src/images/matched_tuples3.png b/docs/src/images/matched_tuples3.png deleted file mode 100644 index f60e11b50..000000000 Binary files a/docs/src/images/matched_tuples3.png and /dev/null differ diff --git a/docs/src/images/mp-diagram.png b/docs/src/images/mp-diagram.png deleted file mode 100644 index d834726fb..000000000 Binary files a/docs/src/images/mp-diagram.png and /dev/null differ diff --git a/docs/src/images/op-restrict.png b/docs/src/images/op-restrict.png deleted file mode 100644 index e686ac94a..000000000 Binary files a/docs/src/images/op-restrict.png and /dev/null differ diff --git a/docs/src/images/outer-example1.png b/docs/src/images/outer-example1.png deleted file mode 100644 index 0a7c7552f..000000000 Binary files a/docs/src/images/outer-example1.png and /dev/null differ diff --git a/docs/src/images/pipeline-database.png b/docs/src/images/pipeline-database.png deleted file mode 100644 index 035df17cb..000000000 Binary files a/docs/src/images/pipeline-database.png and /dev/null differ diff --git a/docs/src/images/pipeline.png b/docs/src/images/pipeline.png deleted file mode 100644 index 0d91f72e9..000000000 Binary files a/docs/src/images/pipeline.png and /dev/null differ diff --git a/docs/src/images/python_collection.png b/docs/src/images/python_collection.png deleted file mode 100644 index 76fd1d7b0..000000000 Binary files a/docs/src/images/python_collection.png and /dev/null differ diff --git a/docs/src/images/queries_example_diagram.png b/docs/src/images/queries_example_diagram.png deleted file mode 100644 index d6aae1377..000000000 Binary files a/docs/src/images/queries_example_diagram.png and /dev/null differ diff --git a/docs/src/images/query_object_preview.png b/docs/src/images/query_object_preview.png deleted file mode 100644 index 16cedc8fc..000000000 Binary files a/docs/src/images/query_object_preview.png and /dev/null differ diff --git a/docs/src/images/restrict-example1.png b/docs/src/images/restrict-example1.png deleted file mode 100644 index 451e68c58..000000000 Binary files a/docs/src/images/restrict-example1.png and /dev/null differ diff --git a/docs/src/images/restrict-example2.png b/docs/src/images/restrict-example2.png deleted file mode 100644 index aa9a4636b..000000000 Binary files a/docs/src/images/restrict-example2.png and /dev/null differ diff --git a/docs/src/images/restrict-example3.png b/docs/src/images/restrict-example3.png deleted file mode 100644 index e8de7f6ca..000000000 Binary files a/docs/src/images/restrict-example3.png and /dev/null differ diff --git a/docs/src/images/shapes_pipeline.svg b/docs/src/images/shapes_pipeline.svg deleted file mode 100644 index 14572c4ce..000000000 --- a/docs/src/images/shapes_pipeline.svg +++ /dev/null @@ -1,36 +0,0 @@ - - -%3 - - - -Area - - -Area - - - - - -Rectangle - - -Rectangle - - - - - -Rectangle->Area - - - - diff --git a/docs/src/images/spawned-classes-ERD.svg b/docs/src/images/spawned-classes-ERD.svg deleted file mode 100644 index 313841e81..000000000 --- a/docs/src/images/spawned-classes-ERD.svg +++ /dev/null @@ -1,147 +0,0 @@ - - - -%3 - - - -Course - - -Course - - - - - -Section - - -Section - - - - - -Course->Section - - - - -Department - - -Department - - - - - -Department->Course - - - - -StudentMajor - - -StudentMajor - - - - - -Department->StudentMajor - - - - -Term - - -Term - - - - - -Term->Section - - - - -CurrentTerm - - -CurrentTerm - - - - - -Term->CurrentTerm - - - - -LetterGrade - - -LetterGrade - - - - - -Grade - - -Grade - - - - - -LetterGrade->Grade - - - - -Enroll - - -Enroll - - - - - -Enroll->Grade - - - - -Student - - -Student - - - - - -Student->Enroll - - - - -Student->StudentMajor - - - - -Section->Enroll - - - - diff --git a/docs/src/images/union-example1.png b/docs/src/images/union-example1.png deleted file mode 100644 index e693e7170..000000000 Binary files a/docs/src/images/union-example1.png and /dev/null differ diff --git a/docs/src/images/union-example2.png b/docs/src/images/union-example2.png deleted file mode 100644 index 82cc5cc51..000000000 Binary files a/docs/src/images/union-example2.png and /dev/null differ diff --git a/docs/src/images/virtual-module-ERD.svg b/docs/src/images/virtual-module-ERD.svg deleted file mode 100644 index 28eb0c481..000000000 --- a/docs/src/images/virtual-module-ERD.svg +++ /dev/null @@ -1,147 +0,0 @@ - - - -%3 - - - -uni.LetterGrade - - -uni.LetterGrade - - - - - -uni.Grade - - -uni.Grade - - - - - -uni.LetterGrade->uni.Grade - - - - -uni.Course - - -uni.Course - - - - - -uni.Section - - -uni.Section - - - - - -uni.Course->uni.Section - - - - -uni.Term - - -uni.Term - - - - - -uni.Term->uni.Section - - - - -uni.CurrentTerm - - -uni.CurrentTerm - - - - - -uni.Term->uni.CurrentTerm - - - - -uni.Enroll - - -uni.Enroll - - - - - -uni.Section->uni.Enroll - - - - -uni.StudentMajor - - -uni.StudentMajor - - - - - -uni.Enroll->uni.Grade - - - - -uni.Department - - -uni.Department - - - - - -uni.Department->uni.Course - - - - -uni.Department->uni.StudentMajor - - - - -uni.Student - - -uni.Student - - - - - -uni.Student->uni.StudentMajor - - - - -uni.Student->uni.Enroll - - - - diff --git a/docs/src/index.md b/docs/src/index.md index 6e3bf2a2d..63b318a1c 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -1,44 +1,44 @@ -# Welcome to DataJoint for Python! +# DataJoint for Python -DataJoint for Python is a framework for scientific workflow management based on -relational principles. DataJoint is built on the foundation of the relational data -model and prescribes a consistent method for organizing, populating, computing, and -querying data. +DataJoint is an open-source Python framework for building scientific data pipelines. +It implements the **Relational Workflow Model**—a paradigm that extends relational +databases with native support for computational workflows. -DataJoint was initially developed in 2009 by Dimitri Yatsenko in Andreas Tolias' Lab at -Baylor College of Medicine for the distributed processing and management of large -volumes of data streaming from regular experiments. Starting in 2011, DataJoint has -been available as an open-source project adopted by other labs and improved through -contributions from several developers. -Presently, the primary developer of DataJoint open-source software is the company [DataJoint](https://datajoint.com){:target="_blank"}. +## Documentation -## Data Pipeline Example +**User documentation** is available at **[docs.datajoint.com](https://docs.datajoint.com)**, including: -![pipeline](https://raw.githubusercontent.com/datajoint/datajoint-python/master/images/pipeline.png) +- Tutorials and getting started guides +- Concepts and explanations +- How-to guides +- API reference -[Yatsenko et al., bioRxiv 2021](https://doi.org/10.1101/2021.03.30.437358){:target="_blank"} +## This Site -## Getting Started +This site contains **developer documentation** for contributors to the DataJoint codebase: -- Install with Conda +- [Contributing Guide](develop.md) — Development environment setup +- [Architecture](architecture/index.md) — Internal design documentation +- [API Reference](api/) — Auto-generated from source - ```bash - conda install -c conda-forge datajoint - ``` +## Quick Links -- Install with pip +| Resource | Link | +|----------|------| +| User Documentation | [docs.datajoint.com](https://docs.datajoint.com) | +| GitHub Repository | [github.com/datajoint/datajoint-python](https://github.com/datajoint/datajoint-python) | +| PyPI Package | [pypi.org/project/datajoint](https://pypi.org/project/datajoint) | +| Issue Tracker | [GitHub Issues](https://github.com/datajoint/datajoint-python/issues) | +| Community | [DataJoint Slack](https://datajoint.slack.com) | - ```bash - pip install datajoint - ``` +## Installation -- [Quick Start Guide](./quick-start.md) +```bash +pip install datajoint +``` -- [Interactive Tutorials](https://github.com/datajoint/datajoint-tutorials){:target="_blank"} on GitHub Codespaces +## License -- [DataJoint Elements](https://docs.datajoint.com/elements/) - Catalog of example pipelines for neuroscience experiments +DataJoint is released under the [Apache 2.0 License](https://github.com/datajoint/datajoint-python/blob/master/LICENSE). -- Contribute - - [Development Environment](./develop) - - - [Guidelines](https://docs.datajoint.com/about/contribute/) +Copyright 2024 DataJoint Inc. and contributors. diff --git a/docs/src/manipulation/delete.md b/docs/src/manipulation/delete.md deleted file mode 100644 index 4e34c69ce..000000000 --- a/docs/src/manipulation/delete.md +++ /dev/null @@ -1,31 +0,0 @@ -# Delete - -The `delete` method deletes entities from a table and all dependent entries in -dependent tables. - -Delete is often used in conjunction with the [restriction](../query/restrict.md) -operator to define the subset of entities to delete. -Delete is performed as an atomic transaction so that partial deletes never occur. - -## Examples - -```python -# delete all entries from tuning.VonMises -tuning.VonMises.delete() - -# delete entries from tuning.VonMises for mouse 1010 -(tuning.VonMises & 'mouse=1010').delete() - -# delete entries from tuning.VonMises except mouse 1010 -(tuning.VonMises - 'mouse=1010').delete() -``` - -## Deleting from part tables - -Entities in a [part table](../design/tables/master-part.md) are usually removed as a -consequence of deleting the master table. - -To enforce this workflow, calling `delete` directly on a part table produces an error. -In some cases, it may be necessary to override this behavior. -To remove entities from a part table without calling `delete` master, use the argument `force_parts=True`. -To include the corresponding entries in the master table, use the argument `force_masters=True`. diff --git a/docs/src/manipulation/index.md b/docs/src/manipulation/index.md deleted file mode 100644 index 295195778..000000000 --- a/docs/src/manipulation/index.md +++ /dev/null @@ -1,9 +0,0 @@ -# Data Manipulation - -Data **manipulation** operations change the state of the data stored in the database -without modifying the structure of the stored data. -These operations include [insert](insert.md), [delete](delete.md), and -[update](update.md). - -Data manipulation operations in DataJoint respect the -[integrity](../design/integrity.md) constraints. diff --git a/docs/src/manipulation/insert.md b/docs/src/manipulation/insert.md deleted file mode 100644 index c64e55f17..000000000 --- a/docs/src/manipulation/insert.md +++ /dev/null @@ -1,94 +0,0 @@ -# Insert - -The `insert` method of DataJoint table objects inserts entities into the table. - -In Python there is a separate method `insert1` to insert one entity at a time. -The entity may have the form of a Python dictionary with key names matching the -attribute names in the table. - -```python -lab.Person.insert1( - dict(username='alice', - first_name='Alice', - last_name='Cooper')) -``` - -The entity also may take the form of a sequence of values in the same order as the -attributes in the table. - -```python -lab.Person.insert1(['alice', 'Alice', 'Cooper']) -``` - -Additionally, the entity may be inserted as a -[NumPy record array](https://docs.scipy.org/doc/numpy/reference/generated/numpy.record.html#numpy.record) - or [Pandas DataFrame](https://pandas.pydata.org/pandas-docs/stable/generated/pandas.DataFrame.html). - -The `insert` method accepts a sequence or a generator of multiple entities and is used -to insert multiple entities at once. - -```python -lab.Person.insert([ - ['alice', 'Alice', 'Cooper'], - ['bob', 'Bob', 'Dylan'], - ['carol', 'Carol', 'Douglas']]) -``` - -Several optional parameters can be used with `insert`: - - `replace` If `True`, replaces the existing entity. - (Default `False`.) - - `skip_duplicates` If `True`, silently skip duplicate inserts. - (Default `False`.) - - `ignore_extra_fields` If `False`, fields that are not in the heading raise an error. - (Default `False`.) - - `allow_direct_insert` If `True`, allows inserts outside of populate calls. - Applies only in auto-populated tables. - (Default `None`.) - -## Batched inserts - -Inserting a set of entities in a single `insert` differs from inserting the same set of -entities one-by-one in a `for` loop in two ways: - -1. Network overhead is reduced. - Network overhead can be tens of milliseconds per query. - Inserting 1000 entities in a single `insert` call may save a few seconds over - inserting them individually. -2. The insert is performed as an all-or-nothing transaction. - If even one insert fails because it violates any constraint, then none of the - entities in the set are inserted. - -However, inserting too many entities in a single query may run against buffer size or -packet size limits of the database server. -Due to these limitations, performing inserts of very large numbers of entities should -be broken up into moderately sized batches, such as a few hundred at a time. - -## Server-side inserts - -Data inserted into a table often come from other tables already present on the database server. -In such cases, data can be [fetched](../query/fetch.md) from the first table and then -inserted into another table, but this results in transfers back and forth between the -database and the local system. -Instead, data can be inserted from one table into another without transfers between the -database and the local system using [queries](../query/principles.md). - -In the example below, a new schema has been created in preparation for phase two of a -project. -Experimental protocols from the first phase of the project will be reused in the second -phase. -Since the entities are already present on the database in the `Protocol` table of the -`phase_one` schema, we can perform a server-side insert into `phase_two.Protocol` -without fetching a local copy. - -```python -# Server-side inserts are faster... -phase_two.Protocol.insert(phase_one.Protocol) - -# ...than fetching before inserting -protocols = phase_one.Protocol.fetch() -phase_two.Protocol.insert(protocols) -``` diff --git a/docs/src/manipulation/transactions.md b/docs/src/manipulation/transactions.md deleted file mode 100644 index 58b9a3167..000000000 --- a/docs/src/manipulation/transactions.md +++ /dev/null @@ -1,36 +0,0 @@ -# Transactions - -In some cases, a sequence of several operations must be performed as a single -operation: -interrupting the sequence of such operations halfway would leave the data in an invalid -state. -While the sequence is in progress, other processes accessing the database will not see -the partial results until the transaction is complete. -The sequence may include [data queries](../query/principles.md) and -[manipulations](index.md). - -In such cases, the sequence of operations may be enclosed in a transaction. - -Transactions are formed using the `transaction` property of the connection object. -The connection object may be obtained from any table object. -The `transaction` property can then be used as a context manager in Python's `with` -statement. - -For example, the following code inserts matching entries for the master table `Session` -and its part table `Session.Experimenter`. - -```python -# get the connection object -connection = Session.connection - -# insert Session and Session.Experimenter entries in a transaction -with connection.transaction: - key = {'subject_id': animal_id, 'session_time': session_time} - Session.insert1({**key, 'brain_region':region, 'cortical_layer':layer}) - Session.Experimenter.insert1({**key, 'experimenter': username}) -``` - -Here, to external observers, both inserts will take effect together upon exiting from -the `with` block or will not have any effect at all. -For example, if the second insert fails due to an error, the first insert will be -rolled back. diff --git a/docs/src/manipulation/update.md b/docs/src/manipulation/update.md deleted file mode 100644 index 7faa7cb87..000000000 --- a/docs/src/manipulation/update.md +++ /dev/null @@ -1,48 +0,0 @@ -# Cautious Update - -In database programming, the **update** operation refers to modifying the values of -individual attributes in an entity within a table without replacing the entire entity. -Such an in-place update mechanism is not part of DataJoint's data manipulation model, -because it circumvents data -[dependency constraints](../design/integrity.md#referential-integrity). - -This is not to say that data cannot be changed once they are part of a pipeline. -In DataJoint, data is changed by replacing entire entities rather than by updating the -values of their attributes. -The process of deleting existing entities and inserting new entities with corrected -values ensures the [integrity](../design/integrity.md) of the data throughout the -pipeline. - -This approach applies specifically to automated tables -(see [Auto-populated tables](../compute/populate.md)). -However, manual tables are often edited outside DataJoint through other interfaces. -It is up to the user's discretion to allow updates in manual tables, and the user must -be cognizant of the fact that updates will not trigger re-computation of dependent data. - -## Usage - -For some cases, it becomes necessary to deliberately correct existing values where a -user has chosen to accept the above responsibility despite the caution. - -The `update1` method accomplishes this if the record already exists. Note that updates -to primary key values are not allowed. - -The method should only be used to fix problems, and not as part of a regular workflow. -When updating an entry, make sure that any information stored in dependent tables that -depends on the update values is properly updated as well. - -## Examples - -```python -# with record as a dict specifying the primary and -# secondary attribute values -table.update1(record) - -# update value in record with id as primary key -table.update1({'id': 1, 'value': 3}) - -# reset value to default with id as primary key -table.update1({'id': 1, 'value': None}) -# or -table.update1({'id': 1}) -``` diff --git a/docs/src/publish-data.md b/docs/src/publish-data.md deleted file mode 100644 index 3ec2d7211..000000000 --- a/docs/src/publish-data.md +++ /dev/null @@ -1,34 +0,0 @@ -# Publishing Data - -DataJoint is a framework for building data pipelines that support rigorous flow of -structured data between experimenters, data scientists, and computing agents *during* -data acquisition and processing within a centralized project. -Publishing final datasets for the outside world may require additional steps and -conversion. - -## Provide access to a DataJoint server - -One approach for publishing data is to grant public access to an existing pipeline. -Then public users will be able to query the data pipelines using DataJoint's query -language and output interfaces just like any other users of the pipeline. -For security, this may require synchronizing the data onto a separate read-only public -server. - -## Containerizing as a DataJoint pipeline - -Containerization platforms such as [Docker](https://www.docker.com/) allow convenient -distribution of environments including database services and data. -It is convenient to publish DataJoint pipelines as a docker container that deploys the -populated DataJoint pipeline. -One example of publishing a DataJoint pipeline as a docker container is -> Sinz, F., Ecker, A.S., Fahey, P., Walker, E., Cobos, E., Froudarakis, E., Yatsenko, D., Pitkow, Z., Reimer, J. and Tolias, A., 2018. Stimulus domain transfer in recurrent models for large scale cortical population prediction on video. In Advances in Neural Information Processing Systems (pp. 7198-7209). https://www.biorxiv.org/content/early/2018/10/25/452672 - -The code and the data can be found at [https://github.com/sinzlab/Sinz2018_NIPS](https://github.com/sinzlab/Sinz2018_NIPS). - -## Exporting into a collection of files - -Another option for publishing and archiving data is to export the data from the -DataJoint pipeline into a collection of files. -DataJoint provides features for exporting and importing sections of the pipeline. -Several ongoing projects are implementing the capability to export from DataJoint -pipelines into [Neurodata Without Borders](https://www.nwb.org/) files. diff --git a/docs/src/query/aggregation.md b/docs/src/query/aggregation.md deleted file mode 100644 index e47fd0b33..000000000 --- a/docs/src/query/aggregation.md +++ /dev/null @@ -1,29 +0,0 @@ -# Aggr - -**Aggregation**, performed with the `aggr` operator, is a special form of `proj` with -the additional feature of allowing aggregation calculations on another table. -It has the form `tab.aggr(other, ...)` where `other` is another table. -Without the argument `other`, `aggr` and `proj` are exactly equivalent. -Aggregation allows adding calculated attributes to each entity in `tab` based on -aggregation functions over attributes in the -[matching](./operators.md#matching-entities) entities of `other`. - -Aggregation functions include `count`, `sum`, `min`, `max`, `avg`, `std`, `variance`, -and others. -Aggregation functions can only be used in the definitions of new attributes within the -`aggr` operator. - -As with `proj`, the output of `aggr` has the same entity class, the same primary key, -and the same number of elements as `tab`. -Primary key attributes are always included in the output and may be renamed, just like -in `proj`. - -## Examples - -```python -# Number of students in each course section -Section.aggr(Enroll, n="count(*)") - -# Average grade in each course -Course.aggr(Grade * LetterGrade, avg_grade="avg(points)") -``` diff --git a/docs/src/query/example-schema.md b/docs/src/query/example-schema.md deleted file mode 100644 index 063e36574..000000000 --- a/docs/src/query/example-schema.md +++ /dev/null @@ -1,112 +0,0 @@ -# Example Schema - -The example schema below contains data for a university enrollment system. -Information about students, departments, courses, etc. are organized in multiple tables. - -Warning: - Empty primary keys, such as in the `CurrentTerm` table, are not yet supported by - DataJoint. - This feature will become available in a future release. - See [Issue #113](https://github.com/datajoint/datajoint-python/issues/113) for more - information. - -```python -@schema -class Student (dj.Manual): -definition = """ -student_id : int unsigned # university ID ---- -first_name : varchar(40) -last_name : varchar(40) -sex : enum('F', 'M', 'U') -date_of_birth : date -home_address : varchar(200) # street address -home_city : varchar(30) -home_state : char(2) # two-letter abbreviation -home_zipcode : char(10) -home_phone : varchar(14) -""" - -@schema -class Department (dj.Manual): -definition = """ -dept : char(6) # abbreviated department name, e.g. BIOL ---- -dept_name : varchar(200) # full department name -dept_address : varchar(200) # mailing address -dept_phone : varchar(14) -""" - -@schema -class StudentMajor (dj.Manual): -definition = """ --> Student ---- --> Department -declare_date : date # when student declared her major -""" - -@schema -class Course (dj.Manual): -definition = """ --> Department -course : int unsigned # course number, e.g. 1010 ---- -course_name : varchar(200) # e.g. "Cell Biology" -credits : decimal(3,1) # number of credits earned by completing the course -""" - -@schema -class Term (dj.Manual): -definition = """ -term_year : year -term : enum('Spring', 'Summer', 'Fall') -""" - -@schema -class Section (dj.Manual): -definition = """ --> Course --> Term -section : char(1) ---- -room : varchar(12) # building and room code -""" - -@schema -class CurrentTerm (dj.Manual): -definition = """ ---- --> Term -""" - -@schema -class Enroll (dj.Manual): -definition = """ --> Section --> Student -""" - -@schema -class LetterGrade (dj.Manual): -definition = """ -grade : char(2) ---- -points : decimal(3,2) -""" - -@schema -class Grade (dj.Manual): -definition = """ --> Enroll ---- --> LetterGrade -""" -``` - -## Example schema diagram - -![University example schema](../images/queries_example_diagram.png){: style="align:center"} - -Example schema for a university database. -Tables contain data on students, departments, courses, etc. diff --git a/docs/src/query/fetch.md b/docs/src/query/fetch.md deleted file mode 100644 index 105d70084..000000000 --- a/docs/src/query/fetch.md +++ /dev/null @@ -1,126 +0,0 @@ -# Fetch - -Data queries in DataJoint comprise two distinct steps: - -1. Construct the `query` object to represent the required data using tables and -[operators](operators.md). -2. Fetch the data from `query` into the workspace of the host language -- described in -this section. - -Note that entities returned by `fetch` methods are not guaranteed to be sorted in any -particular order unless specifically requested. -Furthermore, the order is not guaranteed to be the same in any two queries, and the -contents of two identical queries may change between two sequential invocations unless -they are wrapped in a transaction. -Therefore, if you wish to fetch matching pairs of attributes, do so in one `fetch` call. - -The examples below are based on the [example schema](example-schema.md) for this part -of the documentation. - -## Entire table - -The following statement retrieves the entire table as a NumPy -[recarray](https://docs.scipy.org/doc/numpy/reference/generated/numpy.recarray.html). - -```python -data = query.fetch() -``` - -To retrieve the data as a list of `dict`: - -```python -data = query.fetch(as_dict=True) -``` - -In some cases, the amount of data returned by fetch can be quite large; in these cases -it can be useful to use the `size_on_disk` attribute to determine if running a bare -fetch would be wise. -Please note that it is only currently possible to query the size of entire tables -stored directly in the database at this time. - -## As separate variables - -```python -name, img = query.fetch1('name', 'image') # when query has exactly one entity -name, img = query.fetch('name', 'image') # [name, ...] [image, ...] -``` - -## Primary key values - -```python -keydict = tab.fetch1("KEY") # single key dict when tab has exactly one entity -keylist = tab.fetch("KEY") # list of key dictionaries [{}, ...] -``` - -`KEY` can also used when returning attribute values as separate variables, such that -one of the returned variables contains the entire primary keys. - -## Sorting and limiting the results - -To sort the result, use the `order_by` keyword argument. - -```python -# ascending order: -data = query.fetch(order_by='name') -# descending order: -data = query.fetch(order_by='name desc') -# by name first, year second: -data = query.fetch(order_by=('name desc', 'year')) -# sort by the primary key: -data = query.fetch(order_by='KEY') -# sort by name but for same names order by primary key: -data = query.fetch(order_by=('name', 'KEY desc')) -``` - -The `order_by` argument can be a string specifying the attribute to sort by. By default -the sort is in ascending order. Use `'attr desc'` to sort in descending order by -attribute `attr`. The value can also be a sequence of strings, in which case, the sort -performed on all the attributes jointly in the order specified. - -The special attribute name `'KEY'` represents the primary key attributes in order that -they appear in the index. Otherwise, this name can be used as any other argument. - -If an attribute happens to be a SQL reserved word, it needs to be enclosed in -backquotes. For example: - -```python -data = query.fetch(order_by='`select` desc') -``` - -The `order_by` value is eventually passed to the `ORDER BY` -[clause](https://dev.mysql.com/doc/refman/5.7/en/order-by-optimization.html). - -Similarly, the `limit` and `offset` arguments can be used to limit the result to a -subset of entities. - -For example, one could do the following: - -```python -data = query.fetch(order_by='name', limit=10, offset=5) -``` - -Note that an `offset` cannot be used without specifying a `limit` as well. - -## Usage with Pandas - -The [pandas library](http://pandas.pydata.org/) is a popular library for data analysis -in Python which can easily be used with DataJoint query results. -Since the records returned by `fetch()` are contained within a `numpy.recarray`, they -can be easily converted to `pandas.DataFrame` objects by passing them into the -`pandas.DataFrame` constructor. -For example: - -```python -import pandas as pd -frame = pd.DataFrame(tab.fetch()) -``` - -Calling `fetch()` with the argument `format="frame"` returns results as -`pandas.DataFrame` objects indexed by the table's primary key attributes. - -```python -frame = tab.fetch(format="frame") -``` - -Returning results as a `DataFrame` is not possible when fetching a particular subset of -attributes or when `as_dict` is set to `True`. diff --git a/docs/src/query/iteration.md b/docs/src/query/iteration.md deleted file mode 100644 index 60d95f107..000000000 --- a/docs/src/query/iteration.md +++ /dev/null @@ -1,36 +0,0 @@ -# Iteration - -The DataJoint model primarily handles data as sets, in the form of tables. However, it -can sometimes be useful to access or to perform actions such as visualization upon -individual entities sequentially. In DataJoint this is accomplished through iteration. - -In the simple example below, iteration is used to display the names and values of the -attributes of each entity in the simple table or table expression. - -```python -for entity in table: - print(entity) -``` - -This example illustrates the function of the iterator: DataJoint iterates through the -whole table expression, returning the entire entity during each step. In this case, -each entity will be returned as a `dict` containing all attributes. - -At the start of the above loop, DataJoint internally fetches only the primary keys of -the entities. Since only the primary keys are needed to distinguish between entities, -DataJoint can then iterate over the list of primary keys to execute the loop. At each -step of the loop, DataJoint uses a single primary key to fetch an entire entity for use -in the iteration, such that `print(entity)` will print all attributes of each entity. -By first fetching only the primary keys and then fetching each entity individually, -DataJoint saves memory at the cost of network overhead. This can be particularly useful -for tables containing large amounts of data in secondary attributes. - -The memory savings of the above syntax may not be worth the additional network overhead -in all cases, such as for tables with little data stored as secondary attributes. In -the example below, DataJoint fetches all of the attributes of each entity in a single -call and then iterates over the list of entities stored in memory. - -```python -for entity in table.fetch(as_dict=True): - print(entity) -``` diff --git a/docs/src/query/join.md b/docs/src/query/join.md deleted file mode 100644 index d0ab0eae0..000000000 --- a/docs/src/query/join.md +++ /dev/null @@ -1,37 +0,0 @@ -# Join - -## Join operator `*` - -The Join operator `A * B` combines the matching information in `A` and `B`. -The result contains all matching combinations of entities from both arguments. - -### Principles of joins - -1. The operands `A` and `B` must be **join-compatible**. -2. The primary key of the result is the union of the primary keys of the operands. - -### Examples of joins - -Example 1 : When the operands have no common attributes, the result is the cross -product -- all combinations of entities. - -![join-example1](../images/join-example1.png){: style="width:464px; align:center"} - -Example 2 : When the operands have common attributes, only entities with matching -values are kept. - -![join-example2](../images/join-example2.png){: style="width:689px; align:center"} - -Example 3 : Joining on secondary attribute. - -![join-example3](../images/join-example3.png){: style="width:689px; align:center"} - -### Properties of join - -1. When `A` and `B` have the same attributes, the join `A * B` becomes equivalent to -the set intersection `A` ∩ `B`. - Hence, DataJoint does not need a separate intersection operator. - -2. Commutativity: `A * B` is equivalent to `B * A`. - -3. Associativity: `(A * B) * C` is equivalent to `A * (B * C)`. diff --git a/docs/src/query/operators.md b/docs/src/query/operators.md deleted file mode 100644 index ee3549f35..000000000 --- a/docs/src/query/operators.md +++ /dev/null @@ -1,395 +0,0 @@ -# Operators - -[Data queries](principles.md) have the form of expressions using operators to derive -the desired table. -The expressions themselves do not contain any data. -They represent the desired data symbolically. - -Once a query is formed, the [fetch](fetch.md) methods are used to bring the data into -the local workspace. -Since the expressions are only symbolic representations, repeated `fetch` calls may -yield different results as the state of the database is modified. - -DataJoint implements a complete algebra of operators on tables: - -| operator | notation | meaning | -|------------------------------|----------------|-------------------------------------------------------------------------| -| [join](#join) | A * B | All matching information from A and B | -| [restriction](#restriction) | A & cond | The subset of entities from A that meet the condition | -| [restriction](#restriction) | A - cond | The subset of entities from A that do not meet the condition | -| [proj](#proj) | A.proj(...) | Selects and renames attributes from A or computes new attributes | -| [aggr](#aggr) | A.aggr(B, ...) | Same as projection with computations based on matching information in B | -| [union](#union) | A + B | All unique entities from both A and B | -| [universal set](#universal-set)\*| dj.U() | All unique entities from both A and B | -| [top](#top)\*| dj.Top() | The top rows of A - -\*While not technically query operators, it is useful to discuss Universal Set and Top in the -same context. - -## Principles of relational algebra - -DataJoint's algebra improves upon the classical relational algebra and upon other query -languages to simplify and enhance the construction and interpretation of precise and -efficient data queries. - -1. **Entity integrity**: Data are represented and manipulated in the form of tables -representing [well-formed entity sets](../design/integrity.md). - This applies to the inputs and outputs of query operators. - The output of a query operator is an entity set with a well-defined entity type, a - primary key, unique attribute names, etc. -2. **Algebraic closure**: All operators operate on entity sets and yield entity sets. - Thus query expressions may be used as operands in other expressions or may be - assigned to variables to be used in other expressions. -3. **Attributes are identified by names**: All attributes have explicit names. - This includes results of queries. - Operators use attribute names to determine how to perform the operation. - The order of the attributes is not significant. - -## Matching entities - -Binary operators in DataJoint are based on the concept of **matching entities**; this -phrase will be used throughout the documentation. - - Two entities **match** when they have no common attributes or when their common - attributes contain the same values. - -Here **common attributes** are those that have the same names in both entities. -It is usually assumed that the common attributes are of compatible datatypes to allow -equality comparisons. - -Another way to phrase the same definition is - - Two entities match when they have no common attributes whose values differ. - -It may be conceptually convenient to imagine that all tables always have an additional -invisible attribute, `omega` whose domain comprises only one value, 1. -Then the definition of matching entities is simplified: - - Two entities match when their common attributes contain the same values. - -Matching entities can be **merged** into a single entity without any conflicts of -attribute names and values. - -### Examples - -This is a matching pair of entities: - -![matched_tuples1](../images/matched_tuples1.png){: style="width:366px"} - -and so is this one: - -![matched_tuples2](../images/matched_tuples2.png){: style="width:366px"} - -but these entities do *not* match: - -![matched_tuples3](../images/matched_tuples3.png){: style="width:366px"} - -## Join compatibility - -All binary operators with other tables as their two operands require that the operands -be **join-compatible**, which means that: - -1. All common attributes in both operands (attributes with the same name) must be part -of either the primary key or a foreign key. -2. All common attributes in the two relations must be of a compatible datatype for -equality comparisons. - -## Restriction - -The restriction operator `A & cond` selects the subset of entities from `A` that meet -the condition `cond`. The exclusion operator `A - cond` selects the complement of -restriction, i.e. the subset of entities from `A` that do not meet the condition -`cond`. This means that the restriction and exclusion operators are complementary. -The same query could be constructed using either `A & cond` or `A - Not(cond)`. - -
-![Restriction and exclusion.](../../../images/concepts-operators-restriction.png){: style="height:200px"} -
- -The condition `cond` may be one of the following: - -=== "Python" - - - another table - - a mapping, e.g. `dict` - - an expression in a character string - - a collection of conditions as a `list`, `tuple`, or Pandas `DataFrame` - - a Boolean expression (`True` or `False`) - - an `AndList` - - a `Not` object - - a query expression - -??? Warning "Permissive Operators" - - To circumvent compatibility checks, DataJoint offers permissive operators for - Restriction (`^`) and Join (`@`). Use with Caution. - -## Proj - -The `proj` operator represents **projection** and is used to select attributes -(columns) from a table, to rename them, or to create new calculated attributes. - -1. A simple projection *selects a subset of attributes* of the original -table, which may not include the [primary key](../concepts/glossary#primary-key). - -2. A more complex projection *renames an attribute* in another table. This could be -useful when one table should be referenced multiple times in another. A user table, -could contain all personnel. A project table references one person for the lead and -another the coordinator, both referencing the common personnel pool. - -3. Projection can also perform calculations (as available in -[MySQL](https://dev.mysql.com/doc/refman/5.7/en/functions.html)) on a single attribute. - -## Aggr - -**Aggregation** is a special form of `proj` with the added feature of allowing - aggregation calculations on another table. It has the form `table.aggr - (other, ...)` where `other` is another table. Aggregation allows adding calculated - attributes to each entity in `table` based on aggregation functions over attributes - in the matching entities of `other`. - -Aggregation functions include `count`, `sum`, `min`, `max`, `avg`, `std`, `variance`, -and others. - -## Union - -The result of the union operator `A + B` contains all the entities from both operands. - -[Entity normalization](../design/normalization) requires that `A` and `B` are of the same type, -with with the same [primary key](../concepts/glossary#primary-key), using homologous -attributes. Without secondary attributes, the result is the simple set union. With -secondary attributes, they must have the same names and datatypes. The two operands -must also be **disjoint**, without any duplicate primary key values across both inputs. -These requirements prevent ambiguity of attribute values and preserve entity identity. - -??? Note "Principles of union" - - 1. As in all operators, the order of the attributes in the operands is not - significant. - - 2. Operands `A` and `B` must have the same primary key attributes. Otherwise, an - error will be raised. - - 3. Operands `A` and `B` may not have any common non-key attributes. Otherwise, an - error will be raised. - - 4. The result `A + B` will have the same primary key as `A` and `B`. - - 5. The result `A + B` will have all the non-key attributes from both `A` and `B`. - - 6. For entities that are found in both `A` and `B` (based on the primary key), the - secondary attributes will be filled from the corresponding entities in `A` and - `B`. - - 7. For entities that are only found in either `A` or `B`, the other operand's - secondary attributes will filled with null values. - -For union, order does not matter. - -
-![Union Example 1](../../../images/concepts-operators-union1.png){: style="height:200px"} -
-
-![Union Example 2](../../../images/concepts-operators-union2.png){: style="height:200px"} -
- -??? Note "Properties of union" - - 1. Commutative: `A + B` is equivalent to `B + A`. - 2. Associative: `(A + B) + C` is equivalent to `A + (B + C)`. - -## Universal Set - -All of the above operators are designed to preserve their input type. Some queries may -require creating a new entity type not already represented by existing tables. This -means that the new type must be defined as part of the query. - -Universal sets fulfill this role using `dj.U` notation. They denote the set of all -possible entities with given attributes of any possible datatype. Attributes of -universal sets are allowed to be matched to any namesake attributes, even those that do -not come from the same initial source. - -Universal sets should be used sparingly when no suitable base tables already exist. In -some cases, defining a new base table can make queries clearer and more semantically -constrained. - -The examples below will use the table definitions in [table tiers](../reproduce/table-tiers). - - - -## Top - -Similar to the universal set operator, the top operator uses `dj.Top` notation. It is used to -restrict a query by the given `limit`, `order_by`, and `offset` parameters: - -```python -Session & dj.Top(limit=10, order_by='session_date') -``` - -The result of this expression returns the first 10 rows of `Session` and sorts them -by their `session_date` in ascending order. - -### `order_by` - -| Example | Description | -|-------------------------------------------|---------------------------------------------------------------------------------| -| `order_by="session_date DESC"` | Sort by `session_date` in *descending* order | -| `order_by="KEY"` | Sort by the primary key | -| `order_by="KEY DESC"` | Sort by the primary key in *descending* order | -| `order_by=["subject_id", "session_date"]` | Sort by `subject_id`, then sort matching `subject_id`s by their `session_date` | - -The default values for `dj.Top` parameters are `limit=1`, `order_by="KEY"`, and `offset=0`. - -## Restriction - -`&` and `-` operators permit restriction. - -### By a mapping - -For a [Session table](../reproduce/table-tiers#manual-tables), that has the attribute -`session_date`, we can restrict to sessions from January 1st, 2022: - -```python -Session & {'session_date': "2022-01-01"} -``` - -If there were any typos (e.g., using `sess_date` instead of `session_date`), our query -will return all of the entities of `Session`. - -### By a string - -Conditions may include arithmetic operations, functions, range tests, etc. Restriction -of table `A` by a string containing an attribute not found in table `A` produces an -error. - -```python -Session & 'user = "Alice"' # (1) -Session & 'session_date >= "2022-01-01"' # (2) -``` - -1. All the sessions performed by Alice -2. All of the sessions on or after January 1st, 2022 - -### By a collection - -When `cond` is a collection of conditions, the conditions are applied by logical -disjunction (logical OR). Restricting a table by a collection will return all entities -that meet *any* of the conditions in the collection. - -For example, if we restrict the `Session` table by a collection containing two -conditions, one for user and one for date, the query will return any sessions with a -matching user *or* date. - -A collection can be a list, a tuple, or a Pandas `DataFrame`. - -``` python -cond_list = ['user = "Alice"', 'session_date = "2022-01-01"'] # (1) -cond_tuple = ('user = "Alice"', 'session_date = "2022-01-01"') # (2) -import pandas as pd -cond_frame = pd.DataFrame(data={'user': ['Alice'], 'session_date': ['2022-01-01']}) # (3) - -Session() & ['user = "Alice"', 'session_date = "2022-01-01"'] -``` - -1. A list -2. A tuple -3. A data frame - -`dj.AndList` represents logical conjunction(logical AND). Restricting a table by an -`AndList` will return all entities that meet *all* of the conditions in the list. `A & -dj.AndList([c1, c2, c3])` is equivalent to `A & c1 & c2 & c3`. - -```python -Student() & dj.AndList(['user = "Alice"', 'session_date = "2022-01-01"']) -``` - -The above will show all the sessions that Alice conducted on the given day. - -### By a `Not` object - -The special function `dj.Not` represents logical negation, such that `A & dj.Not -(cond)` is equivalent to `A - cond`. - -### By a query - -Restriction by a query object is a generalization of restriction by a table. The example -below creates a query object corresponding to all the users named Alice. The `Session` -table is then restricted by the query object, returning all the sessions performed by -Alice. - -``` python -query = User & 'user = "Alice"' -Session & query -``` - -## Proj - -Renaming an attribute in python can be done via keyword arguments: - -```python -table.proj(new_attr='old_attr') -``` - -This can be done in the context of a table definition: - -```python -@schema -class Session(dj.Manual): - definition = """ - # Experiment Session - -> Animal - session : smallint # session number for the animal - --- - session_datetime : datetime # YYYY-MM-DD HH:MM:SS - session_start_time : float # seconds relative to session_datetime - session_end_time : float # seconds relative to session_datetime - -> User.proj(experimenter='username') - -> User.proj(supervisor='username') - """ -``` - -Or to rename multiple values in a table with the following syntax: -`Table.proj(*existing_attributes,*renamed_attributes)` - -```python -Session.proj('session','session_date',start='session_start_time',end='session_end_time') -``` - -Projection can also be used to to compute new attributes from existing ones. - -```python -Session.proj(duration='session_end_time-session_start_time') & 'duration > 10' -``` - -## Aggr - -For more complicated calculations, we can use aggregation. - -``` python -Subject.aggr(Session,n="count(*)") # (1) -Subject.aggr(Session,average_start="avg(session_start_time)") # (2) -``` - -1. Number of sessions per subject. -2. Average `session_start_time` for each subject - - - -## Universal set - -Universal sets offer the complete list of combinations of attributes. - -``` python -# All home cities of students -dj.U('laser_wavelength', 'laser_power') & Scan # (1) -dj.U('laser_wavelength', 'laser_power').aggr(Scan, n="count(*)") # (2) -dj.U().aggr(Session, n="max(session)") # (3) -``` - -1. All combinations of wavelength and power. -2. Total number of scans for each combination. -3. Largest session number. - -`dj.U()`, as shown in the last example above, is often useful for integer IDs. -For an example of this process, see the source code for -[Element Array Electrophysiology's `insert_new_params`](https://docs.datajoint.com/elements/element-array-ephys/latest/api/element_array_ephys/ephys_acute/#element_array_ephys.ephys_acute.ClusteringParamSet.insert_new_params). diff --git a/docs/src/query/principles.md b/docs/src/query/principles.md deleted file mode 100644 index 9b9fd284d..000000000 --- a/docs/src/query/principles.md +++ /dev/null @@ -1,81 +0,0 @@ -# Query Principles - -**Data queries** retrieve data from the database. -A data query is performed with the help of a **query object**, which is a symbolic -representation of the query that does not in itself contain any actual data. -The simplest query object is an instance of a **table class**, representing the -contents of an entire table. - -For example, if `experiment.Session` is a DataJoint table class, you can create a query -object to retrieve its entire contents as follows: - -```python -query = experiment.Session() -``` - -More generally, a query object may be formed as a **query expression** constructed by -applying [operators](operators.md) to other query objects. - -For example, the following query retrieves information about all experiments and scans -for mouse 102 (excluding experiments with no scans): - -```python -query = experiment.Session * experiment.Scan & 'animal_id = 102' -``` - -Note that for brevity, query operators can be applied directly to class objects rather -than instance objects so that `experiment.Session` may be used in place of -`experiment.Session()`. - -You can preview the contents of the query in Python, Jupyter Notebook, or MATLAB by -simply displaying the object. -In the image below, the object `query` is first defined as a restriction of the table -`EEG` by values of the attribute `eeg_sample_rate` greater than 1000 Hz. -Displaying the object gives a preview of the entities that will be returned by `query`. -Note that this preview only lists a few of the entities that will be returned. -Also, the preview does not contain any data for attributes of datatype `blob`. - -![Query object preview](../images/query_object_preview.png){: style="align:center"} - -Defining a query object and previewing the entities returned by the query. - -Once the desired query object is formed, the query can be executed using its -[fetch](fetch.md) methods. -To **fetch** means to transfer the data represented by the query object from the -database server into the workspace of the host language. - -```python -s = query.fetch() -``` - -Here fetching from the `query` object produces the NumPy record array `s` of the -queried data. - -## Checking for returned entities - -The preview of the query object shown above displayed only a few of the entities -returned by the query but also displayed the total number of entities that would be -returned. -It can be useful to know the number of entities returned by a query, or even whether a -query will return any entities at all, without having to fetch all the data themselves. - -The `bool` function applied to a query object evaluates to `True` if the query returns -any entities and to `False` if the query result is empty. - -The `len` function applied to a query object determines the number of entities returned -by the query. - -```python -# number of sessions since the start of 2018. -n = len(Session & 'session_date >= "2018-01-01"') -``` - -## Normalization in queries - -Query objects adhere to entity [entity normalization](../design/normalization.md) just -like the stored tables do. -The result of a query is a well-defined entity set with an readily identifiable entity -class and designated primary attributes that jointly distinguish any two entities from -each other. -The query [operators](operators.md) are designed to keep the result normalized even in -complex query expressions. diff --git a/docs/src/query/project.md b/docs/src/query/project.md deleted file mode 100644 index 99e5749c7..000000000 --- a/docs/src/query/project.md +++ /dev/null @@ -1,68 +0,0 @@ -# Proj - -The `proj` operator represents **projection** and is used to select attributes -(columns) from a table, to rename them, or to create new calculated attributes. - -## Simple projection - -The simple projection selects a subset of attributes of the original table. -However, the primary key attributes are always included. - -Using the [example schema](example-schema.md), let table `department` have attributes -**dept**, *dept_name*, *dept_address*, and *dept_phone*. -The primary key attribute is in bold. - -Then `department.proj()` will have attribute **dept**. - -`department.proj('dept')` will have attribute **dept**. - -`department.proj('dept_name', 'dept_phone')` will have attributes **dept**, -*dept_name*, and *dept_phone*. - -## Renaming - -In addition to selecting attributes, `proj` can rename them. -Any attribute can be renamed, including primary key attributes. - -This is done using keyword arguments: -`tab.proj(new_attr='old_attr')` - -For example, let table `tab` have attributes **mouse**, **session**, *session_date*, -*stimulus*, and *behavior*. -The primary key attributes are in bold. - -Then - -```python -tab.proj(animal='mouse', 'stimulus') -``` - -will have attributes **animal**, **session**, and *stimulus*. - -Renaming is often used to control the outcome of a [join](join.md). -For example, let `tab` have attributes **slice**, and **cell**. -Then `tab * tab` will simply yield `tab`. -However, - -```python -tab * tab.proj(other='cell') -``` - -yields all ordered pairs of all cells in each slice. - -## Calculations - -In addition to selecting or renaming attributes, `proj` can compute new attributes from -existing ones. - -For example, let `tab` have attributes `mouse`, `scan`, `surface_z`, and `scan_z`. -To obtain the new attribute `depth` computed as `scan_z - surface_z` and then to -restrict to `depth > 500`: - -```python -tab.proj(depth='scan_z-surface_z') & 'depth > 500' -``` - -Calculations are passed to SQL and are not parsed by DataJoint. -For available functions, you may refer to the -[MySQL documentation](https://dev.mysql.com/doc/refman/8.0/en/functions.html). diff --git a/docs/src/query/query-caching.md b/docs/src/query/query-caching.md deleted file mode 100644 index 124381b63..000000000 --- a/docs/src/query/query-caching.md +++ /dev/null @@ -1,42 +0,0 @@ -# Query Caching - -Query caching allows avoiding repeated queries to the database by caching the results -locally for faster retrieval. - -To enable queries, set the query cache local path in `dj.config`, create the directory, -and activate the query caching. - -```python -# set the query cache path -dj.config['query_cache'] = os.path.expanduser('~/dj_query_cache') - -# access the active connection object for the tables -conn = dj.conn() # if queries co-located with tables -conn = module.schema.connection # if schema co-located with tables -conn = module.table.connection # most flexible - -# activate query caching for a namespace called 'main' -conn.set_query_cache(query_cache='main') -``` - -The `query_cache` argument is an arbitrary string serving to differentiate cache -states; setting a new value will effectively start a new cache, triggering retrieval of -new values once. - -To turn off query caching, use the following: - -```python -conn.set_query_cache(query_cache=None) -# or -conn.set_query_cache() -``` - -While query caching is enabled, any insert or delete calls and any transactions are -disabled and will raise an error. This ensures that stale data are not used for -updating the database in violation of data integrity. - -To clear and remove the query cache, use the following: - -```python -conn.purge_query_cache() -``` diff --git a/docs/src/query/restrict.md b/docs/src/query/restrict.md deleted file mode 100644 index f8b61e641..000000000 --- a/docs/src/query/restrict.md +++ /dev/null @@ -1,205 +0,0 @@ -# Restriction - -## Restriction operators `&` and `-` - -The restriction operator `A & cond` selects the subset of entities from `A` that meet -the condition `cond`. -The exclusion operator `A - cond` selects the complement of restriction, i.e. the -subset of entities from `A` that do not meet the condition `cond`. - -Restriction and exclusion. - -![Restriction and exclusion](../images/op-restrict.png){: style="width:400px; align:center"} - -The condition `cond` may be one of the following: - -+ another table -+ a mapping, e.g. `dict` -+ an expression in a character string -+ a collection of conditions as a `list`, `tuple`, or Pandas `DataFrame` -+ a Boolean expression (`True` or `False`) -+ an `AndList` -+ a `Not` object -+ a query expression - -As the restriction and exclusion operators are complementary, queries can be -constructed using both operators that will return the same results. -For example, the queries `A & cond` and `A - Not(cond)` will return the same entities. - -## Restriction by a table - -When restricting table `A` with another table, written `A & B`, the two tables must be -**join-compatible** (see `join-compatible` in [Operators](./operators.md)). -The result will contain all entities from `A` for which there exist a matching entity -in `B`. -Exclusion of table `A` with table `B`, or `A - B`, will contain all entities from `A` -for which there are no matching entities in `B`. - -Restriction by another table. - -![Restriction by another table](../images/restrict-example1.png){: style="width:546px; align:center"} - -Exclusion by another table. - -![Exclusion by another table](../images/diff-example1.png){: style="width:539px; align:center"} - -### Restriction by a table with no common attributes - -Restriction of table `A` with another table `B` having none of the same attributes as -`A` will simply return all entities in `A`, unless `B` is empty as described below. -Exclusion of table `A` with `B` having no common attributes will return no entities, -unless `B` is empty as described below. - -Restriction by a table having no common attributes. - -![Restriction by a table with no common attributes](../images/restrict-example2.png){: style="width:571px; align:center"} - -Exclusion by a table having no common attributes. - -![Exclusion by a table having no common attributes](../images/diff-example2.png){: style="width:571px; align:center"} - -### Restriction by an empty table - -Restriction of table `A` with an empty table will return no entities regardless of -whether there are any matching attributes. -Exclusion of table `A` with an empty table will return all entities in `A`. - -Restriction by an empty table. - -![Restriction by an empty table](../images/restrict-example3.png){: style="width:563px; align:center"} - -Exclusion by an empty table. - -![Exclusion by an empty table](../images/diff-example3.png){: style="width:571px; align:center"} - -## Restriction by a mapping - -A key-value mapping may be used as an operand in restriction. -For each key that is an attribute in `A`, the paired value is treated as part of an -equality condition. -Any key-value pairs without corresponding attributes in `A` are ignored. - -Restriction by an empty mapping or by a mapping with no keys matching the attributes in -`A` will return all the entities in `A`. -Exclusion by an empty mapping or by a mapping with no matches will return no entities. - -For example, let's say that table `Session` has the attribute `session_date` of -[datatype](../design/tables/attributes.md) `datetime`. -You are interested in sessions from January 1st, 2018, so you write the following -restriction query using a mapping. - -```python -Session & {'session_date': "2018-01-01"} -``` - -Our mapping contains a typo omitting the final `e` from `session_date`, so no keys in -our mapping will match any attribute in `Session`. -As such, our query will return all of the entities of `Session`. - -## Restriction by a string - -Restriction can be performed when `cond` is an explicit condition on attribute values, -expressed as a string. -Such conditions may include arithmetic operations, functions, range tests, etc. -Restriction of table `A` by a string containing an attribute not found in table `A` -produces an error. - -```python -# All the sessions performed by Alice -Session & 'user = "Alice"' - -# All the experiments at least one minute long -Experiment & 'duration >= 60' -``` - -## Restriction by a collection - -A collection can be a list, a tuple, or a Pandas `DataFrame`. - -```python -# a list: -cond_list = ['first_name = "Aaron"', 'last_name = "Aaronson"'] - -# a tuple: -cond_tuple = ('first_name = "Aaron"', 'last_name = "Aaronson"') - -# a dataframe: -import pandas as pd -cond_frame = pd.DataFrame( - data={'first_name': ['Aaron'], 'last_name': ['Aaronson']}) -``` - -When `cond` is a collection of conditions, the conditions are applied by logical -disjunction (logical OR). -Thus, restriction of table `A` by a collection will return all entities in `A` that -meet *any* of the conditions in the collection. -For example, if you restrict the `Student` table by a collection containing two -conditions, one for a first and one for a last name, your query will return any -students with a matching first name *or* a matching last name. - -```python -Student() & ['first_name = "Aaron"', 'last_name = "Aaronson"'] -``` - -Restriction by a collection, returning all entities matching any condition in the collection. - -![Restriction by collection](../images/python_collection.png){: style="align:center"} - -Restriction by an empty collection returns no entities. -Exclusion of table `A` by an empty collection returns all the entities of `A`. - -## Restriction by a Boolean expression - -`A & True` and `A - False` are equivalent to `A`. - -`A & False` and `A - True` are empty. - -## Restriction by an `AndList` - -The special function `dj.AndList` represents logical conjunction (logical AND). -Restriction of table `A` by an `AndList` will return all entities in `A` that meet -*all* of the conditions in the list. -`A & dj.AndList([c1, c2, c3])` is equivalent to `A & c1 & c2 & c3`. -Usually, it is more convenient to simply write out all of the conditions, as -`A & c1 & c2 & c3`. -However, when a list of conditions has already been generated, the list can simply be -passed as the argument to `dj.AndList`. - -Restriction of table `A` by an empty `AndList`, as in `A & dj.AndList([])`, will return -all of the entities in `A`. -Exclusion by an empty `AndList` will return no entities. - -## Restriction by a `Not` object - -The special function `dj.Not` represents logical negation, such that `A & dj.Not(cond)` -is equivalent to `A - cond`. - -## Restriction by a query - -Restriction by a query object is a generalization of restriction by a table (which is -also a query object), because DataJoint queries always produce well-defined entity -sets, as described in [entity normalization](../design/normalization.md). -As such, restriction by queries follows the same behavior as restriction by tables -described above. - -The example below creates a query object corresponding to all the sessions performed by -the user Alice. -The `Experiment` table is then restricted by the query object, returning all the -experiments that are part of sessions performed by Alice. - -```python -query = Session & 'user = "Alice"' -Experiment & query -``` - -## Restriction by `dj.Top` - -Restriction by `dj.Top` returns the number of entities specified by the `limit` -argument. These entities can be returned in the order specified by the `order_by` -argument. And finally, the `offset` argument can be used to offset the returned entities -which is useful for pagination in web applications. - -```python -# Return the first 10 sessions in descending order of session date -Session & dj.Top(limit=10, order_by='session_date DESC') -``` diff --git a/docs/src/query/union.md b/docs/src/query/union.md deleted file mode 100644 index 71f0fa687..000000000 --- a/docs/src/query/union.md +++ /dev/null @@ -1,48 +0,0 @@ -# Union - -The union operator is not yet implemented -- this page serves as the specification for -the upcoming implementation. -Union is rarely needed in practice. - -## Union operator `+` - -The result of the union operator `A + B` contains all the entities from both operands. -[Entity normalization](../design/normalization.md) requires that the operands in a -union both belong to the same entity type with the same primary key using homologous -attributes. -In the absence of any secondary attributes, the result of a union is the simple set union. - -When secondary attributes are present, they must have the same names and datatypes in -both operands. -The two operands must also be **disjoint**, without any duplicate primary key values -across both inputs. -These requirements prevent ambiguity of attribute values and preserve entity identity. - -## Principles of union - -1. As in all operators, the order of the attributes in the operands is not significant. -2. Operands `A` and `B` must have the same primary key attributes. - Otherwise, an error will be raised. -3. Operands `A` and `B` may not have any common non-key attributes. - Otherwise, an error will be raised. -4. The result `A + B` will have the same primary key as `A` and `B`. -5. The result `A + B` will have all the non-key attributes from both `A` and `B`. -6. For entities that are found in both `A` and `B` (based on the primary key), the -secondary attributes will be filled from the corresponding entities in `A` and `B`. -7. For entities that are only found in either `A` or `B`, the other operand's secondary -attributes will filled with null values. - -## Examples of union - -Example 1 : Note that the order of the attributes does not matter. - -![union-example1](../images/union-example1.png){: style="width:404px; align:center"} - -Example 2 : Non-key attributes are combined from both tables and filled with NULLs when missing. - -![union-example2](../images/union-example2.png){: style="width:539px; align:center"} - -## Properties of union - -1. Commutative: `A + B` is equivalent to `B + A`. -2. Associative: `(A + B) + C` is equivalent to `A + (B + C)`. diff --git a/docs/src/query/universals.md b/docs/src/query/universals.md deleted file mode 100644 index a9f12dd96..000000000 --- a/docs/src/query/universals.md +++ /dev/null @@ -1,46 +0,0 @@ -# Universal Sets - -All [query operators](operators.md) are designed to preserve the entity types of their -inputs. -However, some queries require creating a new entity type that is not represented by any -stored tables. -This means that a new entity type must be explicitly defined as part of the query. -Universal sets fulfill this role. - -**Universal sets** are used in DataJoint to define virtual tables with arbitrary -primary key structures for use in query expressions. -A universal set, defined using class `dj.U`, denotes the set of all possible entities -with given attributes of any possible datatype. -Universal sets allow query expressions using virtual tables when no suitable base table exists. -Attributes of universal sets are allowed to be matched to any namesake attributes, even -those that do not come from the same initial source. - -For example, you may like to query the university database for the complete list of -students' home cities, along with the number of students from each city. -The [schema](example-schema.md) for the university database does not have a table for -cities and states. -A virtual table can fill the role of the nonexistent base table, allowing queries that -would not be possible otherwise. - -```python -# All home cities of students -dj.U('home_city', 'home_state') & Student - -# Total number of students from each city -dj.U('home_city', 'home_state').aggr(Student, n="count(*)") - -# Total number of students from each state -U('home_state').aggr(Student, n="count(*)") - -# Total number of students in the database -U().aggr(Student, n="count(*)") -``` - -The result of aggregation on a universal set is restricted to the entities with matches -in the aggregated table, such as `Student` in the example above. -In other words, `X.aggr(A, ...)` is interpreted as `(X & A).aggr(A, ...)` for universal -set `X`. -All attributes of a universal set are considered primary. - -Universal sets should be used sparingly when no suitable base tables already exist. -In some cases, defining a new base table can make queries clearer and more semantically constrained. diff --git a/docs/src/quick-start.md b/docs/src/quick-start.md deleted file mode 100644 index a7f255658..000000000 --- a/docs/src/quick-start.md +++ /dev/null @@ -1,469 +0,0 @@ -# Quick Start Guide - -## Tutorials - -The easiest way to get started is through the [DataJoint -Tutorials](https://github.com/datajoint/datajoint-tutorials). These tutorials are -configured to run using [GitHub Codespaces](https://github.com/features/codespaces) -where the full environment including the database is already set up. - -Advanced users can install DataJoint locally. Please see the installation instructions below. - -## Installation - -First, please [install Python](https://www.python.org/downloads/) version -3.8 or later. - -Next, please install DataJoint via one of the following: - -=== "conda" - - Pre-Requisites - - Ensure you have [conda](https://conda.io/projects/conda/en/latest/user-guide/install/index.html#regular-installation) - installed. - - To add the `conda-forge` channel: - - ```bash - conda config --add channels conda-forge - ``` - - To install: - - ```bash - conda install -c conda-forge datajoint - ``` - -=== "pip + :fontawesome-brands-windows:" - - Pre-Requisites - - Ensure you have [pip](https://pip.pypa.io/en/stable/installation/) installed. - - Install [graphviz](https://graphviz.org/download/#windows) pre-requisite for - diagram visualization. - - To install: - - ```bash - pip install datajoint - ``` - -=== "pip + :fontawesome-brands-apple:" - - Pre-Requisites - - Ensure you have [pip](https://pip.pypa.io/en/stable/installation/) installed. - - Install [graphviz](https://graphviz.org/download/#mac) pre-requisite for - diagram visualization. - - To install: - - ```bash - pip install datajoint - ``` - -=== "pip + :fontawesome-brands-linux:" - - Pre-Requisites - - Ensure you have [pip](https://pip.pypa.io/en/stable/installation/) installed. - - Install [graphviz](https://graphviz.org/download/#linux) pre-requisite for - diagram visualization. - - To install: - - ```bash - pip install datajoint - ``` - -## Connection - -=== "environment variables" - - Before using `datajoint`, set the following environment variables like so: - - ```bash linenums="1" - DJ_HOST={host_address} - DJ_USER={user} - DJ_PASS={password} - ``` - -=== "memory" - - To set connection settings within Python, perform: - - ```python linenums="1" - import datajoint as dj - - dj.config["database.host"] = "{host_address}" - dj.config["database.user"] = "{user}" - dj.config["database.password"] = "{password}" - ``` - - These configuration settings can be saved either locally or system-wide using one - of the following commands: - ```python - dj.config.save_local() - dj.config.save_global() - ``` - -=== "file" - - Before using `datajoint`, create a file named `dj_local_conf.json` in the current - directory like so: - - ```json linenums="1" - { - "database.host": "{host_address}", - "database.user": "{user}", - "database.password": "{password}" - } - ``` - - These settings will be loaded whenever a Python instance is launched from this - directory. To configure settings globally, save a similar file as - `.datajoint_config.json` in your home directory. A local config, if present, will - take precedent over global settings. - -## Data Pipeline Definition - -Let's definite a simple data pipeline. - -```python linenums="1" -import datajoint as dj -schema = dj.Schema(f"{dj.config['database.user']}_shapes") # This statement creates the database schema `{username}_shapes` on the server. - -@schema # The `@schema` decorator for DataJoint classes creates the table on the server. -class Rectangle(dj.Manual): - definition = """ # The table is defined by the the `definition` property. - shape_id: int - --- - shape_height: float - shape_width: float - """ - -@schema -class Area(dj.Computed): - definition = """ - -> Rectangle - --- - shape_area: float - """ - def make(self, key): - rectangle = (Rectangle & key).fetch1() - Area.insert1( - dict( - shape_id=rectangle["shape_id"], - shape_area=rectangle["shape_height"] * rectangle["shape_width"], - ) - ) -``` - -It is a common practice to have a separate Python module for each schema. Therefore, -each such module has only one `dj.Schema` object defined and is usually named -`schema`. - -The `dj.Schema` constructor can take a number of optional parameters -after the schema name. - -- `context` - Dictionary for looking up foreign key references. - Defaults to `None` to use local context. -- `connection` - Specifies the DataJoint connection object. Defaults - to `dj.conn()`. -- `create_schema` - When `False`, the schema object will not create a - schema on the database and will raise an error if one does not - already exist. Defaults to `True`. -- `create_tables` - When `False`, the schema object will not create - tables on the database and will raise errors when accessing missing - tables. Defaults to `True`. - -The `@schema` decorator uses the class name and the data tier to check whether an -appropriate table exists on the database. If a table does not already exist, the -decorator creates one on the database using the definition property. The decorator -attaches the information about the table to the class, and then returns the class. - -## Diagram - -### Display - -The diagram displays the relationship of the data model in the data pipeline. - -This can be done for an entire schema: - -```python -import datajoint as dj -schema = dj.Schema('my_database') -dj.Diagram(schema) -``` - -![pipeline](./images/shapes_pipeline.svg) - -Or for individual or sets of tables: -```python -dj.Diagram(schema.Rectangle) -dj.Diagram(schema.Rectangle) + dj.Diagram(schema.Area) -``` - -What if I don't see the diagram? - -Some Python interfaces may require additional `draw` method. - -```python -dj.Diagram(schema).draw() -``` - -Calling the `.draw()` method is not necessary when working in a Jupyter notebook by -entering `dj.Diagram(schema)` in a notebook cell. The Diagram will automatically -render in the notebook by calling its `_repr_html_` method. A Diagram displayed -without `.draw()` will be rendered as an SVG, and hovering the mouse over a table -will reveal a compact version of the output of the `.describe()` method. - -For more information about diagrams, see [this article](../design/diagrams). - -### Customize - -Adding or subtracting a number to a diagram object adds nodes downstream or upstream, -respectively, in the pipeline. - -```python -(dj.Diagram(schema.Rectangle)+1).draw() # Plot all the tables directly downstream from `schema.Rectangle` -``` - -```python -(dj.Diagram('my_schema')-1+1).draw() # Plot all tables directly downstream of those directly upstream of this schema. -``` - -### Save - -The diagram can be saved as either `png` or `svg`. - -```python -dj.Diagram(schema).save(filename='my-diagram', format='png') -``` - -## Insert data - -Data entry is as easy as providing the appropriate data structure to a permitted -[table](./design/tables/tiers.md). - -Let's add data for a rectangle: - -```python -Rectangle.insert1(dict(shape_id=1, shape_height=2, shape_width=4)) -``` - -Given the following [table definition](./design/tables/declare.md), we can insert data -as tuples, dicts, pandas dataframes, or pathlib `Path` relative paths to local CSV -files. - -```python -mouse_id: int # unique mouse id ---- -dob: date # mouse date of birth -sex: enum('M', 'F', 'U') # sex of mouse - Male, Female, or Unknown -``` - -=== "Tuple" - - ```python - mouse.insert1( (0, '2017-03-01', 'M') ) # Single entry - data = [ - (1, '2016-11-19', 'M'), - (2, '2016-11-20', 'U'), - (5, '2016-12-25', 'F') - ] - mouse.insert(data) # Multi-entry - ``` - -=== "Dict" - - ```python - mouse.insert1( dict(mouse_id=0, dob='2017-03-01', sex='M') ) # Single entry - data = [ - {'mouse_id':1, 'dob':'2016-11-19', 'sex':'M'}, - {'mouse_id':2, 'dob':'2016-11-20', 'sex':'U'}, - {'mouse_id':5, 'dob':'2016-12-25', 'sex':'F'} - ] - mouse.insert(data) # Multi-entry - ``` - -=== "Pandas" - - ```python - import pandas as pd - data = pd.DataFrame( - [[1, "2016-11-19", "M"], [2, "2016-11-20", "U"], [5, "2016-12-25", "F"]], - columns=["mouse_id", "dob", "sex"], - ) - mouse.insert(data) - ``` - -=== "CSV" - - Given the following CSV in the current working directory as `mice.csv` - - ```console - mouse_id,dob,sex - 1,2016-11-19,M - 2,2016-11-20,U - 5,2016-12-25,F - ``` - - We can import as follows: - - ```python - from pathlib import Path - mouse.insert(Path('./mice.csv')) - ``` - -## Run computation - -Let's start the computations on our entity: `Area`. - -```python -Area.populate(display_progress=True) -``` - -The `make` method populates automated tables from inserted data. Read more in the -full article [here](./compute/make.md) - -## Query - -Let's inspect the results. - -```python -Area & "shape_area >= 8" -``` - -| shaped_id | shape_area | -| --- | --- | -| 1 | 8.0 | - -## Fetch - -Data queries in DataJoint comprise two distinct steps: - -1. Construct the `query` object to represent the required data using - tables and [operators](../query/operators). -2. Fetch the data from `query` into the workspace of the host language. - -Note that entities returned by `fetch` methods are not guaranteed to be sorted in any -particular order unless specifically requested. Furthermore, the order is not -guaranteed to be the same in any two queries, and the contents of two identical queries -may change between two sequential invocations unless they are wrapped in a transaction. -Therefore, if you wish to fetch matching pairs of attributes, do so in one `fetch` -call. - -```python -data = query.fetch() -``` - -### Entire table - -A `fetch` command can either retrieve table data as a NumPy -[recarray](https://docs.scipy.org/doc/numpy/reference/generated/numpy.recarray.html) -or a as a list of `dict` - -```python -data = query.fetch() # NumPy recarray -data = query.fetch(as_dict=True) # List of `dict` -``` - -In some cases, the amount of data returned by fetch can be quite large; it can be -useful to use the `size_on_disk` attribute to determine if running a bare fetch -would be wise. Please note that it is only currently possible to query the size of -entire tables stored directly in the database at this time. - -### Separate variables - -```python -name, img = query.fetch1('mouse_id', 'dob') # when query has exactly one entity -name, img = query.fetch('mouse_id', 'dob') # [mouse_id, ...] [dob, ...] -``` - -### Primary key values - -```python -keydict = tab.fetch1("KEY") # single key dict when tab has exactly one entity -keylist = tab.fetch("KEY") # list of key dictionaries [{}, ...] -``` - -`KEY` can also used when returning attribute values as separate -variables, such that one of the returned variables contains the entire -primary keys. - -### Sorting results - -To sort the result, use the `order_by` keyword argument. - -```python -data = query.fetch(order_by='mouse_id') # ascending order -data = query.fetch(order_by='mouse_id desc') # descending order -data = query.fetch(order_by=('mouse_id', 'dob')) # by ID first, dob second -data = query.fetch(order_by='KEY') # sort by the primary key -``` - -The `order_by` argument can be a string specifying the attribute to sort by. By default -the sort is in ascending order. Use `'attr desc'` to sort in descending order by -attribute `attr`. The value can also be a sequence of strings, in which case, the sort -performed on all the attributes jointly in the order specified. - -The special attribute named `'KEY'` represents the primary key attributes in order that -they appear in the index. Otherwise, this name can be used as any other argument. - -If an attribute happens to be a SQL reserved word, it needs to be enclosed in -backquotes. For example: - -```python -data = query.fetch(order_by='`select` desc') -``` - -The `order_by` value is eventually passed to the `ORDER BY` -[clause](https://dev.mysql.com/doc/refman/5.7/en/order-by-optimization.html). - -### Limiting results - -Similar to sorting, the `limit` and `offset` arguments can be used to limit the result -to a subset of entities. - -```python -data = query.fetch(order_by='mouse_id', limit=10, offset=5) -``` - -Note that an `offset` cannot be used without specifying a `limit` as -well. - -### Usage with Pandas - -The `pandas` [library](http://pandas.pydata.org/) is a popular library for data analysis -in Python which can easily be used with DataJoint query results. Since the records -returned by `fetch()` are contained within a `numpy.recarray`, they can be easily -converted to `pandas.DataFrame` objects by passing them into the `pandas.DataFrame` -constructor. For example: - -```python -import pandas as pd -frame = pd.DataFrame(tab.fetch()) -``` - -Calling `fetch()` with the argument `format="frame"` returns results as -`pandas.DataFrame` objects indexed by the table's primary key attributes. - -```python -frame = tab.fetch(format="frame") -``` - -Returning results as a `DataFrame` is not possible when fetching a particular subset of -attributes or when `as_dict` is set to `True`. - -## Drop - -The `drop` method completely removes a table from the database, including its -definition. It also removes all dependent tables, recursively. DataJoint will first -display the tables being dropped and the number of entities in each before prompting -the user for confirmation to proceed. - -The `drop` method is often used during initial design to allow altered -table definitions to take effect. - -```python -# drop the Person table from its schema -Person.drop() -``` diff --git a/docs/src/sysadmin/bulk-storage.md b/docs/src/sysadmin/bulk-storage.md deleted file mode 100644 index 12af44791..000000000 --- a/docs/src/sysadmin/bulk-storage.md +++ /dev/null @@ -1,104 +0,0 @@ -# Bulk Storage Systems - -## Why External Bulk Storage? - -DataJoint supports the storage of large data objects associated with -relational records externally from the MySQL Database itself. This is -significant and useful for a number of reasons. - -### Cost - -One reason is that the high-performance storage commonly used in database systems is -more expensive than typical commodity storage. Therefore, storing the smaller identifying -information typically used in queries on fast, relational database storage and storing -the larger bulk data used for analysis or processing on lower cost commodity storage -enables large savings in storage expense. - -### Flexibility - -Storing bulk data separately also facilitates more flexibility in -usage, since the bulk data can managed using separate maintenance -processes than those in the relational storage. - -For example, larger relational databases may require many hours to be -restored in the event of system failures. If the relational portion of -the data is stored separately, with the larger bulk data stored on -another storage system, this downtime can be reduced to a matter of -minutes. Similarly, due to the lower cost of bulk commodity storage, -more emphasis can be put into redundancy of this data and backups to -help protect the non-relational data. - -### Performance - -Storing the non-relational bulk data separately can have system -performance impacts by removing data transfer, disk I/O, and memory -load from the database server and shifting these to the bulk storage -system. Additionally, DataJoint supports caching of bulk data records -which can allow for faster processing of records which already have -been retrieved in previous queries. - -### Data Sharing - -DataJoint provides pluggable support for different external bulk storage backends, -allowing data sharing by publishing bulk data to S3-Protocol compatible data shares both -in the cloud and on locally managed systems and other common tools for data sharing, -such as Globus, etc. - -## Bulk Storage Scenarios - -Typical bulk storage considerations relate to the cost of the storage -backend per unit of storage, the amount of data which will be stored, -the desired focus of the shared data (system performance, data -flexibility, data sharing), and data access. Some common scenarios are -given in the following table: - -| Scenario | Storage Solution | System Requirements | Notes | -| -- | -- | -- | -- | -| Local Object Cache | Local External Storage | Local Hard Drive | Used to Speed Access to other Storage | -| LAN Object Cache | Network External Storage | Local Network Share | Used to Speed Access to other storage, reduce Cloud/Network Costs/Overhead | -| Local Object Store | Local/Network External Storage | Local/Network Storage | Used to store objects externally from the database | -| Local S3-Compatible Store | Local S3-Compatible Server | Network S3-Server | Used to host S3-Compatible services locally (e.g. minio) for internal use or to lower cloud costs | -| Cloud S3-Compatible Storage | Cloud Provider | Internet Connectivity | Used to reduce/remove requirement for external storage management, data sharing | -| Globus Storage | Globus Endpoint | Local/Local Network Storage, Internet Connectivity | Used for institutional data transfer or publishing. | - -## Bulk Storage Considerations - -Although external bulk storage provides a variety of advantages for -storage cost and data sharing, it also uses slightly different data -input/retrieval semantics and as such has different performance -characteristics. - -### Performance Characteristics - -In the direct database connection scenario, entire result sets are -either added or retrieved from the database in a single stream -action. In the case of external storage, individual record components -are retrieved in a set of sequential actions per record, each one -subject to the network round trip to the given storage medium. As -such, tables using many small records may be ill suited to external -storage usage in the absence of a caching mechanism. While some of -these impacts may be addressed by code changes in a future release of -DataJoint, to some extent, the impact is directly related from needing -to coordinate the activities of the database data stream with the -external storage system, and so cannot be avoided. - -### Network Traffic - -Some of the external storage solutions mentioned above incur cost both -at a data volume and transfer bandwidth level. The number of users -querying the database, data access, and use of caches should be -considered in these cases to reduce this cost if applicable. - -### Data Coherency - -When storing all data directly in the relational data store, it is -relatively easy to ensure that all data in the database is consistent -in the event of system issues such as crash recoveries, since MySQL’s -relational storage engine manages this for you. When using external -storage however, it is important to ensure that any data recoveries of -the database system are paired with a matching point-in-time of the -external storage system. While DataJoint does use hashing to help -facilitate a guarantee that external files are uniquely named -throughout their lifecycle, the pairing of a given relational dataset -against a given filesystem state is loosely coupled, and so an -incorrect pairing could result in processing failures or other issues. diff --git a/docs/src/sysadmin/database-admin.md b/docs/src/sysadmin/database-admin.md deleted file mode 100644 index 352a3af11..000000000 --- a/docs/src/sysadmin/database-admin.md +++ /dev/null @@ -1,364 +0,0 @@ -# Database Administration - -## Hosting - -Let’s say a person, a lab, or a multi-lab consortium decide to use DataJoint as their -data pipeline platform. -What IT resources and support will be required? - -DataJoint uses a MySQL-compatible database server such as MySQL, MariaDB, Percona -Server, or Amazon Aurora to store the structured data used for all relational -operations. -Large blocks of data associated with these records such as multidimensional numeric -arrays (signals, images, scans, movies, etc) can be stored within the database or -stored in additionally configured [bulk storage](../client/stores.md). - -The first decisions you need to make are where this server will be hosted and how it -will be administered. -The server may be hosted on your personal computer, on a dedicated machine in your lab, -or in a cloud-based database service. - -### Cloud hosting - -Increasingly, many teams make use of cloud-hosted database services, which allow great -flexibility and easy administration of the database server. -A cloud hosting option will be provided through https://works.datajoint.com. -DataJoint Works simplifies the setup for labs that wish to host their data pipelines in -the cloud and allows sharing pipelines between multiple groups and locations. -Being an open-source solution, other cloud services such as Amazon RDS can also be used -in this role, albeit with less DataJoint-centric customization. - -### Self hosting - -In the most basic configuration, the relational database management system (database -server) is installed on an individual user's personal computer. -To support a group of users, a specialized machine can be configured as a dedicated -database server. -This server can be accessed by multiple DataJoint clients to query the data and perform -computations. - -For larger groups and multi-site collaborations with heavy workloads, the database -server cluster may be configured in the cloud or on premises. -The following section provides some basic guidelines for these configurations here and -in the subsequent sections of the documentation. - -### General server / hardware support requirements - -The following table lists some likely scenarios for DataJoint database server -deployments and some reasonable estimates of the required computer hardware. -The required IT/systems support needed to ensure smooth operations in the absence of -local database expertise is also listed. - -#### IT infrastructures - -| Usage Scenario | DataJoint Database Computer | Required IT Support | -| -- | -- | -- | -| Single User | Personal Laptop or Workstation | Self-Supported or Ad-Hoc General IT Support | -| Small Group (e.g. 2-10 Users) | Workstation or Small Server | Ad-Hoc General or Experienced IT Support | -| Medium Group (e.g. 10-30 Users) | Small to Medium Server | Ad-Hoc/Part Time Experienced or Specialized IT Support | -| Large Group/Department (e.g. 30-50+ Users) | Medium/Large Server or Multi-Server Replication | Part Time/Dedicated Experienced or Specialized IT Support | -| Multi-Location Collaboration (30+ users, Geographically Distributed) | Large Server, Advanced Replication | Dedicated Specialized IT Support | - -## Configuration - -### Hardware considerations - -As in any computer system, CPU, RAM memory, disk storage, and network speed are -important components of performance. -The relational database component of DataJoint is no exception to this rule. -This section discusses the various factors relating to selecting a server for your -DataJoint pipelines. - -#### CPU - -CPU speed and parallelism (number of cores/threads) will impact the speed of queries -and the number of simultaneous queries which can be efficiently supported by the system. -It is a good rule of thumb to have enough cores to support the number of active users -and background tasks you expect to have running during a typical 'busy' day of usage. -For example, a team of 10 people might want to have 8 cores to support a few active -queries and background tasks. - -#### RAM - -The amount of RAM will impact the amount of DataJoint data kept in memory, allowing for -faster querying of data since the data can be searched and returned to the user without -needing to access the slower disk drives. -It is a good idea to get enough memory to fully store the more important and frequently -accessed portions of your dataset with room to spare, especially if in-database blob -storage is used instead of external [bulk storage](bulk-storage.md). - -#### Disk - -The disk storage for a DataJoint database server should have fast random access, -ideally with flash-based storage to eliminate the rotational delay of mechanical hard -drives. - -#### Networking - -When network connections are used, network speed and latency are important to ensure -that large query results can be quickly transferred across the network and that delays -due to data entry/query round-trip have minimal impact on the runtime of the program. - -#### General recommendations - -DataJoint datasets can consist of many thousands or even millions of records. -Generally speaking one would want to make sure that the relational database system has -sufficient CPU speed and parallelism to support a typical number of concurrent users -and to execute searches quickly. -The system should have enough RAM to store the primary key values of commonly used -tables and operating system caches. -Disk storage should be fast enough to support quick loading of and searching through -the data. -Lastly, network bandwidth must be sufficient to support transferring user records -quickly. - -### Large-scale installations - -Database replication may be beneficial if system downtime or precise database -responsiveness is a concern -Replication can allow for easier coordination of maintenance activities, faster -recovery in the event of system problems, and distribution of the database workload -across server machines to increase throughput and responsiveness. - -#### Multi-master replication - -Multi-master replication configurations allow for all replicas to be used in a read/ -write fashion, with the workload being distributed among all machines. -However, multi-master replication is also more complicated, requiring front-end -machines to distribute the workload, similar performance characteristics on all -replicas to prevent bottlenecks, and redundant network connections to ensure the -replicated machines are always in sync. - -### Recommendations - -It is usually best to go with the simplest solution which can suit the requirements of -the installation, adjusting workloads where possible and adding complexity only as -needs dictate. - -Resource requirements of course depend on the data collection and processing needs of -the given pipeline, but there are general size guidelines that can inform any system -configuration decisions. -A reasonably powerful workstation or small server should support the needs of a small -group (2-10 users). -A medium or large server should support the needs of a larger user community (10-30 -users). -A replicated or distributed setup of 2 or more medium or large servers may be required -in larger cases. -These requirements can be reduced through the use of external or cloud storage, which -is discussed in the subsequent section. - -| Usage Scenario | DataJoint Database Computer | Hardware Recommendation | -| -- | -- | -- | -| Single User | Personal Laptop or Workstation | 4 Cores, 8-16GB or more of RAM, SSD or better storage | -| Small Group (e.g. 2-10 Users) | Workstation or Small Server | 8 or more Cores, 16GB or more of RAM, SSD or better storage | -| Medium Group (e.g. 10-30 Users) | Small to Medium Server | 8-16 or more Cores, 32GB or more of RAM, SSD/RAID or better storage | -| Large Group/Department (e.g. 30-50+ Users) | Medium/Large Server or Multi-Server Replication | 16-32 or more Cores, 64GB or more of RAM, SSD Raid storage, multiple machines | -| Multi-Location Collaboration (30+ users, Geographically Distributed) | Large Server, Advanced Replication | 16-32 or more Cores, 64GB or more of RAM, SSD Raid storage, multiple machines; potentially multiple machines in multiple locations | - -### Docker - -A Docker image is available for a MySQL server configured to work with DataJoint: https://github.com/datajoint/mysql-docker. - -## User Management - -Create user accounts on the MySQL server. For example, if your -username is alice, the SQL code for this step is: - -```mysql -CREATE USER 'alice'@'%' IDENTIFIED BY 'alices-secret-password'; -``` - -Existing users can be listed using the following SQL: - -```mysql -SELECT user, host from mysql.user; -``` - -Teams that use DataJoint typically divide their data into schemas -grouped together by common prefixes. For example, a lab may have a -collection of schemas that begin with `common_`. Some common -processing may be organized into several schemas that begin with -`pipeline_`. Typically each user has all privileges to schemas that -begin with their username. - -For example, alice may have privileges to select and insert data from -the common schemas (but not create new tables), and have all -privileges to the pipeline schemas. - -Then the SQL code to grant her privileges might look like: - -```mysql -GRANT SELECT, INSERT ON `common\_%`.* TO 'alice'@'%'; -GRANT ALL PRIVILEGES ON `pipeline\_%`.* TO 'alice'@'%'; -GRANT ALL PRIVILEGES ON `alice\_%`.* TO 'alice'@'%'; -``` - -To note, the ```ALL PRIVILEGES``` option allows the user to create -and remove databases without administrator intervention. - -Once created, a user's privileges can be listed using the ```SHOW GRANTS``` -statement. - -```mysql -SHOW GRANTS FOR 'alice'@'%'; -``` - -### Grouping with Wildcards - -Depending on the complexity of your installation, using additional -wildcards to group access rules together might make managing user -access rules simpler. For example, the following equivalent -convention: - -```mysql -GRANT ALL PRIVILEGES ON `user_alice\_%`.* TO 'alice'@'%'; -``` - -Could then facilitate using a rule like: - -```mysql -GRANT SELECT ON `user\_%\_%`.* TO 'bob'@'%'; -``` - -to enable `bob` to query all other users tables using the -`user_username_database` convention without needing to explicitly -give him access to `alice\_%`, `charlie\_%`, and so on. - -This convention can be further expanded to create notions of groups -and protected schemas for background processing, etc. For example: - -```mysql -GRANT ALL PRIVILEGES ON `group\_shared\_%`.* TO 'alice'@'%'; -GRANT ALL PRIVILEGES ON `group\_shared\_%`.* TO 'bob'@'%'; - -GRANT ALL PRIVILEGES ON `group\_wonderland\_%`.* TO 'alice'@'%'; -GRANT SELECT ON `group\_wonderland\_%`.* TO 'alice'@'%'; -``` - -could allow both bob an alice to read/write into the -```group\_shared``` databases, but in the case of the -```group\_wonderland``` databases, read write access is restricted -to alice. - -## Backups and Recovery - -Backing up your DataJoint installation is critical to ensuring that your work is safe -and can be continued in the event of system failures, and several mechanisms are -available to use. - -Much like your live installation, your backup will consist of two portions: - -- Backup of the Relational Data -- Backup of optional external bulk storage - -This section primarily deals with backup of the relational data since most of the -optional bulk storage options use "regular" flat-files for storage and can be backed up -via any "normal" disk backup regime. - -There are many options to backup MySQL; subsequent sections discuss a few options. - -### Cloud hosted backups - -In the case of cloud-hosted options, many cloud vendors provide automated backup of -your data, and some facility for downloading such backups externally. -Due to the wide variety of cloud-specific options, discussion of these options falls -outside of the scope of this documentation. -However, since the cloud server is also a MySQL server, other options listed here may -work for your situation. - -### Disk-based backup - -The simplest option for many cases is to perform a disk-level backup of your MySQL -installation using standard disk backup tools. -It should be noted that all database activity should be stopped for the duration of the -backup to prevent errors with the backed up data. -This can be done in one of two ways: - -- Stopping the MySQL server program -- Using database locks - -These methods are required since MySQL data operations can be ongoing in the background -even when no user activity is ongoing. -To use a database lock to perform a backup, the following commands can be used as the -MySQL administrator: - -```mysql -FLUSH TABLES WITH READ LOCK; -UNLOCK TABLES; -``` - -The backup should be performed between the issuing of these two commands, ensuring the -database data is consistent on disk when it is backed up. - -### MySQLDump - -Disk based backups may not be feasible for every installation, or a database may -require constant activity such that stopping it for backups is not feasible. -In such cases, the simplest option is -[MySQLDump](https://dev.mysql.com/doc/mysql-backup-excerpt/8.0/en/using-mysqldump.html), - a command line tool that prints the contents of your database contents in SQL form. - -This tool is generally acceptable for most cases and is especially well suited for -smaller installations due to its simplicity and ease of use. - -For larger installations, the lower speed of MySQLDump can be a limitation, since it -has to convert the database contents to and from SQL rather than dealing with the -database files directly. -Additionally, since backups are performed within a transaction, the backup will be -valid up to the time the backup began rather than to its completion, which can make -ensuring that the latest data are fully backed up more difficult as the time it takes -to run a backup grows. - -### Percona XTraBackup - -The Percona `xtrabackup` tool provides near-realtime backup capability of a MySQL -installation, with extended support for replicated databases, and is a good tool for -backing up larger databases. - -However, this tool requires local disk access as well as reasonably fast backup media, -since it builds an ongoing transaction log in real time to ensure that backups are -valid up to the point of their completion. -This strategy fails if it cannot keep up with the write speed of the database. -Further, the backups it generates are in binary format and include incomplete database -transactions, which require careful attention to detail when restoring. - -As such, this solution is recommended only for advanced use cases or larger databases -where limitations of the other solutions may apply. - -### Locking and DDL issues - -One important thing to note is that at the time of writing, MySQL's transactional -system is not `data definition language` aware, meaning that changes to table -structures occurring during some backup schemes can result in corrupted backup copies. -If schema changes will be occurring during your backup window, it is a good idea to -ensure that appropriate locking mechanisms are used to prevent these changes during -critical steps of the backup process. - -However, on busy installations which cannot be stopped, the use of locks in many backup -utilities may cause issues if your programs expect to write data to the database during -the backup window. - -In such cases it might make sense to review the given backup tools for locking related -options or to use other mechanisms such as replicas or alternate backup tools to -prevent interaction of the database. - -### Replication and snapshots for backup - -Larger databases consisting of many Terabytes of data may take many hours or even days -to backup and restore, and so downtime resulting from system failure can create major -impacts to ongoing work. - -While not backup tools per-se, use of MySQL replication and disk snapshots -can be useful to assist in reducing the downtime resulting from a full database outage. - -Replicas can be configured so that one copy of the data is immediately online in the -event of server crash. -When a server fails in this case, users and programs simply restart and point to the -new server before resuming work. - -Replicas can also reduce the system load generated by regular backup procedures, since -they can be backed up instead of the main server. -Additionally they can allow more flexibility in a given backup scheme, such as allowing -for disk snapshots on a busy system that would not otherwise be able to be stopped. -A replica copy can be stopped temporarily and then resumed while a disk snapshot or -other backup operation occurs. diff --git a/docs/src/sysadmin/external-store.md b/docs/src/sysadmin/external-store.md deleted file mode 100644 index aac61fe24..000000000 --- a/docs/src/sysadmin/external-store.md +++ /dev/null @@ -1,293 +0,0 @@ -# External Store - -DataJoint organizes most of its data in a relational database. -Relational databases excel at representing relationships between entities and storing -structured data. -However, relational databases are not particularly well-suited for storing large -continuous chunks of data such as images, signals, and movies. -An attribute of type `longblob` can contain an object up to 4 GiB in size (after -compression) but storing many such large objects may hamper the performance of queries -on the entire table. -A good rule of thumb is that objects over 10 MiB in size should not be put in the -relational database. -In addition, storing data in cloud-hosted relational databases (e.g. AWS RDS) may be -more expensive than in cloud-hosted simple storage systems (e.g. AWS S3). - -DataJoint allows the use of `external` storage to store large data objects within its -relational framework but outside of the main database. - -Defining an externally-stored attribute is used using the notation `blob@storename` -(see also: [definition syntax](../design/tables/declare.md)) and works the same way as -a `longblob` attribute from the users perspective. However, its data are stored in an -external storage system rather than in the relational database. - -Various systems can play the role of external storage, including a shared file system -accessible to all team members with access to these objects or a cloud storage -solutions such as AWS S3. - -For example, the following table stores motion-aligned two-photon movies. - -```python -# Motion aligned movies --> twophoton.Scan ---- -aligned_movie : blob@external # motion-aligned movie in 'external' store -``` - -All [insert](../manipulation/insert.md) and [fetch](../query/fetch.md) operations work -identically for `external` attributes as they do for `blob` attributes, with the same -serialization protocol. -Similar to `blobs`, `external` attributes cannot be used in restriction conditions. - -Multiple external storage configurations may be used simultaneously with the -`@storename` portion of the attribute definition determining the storage location. - -```python -# Motion aligned movies --> twophoton.Scan ---- -aligned_movie : blob@external-raw # motion-aligned movie in 'external-raw' store -``` - -## Principles of operation - -External storage is organized to emulate individual attribute values in the relational -database. -DataJoint organizes external storage to preserve the same data integrity principles as -in relational storage. - -1. The external storage locations are specified in the DataJoint connection -configuration with one specification for each store. - - ```python - dj.config['stores'] = { - 'external': dict( # 'regular' external storage for this pipeline - protocol='s3', - endpoint='s3.amazonaws.com:9000', - bucket = 'testbucket', - location = 'datajoint-projects/lab1', - access_key='1234567', - secret_key='foaf1234'), - 'external-raw': dict( # 'raw' storage for this pipeline - protocol='file', - location='/net/djblobs/myschema') - } - # external object cache - see fetch operation below for details. - dj.config['cache'] = '/net/djcache' - ``` - -2. Each schema corresponds to a dedicated folder at the storage location with the same -name as the database schema. - -3. Stored objects are identified by the [SHA-256](https://en.wikipedia.org/wiki/SHA-2) -hashes (in web-safe base-64 ASCII) of their serialized contents. - This scheme allows for the same object—used multiple times in the same schema—to be - stored only once. - -4. In the `external-raw` storage, the objects are saved as files with the hash as the -filename. - -5. In the `external` storage, external files are stored in a directory layout -corresponding to the hash of the filename. By default, this corresponds to the first 2 -characters of the hash, followed by the second 2 characters of the hash, followed by -the actual file. - -6. Each database schema has an auxiliary table named `~external_` for each -configured external store. - - It is automatically created the first time external storage is used. - The primary key of `~external_` is the hash of the data (for blobs and - attachments) or of the relative paths to the files for filepath-based storage. - Other attributes are the `count` of references by tables in the schema, the `size` - of the object in bytes, and the `timestamp` of the last event (creation, update, or - deletion). - - Below are sample entries in `~external_`. - - | HASH | size | filepath | contents_hash | timestamp | - | -- | -- | -- | -- | -- | - | 1GEqtEU6JYEOLS4sZHeHDxWQ3JJfLlH VZio1ga25vd2 | 1039536788 | NULL | NULL | 2017-06-07 23:14:01 | - - The fields `filepath` and `contents_hash` relate to the - [filepath](../design/tables/filepath.md) datatype, which will be discussed - separately. - -7. Attributes of type `@` are declared as renamed -[foreign keys](../design/tables/dependencies.md) referencing the -`~external_` table (but are not shown as such to the user). - -8. The [insert](../manipulation/insert.md) operation encodes and hashes the blob data. -If an external object is not present in storage for the same hash, the object is saved -and if the save operation is successful, corresponding entities in table -`~external_` for that store are created. - -9. The [delete](../manipulation/delete.md) operation first deletes the foreign key -reference in the target table. The external table entry and actual external object is -not actually deleted at this time (`soft-delete`). - -10. The [fetch](../query/fetch.md) operation uses the hash values to find the data. - In order to prevent excessive network overhead, a special external store named - `cache` can be configured. - If the `cache` is enabled, the `fetch` operation need not access - `~external_` directly. - Instead `fetch` will retrieve the cached object without downloading directly from - the `real` external store. - -11. Cleanup is performed regularly when the database is in light use or off-line. - -12. DataJoint never removes objects from the local `cache` folder. - The `cache` folder may just be periodically emptied entirely or based on file - access date. - If dedicated `cache` folders are maintained for each schema, then a special - procedure will be provided to remove all objects that are no longer listed in - `~external_`. - -Data removal from external storage is separated from the delete operations to ensure -that data are not lost in race conditions between inserts and deletes of the same -objects, especially in cases of transactional processing or in processes that are -likely to get terminated. -The cleanup steps are performed in a separate process when the risks of race conditions -are minimal. -The process performing the cleanups must be isolated to prevent interruptions resulting -in loss of data integrity. - -## Configuration - -The following steps must be performed to enable external storage: - -1. Assign external location settings for each storage as shown in the -[Step 1](#principles-of-operation) example above. Use `dj.config` for configuration. - - - `protocol` [`s3`, `file`] Specifies whether `s3` or `file` external storage is - desired. - - `endpoint` [`s3`] Specifies the remote endpoint to the external data for all - schemas as well as the target port. - - `bucket` [`s3`] Specifies the appropriate `s3` bucket organization. - - `location` [`s3`, `file`] Specifies the subdirectory within the root or bucket of - store to preserve data. External objects are thus stored remotely with the following - path structure: - `////`. - - `access_key` [`s3`] Specifies the access key credentials for accessing the external - location. - - `secret_key` [`s3`] Specifies the secret key credentials for accessing the external - location. - - `secure` [`s3`] Optional specification to establish secure external storage - connection with TLS (aka SSL, HTTPS). Defaults to `False`. - -2. Optionally, for each schema specify the `cache` folder for local fetch cache. - - This is done by saving the path in the `cache` key of the DataJoint configuration - dictionary: - - ```python - dj.config['cache'] = '/temp/dj-cache' - ``` - -## Cleanup - -Deletion of records containing externally stored blobs is a `soft-delete` which only -removes the database-side records from the database. -To cleanup the external tracking table or the actual external files, a separate process -is provided as follows. - -To remove only the tracking entries in the external table, call `delete` -on the `~external_` table for the external configuration with the argument -`delete_external_files=False`. - -Note: Currently, cleanup operations on a schema's external table are not 100% - transaction safe and so must be run when there is no write activity occurring - in tables which use a given schema / external store pairing. - -```python -schema.external['external_raw'].delete(delete_external_files=False) -``` - -To remove the tracking entries as well as the underlying files, call `delete` -on the external table for the external configuration with the argument -`delete_external_files=True`. - -```python -schema.external['external_raw'].delete(delete_external_files=True) -``` - -Note: Setting `delete_external_files=True` will always attempt to delete - the underlying data file, and so should not typically be used with - the `filepath` datatype. - -## Migration between DataJoint v0.11 and v0.12 - -Note: Please read carefully if you have used external storage in DataJoint v0.11! - -The initial implementation of external storage was reworked for -DataJoint v0.12. These changes are backward-incompatible with DataJoint -v0.11 so care should be taken when upgrading. This section outlines -some details of the change and a general process for upgrading to a -format compatible with DataJoint v0.12 when a schema rebuild is not -desired. - -The primary changes to the external data implementation are: - -- The external object tracking mechanism was modified. Tracking tables -were extended for additional external datatypes and split into -per-store tables to improve database performance in schemas with -many external objects. - -- The external storage format was modified to use a nested subfolder -structure (`folding`) to improve performance and interoperability -with some filesystems that have limitations or performance problems -when storing large numbers of files in single directories. - -Depending on the circumstances, the simplest way to migrate data to -v0.12 may be to drop and repopulate the affected schemas. This will construct -the schema and storage structure in the v0.12 format and save the need for -database migration. When recreation is not possible or is not preferred -to upgrade to DataJoint v0.12, the following process should be followed: - - 1. Stop write activity to all schemas using external storage. - - 2. Perform a full backup of your database(s). - - 3. Upgrade your DataJoint installation to v0.12 - - 4. Adjust your external storage configuration (in `datajoint.config`) - to the new v0.12 configuration format (see above). - - 5. Migrate external tracking tables for each schema to use the new format. For - instance in Python: - - ```python - import datajoint.migrate as migrate - db_schema_name='schema_1' - external_store='raw' - migrate.migrate_dj011_external_blob_storage_to_dj012(db_schema_name, external_store) - ``` - - 6. Verify pipeline functionality after this process has completed. For instance in - Python: - - ```python - x = myschema.TableWithExternal.fetch('external_field', limit=1)[0] - ``` - -Note: This migration function is provided on a best-effort basis, and will - convert the external tracking tables into a format which is compatible - with DataJoint v0.12. While we have attempted to ensure correctness - of the process, all use-cases have not been heavily tested. Please be sure to fully - back-up your data and be prepared to investigate problems with the - migration, should they occur. - -Please note: - -- The migration only migrates the tracking table format and does not -modify the backing file structure to support `folding`. The DataJoint -v0.12 logic is able to work with this format, but to take advantage -of the new backend storage, manual adjustment of the tracking table -and files, or a full rebuild of the schema should be performed. - -- Additional care to ensure all clients are using v0.12 should be -taken after the upgrade. Legacy clients may incorrectly create data -in the old format which would then need to be combined or otherwise -reconciled with the data in v0.12 format. You might wish to take -the opportunity to version-pin your installations so that future -changes requiring controlled upgrades can be coordinated on a system -wide basis. diff --git a/docs/src/tutorials/dj-top.ipynb b/docs/src/tutorials/dj-top.ipynb deleted file mode 100644 index 7ed9f97cc..000000000 --- a/docs/src/tutorials/dj-top.ipynb +++ /dev/null @@ -1,1045 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Using the dj.Top restriction" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "First you will need to [install](../../getting-started/#installation) and [connect](../../getting-started/#connection) to a DataJoint [data pipeline](https://docs.datajoint.com/core/datajoint-python/latest/concepts/data-pipelines/#what-is-a-data-pipeline).\n", - "\n", - "Now let's start by importing the `datajoint` client." - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "[2024-12-20 11:10:20,120][INFO]: Connecting root@127.0.0.1:3306\n", - "[2024-12-20 11:10:20,259][INFO]: Connected root@127.0.0.1:3306\n" - ] - } - ], - "source": [ - "import datajoint as dj\n", - "\n", - "dj.config[\"database.host\"] = \"127.0.0.1\"\n", - "schema = dj.Schema(\"university\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Table Definition" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "@schema\n", - "class Student(dj.Manual):\n", - " definition = \"\"\"\n", - " student_id : int unsigned # university-wide ID number\n", - " ---\n", - " first_name : varchar(40)\n", - " last_name : varchar(40)\n", - " sex : enum('F', 'M', 'U')\n", - " date_of_birth : date\n", - " home_address : varchar(120) # mailing street address\n", - " home_city : varchar(60) # mailing address\n", - " home_state : char(2) # US state acronym: e.g. OH\n", - " home_zip : char(10) # zipcode e.g. 93979-4979\n", - " home_phone : varchar(20) # e.g. 414.657.6883x0881\n", - " \"\"\"" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "@schema\n", - "class Department(dj.Manual):\n", - " definition = \"\"\"\n", - " dept : varchar(6) # abbreviated department name, e.g. BIOL\n", - " ---\n", - " dept_name : varchar(200) # full department name\n", - " dept_address : varchar(200) # mailing address\n", - " dept_phone : varchar(20)\n", - " \"\"\"" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "@schema\n", - "class StudentMajor(dj.Manual):\n", - " definition = \"\"\"\n", - " -> Student\n", - " ---\n", - " -> Department\n", - " declare_date : date # when student declared her major\n", - " \"\"\"" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - "@schema\n", - "class Course(dj.Manual):\n", - " definition = \"\"\"\n", - " -> Department\n", - " course : int unsigned # course number, e.g. 1010\n", - " ---\n", - " course_name : varchar(200) # e.g. \"Neurobiology of Sensation and Movement.\"\n", - " credits : decimal(3,1) # number of credits earned by completing the course\n", - " \"\"\"\n", - "\n", - "\n", - "@schema\n", - "class Term(dj.Manual):\n", - " definition = \"\"\"\n", - " term_year : year\n", - " term : enum('Spring', 'Summer', 'Fall')\n", - " \"\"\"\n", - "\n", - "\n", - "@schema\n", - "class Section(dj.Manual):\n", - " definition = \"\"\"\n", - " -> Course\n", - " -> Term\n", - " section : char(1)\n", - " ---\n", - " auditorium : varchar(12)\n", - " \"\"\"\n", - "\n", - "\n", - "@schema\n", - "class CurrentTerm(dj.Manual):\n", - " definition = \"\"\"\n", - " -> Term\n", - " \"\"\"\n", - "\n", - "\n", - "@schema\n", - "class Enroll(dj.Manual):\n", - " definition = \"\"\"\n", - " -> Student\n", - " -> Section\n", - " \"\"\"\n", - "\n", - "\n", - "@schema\n", - "class LetterGrade(dj.Lookup):\n", - " definition = \"\"\"\n", - " grade : char(2)\n", - " ---\n", - " points : decimal(3,2)\n", - " \"\"\"\n", - " contents = [\n", - " [\"A\", 4.00],\n", - " [\"A-\", 3.67],\n", - " [\"B+\", 3.33],\n", - " [\"B\", 3.00],\n", - " [\"B-\", 2.67],\n", - " [\"C+\", 2.33],\n", - " [\"C\", 2.00],\n", - " [\"C-\", 1.67],\n", - " [\"D+\", 1.33],\n", - " [\"D\", 1.00],\n", - " [\"F\", 0.00],\n", - " ]\n", - "\n", - "\n", - "@schema\n", - "class Grade(dj.Manual):\n", - " definition = \"\"\"\n", - " -> Enroll\n", - " ---\n", - " -> LetterGrade\n", - " \"\"\"" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Insert" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [], - "source": [ - "from tqdm import tqdm\n", - "import faker\n", - "import random\n", - "import datetime\n", - "\n", - "fake = faker.Faker()" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [], - "source": [ - "def yield_students():\n", - " fake_name = {\"F\": fake.name_female, \"M\": fake.name_male}\n", - " while True: # ignore invalid values\n", - " try:\n", - " sex = random.choice((\"F\", \"M\"))\n", - " first_name, last_name = fake_name[sex]().split(\" \")[:2]\n", - " street_address, city = fake.address().split(\"\\n\")\n", - " city, state = city.split(\", \")\n", - " state, zipcode = state.split(\" \")\n", - " except ValueError:\n", - " continue\n", - " else:\n", - " yield dict(\n", - " first_name=first_name,\n", - " last_name=last_name,\n", - " sex=sex,\n", - " home_address=street_address,\n", - " home_city=city,\n", - " home_state=state,\n", - " home_zip=zipcode,\n", - " date_of_birth=str(\n", - " fake.date_time_between(start_date=\"-35y\", end_date=\"-15y\").date()\n", - " ),\n", - " home_phone=fake.phone_number()[:20],\n", - " )" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [], - "source": [ - "Student.insert(dict(k, student_id=i) for i, k in zip(range(100, 300), yield_students()))\n", - "\n", - "Department.insert(\n", - " dict(\n", - " dept=dept,\n", - " dept_name=name,\n", - " dept_address=fake.address(),\n", - " dept_phone=fake.phone_number()[:20],\n", - " )\n", - " for dept, name in [\n", - " [\"CS\", \"Computer Science\"],\n", - " [\"BIOL\", \"Life Sciences\"],\n", - " [\"PHYS\", \"Physics\"],\n", - " [\"MATH\", \"Mathematics\"],\n", - " ]\n", - ")\n", - "\n", - "StudentMajor.insert(\n", - " {**s, **d, \"declare_date\": fake.date_between(start_date=datetime.date(1999, 1, 1))}\n", - " for s, d in zip(\n", - " Student.fetch(\"KEY\"), random.choices(Department.fetch(\"KEY\"), k=len(Student()))\n", - " )\n", - " if random.random() < 0.75\n", - ")\n", - "\n", - "# from https://www.utah.edu/\n", - "Course.insert(\n", - " [\n", - " [\"BIOL\", 1006, \"World of Dinosaurs\", 3],\n", - " [\"BIOL\", 1010, \"Biology in the 21st Century\", 3],\n", - " [\"BIOL\", 1030, \"Human Biology\", 3],\n", - " [\"BIOL\", 1210, \"Principles of Biology\", 4],\n", - " [\"BIOL\", 2010, \"Evolution & Diversity of Life\", 3],\n", - " [\"BIOL\", 2020, \"Principles of Cell Biology\", 3],\n", - " [\"BIOL\", 2021, \"Principles of Cell Science\", 4],\n", - " [\"BIOL\", 2030, \"Principles of Genetics\", 3],\n", - " [\"BIOL\", 2210, \"Human Genetics\", 3],\n", - " [\"BIOL\", 2325, \"Human Anatomy\", 4],\n", - " [\"BIOL\", 2330, \"Plants & Society\", 3],\n", - " [\"BIOL\", 2355, \"Field Botany\", 2],\n", - " [\"BIOL\", 2420, \"Human Physiology\", 4],\n", - " [\"PHYS\", 2040, \"Classcal Theoretical Physics II\", 4],\n", - " [\"PHYS\", 2060, \"Quantum Mechanics\", 3],\n", - " [\"PHYS\", 2100, \"General Relativity and Cosmology\", 3],\n", - " [\"PHYS\", 2140, \"Statistical Mechanics\", 4],\n", - " [\"PHYS\", 2210, \"Physics for Scientists and Engineers I\", 4],\n", - " [\"PHYS\", 2220, \"Physics for Scientists and Engineers II\", 4],\n", - " [\"PHYS\", 3210, \"Physics for Scientists I (Honors)\", 4],\n", - " [\"PHYS\", 3220, \"Physics for Scientists II (Honors)\", 4],\n", - " [\"MATH\", 1250, \"Calculus for AP Students I\", 4],\n", - " [\"MATH\", 1260, \"Calculus for AP Students II\", 4],\n", - " [\"MATH\", 1210, \"Calculus I\", 4],\n", - " [\"MATH\", 1220, \"Calculus II\", 4],\n", - " [\"MATH\", 2210, \"Calculus III\", 3],\n", - " [\"MATH\", 2270, \"Linear Algebra\", 4],\n", - " [\"MATH\", 2280, \"Introduction to Differential Equations\", 4],\n", - " [\"MATH\", 3210, \"Foundations of Analysis I\", 4],\n", - " [\"MATH\", 3220, \"Foundations of Analysis II\", 4],\n", - " [\"CS\", 1030, \"Foundations of Computer Science\", 3],\n", - " [\"CS\", 1410, \"Introduction to Object-Oriented Programming\", 4],\n", - " [\"CS\", 2420, \"Introduction to Algorithms & Data Structures\", 4],\n", - " [\"CS\", 2100, \"Discrete Structures\", 3],\n", - " [\"CS\", 3500, \"Software Practice\", 4],\n", - " [\"CS\", 3505, \"Software Practice II\", 3],\n", - " [\"CS\", 3810, \"Computer Organization\", 4],\n", - " [\"CS\", 4400, \"Computer Systems\", 4],\n", - " [\"CS\", 4150, \"Algorithms\", 3],\n", - " [\"CS\", 3100, \"Models of Computation\", 3],\n", - " [\"CS\", 3200, \"Introduction to Scientific Computing\", 3],\n", - " [\"CS\", 4000, \"Senior Capstone Project - Design Phase\", 3],\n", - " [\"CS\", 4500, \"Senior Capstone Project\", 3],\n", - " [\"CS\", 4940, \"Undergraduate Research\", 3],\n", - " [\"CS\", 4970, \"Computer Science Bachelors Thesis\", 3],\n", - " ]\n", - ")\n", - "\n", - "Term.insert(\n", - " dict(term_year=year, term=term)\n", - " for year in range(1999, 2019)\n", - " for term in [\"Spring\", \"Summer\", \"Fall\"]\n", - ")\n", - "\n", - "Term().fetch(order_by=(\"term_year DESC\", \"term DESC\"), as_dict=True, limit=1)[0]\n", - "\n", - "CurrentTerm().insert1(\n", - " {**Term().fetch(order_by=(\"term_year DESC\", \"term DESC\"), as_dict=True, limit=1)[0]}\n", - ")\n", - "\n", - "\n", - "def make_section(prob):\n", - " for c in (Course * Term).proj():\n", - " for sec in \"abcd\":\n", - " if random.random() < prob:\n", - " break\n", - " yield {\n", - " **c,\n", - " \"section\": sec,\n", - " \"auditorium\": random.choice(\"ABCDEF\") + str(random.randint(1, 100)),\n", - " }\n", - "\n", - "\n", - "Section.insert(make_section(0.5))" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|██████████| 200/200 [00:27<00:00, 7.17it/s]\n" - ] - } - ], - "source": [ - "# Enrollment\n", - "terms = Term().fetch(\"KEY\")\n", - "quit_prob = 0.1\n", - "for student in tqdm(Student.fetch(\"KEY\")):\n", - " start_term = random.randrange(len(terms))\n", - " for term in terms[start_term:]:\n", - " if random.random() < quit_prob:\n", - " break\n", - " else:\n", - " sections = ((Section & term) - (Course & (Enroll & student))).fetch(\"KEY\")\n", - " if sections:\n", - " Enroll.insert(\n", - " {**student, **section}\n", - " for section in random.sample(\n", - " sections, random.randrange(min(5, len(sections)))\n", - " )\n", - " )\n", - "\n", - "# assign random grades\n", - "grades = LetterGrade.fetch(\"grade\")\n", - "\n", - "grade_keys = Enroll.fetch(\"KEY\")\n", - "random.shuffle(grade_keys)\n", - "grade_keys = grade_keys[: len(grade_keys) * 9 // 10]\n", - "\n", - "Grade.insert(\n", - " {**key, \"grade\": grade}\n", - " for key, grade in zip(grade_keys, random.choices(grades, k=len(grade_keys)))\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# dj.Top Restriction" - ] - }, - { - "cell_type": "code", - "execution_count": 29, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - " \n", - " \n", - " \n", - " \n", - "
\n", - " \n", - " \n", - " \n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "
\n", - "

student_id

\n", - " university-wide ID number\n", - "
\n", - "

dept

\n", - " abbreviated department name, e.g. BIOL\n", - "
\n", - "

course

\n", - " course number, e.g. 1010\n", - "
\n", - "

term_year

\n", - " \n", - "
\n", - "

term

\n", - " \n", - "
\n", - "

section

\n", - " \n", - "
\n", - "

grade

\n", - " \n", - "
\n", - "

points

\n", - " \n", - "
100MATH22802018FallaA-3.67
191MATH22102018SpringbA4.00
211CS21002018FallaA4.00
273PHYS21002018SpringaA4.00
282BIOL20212018SpringdA4.00
\n", - " \n", - "

Total: 5

\n", - " " - ], - "text/plain": [ - "*student_id *dept *course *term_year *term *section *grade points \n", - "+------------+ +------+ +--------+ +-----------+ +--------+ +---------+ +-------+ +--------+\n", - "100 MATH 2280 2018 Fall a A- 3.67 \n", - "191 MATH 2210 2018 Spring b A 4.00 \n", - "211 CS 2100 2018 Fall a A 4.00 \n", - "273 PHYS 2100 2018 Spring a A 4.00 \n", - "282 BIOL 2021 2018 Spring d A 4.00 \n", - " (Total: 5)" - ] - }, - "execution_count": 29, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "(Grade * LetterGrade) & \"term_year='2018'\" & dj.Top(\n", - " limit=5, order_by=\"points DESC\", offset=5\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 35, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "\"SELECT `grade`,`student_id`,`dept`,`course`,`term_year`,`term`,`section`,`points` FROM `university`.`#letter_grade` NATURAL JOIN `university`.`grade` WHERE ( (term_year='2018')) ORDER BY `points` DESC LIMIT 10\"" - ] - }, - "execution_count": 35, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "(\n", - " (LetterGrade * Grade)\n", - " & \"term_year='2018'\"\n", - " & dj.Top(limit=10, order_by=\"points DESC\", offset=0)\n", - ").make_sql()" - ] - }, - { - "cell_type": "code", - "execution_count": 44, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "\"SELECT `student_id`,`dept`,`course`,`term_year`,`term`,`section`,`grade`,`points` FROM `university`.`grade` NATURAL JOIN `university`.`#letter_grade` WHERE ( (term_year='2018')) ORDER BY `points` DESC LIMIT 20\"" - ] - }, - "execution_count": 44, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "(\n", - " (Grade * LetterGrade)\n", - " & \"term_year='2018'\"\n", - " & dj.Top(limit=20, order_by=\"points DESC\", offset=0)\n", - ").make_sql()" - ] - }, - { - "cell_type": "code", - "execution_count": 47, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - " \n", - " \n", - " \n", - " \n", - "
\n", - " \n", - " \n", - " \n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "
\n", - "

student_id

\n", - " university-wide ID number\n", - "
\n", - "

dept

\n", - " abbreviated department name, e.g. BIOL\n", - "
\n", - "

course

\n", - " course number, e.g. 1010\n", - "
\n", - "

term_year

\n", - " \n", - "
\n", - "

term

\n", - " \n", - "
\n", - "

section

\n", - " \n", - "
\n", - "

grade

\n", - " \n", - "
\n", - "

points

\n", - " \n", - "
100CS32002018FallcA4.00
100MATH22802018FallaA-3.67
100PHYS22102018SpringdA4.00
122CS10302018FallcB+3.33
131BIOL20302018SpringaA4.00
131CS32002018FallbB+3.33
136BIOL22102018SpringcB+3.33
136MATH22102018FallbB+3.33
141BIOL20102018SummercB+3.33
141CS24202018FallbA4.00
141CS32002018FallbA-3.67
182CS14102018SummercA-3.67
\n", - "

...

\n", - "

Total: 20

\n", - " " - ], - "text/plain": [ - "*student_id *dept *course *term_year *term *section *grade points \n", - "+------------+ +------+ +--------+ +-----------+ +--------+ +---------+ +-------+ +--------+\n", - "100 CS 3200 2018 Fall c A 4.00 \n", - "100 MATH 2280 2018 Fall a A- 3.67 \n", - "100 PHYS 2210 2018 Spring d A 4.00 \n", - "122 CS 1030 2018 Fall c B+ 3.33 \n", - "131 BIOL 2030 2018 Spring a A 4.00 \n", - "131 CS 3200 2018 Fall b B+ 3.33 \n", - "136 BIOL 2210 2018 Spring c B+ 3.33 \n", - "136 MATH 2210 2018 Fall b B+ 3.33 \n", - "141 BIOL 2010 2018 Summer c B+ 3.33 \n", - "141 CS 2420 2018 Fall b A 4.00 \n", - "141 CS 3200 2018 Fall b A- 3.67 \n", - "182 CS 1410 2018 Summer c A- 3.67 \n", - " ...\n", - " (Total: 20)" - ] - }, - "execution_count": 47, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "(Grade * LetterGrade) & \"term_year='2018'\" & dj.Top(\n", - " limit=20, order_by=\"points DESC\", offset=0\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 41, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - " \n", - " \n", - " \n", - " \n", - "
\n", - " \n", - " \n", - " \n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "
\n", - "

grade

\n", - " \n", - "
\n", - "

student_id

\n", - " university-wide ID number\n", - "
\n", - "

dept

\n", - " abbreviated department name, e.g. BIOL\n", - "
\n", - "

course

\n", - " course number, e.g. 1010\n", - "
\n", - "

term_year

\n", - " \n", - "
\n", - "

term

\n", - " \n", - "
\n", - "

section

\n", - " \n", - "
\n", - "

points

\n", - " \n", - "
A100CS32002018Fallc4.00
A100PHYS22102018Springd4.00
A131BIOL20302018Springa4.00
A141CS24202018Fallb4.00
A186PHYS22102018Springa4.00
A191MATH22102018Springb4.00
A211CS21002018Falla4.00
A273PHYS21002018Springa4.00
A282BIOL20212018Springd4.00
A-100MATH22802018Falla3.67
A-141CS32002018Fallb3.67
A-182CS14102018Summerc3.67
\n", - "

...

\n", - "

Total: 20

\n", - " " - ], - "text/plain": [ - "*grade *student_id *dept *course *term_year *term *section points \n", - "+-------+ +------------+ +------+ +--------+ +-----------+ +--------+ +---------+ +--------+\n", - "A 100 CS 3200 2018 Fall c 4.00 \n", - "A 100 PHYS 2210 2018 Spring d 4.00 \n", - "A 131 BIOL 2030 2018 Spring a 4.00 \n", - "A 141 CS 2420 2018 Fall b 4.00 \n", - "A 186 PHYS 2210 2018 Spring a 4.00 \n", - "A 191 MATH 2210 2018 Spring b 4.00 \n", - "A 211 CS 2100 2018 Fall a 4.00 \n", - "A 273 PHYS 2100 2018 Spring a 4.00 \n", - "A 282 BIOL 2021 2018 Spring d 4.00 \n", - "A- 100 MATH 2280 2018 Fall a 3.67 \n", - "A- 141 CS 3200 2018 Fall b 3.67 \n", - "A- 182 CS 1410 2018 Summer c 3.67 \n", - " ...\n", - " (Total: 20)" - ] - }, - "execution_count": 41, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "(LetterGrade * Grade) & \"term_year='2018'\" & dj.Top(\n", - " limit=20, order_by=\"points DESC\", offset=0\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "elements", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.8" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/docs/src/tutorials/json.ipynb b/docs/src/tutorials/json.ipynb deleted file mode 100644 index 9c5feebf6..000000000 --- a/docs/src/tutorials/json.ipynb +++ /dev/null @@ -1,1080 +0,0 @@ -{ - "cells": [ - { - "attachments": {}, - "cell_type": "markdown", - "id": "7fe24127-c0d0-4ff8-96b4-6ab0d9307e73", - "metadata": {}, - "source": [ - "# Using the json type" - ] - }, - { - "cell_type": "markdown", - "id": "62450023", - "metadata": {}, - "source": [ - "> ⚠️ Note the following before using the `json` type\n", - "> - Supported only for MySQL >= 8.0 when [JSON_VALUE](https://dev.mysql.com/doc/refman/8.0/en/json-search-functions.html#function_json-value) introduced.\n", - "> - Equivalent Percona is fully-compatible.\n", - "> - MariaDB is not supported since [JSON_VALUE](https://mariadb.com/kb/en/json_value/#syntax) does not allow type specification like MySQL's.\n", - "> - Not yet supported in DataJoint MATLAB" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "id": "67cf93d2", - "metadata": {}, - "source": [ - "First you will need to [install](../../getting-started/#installation) and [connect](../../getting-started/#connection) to a DataJoint [data pipeline](https://docs.datajoint.com/core/datajoint-python/latest/concepts/data-pipelines/#what-is-a-data-pipeline).\n", - "\n", - "Now let's start by importing the `datajoint` client." - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "id": "bc0b6f54-8f11-45f4-bf8d-e1058ee0056f", - "metadata": {}, - "outputs": [], - "source": [ - "import datajoint as dj" - ] - }, - { - "cell_type": "markdown", - "id": "3544cab9-f2db-458a-9431-939bea5affc5", - "metadata": {}, - "source": [ - "## Table Definition" - ] - }, - { - "cell_type": "markdown", - "id": "a2998c71", - "metadata": {}, - "source": [ - "For this exercise, let's imagine we work for an awesome company that is organizing a fun RC car race across various teams in the company. Let's see which team has the fastest car! 🏎️\n", - "\n", - "This establishes 2 important entities: a `Team` and a `Car`. Normally the entities are mapped to their own dedicated table, however, let's assume that `Team` is well-structured but `Car` is less structured than we'd prefer. In other words, the structure for what makes up a *car* is varying too much between entries (perhaps because users of the pipeline haven't agreed yet on the definition? 🤷).\n", - "\n", - "This would make it a good use-case to keep `Team` as a table but make `Car` a `json` type defined within the `Team` table.\n", - "\n", - "Let's begin." - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "dc318298-b819-4f06-abbd-7bb7544dd431", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "[2023-02-12 00:14:33,027][INFO]: Connecting root@fakeservices.datajoint.io:3306\n", - "[2023-02-12 00:14:33,039][INFO]: Connected root@fakeservices.datajoint.io:3306\n" - ] - } - ], - "source": [ - "schema = dj.Schema(f\"{dj.config['database.user']}_json\")" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "4aaf96db-85d9-4e94-a4c3-3558f4cc6671", - "metadata": {}, - "outputs": [], - "source": [ - "@schema\n", - "class Team(dj.Lookup):\n", - " definition = \"\"\"\n", - " # A team within a company\n", - " name: varchar(40) # team name\n", - " ---\n", - " car=null: json # A car belonging to a team (null to allow registering first but specifying car later)\n", - " \n", - " unique index(car.length:decimal(4, 1)) # Add an index if this key is frequently accessed\n", - " \"\"\"" - ] - }, - { - "cell_type": "markdown", - "id": "640bf7a7-9e07-4953-9c8a-304e55c467f8", - "metadata": {}, - "source": [ - "## Insert" - ] - }, - { - "cell_type": "markdown", - "id": "7081e577", - "metadata": {}, - "source": [ - "Let's suppose that engineering is first up to register their car." - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "30f0d62e", - "metadata": {}, - "outputs": [], - "source": [ - "Team.insert1(\n", - " {\n", - " \"name\": \"engineering\",\n", - " \"car\": {\n", - " \"name\": \"Rever\",\n", - " \"length\": 20.5,\n", - " \"inspected\": True,\n", - " \"tire_pressure\": [32, 31, 33, 34],\n", - " \"headlights\": [\n", - " {\n", - " \"side\": \"left\",\n", - " \"hyper_white\": None,\n", - " },\n", - " {\n", - " \"side\": \"right\",\n", - " \"hyper_white\": None,\n", - " },\n", - " ],\n", - " },\n", - " }\n", - ")" - ] - }, - { - "cell_type": "markdown", - "id": "ee5e4dcf", - "metadata": {}, - "source": [ - "Next, business and marketing teams are up and register their cars.\n", - "\n", - "A few points to notice below:\n", - "- The person signing up on behalf of marketing does not know the specifics of the car during registration but another team member will be updating this soon before the race.\n", - "- Notice how the `business` and `engineering` teams appear to specify the same property but refer to it as `safety_inspected` and `inspected` respectfully." - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "id": "b532e16c", - "metadata": {}, - "outputs": [], - "source": [ - "Team.insert(\n", - " [\n", - " {\n", - " \"name\": \"marketing\",\n", - " \"car\": None,\n", - " },\n", - " {\n", - " \"name\": \"business\",\n", - " \"car\": {\n", - " \"name\": \"Chaching\",\n", - " \"length\": 100,\n", - " \"safety_inspected\": False,\n", - " \"tire_pressure\": [34, 30, 27, 32],\n", - " \"headlights\": [\n", - " {\n", - " \"side\": \"left\",\n", - " \"hyper_white\": True,\n", - " },\n", - " {\n", - " \"side\": \"right\",\n", - " \"hyper_white\": True,\n", - " },\n", - " ],\n", - " },\n", - " },\n", - " ]\n", - ")" - ] - }, - { - "cell_type": "markdown", - "id": "57365de7", - "metadata": {}, - "source": [ - "We can preview the table data much like normal but notice how the value of `car` behaves like other BLOB-like attributes." - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "id": "0e3b517c", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - " \n", - " \n", - " \n", - " A team within a company\n", - "
\n", - " \n", - " \n", - " \n", - "\n", - "\n", - "\n", - "
\n", - "

name

\n", - " team name\n", - "
\n", - "

car

\n", - " A car belonging to a team (null to allow registering first but specifying car later)\n", - "
marketing=BLOB=
engineering=BLOB=
business=BLOB=
\n", - " \n", - "

Total: 3

\n", - " " - ], - "text/plain": [ - "*name car \n", - "+------------+ +--------+\n", - "marketing =BLOB= \n", - "engineering =BLOB= \n", - "business =BLOB= \n", - " (Total: 3)" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "Team()" - ] - }, - { - "cell_type": "markdown", - "id": "c95cbbee-4ef7-4870-ad42-a60345a3644f", - "metadata": {}, - "source": [ - "## Restriction" - ] - }, - { - "cell_type": "markdown", - "id": "8b454996", - "metadata": {}, - "source": [ - "Now let's see what kinds of queries we can form to demostrate how we can query this pipeline." - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "id": "81efda24", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - " \n", - " \n", - " \n", - " A team within a company\n", - "
\n", - " \n", - " \n", - " \n", - "\n", - "
\n", - "

name

\n", - " team name\n", - "
\n", - "

car

\n", - " A car belonging to a team (null to allow registering first but specifying car later)\n", - "
business=BLOB=
\n", - " \n", - "

Total: 1

\n", - " " - ], - "text/plain": [ - "*name car \n", - "+----------+ +--------+\n", - "business =BLOB= \n", - " (Total: 1)" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# Which team has a `car` equal to 100 inches long?\n", - "Team & {\"car.length\": 100}" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "id": "fd7b855d", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - " \n", - " \n", - " \n", - " A team within a company\n", - "
\n", - " \n", - " \n", - " \n", - "\n", - "
\n", - "

name

\n", - " team name\n", - "
\n", - "

car

\n", - " A car belonging to a team (null to allow registering first but specifying car later)\n", - "
engineering=BLOB=
\n", - " \n", - "

Total: 1

\n", - " " - ], - "text/plain": [ - "*name car \n", - "+------------+ +--------+\n", - "engineering =BLOB= \n", - " (Total: 1)" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# Which team has a `car` less than 50 inches long?\n", - "Team & \"car->>'$.length' < 50\"" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "id": "b76ebb75", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - " \n", - " \n", - " \n", - " A team within a company\n", - "
\n", - " \n", - " \n", - " \n", - "\n", - "
\n", - "

name

\n", - " team name\n", - "
\n", - "

car

\n", - " A car belonging to a team (null to allow registering first but specifying car later)\n", - "
engineering=BLOB=
\n", - " \n", - "

Total: 1

\n", - " " - ], - "text/plain": [ - "*name car \n", - "+------------+ +--------+\n", - "engineering =BLOB= \n", - " (Total: 1)" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# Any team that has had their car inspected?\n", - "Team & [{\"car.inspected:unsigned\": True}, {\"car.safety_inspected:unsigned\": True}]" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "id": "b787784c", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - " \n", - " \n", - " \n", - " A team within a company\n", - "
\n", - " \n", - " \n", - " \n", - "\n", - "\n", - "
\n", - "

name

\n", - " team name\n", - "
\n", - "

car

\n", - " A car belonging to a team (null to allow registering first but specifying car later)\n", - "
engineering=BLOB=
marketing=BLOB=
\n", - " \n", - "

Total: 2

\n", - " " - ], - "text/plain": [ - "*name car \n", - "+------------+ +--------+\n", - "engineering =BLOB= \n", - "marketing =BLOB= \n", - " (Total: 2)" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# Which teams do not have hyper white lights for their first head light?\n", - "Team & {\"car.headlights[0].hyper_white\": None}" - ] - }, - { - "cell_type": "markdown", - "id": "5bcf0b5d", - "metadata": {}, - "source": [ - "Notice that the previous query will satisfy the `None` check if it experiences any of the following scenarious:\n", - "- if entire record missing (`marketing` satisfies this)\n", - "- JSON key is missing\n", - "- JSON value is set to JSON `null` (`engineering` satisfies this)" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "id": "bcf1682e-a0c7-4c2f-826b-0aec9052a694", - "metadata": {}, - "source": [ - "## Projection" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "id": "daea110e", - "metadata": {}, - "source": [ - "Projections can be quite useful with the `json` type since we can extract out just what we need. This allows greater query flexibility but more importantly, for us to be able to fetch only what is pertinent." - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "id": "8fb8334a", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - " \n", - " \n", - " \n", - " \n", - "
\n", - " \n", - " \n", - " \n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "
\n", - "

name

\n", - " team name\n", - "
\n", - "

car_name

\n", - " calculated attribute\n", - "
\n", - "

car_length

\n", - " calculated attribute\n", - "
businessChaching100
engineeringRever20.5
marketingNoneNone
\n", - " \n", - "

Total: 3

\n", - " " - ], - "text/plain": [ - "*name car_name car_length \n", - "+------------+ +----------+ +------------+\n", - "business Chaching 100 \n", - "engineering Rever 20.5 \n", - "marketing None None \n", - " (Total: 3)" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# Only interested in the car names and the length but let the type be inferred\n", - "q_untyped = Team.proj(\n", - " car_name=\"car.name\",\n", - " car_length=\"car.length\",\n", - ")\n", - "q_untyped" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "id": "bb5f0448", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[{'name': 'business', 'car_name': 'Chaching', 'car_length': '100'},\n", - " {'name': 'engineering', 'car_name': 'Rever', 'car_length': '20.5'},\n", - " {'name': 'marketing', 'car_name': None, 'car_length': None}]" - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "q_untyped.fetch(as_dict=True)" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "id": "a307dfd7", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - " \n", - " \n", - " \n", - " \n", - "
\n", - " \n", - " \n", - " \n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "
\n", - "

name

\n", - " team name\n", - "
\n", - "

car_name

\n", - " calculated attribute\n", - "
\n", - "

car_length

\n", - " calculated attribute\n", - "
businessChaching100.0
engineeringRever20.5
marketingNoneNone
\n", - " \n", - "

Total: 3

\n", - " " - ], - "text/plain": [ - "*name car_name car_length \n", - "+------------+ +----------+ +------------+\n", - "business Chaching 100.0 \n", - "engineering Rever 20.5 \n", - "marketing None None \n", - " (Total: 3)" - ] - }, - "execution_count": 13, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# Nevermind, I'll specify the type explicitly\n", - "q_typed = Team.proj(\n", - " car_name=\"car.name\",\n", - " car_length=\"car.length:float\",\n", - ")\n", - "q_typed" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "id": "8a93dbf9", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[{'name': 'business', 'car_name': 'Chaching', 'car_length': 100.0},\n", - " {'name': 'engineering', 'car_name': 'Rever', 'car_length': 20.5},\n", - " {'name': 'marketing', 'car_name': None, 'car_length': None}]" - ] - }, - "execution_count": 14, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "q_typed.fetch(as_dict=True)" - ] - }, - { - "cell_type": "markdown", - "id": "62dd0239-fa70-4369-81eb-3d46c5053fee", - "metadata": {}, - "source": [ - "## Describe" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "id": "73d9df01", - "metadata": {}, - "source": [ - "Lastly, the `.describe()` function on the `Team` table can help us generate the table's definition. This is useful if we are connected directly to the pipeline without the original source." - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "id": "0e739932", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "# A team within a company\n", - "name : varchar(40) # team name\n", - "---\n", - "car=null : json # A car belonging to a team (null to allow registering first but specifying car later)\n", - "UNIQUE INDEX ((json_value(`car`, _utf8mb4'$.length' returning decimal(4, 1))))\n", - "\n" - ] - } - ], - "source": [ - "rebuilt_definition = Team.describe()\n", - "print(rebuilt_definition)" - ] - }, - { - "cell_type": "markdown", - "id": "be1070d5-765b-4bc2-92de-8a6ffd885984", - "metadata": {}, - "source": [ - "## Cleanup" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "id": "cb959927", - "metadata": {}, - "source": [ - "Finally, let's clean up what we created in this tutorial." - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "id": "d9cc28a3-3ffd-4126-b7e9-bc6365040b93", - "metadata": {}, - "outputs": [], - "source": [ - "schema.drop()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "68ad4340", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "all_purposes", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.18" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/pixi.lock b/pixi.lock new file mode 100644 index 000000000..dcc82c2b5 --- /dev/null +++ b/pixi.lock @@ -0,0 +1,6128 @@ +version: 6 +environments: + default: + channels: + - url: https://conda.anaconda.org/conda-forge/ + indexes: + - https://pypi.org/simple + packages: + linux-64: + - conda: https://conda.anaconda.org/conda-forge/linux-64/_libgcc_mutex-0.1-conda_forge.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/linux-64/_openmp_mutex-4.5-2_gnu.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/noarch/adwaita-icon-theme-48.1-unix_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/at-spi2-atk-2.38.0-h0630a04_3.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/linux-64/at-spi2-core-2.40.3-h0630a04_0.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/linux-64/atk-1.0-2.38.0-h04ea711_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/bzip2-1.0.8-hda65f42_8.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025.8.3-hbd8a1cb_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/cairo-1.18.4-h3394656_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/dbus-1.16.2-h3c4dab8_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/epoxy-1.5.10-h166bdaf_1.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/noarch/font-ttf-dejavu-sans-mono-2.37-hab24e00_0.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/noarch/font-ttf-inconsolata-3.000-h77eed37_0.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/noarch/font-ttf-source-code-pro-2.038-h77eed37_0.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/noarch/font-ttf-ubuntu-0.83-h77eed37_3.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/fontconfig-2.15.0-h7e30c49_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/fonts-conda-ecosystem-1-0.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/noarch/fonts-conda-forge-1-0.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/linux-64/freetype-2.14.1-ha770c72_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/fribidi-1.0.16-hb03c661_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/gdk-pixbuf-2.44.1-h2b0a6b4_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/glib-tools-2.86.0-hf516916_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/graphite2-1.3.14-hecca717_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/graphviz-13.1.2-h87b6fe6_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/gtk3-3.24.43-h0c6a113_5.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/gts-0.7.6-h977cf35_4.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/harfbuzz-11.5.0-h15599e2_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/hicolor-icon-theme-0.17-ha770c72_2.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/linux-64/icu-75.1-he02047a_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/keyutils-1.6.3-hb9d3cd8_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/krb5-1.21.3-h659f571_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/ld_impl_linux-64-2.44-h1423503_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/lerc-4.0.0-h0aef613_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libcups-2.3.3-hb8b1518_5.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libdeflate-1.24-h86f0d12_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libedit-3.1.20250104-pl5321h7949ede_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libexpat-2.7.1-hecca717_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libffi-3.4.6-h2dba641_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libfreetype-2.14.1-ha770c72_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libfreetype6-2.14.1-h73754d4_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libgcc-15.1.0-h767d61c_5.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libgcc-ng-15.1.0-h69a702a_5.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libgd-2.3.3-h6f5c62b_11.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libglib-2.86.0-h1fed272_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libgomp-15.1.0-h767d61c_5.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libiconv-1.18-h3b78370_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libjpeg-turbo-3.1.0-hb9d3cd8_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/liblzma-5.8.1-hb9d3cd8_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libmpdec-4.0.0-hb9d3cd8_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libpng-1.6.50-h421ea60_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/librsvg-2.58.4-he92a37e_3.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libsqlite-3.50.4-h0c1763c_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-15.1.0-h8f9b012_5.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-ng-15.1.0-h4852527_5.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libtiff-4.7.0-h8261f1e_6.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libuuid-2.41.1-he9a06e4_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libwebp-base-1.6.0-hd42ef1d_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libxcb-1.17.0-h8a09558_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libxkbcommon-1.11.0-he8b52b9_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libxml2-2.13.8-h04c0eec_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libzlib-1.3.1-hb9d3cd8_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/ncurses-6.5-h2d0b736_3.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/openssl-3.5.2-h26f9b46_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/pango-1.56.4-hadf4263_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/pcre2-10.46-h1321c63_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/pixman-0.46.4-h54a6638_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/pthread-stubs-0.4-hb9d3cd8_1002.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/python-3.13.7-h2b335a9_100_cp313.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python_abi-3.13-8_cp313.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/readline-8.2-h8c095d6_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/tk-8.6.13-noxft_hd72426e_102.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2025b-h78e105d_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/wayland-1.24.0-h3e06ad9_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/xkeyboard-config-2.45-hb9d3cd8_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/xorg-libice-1.1.2-hb9d3cd8_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/xorg-libsm-1.2.6-he73a12e_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/xorg-libx11-1.8.12-h4f16b4b_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/xorg-libxau-1.0.12-hb9d3cd8_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/xorg-libxcomposite-0.4.6-hb9d3cd8_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/xorg-libxcursor-1.2.3-hb9d3cd8_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/xorg-libxdamage-1.1.6-hb9d3cd8_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/xorg-libxdmcp-1.1.5-hb9d3cd8_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/xorg-libxext-1.3.6-hb9d3cd8_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/xorg-libxfixes-6.0.1-hb9d3cd8_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/xorg-libxi-1.8.2-hb9d3cd8_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/xorg-libxinerama-1.1.5-h5888daf_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/xorg-libxrandr-1.5.4-hb9d3cd8_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/xorg-libxrender-0.9.12-hb9d3cd8_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/xorg-libxtst-1.2.5-hb9d3cd8_3.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/zstd-1.5.7-hb8e6e7a_2.conda + - pypi: https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/4f/d3/a8b22fa575b297cd6e3e3b0155c7e25db170edf1c74783d6a31a2490b8d9/argon2_cffi-25.1.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/09/52/94108adfdd6e2ddf58be64f959a0b9c7d4ef2fa71086c38356d22dc501ea/argon2_cffi_bindings-25.1.0-cp39-abi3-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/25/8a/c46dcc25341b5bce5472c718902eb3d38600a903b14fa6aeecef3f21a46f/asttokens-3.0.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e5/48/1549795ba7742c948d2ad169c1c8cdbae65bc450d6cd753d124b17c8cd32/certifi-2025.8.3-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/98/df/0a1755e750013a2081e863e7cd37e0cdd02664372c754e5560099eb7aa44/cffi-2.0.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/4b/32/e0f13a1c5b0f8572d0ec6ae2f6c677b7991fafd95da523159c19eff0696a/contourpy-1.3.3-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/e7/05/c19819d5e3d95294a6f5947fb9b9629efb316b96de511b418c53d245aae6/cycler-0.12.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/4e/8c/f3147f5c4b73e7550fe5f9352eaa956ae838d5c51eb58e7a25b9f3e2643b/decorator-5.2.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/f7/e6/efe534ef0952b531b630780e19cabd416e2032697019d5295defc6ef9bd9/deepdiff-8.6.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/c1/ea/53f2148663b321f21b5a606bd5f191517cf40b7072c0497d3c92c4a13b1e/executing-2.2.1-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/f5/11/02ebebb09ff2104b690457cb7bc6ed700c9e0ce88cf581486bb0a5d3c88b/faker-37.8.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/f2/9f/bf231c2a3fac99d1d7f1d89c76594f158693f981a4aa02be406e9f036832/fonttools-4.59.2-cp313-cp313-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/08/2a/5628a99d04acb2d2f2e749cdf4ea571d2575e898df0528a090948018b726/ipython-9.5.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/d9/33/1f075bf72b0b747cb3288d011319aaf64083cf2efef8354174e3ed4540e2/ipython_pygments_lexers-1.1.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/c0/5a/9cac0c82afec3d09ccd97c8b6502d48f165f9124db81b4bcb90b4af974ee/jedi-0.19.2-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e9/e9/f218a2cb3a9ffbe324ca29a9e399fa2d2866d7f348ec3a88df87fc248fc5/kiwisolver-1.4.9-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/e5/b8/9eea6630198cb303d131d95d285a024b3b8645b1763a2916fddb44ca8760/matplotlib-3.10.6-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/8f/8e/9ad090d3553c280a8060fbf6e24dc1c0c29704ee7d1c372f0c174aa59285/matplotlib_inline-0.1.7-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/89/a3/00260f8df72b51afa1f182dd609533c77fa2407918c4c2813d87b4a56725/minio-7.2.16-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/eb/8d/776adee7bbf76365fdd7f2552710282c79a4ead5d2a46408c9043a2b70ba/networkx-3.5-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/9a/a5/bf3db6e66c4b160d6ea10b534c381a1955dfab34cb1017ea93aa33c70ed3/numpy-2.3.3-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/12/27/fb8d7338b4d551900fa3e580acbe7a0cf655d940e164cb5c00ec31961094/orderly_set-5.5.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/20/12/38679034af332785aac8774540895e234f4d07f7545804097de4b666afd8/packaging-25.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/8f/52/0634adaace9be2d8cac9ef78f05c47f3a675882e068438b9d7ec7ef0c13f/pandas-2.3.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/16/32/f8e3c85d1d5250232a5d3477a2a28cc291968ff175caeadaf3cc19ce0e4a/parso-0.8.5-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/9e/c3/059298687310d527a58bb01f3b1965787ee3b40dce76752eda8b44e9a2c5/pexpect-4.9.0-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/d5/1c/a2a29649c0b1983d3ef57ee87a66487fdeb45132df66ab30dd37f7dbe162/pillow-11.3.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/84/03/0d3ce49e2505ae70cf43bc5bb3033955d2fc9f932163e84dc0779cc47f48/prompt_toolkit-3.0.52-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/22/a6/858897256d0deac81a172289110f31629fc4cee19b6f01283303e18c8db3/ptyprocess-0.7.0-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/8e/37/efad0257dc6e593a18957422533ff0f87ede7c9c6ea010a2177d738fb82f/pure_eval-0.2.3-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/a0/e3/59cd50310fc9b59512193629e1984c1f95e5c8ae6e5d8c69532ccc65a7fe/pycparser-2.23-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/5f/e9/a09476d436d0ff1402ac3867d933c61805ec2326c6ea557aeeac3825604e/pycryptodome-3.23.0-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/5a/87/b70ad306ebb6f9b585f114d0ac2137d792b48be34d732d60e597c2f8465a/pydantic-2.12.5-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/cf/4e/35a80cae583a37cf15604b44240e45c05e04e86f9cfd766623149297e971/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/c1/60/5d4751ba3f4a40a6891f24eec885f51afd78d208498268c734e256fb13c4/pydantic_settings-2.12.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/7e/32/a7125fb28c4261a627f999d5fb4afff25b523800faed2c30979949d6facd/pydot-4.0.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/c7/21/705964c7812476f378728bdf590ca4b771ec72385c533964653c68e86bdc/pygments-2.19.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/7c/4c/ad33b92b9864cbde84f259d5df035a6447f91891f5be77788e2a3892bce3/pymysql-1.1.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/53/b8/fbab973592e23ae313042d450fc26fa24282ebffba21ba373786e1ce63b4/pyparsing-3.2.4-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/14/1b/a298b06749107c305e1fe0f814c6c74aea7b2f1e10989cb30f544a1b3253/python_dotenv-1.2.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/81/c4/34e93fe5f5429d7570ec1fa436f1986fb1f00c3e0f43a589fe2bbcd22c3f/pytz-2025.2-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/a3/dc/17031897dae0efacfea57dfd3a82fdd2a2aeb58e0ff71b77b87e44edc772/setuptools-80.9.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/f1/7b/ce1eafaf1a76852e2ec9b22edecf1daa58175c090266e9f6c64afcd81d91/stack_data-0.6.3-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/d0/30/dc54f88dd4a2b5dc8a0279bdd7270e735851848b762aeb1c1184ed1f6b14/tqdm-4.67.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/00/c0/8f5d070730d7836adc9c9b6408dec68c6ced86b304a9b26a14df072a6e8c/traitlets-5.14.3-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/5c/23/c7abc0ca0a1526a0774eca151daeb8de62ec457e77262b66b359c3c7679e/tzdata-2025.2-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/a7/c2/fe1e52489ae3122415c51f387e221dd0773709bad6c6cdaa599e8a2c5185/urllib3-2.5.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/fd/84/fd2ba7aafacbad3c4201d395674fc6348826569da3c0937e75505ead3528/wcwidth-0.2.13-py2.py3-none-any.whl + - pypi: ./ + linux-aarch64: + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/_openmp_mutex-4.5-2_gnu.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/noarch/adwaita-icon-theme-49.0-unix_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/at-spi2-atk-2.38.0-h1f2db35_3.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/at-spi2-core-2.40.3-h1f2db35_0.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/atk-1.0-2.38.0-hedc4a1f_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/bzip2-1.0.8-h4777abc_8.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025.10.5-hbd8a1cb_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/cairo-1.18.4-h83712da_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/dbus-1.16.2-heda779d_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/epoxy-1.5.10-he30d5cf_2.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/font-ttf-dejavu-sans-mono-2.37-hab24e00_0.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/noarch/font-ttf-inconsolata-3.000-h77eed37_0.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/noarch/font-ttf-source-code-pro-2.038-h77eed37_0.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/noarch/font-ttf-ubuntu-0.83-h77eed37_3.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/fontconfig-2.15.0-h8dda3cd_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/fonts-conda-ecosystem-1-0.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/noarch/fonts-conda-forge-1-hc364b38_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/freetype-2.14.1-h8af1aa0_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/fribidi-1.0.16-he30d5cf_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/gdk-pixbuf-2.44.4-h90308e0_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/glib-tools-2.86.1-hc87f4d4_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/graphite2-1.3.14-hfae3067_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/graphviz-13.1.2-hdb06ba2_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/gtk3-3.24.43-h4cd1324_6.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/gts-0.7.6-he293c15_4.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/harfbuzz-12.2.0-he4899c9_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/hicolor-icon-theme-0.17-h8af1aa0_2.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/icu-75.1-hf9b3779_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/keyutils-1.6.3-h86ecc28_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/krb5-1.21.3-h50a48e9_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/ld_impl_linux-aarch64-2.44-hd32f0e1_5.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/lerc-4.0.0-hfdc4d58_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libcups-2.3.3-h5cdc715_5.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libdeflate-1.25-h1af38f5_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libdrm-2.4.125-he30d5cf_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libedit-3.1.20250104-pl5321h976ea20_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libegl-1.7.0-hd24410f_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libegl-devel-1.7.0-hd24410f_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libexpat-2.7.1-hfae3067_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libffi-3.5.2-hd65408f_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libfreetype-2.14.1-h8af1aa0_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libfreetype6-2.14.1-hdae7a39_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libgcc-15.2.0-he277a41_7.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libgcc-ng-15.2.0-he9431aa_7.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libgd-2.3.3-hc8d7b1d_11.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libgl-1.7.0-hd24410f_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libgl-devel-1.7.0-hd24410f_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libglib-2.86.1-he84ff74_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libglvnd-1.7.0-hd24410f_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libglx-1.7.0-hd24410f_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libglx-devel-1.7.0-hd24410f_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libgomp-15.2.0-he277a41_7.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libiconv-1.18-h90929bb_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libjpeg-turbo-3.1.2-he30d5cf_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/liblzma-5.8.1-h86ecc28_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libmpdec-4.0.0-h86ecc28_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libpciaccess-0.18-h86ecc28_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libpng-1.6.50-h1abf092_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/librsvg-2.60.0-h8171147_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libsqlite-3.51.0-h022381a_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libstdcxx-15.2.0-h3f4de04_7.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libstdcxx-ng-15.2.0-hf1166c9_7.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libtiff-4.7.1-hdb009f0_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libuuid-2.41.2-h3e4203c_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libwebp-base-1.6.0-ha2e29f5_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libxcb-1.17.0-h262b8f6_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libxkbcommon-1.13.0-h3c6a4c8_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libxml2-16-2.15.1-h8591a01_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libxml2-2.15.1-h788dabe_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libzlib-1.3.1-h86ecc28_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/ncurses-6.5-ha32ae93_3.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/openssl-3.5.4-h8e36d6e_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/pango-1.56.4-he55ef5b_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/pcre2-10.46-h15761aa_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/pixman-0.46.4-h7ac5ae9_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/pthread-stubs-0.4-h86ecc28_1002.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/python-3.13.9-h4c0d347_101_cp313.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python_abi-3.13-8_cp313.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/readline-8.2-h8382b9d_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/tk-8.6.13-noxft_h5688188_102.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2025b-h78e105d_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/wayland-1.24.0-h4f8a99f_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/xkeyboard-config-2.46-he30d5cf_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/xorg-libice-1.1.2-h86ecc28_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/xorg-libsm-1.2.6-h0808dbd_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/xorg-libx11-1.8.12-hca56bd8_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/xorg-libxau-1.0.12-h86ecc28_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/xorg-libxcomposite-0.4.6-h86ecc28_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/xorg-libxcursor-1.2.3-h86ecc28_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/xorg-libxdamage-1.1.6-h86ecc28_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/xorg-libxdmcp-1.1.5-h57736b2_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/xorg-libxext-1.3.6-h57736b2_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/xorg-libxfixes-6.0.2-he30d5cf_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/xorg-libxi-1.8.2-h57736b2_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/xorg-libxinerama-1.1.5-h5ad3122_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/xorg-libxrandr-1.5.4-h86ecc28_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/xorg-libxrender-0.9.12-h86ecc28_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/xorg-libxtst-1.2.5-h57736b2_3.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/xorg-libxxf86vm-1.1.6-h86ecc28_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/xorg-xorgproto-2024.1-h86ecc28_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/zstd-1.5.7-hbcf94c1_2.conda + - pypi: https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/4f/d3/a8b22fa575b297cd6e3e3b0155c7e25db170edf1c74783d6a31a2490b8d9/argon2_cffi-25.1.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/c1/93/44365f3d75053e53893ec6d733e4a5e3147502663554b4d864587c7828a7/argon2_cffi_bindings-25.1.0-cp39-abi3-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl + - pypi: https://files.pythonhosted.org/packages/25/8a/c46dcc25341b5bce5472c718902eb3d38600a903b14fa6aeecef3f21a46f/asttokens-3.0.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e4/37/af0d2ef3967ac0d6113837b44a4f0bfe1328c2b9763bd5b1744520e5cfed/certifi-2025.10.5-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/a9/f5/a2c23eb03b61a0b8747f211eb716446c826ad66818ddc7810cc2cc19b3f2/cffi-2.0.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl + - pypi: https://files.pythonhosted.org/packages/73/23/90e31ceeed1de63058a02cb04b12f2de4b40e3bef5e082a7c18d9c8ae281/contourpy-1.3.3-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl + - pypi: https://files.pythonhosted.org/packages/e7/05/c19819d5e3d95294a6f5947fb9b9629efb316b96de511b418c53d245aae6/cycler-0.12.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/4e/8c/f3147f5c4b73e7550fe5f9352eaa956ae838d5c51eb58e7a25b9f3e2643b/decorator-5.2.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/f7/e6/efe534ef0952b531b630780e19cabd416e2032697019d5295defc6ef9bd9/deepdiff-8.6.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/c1/ea/53f2148663b321f21b5a606bd5f191517cf40b7072c0497d3c92c4a13b1e/executing-2.2.1-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/8e/98/2c050dec90e295a524c9b65c4cb9e7c302386a296b2938710448cbd267d5/faker-37.12.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/04/05/06b1455e4bc653fcb2117ac3ef5fa3a8a14919b93c60742d04440605d058/fonttools-4.60.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl + - pypi: https://files.pythonhosted.org/packages/05/aa/62893d6a591d337aa59dcc4c6f6c842f1fe20cd72c8c5c1f980255243252/ipython-9.7.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/d9/33/1f075bf72b0b747cb3288d011319aaf64083cf2efef8354174e3ed4540e2/ipython_pygments_lexers-1.1.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/c0/5a/9cac0c82afec3d09ccd97c8b6502d48f165f9124db81b4bcb90b4af974ee/jedi-0.19.2-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/d9/28/aac26d4c882f14de59041636292bc838db8961373825df23b8eeb807e198/kiwisolver-1.4.9-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl + - pypi: https://files.pythonhosted.org/packages/d0/7f/ccdca06f4c2e6c7989270ed7829b8679466682f4cfc0f8c9986241c023b6/matplotlib-3.10.7-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl + - pypi: https://files.pythonhosted.org/packages/af/33/ee4519fa02ed11a94aef9559552f3b17bb863f2ecfe1a35dc7f548cde231/matplotlib_inline-0.2.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/7d/ae/f32695da4f93de50dd7075100dab8cf689a9d96270f58ce6f940fd044a3e/minio-7.2.18-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/eb/8d/776adee7bbf76365fdd7f2552710282c79a4ead5d2a46408c9043a2b70ba/networkx-3.5-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/3e/d1/913fe563820f3c6b079f992458f7331278dcd7ba8427e8e745af37ddb44f/numpy-2.3.4-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl + - pypi: https://files.pythonhosted.org/packages/12/27/fb8d7338b4d551900fa3e580acbe7a0cf655d940e164cb5c00ec31961094/orderly_set-5.5.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/20/12/38679034af332785aac8774540895e234f4d07f7545804097de4b666afd8/packaging-25.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/16/87/9472cf4a487d848476865321de18cc8c920b8cab98453ab79dbbc98db63a/pandas-2.3.3-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl + - pypi: https://files.pythonhosted.org/packages/16/32/f8e3c85d1d5250232a5d3477a2a28cc291968ff175caeadaf3cc19ce0e4a/parso-0.8.5-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/9e/c3/059298687310d527a58bb01f3b1965787ee3b40dce76752eda8b44e9a2c5/pexpect-4.9.0-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/20/39/c685d05c06deecfd4e2d1950e9a908aa2ca8bc4e6c3b12d93b9cafbd7837/pillow-12.0.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl + - pypi: https://files.pythonhosted.org/packages/84/03/0d3ce49e2505ae70cf43bc5bb3033955d2fc9f932163e84dc0779cc47f48/prompt_toolkit-3.0.52-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/22/a6/858897256d0deac81a172289110f31629fc4cee19b6f01283303e18c8db3/ptyprocess-0.7.0-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/8e/37/efad0257dc6e593a18957422533ff0f87ede7c9c6ea010a2177d738fb82f/pure_eval-0.2.3-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/a0/e3/59cd50310fc9b59512193629e1984c1f95e5c8ae6e5d8c69532ccc65a7fe/pycparser-2.23-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/50/52/adaf4c8c100a8c49d2bd058e5b551f73dfd8cb89eb4911e25a0c469b6b4e/pycryptodome-3.23.0-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl + - pypi: https://files.pythonhosted.org/packages/5a/87/b70ad306ebb6f9b585f114d0ac2137d792b48be34d732d60e597c2f8465a/pydantic-2.12.5-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/15/df/a4c740c0943e93e6500f9eb23f4ca7ec9bf71b19e608ae5b579678c8d02f/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl + - pypi: https://files.pythonhosted.org/packages/c1/60/5d4751ba3f4a40a6891f24eec885f51afd78d208498268c734e256fb13c4/pydantic_settings-2.12.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/7e/32/a7125fb28c4261a627f999d5fb4afff25b523800faed2c30979949d6facd/pydot-4.0.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/c7/21/705964c7812476f378728bdf590ca4b771ec72385c533964653c68e86bdc/pygments-2.19.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/7c/4c/ad33b92b9864cbde84f259d5df035a6447f91891f5be77788e2a3892bce3/pymysql-1.1.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/10/5e/1aa9a93198c6b64513c9d7752de7422c06402de6600a8767da1524f9570b/pyparsing-3.2.5-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/14/1b/a298b06749107c305e1fe0f814c6c74aea7b2f1e10989cb30f544a1b3253/python_dotenv-1.2.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/81/c4/34e93fe5f5429d7570ec1fa436f1986fb1f00c3e0f43a589fe2bbcd22c3f/pytz-2025.2-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/a3/dc/17031897dae0efacfea57dfd3a82fdd2a2aeb58e0ff71b77b87e44edc772/setuptools-80.9.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/f1/7b/ce1eafaf1a76852e2ec9b22edecf1daa58175c090266e9f6c64afcd81d91/stack_data-0.6.3-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/d0/30/dc54f88dd4a2b5dc8a0279bdd7270e735851848b762aeb1c1184ed1f6b14/tqdm-4.67.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/00/c0/8f5d070730d7836adc9c9b6408dec68c6ced86b304a9b26a14df072a6e8c/traitlets-5.14.3-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/5c/23/c7abc0ca0a1526a0774eca151daeb8de62ec457e77262b66b359c3c7679e/tzdata-2025.2-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/a7/c2/fe1e52489ae3122415c51f387e221dd0773709bad6c6cdaa599e8a2c5185/urllib3-2.5.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/af/b5/123f13c975e9f27ab9c0770f514345bd406d0e8d3b7a0723af9d43f710af/wcwidth-0.2.14-py2.py3-none-any.whl + - pypi: ./ + osx-arm64: + - conda: https://conda.anaconda.org/conda-forge/noarch/adwaita-icon-theme-49.0-unix_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/atk-1.0-2.38.0-hd03087b_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/bzip2-1.0.8-hd037594_8.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025.10.5-hbd8a1cb_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/cairo-1.18.4-h6a3b0d2_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/epoxy-1.5.10-hc919400_2.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/font-ttf-dejavu-sans-mono-2.37-hab24e00_0.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/noarch/font-ttf-inconsolata-3.000-h77eed37_0.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/noarch/font-ttf-source-code-pro-2.038-h77eed37_0.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/noarch/font-ttf-ubuntu-0.83-h77eed37_3.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/fontconfig-2.15.0-h1383a14_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/fonts-conda-ecosystem-1-0.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/noarch/fonts-conda-forge-1-0.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/freetype-2.14.1-hce30654_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/fribidi-1.0.16-hc919400_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/gdk-pixbuf-2.44.4-h7542897_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/glib-tools-2.86.1-hb9d6e3a_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/graphite2-1.3.14-hec049ff_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/graphviz-13.1.2-hcd33d8b_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/gtk3-3.24.43-h5febe37_6.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/gts-0.7.6-he42f4ea_4.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/harfbuzz-12.1.0-haf38c7b_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/hicolor-icon-theme-0.17-hce30654_2.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/icu-75.1-hfee45f7_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/lerc-4.0.0-hd64df32_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libcxx-21.1.4-hf598326_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libdeflate-1.24-h5773f1b_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libexpat-2.7.1-hec049ff_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libffi-3.5.2-he5f378a_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libfreetype-2.14.1-hce30654_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libfreetype6-2.14.1-h6da58f4_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libgd-2.3.3-hb2c3a21_11.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libglib-2.86.1-he69a767_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libiconv-1.18-h23cfdf5_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libintl-0.25.1-h493aca8_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libjpeg-turbo-3.1.0-h5505292_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/liblzma-5.8.1-h39f12f2_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libmpdec-4.0.0-h5505292_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libpng-1.6.50-h280e0eb_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/librsvg-2.60.0-h5c55ec3_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libsqlite-3.50.4-h4237e3c_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libtiff-4.7.1-h7dc4979_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libwebp-base-1.6.0-h07db88b_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libxml2-16-2.15.1-h0ff4647_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libzlib-1.3.1-h8359307_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/ncurses-6.5-h5e97a16_3.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/openssl-3.5.4-h5503f6c_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/pango-1.56.4-h875632e_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/pcre2-10.46-h7125dd6_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/pixman-0.46.4-h81086ad_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/python-3.13.9-hfc2f54d_101_cp313.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python_abi-3.13-8_cp313.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/readline-8.2-h1d1bf99_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/tk-8.6.13-h892fb3f_2.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2025b-h78e105d_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/zstd-1.5.7-h6491c7d_2.conda + - pypi: https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/4f/d3/a8b22fa575b297cd6e3e3b0155c7e25db170edf1c74783d6a31a2490b8d9/argon2_cffi-25.1.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/b6/02/d297943bcacf05e4f2a94ab6f462831dc20158614e5d067c35d4e63b9acb/argon2_cffi_bindings-25.1.0-cp39-abi3-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/25/8a/c46dcc25341b5bce5472c718902eb3d38600a903b14fa6aeecef3f21a46f/asttokens-3.0.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e4/37/af0d2ef3967ac0d6113837b44a4f0bfe1328c2b9763bd5b1744520e5cfed/certifi-2025.10.5-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/4a/d2/a6c0296814556c68ee32009d9c2ad4f85f2707cdecfd7727951ec228005d/cffi-2.0.0-cp313-cp313-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/96/e4/7adcd9c8362745b2210728f209bfbcf7d91ba868a2c5f40d8b58f54c509b/contourpy-1.3.3-cp313-cp313-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/e7/05/c19819d5e3d95294a6f5947fb9b9629efb316b96de511b418c53d245aae6/cycler-0.12.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/4e/8c/f3147f5c4b73e7550fe5f9352eaa956ae838d5c51eb58e7a25b9f3e2643b/decorator-5.2.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/f7/e6/efe534ef0952b531b630780e19cabd416e2032697019d5295defc6ef9bd9/deepdiff-8.6.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/c1/ea/53f2148663b321f21b5a606bd5f191517cf40b7072c0497d3c92c4a13b1e/executing-2.2.1-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/8e/98/2c050dec90e295a524c9b65c4cb9e7c302386a296b2938710448cbd267d5/faker-37.12.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/7c/5b/cdd2c612277b7ac7ec8c0c9bc41812c43dc7b2d5f2b0897e15fdf5a1f915/fonttools-4.60.1-cp313-cp313-macosx_10_13_universal2.whl + - pypi: https://files.pythonhosted.org/packages/48/c5/d5e07995077e48220269c28a221e168c91123ad5ceee44d548f54a057fc0/ipython-9.6.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/d9/33/1f075bf72b0b747cb3288d011319aaf64083cf2efef8354174e3ed4540e2/ipython_pygments_lexers-1.1.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/c0/5a/9cac0c82afec3d09ccd97c8b6502d48f165f9124db81b4bcb90b4af974ee/jedi-0.19.2-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/2d/7a/9d90a151f558e29c3936b8a47ac770235f436f2120aca41a6d5f3d62ae8d/kiwisolver-1.4.9-cp313-cp313-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/bc/d0/b3d3338d467d3fc937f0bb7f256711395cae6f78e22cef0656159950adf0/matplotlib-3.10.7-cp313-cp313-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/af/33/ee4519fa02ed11a94aef9559552f3b17bb863f2ecfe1a35dc7f548cde231/matplotlib_inline-0.2.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/7d/ae/f32695da4f93de50dd7075100dab8cf689a9d96270f58ce6f940fd044a3e/minio-7.2.18-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/eb/8d/776adee7bbf76365fdd7f2552710282c79a4ead5d2a46408c9043a2b70ba/networkx-3.5-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/3e/46/bdd3370dcea2f95ef14af79dbf81e6927102ddf1cc54adc0024d61252fd9/numpy-2.3.4-cp313-cp313-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/12/27/fb8d7338b4d551900fa3e580acbe7a0cf655d940e164cb5c00ec31961094/orderly_set-5.5.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/20/12/38679034af332785aac8774540895e234f4d07f7545804097de4b666afd8/packaging-25.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/31/94/72fac03573102779920099bcac1c3b05975c2cb5f01eac609faf34bed1ca/pandas-2.3.3-cp313-cp313-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/16/32/f8e3c85d1d5250232a5d3477a2a28cc291968ff175caeadaf3cc19ce0e4a/parso-0.8.5-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/9e/c3/059298687310d527a58bb01f3b1965787ee3b40dce76752eda8b44e9a2c5/pexpect-4.9.0-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/83/06/48eab21dd561de2914242711434c0c0eb992ed08ff3f6107a5f44527f5e9/pillow-12.0.0-cp313-cp313-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/84/03/0d3ce49e2505ae70cf43bc5bb3033955d2fc9f932163e84dc0779cc47f48/prompt_toolkit-3.0.52-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/22/a6/858897256d0deac81a172289110f31629fc4cee19b6f01283303e18c8db3/ptyprocess-0.7.0-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/8e/37/efad0257dc6e593a18957422533ff0f87ede7c9c6ea010a2177d738fb82f/pure_eval-0.2.3-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/a0/e3/59cd50310fc9b59512193629e1984c1f95e5c8ae6e5d8c69532ccc65a7fe/pycparser-2.23-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/db/6c/a1f71542c969912bb0e106f64f60a56cc1f0fabecf9396f45accbe63fa68/pycryptodome-3.23.0-cp37-abi3-macosx_10_9_universal2.whl + - pypi: https://files.pythonhosted.org/packages/5a/87/b70ad306ebb6f9b585f114d0ac2137d792b48be34d732d60e597c2f8465a/pydantic-2.12.5-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/94/02/abfa0e0bda67faa65fef1c84971c7e45928e108fe24333c81f3bfe35d5f5/pydantic_core-2.41.5-cp313-cp313-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/c1/60/5d4751ba3f4a40a6891f24eec885f51afd78d208498268c734e256fb13c4/pydantic_settings-2.12.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/7e/32/a7125fb28c4261a627f999d5fb4afff25b523800faed2c30979949d6facd/pydot-4.0.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/c7/21/705964c7812476f378728bdf590ca4b771ec72385c533964653c68e86bdc/pygments-2.19.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/7c/4c/ad33b92b9864cbde84f259d5df035a6447f91891f5be77788e2a3892bce3/pymysql-1.1.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/10/5e/1aa9a93198c6b64513c9d7752de7422c06402de6600a8767da1524f9570b/pyparsing-3.2.5-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/14/1b/a298b06749107c305e1fe0f814c6c74aea7b2f1e10989cb30f544a1b3253/python_dotenv-1.2.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/81/c4/34e93fe5f5429d7570ec1fa436f1986fb1f00c3e0f43a589fe2bbcd22c3f/pytz-2025.2-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/a3/dc/17031897dae0efacfea57dfd3a82fdd2a2aeb58e0ff71b77b87e44edc772/setuptools-80.9.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/f1/7b/ce1eafaf1a76852e2ec9b22edecf1daa58175c090266e9f6c64afcd81d91/stack_data-0.6.3-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/d0/30/dc54f88dd4a2b5dc8a0279bdd7270e735851848b762aeb1c1184ed1f6b14/tqdm-4.67.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/00/c0/8f5d070730d7836adc9c9b6408dec68c6ced86b304a9b26a14df072a6e8c/traitlets-5.14.3-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/5c/23/c7abc0ca0a1526a0774eca151daeb8de62ec457e77262b66b359c3c7679e/tzdata-2025.2-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/a7/c2/fe1e52489ae3122415c51f387e221dd0773709bad6c6cdaa599e8a2c5185/urllib3-2.5.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/af/b5/123f13c975e9f27ab9c0770f514345bd406d0e8d3b7a0723af9d43f710af/wcwidth-0.2.14-py2.py3-none-any.whl + - pypi: ./ + dev: + channels: + - url: https://conda.anaconda.org/conda-forge/ + indexes: + - https://pypi.org/simple + packages: + linux-64: + - conda: https://conda.anaconda.org/conda-forge/linux-64/_libgcc_mutex-0.1-conda_forge.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/linux-64/_openmp_mutex-4.5-2_gnu.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/noarch/adwaita-icon-theme-48.1-unix_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/at-spi2-atk-2.38.0-h0630a04_3.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/linux-64/at-spi2-core-2.40.3-h0630a04_0.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/linux-64/atk-1.0-2.38.0-h04ea711_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/bzip2-1.0.8-hda65f42_8.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025.8.3-hbd8a1cb_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/cairo-1.18.4-h3394656_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/dbus-1.16.2-h3c4dab8_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/epoxy-1.5.10-h166bdaf_1.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/noarch/font-ttf-dejavu-sans-mono-2.37-hab24e00_0.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/noarch/font-ttf-inconsolata-3.000-h77eed37_0.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/noarch/font-ttf-source-code-pro-2.038-h77eed37_0.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/noarch/font-ttf-ubuntu-0.83-h77eed37_3.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/fontconfig-2.15.0-h7e30c49_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/fonts-conda-ecosystem-1-0.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/noarch/fonts-conda-forge-1-0.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/linux-64/freetype-2.14.1-ha770c72_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/fribidi-1.0.16-hb03c661_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/gdk-pixbuf-2.44.1-h2b0a6b4_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/glib-tools-2.86.0-hf516916_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/graphite2-1.3.14-hecca717_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/graphviz-13.1.2-h87b6fe6_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/gtk3-3.24.43-h0c6a113_5.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/gts-0.7.6-h977cf35_4.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/harfbuzz-11.5.0-h15599e2_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/hicolor-icon-theme-0.17-ha770c72_2.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/linux-64/icu-75.1-he02047a_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/keyutils-1.6.3-hb9d3cd8_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/krb5-1.21.3-h659f571_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/ld_impl_linux-64-2.44-h1423503_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/lerc-4.0.0-h0aef613_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libcups-2.3.3-hb8b1518_5.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libdeflate-1.24-h86f0d12_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libedit-3.1.20250104-pl5321h7949ede_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libexpat-2.7.1-hecca717_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libffi-3.4.6-h2dba641_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libfreetype-2.14.1-ha770c72_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libfreetype6-2.14.1-h73754d4_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libgcc-15.1.0-h767d61c_5.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libgcc-ng-15.1.0-h69a702a_5.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libgd-2.3.3-h6f5c62b_11.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libglib-2.86.0-h1fed272_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libgomp-15.1.0-h767d61c_5.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libiconv-1.18-h3b78370_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libjpeg-turbo-3.1.0-hb9d3cd8_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/liblzma-5.8.1-hb9d3cd8_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libmpdec-4.0.0-hb9d3cd8_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libpng-1.6.50-h421ea60_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/librsvg-2.58.4-he92a37e_3.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libsqlite-3.50.4-h0c1763c_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-15.1.0-h8f9b012_5.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-ng-15.1.0-h4852527_5.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libtiff-4.7.0-h8261f1e_6.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libuuid-2.41.1-he9a06e4_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libwebp-base-1.6.0-hd42ef1d_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libxcb-1.17.0-h8a09558_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libxkbcommon-1.11.0-he8b52b9_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libxml2-2.13.8-h04c0eec_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libzlib-1.3.1-hb9d3cd8_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/ncurses-6.5-h2d0b736_3.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/openssl-3.5.2-h26f9b46_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/pango-1.56.4-hadf4263_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/pcre2-10.46-h1321c63_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/pixman-0.46.4-h54a6638_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/pthread-stubs-0.4-hb9d3cd8_1002.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/python-3.13.7-h2b335a9_100_cp313.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python_abi-3.13-8_cp313.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/readline-8.2-h8c095d6_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/tk-8.6.13-noxft_hd72426e_102.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2025b-h78e105d_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/wayland-1.24.0-h3e06ad9_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/xkeyboard-config-2.45-hb9d3cd8_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/xorg-libice-1.1.2-hb9d3cd8_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/xorg-libsm-1.2.6-he73a12e_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/xorg-libx11-1.8.12-h4f16b4b_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/xorg-libxau-1.0.12-hb9d3cd8_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/xorg-libxcomposite-0.4.6-hb9d3cd8_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/xorg-libxcursor-1.2.3-hb9d3cd8_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/xorg-libxdamage-1.1.6-hb9d3cd8_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/xorg-libxdmcp-1.1.5-hb9d3cd8_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/xorg-libxext-1.3.6-hb9d3cd8_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/xorg-libxfixes-6.0.1-hb9d3cd8_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/xorg-libxi-1.8.2-hb9d3cd8_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/xorg-libxinerama-1.1.5-h5888daf_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/xorg-libxrandr-1.5.4-hb9d3cd8_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/xorg-libxrender-0.9.12-hb9d3cd8_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/xorg-libxtst-1.2.5-hb9d3cd8_3.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/zstd-1.5.7-hb8e6e7a_2.conda + - pypi: https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/4f/d3/a8b22fa575b297cd6e3e3b0155c7e25db170edf1c74783d6a31a2490b8d9/argon2_cffi-25.1.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/09/52/94108adfdd6e2ddf58be64f959a0b9c7d4ef2fa71086c38356d22dc501ea/argon2_cffi_bindings-25.1.0-cp39-abi3-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/25/8a/c46dcc25341b5bce5472c718902eb3d38600a903b14fa6aeecef3f21a46f/asttokens-3.0.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e5/48/1549795ba7742c948d2ad169c1c8cdbae65bc450d6cd753d124b17c8cd32/certifi-2025.8.3-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/98/df/0a1755e750013a2081e863e7cd37e0cdd02664372c754e5560099eb7aa44/cffi-2.0.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/c5/55/51844dd50c4fc7a33b653bfaba4c2456f06955289ca770a5dbd5fd267374/cfgv-3.4.0-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/20/01/b394922252051e97aab231d416c86da3d8a6d781eeadcdca1082867de64e/codespell-2.4.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/4b/32/e0f13a1c5b0f8572d0ec6ae2f6c677b7991fafd95da523159c19eff0696a/contourpy-1.3.3-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/73/dd/508420fb47d09d904d962f123221bc249f64b5e56aa93d5f5f7603be475f/coverage-7.10.6-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/e7/05/c19819d5e3d95294a6f5947fb9b9629efb316b96de511b418c53d245aae6/cycler-0.12.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/4e/8c/f3147f5c4b73e7550fe5f9352eaa956ae838d5c51eb58e7a25b9f3e2643b/decorator-5.2.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/f7/e6/efe534ef0952b531b630780e19cabd416e2032697019d5295defc6ef9bd9/deepdiff-8.6.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/33/6b/e0547afaf41bf2c42e52430072fa5658766e3d65bd4b03a563d1b6336f57/distlib-0.4.0-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/c1/ea/53f2148663b321f21b5a606bd5f191517cf40b7072c0497d3c92c4a13b1e/executing-2.2.1-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/f5/11/02ebebb09ff2104b690457cb7bc6ed700c9e0ce88cf581486bb0a5d3c88b/faker-37.8.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/42/14/42b2651a2f46b022ccd948bca9f2d5af0fd8929c4eec235b8d6d844fbe67/filelock-3.19.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/f2/9f/bf231c2a3fac99d1d7f1d89c76594f158693f981a4aa02be406e9f036832/fonttools-4.59.2-cp313-cp313-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/e5/ae/2ad30f4652712c82f1c23423d79136fbce338932ad166d70c1efb86a5998/identify-2.6.14-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/2c/e1/e6716421ea10d38022b952c159d5161ca1193197fb744506875fbb87ea7b/iniconfig-2.1.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/08/2a/5628a99d04acb2d2f2e749cdf4ea571d2575e898df0528a090948018b726/ipython-9.5.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/d9/33/1f075bf72b0b747cb3288d011319aaf64083cf2efef8354174e3ed4540e2/ipython_pygments_lexers-1.1.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/c0/5a/9cac0c82afec3d09ccd97c8b6502d48f165f9124db81b4bcb90b4af974ee/jedi-0.19.2-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e9/e9/f218a2cb3a9ffbe324ca29a9e399fa2d2866d7f348ec3a88df87fc248fc5/kiwisolver-1.4.9-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/e5/b8/9eea6630198cb303d131d95d285a024b3b8645b1763a2916fddb44ca8760/matplotlib-3.10.6-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/8f/8e/9ad090d3553c280a8060fbf6e24dc1c0c29704ee7d1c372f0c174aa59285/matplotlib_inline-0.1.7-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/89/a3/00260f8df72b51afa1f182dd609533c77fa2407918c4c2813d87b4a56725/minio-7.2.16-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/eb/8d/776adee7bbf76365fdd7f2552710282c79a4ead5d2a46408c9043a2b70ba/networkx-3.5-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/d2/1d/1b658dbd2b9fa9c4c9f32accbfc0205d532c8c6194dc0f2a4c0428e7128a/nodeenv-1.9.1-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/9a/a5/bf3db6e66c4b160d6ea10b534c381a1955dfab34cb1017ea93aa33c70ed3/numpy-2.3.3-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/12/27/fb8d7338b4d551900fa3e580acbe7a0cf655d940e164cb5c00ec31961094/orderly_set-5.5.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/20/12/38679034af332785aac8774540895e234f4d07f7545804097de4b666afd8/packaging-25.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/8f/52/0634adaace9be2d8cac9ef78f05c47f3a675882e068438b9d7ec7ef0c13f/pandas-2.3.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/16/32/f8e3c85d1d5250232a5d3477a2a28cc291968ff175caeadaf3cc19ce0e4a/parso-0.8.5-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/9e/c3/059298687310d527a58bb01f3b1965787ee3b40dce76752eda8b44e9a2c5/pexpect-4.9.0-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/d5/1c/a2a29649c0b1983d3ef57ee87a66487fdeb45132df66ab30dd37f7dbe162/pillow-11.3.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/40/4b/2028861e724d3bd36227adfa20d3fd24c3fc6d52032f4a93c133be5d17ce/platformdirs-4.4.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/5b/a5/987a405322d78a73b66e39e4a90e4ef156fd7141bf71df987e50717c321b/pre_commit-4.3.0-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/84/03/0d3ce49e2505ae70cf43bc5bb3033955d2fc9f932163e84dc0779cc47f48/prompt_toolkit-3.0.52-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/22/a6/858897256d0deac81a172289110f31629fc4cee19b6f01283303e18c8db3/ptyprocess-0.7.0-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/8e/37/efad0257dc6e593a18957422533ff0f87ede7c9c6ea010a2177d738fb82f/pure_eval-0.2.3-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/a0/e3/59cd50310fc9b59512193629e1984c1f95e5c8ae6e5d8c69532ccc65a7fe/pycparser-2.23-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/5f/e9/a09476d436d0ff1402ac3867d933c61805ec2326c6ea557aeeac3825604e/pycryptodome-3.23.0-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/5a/87/b70ad306ebb6f9b585f114d0ac2137d792b48be34d732d60e597c2f8465a/pydantic-2.12.5-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/cf/4e/35a80cae583a37cf15604b44240e45c05e04e86f9cfd766623149297e971/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/c1/60/5d4751ba3f4a40a6891f24eec885f51afd78d208498268c734e256fb13c4/pydantic_settings-2.12.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/7e/32/a7125fb28c4261a627f999d5fb4afff25b523800faed2c30979949d6facd/pydot-4.0.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/c7/21/705964c7812476f378728bdf590ca4b771ec72385c533964653c68e86bdc/pygments-2.19.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/7c/4c/ad33b92b9864cbde84f259d5df035a6447f91891f5be77788e2a3892bce3/pymysql-1.1.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/53/b8/fbab973592e23ae313042d450fc26fa24282ebffba21ba373786e1ce63b4/pyparsing-3.2.4-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/a8/a4/20da314d277121d6534b3a980b29035dcd51e6744bd79075a6ce8fa4eb8d/pytest-8.4.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/ee/49/1377b49de7d0c1ce41292161ea0f721913fa8722c19fb9c1e3aa0367eecb/pytest_cov-7.0.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/14/1b/a298b06749107c305e1fe0f814c6c74aea7b2f1e10989cb30f544a1b3253/python_dotenv-1.2.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/81/c4/34e93fe5f5429d7570ec1fa436f1986fb1f00c3e0f43a589fe2bbcd22c3f/pytz-2025.2-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/04/24/b7721e4845c2f162d26f50521b825fb061bc0a5afcf9a386840f23ea19fa/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/f3/51/0489a6a5595b7760b5dbac0dd82852b510326e7d88d51dbffcd2e07e3ff3/ruff-0.14.9-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/a3/dc/17031897dae0efacfea57dfd3a82fdd2a2aeb58e0ff71b77b87e44edc772/setuptools-80.9.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/f1/7b/ce1eafaf1a76852e2ec9b22edecf1daa58175c090266e9f6c64afcd81d91/stack_data-0.6.3-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/d0/30/dc54f88dd4a2b5dc8a0279bdd7270e735851848b762aeb1c1184ed1f6b14/tqdm-4.67.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/00/c0/8f5d070730d7836adc9c9b6408dec68c6ced86b304a9b26a14df072a6e8c/traitlets-5.14.3-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/5c/23/c7abc0ca0a1526a0774eca151daeb8de62ec457e77262b66b359c3c7679e/tzdata-2025.2-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/a7/c2/fe1e52489ae3122415c51f387e221dd0773709bad6c6cdaa599e8a2c5185/urllib3-2.5.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/76/06/04c8e804f813cf972e3262f3f8584c232de64f0cde9f703b46cf53a45090/virtualenv-20.34.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/fd/84/fd2ba7aafacbad3c4201d395674fc6348826569da3c0937e75505ead3528/wcwidth-0.2.13-py2.py3-none-any.whl + - pypi: ./ + linux-aarch64: + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/_openmp_mutex-4.5-2_gnu.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/noarch/adwaita-icon-theme-49.0-unix_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/at-spi2-atk-2.38.0-h1f2db35_3.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/at-spi2-core-2.40.3-h1f2db35_0.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/atk-1.0-2.38.0-hedc4a1f_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/bzip2-1.0.8-h4777abc_8.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025.10.5-hbd8a1cb_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/cairo-1.18.4-h83712da_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/dbus-1.16.2-heda779d_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/epoxy-1.5.10-he30d5cf_2.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/font-ttf-dejavu-sans-mono-2.37-hab24e00_0.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/noarch/font-ttf-inconsolata-3.000-h77eed37_0.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/noarch/font-ttf-source-code-pro-2.038-h77eed37_0.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/noarch/font-ttf-ubuntu-0.83-h77eed37_3.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/fontconfig-2.15.0-h8dda3cd_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/fonts-conda-ecosystem-1-0.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/noarch/fonts-conda-forge-1-hc364b38_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/freetype-2.14.1-h8af1aa0_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/fribidi-1.0.16-he30d5cf_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/gdk-pixbuf-2.44.4-h90308e0_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/glib-tools-2.86.1-hc87f4d4_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/graphite2-1.3.14-hfae3067_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/graphviz-13.1.2-hdb06ba2_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/gtk3-3.24.43-h4cd1324_6.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/gts-0.7.6-he293c15_4.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/harfbuzz-12.2.0-he4899c9_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/hicolor-icon-theme-0.17-h8af1aa0_2.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/icu-75.1-hf9b3779_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/keyutils-1.6.3-h86ecc28_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/krb5-1.21.3-h50a48e9_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/ld_impl_linux-aarch64-2.44-hd32f0e1_5.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/lerc-4.0.0-hfdc4d58_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libcups-2.3.3-h5cdc715_5.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libdeflate-1.25-h1af38f5_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libdrm-2.4.125-he30d5cf_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libedit-3.1.20250104-pl5321h976ea20_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libegl-1.7.0-hd24410f_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libegl-devel-1.7.0-hd24410f_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libexpat-2.7.1-hfae3067_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libffi-3.5.2-hd65408f_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libfreetype-2.14.1-h8af1aa0_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libfreetype6-2.14.1-hdae7a39_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libgcc-15.2.0-he277a41_7.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libgcc-ng-15.2.0-he9431aa_7.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libgd-2.3.3-hc8d7b1d_11.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libgl-1.7.0-hd24410f_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libgl-devel-1.7.0-hd24410f_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libglib-2.86.1-he84ff74_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libglvnd-1.7.0-hd24410f_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libglx-1.7.0-hd24410f_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libglx-devel-1.7.0-hd24410f_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libgomp-15.2.0-he277a41_7.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libiconv-1.18-h90929bb_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libjpeg-turbo-3.1.2-he30d5cf_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/liblzma-5.8.1-h86ecc28_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libmpdec-4.0.0-h86ecc28_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libpciaccess-0.18-h86ecc28_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libpng-1.6.50-h1abf092_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/librsvg-2.60.0-h8171147_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libsqlite-3.51.0-h022381a_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libstdcxx-15.2.0-h3f4de04_7.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libstdcxx-ng-15.2.0-hf1166c9_7.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libtiff-4.7.1-hdb009f0_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libuuid-2.41.2-h3e4203c_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libwebp-base-1.6.0-ha2e29f5_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libxcb-1.17.0-h262b8f6_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libxkbcommon-1.13.0-h3c6a4c8_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libxml2-16-2.15.1-h8591a01_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libxml2-2.15.1-h788dabe_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libzlib-1.3.1-h86ecc28_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/ncurses-6.5-ha32ae93_3.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/openssl-3.5.4-h8e36d6e_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/pango-1.56.4-he55ef5b_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/pcre2-10.46-h15761aa_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/pixman-0.46.4-h7ac5ae9_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/pthread-stubs-0.4-h86ecc28_1002.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/python-3.13.9-h4c0d347_101_cp313.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python_abi-3.13-8_cp313.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/readline-8.2-h8382b9d_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/tk-8.6.13-noxft_h5688188_102.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2025b-h78e105d_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/wayland-1.24.0-h4f8a99f_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/xkeyboard-config-2.46-he30d5cf_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/xorg-libice-1.1.2-h86ecc28_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/xorg-libsm-1.2.6-h0808dbd_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/xorg-libx11-1.8.12-hca56bd8_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/xorg-libxau-1.0.12-h86ecc28_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/xorg-libxcomposite-0.4.6-h86ecc28_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/xorg-libxcursor-1.2.3-h86ecc28_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/xorg-libxdamage-1.1.6-h86ecc28_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/xorg-libxdmcp-1.1.5-h57736b2_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/xorg-libxext-1.3.6-h57736b2_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/xorg-libxfixes-6.0.2-he30d5cf_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/xorg-libxi-1.8.2-h57736b2_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/xorg-libxinerama-1.1.5-h5ad3122_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/xorg-libxrandr-1.5.4-h86ecc28_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/xorg-libxrender-0.9.12-h86ecc28_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/xorg-libxtst-1.2.5-h57736b2_3.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/xorg-libxxf86vm-1.1.6-h86ecc28_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/xorg-xorgproto-2024.1-h86ecc28_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/zstd-1.5.7-hbcf94c1_2.conda + - pypi: https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/4f/d3/a8b22fa575b297cd6e3e3b0155c7e25db170edf1c74783d6a31a2490b8d9/argon2_cffi-25.1.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/c1/93/44365f3d75053e53893ec6d733e4a5e3147502663554b4d864587c7828a7/argon2_cffi_bindings-25.1.0-cp39-abi3-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl + - pypi: https://files.pythonhosted.org/packages/25/8a/c46dcc25341b5bce5472c718902eb3d38600a903b14fa6aeecef3f21a46f/asttokens-3.0.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e4/37/af0d2ef3967ac0d6113837b44a4f0bfe1328c2b9763bd5b1744520e5cfed/certifi-2025.10.5-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/a9/f5/a2c23eb03b61a0b8747f211eb716446c826ad66818ddc7810cc2cc19b3f2/cffi-2.0.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl + - pypi: https://files.pythonhosted.org/packages/c5/55/51844dd50c4fc7a33b653bfaba4c2456f06955289ca770a5dbd5fd267374/cfgv-3.4.0-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/20/01/b394922252051e97aab231d416c86da3d8a6d781eeadcdca1082867de64e/codespell-2.4.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/73/23/90e31ceeed1de63058a02cb04b12f2de4b40e3bef5e082a7c18d9c8ae281/contourpy-1.3.3-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl + - pypi: https://files.pythonhosted.org/packages/2e/7a/34c9402ad12bce609be4be1146a7d22a7fae8e9d752684b6315cce552a65/coverage-7.11.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl + - pypi: https://files.pythonhosted.org/packages/e7/05/c19819d5e3d95294a6f5947fb9b9629efb316b96de511b418c53d245aae6/cycler-0.12.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/4e/8c/f3147f5c4b73e7550fe5f9352eaa956ae838d5c51eb58e7a25b9f3e2643b/decorator-5.2.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/f7/e6/efe534ef0952b531b630780e19cabd416e2032697019d5295defc6ef9bd9/deepdiff-8.6.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/33/6b/e0547afaf41bf2c42e52430072fa5658766e3d65bd4b03a563d1b6336f57/distlib-0.4.0-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/c1/ea/53f2148663b321f21b5a606bd5f191517cf40b7072c0497d3c92c4a13b1e/executing-2.2.1-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/8e/98/2c050dec90e295a524c9b65c4cb9e7c302386a296b2938710448cbd267d5/faker-37.12.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/76/91/7216b27286936c16f5b4d0c530087e4a54eead683e6b0b73dd0c64844af6/filelock-3.20.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/04/05/06b1455e4bc653fcb2117ac3ef5fa3a8a14919b93c60742d04440605d058/fonttools-4.60.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl + - pypi: https://files.pythonhosted.org/packages/0f/1c/e5fd8f973d4f375adb21565739498e2e9a1e54c858a97b9a8ccfdc81da9b/identify-2.6.15-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/05/aa/62893d6a591d337aa59dcc4c6f6c842f1fe20cd72c8c5c1f980255243252/ipython-9.7.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/d9/33/1f075bf72b0b747cb3288d011319aaf64083cf2efef8354174e3ed4540e2/ipython_pygments_lexers-1.1.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/c0/5a/9cac0c82afec3d09ccd97c8b6502d48f165f9124db81b4bcb90b4af974ee/jedi-0.19.2-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/d9/28/aac26d4c882f14de59041636292bc838db8961373825df23b8eeb807e198/kiwisolver-1.4.9-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl + - pypi: https://files.pythonhosted.org/packages/d0/7f/ccdca06f4c2e6c7989270ed7829b8679466682f4cfc0f8c9986241c023b6/matplotlib-3.10.7-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl + - pypi: https://files.pythonhosted.org/packages/af/33/ee4519fa02ed11a94aef9559552f3b17bb863f2ecfe1a35dc7f548cde231/matplotlib_inline-0.2.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/7d/ae/f32695da4f93de50dd7075100dab8cf689a9d96270f58ce6f940fd044a3e/minio-7.2.18-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/eb/8d/776adee7bbf76365fdd7f2552710282c79a4ead5d2a46408c9043a2b70ba/networkx-3.5-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/d2/1d/1b658dbd2b9fa9c4c9f32accbfc0205d532c8c6194dc0f2a4c0428e7128a/nodeenv-1.9.1-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/3e/d1/913fe563820f3c6b079f992458f7331278dcd7ba8427e8e745af37ddb44f/numpy-2.3.4-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl + - pypi: https://files.pythonhosted.org/packages/12/27/fb8d7338b4d551900fa3e580acbe7a0cf655d940e164cb5c00ec31961094/orderly_set-5.5.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/20/12/38679034af332785aac8774540895e234f4d07f7545804097de4b666afd8/packaging-25.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/16/87/9472cf4a487d848476865321de18cc8c920b8cab98453ab79dbbc98db63a/pandas-2.3.3-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl + - pypi: https://files.pythonhosted.org/packages/16/32/f8e3c85d1d5250232a5d3477a2a28cc291968ff175caeadaf3cc19ce0e4a/parso-0.8.5-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/9e/c3/059298687310d527a58bb01f3b1965787ee3b40dce76752eda8b44e9a2c5/pexpect-4.9.0-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/20/39/c685d05c06deecfd4e2d1950e9a908aa2ca8bc4e6c3b12d93b9cafbd7837/pillow-12.0.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl + - pypi: https://files.pythonhosted.org/packages/73/cb/ac7874b3e5d58441674fb70742e6c374b28b0c7cb988d37d991cde47166c/platformdirs-4.5.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/27/11/574fe7d13acf30bfd0a8dd7fa1647040f2b8064f13f43e8c963b1e65093b/pre_commit-4.4.0-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/84/03/0d3ce49e2505ae70cf43bc5bb3033955d2fc9f932163e84dc0779cc47f48/prompt_toolkit-3.0.52-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/22/a6/858897256d0deac81a172289110f31629fc4cee19b6f01283303e18c8db3/ptyprocess-0.7.0-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/8e/37/efad0257dc6e593a18957422533ff0f87ede7c9c6ea010a2177d738fb82f/pure_eval-0.2.3-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/a0/e3/59cd50310fc9b59512193629e1984c1f95e5c8ae6e5d8c69532ccc65a7fe/pycparser-2.23-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/50/52/adaf4c8c100a8c49d2bd058e5b551f73dfd8cb89eb4911e25a0c469b6b4e/pycryptodome-3.23.0-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl + - pypi: https://files.pythonhosted.org/packages/5a/87/b70ad306ebb6f9b585f114d0ac2137d792b48be34d732d60e597c2f8465a/pydantic-2.12.5-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/15/df/a4c740c0943e93e6500f9eb23f4ca7ec9bf71b19e608ae5b579678c8d02f/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl + - pypi: https://files.pythonhosted.org/packages/c1/60/5d4751ba3f4a40a6891f24eec885f51afd78d208498268c734e256fb13c4/pydantic_settings-2.12.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/7e/32/a7125fb28c4261a627f999d5fb4afff25b523800faed2c30979949d6facd/pydot-4.0.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/c7/21/705964c7812476f378728bdf590ca4b771ec72385c533964653c68e86bdc/pygments-2.19.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/7c/4c/ad33b92b9864cbde84f259d5df035a6447f91891f5be77788e2a3892bce3/pymysql-1.1.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/10/5e/1aa9a93198c6b64513c9d7752de7422c06402de6600a8767da1524f9570b/pyparsing-3.2.5-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/72/99/cafef234114a3b6d9f3aaed0723b437c40c57bdb7b3e4c3a575bc4890052/pytest-9.0.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/ee/49/1377b49de7d0c1ce41292161ea0f721913fa8722c19fb9c1e3aa0367eecb/pytest_cov-7.0.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/14/1b/a298b06749107c305e1fe0f814c6c74aea7b2f1e10989cb30f544a1b3253/python_dotenv-1.2.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/81/c4/34e93fe5f5429d7570ec1fa436f1986fb1f00c3e0f43a589fe2bbcd22c3f/pytz-2025.2-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/50/31/b20f376d3f810b9b2371e72ef5adb33879b25edb7a6d072cb7ca0c486398/pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl + - pypi: https://files.pythonhosted.org/packages/9e/e9/08840ff5127916bb989c86f18924fd568938b06f58b60e206176f327c0fe/ruff-0.14.9-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl + - pypi: https://files.pythonhosted.org/packages/a3/dc/17031897dae0efacfea57dfd3a82fdd2a2aeb58e0ff71b77b87e44edc772/setuptools-80.9.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/f1/7b/ce1eafaf1a76852e2ec9b22edecf1daa58175c090266e9f6c64afcd81d91/stack_data-0.6.3-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/d0/30/dc54f88dd4a2b5dc8a0279bdd7270e735851848b762aeb1c1184ed1f6b14/tqdm-4.67.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/00/c0/8f5d070730d7836adc9c9b6408dec68c6ced86b304a9b26a14df072a6e8c/traitlets-5.14.3-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/5c/23/c7abc0ca0a1526a0774eca151daeb8de62ec457e77262b66b359c3c7679e/tzdata-2025.2-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/a7/c2/fe1e52489ae3122415c51f387e221dd0773709bad6c6cdaa599e8a2c5185/urllib3-2.5.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/79/0c/c05523fa3181fdf0c9c52a6ba91a23fbf3246cc095f26f6516f9c60e6771/virtualenv-20.35.4-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/af/b5/123f13c975e9f27ab9c0770f514345bd406d0e8d3b7a0723af9d43f710af/wcwidth-0.2.14-py2.py3-none-any.whl + - pypi: ./ + osx-arm64: + - conda: https://conda.anaconda.org/conda-forge/noarch/adwaita-icon-theme-49.0-unix_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/atk-1.0-2.38.0-hd03087b_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/bzip2-1.0.8-hd037594_8.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025.10.5-hbd8a1cb_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/cairo-1.18.4-h6a3b0d2_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/epoxy-1.5.10-hc919400_2.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/font-ttf-dejavu-sans-mono-2.37-hab24e00_0.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/noarch/font-ttf-inconsolata-3.000-h77eed37_0.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/noarch/font-ttf-source-code-pro-2.038-h77eed37_0.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/noarch/font-ttf-ubuntu-0.83-h77eed37_3.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/fontconfig-2.15.0-h1383a14_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/fonts-conda-ecosystem-1-0.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/noarch/fonts-conda-forge-1-0.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/freetype-2.14.1-hce30654_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/fribidi-1.0.16-hc919400_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/gdk-pixbuf-2.44.4-h7542897_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/glib-tools-2.86.1-hb9d6e3a_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/graphite2-1.3.14-hec049ff_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/graphviz-13.1.2-hcd33d8b_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/gtk3-3.24.43-h5febe37_6.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/gts-0.7.6-he42f4ea_4.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/harfbuzz-12.1.0-haf38c7b_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/hicolor-icon-theme-0.17-hce30654_2.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/icu-75.1-hfee45f7_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/lerc-4.0.0-hd64df32_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libcxx-21.1.4-hf598326_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libdeflate-1.24-h5773f1b_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libexpat-2.7.1-hec049ff_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libffi-3.5.2-he5f378a_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libfreetype-2.14.1-hce30654_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libfreetype6-2.14.1-h6da58f4_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libgd-2.3.3-hb2c3a21_11.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libglib-2.86.1-he69a767_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libiconv-1.18-h23cfdf5_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libintl-0.25.1-h493aca8_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libjpeg-turbo-3.1.0-h5505292_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/liblzma-5.8.1-h39f12f2_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libmpdec-4.0.0-h5505292_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libpng-1.6.50-h280e0eb_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/librsvg-2.60.0-h5c55ec3_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libsqlite-3.50.4-h4237e3c_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libtiff-4.7.1-h7dc4979_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libwebp-base-1.6.0-h07db88b_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libxml2-16-2.15.1-h0ff4647_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libzlib-1.3.1-h8359307_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/ncurses-6.5-h5e97a16_3.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/openssl-3.5.4-h5503f6c_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/pango-1.56.4-h875632e_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/pcre2-10.46-h7125dd6_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/pixman-0.46.4-h81086ad_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/python-3.13.9-hfc2f54d_101_cp313.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python_abi-3.13-8_cp313.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/readline-8.2-h1d1bf99_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/tk-8.6.13-h892fb3f_2.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2025b-h78e105d_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/zstd-1.5.7-h6491c7d_2.conda + - pypi: https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/4f/d3/a8b22fa575b297cd6e3e3b0155c7e25db170edf1c74783d6a31a2490b8d9/argon2_cffi-25.1.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/b6/02/d297943bcacf05e4f2a94ab6f462831dc20158614e5d067c35d4e63b9acb/argon2_cffi_bindings-25.1.0-cp39-abi3-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/25/8a/c46dcc25341b5bce5472c718902eb3d38600a903b14fa6aeecef3f21a46f/asttokens-3.0.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e4/37/af0d2ef3967ac0d6113837b44a4f0bfe1328c2b9763bd5b1744520e5cfed/certifi-2025.10.5-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/4a/d2/a6c0296814556c68ee32009d9c2ad4f85f2707cdecfd7727951ec228005d/cffi-2.0.0-cp313-cp313-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/c5/55/51844dd50c4fc7a33b653bfaba4c2456f06955289ca770a5dbd5fd267374/cfgv-3.4.0-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/20/01/b394922252051e97aab231d416c86da3d8a6d781eeadcdca1082867de64e/codespell-2.4.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/96/e4/7adcd9c8362745b2210728f209bfbcf7d91ba868a2c5f40d8b58f54c509b/contourpy-1.3.3-cp313-cp313-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/96/5d/dc5fa98fea3c175caf9d360649cb1aa3715e391ab00dc78c4c66fabd7356/coverage-7.11.0-cp313-cp313-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/e7/05/c19819d5e3d95294a6f5947fb9b9629efb316b96de511b418c53d245aae6/cycler-0.12.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/4e/8c/f3147f5c4b73e7550fe5f9352eaa956ae838d5c51eb58e7a25b9f3e2643b/decorator-5.2.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/f7/e6/efe534ef0952b531b630780e19cabd416e2032697019d5295defc6ef9bd9/deepdiff-8.6.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/33/6b/e0547afaf41bf2c42e52430072fa5658766e3d65bd4b03a563d1b6336f57/distlib-0.4.0-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/c1/ea/53f2148663b321f21b5a606bd5f191517cf40b7072c0497d3c92c4a13b1e/executing-2.2.1-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/8e/98/2c050dec90e295a524c9b65c4cb9e7c302386a296b2938710448cbd267d5/faker-37.12.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/76/91/7216b27286936c16f5b4d0c530087e4a54eead683e6b0b73dd0c64844af6/filelock-3.20.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/7c/5b/cdd2c612277b7ac7ec8c0c9bc41812c43dc7b2d5f2b0897e15fdf5a1f915/fonttools-4.60.1-cp313-cp313-macosx_10_13_universal2.whl + - pypi: https://files.pythonhosted.org/packages/0f/1c/e5fd8f973d4f375adb21565739498e2e9a1e54c858a97b9a8ccfdc81da9b/identify-2.6.15-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/48/c5/d5e07995077e48220269c28a221e168c91123ad5ceee44d548f54a057fc0/ipython-9.6.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/d9/33/1f075bf72b0b747cb3288d011319aaf64083cf2efef8354174e3ed4540e2/ipython_pygments_lexers-1.1.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/c0/5a/9cac0c82afec3d09ccd97c8b6502d48f165f9124db81b4bcb90b4af974ee/jedi-0.19.2-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/2d/7a/9d90a151f558e29c3936b8a47ac770235f436f2120aca41a6d5f3d62ae8d/kiwisolver-1.4.9-cp313-cp313-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/bc/d0/b3d3338d467d3fc937f0bb7f256711395cae6f78e22cef0656159950adf0/matplotlib-3.10.7-cp313-cp313-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/af/33/ee4519fa02ed11a94aef9559552f3b17bb863f2ecfe1a35dc7f548cde231/matplotlib_inline-0.2.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/7d/ae/f32695da4f93de50dd7075100dab8cf689a9d96270f58ce6f940fd044a3e/minio-7.2.18-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/eb/8d/776adee7bbf76365fdd7f2552710282c79a4ead5d2a46408c9043a2b70ba/networkx-3.5-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/d2/1d/1b658dbd2b9fa9c4c9f32accbfc0205d532c8c6194dc0f2a4c0428e7128a/nodeenv-1.9.1-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/3e/46/bdd3370dcea2f95ef14af79dbf81e6927102ddf1cc54adc0024d61252fd9/numpy-2.3.4-cp313-cp313-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/12/27/fb8d7338b4d551900fa3e580acbe7a0cf655d940e164cb5c00ec31961094/orderly_set-5.5.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/20/12/38679034af332785aac8774540895e234f4d07f7545804097de4b666afd8/packaging-25.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/31/94/72fac03573102779920099bcac1c3b05975c2cb5f01eac609faf34bed1ca/pandas-2.3.3-cp313-cp313-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/16/32/f8e3c85d1d5250232a5d3477a2a28cc291968ff175caeadaf3cc19ce0e4a/parso-0.8.5-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/9e/c3/059298687310d527a58bb01f3b1965787ee3b40dce76752eda8b44e9a2c5/pexpect-4.9.0-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/83/06/48eab21dd561de2914242711434c0c0eb992ed08ff3f6107a5f44527f5e9/pillow-12.0.0-cp313-cp313-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/73/cb/ac7874b3e5d58441674fb70742e6c374b28b0c7cb988d37d991cde47166c/platformdirs-4.5.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/5b/a5/987a405322d78a73b66e39e4a90e4ef156fd7141bf71df987e50717c321b/pre_commit-4.3.0-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/84/03/0d3ce49e2505ae70cf43bc5bb3033955d2fc9f932163e84dc0779cc47f48/prompt_toolkit-3.0.52-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/22/a6/858897256d0deac81a172289110f31629fc4cee19b6f01283303e18c8db3/ptyprocess-0.7.0-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/8e/37/efad0257dc6e593a18957422533ff0f87ede7c9c6ea010a2177d738fb82f/pure_eval-0.2.3-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/a0/e3/59cd50310fc9b59512193629e1984c1f95e5c8ae6e5d8c69532ccc65a7fe/pycparser-2.23-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/db/6c/a1f71542c969912bb0e106f64f60a56cc1f0fabecf9396f45accbe63fa68/pycryptodome-3.23.0-cp37-abi3-macosx_10_9_universal2.whl + - pypi: https://files.pythonhosted.org/packages/5a/87/b70ad306ebb6f9b585f114d0ac2137d792b48be34d732d60e597c2f8465a/pydantic-2.12.5-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/94/02/abfa0e0bda67faa65fef1c84971c7e45928e108fe24333c81f3bfe35d5f5/pydantic_core-2.41.5-cp313-cp313-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/c1/60/5d4751ba3f4a40a6891f24eec885f51afd78d208498268c734e256fb13c4/pydantic_settings-2.12.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/7e/32/a7125fb28c4261a627f999d5fb4afff25b523800faed2c30979949d6facd/pydot-4.0.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/c7/21/705964c7812476f378728bdf590ca4b771ec72385c533964653c68e86bdc/pygments-2.19.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/7c/4c/ad33b92b9864cbde84f259d5df035a6447f91891f5be77788e2a3892bce3/pymysql-1.1.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/10/5e/1aa9a93198c6b64513c9d7752de7422c06402de6600a8767da1524f9570b/pyparsing-3.2.5-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/a8/a4/20da314d277121d6534b3a980b29035dcd51e6744bd79075a6ce8fa4eb8d/pytest-8.4.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/ee/49/1377b49de7d0c1ce41292161ea0f721913fa8722c19fb9c1e3aa0367eecb/pytest_cov-7.0.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/14/1b/a298b06749107c305e1fe0f814c6c74aea7b2f1e10989cb30f544a1b3253/python_dotenv-1.2.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/81/c4/34e93fe5f5429d7570ec1fa436f1986fb1f00c3e0f43a589fe2bbcd22c3f/pytz-2025.2-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/b1/16/95309993f1d3748cd644e02e38b75d50cbc0d9561d21f390a76242ce073f/pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/7d/f8/2be49047f929d6965401855461e697ab185e1a6a683d914c5c19c7962d9e/ruff-0.14.9-py3-none-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/a3/dc/17031897dae0efacfea57dfd3a82fdd2a2aeb58e0ff71b77b87e44edc772/setuptools-80.9.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/f1/7b/ce1eafaf1a76852e2ec9b22edecf1daa58175c090266e9f6c64afcd81d91/stack_data-0.6.3-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/d0/30/dc54f88dd4a2b5dc8a0279bdd7270e735851848b762aeb1c1184ed1f6b14/tqdm-4.67.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/00/c0/8f5d070730d7836adc9c9b6408dec68c6ced86b304a9b26a14df072a6e8c/traitlets-5.14.3-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/5c/23/c7abc0ca0a1526a0774eca151daeb8de62ec457e77262b66b359c3c7679e/tzdata-2025.2-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/a7/c2/fe1e52489ae3122415c51f387e221dd0773709bad6c6cdaa599e8a2c5185/urllib3-2.5.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/79/0c/c05523fa3181fdf0c9c52a6ba91a23fbf3246cc095f26f6516f9c60e6771/virtualenv-20.35.4-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/af/b5/123f13c975e9f27ab9c0770f514345bd406d0e8d3b7a0723af9d43f710af/wcwidth-0.2.14-py2.py3-none-any.whl + - pypi: ./ + test: + channels: + - url: https://conda.anaconda.org/conda-forge/ + indexes: + - https://pypi.org/simple + packages: + linux-64: + - conda: https://conda.anaconda.org/conda-forge/linux-64/_libgcc_mutex-0.1-conda_forge.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/linux-64/_openmp_mutex-4.5-2_gnu.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/noarch/adwaita-icon-theme-48.1-unix_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/at-spi2-atk-2.38.0-h0630a04_3.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/linux-64/at-spi2-core-2.40.3-h0630a04_0.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/linux-64/atk-1.0-2.38.0-h04ea711_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/bzip2-1.0.8-hda65f42_8.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025.8.3-hbd8a1cb_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/cairo-1.18.4-h3394656_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/dbus-1.16.2-h3c4dab8_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/epoxy-1.5.10-h166bdaf_1.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/noarch/font-ttf-dejavu-sans-mono-2.37-hab24e00_0.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/noarch/font-ttf-inconsolata-3.000-h77eed37_0.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/noarch/font-ttf-source-code-pro-2.038-h77eed37_0.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/noarch/font-ttf-ubuntu-0.83-h77eed37_3.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/fontconfig-2.15.0-h7e30c49_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/fonts-conda-ecosystem-1-0.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/noarch/fonts-conda-forge-1-0.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/linux-64/freetype-2.14.1-ha770c72_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/fribidi-1.0.16-hb03c661_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/gdk-pixbuf-2.44.1-h2b0a6b4_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/glib-tools-2.86.0-hf516916_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/graphite2-1.3.14-hecca717_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/graphviz-13.1.2-h87b6fe6_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/gtk3-3.24.43-h0c6a113_5.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/gts-0.7.6-h977cf35_4.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/harfbuzz-11.5.0-h15599e2_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/hicolor-icon-theme-0.17-ha770c72_2.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/linux-64/icu-75.1-he02047a_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/keyutils-1.6.3-hb9d3cd8_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/krb5-1.21.3-h659f571_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/ld_impl_linux-64-2.44-h1423503_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/lerc-4.0.0-h0aef613_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libcups-2.3.3-hb8b1518_5.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libdeflate-1.24-h86f0d12_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libedit-3.1.20250104-pl5321h7949ede_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libexpat-2.7.1-hecca717_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libffi-3.4.6-h2dba641_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libfreetype-2.14.1-ha770c72_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libfreetype6-2.14.1-h73754d4_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libgcc-15.1.0-h767d61c_5.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libgcc-ng-15.1.0-h69a702a_5.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libgd-2.3.3-h6f5c62b_11.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libglib-2.86.0-h1fed272_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libgomp-15.1.0-h767d61c_5.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libiconv-1.18-h3b78370_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libjpeg-turbo-3.1.0-hb9d3cd8_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/liblzma-5.8.1-hb9d3cd8_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libmpdec-4.0.0-hb9d3cd8_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libpng-1.6.50-h421ea60_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/librsvg-2.58.4-he92a37e_3.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libsqlite-3.50.4-h0c1763c_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-15.1.0-h8f9b012_5.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-ng-15.1.0-h4852527_5.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libtiff-4.7.0-h8261f1e_6.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libuuid-2.41.1-he9a06e4_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libwebp-base-1.6.0-hd42ef1d_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libxcb-1.17.0-h8a09558_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libxkbcommon-1.11.0-he8b52b9_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libxml2-2.13.8-h04c0eec_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libzlib-1.3.1-hb9d3cd8_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/ncurses-6.5-h2d0b736_3.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/openssl-3.5.2-h26f9b46_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/pango-1.56.4-hadf4263_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/pcre2-10.46-h1321c63_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/pixman-0.46.4-h54a6638_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/pthread-stubs-0.4-hb9d3cd8_1002.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/python-3.13.7-h2b335a9_100_cp313.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python_abi-3.13-8_cp313.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/readline-8.2-h8c095d6_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/tk-8.6.13-noxft_hd72426e_102.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2025b-h78e105d_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/wayland-1.24.0-h3e06ad9_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/xkeyboard-config-2.45-hb9d3cd8_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/xorg-libice-1.1.2-hb9d3cd8_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/xorg-libsm-1.2.6-he73a12e_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/xorg-libx11-1.8.12-h4f16b4b_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/xorg-libxau-1.0.12-hb9d3cd8_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/xorg-libxcomposite-0.4.6-hb9d3cd8_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/xorg-libxcursor-1.2.3-hb9d3cd8_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/xorg-libxdamage-1.1.6-hb9d3cd8_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/xorg-libxdmcp-1.1.5-hb9d3cd8_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/xorg-libxext-1.3.6-hb9d3cd8_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/xorg-libxfixes-6.0.1-hb9d3cd8_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/xorg-libxi-1.8.2-hb9d3cd8_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/xorg-libxinerama-1.1.5-h5888daf_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/xorg-libxrandr-1.5.4-hb9d3cd8_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/xorg-libxrender-0.9.12-hb9d3cd8_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/xorg-libxtst-1.2.5-hb9d3cd8_3.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/zstd-1.5.7-hb8e6e7a_2.conda + - pypi: https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/4f/d3/a8b22fa575b297cd6e3e3b0155c7e25db170edf1c74783d6a31a2490b8d9/argon2_cffi-25.1.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/09/52/94108adfdd6e2ddf58be64f959a0b9c7d4ef2fa71086c38356d22dc501ea/argon2_cffi_bindings-25.1.0-cp39-abi3-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/25/8a/c46dcc25341b5bce5472c718902eb3d38600a903b14fa6aeecef3f21a46f/asttokens-3.0.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e5/48/1549795ba7742c948d2ad169c1c8cdbae65bc450d6cd753d124b17c8cd32/certifi-2025.8.3-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/98/df/0a1755e750013a2081e863e7cd37e0cdd02664372c754e5560099eb7aa44/cffi-2.0.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/7e/95/42aa2156235cbc8fa61208aded06ef46111c4d3f0de233107b3f38631803/charset_normalizer-3.4.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/4b/32/e0f13a1c5b0f8572d0ec6ae2f6c677b7991fafd95da523159c19eff0696a/contourpy-1.3.3-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/73/dd/508420fb47d09d904d962f123221bc249f64b5e56aa93d5f5f7603be475f/coverage-7.10.6-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/e7/05/c19819d5e3d95294a6f5947fb9b9629efb316b96de511b418c53d245aae6/cycler-0.12.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/4e/8c/f3147f5c4b73e7550fe5f9352eaa956ae838d5c51eb58e7a25b9f3e2643b/decorator-5.2.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/f7/e6/efe534ef0952b531b630780e19cabd416e2032697019d5295defc6ef9bd9/deepdiff-8.6.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e3/26/57c6fb270950d476074c087527a558ccb6f4436657314bfb6cdf484114c4/docker-7.1.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/c1/ea/53f2148663b321f21b5a606bd5f191517cf40b7072c0497d3c92c4a13b1e/executing-2.2.1-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/f5/11/02ebebb09ff2104b690457cb7bc6ed700c9e0ce88cf581486bb0a5d3c88b/faker-37.8.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/f2/9f/bf231c2a3fac99d1d7f1d89c76594f158693f981a4aa02be406e9f036832/fonttools-4.59.2-cp313-cp313-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/91/4c/e0ce1ef95d4000ebc1c11801f9b944fa5910ecc15b5e351865763d8657f8/graphviz-0.21-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/76/c6/c88e154df9c4e1a2a66ccf0005a88dfb2650c1dffb6f5ce603dfbd452ce3/idna-3.10-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/2c/e1/e6716421ea10d38022b952c159d5161ca1193197fb744506875fbb87ea7b/iniconfig-2.1.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/08/2a/5628a99d04acb2d2f2e749cdf4ea571d2575e898df0528a090948018b726/ipython-9.5.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/d9/33/1f075bf72b0b747cb3288d011319aaf64083cf2efef8354174e3ed4540e2/ipython_pygments_lexers-1.1.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/c0/5a/9cac0c82afec3d09ccd97c8b6502d48f165f9124db81b4bcb90b4af974ee/jedi-0.19.2-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e9/e9/f218a2cb3a9ffbe324ca29a9e399fa2d2866d7f348ec3a88df87fc248fc5/kiwisolver-1.4.9-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/e5/b8/9eea6630198cb303d131d95d285a024b3b8645b1763a2916fddb44ca8760/matplotlib-3.10.6-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/8f/8e/9ad090d3553c280a8060fbf6e24dc1c0c29704ee7d1c372f0c174aa59285/matplotlib_inline-0.1.7-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/89/a3/00260f8df72b51afa1f182dd609533c77fa2407918c4c2813d87b4a56725/minio-7.2.16-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/eb/8d/776adee7bbf76365fdd7f2552710282c79a4ead5d2a46408c9043a2b70ba/networkx-3.5-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/9a/a5/bf3db6e66c4b160d6ea10b534c381a1955dfab34cb1017ea93aa33c70ed3/numpy-2.3.3-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/12/27/fb8d7338b4d551900fa3e580acbe7a0cf655d940e164cb5c00ec31961094/orderly_set-5.5.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/20/12/38679034af332785aac8774540895e234f4d07f7545804097de4b666afd8/packaging-25.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/8f/52/0634adaace9be2d8cac9ef78f05c47f3a675882e068438b9d7ec7ef0c13f/pandas-2.3.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/16/32/f8e3c85d1d5250232a5d3477a2a28cc291968ff175caeadaf3cc19ce0e4a/parso-0.8.5-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/9e/c3/059298687310d527a58bb01f3b1965787ee3b40dce76752eda8b44e9a2c5/pexpect-4.9.0-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/d5/1c/a2a29649c0b1983d3ef57ee87a66487fdeb45132df66ab30dd37f7dbe162/pillow-11.3.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/84/03/0d3ce49e2505ae70cf43bc5bb3033955d2fc9f932163e84dc0779cc47f48/prompt_toolkit-3.0.52-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/22/a6/858897256d0deac81a172289110f31629fc4cee19b6f01283303e18c8db3/ptyprocess-0.7.0-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/8e/37/efad0257dc6e593a18957422533ff0f87ede7c9c6ea010a2177d738fb82f/pure_eval-0.2.3-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/a0/e3/59cd50310fc9b59512193629e1984c1f95e5c8ae6e5d8c69532ccc65a7fe/pycparser-2.23-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/5f/e9/a09476d436d0ff1402ac3867d933c61805ec2326c6ea557aeeac3825604e/pycryptodome-3.23.0-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/5a/87/b70ad306ebb6f9b585f114d0ac2137d792b48be34d732d60e597c2f8465a/pydantic-2.12.5-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/cf/4e/35a80cae583a37cf15604b44240e45c05e04e86f9cfd766623149297e971/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/c1/60/5d4751ba3f4a40a6891f24eec885f51afd78d208498268c734e256fb13c4/pydantic_settings-2.12.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/7e/32/a7125fb28c4261a627f999d5fb4afff25b523800faed2c30979949d6facd/pydot-4.0.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/c7/21/705964c7812476f378728bdf590ca4b771ec72385c533964653c68e86bdc/pygments-2.19.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/7c/4c/ad33b92b9864cbde84f259d5df035a6447f91891f5be77788e2a3892bce3/pymysql-1.1.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/53/b8/fbab973592e23ae313042d450fc26fa24282ebffba21ba373786e1ce63b4/pyparsing-3.2.4-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/a8/a4/20da314d277121d6534b3a980b29035dcd51e6744bd79075a6ce8fa4eb8d/pytest-8.4.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/ee/49/1377b49de7d0c1ce41292161ea0f721913fa8722c19fb9c1e3aa0367eecb/pytest_cov-7.0.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/de/b8/87cfb16045c9d4092cfcf526135d73b88101aac83bc1adcf82dfb5fd3833/pytest_env-1.1.5-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/14/1b/a298b06749107c305e1fe0f814c6c74aea7b2f1e10989cb30f544a1b3253/python_dotenv-1.2.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/81/c4/34e93fe5f5429d7570ec1fa436f1986fb1f00c3e0f43a589fe2bbcd22c3f/pytz-2025.2-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/1e/db/4254e3eabe8020b458f1a747140d32277ec7a271daf1d235b70dc0b4e6e3/requests-2.32.5-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/a3/dc/17031897dae0efacfea57dfd3a82fdd2a2aeb58e0ff71b77b87e44edc772/setuptools-80.9.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/f1/7b/ce1eafaf1a76852e2ec9b22edecf1daa58175c090266e9f6c64afcd81d91/stack_data-0.6.3-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/d0/30/dc54f88dd4a2b5dc8a0279bdd7270e735851848b762aeb1c1184ed1f6b14/tqdm-4.67.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/00/c0/8f5d070730d7836adc9c9b6408dec68c6ced86b304a9b26a14df072a6e8c/traitlets-5.14.3-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/5c/23/c7abc0ca0a1526a0774eca151daeb8de62ec457e77262b66b359c3c7679e/tzdata-2025.2-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/a7/c2/fe1e52489ae3122415c51f387e221dd0773709bad6c6cdaa599e8a2c5185/urllib3-2.5.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/fd/84/fd2ba7aafacbad3c4201d395674fc6348826569da3c0937e75505ead3528/wcwidth-0.2.13-py2.py3-none-any.whl + - pypi: ./ + linux-aarch64: + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/_openmp_mutex-4.5-2_gnu.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/noarch/adwaita-icon-theme-49.0-unix_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/at-spi2-atk-2.38.0-h1f2db35_3.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/at-spi2-core-2.40.3-h1f2db35_0.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/atk-1.0-2.38.0-hedc4a1f_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/bzip2-1.0.8-h4777abc_8.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025.10.5-hbd8a1cb_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/cairo-1.18.4-h83712da_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/dbus-1.16.2-heda779d_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/epoxy-1.5.10-he30d5cf_2.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/font-ttf-dejavu-sans-mono-2.37-hab24e00_0.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/noarch/font-ttf-inconsolata-3.000-h77eed37_0.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/noarch/font-ttf-source-code-pro-2.038-h77eed37_0.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/noarch/font-ttf-ubuntu-0.83-h77eed37_3.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/fontconfig-2.15.0-h8dda3cd_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/fonts-conda-ecosystem-1-0.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/noarch/fonts-conda-forge-1-hc364b38_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/freetype-2.14.1-h8af1aa0_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/fribidi-1.0.16-he30d5cf_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/gdk-pixbuf-2.44.4-h90308e0_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/glib-tools-2.86.1-hc87f4d4_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/graphite2-1.3.14-hfae3067_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/graphviz-13.1.2-hdb06ba2_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/gtk3-3.24.43-h4cd1324_6.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/gts-0.7.6-he293c15_4.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/harfbuzz-12.2.0-he4899c9_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/hicolor-icon-theme-0.17-h8af1aa0_2.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/icu-75.1-hf9b3779_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/keyutils-1.6.3-h86ecc28_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/krb5-1.21.3-h50a48e9_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/ld_impl_linux-aarch64-2.44-hd32f0e1_5.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/lerc-4.0.0-hfdc4d58_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libcups-2.3.3-h5cdc715_5.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libdeflate-1.25-h1af38f5_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libdrm-2.4.125-he30d5cf_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libedit-3.1.20250104-pl5321h976ea20_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libegl-1.7.0-hd24410f_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libegl-devel-1.7.0-hd24410f_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libexpat-2.7.1-hfae3067_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libffi-3.5.2-hd65408f_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libfreetype-2.14.1-h8af1aa0_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libfreetype6-2.14.1-hdae7a39_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libgcc-15.2.0-he277a41_7.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libgcc-ng-15.2.0-he9431aa_7.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libgd-2.3.3-hc8d7b1d_11.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libgl-1.7.0-hd24410f_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libgl-devel-1.7.0-hd24410f_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libglib-2.86.1-he84ff74_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libglvnd-1.7.0-hd24410f_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libglx-1.7.0-hd24410f_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libglx-devel-1.7.0-hd24410f_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libgomp-15.2.0-he277a41_7.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libiconv-1.18-h90929bb_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libjpeg-turbo-3.1.2-he30d5cf_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/liblzma-5.8.1-h86ecc28_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libmpdec-4.0.0-h86ecc28_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libpciaccess-0.18-h86ecc28_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libpng-1.6.50-h1abf092_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/librsvg-2.60.0-h8171147_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libsqlite-3.51.0-h022381a_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libstdcxx-15.2.0-h3f4de04_7.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libstdcxx-ng-15.2.0-hf1166c9_7.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libtiff-4.7.1-hdb009f0_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libuuid-2.41.2-h3e4203c_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libwebp-base-1.6.0-ha2e29f5_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libxcb-1.17.0-h262b8f6_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libxkbcommon-1.13.0-h3c6a4c8_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libxml2-16-2.15.1-h8591a01_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libxml2-2.15.1-h788dabe_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libzlib-1.3.1-h86ecc28_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/ncurses-6.5-ha32ae93_3.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/openssl-3.5.4-h8e36d6e_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/pango-1.56.4-he55ef5b_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/pcre2-10.46-h15761aa_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/pixman-0.46.4-h7ac5ae9_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/pthread-stubs-0.4-h86ecc28_1002.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/python-3.13.9-h4c0d347_101_cp313.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python_abi-3.13-8_cp313.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/readline-8.2-h8382b9d_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/tk-8.6.13-noxft_h5688188_102.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2025b-h78e105d_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/wayland-1.24.0-h4f8a99f_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/xkeyboard-config-2.46-he30d5cf_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/xorg-libice-1.1.2-h86ecc28_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/xorg-libsm-1.2.6-h0808dbd_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/xorg-libx11-1.8.12-hca56bd8_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/xorg-libxau-1.0.12-h86ecc28_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/xorg-libxcomposite-0.4.6-h86ecc28_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/xorg-libxcursor-1.2.3-h86ecc28_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/xorg-libxdamage-1.1.6-h86ecc28_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/xorg-libxdmcp-1.1.5-h57736b2_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/xorg-libxext-1.3.6-h57736b2_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/xorg-libxfixes-6.0.2-he30d5cf_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/xorg-libxi-1.8.2-h57736b2_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/xorg-libxinerama-1.1.5-h5ad3122_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/xorg-libxrandr-1.5.4-h86ecc28_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/xorg-libxrender-0.9.12-h86ecc28_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/xorg-libxtst-1.2.5-h57736b2_3.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/xorg-libxxf86vm-1.1.6-h86ecc28_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/xorg-xorgproto-2024.1-h86ecc28_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/zstd-1.5.7-hbcf94c1_2.conda + - pypi: https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/4f/d3/a8b22fa575b297cd6e3e3b0155c7e25db170edf1c74783d6a31a2490b8d9/argon2_cffi-25.1.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/c1/93/44365f3d75053e53893ec6d733e4a5e3147502663554b4d864587c7828a7/argon2_cffi_bindings-25.1.0-cp39-abi3-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl + - pypi: https://files.pythonhosted.org/packages/25/8a/c46dcc25341b5bce5472c718902eb3d38600a903b14fa6aeecef3f21a46f/asttokens-3.0.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e4/37/af0d2ef3967ac0d6113837b44a4f0bfe1328c2b9763bd5b1744520e5cfed/certifi-2025.10.5-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/a9/f5/a2c23eb03b61a0b8747f211eb716446c826ad66818ddc7810cc2cc19b3f2/cffi-2.0.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl + - pypi: https://files.pythonhosted.org/packages/7d/62/73a6d7450829655a35bb88a88fca7d736f9882a27eacdca2c6d505b57e2e/charset_normalizer-3.4.4-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl + - pypi: https://files.pythonhosted.org/packages/73/23/90e31ceeed1de63058a02cb04b12f2de4b40e3bef5e082a7c18d9c8ae281/contourpy-1.3.3-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl + - pypi: https://files.pythonhosted.org/packages/2e/7a/34c9402ad12bce609be4be1146a7d22a7fae8e9d752684b6315cce552a65/coverage-7.11.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl + - pypi: https://files.pythonhosted.org/packages/e7/05/c19819d5e3d95294a6f5947fb9b9629efb316b96de511b418c53d245aae6/cycler-0.12.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/4e/8c/f3147f5c4b73e7550fe5f9352eaa956ae838d5c51eb58e7a25b9f3e2643b/decorator-5.2.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/f7/e6/efe534ef0952b531b630780e19cabd416e2032697019d5295defc6ef9bd9/deepdiff-8.6.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e3/26/57c6fb270950d476074c087527a558ccb6f4436657314bfb6cdf484114c4/docker-7.1.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/c1/ea/53f2148663b321f21b5a606bd5f191517cf40b7072c0497d3c92c4a13b1e/executing-2.2.1-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/8e/98/2c050dec90e295a524c9b65c4cb9e7c302386a296b2938710448cbd267d5/faker-37.12.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/04/05/06b1455e4bc653fcb2117ac3ef5fa3a8a14919b93c60742d04440605d058/fonttools-4.60.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl + - pypi: https://files.pythonhosted.org/packages/91/4c/e0ce1ef95d4000ebc1c11801f9b944fa5910ecc15b5e351865763d8657f8/graphviz-0.21-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/0e/61/66938bbb5fc52dbdf84594873d5b51fb1f7c7794e9c0f5bd885f30bc507b/idna-3.11-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/05/aa/62893d6a591d337aa59dcc4c6f6c842f1fe20cd72c8c5c1f980255243252/ipython-9.7.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/d9/33/1f075bf72b0b747cb3288d011319aaf64083cf2efef8354174e3ed4540e2/ipython_pygments_lexers-1.1.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/c0/5a/9cac0c82afec3d09ccd97c8b6502d48f165f9124db81b4bcb90b4af974ee/jedi-0.19.2-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/d9/28/aac26d4c882f14de59041636292bc838db8961373825df23b8eeb807e198/kiwisolver-1.4.9-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl + - pypi: https://files.pythonhosted.org/packages/d0/7f/ccdca06f4c2e6c7989270ed7829b8679466682f4cfc0f8c9986241c023b6/matplotlib-3.10.7-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl + - pypi: https://files.pythonhosted.org/packages/af/33/ee4519fa02ed11a94aef9559552f3b17bb863f2ecfe1a35dc7f548cde231/matplotlib_inline-0.2.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/7d/ae/f32695da4f93de50dd7075100dab8cf689a9d96270f58ce6f940fd044a3e/minio-7.2.18-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/eb/8d/776adee7bbf76365fdd7f2552710282c79a4ead5d2a46408c9043a2b70ba/networkx-3.5-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/3e/d1/913fe563820f3c6b079f992458f7331278dcd7ba8427e8e745af37ddb44f/numpy-2.3.4-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl + - pypi: https://files.pythonhosted.org/packages/12/27/fb8d7338b4d551900fa3e580acbe7a0cf655d940e164cb5c00ec31961094/orderly_set-5.5.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/20/12/38679034af332785aac8774540895e234f4d07f7545804097de4b666afd8/packaging-25.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/16/87/9472cf4a487d848476865321de18cc8c920b8cab98453ab79dbbc98db63a/pandas-2.3.3-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl + - pypi: https://files.pythonhosted.org/packages/16/32/f8e3c85d1d5250232a5d3477a2a28cc291968ff175caeadaf3cc19ce0e4a/parso-0.8.5-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/9e/c3/059298687310d527a58bb01f3b1965787ee3b40dce76752eda8b44e9a2c5/pexpect-4.9.0-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/20/39/c685d05c06deecfd4e2d1950e9a908aa2ca8bc4e6c3b12d93b9cafbd7837/pillow-12.0.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl + - pypi: https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/84/03/0d3ce49e2505ae70cf43bc5bb3033955d2fc9f932163e84dc0779cc47f48/prompt_toolkit-3.0.52-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/22/a6/858897256d0deac81a172289110f31629fc4cee19b6f01283303e18c8db3/ptyprocess-0.7.0-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/8e/37/efad0257dc6e593a18957422533ff0f87ede7c9c6ea010a2177d738fb82f/pure_eval-0.2.3-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/a0/e3/59cd50310fc9b59512193629e1984c1f95e5c8ae6e5d8c69532ccc65a7fe/pycparser-2.23-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/50/52/adaf4c8c100a8c49d2bd058e5b551f73dfd8cb89eb4911e25a0c469b6b4e/pycryptodome-3.23.0-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl + - pypi: https://files.pythonhosted.org/packages/5a/87/b70ad306ebb6f9b585f114d0ac2137d792b48be34d732d60e597c2f8465a/pydantic-2.12.5-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/15/df/a4c740c0943e93e6500f9eb23f4ca7ec9bf71b19e608ae5b579678c8d02f/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl + - pypi: https://files.pythonhosted.org/packages/c1/60/5d4751ba3f4a40a6891f24eec885f51afd78d208498268c734e256fb13c4/pydantic_settings-2.12.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/7e/32/a7125fb28c4261a627f999d5fb4afff25b523800faed2c30979949d6facd/pydot-4.0.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/c7/21/705964c7812476f378728bdf590ca4b771ec72385c533964653c68e86bdc/pygments-2.19.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/7c/4c/ad33b92b9864cbde84f259d5df035a6447f91891f5be77788e2a3892bce3/pymysql-1.1.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/10/5e/1aa9a93198c6b64513c9d7752de7422c06402de6600a8767da1524f9570b/pyparsing-3.2.5-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/72/99/cafef234114a3b6d9f3aaed0723b437c40c57bdb7b3e4c3a575bc4890052/pytest-9.0.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/ee/49/1377b49de7d0c1ce41292161ea0f721913fa8722c19fb9c1e3aa0367eecb/pytest_cov-7.0.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/27/98/822b924a4a3eb58aacba84444c7439fce32680592f394de26af9c76e2569/pytest_env-1.2.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/14/1b/a298b06749107c305e1fe0f814c6c74aea7b2f1e10989cb30f544a1b3253/python_dotenv-1.2.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/81/c4/34e93fe5f5429d7570ec1fa436f1986fb1f00c3e0f43a589fe2bbcd22c3f/pytz-2025.2-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/1e/db/4254e3eabe8020b458f1a747140d32277ec7a271daf1d235b70dc0b4e6e3/requests-2.32.5-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/a3/dc/17031897dae0efacfea57dfd3a82fdd2a2aeb58e0ff71b77b87e44edc772/setuptools-80.9.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/f1/7b/ce1eafaf1a76852e2ec9b22edecf1daa58175c090266e9f6c64afcd81d91/stack_data-0.6.3-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/d0/30/dc54f88dd4a2b5dc8a0279bdd7270e735851848b762aeb1c1184ed1f6b14/tqdm-4.67.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/00/c0/8f5d070730d7836adc9c9b6408dec68c6ced86b304a9b26a14df072a6e8c/traitlets-5.14.3-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/5c/23/c7abc0ca0a1526a0774eca151daeb8de62ec457e77262b66b359c3c7679e/tzdata-2025.2-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/a7/c2/fe1e52489ae3122415c51f387e221dd0773709bad6c6cdaa599e8a2c5185/urllib3-2.5.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/af/b5/123f13c975e9f27ab9c0770f514345bd406d0e8d3b7a0723af9d43f710af/wcwidth-0.2.14-py2.py3-none-any.whl + - pypi: ./ + osx-arm64: + - conda: https://conda.anaconda.org/conda-forge/noarch/adwaita-icon-theme-49.0-unix_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/atk-1.0-2.38.0-hd03087b_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/bzip2-1.0.8-hd037594_8.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025.10.5-hbd8a1cb_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/cairo-1.18.4-h6a3b0d2_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/epoxy-1.5.10-hc919400_2.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/font-ttf-dejavu-sans-mono-2.37-hab24e00_0.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/noarch/font-ttf-inconsolata-3.000-h77eed37_0.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/noarch/font-ttf-source-code-pro-2.038-h77eed37_0.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/noarch/font-ttf-ubuntu-0.83-h77eed37_3.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/fontconfig-2.15.0-h1383a14_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/fonts-conda-ecosystem-1-0.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/noarch/fonts-conda-forge-1-0.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/freetype-2.14.1-hce30654_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/fribidi-1.0.16-hc919400_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/gdk-pixbuf-2.44.4-h7542897_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/glib-tools-2.86.1-hb9d6e3a_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/graphite2-1.3.14-hec049ff_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/graphviz-13.1.2-hcd33d8b_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/gtk3-3.24.43-h5febe37_6.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/gts-0.7.6-he42f4ea_4.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/harfbuzz-12.1.0-haf38c7b_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/hicolor-icon-theme-0.17-hce30654_2.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/icu-75.1-hfee45f7_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/lerc-4.0.0-hd64df32_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libcxx-21.1.4-hf598326_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libdeflate-1.24-h5773f1b_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libexpat-2.7.1-hec049ff_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libffi-3.5.2-he5f378a_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libfreetype-2.14.1-hce30654_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libfreetype6-2.14.1-h6da58f4_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libgd-2.3.3-hb2c3a21_11.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libglib-2.86.1-he69a767_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libiconv-1.18-h23cfdf5_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libintl-0.25.1-h493aca8_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libjpeg-turbo-3.1.0-h5505292_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/liblzma-5.8.1-h39f12f2_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libmpdec-4.0.0-h5505292_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libpng-1.6.50-h280e0eb_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/librsvg-2.60.0-h5c55ec3_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libsqlite-3.50.4-h4237e3c_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libtiff-4.7.1-h7dc4979_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libwebp-base-1.6.0-h07db88b_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libxml2-16-2.15.1-h0ff4647_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libzlib-1.3.1-h8359307_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/ncurses-6.5-h5e97a16_3.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/openssl-3.5.4-h5503f6c_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/pango-1.56.4-h875632e_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/pcre2-10.46-h7125dd6_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/pixman-0.46.4-h81086ad_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/python-3.13.9-hfc2f54d_101_cp313.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python_abi-3.13-8_cp313.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/readline-8.2-h1d1bf99_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/tk-8.6.13-h892fb3f_2.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2025b-h78e105d_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/zstd-1.5.7-h6491c7d_2.conda + - pypi: https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/4f/d3/a8b22fa575b297cd6e3e3b0155c7e25db170edf1c74783d6a31a2490b8d9/argon2_cffi-25.1.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/b6/02/d297943bcacf05e4f2a94ab6f462831dc20158614e5d067c35d4e63b9acb/argon2_cffi_bindings-25.1.0-cp39-abi3-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/25/8a/c46dcc25341b5bce5472c718902eb3d38600a903b14fa6aeecef3f21a46f/asttokens-3.0.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e4/37/af0d2ef3967ac0d6113837b44a4f0bfe1328c2b9763bd5b1744520e5cfed/certifi-2025.10.5-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/4a/d2/a6c0296814556c68ee32009d9c2ad4f85f2707cdecfd7727951ec228005d/cffi-2.0.0-cp313-cp313-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/97/45/4b3a1239bbacd321068ea6e7ac28875b03ab8bc0aa0966452db17cd36714/charset_normalizer-3.4.4-cp313-cp313-macosx_10_13_universal2.whl + - pypi: https://files.pythonhosted.org/packages/96/e4/7adcd9c8362745b2210728f209bfbcf7d91ba868a2c5f40d8b58f54c509b/contourpy-1.3.3-cp313-cp313-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/96/5d/dc5fa98fea3c175caf9d360649cb1aa3715e391ab00dc78c4c66fabd7356/coverage-7.11.0-cp313-cp313-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/e7/05/c19819d5e3d95294a6f5947fb9b9629efb316b96de511b418c53d245aae6/cycler-0.12.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/4e/8c/f3147f5c4b73e7550fe5f9352eaa956ae838d5c51eb58e7a25b9f3e2643b/decorator-5.2.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/f7/e6/efe534ef0952b531b630780e19cabd416e2032697019d5295defc6ef9bd9/deepdiff-8.6.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e3/26/57c6fb270950d476074c087527a558ccb6f4436657314bfb6cdf484114c4/docker-7.1.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/c1/ea/53f2148663b321f21b5a606bd5f191517cf40b7072c0497d3c92c4a13b1e/executing-2.2.1-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/8e/98/2c050dec90e295a524c9b65c4cb9e7c302386a296b2938710448cbd267d5/faker-37.12.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/7c/5b/cdd2c612277b7ac7ec8c0c9bc41812c43dc7b2d5f2b0897e15fdf5a1f915/fonttools-4.60.1-cp313-cp313-macosx_10_13_universal2.whl + - pypi: https://files.pythonhosted.org/packages/91/4c/e0ce1ef95d4000ebc1c11801f9b944fa5910ecc15b5e351865763d8657f8/graphviz-0.21-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/0e/61/66938bbb5fc52dbdf84594873d5b51fb1f7c7794e9c0f5bd885f30bc507b/idna-3.11-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/48/c5/d5e07995077e48220269c28a221e168c91123ad5ceee44d548f54a057fc0/ipython-9.6.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/d9/33/1f075bf72b0b747cb3288d011319aaf64083cf2efef8354174e3ed4540e2/ipython_pygments_lexers-1.1.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/c0/5a/9cac0c82afec3d09ccd97c8b6502d48f165f9124db81b4bcb90b4af974ee/jedi-0.19.2-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/2d/7a/9d90a151f558e29c3936b8a47ac770235f436f2120aca41a6d5f3d62ae8d/kiwisolver-1.4.9-cp313-cp313-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/bc/d0/b3d3338d467d3fc937f0bb7f256711395cae6f78e22cef0656159950adf0/matplotlib-3.10.7-cp313-cp313-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/af/33/ee4519fa02ed11a94aef9559552f3b17bb863f2ecfe1a35dc7f548cde231/matplotlib_inline-0.2.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/7d/ae/f32695da4f93de50dd7075100dab8cf689a9d96270f58ce6f940fd044a3e/minio-7.2.18-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/eb/8d/776adee7bbf76365fdd7f2552710282c79a4ead5d2a46408c9043a2b70ba/networkx-3.5-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/3e/46/bdd3370dcea2f95ef14af79dbf81e6927102ddf1cc54adc0024d61252fd9/numpy-2.3.4-cp313-cp313-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/12/27/fb8d7338b4d551900fa3e580acbe7a0cf655d940e164cb5c00ec31961094/orderly_set-5.5.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/20/12/38679034af332785aac8774540895e234f4d07f7545804097de4b666afd8/packaging-25.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/31/94/72fac03573102779920099bcac1c3b05975c2cb5f01eac609faf34bed1ca/pandas-2.3.3-cp313-cp313-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/16/32/f8e3c85d1d5250232a5d3477a2a28cc291968ff175caeadaf3cc19ce0e4a/parso-0.8.5-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/9e/c3/059298687310d527a58bb01f3b1965787ee3b40dce76752eda8b44e9a2c5/pexpect-4.9.0-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/83/06/48eab21dd561de2914242711434c0c0eb992ed08ff3f6107a5f44527f5e9/pillow-12.0.0-cp313-cp313-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/84/03/0d3ce49e2505ae70cf43bc5bb3033955d2fc9f932163e84dc0779cc47f48/prompt_toolkit-3.0.52-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/22/a6/858897256d0deac81a172289110f31629fc4cee19b6f01283303e18c8db3/ptyprocess-0.7.0-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/8e/37/efad0257dc6e593a18957422533ff0f87ede7c9c6ea010a2177d738fb82f/pure_eval-0.2.3-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/a0/e3/59cd50310fc9b59512193629e1984c1f95e5c8ae6e5d8c69532ccc65a7fe/pycparser-2.23-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/db/6c/a1f71542c969912bb0e106f64f60a56cc1f0fabecf9396f45accbe63fa68/pycryptodome-3.23.0-cp37-abi3-macosx_10_9_universal2.whl + - pypi: https://files.pythonhosted.org/packages/5a/87/b70ad306ebb6f9b585f114d0ac2137d792b48be34d732d60e597c2f8465a/pydantic-2.12.5-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/94/02/abfa0e0bda67faa65fef1c84971c7e45928e108fe24333c81f3bfe35d5f5/pydantic_core-2.41.5-cp313-cp313-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/c1/60/5d4751ba3f4a40a6891f24eec885f51afd78d208498268c734e256fb13c4/pydantic_settings-2.12.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/7e/32/a7125fb28c4261a627f999d5fb4afff25b523800faed2c30979949d6facd/pydot-4.0.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/c7/21/705964c7812476f378728bdf590ca4b771ec72385c533964653c68e86bdc/pygments-2.19.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/7c/4c/ad33b92b9864cbde84f259d5df035a6447f91891f5be77788e2a3892bce3/pymysql-1.1.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/10/5e/1aa9a93198c6b64513c9d7752de7422c06402de6600a8767da1524f9570b/pyparsing-3.2.5-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/a8/a4/20da314d277121d6534b3a980b29035dcd51e6744bd79075a6ce8fa4eb8d/pytest-8.4.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/ee/49/1377b49de7d0c1ce41292161ea0f721913fa8722c19fb9c1e3aa0367eecb/pytest_cov-7.0.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/27/98/822b924a4a3eb58aacba84444c7439fce32680592f394de26af9c76e2569/pytest_env-1.2.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/14/1b/a298b06749107c305e1fe0f814c6c74aea7b2f1e10989cb30f544a1b3253/python_dotenv-1.2.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/81/c4/34e93fe5f5429d7570ec1fa436f1986fb1f00c3e0f43a589fe2bbcd22c3f/pytz-2025.2-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/1e/db/4254e3eabe8020b458f1a747140d32277ec7a271daf1d235b70dc0b4e6e3/requests-2.32.5-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/a3/dc/17031897dae0efacfea57dfd3a82fdd2a2aeb58e0ff71b77b87e44edc772/setuptools-80.9.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/f1/7b/ce1eafaf1a76852e2ec9b22edecf1daa58175c090266e9f6c64afcd81d91/stack_data-0.6.3-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/d0/30/dc54f88dd4a2b5dc8a0279bdd7270e735851848b762aeb1c1184ed1f6b14/tqdm-4.67.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/00/c0/8f5d070730d7836adc9c9b6408dec68c6ced86b304a9b26a14df072a6e8c/traitlets-5.14.3-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/5c/23/c7abc0ca0a1526a0774eca151daeb8de62ec457e77262b66b359c3c7679e/tzdata-2025.2-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/a7/c2/fe1e52489ae3122415c51f387e221dd0773709bad6c6cdaa599e8a2c5185/urllib3-2.5.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/af/b5/123f13c975e9f27ab9c0770f514345bd406d0e8d3b7a0723af9d43f710af/wcwidth-0.2.14-py2.py3-none-any.whl + - pypi: ./ +packages: +- conda: https://conda.anaconda.org/conda-forge/linux-64/_libgcc_mutex-0.1-conda_forge.tar.bz2 + sha256: fe51de6107f9edc7aa4f786a70f4a883943bc9d39b3bb7307c04c41410990726 + md5: d7c89558ba9fa0495403155b64376d81 + license: None + purls: [] + size: 2562 + timestamp: 1578324546067 +- conda: https://conda.anaconda.org/conda-forge/linux-64/_openmp_mutex-4.5-2_gnu.tar.bz2 + build_number: 16 + sha256: fbe2c5e56a653bebb982eda4876a9178aedfc2b545f25d0ce9c4c0b508253d22 + md5: 73aaf86a425cc6e73fcf236a5a46396d + depends: + - _libgcc_mutex 0.1 conda_forge + - libgomp >=7.5.0 + constrains: + - openmp_impl 9999 + license: BSD-3-Clause + license_family: BSD + purls: [] + size: 23621 + timestamp: 1650670423406 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/_openmp_mutex-4.5-2_gnu.tar.bz2 + build_number: 16 + sha256: 3702bef2f0a4d38bd8288bbe54aace623602a1343c2cfbefd3fa188e015bebf0 + md5: 6168d71addc746e8f2b8d57dfd2edcea + depends: + - libgomp >=7.5.0 + constrains: + - openmp_impl 9999 + license: BSD-3-Clause + license_family: BSD + purls: [] + size: 23712 + timestamp: 1650670790230 +- conda: https://conda.anaconda.org/conda-forge/noarch/adwaita-icon-theme-48.1-unix_1.conda + sha256: f52307d3ff839bf4a001cb14b3944f169e46e37982a97c3d52cbf48a0cfe2327 + md5: 388097ca1f27fc28e0ef1986dd311891 + depends: + - __unix + - hicolor-icon-theme + - librsvg + license: LGPL-3.0-or-later OR CC-BY-SA-3.0 + license_family: LGPL + purls: [] + size: 621553 + timestamp: 1755882037787 +- conda: https://conda.anaconda.org/conda-forge/noarch/adwaita-icon-theme-49.0-unix_0.conda + sha256: a362b4f5c96a0bf4def96be1a77317e2730af38915eb9bec85e2a92836501ed7 + md5: b3f0179590f3c0637b7eb5309898f79e + depends: + - __unix + - hicolor-icon-theme + - librsvg + license: LGPL-3.0-or-later OR CC-BY-SA-3.0 + license_family: LGPL + purls: [] + size: 631452 + timestamp: 1758743294412 +- pypi: https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl + name: annotated-types + version: 0.7.0 + sha256: 1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53 + requires_dist: + - typing-extensions>=4.0.0 ; python_full_version < '3.9' + requires_python: '>=3.8' +- pypi: https://files.pythonhosted.org/packages/4f/d3/a8b22fa575b297cd6e3e3b0155c7e25db170edf1c74783d6a31a2490b8d9/argon2_cffi-25.1.0-py3-none-any.whl + name: argon2-cffi + version: 25.1.0 + sha256: fdc8b074db390fccb6eb4a3604ae7231f219aa669a2652e0f20e16ba513d5741 + requires_dist: + - argon2-cffi-bindings + requires_python: '>=3.8' +- pypi: https://files.pythonhosted.org/packages/09/52/94108adfdd6e2ddf58be64f959a0b9c7d4ef2fa71086c38356d22dc501ea/argon2_cffi_bindings-25.1.0-cp39-abi3-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl + name: argon2-cffi-bindings + version: 25.1.0 + sha256: d3e924cfc503018a714f94a49a149fdc0b644eaead5d1f089330399134fa028a + requires_dist: + - cffi>=1.0.1 ; python_full_version < '3.14' + - cffi>=2.0.0b1 ; python_full_version >= '3.14' + requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/b6/02/d297943bcacf05e4f2a94ab6f462831dc20158614e5d067c35d4e63b9acb/argon2_cffi_bindings-25.1.0-cp39-abi3-macosx_11_0_arm64.whl + name: argon2-cffi-bindings + version: 25.1.0 + sha256: 7aef0c91e2c0fbca6fc68e7555aa60ef7008a739cbe045541e438373bc54d2b0 + requires_dist: + - cffi>=1.0.1 ; python_full_version < '3.14' + - cffi>=2.0.0b1 ; python_full_version >= '3.14' + requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/c1/93/44365f3d75053e53893ec6d733e4a5e3147502663554b4d864587c7828a7/argon2_cffi_bindings-25.1.0-cp39-abi3-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl + name: argon2-cffi-bindings + version: 25.1.0 + sha256: 1e021e87faa76ae0d413b619fe2b65ab9a037f24c60a1e6cc43457ae20de6dc6 + requires_dist: + - cffi>=1.0.1 ; python_full_version < '3.14' + - cffi>=2.0.0b1 ; python_full_version >= '3.14' + requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/25/8a/c46dcc25341b5bce5472c718902eb3d38600a903b14fa6aeecef3f21a46f/asttokens-3.0.0-py3-none-any.whl + name: asttokens + version: 3.0.0 + sha256: e3078351a059199dd5138cb1c706e6430c05eff2ff136af5eb4790f9d28932e2 + requires_dist: + - astroid>=2,<4 ; extra == 'astroid' + - astroid>=2,<4 ; extra == 'test' + - pytest ; extra == 'test' + - pytest-cov ; extra == 'test' + - pytest-xdist ; extra == 'test' + requires_python: '>=3.8' +- conda: https://conda.anaconda.org/conda-forge/linux-64/at-spi2-atk-2.38.0-h0630a04_3.tar.bz2 + sha256: 26ab9386e80bf196e51ebe005da77d57decf6d989b4f34d96130560bc133479c + md5: 6b889f174df1e0f816276ae69281af4d + depends: + - at-spi2-core >=2.40.0,<2.41.0a0 + - atk-1.0 >=2.36.0 + - dbus >=1.13.6,<2.0a0 + - libgcc-ng >=9.3.0 + - libglib >=2.68.1,<3.0a0 + license: LGPL-2.1-or-later + license_family: LGPL + purls: [] + size: 339899 + timestamp: 1619122953439 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/at-spi2-atk-2.38.0-h1f2db35_3.tar.bz2 + sha256: c2c2c998d49c061e390537f929e77ce6b023ef22b51a0f55692d6df7327f3358 + md5: 4ea9d4634f3b054549be5e414291801e + depends: + - at-spi2-core >=2.40.0,<2.41.0a0 + - atk-1.0 >=2.36.0 + - dbus >=1.13.6,<2.0a0 + - libgcc-ng >=9.3.0 + - libglib >=2.68.1,<3.0a0 + license: LGPL-2.1-or-later + license_family: LGPL + purls: [] + size: 322172 + timestamp: 1619123713021 +- conda: https://conda.anaconda.org/conda-forge/linux-64/at-spi2-core-2.40.3-h0630a04_0.tar.bz2 + sha256: c4f9b66bd94c40d8f1ce1fad2d8b46534bdefda0c86e3337b28f6c25779f258d + md5: 8cb2fc4cd6cc63f1369cfa318f581cc3 + depends: + - dbus >=1.13.6,<2.0a0 + - libgcc-ng >=9.3.0 + - libglib >=2.68.3,<3.0a0 + - xorg-libx11 + - xorg-libxi + - xorg-libxtst + license: LGPL-2.1-or-later + license_family: LGPL + purls: [] + size: 658390 + timestamp: 1625848454791 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/at-spi2-core-2.40.3-h1f2db35_0.tar.bz2 + sha256: cd48de9674a20133e70a643476accc1a63360c921ab49477638364877937a40d + md5: a12602a94ee402b57063ef74e82016c0 + depends: + - dbus >=1.13.6,<2.0a0 + - libgcc-ng >=9.3.0 + - libglib >=2.68.3,<3.0a0 + - xorg-libx11 + - xorg-libxi + - xorg-libxtst + license: LGPL-2.1-or-later + license_family: LGPL + purls: [] + size: 622407 + timestamp: 1625848355776 +- conda: https://conda.anaconda.org/conda-forge/linux-64/atk-1.0-2.38.0-h04ea711_2.conda + sha256: df682395d05050cd1222740a42a551281210726a67447e5258968dd55854302e + md5: f730d54ba9cd543666d7220c9f7ed563 + depends: + - libgcc-ng >=12 + - libglib >=2.80.0,<3.0a0 + - libstdcxx-ng >=12 + constrains: + - atk-1.0 2.38.0 + license: LGPL-2.0-or-later + license_family: LGPL + purls: [] + size: 355900 + timestamp: 1713896169874 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/atk-1.0-2.38.0-hedc4a1f_2.conda + sha256: 69f70048a1a915be7b8ad5d2cbb7bf020baa989b5506e45a676ef4ef5106c4f0 + md5: 9308557e2328f944bd5809c5630761af + depends: + - libgcc-ng >=12 + - libglib >=2.80.0,<3.0a0 + - libstdcxx-ng >=12 + constrains: + - atk-1.0 2.38.0 + license: LGPL-2.0-or-later + license_family: LGPL + purls: [] + size: 358327 + timestamp: 1713898303194 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/atk-1.0-2.38.0-hd03087b_2.conda + sha256: b0747f9b1bc03d1932b4d8c586f39a35ac97e7e72fe6e63f2b2a2472d466f3c1 + md5: 57301986d02d30d6805fdce6c99074ee + depends: + - __osx >=11.0 + - libcxx >=16 + - libglib >=2.80.0,<3.0a0 + - libintl >=0.22.5,<1.0a0 + constrains: + - atk-1.0 2.38.0 + license: LGPL-2.0-or-later + license_family: LGPL + purls: [] + size: 347530 + timestamp: 1713896411580 +- conda: https://conda.anaconda.org/conda-forge/linux-64/bzip2-1.0.8-hda65f42_8.conda + sha256: c30daba32ddebbb7ded490f0e371eae90f51e72db620554089103b4a6934b0d5 + md5: 51a19bba1b8ebfb60df25cde030b7ebc + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=14 + license: bzip2-1.0.6 + license_family: BSD + purls: [] + size: 260341 + timestamp: 1757437258798 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/bzip2-1.0.8-h4777abc_8.conda + sha256: d2a296aa0b5f38ed9c264def6cf775c0ccb0f110ae156fcde322f3eccebf2e01 + md5: 2921ac0b541bf37c69e66bd6d9a43bca + depends: + - libgcc >=14 + license: bzip2-1.0.6 + license_family: BSD + purls: [] + size: 192536 + timestamp: 1757437302703 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/bzip2-1.0.8-hd037594_8.conda + sha256: b456200636bd5fecb2bec63f7e0985ad2097cf1b83d60ce0b6968dffa6d02aa1 + md5: 58fd217444c2a5701a44244faf518206 + depends: + - __osx >=11.0 + license: bzip2-1.0.6 + license_family: BSD + purls: [] + size: 125061 + timestamp: 1757437486465 +- conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025.10.5-hbd8a1cb_0.conda + sha256: 3b5ad78b8bb61b6cdc0978a6a99f8dfb2cc789a451378d054698441005ecbdb6 + md5: f9e5fbc24009179e8b0409624691758a + depends: + - __unix + license: ISC + purls: [] + size: 155907 + timestamp: 1759649036195 +- conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025.8.3-hbd8a1cb_0.conda + sha256: 837b795a2bb39b75694ba910c13c15fa4998d4bb2a622c214a6a5174b2ae53d1 + md5: 74784ee3d225fc3dca89edb635b4e5cc + depends: + - __unix + license: ISC + purls: [] + size: 154402 + timestamp: 1754210968730 +- conda: https://conda.anaconda.org/conda-forge/linux-64/cairo-1.18.4-h3394656_0.conda + sha256: 3bd6a391ad60e471de76c0e9db34986c4b5058587fbf2efa5a7f54645e28c2c7 + md5: 09262e66b19567aff4f592fb53b28760 + depends: + - __glibc >=2.17,<3.0.a0 + - fontconfig >=2.15.0,<3.0a0 + - fonts-conda-ecosystem + - freetype >=2.12.1,<3.0a0 + - icu >=75.1,<76.0a0 + - libexpat >=2.6.4,<3.0a0 + - libgcc >=13 + - libglib >=2.82.2,<3.0a0 + - libpng >=1.6.47,<1.7.0a0 + - libstdcxx >=13 + - libxcb >=1.17.0,<2.0a0 + - libzlib >=1.3.1,<2.0a0 + - pixman >=0.44.2,<1.0a0 + - xorg-libice >=1.1.2,<2.0a0 + - xorg-libsm >=1.2.5,<2.0a0 + - xorg-libx11 >=1.8.11,<2.0a0 + - xorg-libxext >=1.3.6,<2.0a0 + - xorg-libxrender >=0.9.12,<0.10.0a0 + license: LGPL-2.1-only or MPL-1.1 + purls: [] + size: 978114 + timestamp: 1741554591855 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/cairo-1.18.4-h83712da_0.conda + sha256: 37cfff940d2d02259afdab75eb2dbac42cf830adadee78d3733d160a1de2cc66 + md5: cd55953a67ec727db5dc32b167201aa6 + depends: + - fontconfig >=2.15.0,<3.0a0 + - fonts-conda-ecosystem + - freetype >=2.12.1,<3.0a0 + - icu >=75.1,<76.0a0 + - libexpat >=2.6.4,<3.0a0 + - libgcc >=13 + - libglib >=2.82.2,<3.0a0 + - libpng >=1.6.47,<1.7.0a0 + - libstdcxx >=13 + - libxcb >=1.17.0,<2.0a0 + - libzlib >=1.3.1,<2.0a0 + - pixman >=0.44.2,<1.0a0 + - xorg-libice >=1.1.2,<2.0a0 + - xorg-libsm >=1.2.5,<2.0a0 + - xorg-libx11 >=1.8.11,<2.0a0 + - xorg-libxext >=1.3.6,<2.0a0 + - xorg-libxrender >=0.9.12,<0.10.0a0 + license: LGPL-2.1-only or MPL-1.1 + purls: [] + size: 966667 + timestamp: 1741554768968 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/cairo-1.18.4-h6a3b0d2_0.conda + sha256: 00439d69bdd94eaf51656fdf479e0c853278439d22ae151cabf40eb17399d95f + md5: 38f6df8bc8c668417b904369a01ba2e2 + depends: + - __osx >=11.0 + - fontconfig >=2.15.0,<3.0a0 + - fonts-conda-ecosystem + - freetype >=2.12.1,<3.0a0 + - icu >=75.1,<76.0a0 + - libcxx >=18 + - libexpat >=2.6.4,<3.0a0 + - libglib >=2.82.2,<3.0a0 + - libpng >=1.6.47,<1.7.0a0 + - libzlib >=1.3.1,<2.0a0 + - pixman >=0.44.2,<1.0a0 + license: LGPL-2.1-only or MPL-1.1 + purls: [] + size: 896173 + timestamp: 1741554795915 +- pypi: https://files.pythonhosted.org/packages/e5/48/1549795ba7742c948d2ad169c1c8cdbae65bc450d6cd753d124b17c8cd32/certifi-2025.8.3-py3-none-any.whl + name: certifi + version: 2025.8.3 + sha256: f6c12493cfb1b06ba2ff328595af9350c65d6644968e5d3a2ffd78699af217a5 + requires_python: '>=3.7' +- pypi: https://files.pythonhosted.org/packages/e4/37/af0d2ef3967ac0d6113837b44a4f0bfe1328c2b9763bd5b1744520e5cfed/certifi-2025.10.5-py3-none-any.whl + name: certifi + version: 2025.10.5 + sha256: 0f212c2744a9bb6de0c56639a6f68afe01ecd92d91f14ae897c4fe7bbeeef0de + requires_python: '>=3.7' +- pypi: https://files.pythonhosted.org/packages/4a/d2/a6c0296814556c68ee32009d9c2ad4f85f2707cdecfd7727951ec228005d/cffi-2.0.0-cp313-cp313-macosx_11_0_arm64.whl + name: cffi + version: 2.0.0 + sha256: 45d5e886156860dc35862657e1494b9bae8dfa63bf56796f2fb56e1679fc0bca + requires_dist: + - pycparser ; implementation_name != 'PyPy' + requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/98/df/0a1755e750013a2081e863e7cd37e0cdd02664372c754e5560099eb7aa44/cffi-2.0.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl + name: cffi + version: 2.0.0 + sha256: c8d3b5532fc71b7a77c09192b4a5a200ea992702734a2e9279a37f2478236f26 + requires_dist: + - pycparser ; implementation_name != 'PyPy' + requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/a9/f5/a2c23eb03b61a0b8747f211eb716446c826ad66818ddc7810cc2cc19b3f2/cffi-2.0.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl + name: cffi + version: 2.0.0 + sha256: d48a880098c96020b02d5a1f7d9251308510ce8858940e6fa99ece33f610838b + requires_dist: + - pycparser ; implementation_name != 'PyPy' + requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/c5/55/51844dd50c4fc7a33b653bfaba4c2456f06955289ca770a5dbd5fd267374/cfgv-3.4.0-py2.py3-none-any.whl + name: cfgv + version: 3.4.0 + sha256: b7265b1f29fd3316bfcd2b330d63d024f2bfd8bcb8b0272f8e19a504856c48f9 + requires_python: '>=3.8' +- pypi: https://files.pythonhosted.org/packages/7e/95/42aa2156235cbc8fa61208aded06ef46111c4d3f0de233107b3f38631803/charset_normalizer-3.4.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl + name: charset-normalizer + version: 3.4.3 + sha256: 416175faf02e4b0810f1f38bcb54682878a4af94059a1cd63b8747244420801f + requires_python: '>=3.7' +- pypi: https://files.pythonhosted.org/packages/7d/62/73a6d7450829655a35bb88a88fca7d736f9882a27eacdca2c6d505b57e2e/charset_normalizer-3.4.4-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl + name: charset-normalizer + version: 3.4.4 + sha256: 6b39f987ae8ccdf0d2642338faf2abb1862340facc796048b604ef14919e55ed + requires_python: '>=3.7' +- pypi: https://files.pythonhosted.org/packages/97/45/4b3a1239bbacd321068ea6e7ac28875b03ab8bc0aa0966452db17cd36714/charset_normalizer-3.4.4-cp313-cp313-macosx_10_13_universal2.whl + name: charset-normalizer + version: 3.4.4 + sha256: e1f185f86a6f3403aa2420e815904c67b2f9ebc443f045edd0de921108345794 + requires_python: '>=3.7' +- pypi: https://files.pythonhosted.org/packages/20/01/b394922252051e97aab231d416c86da3d8a6d781eeadcdca1082867de64e/codespell-2.4.1-py3-none-any.whl + name: codespell + version: 2.4.1 + sha256: 3dadafa67df7e4a3dbf51e0d7315061b80d265f9552ebd699b3dd6834b47e425 + requires_dist: + - build ; extra == 'dev' + - chardet ; extra == 'dev' + - pre-commit ; extra == 'dev' + - pytest ; extra == 'dev' + - pytest-cov ; extra == 'dev' + - pytest-dependency ; extra == 'dev' + - pygments ; extra == 'dev' + - ruff ; extra == 'dev' + - tomli ; extra == 'dev' + - twine ; extra == 'dev' + - chardet ; extra == 'hard-encoding-detection' + - tomli ; python_full_version < '3.11' and extra == 'toml' + - chardet>=5.1.0 ; extra == 'types' + - mypy ; extra == 'types' + - pytest ; extra == 'types' + - pytest-cov ; extra == 'types' + - pytest-dependency ; extra == 'types' + requires_python: '>=3.8' +- pypi: https://files.pythonhosted.org/packages/4b/32/e0f13a1c5b0f8572d0ec6ae2f6c677b7991fafd95da523159c19eff0696a/contourpy-1.3.3-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl + name: contourpy + version: 1.3.3 + sha256: 4debd64f124ca62069f313a9cb86656ff087786016d76927ae2cf37846b006c9 + requires_dist: + - numpy>=1.25 + - furo ; extra == 'docs' + - sphinx>=7.2 ; extra == 'docs' + - sphinx-copybutton ; extra == 'docs' + - bokeh ; extra == 'bokeh' + - selenium ; extra == 'bokeh' + - contourpy[bokeh,docs] ; extra == 'mypy' + - bokeh ; extra == 'mypy' + - docutils-stubs ; extra == 'mypy' + - mypy==1.17.0 ; extra == 'mypy' + - types-pillow ; extra == 'mypy' + - contourpy[test-no-images] ; extra == 'test' + - matplotlib ; extra == 'test' + - pillow ; extra == 'test' + - pytest ; extra == 'test-no-images' + - pytest-cov ; extra == 'test-no-images' + - pytest-rerunfailures ; extra == 'test-no-images' + - pytest-xdist ; extra == 'test-no-images' + - wurlitzer ; extra == 'test-no-images' + requires_python: '>=3.11' +- pypi: https://files.pythonhosted.org/packages/73/23/90e31ceeed1de63058a02cb04b12f2de4b40e3bef5e082a7c18d9c8ae281/contourpy-1.3.3-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl + name: contourpy + version: 1.3.3 + sha256: 348ac1f5d4f1d66d3322420f01d42e43122f43616e0f194fc1c9f5d830c5b286 + requires_dist: + - numpy>=1.25 + - furo ; extra == 'docs' + - sphinx>=7.2 ; extra == 'docs' + - sphinx-copybutton ; extra == 'docs' + - bokeh ; extra == 'bokeh' + - selenium ; extra == 'bokeh' + - contourpy[bokeh,docs] ; extra == 'mypy' + - bokeh ; extra == 'mypy' + - docutils-stubs ; extra == 'mypy' + - mypy==1.17.0 ; extra == 'mypy' + - types-pillow ; extra == 'mypy' + - contourpy[test-no-images] ; extra == 'test' + - matplotlib ; extra == 'test' + - pillow ; extra == 'test' + - pytest ; extra == 'test-no-images' + - pytest-cov ; extra == 'test-no-images' + - pytest-rerunfailures ; extra == 'test-no-images' + - pytest-xdist ; extra == 'test-no-images' + - wurlitzer ; extra == 'test-no-images' + requires_python: '>=3.11' +- pypi: https://files.pythonhosted.org/packages/96/e4/7adcd9c8362745b2210728f209bfbcf7d91ba868a2c5f40d8b58f54c509b/contourpy-1.3.3-cp313-cp313-macosx_11_0_arm64.whl + name: contourpy + version: 1.3.3 + sha256: d002b6f00d73d69333dac9d0b8d5e84d9724ff9ef044fd63c5986e62b7c9e1b1 + requires_dist: + - numpy>=1.25 + - furo ; extra == 'docs' + - sphinx>=7.2 ; extra == 'docs' + - sphinx-copybutton ; extra == 'docs' + - bokeh ; extra == 'bokeh' + - selenium ; extra == 'bokeh' + - contourpy[bokeh,docs] ; extra == 'mypy' + - bokeh ; extra == 'mypy' + - docutils-stubs ; extra == 'mypy' + - mypy==1.17.0 ; extra == 'mypy' + - types-pillow ; extra == 'mypy' + - contourpy[test-no-images] ; extra == 'test' + - matplotlib ; extra == 'test' + - pillow ; extra == 'test' + - pytest ; extra == 'test-no-images' + - pytest-cov ; extra == 'test-no-images' + - pytest-rerunfailures ; extra == 'test-no-images' + - pytest-xdist ; extra == 'test-no-images' + - wurlitzer ; extra == 'test-no-images' + requires_python: '>=3.11' +- pypi: https://files.pythonhosted.org/packages/73/dd/508420fb47d09d904d962f123221bc249f64b5e56aa93d5f5f7603be475f/coverage-7.10.6-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl + name: coverage + version: 7.10.6 + sha256: 0f3f56e4cb573755e96a16501a98bf211f100463d70275759e73f3cbc00d4f27 + requires_dist: + - tomli ; python_full_version <= '3.11' and extra == 'toml' + requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/96/5d/dc5fa98fea3c175caf9d360649cb1aa3715e391ab00dc78c4c66fabd7356/coverage-7.11.0-cp313-cp313-macosx_11_0_arm64.whl + name: coverage + version: 7.11.0 + sha256: f39ae2f63f37472c17b4990f794035c9890418b1b8cca75c01193f3c8d3e01be + requires_dist: + - tomli ; python_full_version <= '3.11' and extra == 'toml' + requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/2e/7a/34c9402ad12bce609be4be1146a7d22a7fae8e9d752684b6315cce552a65/coverage-7.11.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl + name: coverage + version: 7.11.2 + sha256: 811bff1f93566a8556a9aeb078bd82573e37f4d802a185fba4cbe75468615050 + requires_dist: + - tomli ; python_full_version <= '3.11' and extra == 'toml' + requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/e7/05/c19819d5e3d95294a6f5947fb9b9629efb316b96de511b418c53d245aae6/cycler-0.12.1-py3-none-any.whl + name: cycler + version: 0.12.1 + sha256: 85cef7cff222d8644161529808465972e51340599459b8ac3ccbac5a854e0d30 + requires_dist: + - ipython ; extra == 'docs' + - matplotlib ; extra == 'docs' + - numpydoc ; extra == 'docs' + - sphinx ; extra == 'docs' + - pytest ; extra == 'tests' + - pytest-cov ; extra == 'tests' + - pytest-xdist ; extra == 'tests' + requires_python: '>=3.8' +- pypi: ./ + name: datajoint + version: 0.14.6 + sha256: f761bb719d6afe0361d7e564bcc950ea76c79fbee9c334032459d0d4437a6423 + requires_dist: + - numpy + - pymysql>=0.7.2 + - deepdiff + - pyparsing + - ipython + - pandas + - tqdm + - networkx + - pydot + - minio>=7.0.0 + - matplotlib + - faker + - urllib3 + - setuptools + - pydantic-settings>=2.0.0 + - pre-commit ; extra == 'dev' + - ruff ; extra == 'dev' + - codespell ; extra == 'dev' + - pytest ; extra == 'dev' + - pytest-cov ; extra == 'dev' + requires_python: '>=3.9,<3.14' + editable: true +- conda: https://conda.anaconda.org/conda-forge/linux-64/dbus-1.16.2-h3c4dab8_0.conda + sha256: 3b988146a50e165f0fa4e839545c679af88e4782ec284cc7b6d07dd226d6a068 + md5: 679616eb5ad4e521c83da4650860aba7 + depends: + - libstdcxx >=13 + - libgcc >=13 + - __glibc >=2.17,<3.0.a0 + - libgcc >=13 + - libexpat >=2.7.0,<3.0a0 + - libzlib >=1.3.1,<2.0a0 + - libglib >=2.84.2,<3.0a0 + license: GPL-2.0-or-later + license_family: GPL + purls: [] + size: 437860 + timestamp: 1747855126005 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/dbus-1.16.2-heda779d_0.conda + sha256: 5c9166bbbe1ea7d0685a1549aad4ea887b1eb3a07e752389f86b185ef8eac99a + md5: 9203b74bb1f3fa0d6f308094b3b44c1e + depends: + - libgcc >=13 + - libstdcxx >=13 + - libgcc >=13 + - libexpat >=2.7.0,<3.0a0 + - libglib >=2.84.2,<3.0a0 + - libzlib >=1.3.1,<2.0a0 + license: GPL-2.0-or-later + license_family: GPL + purls: [] + size: 469781 + timestamp: 1747855172617 +- pypi: https://files.pythonhosted.org/packages/4e/8c/f3147f5c4b73e7550fe5f9352eaa956ae838d5c51eb58e7a25b9f3e2643b/decorator-5.2.1-py3-none-any.whl + name: decorator + version: 5.2.1 + sha256: d316bb415a2d9e2d2b3abcc4084c6502fc09240e292cd76a76afc106a1c8e04a + requires_python: '>=3.8' +- pypi: https://files.pythonhosted.org/packages/f7/e6/efe534ef0952b531b630780e19cabd416e2032697019d5295defc6ef9bd9/deepdiff-8.6.1-py3-none-any.whl + name: deepdiff + version: 8.6.1 + sha256: ee8708a7f7d37fb273a541fa24ad010ed484192cd0c4ffc0fa0ed5e2d4b9e78b + requires_dist: + - orderly-set>=5.4.1,<6 + - click~=8.1.0 ; extra == 'cli' + - pyyaml~=6.0.0 ; extra == 'cli' + - coverage~=7.6.0 ; extra == 'coverage' + - bump2version~=1.0.0 ; extra == 'dev' + - jsonpickle~=4.0.0 ; extra == 'dev' + - ipdb~=0.13.0 ; extra == 'dev' + - numpy~=2.2.0 ; python_full_version >= '3.10' and extra == 'dev' + - numpy~=2.0 ; python_full_version < '3.10' and extra == 'dev' + - python-dateutil~=2.9.0 ; extra == 'dev' + - orjson~=3.10.0 ; extra == 'dev' + - tomli~=2.2.0 ; extra == 'dev' + - tomli-w~=1.2.0 ; extra == 'dev' + - pandas~=2.2.0 ; extra == 'dev' + - polars~=1.21.0 ; extra == 'dev' + - nox==2025.5.1 ; extra == 'dev' + - uuid6==2025.0.1 ; extra == 'dev' + - sphinx~=6.2.0 ; extra == 'docs' + - sphinx-sitemap~=2.6.0 ; extra == 'docs' + - sphinxemoji~=0.3.0 ; extra == 'docs' + - orjson ; extra == 'optimize' + - flake8~=7.1.0 ; extra == 'static' + - flake8-pyproject~=1.2.3 ; extra == 'static' + - pydantic~=2.10.0 ; extra == 'static' + - pytest~=8.3.0 ; extra == 'test' + - pytest-benchmark~=5.1.0 ; extra == 'test' + - pytest-cov~=6.0.0 ; extra == 'test' + - python-dotenv~=1.0.0 ; extra == 'test' + requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/33/6b/e0547afaf41bf2c42e52430072fa5658766e3d65bd4b03a563d1b6336f57/distlib-0.4.0-py2.py3-none-any.whl + name: distlib + version: 0.4.0 + sha256: 9659f7d87e46584a30b5780e43ac7a2143098441670ff0a49d5f9034c54a6c16 +- pypi: https://files.pythonhosted.org/packages/e3/26/57c6fb270950d476074c087527a558ccb6f4436657314bfb6cdf484114c4/docker-7.1.0-py3-none-any.whl + name: docker + version: 7.1.0 + sha256: c96b93b7f0a746f9e77d325bcfb87422a3d8bd4f03136ae8a85b37f1898d5fc0 + requires_dist: + - pywin32>=304 ; sys_platform == 'win32' + - requests>=2.26.0 + - urllib3>=1.26.0 + - coverage==7.2.7 ; extra == 'dev' + - pytest-cov==4.1.0 ; extra == 'dev' + - pytest-timeout==2.1.0 ; extra == 'dev' + - pytest==7.4.2 ; extra == 'dev' + - ruff==0.1.8 ; extra == 'dev' + - myst-parser==0.18.0 ; extra == 'docs' + - sphinx==5.1.1 ; extra == 'docs' + - paramiko>=2.4.3 ; extra == 'ssh' + - websocket-client>=1.3.0 ; extra == 'websockets' + requires_python: '>=3.8' +- conda: https://conda.anaconda.org/conda-forge/linux-64/epoxy-1.5.10-h166bdaf_1.tar.bz2 + sha256: 1e58ee2ed0f4699be202f23d49b9644b499836230da7dd5b2f63e6766acff89e + md5: a089d06164afd2d511347d3f87214e0b + depends: + - libgcc-ng >=10.3.0 + license: MIT + license_family: MIT + purls: [] + size: 1440699 + timestamp: 1648505042260 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/epoxy-1.5.10-he30d5cf_2.conda + sha256: aa562cdd72d2d15b0f2ee4565c8e34f18b52f7135a3f3b1ce727c202425c3bec + md5: 1c50e7c46ccefffe918ac974fa1a6752 + depends: + - libdrm >=2.4.125,<2.5.0a0 + - libegl >=1.7.0,<2.0a0 + - libegl-devel + - libgcc >=14 + - libgl >=1.7.0,<2.0a0 + - libgl-devel + - libglx >=1.7.0,<2.0a0 + - libglx-devel + - xorg-libx11 >=1.8.12,<2.0a0 + - xorg-libxdamage >=1.1.6,<2.0a0 + - xorg-libxext >=1.3.6,<2.0a0 + - xorg-libxfixes >=6.0.1,<7.0a0 + - xorg-libxxf86vm >=1.1.6,<2.0a0 + license: MIT + license_family: MIT + purls: [] + size: 422103 + timestamp: 1758743388115 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/epoxy-1.5.10-hc919400_2.conda + sha256: ba685b87529c95a4bf9de140a33d703d57dc46b036e9586ed26890de65c1c0d5 + md5: 3b87dabebe54c6d66a07b97b53ac5874 + depends: + - __osx >=11.0 + license: MIT + license_family: MIT + purls: [] + size: 296347 + timestamp: 1758743805063 +- pypi: https://files.pythonhosted.org/packages/c1/ea/53f2148663b321f21b5a606bd5f191517cf40b7072c0497d3c92c4a13b1e/executing-2.2.1-py2.py3-none-any.whl + name: executing + version: 2.2.1 + sha256: 760643d3452b4d777d295bb167ccc74c64a81df23fb5e08eff250c425a4b2017 + requires_dist: + - asttokens>=2.1.0 ; extra == 'tests' + - ipython ; extra == 'tests' + - pytest ; extra == 'tests' + - coverage ; extra == 'tests' + - coverage-enable-subprocess ; extra == 'tests' + - littleutils ; extra == 'tests' + - rich ; python_full_version >= '3.11' and extra == 'tests' + requires_python: '>=3.8' +- pypi: https://files.pythonhosted.org/packages/f5/11/02ebebb09ff2104b690457cb7bc6ed700c9e0ce88cf581486bb0a5d3c88b/faker-37.8.0-py3-none-any.whl + name: faker + version: 37.8.0 + sha256: b08233118824423b5fc239f7dd51f145e7018082b4164f8da6a9994e1f1ae793 + requires_dist: + - tzdata + requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/8e/98/2c050dec90e295a524c9b65c4cb9e7c302386a296b2938710448cbd267d5/faker-37.12.0-py3-none-any.whl + name: faker + version: 37.12.0 + sha256: afe7ccc038da92f2fbae30d8e16d19d91e92e242f8401ce9caf44de892bab4c4 + requires_dist: + - tzdata + requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/42/14/42b2651a2f46b022ccd948bca9f2d5af0fd8929c4eec235b8d6d844fbe67/filelock-3.19.1-py3-none-any.whl + name: filelock + version: 3.19.1 + sha256: d38e30481def20772f5baf097c122c3babc4fcdb7e14e57049eb9d88c6dc017d + requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/76/91/7216b27286936c16f5b4d0c530087e4a54eead683e6b0b73dd0c64844af6/filelock-3.20.0-py3-none-any.whl + name: filelock + version: 3.20.0 + sha256: 339b4732ffda5cd79b13f4e2711a31b0365ce445d95d243bb996273d072546a2 + requires_python: '>=3.10' +- conda: https://conda.anaconda.org/conda-forge/noarch/font-ttf-dejavu-sans-mono-2.37-hab24e00_0.tar.bz2 + sha256: 58d7f40d2940dd0a8aa28651239adbf5613254df0f75789919c4e6762054403b + md5: 0c96522c6bdaed4b1566d11387caaf45 + license: BSD-3-Clause + license_family: BSD + purls: [] + size: 397370 + timestamp: 1566932522327 +- conda: https://conda.anaconda.org/conda-forge/noarch/font-ttf-inconsolata-3.000-h77eed37_0.tar.bz2 + sha256: c52a29fdac682c20d252facc50f01e7c2e7ceac52aa9817aaf0bb83f7559ec5c + md5: 34893075a5c9e55cdafac56607368fc6 + license: OFL-1.1 + license_family: Other + purls: [] + size: 96530 + timestamp: 1620479909603 +- conda: https://conda.anaconda.org/conda-forge/noarch/font-ttf-source-code-pro-2.038-h77eed37_0.tar.bz2 + sha256: 00925c8c055a2275614b4d983e1df637245e19058d79fc7dd1a93b8d9fb4b139 + md5: 4d59c254e01d9cde7957100457e2d5fb + license: OFL-1.1 + license_family: Other + purls: [] + size: 700814 + timestamp: 1620479612257 +- conda: https://conda.anaconda.org/conda-forge/noarch/font-ttf-ubuntu-0.83-h77eed37_3.conda + sha256: 2821ec1dc454bd8b9a31d0ed22a7ce22422c0aef163c59f49dfdf915d0f0ca14 + md5: 49023d73832ef61042f6a237cb2687e7 + license: LicenseRef-Ubuntu-Font-Licence-Version-1.0 + license_family: Other + purls: [] + size: 1620504 + timestamp: 1727511233259 +- conda: https://conda.anaconda.org/conda-forge/linux-64/fontconfig-2.15.0-h7e30c49_1.conda + sha256: 7093aa19d6df5ccb6ca50329ef8510c6acb6b0d8001191909397368b65b02113 + md5: 8f5b0b297b59e1ac160ad4beec99dbee + depends: + - __glibc >=2.17,<3.0.a0 + - freetype >=2.12.1,<3.0a0 + - libexpat >=2.6.3,<3.0a0 + - libgcc >=13 + - libuuid >=2.38.1,<3.0a0 + - libzlib >=1.3.1,<2.0a0 + license: MIT + license_family: MIT + purls: [] + size: 265599 + timestamp: 1730283881107 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/fontconfig-2.15.0-h8dda3cd_1.conda + sha256: fe023bb8917c8a3138af86ef537b70c8c5d60c44f93946a87d1e8bb1a6634b55 + md5: 112b71b6af28b47c624bcbeefeea685b + depends: + - freetype >=2.12.1,<3.0a0 + - libexpat >=2.6.3,<3.0a0 + - libgcc >=13 + - libuuid >=2.38.1,<3.0a0 + - libzlib >=1.3.1,<2.0a0 + license: MIT + license_family: MIT + purls: [] + size: 277832 + timestamp: 1730284967179 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/fontconfig-2.15.0-h1383a14_1.conda + sha256: f79d3d816fafbd6a2b0f75ebc3251a30d3294b08af9bb747194121f5efa364bc + md5: 7b29f48742cea5d1ccb5edd839cb5621 + depends: + - __osx >=11.0 + - freetype >=2.12.1,<3.0a0 + - libexpat >=2.6.3,<3.0a0 + - libzlib >=1.3.1,<2.0a0 + license: MIT + license_family: MIT + purls: [] + size: 234227 + timestamp: 1730284037572 +- conda: https://conda.anaconda.org/conda-forge/noarch/fonts-conda-ecosystem-1-0.tar.bz2 + sha256: a997f2f1921bb9c9d76e6fa2f6b408b7fa549edd349a77639c9fe7a23ea93e61 + md5: fee5683a3f04bd15cbd8318b096a27ab + depends: + - fonts-conda-forge + license: BSD-3-Clause + license_family: BSD + purls: [] + size: 3667 + timestamp: 1566974674465 +- conda: https://conda.anaconda.org/conda-forge/noarch/fonts-conda-forge-1-0.tar.bz2 + sha256: 53f23a3319466053818540bcdf2091f253cbdbab1e0e9ae7b9e509dcaa2a5e38 + md5: f766549260d6815b0c52253f1fb1bb29 + depends: + - font-ttf-dejavu-sans-mono + - font-ttf-inconsolata + - font-ttf-source-code-pro + - font-ttf-ubuntu + license: BSD-3-Clause + license_family: BSD + purls: [] + size: 4102 + timestamp: 1566932280397 +- conda: https://conda.anaconda.org/conda-forge/noarch/fonts-conda-forge-1-hc364b38_1.conda + sha256: 54eea8469786bc2291cc40bca5f46438d3e062a399e8f53f013b6a9f50e98333 + md5: a7970cd949a077b7cb9696379d338681 + depends: + - font-ttf-ubuntu + - font-ttf-inconsolata + - font-ttf-dejavu-sans-mono + - font-ttf-source-code-pro + license: BSD-3-Clause + license_family: BSD + purls: [] + size: 4059 + timestamp: 1762351264405 +- pypi: https://files.pythonhosted.org/packages/f2/9f/bf231c2a3fac99d1d7f1d89c76594f158693f981a4aa02be406e9f036832/fonttools-4.59.2-cp313-cp313-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl + name: fonttools + version: 4.59.2 + sha256: 6235fc06bcbdb40186f483ba9d5d68f888ea68aa3c8dac347e05a7c54346fbc8 + requires_dist: + - lxml>=4.0 ; extra == 'lxml' + - brotli>=1.0.1 ; platform_python_implementation == 'CPython' and extra == 'woff' + - brotlicffi>=0.8.0 ; platform_python_implementation != 'CPython' and extra == 'woff' + - zopfli>=0.1.4 ; extra == 'woff' + - unicodedata2>=15.1.0 ; python_full_version < '3.13' and extra == 'unicode' + - lz4>=1.7.4.2 ; extra == 'graphite' + - scipy ; platform_python_implementation != 'PyPy' and extra == 'interpolatable' + - munkres ; platform_python_implementation == 'PyPy' and extra == 'interpolatable' + - pycairo ; extra == 'interpolatable' + - matplotlib ; extra == 'plot' + - sympy ; extra == 'symfont' + - xattr ; sys_platform == 'darwin' and extra == 'type1' + - skia-pathops>=0.5.0 ; extra == 'pathops' + - uharfbuzz>=0.23.0 ; extra == 'repacker' + - lxml>=4.0 ; extra == 'all' + - brotli>=1.0.1 ; platform_python_implementation == 'CPython' and extra == 'all' + - brotlicffi>=0.8.0 ; platform_python_implementation != 'CPython' and extra == 'all' + - zopfli>=0.1.4 ; extra == 'all' + - unicodedata2>=15.1.0 ; python_full_version < '3.13' and extra == 'all' + - lz4>=1.7.4.2 ; extra == 'all' + - scipy ; platform_python_implementation != 'PyPy' and extra == 'all' + - munkres ; platform_python_implementation == 'PyPy' and extra == 'all' + - pycairo ; extra == 'all' + - matplotlib ; extra == 'all' + - sympy ; extra == 'all' + - xattr ; sys_platform == 'darwin' and extra == 'all' + - skia-pathops>=0.5.0 ; extra == 'all' + - uharfbuzz>=0.23.0 ; extra == 'all' + requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/04/05/06b1455e4bc653fcb2117ac3ef5fa3a8a14919b93c60742d04440605d058/fonttools-4.60.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl + name: fonttools + version: 4.60.1 + sha256: 2409d5fb7b55fd70f715e6d34e7a6e4f7511b8ad29a49d6df225ee76da76dd77 + requires_dist: + - lxml>=4.0 ; extra == 'lxml' + - brotli>=1.0.1 ; platform_python_implementation == 'CPython' and extra == 'woff' + - brotlicffi>=0.8.0 ; platform_python_implementation != 'CPython' and extra == 'woff' + - zopfli>=0.1.4 ; extra == 'woff' + - unicodedata2>=15.1.0 ; python_full_version < '3.13' and extra == 'unicode' + - lz4>=1.7.4.2 ; extra == 'graphite' + - scipy ; platform_python_implementation != 'PyPy' and extra == 'interpolatable' + - munkres ; platform_python_implementation == 'PyPy' and extra == 'interpolatable' + - pycairo ; extra == 'interpolatable' + - matplotlib ; extra == 'plot' + - sympy ; extra == 'symfont' + - xattr ; sys_platform == 'darwin' and extra == 'type1' + - skia-pathops>=0.5.0 ; extra == 'pathops' + - uharfbuzz>=0.23.0 ; extra == 'repacker' + - lxml>=4.0 ; extra == 'all' + - brotli>=1.0.1 ; platform_python_implementation == 'CPython' and extra == 'all' + - brotlicffi>=0.8.0 ; platform_python_implementation != 'CPython' and extra == 'all' + - zopfli>=0.1.4 ; extra == 'all' + - unicodedata2>=15.1.0 ; python_full_version < '3.13' and extra == 'all' + - lz4>=1.7.4.2 ; extra == 'all' + - scipy ; platform_python_implementation != 'PyPy' and extra == 'all' + - munkres ; platform_python_implementation == 'PyPy' and extra == 'all' + - pycairo ; extra == 'all' + - matplotlib ; extra == 'all' + - sympy ; extra == 'all' + - xattr ; sys_platform == 'darwin' and extra == 'all' + - skia-pathops>=0.5.0 ; extra == 'all' + - uharfbuzz>=0.23.0 ; extra == 'all' + requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/7c/5b/cdd2c612277b7ac7ec8c0c9bc41812c43dc7b2d5f2b0897e15fdf5a1f915/fonttools-4.60.1-cp313-cp313-macosx_10_13_universal2.whl + name: fonttools + version: 4.60.1 + sha256: 6f68576bb4bbf6060c7ab047b1574a1ebe5c50a17de62830079967b211059ebb + requires_dist: + - lxml>=4.0 ; extra == 'lxml' + - brotli>=1.0.1 ; platform_python_implementation == 'CPython' and extra == 'woff' + - brotlicffi>=0.8.0 ; platform_python_implementation != 'CPython' and extra == 'woff' + - zopfli>=0.1.4 ; extra == 'woff' + - unicodedata2>=15.1.0 ; python_full_version < '3.13' and extra == 'unicode' + - lz4>=1.7.4.2 ; extra == 'graphite' + - scipy ; platform_python_implementation != 'PyPy' and extra == 'interpolatable' + - munkres ; platform_python_implementation == 'PyPy' and extra == 'interpolatable' + - pycairo ; extra == 'interpolatable' + - matplotlib ; extra == 'plot' + - sympy ; extra == 'symfont' + - xattr ; sys_platform == 'darwin' and extra == 'type1' + - skia-pathops>=0.5.0 ; extra == 'pathops' + - uharfbuzz>=0.23.0 ; extra == 'repacker' + - lxml>=4.0 ; extra == 'all' + - brotli>=1.0.1 ; platform_python_implementation == 'CPython' and extra == 'all' + - brotlicffi>=0.8.0 ; platform_python_implementation != 'CPython' and extra == 'all' + - zopfli>=0.1.4 ; extra == 'all' + - unicodedata2>=15.1.0 ; python_full_version < '3.13' and extra == 'all' + - lz4>=1.7.4.2 ; extra == 'all' + - scipy ; platform_python_implementation != 'PyPy' and extra == 'all' + - munkres ; platform_python_implementation == 'PyPy' and extra == 'all' + - pycairo ; extra == 'all' + - matplotlib ; extra == 'all' + - sympy ; extra == 'all' + - xattr ; sys_platform == 'darwin' and extra == 'all' + - skia-pathops>=0.5.0 ; extra == 'all' + - uharfbuzz>=0.23.0 ; extra == 'all' + requires_python: '>=3.9' +- conda: https://conda.anaconda.org/conda-forge/linux-64/freetype-2.14.1-ha770c72_0.conda + sha256: bf8e4dffe46f7d25dc06f31038cacb01672c47b9f45201f065b0f4d00ab0a83e + md5: 4afc585cd97ba8a23809406cd8a9eda8 + depends: + - libfreetype 2.14.1 ha770c72_0 + - libfreetype6 2.14.1 h73754d4_0 + license: GPL-2.0-only OR FTL + purls: [] + size: 173114 + timestamp: 1757945422243 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/freetype-2.14.1-h8af1aa0_0.conda + sha256: 9f8de35e95ce301cecfe01bc9d539c7cc045146ffba55efe9733ff77ad1cfb21 + md5: 0c8f36ebd3678eed1685f0fc93fc2175 + depends: + - libfreetype 2.14.1 h8af1aa0_0 + - libfreetype6 2.14.1 hdae7a39_0 + license: GPL-2.0-only OR FTL + purls: [] + size: 173174 + timestamp: 1757945489158 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/freetype-2.14.1-hce30654_0.conda + sha256: 14427aecd72e973a73d5f9dfd0e40b6bc3791d253de09b7bf233f6a9a190fd17 + md5: 1ec9a1ee7a2c9339774ad9bb6fe6caec + depends: + - libfreetype 2.14.1 hce30654_0 + - libfreetype6 2.14.1 h6da58f4_0 + license: GPL-2.0-only OR FTL + purls: [] + size: 173399 + timestamp: 1757947175403 +- conda: https://conda.anaconda.org/conda-forge/linux-64/fribidi-1.0.16-hb03c661_0.conda + sha256: 858283ff33d4c033f4971bf440cebff217d5552a5222ba994c49be990dacd40d + md5: f9f81ea472684d75b9dd8d0b328cf655 + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=14 + license: LGPL-2.1-or-later + purls: [] + size: 61244 + timestamp: 1757438574066 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/fribidi-1.0.16-he30d5cf_0.conda + sha256: 1bfcd715bcb49a0b22d5d1899a22c6ff884b06f8e141eb746f3949752469a422 + md5: f3ac54914f7d3e1d68cb8d891765e5f9 + depends: + - libgcc >=14 + license: LGPL-2.1-or-later + purls: [] + size: 62909 + timestamp: 1757438620177 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/fribidi-1.0.16-hc919400_0.conda + sha256: d856dc6744ecfba78c5f7df3378f03a75c911aadac803fa2b41a583667b4b600 + md5: 04bdce8d93a4ed181d1d726163c2d447 + depends: + - __osx >=11.0 + license: LGPL-2.1-or-later + purls: [] + size: 59391 + timestamp: 1757438897523 +- conda: https://conda.anaconda.org/conda-forge/linux-64/gdk-pixbuf-2.44.1-h2b0a6b4_0.conda + sha256: b827285fe001806beeddcc30953d2bd07869aeb0efe4581d56432c92c06b0c48 + md5: 2935d9c0526277bd42373cf23d49d51f + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=14 + - libglib >=2.86.0,<3.0a0 + - libjpeg-turbo >=3.1.0,<4.0a0 + - liblzma >=5.8.1,<6.0a0 + - libpng >=1.6.50,<1.7.0a0 + - libtiff >=4.7.0,<4.8.0a0 + license: LGPL-2.1-or-later + license_family: LGPL + purls: [] + size: 579596 + timestamp: 1757867209855 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/gdk-pixbuf-2.44.4-h90308e0_0.conda + sha256: 78a1d69c3d0da73b4d54a35001abd4e273605180d21365b4f31e9a241d9fb715 + md5: 4c8c0d2f7620467869d41f29304362dc + depends: + - libgcc >=14 + - libglib >=2.86.0,<3.0a0 + - libjpeg-turbo >=3.1.0,<4.0a0 + - liblzma >=5.8.1,<6.0a0 + - libpng >=1.6.50,<1.7.0a0 + - libtiff >=4.7.1,<4.8.0a0 + license: LGPL-2.1-or-later + license_family: LGPL + purls: [] + size: 580454 + timestamp: 1761083738779 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/gdk-pixbuf-2.44.4-h7542897_0.conda + sha256: 1164ba63360736439c6e50f2d390e93f04df86901e7711de41072a32d9b8bfc9 + md5: 0b349c0400357e701cf2fa69371e5d39 + depends: + - __osx >=11.0 + - libglib >=2.86.0,<3.0a0 + - libintl >=0.25.1,<1.0a0 + - libjpeg-turbo >=3.1.0,<4.0a0 + - liblzma >=5.8.1,<6.0a0 + - libpng >=1.6.50,<1.7.0a0 + - libtiff >=4.7.1,<4.8.0a0 + license: LGPL-2.1-or-later + license_family: LGPL + purls: [] + size: 544149 + timestamp: 1761082904334 +- conda: https://conda.anaconda.org/conda-forge/linux-64/glib-tools-2.86.0-hf516916_0.conda + sha256: b77316bd5c8680bde4e5a7ab7013c8f0f10c1702cc6c3b0fd0fac3923a31fec3 + md5: 1a8e49615381c381659de1bc6a3bf9ec + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=14 + - libglib 2.86.0 h1fed272_0 + license: LGPL-2.1-or-later + purls: [] + size: 117284 + timestamp: 1757403341964 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/glib-tools-2.86.1-hc87f4d4_1.conda + sha256: 59d89ed84223775b4354c2bc0fc51c465ee1caf53607bf7eae868b0aca4b5a9e + md5: eabd2c76bb4cbf80fd78bb5e7d8122d7 + depends: + - libgcc >=14 + - libglib 2.86.1 he84ff74_1 + license: LGPL-2.1-or-later + purls: [] + size: 126254 + timestamp: 1761874152194 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/glib-tools-2.86.1-hb9d6e3a_1.conda + sha256: 6492472d76db47d85699c895acbe6b578ee0d4a964490388e71aec8777c0e9ec + md5: 5a90e74e57c0d1e2381ce1246b0a2125 + depends: + - __osx >=11.0 + - libglib 2.86.1 he69a767_1 + - libintl >=0.25.1,<1.0a0 + license: LGPL-2.1-or-later + purls: [] + size: 101419 + timestamp: 1761875708283 +- conda: https://conda.anaconda.org/conda-forge/linux-64/graphite2-1.3.14-hecca717_2.conda + sha256: 25ba37da5c39697a77fce2c9a15e48cf0a84f1464ad2aafbe53d8357a9f6cc8c + md5: 2cd94587f3a401ae05e03a6caf09539d + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=14 + - libstdcxx >=14 + license: LGPL-2.0-or-later + license_family: LGPL + purls: [] + size: 99596 + timestamp: 1755102025473 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/graphite2-1.3.14-hfae3067_2.conda + sha256: c9b1781fe329e0b77c5addd741e58600f50bef39321cae75eba72f2f381374b7 + md5: 4aa540e9541cc9d6581ab23ff2043f13 + depends: + - libgcc >=14 + - libstdcxx >=14 + license: LGPL-2.0-or-later + license_family: LGPL + purls: [] + size: 102400 + timestamp: 1755102000043 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/graphite2-1.3.14-hec049ff_2.conda + sha256: c507ae9989dbea7024aa6feaebb16cbf271faac67ac3f0342ef1ab747c20475d + md5: 0fc46fee39e88bbcf5835f71a9d9a209 + depends: + - __osx >=11.0 + - libcxx >=19 + license: LGPL-2.0-or-later + license_family: LGPL + purls: [] + size: 81202 + timestamp: 1755102333712 +- pypi: https://files.pythonhosted.org/packages/91/4c/e0ce1ef95d4000ebc1c11801f9b944fa5910ecc15b5e351865763d8657f8/graphviz-0.21-py3-none-any.whl + name: graphviz + version: '0.21' + sha256: 54f33de9f4f911d7e84e4191749cac8cc5653f815b06738c54db9a15ab8b1e42 + requires_dist: + - build ; extra == 'dev' + - wheel ; extra == 'dev' + - twine ; extra == 'dev' + - flake8 ; extra == 'dev' + - flake8-pyproject ; extra == 'dev' + - pep8-naming ; extra == 'dev' + - tox>=3 ; extra == 'dev' + - pytest>=7,<8.1 ; extra == 'test' + - pytest-mock>=3 ; extra == 'test' + - pytest-cov ; extra == 'test' + - coverage ; extra == 'test' + - sphinx>=5,<7 ; extra == 'docs' + - sphinx-autodoc-typehints ; extra == 'docs' + - sphinx-rtd-theme>=0.2.5 ; extra == 'docs' + requires_python: '>=3.9' +- conda: https://conda.anaconda.org/conda-forge/linux-64/graphviz-13.1.2-h87b6fe6_0.conda + sha256: efbd7d483f3d79b7882515ccf229eceb7f4ff636ea2019044e98243722f428be + md5: 0adddc9b820f596638d8b0ff9e3b4823 + depends: + - __glibc >=2.17,<3.0.a0 + - adwaita-icon-theme + - cairo >=1.18.4,<2.0a0 + - fonts-conda-ecosystem + - gdk-pixbuf >=2.42.12,<3.0a0 + - gtk3 >=3.24.43,<4.0a0 + - gts >=0.7.6,<0.8.0a0 + - libexpat >=2.7.1,<3.0a0 + - libgcc >=14 + - libgd >=2.3.3,<2.4.0a0 + - libglib >=2.84.3,<3.0a0 + - librsvg >=2.58.4,<3.0a0 + - libstdcxx >=14 + - libwebp-base >=1.6.0,<2.0a0 + - libzlib >=1.3.1,<2.0a0 + - pango >=1.56.4,<2.0a0 + license: EPL-1.0 + license_family: Other + purls: [] + size: 2427887 + timestamp: 1754732581595 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/graphviz-13.1.2-hdb06ba2_0.conda + sha256: 15f0f8bc5b5fc1c51be13f0dd4e2dcfb4cd6555e75b18656d51def0d8b7e4db2 + md5: 52fc4ad5de8b211077edfa9e657f6cab + depends: + - adwaita-icon-theme + - cairo >=1.18.4,<2.0a0 + - fonts-conda-ecosystem + - gdk-pixbuf >=2.42.12,<3.0a0 + - gtk3 >=3.24.43,<4.0a0 + - gts >=0.7.6,<0.8.0a0 + - libexpat >=2.7.1,<3.0a0 + - libgcc >=14 + - libgd >=2.3.3,<2.4.0a0 + - libglib >=2.84.3,<3.0a0 + - librsvg >=2.58.4,<3.0a0 + - libstdcxx >=14 + - libwebp-base >=1.6.0,<2.0a0 + - libzlib >=1.3.1,<2.0a0 + - pango >=1.56.4,<2.0a0 + license: EPL-1.0 + license_family: Other + purls: [] + size: 2557826 + timestamp: 1754732391605 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/graphviz-13.1.2-hcd33d8b_0.conda + sha256: f25e1828d02ebd78214966f483cfca5ac6a7b18824369c748d8cda99c66ff588 + md5: 81ab85a5a8481667660c7ce6e84bd681 + depends: + - __osx >=11.0 + - adwaita-icon-theme + - cairo >=1.18.4,<2.0a0 + - fonts-conda-ecosystem + - gdk-pixbuf >=2.42.12,<3.0a0 + - gtk3 >=3.24.43,<4.0a0 + - gts >=0.7.6,<0.8.0a0 + - libcxx >=19 + - libexpat >=2.7.1,<3.0a0 + - libgd >=2.3.3,<2.4.0a0 + - libglib >=2.84.3,<3.0a0 + - librsvg >=2.58.4,<3.0a0 + - libwebp-base >=1.6.0,<2.0a0 + - libzlib >=1.3.1,<2.0a0 + - pango >=1.56.4,<2.0a0 + license: EPL-1.0 + license_family: Other + purls: [] + size: 2201370 + timestamp: 1754732518951 +- conda: https://conda.anaconda.org/conda-forge/linux-64/gtk3-3.24.43-h0c6a113_5.conda + sha256: d36263cbcbce34ec463ce92bd72efa198b55d987959eab6210cc256a0e79573b + md5: 67d00e9cfe751cfe581726c5eff7c184 + depends: + - __glibc >=2.17,<3.0.a0 + - at-spi2-atk >=2.38.0,<3.0a0 + - atk-1.0 >=2.38.0 + - cairo >=1.18.4,<2.0a0 + - epoxy >=1.5.10,<1.6.0a0 + - fontconfig >=2.15.0,<3.0a0 + - fonts-conda-ecosystem + - fribidi >=1.0.10,<2.0a0 + - gdk-pixbuf >=2.42.12,<3.0a0 + - glib-tools + - harfbuzz >=11.0.0,<12.0a0 + - hicolor-icon-theme + - libcups >=2.3.3,<2.4.0a0 + - libcups >=2.3.3,<3.0a0 + - libexpat >=2.6.4,<3.0a0 + - libgcc >=13 + - libglib >=2.84.0,<3.0a0 + - liblzma >=5.6.4,<6.0a0 + - libxkbcommon >=1.8.1,<2.0a0 + - libzlib >=1.3.1,<2.0a0 + - pango >=1.56.3,<2.0a0 + - wayland >=1.23.1,<2.0a0 + - xorg-libx11 >=1.8.12,<2.0a0 + - xorg-libxcomposite >=0.4.6,<1.0a0 + - xorg-libxcursor >=1.2.3,<2.0a0 + - xorg-libxdamage >=1.1.6,<2.0a0 + - xorg-libxext >=1.3.6,<2.0a0 + - xorg-libxfixes >=6.0.1,<7.0a0 + - xorg-libxi >=1.8.2,<2.0a0 + - xorg-libxinerama >=1.1.5,<1.2.0a0 + - xorg-libxrandr >=1.5.4,<2.0a0 + - xorg-libxrender >=0.9.12,<0.10.0a0 + license: LGPL-2.0-or-later + license_family: LGPL + purls: [] + size: 5585389 + timestamp: 1743405684985 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/gtk3-3.24.43-h4cd1324_6.conda + sha256: 5b8c5255d88d97083095790765dfafda6ce99daa8dcaaa8c0b668e82c5b73187 + md5: 124842a6e0b59cbd121233346bd56e33 + depends: + - at-spi2-atk >=2.38.0,<3.0a0 + - atk-1.0 >=2.38.0 + - cairo >=1.18.4,<2.0a0 + - epoxy >=1.5.10,<1.6.0a0 + - fontconfig >=2.15.0,<3.0a0 + - fonts-conda-ecosystem + - fribidi >=1.0.16,<2.0a0 + - gdk-pixbuf >=2.44.4,<3.0a0 + - glib-tools + - harfbuzz >=11.5.1 + - hicolor-icon-theme + - libcups >=2.3.3,<2.4.0a0 + - libcups >=2.3.3,<3.0a0 + - libexpat >=2.7.1,<3.0a0 + - libgcc >=14 + - libglib >=2.86.0,<3.0a0 + - liblzma >=5.8.1,<6.0a0 + - libxkbcommon >=1.12.2,<2.0a0 + - libzlib >=1.3.1,<2.0a0 + - pango >=1.56.4,<2.0a0 + - wayland >=1.24.0,<2.0a0 + - xorg-libx11 >=1.8.12,<2.0a0 + - xorg-libxcomposite >=0.4.6,<1.0a0 + - xorg-libxcursor >=1.2.3,<2.0a0 + - xorg-libxdamage >=1.1.6,<2.0a0 + - xorg-libxext >=1.3.6,<2.0a0 + - xorg-libxfixes >=6.0.2,<7.0a0 + - xorg-libxi >=1.8.2,<2.0a0 + - xorg-libxinerama >=1.1.5,<1.2.0a0 + - xorg-libxrandr >=1.5.4,<2.0a0 + - xorg-libxrender >=0.9.12,<0.10.0a0 + license: LGPL-2.0-or-later + license_family: LGPL + purls: [] + size: 5660172 + timestamp: 1761334356772 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/gtk3-3.24.43-h5febe37_6.conda + sha256: bd66a3325bf3ce63ada3bf12eaafcfe036698741ee4bb595e83e5fdd3dba9f3d + md5: a99f96906158ebae5e3c0904bcd45145 + depends: + - __osx >=11.0 + - atk-1.0 >=2.38.0 + - cairo >=1.18.4,<2.0a0 + - epoxy >=1.5.10,<1.6.0a0 + - fribidi >=1.0.16,<2.0a0 + - gdk-pixbuf >=2.44.4,<3.0a0 + - glib-tools + - harfbuzz >=11.5.1 + - hicolor-icon-theme + - libexpat >=2.7.1,<3.0a0 + - libglib >=2.86.0,<3.0a0 + - libintl >=0.25.1,<1.0a0 + - liblzma >=5.8.1,<6.0a0 + - libzlib >=1.3.1,<2.0a0 + - pango >=1.56.4,<2.0a0 + license: LGPL-2.0-or-later + license_family: LGPL + purls: [] + size: 4768791 + timestamp: 1761328318680 +- conda: https://conda.anaconda.org/conda-forge/linux-64/gts-0.7.6-h977cf35_4.conda + sha256: b5cd16262fefb836f69dc26d879b6508d29f8a5c5948a966c47fe99e2e19c99b + md5: 4d8df0b0db060d33c9a702ada998a8fe + depends: + - libgcc-ng >=12 + - libglib >=2.76.3,<3.0a0 + - libstdcxx-ng >=12 + license: LGPL-2.0-or-later + license_family: LGPL + purls: [] + size: 318312 + timestamp: 1686545244763 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/gts-0.7.6-he293c15_4.conda + sha256: 1e9cc30d1c746d5a3399a279f5f642a953f37d9f9c82fd4d55b301e9c2a23f7c + md5: 2aeaeddbd89e84b60165463225814cfc + depends: + - libgcc-ng >=12 + - libglib >=2.76.3,<3.0a0 + - libstdcxx-ng >=12 + license: LGPL-2.0-or-later + license_family: LGPL + purls: [] + size: 332673 + timestamp: 1686545222091 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/gts-0.7.6-he42f4ea_4.conda + sha256: e0f8c7bc1b9ea62ded78ffa848e37771eeaaaf55b3146580513c7266862043ba + md5: 21b4dd3098f63a74cf2aa9159cbef57d + depends: + - libcxx >=15.0.7 + - libglib >=2.76.3,<3.0a0 + license: LGPL-2.0-or-later + license_family: LGPL + purls: [] + size: 304331 + timestamp: 1686545503242 +- conda: https://conda.anaconda.org/conda-forge/linux-64/harfbuzz-11.5.0-h15599e2_0.conda + sha256: 04d33cef3345ce6e3fbbfb5539ebc8a3730026ea94ce6ace1f8f8d3551fa079c + md5: 47599428437d622bfee24fbd06a2d0b4 + depends: + - __glibc >=2.17,<3.0.a0 + - cairo >=1.18.4,<2.0a0 + - graphite2 >=1.3.14,<2.0a0 + - icu >=75.1,<76.0a0 + - libexpat >=2.7.1,<3.0a0 + - libfreetype >=2.14.0 + - libfreetype6 >=2.14.0 + - libgcc >=14 + - libglib >=2.86.0,<3.0a0 + - libstdcxx >=14 + - libzlib >=1.3.1,<2.0a0 + license: MIT + purls: [] + size: 2048134 + timestamp: 1757867460348 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/harfbuzz-12.2.0-he4899c9_0.conda + sha256: 5cfd74a3fbce0921af5beff93a3fe7edc5b1344d9b9668b2de1c1be932b54993 + md5: 1437bf9690976948f90175a65407b65f + depends: + - cairo >=1.18.4,<2.0a0 + - graphite2 >=1.3.14,<2.0a0 + - icu >=75.1,<76.0a0 + - libexpat >=2.7.1,<3.0a0 + - libfreetype >=2.14.1 + - libfreetype6 >=2.14.1 + - libgcc >=14 + - libglib >=2.86.1,<3.0a0 + - libstdcxx >=14 + - libzlib >=1.3.1,<2.0a0 + license: MIT + license_family: MIT + purls: [] + size: 2156041 + timestamp: 1762376447693 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/harfbuzz-12.1.0-haf38c7b_0.conda + sha256: 8f2fac3e74608af55334ab9e77e9db9112c9078858aa938d191481d873a902d3 + md5: 3fd0b257d246ddedd1f1496e5246958d + depends: + - __osx >=11.0 + - cairo >=1.18.4,<2.0a0 + - graphite2 >=1.3.14,<2.0a0 + - icu >=75.1,<76.0a0 + - libcxx >=19 + - libexpat >=2.7.1,<3.0a0 + - libfreetype >=2.14.1 + - libfreetype6 >=2.14.1 + - libglib >=2.86.0,<3.0a0 + - libzlib >=1.3.1,<2.0a0 + license: MIT + license_family: MIT + purls: [] + size: 1548996 + timestamp: 1759366687572 +- conda: https://conda.anaconda.org/conda-forge/linux-64/hicolor-icon-theme-0.17-ha770c72_2.tar.bz2 + sha256: 336f29ceea9594f15cc8ec4c45fdc29e10796573c697ee0d57ebb7edd7e92043 + md5: bbf6f174dcd3254e19a2f5d2295ce808 + license: GPL-2.0-or-later + license_family: GPL + purls: [] + size: 13841 + timestamp: 1605162808667 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/hicolor-icon-theme-0.17-h8af1aa0_2.tar.bz2 + sha256: 479a0f95cf3e7d7db795fb7a14337cab73c2c926a5599c8512a3e8f8466f9e54 + md5: 331add9f855e921695d7b569aa23d5ec + license: GPL-2.0-or-later + license_family: GPL + purls: [] + size: 13896 + timestamp: 1605162856037 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/hicolor-icon-theme-0.17-hce30654_2.tar.bz2 + sha256: 286e33fb452f61133a3a61d002890235d1d1378554218ab063d6870416440281 + md5: 237b05b7eb284d7eebc3c5d93f5e4bca + license: GPL-2.0-or-later + license_family: GPL + purls: [] + size: 13800 + timestamp: 1611053664863 +- conda: https://conda.anaconda.org/conda-forge/linux-64/icu-75.1-he02047a_0.conda + sha256: 71e750d509f5fa3421087ba88ef9a7b9be11c53174af3aa4d06aff4c18b38e8e + md5: 8b189310083baabfb622af68fd9d3ae3 + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc-ng >=12 + - libstdcxx-ng >=12 + license: MIT + license_family: MIT + purls: [] + size: 12129203 + timestamp: 1720853576813 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/icu-75.1-hf9b3779_0.conda + sha256: 813298f2e54ef087dbfc9cc2e56e08ded41de65cff34c639cc8ba4e27e4540c9 + md5: 268203e8b983fddb6412b36f2024e75c + depends: + - libgcc-ng >=12 + - libstdcxx-ng >=12 + license: MIT + license_family: MIT + purls: [] + size: 12282786 + timestamp: 1720853454991 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/icu-75.1-hfee45f7_0.conda + sha256: 9ba12c93406f3df5ab0a43db8a4b4ef67a5871dfd401010fbe29b218b2cbe620 + md5: 5eb22c1d7b3fc4abb50d92d621583137 + depends: + - __osx >=11.0 + license: MIT + license_family: MIT + purls: [] + size: 11857802 + timestamp: 1720853997952 +- pypi: https://files.pythonhosted.org/packages/e5/ae/2ad30f4652712c82f1c23423d79136fbce338932ad166d70c1efb86a5998/identify-2.6.14-py2.py3-none-any.whl + name: identify + version: 2.6.14 + sha256: 11a073da82212c6646b1f39bb20d4483bfb9543bd5566fec60053c4bb309bf2e + requires_dist: + - ukkonen ; extra == 'license' + requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/0f/1c/e5fd8f973d4f375adb21565739498e2e9a1e54c858a97b9a8ccfdc81da9b/identify-2.6.15-py2.py3-none-any.whl + name: identify + version: 2.6.15 + sha256: 1181ef7608e00704db228516541eb83a88a9f94433a8c80bb9b5bd54b1d81757 + requires_dist: + - ukkonen ; extra == 'license' + requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/76/c6/c88e154df9c4e1a2a66ccf0005a88dfb2650c1dffb6f5ce603dfbd452ce3/idna-3.10-py3-none-any.whl + name: idna + version: '3.10' + sha256: 946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3 + requires_dist: + - ruff>=0.6.2 ; extra == 'all' + - mypy>=1.11.2 ; extra == 'all' + - pytest>=8.3.2 ; extra == 'all' + - flake8>=7.1.1 ; extra == 'all' + requires_python: '>=3.6' +- pypi: https://files.pythonhosted.org/packages/0e/61/66938bbb5fc52dbdf84594873d5b51fb1f7c7794e9c0f5bd885f30bc507b/idna-3.11-py3-none-any.whl + name: idna + version: '3.11' + sha256: 771a87f49d9defaf64091e6e6fe9c18d4833f140bd19464795bc32d966ca37ea + requires_dist: + - ruff>=0.6.2 ; extra == 'all' + - mypy>=1.11.2 ; extra == 'all' + - pytest>=8.3.2 ; extra == 'all' + - flake8>=7.1.1 ; extra == 'all' + requires_python: '>=3.8' +- pypi: https://files.pythonhosted.org/packages/2c/e1/e6716421ea10d38022b952c159d5161ca1193197fb744506875fbb87ea7b/iniconfig-2.1.0-py3-none-any.whl + name: iniconfig + version: 2.1.0 + sha256: 9deba5723312380e77435581c6bf4935c94cbfab9b1ed33ef8d238ea168eb760 + requires_python: '>=3.8' +- pypi: https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl + name: iniconfig + version: 2.3.0 + sha256: f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12 + requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/08/2a/5628a99d04acb2d2f2e749cdf4ea571d2575e898df0528a090948018b726/ipython-9.5.0-py3-none-any.whl + name: ipython + version: 9.5.0 + sha256: 88369ffa1d5817d609120daa523a6da06d02518e582347c29f8451732a9c5e72 + requires_dist: + - colorama ; sys_platform == 'win32' + - decorator + - ipython-pygments-lexers + - jedi>=0.16 + - matplotlib-inline + - pexpect>4.3 ; sys_platform != 'emscripten' and sys_platform != 'win32' + - prompt-toolkit>=3.0.41,<3.1.0 + - pygments>=2.4.0 + - stack-data + - traitlets>=5.13.0 + - typing-extensions>=4.6 ; python_full_version < '3.12' + - black ; extra == 'black' + - docrepr ; extra == 'doc' + - exceptiongroup ; extra == 'doc' + - intersphinx-registry ; extra == 'doc' + - ipykernel ; extra == 'doc' + - ipython[test] ; extra == 'doc' + - matplotlib ; extra == 'doc' + - setuptools>=18.5 ; extra == 'doc' + - sphinx-toml==0.0.4 ; extra == 'doc' + - sphinx-rtd-theme ; extra == 'doc' + - sphinx>=1.3 ; extra == 'doc' + - typing-extensions ; extra == 'doc' + - pytest ; extra == 'test' + - pytest-asyncio ; extra == 'test' + - testpath ; extra == 'test' + - packaging ; extra == 'test' + - ipython[test] ; extra == 'test-extra' + - curio ; extra == 'test-extra' + - jupyter-ai ; extra == 'test-extra' + - matplotlib!=3.2.0 ; extra == 'test-extra' + - nbformat ; extra == 'test-extra' + - nbclient ; extra == 'test-extra' + - ipykernel ; extra == 'test-extra' + - numpy>=1.23 ; extra == 'test-extra' + - pandas ; extra == 'test-extra' + - trio ; extra == 'test-extra' + - matplotlib ; extra == 'matplotlib' + - ipython[doc,matplotlib,test,test-extra] ; extra == 'all' + requires_python: '>=3.11' +- pypi: https://files.pythonhosted.org/packages/48/c5/d5e07995077e48220269c28a221e168c91123ad5ceee44d548f54a057fc0/ipython-9.6.0-py3-none-any.whl + name: ipython + version: 9.6.0 + sha256: 5f77efafc886d2f023442479b8149e7d86547ad0a979e9da9f045d252f648196 + requires_dist: + - colorama ; sys_platform == 'win32' + - decorator + - ipython-pygments-lexers + - jedi>=0.16 + - matplotlib-inline + - pexpect>4.3 ; sys_platform != 'emscripten' and sys_platform != 'win32' + - prompt-toolkit>=3.0.41,<3.1.0 + - pygments>=2.4.0 + - stack-data + - traitlets>=5.13.0 + - typing-extensions>=4.6 ; python_full_version < '3.12' + - black ; extra == 'black' + - docrepr ; extra == 'doc' + - exceptiongroup ; extra == 'doc' + - intersphinx-registry ; extra == 'doc' + - ipykernel ; extra == 'doc' + - ipython[matplotlib,test] ; extra == 'doc' + - setuptools>=61.2 ; extra == 'doc' + - sphinx-toml==0.0.4 ; extra == 'doc' + - sphinx-rtd-theme ; extra == 'doc' + - sphinx>=1.3 ; extra == 'doc' + - typing-extensions ; extra == 'doc' + - pytest ; extra == 'test' + - pytest-asyncio ; extra == 'test' + - testpath ; extra == 'test' + - packaging ; extra == 'test' + - ipython[test] ; extra == 'test-extra' + - curio ; extra == 'test-extra' + - jupyter-ai ; extra == 'test-extra' + - ipython[matplotlib] ; extra == 'test-extra' + - nbformat ; extra == 'test-extra' + - nbclient ; extra == 'test-extra' + - ipykernel ; extra == 'test-extra' + - numpy>=1.25 ; extra == 'test-extra' + - pandas>2.0 ; extra == 'test-extra' + - trio ; extra == 'test-extra' + - matplotlib>3.7 ; extra == 'matplotlib' + - ipython[doc,matplotlib,test,test-extra] ; extra == 'all' + requires_python: '>=3.11' +- pypi: https://files.pythonhosted.org/packages/05/aa/62893d6a591d337aa59dcc4c6f6c842f1fe20cd72c8c5c1f980255243252/ipython-9.7.0-py3-none-any.whl + name: ipython + version: 9.7.0 + sha256: bce8ac85eb9521adc94e1845b4c03d88365fd6ac2f4908ec4ed1eb1b0a065f9f + requires_dist: + - colorama>=0.4.4 ; sys_platform == 'win32' + - decorator>=4.3.2 + - ipython-pygments-lexers>=1.0.0 + - jedi>=0.18.1 + - matplotlib-inline>=0.1.5 + - pexpect>4.3 ; sys_platform != 'emscripten' and sys_platform != 'win32' + - prompt-toolkit>=3.0.41,<3.1.0 + - pygments>=2.11.0 + - stack-data>=0.6.0 + - traitlets>=5.13.0 + - typing-extensions>=4.6 ; python_full_version < '3.12' + - black ; extra == 'black' + - docrepr ; extra == 'doc' + - exceptiongroup ; extra == 'doc' + - intersphinx-registry ; extra == 'doc' + - ipykernel ; extra == 'doc' + - ipython[matplotlib,test] ; extra == 'doc' + - setuptools>=70.0 ; extra == 'doc' + - sphinx-toml==0.0.4 ; extra == 'doc' + - sphinx-rtd-theme>=0.1.8 ; extra == 'doc' + - sphinx>=8.0 ; extra == 'doc' + - typing-extensions ; extra == 'doc' + - pytest>=7.0.0 ; extra == 'test' + - pytest-asyncio>=1.0.0 ; extra == 'test' + - testpath>=0.2 ; extra == 'test' + - packaging>=20.1.0 ; extra == 'test' + - setuptools>=61.2 ; extra == 'test' + - ipython[test] ; extra == 'test-extra' + - curio ; extra == 'test-extra' + - jupyter-ai ; extra == 'test-extra' + - ipython[matplotlib] ; extra == 'test-extra' + - nbformat ; extra == 'test-extra' + - nbclient ; extra == 'test-extra' + - ipykernel>6.30 ; extra == 'test-extra' + - numpy>=1.27 ; extra == 'test-extra' + - pandas>2.1 ; extra == 'test-extra' + - trio>=0.1.0 ; extra == 'test-extra' + - matplotlib>3.9 ; extra == 'matplotlib' + - ipython[doc,matplotlib,test,test-extra] ; extra == 'all' + requires_python: '>=3.11' +- pypi: https://files.pythonhosted.org/packages/d9/33/1f075bf72b0b747cb3288d011319aaf64083cf2efef8354174e3ed4540e2/ipython_pygments_lexers-1.1.1-py3-none-any.whl + name: ipython-pygments-lexers + version: 1.1.1 + sha256: a9462224a505ade19a605f71f8fa63c2048833ce50abc86768a0d81d876dc81c + requires_dist: + - pygments + requires_python: '>=3.8' +- pypi: https://files.pythonhosted.org/packages/c0/5a/9cac0c82afec3d09ccd97c8b6502d48f165f9124db81b4bcb90b4af974ee/jedi-0.19.2-py2.py3-none-any.whl + name: jedi + version: 0.19.2 + sha256: a8ef22bde8490f57fe5c7681a3c83cb58874daf72b4784de3cce5b6ef6edb5b9 + requires_dist: + - parso>=0.8.4,<0.9.0 + - jinja2==2.11.3 ; extra == 'docs' + - markupsafe==1.1.1 ; extra == 'docs' + - pygments==2.8.1 ; extra == 'docs' + - alabaster==0.7.12 ; extra == 'docs' + - babel==2.9.1 ; extra == 'docs' + - chardet==4.0.0 ; extra == 'docs' + - commonmark==0.8.1 ; extra == 'docs' + - docutils==0.17.1 ; extra == 'docs' + - future==0.18.2 ; extra == 'docs' + - idna==2.10 ; extra == 'docs' + - imagesize==1.2.0 ; extra == 'docs' + - mock==1.0.1 ; extra == 'docs' + - packaging==20.9 ; extra == 'docs' + - pyparsing==2.4.7 ; extra == 'docs' + - pytz==2021.1 ; extra == 'docs' + - readthedocs-sphinx-ext==2.1.4 ; extra == 'docs' + - recommonmark==0.5.0 ; extra == 'docs' + - requests==2.25.1 ; extra == 'docs' + - six==1.15.0 ; extra == 'docs' + - snowballstemmer==2.1.0 ; extra == 'docs' + - sphinx-rtd-theme==0.4.3 ; extra == 'docs' + - sphinx==1.8.5 ; extra == 'docs' + - sphinxcontrib-serializinghtml==1.1.4 ; extra == 'docs' + - sphinxcontrib-websupport==1.2.4 ; extra == 'docs' + - urllib3==1.26.4 ; extra == 'docs' + - flake8==5.0.4 ; extra == 'qa' + - mypy==0.971 ; extra == 'qa' + - types-setuptools==67.2.0.1 ; extra == 'qa' + - django ; extra == 'testing' + - attrs ; extra == 'testing' + - colorama ; extra == 'testing' + - docopt ; extra == 'testing' + - pytest<9.0.0 ; extra == 'testing' + requires_python: '>=3.6' +- conda: https://conda.anaconda.org/conda-forge/linux-64/keyutils-1.6.3-hb9d3cd8_0.conda + sha256: 0960d06048a7185d3542d850986d807c6e37ca2e644342dd0c72feefcf26c2a4 + md5: b38117a3c920364aff79f870c984b4a3 + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=13 + license: LGPL-2.1-or-later + purls: [] + size: 134088 + timestamp: 1754905959823 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/keyutils-1.6.3-h86ecc28_0.conda + sha256: 5ce830ca274b67de11a7075430a72020c1fb7d486161a82839be15c2b84e9988 + md5: e7df0aab10b9cbb73ab2a467ebfaf8c7 + depends: + - libgcc >=13 + license: LGPL-2.1-or-later + purls: [] + size: 129048 + timestamp: 1754906002667 +- pypi: https://files.pythonhosted.org/packages/2d/7a/9d90a151f558e29c3936b8a47ac770235f436f2120aca41a6d5f3d62ae8d/kiwisolver-1.4.9-cp313-cp313-macosx_11_0_arm64.whl + name: kiwisolver + version: 1.4.9 + sha256: 1a12cf6398e8a0a001a059747a1cbf24705e18fe413bc22de7b3d15c67cffe3f + requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/d9/28/aac26d4c882f14de59041636292bc838db8961373825df23b8eeb807e198/kiwisolver-1.4.9-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl + name: kiwisolver + version: 1.4.9 + sha256: 5656aa670507437af0207645273ccdfee4f14bacd7f7c67a4306d0dcaeaf6eed + requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/e9/e9/f218a2cb3a9ffbe324ca29a9e399fa2d2866d7f348ec3a88df87fc248fc5/kiwisolver-1.4.9-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl + name: kiwisolver + version: 1.4.9 + sha256: b67e6efbf68e077dd71d1a6b37e43e1a99d0bff1a3d51867d45ee8908b931098 + requires_python: '>=3.10' +- conda: https://conda.anaconda.org/conda-forge/linux-64/krb5-1.21.3-h659f571_0.conda + sha256: 99df692f7a8a5c27cd14b5fb1374ee55e756631b9c3d659ed3ee60830249b238 + md5: 3f43953b7d3fb3aaa1d0d0723d91e368 + depends: + - keyutils >=1.6.1,<2.0a0 + - libedit >=3.1.20191231,<3.2.0a0 + - libedit >=3.1.20191231,<4.0a0 + - libgcc-ng >=12 + - libstdcxx-ng >=12 + - openssl >=3.3.1,<4.0a0 + license: MIT + license_family: MIT + purls: [] + size: 1370023 + timestamp: 1719463201255 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/krb5-1.21.3-h50a48e9_0.conda + sha256: 0ec272afcf7ea7fbf007e07a3b4678384b7da4047348107b2ae02630a570a815 + md5: 29c10432a2ca1472b53f299ffb2ffa37 + depends: + - keyutils >=1.6.1,<2.0a0 + - libedit >=3.1.20191231,<3.2.0a0 + - libedit >=3.1.20191231,<4.0a0 + - libgcc-ng >=12 + - libstdcxx-ng >=12 + - openssl >=3.3.1,<4.0a0 + license: MIT + license_family: MIT + purls: [] + size: 1474620 + timestamp: 1719463205834 +- conda: https://conda.anaconda.org/conda-forge/linux-64/ld_impl_linux-64-2.44-h1423503_1.conda + sha256: 1a620f27d79217c1295049ba214c2f80372062fd251b569e9873d4a953d27554 + md5: 0be7c6e070c19105f966d3758448d018 + depends: + - __glibc >=2.17,<3.0.a0 + constrains: + - binutils_impl_linux-64 2.44 + license: GPL-3.0-only + license_family: GPL + purls: [] + size: 676044 + timestamp: 1752032747103 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/ld_impl_linux-aarch64-2.44-hd32f0e1_5.conda + sha256: cc03f3e2d5d48f1193a2d0822971b085d583327d6e20f2a5cf7d030ffdb35f9a + md5: 7c87c0b72575b30626a6dc5b49229f0c + depends: + - zstd >=1.5.7,<1.6.0a0 + constrains: + - binutils_impl_linux-aarch64 2.44 + license: GPL-3.0-only + purls: [] + size: 782949 + timestamp: 1762674873740 +- conda: https://conda.anaconda.org/conda-forge/linux-64/lerc-4.0.0-h0aef613_1.conda + sha256: 412381a43d5ff9bbed82cd52a0bbca5b90623f62e41007c9c42d3870c60945ff + md5: 9344155d33912347b37f0ae6c410a835 + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=13 + - libstdcxx >=13 + license: Apache-2.0 + license_family: Apache + purls: [] + size: 264243 + timestamp: 1745264221534 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/lerc-4.0.0-hfdc4d58_1.conda + sha256: f01df5bbf97783fac9b89be602b4d02f94353f5221acfd80c424ec1c9a8d276c + md5: 60dceb7e876f4d74a9cbd42bbbc6b9cf + depends: + - libgcc >=13 + - libstdcxx >=13 + license: Apache-2.0 + license_family: Apache + purls: [] + size: 227184 + timestamp: 1745265544057 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/lerc-4.0.0-hd64df32_1.conda + sha256: 12361697f8ffc9968907d1a7b5830e34c670e4a59b638117a2cdfed8f63a38f8 + md5: a74332d9b60b62905e3d30709df08bf1 + depends: + - __osx >=11.0 + - libcxx >=18 + license: Apache-2.0 + license_family: Apache + purls: [] + size: 188306 + timestamp: 1745264362794 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libcups-2.3.3-hb8b1518_5.conda + sha256: cb83980c57e311783ee831832eb2c20ecb41e7dee6e86e8b70b8cef0e43eab55 + md5: d4a250da4737ee127fb1fa6452a9002e + depends: + - __glibc >=2.17,<3.0.a0 + - krb5 >=1.21.3,<1.22.0a0 + - libgcc >=13 + - libstdcxx >=13 + - libzlib >=1.3.1,<2.0a0 + license: Apache-2.0 + license_family: Apache + purls: [] + size: 4523621 + timestamp: 1749905341688 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libcups-2.3.3-h5cdc715_5.conda + sha256: f3282d27be35e5d29b5b798e5136427ec798916ee6374499be7b7682c8582b72 + md5: ac0333d338076ef19170938bbaf97582 + depends: + - krb5 >=1.21.3,<1.22.0a0 + - libgcc >=13 + - libstdcxx >=13 + - libzlib >=1.3.1,<2.0a0 + license: Apache-2.0 + license_family: Apache + purls: [] + size: 4550533 + timestamp: 1749906839681 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libcxx-21.1.4-hf598326_2.conda + sha256: 0a0765cc8b6000e7f7be879c12825583d046ef22ab95efc7c5f8622e4b3302d5 + md5: 4346830dcc0c0e930328fddb0b829f63 + depends: + - __osx >=11.0 + license: Apache-2.0 WITH LLVM-exception + license_family: Apache + purls: [] + size: 568742 + timestamp: 1761852287381 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libdeflate-1.24-h86f0d12_0.conda + sha256: 8420748ea1cc5f18ecc5068b4f24c7a023cc9b20971c99c824ba10641fb95ddf + md5: 64f0c503da58ec25ebd359e4d990afa8 + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=13 + license: MIT + license_family: MIT + purls: [] + size: 72573 + timestamp: 1747040452262 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libdeflate-1.25-h1af38f5_0.conda + sha256: 48814b73bd462da6eed2e697e30c060ae16af21e9fbed30d64feaf0aad9da392 + md5: a9138815598fe6b91a1d6782ca657b0c + depends: + - libgcc >=14 + license: MIT + license_family: MIT + purls: [] + size: 71117 + timestamp: 1761979776756 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libdeflate-1.24-h5773f1b_0.conda + sha256: 417d52b19c679e1881cce3f01cad3a2d542098fa2d6df5485aac40f01aede4d1 + md5: 3baf58a5a87e7c2f4d243ce2f8f2fe5c + depends: + - __osx >=11.0 + license: MIT + license_family: MIT + purls: [] + size: 54790 + timestamp: 1747040549847 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libdrm-2.4.125-he30d5cf_1.conda + sha256: 4e6cdb5dd37db794b88bec714b4418a0435b04d14e9f7afc8cc32f2a3ced12f2 + md5: 2079727b538f6dd16f3fa579d4c3c53f + depends: + - libgcc >=14 + - libpciaccess >=0.18,<0.19.0a0 + license: MIT + license_family: MIT + purls: [] + size: 344548 + timestamp: 1757212128414 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libedit-3.1.20250104-pl5321h7949ede_0.conda + sha256: d789471216e7aba3c184cd054ed61ce3f6dac6f87a50ec69291b9297f8c18724 + md5: c277e0a4d549b03ac1e9d6cbbe3d017b + depends: + - ncurses + - __glibc >=2.17,<3.0.a0 + - libgcc >=13 + - ncurses >=6.5,<7.0a0 + license: BSD-2-Clause + license_family: BSD + purls: [] + size: 134676 + timestamp: 1738479519902 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libedit-3.1.20250104-pl5321h976ea20_0.conda + sha256: c0b27546aa3a23d47919226b3a1635fccdb4f24b94e72e206a751b33f46fd8d6 + md5: fb640d776fc92b682a14e001980825b1 + depends: + - ncurses + - libgcc >=13 + - ncurses >=6.5,<7.0a0 + license: BSD-2-Clause + license_family: BSD + purls: [] + size: 148125 + timestamp: 1738479808948 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libegl-1.7.0-hd24410f_2.conda + sha256: 8962abf38a58c235611ce356b9899f6caeb0352a8bce631b0bcc59352fda455e + md5: cf105bce884e4ef8c8ccdca9fe6695e7 + depends: + - libglvnd 1.7.0 hd24410f_2 + license: LicenseRef-libglvnd + purls: [] + size: 53551 + timestamp: 1731330990477 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libegl-devel-1.7.0-hd24410f_2.conda + sha256: 9c8e9d2289316741d037f0c5003de42488780d181453543f75497dd5a4891c7c + md5: cd8877e3833ba1bfac2fbaa5ae72c226 + depends: + - libegl 1.7.0 hd24410f_2 + - libgl-devel 1.7.0 hd24410f_2 + - xorg-libx11 + license: LicenseRef-libglvnd + purls: [] + size: 30397 + timestamp: 1731331017398 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libexpat-2.7.1-hecca717_0.conda + sha256: da2080da8f0288b95dd86765c801c6e166c4619b910b11f9a8446fb852438dc2 + md5: 4211416ecba1866fab0c6470986c22d6 + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=14 + constrains: + - expat 2.7.1.* + license: MIT + license_family: MIT + purls: [] + size: 74811 + timestamp: 1752719572741 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libexpat-2.7.1-hfae3067_0.conda + sha256: 378cabff44ea83ce4d9f9c59f47faa8d822561d39166608b3e65d1e06c927415 + md5: f75d19f3755461db2eb69401f5514f4c + depends: + - libgcc >=14 + constrains: + - expat 2.7.1.* + license: MIT + license_family: MIT + purls: [] + size: 74309 + timestamp: 1752719762749 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libexpat-2.7.1-hec049ff_0.conda + sha256: 8fbb17a56f51e7113ed511c5787e0dec0d4b10ef9df921c4fd1cccca0458f648 + md5: b1ca5f21335782f71a8bd69bdc093f67 + depends: + - __osx >=11.0 + constrains: + - expat 2.7.1.* + license: MIT + license_family: MIT + purls: [] + size: 65971 + timestamp: 1752719657566 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libffi-3.4.6-h2dba641_1.conda + sha256: 764432d32db45466e87f10621db5b74363a9f847d2b8b1f9743746cd160f06ab + md5: ede4673863426c0883c0063d853bbd85 + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=13 + license: MIT + license_family: MIT + purls: [] + size: 57433 + timestamp: 1743434498161 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libffi-3.5.2-hd65408f_0.conda + sha256: 6c3332e78a975e092e54f87771611db81dcb5515a3847a3641021621de76caea + md5: 0c5ad486dcfb188885e3cf8ba209b97b + depends: + - libgcc >=14 + license: MIT + license_family: MIT + purls: [] + size: 55586 + timestamp: 1760295405021 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libffi-3.5.2-he5f378a_0.conda + sha256: 9b8acdf42df61b7bfe8bdc545c016c29e61985e79748c64ad66df47dbc2e295f + md5: 411ff7cd5d1472bba0f55c0faf04453b + depends: + - __osx >=11.0 + license: MIT + license_family: MIT + purls: [] + size: 40251 + timestamp: 1760295839166 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libfreetype-2.14.1-ha770c72_0.conda + sha256: 4641d37faeb97cf8a121efafd6afd040904d4bca8c46798122f417c31d5dfbec + md5: f4084e4e6577797150f9b04a4560ceb0 + depends: + - libfreetype6 >=2.14.1 + license: GPL-2.0-only OR FTL + purls: [] + size: 7664 + timestamp: 1757945417134 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libfreetype-2.14.1-h8af1aa0_0.conda + sha256: 342c07e4be3d09d04b531c889182a11a488e7e9ba4b75f642040e4681c1e9b98 + md5: 1e61fb236ccd3d6ccaf9e91cb2d7e12d + depends: + - libfreetype6 >=2.14.1 + license: GPL-2.0-only OR FTL + purls: [] + size: 7753 + timestamp: 1757945484817 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libfreetype-2.14.1-hce30654_0.conda + sha256: 9de25a86066f078822d8dd95a83048d7dc2897d5d655c0e04a8a54fca13ef1ef + md5: f35fb38e89e2776994131fbf961fa44b + depends: + - libfreetype6 >=2.14.1 + license: GPL-2.0-only OR FTL + purls: [] + size: 7810 + timestamp: 1757947168537 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libfreetype6-2.14.1-h73754d4_0.conda + sha256: 4a7af818a3179fafb6c91111752954e29d3a2a950259c14a2fc7ba40a8b03652 + md5: 8e7251989bca326a28f4a5ffbd74557a + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=14 + - libpng >=1.6.50,<1.7.0a0 + - libzlib >=1.3.1,<2.0a0 + constrains: + - freetype >=2.14.1 + license: GPL-2.0-only OR FTL + purls: [] + size: 386739 + timestamp: 1757945416744 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libfreetype6-2.14.1-hdae7a39_0.conda + sha256: cedc83d9733363aca353872c3bfed2e188aa7caf57b57842ba0c6d2765652b7c + md5: 9c2f56b6e011c6d8010ff43b796aab2f + depends: + - libgcc >=14 + - libpng >=1.6.50,<1.7.0a0 + - libzlib >=1.3.1,<2.0a0 + constrains: + - freetype >=2.14.1 + license: GPL-2.0-only OR FTL + purls: [] + size: 423210 + timestamp: 1757945484108 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libfreetype6-2.14.1-h6da58f4_0.conda + sha256: cc4aec4c490123c0f248c1acd1aeab592afb6a44b1536734e20937cda748f7cd + md5: 6d4ede03e2a8e20eb51f7f681d2a2550 + depends: + - __osx >=11.0 + - libpng >=1.6.50,<1.7.0a0 + - libzlib >=1.3.1,<2.0a0 + constrains: + - freetype >=2.14.1 + license: GPL-2.0-only OR FTL + purls: [] + size: 346703 + timestamp: 1757947166116 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libgcc-15.1.0-h767d61c_5.conda + sha256: 0caed73aac3966bfbf5710e06c728a24c6c138605121a3dacb2e03440e8baa6a + md5: 264fbfba7fb20acf3b29cde153e345ce + depends: + - __glibc >=2.17,<3.0.a0 + - _openmp_mutex >=4.5 + constrains: + - libgomp 15.1.0 h767d61c_5 + - libgcc-ng ==15.1.0=*_5 + license: GPL-3.0-only WITH GCC-exception-3.1 + license_family: GPL + purls: [] + size: 824191 + timestamp: 1757042543820 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libgcc-15.2.0-he277a41_7.conda + sha256: 616f5960930ad45b48c57f49c3adddefd9423674b331887ef0e69437798c214b + md5: afa05d91f8d57dd30985827a09c21464 + depends: + - _openmp_mutex >=4.5 + constrains: + - libgomp 15.2.0 he277a41_7 + - libgcc-ng ==15.2.0=*_7 + license: GPL-3.0-only WITH GCC-exception-3.1 + license_family: GPL + purls: [] + size: 510719 + timestamp: 1759967448307 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libgcc-ng-15.1.0-h69a702a_5.conda + sha256: f54bb9c3be12b24be327f4c1afccc2969712e0b091cdfbd1d763fb3e61cda03f + md5: 069afdf8ea72504e48d23ae1171d951c + depends: + - libgcc 15.1.0 h767d61c_5 + license: GPL-3.0-only WITH GCC-exception-3.1 + license_family: GPL + purls: [] + size: 29187 + timestamp: 1757042549554 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libgcc-ng-15.2.0-he9431aa_7.conda + sha256: 7d98979b2b5698330007b0146b8b4b95b3790378de12129ce13c9fc88c1ef45a + md5: a5ce1f0a32f02c75c11580c5b2f9258a + depends: + - libgcc 15.2.0 he277a41_7 + license: GPL-3.0-only WITH GCC-exception-3.1 + license_family: GPL + purls: [] + size: 29261 + timestamp: 1759967452303 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libgd-2.3.3-h6f5c62b_11.conda + sha256: 19e5be91445db119152217e8e8eec4fd0499d854acc7d8062044fb55a70971cd + md5: 68fc66282364981589ef36868b1a7c78 + depends: + - __glibc >=2.17,<3.0.a0 + - fontconfig >=2.15.0,<3.0a0 + - fonts-conda-ecosystem + - freetype >=2.12.1,<3.0a0 + - icu >=75.1,<76.0a0 + - libexpat >=2.6.4,<3.0a0 + - libgcc >=13 + - libjpeg-turbo >=3.0.0,<4.0a0 + - libpng >=1.6.45,<1.7.0a0 + - libtiff >=4.7.0,<4.8.0a0 + - libwebp-base >=1.5.0,<2.0a0 + - libzlib >=1.3.1,<2.0a0 + license: GD + license_family: BSD + purls: [] + size: 177082 + timestamp: 1737548051015 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libgd-2.3.3-hc8d7b1d_11.conda + sha256: 7e199bb390f985b34aee38cdb1f0d166abc09ed44bd703a1b91a3c6cd9912d45 + md5: d256b0311b7a207a2c6b68d2b399f707 + depends: + - fontconfig >=2.15.0,<3.0a0 + - fonts-conda-ecosystem + - freetype >=2.12.1,<3.0a0 + - icu >=75.1,<76.0a0 + - libexpat >=2.6.4,<3.0a0 + - libgcc >=13 + - libjpeg-turbo >=3.0.0,<4.0a0 + - libpng >=1.6.45,<1.7.0a0 + - libtiff >=4.7.0,<4.8.0a0 + - libwebp-base >=1.5.0,<2.0a0 + - libzlib >=1.3.1,<2.0a0 + license: GD + license_family: BSD + purls: [] + size: 191033 + timestamp: 1737548098172 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libgd-2.3.3-hb2c3a21_11.conda + sha256: be038eb8dfe296509aee2df21184c72cb76285b0340448525664bc396aa6146d + md5: 4581aa3cfcd1a90967ed02d4a9f3db4b + depends: + - __osx >=11.0 + - fontconfig >=2.15.0,<3.0a0 + - fonts-conda-ecosystem + - freetype >=2.12.1,<3.0a0 + - icu >=75.1,<76.0a0 + - libexpat >=2.6.4,<3.0a0 + - libiconv >=1.17,<2.0a0 + - libjpeg-turbo >=3.0.0,<4.0a0 + - libpng >=1.6.45,<1.7.0a0 + - libtiff >=4.7.0,<4.8.0a0 + - libwebp-base >=1.5.0,<2.0a0 + - libzlib >=1.3.1,<2.0a0 + license: GD + license_family: BSD + purls: [] + size: 156868 + timestamp: 1737548290283 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libgl-1.7.0-hd24410f_2.conda + sha256: 3e954380f16255d1c8ae5da3bd3044d3576a0e1ac2e3c3ff2fe8f2f1ad2e467a + md5: 0d00176464ebb25af83d40736a2cd3bb + depends: + - libglvnd 1.7.0 hd24410f_2 + - libglx 1.7.0 hd24410f_2 + license: LicenseRef-libglvnd + purls: [] + size: 145442 + timestamp: 1731331005019 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libgl-devel-1.7.0-hd24410f_2.conda + sha256: ec5c3125b38295bad8acc80f793b8ee217ccb194338d73858be278db50ea82f1 + md5: 5d8323dff6a93596fb6f985cf6e8521a + depends: + - libgl 1.7.0 hd24410f_2 + - libglx-devel 1.7.0 hd24410f_2 + license: LicenseRef-libglvnd + purls: [] + size: 113925 + timestamp: 1731331014056 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libglib-2.86.0-h1fed272_0.conda + sha256: 33336bd55981be938f4823db74291e1323454491623de0be61ecbe6cf3a4619c + md5: b8e4c93f4ab70c3b6f6499299627dbdc + depends: + - __glibc >=2.17,<3.0.a0 + - libffi >=3.4.6,<3.5.0a0 + - libgcc >=14 + - libiconv >=1.18,<2.0a0 + - libzlib >=1.3.1,<2.0a0 + - pcre2 >=10.46,<10.47.0a0 + constrains: + - glib 2.86.0 *_0 + license: LGPL-2.1-or-later + purls: [] + size: 3978602 + timestamp: 1757403291664 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libglib-2.86.1-he84ff74_1.conda + sha256: 5212c30d9e14a9480c7d25bf93ccca4db23d3794430c9be90e13124d9a8b1687 + md5: f0fc1b2fa2e68b1309852e5c3c8e011d + depends: + - libffi >=3.5.2,<3.6.0a0 + - libgcc >=14 + - libiconv >=1.18,<2.0a0 + - libzlib >=1.3.1,<2.0a0 + - pcre2 >=10.46,<10.47.0a0 + constrains: + - glib 2.86.1 *_1 + license: LGPL-2.1-or-later + purls: [] + size: 4040523 + timestamp: 1761874121589 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libglib-2.86.1-he69a767_1.conda + sha256: 253ac4eca90006b19571f8c4766e8ebdad0f01f44de1bfa0472d3df9be9c8ac8 + md5: acff031bb5b97602d2b7ef913a8ea076 + depends: + - __osx >=11.0 + - libffi >=3.5.2,<3.6.0a0 + - libiconv >=1.18,<2.0a0 + - libintl >=0.25.1,<1.0a0 + - libzlib >=1.3.1,<2.0a0 + - pcre2 >=10.46,<10.47.0a0 + constrains: + - glib 2.86.1 *_1 + license: LGPL-2.1-or-later + purls: [] + size: 3677659 + timestamp: 1761875607047 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libglvnd-1.7.0-hd24410f_2.conda + sha256: 57ec3898a923d4bcc064669e90e8abfc4d1d945a13639470ba5f3748bd3090da + md5: 9e115653741810778c9a915a2f8439e7 + license: LicenseRef-libglvnd + purls: [] + size: 152135 + timestamp: 1731330986070 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libglx-1.7.0-hd24410f_2.conda + sha256: 6591af640cb05a399fab47646025f8b1e1a06a0d4bbb4d2e320d6629b47a1c61 + md5: 1d4269e233636148696a67e2d30dad2a + depends: + - libglvnd 1.7.0 hd24410f_2 + - xorg-libx11 >=1.8.9,<2.0a0 + license: LicenseRef-libglvnd + purls: [] + size: 77736 + timestamp: 1731330998960 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libglx-devel-1.7.0-hd24410f_2.conda + sha256: 4bc28ecc38f30ca1ac66a8fb6c5703f4d888381ec46d3938b7c3383210061ec5 + md5: 1f9ddbb175a63401662d1c6222cef6ff + depends: + - libglx 1.7.0 hd24410f_2 + - xorg-libx11 >=1.8.9,<2.0a0 + - xorg-xorgproto + license: LicenseRef-libglvnd + purls: [] + size: 26362 + timestamp: 1731331008489 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libgomp-15.1.0-h767d61c_5.conda + sha256: 125051d51a8c04694d0830f6343af78b556dd88cc249dfec5a97703ebfb1832d + md5: dcd5ff1940cd38f6df777cac86819d60 + depends: + - __glibc >=2.17,<3.0.a0 + license: GPL-3.0-only WITH GCC-exception-3.1 + license_family: GPL + purls: [] + size: 447215 + timestamp: 1757042483384 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libgomp-15.2.0-he277a41_7.conda + sha256: 0a024f1e4796f5d90fb8e8555691dad1b3bdfc6ac3c2cd14d876e30f805fcac7 + md5: 34cef4753287c36441f907d5fdd78d42 + license: GPL-3.0-only WITH GCC-exception-3.1 + license_family: GPL + purls: [] + size: 450308 + timestamp: 1759967379407 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libiconv-1.18-h3b78370_2.conda + sha256: c467851a7312765447155e071752d7bf9bf44d610a5687e32706f480aad2833f + md5: 915f5995e94f60e9a4826e0b0920ee88 + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=14 + license: LGPL-2.1-only + purls: [] + size: 790176 + timestamp: 1754908768807 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libiconv-1.18-h90929bb_2.conda + sha256: 1473451cd282b48d24515795a595801c9b65b567fe399d7e12d50b2d6cdb04d9 + md5: 5a86bf847b9b926f3a4f203339748d78 + depends: + - libgcc >=14 + license: LGPL-2.1-only + purls: [] + size: 791226 + timestamp: 1754910975665 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libiconv-1.18-h23cfdf5_2.conda + sha256: de0336e800b2af9a40bdd694b03870ac4a848161b35c8a2325704f123f185f03 + md5: 4d5a7445f0b25b6a3ddbb56e790f5251 + depends: + - __osx >=11.0 + license: LGPL-2.1-only + purls: [] + size: 750379 + timestamp: 1754909073836 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libintl-0.25.1-h493aca8_0.conda + sha256: 99d2cebcd8f84961b86784451b010f5f0a795ed1c08f1e7c76fbb3c22abf021a + md5: 5103f6a6b210a3912faf8d7db516918c + depends: + - __osx >=11.0 + - libiconv >=1.18,<2.0a0 + license: LGPL-2.1-or-later + purls: [] + size: 90957 + timestamp: 1751558394144 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libjpeg-turbo-3.1.0-hb9d3cd8_0.conda + sha256: 98b399287e27768bf79d48faba8a99a2289748c65cd342ca21033fab1860d4a4 + md5: 9fa334557db9f63da6c9285fd2a48638 + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=13 + constrains: + - jpeg <0.0.0a + license: IJG AND BSD-3-Clause AND Zlib + purls: [] + size: 628947 + timestamp: 1745268527144 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libjpeg-turbo-3.1.2-he30d5cf_0.conda + sha256: 84064c7c53a64291a585d7215fe95ec42df74203a5bf7615d33d49a3b0f08bb6 + md5: 5109d7f837a3dfdf5c60f60e311b041f + depends: + - libgcc >=14 + constrains: + - jpeg <0.0.0a + license: IJG AND BSD-3-Clause AND Zlib + purls: [] + size: 691818 + timestamp: 1762094728337 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libjpeg-turbo-3.1.0-h5505292_0.conda + sha256: 78df2574fa6aa5b6f5fc367c03192f8ddf8e27dc23641468d54e031ff560b9d4 + md5: 01caa4fbcaf0e6b08b3aef1151e91745 + depends: + - __osx >=11.0 + constrains: + - jpeg <0.0.0a + license: IJG AND BSD-3-Clause AND Zlib + purls: [] + size: 553624 + timestamp: 1745268405713 +- conda: https://conda.anaconda.org/conda-forge/linux-64/liblzma-5.8.1-hb9d3cd8_2.conda + sha256: f2591c0069447bbe28d4d696b7fcb0c5bd0b4ac582769b89addbcf26fb3430d8 + md5: 1a580f7796c7bf6393fddb8bbbde58dc + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=13 + constrains: + - xz 5.8.1.* + license: 0BSD + purls: [] + size: 112894 + timestamp: 1749230047870 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/liblzma-5.8.1-h86ecc28_2.conda + sha256: 498ea4b29155df69d7f20990a7028d75d91dbea24d04b2eb8a3d6ef328806849 + md5: 7d362346a479256857ab338588190da0 + depends: + - libgcc >=13 + constrains: + - xz 5.8.1.* + license: 0BSD + purls: [] + size: 125103 + timestamp: 1749232230009 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/liblzma-5.8.1-h39f12f2_2.conda + sha256: 0cb92a9e026e7bd4842f410a5c5c665c89b2eb97794ffddba519a626b8ce7285 + md5: d6df911d4564d77c4374b02552cb17d1 + depends: + - __osx >=11.0 + constrains: + - xz 5.8.1.* + license: 0BSD + purls: [] + size: 92286 + timestamp: 1749230283517 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libmpdec-4.0.0-hb9d3cd8_0.conda + sha256: 3aa92d4074d4063f2a162cd8ecb45dccac93e543e565c01a787e16a43501f7ee + md5: c7e925f37e3b40d893459e625f6a53f1 + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=13 + license: BSD-2-Clause + license_family: BSD + purls: [] + size: 91183 + timestamp: 1748393666725 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libmpdec-4.0.0-h86ecc28_0.conda + sha256: ef8697f934c80b347bf9d7ed45650928079e303bad01bd064995b0e3166d6e7a + md5: 78cfed3f76d6f3f279736789d319af76 + depends: + - libgcc >=13 + license: BSD-2-Clause + license_family: BSD + purls: [] + size: 114064 + timestamp: 1748393729243 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libmpdec-4.0.0-h5505292_0.conda + sha256: 0a1875fc1642324ebd6c4ac864604f3f18f57fbcf558a8264f6ced028a3c75b2 + md5: 85ccccb47823dd9f7a99d2c7f530342f + depends: + - __osx >=11.0 + license: BSD-2-Clause + license_family: BSD + purls: [] + size: 71829 + timestamp: 1748393749336 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libpciaccess-0.18-h86ecc28_0.conda + sha256: 7641dfdfe9bda7069ae94379e9924892f0b6604c1a016a3f76b230433bb280f2 + md5: 5044e160c5306968d956c2a0a2a440d6 + depends: + - libgcc >=13 + license: MIT + license_family: MIT + purls: [] + size: 29512 + timestamp: 1749901899881 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libpng-1.6.50-h421ea60_1.conda + sha256: e75a2723000ce3a4b9fd9b9b9ce77553556c93e475a4657db6ed01abc02ea347 + md5: 7af8e91b0deb5f8e25d1a595dea79614 + depends: + - libgcc >=14 + - __glibc >=2.17,<3.0.a0 + - libzlib >=1.3.1,<2.0a0 + license: zlib-acknowledgement + purls: [] + size: 317390 + timestamp: 1753879899951 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libpng-1.6.50-h1abf092_1.conda + sha256: e1effd7335ec101bb124f41a5f79fabb5e7b858eafe0f2db4401fb90c51505a7 + md5: ed42935ac048d73109163d653d9445a0 + depends: + - libgcc >=14 + - libzlib >=1.3.1,<2.0a0 + license: zlib-acknowledgement + purls: [] + size: 339168 + timestamp: 1753879915462 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libpng-1.6.50-h280e0eb_1.conda + sha256: a2e0240fb0c79668047b528976872307ea80cb330baf8bf6624ac2c6443449df + md5: 4d0f5ce02033286551a32208a5519884 + depends: + - __osx >=11.0 + - libzlib >=1.3.1,<2.0a0 + license: zlib-acknowledgement + purls: [] + size: 287056 + timestamp: 1753879907258 +- conda: https://conda.anaconda.org/conda-forge/linux-64/librsvg-2.58.4-he92a37e_3.conda + sha256: a45ef03e6e700cc6ac6c375e27904531cf8ade27eb3857e080537ff283fb0507 + md5: d27665b20bc4d074b86e628b3ba5ab8b + depends: + - __glibc >=2.17,<3.0.a0 + - cairo >=1.18.4,<2.0a0 + - freetype >=2.13.3,<3.0a0 + - gdk-pixbuf >=2.42.12,<3.0a0 + - harfbuzz >=11.0.0,<12.0a0 + - libgcc >=13 + - libglib >=2.84.0,<3.0a0 + - libpng >=1.6.47,<1.7.0a0 + - libxml2 >=2.13.7,<2.14.0a0 + - pango >=1.56.3,<2.0a0 + constrains: + - __glibc >=2.17 + license: LGPL-2.1-or-later + purls: [] + size: 6543651 + timestamp: 1743368725313 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/librsvg-2.60.0-h8171147_0.conda + sha256: b6cb38e95a447a04e624b6070981899e18c03f71915476fe024dadf384f48f15 + md5: 7e4a8318e73ba685615f90bff926bfe4 + depends: + - cairo >=1.18.4,<2.0a0 + - gdk-pixbuf >=2.44.3,<3.0a0 + - libgcc >=14 + - libglib >=2.86.0,<3.0a0 + - libxml2-16 >=2.14.6 + - pango >=1.56.4,<2.0a0 + constrains: + - __glibc >=2.17 + license: LGPL-2.1-or-later + purls: [] + size: 2995492 + timestamp: 1759335330016 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/librsvg-2.60.0-h5c55ec3_0.conda + sha256: ca5a2de5d3f68e8d6443ea1bf193c1596a278e6f86018017c0ccd4928eaf8971 + md5: 05ad1d6b6fb3b384f7a07128025725cb + depends: + - __osx >=11.0 + - cairo >=1.18.4,<2.0a0 + - gdk-pixbuf >=2.44.3,<3.0a0 + - libglib >=2.86.0,<3.0a0 + - libxml2-16 >=2.14.6 + - pango >=1.56.4,<2.0a0 + constrains: + - __osx >=11.0 + license: LGPL-2.1-or-later + purls: [] + size: 2344343 + timestamp: 1759328503184 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libsqlite-3.50.4-h0c1763c_0.conda + sha256: 6d9c32fc369af5a84875725f7ddfbfc2ace795c28f246dc70055a79f9b2003da + md5: 0b367fad34931cb79e0d6b7e5c06bb1c + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=14 + - libzlib >=1.3.1,<2.0a0 + license: blessing + purls: [] + size: 932581 + timestamp: 1753948484112 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libsqlite-3.51.0-h022381a_0.conda + sha256: f66a40b6e07a6f8ce6ccbd38d079b7394217d8f8ae0a05efa644aa0a40140671 + md5: 8920ce2226463a3815e2183c8b5008b8 + depends: + - libgcc >=14 + - libzlib >=1.3.1,<2.0a0 + license: blessing + purls: [] + size: 938476 + timestamp: 1762299829629 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libsqlite-3.50.4-h4237e3c_0.conda + sha256: 802ebe62e6bc59fc26b26276b793e0542cfff2d03c086440aeaf72fb8bbcec44 + md5: 1dcb0468f5146e38fae99aef9656034b + depends: + - __osx >=11.0 + - icu >=75.1,<76.0a0 + - libzlib >=1.3.1,<2.0a0 + license: blessing + purls: [] + size: 902645 + timestamp: 1753948599139 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-15.1.0-h8f9b012_5.conda + sha256: 0f5f61cab229b6043541c13538d75ce11bd96fb2db76f94ecf81997b1fde6408 + md5: 4e02a49aaa9d5190cb630fa43528fbe6 + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc 15.1.0 h767d61c_5 + license: GPL-3.0-only WITH GCC-exception-3.1 + license_family: GPL + purls: [] + size: 3896432 + timestamp: 1757042571458 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libstdcxx-15.2.0-h3f4de04_7.conda + sha256: 4c6d1a2ae58044112233a57103bbf06000bd4c2aad44a0fd3b464b05fa8df514 + md5: 6a2f0ee17851251a85fbebafbe707d2d + depends: + - libgcc 15.2.0 he277a41_7 + constrains: + - libstdcxx-ng ==15.2.0=*_7 + license: GPL-3.0-only WITH GCC-exception-3.1 + license_family: GPL + purls: [] + size: 3831785 + timestamp: 1759967470295 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-ng-15.1.0-h4852527_5.conda + sha256: 7b8cabbf0ab4fe3581ca28fe8ca319f964078578a51dd2ca3f703c1d21ba23ff + md5: 8bba50c7f4679f08c861b597ad2bda6b + depends: + - libstdcxx 15.1.0 h8f9b012_5 + license: GPL-3.0-only WITH GCC-exception-3.1 + license_family: GPL + purls: [] + size: 29233 + timestamp: 1757042603319 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libstdcxx-ng-15.2.0-hf1166c9_7.conda + sha256: 26fc1bdb39042f27302b363785fea6f6b9607f9c2f5eb949c6ae0bdbb8599574 + md5: 9e5deec886ad32f3c6791b3b75c78681 + depends: + - libstdcxx 15.2.0 h3f4de04_7 + license: GPL-3.0-only WITH GCC-exception-3.1 + license_family: GPL + purls: [] + size: 29341 + timestamp: 1759967498023 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libtiff-4.7.0-h8261f1e_6.conda + sha256: c62694cd117548d810d2803da6d9063f78b1ffbf7367432c5388ce89474e9ebe + md5: b6093922931b535a7ba566b6f384fbe6 + depends: + - __glibc >=2.17,<3.0.a0 + - lerc >=4.0.0,<5.0a0 + - libdeflate >=1.24,<1.25.0a0 + - libgcc >=14 + - libjpeg-turbo >=3.1.0,<4.0a0 + - liblzma >=5.8.1,<6.0a0 + - libstdcxx >=14 + - libwebp-base >=1.6.0,<2.0a0 + - libzlib >=1.3.1,<2.0a0 + - zstd >=1.5.7,<1.6.0a0 + license: HPND + purls: [] + size: 433078 + timestamp: 1755011934951 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libtiff-4.7.1-hdb009f0_1.conda + sha256: 7ff79470db39e803e21b8185bc8f19c460666d5557b1378d1b1e857d929c6b39 + md5: 8c6fd84f9c87ac00636007c6131e457d + depends: + - lerc >=4.0.0,<5.0a0 + - libdeflate >=1.25,<1.26.0a0 + - libgcc >=14 + - libjpeg-turbo >=3.1.0,<4.0a0 + - liblzma >=5.8.1,<6.0a0 + - libstdcxx >=14 + - libwebp-base >=1.6.0,<2.0a0 + - libzlib >=1.3.1,<2.0a0 + - zstd >=1.5.7,<1.6.0a0 + license: HPND + purls: [] + size: 488407 + timestamp: 1762022048105 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libtiff-4.7.1-h7dc4979_0.conda + sha256: 6bc1b601f0d3ee853acd23884a007ac0a0290f3609dabb05a47fc5a0295e2b53 + md5: 2bb9e04e2da869125e2dc334d665f00d + depends: + - __osx >=11.0 + - lerc >=4.0.0,<5.0a0 + - libcxx >=19 + - libdeflate >=1.24,<1.25.0a0 + - libjpeg-turbo >=3.1.0,<4.0a0 + - liblzma >=5.8.1,<6.0a0 + - libwebp-base >=1.6.0,<2.0a0 + - libzlib >=1.3.1,<2.0a0 + - zstd >=1.5.7,<1.6.0a0 + license: HPND + purls: [] + size: 373640 + timestamp: 1758278641520 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libuuid-2.41.1-he9a06e4_0.conda + sha256: 776e28735cee84b97e4d05dd5d67b95221a3e2c09b8b13e3d6dbe6494337d527 + md5: af930c65e9a79a3423d6d36e265cef65 + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=14 + license: BSD-3-Clause + license_family: BSD + purls: [] + size: 37087 + timestamp: 1757334557450 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libuuid-2.41.2-h3e4203c_0.conda + sha256: 7aed28ac04e0298bf8f7ad44a23d6f8ee000aa0445807344b16fceedc67cce0f + md5: 3a68e44fdf2a2811672520fdd62996bd + depends: + - libgcc >=14 + license: BSD-3-Clause + license_family: BSD + purls: [] + size: 39172 + timestamp: 1758626850999 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libwebp-base-1.6.0-hd42ef1d_0.conda + sha256: 3aed21ab28eddffdaf7f804f49be7a7d701e8f0e46c856d801270b470820a37b + md5: aea31d2e5b1091feca96fcfe945c3cf9 + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=14 + constrains: + - libwebp 1.6.0 + license: BSD-3-Clause + license_family: BSD + purls: [] + size: 429011 + timestamp: 1752159441324 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libwebp-base-1.6.0-ha2e29f5_0.conda + sha256: b03700a1f741554e8e5712f9b06dd67e76f5301292958cd3cb1ac8c6fdd9ed25 + md5: 24e92d0942c799db387f5c9d7b81f1af + depends: + - libgcc >=14 + constrains: + - libwebp 1.6.0 + license: BSD-3-Clause + license_family: BSD + purls: [] + size: 359496 + timestamp: 1752160685488 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libwebp-base-1.6.0-h07db88b_0.conda + sha256: a4de3f371bb7ada325e1f27a4ef7bcc81b2b6a330e46fac9c2f78ac0755ea3dd + md5: e5e7d467f80da752be17796b87fe6385 + depends: + - __osx >=11.0 + constrains: + - libwebp 1.6.0 + license: BSD-3-Clause + license_family: BSD + purls: [] + size: 294974 + timestamp: 1752159906788 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libxcb-1.17.0-h8a09558_0.conda + sha256: 666c0c431b23c6cec6e492840b176dde533d48b7e6fb8883f5071223433776aa + md5: 92ed62436b625154323d40d5f2f11dd7 + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=13 + - pthread-stubs + - xorg-libxau >=1.0.11,<2.0a0 + - xorg-libxdmcp + license: MIT + license_family: MIT + purls: [] + size: 395888 + timestamp: 1727278577118 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libxcb-1.17.0-h262b8f6_0.conda + sha256: 461cab3d5650ac6db73a367de5c8eca50363966e862dcf60181d693236b1ae7b + md5: cd14ee5cca2464a425b1dbfc24d90db2 + depends: + - libgcc >=13 + - pthread-stubs + - xorg-libxau >=1.0.11,<2.0a0 + - xorg-libxdmcp + license: MIT + license_family: MIT + purls: [] + size: 397493 + timestamp: 1727280745441 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libxkbcommon-1.11.0-he8b52b9_0.conda + sha256: 23f47e86cc1386e7f815fa9662ccedae151471862e971ea511c5c886aa723a54 + md5: 74e91c36d0eef3557915c68b6c2bef96 + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=14 + - libstdcxx >=14 + - libxcb >=1.17.0,<2.0a0 + - libxml2 >=2.13.8,<2.14.0a0 + - xkeyboard-config + - xorg-libxau >=1.0.12,<2.0a0 + license: MIT/X11 Derivative + license_family: MIT + purls: [] + size: 791328 + timestamp: 1754703902365 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libxkbcommon-1.13.0-h3c6a4c8_0.conda + sha256: c197e58ba06fa9ac73fcbdc20f9a78ba0164f61879d127bb2f7d0d4be346216a + md5: a7c78be36bf59b4ba44ad2f2f8b92b37 + depends: + - libgcc >=14 + - libstdcxx >=14 + - libxcb >=1.17.0,<2.0a0 + - libxml2 + - libxml2-16 >=2.14.6 + - xkeyboard-config + - xorg-libxau >=1.0.12,<2.0a0 + license: MIT/X11 Derivative + license_family: MIT + purls: [] + size: 862682 + timestamp: 1762341934465 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libxml2-2.13.8-h04c0eec_1.conda + sha256: 03deb1ec6edfafc5aaeecadfc445ee436fecffcda11fcd97fde9b6632acb583f + md5: 10bcbd05e1c1c9d652fccb42b776a9fa + depends: + - __glibc >=2.17,<3.0.a0 + - icu >=75.1,<76.0a0 + - libgcc >=14 + - libiconv >=1.18,<2.0a0 + - liblzma >=5.8.1,<6.0a0 + - libzlib >=1.3.1,<2.0a0 + license: MIT + license_family: MIT + purls: [] + size: 698448 + timestamp: 1754315344761 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libxml2-2.15.1-h788dabe_0.conda + sha256: db0a568e0853ee38b7a4db1cb4ee76e57fe7c32ccb1d5b75f6618a1041d3c6e4 + md5: a0e7779b7625b88e37df9bd73f0638dc + depends: + - icu >=75.1,<76.0a0 + - libgcc >=14 + - libiconv >=1.18,<2.0a0 + - liblzma >=5.8.1,<6.0a0 + - libxml2-16 2.15.1 h8591a01_0 + - libzlib >=1.3.1,<2.0a0 + license: MIT + license_family: MIT + purls: [] + size: 47192 + timestamp: 1761015739999 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libxml2-16-2.15.1-h8591a01_0.conda + sha256: 7a13450bce2eeba8f8fb691868b79bf0891377b707493a527bd930d64d9b98af + md5: e7177c6fbbf815da7b215b4cc3e70208 + depends: + - icu >=75.1,<76.0a0 + - libgcc >=14 + - libiconv >=1.18,<2.0a0 + - liblzma >=5.8.1,<6.0a0 + - libzlib >=1.3.1,<2.0a0 + constrains: + - libxml2 2.15.1 + license: MIT + license_family: MIT + purls: [] + size: 597078 + timestamp: 1761015734476 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libxml2-16-2.15.1-h0ff4647_0.conda + sha256: ebe2dd9da94280ad43da936efa7127d329b559f510670772debc87602b49b06d + md5: 438c97d1e9648dd7342f86049dd44638 + depends: + - __osx >=11.0 + - icu >=75.1,<76.0a0 + - libiconv >=1.18,<2.0a0 + - liblzma >=5.8.1,<6.0a0 + - libzlib >=1.3.1,<2.0a0 + constrains: + - libxml2 2.15.1 + license: MIT + license_family: MIT + purls: [] + size: 464952 + timestamp: 1761016087733 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libzlib-1.3.1-hb9d3cd8_2.conda + sha256: d4bfe88d7cb447768e31650f06257995601f89076080e76df55e3112d4e47dc4 + md5: edb0dca6bc32e4f4789199455a1dbeb8 + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=13 + constrains: + - zlib 1.3.1 *_2 + license: Zlib + license_family: Other + purls: [] + size: 60963 + timestamp: 1727963148474 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libzlib-1.3.1-h86ecc28_2.conda + sha256: 5a2c1eeef69342e88a98d1d95bff1603727ab1ff4ee0e421522acd8813439b84 + md5: 08aad7cbe9f5a6b460d0976076b6ae64 + depends: + - libgcc >=13 + constrains: + - zlib 1.3.1 *_2 + license: Zlib + license_family: Other + purls: [] + size: 66657 + timestamp: 1727963199518 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libzlib-1.3.1-h8359307_2.conda + sha256: ce34669eadaba351cd54910743e6a2261b67009624dbc7daeeafdef93616711b + md5: 369964e85dc26bfe78f41399b366c435 + depends: + - __osx >=11.0 + constrains: + - zlib 1.3.1 *_2 + license: Zlib + license_family: Other + purls: [] + size: 46438 + timestamp: 1727963202283 +- pypi: https://files.pythonhosted.org/packages/e5/b8/9eea6630198cb303d131d95d285a024b3b8645b1763a2916fddb44ca8760/matplotlib-3.10.6-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl + name: matplotlib + version: 3.10.6 + sha256: 84e82d9e0fd70c70bc55739defbd8055c54300750cbacf4740c9673a24d6933a + requires_dist: + - contourpy>=1.0.1 + - cycler>=0.10 + - fonttools>=4.22.0 + - kiwisolver>=1.3.1 + - numpy>=1.23 + - packaging>=20.0 + - pillow>=8 + - pyparsing>=2.3.1 + - python-dateutil>=2.7 + - meson-python>=0.13.1,<0.17.0 ; extra == 'dev' + - pybind11>=2.13.2,!=2.13.3 ; extra == 'dev' + - setuptools-scm>=7 ; extra == 'dev' + - setuptools>=64 ; extra == 'dev' + requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/bc/d0/b3d3338d467d3fc937f0bb7f256711395cae6f78e22cef0656159950adf0/matplotlib-3.10.7-cp313-cp313-macosx_11_0_arm64.whl + name: matplotlib + version: 3.10.7 + sha256: 37a1fea41153dd6ee061d21ab69c9cf2cf543160b1b85d89cd3d2e2a7902ca4c + requires_dist: + - contourpy>=1.0.1 + - cycler>=0.10 + - fonttools>=4.22.0 + - kiwisolver>=1.3.1 + - numpy>=1.23 + - packaging>=20.0 + - pillow>=8 + - pyparsing>=3 + - python-dateutil>=2.7 + - meson-python>=0.13.1,<0.17.0 ; extra == 'dev' + - pybind11>=2.13.2,!=2.13.3 ; extra == 'dev' + - setuptools-scm>=7 ; extra == 'dev' + - setuptools>=64 ; extra == 'dev' + requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/d0/7f/ccdca06f4c2e6c7989270ed7829b8679466682f4cfc0f8c9986241c023b6/matplotlib-3.10.7-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl + name: matplotlib + version: 3.10.7 + sha256: 22df30ffaa89f6643206cf13877191c63a50e8f800b038bc39bee9d2d4957632 + requires_dist: + - contourpy>=1.0.1 + - cycler>=0.10 + - fonttools>=4.22.0 + - kiwisolver>=1.3.1 + - numpy>=1.23 + - packaging>=20.0 + - pillow>=8 + - pyparsing>=3 + - python-dateutil>=2.7 + - meson-python>=0.13.1,<0.17.0 ; extra == 'dev' + - pybind11>=2.13.2,!=2.13.3 ; extra == 'dev' + - setuptools-scm>=7 ; extra == 'dev' + - setuptools>=64 ; extra == 'dev' + requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/8f/8e/9ad090d3553c280a8060fbf6e24dc1c0c29704ee7d1c372f0c174aa59285/matplotlib_inline-0.1.7-py3-none-any.whl + name: matplotlib-inline + version: 0.1.7 + sha256: df192d39a4ff8f21b1895d72e6a13f5fcc5099f00fa84384e0ea28c2cc0653ca + requires_dist: + - traitlets + requires_python: '>=3.8' +- pypi: https://files.pythonhosted.org/packages/af/33/ee4519fa02ed11a94aef9559552f3b17bb863f2ecfe1a35dc7f548cde231/matplotlib_inline-0.2.1-py3-none-any.whl + name: matplotlib-inline + version: 0.2.1 + sha256: d56ce5156ba6085e00a9d54fead6ed29a9c47e215cd1bba2e976ef39f5710a76 + requires_dist: + - traitlets + - flake8 ; extra == 'test' + - nbdime ; extra == 'test' + - nbval ; extra == 'test' + - notebook ; extra == 'test' + - pytest ; extra == 'test' + requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/89/a3/00260f8df72b51afa1f182dd609533c77fa2407918c4c2813d87b4a56725/minio-7.2.16-py3-none-any.whl + name: minio + version: 7.2.16 + sha256: 9288ab988ca57c181eb59a4c96187b293131418e28c164392186c2b89026b223 + requires_dist: + - argon2-cffi + - certifi + - pycryptodome + - typing-extensions + - urllib3 + requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/7d/ae/f32695da4f93de50dd7075100dab8cf689a9d96270f58ce6f940fd044a3e/minio-7.2.18-py3-none-any.whl + name: minio + version: 7.2.18 + sha256: f23a6edbff8d0bc4b5c1a61b2628a01c5a3342aefc613ff9c276012e6321108f + requires_dist: + - argon2-cffi + - certifi + - pycryptodome + - typing-extensions + - urllib3 + requires_python: '>=3.9' +- conda: https://conda.anaconda.org/conda-forge/linux-64/ncurses-6.5-h2d0b736_3.conda + sha256: 3fde293232fa3fca98635e1167de6b7c7fda83caf24b9d6c91ec9eefb4f4d586 + md5: 47e340acb35de30501a76c7c799c41d7 + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=13 + license: X11 AND BSD-3-Clause + purls: [] + size: 891641 + timestamp: 1738195959188 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/ncurses-6.5-ha32ae93_3.conda + sha256: 91cfb655a68b0353b2833521dc919188db3d8a7f4c64bea2c6a7557b24747468 + md5: 182afabe009dc78d8b73100255ee6868 + depends: + - libgcc >=13 + license: X11 AND BSD-3-Clause + purls: [] + size: 926034 + timestamp: 1738196018799 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/ncurses-6.5-h5e97a16_3.conda + sha256: 2827ada40e8d9ca69a153a45f7fd14f32b2ead7045d3bbb5d10964898fe65733 + md5: 068d497125e4bf8a66bf707254fff5ae + depends: + - __osx >=11.0 + license: X11 AND BSD-3-Clause + purls: [] + size: 797030 + timestamp: 1738196177597 +- pypi: https://files.pythonhosted.org/packages/eb/8d/776adee7bbf76365fdd7f2552710282c79a4ead5d2a46408c9043a2b70ba/networkx-3.5-py3-none-any.whl + name: networkx + version: '3.5' + sha256: 0030d386a9a06dee3565298b4a734b68589749a544acbb6c412dc9e2489ec6ec + requires_dist: + - numpy>=1.25 ; extra == 'default' + - scipy>=1.11.2 ; extra == 'default' + - matplotlib>=3.8 ; extra == 'default' + - pandas>=2.0 ; extra == 'default' + - pre-commit>=4.1 ; extra == 'developer' + - mypy>=1.15 ; extra == 'developer' + - sphinx>=8.0 ; extra == 'doc' + - pydata-sphinx-theme>=0.16 ; extra == 'doc' + - sphinx-gallery>=0.18 ; extra == 'doc' + - numpydoc>=1.8.0 ; extra == 'doc' + - pillow>=10 ; extra == 'doc' + - texext>=0.6.7 ; extra == 'doc' + - myst-nb>=1.1 ; extra == 'doc' + - intersphinx-registry ; extra == 'doc' + - osmnx>=2.0.0 ; extra == 'example' + - momepy>=0.7.2 ; extra == 'example' + - contextily>=1.6 ; extra == 'example' + - seaborn>=0.13 ; extra == 'example' + - cairocffi>=1.7 ; extra == 'example' + - igraph>=0.11 ; extra == 'example' + - scikit-learn>=1.5 ; extra == 'example' + - lxml>=4.6 ; extra == 'extra' + - pygraphviz>=1.14 ; extra == 'extra' + - pydot>=3.0.1 ; extra == 'extra' + - sympy>=1.10 ; extra == 'extra' + - pytest>=7.2 ; extra == 'test' + - pytest-cov>=4.0 ; extra == 'test' + - pytest-xdist>=3.0 ; extra == 'test' + - pytest-mpl ; extra == 'test-extras' + - pytest-randomly ; extra == 'test-extras' + requires_python: '>=3.11' +- pypi: https://files.pythonhosted.org/packages/d2/1d/1b658dbd2b9fa9c4c9f32accbfc0205d532c8c6194dc0f2a4c0428e7128a/nodeenv-1.9.1-py2.py3-none-any.whl + name: nodeenv + version: 1.9.1 + sha256: ba11c9782d29c27c70ffbdda2d7415098754709be8a7056d79a737cd901155c9 + requires_python: '>=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*' +- pypi: https://files.pythonhosted.org/packages/9a/a5/bf3db6e66c4b160d6ea10b534c381a1955dfab34cb1017ea93aa33c70ed3/numpy-2.3.3-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl + name: numpy + version: 2.3.3 + sha256: 5b83648633d46f77039c29078751f80da65aa64d5622a3cd62aaef9d835b6c93 + requires_python: '>=3.11' +- pypi: https://files.pythonhosted.org/packages/3e/46/bdd3370dcea2f95ef14af79dbf81e6927102ddf1cc54adc0024d61252fd9/numpy-2.3.4-cp313-cp313-macosx_11_0_arm64.whl + name: numpy + version: 2.3.4 + sha256: a13fc473b6db0be619e45f11f9e81260f7302f8d180c49a22b6e6120022596b3 + requires_python: '>=3.11' +- pypi: https://files.pythonhosted.org/packages/3e/d1/913fe563820f3c6b079f992458f7331278dcd7ba8427e8e745af37ddb44f/numpy-2.3.4-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl + name: numpy + version: 2.3.4 + sha256: 4ee6a571d1e4f0ea6d5f22d6e5fbd6ed1dc2b18542848e1e7301bd190500c9d7 + requires_python: '>=3.11' +- conda: https://conda.anaconda.org/conda-forge/linux-64/openssl-3.5.2-h26f9b46_0.conda + sha256: c9f54d4e8212f313be7b02eb962d0cb13a8dae015683a403d3accd4add3e520e + md5: ffffb341206dd0dab0c36053c048d621 + depends: + - __glibc >=2.17,<3.0.a0 + - ca-certificates + - libgcc >=14 + license: Apache-2.0 + license_family: Apache + purls: [] + size: 3128847 + timestamp: 1754465526100 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/openssl-3.5.4-h8e36d6e_0.conda + sha256: a24b318733c98903e2689adc7ef73448e27cbb10806852032c023f0ea4446fc5 + md5: 9303e8887afe539f78517951ce25cd13 + depends: + - ca-certificates + - libgcc >=14 + license: Apache-2.0 + license_family: Apache + purls: [] + size: 3644584 + timestamp: 1759326000128 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/openssl-3.5.4-h5503f6c_0.conda + sha256: f0512629f9589392c2fb9733d11e753d0eab8fc7602f96e4d7f3bd95c783eb07 + md5: 71118318f37f717eefe55841adb172fd + depends: + - __osx >=11.0 + - ca-certificates + license: Apache-2.0 + license_family: Apache + purls: [] + size: 3067808 + timestamp: 1759324763146 +- pypi: https://files.pythonhosted.org/packages/12/27/fb8d7338b4d551900fa3e580acbe7a0cf655d940e164cb5c00ec31961094/orderly_set-5.5.0-py3-none-any.whl + name: orderly-set + version: 5.5.0 + sha256: 46f0b801948e98f427b412fcabb831677194c05c3b699b80de260374baa0b1e7 + requires_dist: + - coverage~=7.6.0 ; extra == 'coverage' + - bump2version~=1.0.0 ; extra == 'dev' + - ipdb~=0.13.0 ; extra == 'dev' + - orjson ; extra == 'optimize' + - flake8~=7.1.0 ; extra == 'static' + - flake8-pyproject~=1.2.3 ; extra == 'static' + - pytest~=8.3.0 ; extra == 'test' + - pytest-benchmark~=5.1.0 ; extra == 'test' + - pytest-cov~=6.0.0 ; extra == 'test' + - python-dotenv~=1.0.0 ; extra == 'test' + requires_python: '>=3.8' +- pypi: https://files.pythonhosted.org/packages/20/12/38679034af332785aac8774540895e234f4d07f7545804097de4b666afd8/packaging-25.0-py3-none-any.whl + name: packaging + version: '25.0' + sha256: 29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484 + requires_python: '>=3.8' +- pypi: https://files.pythonhosted.org/packages/8f/52/0634adaace9be2d8cac9ef78f05c47f3a675882e068438b9d7ec7ef0c13f/pandas-2.3.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + name: pandas + version: 2.3.2 + sha256: 4ac8c320bded4718b298281339c1a50fb00a6ba78cb2a63521c39bec95b0209b + requires_dist: + - numpy>=1.22.4 ; python_full_version < '3.11' + - numpy>=1.23.2 ; python_full_version == '3.11.*' + - numpy>=1.26.0 ; python_full_version >= '3.12' + - python-dateutil>=2.8.2 + - pytz>=2020.1 + - tzdata>=2022.7 + - hypothesis>=6.46.1 ; extra == 'test' + - pytest>=7.3.2 ; extra == 'test' + - pytest-xdist>=2.2.0 ; extra == 'test' + - pyarrow>=10.0.1 ; extra == 'pyarrow' + - bottleneck>=1.3.6 ; extra == 'performance' + - numba>=0.56.4 ; extra == 'performance' + - numexpr>=2.8.4 ; extra == 'performance' + - scipy>=1.10.0 ; extra == 'computation' + - xarray>=2022.12.0 ; extra == 'computation' + - fsspec>=2022.11.0 ; extra == 'fss' + - s3fs>=2022.11.0 ; extra == 'aws' + - gcsfs>=2022.11.0 ; extra == 'gcp' + - pandas-gbq>=0.19.0 ; extra == 'gcp' + - odfpy>=1.4.1 ; extra == 'excel' + - openpyxl>=3.1.0 ; extra == 'excel' + - python-calamine>=0.1.7 ; extra == 'excel' + - pyxlsb>=1.0.10 ; extra == 'excel' + - xlrd>=2.0.1 ; extra == 'excel' + - xlsxwriter>=3.0.5 ; extra == 'excel' + - pyarrow>=10.0.1 ; extra == 'parquet' + - pyarrow>=10.0.1 ; extra == 'feather' + - tables>=3.8.0 ; extra == 'hdf5' + - pyreadstat>=1.2.0 ; extra == 'spss' + - sqlalchemy>=2.0.0 ; extra == 'postgresql' + - psycopg2>=2.9.6 ; extra == 'postgresql' + - adbc-driver-postgresql>=0.8.0 ; extra == 'postgresql' + - sqlalchemy>=2.0.0 ; extra == 'mysql' + - pymysql>=1.0.2 ; extra == 'mysql' + - sqlalchemy>=2.0.0 ; extra == 'sql-other' + - adbc-driver-postgresql>=0.8.0 ; extra == 'sql-other' + - adbc-driver-sqlite>=0.8.0 ; extra == 'sql-other' + - beautifulsoup4>=4.11.2 ; extra == 'html' + - html5lib>=1.1 ; extra == 'html' + - lxml>=4.9.2 ; extra == 'html' + - lxml>=4.9.2 ; extra == 'xml' + - matplotlib>=3.6.3 ; extra == 'plot' + - jinja2>=3.1.2 ; extra == 'output-formatting' + - tabulate>=0.9.0 ; extra == 'output-formatting' + - pyqt5>=5.15.9 ; extra == 'clipboard' + - qtpy>=2.3.0 ; extra == 'clipboard' + - zstandard>=0.19.0 ; extra == 'compression' + - dataframe-api-compat>=0.1.7 ; extra == 'consortium-standard' + - adbc-driver-postgresql>=0.8.0 ; extra == 'all' + - adbc-driver-sqlite>=0.8.0 ; extra == 'all' + - beautifulsoup4>=4.11.2 ; extra == 'all' + - bottleneck>=1.3.6 ; extra == 'all' + - dataframe-api-compat>=0.1.7 ; extra == 'all' + - fastparquet>=2022.12.0 ; extra == 'all' + - fsspec>=2022.11.0 ; extra == 'all' + - gcsfs>=2022.11.0 ; extra == 'all' + - html5lib>=1.1 ; extra == 'all' + - hypothesis>=6.46.1 ; extra == 'all' + - jinja2>=3.1.2 ; extra == 'all' + - lxml>=4.9.2 ; extra == 'all' + - matplotlib>=3.6.3 ; extra == 'all' + - numba>=0.56.4 ; extra == 'all' + - numexpr>=2.8.4 ; extra == 'all' + - odfpy>=1.4.1 ; extra == 'all' + - openpyxl>=3.1.0 ; extra == 'all' + - pandas-gbq>=0.19.0 ; extra == 'all' + - psycopg2>=2.9.6 ; extra == 'all' + - pyarrow>=10.0.1 ; extra == 'all' + - pymysql>=1.0.2 ; extra == 'all' + - pyqt5>=5.15.9 ; extra == 'all' + - pyreadstat>=1.2.0 ; extra == 'all' + - pytest>=7.3.2 ; extra == 'all' + - pytest-xdist>=2.2.0 ; extra == 'all' + - python-calamine>=0.1.7 ; extra == 'all' + - pyxlsb>=1.0.10 ; extra == 'all' + - qtpy>=2.3.0 ; extra == 'all' + - scipy>=1.10.0 ; extra == 'all' + - s3fs>=2022.11.0 ; extra == 'all' + - sqlalchemy>=2.0.0 ; extra == 'all' + - tables>=3.8.0 ; extra == 'all' + - tabulate>=0.9.0 ; extra == 'all' + - xarray>=2022.12.0 ; extra == 'all' + - xlrd>=2.0.1 ; extra == 'all' + - xlsxwriter>=3.0.5 ; extra == 'all' + - zstandard>=0.19.0 ; extra == 'all' + requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/16/87/9472cf4a487d848476865321de18cc8c920b8cab98453ab79dbbc98db63a/pandas-2.3.3-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl + name: pandas + version: 2.3.3 + sha256: e32e7cc9af0f1cc15548288a51a3b681cc2a219faa838e995f7dc53dbab1062d + requires_dist: + - numpy>=1.22.4 ; python_full_version < '3.11' + - numpy>=1.23.2 ; python_full_version == '3.11.*' + - numpy>=1.26.0 ; python_full_version >= '3.12' + - python-dateutil>=2.8.2 + - pytz>=2020.1 + - tzdata>=2022.7 + - hypothesis>=6.46.1 ; extra == 'test' + - pytest>=7.3.2 ; extra == 'test' + - pytest-xdist>=2.2.0 ; extra == 'test' + - pyarrow>=10.0.1 ; extra == 'pyarrow' + - bottleneck>=1.3.6 ; extra == 'performance' + - numba>=0.56.4 ; extra == 'performance' + - numexpr>=2.8.4 ; extra == 'performance' + - scipy>=1.10.0 ; extra == 'computation' + - xarray>=2022.12.0 ; extra == 'computation' + - fsspec>=2022.11.0 ; extra == 'fss' + - s3fs>=2022.11.0 ; extra == 'aws' + - gcsfs>=2022.11.0 ; extra == 'gcp' + - pandas-gbq>=0.19.0 ; extra == 'gcp' + - odfpy>=1.4.1 ; extra == 'excel' + - openpyxl>=3.1.0 ; extra == 'excel' + - python-calamine>=0.1.7 ; extra == 'excel' + - pyxlsb>=1.0.10 ; extra == 'excel' + - xlrd>=2.0.1 ; extra == 'excel' + - xlsxwriter>=3.0.5 ; extra == 'excel' + - pyarrow>=10.0.1 ; extra == 'parquet' + - pyarrow>=10.0.1 ; extra == 'feather' + - tables>=3.8.0 ; extra == 'hdf5' + - pyreadstat>=1.2.0 ; extra == 'spss' + - sqlalchemy>=2.0.0 ; extra == 'postgresql' + - psycopg2>=2.9.6 ; extra == 'postgresql' + - adbc-driver-postgresql>=0.8.0 ; extra == 'postgresql' + - sqlalchemy>=2.0.0 ; extra == 'mysql' + - pymysql>=1.0.2 ; extra == 'mysql' + - sqlalchemy>=2.0.0 ; extra == 'sql-other' + - adbc-driver-postgresql>=0.8.0 ; extra == 'sql-other' + - adbc-driver-sqlite>=0.8.0 ; extra == 'sql-other' + - beautifulsoup4>=4.11.2 ; extra == 'html' + - html5lib>=1.1 ; extra == 'html' + - lxml>=4.9.2 ; extra == 'html' + - lxml>=4.9.2 ; extra == 'xml' + - matplotlib>=3.6.3 ; extra == 'plot' + - jinja2>=3.1.2 ; extra == 'output-formatting' + - tabulate>=0.9.0 ; extra == 'output-formatting' + - pyqt5>=5.15.9 ; extra == 'clipboard' + - qtpy>=2.3.0 ; extra == 'clipboard' + - zstandard>=0.19.0 ; extra == 'compression' + - dataframe-api-compat>=0.1.7 ; extra == 'consortium-standard' + - adbc-driver-postgresql>=0.8.0 ; extra == 'all' + - adbc-driver-sqlite>=0.8.0 ; extra == 'all' + - beautifulsoup4>=4.11.2 ; extra == 'all' + - bottleneck>=1.3.6 ; extra == 'all' + - dataframe-api-compat>=0.1.7 ; extra == 'all' + - fastparquet>=2022.12.0 ; extra == 'all' + - fsspec>=2022.11.0 ; extra == 'all' + - gcsfs>=2022.11.0 ; extra == 'all' + - html5lib>=1.1 ; extra == 'all' + - hypothesis>=6.46.1 ; extra == 'all' + - jinja2>=3.1.2 ; extra == 'all' + - lxml>=4.9.2 ; extra == 'all' + - matplotlib>=3.6.3 ; extra == 'all' + - numba>=0.56.4 ; extra == 'all' + - numexpr>=2.8.4 ; extra == 'all' + - odfpy>=1.4.1 ; extra == 'all' + - openpyxl>=3.1.0 ; extra == 'all' + - pandas-gbq>=0.19.0 ; extra == 'all' + - psycopg2>=2.9.6 ; extra == 'all' + - pyarrow>=10.0.1 ; extra == 'all' + - pymysql>=1.0.2 ; extra == 'all' + - pyqt5>=5.15.9 ; extra == 'all' + - pyreadstat>=1.2.0 ; extra == 'all' + - pytest>=7.3.2 ; extra == 'all' + - pytest-xdist>=2.2.0 ; extra == 'all' + - python-calamine>=0.1.7 ; extra == 'all' + - pyxlsb>=1.0.10 ; extra == 'all' + - qtpy>=2.3.0 ; extra == 'all' + - scipy>=1.10.0 ; extra == 'all' + - s3fs>=2022.11.0 ; extra == 'all' + - sqlalchemy>=2.0.0 ; extra == 'all' + - tables>=3.8.0 ; extra == 'all' + - tabulate>=0.9.0 ; extra == 'all' + - xarray>=2022.12.0 ; extra == 'all' + - xlrd>=2.0.1 ; extra == 'all' + - xlsxwriter>=3.0.5 ; extra == 'all' + - zstandard>=0.19.0 ; extra == 'all' + requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/31/94/72fac03573102779920099bcac1c3b05975c2cb5f01eac609faf34bed1ca/pandas-2.3.3-cp313-cp313-macosx_11_0_arm64.whl + name: pandas + version: 2.3.3 + sha256: bdcd9d1167f4885211e401b3036c0c8d9e274eee67ea8d0758a256d60704cfe8 + requires_dist: + - numpy>=1.22.4 ; python_full_version < '3.11' + - numpy>=1.23.2 ; python_full_version == '3.11.*' + - numpy>=1.26.0 ; python_full_version >= '3.12' + - python-dateutil>=2.8.2 + - pytz>=2020.1 + - tzdata>=2022.7 + - hypothesis>=6.46.1 ; extra == 'test' + - pytest>=7.3.2 ; extra == 'test' + - pytest-xdist>=2.2.0 ; extra == 'test' + - pyarrow>=10.0.1 ; extra == 'pyarrow' + - bottleneck>=1.3.6 ; extra == 'performance' + - numba>=0.56.4 ; extra == 'performance' + - numexpr>=2.8.4 ; extra == 'performance' + - scipy>=1.10.0 ; extra == 'computation' + - xarray>=2022.12.0 ; extra == 'computation' + - fsspec>=2022.11.0 ; extra == 'fss' + - s3fs>=2022.11.0 ; extra == 'aws' + - gcsfs>=2022.11.0 ; extra == 'gcp' + - pandas-gbq>=0.19.0 ; extra == 'gcp' + - odfpy>=1.4.1 ; extra == 'excel' + - openpyxl>=3.1.0 ; extra == 'excel' + - python-calamine>=0.1.7 ; extra == 'excel' + - pyxlsb>=1.0.10 ; extra == 'excel' + - xlrd>=2.0.1 ; extra == 'excel' + - xlsxwriter>=3.0.5 ; extra == 'excel' + - pyarrow>=10.0.1 ; extra == 'parquet' + - pyarrow>=10.0.1 ; extra == 'feather' + - tables>=3.8.0 ; extra == 'hdf5' + - pyreadstat>=1.2.0 ; extra == 'spss' + - sqlalchemy>=2.0.0 ; extra == 'postgresql' + - psycopg2>=2.9.6 ; extra == 'postgresql' + - adbc-driver-postgresql>=0.8.0 ; extra == 'postgresql' + - sqlalchemy>=2.0.0 ; extra == 'mysql' + - pymysql>=1.0.2 ; extra == 'mysql' + - sqlalchemy>=2.0.0 ; extra == 'sql-other' + - adbc-driver-postgresql>=0.8.0 ; extra == 'sql-other' + - adbc-driver-sqlite>=0.8.0 ; extra == 'sql-other' + - beautifulsoup4>=4.11.2 ; extra == 'html' + - html5lib>=1.1 ; extra == 'html' + - lxml>=4.9.2 ; extra == 'html' + - lxml>=4.9.2 ; extra == 'xml' + - matplotlib>=3.6.3 ; extra == 'plot' + - jinja2>=3.1.2 ; extra == 'output-formatting' + - tabulate>=0.9.0 ; extra == 'output-formatting' + - pyqt5>=5.15.9 ; extra == 'clipboard' + - qtpy>=2.3.0 ; extra == 'clipboard' + - zstandard>=0.19.0 ; extra == 'compression' + - dataframe-api-compat>=0.1.7 ; extra == 'consortium-standard' + - adbc-driver-postgresql>=0.8.0 ; extra == 'all' + - adbc-driver-sqlite>=0.8.0 ; extra == 'all' + - beautifulsoup4>=4.11.2 ; extra == 'all' + - bottleneck>=1.3.6 ; extra == 'all' + - dataframe-api-compat>=0.1.7 ; extra == 'all' + - fastparquet>=2022.12.0 ; extra == 'all' + - fsspec>=2022.11.0 ; extra == 'all' + - gcsfs>=2022.11.0 ; extra == 'all' + - html5lib>=1.1 ; extra == 'all' + - hypothesis>=6.46.1 ; extra == 'all' + - jinja2>=3.1.2 ; extra == 'all' + - lxml>=4.9.2 ; extra == 'all' + - matplotlib>=3.6.3 ; extra == 'all' + - numba>=0.56.4 ; extra == 'all' + - numexpr>=2.8.4 ; extra == 'all' + - odfpy>=1.4.1 ; extra == 'all' + - openpyxl>=3.1.0 ; extra == 'all' + - pandas-gbq>=0.19.0 ; extra == 'all' + - psycopg2>=2.9.6 ; extra == 'all' + - pyarrow>=10.0.1 ; extra == 'all' + - pymysql>=1.0.2 ; extra == 'all' + - pyqt5>=5.15.9 ; extra == 'all' + - pyreadstat>=1.2.0 ; extra == 'all' + - pytest>=7.3.2 ; extra == 'all' + - pytest-xdist>=2.2.0 ; extra == 'all' + - python-calamine>=0.1.7 ; extra == 'all' + - pyxlsb>=1.0.10 ; extra == 'all' + - qtpy>=2.3.0 ; extra == 'all' + - scipy>=1.10.0 ; extra == 'all' + - s3fs>=2022.11.0 ; extra == 'all' + - sqlalchemy>=2.0.0 ; extra == 'all' + - tables>=3.8.0 ; extra == 'all' + - tabulate>=0.9.0 ; extra == 'all' + - xarray>=2022.12.0 ; extra == 'all' + - xlrd>=2.0.1 ; extra == 'all' + - xlsxwriter>=3.0.5 ; extra == 'all' + - zstandard>=0.19.0 ; extra == 'all' + requires_python: '>=3.9' +- conda: https://conda.anaconda.org/conda-forge/linux-64/pango-1.56.4-hadf4263_0.conda + sha256: 3613774ad27e48503a3a6a9d72017087ea70f1426f6e5541dbdb59a3b626eaaf + md5: 79f71230c069a287efe3a8614069ddf1 + depends: + - __glibc >=2.17,<3.0.a0 + - cairo >=1.18.4,<2.0a0 + - fontconfig >=2.15.0,<3.0a0 + - fonts-conda-ecosystem + - fribidi >=1.0.10,<2.0a0 + - harfbuzz >=11.0.1 + - libexpat >=2.7.0,<3.0a0 + - libfreetype >=2.13.3 + - libfreetype6 >=2.13.3 + - libgcc >=13 + - libglib >=2.84.2,<3.0a0 + - libpng >=1.6.49,<1.7.0a0 + - libzlib >=1.3.1,<2.0a0 + license: LGPL-2.1-or-later + purls: [] + size: 455420 + timestamp: 1751292466873 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/pango-1.56.4-he55ef5b_0.conda + sha256: dd36cd5b6bc1c2988291a6db9fa4eb8acade9b487f6f1da4eaa65a1eebb0a12d + md5: a22cc88bf6059c9bcc158c94c9aab5b8 + depends: + - cairo >=1.18.4,<2.0a0 + - fontconfig >=2.15.0,<3.0a0 + - fonts-conda-ecosystem + - fribidi >=1.0.10,<2.0a0 + - harfbuzz >=11.0.1 + - libexpat >=2.7.0,<3.0a0 + - libfreetype >=2.13.3 + - libfreetype6 >=2.13.3 + - libgcc >=13 + - libglib >=2.84.2,<3.0a0 + - libpng >=1.6.49,<1.7.0a0 + - libzlib >=1.3.1,<2.0a0 + license: LGPL-2.1-or-later + purls: [] + size: 468811 + timestamp: 1751293869070 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/pango-1.56.4-h875632e_0.conda + sha256: 705484ad60adee86cab1aad3d2d8def03a699ece438c864e8ac995f6f66401a6 + md5: 7d57f8b4b7acfc75c777bc231f0d31be + depends: + - __osx >=11.0 + - cairo >=1.18.4,<2.0a0 + - fontconfig >=2.15.0,<3.0a0 + - fonts-conda-ecosystem + - fribidi >=1.0.10,<2.0a0 + - harfbuzz >=11.0.1 + - libexpat >=2.7.0,<3.0a0 + - libfreetype >=2.13.3 + - libfreetype6 >=2.13.3 + - libglib >=2.84.2,<3.0a0 + - libpng >=1.6.49,<1.7.0a0 + - libzlib >=1.3.1,<2.0a0 + license: LGPL-2.1-or-later + purls: [] + size: 426931 + timestamp: 1751292636271 +- pypi: https://files.pythonhosted.org/packages/16/32/f8e3c85d1d5250232a5d3477a2a28cc291968ff175caeadaf3cc19ce0e4a/parso-0.8.5-py2.py3-none-any.whl + name: parso + version: 0.8.5 + sha256: 646204b5ee239c396d040b90f9e272e9a8017c630092bf59980beb62fd033887 + requires_dist: + - pytest ; extra == 'testing' + - docopt ; extra == 'testing' + - flake8==5.0.4 ; extra == 'qa' + - mypy==0.971 ; extra == 'qa' + - types-setuptools==67.2.0.1 ; extra == 'qa' + requires_python: '>=3.6' +- conda: https://conda.anaconda.org/conda-forge/linux-64/pcre2-10.46-h1321c63_0.conda + sha256: 5c7380c8fd3ad5fc0f8039069a45586aa452cf165264bc5a437ad80397b32934 + md5: 7fa07cb0fb1b625a089ccc01218ee5b1 + depends: + - __glibc >=2.17,<3.0.a0 + - bzip2 >=1.0.8,<2.0a0 + - libgcc >=14 + - libzlib >=1.3.1,<2.0a0 + license: BSD-3-Clause + license_family: BSD + purls: [] + size: 1209177 + timestamp: 1756742976157 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/pcre2-10.46-h15761aa_0.conda + sha256: 75800e60e0e44d957c691a964085f56c9ac37dcd75e6c6904809d7b68f39e4ea + md5: 5128cb5188b630a58387799ea1366e37 + depends: + - bzip2 >=1.0.8,<2.0a0 + - libgcc >=14 + - libzlib >=1.3.1,<2.0a0 + license: BSD-3-Clause + license_family: BSD + purls: [] + size: 1161914 + timestamp: 1756742893031 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/pcre2-10.46-h7125dd6_0.conda + sha256: 5bf2eeaa57aab6e8e95bea6bd6bb2a739f52eb10572d8ed259d25864d3528240 + md5: 0e6e82c3cc3835f4692022e9b9cd5df8 + depends: + - __osx >=11.0 + - bzip2 >=1.0.8,<2.0a0 + - libzlib >=1.3.1,<2.0a0 + license: BSD-3-Clause + license_family: BSD + purls: [] + size: 835080 + timestamp: 1756743041908 +- pypi: https://files.pythonhosted.org/packages/9e/c3/059298687310d527a58bb01f3b1965787ee3b40dce76752eda8b44e9a2c5/pexpect-4.9.0-py2.py3-none-any.whl + name: pexpect + version: 4.9.0 + sha256: 7236d1e080e4936be2dc3e326cec0af72acf9212a7e1d060210e70a47e253523 + requires_dist: + - ptyprocess>=0.5 +- pypi: https://files.pythonhosted.org/packages/d5/1c/a2a29649c0b1983d3ef57ee87a66487fdeb45132df66ab30dd37f7dbe162/pillow-11.3.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl + name: pillow + version: 11.3.0 + sha256: 13f87d581e71d9189ab21fe0efb5a23e9f28552d5be6979e84001d3b8505abe8 + requires_dist: + - furo ; extra == 'docs' + - olefile ; extra == 'docs' + - sphinx>=8.2 ; extra == 'docs' + - sphinx-autobuild ; extra == 'docs' + - sphinx-copybutton ; extra == 'docs' + - sphinx-inline-tabs ; extra == 'docs' + - sphinxext-opengraph ; extra == 'docs' + - olefile ; extra == 'fpx' + - olefile ; extra == 'mic' + - pyarrow ; extra == 'test-arrow' + - check-manifest ; extra == 'tests' + - coverage>=7.4.2 ; extra == 'tests' + - defusedxml ; extra == 'tests' + - markdown2 ; extra == 'tests' + - olefile ; extra == 'tests' + - packaging ; extra == 'tests' + - pyroma ; extra == 'tests' + - pytest ; extra == 'tests' + - pytest-cov ; extra == 'tests' + - pytest-timeout ; extra == 'tests' + - pytest-xdist ; extra == 'tests' + - trove-classifiers>=2024.10.12 ; extra == 'tests' + - typing-extensions ; python_full_version < '3.10' and extra == 'typing' + - defusedxml ; extra == 'xmp' + requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/20/39/c685d05c06deecfd4e2d1950e9a908aa2ca8bc4e6c3b12d93b9cafbd7837/pillow-12.0.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl + name: pillow + version: 12.0.0 + sha256: 0b817e7035ea7f6b942c13aa03bb554fc44fea70838ea21f8eb31c638326584e + requires_dist: + - furo ; extra == 'docs' + - olefile ; extra == 'docs' + - sphinx>=8.2 ; extra == 'docs' + - sphinx-autobuild ; extra == 'docs' + - sphinx-copybutton ; extra == 'docs' + - sphinx-inline-tabs ; extra == 'docs' + - sphinxext-opengraph ; extra == 'docs' + - olefile ; extra == 'fpx' + - olefile ; extra == 'mic' + - arro3-compute ; extra == 'test-arrow' + - arro3-core ; extra == 'test-arrow' + - nanoarrow ; extra == 'test-arrow' + - pyarrow ; extra == 'test-arrow' + - check-manifest ; extra == 'tests' + - coverage>=7.4.2 ; extra == 'tests' + - defusedxml ; extra == 'tests' + - markdown2 ; extra == 'tests' + - olefile ; extra == 'tests' + - packaging ; extra == 'tests' + - pyroma>=5 ; extra == 'tests' + - pytest ; extra == 'tests' + - pytest-cov ; extra == 'tests' + - pytest-timeout ; extra == 'tests' + - pytest-xdist ; extra == 'tests' + - trove-classifiers>=2024.10.12 ; extra == 'tests' + - defusedxml ; extra == 'xmp' + requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/83/06/48eab21dd561de2914242711434c0c0eb992ed08ff3f6107a5f44527f5e9/pillow-12.0.0-cp313-cp313-macosx_11_0_arm64.whl + name: pillow + version: 12.0.0 + sha256: 5193fde9a5f23c331ea26d0cf171fbf67e3f247585f50c08b3e205c7aeb4589b + requires_dist: + - furo ; extra == 'docs' + - olefile ; extra == 'docs' + - sphinx>=8.2 ; extra == 'docs' + - sphinx-autobuild ; extra == 'docs' + - sphinx-copybutton ; extra == 'docs' + - sphinx-inline-tabs ; extra == 'docs' + - sphinxext-opengraph ; extra == 'docs' + - olefile ; extra == 'fpx' + - olefile ; extra == 'mic' + - arro3-compute ; extra == 'test-arrow' + - arro3-core ; extra == 'test-arrow' + - nanoarrow ; extra == 'test-arrow' + - pyarrow ; extra == 'test-arrow' + - check-manifest ; extra == 'tests' + - coverage>=7.4.2 ; extra == 'tests' + - defusedxml ; extra == 'tests' + - markdown2 ; extra == 'tests' + - olefile ; extra == 'tests' + - packaging ; extra == 'tests' + - pyroma>=5 ; extra == 'tests' + - pytest ; extra == 'tests' + - pytest-cov ; extra == 'tests' + - pytest-timeout ; extra == 'tests' + - pytest-xdist ; extra == 'tests' + - trove-classifiers>=2024.10.12 ; extra == 'tests' + - defusedxml ; extra == 'xmp' + requires_python: '>=3.10' +- conda: https://conda.anaconda.org/conda-forge/linux-64/pixman-0.46.4-h54a6638_1.conda + sha256: 43d37bc9ca3b257c5dd7bf76a8426addbdec381f6786ff441dc90b1a49143b6a + md5: c01af13bdc553d1a8fbfff6e8db075f0 + depends: + - libgcc >=14 + - libstdcxx >=14 + - libgcc >=14 + - __glibc >=2.17,<3.0.a0 + license: MIT + license_family: MIT + purls: [] + size: 450960 + timestamp: 1754665235234 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/pixman-0.46.4-h7ac5ae9_1.conda + sha256: e6b0846a998f2263629cfeac7bca73565c35af13251969f45d385db537a514e4 + md5: 1587081d537bd4ae77d1c0635d465ba5 + depends: + - libgcc >=14 + - libstdcxx >=14 + - libgcc >=14 + license: MIT + license_family: MIT + purls: [] + size: 357913 + timestamp: 1754665583353 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/pixman-0.46.4-h81086ad_1.conda + sha256: 29c9b08a9b8b7810f9d4f159aecfd205fce051633169040005c0b7efad4bc718 + md5: 17c3d745db6ea72ae2fce17e7338547f + depends: + - __osx >=11.0 + - libcxx >=19 + license: MIT + license_family: MIT + purls: [] + size: 248045 + timestamp: 1754665282033 +- pypi: https://files.pythonhosted.org/packages/40/4b/2028861e724d3bd36227adfa20d3fd24c3fc6d52032f4a93c133be5d17ce/platformdirs-4.4.0-py3-none-any.whl + name: platformdirs + version: 4.4.0 + sha256: abd01743f24e5287cd7a5db3752faf1a2d65353f38ec26d98e25a6db65958c85 + requires_dist: + - furo>=2024.8.6 ; extra == 'docs' + - proselint>=0.14 ; extra == 'docs' + - sphinx-autodoc-typehints>=3 ; extra == 'docs' + - sphinx>=8.1.3 ; extra == 'docs' + - appdirs==1.4.4 ; extra == 'test' + - covdefaults>=2.3 ; extra == 'test' + - pytest-cov>=6 ; extra == 'test' + - pytest-mock>=3.14 ; extra == 'test' + - pytest>=8.3.4 ; extra == 'test' + - mypy>=1.14.1 ; extra == 'type' + requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/73/cb/ac7874b3e5d58441674fb70742e6c374b28b0c7cb988d37d991cde47166c/platformdirs-4.5.0-py3-none-any.whl + name: platformdirs + version: 4.5.0 + sha256: e578a81bb873cbb89a41fcc904c7ef523cc18284b7e3b3ccf06aca1403b7ebd3 + requires_dist: + - furo>=2025.9.25 ; extra == 'docs' + - proselint>=0.14 ; extra == 'docs' + - sphinx-autodoc-typehints>=3.2 ; extra == 'docs' + - sphinx>=8.2.3 ; extra == 'docs' + - appdirs==1.4.4 ; extra == 'test' + - covdefaults>=2.3 ; extra == 'test' + - pytest-cov>=7 ; extra == 'test' + - pytest-mock>=3.15.1 ; extra == 'test' + - pytest>=8.4.2 ; extra == 'test' + - mypy>=1.18.2 ; extra == 'type' + requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl + name: pluggy + version: 1.6.0 + sha256: e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746 + requires_dist: + - pre-commit ; extra == 'dev' + - tox ; extra == 'dev' + - pytest ; extra == 'testing' + - pytest-benchmark ; extra == 'testing' + - coverage ; extra == 'testing' + requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/5b/a5/987a405322d78a73b66e39e4a90e4ef156fd7141bf71df987e50717c321b/pre_commit-4.3.0-py2.py3-none-any.whl + name: pre-commit + version: 4.3.0 + sha256: 2b0747ad7e6e967169136edffee14c16e148a778a54e4f967921aa1ebf2308d8 + requires_dist: + - cfgv>=2.0.0 + - identify>=1.0.0 + - nodeenv>=0.11.1 + - pyyaml>=5.1 + - virtualenv>=20.10.0 + requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/27/11/574fe7d13acf30bfd0a8dd7fa1647040f2b8064f13f43e8c963b1e65093b/pre_commit-4.4.0-py2.py3-none-any.whl + name: pre-commit + version: 4.4.0 + sha256: b35ea52957cbf83dcc5d8ee636cbead8624e3a15fbfa61a370e42158ac8a5813 + requires_dist: + - cfgv>=2.0.0 + - identify>=1.0.0 + - nodeenv>=0.11.1 + - pyyaml>=5.1 + - virtualenv>=20.10.0 + requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/84/03/0d3ce49e2505ae70cf43bc5bb3033955d2fc9f932163e84dc0779cc47f48/prompt_toolkit-3.0.52-py3-none-any.whl + name: prompt-toolkit + version: 3.0.52 + sha256: 9aac639a3bbd33284347de5ad8d68ecc044b91a762dc39b7c21095fcd6a19955 + requires_dist: + - wcwidth + requires_python: '>=3.8' +- conda: https://conda.anaconda.org/conda-forge/linux-64/pthread-stubs-0.4-hb9d3cd8_1002.conda + sha256: 9c88f8c64590e9567c6c80823f0328e58d3b1efb0e1c539c0315ceca764e0973 + md5: b3c17d95b5a10c6e64a21fa17573e70e + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=13 + license: MIT + license_family: MIT + purls: [] + size: 8252 + timestamp: 1726802366959 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/pthread-stubs-0.4-h86ecc28_1002.conda + sha256: 977dfb0cb3935d748521dd80262fe7169ab82920afd38ed14b7fee2ea5ec01ba + md5: bb5a90c93e3bac3d5690acf76b4a6386 + depends: + - libgcc >=13 + license: MIT + license_family: MIT + purls: [] + size: 8342 + timestamp: 1726803319942 +- pypi: https://files.pythonhosted.org/packages/22/a6/858897256d0deac81a172289110f31629fc4cee19b6f01283303e18c8db3/ptyprocess-0.7.0-py2.py3-none-any.whl + name: ptyprocess + version: 0.7.0 + sha256: 4b41f3967fce3af57cc7e94b888626c18bf37a083e3651ca8feeb66d492fef35 +- pypi: https://files.pythonhosted.org/packages/8e/37/efad0257dc6e593a18957422533ff0f87ede7c9c6ea010a2177d738fb82f/pure_eval-0.2.3-py3-none-any.whl + name: pure-eval + version: 0.2.3 + sha256: 1db8e35b67b3d218d818ae653e27f06c3aa420901fa7b081ca98cbedc874e0d0 + requires_dist: + - pytest ; extra == 'tests' +- pypi: https://files.pythonhosted.org/packages/a0/e3/59cd50310fc9b59512193629e1984c1f95e5c8ae6e5d8c69532ccc65a7fe/pycparser-2.23-py3-none-any.whl + name: pycparser + version: '2.23' + sha256: e5c6e8d3fbad53479cab09ac03729e0a9faf2bee3db8208a550daf5af81a5934 + requires_python: '>=3.8' +- pypi: https://files.pythonhosted.org/packages/50/52/adaf4c8c100a8c49d2bd058e5b551f73dfd8cb89eb4911e25a0c469b6b4e/pycryptodome-3.23.0-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl + name: pycryptodome + version: 3.23.0 + sha256: 67bd81fcbe34f43ad9422ee8fd4843c8e7198dd88dd3d40e6de42ee65fbe1490 + requires_python: '>=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*' +- pypi: https://files.pythonhosted.org/packages/5f/e9/a09476d436d0ff1402ac3867d933c61805ec2326c6ea557aeeac3825604e/pycryptodome-3.23.0-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + name: pycryptodome + version: 3.23.0 + sha256: c8987bd3307a39bc03df5c8e0e3d8be0c4c3518b7f044b0f4c15d1aa78f52575 + requires_python: '>=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*' +- pypi: https://files.pythonhosted.org/packages/db/6c/a1f71542c969912bb0e106f64f60a56cc1f0fabecf9396f45accbe63fa68/pycryptodome-3.23.0-cp37-abi3-macosx_10_9_universal2.whl + name: pycryptodome + version: 3.23.0 + sha256: 187058ab80b3281b1de11c2e6842a357a1f71b42cb1e15bce373f3d238135c27 + requires_python: '>=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*' +- pypi: https://files.pythonhosted.org/packages/5a/87/b70ad306ebb6f9b585f114d0ac2137d792b48be34d732d60e597c2f8465a/pydantic-2.12.5-py3-none-any.whl + name: pydantic + version: 2.12.5 + sha256: e561593fccf61e8a20fc46dfc2dfe075b8be7d0188df33f221ad1f0139180f9d + requires_dist: + - annotated-types>=0.6.0 + - pydantic-core==2.41.5 + - typing-extensions>=4.14.1 + - typing-inspection>=0.4.2 + - email-validator>=2.0.0 ; extra == 'email' + - tzdata ; python_full_version >= '3.9' and sys_platform == 'win32' and extra == 'timezone' + requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/15/df/a4c740c0943e93e6500f9eb23f4ca7ec9bf71b19e608ae5b579678c8d02f/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl + name: pydantic-core + version: 2.41.5 + sha256: 0cbaad15cb0c90aa221d43c00e77bb33c93e8d36e0bf74760cd00e732d10a6a0 + requires_dist: + - typing-extensions>=4.14.1 + requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/94/02/abfa0e0bda67faa65fef1c84971c7e45928e108fe24333c81f3bfe35d5f5/pydantic_core-2.41.5-cp313-cp313-macosx_11_0_arm64.whl + name: pydantic-core + version: 2.41.5 + sha256: 112e305c3314f40c93998e567879e887a3160bb8689ef3d2c04b6cc62c33ac34 + requires_dist: + - typing-extensions>=4.14.1 + requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/cf/4e/35a80cae583a37cf15604b44240e45c05e04e86f9cfd766623149297e971/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + name: pydantic-core + version: 2.41.5 + sha256: 406bf18d345822d6c21366031003612b9c77b3e29ffdb0f612367352aab7d586 + requires_dist: + - typing-extensions>=4.14.1 + requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/c1/60/5d4751ba3f4a40a6891f24eec885f51afd78d208498268c734e256fb13c4/pydantic_settings-2.12.0-py3-none-any.whl + name: pydantic-settings + version: 2.12.0 + sha256: fddb9fd99a5b18da837b29710391e945b1e30c135477f484084ee513adb93809 + requires_dist: + - pydantic>=2.7.0 + - python-dotenv>=0.21.0 + - typing-inspection>=0.4.0 + - boto3-stubs[secretsmanager] ; extra == 'aws-secrets-manager' + - boto3>=1.35.0 ; extra == 'aws-secrets-manager' + - azure-identity>=1.16.0 ; extra == 'azure-key-vault' + - azure-keyvault-secrets>=4.8.0 ; extra == 'azure-key-vault' + - google-cloud-secret-manager>=2.23.1 ; extra == 'gcp-secret-manager' + - tomli>=2.0.1 ; extra == 'toml' + - pyyaml>=6.0.1 ; extra == 'yaml' + requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/7e/32/a7125fb28c4261a627f999d5fb4afff25b523800faed2c30979949d6facd/pydot-4.0.1-py3-none-any.whl + name: pydot + version: 4.0.1 + sha256: 869c0efadd2708c0be1f916eb669f3d664ca684bc57ffb7ecc08e70d5e93fee6 + requires_dist: + - pyparsing>=3.1.0 + - ruff ; extra == 'lint' + - mypy ; extra == 'types' + - pydot[lint] ; extra == 'dev' + - pydot[types] ; extra == 'dev' + - chardet ; extra == 'dev' + - parameterized ; extra == 'dev' + - pydot[dev] ; extra == 'tests' + - tox ; extra == 'tests' + - pytest ; extra == 'tests' + - pytest-cov ; extra == 'tests' + - pytest-xdist[psutil] ; extra == 'tests' + - zest-releaser[recommended] ; extra == 'release' + requires_python: '>=3.8' +- pypi: https://files.pythonhosted.org/packages/c7/21/705964c7812476f378728bdf590ca4b771ec72385c533964653c68e86bdc/pygments-2.19.2-py3-none-any.whl + name: pygments + version: 2.19.2 + sha256: 86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b + requires_dist: + - colorama>=0.4.6 ; extra == 'windows-terminal' + requires_python: '>=3.8' +- pypi: https://files.pythonhosted.org/packages/7c/4c/ad33b92b9864cbde84f259d5df035a6447f91891f5be77788e2a3892bce3/pymysql-1.1.2-py3-none-any.whl + name: pymysql + version: 1.1.2 + sha256: e6b1d89711dd51f8f74b1631fe08f039e7d76cf67a42a323d3178f0f25762ed9 + requires_dist: + - cryptography ; extra == 'rsa' + - pynacl>=1.4.0 ; extra == 'ed25519' + requires_python: '>=3.8' +- pypi: https://files.pythonhosted.org/packages/53/b8/fbab973592e23ae313042d450fc26fa24282ebffba21ba373786e1ce63b4/pyparsing-3.2.4-py3-none-any.whl + name: pyparsing + version: 3.2.4 + sha256: 91d0fcde680d42cd031daf3a6ba20da3107e08a75de50da58360e7d94ab24d36 + requires_dist: + - railroad-diagrams ; extra == 'diagrams' + - jinja2 ; extra == 'diagrams' + requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/10/5e/1aa9a93198c6b64513c9d7752de7422c06402de6600a8767da1524f9570b/pyparsing-3.2.5-py3-none-any.whl + name: pyparsing + version: 3.2.5 + sha256: e38a4f02064cf41fe6593d328d0512495ad1f3d8a91c4f73fc401b3079a59a5e + requires_dist: + - railroad-diagrams ; extra == 'diagrams' + - jinja2 ; extra == 'diagrams' + requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/a8/a4/20da314d277121d6534b3a980b29035dcd51e6744bd79075a6ce8fa4eb8d/pytest-8.4.2-py3-none-any.whl + name: pytest + version: 8.4.2 + sha256: 872f880de3fc3a5bdc88a11b39c9710c3497a547cfa9320bc3c5e62fbf272e79 + requires_dist: + - colorama>=0.4 ; sys_platform == 'win32' + - exceptiongroup>=1 ; python_full_version < '3.11' + - iniconfig>=1 + - packaging>=20 + - pluggy>=1.5,<2 + - pygments>=2.7.2 + - tomli>=1 ; python_full_version < '3.11' + - argcomplete ; extra == 'dev' + - attrs>=19.2 ; extra == 'dev' + - hypothesis>=3.56 ; extra == 'dev' + - mock ; extra == 'dev' + - requests ; extra == 'dev' + - setuptools ; extra == 'dev' + - xmlschema ; extra == 'dev' + requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/72/99/cafef234114a3b6d9f3aaed0723b437c40c57bdb7b3e4c3a575bc4890052/pytest-9.0.0-py3-none-any.whl + name: pytest + version: 9.0.0 + sha256: e5ccdf10b0bac554970ee88fc1a4ad0ee5d221f8ef22321f9b7e4584e19d7f96 + requires_dist: + - colorama>=0.4 ; sys_platform == 'win32' + - exceptiongroup>=1 ; python_full_version < '3.11' + - iniconfig>=1.0.1 + - packaging>=22 + - pluggy>=1.5,<2 + - pygments>=2.7.2 + - tomli>=1 ; python_full_version < '3.11' + - argcomplete ; extra == 'dev' + - attrs>=19.2 ; extra == 'dev' + - hypothesis>=3.56 ; extra == 'dev' + - mock ; extra == 'dev' + - requests ; extra == 'dev' + - setuptools ; extra == 'dev' + - xmlschema ; extra == 'dev' + requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/ee/49/1377b49de7d0c1ce41292161ea0f721913fa8722c19fb9c1e3aa0367eecb/pytest_cov-7.0.0-py3-none-any.whl + name: pytest-cov + version: 7.0.0 + sha256: 3b8e9558b16cc1479da72058bdecf8073661c7f57f7d3c5f22a1c23507f2d861 + requires_dist: + - coverage[toml]>=7.10.6 + - pluggy>=1.2 + - pytest>=7 + - process-tests ; extra == 'testing' + - pytest-xdist ; extra == 'testing' + - virtualenv ; extra == 'testing' + requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/de/b8/87cfb16045c9d4092cfcf526135d73b88101aac83bc1adcf82dfb5fd3833/pytest_env-1.1.5-py3-none-any.whl + name: pytest-env + version: 1.1.5 + sha256: ce90cf8772878515c24b31cd97c7fa1f4481cd68d588419fd45f10ecaee6bc30 + requires_dist: + - pytest>=8.3.3 + - tomli>=2.0.1 ; python_full_version < '3.11' + - covdefaults>=2.3 ; extra == 'testing' + - coverage>=7.6.1 ; extra == 'testing' + - pytest-mock>=3.14 ; extra == 'testing' + requires_python: '>=3.8' +- pypi: https://files.pythonhosted.org/packages/27/98/822b924a4a3eb58aacba84444c7439fce32680592f394de26af9c76e2569/pytest_env-1.2.0-py3-none-any.whl + name: pytest-env + version: 1.2.0 + sha256: d7e5b7198f9b83c795377c09feefa45d56083834e60d04767efd64819fc9da00 + requires_dist: + - pytest>=8.4.2 + - tomli>=2.2.1 ; python_full_version < '3.11' + - covdefaults>=2.3 ; extra == 'testing' + - coverage>=7.10.7 ; extra == 'testing' + - pytest-mock>=3.15.1 ; extra == 'testing' + requires_python: '>=3.10' +- conda: https://conda.anaconda.org/conda-forge/linux-64/python-3.13.7-h2b335a9_100_cp313.conda + build_number: 100 + sha256: 16cc30a5854f31ca6c3688337d34e37a79cdc518a06375fe3482ea8e2d6b34c8 + md5: 724dcf9960e933838247971da07fe5cf + depends: + - __glibc >=2.17,<3.0.a0 + - bzip2 >=1.0.8,<2.0a0 + - ld_impl_linux-64 >=2.36.1 + - libexpat >=2.7.1,<3.0a0 + - libffi >=3.4.6,<3.5.0a0 + - libgcc >=14 + - liblzma >=5.8.1,<6.0a0 + - libmpdec >=4.0.0,<5.0a0 + - libsqlite >=3.50.4,<4.0a0 + - libuuid >=2.38.1,<3.0a0 + - libzlib >=1.3.1,<2.0a0 + - ncurses >=6.5,<7.0a0 + - openssl >=3.5.2,<4.0a0 + - python_abi 3.13.* *_cp313 + - readline >=8.2,<9.0a0 + - tk >=8.6.13,<8.7.0a0 + - tzdata + license: Python-2.0 + purls: [] + size: 33583088 + timestamp: 1756911465277 + python_site_packages_path: lib/python3.13/site-packages +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/python-3.13.9-h4c0d347_101_cp313.conda + build_number: 101 + sha256: 95f11d8f8e8007ead0927ff15401a9a48a28df92b284f41a08824955c009e974 + md5: b62a2e7c210e4bffa9aaa041f7152a25 + depends: + - bzip2 >=1.0.8,<2.0a0 + - ld_impl_linux-aarch64 >=2.36.1 + - libexpat >=2.7.1,<3.0a0 + - libffi >=3.5.2,<3.6.0a0 + - libgcc >=14 + - liblzma >=5.8.1,<6.0a0 + - libmpdec >=4.0.0,<5.0a0 + - libsqlite >=3.50.4,<4.0a0 + - libuuid >=2.41.2,<3.0a0 + - libzlib >=1.3.1,<2.0a0 + - ncurses >=6.5,<7.0a0 + - openssl >=3.5.4,<4.0a0 + - python_abi 3.13.* *_cp313 + - readline >=8.2,<9.0a0 + - tk >=8.6.13,<8.7.0a0 + - tzdata + license: Python-2.0 + purls: [] + size: 33737136 + timestamp: 1761175607146 + python_site_packages_path: lib/python3.13/site-packages +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/python-3.13.9-hfc2f54d_101_cp313.conda + build_number: 101 + sha256: 516229f780b98783a5ef4112a5a4b5e5647d4f0177c4621e98aa60bb9bc32f98 + md5: a4241bce59eecc74d4d2396e108c93b8 + depends: + - __osx >=11.0 + - bzip2 >=1.0.8,<2.0a0 + - libexpat >=2.7.1,<3.0a0 + - libffi >=3.5.2,<3.6.0a0 + - liblzma >=5.8.1,<6.0a0 + - libmpdec >=4.0.0,<5.0a0 + - libsqlite >=3.50.4,<4.0a0 + - libzlib >=1.3.1,<2.0a0 + - ncurses >=6.5,<7.0a0 + - openssl >=3.5.4,<4.0a0 + - python_abi 3.13.* *_cp313 + - readline >=8.2,<9.0a0 + - tk >=8.6.13,<8.7.0a0 + - tzdata + license: Python-2.0 + purls: [] + size: 11915380 + timestamp: 1761176793936 + python_site_packages_path: lib/python3.13/site-packages +- pypi: https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl + name: python-dateutil + version: 2.9.0.post0 + sha256: a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427 + requires_dist: + - six>=1.5 + requires_python: '>=2.7,!=3.0.*,!=3.1.*,!=3.2.*' +- pypi: https://files.pythonhosted.org/packages/14/1b/a298b06749107c305e1fe0f814c6c74aea7b2f1e10989cb30f544a1b3253/python_dotenv-1.2.1-py3-none-any.whl + name: python-dotenv + version: 1.2.1 + sha256: b81ee9561e9ca4004139c6cbba3a238c32b03e4894671e181b671e8cb8425d61 + requires_dist: + - click>=5.0 ; extra == 'cli' + requires_python: '>=3.9' +- conda: https://conda.anaconda.org/conda-forge/noarch/python_abi-3.13-8_cp313.conda + build_number: 8 + sha256: 210bffe7b121e651419cb196a2a63687b087497595c9be9d20ebe97dd06060a7 + md5: 94305520c52a4aa3f6c2b1ff6008d9f8 + constrains: + - python 3.13.* *_cp313 + license: BSD-3-Clause + license_family: BSD + purls: [] + size: 7002 + timestamp: 1752805902938 +- pypi: https://files.pythonhosted.org/packages/81/c4/34e93fe5f5429d7570ec1fa436f1986fb1f00c3e0f43a589fe2bbcd22c3f/pytz-2025.2-py2.py3-none-any.whl + name: pytz + version: '2025.2' + sha256: 5ddf76296dd8c44c26eb8f4b6f35488f3ccbf6fbbd7adee0b7262d43f0ec2f00 +- pypi: https://files.pythonhosted.org/packages/04/24/b7721e4845c2f162d26f50521b825fb061bc0a5afcf9a386840f23ea19fa/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + name: pyyaml + version: 6.0.2 + sha256: 70b189594dbe54f75ab3a1acec5f1e3faa7e8cf2f1e08d9b561cb41b845f69d5 + requires_python: '>=3.8' +- pypi: https://files.pythonhosted.org/packages/50/31/b20f376d3f810b9b2371e72ef5adb33879b25edb7a6d072cb7ca0c486398/pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl + name: pyyaml + version: 6.0.3 + sha256: ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c + requires_python: '>=3.8' +- pypi: https://files.pythonhosted.org/packages/b1/16/95309993f1d3748cd644e02e38b75d50cbc0d9561d21f390a76242ce073f/pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl + name: pyyaml + version: 6.0.3 + sha256: 2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1 + requires_python: '>=3.8' +- conda: https://conda.anaconda.org/conda-forge/linux-64/readline-8.2-h8c095d6_2.conda + sha256: 2d6d0c026902561ed77cd646b5021aef2d4db22e57a5b0178dfc669231e06d2c + md5: 283b96675859b20a825f8fa30f311446 + depends: + - libgcc >=13 + - ncurses >=6.5,<7.0a0 + license: GPL-3.0-only + license_family: GPL + purls: [] + size: 282480 + timestamp: 1740379431762 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/readline-8.2-h8382b9d_2.conda + sha256: 54bed3a3041befaa9f5acde4a37b1a02f44705b7796689574bcf9d7beaad2959 + md5: c0f08fc2737967edde1a272d4bf41ed9 + depends: + - libgcc >=13 + - ncurses >=6.5,<7.0a0 + license: GPL-3.0-only + license_family: GPL + purls: [] + size: 291806 + timestamp: 1740380591358 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/readline-8.2-h1d1bf99_2.conda + sha256: 7db04684d3904f6151eff8673270922d31da1eea7fa73254d01c437f49702e34 + md5: 63ef3f6e6d6d5c589e64f11263dc5676 + depends: + - ncurses >=6.5,<7.0a0 + license: GPL-3.0-only + license_family: GPL + purls: [] + size: 252359 + timestamp: 1740379663071 +- pypi: https://files.pythonhosted.org/packages/1e/db/4254e3eabe8020b458f1a747140d32277ec7a271daf1d235b70dc0b4e6e3/requests-2.32.5-py3-none-any.whl + name: requests + version: 2.32.5 + sha256: 2462f94637a34fd532264295e186976db0f5d453d1cdd31473c85a6a161affb6 + requires_dist: + - charset-normalizer>=2,<4 + - idna>=2.5,<4 + - urllib3>=1.21.1,<3 + - certifi>=2017.4.17 + - pysocks>=1.5.6,!=1.5.7 ; extra == 'socks' + - chardet>=3.0.2,<6 ; extra == 'use-chardet-on-py3' + requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/7d/f8/2be49047f929d6965401855461e697ab185e1a6a683d914c5c19c7962d9e/ruff-0.14.9-py3-none-macosx_11_0_arm64.whl + name: ruff + version: 0.14.9 + sha256: d5dc3473c3f0e4a1008d0ef1d75cee24a48e254c8bed3a7afdd2b4392657ed2c + requires_python: '>=3.7' +- pypi: https://files.pythonhosted.org/packages/9e/e9/08840ff5127916bb989c86f18924fd568938b06f58b60e206176f327c0fe/ruff-0.14.9-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl + name: ruff + version: 0.14.9 + sha256: 84bf7c698fc8f3cb8278830fb6b5a47f9bcc1ed8cb4f689b9dd02698fa840697 + requires_python: '>=3.7' +- pypi: https://files.pythonhosted.org/packages/f3/51/0489a6a5595b7760b5dbac0dd82852b510326e7d88d51dbffcd2e07e3ff3/ruff-0.14.9-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + name: ruff + version: 0.14.9 + sha256: 72034534e5b11e8a593f517b2f2f2b273eb68a30978c6a2d40473ad0aaa4cb4a + requires_python: '>=3.7' +- pypi: https://files.pythonhosted.org/packages/a3/dc/17031897dae0efacfea57dfd3a82fdd2a2aeb58e0ff71b77b87e44edc772/setuptools-80.9.0-py3-none-any.whl + name: setuptools + version: 80.9.0 + sha256: 062d34222ad13e0cc312a4c02d73f059e86a4acbfbdea8f8f76b28c99f306922 + requires_dist: + - pytest>=6,!=8.1.* ; extra == 'test' + - virtualenv>=13.0.0 ; extra == 'test' + - wheel>=0.44.0 ; extra == 'test' + - pip>=19.1 ; extra == 'test' + - packaging>=24.2 ; extra == 'test' + - jaraco-envs>=2.2 ; extra == 'test' + - pytest-xdist>=3 ; extra == 'test' + - jaraco-path>=3.7.2 ; extra == 'test' + - build[virtualenv]>=1.0.3 ; extra == 'test' + - filelock>=3.4.0 ; extra == 'test' + - ini2toml[lite]>=0.14 ; extra == 'test' + - tomli-w>=1.0.0 ; extra == 'test' + - pytest-timeout ; extra == 'test' + - pytest-perf ; sys_platform != 'cygwin' and extra == 'test' + - jaraco-develop>=7.21 ; python_full_version >= '3.9' and sys_platform != 'cygwin' and extra == 'test' + - pytest-home>=0.5 ; extra == 'test' + - pytest-subprocess ; extra == 'test' + - pyproject-hooks!=1.1 ; extra == 'test' + - jaraco-test>=5.5 ; extra == 'test' + - sphinx>=3.5 ; extra == 'doc' + - jaraco-packaging>=9.3 ; extra == 'doc' + - rst-linker>=1.9 ; extra == 'doc' + - furo ; extra == 'doc' + - sphinx-lint ; extra == 'doc' + - jaraco-tidelift>=1.4 ; extra == 'doc' + - pygments-github-lexers==0.0.5 ; extra == 'doc' + - sphinx-favicon ; extra == 'doc' + - sphinx-inline-tabs ; extra == 'doc' + - sphinx-reredirects ; extra == 'doc' + - sphinxcontrib-towncrier ; extra == 'doc' + - sphinx-notfound-page>=1,<2 ; extra == 'doc' + - pyproject-hooks!=1.1 ; extra == 'doc' + - towncrier<24.7 ; extra == 'doc' + - packaging>=24.2 ; extra == 'core' + - more-itertools>=8.8 ; extra == 'core' + - jaraco-text>=3.7 ; extra == 'core' + - importlib-metadata>=6 ; python_full_version < '3.10' and extra == 'core' + - tomli>=2.0.1 ; python_full_version < '3.11' and extra == 'core' + - wheel>=0.43.0 ; extra == 'core' + - platformdirs>=4.2.2 ; extra == 'core' + - jaraco-functools>=4 ; extra == 'core' + - more-itertools ; extra == 'core' + - pytest-checkdocs>=2.4 ; extra == 'check' + - pytest-ruff>=0.2.1 ; sys_platform != 'cygwin' and extra == 'check' + - ruff>=0.8.0 ; sys_platform != 'cygwin' and extra == 'check' + - pytest-cov ; extra == 'cover' + - pytest-enabler>=2.2 ; extra == 'enabler' + - pytest-mypy ; extra == 'type' + - mypy==1.14.* ; extra == 'type' + - importlib-metadata>=7.0.2 ; python_full_version < '3.10' and extra == 'type' + - jaraco-develop>=7.21 ; sys_platform != 'cygwin' and extra == 'type' + requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl + name: six + version: 1.17.0 + sha256: 4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274 + requires_python: '>=2.7,!=3.0.*,!=3.1.*,!=3.2.*' +- pypi: https://files.pythonhosted.org/packages/f1/7b/ce1eafaf1a76852e2ec9b22edecf1daa58175c090266e9f6c64afcd81d91/stack_data-0.6.3-py3-none-any.whl + name: stack-data + version: 0.6.3 + sha256: d5558e0c25a4cb0853cddad3d77da9891a08cb85dd9f9f91b9f8cd66e511e695 + requires_dist: + - executing>=1.2.0 + - asttokens>=2.1.0 + - pure-eval + - pytest ; extra == 'tests' + - typeguard ; extra == 'tests' + - pygments ; extra == 'tests' + - littleutils ; extra == 'tests' + - cython ; extra == 'tests' +- conda: https://conda.anaconda.org/conda-forge/linux-64/tk-8.6.13-noxft_hd72426e_102.conda + sha256: a84ff687119e6d8752346d1d408d5cf360dee0badd487a472aa8ddedfdc219e1 + md5: a0116df4f4ed05c303811a837d5b39d8 + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=13 + - libzlib >=1.3.1,<2.0a0 + license: TCL + license_family: BSD + purls: [] + size: 3285204 + timestamp: 1748387766691 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/tk-8.6.13-noxft_h5688188_102.conda + sha256: 46e10488e9254092c655257c18fcec0a9864043bdfbe935a9fbf4fb2028b8514 + md5: 2562c9bfd1de3f9c590f0fe53858d85c + depends: + - libgcc >=13 + - libzlib >=1.3.1,<2.0a0 + license: TCL + license_family: BSD + purls: [] + size: 3342845 + timestamp: 1748393219221 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/tk-8.6.13-h892fb3f_2.conda + sha256: cb86c522576fa95c6db4c878849af0bccfd3264daf0cc40dd18e7f4a7bfced0e + md5: 7362396c170252e7b7b0c8fb37fe9c78 + depends: + - __osx >=11.0 + - libzlib >=1.3.1,<2.0a0 + license: TCL + license_family: BSD + purls: [] + size: 3125538 + timestamp: 1748388189063 +- pypi: https://files.pythonhosted.org/packages/d0/30/dc54f88dd4a2b5dc8a0279bdd7270e735851848b762aeb1c1184ed1f6b14/tqdm-4.67.1-py3-none-any.whl + name: tqdm + version: 4.67.1 + sha256: 26445eca388f82e72884e0d580d5464cd801a3ea01e63e5601bdff9ba6a48de2 + requires_dist: + - colorama ; sys_platform == 'win32' + - pytest>=6 ; extra == 'dev' + - pytest-cov ; extra == 'dev' + - pytest-timeout ; extra == 'dev' + - pytest-asyncio>=0.24 ; extra == 'dev' + - nbval ; extra == 'dev' + - requests ; extra == 'discord' + - slack-sdk ; extra == 'slack' + - requests ; extra == 'telegram' + - ipywidgets>=6 ; extra == 'notebook' + requires_python: '>=3.7' +- pypi: https://files.pythonhosted.org/packages/00/c0/8f5d070730d7836adc9c9b6408dec68c6ced86b304a9b26a14df072a6e8c/traitlets-5.14.3-py3-none-any.whl + name: traitlets + version: 5.14.3 + sha256: b74e89e397b1ed28cc831db7aea759ba6640cb3de13090ca145426688ff1ac4f + requires_dist: + - myst-parser ; extra == 'docs' + - pydata-sphinx-theme ; extra == 'docs' + - sphinx ; extra == 'docs' + - argcomplete>=3.0.3 ; extra == 'test' + - mypy>=1.7.0 ; extra == 'test' + - pre-commit ; extra == 'test' + - pytest-mock ; extra == 'test' + - pytest-mypy-testing ; extra == 'test' + - pytest>=7.0,<8.2 ; extra == 'test' + requires_python: '>=3.8' +- pypi: https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl + name: typing-extensions + version: 4.15.0 + sha256: f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548 + requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl + name: typing-inspection + version: 0.4.2 + sha256: 4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7 + requires_dist: + - typing-extensions>=4.12.0 + requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/5c/23/c7abc0ca0a1526a0774eca151daeb8de62ec457e77262b66b359c3c7679e/tzdata-2025.2-py2.py3-none-any.whl + name: tzdata + version: '2025.2' + sha256: 1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8 + requires_python: '>=2' +- conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2025b-h78e105d_0.conda + sha256: 5aaa366385d716557e365f0a4e9c3fca43ba196872abbbe3d56bb610d131e192 + md5: 4222072737ccff51314b5ece9c7d6f5a + license: LicenseRef-Public-Domain + purls: [] + size: 122968 + timestamp: 1742727099393 +- pypi: https://files.pythonhosted.org/packages/a7/c2/fe1e52489ae3122415c51f387e221dd0773709bad6c6cdaa599e8a2c5185/urllib3-2.5.0-py3-none-any.whl + name: urllib3 + version: 2.5.0 + sha256: e6b01673c0fa6a13e374b50871808eb3bf7046c4b125b216f6bf1cc604cff0dc + requires_dist: + - brotli>=1.0.9 ; platform_python_implementation == 'CPython' and extra == 'brotli' + - brotlicffi>=0.8.0 ; platform_python_implementation != 'CPython' and extra == 'brotli' + - h2>=4,<5 ; extra == 'h2' + - pysocks>=1.5.6,!=1.5.7,<2.0 ; extra == 'socks' + - zstandard>=0.18.0 ; extra == 'zstd' + requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/76/06/04c8e804f813cf972e3262f3f8584c232de64f0cde9f703b46cf53a45090/virtualenv-20.34.0-py3-none-any.whl + name: virtualenv + version: 20.34.0 + sha256: 341f5afa7eee943e4984a9207c025feedd768baff6753cd660c857ceb3e36026 + requires_dist: + - distlib>=0.3.7,<1 + - filelock>=3.12.2,<4 + - importlib-metadata>=6.6 ; python_full_version < '3.8' + - platformdirs>=3.9.1,<5 + - typing-extensions>=4.13.2 ; python_full_version < '3.11' + - furo>=2023.7.26 ; extra == 'docs' + - proselint>=0.13 ; extra == 'docs' + - sphinx>=7.1.2,!=7.3 ; extra == 'docs' + - sphinx-argparse>=0.4 ; extra == 'docs' + - sphinxcontrib-towncrier>=0.2.1a0 ; extra == 'docs' + - towncrier>=23.6 ; extra == 'docs' + - covdefaults>=2.3 ; extra == 'test' + - coverage-enable-subprocess>=1 ; extra == 'test' + - coverage>=7.2.7 ; extra == 'test' + - flaky>=3.7 ; extra == 'test' + - packaging>=23.1 ; extra == 'test' + - pytest-env>=0.8.2 ; extra == 'test' + - pytest-freezer>=0.4.8 ; (python_full_version >= '3.13' and platform_python_implementation == 'CPython' and sys_platform == 'win32' and extra == 'test') or (platform_python_implementation == 'GraalVM' and extra == 'test') or (platform_python_implementation == 'PyPy' and extra == 'test') + - pytest-mock>=3.11.1 ; extra == 'test' + - pytest-randomly>=3.12 ; extra == 'test' + - pytest-timeout>=2.1 ; extra == 'test' + - pytest>=7.4 ; extra == 'test' + - setuptools>=68 ; extra == 'test' + - time-machine>=2.10 ; platform_python_implementation == 'CPython' and extra == 'test' + requires_python: '>=3.8' +- pypi: https://files.pythonhosted.org/packages/79/0c/c05523fa3181fdf0c9c52a6ba91a23fbf3246cc095f26f6516f9c60e6771/virtualenv-20.35.4-py3-none-any.whl + name: virtualenv + version: 20.35.4 + sha256: c21c9cede36c9753eeade68ba7d523529f228a403463376cf821eaae2b650f1b + requires_dist: + - distlib>=0.3.7,<1 + - filelock>=3.12.2,<4 + - importlib-metadata>=6.6 ; python_full_version < '3.8' + - platformdirs>=3.9.1,<5 + - typing-extensions>=4.13.2 ; python_full_version < '3.11' + - furo>=2023.7.26 ; extra == 'docs' + - proselint>=0.13 ; extra == 'docs' + - sphinx>=7.1.2,!=7.3 ; extra == 'docs' + - sphinx-argparse>=0.4 ; extra == 'docs' + - sphinxcontrib-towncrier>=0.2.1a0 ; extra == 'docs' + - towncrier>=23.6 ; extra == 'docs' + - covdefaults>=2.3 ; extra == 'test' + - coverage-enable-subprocess>=1 ; extra == 'test' + - coverage>=7.2.7 ; extra == 'test' + - flaky>=3.7 ; extra == 'test' + - packaging>=23.1 ; extra == 'test' + - pytest-env>=0.8.2 ; extra == 'test' + - pytest-freezer>=0.4.8 ; (python_full_version >= '3.13' and platform_python_implementation == 'CPython' and sys_platform == 'win32' and extra == 'test') or (platform_python_implementation == 'GraalVM' and extra == 'test') or (platform_python_implementation == 'PyPy' and extra == 'test') + - pytest-mock>=3.11.1 ; extra == 'test' + - pytest-randomly>=3.12 ; extra == 'test' + - pytest-timeout>=2.1 ; extra == 'test' + - pytest>=7.4 ; extra == 'test' + - setuptools>=68 ; extra == 'test' + - time-machine>=2.10 ; platform_python_implementation == 'CPython' and extra == 'test' + requires_python: '>=3.8' +- conda: https://conda.anaconda.org/conda-forge/linux-64/wayland-1.24.0-h3e06ad9_0.conda + sha256: ba673427dcd480cfa9bbc262fd04a9b1ad2ed59a159bd8f7e750d4c52282f34c + md5: 0f2ca7906bf166247d1d760c3422cb8a + depends: + - __glibc >=2.17,<3.0.a0 + - libexpat >=2.7.0,<3.0a0 + - libffi >=3.4.6,<3.5.0a0 + - libgcc >=13 + - libstdcxx >=13 + license: MIT + license_family: MIT + purls: [] + size: 330474 + timestamp: 1751817998141 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/wayland-1.24.0-h4f8a99f_1.conda + sha256: d94af8f287db764327ac7b48f6c0cd5c40da6ea2606afd34ac30671b7c85d8ee + md5: f6966cb1f000c230359ae98c29e37d87 + depends: + - libexpat >=2.7.1,<3.0a0 + - libffi >=3.5.2,<3.6.0a0 + - libgcc >=14 + - libstdcxx >=14 + license: MIT + license_family: MIT + purls: [] + size: 331480 + timestamp: 1761174368396 +- pypi: https://files.pythonhosted.org/packages/fd/84/fd2ba7aafacbad3c4201d395674fc6348826569da3c0937e75505ead3528/wcwidth-0.2.13-py2.py3-none-any.whl + name: wcwidth + version: 0.2.13 + sha256: 3da69048e4540d84af32131829ff948f1e022c1c6bdb8d6102117aac784f6859 + requires_dist: + - backports-functools-lru-cache>=1.2.1 ; python_full_version < '3.2' +- pypi: https://files.pythonhosted.org/packages/af/b5/123f13c975e9f27ab9c0770f514345bd406d0e8d3b7a0723af9d43f710af/wcwidth-0.2.14-py2.py3-none-any.whl + name: wcwidth + version: 0.2.14 + sha256: a7bb560c8aee30f9957e5f9895805edd20602f2d7f720186dfd906e82b4982e1 + requires_python: '>=3.6' +- conda: https://conda.anaconda.org/conda-forge/linux-64/xkeyboard-config-2.45-hb9d3cd8_0.conda + sha256: a5d4af601f71805ec67403406e147c48d6bad7aaeae92b0622b7e2396842d3fe + md5: 397a013c2dc5145a70737871aaa87e98 + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=13 + - xorg-libx11 >=1.8.12,<2.0a0 + license: MIT + license_family: MIT + purls: [] + size: 392406 + timestamp: 1749375847832 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/xkeyboard-config-2.46-he30d5cf_0.conda + sha256: c440a757d210e84c7f315ac3b034266980a8b4c986600649d296b9198b5b4f5e + md5: 9524f30d9dea7dd5d6ead43a8823b6c2 + depends: + - libgcc >=14 + - xorg-libx11 >=1.8.12,<2.0a0 + license: MIT + license_family: MIT + purls: [] + size: 396706 + timestamp: 1759543850920 +- conda: https://conda.anaconda.org/conda-forge/linux-64/xorg-libice-1.1.2-hb9d3cd8_0.conda + sha256: c12396aabb21244c212e488bbdc4abcdef0b7404b15761d9329f5a4a39113c4b + md5: fb901ff28063514abb6046c9ec2c4a45 + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=13 + license: MIT + license_family: MIT + purls: [] + size: 58628 + timestamp: 1734227592886 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/xorg-libice-1.1.2-h86ecc28_0.conda + sha256: a2ba1864403c7eb4194dacbfe2777acf3d596feae43aada8d1b478617ce45031 + md5: c8d8ec3e00cd0fd8a231789b91a7c5b7 + depends: + - libgcc >=13 + license: MIT + license_family: MIT + purls: [] + size: 60433 + timestamp: 1734229908988 +- conda: https://conda.anaconda.org/conda-forge/linux-64/xorg-libsm-1.2.6-he73a12e_0.conda + sha256: 277841c43a39f738927145930ff963c5ce4c4dacf66637a3d95d802a64173250 + md5: 1c74ff8c35dcadf952a16f752ca5aa49 + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=13 + - libuuid >=2.38.1,<3.0a0 + - xorg-libice >=1.1.2,<2.0a0 + license: MIT + license_family: MIT + purls: [] + size: 27590 + timestamp: 1741896361728 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/xorg-libsm-1.2.6-h0808dbd_0.conda + sha256: b86a819cd16f90c01d9d81892155126d01555a20dabd5f3091da59d6309afd0a + md5: 2d1409c50882819cb1af2de82e2b7208 + depends: + - libgcc >=13 + - libuuid >=2.38.1,<3.0a0 + - xorg-libice >=1.1.2,<2.0a0 + license: MIT + license_family: MIT + purls: [] + size: 28701 + timestamp: 1741897678254 +- conda: https://conda.anaconda.org/conda-forge/linux-64/xorg-libx11-1.8.12-h4f16b4b_0.conda + sha256: 51909270b1a6c5474ed3978628b341b4d4472cd22610e5f22b506855a5e20f67 + md5: db038ce880f100acc74dba10302b5630 + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=13 + - libxcb >=1.17.0,<2.0a0 + license: MIT + license_family: MIT + purls: [] + size: 835896 + timestamp: 1741901112627 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/xorg-libx11-1.8.12-hca56bd8_0.conda + sha256: 452977d8ad96f04ec668ba74f46e70a53e00f99c0e0307956aeca75894c8131d + md5: 3df132f0048b9639bc091ef22937c111 + depends: + - libgcc >=13 + - libxcb >=1.17.0,<2.0a0 + license: MIT + license_family: MIT + purls: [] + size: 864850 + timestamp: 1741901264068 +- conda: https://conda.anaconda.org/conda-forge/linux-64/xorg-libxau-1.0.12-hb9d3cd8_0.conda + sha256: ed10c9283974d311855ae08a16dfd7e56241fac632aec3b92e3cfe73cff31038 + md5: f6ebe2cb3f82ba6c057dde5d9debe4f7 + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=13 + license: MIT + license_family: MIT + purls: [] + size: 14780 + timestamp: 1734229004433 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/xorg-libxau-1.0.12-h86ecc28_0.conda + sha256: 7829a0019b99ba462aece7592d2d7f42e12d12ccd3b9614e529de6ddba453685 + md5: d5397424399a66d33c80b1f2345a36a6 + depends: + - libgcc >=13 + license: MIT + license_family: MIT + purls: [] + size: 15873 + timestamp: 1734230458294 +- conda: https://conda.anaconda.org/conda-forge/linux-64/xorg-libxcomposite-0.4.6-hb9d3cd8_2.conda + sha256: 753f73e990c33366a91fd42cc17a3d19bb9444b9ca5ff983605fa9e953baf57f + md5: d3c295b50f092ab525ffe3c2aa4b7413 + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=13 + - xorg-libx11 >=1.8.10,<2.0a0 + - xorg-libxfixes >=6.0.1,<7.0a0 + license: MIT + license_family: MIT + purls: [] + size: 13603 + timestamp: 1727884600744 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/xorg-libxcomposite-0.4.6-h86ecc28_2.conda + sha256: 0cb82160412adb6d83f03cf50e807a8e944682d556b2215992a6fbe9ced18bc0 + md5: 86051eee0766c3542be24844a9c3cf36 + depends: + - libgcc >=13 + - xorg-libx11 >=1.8.9,<2.0a0 + - xorg-libxfixes >=6.0.1,<7.0a0 + license: MIT + license_family: MIT + purls: [] + size: 13982 + timestamp: 1727884626338 +- conda: https://conda.anaconda.org/conda-forge/linux-64/xorg-libxcursor-1.2.3-hb9d3cd8_0.conda + sha256: 832f538ade441b1eee863c8c91af9e69b356cd3e9e1350fff4fe36cc573fc91a + md5: 2ccd714aa2242315acaf0a67faea780b + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=13 + - xorg-libx11 >=1.8.10,<2.0a0 + - xorg-libxfixes >=6.0.1,<7.0a0 + - xorg-libxrender >=0.9.11,<0.10.0a0 + license: MIT + license_family: MIT + purls: [] + size: 32533 + timestamp: 1730908305254 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/xorg-libxcursor-1.2.3-h86ecc28_0.conda + sha256: c5d3692520762322a9598e7448492309f5ee9d8f3aff72d787cf06e77c42507f + md5: f2054759c2203d12d0007005e1f1296d + depends: + - libgcc >=13 + - xorg-libx11 >=1.8.9,<2.0a0 + - xorg-libxfixes >=6.0.1,<7.0a0 + - xorg-libxrender >=0.9.11,<0.10.0a0 + license: MIT + license_family: MIT + purls: [] + size: 34596 + timestamp: 1730908388714 +- conda: https://conda.anaconda.org/conda-forge/linux-64/xorg-libxdamage-1.1.6-hb9d3cd8_0.conda + sha256: 43b9772fd6582bf401846642c4635c47a9b0e36ca08116b3ec3df36ab96e0ec0 + md5: b5fcc7172d22516e1f965490e65e33a4 + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=13 + - xorg-libx11 >=1.8.10,<2.0a0 + - xorg-libxext >=1.3.6,<2.0a0 + - xorg-libxfixes >=6.0.1,<7.0a0 + license: MIT + license_family: MIT + purls: [] + size: 13217 + timestamp: 1727891438799 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/xorg-libxdamage-1.1.6-h86ecc28_0.conda + sha256: 3afaa2f43eb4cb679fc0c3d9d7c50f0f2c80cc5d3df01d5d5fd60655d0bfa9be + md5: d5773c4e4d64428d7ddaa01f6f845dc7 + depends: + - libgcc >=13 + - xorg-libx11 >=1.8.9,<2.0a0 + - xorg-libxext >=1.3.6,<2.0a0 + - xorg-libxfixes >=6.0.1,<7.0a0 + license: MIT + license_family: MIT + purls: [] + size: 13794 + timestamp: 1727891406431 +- conda: https://conda.anaconda.org/conda-forge/linux-64/xorg-libxdmcp-1.1.5-hb9d3cd8_0.conda + sha256: 6b250f3e59db07c2514057944a3ea2044d6a8cdde8a47b6497c254520fade1ee + md5: 8035c64cb77ed555e3f150b7b3972480 + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=13 + license: MIT + license_family: MIT + purls: [] + size: 19901 + timestamp: 1727794976192 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/xorg-libxdmcp-1.1.5-h57736b2_0.conda + sha256: efcc150da5926cf244f757b8376d96a4db78bc15b8d90ca9f56ac6e75755971f + md5: 25a5a7b797fe6e084e04ffe2db02fc62 + depends: + - libgcc >=13 + license: MIT + license_family: MIT + purls: [] + size: 20615 + timestamp: 1727796660574 +- conda: https://conda.anaconda.org/conda-forge/linux-64/xorg-libxext-1.3.6-hb9d3cd8_0.conda + sha256: da5dc921c017c05f38a38bd75245017463104457b63a1ce633ed41f214159c14 + md5: febbab7d15033c913d53c7a2c102309d + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=13 + - xorg-libx11 >=1.8.10,<2.0a0 + license: MIT + license_family: MIT + purls: [] + size: 50060 + timestamp: 1727752228921 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/xorg-libxext-1.3.6-h57736b2_0.conda + sha256: 8e216b024f52e367463b4173f237af97cf7053c77d9ce3e958bc62473a053f71 + md5: bd1e86dd8aa3afd78a4bfdb4ef918165 + depends: + - libgcc >=13 + - xorg-libx11 >=1.8.9,<2.0a0 + license: MIT + license_family: MIT + purls: [] + size: 50746 + timestamp: 1727754268156 +- conda: https://conda.anaconda.org/conda-forge/linux-64/xorg-libxfixes-6.0.1-hb9d3cd8_0.conda + sha256: 2fef37e660985794617716eb915865ce157004a4d567ed35ec16514960ae9271 + md5: 4bdb303603e9821baf5fe5fdff1dc8f8 + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=13 + - xorg-libx11 >=1.8.10,<2.0a0 + license: MIT + license_family: MIT + purls: [] + size: 19575 + timestamp: 1727794961233 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/xorg-libxfixes-6.0.2-he30d5cf_0.conda + sha256: 8cb9c88e25c57e47419e98f04f9ef3154ad96b9f858c88c570c7b91216a64d0e + md5: e8b4056544341daf1d415eaeae7a040c + depends: + - libgcc >=14 + - xorg-libx11 >=1.8.12,<2.0a0 + license: MIT + license_family: MIT + purls: [] + size: 20704 + timestamp: 1759284028146 +- conda: https://conda.anaconda.org/conda-forge/linux-64/xorg-libxi-1.8.2-hb9d3cd8_0.conda + sha256: 1a724b47d98d7880f26da40e45f01728e7638e6ec69f35a3e11f92acd05f9e7a + md5: 17dcc85db3c7886650b8908b183d6876 + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=13 + - xorg-libx11 >=1.8.10,<2.0a0 + - xorg-libxext >=1.3.6,<2.0a0 + - xorg-libxfixes >=6.0.1,<7.0a0 + license: MIT + license_family: MIT + purls: [] + size: 47179 + timestamp: 1727799254088 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/xorg-libxi-1.8.2-h57736b2_0.conda + sha256: 7b587407ecb9ccd2bbaf0fb94c5dbdde4d015346df063e9502dc0ce2b682fb5e + md5: eeee3bdb31c6acde2b81ad1b8c287087 + depends: + - libgcc >=13 + - xorg-libx11 >=1.8.9,<2.0a0 + - xorg-libxext >=1.3.6,<2.0a0 + - xorg-libxfixes >=6.0.1,<7.0a0 + license: MIT + license_family: MIT + purls: [] + size: 48197 + timestamp: 1727801059062 +- conda: https://conda.anaconda.org/conda-forge/linux-64/xorg-libxinerama-1.1.5-h5888daf_1.conda + sha256: 1b9141c027f9d84a9ee5eb642a0c19457c788182a5a73c5a9083860ac5c20a8c + md5: 5e2eb9bf77394fc2e5918beefec9f9ab + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=13 + - libstdcxx >=13 + - xorg-libx11 >=1.8.10,<2.0a0 + - xorg-libxext >=1.3.6,<2.0a0 + license: MIT + license_family: MIT + purls: [] + size: 13891 + timestamp: 1727908521531 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/xorg-libxinerama-1.1.5-h5ad3122_1.conda + sha256: 5f84f820397db504e187754665d48d385e0a2a49f07ffc2372c7f42fa36dd972 + md5: a7b99f104e14b99ca773d2fe2d195585 + depends: + - libgcc >=13 + - libstdcxx >=13 + - xorg-libx11 >=1.8.9,<2.0a0 + - xorg-libxext >=1.3.6,<2.0a0 + license: MIT + license_family: MIT + purls: [] + size: 14388 + timestamp: 1727908606602 +- conda: https://conda.anaconda.org/conda-forge/linux-64/xorg-libxrandr-1.5.4-hb9d3cd8_0.conda + sha256: ac0f037e0791a620a69980914a77cb6bb40308e26db11698029d6708f5aa8e0d + md5: 2de7f99d6581a4a7adbff607b5c278ca + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=13 + - xorg-libx11 >=1.8.10,<2.0a0 + - xorg-libxext >=1.3.6,<2.0a0 + - xorg-libxrender >=0.9.11,<0.10.0a0 + license: MIT + license_family: MIT + purls: [] + size: 29599 + timestamp: 1727794874300 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/xorg-libxrandr-1.5.4-h86ecc28_0.conda + sha256: b2588a2b101d1b0a4e852532c8b9c92c59ef584fc762dd700567bdbf8cd00650 + md5: dd3e74283a082381aa3860312e3c721e + depends: + - libgcc >=13 + - xorg-libx11 >=1.8.9,<2.0a0 + - xorg-libxext >=1.3.6,<2.0a0 + - xorg-libxrender >=0.9.11,<0.10.0a0 + license: MIT + license_family: MIT + purls: [] + size: 30197 + timestamp: 1727794957221 +- conda: https://conda.anaconda.org/conda-forge/linux-64/xorg-libxrender-0.9.12-hb9d3cd8_0.conda + sha256: 044c7b3153c224c6cedd4484dd91b389d2d7fd9c776ad0f4a34f099b3389f4a1 + md5: 96d57aba173e878a2089d5638016dc5e + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=13 + - xorg-libx11 >=1.8.10,<2.0a0 + license: MIT + license_family: MIT + purls: [] + size: 33005 + timestamp: 1734229037766 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/xorg-libxrender-0.9.12-h86ecc28_0.conda + sha256: ffd77ee860c9635a28cfda46163dcfe9224dc6248c62404c544ae6b564a0be1f + md5: ae2c2dd0e2d38d249887727db2af960e + depends: + - libgcc >=13 + - xorg-libx11 >=1.8.10,<2.0a0 + license: MIT + license_family: MIT + purls: [] + size: 33649 + timestamp: 1734229123157 +- conda: https://conda.anaconda.org/conda-forge/linux-64/xorg-libxtst-1.2.5-hb9d3cd8_3.conda + sha256: 752fdaac5d58ed863bbf685bb6f98092fe1a488ea8ebb7ed7b606ccfce08637a + md5: 7bbe9a0cc0df0ac5f5a8ad6d6a11af2f + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=13 + - xorg-libx11 >=1.8.10,<2.0a0 + - xorg-libxext >=1.3.6,<2.0a0 + - xorg-libxi >=1.7.10,<2.0a0 + license: MIT + license_family: MIT + purls: [] + size: 32808 + timestamp: 1727964811275 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/xorg-libxtst-1.2.5-h57736b2_3.conda + sha256: 6eaffce5a34fc0a16a21ddeaefb597e792a263b1b0c387c1ce46b0a967d558e1 + md5: c05698071b5c8e0da82a282085845860 + depends: + - libgcc >=13 + - xorg-libx11 >=1.8.9,<2.0a0 + - xorg-libxext >=1.3.6,<2.0a0 + - xorg-libxi >=1.7.10,<2.0a0 + license: MIT + license_family: MIT + purls: [] + size: 33786 + timestamp: 1727964907993 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/xorg-libxxf86vm-1.1.6-h86ecc28_0.conda + sha256: 012f0d1fd9fb1d949e0dccc0b28d9dd5a8895a1f3e2a7edc1fa2e1b33fc0f233 + md5: d745faa2d7c15092652e40a22bb261ed + depends: + - libgcc >=13 + - xorg-libx11 >=1.8.10,<2.0a0 + - xorg-libxext >=1.3.6,<2.0a0 + license: MIT + license_family: MIT + purls: [] + size: 18185 + timestamp: 1734214652726 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/xorg-xorgproto-2024.1-h86ecc28_1.conda + sha256: 3dbbf4cdb5ad82d3479ab2aa68ae67de486a6d57d67f0402d8e55869f6f13aec + md5: 91cef7867bf2b47f614597b59705ff56 + depends: + - libgcc >=13 + license: MIT + license_family: MIT + purls: [] + size: 566948 + timestamp: 1726847598167 +- conda: https://conda.anaconda.org/conda-forge/linux-64/zstd-1.5.7-hb8e6e7a_2.conda + sha256: a4166e3d8ff4e35932510aaff7aa90772f84b4d07e9f6f83c614cba7ceefe0eb + md5: 6432cb5d4ac0046c3ac0a8a0f95842f9 + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=13 + - libstdcxx >=13 + - libzlib >=1.3.1,<2.0a0 + license: BSD-3-Clause + license_family: BSD + purls: [] + size: 567578 + timestamp: 1742433379869 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/zstd-1.5.7-hbcf94c1_2.conda + sha256: 0812e7b45f087cfdd288690ada718ce5e13e8263312e03b643dd7aa50d08b51b + md5: 5be90c5a3e4b43c53e38f50a85e11527 + depends: + - libgcc >=13 + - libstdcxx >=13 + - libzlib >=1.3.1,<2.0a0 + license: BSD-3-Clause + license_family: BSD + purls: [] + size: 551176 + timestamp: 1742433378347 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/zstd-1.5.7-h6491c7d_2.conda + sha256: 0d02046f57f7a1a3feae3e9d1aa2113788311f3cf37a3244c71e61a93177ba67 + md5: e6f69c7bcccdefa417f056fa593b40f0 + depends: + - __osx >=11.0 + - libzlib >=1.3.1,<2.0a0 + license: BSD-3-Clause + license_family: BSD + purls: [] + size: 399979 + timestamp: 1742433432699 diff --git a/pyproject.toml b/pyproject.toml index b98361fe8..ef9e622a2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,13 +16,15 @@ dependencies = [ "tqdm", "networkx", "pydot", - "minio>=7.0.0", + "fsspec>=2023.1.0", "matplotlib", "faker", "urllib3", "setuptools", + "pydantic-settings>=2.0.0", ] -requires-python = ">=3.9,<4.0" + +requires-python = ">=3.10,<3.14" authors = [ {name = "Dimitri Yatsenko", email = "dimitri@datajoint.com"}, {name = "Thinh Nguyen", email = "thinh@datajoint.com"}, @@ -37,22 +39,25 @@ maintainers = [ # manually sync here: https://docs.datajoint.com/core/datajoint-python/latest/#welcome-to-datajoint-for-python description = "DataJoint for Python is a framework for scientific workflow management based on relational principles. DataJoint is built on the foundation of the relational data model and prescribes a consistent method for organizing, populating, computing, and querying data." readme = "README.md" -license = {file = "LICENSE.txt"} +license = {file = "LICENSE"} keywords = [ - "database", - "automated", - "automation", - "compute", - "data", - "pipeline", - "workflow", - "scientific", - "science", - "research", - "neuroscience", - "bioinformatics", - "bio-informatics", "datajoint", + "data-pipelines", + "workflow-management", + "data-engineering", + "scientific-computing", + "neuroscience", + "research-software", + "data-integrity", + "reproducibility", + "declarative", + "etl", + "object-storage", + "schema-management", + "data-lineage", + "relational-model", + "mysql", + "postgresql", ] # https://pypi.org/classifiers/ classifiers = [ @@ -60,7 +65,7 @@ classifiers = [ "Development Status :: 5 - Production/Stable", "Intended Audience :: Science/Research", "Intended Audience :: Healthcare Industry", - "License :: OSI Approved :: GNU Lesser General Public License v2 or later (LGPLv2+)", + "License :: OSI Approved :: Apache Software License", "Topic :: Software Development :: Libraries :: Python Modules", "Topic :: Scientific/Engineering", "Topic :: Scientific/Engineering :: Bio-Informatics", @@ -78,27 +83,183 @@ Repository = "https://github.com/datajoint/datajoint-python" dj = "datajoint.cli:cli" datajoint = "datajoint.cli:cli" +[dependency-groups] +test = [ + "pytest", + "pytest-cov", + "requests", + "graphviz", + "testcontainers[mysql,minio]>=4.0", + "polars>=0.20.0", + "pyarrow>=14.0.0", +] + [project.optional-dependencies] +s3 = ["s3fs>=2023.1.0"] +gcs = ["gcsfs>=2023.1.0"] +azure = ["adlfs>=2023.1.0"] +polars = ["polars>=0.20.0"] +arrow = ["pyarrow>=14.0.0"] test = [ "pytest", "pytest-cov", + "requests", + "s3fs>=2023.1.0", + "testcontainers[mysql,minio]>=4.0", + "polars>=0.20.0", + "pyarrow>=14.0.0", ] dev = [ "pre-commit", - "black==24.2.0", - "flake8", - "isort", + "ruff", "codespell", # including test "pytest", "pytest-cov", + "polars>=0.20.0", + "pyarrow>=14.0.0", +] + +[tool.ruff] +# Equivalent to flake8 configuration +line-length = 127 +target-version = "py310" + +[tool.ruff.lint] +# Enable specific rule sets equivalent to flake8 configuration +select = [ + "E", # pycodestyle errors + "W", # pycodestyle warnings + "F", # pyflakes + "C90", # mccabe complexity +] + +# Ignore specific rules (equivalent to flake8 --ignore) +ignore = [ + "E203", # whitespace before ':' + "E722", # bare except +] + +# Per-file ignores (equivalent to flake8 --per-file-ignores) +[tool.ruff.lint.per-file-ignores] +"datajoint/diagram.py" = ["C901"] # function too complex +"tests/integration/test_blob_matlab.py" = ["E501"] # SQL hex strings cannot be broken across lines + +[tool.ruff.lint.mccabe] +# Maximum complexity (equivalent to flake8 --max-complexity) +max-complexity = 62 + +[tool.ruff.format] +# Use black-compatible formatting +quote-style = "double" +indent-style = "space" +line-ending = "auto" + +[tool.mypy] +python_version = "3.10" +ignore_missing_imports = true +# Start with lenient settings, gradually enable stricter checks +warn_return_any = false +warn_unused_ignores = false +disallow_untyped_defs = false +disallow_incomplete_defs = false +check_untyped_defs = true + +# Modules with complete type coverage - strict checking enabled +[[tool.mypy.overrides]] +module = [ + "datajoint.content_registry", + "datajoint.errors", + "datajoint.hash", ] +disallow_untyped_defs = true +disallow_incomplete_defs = true +warn_return_any = true -[tool.isort] -profile = "black" +# Modules excluded from type checking until fully typed +[[tool.mypy.overrides]] +module = [ + "datajoint.admin", + "datajoint.autopopulate", + "datajoint.blob", + "datajoint.builtin_codecs", + "datajoint.cli", + "datajoint.codecs", + "datajoint.condition", + "datajoint.connection", + "datajoint.declare", + "datajoint.dependencies", + "datajoint.diagram", + "datajoint.expression", + "datajoint.gc", + "datajoint.heading", + "datajoint.jobs", + "datajoint.lineage", + "datajoint.logging", + "datajoint.migrate", + "datajoint.objectref", + "datajoint.preview", + "datajoint.schemas", + "datajoint.settings", + "datajoint.staged_insert", + "datajoint.storage", + "datajoint.table", + "datajoint.user_tables", + "datajoint.utils", +] +ignore_errors = true [tool.setuptools] packages = ["datajoint"] +package-dir = {"" = "src"} [tool.setuptools.dynamic] version = { attr = "datajoint.version.__version__"} + +[tool.codespell] +skip = ".git,*.pdf,*.svg,*.csv,*.ipynb,*.drawio" +# Rever -- nobody knows +# numer -- numerator variable +# astroid -- Python library name (not "asteroid") +ignore-words-list = "rever,numer,astroid" + +[tool.pytest.ini_options] +markers = [ + "requires_mysql: marks tests as requiring MySQL database (deselect with '-m \"not requires_mysql\"')", + "requires_minio: marks tests as requiring MinIO object storage (deselect with '-m \"not requires_minio\"')", +] + + +[tool.pixi.workspace] +channels = ["conda-forge"] +platforms = ["linux-64", "osx-arm64", "linux-aarch64"] + +[tool.pixi.pypi-dependencies] +datajoint = { path = ".", editable = true } + +[tool.pixi.feature.test.pypi-dependencies] +datajoint = { path = ".", editable = true, extras = ["test"] } + +[tool.pixi.feature.dev.pypi-dependencies] +datajoint = { path = ".", editable = true, extras = ["dev", "test"] } + +[tool.pixi.environments] +default = { solve-group = "default" } +dev = { features = ["dev"], solve-group = "default" } +test = { features = ["test"], solve-group = "default" } + +[tool.pixi.tasks] +# Tests use testcontainers - no manual setup required +test = "pytest tests/" +test-cov = "pytest --cov-report term-missing --cov=datajoint tests/" +# Optional: use external containers (docker-compose) instead of testcontainers +services-up = "docker compose up -d db minio" +services-down = "docker compose down" +test-external = { cmd = "DJ_USE_EXTERNAL_CONTAINERS=1 pytest tests/", depends-on = ["services-up"] } + +[tool.pixi.dependencies] +python = ">=3.10,<3.14" +graphviz = ">=13.1.2,<14" + +[tool.pixi.activation] +scripts=["activate.sh"] \ No newline at end of file diff --git a/src/datajoint/__init__.py b/src/datajoint/__init__.py new file mode 100644 index 000000000..ae8d308d2 --- /dev/null +++ b/src/datajoint/__init__.py @@ -0,0 +1,120 @@ +""" +DataJoint for Python — a framework for scientific data pipelines. + +DataJoint introduces the Relational Workflow Model, where your database schema +is an executable specification of your workflow. Tables represent workflow steps, +foreign keys encode dependencies, and computations are declarative. + +Documentation: https://docs.datajoint.com +Source: https://github.com/datajoint/datajoint-python + +Copyright 2014-2026 DataJoint Inc. and contributors. +Licensed under the Apache License, Version 2.0. + +If DataJoint contributes to a publication, please cite: +https://doi.org/10.1101/031658 +""" + +__author__ = "DataJoint Contributors" +__date__ = "November 7, 2020" +__all__ = [ + "__author__", + "__version__", + "config", + "conn", + "Connection", + "Schema", + "schema", + "VirtualModule", + "virtual_schema", + "list_schemas", + "Table", + "FreeTable", + "Manual", + "Lookup", + "Imported", + "Computed", + "Part", + "Not", + "AndList", + "Top", + "U", + "Diagram", + "Di", + "ERD", + "kill", + "MatCell", + "MatStruct", + # Codec API + "Codec", + "list_codecs", + "get_codec", + "errors", + "migrate", + "DataJointError", + "key", + "key_hash", + "logger", + "cli", + "ObjectRef", + "ValidationResult", +] + +# ============================================================================= +# Eager imports — core functionality needed immediately +# ============================================================================= +from . import errors +from . import migrate +from .codecs import ( + Codec, + get_codec, + list_codecs, +) +from .blob import MatCell, MatStruct +from .connection import Connection, conn +from .errors import DataJointError +from .expression import AndList, Not, Top, U +from .hash import key_hash +from .logging import logger +from .objectref import ObjectRef +from .schemas import Schema, VirtualModule, list_schemas, virtual_schema +from .settings import config +from .table import FreeTable, Table, ValidationResult +from .user_tables import Computed, Imported, Lookup, Manual, Part +from .version import __version__ + +schema = Schema # Alias for Schema + +# ============================================================================= +# Lazy imports — heavy dependencies loaded on first access +# ============================================================================= +# These modules import heavy dependencies (networkx, matplotlib, click, pymysql) +# that slow down `import datajoint`. They are loaded on demand. + +_lazy_modules = { + # Diagram imports networkx and matplotlib + "Diagram": (".diagram", "Diagram"), + "Di": (".diagram", "Diagram"), + "ERD": (".diagram", "Diagram"), + "diagram": (".diagram", None), # Return the module itself + # kill imports pymysql via connection + "kill": (".admin", "kill"), + # cli imports click + "cli": (".cli", "cli"), +} + + +def __getattr__(name: str): + """Lazy import for heavy dependencies.""" + if name in _lazy_modules: + module_path, attr_name = _lazy_modules[name] + import importlib + + module = importlib.import_module(module_path, __package__) + # If attr_name is None, return the module itself + attr = module if attr_name is None else getattr(module, attr_name) + # Cache in module __dict__ to avoid repeated __getattr__ calls + # and to override the submodule that importlib adds automatically + globals()[name] = attr + return attr + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/src/datajoint/admin.py b/src/datajoint/admin.py new file mode 100644 index 000000000..275e9a823 --- /dev/null +++ b/src/datajoint/admin.py @@ -0,0 +1,101 @@ +import logging + +import pymysql + +from .connection import conn + +logger = logging.getLogger(__name__.split(".")[0]) + + +def kill(restriction=None, connection=None, order_by=None): + """ + View and kill database connections interactively. + + Displays a list of active connections and prompts for connections to kill. + + Parameters + ---------- + restriction : str, optional + SQL WHERE clause to filter connections. Can use any attribute from + information_schema.processlist: ID, USER, HOST, DB, COMMAND, TIME, STATE, INFO. + connection : Connection, optional + A datajoint.Connection object. Defaults to datajoint.conn(). + order_by : str or list[str], optional + Attribute(s) to order results by. Defaults to 'id'. + + Examples + -------- + >>> dj.kill('HOST LIKE "%compute%"') # List connections from hosts containing "compute" + >>> dj.kill('TIME > 600') # List connections idle for more than 10 minutes + """ + + if connection is None: + connection = conn() + + if order_by is not None and not isinstance(order_by, str): + order_by = ",".join(order_by) + + query = ( + "SELECT * FROM information_schema.processlist WHERE id <> CONNECTION_ID()" + + ("" if restriction is None else " AND (%s)" % restriction) + + (" ORDER BY %s" % (order_by or "id")) + ) + + while True: + print(" ID USER HOST STATE TIME INFO") + print("+--+ +----------+ +-----------+ +-----------+ +-----+") + cur = ({k.lower(): v for k, v in elem.items()} for elem in connection.query(query, as_dict=True)) + for process in cur: + try: + print("{id:>4d} {user:<12s} {host:<12s} {state:<12s} {time:>7d} {info}".format(**process)) + except TypeError: + print(process) + response = input('process to kill or "q" to quit > ') + if response == "q": + break + if response: + try: + pid = int(response) + except ValueError: + pass # ignore non-numeric input + else: + try: + connection.query("kill %d" % pid) + except pymysql.err.InternalError: + logger.warn("Process not found") + + +def kill_quick(restriction=None, connection=None): + """ + Kill database connections without prompting. + + Parameters + ---------- + restriction : str, optional + SQL WHERE clause to filter connections. Can use any attribute from + information_schema.processlist: ID, USER, HOST, DB, COMMAND, TIME, STATE, INFO. + connection : Connection, optional + A datajoint.Connection object. Defaults to datajoint.conn(). + + Returns + ------- + int + Number of terminated connections. + + Examples + -------- + >>> dj.kill_quick('HOST LIKE "%compute%"') # Kill connections from hosts with "compute" + """ + if connection is None: + connection = conn() + + query = "SELECT * FROM information_schema.processlist WHERE id <> CONNECTION_ID()" + ( + "" if restriction is None else " AND (%s)" % restriction + ) + + cur = ({k.lower(): v for k, v in elem.items()} for elem in connection.query(query, as_dict=True)) + nkill = 0 + for process in cur: + connection.query("kill %d" % process["id"]) + nkill += 1 + return nkill diff --git a/src/datajoint/autopopulate.py b/src/datajoint/autopopulate.py new file mode 100644 index 000000000..80c669079 --- /dev/null +++ b/src/datajoint/autopopulate.py @@ -0,0 +1,768 @@ +"""This module defines class dj.AutoPopulate""" + +from __future__ import annotations + +import contextlib +import datetime +import inspect +import logging +import multiprocessing as mp +import signal +import traceback +from typing import TYPE_CHECKING, Any, Generator + +import deepdiff +from tqdm import tqdm + +from .errors import DataJointError, LostConnectionError +from .expression import AndList, QueryExpression + +if TYPE_CHECKING: + from .jobs import Job + from .table import Table + +# noinspection PyExceptionInherit,PyCallingNonCallable + +logger = logging.getLogger(__name__.split(".")[0]) + + +# --- helper functions for multiprocessing -- + + +def _initialize_populate(table: Table, jobs: Job | None, populate_kwargs: dict[str, Any]) -> None: + """ + Initialize a worker process for multiprocessing. + + Saves the unpickled table to the current process and reconnects to database. + + Parameters + ---------- + table : Table + Table instance to populate. + jobs : Job or None + Job management object or None for direct mode. + populate_kwargs : dict + Arguments for _populate1(). + """ + process = mp.current_process() + process.table = table + process.jobs = jobs + process.populate_kwargs = populate_kwargs + table.connection.connect() # reconnect + + +def _call_populate1(key: dict[str, Any]) -> bool | tuple[dict[str, Any], Any]: + """ + Call _populate1() for a single key in the worker process. + + Parameters + ---------- + key : dict + Primary key specifying job to compute. + + Returns + ------- + bool or tuple + Result from _populate1(). + """ + process = mp.current_process() + return process.table._populate1(key, process.jobs, **process.populate_kwargs) + + +class AutoPopulate: + """ + Mixin class that adds automated population to Table classes. + + Auto-populated tables (Computed, Imported) inherit from both Table and + AutoPopulate. They must implement the ``make()`` method that computes + and inserts data for one primary key. + + Attributes + ---------- + key_source : QueryExpression + Query yielding keys to be populated. Default is join of FK parents. + jobs : Job + Job table (``~~table_name``) for distributed processing. + + Notes + ----- + Subclasses may override ``key_source`` to customize population scope. + """ + + _key_source = None + _allow_insert = False + _jobs = None + + class _JobsDescriptor: + """Descriptor allowing jobs access on both class and instance.""" + + def __get__(self, obj, objtype=None): + """ + Access the job table for this auto-populated table. + + The job table (``~~table_name``) is created lazily on first access. + It tracks job status, priority, scheduling, and error information + for distributed populate operations. + + Can be accessed on either the class or an instance:: + + # Both work equivalently + Analysis.jobs.refresh() + Analysis().jobs.refresh() + + Returns + ------- + Job + Job management object for this table. + """ + if obj is None: + # Accessed on class - instantiate first + obj = objtype() + if obj._jobs is None: + from .jobs import Job + + obj._jobs = Job(obj) + if not obj._jobs.is_declared: + obj._jobs.declare() + return obj._jobs + + jobs: Job = _JobsDescriptor() + + def _declare_check(self, primary_key: list[str], fk_attribute_map: dict[str, tuple[str, str]]) -> None: + """ + Validate FK-only primary key constraint for auto-populated tables. + + Auto-populated tables (Computed/Imported) must derive all primary key + attributes from foreign key references. This ensures proper job granularity + for distributed populate operations. + + Parameters + ---------- + primary_key : list + List of primary key attribute names. + fk_attribute_map : dict + Mapping of child_attr -> (parent_table, parent_attr). + + Raises + ------ + DataJointError + If native (non-FK) PK attributes are found, unless bypassed via + ``dj.config.jobs.allow_new_pk_fields_in_computed_tables = True``. + """ + from .settings import config + + # Check if validation is bypassed + if config.jobs.allow_new_pk_fields_in_computed_tables: + return + + # Check for native (non-FK) primary key attributes + native_pk_attrs = [attr for attr in primary_key if attr not in fk_attribute_map] + + if native_pk_attrs: + raise DataJointError( + f"Auto-populated table `{self.full_table_name}` has non-FK primary key " + f"attribute(s): {', '.join(native_pk_attrs)}. " + f"Computed and Imported tables must derive all primary key attributes " + f"from foreign key references. The make() method is called once per entity " + f"(row) in the table. If you need to compute multiple entities per job, " + f"define a Part table to store them. " + f"To bypass this restriction, set: dj.config.jobs.allow_new_pk_fields_in_computed_tables = True" + ) + + @property + def key_source(self) -> QueryExpression: + """ + Query expression yielding keys to be populated. + + Returns the primary key values to be passed sequentially to ``make()`` + when ``populate()`` is called. The default is the join of parent tables + referenced from the primary key. + + Returns + ------- + QueryExpression + Expression yielding keys for population. + + Notes + ----- + Subclasses may override to change the scope or granularity of make calls. + """ + + def _rename_attributes(table, props): + return ( + table.proj(**{attr: ref for attr, ref in props["attr_map"].items() if attr != ref}) + if props["aliased"] + else table.proj() + ) + + if self._key_source is None: + parents = self.parents(primary=True, as_objects=True, foreign_key_info=True) + if not parents: + raise DataJointError("A table must have dependencies from its primary key for auto-populate to work") + self._key_source = _rename_attributes(*parents[0]) + for q in parents[1:]: + self._key_source *= _rename_attributes(*q) + return self._key_source + + def make(self, key: dict[str, Any]) -> None | Generator[Any, Any, None]: + """ + Compute and insert data for one key. + + Must be implemented by subclasses to perform automated computation. + The method implements three steps: + + 1. Fetch data from parent tables, restricted by the given key + 2. Compute secondary attributes based on the fetched data + 3. Insert the new row(s) into the current table + + Parameters + ---------- + key : dict + Primary key value identifying the entity to compute. + + Raises + ------ + NotImplementedError + If neither ``make()`` nor the tripartite methods are implemented. + + Notes + ----- + **Simple make**: Implement as a regular method that performs all three + steps in a single database transaction. Must return None. + + **Tripartite make**: For long-running computations, implement: + + - ``make_fetch(key)``: Fetch data from parent tables + - ``make_compute(key, *fetched_data)``: Compute results + - ``make_insert(key, *computed_result)``: Insert results + + The tripartite pattern allows computation outside the transaction, + with referential integrity checking before commit. + """ + + if not (hasattr(self, "make_fetch") and hasattr(self, "make_insert") and hasattr(self, "make_compute")): + # user must implement `make` + raise NotImplementedError( + "Subclasses of AutoPopulate must implement the method `make` " + "or (`make_fetch` + `make_compute` + `make_insert`)" + ) + + # User has implemented `_fetch`, `_compute`, and `_insert` methods instead + + # Step 1: Fetch data from parent tables + fetched_data = self.make_fetch(key) # fetched_data is a tuple + computed_result = yield fetched_data # passed as input into make_compute + + # Step 2: If computed result is not passed in, compute the result + if computed_result is None: + # this is only executed in the first invocation + computed_result = self.make_compute(key, *fetched_data) + yield computed_result # this is passed to the second invocation of make + + # Step 3: Insert the computed result into the current table. + self.make_insert(key, *computed_result) + yield + + def _jobs_to_do(self, restrictions: tuple) -> QueryExpression: + """ + Return the query yielding keys to be computed. + + Parameters + ---------- + restrictions : tuple + Conditions to filter key_source. + + Returns + ------- + QueryExpression + Keys derived from key_source that need computation. + """ + if self.restriction: + raise DataJointError( + "Cannot call populate on a restricted table. Instead, pass conditions to populate() as arguments." + ) + todo = self.key_source + + # key_source is a QueryExpression subclass -- trigger instantiation + if inspect.isclass(todo) and issubclass(todo, QueryExpression): + todo = todo() + + if not isinstance(todo, QueryExpression): + raise DataJointError("Invalid key_source value") + + try: + # check if target lacks any attributes from the primary key of key_source + raise DataJointError( + "The populate target lacks attribute %s " + "from the primary key of key_source" + % next(name for name in todo.heading.primary_key if name not in self.heading) + ) + except StopIteration: + pass + return (todo & AndList(restrictions)).proj() + + def populate( + self, + *restrictions: Any, + suppress_errors: bool = False, + return_exception_objects: bool = False, + reserve_jobs: bool = False, + max_calls: int | None = None, + display_progress: bool = False, + processes: int = 1, + make_kwargs: dict[str, Any] | None = None, + priority: int | None = None, + refresh: bool | None = None, + ) -> dict[str, Any]: + """ + Populate the table by calling ``make()`` for unpopulated keys. + + Calls ``make(key)`` for every primary key in ``key_source`` for which + there is not already a row in this table. + + Parameters + ---------- + *restrictions + Conditions to filter key_source. + suppress_errors : bool, optional + If True, collect errors instead of raising. Default False. + return_exception_objects : bool, optional + If True, return exception objects instead of messages. Default False. + reserve_jobs : bool, optional + If True, use job table for distributed processing. Default False. + max_calls : int, optional + Maximum number of ``make()`` calls. + display_progress : bool, optional + If True, show progress bar. Default False. + processes : int, optional + Number of worker processes. Default 1. + make_kwargs : dict, optional + Keyword arguments passed to each ``make()`` call. + priority : int, optional + (Distributed mode) Only process jobs at this priority or higher. + refresh : bool, optional + (Distributed mode) Refresh job queue before processing. + Default from ``config.jobs.auto_refresh``. + + Returns + ------- + dict + ``{"success_count": int, "error_list": list}``. + + Notes + ----- + **Direct mode** (``reserve_jobs=False``): Keys computed from + ``(key_source & restrictions) - target``. No job table. Suitable for + single-worker, development, and debugging. + + **Distributed mode** (``reserve_jobs=True``): Uses job table + (``~~table_name``) for multi-worker coordination with priority and + status tracking. + """ + if self.connection.in_transaction: + raise DataJointError("Populate cannot be called during a transaction.") + + if reserve_jobs: + return self._populate_distributed( + *restrictions, + suppress_errors=suppress_errors, + return_exception_objects=return_exception_objects, + max_calls=max_calls, + display_progress=display_progress, + processes=processes, + make_kwargs=make_kwargs, + priority=priority, + refresh=refresh, + ) + else: + return self._populate_direct( + *restrictions, + suppress_errors=suppress_errors, + return_exception_objects=return_exception_objects, + max_calls=max_calls, + display_progress=display_progress, + processes=processes, + make_kwargs=make_kwargs, + ) + + def _populate_direct( + self, + *restrictions, + suppress_errors, + return_exception_objects, + max_calls, + display_progress, + processes, + make_kwargs, + ): + """ + Populate without job table coordination. + + Computes keys directly from key_source, suitable for single-worker + execution, development, and debugging. + """ + keys = (self._jobs_to_do(restrictions) - self).keys() + + logger.debug("Found %d keys to populate" % len(keys)) + + keys = keys[:max_calls] + nkeys = len(keys) + + error_list = [] + success_list = [] + + if nkeys: + processes = min(_ for _ in (processes, nkeys, mp.cpu_count()) if _) + + populate_kwargs = dict( + suppress_errors=suppress_errors, + return_exception_objects=return_exception_objects, + make_kwargs=make_kwargs, + ) + + if processes == 1: + for key in tqdm(keys, desc=self.__class__.__name__) if display_progress else keys: + status = self._populate1(key, jobs=None, **populate_kwargs) + if status is True: + success_list.append(1) + elif isinstance(status, tuple): + error_list.append(status) + else: + assert status is False + else: + # spawn multiple processes + self.connection.close() + del self.connection._conn.ctx # SSLContext is not pickleable + with ( + mp.Pool(processes, _initialize_populate, (self, None, populate_kwargs)) as pool, + tqdm(desc="Processes: ", total=nkeys) if display_progress else contextlib.nullcontext() as progress_bar, + ): + for status in pool.imap(_call_populate1, keys, chunksize=1): + if status is True: + success_list.append(1) + elif isinstance(status, tuple): + error_list.append(status) + else: + assert status is False + if display_progress: + progress_bar.update() + self.connection.connect() + + return { + "success_count": sum(success_list), + "error_list": error_list, + } + + def _populate_distributed( + self, + *restrictions, + suppress_errors, + return_exception_objects, + max_calls, + display_progress, + processes, + make_kwargs, + priority, + refresh, + ): + """ + Populate with job table coordination. + + Uses job table for multi-worker coordination, priority scheduling, + and status tracking. + """ + from .settings import config + + # Define a signal handler for SIGTERM + def handler(signum, frame): + logger.info("Populate terminated by SIGTERM") + raise SystemExit("SIGTERM received") + + old_handler = signal.signal(signal.SIGTERM, handler) + + try: + # Refresh job queue if configured + if refresh is None: + refresh = config.jobs.auto_refresh + if refresh: + self.jobs.refresh(*restrictions, priority=priority) + + # Fetch pending jobs ordered by priority (use NOW(3) to match CURRENT_TIMESTAMP(3) precision) + pending_query = self.jobs.pending & "scheduled_time <= NOW(3)" + if priority is not None: + pending_query = pending_query & f"priority <= {priority}" + + keys = pending_query.keys(order_by="priority ASC, scheduled_time ASC", limit=max_calls) + + logger.debug("Found %d pending jobs to populate" % len(keys)) + + nkeys = len(keys) + error_list = [] + success_list = [] + + if nkeys: + processes = min(_ for _ in (processes, nkeys, mp.cpu_count()) if _) + + populate_kwargs = dict( + suppress_errors=suppress_errors, + return_exception_objects=return_exception_objects, + make_kwargs=make_kwargs, + ) + + if processes == 1: + for key in tqdm(keys, desc=self.__class__.__name__) if display_progress else keys: + status = self._populate1(key, jobs=self.jobs, **populate_kwargs) + if status is True: + success_list.append(1) + elif isinstance(status, tuple): + error_list.append(status) + # status is False means job was already reserved + else: + # spawn multiple processes + self.connection.close() + del self.connection._conn.ctx # SSLContext is not pickleable + with ( + mp.Pool(processes, _initialize_populate, (self, self.jobs, populate_kwargs)) as pool, + tqdm(desc="Processes: ", total=nkeys) + if display_progress + else contextlib.nullcontext() as progress_bar, + ): + for status in pool.imap(_call_populate1, keys, chunksize=1): + if status is True: + success_list.append(1) + elif isinstance(status, tuple): + error_list.append(status) + if display_progress: + progress_bar.update() + self.connection.connect() + + return { + "success_count": sum(success_list), + "error_list": error_list, + } + finally: + signal.signal(signal.SIGTERM, old_handler) + + def _populate1( + self, + key: dict[str, Any], + jobs: Job | None, + suppress_errors: bool, + return_exception_objects: bool, + make_kwargs: dict[str, Any] | None = None, + ) -> bool | tuple[dict[str, Any], Any]: + """ + Populate table for one key, calling make() inside a transaction. + + Parameters + ---------- + key : dict + Primary key specifying the job to populate. + jobs : Job or None + Job object for distributed mode, None for direct mode. + suppress_errors : bool + If True, errors are suppressed and returned. + return_exception_objects : bool + If True, return exception objects instead of messages. + make_kwargs : dict, optional + Keyword arguments passed to ``make()``. + + Returns + ------- + bool or tuple + True if make() succeeded, False if skipped (already done or reserved), + (key, error) tuple if suppress_errors=True and error occurred. + """ + import time + + # use the legacy `_make_tuples` callback. + make = self._make_tuples if hasattr(self, "_make_tuples") else self.make + + # Try to reserve the job (distributed mode only) + if jobs is not None and not jobs.reserve(key): + return False + + start_time = time.time() + + # if make is a generator, transaction can be delayed until the final stage + is_generator = inspect.isgeneratorfunction(make) + if not is_generator: + self.connection.start_transaction() + + if key in self: # already populated + if not is_generator: + self.connection.cancel_transaction() + if jobs is not None: + jobs.complete(key) + return False + + logger.debug(f"Making {key} -> {self.full_table_name}") + self.__class__._allow_insert = True + + try: + if not is_generator: + make(dict(key), **(make_kwargs or {})) + else: + # tripartite make - transaction is delayed until the final stage + gen = make(dict(key), **(make_kwargs or {})) + fetched_data = next(gen) + fetch_hash = deepdiff.DeepHash(fetched_data, ignore_iterable_order=False)[fetched_data] + computed_result = next(gen) # perform the computation + # fetch and insert inside a transaction + self.connection.start_transaction() + gen = make(dict(key), **(make_kwargs or {})) # restart make + fetched_data = next(gen) + if ( + fetch_hash != deepdiff.DeepHash(fetched_data, ignore_iterable_order=False)[fetched_data] + ): # raise error if fetched data has changed + raise DataJointError("Referential integrity failed! The `make_fetch` data has changed") + gen.send(computed_result) # insert + + except (KeyboardInterrupt, SystemExit, Exception) as error: + try: + self.connection.cancel_transaction() + except LostConnectionError: + pass + error_message = "{exception}{msg}".format( + exception=error.__class__.__name__, + msg=": " + str(error) if str(error) else "", + ) + logger.debug(f"Error making {key} -> {self.full_table_name} - {error_message}") + if jobs is not None: + jobs.error(key, error_message=error_message, error_stack=traceback.format_exc()) + if not suppress_errors or isinstance(error, SystemExit): + raise + else: + logger.error(error) + return key, error if return_exception_objects else error_message + else: + self.connection.commit_transaction() + duration = time.time() - start_time + logger.debug(f"Success making {key} -> {self.full_table_name}") + + # Update hidden job metadata if table has the columns + if self._has_job_metadata_attrs(): + from .jobs import _get_job_version + + self._update_job_metadata( + key, + start_time=datetime.datetime.fromtimestamp(start_time), + duration=duration, + version=_get_job_version(), + ) + + if jobs is not None: + jobs.complete(key, duration=duration) + return True + finally: + self.__class__._allow_insert = False + + def progress(self, *restrictions: Any, display: bool = False) -> tuple[int, int]: + """ + Report the progress of populating the table. + + Uses a single aggregation query to efficiently compute both total and + remaining counts. + + Parameters + ---------- + *restrictions + Conditions to restrict key_source. + display : bool, optional + If True, log the progress. Default False. + + Returns + ------- + tuple + (remaining, total) - number of keys yet to populate and total keys. + """ + todo = self._jobs_to_do(restrictions) + + # Get primary key attributes from key_source for join condition + # These are the "job keys" - the granularity at which populate() works + pk_attrs = todo.primary_key + assert pk_attrs, "key_source must have a primary key" + + # Find common attributes between key_source and self for the join + # This handles cases where self has additional PK attributes + common_attrs = [attr for attr in pk_attrs if attr in self.heading.names] + + if not common_attrs: + # No common attributes - fall back to two-query method + total = len(todo) + remaining = len(todo - self) + else: + # Build a single query that computes both total and remaining + # Using LEFT JOIN with COUNT(DISTINCT) to handle 1:many relationships + todo_sql = todo.make_sql() + target_sql = self.make_sql() + + # Build join condition on common attributes + join_cond = " AND ".join(f"`$ks`.`{attr}` = `$tgt`.`{attr}`" for attr in common_attrs) + + # Build DISTINCT key expression for counting unique jobs + # Use CONCAT for composite keys to create a single distinct value + if len(pk_attrs) == 1: + distinct_key = f"`$ks`.`{pk_attrs[0]}`" + null_check = f"`$tgt`.`{common_attrs[0]}`" + else: + distinct_key = "CONCAT_WS('|', {})".format(", ".join(f"`$ks`.`{attr}`" for attr in pk_attrs)) + null_check = f"`$tgt`.`{common_attrs[0]}`" + + # Single aggregation query: + # - COUNT(DISTINCT key) gives total unique jobs in key_source + # - Remaining = jobs where no matching target row exists + sql = f""" + SELECT + COUNT(DISTINCT {distinct_key}) AS total, + COUNT(DISTINCT CASE WHEN {null_check} IS NULL THEN {distinct_key} END) AS remaining + FROM ({todo_sql}) AS `$ks` + LEFT JOIN ({target_sql}) AS `$tgt` ON {join_cond} + """ + + result = self.connection.query(sql).fetchone() + total, remaining = result + + if display: + logger.info( + "%-20s" % self.__class__.__name__ + + " Completed %d of %d (%2.1f%%) %s" + % ( + total - remaining, + total, + 100 - 100 * remaining / (total + 1e-12), + datetime.datetime.strftime(datetime.datetime.now(), "%Y-%m-%d %H:%M:%S"), + ), + ) + return remaining, total + + def _has_job_metadata_attrs(self): + """Check if table has hidden job metadata columns.""" + # Access _attributes directly to include hidden attributes + all_attrs = self.heading._attributes + return all_attrs is not None and "_job_start_time" in all_attrs + + def _update_job_metadata(self, key, start_time, duration, version): + """ + Update hidden job metadata for the given key. + + Parameters + ---------- + key : dict + Primary key identifying the row(s) to update. + start_time : datetime + When computation started. + duration : float + Computation duration in seconds. + version : str + Code version (truncated to 64 chars). + """ + from .condition import make_condition + + pk_condition = make_condition(self, key, set()) + self.connection.query( + f"UPDATE {self.full_table_name} SET " + "`_job_start_time`=%s, `_job_duration`=%s, `_job_version`=%s " + f"WHERE {pk_condition}", + args=(start_time, duration, version[:64] if version else ""), + ) diff --git a/datajoint/blob.py b/src/datajoint/blob.py similarity index 70% rename from datajoint/blob.py rename to src/datajoint/blob.py index 639789680..292350ad7 100644 --- a/datajoint/blob.py +++ b/src/datajoint/blob.py @@ -1,8 +1,13 @@ """ -(De)serialization methods for basic datatypes and numpy.ndarrays with provisions for mutual -compatibility with Matlab-based serialization implemented by mYm. +Binary serialization for DataJoint blob storage. + +Provides (de)serialization for Python/NumPy objects with backward compatibility +for MATLAB mYm-format blobs. Supports arrays, scalars, structs, cells, and +Python built-in types (dict, list, tuple, set, datetime, UUID, Decimal). """ +from __future__ import annotations + import collections import datetime import uuid @@ -13,7 +18,6 @@ import numpy as np from .errors import DataJointError -from .settings import config deserialize_lookup = { 0: {"dtype": None, "scalar_type": "UNKNOWN"}, @@ -56,8 +60,6 @@ compression = {b"ZL123\0": zlib.decompress} -bypass_serialization = False # runtime setting to bypass blob (en|de)code - # runtime setting to read integers as 32-bit to read blobs created by the 32-bit # version of the mYm library for MATLAB use_32bit_dims = False @@ -72,37 +74,74 @@ def len_u32(obj): class MatCell(np.ndarray): - """a numpy ndarray representing a Matlab cell array""" + """ + NumPy ndarray subclass representing a MATLAB cell array. + + Used to distinguish cell arrays from regular arrays during serialization + for MATLAB compatibility. + """ pass class MatStruct(np.recarray): - """numpy.recarray representing a Matlab struct array""" + """ + NumPy recarray subclass representing a MATLAB struct array. + + Used to distinguish struct arrays from regular recarrays during + serialization for MATLAB compatibility. + """ pass class Blob: - def __init__(self, squeeze=False): + """ + Binary serializer/deserializer for DataJoint blob storage. + + Handles packing Python objects into binary format and unpacking binary + data back to Python objects. Supports two protocols: + + - ``mYm``: Original MATLAB-compatible format (default) + - ``dj0``: Extended format for Python-specific types + + Parameters + ---------- + squeeze : bool, optional + If True, remove singleton dimensions from arrays and convert + 0-dimensional arrays to scalars. Default False. + + Attributes + ---------- + protocol : bytes or None + Current serialization protocol (``b"mYm\\0"`` or ``b"dj0\\0"``). + """ + + def __init__(self, squeeze: bool = False) -> None: self._squeeze = squeeze self._blob = None self._pos = 0 self.protocol = None - def set_dj0(self): - if not config.get("enable_python_native_blobs"): - raise DataJointError( - """v0.12+ python native blobs disabled. - See also: https://github.com/datajoint/datajoint-python#python-native-blobs""" - ) - + def set_dj0(self) -> None: + """Switch to dj0 protocol for extended type support.""" self.protocol = b"dj0\0" # when using new blob features - def squeeze(self, array, convert_to_scalar=True): + def squeeze(self, array: np.ndarray, convert_to_scalar: bool = True) -> np.ndarray: """ - Simplify the input array - squeeze out all singleton dimensions. - If convert_to_scalar, then convert zero-dimensional arrays to scalars + Remove singleton dimensions from an array. + + Parameters + ---------- + array : np.ndarray + Input array. + convert_to_scalar : bool, optional + If True, convert 0-dimensional arrays to Python scalars. Default True. + + Returns + ------- + np.ndarray or scalar + Squeezed array or scalar value. """ if not self._squeeze: return array @@ -113,9 +152,7 @@ def unpack(self, blob): self._blob = blob try: # decompress - prefix = next( - p for p in compression if self._blob[self._pos :].startswith(p) - ) + prefix = next(p for p in compression if self._blob[self._pos :].startswith(p)) except StopIteration: pass # assume uncompressed but could be unrecognized compression else: @@ -157,10 +194,7 @@ def read_blob(self, n_bytes=None): "u": self.read_uuid, # UUID }[data_structure_code] except KeyError: - raise DataJointError( - 'Unknown data structure code "%s". Upgrade datajoint.' - % data_structure_code - ) + raise DataJointError('Unknown data structure code "%s". Upgrade datajoint.' % data_structure_code) v = call() if n_bytes is not None and self._pos - start != n_bytes: raise DataJointError("Blob length check failed! Invalid blob") @@ -215,9 +249,7 @@ def pack_blob(self, obj): return self.pack_set(obj) if obj is None: return self.pack_none() - raise DataJointError( - "Packing object of type %s currently not supported!" % type(obj) - ) + raise DataJointError("Packing object of type %s currently not supported!" % type(obj)) def read_array(self): n_dims = int(self.read_value()) @@ -241,11 +273,7 @@ def read_array(self): data = data[::2].astype("U1") if n_dims == 2 and shape[0] == 1 or n_dims == 1: compact = data.squeeze() - data = ( - compact - if compact.shape == () - else np.array("".join(data.squeeze())) - ) + data = compact if compact.shape == () else np.array("".join(data.squeeze())) shape = (1,) else: data = self.read_value(dtype, count=n_elem) @@ -253,17 +281,23 @@ def read_array(self): data = data + 1j * self.read_value(dtype, count=n_elem) return self.squeeze(data.reshape(shape, order="F")) - def pack_array(self, array): + def pack_array(self, array: np.ndarray) -> bytes: """ - Serialize an np.ndarray into bytes. Scalars are encoded with ndim=0. + Serialize a NumPy array into bytes. + + Parameters + ---------- + array : np.ndarray + Array to serialize. Scalars are encoded with ndim=0. + + Returns + ------- + bytes + Serialized array data. """ if "datetime64" in array.dtype.name: self.set_dj0() - blob = ( - b"A" - + np.uint64(array.ndim).tobytes() - + np.array(array.shape, dtype=np.uint64).tobytes() - ) + blob = b"A" + np.uint64(array.ndim).tobytes() + np.array(array.shape, dtype=np.uint64).tobytes() is_complex = np.iscomplexobj(array) if is_complex: array, imaginary = np.real(array), np.imag(array) @@ -277,19 +311,11 @@ def pack_array(self, array): raise DataJointError(f"Type {array.dtype} is ambiguous or unknown") blob += np.array([type_id, is_complex], dtype=np.uint32).tobytes() - if ( - array.dtype.char == "U" - or serialize_lookup[array.dtype]["scalar_type"] == "VOID" - ): - blob += b"".join( - len_u64(it) + it - for it in (self.pack_blob(e) for e in array.flatten(order="F")) - ) + if array.dtype.char == "U" or serialize_lookup[array.dtype]["scalar_type"] == "VOID": + blob += b"".join(len_u64(it) + it for it in (self.pack_blob(e) for e in array.flatten(order="F"))) self.set_dj0() # not supported by original mym elif serialize_lookup[array.dtype]["scalar_type"] == "CHAR": - blob += ( - array.view(np.uint8).astype(np.uint16).tobytes() - ) # convert to 16-bit chars for MATLAB + blob += array.view(np.uint8).astype(np.uint16).tobytes() # convert to 16-bit chars for MATLAB else: # numeric arrays if array.ndim == 0: # not supported by original mym self.set_dj0() @@ -323,34 +349,22 @@ def pack_recarray(self, array): + "\0".join(array.dtype.names).encode() # number of fields + b"\0" + b"".join( # field names - ( - self.pack_recarray(array[f]) - if array[f].dtype.fields - else self.pack_array(array[f]) - ) + (self.pack_recarray(array[f]) if array[f].dtype.fields else self.pack_array(array[f])) for f in array.dtype.names ) ) def read_sparse_array(self): - raise DataJointError( - "datajoint-python does not yet support sparse arrays. Issue (#590)" - ) + raise DataJointError("datajoint-python does not yet support sparse arrays. Issue (#590)") def read_int(self): - return int.from_bytes( - self.read_binary(self.read_value("uint16")), byteorder="little", signed=True - ) + return int.from_bytes(self.read_binary(self.read_value("uint16")), byteorder="little", signed=True) @staticmethod def pack_int(v): n_bytes = v.bit_length() // 8 + 1 assert 0 < n_bytes <= 0xFFFF, "Integers are limited to 65535 bytes" - return ( - b"\x0a" - + np.uint16(n_bytes).tobytes() - + v.to_bytes(n_bytes, byteorder="little", signed=True) - ) + return b"\x0a" + np.uint16(n_bytes).tobytes() + v.to_bytes(n_bytes, byteorder="little", signed=True) def read_bool(self): return bool(self.read_value("bool")) @@ -404,50 +418,32 @@ def pack_none(): return b"\xff" def read_tuple(self): - return tuple( - self.read_blob(self.read_value()) for _ in range(self.read_value()) - ) + return tuple(self.read_blob(self.read_value()) for _ in range(self.read_value())) def pack_tuple(self, t): - return ( - b"\1" - + len_u64(t) - + b"".join(len_u64(it) + it for it in (self.pack_blob(i) for i in t)) - ) + return b"\1" + len_u64(t) + b"".join(len_u64(it) + it for it in (self.pack_blob(i) for i in t)) def read_list(self): return list(self.read_blob(self.read_value()) for _ in range(self.read_value())) def pack_list(self, t): - return ( - b"\2" - + len_u64(t) - + b"".join(len_u64(it) + it for it in (self.pack_blob(i) for i in t)) - ) + return b"\2" + len_u64(t) + b"".join(len_u64(it) + it for it in (self.pack_blob(i) for i in t)) def read_set(self): return set(self.read_blob(self.read_value()) for _ in range(self.read_value())) def pack_set(self, t): - return ( - b"\3" - + len_u64(t) - + b"".join(len_u64(it) + it for it in (self.pack_blob(i) for i in t)) - ) + return b"\3" + len_u64(t) + b"".join(len_u64(it) + it for it in (self.pack_blob(i) for i in t)) def read_dict(self): - return dict( - (self.read_blob(self.read_value()), self.read_blob(self.read_value())) - for _ in range(self.read_value()) - ) + return dict((self.read_blob(self.read_value()), self.read_blob(self.read_value())) for _ in range(self.read_value())) def pack_dict(self, d): return ( b"\4" + len_u64(d) + b"".join( - b"".join((len_u64(it) + it) for it in packed) - for packed in (map(self.pack_blob, pair) for pair in d.items()) + b"".join((len_u64(it) + it) for it in packed) for packed in (map(self.pack_blob, pair) for pair in d.items()) ) ) @@ -460,16 +456,9 @@ def read_struct(self): if not n_fields: return np.array(None) # empty array field_names = [self.read_zero_terminated_string() for _ in range(n_fields)] - raw_data = [ - tuple( - self.read_blob(n_bytes=int(self.read_value())) for _ in range(n_fields) - ) - for __ in range(n_elem) - ] + raw_data = [tuple(self.read_blob(n_bytes=int(self.read_value())) for _ in range(n_fields)) for __ in range(n_elem)] data = np.array(raw_data, dtype=list(zip(field_names, repeat(object)))) - return self.squeeze( - data.reshape(shape, order="F"), convert_to_scalar=False - ).view(MatStruct) + return self.squeeze(data.reshape(shape, order="F"), convert_to_scalar=False).view(MatStruct) def pack_struct(self, array): """Serialize a Matlab struct array""" @@ -480,43 +469,47 @@ def pack_struct(self, array): + "\0".join(array.dtype.names).encode() # number of fields + b"\0" + b"".join( # field names - len_u64(it) + it - for it in ( - self.pack_blob(e) for rec in array.flatten(order="F") for e in rec - ) + len_u64(it) + it for it in (self.pack_blob(e) for rec in array.flatten(order="F") for e in rec) ) ) # values def read_cell_array(self): - """deserialize MATLAB cell array""" + """ + Deserialize MATLAB cell array. + + Handles edge cases from MATLAB: + - Empty cell arrays ({}) + - Cell arrays with empty elements ({[], [], []}) + - Nested arrays ({[1,2], [3,4,5]}) - ragged arrays + - Cell matrices with mixed content + """ n_dims = self.read_value() shape = self.read_value(count=n_dims) n_elem = int(np.prod(shape)) result = [self.read_blob(n_bytes=self.read_value()) for _ in range(n_elem)] - return ( - self.squeeze( - np.array(result).reshape(shape, order="F"), convert_to_scalar=False - ) - ).view(MatCell) + + # Handle empty cell array + if n_elem == 0: + return np.empty(0, dtype=object).view(MatCell) + + # Use object dtype to handle ragged/nested arrays without reshape errors. + # This avoids NumPy's array homogeneity requirements that cause failures + # with MATLAB cell arrays containing arrays of different sizes. + arr = np.empty(n_elem, dtype=object) + arr[:] = result + return self.squeeze(arr.reshape(shape, order="F"), convert_to_scalar=False).view(MatCell) def pack_cell_array(self, array): return ( b"C" + np.array((array.ndim,) + array.shape, dtype=np.uint64).tobytes() - + b"".join( - len_u64(it) + it - for it in (self.pack_blob(e) for e in array.flatten(order="F")) - ) + + b"".join(len_u64(it) + it for it in (self.pack_blob(e) for e in array.flatten(order="F"))) ) def read_datetime(self): """deserialize datetime.date, .time, or .datetime""" date, time = self.read_value("int32"), self.read_value("int64") - date = ( - datetime.date(year=date // 10000, month=(date // 100) % 100, day=date % 100) - if date >= 0 - else None - ) + date = datetime.date(year=date // 10000, month=(date // 100) % 100, day=date % 100) if date >= 0 else None time = ( datetime.time( hour=(time // 10000000000) % 100, @@ -538,14 +531,9 @@ def pack_datetime(d): else: date, time = None, d return b"t" + ( - np.int32( - -1 if date is None else (date.year * 100 + date.month) * 100 + date.day - ).tobytes() + np.int32(-1 if date is None else (date.year * 100 + date.month) * 100 + date.day).tobytes() + np.int64( - -1 - if time is None - else ((time.hour * 100 + time.minute) * 100 + time.second) * 1000000 - + time.microsecond + -1 if time is None else ((time.hour * 100 + time.minute) * 100 + time.second) * 1000000 + time.microsecond ).tobytes() ) @@ -576,9 +564,7 @@ def read_binary(self, size): def pack(self, obj, compress): self.protocol = b"mYm\0" # will be replaced with dj0 if new features are used - blob = self.pack_blob( - obj - ) # this may reset the protocol and must precede protocol evaluation + blob = self.pack_blob(obj) # this may reset the protocol and must precede protocol evaluation blob = self.protocol + blob if compress and len(blob) > 1000: compressed = b"ZL123\0" + len_u64(blob) + zlib.compress(blob) @@ -587,22 +573,60 @@ def pack(self, obj, compress): return blob -def pack(obj, compress=True): - if bypass_serialization: - # provide a way to move blobs quickly without de/serialization - assert isinstance(obj, bytes) and obj.startswith( - (b"ZL123\0", b"mYm\0", b"dj0\0") - ) - return obj +def pack(obj, compress: bool = True) -> bytes: + """ + Serialize a Python object to binary blob format. + + Parameters + ---------- + obj : any + Object to serialize. Supports NumPy arrays, Python scalars, + collections (dict, list, tuple, set), datetime objects, UUID, + Decimal, and MATLAB-compatible MatCell/MatStruct. + compress : bool, optional + If True (default), compress blobs larger than 1000 bytes using zlib. + + Returns + ------- + bytes + Serialized binary data. + + Raises + ------ + DataJointError + If the object type is not supported. + + Examples + -------- + >>> data = np.array([1, 2, 3]) + >>> blob = pack(data) + >>> unpacked = unpack(blob) + """ return Blob().pack(obj, compress=compress) -def unpack(blob, squeeze=False): - if bypass_serialization: - # provide a way to move blobs quickly without de/serialization - assert isinstance(blob, bytes) and blob.startswith( - (b"ZL123\0", b"mYm\0", b"dj0\0") - ) - return blob +def unpack(blob: bytes, squeeze: bool = False): + """ + Deserialize a binary blob to a Python object. + + Parameters + ---------- + blob : bytes + Binary data from ``pack()`` or MATLAB mYm serialization. + squeeze : bool, optional + If True, remove singleton dimensions from arrays. Default False. + + Returns + ------- + any + Deserialized Python object. + + Examples + -------- + >>> blob = pack({'a': 1, 'b': [1, 2, 3]}) + >>> data = unpack(blob) + >>> data['b'] + [1, 2, 3] + """ if blob is not None: return Blob(squeeze=squeeze).unpack(blob) diff --git a/src/datajoint/builtin_codecs.py b/src/datajoint/builtin_codecs.py new file mode 100644 index 000000000..66589dc36 --- /dev/null +++ b/src/datajoint/builtin_codecs.py @@ -0,0 +1,690 @@ +""" +Built-in DataJoint codecs. + +This module defines the standard codecs that ship with DataJoint. +These serve as both useful built-in codecs and as examples for users who +want to create their own custom codecs. + +Built-in Codecs: + - ````: Serialize Python objects (internal) or external with dedup + - ````: Hash-addressed storage with MD5 deduplication + - ````: Path-addressed storage for files/folders (Zarr, HDF5) + - ````: File attachment (internal) or external with dedup + - ````: Reference to existing file in store + +Example - Creating a Custom Codec: + Here's how to define your own codec, modeled after the built-in codecs:: + + import datajoint as dj + import networkx as nx + + class GraphCodec(dj.Codec): + '''Store NetworkX graphs as edge lists.''' + + name = "graph" # Use as in definitions + + def get_dtype(self, is_external: bool) -> str: + return "" # Compose with blob for serialization + + def encode(self, graph, *, key=None, store_name=None): + # Convert graph to a serializable format + return { + 'nodes': list(graph.nodes(data=True)), + 'edges': list(graph.edges(data=True)), + } + + def decode(self, stored, *, key=None): + # Reconstruct graph from stored format + G = nx.Graph() + G.add_nodes_from(stored['nodes']) + G.add_edges_from(stored['edges']) + return G + + def validate(self, value): + if not isinstance(value, nx.Graph): + raise TypeError(f"Expected nx.Graph, got {type(value).__name__}") + + # Now use in table definitions: + @schema + class Networks(dj.Manual): + definition = ''' + network_id : int + --- + topology : + ''' +""" + +from __future__ import annotations + +from typing import Any + +from .codecs import Codec +from .errors import DataJointError + + +# ============================================================================= +# Blob Codec - DataJoint's native serialization +# ============================================================================= + + +class BlobCodec(Codec): + """ + Serialize Python objects using DataJoint's blob format. + + The ```` codec handles serialization of arbitrary Python objects + including NumPy arrays, dictionaries, lists, datetime objects, and UUIDs. + + Supports both internal and external storage: + - ````: Stored in database (bytes → LONGBLOB) + - ````: Stored externally via ```` with deduplication + - ````: Stored in specific named store + + Format Features: + - Protocol headers (``mYm`` for MATLAB-compatible, ``dj0`` for Python-native) + - Optional zlib compression for data > 1KB + - Support for nested structures + + Example:: + + @schema + class ProcessedData(dj.Manual): + definition = ''' + data_id : int + --- + small_result : # internal (in database) + large_result : # external (default store) + archive : # external (specific store) + ''' + + # Insert any serializable object + table.insert1({'data_id': 1, 'small_result': {'scores': [0.9, 0.8]}}) + """ + + name = "blob" + + def get_dtype(self, is_external: bool) -> str: + """Return bytes for internal, for external storage.""" + return "" if is_external else "bytes" + + def encode(self, value: Any, *, key: dict | None = None, store_name: str | None = None) -> bytes: + """Serialize a Python object to DataJoint's blob format.""" + from . import blob + + return blob.pack(value, compress=True) + + def decode(self, stored: bytes, *, key: dict | None = None) -> Any: + """Deserialize blob bytes back to a Python object.""" + from . import blob + + return blob.unpack(stored, squeeze=False) + + +# ============================================================================= +# Hash-Addressed Storage Codec +# ============================================================================= + + +class HashCodec(Codec): + """ + Hash-addressed storage with MD5 deduplication. + + The ```` codec stores raw bytes using content-addressed storage. + Data is identified by its MD5 hash and stored in a hierarchical directory: + ``_hash/{hash[:2]}/{hash[2:4]}/{hash}`` + + The database column stores JSON metadata: ``{hash, store, size}``. + Duplicate content is automatically deduplicated. + + External only - requires @ modifier. + + Example:: + + @schema + class RawContent(dj.Manual): + definition = ''' + content_id : int + --- + data : + ''' + + # Insert raw bytes + table.insert1({'content_id': 1, 'data': b'raw binary content'}) + + Note: + This codec accepts only ``bytes``. For Python objects, use ````. + Typically used indirectly via ```` or ```` rather than directly. + """ + + name = "hash" + + def get_dtype(self, is_external: bool) -> str: + """Hash storage is external only.""" + if not is_external: + raise DataJointError(" requires @ (external storage only)") + return "json" + + def encode(self, value: bytes, *, key: dict | None = None, store_name: str | None = None) -> dict: + """ + Store content and return metadata. + + Parameters + ---------- + value : bytes + Raw bytes to store. + key : dict, optional + Primary key values (unused). + store_name : str, optional + Store to use. If None, uses default store. + + Returns + ------- + dict + Metadata dict: ``{hash, store, size}``. + """ + from .content_registry import put_content + + return put_content(value, store_name=store_name) + + def decode(self, stored: dict, *, key: dict | None = None) -> bytes: + """ + Retrieve content by hash. + + Parameters + ---------- + stored : dict + Metadata dict with ``'hash'`` and optionally ``'store'``. + key : dict, optional + Primary key values (unused). + + Returns + ------- + bytes + Original bytes. + """ + from .content_registry import get_content + + return get_content(stored["hash"], store_name=stored.get("store")) + + def validate(self, value: Any) -> None: + """Validate that value is bytes.""" + if not isinstance(value, bytes): + raise TypeError(f" expects bytes, got {type(value).__name__}") + + +# ============================================================================= +# Path-Addressed Storage Codec (OAS - Object-Augmented Schema) +# ============================================================================= + + +class ObjectCodec(Codec): + """ + Path-addressed storage for files and folders. + + The ```` codec provides managed file/folder storage where the path + is derived from the primary key: ``{schema}/{table}/{pk}/{field}/`` + + Unlike ```` (hash-addressed), each row has its own storage path, + and content is deleted when the row is deleted. This is ideal for: + + - Zarr arrays (hierarchical chunked data) + - HDF5 files + - Complex multi-file outputs + - Any content that shouldn't be deduplicated + + External only - requires @ modifier. + + Example:: + + @schema + class Analysis(dj.Computed): + definition = ''' + -> Recording + --- + results : + ''' + + def make(self, key): + # Store a file + self.insert1({**key, 'results': '/path/to/results.zarr'}) + + # Fetch returns ObjectRef for lazy access + ref = (Analysis & key).fetch1('results') + ref.path # Storage path + ref.read() # Read file content + ref.fsmap # For zarr.open(ref.fsmap) + + Storage Structure: + Objects are stored at:: + + {store_root}/{schema}/{table}/{pk}/{field}/ + + Comparison with ````:: + + | Aspect | | | + |----------------|-------------------|---------------------| + | Addressing | Path (by PK) | Hash (by content) | + | Deduplication | No | Yes | + | Deletion | With row | GC when unreferenced| + | Use case | Zarr, HDF5 | Blobs, attachments | + """ + + name = "object" + + def get_dtype(self, is_external: bool) -> str: + """Object storage is external only.""" + if not is_external: + raise DataJointError(" requires @ (external storage only)") + return "json" + + def encode( + self, + value: Any, + *, + key: dict | None = None, + store_name: str | None = None, + ) -> dict: + """ + Store content and return metadata. + + Parameters + ---------- + value : bytes, str, or Path + Content to store: bytes (raw data), or str/Path (file/folder to upload). + key : dict, optional + Context for path construction with keys ``_schema``, ``_table``, + ``_field``, plus primary key values. + store_name : str, optional + Store to use. If None, uses default store. + + Returns + ------- + dict + Metadata dict suitable for ``ObjectRef.from_json()``. + """ + from datetime import datetime, timezone + from pathlib import Path + + from .content_registry import get_store_backend + from .storage import build_object_path + + # Extract context from key + key = key or {} + schema = key.pop("_schema", "unknown") + table = key.pop("_table", "unknown") + field = key.pop("_field", "data") + primary_key = {k: v for k, v in key.items() if not k.startswith("_")} + + # Check for pre-computed metadata (from staged insert) + if isinstance(value, dict) and "path" in value: + # Already encoded, pass through + return value + + # Determine content type and extension + is_dir = False + ext = None + size = None + item_count = None + + if isinstance(value, bytes): + content = value + size = len(content) + elif isinstance(value, tuple) and len(value) == 2: + # Tuple format: (extension, data) where data is bytes or file-like + ext, data = value + if hasattr(data, "read"): + content = data.read() + else: + content = data + size = len(content) + elif isinstance(value, (str, Path)): + source_path = Path(value) + if not source_path.exists(): + raise DataJointError(f"Source path not found: {source_path}") + is_dir = source_path.is_dir() + ext = source_path.suffix if not is_dir else None + if is_dir: + # For directories, we'll upload later + content = None + # Count items in directory + item_count = sum(1 for _ in source_path.rglob("*") if _.is_file()) + else: + content = source_path.read_bytes() + size = len(content) + else: + raise TypeError(f" expects bytes or path, got {type(value).__name__}") + + # Build storage path + path, token = build_object_path( + schema=schema, + table=table, + field=field, + primary_key=primary_key, + ext=ext, + ) + + # Get storage backend + backend = get_store_backend(store_name) + + # Upload content + if is_dir: + # Upload directory recursively + source_path = Path(value) + backend.put_folder(str(source_path), path) + # Compute size by summing all files + size = sum(f.stat().st_size for f in source_path.rglob("*") if f.is_file()) + else: + backend.put_buffer(content, path) + + # Build metadata + timestamp = datetime.now(timezone.utc) + metadata = { + "path": path, + "store": store_name, + "size": size, + "ext": ext, + "is_dir": is_dir, + "item_count": item_count, + "timestamp": timestamp.isoformat(), + } + + return metadata + + def decode(self, stored: dict, *, key: dict | None = None) -> Any: + """ + Create ObjectRef handle for lazy access. + + Parameters + ---------- + stored : dict + Metadata dict from database. + key : dict, optional + Primary key values (unused). + + Returns + ------- + ObjectRef + Handle for accessing the stored content. + """ + from .objectref import ObjectRef + from .content_registry import get_store_backend + + store_name = stored.get("store") + backend = get_store_backend(store_name) + return ObjectRef.from_json(stored, backend=backend) + + def validate(self, value: Any) -> None: + """Validate value is bytes, path, dict metadata, or (ext, data) tuple.""" + from pathlib import Path + + if isinstance(value, bytes): + return + if isinstance(value, (str, Path)): + # Could be a path or pre-encoded JSON string + return + if isinstance(value, tuple) and len(value) == 2: + # Tuple format: (extension, data) + return + if isinstance(value, dict) and "path" in value: + # Pre-computed metadata dict (from staged insert) + return + raise TypeError(f" expects bytes or path, got {type(value).__name__}") + + +# ============================================================================= +# File Attachment Codecs +# ============================================================================= + + +class AttachCodec(Codec): + """ + File attachment with filename preserved. + + Supports both internal and external storage: + - ````: Stored in database (bytes → LONGBLOB) + - ````: Stored externally via ```` with deduplication + - ````: Stored in specific named store + + The filename is preserved and the file is extracted to the configured + download path on fetch. + + Example:: + + @schema + class Documents(dj.Manual): + definition = ''' + doc_id : int + --- + config : # internal (small file in DB) + dataset : # external (default store) + archive : # external (specific store) + ''' + + # Insert a file + table.insert1({'doc_id': 1, 'config': '/path/to/config.json'}) + + # Fetch extracts to download_path and returns local path + local_path = (table & 'doc_id=1').fetch1('config') + + Storage Format (internal): + The blob contains: ``filename\\0contents`` + - Filename (UTF-8 encoded) + null byte + raw file contents + """ + + name = "attach" + + def get_dtype(self, is_external: bool) -> str: + """Return bytes for internal, for external storage.""" + return "" if is_external else "bytes" + + def encode(self, value: Any, *, key: dict | None = None, store_name: str | None = None) -> bytes: + """ + Read file and encode as filename + contents. + + Parameters + ---------- + value : str or Path + Path to file. + key : dict, optional + Primary key values (unused). + store_name : str, optional + Unused for internal storage. + + Returns + ------- + bytes + Filename (UTF-8) + null byte + file contents. + """ + from pathlib import Path + + path = Path(value) + if not path.exists(): + raise FileNotFoundError(f"Attachment file not found: {path}") + if path.is_dir(): + raise IsADirectoryError(f" does not support directories: {path}") + + filename = path.name + contents = path.read_bytes() + return filename.encode("utf-8") + b"\x00" + contents + + def decode(self, stored: bytes, *, key: dict | None = None) -> str: + """ + Extract file to download path and return local path. + + Parameters + ---------- + stored : bytes + Blob containing filename + null + contents. + key : dict, optional + Primary key values (unused). + + Returns + ------- + str + Path to extracted file. + """ + from pathlib import Path + + from .settings import config + + # Split on first null byte + null_pos = stored.index(b"\x00") + filename = stored[:null_pos].decode("utf-8") + contents = stored[null_pos + 1 :] + + # Write to download path + download_path = Path(config.get("download_path", ".")) + download_path.mkdir(parents=True, exist_ok=True) + local_path = download_path / filename + + # Handle filename collision - if file exists with different content, add suffix + if local_path.exists(): + existing_contents = local_path.read_bytes() + if existing_contents != contents: + # Find unique filename + stem = local_path.stem + suffix = local_path.suffix + counter = 1 + while local_path.exists() and local_path.read_bytes() != contents: + local_path = download_path / f"{stem}_{counter}{suffix}" + counter += 1 + + # Only write if file doesn't exist or has different content + if not local_path.exists(): + local_path.write_bytes(contents) + + return str(local_path) + + def validate(self, value: Any) -> None: + """Validate that value is a valid file path.""" + from pathlib import Path + + if not isinstance(value, (str, Path)): + raise TypeError(f" expects a file path, got {type(value).__name__}") + + +# ============================================================================= +# Filepath Reference Codec +# ============================================================================= + + +class FilepathCodec(Codec): + """ + Reference to existing file in configured store. + + The ```` codec stores a reference to a file that already + exists in the storage backend. Unlike ```` or ````, no + file copying occurs - only the path is recorded. + + External only - requires @store. + + This is useful when: + - Files are managed externally (e.g., by acquisition software) + - Files are too large to copy + - You want to reference shared datasets + + Example:: + + @schema + class Recordings(dj.Manual): + definition = ''' + recording_id : int + --- + raw_data : + ''' + + # Reference an existing file (no copy) + table.insert1({'recording_id': 1, 'raw_data': 'subject01/session001/data.bin'}) + + # Fetch returns ObjectRef for lazy access + ref = (table & 'recording_id=1').fetch1('raw_data') + ref.read() # Read file content + ref.download() # Download to local path + + Storage Format: + JSON metadata: ``{path, store}`` + + Warning: + The file must exist in the store at the specified path. + DataJoint does not manage the lifecycle of referenced files. + """ + + name = "filepath" + + def get_dtype(self, is_external: bool) -> str: + """Filepath is external only.""" + if not is_external: + raise DataJointError(" requires @store") + return "json" + + def encode(self, value: Any, *, key: dict | None = None, store_name: str | None = None) -> dict: + """ + Store path reference as JSON metadata. + + Parameters + ---------- + value : str + Relative path within the store. + key : dict, optional + Primary key values (unused). + store_name : str, optional + Store where the file exists. + + Returns + ------- + dict + Metadata dict: ``{path, store}``. + """ + from datetime import datetime, timezone + + from .content_registry import get_store_backend + + path = str(value) + + # Optionally verify file exists + backend = get_store_backend(store_name) + if not backend.exists(path): + raise FileNotFoundError(f"File not found in store '{store_name or 'default'}': {path}") + + # Get file info + try: + size = backend.size(path) + except Exception: + size = None + + return { + "path": path, + "store": store_name, + "size": size, + "is_dir": False, + "timestamp": datetime.now(timezone.utc).isoformat(), + } + + def decode(self, stored: dict, *, key: dict | None = None) -> Any: + """ + Create ObjectRef handle for lazy access. + + Parameters + ---------- + stored : dict + Metadata dict with path and store. + key : dict, optional + Primary key values (unused). + + Returns + ------- + ObjectRef + Handle for accessing the file. + """ + from .objectref import ObjectRef + from .content_registry import get_store_backend + + store_name = stored.get("store") + backend = get_store_backend(store_name) + return ObjectRef.from_json(stored, backend=backend) + + def validate(self, value: Any) -> None: + """Validate that value is a path string or Path object.""" + from pathlib import Path + + if not isinstance(value, (str, Path)): + raise TypeError(f" expects a path string or Path, got {type(value).__name__}") diff --git a/src/datajoint/cli.py b/src/datajoint/cli.py new file mode 100644 index 000000000..c77cca686 --- /dev/null +++ b/src/datajoint/cli.py @@ -0,0 +1,125 @@ +""" +DataJoint command-line interface. + +Provides a Python REPL with DataJoint pre-loaded and optional schema access. + +Usage:: + + # Start REPL with database credentials + dj --user root --password secret --host localhost:3306 + + # Load schemas as virtual modules + dj -s my_lab:lab -s my_analysis:analysis + + # In the REPL + >>> lab.Subject.to_dicts() + >>> dj.Diagram(lab.schema) +""" + +from __future__ import annotations + +import argparse +from code import interact +from collections import ChainMap + +import datajoint as dj + + +def cli(args: list[str] | None = None) -> None: + """ + DataJoint command-line interface. + + Starts an interactive Python REPL with DataJoint imported and configured. + Optionally loads database schemas as virtual modules for quick exploration. + + Parameters + ---------- + args : list[str], optional + Command-line arguments. If None, reads from sys.argv. + + Examples + -------- + From the command line:: + + $ dj --host localhost:3306 --user root --password secret + $ dj -s my_lab:lab -s my_analysis:analysis + + Programmatically:: + + >>> from datajoint.cli import cli + >>> cli(["--version"]) + """ + parser = argparse.ArgumentParser( + prog="dj", + description="DataJoint interactive console. Start a Python REPL with DataJoint pre-loaded.", + epilog="Example: dj -s my_lab:lab --host localhost:3306", + ) + parser.add_argument( + "-V", + "--version", + action="version", + version=f"{dj.__name__} {dj.__version__}", + ) + parser.add_argument( + "-u", + "--user", + type=str, + default=None, + help="Database username (default: from config)", + ) + parser.add_argument( + "-p", + "--password", + type=str, + default=None, + help="Database password (default: from config)", + ) + parser.add_argument( + "--host", + type=str, + default=None, + help="Database host as host:port (default: from config)", + ) + parser.add_argument( + "-s", + "--schemas", + nargs="+", + type=str, + metavar="DB:ALIAS", + help="Load schemas as virtual modules. Format: schema_name:alias", + ) + + kwargs = vars(parser.parse_args(args)) + + # Apply credentials to config + if kwargs["user"]: + dj.config["database.user"] = kwargs["user"] + if kwargs["password"]: + dj.config["database.password"] = kwargs["password"] + if kwargs["host"]: + dj.config["database.host"] = kwargs["host"] + + # Load requested schemas + mods: dict[str, dj.VirtualModule] = {} + if kwargs["schemas"]: + for vm in kwargs["schemas"]: + if ":" not in vm: + parser.error(f"Invalid schema format '{vm}'. Use schema_name:alias") + schema_name, alias = vm.split(":", 1) + mods[alias] = dj.VirtualModule(alias, schema_name) + + # Build banner + banner = f"DataJoint {dj.__version__} REPL\n" + banner += "Type 'dj.' and press Tab for available functions.\n" + if mods: + banner += "\nLoaded schemas:\n" + for alias in mods: + banner += f" {alias} -> {mods[alias].schema.database}\n" + + # Start interactive session + interact(banner, local=dict(ChainMap(mods, {"dj": dj}, globals()))) + raise SystemExit + + +if __name__ == "__main__": + cli() diff --git a/src/datajoint/codecs.py b/src/datajoint/codecs.py new file mode 100644 index 000000000..211308d1c --- /dev/null +++ b/src/datajoint/codecs.py @@ -0,0 +1,585 @@ +""" +Codec type system for DataJoint. + +This module provides the Codec base class for creating custom data types +that extend DataJoint's native type system. Codecs provide encode/decode +semantics for complex Python objects. + +Codecs auto-register when subclassed - no decorator needed (Python 3.10+). + +Example: + class GraphCodec(dj.Codec): + name = "graph" + + def get_dtype(self, is_external: bool) -> str: + return "" + + def encode(self, graph, *, key=None, store_name=None): + return {'nodes': list(graph.nodes()), 'edges': list(graph.edges())} + + def decode(self, stored, *, key=None): + import networkx as nx + G = nx.Graph() + G.add_nodes_from(stored['nodes']) + G.add_edges_from(stored['edges']) + return G + + # Then use in table definitions: + class MyTable(dj.Manual): + definition = ''' + id : uint16 + --- + data : + ''' +""" + +from __future__ import annotations + +import logging +from abc import ABC, abstractmethod +from typing import Any + +from .errors import DataJointError + +logger = logging.getLogger(__name__.split(".")[0]) + +# Global codec registry - maps name to Codec instance +_codec_registry: dict[str, Codec] = {} +_entry_points_loaded: bool = False + + +class Codec(ABC): + """ + Base class for codec types. Subclasses auto-register by name. + + Requires Python 3.10+. + + Attributes + ---------- + name : str or None + Unique identifier used in ```` syntax. Must be set by subclasses. + + Examples + -------- + >>> class GraphCodec(dj.Codec): + ... name = "graph" + ... + ... def get_dtype(self, is_external: bool) -> str: + ... return "" + ... + ... def encode(self, graph, *, key=None, store_name=None): + ... return {'nodes': list(graph.nodes()), 'edges': list(graph.edges())} + ... + ... def decode(self, stored, *, key=None): + ... import networkx as nx + ... G = nx.Graph() + ... G.add_nodes_from(stored['nodes']) + ... G.add_edges_from(stored['edges']) + ... return G + + Use in table definitions:: + + class Connectivity(dj.Manual): + definition = ''' + id : uint16 + --- + graph_data : + ''' + + Skip auto-registration for abstract base classes:: + + class ExternalOnlyCodec(dj.Codec, register=False): + '''Abstract base - not registered.''' + ... + """ + + name: str | None = None # Must be set by concrete subclasses + + def __init_subclass__(cls, *, register: bool = True, **kwargs): + """Auto-register concrete codecs when subclassed.""" + super().__init_subclass__(**kwargs) + + if not register: + return # Skip registration for abstract bases + + if cls.name is None: + return # Skip registration if no name (abstract) + + if not isinstance(cls.name, str) or not cls.name: + raise DataJointError(f"Codec name must be a non-empty string, got {cls.name!r}") + + if cls.name in _codec_registry: + existing = _codec_registry[cls.name] + if type(existing) is not cls: + raise DataJointError( + f"Codec <{cls.name}> already registered by {type(existing).__module__}.{type(existing).__name__}" + ) + return # Same class, idempotent + + _codec_registry[cls.name] = cls() + logger.debug(f"Registered codec <{cls.name}> from {cls.__module__}.{cls.__name__}") + + @abstractmethod + def get_dtype(self, is_external: bool) -> str: + """ + Return the storage dtype for this codec. + + Parameters + ---------- + is_external : bool + True if ``@`` modifier present (external storage). + + Returns + ------- + str + A core type (e.g., ``"bytes"``, ``"json"``) or another codec + (e.g., ``""``). + + Raises + ------ + DataJointError + If external storage not supported but requested. + """ + ... + + @abstractmethod + def encode(self, value: Any, *, key: dict | None = None, store_name: str | None = None) -> Any: + """ + Encode Python value for storage. + + Parameters + ---------- + value : any + The Python object to store. + key : dict, optional + Primary key values. May be needed for path construction. + store_name : str, optional + Target store name for external storage. + + Returns + ------- + any + Value in the format expected by the dtype. + """ + ... + + @abstractmethod + def decode(self, stored: Any, *, key: dict | None = None) -> Any: + """ + Decode stored value back to Python. + + Parameters + ---------- + stored : any + Data retrieved from storage. + key : dict, optional + Primary key values. + + Returns + ------- + any + The reconstructed Python object. + """ + ... + + def validate(self, value: Any) -> None: + """ + Validate a value before encoding. + + Override this method to add type checking or domain constraints. + Called automatically before ``encode()`` during INSERT operations. + The default implementation accepts any value. + + Parameters + ---------- + value : any + The value to validate. + + Raises + ------ + TypeError + If the value has an incompatible type. + ValueError + If the value fails domain validation. + """ + pass + + def __repr__(self) -> str: + return f"<{self.__class__.__name__}(name={self.name!r})>" + + +def parse_type_spec(spec: str) -> tuple[str, str | None]: + """ + Parse a type specification into type name and optional store parameter. + + Parameters + ---------- + spec : str + Type specification string, with or without angle brackets. + + Returns + ------- + tuple[str, str | None] + ``(type_name, store_name)``. ``store_name`` is None if not specified, + empty string if ``@`` present without name (default store). + + Examples + -------- + >>> parse_type_spec("") + ("blob", None) + >>> parse_type_spec("") + ("blob", "cold") + >>> parse_type_spec("") + ("blob", "") + """ + # Strip angle brackets + spec = spec.strip("<>").strip() + + if "@" in spec: + type_name, store_name = spec.split("@", 1) + return type_name.strip(), store_name.strip() + + return spec, None + + +def unregister_codec(name: str) -> None: + """ + Remove a codec from the registry. + + Primarily useful for testing. Use with caution in production code. + + Parameters + ---------- + name : str + The codec name to unregister. + + Raises + ------ + DataJointError + If the codec is not registered. + """ + name = name.strip("<>") + if name not in _codec_registry: + raise DataJointError(f"Codec <{name}> is not registered") + del _codec_registry[name] + + +def get_codec(name: str) -> Codec: + """ + Retrieve a registered codec by name. + + Looks up the codec in the explicit registry first, then attempts + to load from installed packages via entry points. + + Parameters + ---------- + name : str + The codec name, with or without angle brackets. + Store parameters (e.g., ``""``) are stripped. + + Returns + ------- + Codec + The registered Codec instance. + + Raises + ------ + DataJointError + If the codec is not found. + """ + # Strip angle brackets and store parameter + type_name, _ = parse_type_spec(name) + + # Check explicit registry first + if type_name in _codec_registry: + return _codec_registry[type_name] + + # Lazy-load entry points + _load_entry_points() + + if type_name in _codec_registry: + return _codec_registry[type_name] + + raise DataJointError( + f"Unknown codec: <{type_name}>. Ensure the codec is defined (inherit from dj.Codec with name='{type_name}')." + ) + + +def list_codecs() -> list[str]: + """ + List all registered codec names. + + Returns + ------- + list[str] + Sorted list of registered codec names. + """ + _load_entry_points() + return sorted(_codec_registry.keys()) + + +def is_codec_registered(name: str) -> bool: + """ + Check if a codec name is registered. + + Parameters + ---------- + name : str + The codec name to check (store parameters are ignored). + + Returns + ------- + bool + True if the codec is registered. + """ + type_name, _ = parse_type_spec(name) + if type_name in _codec_registry: + return True + _load_entry_points() + return type_name in _codec_registry + + +def _load_entry_points() -> None: + """ + Load codecs from installed packages via entry points. + + Codecs are discovered from the ``datajoint.codecs`` entry point group + (also checks legacy ``datajoint.types`` for backward compatibility). + + Packages declare codecs in pyproject.toml:: + + [project.entry-points."datajoint.codecs"] + zarr_array = "dj_zarr:ZarrArrayCodec" + + This function is idempotent - entry points are only loaded once. + """ + global _entry_points_loaded + if _entry_points_loaded: + return + + _entry_points_loaded = True + + try: + from importlib.metadata import entry_points + except ImportError: + logger.debug("importlib.metadata not available, skipping entry point discovery") + return + + # Load from both new and legacy entry point groups + for group in ("datajoint.codecs", "datajoint.types"): + try: + eps = entry_points(group=group) + except TypeError: + # Older API fallback + eps = entry_points().get(group, []) + + for ep in eps: + if ep.name in _codec_registry: + # Already registered explicitly, skip entry point + continue + try: + codec_class = ep.load() + # The class should auto-register via __init_subclass__ + # But if it's an old-style class, manually register + if ep.name not in _codec_registry and hasattr(codec_class, "name"): + _codec_registry[ep.name] = codec_class() + logger.debug(f"Loaded codec <{ep.name}> from entry point {ep.value}") + except Exception as e: + logger.warning(f"Failed to load codec '{ep.name}' from {ep.value}: {e}") + + +def resolve_dtype( + dtype: str, seen: set[str] | None = None, store_name: str | None = None +) -> tuple[str, list[Codec], str | None]: + """ + Resolve a dtype string, following codec chains. + + If dtype references another codec (e.g., ``""``), recursively + resolves to find the ultimate storage type. Store parameters are propagated + through the chain. + + Parameters + ---------- + dtype : str + The dtype string to resolve (e.g., ``""``, ``""``, ``"bytes"``). + seen : set[str], optional + Set of already-seen codec names (for cycle detection). + store_name : str, optional + Store name from outer type specification (propagated inward). + + Returns + ------- + tuple[str, list[Codec], str | None] + ``(final_storage_type, codec_chain, resolved_store_name)``. + Chain is ordered from outermost to innermost codec. + + Raises + ------ + DataJointError + If a circular type reference is detected. + + Examples + -------- + >>> resolve_dtype("") + ("bytes", [BlobCodec], None) + >>> resolve_dtype("") + ("", [BlobCodec], "cold") + >>> resolve_dtype("bytes") + ("bytes", [], None) + """ + if seen is None: + seen = set() + + chain: list[Codec] = [] + + # Check if dtype is a codec reference + if dtype.startswith("<") and dtype.endswith(">"): + type_name, dtype_store = parse_type_spec(dtype) + + # Store from this level overrides inherited store + # Empty string means default store (@), None means no store specified + if dtype_store is not None: + effective_store = dtype_store + else: + effective_store = store_name + + if type_name in seen: + raise DataJointError(f"Circular codec reference detected: <{type_name}>") + + seen.add(type_name) + codec = get_codec(type_name) + chain.append(codec) + + # Determine if external based on whether @ is present + is_external = effective_store is not None + + # Get the inner dtype from the codec + inner_dtype = codec.get_dtype(is_external) + + # Recursively resolve the inner dtype, propagating store + final_dtype, inner_chain, resolved_store = resolve_dtype(inner_dtype, seen, effective_store) + chain.extend(inner_chain) + return final_dtype, chain, resolved_store + + # Not a codec - check if it has a store suffix (e.g., "blob@store") + if "@" in dtype: + base_type, dtype_store = dtype.split("@", 1) + effective_store = dtype_store if dtype_store else store_name + return base_type, chain, effective_store + + # Plain type - return as-is with propagated store + return dtype, chain, store_name + + +def lookup_codec(codec_spec: str) -> tuple[Codec, str | None]: + """ + Look up a codec from a type specification string. + + Parses a codec specification (e.g., ``""``) and returns + the codec instance along with any store name. + + Parameters + ---------- + codec_spec : str + The codec specification, with or without angle brackets. + May include store parameter (e.g., ``""``). + + Returns + ------- + tuple[Codec, str | None] + ``(codec_instance, store_name)`` or ``(codec_instance, None)``. + + Raises + ------ + DataJointError + If the codec is not found. + """ + type_name, store_name = parse_type_spec(codec_spec) + + if is_codec_registered(type_name): + return get_codec(type_name), store_name + + raise DataJointError(f"Codec <{type_name}> is not registered. Define a Codec subclass with name='{{type_name}}'.") + + +# ============================================================================= +# Decode Helper +# ============================================================================= + + +def decode_attribute(attr, data, squeeze: bool = False): + """ + Decode raw database value using attribute's codec or native type handling. + + This is the central decode function used by all fetch methods. It handles: + - Codec chains (e.g., → → bytes) + - Native type conversions (JSON, UUID) + - External storage downloads (via config["download_path"]) + + Args: + attr: Attribute from the table's heading. + data: Raw value fetched from the database. + squeeze: If True, remove singleton dimensions from numpy arrays. + + Returns: + Decoded Python value. + """ + import json + import uuid as uuid_module + + import numpy as np + + if data is None: + return None + + if attr.codec: + # Get store if present for external storage + store = getattr(attr, "store", None) + if store is not None: + dtype_spec = f"<{attr.codec.name}@{store}>" + else: + dtype_spec = f"<{attr.codec.name}>" + + final_dtype, type_chain, _ = resolve_dtype(dtype_spec) + + # Process the final storage type (what's in the database) + if final_dtype.lower() == "json": + data = json.loads(data) + elif final_dtype.lower() in ("longblob", "blob", "mediumblob", "tinyblob"): + pass # Blob data is already bytes + elif final_dtype.lower() == "binary(16)": + data = uuid_module.UUID(bytes=data) + + # Apply decoders in reverse order: innermost first, then outermost + for codec in reversed(type_chain): + data = codec.decode(data, key=None) + + # Squeeze arrays if requested + if squeeze and isinstance(data, np.ndarray): + data = data.squeeze() + + return data + + # No codec - handle native types + if attr.json: + return json.loads(data) + + if attr.uuid: + import uuid as uuid_module + + return uuid_module.UUID(bytes=data) + + if attr.is_blob: + return data # Raw bytes + + # Native types - pass through unchanged + return data + + +# ============================================================================= +# Auto-register built-in codecs +# ============================================================================= + +# Import builtin_codecs module to register built-in codecs +# This import has a side effect: it registers the codecs via __init_subclass__ +from . import builtin_codecs as _builtin_codecs # noqa: F401, E402 diff --git a/src/datajoint/condition.py b/src/datajoint/condition.py new file mode 100644 index 000000000..ea7d7a504 --- /dev/null +++ b/src/datajoint/condition.py @@ -0,0 +1,496 @@ +""" +SQL WHERE clause generation from DataJoint restriction conditions. + +This module provides utilities for converting various restriction formats +(dicts, strings, QueryExpressions) into SQL WHERE clauses. +""" + +from __future__ import annotations + +import collections +import datetime +import decimal +import inspect +import json +import logging +import re +import uuid +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any + +import numpy +import pandas + +from .errors import DataJointError + +if TYPE_CHECKING: + from .expression import QueryExpression + +logger = logging.getLogger(__name__.split(".")[0]) + +JSON_PATTERN = re.compile(r"^(?P\w+)(\.(?P[\w.*\[\]]+))?(:(?P[\w(,\s)]+))?$") + + +def translate_attribute(key: str) -> tuple[dict | None, str]: + """ + Translate an attribute key, handling JSON path notation. + + Parameters + ---------- + key : str + Attribute name, optionally with JSON path (e.g., ``"attr.path.field"``). + + Returns + ------- + tuple + (match_dict, sql_expression) where match_dict contains parsed + components or None if no JSON path. + """ + match = JSON_PATTERN.match(key) + if match is None: + return match, key + match = match.groupdict() + if match["path"] is None: + return match, match["attr"] + else: + return match, "json_value(`{}`, _utf8mb4'$.{}'{})".format( + *[((f" returning {v}" if k == "type" else v) if v else "") for k, v in match.items()] + ) + + +class PromiscuousOperand: + """ + Wrapper to bypass join compatibility checking. + + Used when you want to force a natural join without semantic matching. + + Parameters + ---------- + operand : QueryExpression + The operand to wrap. + """ + + def __init__(self, operand: QueryExpression) -> None: + self.operand = operand + + +class AndList(list): + """ + List of conditions combined with logical AND. + + All conditions in the list are AND-ed together. Other collections + (lists, sets, QueryExpressions) are OR-ed. + + Examples + -------- + >>> expr & dj.AndList((cond1, cond2, cond3)) + # equivalent to + >>> expr & cond1 & cond2 & cond3 + """ + + def append(self, restriction: Any) -> None: + if isinstance(restriction, AndList): + # extend to reduce nesting + self.extend(restriction) + else: + super().append(restriction) + + +@dataclass +class Top: + """ + Restrict query to top N entities with ordering. + + In SQL, corresponds to ``ORDER BY ... LIMIT ... OFFSET``. + + Parameters + ---------- + limit : int, optional + Maximum number of rows to return. Default 1. + order_by : str or list[str] or None, optional + Attributes to order by. ``"KEY"`` for primary key order. + ``None`` means inherit ordering from an existing Top (or default to KEY). + Default ``"KEY"``. + offset : int, optional + Number of rows to skip. Default 0. + + Examples + -------- + >>> query & dj.Top(5) # Top 5 by primary key + >>> query & dj.Top(10, 'score DESC') # Top 10 by score descending + >>> query & dj.Top(10, order_by=None) # Top 10, inherit existing order + >>> query & dj.Top(5, offset=10) # Skip 10, take 5 + """ + + limit: int | None = 1 + order_by: str | list[str] | None = "KEY" + offset: int = 0 + + def __post_init__(self) -> None: + self.offset = self.offset or 0 + + if self.limit is not None and not isinstance(self.limit, int): + raise TypeError("Top limit must be an integer") + if self.order_by is not None: + if not isinstance(self.order_by, (str, collections.abc.Sequence)) or not all( + isinstance(r, str) for r in self.order_by + ): + raise TypeError("Top order_by attributes must all be strings") + if isinstance(self.order_by, str): + self.order_by = [self.order_by] + if not isinstance(self.offset, int): + raise TypeError("The offset argument must be an integer") + if self.offset and self.limit is None: + self.limit = 999999999999 # arbitrary large number to allow query + + def merge(self, other: "Top") -> "Top": + """ + Merge another Top into this one (when other inherits ordering). + + Used when ``other.order_by`` is None or matches ``self.order_by``. + + Parameters + ---------- + other : Top + The Top to merge. Its order_by should be None or equal to self.order_by. + + Returns + ------- + Top + New Top with merged limit/offset and preserved ordering. + """ + # Compute effective limit (minimum of defined limits) + if self.limit is None and other.limit is None: + new_limit = None + elif self.limit is None: + new_limit = other.limit + elif other.limit is None: + new_limit = self.limit + else: + new_limit = min(self.limit, other.limit) + + return Top( + limit=new_limit, + order_by=self.order_by, # preserve existing ordering + offset=self.offset + other.offset, # offsets add + ) + + +class Not: + """ + Invert a restriction condition. + + Parameters + ---------- + restriction : any + Restriction condition to negate. + + Examples + -------- + >>> table - condition # equivalent to table & Not(condition) + """ + + def __init__(self, restriction: Any) -> None: + self.restriction = restriction + + +def assert_join_compatibility( + expr1: QueryExpression, + expr2: QueryExpression, + semantic_check: bool = True, +) -> None: + """ + Check if two expressions are join-compatible. + + Parameters + ---------- + expr1 : QueryExpression + First expression. + expr2 : QueryExpression + Second expression. + semantic_check : bool, optional + If True (default), use semantic matching and error on non-homologous + namesakes (same name, different lineage). If False, use natural join. + + Raises + ------ + DataJointError + If semantic_check is True and expressions have non-homologous namesakes. + + Notes + ----- + With semantic_check=True: + Prevents accidental joins on attributes that share names but represent + different entities. If ~lineage table doesn't exist, a warning is issued. + + With semantic_check=False: + All namesake attributes are matched (natural join behavior). + """ + from .expression import QueryExpression, U + + for rel in (expr1, expr2): + if not isinstance(rel, (U, QueryExpression)): + raise DataJointError("Object %r is not a QueryExpression and cannot be joined." % rel) + + # dj.U is always compatible (it represents all possible lineages) + if isinstance(expr1, U) or isinstance(expr2, U): + return + + if semantic_check: + # Check if lineage tracking is available for both expressions + if not expr1.heading.lineage_available or not expr2.heading.lineage_available: + logger.warning( + "Semantic check disabled: ~lineage table not found. " + "To enable semantic matching, rebuild lineage with: " + "schema.rebuild_lineage()" + ) + return + + # Error on non-homologous namesakes + namesakes = set(expr1.heading.names) & set(expr2.heading.names) + for name in namesakes: + lineage1 = expr1.heading[name].lineage + lineage2 = expr2.heading[name].lineage + # Semantic match requires both lineages to be non-None and equal + if lineage1 is None or lineage2 is None or lineage1 != lineage2: + raise DataJointError( + f"Cannot join on attribute `{name}`: " + f"different lineages ({lineage1} vs {lineage2}). " + f"Use .proj() to rename one of the attributes." + ) + + +def make_condition( + query_expression: QueryExpression, + condition: Any, + columns: set[str], + semantic_check: bool = True, +) -> str | bool: + """ + Translate a restriction into an SQL WHERE clause condition. + + Parameters + ---------- + query_expression : QueryExpression + The expression to apply the condition to. + condition : any + Valid restriction: str, dict, bool, QueryExpression, AndList, + numpy.void, pandas.DataFrame, or iterable of restrictions. + columns : set[str] + Set passed by reference to collect column names used in the condition. + semantic_check : bool, optional + If True (default), use semantic matching and error on conflicts. + + Returns + ------- + str or bool + SQL condition string, or bool if condition evaluates to constant. + + Notes + ----- + Restriction types are processed as follows: + + - ``str``: Used directly as SQL condition + - ``dict``: AND of equality conditions for matching attributes + - ``bool``: Returns the boolean value (possibly negated) + - ``QueryExpression``: Generates subquery (semijoin/antijoin) + - ``AndList``: AND of all conditions + - ``list/set/tuple``: OR of all conditions + - ``numpy.void``: Like dict, from record array + - ``pandas.DataFrame``: Converted to records, then OR-ed + """ + from .expression import Aggregation, QueryExpression, U + + def prep_value(k, v): + """prepare SQL condition""" + key_match, k = translate_attribute(k) + if key_match["path"] is None: + k = f"`{k}`" + if query_expression.heading[key_match["attr"]].json and key_match["path"] is not None and isinstance(v, dict): + return f"{k}='{json.dumps(v)}'" + if v is None: + return f"{k} IS NULL" + if query_expression.heading[key_match["attr"]].uuid: + if not isinstance(v, uuid.UUID): + try: + v = uuid.UUID(v) + except (AttributeError, ValueError): + raise DataJointError("Badly formed UUID {v} in restriction by `{k}`".format(k=k, v=v)) + return f"{k}=X'{v.bytes.hex()}'" + if isinstance( + v, + ( + datetime.date, + datetime.datetime, + datetime.time, + decimal.Decimal, + list, + ), + ): + return f'{k}="{v}"' + if isinstance(v, str): + v = v.replace("%", "%%").replace("\\", "\\\\") + return f'{k}="{v}"' + return f"{k}={v}" + + def combine_conditions(negate, conditions): + return f"{'NOT ' if negate else ''} ({')AND('.join(conditions)})" + + negate = False + while isinstance(condition, Not): + negate = not negate + condition = condition.restriction + + # restrict by string + if isinstance(condition, str): + columns.update(extract_column_names(condition)) + return combine_conditions(negate, conditions=[condition.strip().replace("%", "%%")]) # escape %, see issue #376 + + # restrict by AndList + if isinstance(condition, AndList): + # omit all conditions that evaluate to True + items = [ + item + for item in (make_condition(query_expression, cond, columns, semantic_check) for cond in condition) + if item is not True + ] + if any(item is False for item in items): + return negate # if any item is False, the whole thing is False + if not items: + return not negate # and empty AndList is True + return combine_conditions(negate, conditions=items) + + # restriction by dj.U evaluates to True + if isinstance(condition, U): + return not negate + + # restrict by boolean + if isinstance(condition, bool): + return negate != condition + + # restrict by a mapping/dict -- convert to an AndList of string equality conditions + if isinstance(condition, collections.abc.Mapping): + common_attributes = set(c.split(".", 1)[0] for c in condition).intersection(query_expression.heading.names) + if not common_attributes: + return not negate # no matching attributes -> evaluates to True + columns.update(common_attributes) + return combine_conditions( + negate, + conditions=[ + prep_value(k, v) + for k, v in condition.items() + if k.split(".", 1)[0] in common_attributes # handle json indexing + ], + ) + + # restrict by a numpy record -- convert to an AndList of string equality conditions + if isinstance(condition, numpy.void): + common_attributes = set(condition.dtype.fields).intersection(query_expression.heading.names) + if not common_attributes: + return not negate # no matching attributes -> evaluate to True + columns.update(common_attributes) + return combine_conditions( + negate, + conditions=[prep_value(k, condition[k]) for k in common_attributes], + ) + + # restrict by a QueryExpression subclass -- trigger instantiation and move on + if inspect.isclass(condition) and issubclass(condition, QueryExpression): + condition = condition() + + # restrict by another expression (aka semijoin and antijoin) + if isinstance(condition, QueryExpression): + assert_join_compatibility(query_expression, condition, semantic_check=semantic_check) + # Match on all non-hidden namesakes (hidden attributes excluded) + common_attributes = [q for q in condition.heading.names if q in query_expression.heading.names] + columns.update(common_attributes) + if isinstance(condition, Aggregation): + condition = condition.make_subquery() + return ( + # without common attributes, any non-empty set matches everything + (not negate if condition else negate) + if not common_attributes + else "({fields}) {not_}in ({subquery})".format( + fields="`" + "`,`".join(common_attributes) + "`", + not_="not " if negate else "", + subquery=condition.make_sql(common_attributes), + ) + ) + + # restrict by pandas.DataFrames + if isinstance(condition, pandas.DataFrame): + condition = condition.to_records() # convert to numpy.recarray and move on + + # if iterable (but not a string, a QueryExpression, or an AndList), treat as an OrList + try: + or_list = [make_condition(query_expression, q, columns, semantic_check) for q in condition] + except TypeError: + raise DataJointError("Invalid restriction type %r" % condition) + else: + or_list = [item for item in or_list if item is not False] # ignore False conditions + if any(item is True for item in or_list): # if any item is True, entirely True + return not negate + return f"{'NOT ' if negate else ''} ({' OR '.join(or_list)})" if or_list else negate + + +def extract_column_names(sql_expression: str) -> set[str]: + r""" + Extract column names from an SQL expression. + + Parameters + ---------- + sql_expression : str + SQL expression (e.g., WHERE clause) to parse. + + Returns + ------- + set[str] + Set of extracted column names. + + Notes + ----- + Parsing is MySQL-specific. Identifies columns by: + + 1. Names in backticks (``\`column\```) + 2. Bare identifiers not followed by ``(`` (excludes functions) + 3. Excludes SQL reserved words (IS, IN, AND, OR, etc.) + """ + assert isinstance(sql_expression, str) + result = set() + s = sql_expression # for terseness + # remove escaped quotes + s = re.sub(r"(\\\")|(\\\')", "", s) + # remove quoted text + s = re.sub(r"'[^']*'", "", s) + s = re.sub(r'"[^"]*"', "", s) + # find all tokens in back quotes and remove them + result.update(re.findall(r"`([a-z][a-z_0-9]*)`", s)) + s = re.sub(r"`[a-z][a-z_0-9]*`", "", s) + # remove space before parentheses + s = re.sub(r"\s*\(", "(", s) + # remove tokens followed by ( since they must be functions + s = re.sub(r"(\b[a-z][a-z_0-9]*)\(", "(", s) + remaining_tokens = set(re.findall(r"\b[a-z][a-z_0-9]*\b", s)) + # update result removing reserved words + result.update( + remaining_tokens + - { + "is", + "in", + "between", + "like", + "and", + "or", + "null", + "not", + "interval", + "second", + "minute", + "hour", + "day", + "month", + "week", + "year", + } + ) + return result diff --git a/src/datajoint/connection.py b/src/datajoint/connection.py new file mode 100644 index 000000000..57301c2f3 --- /dev/null +++ b/src/datajoint/connection.py @@ -0,0 +1,536 @@ +""" +This module contains the Connection class that manages the connection to the database, and +the ``conn`` function that provides access to a persistent connection in datajoint. +""" + +from __future__ import annotations + +import logging +import pathlib +import re +import warnings +from contextlib import contextmanager +from getpass import getpass +from typing import Callable + +import pymysql as client + +from . import errors +from .blob import pack, unpack +from .dependencies import Dependencies +from .hash import uuid_from_buffer +from .settings import config +from .version import __version__ + +logger = logging.getLogger(__name__.split(".")[0]) +query_log_max_length = 300 + + +cache_key = "query_cache" # the key to lookup the query_cache folder in dj.config + + +def translate_query_error(client_error: Exception, query: str) -> Exception: + """ + Translate client error to the corresponding DataJoint exception. + + Parameters + ---------- + client_error : Exception + The exception raised by the client interface. + query : str + SQL query with placeholders. + + Returns + ------- + Exception + An instance of the corresponding DataJoint error subclass, + or the original error if no mapping exists. + """ + logger.debug("type: {}, args: {}".format(type(client_error), client_error.args)) + + err, *args = client_error.args + + match err: + # Loss of connection errors + case 0 | "(0, '')": + return errors.LostConnectionError("Server connection lost due to an interface error.", *args) + case 2006: + return errors.LostConnectionError("Connection timed out", *args) + case 2013: + return errors.LostConnectionError("Server connection lost", *args) + + # Access errors + case 1044 | 1142: + return errors.AccessError("Insufficient privileges.", args[0], query) + + # Integrity errors + case 1062: + return errors.DuplicateError(*args) + case 1217 | 1451 | 1452 | 3730: + # 1217: Cannot delete parent row (FK constraint) + # 1451: Cannot delete/update parent row (FK constraint) + # 1452: Cannot add/update child row (FK constraint) + # 3730: Cannot drop table referenced by FK constraint + return errors.IntegrityError(*args) + + # Syntax errors + case 1064: + return errors.QuerySyntaxError(args[0], query) + + # Existence errors + case 1146: + return errors.MissingTableError(args[0], query) + case 1364: + return errors.MissingAttributeError(*args) + case 1054: + return errors.UnknownAttributeError(*args) + + # All other errors pass through unchanged + case _: + return client_error + + +def conn( + host: str | None = None, + user: str | None = None, + password: str | None = None, + *, + init_fun: Callable | None = None, + reset: bool = False, + use_tls: bool | dict | None = None, +) -> Connection: + """ + Return a persistent connection object shared by multiple modules. + + If the connection is not yet established or reset=True, a new connection is set up. + If connection information is not provided, it is taken from config. + + Parameters + ---------- + host : str, optional + Database hostname. + user : str, optional + MySQL username. + password : str, optional + MySQL password. Prompts if not provided. + init_fun : callable, optional + Initialization function called after connection. + reset : bool, optional + If True, reset existing connection. Default False. + use_tls : bool or dict, optional + TLS encryption option: True (required), False (no TLS), + None (preferred, default), or dict for manual configuration. + + Returns + ------- + Connection + Persistent database connection. + """ + if not hasattr(conn, "connection") or reset: + host = host if host is not None else config["database.host"] + user = user if user is not None else config["database.user"] + password = password if password is not None else config["database.password"] + if user is None: + user = input("Please enter DataJoint username: ") + if password is None: + password = getpass(prompt="Please enter DataJoint password: ") + init_fun = init_fun if init_fun is not None else config["connection.init_function"] + use_tls = use_tls if use_tls is not None else config["database.use_tls"] + conn.connection = Connection(host, user, password, None, init_fun, use_tls) + return conn.connection + + +class EmulatedCursor: + """acts like a cursor""" + + def __init__(self, data): + self._data = data + self._iter = iter(self._data) + + def __iter__(self): + return self + + def __next__(self): + return next(self._iter) + + def fetchall(self): + return self._data + + def fetchone(self): + return next(self._iter) + + @property + def rowcount(self): + return len(self._data) + + +class Connection: + """ + Manages a connection to a database server. + + Catalogues schemas, tables, and their dependencies (foreign keys). + Most parameters should be set in the configuration file. + + Parameters + ---------- + host : str + Hostname, may include port as ``hostname:port``. + user : str + Database username. + password : str + Database password. + port : int, optional + Port number. Overridden if specified in host. + init_fun : str, optional + SQL initialization command. + use_tls : bool or dict, optional + TLS encryption option. + + Attributes + ---------- + schemas : dict + Registered schema objects. + dependencies : Dependencies + Foreign key dependency graph. + """ + + def __init__( + self, + host: str, + user: str, + password: str, + port: int | None = None, + init_fun: str | None = None, + use_tls: bool | dict | None = None, + ) -> None: + if ":" in host: + # the port in the hostname overrides the port argument + host, port = host.split(":") + port = int(port) + elif port is None: + port = config["database.port"] + self.conn_info = dict(host=host, port=port, user=user, passwd=password) + if use_tls is not False: + self.conn_info["ssl"] = use_tls if isinstance(use_tls, dict) else {"ssl": {}} + self.conn_info["ssl_input"] = use_tls + self.init_fun = init_fun + self._conn = None + self._query_cache = None + self.connect() + if self.is_connected: + logger.info("DataJoint {version} connected to {user}@{host}:{port}".format(version=__version__, **self.conn_info)) + self.connection_id = self.query("SELECT connection_id()").fetchone()[0] + else: + raise errors.LostConnectionError("Connection failed {user}@{host}:{port}".format(**self.conn_info)) + self._in_transaction = False + self.schemas = dict() + self.dependencies = Dependencies(self) + + def __eq__(self, other): + return self.conn_info == other.conn_info + + def __repr__(self): + connected = "connected" if self.is_connected else "disconnected" + return "DataJoint connection ({connected}) {user}@{host}:{port}".format(connected=connected, **self.conn_info) + + def connect(self) -> None: + """Establish or re-establish connection to the database server.""" + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", ".*deprecated.*") + try: + self._conn = client.connect( + init_command=self.init_fun, + sql_mode="NO_ZERO_DATE,NO_ZERO_IN_DATE,ERROR_FOR_DIVISION_BY_ZERO," + "STRICT_ALL_TABLES,NO_ENGINE_SUBSTITUTION,ONLY_FULL_GROUP_BY", + charset=config["connection.charset"], + **{k: v for k, v in self.conn_info.items() if k not in ["ssl_input"]}, + ) + except client.err.InternalError: + self._conn = client.connect( + init_command=self.init_fun, + sql_mode="NO_ZERO_DATE,NO_ZERO_IN_DATE,ERROR_FOR_DIVISION_BY_ZERO," + "STRICT_ALL_TABLES,NO_ENGINE_SUBSTITUTION,ONLY_FULL_GROUP_BY", + charset=config["connection.charset"], + **{ + k: v + for k, v in self.conn_info.items() + if not (k == "ssl_input" or k == "ssl" and self.conn_info["ssl_input"] is None) + }, + ) + self._conn.autocommit(True) + + def set_query_cache(self, query_cache: str | None = None) -> None: + """ + Enable query caching mode. + + When enabled: + 1. Only SELECT queries are allowed + 2. Results are cached under ``dj.config['query_cache']`` + 3. Cache key differentiates cache states + + Parameters + ---------- + query_cache : str, optional + String to initialize the hash for query results. + None disables caching. + """ + self._query_cache = query_cache + + def purge_query_cache(self) -> None: + """Delete all cached query results.""" + if isinstance(config.get(cache_key), str) and pathlib.Path(config[cache_key]).is_dir(): + for path in pathlib.Path(config[cache_key]).iterdir(): + if not path.is_dir(): + path.unlink() + + def close(self) -> None: + """Close the database connection.""" + self._conn.close() + + def __enter__(self) -> "Connection": + """ + Enter context manager. + + Returns + ------- + Connection + This connection object. + + Examples + -------- + >>> with dj.Connection(host, user, password) as conn: + ... schema = dj.schema('my_schema', connection=conn) + ... # perform operations + ... # connection automatically closed + """ + return self + + def __exit__(self, exc_type, exc_val, exc_tb) -> bool: + """ + Exit context manager and close connection. + + Parameters + ---------- + exc_type : type or None + Exception type if an exception was raised. + exc_val : Exception or None + Exception instance if an exception was raised. + exc_tb : traceback or None + Traceback if an exception was raised. + + Returns + ------- + bool + False to propagate exceptions. + """ + self.close() + return False + + def register(self, schema) -> None: + """ + Register a schema with this connection. + + Parameters + ---------- + schema : Schema + Schema object to register. + """ + self.schemas[schema.database] = schema + self.dependencies.clear() + + def ping(self) -> None: + """ + Ping the server to verify connection is alive. + + Raises + ------ + Exception + If the connection is closed. + """ + self._conn.ping(reconnect=False) + + @property + def is_connected(self) -> bool: + """ + Check if connected to the database server. + + Returns + ------- + bool + True if connected. + """ + try: + self.ping() + except: + return False + return True + + @staticmethod + def _execute_query(cursor, query, args, suppress_warnings): + try: + with warnings.catch_warnings(): + if suppress_warnings: + # suppress all warnings arising from underlying SQL library + warnings.simplefilter("ignore") + cursor.execute(query, args) + except client.err.Error as err: + raise translate_query_error(err, query) + + def query( + self, + query: str, + args: tuple = (), + *, + as_dict: bool = False, + suppress_warnings: bool = True, + reconnect: bool | None = None, + ): + """ + Execute a SQL query and return the cursor. + + Parameters + ---------- + query : str + SQL query to execute. + args : tuple, optional + Query parameters for prepared statement. + as_dict : bool, optional + If True, return rows as dictionaries. Default False. + suppress_warnings : bool, optional + If True, suppress SQL library warnings. Default True. + reconnect : bool, optional + If True, reconnect if disconnected. None uses config setting. + + Returns + ------- + cursor + Database cursor with query results. + + Raises + ------ + DataJointError + If non-SELECT query during query caching mode. + """ + # check cache first: + use_query_cache = bool(self._query_cache) + if use_query_cache and not re.match(r"\s*(SELECT|SHOW)", query): + raise errors.DataJointError("Only SELECT queries are allowed when query caching is on.") + if use_query_cache: + if not config[cache_key]: + raise errors.DataJointError(f"Provide filepath dj.config['{cache_key}'] when using query caching.") + hash_ = uuid_from_buffer((str(self._query_cache) + re.sub(r"`\$\w+`", "", query)).encode() + pack(args)) + cache_path = pathlib.Path(config[cache_key]) / str(hash_) + try: + buffer = cache_path.read_bytes() + except FileNotFoundError: + pass # proceed to query the database + else: + return EmulatedCursor(unpack(buffer)) + + if reconnect is None: + reconnect = config["database.reconnect"] + logger.debug("Executing SQL:" + query[:query_log_max_length]) + cursor_class = client.cursors.DictCursor if as_dict else client.cursors.Cursor + cursor = self._conn.cursor(cursor=cursor_class) + try: + self._execute_query(cursor, query, args, suppress_warnings) + except errors.LostConnectionError: + if not reconnect: + raise + logger.warning("Reconnecting to MySQL server.") + self.connect() + if self._in_transaction: + self.cancel_transaction() + raise errors.LostConnectionError("Connection was lost during a transaction.") + logger.debug("Re-executing") + cursor = self._conn.cursor(cursor=cursor_class) + self._execute_query(cursor, query, args, suppress_warnings) + + if use_query_cache: + data = cursor.fetchall() + cache_path.write_bytes(pack(data)) + return EmulatedCursor(data) + + return cursor + + def get_user(self) -> str: + """ + Get the current user and host. + + Returns + ------- + str + User name and host as ``'user@host'``. + """ + return self.query("SELECT user()").fetchone()[0] + + # ---------- transaction processing + @property + def in_transaction(self) -> bool: + """ + Check if a transaction is open. + + Returns + ------- + bool + True if a transaction is in progress. + """ + self._in_transaction = self._in_transaction and self.is_connected + return self._in_transaction + + def start_transaction(self) -> None: + """ + Start a new transaction. + + Raises + ------ + DataJointError + If a transaction is already in progress. + """ + if self.in_transaction: + raise errors.DataJointError("Nested connections are not supported.") + self.query("START TRANSACTION WITH CONSISTENT SNAPSHOT") + self._in_transaction = True + logger.debug("Transaction started") + + def cancel_transaction(self) -> None: + """Cancel the current transaction and roll back all changes.""" + self.query("ROLLBACK") + self._in_transaction = False + logger.debug("Transaction cancelled. Rolling back ...") + + def commit_transaction(self) -> None: + """Commit all changes and close the transaction.""" + self.query("COMMIT") + self._in_transaction = False + logger.debug("Transaction committed and closed.") + + # -------- context manager for transactions + @property + @contextmanager + def transaction(self): + """ + Context manager for transactions. + + Opens a transaction and automatically commits on success or rolls back + on exception. + + Yields + ------ + Connection + This connection object. + + Examples + -------- + >>> with dj.conn().transaction: + ... # All operations here are in one transaction + ... table.insert(data) + """ + try: + self.start_transaction() + yield self + except: + self.cancel_transaction() + raise + else: + self.commit_transaction() diff --git a/src/datajoint/content_registry.py b/src/datajoint/content_registry.py new file mode 100644 index 000000000..70b38324a --- /dev/null +++ b/src/datajoint/content_registry.py @@ -0,0 +1,231 @@ +""" +Content-addressed storage registry for DataJoint. + +This module provides content-addressed storage with deduplication for the +Codec. Content is identified by its MD5 hash and stored in a hierarchical +directory structure: _hash/{hash[:2]}/{hash[2:4]}/{hash} + +The ContentRegistry tracks stored content for garbage collection purposes. +""" + +import hashlib +import logging +from typing import Any + +from .errors import DataJointError +from .settings import config +from .storage import StorageBackend + +logger = logging.getLogger(__name__.split(".")[0]) + + +def compute_content_hash(data: bytes) -> str: + """ + Compute SHA256 hash of content. + + Parameters + ---------- + data : bytes + Content bytes. + + Returns + ------- + str + Hex-encoded SHA256 hash (64 characters). + """ + return hashlib.sha256(data).hexdigest() + + +def build_content_path(content_hash: str) -> str: + """ + Build the storage path for content-addressed storage. + + Content is stored in a hierarchical structure to avoid too many files + in a single directory: _content/{hash[:2]}/{hash[2:4]}/{hash} + + Parameters + ---------- + content_hash : str + SHA256 hex hash (64 characters). + + Returns + ------- + str + Relative path within the store. + """ + if len(content_hash) != 64: + raise DataJointError(f"Invalid content hash length: {len(content_hash)} (expected 64)") + return f"_content/{content_hash[:2]}/{content_hash[2:4]}/{content_hash}" + + +def get_store_backend(store_name: str | None = None) -> StorageBackend: + """ + Get a StorageBackend for content storage. + + Parameters + ---------- + store_name : str, optional + Name of the store to use. If None, uses the default object storage + configuration or the configured default_store. + + Returns + ------- + StorageBackend + StorageBackend instance. + """ + # If store_name is None, check for configured default_store + if store_name is None and config.object_storage.default_store: + store_name = config.object_storage.default_store + + # get_object_store_spec handles None by returning default object_storage config + spec = config.get_object_store_spec(store_name) + return StorageBackend(spec) + + +def put_content(data: bytes, store_name: str | None = None) -> dict[str, Any]: + """ + Store content using content-addressed storage. + + If the content already exists (same hash), it is not re-uploaded. + Returns metadata including the hash, store, and size. + + Parameters + ---------- + data : bytes + Content bytes to store. + store_name : str, optional + Name of the store. If None, uses default store. + + Returns + ------- + dict[str, Any] + Metadata dict with keys: hash, store, size. + """ + content_hash = compute_content_hash(data) + path = build_content_path(content_hash) + + backend = get_store_backend(store_name) + + # Check if content already exists (deduplication) + if not backend.exists(path): + backend.put_buffer(data, path) + logger.debug(f"Stored new content: {content_hash[:16]}... ({len(data)} bytes)") + else: + logger.debug(f"Content already exists: {content_hash[:16]}...") + + return { + "hash": content_hash, + "store": store_name, + "size": len(data), + } + + +def get_content(content_hash: str, store_name: str | None = None) -> bytes: + """ + Retrieve content by its hash. + + Parameters + ---------- + content_hash : str + SHA256 hex hash of the content. + store_name : str, optional + Name of the store. If None, uses default store. + + Returns + ------- + bytes + Content bytes. + + Raises + ------ + MissingExternalFile + If content is not found. + DataJointError + If hash verification fails. + """ + path = build_content_path(content_hash) + backend = get_store_backend(store_name) + + data = backend.get_buffer(path) + + # Verify hash (optional but recommended for integrity) + actual_hash = compute_content_hash(data) + if actual_hash != content_hash: + raise DataJointError(f"Content hash mismatch: expected {content_hash[:16]}..., got {actual_hash[:16]}...") + + return data + + +def content_exists(content_hash: str, store_name: str | None = None) -> bool: + """ + Check if content exists in storage. + + Parameters + ---------- + content_hash : str + SHA256 hex hash of the content. + store_name : str, optional + Name of the store. If None, uses default store. + + Returns + ------- + bool + True if content exists. + """ + path = build_content_path(content_hash) + backend = get_store_backend(store_name) + return backend.exists(path) + + +def delete_content(content_hash: str, store_name: str | None = None) -> bool: + """ + Delete content from storage. + + This should only be called after verifying no references exist. + Use garbage collection to safely remove unreferenced content. + + Parameters + ---------- + content_hash : str + SHA256 hex hash of the content. + store_name : str, optional + Name of the store. If None, uses default store. + + Returns + ------- + bool + True if content was deleted, False if it didn't exist. + + Warnings + -------- + This permanently deletes content. Ensure no references exist first. + """ + path = build_content_path(content_hash) + backend = get_store_backend(store_name) + + if backend.exists(path): + backend.remove(path) + logger.debug(f"Deleted content: {content_hash[:16]}...") + return True + return False + + +def get_content_size(content_hash: str, store_name: str | None = None) -> int: + """ + Get the size of stored content. + + Parameters + ---------- + content_hash : str + SHA256 hex hash of the content. + store_name : str, optional + Name of the store. If None, uses default store. + + Returns + ------- + int + Size in bytes. + """ + path = build_content_path(content_hash) + backend = get_store_backend(store_name) + return backend.size(path) diff --git a/src/datajoint/declare.py b/src/datajoint/declare.py new file mode 100644 index 000000000..d86e90ed9 --- /dev/null +++ b/src/datajoint/declare.py @@ -0,0 +1,758 @@ +""" +Table definition parsing and SQL generation. + +This module converts DataJoint table definitions into MySQL CREATE TABLE +statements, handling type mapping, foreign key resolution, and index creation. +""" + +from __future__ import annotations + +import logging +import re + +import pyparsing as pp + +from .codecs import lookup_codec +from .condition import translate_attribute +from .errors import DataJointError +from .settings import config + +# Core DataJoint types - scientist-friendly names that are fully supported +# These are recorded in field comments using :type: syntax for reconstruction +# Format: pattern_name -> (regex_pattern, mysql_type or None if same as matched) +CORE_TYPES = { + # Numeric types (aliased to native SQL) + "float32": (r"float32$", "float"), + "float64": (r"float64$", "double"), + "int64": (r"int64$", "bigint"), + "uint64": (r"uint64$", "bigint unsigned"), + "int32": (r"int32$", "int"), + "uint32": (r"uint32$", "int unsigned"), + "int16": (r"int16$", "smallint"), + "uint16": (r"uint16$", "smallint unsigned"), + "int8": (r"int8$", "tinyint"), + "uint8": (r"uint8$", "tinyint unsigned"), + "bool": (r"bool$", "tinyint"), + # UUID (stored as binary) + "uuid": (r"uuid$", "binary(16)"), + # JSON + "json": (r"json$", None), # json passes through as-is + # Binary (bytes maps to longblob in MySQL, bytea in PostgreSQL) + "bytes": (r"bytes$", "longblob"), + # Temporal + "date": (r"date$", None), + "datetime": (r"datetime(\s*\(\d+\))?$", None), # datetime with optional fractional seconds precision + # String types (with parameters) + "char": (r"char\s*\(\d+\)$", None), + "varchar": (r"varchar\s*\(\d+\)$", None), + # Enumeration + "enum": (r"enum\s*\(.+\)$", None), + # Fixed-point decimal + "decimal": (r"decimal\s*\(\d+\s*,\s*\d+\)$", None), +} + +# Compile core type patterns +CORE_TYPE_PATTERNS = {name: re.compile(pattern, re.I) for name, (pattern, _) in CORE_TYPES.items()} + +# Get SQL mapping for core types +CORE_TYPE_SQL = {name: sql_type for name, (_, sql_type) in CORE_TYPES.items()} + +MAX_TABLE_NAME_LENGTH = 64 +CONSTANT_LITERALS = { + "CURRENT_TIMESTAMP", + "NULL", +} # SQL literals to be used without quotes (case insensitive) + +# Type patterns for declaration parsing +TYPE_PATTERN = { + k: re.compile(v, re.I) + for k, v in dict( + # Core DataJoint types + **{name.upper(): pattern for name, (pattern, _) in CORE_TYPES.items()}, + # Native SQL types (passthrough with warning for non-standard use) + INTEGER=r"((tiny|small|medium|big|)int|integer)(\s*\(.+\))?(\s+unsigned)?(\s+auto_increment)?|serial$", + NUMERIC=r"numeric(\s*\(.+\))?(\s+unsigned)?$", # numeric is SQL alias, use decimal instead + FLOAT=r"(double|float|real)(\s*\(.+\))?(\s+unsigned)?$", + STRING=r"(var)?char\s*\(.+\)$", # Catches char/varchar not matched by core types + TEMPORAL=r"(time|timestamp|year)(\s*\(.+\))?$", # time, timestamp, year (not date/datetime) + NATIVE_BLOB=r"(tiny|small|medium|long)blob$", # Specific blob variants + NATIVE_TEXT=r"(tiny|small|medium|long)?text$", # Native text types (not portable) + # Codecs use angle brackets + CODEC=r"<.+>$", + ).items() +} + +# Core types are stored in attribute comment for reconstruction +CORE_TYPE_NAMES = {name.upper() for name in CORE_TYPES} + +# Special types that need comment storage (core types + adapted) +SPECIAL_TYPES = CORE_TYPE_NAMES | {"CODEC"} + +# Native SQL types that pass through (with optional warning) +NATIVE_TYPES = set(TYPE_PATTERN) - SPECIAL_TYPES + +assert SPECIAL_TYPES <= set(TYPE_PATTERN) + + +def match_type(attribute_type: str) -> str: + """ + Match an attribute type string to its category. + + Parameters + ---------- + attribute_type : str + The type string from the table definition (e.g., ``"float32"``, ``"varchar(255)"``). + + Returns + ------- + str + Category name from TYPE_PATTERN (e.g., ``"FLOAT32"``, ``"STRING"``, ``"CODEC"``). + + Raises + ------ + DataJointError + If the type string doesn't match any known pattern. + """ + try: + return next(category for category, pattern in TYPE_PATTERN.items() if pattern.match(attribute_type)) + except StopIteration: + raise DataJointError("Unsupported attribute type {type}".format(type=attribute_type)) + + +logger = logging.getLogger(__name__.split(".")[0]) + + +def build_foreign_key_parser() -> pp.ParserElement: + """ + Build a pyparsing parser for foreign key definitions. + + Returns + ------- + pp.ParserElement + Parser that extracts ``options`` and ``ref_table`` from lines like + ``-> [nullable] ParentTable``. + """ + arrow = pp.Literal("->").suppress() + lbracket = pp.Literal("[").suppress() + rbracket = pp.Literal("]").suppress() + option = pp.Word(pp.srange("[a-zA-Z]")) + options = pp.Optional(lbracket + pp.DelimitedList(option) + rbracket).set_results_name("options") + ref_table = pp.restOfLine.set_results_name("ref_table") + return arrow + options + ref_table + + +def build_attribute_parser() -> pp.ParserElement: + """ + Build a pyparsing parser for attribute definitions. + + Returns + ------- + pp.ParserElement + Parser that extracts ``name``, ``type``, ``default``, and ``comment`` + from attribute definition lines. + """ + quoted = pp.QuotedString('"') ^ pp.QuotedString("'") + colon = pp.Literal(":").suppress() + attribute_name = pp.Word(pp.srange("[a-z]"), pp.srange("[a-z0-9_]")).set_results_name("name") + data_type = ( + pp.Combine(pp.Word(pp.alphas) + pp.SkipTo("#", ignore=quoted)) + ^ pp.QuotedString("<", end_quote_char=">", unquote_results=False) + ).set_results_name("type") + default = pp.Literal("=").suppress() + pp.SkipTo(colon, ignore=quoted).set_results_name("default") + comment = pp.Literal("#").suppress() + pp.restOfLine.set_results_name("comment") + return attribute_name + pp.Optional(default) + colon + data_type + comment + + +foreign_key_parser = build_foreign_key_parser() +attribute_parser = build_attribute_parser() + + +def is_foreign_key(line: str) -> bool: + """ + Check if a definition line is a foreign key reference. + + Parameters + ---------- + line : str + A line from the table definition. + + Returns + ------- + bool + True if the line appears to be a foreign key definition (contains ``->`` + not inside quotes or comments). + """ + arrow_position = line.find("->") + return arrow_position >= 0 and not any(c in line[:arrow_position] for c in "\"#'") + + +def compile_foreign_key( + line: str, + context: dict, + attributes: list[str], + primary_key: list[str] | None, + attr_sql: list[str], + foreign_key_sql: list[str], + index_sql: list[str], + fk_attribute_map: dict[str, tuple[str, str]] | None = None, +) -> None: + """ + Parse a foreign key line and update declaration components. + + Parameters + ---------- + line : str + A foreign key line from the table definition (e.g., ``"-> Parent"``). + context : dict + Namespace containing referenced table objects. + attributes : list[str] + Attribute names already declared. Updated in place with new FK attributes. + primary_key : list[str] or None + Primary key attributes so far. None if in dependent section. + Updated in place with FK attributes when not None. + attr_sql : list[str] + SQL attribute definitions. Updated in place. + foreign_key_sql : list[str] + SQL FOREIGN KEY constraints. Updated in place. + index_sql : list[str] + SQL INDEX declarations. Updated in place. + fk_attribute_map : dict, optional + Mapping of ``child_attr -> (parent_table, parent_attr)``. Updated in place. + + Raises + ------ + DataJointError + If the foreign key reference cannot be resolved or options are invalid. + """ + # Parse and validate + from .expression import QueryExpression + from .table import Table + + try: + result = foreign_key_parser.parse_string(line) + except pp.ParseException as err: + raise DataJointError('Parsing error in line "%s". %s.' % (line, err)) + + try: + ref = eval(result.ref_table, context) + except Exception: + raise DataJointError("Foreign key reference %s could not be resolved" % result.ref_table) + + options = [opt.upper() for opt in result.options] + for opt in options: # check for invalid options + if opt not in {"NULLABLE", "UNIQUE"}: + raise DataJointError('Invalid foreign key option "{opt}"'.format(opt=opt)) + is_nullable = "NULLABLE" in options + is_unique = "UNIQUE" in options + if is_nullable and primary_key is not None: + raise DataJointError('Primary dependencies cannot be nullable in line "{line}"'.format(line=line)) + + if isinstance(ref, type) and issubclass(ref, Table): + ref = ref() + + # check that dependency is of a supported type + if ( + not isinstance(ref, QueryExpression) + or len(ref.restriction) + or len(ref.support) != 1 + or not isinstance(ref.support[0], str) + ): + raise DataJointError('Dependency "%s" is not supported (yet). Use a base table or its projection.' % result.ref_table) + + # declare new foreign key attributes + for attr in ref.primary_key: + if attr not in attributes: + attributes.append(attr) + if primary_key is not None: + primary_key.append(attr) + attr_sql.append(ref.heading[attr].sql.replace("NOT NULL ", "", int(is_nullable))) + # Track FK attribute mapping for lineage: child_attr -> (parent_table, parent_attr) + if fk_attribute_map is not None: + parent_table = ref.support[0] # e.g., `schema`.`table` + parent_attr = ref.heading[attr].original_name + fk_attribute_map[attr] = (parent_table, parent_attr) + + # declare the foreign key + foreign_key_sql.append( + "FOREIGN KEY (`{fk}`) REFERENCES {ref} (`{pk}`) ON UPDATE CASCADE ON DELETE RESTRICT".format( + fk="`,`".join(ref.primary_key), + pk="`,`".join(ref.heading[name].original_name for name in ref.primary_key), + ref=ref.support[0], + ) + ) + + # declare unique index + if is_unique: + index_sql.append("UNIQUE INDEX ({attrs})".format(attrs=",".join("`%s`" % attr for attr in ref.primary_key))) + + +def prepare_declare( + definition: str, context: dict +) -> tuple[str, list[str], list[str], list[str], list[str], list[str], dict[str, tuple[str, str]]]: + """ + Parse a table definition into its components. + + Parameters + ---------- + definition : str + DataJoint table definition string. + context : dict + Namespace for resolving foreign key references. + + Returns + ------- + tuple + Seven-element tuple containing: + + - table_comment : str + - primary_key : list[str] + - attribute_sql : list[str] + - foreign_key_sql : list[str] + - index_sql : list[str] + - external_stores : list[str] + - fk_attribute_map : dict[str, tuple[str, str]] + """ + # split definition into lines + definition = re.split(r"\s*\n\s*", definition.strip()) + # check for optional table comment + table_comment = definition.pop(0)[1:].strip() if definition[0].startswith("#") else "" + if table_comment.startswith(":"): + raise DataJointError('Table comment must not start with a colon ":"') + in_key = True # parse primary keys + primary_key = [] + attributes = [] + attribute_sql = [] + foreign_key_sql = [] + index_sql = [] + external_stores = [] + fk_attribute_map = {} # child_attr -> (parent_table, parent_attr) + + for line in definition: + if not line or line.startswith("#"): # ignore additional comments + pass + elif line.startswith("---") or line.startswith("___"): + in_key = False # start parsing dependent attributes + elif is_foreign_key(line): + compile_foreign_key( + line, + context, + attributes, + primary_key if in_key else None, + attribute_sql, + foreign_key_sql, + index_sql, + fk_attribute_map, + ) + elif re.match(r"^(unique\s+)?index\s*.*$", line, re.I): # index + compile_index(line, index_sql) + else: + name, sql, store = compile_attribute(line, in_key, foreign_key_sql, context) + if store: + external_stores.append(store) + if in_key and name not in primary_key: + primary_key.append(name) + if name not in attributes: + attributes.append(name) + attribute_sql.append(sql) + + return ( + table_comment, + primary_key, + attribute_sql, + foreign_key_sql, + index_sql, + external_stores, + fk_attribute_map, + ) + + +def declare( + full_table_name: str, definition: str, context: dict +) -> tuple[str, list[str], list[str], dict[str, tuple[str, str]]]: + r""" + Parse a definition and generate SQL CREATE TABLE statement. + + Parameters + ---------- + full_table_name : str + Fully qualified table name (e.g., ```\`schema\`.\`table\```). + definition : str + DataJoint table definition string. + context : dict + Namespace for resolving foreign key references. + + Returns + ------- + tuple + Four-element tuple: + + - sql : str - SQL CREATE TABLE statement + - external_stores : list[str] - External store names used + - primary_key : list[str] - Primary key attribute names + - fk_attribute_map : dict - FK attribute lineage mapping + + Raises + ------ + DataJointError + If table name exceeds max length or has no primary key. + """ + table_name = full_table_name.strip("`").split(".")[1] + if len(table_name) > MAX_TABLE_NAME_LENGTH: + raise DataJointError( + "Table name `{name}` exceeds the max length of {max_length}".format( + name=table_name, max_length=MAX_TABLE_NAME_LENGTH + ) + ) + + ( + table_comment, + primary_key, + attribute_sql, + foreign_key_sql, + index_sql, + external_stores, + fk_attribute_map, + ) = prepare_declare(definition, context) + + # Add hidden job metadata for Computed/Imported tables (not parts) + # Note: table_name may still have backticks, strip them for prefix checking + clean_table_name = table_name.strip("`") + if config.jobs.add_job_metadata: + # Check if this is a Computed (__) or Imported (_) table, but not a Part (contains __ in middle) + is_computed = clean_table_name.startswith("__") and "__" not in clean_table_name[2:] + is_imported = clean_table_name.startswith("_") and not clean_table_name.startswith("__") + if is_computed or is_imported: + job_metadata_sql = [ + "`_job_start_time` datetime(3) DEFAULT NULL", + "`_job_duration` float DEFAULT NULL", + "`_job_version` varchar(64) DEFAULT ''", + ] + attribute_sql.extend(job_metadata_sql) + + if not primary_key: + raise DataJointError("Table must have a primary key") + + sql = ( + "CREATE TABLE IF NOT EXISTS %s (\n" % full_table_name + + ",\n".join(attribute_sql + ["PRIMARY KEY (`" + "`,`".join(primary_key) + "`)"] + foreign_key_sql + index_sql) + + '\n) ENGINE=InnoDB, COMMENT "%s"' % table_comment + ) + return sql, external_stores, primary_key, fk_attribute_map + + +def _make_attribute_alter(new: list[str], old: list[str], primary_key: list[str]) -> list[str]: + """ + Generate SQL ALTER commands for attribute changes. + + Parameters + ---------- + new : list[str] + New attribute SQL declarations. + old : list[str] + Old attribute SQL declarations. + primary_key : list[str] + Primary key attribute names (cannot be altered). + + Returns + ------- + list[str] + SQL ALTER commands (ADD, MODIFY, CHANGE, DROP). + + Raises + ------ + DataJointError + If an attribute is renamed twice or renamed from non-existent attribute. + """ + # parse attribute names + name_regexp = re.compile(r"^`(?P\w+)`") + original_regexp = re.compile(r'COMMENT "{\s*(?P\w+)\s*}') + matched = ((name_regexp.match(d), original_regexp.search(d)) for d in new) + new_names = dict((d.group("name"), n and n.group("name")) for d, n in matched) + old_names = [name_regexp.search(d).group("name") for d in old] + + # verify that original names are only used once + renamed = set() + for v in new_names.values(): + if v: + if v in renamed: + raise DataJointError("Alter attempted to rename attribute {%s} twice." % v) + renamed.add(v) + + # verify that all renamed attributes existed in the old definition + try: + raise DataJointError( + "Attribute {} does not exist in the original definition".format( + next(attr for attr in renamed if attr not in old_names) + ) + ) + except StopIteration: + pass + + # dropping attributes + to_drop = [n for n in old_names if n not in renamed and n not in new_names] + sql = ["DROP `%s`" % n for n in to_drop] + old_names = [name for name in old_names if name not in to_drop] + + # add or change attributes in order + prev = None + for new_def, (new_name, old_name) in zip(new, new_names.items()): + if new_name not in primary_key: + after = None # if None, then must include the AFTER clause + if prev: + try: + idx = old_names.index(old_name or new_name) + except ValueError: + after = prev[0] + else: + if idx >= 1 and old_names[idx - 1] != (prev[1] or prev[0]): + after = prev[0] + if new_def not in old or after: + sql.append( + "{command} {new_def} {after}".format( + command=( + "ADD" + if (old_name or new_name) not in old_names + else "MODIFY" + if not old_name + else "CHANGE `%s`" % old_name + ), + new_def=new_def, + after="" if after is None else "AFTER `%s`" % after, + ) + ) + prev = new_name, old_name + + return sql + + +def alter(definition: str, old_definition: str, context: dict) -> tuple[list[str], list[str]]: + """ + Generate SQL ALTER commands for table definition changes. + + Parameters + ---------- + definition : str + New table definition. + old_definition : str + Current table definition. + context : dict + Namespace for resolving foreign key references. + + Returns + ------- + tuple + Two-element tuple: + + - sql : list[str] - SQL ALTER commands + - new_stores : list[str] - New external stores used + + Raises + ------ + NotImplementedError + If attempting to alter primary key, foreign keys, or indexes. + """ + ( + table_comment, + primary_key, + attribute_sql, + foreign_key_sql, + index_sql, + external_stores, + _fk_attribute_map, + ) = prepare_declare(definition, context) + ( + table_comment_, + primary_key_, + attribute_sql_, + foreign_key_sql_, + index_sql_, + external_stores_, + _fk_attribute_map_, + ) = prepare_declare(old_definition, context) + + # analyze differences between declarations + sql = list() + if primary_key != primary_key_: + raise NotImplementedError("table.alter cannot alter the primary key (yet).") + if foreign_key_sql != foreign_key_sql_: + raise NotImplementedError("table.alter cannot alter foreign keys (yet).") + if index_sql != index_sql_: + raise NotImplementedError("table.alter cannot alter indexes (yet)") + if attribute_sql != attribute_sql_: + sql.extend(_make_attribute_alter(attribute_sql, attribute_sql_, primary_key)) + if table_comment != table_comment_: + sql.append('COMMENT="%s"' % table_comment) + return sql, [e for e in external_stores if e not in external_stores_] + + +def compile_index(line: str, index_sql: list[str]) -> None: + """ + Parse an index declaration and append SQL to index_sql. + + Parameters + ---------- + line : str + Index declaration line (e.g., ``"index(attr1, attr2)"`` or + ``"unique index(attr)"``). + index_sql : list[str] + List of index SQL declarations. Updated in place. + + Raises + ------ + DataJointError + If the index syntax is invalid. + """ + + def format_attribute(attr): + match, attr = translate_attribute(attr) + if match is None: + return attr + if match["path"] is None: + return f"`{attr}`" + return f"({attr})" + + match = re.match(r"(?Punique\s+)?index\s*\(\s*(?P.*)\)", line, re.I) + if match is None: + raise DataJointError(f'Table definition syntax error in line "{line}"') + match = match.groupdict() + + attr_list = re.findall(r"(?:[^,(]|\([^)]*\))+", match["args"]) + index_sql.append( + "{unique}index ({attrs})".format( + unique="unique " if match["unique"] else "", + attrs=",".join(format_attribute(a.strip()) for a in attr_list), + ) + ) + + +def substitute_special_type(match: dict, category: str, foreign_key_sql: list[str], context: dict) -> None: + """ + Substitute special types with their native SQL equivalents. + + Special types include core DataJoint types (``float32`` → ``float``, + ``uuid`` → ``binary(16)``, ``bytes`` → ``longblob``) and codec types + (angle bracket syntax like ````). + + Parameters + ---------- + match : dict + Parsed attribute with keys ``"type"``, ``"comment"``, etc. + Modified in place with substituted type. + category : str + Type category from TYPE_PATTERN (e.g., ``"FLOAT32"``, ``"CODEC"``). + foreign_key_sql : list[str] + Foreign key declarations (unused, kept for API compatibility). + context : dict + Namespace for codec lookup (unused, kept for API compatibility). + """ + if category == "CODEC": + # Codec - resolve to underlying dtype + codec, store_name = lookup_codec(match["type"]) + if store_name is not None: + match["store"] = store_name + # Determine if external storage is used (store_name is present, even if empty string for default) + is_external = store_name is not None + inner_dtype = codec.get_dtype(is_external=is_external) + + # If inner dtype is a codec without store, propagate the store from outer type + # e.g., returns , we need to resolve as + if inner_dtype.startswith("<") and "@" not in inner_dtype and match.get("store") is not None: + # Append store to the inner dtype + inner_dtype = inner_dtype[:-1] + "@" + match["store"] + ">" + + match["type"] = inner_dtype + # Recursively resolve if dtype is also a special type + category = match_type(match["type"]) + if category in SPECIAL_TYPES: + substitute_special_type(match, category, foreign_key_sql, context) + elif category in CORE_TYPE_NAMES: + # Core DataJoint type - substitute with native SQL type if mapping exists + core_name = category.lower() + sql_type = CORE_TYPE_SQL.get(core_name) + if sql_type is not None: + match["type"] = sql_type + # else: type passes through as-is (json, date, datetime, char, varchar, enum) + else: + assert False, f"Unknown special type: {category}" + + +def compile_attribute(line: str, in_key: bool, foreign_key_sql: list[str], context: dict) -> tuple[str, str, str | None]: + """ + Convert an attribute definition from DataJoint format to SQL. + + Parameters + ---------- + line : str + Attribute definition line (e.g., ``"session_id : int32 # unique session"``). + in_key : bool + True if the attribute is part of the primary key. + foreign_key_sql : list[str] + Foreign key declarations (passed to type substitution). + context : dict + Namespace for codec lookup. + + Returns + ------- + tuple + Three-element tuple: + + - name : str - Attribute name + - sql : str - SQL column declaration + - store : str or None - External store name if applicable + + Raises + ------ + DataJointError + If syntax is invalid, primary key is nullable, or blob has invalid default. + """ + try: + match = attribute_parser.parse_string(line + "#", parse_all=True) + except pp.ParseException as err: + raise DataJointError( + "Declaration error in position {pos} in line:\n {line}\n{msg}".format( + line=err.args[0], pos=err.args[1], msg=err.args[2] + ) + ) + match["comment"] = match["comment"].rstrip("#") + if "default" not in match: + match["default"] = "" + match = {k: v.strip() for k, v in match.items()} + match["nullable"] = match["default"].lower() == "null" + + if match["nullable"]: + if in_key: + raise DataJointError('Primary key attributes cannot be nullable in line "%s"' % line) + match["default"] = "DEFAULT NULL" # nullable attributes default to null + else: + if match["default"]: + quote = match["default"].split("(")[0].upper() not in CONSTANT_LITERALS and match["default"][0] not in "\"'" + match["default"] = "NOT NULL DEFAULT " + ('"%s"' if quote else "%s") % match["default"] + else: + match["default"] = "NOT NULL" + + match["comment"] = match["comment"].replace('"', '\\"') # escape double quotes in comment + + if match["comment"].startswith(":"): + raise DataJointError('An attribute comment must not start with a colon in comment "{comment}"'.format(**match)) + + category = match_type(match["type"]) + + if category in SPECIAL_TYPES: + # Core types and Codecs are recorded in comment for reconstruction + match["comment"] = ":{type}:{comment}".format(**match) + substitute_special_type(match, category, foreign_key_sql, context) + elif category in NATIVE_TYPES: + # Native type - warn user + logger.warning( + f"Native type '{match['type']}' is used in attribute '{match['name']}'. " + "Consider using a core DataJoint type for better portability." + ) + + # Check for invalid default values on blob types (after type substitution) + # Note: blob → longblob, so check for NATIVE_BLOB or longblob result + final_type = match["type"].lower() + if ("blob" in final_type) and match["default"] not in {"DEFAULT NULL", "NOT NULL"}: + raise DataJointError("The default value for blob attributes can only be NULL in:\n{line}".format(line=line)) + + sql = ("`{name}` {type} {default}" + (' COMMENT "{comment}"' if match["comment"] else "")).format(**match) + return match["name"], sql, match.get("store") diff --git a/datajoint/dependencies.py b/src/datajoint/dependencies.py similarity index 55% rename from datajoint/dependencies.py rename to src/datajoint/dependencies.py index 7496ee600..621011426 100644 --- a/datajoint/dependencies.py +++ b/src/datajoint/dependencies.py @@ -1,3 +1,13 @@ +""" +Foreign key dependency graph for DataJoint schemas. + +This module provides the Dependencies class that tracks foreign key +relationships between tables and supports topological sorting for +proper ordering of operations like delete and drop. +""" + +from __future__ import annotations + import itertools import re from collections import defaultdict @@ -7,18 +17,37 @@ from .errors import DataJointError -def extract_master(part_table): - """ - given a part table name, return master part. None if not a part table +def extract_master(part_table: str) -> str | None: + r""" + Extract master table name from a part table name. + + Parameters + ---------- + part_table : str + Full table name (e.g., ```\`schema\`.\`master__part\```). + + Returns + ------- + str or None + Master table name if part_table is a part table, None otherwise. """ match = re.match(r"(?P`\w+`.`#?\w+)__\w+`", part_table) return match["master"] + "`" if match else None -def topo_sort(graph): +def topo_sort(graph: nx.DiGraph) -> list[str]: """ - topological sort of a dependency graph that keeps part tables together with their masters - :return: list of table names in topological order + Topological sort keeping part tables with their masters. + + Parameters + ---------- + graph : nx.DiGraph + Dependency graph. + + Returns + ------- + list[str] + Table names in topological order with parts following masters. """ graph = nx.DiGraph(graph) # make a copy @@ -69,28 +98,52 @@ def topo_sort(graph): class Dependencies(nx.DiGraph): """ - The graph of dependencies (foreign keys) between loaded tables. + Graph of foreign key dependencies between loaded tables. - Note: the 'connection' argument should normally be supplied; - Empty use is permitted to facilitate use of networkx algorithms which - internally create objects with the expectation of empty constructors. - See also: https://github.com/datajoint/datajoint-python/pull/443 + Extends NetworkX DiGraph to track foreign key relationships and + support operations like cascade delete and topological ordering. + + Parameters + ---------- + connection : Connection, optional + Database connection. May be None to support NetworkX algorithms + that create objects with empty constructors. + + Attributes + ---------- + _conn : Connection or None + Database connection. + _loaded : bool + Whether dependencies have been loaded from the database. + + Notes + ----- + Empty constructor use is permitted to facilitate NetworkX algorithms. + See: https://github.com/datajoint/datajoint-python/pull/443 """ - def __init__(self, connection=None): + def __init__(self, connection=None) -> None: self._conn = connection self._node_alias_count = itertools.count() self._loaded = False super().__init__(self) - def clear(self): + def clear(self) -> None: + """Clear the graph and reset loaded state.""" self._loaded = False super().clear() - def load(self, force=True): + def load(self, force: bool = True) -> None: """ Load dependencies for all loaded schemas. - This method gets called before any operation that requires dependencies: delete, drop, populate, progress. + + Called before operations requiring dependencies: delete, drop, + populate, progress. + + Parameters + ---------- + force : bool, optional + If True (default), reload even if already loaded. """ # reload from scratch to prevent duplication of renamed edges if self._loaded and not force: @@ -105,9 +158,7 @@ def load(self, force=True): concat('`', table_schema, '`.`', table_name, '`') as tab, column_name FROM information_schema.key_column_usage WHERE table_name not LIKE "~%%" AND table_schema in ('{schemas}') AND constraint_name="PRIMARY" - """.format( - schemas="','".join(self._conn.schemas) - ) + """.format(schemas="','".join(self._conn.schemas)) ) pks = defaultdict(set) for key in keys: @@ -129,9 +180,7 @@ def load(self, force=True): FROM information_schema.key_column_usage WHERE referenced_table_name NOT LIKE "~%%" AND (referenced_table_schema in ('{schemas}') OR referenced_table_schema is not NULL AND table_schema in ('{schemas}')) - """.format( - schemas="','".join(self._conn.schemas) - ), + """.format(schemas="','".join(self._conn.schemas)), as_dict=True, ) ) @@ -169,53 +218,90 @@ def load(self, force=True): raise DataJointError("DataJoint can only work with acyclic dependencies") self._loaded = True - def topo_sort(self): - """:return: list of tables names in topological order""" - return topo_sort(self) + def topo_sort(self) -> list[str]: + """ + Return table names in topological order. - def parents(self, table_name, primary=None): + Returns + ------- + list[str] + Table names sorted topologically. """ - :param table_name: `schema`.`table` - :param primary: if None, then all parents are returned. If True, then only foreign keys composed of - primary key attributes are considered. If False, the only foreign keys including at least one non-primary - attribute are considered. - :return: dict of tables referenced by the foreign keys of table + return topo_sort(self) + + def parents(self, table_name: str, primary: bool | None = None) -> dict: + r""" + Get tables referenced by this table's foreign keys. + + Parameters + ---------- + table_name : str + Full table name (```\`schema\`.\`table\```). + primary : bool, optional + If None, return all parents. If True, only FK composed entirely + of primary key attributes. If False, only FK with at least one + non-primary attribute. + + Returns + ------- + dict + Mapping of parent table name to edge properties. """ self.load(force=False) - return { - p[0]: p[2] - for p in self.in_edges(table_name, data=True) - if primary is None or p[2]["primary"] == primary - } + return {p[0]: p[2] for p in self.in_edges(table_name, data=True) if primary is None or p[2]["primary"] == primary} - def children(self, table_name, primary=None): - """ - :param table_name: `schema`.`table` - :param primary: if None, then all children are returned. If True, then only foreign keys composed of - primary key attributes are considered. If False, the only foreign keys including at least one non-primary - attribute are considered. - :return: dict of tables referencing the table through foreign keys + def children(self, table_name: str, primary: bool | None = None) -> dict: + r""" + Get tables that reference this table through foreign keys. + + Parameters + ---------- + table_name : str + Full table name (```\`schema\`.\`table\```). + primary : bool, optional + If None, return all children. If True, only FK composed entirely + of primary key attributes. If False, only FK with at least one + non-primary attribute. + + Returns + ------- + dict + Mapping of child table name to edge properties. """ self.load(force=False) - return { - p[1]: p[2] - for p in self.out_edges(table_name, data=True) - if primary is None or p[2]["primary"] == primary - } + return {p[1]: p[2] for p in self.out_edges(table_name, data=True) if primary is None or p[2]["primary"] == primary} - def descendants(self, full_table_name): - """ - :param full_table_name: In form `schema`.`table_name` - :return: all dependent tables sorted in topological order. Self is included. + def descendants(self, full_table_name: str) -> list[str]: + r""" + Get all dependent tables in topological order. + + Parameters + ---------- + full_table_name : str + Full table name (```\`schema\`.\`table_name\```). + + Returns + ------- + list[str] + Dependent tables in topological order. Self is included first. """ self.load(force=False) nodes = self.subgraph(nx.descendants(self, full_table_name)) return [full_table_name] + nodes.topo_sort() - def ancestors(self, full_table_name): - """ - :param full_table_name: In form `schema`.`table_name` - :return: all dependent tables sorted in topological order. Self is included. + def ancestors(self, full_table_name: str) -> list[str]: + r""" + Get all ancestor tables in reverse topological order. + + Parameters + ---------- + full_table_name : str + Full table name (```\`schema\`.\`table_name\```). + + Returns + ------- + list[str] + Ancestor tables in reverse topological order. Self is included last. """ self.load(force=False) nodes = self.subgraph(nx.ancestors(self, full_table_name)) diff --git a/datajoint/diagram.py b/src/datajoint/diagram.py similarity index 61% rename from datajoint/diagram.py rename to src/datajoint/diagram.py index aa505fb54..de211df8f 100644 --- a/datajoint/diagram.py +++ b/src/datajoint/diagram.py @@ -1,3 +1,12 @@ +""" +Diagram visualization for DataJoint schemas. + +This module provides the Diagram class for visualizing schema structure +as directed acyclic graphs showing tables and their foreign key relationships. +""" + +from __future__ import annotations + import functools import inspect import io @@ -28,51 +37,62 @@ logger = logging.getLogger(__name__.split(".")[0]) -if not diagram_active: +if not diagram_active: # noqa: C901 class Diagram: """ - Entity relationship diagram, currently disabled due to the lack of required packages: matplotlib and pygraphviz. + Schema diagram (disabled). - To enable Diagram feature, please install both matplotlib and pygraphviz. For instructions on how to install - these two packages, refer to https://docs.datajoint.com/core/datajoint-python/0.14/client/install/ + Diagram visualization requires matplotlib and pygraphviz packages. + Install them to enable this feature. + + See Also + -------- + https://docs.datajoint.com/core/datajoint-python/0.14/client/install/ """ - def __init__(self, *args, **kwargs): - logger.warning( - "Please install matplotlib and pygraphviz libraries to enable the Diagram feature." - ) + def __init__(self, *args, **kwargs) -> None: + logger.warning("Please install matplotlib and pygraphviz libraries to enable the Diagram feature.") else: class Diagram(nx.DiGraph): """ - Schema diagram showing tables and foreign keys between in the form of a directed - acyclic graph (DAG). The diagram is derived from the connection.dependencies object. - - Usage: - - >>> diag = Diagram(source) - - source can be a table object, a table class, a schema, or a module that has a schema. - + Schema diagram as a directed acyclic graph (DAG). + + Visualizes tables and foreign key relationships derived from + ``connection.dependencies``. + + Parameters + ---------- + source : Table, Schema, or module + A table object, table class, schema, or module with a schema. + context : dict, optional + Namespace for resolving table class names. If None, uses caller's + frame globals/locals. + + Examples + -------- + >>> diag = dj.Diagram(schema.MyTable) >>> diag.draw() - draws the diagram using pyplot + Operators: - diag1 + diag2 - combines the two diagrams. - diag1 - diag2 - difference between diagrams - diag1 * diag2 - intersection of diagrams - diag + n - expands n levels of successors - diag - n - expands n levels of predecessors - Thus dj.Diagram(schema.Table)+1-1 defines the diagram of immediate ancestors and descendants of schema.Table + - ``diag1 + diag2`` - union of diagrams + - ``diag1 - diag2`` - difference of diagrams + - ``diag1 * diag2`` - intersection of diagrams + - ``diag + n`` - expand n levels of successors (children) + - ``diag - n`` - expand n levels of predecessors (parents) - Note that diagram + 1 - 1 may differ from diagram - 1 + 1 and so forth. - Only those tables that are loaded in the connection object are displayed - """ + >>> dj.Diagram(schema.Table) + 1 - 1 # immediate ancestors and descendants - def __init__(self, source, context=None): + Notes + ----- + ``diagram + 1 - 1`` may differ from ``diagram - 1 + 1``. + Only tables loaded in the connection are displayed. + """ + def __init__(self, source, context=None) -> None: if isinstance(source, Diagram): # copy constructor self.nodes_to_show = set(source.nodes_to_show) @@ -95,9 +115,7 @@ def __init__(self, source, context=None): try: connection = source.schema.connection except AttributeError: - raise DataJointError( - "Could not find database connection in %s" % repr(source[0]) - ) + raise DataJointError("Could not find database connection in %s" % repr(source[0])) # initialize graph from dependencies connection.dependencies.load() @@ -114,55 +132,60 @@ def __init__(self, source, context=None): try: database = source.schema.database except AttributeError: - raise DataJointError( - "Cannot plot Diagram for %s" % repr(source) - ) + raise DataJointError("Cannot plot Diagram for %s" % repr(source)) for node in self: if node.startswith("`%s`" % database): self.nodes_to_show.add(node) @classmethod - def from_sequence(cls, sequence): + def from_sequence(cls, sequence) -> "Diagram": """ - The join Diagram for all objects in sequence + Create combined Diagram from a sequence of sources. + + Parameters + ---------- + sequence : iterable + Sequence of table objects, classes, or schemas. - :param sequence: a sequence (e.g. list, tuple) - :return: Diagram(arg1) + ... + Diagram(argn) + Returns + ------- + Diagram + Union of diagrams: ``Diagram(arg1) + ... + Diagram(argn)``. """ return functools.reduce(lambda x, y: x + y, map(Diagram, sequence)) - def add_parts(self): + def add_parts(self) -> "Diagram": """ - Adds to the diagram the part tables of all master tables already in the diagram - :return: + Add part tables of all masters already in the diagram. + + Returns + ------- + Diagram + New diagram with part tables included. """ def is_part(part, master): - """ - :param part: `database`.`table_name` - :param master: `database`.`table_name` - :return: True if part is part of master. - """ part = [s.strip("`") for s in part.split(".")] master = [s.strip("`") for s in master.split(".")] - return ( - master[0] == part[0] - and master[1] + "__" == part[1][: len(master[1]) + 2] - ) + return master[0] == part[0] and master[1] + "__" == part[1][: len(master[1]) + 2] self = Diagram(self) # copy - self.nodes_to_show.update( - n - for n in self.nodes() - if any(is_part(n, m) for m in self.nodes_to_show) - ) + self.nodes_to_show.update(n for n in self.nodes() if any(is_part(n, m) for m in self.nodes_to_show)) return self - def __add__(self, arg): + def __add__(self, arg) -> "Diagram": """ - :param arg: either another Diagram or a positive integer. - :return: Union of the diagrams when arg is another Diagram - or an expansion downstream when arg is a positive integer. + Union or downstream expansion. + + Parameters + ---------- + arg : Diagram or int + Another Diagram for union, or positive int for downstream expansion. + + Returns + ------- + Diagram + Combined or expanded diagram. """ self = Diagram(self) # copy try: @@ -172,25 +195,27 @@ def __add__(self, arg): self.nodes_to_show.add(arg.full_table_name) except AttributeError: for i in range(arg): - new = nx.algorithms.boundary.node_boundary( - self, self.nodes_to_show - ) + new = nx.algorithms.boundary.node_boundary(self, self.nodes_to_show) if not new: break # add nodes referenced by aliased nodes - new.update( - nx.algorithms.boundary.node_boundary( - self, (a for a in new if a.isdigit()) - ) - ) + new.update(nx.algorithms.boundary.node_boundary(self, (a for a in new if a.isdigit()))) self.nodes_to_show.update(new) return self - def __sub__(self, arg): + def __sub__(self, arg) -> "Diagram": """ - :param arg: either another Diagram or a positive integer. - :return: Difference of the diagrams when arg is another Diagram or - an expansion upstream when arg is a positive integer. + Difference or upstream expansion. + + Parameters + ---------- + arg : Diagram or int + Another Diagram for difference, or positive int for upstream expansion. + + Returns + ------- + Diagram + Reduced or expanded diagram. """ self = Diagram(self) # copy try: @@ -201,86 +226,86 @@ def __sub__(self, arg): except AttributeError: for i in range(arg): graph = nx.DiGraph(self).reverse() - new = nx.algorithms.boundary.node_boundary( - graph, self.nodes_to_show - ) + new = nx.algorithms.boundary.node_boundary(graph, self.nodes_to_show) if not new: break # add nodes referenced by aliased nodes - new.update( - nx.algorithms.boundary.node_boundary( - graph, (a for a in new if a.isdigit()) - ) - ) + new.update(nx.algorithms.boundary.node_boundary(graph, (a for a in new if a.isdigit()))) self.nodes_to_show.update(new) return self - def __mul__(self, arg): + def __mul__(self, arg) -> "Diagram": """ - Intersection of two diagrams - :param arg: another Diagram - :return: a new Diagram comprising nodes that are present in both operands. + Intersection of two diagrams. + + Parameters + ---------- + arg : Diagram + Another Diagram. + + Returns + ------- + Diagram + Diagram with nodes present in both operands. """ self = Diagram(self) # copy self.nodes_to_show.intersection_update(arg.nodes_to_show) return self - def topo_sort(self): - """return nodes in lexicographical topological order""" + def topo_sort(self) -> list[str]: + """ + Return nodes in topological order. + + Returns + ------- + list[str] + Node names in topological order. + """ return topo_sort(self) - def _make_graph(self): + def _make_graph(self) -> nx.DiGraph: """ - Make the self.graph - a graph object ready for drawing + Build graph object ready for drawing. + + Returns + ------- + nx.DiGraph + Graph with nodes relabeled to class names. """ # mark "distinguished" tables, i.e. those that introduce new primary key # attributes for name in self.nodes_to_show: foreign_attributes = set( - attr - for p in self.in_edges(name, data=True) - for attr in p[2]["attr_map"] - if p[2]["primary"] + attr for p in self.in_edges(name, data=True) for attr in p[2]["attr_map"] if p[2]["primary"] ) self.nodes[name]["distinguished"] = ( - "primary_key" in self.nodes[name] - and foreign_attributes < self.nodes[name]["primary_key"] + "primary_key" in self.nodes[name] and foreign_attributes < self.nodes[name]["primary_key"] ) # include aliased nodes that are sandwiched between two displayed nodes - gaps = set( - nx.algorithms.boundary.node_boundary(self, self.nodes_to_show) - ).intersection( - nx.algorithms.boundary.node_boundary( - nx.DiGraph(self).reverse(), self.nodes_to_show - ) + gaps = set(nx.algorithms.boundary.node_boundary(self, self.nodes_to_show)).intersection( + nx.algorithms.boundary.node_boundary(nx.DiGraph(self).reverse(), self.nodes_to_show) ) nodes = self.nodes_to_show.union(a for a in gaps if a.isdigit) # construct subgraph and rename nodes to class names graph = nx.DiGraph(nx.DiGraph(self).subgraph(nodes)) - nx.set_node_attributes( - graph, name="node_type", values={n: _get_tier(n) for n in graph} - ) + nx.set_node_attributes(graph, name="node_type", values={n: _get_tier(n) for n in graph}) # relabel nodes to class names - mapping = { - node: lookup_class_name(node, self.context) or node - for node in graph.nodes() - } + mapping = {node: lookup_class_name(node, self.context) or node for node in graph.nodes()} new_names = [mapping.values()] if len(new_names) > len(set(new_names)): - raise DataJointError( - "Some classes have identical names. The Diagram cannot be plotted." - ) + raise DataJointError("Some classes have identical names. The Diagram cannot be plotted.") nx.relabel_nodes(graph, mapping, copy=False) return graph @staticmethod - def _encapsulate_edge_attributes(graph): + def _encapsulate_edge_attributes(graph: nx.DiGraph) -> None: """ - Modifies the `nx.Graph`'s edge attribute `attr_map` to be a string representation - of the attribute map, and encapsulates the string in double quotes. - Changes the graph in place. + Encapsulate edge attr_map in double quotes for pydot compatibility. + + Modifies graph in place. - Implements workaround described in + See Also + -------- https://github.com/pydot/pydot/issues/258#issuecomment-795798099 """ for u, v, *_, edgedata in graph.edges(data=True): @@ -288,13 +313,14 @@ def _encapsulate_edge_attributes(graph): graph.edges[u, v]["attr_map"] = '"{0}"'.format(edgedata["attr_map"]) @staticmethod - def _encapsulate_node_names(graph): + def _encapsulate_node_names(graph: nx.DiGraph) -> None: """ - Modifies the `nx.Graph`'s node names string representations encapsulated in - double quotes. - Changes the graph in place. + Encapsulate node names in double quotes for pydot compatibility. - Implements workaround described in + Modifies graph in place. + + See Also + -------- https://github.com/datajoint/datajoint-python/pull/1176 """ nx.relabel_nodes( @@ -366,10 +392,7 @@ def make_dot(self): fixed=False, ), } - node_props = { - node: label_props[d["node_type"]] - for node, d in dict(graph.nodes(data=True)).items() - } + node_props = {node: label_props[d["node_type"]] for node, d in dict(graph.nodes(data=True)).items()} self._encapsulate_node_names(graph) self._encapsulate_edge_attributes(graph) @@ -390,24 +413,12 @@ def make_dot(self): assert issubclass(cls, Table) description = cls().describe(context=self.context).split("\n") description = ( - ( - "-" * 30 - if q.startswith("---") - else ( - q.replace("->", "→") - if "->" in q - else q.split(":")[0] - ) - ) + ("-" * 30 if q.startswith("---") else (q.replace("->", "→") if "->" in q else q.split(":")[0])) for q in description if not q.startswith("#") ) node.set_tooltip(" ".join(description)) - node.set_label( - "<" + name + ">" - if node.get("distinguished") == "True" - else name - ) + node.set_label("<" + name + ">" if node.get("distinguished") == "True" else name) node.set_color(props["color"]) node.set_style("filled") @@ -417,15 +428,10 @@ def make_dot(self): dest = edge.get_destination() props = graph.get_edge_data(src, dest) if props is None: - raise DataJointError( - "Could not find edge with source " - "'{}' and destination '{}'".format(src, dest) - ) + raise DataJointError("Could not find edge with source '{}' and destination '{}'".format(src, dest)) edge.set_color("#00000040") edge.set_style("solid" if props["primary"] else "dashed") - master_part = graph.nodes[dest][ - "node_type" - ] is Part and dest.startswith(src + ".") + master_part = graph.nodes[dest]["node_type"] is Part and dest.startswith(src + ".") edge.set_weight(3 if master_part else 1) edge.set_arrowhead("none") edge.set_penwidth(0.75 if props["multi"] else 2) @@ -457,7 +463,22 @@ def draw(self): else: raise DataJointError("pyplot was not imported") - def save(self, filename, format=None): + def save(self, filename: str, format: str | None = None) -> None: + """ + Save diagram to file. + + Parameters + ---------- + filename : str + Output filename. + format : str, optional + File format (``'png'`` or ``'svg'``). Inferred from extension if None. + + Raises + ------ + DataJointError + If format is unsupported. + """ if format is None: if filename.lower().endswith(".png"): format = "png" diff --git a/src/datajoint/errors.py b/src/datajoint/errors.py new file mode 100644 index 000000000..7e10f021d --- /dev/null +++ b/src/datajoint/errors.py @@ -0,0 +1,74 @@ +""" +Exception classes for the DataJoint library. + +This module defines the exception hierarchy for DataJoint errors. +""" + +from __future__ import annotations + + +# --- Top Level --- +class DataJointError(Exception): + """Base class for errors specific to DataJoint internal operation.""" + + def suggest(self, *args: object) -> "DataJointError": + """ + Regenerate the exception with additional arguments. + + Parameters + ---------- + *args : object + Additional arguments to append to the exception. + + Returns + ------- + DataJointError + A new exception of the same type with the additional arguments. + """ + return self.__class__(*(self.args + args)) + + +# --- Second Level --- +class LostConnectionError(DataJointError): + """Loss of server connection.""" + + +class QueryError(DataJointError): + """Errors arising from queries to the database.""" + + +# --- Third Level: QueryErrors --- +class QuerySyntaxError(QueryError): + """Errors arising from incorrect query syntax.""" + + +class AccessError(QueryError): + """User access error: insufficient privileges.""" + + +class MissingTableError(DataJointError): + """Query on a table that has not been declared.""" + + +class DuplicateError(QueryError): + """Integrity error caused by a duplicate entry into a unique key.""" + + +class IntegrityError(QueryError): + """Integrity error triggered by foreign key constraints.""" + + +class UnknownAttributeError(QueryError): + """User requests an attribute name not found in query heading.""" + + +class MissingAttributeError(QueryError): + """Required attribute value not provided in INSERT.""" + + +class MissingExternalFile(DataJointError): + """External file managed by DataJoint is no longer accessible.""" + + +class BucketInaccessible(DataJointError): + """S3 bucket is inaccessible.""" diff --git a/src/datajoint/expression.py b/src/datajoint/expression.py new file mode 100644 index 000000000..354e3ef35 --- /dev/null +++ b/src/datajoint/expression.py @@ -0,0 +1,1247 @@ +import copy +import inspect +import logging +import re +from itertools import count + +from .condition import ( + AndList, + Not, + Top, + assert_join_compatibility, + extract_column_names, + make_condition, + translate_attribute, +) +from .declare import CONSTANT_LITERALS +import numpy as np +import pandas + +from .errors import DataJointError +from .codecs import decode_attribute +from .preview import preview, repr_html +from .settings import config + +logger = logging.getLogger(__name__.split(".")[0]) + + +class QueryExpression: + """ + QueryExpression implements query operators to derive new entity set from its input. + A QueryExpression object generates a SELECT statement in SQL. + QueryExpression operators are restrict, join, proj, aggr, and union. + + A QueryExpression object has a support, a restriction (an AndList), and heading. + Property `heading` (type dj.Heading) contains information about the attributes. + It is loaded from the database and updated by proj. + + Property `support` is the list of table names or other QueryExpressions to be joined. + + The restriction is applied first without having access to the attributes generated by the projection. + Then projection is applied by selecting modifying the heading attribute. + + Application of operators does not always lead to the creation of a subquery. + A subquery is generated when: + 1. A restriction is applied on any computed or renamed attributes + 2. A projection is applied remapping remapped attributes + 3. Subclasses: Join, Aggregation, and Union have additional specific rules. + """ + + _restriction = None + _restriction_attributes = None + _joins = [] # list of (is_left: bool, using_attrs: list[str]) for each join + _original_heading = None # heading before projections + + # subclasses or instantiators must provide values + _connection = None + _heading = None + _support = None + _top = None + + # If the query will be using distinct + _distinct = False + + @property + def connection(self): + """a dj.Connection object""" + assert self._connection is not None + return self._connection + + @property + def support(self): + """A list of table names or subqueries to from the FROM clause""" + assert self._support is not None + return self._support + + @property + def heading(self): + """a dj.Heading object, reflects the effects of the projection operator .proj""" + return self._heading + + @property + def original_heading(self): + """a dj.Heading object reflecting the attributes before projection""" + return self._original_heading or self.heading + + @property + def restriction(self): + """a AndList object of restrictions applied to input to produce the result""" + if self._restriction is None: + self._restriction = AndList() + return self._restriction + + @property + def restriction_attributes(self): + """the set of attribute names invoked in the WHERE clause""" + if self._restriction_attributes is None: + self._restriction_attributes = set() + return self._restriction_attributes + + @property + def primary_key(self): + return self.heading.primary_key + + _subquery_alias_count = count() # count for alias names used in the FROM clause + + def from_clause(self): + support = ( + ( + "(" + src.make_sql() + ") as `$%x`" % next(self._subquery_alias_count) + if isinstance(src, QueryExpression) + else src + ) + for src in self.support + ) + clause = next(support) + for s, (is_left, using_attrs) in zip(support, self._joins): + left_kw = "LEFT " if is_left else "" + if using_attrs: + using = "USING ({})".format(", ".join(f"`{a}`" for a in using_attrs)) + clause += f" {left_kw}JOIN {s} {using}" + else: + # Cross join (no common non-hidden attributes) + if is_left: + clause += f" LEFT JOIN {s} ON TRUE" + else: + clause += f" CROSS JOIN {s}" + return clause + + def where_clause(self): + return "" if not self.restriction else " WHERE (%s)" % ")AND(".join(str(s) for s in self.restriction) + + def sorting_clauses(self): + if not self._top: + return "" + # Default to KEY ordering if order_by is None (inherit with no existing order) + order_by = self._top.order_by if self._top.order_by is not None else ["KEY"] + clause = ", ".join(_wrap_attributes(_flatten_attribute_list(self.primary_key, order_by))) + if clause: + clause = f" ORDER BY {clause}" + if self._top.limit is not None: + clause += f" LIMIT {self._top.limit}{f' OFFSET {self._top.offset}' if self._top.offset else ''}" + + return clause + + def make_sql(self, fields=None): + """ + Make the SQL SELECT statement. + + :param fields: used to explicitly set the select attributes + """ + return "SELECT {distinct}{fields} FROM {from_}{where}{sorting}".format( + distinct="DISTINCT " if self._distinct else "", + fields=self.heading.as_sql(fields or self.heading.names), + from_=self.from_clause(), + where=self.where_clause(), + sorting=self.sorting_clauses(), + ) + + # --------- query operators ----------- + def make_subquery(self): + """create a new SELECT statement where self is the FROM clause""" + result = QueryExpression() + result._connection = self.connection + result._support = [self] + result._heading = self.heading.make_subquery_heading() + return result + + def restrict(self, restriction, semantic_check=True): + """ + Produces a new expression with the new restriction applied. + + :param restriction: a sequence or an array (treated as OR list), another QueryExpression, + an SQL condition string, or an AndList. + :param semantic_check: If True (default), use semantic matching - only match on + homologous namesakes and error on non-homologous namesakes. + If False, use natural matching on all namesakes (no lineage checking). + :return: A new QueryExpression with the restriction applied. + + rel.restrict(restriction) is equivalent to rel & restriction. + rel.restrict(Not(restriction)) is equivalent to rel - restriction + + The primary key of the result is unaffected. + Successive restrictions are combined as logical AND: r & a & b is equivalent to r & AndList((a, b)) + Any QueryExpression, collection, or sequence other than an AndList are treated as OrLists + (logical disjunction of conditions) + Inverse restriction is accomplished by either using the subtraction operator or the Not class. + + The expressions in each row equivalent: + + rel & True rel + rel & False the empty entity set + rel & 'TRUE' rel + rel & 'FALSE' the empty entity set + rel - cond rel & Not(cond) + rel - 'TRUE' rel & False + rel - 'FALSE' rel + rel & AndList((cond1,cond2)) rel & cond1 & cond2 + rel & AndList() rel + rel & [cond1, cond2] rel & OrList((cond1, cond2)) + rel & [] rel & False + rel & None rel & False + rel & any_empty_entity_set rel & False + rel - AndList((cond1,cond2)) rel & [Not(cond1), Not(cond2)] + rel - [cond1, cond2] rel & Not(cond1) & Not(cond2) + rel - AndList() rel & False + rel - [] rel + rel - None rel + rel - any_empty_entity_set rel + + When arg is another QueryExpression, the restriction rel & arg restricts rel to elements that match at least + one element in arg (hence arg is treated as an OrList). + Conversely, rel - arg restricts rel to elements that do not match any elements in arg. + Two elements match when their common attributes have equal values or when they have no common attributes. + All shared attributes must be in the primary key of either rel or arg or both or an error will be raised. + + QueryExpression.restrict is the only access point that modifies restrictions. All other operators must + ultimately call restrict() + """ + attributes = set() + if isinstance(restriction, Top): + if self._top is None: + # No existing Top — apply new one directly + result = copy.copy(self) + result._top = restriction + elif restriction.order_by is None or restriction.order_by == self._top.order_by: + # Merge: new Top inherits or matches existing ordering + result = copy.copy(self) + result._top = self._top.merge(restriction) + else: + # Different ordering — need subquery + result = self.make_subquery() + result._top = restriction + return result + new_condition = make_condition(self, restriction, attributes, semantic_check=semantic_check) + if new_condition is True: + return self # restriction has no effect, return the same object + # check that all attributes in condition are present in the query + try: + raise DataJointError( + "Attribute `%s` is not found in query." % next(attr for attr in attributes if attr not in self.heading.names) + ) + except StopIteration: + pass # all ok + # If the new condition uses any new attributes, a subquery is required. + # However, Aggregation's HAVING statement works fine with aliased attributes. + need_subquery = ( + isinstance(self, Union) or (not isinstance(self, Aggregation) and self.heading.new_attributes) or self._top + ) + if need_subquery: + result = self.make_subquery() + else: + result = copy.copy(self) + result._restriction = AndList(self.restriction) # copy to preserve the original + result.restriction.append(new_condition) + result.restriction_attributes.update(attributes) + return result + + def restrict_in_place(self, restriction): + self.__dict__.update(self.restrict(restriction).__dict__) + + def __and__(self, restriction): + """ + Restriction operator e.g. ``q1 & q2``. + :return: a restricted copy of the input argument + See QueryExpression.restrict for more detail. + """ + return self.restrict(restriction) + + def __xor__(self, restriction): + """The ^ operator has been removed in DataJoint 2.0.""" + raise DataJointError( + "The ^ operator has been removed in DataJoint 2.0. " + "Use .restrict(other, semantic_check=False) for restrictions without semantic checking." + ) + + def __sub__(self, restriction): + """ + Inverted restriction e.g. ``q1 - q2``. + :return: a restricted copy of the input argument + See QueryExpression.restrict for more detail. + """ + return self.restrict(Not(restriction)) + + def __neg__(self): + """ + Convert between restriction and inverted restriction e.g. ``-q1``. + :return: target restriction + See QueryExpression.restrict for more detail. + """ + if isinstance(self, Not): + return self.restriction + return Not(self) + + def __mul__(self, other): + """ + join of query expressions `self` and `other` e.g. ``q1 * q2``. + """ + return self.join(other) + + def __matmul__(self, other): + """The @ operator has been removed in DataJoint 2.0.""" + raise DataJointError( + "The @ operator has been removed in DataJoint 2.0. " + "Use .join(other, semantic_check=False) for joins without semantic checking." + ) + + def join(self, other, semantic_check=True, left=False, allow_nullable_pk=False): + """ + Create the joined QueryExpression. + + :param other: QueryExpression to join with + :param semantic_check: If True (default), use semantic matching - only match on + homologous namesakes (same lineage) and error on non-homologous namesakes. + If False, use natural join on all namesakes (no lineage checking). + :param left: If True, perform a left join (retain all rows from self) + :param allow_nullable_pk: If True, bypass the left join constraint that requires + self to determine other. When bypassed, the result PK is the union of both + operands' PKs, and PK attributes from the right operand could be NULL. + Used internally by aggregation when exclude_nonmatching=False. + :return: The joined QueryExpression + + a * b is short for a.join(b) + """ + # Joining with U is no longer supported + if isinstance(other, U): + raise DataJointError( + "table * dj.U(...) is no longer supported in DataJoint 2.0. " + "This pattern is no longer necessary with the new semantic matching system." + ) + if inspect.isclass(other) and issubclass(other, QueryExpression): + other = other() # instantiate + if not isinstance(other, QueryExpression): + raise DataJointError("The argument of join must be a QueryExpression") + assert_join_compatibility(self, other, semantic_check=semantic_check) + + # Left join constraint: requires self → other (left operand determines right) + # This ensures the result's PK (which is PK(self) for left joins) can't have NULLs + nullable_pk = False + if left and not self.heading.determines(other.heading): + if allow_nullable_pk: + nullable_pk = True + else: + undetermined = [attr for attr in other.heading.primary_key if attr not in self.heading.names] + raise DataJointError( + "Left join requires the left operand to determine the right operand (A → B). " + f"The following attributes from the right operand's primary key are not " + f"in the left operand: {undetermined}. " + "Use an inner join or restructure the query." + ) + + # Always join on all non-hidden namesakes + join_attributes = set(n for n in self.heading.names if n in other.heading.names) + # needs subquery if self's FROM clause has common attributes with other's FROM clause + need_subquery1 = need_subquery2 = bool( + (set(self.original_heading.names) & set(other.original_heading.names)) - join_attributes + ) + # need subquery if any of the join attributes are derived + need_subquery1 = ( + need_subquery1 + or isinstance(self, Aggregation) + or any(n in self.heading.new_attributes for n in join_attributes) + or isinstance(self, Union) + ) + need_subquery2 = ( + need_subquery2 + or isinstance(other, Aggregation) + or any(n in other.heading.new_attributes for n in join_attributes) + or isinstance(self, Union) + ) + # With USING clause (instead of NATURAL JOIN), we need subqueries when + # joining with multi-table expressions to ensure correct column matching + if len(self.support) > 1 and join_attributes: + need_subquery1 = True + if len(other.support) > 1 and join_attributes: + need_subquery2 = True + if need_subquery1: + self = self.make_subquery() + if need_subquery2: + other = other.make_subquery() + result = QueryExpression() + result._connection = self.connection + result._support = self.support + other.support + # Store join info: (is_left, using_attrs) - using_attrs excludes hidden attributes + using_attrs = [n for n in self.heading.names if n in other.heading.names] + result._joins = self._joins + [(left, using_attrs)] + other._joins + result._heading = self.heading.join(other.heading, nullable_pk=nullable_pk) + result._restriction = AndList(self.restriction) + result._restriction.append(other.restriction) + result._original_heading = self.original_heading.join(other.original_heading, nullable_pk=nullable_pk) + assert len(result.support) == len(result._joins) + 1 + return result + + def extend(self, other, semantic_check=True): + """ + Extend self with attributes from other. + + The extend operation adds attributes from `other` to `self` while preserving + self's entity identity. It is semantically equivalent to `self.join(other, left=True)` + but expresses a clearer intent: extending an entity set with additional attributes + rather than combining two entity sets. + + Requirements: + self → other: Every attribute in other's primary key must exist in self. + This ensures: + - All rows of self are preserved (no filtering) + - Self's primary key remains the result's primary key (no NULL PKs) + - The operation is a true extension, not a Cartesian product + + Conceptual model: + Unlike a general join (Cartesian product restricted by matching attributes), + extend is closer to projection—it adds new attributes to existing entities + without changing which entities are in the result. + + Example: + # Session determines Trial (session_id is in Trial's PK) + # But Trial does NOT determine Session (trial_num not in Session) + + # Valid: extend trials with session info + Trial.extend(Session) # Adds 'date' from Session to each Trial + + # Invalid: Session cannot extend to Trial + Session.extend(Trial) # Error: trial_num not in Session + + :param other: QueryExpression whose attributes will extend self + :param semantic_check: If True (default), require homologous namesakes. + If False, match on all namesakes without lineage checking. + :return: Extended QueryExpression with self's PK and combined attributes + :raises DataJointError: If self does not determine other + """ + return self.join(other, semantic_check=semantic_check, left=True) + + def __add__(self, other): + """union e.g. ``q1 + q2``.""" + return Union.create(self, other) + + def proj(self, *attributes, **named_attributes): + """ + Projection operator. + + :param attributes: attributes to be included in the result. (The primary key is already included). + :param named_attributes: new attributes computed or renamed from existing attributes. + :return: the projected expression. + Primary key attributes cannot be excluded but may be renamed. + If the attribute list contains an Ellipsis ..., then all secondary attributes are included too + Prefixing an attribute name with a dash '-attr' removes the attribute from the list if present. + Keyword arguments can be used to rename attributes as in name='attr', duplicate them as in name='(attr)', or + self.proj(...) or self.proj(Ellipsis) -- include all attributes (return self) + self.proj() -- include only primary key + self.proj('attr1', 'attr2') -- include primary key and attributes attr1 and attr2 + self.proj(..., '-attr1', '-attr2') -- include all attributes except attr1 and attr2 + self.proj(name1='attr1') -- include primary key and 'attr1' renamed as name1 + self.proj('attr1', dup='(attr1)') -- include primary key and attribute attr1 twice, with the duplicate 'dup' + self.proj(k='abs(attr1)') adds the new attribute k with the value computed as an expression (SQL syntax) + from other attributes available before the projection. + Each attribute name can only be used once. + """ + named_attributes = {k: translate_attribute(v)[1] for k, v in named_attributes.items()} + # new attributes in parentheses are included again with the new name without removing original + duplication_pattern = re.compile(rf"^\s*\(\s*(?!{'|'.join(CONSTANT_LITERALS)})(?P[a-zA-Z_]\w*)\s*\)\s*$") + # attributes without parentheses renamed + rename_pattern = re.compile(rf"^\s*(?!{'|'.join(CONSTANT_LITERALS)})(?P[a-zA-Z_]\w*)\s*$") + replicate_map = { + k: m.group("name") for k, m in ((k, duplication_pattern.match(v)) for k, v in named_attributes.items()) if m + } + rename_map = {k: m.group("name") for k, m in ((k, rename_pattern.match(v)) for k, v in named_attributes.items()) if m} + compute_map = { + k: v for k, v in named_attributes.items() if not duplication_pattern.match(v) and not rename_pattern.match(v) + } + attributes = set(attributes) + # include primary key + attributes.update((k for k in self.primary_key if k not in rename_map.values())) + # include all secondary attributes with Ellipsis + if Ellipsis in attributes: + attributes.discard(Ellipsis) + attributes.update( + (a for a in self.heading.secondary_attributes if a not in attributes and a not in rename_map.values()) + ) + try: + raise DataJointError( + "%s is not a valid data type for an attribute in .proj" % next(a for a in attributes if not isinstance(a, str)) + ) + except StopIteration: + pass # normal case + # remove excluded attributes, specified as `-attr' + excluded = set(a for a in attributes if a.strip().startswith("-")) + attributes.difference_update(excluded) + excluded = set(a.lstrip("-").strip() for a in excluded) + attributes.difference_update(excluded) + try: + raise DataJointError( + "Cannot exclude primary key attribute %s", + next(a for a in excluded if a in self.primary_key), + ) + except StopIteration: + pass # all ok + # check that all attributes exist in heading + try: + raise DataJointError("Attribute `%s` not found." % next(a for a in attributes if a not in self.heading.names)) + except StopIteration: + pass # all ok + + # check that all mentioned names are present in heading + mentions = attributes.union(replicate_map.values()).union(rename_map.values()) + try: + raise DataJointError("Attribute '%s' not found." % next(a for a in mentions if not self.heading.names)) + except StopIteration: + pass # all ok + + # check that newly created attributes do not clash with any other selected attributes + try: + raise DataJointError( + "Attribute `%s` already exists" + % next(a for a in rename_map if a in attributes.union(compute_map).union(replicate_map)) + ) + except StopIteration: + pass # all ok + try: + raise DataJointError( + "Attribute `%s` already exists" + % next(a for a in compute_map if a in attributes.union(rename_map).union(replicate_map)) + ) + except StopIteration: + pass # all ok + try: + raise DataJointError( + "Attribute `%s` already exists" + % next(a for a in replicate_map if a in attributes.union(rename_map).union(compute_map)) + ) + except StopIteration: + pass # all ok + + # need a subquery if the projection remaps any remapped attributes + used = set(q for v in compute_map.values() for q in extract_column_names(v)) + used.update(rename_map.values()) + used.update(replicate_map.values()) + used.intersection_update(self.heading.names) + need_subquery = isinstance(self, Union) or any(self.heading[name].attribute_expression is not None for name in used) + if not need_subquery and self.restriction: + # need a subquery if the restriction applies to attributes that have been renamed + need_subquery = any(name in self.restriction_attributes for name in self.heading.new_attributes) + + result = self.make_subquery() if need_subquery else copy.copy(self) + result._original_heading = result.original_heading + result._heading = result.heading.select( + attributes, + rename_map=dict(**rename_map, **replicate_map), + compute_map=compute_map, + ) + return result + + def aggr(self, group, *attributes, exclude_nonmatching=False, **named_attributes): + """ + Aggregation/grouping operation, similar to proj but with computations over a grouped relation. + + By default, keeps all rows from self (like proj). Use exclude_nonmatching=True to + keep only rows that have matches in group. + + :param group: The query expression to be aggregated. + :param exclude_nonmatching: If True, exclude rows from self that have no matching + entries in group (INNER JOIN). Default False keeps all rows (LEFT JOIN). + :param named_attributes: computations of the form new_attribute="sql expression on attributes of group" + :return: The derived query expression + + Example:: + + # Count sessions per subject (keeps all subjects, even those with 0 sessions) + Subject.aggr(Session, n="count(*)") + + # Count sessions per subject (only subjects with at least one session) + Subject.aggr(Session, n="count(*)", exclude_nonmatching=True) + """ + if Ellipsis in attributes: + # expand ellipsis to include only attributes from the left table + attributes = set(attributes) + attributes.discard(Ellipsis) + attributes.update(self.heading.secondary_attributes) + keep_all_rows = not exclude_nonmatching + return Aggregation.create(self, group=group, keep_all_rows=keep_all_rows).proj(*attributes, **named_attributes) + + aggregate = aggr # alias for aggr + + # ---------- Fetch operators -------------------- + @property + def fetch(self): + """ + The fetch() method has been removed in DataJoint 2.0. + + Use the new explicit output methods instead: + - table.to_dicts() # list of dictionaries + - table.to_pandas() # pandas DataFrame + - table.to_arrays() # numpy structured array + - table.to_arrays('a', 'b') # tuple of numpy arrays + - table.keys() # primary keys as list[dict] + - table.to_polars() # polars DataFrame (requires pip install datajoint[polars]) + - table.to_arrow() # PyArrow Table (requires pip install datajoint[arrow]) + + For single-row fetch, use fetch1() which is unchanged. + + See migration guide: https://docs.datajoint.com/migration/fetch-api + """ + raise AttributeError( + "fetch() has been removed in DataJoint 2.0. " + "Use to_dicts(), to_pandas(), to_arrays(), or keys() instead. " + "See table.fetch.__doc__ for details." + ) + + def fetch1(self, *attrs, squeeze=False): + """ + Fetch exactly one row from the query result. + + If no attributes are specified, returns the result as a dict. + If attributes are specified, returns the corresponding values as a tuple. + + :param attrs: attribute names to fetch (if empty, fetch all as dict) + :param squeeze: if True, remove extra dimensions from arrays + :return: dict (no attrs) or tuple/value (with attrs) + :raises DataJointError: if not exactly one row in result + + Examples:: + + d = table.fetch1() # returns dict with all attributes + a, b = table.fetch1('a', 'b') # returns tuple of attribute values + value = table.fetch1('a') # returns single value + """ + heading = self.heading + + if not attrs: + # Fetch all attributes, return as dict + cursor = self.cursor(as_dict=True) + row = cursor.fetchone() + if not row or cursor.fetchone(): + raise DataJointError("fetch1 requires exactly one tuple in the input set.") + return {name: decode_attribute(heading[name], row[name], squeeze=squeeze) for name in heading.names} + else: + # Handle "KEY" specially - it means primary key columns + def is_key(attr): + return attr == "KEY" + + has_key = any(is_key(a) for a in attrs) + + if has_key and len(attrs) == 1: + # Just fetching KEY - return the primary key dict + keys = self.keys() + if len(keys) != 1: + raise DataJointError(f"fetch1 should only return one tuple. {len(keys)} tuples found") + return keys[0] + + # Fetch specific attributes, return as tuple + # Replace KEY with primary key columns for projection + proj_attrs = [] + for attr in attrs: + if is_key(attr): + proj_attrs.extend(self.primary_key) + else: + proj_attrs.append(attr) + + dicts = self.proj(*proj_attrs).to_dicts(squeeze=squeeze) + if len(dicts) != 1: + raise DataJointError(f"fetch1 should only return one tuple. {len(dicts)} tuples found") + row = dicts[0] + + # Build result values, handling KEY specially + values = [] + for attr in attrs: + if is_key(attr): + # Return dict of primary key columns + values.append({k: row[k] for k in self.primary_key}) + else: + values.append(row[attr]) + + return values[0] if len(attrs) == 1 else tuple(values) + + def _apply_top(self, order_by=None, limit=None, offset=None): + """Apply order_by, limit, offset if specified, return modified expression.""" + if order_by is not None or limit is not None or offset is not None: + return self.restrict(Top(limit, order_by, offset)) + return self + + def to_dicts(self, order_by=None, limit=None, offset=None, squeeze=False): + """ + Fetch all rows as a list of dictionaries. + + :param order_by: attribute(s) to order by, or "KEY"/"KEY DESC" + :param limit: maximum number of rows to return + :param offset: number of rows to skip + :param squeeze: if True, remove extra dimensions from arrays + :return: list of dictionaries, one per row + + For external storage types (attachments, filepaths), files are downloaded + to config["download_path"]. Use config.override() to change:: + + with dj.config.override(download_path="/data"): + data = table.to_dicts() + """ + expr = self._apply_top(order_by, limit, offset) + cursor = expr.cursor(as_dict=True) + heading = expr.heading + return [{name: decode_attribute(heading[name], row[name], squeeze) for name in heading.names} for row in cursor] + + def to_pandas(self, order_by=None, limit=None, offset=None, squeeze=False): + """ + Fetch all rows as a pandas DataFrame with primary key as index. + + :param order_by: attribute(s) to order by, or "KEY"/"KEY DESC" + :param limit: maximum number of rows to return + :param offset: number of rows to skip + :param squeeze: if True, remove extra dimensions from arrays + :return: pandas DataFrame with primary key columns as index + """ + dicts = self.to_dicts(order_by=order_by, limit=limit, offset=offset, squeeze=squeeze) + df = pandas.DataFrame(dicts) + if len(df) > 0 and self.primary_key: + df = df.set_index(self.primary_key) + return df + + def to_polars(self, order_by=None, limit=None, offset=None, squeeze=False): + """ + Fetch all rows as a polars DataFrame. + + Requires polars: pip install datajoint[polars] + + :param order_by: attribute(s) to order by, or "KEY"/"KEY DESC" + :param limit: maximum number of rows to return + :param offset: number of rows to skip + :param squeeze: if True, remove extra dimensions from arrays + :return: polars DataFrame + """ + try: + import polars + except ImportError: + raise ImportError("polars is required for to_polars(). Install with: pip install datajoint[polars]") + dicts = self.to_dicts(order_by=order_by, limit=limit, offset=offset, squeeze=squeeze) + return polars.DataFrame(dicts) + + def to_arrow(self, order_by=None, limit=None, offset=None, squeeze=False): + """ + Fetch all rows as a PyArrow Table. + + Requires pyarrow: pip install datajoint[arrow] + + :param order_by: attribute(s) to order by, or "KEY"/"KEY DESC" + :param limit: maximum number of rows to return + :param offset: number of rows to skip + :param squeeze: if True, remove extra dimensions from arrays + :return: pyarrow Table + """ + try: + import pyarrow + except ImportError: + raise ImportError("pyarrow is required for to_arrow(). Install with: pip install datajoint[arrow]") + dicts = self.to_dicts(order_by=order_by, limit=limit, offset=offset, squeeze=squeeze) + if not dicts: + return pyarrow.table({}) + return pyarrow.Table.from_pylist(dicts) + + def to_arrays(self, *attrs, include_key=False, order_by=None, limit=None, offset=None, squeeze=False): + """ + Fetch data as numpy arrays. + + If no attrs specified, returns a numpy structured array (recarray) of all columns. + If attrs specified, returns a tuple of numpy arrays (one per attribute). + + :param attrs: attribute names to fetch (if empty, fetch all) + :param include_key: if True and attrs specified, prepend primary keys as list of dicts + :param order_by: attribute(s) to order by, or "KEY"/"KEY DESC" + :param limit: maximum number of rows to return + :param offset: number of rows to skip + :param squeeze: if True, remove extra dimensions from arrays + :return: numpy recarray (no attrs) or tuple of arrays (with attrs). + With include_key=True: (keys, *arrays) where keys is list[dict] + + Examples:: + + # Fetch as structured array + data = table.to_arrays() + + # Fetch specific columns as separate arrays + a, b = table.to_arrays('a', 'b') + + # Fetch with primary keys for later restrictions + keys, a, b = table.to_arrays('a', 'b', include_key=True) + # keys = [{'id': 1}, {'id': 2}, ...] # same format as table.keys() + """ + from functools import partial + + expr = self._apply_top(order_by, limit, offset) + heading = expr.heading + + if attrs: + # Fetch specific attributes as tuple of arrays + if include_key: + fetch_attrs = list(expr.primary_key) + [a for a in attrs if a not in expr.primary_key] + else: + fetch_attrs = list(attrs) + + # Project to only needed columns + projected = expr.proj(*fetch_attrs) + dicts = projected.to_dicts(squeeze=squeeze) + + # Extract keys if requested + if include_key: + keys = [{k: d[k] for k in expr.primary_key} for d in dicts] + + # Extract arrays for requested attributes + result_arrays = [] + for attr in attrs: + values = [d[attr] for d in dicts] + # Try to create a homogeneous array, fall back to object array for variable-size data + try: + arr = np.array(values) + except ValueError: + # Variable-size data (e.g., arrays of different shapes) + arr = np.array(values, dtype=object) + result_arrays.append(arr) + + if include_key: + return (keys, *result_arrays) + return result_arrays[0] if len(attrs) == 1 else tuple(result_arrays) + else: + # Fetch all columns as structured array + get = partial(decode_attribute, squeeze=squeeze) + cursor = expr.cursor(as_dict=False) + rows = list(cursor.fetchall()) + + if not rows: + return np.array([], dtype=heading.as_dtype) + + # Build dtype, detecting blob types from first row + import numbers + + record_type = np.dtype( + [ + (name, type(value)) + if heading[name].is_blob and isinstance(value, numbers.Number) + else (name, heading.as_dtype[name]) + for value, name in zip(rows[0], heading.as_dtype.names) + ] + ) + + ret = np.array(rows, dtype=record_type) + # Decode blobs and codecs + for name in heading: + ret[name] = list(map(partial(get, heading[name]), ret[name])) + return ret + + def keys(self, order_by=None, limit=None, offset=None): + """ + Fetch primary key values as a list of dictionaries. + + :param order_by: attribute(s) to order by, or "KEY"/"KEY DESC" + :param limit: maximum number of rows to return + :param offset: number of rows to skip + :return: list of dictionaries containing only primary key columns + """ + return self.proj().to_dicts(order_by=order_by, limit=limit, offset=offset) + + def head(self, limit=25): + """ + Preview the first few entries from query expression. + + :param limit: number of entries (default 25) + :return: list of dictionaries + """ + return self.to_dicts(order_by="KEY", limit=limit) + + def tail(self, limit=25): + """ + Preview the last few entries from query expression. + + :param limit: number of entries (default 25) + :return: list of dictionaries + """ + return list(reversed(self.to_dicts(order_by="KEY DESC", limit=limit))) + + def __len__(self): + """:return: number of elements in the result set e.g. ``len(q1)``.""" + result = self.make_subquery() if self._top else copy.copy(self) + has_left_join = any(is_left for is_left, _ in result._joins) + return result.connection.query( + "SELECT {select_} FROM {from_}{where}".format( + select_=( + "count(*)" + if has_left_join + else "count(DISTINCT {fields})".format( + fields=result.heading.as_sql(result.primary_key, include_aliases=False) + ) + ), + from_=result.from_clause(), + where=result.where_clause(), + ) + ).fetchone()[0] + + def __bool__(self): + """ + :return: True if the result is not empty. Equivalent to len(self) > 0 but often + faster e.g. ``bool(q1)``. + """ + return bool( + self.connection.query( + "SELECT EXISTS(SELECT 1 FROM {from_}{where})".format(from_=self.from_clause(), where=self.where_clause()) + ).fetchone()[0] + ) + + def __contains__(self, item): + """ + returns True if the restriction in item matches any entries in self + e.g. ``restriction in q1``. + + :param item: any restriction + (item in query_expression) is equivalent to bool(query_expression & item) but may be + executed more efficiently. + """ + return bool(self & item) # May be optimized e.g. using an EXISTS query + + def __iter__(self): + """ + Lazy streaming iterator over rows as dictionaries. + + Yields one row at a time from a single database cursor, efficiently + streaming data without loading all rows into memory. + + :yields: dict for each row + """ + cursor = self.cursor(as_dict=True) + heading = self.heading + for row in cursor: + yield {name: decode_attribute(heading[name], row[name], squeeze=False) for name in heading.names} + + def cursor(self, as_dict=False): + """ + Execute the query and return a database cursor. + + :param as_dict: if True, rows are returned as dictionaries + :return: database query cursor + """ + sql = self.make_sql() + logger.debug(sql) + return self.connection.query(sql, as_dict=as_dict) + + def __repr__(self): + """ + returns the string representation of a QueryExpression object e.g. ``str(q1)``. + + :param self: A query expression + :type self: :class:`QueryExpression` + :rtype: str + """ + return super().__repr__() if config["loglevel"].lower() == "debug" else self.preview() + + def preview(self, limit=None, width=None): + """:return: a string of preview of the contents of the query.""" + return preview(self, limit, width) + + def _repr_html_(self): + """:return: HTML to display table in Jupyter notebook.""" + return repr_html(self) + + +class Aggregation(QueryExpression): + """ + Aggregation.create(arg, group, comp1='calc1', ..., compn='calcn') yields an entity set + with primary key from arg. + The computed arguments comp1, ..., compn use aggregation calculations on the attributes of + group or simple projections and calculations on the attributes of arg. + Aggregation is used QueryExpression.aggr and U.aggr. + Aggregation is a private class in DataJoint, not exposed to users. + """ + + _left_restrict = None # the pre-GROUP BY conditions for the WHERE clause + _subquery_alias_count = count() + + @classmethod + def create(cls, groupby, group, keep_all_rows=False): + """ + Create an aggregation expression. + + :param groupby: The expression to GROUP BY (determines the result's primary key) + :param group: The expression to aggregate over + :param keep_all_rows: If True, use left join to keep all rows from groupby + """ + if inspect.isclass(group) and issubclass(group, QueryExpression): + group = group() # instantiate if a class + assert isinstance(group, QueryExpression) + + # Aggregation requires group → groupby: every attribute in groupby's PK + # must be in group, so we can GROUP BY groupby's PK. + # Skip check for U (universal set) which doesn't have a heading until joined. + if not isinstance(groupby, U) and not group.heading.determines(groupby.heading): + missing = [attr for attr in groupby.heading.primary_key if attr not in group.heading.names] + raise DataJointError( + "Aggregation requires the group expression to contain all primary key " + f"attributes of the grouping expression. Missing attributes: {missing}." + ) + + if keep_all_rows and len(group.support) > 1 or group.heading.new_attributes: + group = group.make_subquery() # subquery if left joining a join + # When keep_all_rows=True, we use a left join which normally requires A → B. + # Aggregation has the opposite requirement (B → A). We bypass the left join + # constraint because GROUP BY resets the PK to groupby's PK (never NULL). + join = groupby.join(group, left=keep_all_rows, allow_nullable_pk=keep_all_rows) + result = cls() + result._connection = join.connection + result._heading = join.heading.set_primary_key(groupby.primary_key) + result._support = join.support + result._joins = join._joins + result._left_restrict = join.restriction # WHERE clause applied before GROUP BY + result._grouping_attributes = result.primary_key + + return result + + def where_clause(self): + return "" if not self._left_restrict else " WHERE (%s)" % ")AND(".join(str(s) for s in self._left_restrict) + + def make_sql(self, fields=None): + fields = self.heading.as_sql(fields or self.heading.names) + assert self._grouping_attributes or not self.restriction + distinct = set(self.heading.names) == set(self.primary_key) + return "SELECT {distinct}{fields} FROM {from_}{where}{group_by}{sorting}".format( + distinct="DISTINCT " if distinct else "", + fields=fields, + from_=self.from_clause(), + where=self.where_clause(), + group_by=( + "" + if not self.primary_key + else ( + " GROUP BY `%s`" % "`,`".join(self._grouping_attributes) + + ("" if not self.restriction else " HAVING (%s)" % ")AND(".join(self.restriction)) + ) + ), + sorting=self.sorting_clauses(), + ) + + def __len__(self): + return self.connection.query( + "SELECT count(1) FROM ({subquery}) `${alias:x}`".format( + subquery=self.make_sql(), alias=next(self._subquery_alias_count) + ) + ).fetchone()[0] + + def __bool__(self): + return bool(self.connection.query("SELECT EXISTS({sql})".format(sql=self.make_sql())).fetchone()[0]) + + +class Union(QueryExpression): + """ + Union is the private DataJoint class that implements the union operator. + """ + + __count = count() + + @classmethod + def create(cls, arg1, arg2): + if inspect.isclass(arg2) and issubclass(arg2, QueryExpression): + arg2 = arg2() # instantiate if a class + if not isinstance(arg2, QueryExpression): + raise DataJointError("A QueryExpression can only be unioned with another QueryExpression") + if arg1.connection != arg2.connection: + raise DataJointError("Cannot operate on QueryExpressions originating from different connections.") + if set(arg1.primary_key) != set(arg2.primary_key): + raise DataJointError("The operands of a union must share the same primary key.") + if set(arg1.heading.secondary_attributes) & set(arg2.heading.secondary_attributes): + raise DataJointError("The operands of a union must not share any secondary attributes.") + result = cls() + result._connection = arg1.connection + result._heading = arg1.heading.join(arg2.heading) + result._support = [arg1, arg2] + return result + + def make_sql(self): + arg1, arg2 = self._support + if not arg1.heading.secondary_attributes and not arg2.heading.secondary_attributes: + # no secondary attributes: use UNION DISTINCT + fields = arg1.primary_key + return "SELECT * FROM (({sql1}) UNION ({sql2})) as `_u{alias}{sorting}`".format( + sql1=(arg1.make_sql() if isinstance(arg1, Union) else arg1.make_sql(fields)), + sql2=(arg2.make_sql() if isinstance(arg2, Union) else arg2.make_sql(fields)), + alias=next(self.__count), + sorting=self.sorting_clauses(), + ) + # with secondary attributes, use union of left join with antijoin + fields = self.heading.names + sql1 = arg1.join(arg2, left=True).make_sql(fields) + sql2 = (arg2 - arg1).proj(..., **{k: "NULL" for k in arg1.heading.secondary_attributes}).make_sql(fields) + return "({sql1}) UNION ({sql2})".format(sql1=sql1, sql2=sql2) + + def from_clause(self): + """The union does not use a FROM clause""" + assert False + + def where_clause(self): + """The union does not use a WHERE clause""" + assert False + + def __len__(self): + return self.connection.query( + "SELECT count(1) FROM ({subquery}) `${alias:x}`".format( + subquery=self.make_sql(), + alias=next(QueryExpression._subquery_alias_count), + ) + ).fetchone()[0] + + def __bool__(self): + return bool(self.connection.query("SELECT EXISTS({sql})".format(sql=self.make_sql())).fetchone()[0]) + + +class U: + """ + dj.U objects are the universal sets representing all possible values of their attributes. + dj.U objects cannot be queried on their own but are useful for forming some queries. + dj.U('attr1', ..., 'attrn') represents the universal set with the primary key attributes attr1 ... attrn. + The universal set is the set of all possible combinations of values of the attributes. + Without any attributes, dj.U() represents the set with one element that has no attributes. + + Restriction: + + dj.U can be used to enumerate unique combinations of values of attributes from other expressions. + + The following expression yields all unique combinations of contrast and brightness found in the `stimulus` set: + + >>> dj.U('contrast', 'brightness') & stimulus + + Aggregation: + + In aggregation, dj.U is used for summary calculation over an entire set: + + The following expression yields one element with one attribute `s` containing the total number of elements in + query expression `expr`: + + >>> dj.U().aggr(expr, n='count(*)') + + The following expressions both yield one element containing the number `n` of distinct values of attribute `attr` in + query expression `expr`. + + >>> dj.U().aggr(expr, n='count(distinct attr)') + >>> dj.U().aggr(dj.U('attr').aggr(expr), 'n=count(*)') + + The following expression yields one element and one attribute `s` containing the sum of values of attribute `attr` + over entire result set of expression `expr`: + + >>> dj.U().aggr(expr, s='sum(attr)') + + The following expression yields the set of all unique combinations of attributes `attr1`, `attr2` and the number of + their occurrences in the result set of query expression `expr`. + + >>> dj.U(attr1,attr2).aggr(expr, n='count(*)') + + Joins: + + If expression `expr` has attributes 'attr1' and 'attr2', then expr * dj.U('attr1','attr2') yields the same result + as `expr` but `attr1` and `attr2` are promoted to the the primary key. This is useful for producing a join on + non-primary key attributes. + For example, if `attr` is in both expr1 and expr2 but not in their primary keys, then expr1 * expr2 will throw + an error because in most cases, it does not make sense to join on non-primary key attributes and users must first + rename `attr` in one of the operands. The expression dj.U('attr') * rel1 * rel2 overrides this constraint. + """ + + def __init__(self, *primary_key): + self._primary_key = primary_key + + @property + def primary_key(self): + return self._primary_key + + def __and__(self, other): + if inspect.isclass(other) and issubclass(other, QueryExpression): + other = other() # instantiate if a class + if not isinstance(other, QueryExpression): + raise DataJointError("Set U can only be restricted with a QueryExpression.") + result = copy.copy(other) + result._distinct = True + result._heading = result.heading.set_primary_key(self.primary_key) + result = result.proj() + return result + + def __mul__(self, other): + """The * operator with dj.U has been removed in DataJoint 2.0.""" + raise DataJointError( + "dj.U(...) * table is no longer supported in DataJoint 2.0. " + "This pattern is no longer necessary with the new semantic matching system." + ) + + def __sub__(self, other): + """Anti-restriction with dj.U produces an infinite set.""" + raise DataJointError( + "dj.U(...) - table produces an infinite set and is not supported. " + "Consider using a different approach for your query." + ) + + def aggr(self, group, **named_attributes): + """ + Aggregation of the type U('attr1','attr2').aggr(group, computation="QueryExpression") + has the primary key ('attr1','attr2') and performs aggregation computations for all matching elements of `group`. + + Note: exclude_nonmatching is always True for dj.U (cannot keep all rows from infinite set). + + :param group: The query expression to be aggregated. + :param named_attributes: computations of the form new_attribute="sql expression on attributes of group" + :return: The derived query expression + """ + if named_attributes.pop("exclude_nonmatching", True) is False: + raise DataJointError("Cannot set exclude_nonmatching=False when aggregating on a universal set.") + + if inspect.isclass(group) and issubclass(group, QueryExpression): + group = group() + if not isinstance(group, QueryExpression): + raise DataJointError("dj.U.aggr requires a QueryExpression as the group argument.") + + # Verify U's primary key attributes exist in group + missing = [attr for attr in self.primary_key if attr not in group.heading.names] + if missing: + raise DataJointError(f"Attributes {missing} not found in the group expression.") + + # Create Aggregation directly without join - just group by U's primary key + result = Aggregation() + result._connection = group.connection + result._heading = group.heading.set_primary_key(list(self.primary_key)) + result._support = group.support + result._joins = group._joins + result._left_restrict = group.restriction + result._grouping_attributes = list(self.primary_key) + + return result.proj(**named_attributes) + + aggregate = aggr # alias for aggr + + +def _flatten_attribute_list(primary_key, attrs): + """ + :param primary_key: list of attributes in primary key + :param attrs: list of attribute names, which may include "KEY", "KEY DESC" or "KEY ASC" + :return: generator of attributes where "KEY" is replaced with its component attributes + """ + for a in attrs: + if re.match(r"^\s*KEY(\s+[aA][Ss][Cc])?\s*$", a): + if primary_key: + yield from primary_key + elif re.match(r"^\s*KEY\s+[Dd][Ee][Ss][Cc]\s*$", a): + if primary_key: + yield from (q + " DESC" for q in primary_key) + else: + yield a + + +def _wrap_attributes(attr): + for entry in attr: # wrap attribute names in backquotes + yield re.sub(r"\b((?!asc|desc)\w+)\b", r"`\1`", entry, flags=re.IGNORECASE) diff --git a/src/datajoint/gc.py b/src/datajoint/gc.py new file mode 100644 index 000000000..7570e6f24 --- /dev/null +++ b/src/datajoint/gc.py @@ -0,0 +1,661 @@ +""" +Garbage collection for external storage. + +This module provides utilities to identify and remove orphaned content +from external storage. Content becomes orphaned when all database rows +referencing it are deleted. + +Supports two storage patterns: +- Content-addressed storage: , , + Stored at: _content/{hash[:2]}/{hash[2:4]}/{hash} + +- Path-addressed storage: + Stored at: {schema}/{table}/objects/{pk}/{field}_{token}/ + +Usage: + import datajoint as dj + + # Scan schemas and find orphaned content + stats = dj.gc.scan(schema1, schema2, store_name='mystore') + + # Remove orphaned content (dry_run=False to actually delete) + stats = dj.gc.collect(schema1, schema2, store_name='mystore', dry_run=True) +""" + +from __future__ import annotations + +import json +import logging +from typing import TYPE_CHECKING, Any + +from .content_registry import delete_content, get_store_backend +from .errors import DataJointError + +if TYPE_CHECKING: + from .schemas import Schema + +logger = logging.getLogger(__name__.split(".")[0]) + + +def _uses_content_storage(attr) -> bool: + """ + Check if an attribute uses content-addressed storage. + + This includes types that chain to for external storage: + - directly + - (chains to ) + - (chains to ) + + Parameters + ---------- + attr : Attribute + Attribute from table heading. + + Returns + ------- + bool + True if the attribute stores content hashes. + """ + if not attr.codec: + return False + + # Check if this type uses content storage + codec_name = getattr(attr.codec, "name", "") + store = getattr(attr, "store", None) + + # always uses content storage (external only) + if codec_name == "hash": + return True + + # and use content storage when external (has store) + if codec_name in ("blob", "attach") and store is not None: + return True + + return False + + +def _uses_object_storage(attr) -> bool: + """ + Check if an attribute uses path-addressed object storage. + + Parameters + ---------- + attr : Attribute + Attribute from table heading. + + Returns + ------- + bool + True if the attribute stores object paths. + """ + if not attr.codec: + return False + + codec_name = getattr(attr.codec, "name", "") + return codec_name == "object" + + +def _extract_content_refs(value: Any) -> list[tuple[str, str | None]]: + """ + Extract content references from a stored value. + + Parameters + ---------- + value : Any + The stored value (could be JSON string or dict). + + Returns + ------- + list[tuple[str, str | None]] + List of (content_hash, store_name) tuples. + """ + refs = [] + + if value is None: + return refs + + # Parse JSON if string + if isinstance(value, str): + try: + value = json.loads(value) + except (json.JSONDecodeError, TypeError): + return refs + + # Extract hash from dict + if isinstance(value, dict) and "hash" in value: + refs.append((value["hash"], value.get("store"))) + + return refs + + +def _extract_object_refs(value: Any) -> list[tuple[str, str | None]]: + """ + Extract object path references from a stored value. + + Parameters + ---------- + value : Any + The stored value (could be JSON string or dict). + + Returns + ------- + list[tuple[str, str | None]] + List of (path, store_name) tuples. + """ + refs = [] + + if value is None: + return refs + + # Parse JSON if string + if isinstance(value, str): + try: + value = json.loads(value) + except (json.JSONDecodeError, TypeError): + return refs + + # Extract path from dict + if isinstance(value, dict) and "path" in value: + refs.append((value["path"], value.get("store"))) + + return refs + + +def scan_references( + *schemas: "Schema", + store_name: str | None = None, + verbose: bool = False, +) -> set[str]: + """ + Scan schemas for content references. + + Examines all tables in the given schemas and extracts content hashes + from columns that use content-addressed storage (, , ). + + Parameters + ---------- + *schemas : Schema + Schema instances to scan. + store_name : str, optional + Only include references to this store (None = all stores). + verbose : bool, optional + Print progress information. + + Returns + ------- + set[str] + Set of content hashes that are referenced. + """ + referenced: set[str] = set() + + for schema in schemas: + if verbose: + logger.info(f"Scanning schema: {schema.database}") + + # Get all tables in schema + for table_name in schema.list_tables(): + try: + # Get table class + table = schema.get_table(table_name) + + # Check each attribute for content storage + for attr_name, attr in table.heading.attributes.items(): + if not _uses_content_storage(attr): + continue + + if verbose: + logger.info(f" Scanning {table_name}.{attr_name}") + + # Fetch all values for this attribute + # Use to_arrays to get attribute values + try: + values = table.to_arrays(attr_name) + for value in values: + for content_hash, ref_store in _extract_content_refs(value): + # Filter by store if specified + if store_name is None or ref_store == store_name: + referenced.add(content_hash) + except Exception as e: + logger.warning(f"Error scanning {table_name}.{attr_name}: {e}") + + except Exception as e: + logger.warning(f"Error accessing table {table_name}: {e}") + + return referenced + + +def scan_object_references( + *schemas: "Schema", + store_name: str | None = None, + verbose: bool = False, +) -> set[str]: + """ + Scan schemas for object path references. + + Examines all tables in the given schemas and extracts object paths + from columns that use path-addressed storage (). + + Parameters + ---------- + *schemas : Schema + Schema instances to scan. + store_name : str, optional + Only include references to this store (None = all stores). + verbose : bool, optional + Print progress information. + + Returns + ------- + set[str] + Set of object paths that are referenced. + """ + referenced: set[str] = set() + + for schema in schemas: + if verbose: + logger.info(f"Scanning schema for objects: {schema.database}") + + # Get all tables in schema + for table_name in schema.list_tables(): + try: + # Get table class + table = schema.get_table(table_name) + + # Check each attribute for object storage + for attr_name, attr in table.heading.attributes.items(): + if not _uses_object_storage(attr): + continue + + if verbose: + logger.info(f" Scanning {table_name}.{attr_name}") + + # Fetch all values for this attribute + try: + values = table.to_arrays(attr_name) + for value in values: + for path, ref_store in _extract_object_refs(value): + # Filter by store if specified + if store_name is None or ref_store == store_name: + referenced.add(path) + except Exception as e: + logger.warning(f"Error scanning {table_name}.{attr_name}: {e}") + + except Exception as e: + logger.warning(f"Error accessing table {table_name}: {e}") + + return referenced + + +def list_stored_content(store_name: str | None = None) -> dict[str, int]: + """ + List all content hashes in storage. + + Scans the _content/ directory in the specified store and returns + all content hashes found. + + Parameters + ---------- + store_name : str, optional + Store to scan (None = default store). + + Returns + ------- + dict[str, int] + Dict mapping content_hash to size in bytes. + """ + backend = get_store_backend(store_name) + stored: dict[str, int] = {} + + # Content is stored at _content/{hash[:2]}/{hash[2:4]}/{hash} + content_prefix = "_content/" + + try: + # List all files under _content/ + full_prefix = backend._full_path(content_prefix) + + for root, dirs, files in backend.fs.walk(full_prefix): + for filename in files: + # Skip manifest files + if filename.endswith(".manifest.json"): + continue + + # The filename is the full hash + content_hash = filename + + # Validate it looks like a hash (64 hex chars) + if len(content_hash) == 64 and all(c in "0123456789abcdef" for c in content_hash): + try: + file_path = f"{root}/{filename}" + size = backend.fs.size(file_path) + stored[content_hash] = size + except Exception: + stored[content_hash] = 0 + + except FileNotFoundError: + # No _content/ directory exists yet + pass + except Exception as e: + logger.warning(f"Error listing stored content: {e}") + + return stored + + +def list_stored_objects(store_name: str | None = None) -> dict[str, int]: + """ + List all object paths in storage. + + Scans for directories matching the object storage pattern: + {schema}/{table}/objects/{pk}/{field}_{token}/ + + Parameters + ---------- + store_name : str, optional + Store to scan (None = default store). + + Returns + ------- + dict[str, int] + Dict mapping object_path to size in bytes. + """ + backend = get_store_backend(store_name) + stored: dict[str, int] = {} + + try: + # Walk the storage looking for /objects/ directories + full_prefix = backend._full_path("") + + for root, dirs, files in backend.fs.walk(full_prefix): + # Skip _content directory + if "_content" in root: + continue + + # Look for "objects" directory pattern + if "/objects/" in root: + # This could be an object storage path + # Path pattern: {schema}/{table}/objects/{pk}/{field}_{token} + relative_path = root.replace(full_prefix, "").lstrip("/") + + # Calculate total size of this object directory + total_size = 0 + for file in files: + try: + file_path = f"{root}/{file}" + total_size += backend.fs.size(file_path) + except Exception: + pass + + # Only count directories with files (actual objects) + if total_size > 0 or files: + stored[relative_path] = total_size + + except FileNotFoundError: + pass + except Exception as e: + logger.warning(f"Error listing stored objects: {e}") + + return stored + + +def delete_object(path: str, store_name: str | None = None) -> bool: + """ + Delete an object directory from storage. + + Parameters + ---------- + path : str + Object path (relative to store root). + store_name : str, optional + Store name (None = default store). + + Returns + ------- + bool + True if deleted, False if not found. + """ + backend = get_store_backend(store_name) + + try: + full_path = backend._full_path(path) + if backend.fs.exists(full_path): + # Remove entire directory tree + backend.fs.rm(full_path, recursive=True) + logger.debug(f"Deleted object: {path}") + return True + except Exception as e: + logger.warning(f"Error deleting object {path}: {e}") + + return False + + +def scan( + *schemas: "Schema", + store_name: str | None = None, + verbose: bool = False, +) -> dict[str, Any]: + """ + Scan for orphaned content and objects without deleting. + + Scans both content-addressed storage (for , , ) + and path-addressed storage (for ). + + Parameters + ---------- + *schemas : Schema + Schema instances to scan. + store_name : str, optional + Store to check (None = default store). + verbose : bool, optional + Print progress information. + + Returns + ------- + dict[str, Any] + Dict with scan statistics: + + - content_referenced: Number of content items referenced in database + - content_stored: Number of content items in storage + - content_orphaned: Number of unreferenced content items + - content_orphaned_bytes: Total size of orphaned content + - orphaned_hashes: List of orphaned content hashes + - object_referenced: Number of objects referenced in database + - object_stored: Number of objects in storage + - object_orphaned: Number of unreferenced objects + - object_orphaned_bytes: Total size of orphaned objects + - orphaned_paths: List of orphaned object paths + """ + if not schemas: + raise DataJointError("At least one schema must be provided") + + # --- Content-addressed storage --- + content_referenced = scan_references(*schemas, store_name=store_name, verbose=verbose) + content_stored = list_stored_content(store_name) + orphaned_hashes = set(content_stored.keys()) - content_referenced + content_orphaned_bytes = sum(content_stored.get(h, 0) for h in orphaned_hashes) + + # --- Path-addressed storage (objects) --- + object_referenced = scan_object_references(*schemas, store_name=store_name, verbose=verbose) + object_stored = list_stored_objects(store_name) + orphaned_paths = set(object_stored.keys()) - object_referenced + object_orphaned_bytes = sum(object_stored.get(p, 0) for p in orphaned_paths) + + return { + # Content-addressed storage stats + "content_referenced": len(content_referenced), + "content_stored": len(content_stored), + "content_orphaned": len(orphaned_hashes), + "content_orphaned_bytes": content_orphaned_bytes, + "orphaned_hashes": sorted(orphaned_hashes), + # Path-addressed storage stats + "object_referenced": len(object_referenced), + "object_stored": len(object_stored), + "object_orphaned": len(orphaned_paths), + "object_orphaned_bytes": object_orphaned_bytes, + "orphaned_paths": sorted(orphaned_paths), + # Combined totals + "referenced": len(content_referenced) + len(object_referenced), + "stored": len(content_stored) + len(object_stored), + "orphaned": len(orphaned_hashes) + len(orphaned_paths), + "orphaned_bytes": content_orphaned_bytes + object_orphaned_bytes, + } + + +def collect( + *schemas: "Schema", + store_name: str | None = None, + dry_run: bool = True, + verbose: bool = False, +) -> dict[str, Any]: + """ + Remove orphaned content and objects from storage. + + Scans the given schemas for content and object references, then removes any + storage items that are not referenced. + + Parameters + ---------- + *schemas : Schema + Schema instances to scan. + store_name : str, optional + Store to clean (None = default store). + dry_run : bool, optional + If True, report what would be deleted without deleting. Default True. + verbose : bool, optional + Print progress information. + + Returns + ------- + dict[str, Any] + Dict with collection statistics: + + - referenced: Total items referenced in database + - stored: Total items in storage + - orphaned: Total unreferenced items + - content_deleted: Number of content items deleted + - object_deleted: Number of object items deleted + - deleted: Total items deleted (0 if dry_run) + - bytes_freed: Bytes freed (0 if dry_run) + - errors: Number of deletion errors + """ + # First scan to find orphaned content and objects + stats = scan(*schemas, store_name=store_name, verbose=verbose) + + content_deleted = 0 + object_deleted = 0 + bytes_freed = 0 + errors = 0 + + if not dry_run: + # Delete orphaned content (hash-addressed) + if stats["content_orphaned"] > 0: + content_stored = list_stored_content(store_name) + + for content_hash in stats["orphaned_hashes"]: + try: + size = content_stored.get(content_hash, 0) + if delete_content(content_hash, store_name): + content_deleted += 1 + bytes_freed += size + if verbose: + logger.info(f"Deleted content: {content_hash[:16]}... ({size} bytes)") + except Exception as e: + errors += 1 + logger.warning(f"Failed to delete content {content_hash[:16]}...: {e}") + + # Delete orphaned objects (path-addressed) + if stats["object_orphaned"] > 0: + object_stored = list_stored_objects(store_name) + + for path in stats["orphaned_paths"]: + try: + size = object_stored.get(path, 0) + if delete_object(path, store_name): + object_deleted += 1 + bytes_freed += size + if verbose: + logger.info(f"Deleted object: {path} ({size} bytes)") + except Exception as e: + errors += 1 + logger.warning(f"Failed to delete object {path}: {e}") + + return { + "referenced": stats["referenced"], + "stored": stats["stored"], + "orphaned": stats["orphaned"], + "content_deleted": content_deleted, + "object_deleted": object_deleted, + "deleted": content_deleted + object_deleted, + "bytes_freed": bytes_freed, + "errors": errors, + "dry_run": dry_run, + # Include detailed stats + "content_orphaned": stats["content_orphaned"], + "object_orphaned": stats["object_orphaned"], + } + + +def format_stats(stats: dict[str, Any]) -> str: + """ + Format GC statistics as a human-readable string. + + Parameters + ---------- + stats : dict[str, Any] + Statistics dict from scan() or collect(). + + Returns + ------- + str + Formatted string. + """ + lines = ["External Storage Statistics:"] + + # Show content-addressed storage stats if present + if "content_referenced" in stats: + lines.append("") + lines.append("Content-Addressed Storage (, , ):") + lines.append(f" Referenced: {stats['content_referenced']}") + lines.append(f" Stored: {stats['content_stored']}") + lines.append(f" Orphaned: {stats['content_orphaned']}") + if "content_orphaned_bytes" in stats: + size_mb = stats["content_orphaned_bytes"] / (1024 * 1024) + lines.append(f" Orphaned size: {size_mb:.2f} MB") + + # Show path-addressed storage stats if present + if "object_referenced" in stats: + lines.append("") + lines.append("Path-Addressed Storage ():") + lines.append(f" Referenced: {stats['object_referenced']}") + lines.append(f" Stored: {stats['object_stored']}") + lines.append(f" Orphaned: {stats['object_orphaned']}") + if "object_orphaned_bytes" in stats: + size_mb = stats["object_orphaned_bytes"] / (1024 * 1024) + lines.append(f" Orphaned size: {size_mb:.2f} MB") + + # Show totals + lines.append("") + lines.append("Totals:") + lines.append(f" Referenced in database: {stats['referenced']}") + lines.append(f" Stored in backend: {stats['stored']}") + lines.append(f" Orphaned (unreferenced): {stats['orphaned']}") + + if "orphaned_bytes" in stats: + size_mb = stats["orphaned_bytes"] / (1024 * 1024) + lines.append(f" Orphaned size: {size_mb:.2f} MB") + + # Show deletion results if this is from collect() + if "deleted" in stats: + lines.append("") + if stats.get("dry_run", True): + lines.append(" [DRY RUN - no changes made]") + else: + lines.append(f" Deleted: {stats['deleted']}") + if "content_deleted" in stats: + lines.append(f" Content: {stats['content_deleted']}") + if "object_deleted" in stats: + lines.append(f" Objects: {stats['object_deleted']}") + freed_mb = stats["bytes_freed"] / (1024 * 1024) + lines.append(f" Bytes freed: {freed_mb:.2f} MB") + if stats.get("errors", 0) > 0: + lines.append(f" Errors: {stats['errors']}") + + return "\n".join(lines) diff --git a/src/datajoint/hash.py b/src/datajoint/hash.py new file mode 100644 index 000000000..2a58e9bf4 --- /dev/null +++ b/src/datajoint/hash.py @@ -0,0 +1,29 @@ +from __future__ import annotations + +import hashlib +import uuid +from typing import Any + + +def key_hash(mapping: dict[str, Any]) -> str: + """ + 32-byte hash of the mapping's key values sorted by the key name. + This is often used to convert a long primary key value into a shorter hash. + """ + hashed = hashlib.md5() + for k, v in sorted(mapping.items()): + hashed.update(str(v).encode()) + return hashed.hexdigest() + + +def uuid_from_buffer(buffer: bytes = b"", *, init_string: str = "") -> uuid.UUID: + """ + Compute MD5 hash of buffer data, returned as UUID. + + :param buffer: bytes to hash + :param init_string: string to initialize the checksum (for namespacing) + :return: UUID based on MD5 digest + """ + hashed = hashlib.md5(init_string.encode()) + hashed.update(buffer) + return uuid.UUID(bytes=hashed.digest()) diff --git a/src/datajoint/heading.py b/src/datajoint/heading.py new file mode 100644 index 000000000..96383170b --- /dev/null +++ b/src/datajoint/heading.py @@ -0,0 +1,640 @@ +""" +Heading management for DataJoint tables. + +This module provides the Heading class for managing table column metadata, +including attribute types, constraints, and lineage information. +""" + +from __future__ import annotations + +import logging +import re +from collections import defaultdict, namedtuple +from itertools import chain +from typing import TYPE_CHECKING, Any + +import numpy as np + +from .codecs import lookup_codec +from .codecs import Codec +from .declare import ( + CORE_TYPE_NAMES, + SPECIAL_TYPES, + TYPE_PATTERN, +) +from .errors import DataJointError +from .lineage import get_table_lineages, lineage_table_exists + +if TYPE_CHECKING: + pass + + +class _MissingType(Codec, register=False): + """Placeholder for missing/unregistered codecs. Raises error on use.""" + + def __init__(self, codec_name: str): + self._codec_name = codec_name + + @property + def name(self) -> str: + return self._codec_name + + def get_dtype(self, is_external: bool) -> str: + raise DataJointError( + f"Codec <{self._codec_name}> is not registered. Define a Codec subclass with name='{self._codec_name}'." + ) + + def encode(self, value, *, key=None, store_name=None): + raise DataJointError( + f"Codec <{self._codec_name}> is not registered. Define a Codec subclass with name='{self._codec_name}'." + ) + + def decode(self, stored, *, key=None): + raise DataJointError( + f"Codec <{self._codec_name}> is not registered. Define a Codec subclass with name='{self._codec_name}'." + ) + + +logger = logging.getLogger(__name__.split(".")[0]) + +default_attribute_properties = dict( # these default values are set in computed attributes + name=None, + type="expression", + original_type=None, # For core types, stores the alias (e.g., "uuid") while type has db type ("binary(16)") + in_key=False, + nullable=False, + default=None, + comment="calculated attribute", + autoincrement=False, + numeric=None, + string=None, + uuid=False, + json=None, + is_blob=False, + is_hidden=False, + codec=None, + store=None, + unsupported=False, + attribute_expression=None, + dtype=object, + lineage=None, # Origin of attribute, e.g. "schema.table.attr" for semantic matching +) + + +class Attribute(namedtuple("_Attribute", default_attribute_properties)): + """ + Properties of a table column (attribute). + + Attributes + ---------- + name : str + Attribute name. + type : str + Database type string. + in_key : bool + True if part of primary key. + nullable : bool + True if NULL values allowed. + default : any + Default value. + comment : str + Attribute comment/description. + codec : Codec + Codec for encoding/decoding values. + lineage : str + Origin of attribute for semantic matching. + """ + + def todict(self) -> dict[str, Any]: + """Convert to dictionary.""" + return dict((name, self[i]) for i, name in enumerate(self._fields)) + + @property + def sql_type(self) -> str: + """ + Return the SQL datatype string. + + Returns + ------- + str + Database type (usually same as self.type). + """ + # UUID is now a core type alias - already resolved to binary(16) + return self.type + + @property + def sql_comment(self) -> str: + """ + Return the full SQL comment including type markers. + + Returns + ------- + str + Comment with optional ``:uuid:`` prefix. + """ + # UUID info is stored in the comment for reconstruction + return (":uuid:" if self.uuid else "") + self.comment + + @property + def sql(self) -> str: + """ + Generate SQL clause for this attribute in CREATE TABLE. + + Used for declaring foreign keys in referencing tables. + Default values are not included. + + Returns + ------- + str + SQL attribute declaration. + """ + return '`{name}` {type} NOT NULL COMMENT "{comment}"'.format( + name=self.name, type=self.sql_type, comment=self.sql_comment + ) + + @property + def original_name(self) -> str: + """ + Return the original attribute name before any renaming. + + Returns + ------- + str + Original name from attribute_expression or current name. + """ + if self.attribute_expression is None: + return self.name + assert self.attribute_expression.startswith("`") + return self.attribute_expression.strip("`") + + +class Heading: + """ + Table heading containing column metadata. + + Manages attribute information including names, types, constraints, + and lineage for semantic matching. + + Parameters + ---------- + attribute_specs : list, optional + List of attribute specification dictionaries. + table_info : dict, optional + Database table information for lazy loading. + lineage_available : bool, optional + Whether lineage information is available. Default True. + + Attributes + ---------- + attributes : dict + Mapping of attribute names to Attribute objects. + """ + + def __init__( + self, + attribute_specs: list[dict] | None = None, + table_info: dict | None = None, + lineage_available: bool = True, + ) -> None: + self.indexes = None + self.table_info = table_info + self._table_status = None + self._lineage_available = lineage_available + self._attributes = None if attribute_specs is None else dict((q["name"], Attribute(**q)) for q in attribute_specs) + + @property + def lineage_available(self) -> bool: + """Whether lineage tracking is available for this heading's schema.""" + return self._lineage_available + + def __len__(self) -> int: + return 0 if self.attributes is None else len(self.attributes) + + @property + def table_status(self) -> dict | None: + """Table status information from database.""" + if self.table_info is None: + return None + if self._table_status is None: + self._init_from_database() + return self._table_status + + @property + def attributes(self) -> dict[str, Attribute]: + """ + Mapping of attribute names to Attribute objects. + + Excludes hidden attributes (names starting with ``_``). + """ + if self._attributes is None: + self._init_from_database() # lazy loading from database + return {k: v for k, v in self._attributes.items() if not v.is_hidden} + + @property + def names(self) -> list[str]: + """List of visible attribute names.""" + return [k for k in self.attributes] + + @property + def primary_key(self) -> list[str]: + """List of primary key attribute names.""" + return [k for k, v in self.attributes.items() if v.in_key] + + @property + def secondary_attributes(self) -> list[str]: + """List of non-primary-key attribute names.""" + return [k for k, v in self.attributes.items() if not v.in_key] + + def determines(self, other: Heading) -> bool: + """ + Check if self determines other (self → other). + + A determines B iff every attribute in PK(B) is in A. This means + knowing A's primary key is sufficient to determine B's primary key + through functional dependencies. + + Parameters + ---------- + other : Heading + Another Heading object. + + Returns + ------- + bool + True if self determines other. + """ + self_attrs = set(self.names) + return all(attr in self_attrs for attr in other.primary_key) + + @property + def blobs(self) -> list[str]: + """List of blob attribute names.""" + return [k for k, v in self.attributes.items() if v.is_blob] + + @property + def non_blobs(self) -> list[str]: + """Attributes that are not blobs or JSON.""" + return [k for k, v in self.attributes.items() if not (v.is_blob or v.json)] + + @property + def new_attributes(self) -> list[str]: + """Attributes with computed expressions (projections).""" + return [k for k, v in self.attributes.items() if v.attribute_expression is not None] + + def __getitem__(self, name: str) -> Attribute: + """Get attribute by name.""" + return self.attributes[name] + + def __repr__(self) -> str: + """Return heading in DataJoint declaration format.""" + in_key = True + ret = "" + if self._table_status is not None: + ret += "# " + self.table_status["comment"] + "\n" + for v in self.attributes.values(): + if in_key and not v.in_key: + ret += "---\n" + in_key = False + ret += "%-20s : %-28s # %s\n" % ( + v.name if v.default is None else "%s=%s" % (v.name, v.default), + "%s%s" % (v.type, "auto_increment" if v.autoincrement else ""), + v.comment, + ) + return ret + + @property + def has_autoincrement(self) -> bool: + """Check if any attribute has auto_increment.""" + return any(e.autoincrement for e in self.attributes.values()) + + @property + def as_dtype(self) -> np.dtype: + """ + Return heading as a numpy dtype. + + Returns + ------- + numpy.dtype + Structured dtype for creating numpy arrays. + """ + return np.dtype(dict(names=self.names, formats=[v.dtype for v in self.attributes.values()])) + + def as_sql(self, fields: list[str], include_aliases: bool = True) -> str: + """ + Generate SQL SELECT clause for specified fields. + + Parameters + ---------- + fields : list[str] + Attribute names to include. + include_aliases : bool, optional + Include AS clauses for computed attributes. Default True. + + Returns + ------- + str + Comma-separated SQL field list. + """ + return ",".join( + ( + "`%s`" % name + if self.attributes[name].attribute_expression is None + else self.attributes[name].attribute_expression + (" as `%s`" % name if include_aliases else "") + ) + for name in fields + ) + + def __iter__(self): + return iter(self.attributes) + + def _init_from_database(self) -> None: + """Initialize heading from an existing database table.""" + conn, database, table_name, context = (self.table_info[k] for k in ("conn", "database", "table_name", "context")) + info = conn.query( + 'SHOW TABLE STATUS FROM `{database}` WHERE name="{table_name}"'.format(table_name=table_name, database=database), + as_dict=True, + ).fetchone() + if info is None: + raise DataJointError( + "The table `{database}`.`{table_name}` is not defined.".format(table_name=table_name, database=database) + ) + self._table_status = {k.lower(): v for k, v in info.items()} + cur = conn.query( + "SHOW FULL COLUMNS FROM `{table_name}` IN `{database}`".format(table_name=table_name, database=database), + as_dict=True, + ) + + attributes = cur.fetchall() + + rename_map = { + "Field": "name", + "Type": "type", + "Null": "nullable", + "Default": "default", + "Key": "in_key", + "Comment": "comment", + } + + fields_to_drop = ("Privileges", "Collation") + + # rename and drop attributes + attributes = [ + {rename_map[k] if k in rename_map else k: v for k, v in x.items() if k not in fields_to_drop} for x in attributes + ] + numeric_types = { + ("float", False): np.float64, + ("float", True): np.float64, + ("double", False): np.float64, + ("double", True): np.float64, + ("tinyint", False): np.int64, + ("tinyint", True): np.int64, + ("smallint", False): np.int64, + ("smallint", True): np.int64, + ("mediumint", False): np.int64, + ("mediumint", True): np.int64, + ("int", False): np.int64, + ("int", True): np.int64, + ("bigint", False): np.int64, + ("bigint", True): np.uint64, + } + + sql_literals = ["CURRENT_TIMESTAMP"] + + # additional attribute properties + for attr in attributes: + attr.update( + in_key=(attr["in_key"] == "PRI"), + nullable=attr["nullable"] == "YES", + autoincrement=bool(re.search(r"auto_increment", attr["Extra"], flags=re.I)), + numeric=any(TYPE_PATTERN[t].match(attr["type"]) for t in ("DECIMAL", "INTEGER", "FLOAT")), + string=any(TYPE_PATTERN[t].match(attr["type"]) for t in ("ENUM", "TEMPORAL", "STRING")), + is_blob=any(TYPE_PATTERN[t].match(attr["type"]) for t in ("BYTES", "NATIVE_BLOB")), + uuid=False, + json=bool(TYPE_PATTERN["JSON"].match(attr["type"])), + codec=None, + store=None, + attribute_expression=None, + is_hidden=attr["name"].startswith("_"), + original_type=None, # May be set later for core type aliases + ) + + if any(TYPE_PATTERN[t].match(attr["type"]) for t in ("INTEGER", "FLOAT")): + attr["type"] = re.sub(r"\(\d+\)", "", attr["type"], count=1) # strip size off integers and floats + attr["unsupported"] = not any((attr["is_blob"], attr["numeric"], attr["numeric"])) + attr.pop("Extra") + + # process custom DataJoint types stored in comment + special = re.match(r":(?P[^:]+):(?P.*)", attr["comment"]) + if special: + special = special.groupdict() + attr["comment"] = special["comment"] # Always update the comment + # Only update the type for adapted types (angle brackets) + # Core types (uuid, float32, etc.) keep the database type for SQL + if special["type"].startswith("<"): + attr["type"] = special["type"] + else: + # Store the original type name for display but keep db_type for SQL + attr["original_type"] = special["type"] + + # process Codecs (types in angle brackets) + if special and TYPE_PATTERN["CODEC"].match(attr["type"]): + # Context can be None for built-in types that are globally registered + codec_spec = special["type"] + try: + codec_instance, codec_store = lookup_codec(codec_spec) + attr["codec"] = codec_instance + if codec_store is not None: + attr["store"] = codec_store + except DataJointError: + # if no codec, then delay the error until the first invocation + attr["codec"] = _MissingType(codec_spec) + else: + # Determine if external storage based on store presence + is_external = attr.get("store") is not None + attr["type"] = attr["codec"].get_dtype(is_external=is_external) + if not any(r.match(attr["type"]) for r in TYPE_PATTERN.values()): + raise DataJointError(f"Invalid dtype '{attr['type']}' in codec <{codec_spec}>.") + # Update is_blob based on resolved dtype (check both BYTES and NATIVE_BLOB patterns) + attr["is_blob"] = any(TYPE_PATTERN[t].match(attr["type"]) for t in ("BYTES", "NATIVE_BLOB")) + + # Handle core type aliases (uuid, float32, etc.) + if special: + # Check original_type for core type detection (not attr["type"] which is now db type) + original_type = attr["original_type"] or attr["type"] + try: + category = next(c for c in SPECIAL_TYPES if TYPE_PATTERN[c].match(original_type)) + except StopIteration: + if original_type.startswith("external"): + raise DataJointError( + f"Legacy datatype `{original_type}`. Migrate your external stores to datajoint 0.12: " + "https://docs.datajoint.io/python/admin/5-blob-config.html#migration-between-datajoint-v0-11-and-v0-12" + ) + # Not a special type - that's fine, could be native passthrough + category = None + + if category == "UUID": + attr["uuid"] = True + elif category in CORE_TYPE_NAMES: + # Core type alias - already resolved in DB + pass + + # Check primary key constraints + if attr["in_key"] and (attr["is_blob"] or attr["json"]): + raise DataJointError("Blob or JSON attributes are not allowed in the primary key") + + if attr["string"] and attr["default"] is not None and attr["default"] not in sql_literals: + attr["default"] = '"%s"' % attr["default"] + + if attr["nullable"]: # nullable fields always default to null + attr["default"] = "null" + + # fill out dtype. All floats and non-nullable integers are turned into specific dtypes + attr["dtype"] = object + if attr["numeric"] and not attr["codec"]: + is_integer = TYPE_PATTERN["INTEGER"].match(attr["type"]) + is_float = TYPE_PATTERN["FLOAT"].match(attr["type"]) + if is_integer and not attr["nullable"] or is_float: + is_unsigned = bool(re.match("sunsigned", attr["type"], flags=re.I)) + t = re.sub(r"\(.*\)", "", attr["type"]) # remove parentheses + t = re.sub(r" unsigned$", "", t) # remove unsigned + assert (t, is_unsigned) in numeric_types, "dtype not found for type %s" % t + attr["dtype"] = numeric_types[(t, is_unsigned)] + + if attr["codec"]: + # restore codec type name for display + attr["type"] = codec_spec + + # Load lineage information for semantic matching from ~lineage table + self._lineage_available = lineage_table_exists(conn, database) + if self._lineage_available: + lineages = get_table_lineages(conn, database, table_name) + for attr in attributes: + attr["lineage"] = lineages.get(attr["name"]) + else: + for attr in attributes: + attr["lineage"] = None + + self._attributes = dict(((q["name"], Attribute(**q)) for q in attributes)) + + # Read and tabulate secondary indexes + keys = defaultdict(dict) + for item in conn.query( + "SHOW KEYS FROM `{db}`.`{tab}`".format(db=database, tab=table_name), + as_dict=True, + ): + if item["Key_name"] != "PRIMARY": + keys[item["Key_name"]][item["Seq_in_index"]] = dict( + column=item["Column_name"] or f"({item['Expression']})".replace(r"\'", "'"), + unique=(item["Non_unique"] == 0), + nullable=item["Null"].lower() == "yes", + ) + self.indexes = { + tuple(item[k]["column"] for k in sorted(item.keys())): dict( + unique=item[1]["unique"], + nullable=any(v["nullable"] for v in item.values()), + ) + for item in keys.values() + } + + def select(self, select_list, rename_map=None, compute_map=None): + """ + derive a new heading by selecting, renaming, or computing attributes. + In relational algebra these operators are known as project, rename, and extend. + + :param select_list: the full list of existing attributes to include + :param rename_map: dictionary of renamed attributes: keys=new names, values=old names + :param compute_map: a direction of computed attributes + This low-level method performs no error checking. + """ + rename_map = rename_map or {} + compute_map = compute_map or {} + copy_attrs = list() + for name in self.attributes: + if name in select_list: + copy_attrs.append(self.attributes[name].todict()) + copy_attrs.extend( + ( + dict( + self.attributes[old_name].todict(), + name=new_name, + attribute_expression="`%s`" % old_name, + ) + for new_name, old_name in rename_map.items() + if old_name == name + ) + ) + compute_attrs = ( + dict(default_attribute_properties, name=new_name, attribute_expression=expr) + for new_name, expr in compute_map.items() + ) + return Heading(chain(copy_attrs, compute_attrs), lineage_available=self._lineage_available) + + def _join_dependent(self, dependent): + """Build attribute list when self → dependent: PK = PK(self), self's attrs first.""" + return ( + [self.attributes[name].todict() for name in self.primary_key] + + [self.attributes[name].todict() for name in self.secondary_attributes] + + [dependent.attributes[name].todict() for name in dependent.names if name not in self.attributes] + ) + + def join(self, other, nullable_pk=False): + """ + Join two headings into a new one. + + The primary key of the result depends on functional dependencies: + - A → B: PK = PK(A), A's attributes first + - B → A (not A → B): PK = PK(B), B's attributes first + - Both: PK = PK(A), left operand takes precedence + - Neither: PK = PK(A) ∪ PK(B), A's PK first then B's new PK attrs + + :param nullable_pk: If True, skip PK optimization and use combined PK from both + operands. Used for left joins that bypass the A → B constraint, where the + right operand's PK attributes could be NULL. + + It assumes that self and other are headings that share no common dependent attributes. + """ + if nullable_pk: + a_determines_b = b_determines_a = False + else: + a_determines_b = self.determines(other) + b_determines_a = other.determines(self) + + if a_determines_b: + attrs = self._join_dependent(other) + elif b_determines_a: + attrs = other._join_dependent(self) + else: + # Neither direction: PK = PK(A) ∪ PK(B) + self_pk_set = set(self.primary_key) + other_pk_set = set(other.primary_key) + attrs = ( + [self.attributes[name].todict() for name in self.primary_key] + + [dict(other.attributes[name].todict(), in_key=True) for name in other.primary_key if name not in self_pk_set] + + [self.attributes[name].todict() for name in self.secondary_attributes if name not in other_pk_set] + + [other.attributes[name].todict() for name in other.secondary_attributes if name not in self_pk_set] + ) + + return Heading(attrs, lineage_available=self._lineage_available and other._lineage_available) + + def set_primary_key(self, primary_key): + """ + Create a new heading with the specified primary key. + This low-level method performs no error checking. + """ + return Heading( + chain( + (dict(self.attributes[name].todict(), in_key=True) for name in primary_key), + (dict(self.attributes[name].todict(), in_key=False) for name in self.names if name not in primary_key), + ), + lineage_available=self._lineage_available, + ) + + def make_subquery_heading(self): + """ + Create a new heading with removed attribute sql_expressions. + Used by subqueries, which resolve the sql_expressions. + """ + return Heading( + (dict(v.todict(), attribute_expression=None) for v in self.attributes.values()), + lineage_available=self._lineage_available, + ) diff --git a/src/datajoint/jobs.py b/src/datajoint/jobs.py new file mode 100644 index 000000000..70c24f354 --- /dev/null +++ b/src/datajoint/jobs.py @@ -0,0 +1,594 @@ +""" +Job queue management for AutoPopulate 2.0. + +Each auto-populated table (Computed/Imported) has an associated jobs table +with the naming pattern ``~~table_name``. The jobs table tracks job status, +priority, scheduling, and error information. +""" + +from __future__ import annotations + +import logging +import os +import platform +import subprocess + +from .condition import AndList +from .errors import DataJointError, DuplicateError +from .heading import Heading +from .table import Table + +ERROR_MESSAGE_LENGTH = 2047 +TRUNCATION_APPENDIX = "...truncated" + +logger = logging.getLogger(__name__.split(".")[0]) + + +def _get_job_version() -> str: + """ + Get version string based on config settings. + + Returns + ------- + str + Version string, or empty string if version tracking disabled. + """ + from .settings import config + + method = config.jobs.version_method + if method is None or method == "none": + return "" + elif method == "git": + try: + result = subprocess.run( + ["git", "rev-parse", "--short", "HEAD"], + capture_output=True, + text=True, + timeout=5, + ) + return result.stdout.strip() if result.returncode == 0 else "" + except Exception: + return "" + return "" + + +class Job(Table): + """ + Per-table job queue for AutoPopulate 2.0. + + Each auto-populated table (Computed/Imported) has an associated job table + with the naming pattern ``~~table_name``. The job table tracks job status, + priority, scheduling, and error information. + + Parameters + ---------- + target_table : Table + The Computed/Imported table instance this jobs table manages. + + Attributes + ---------- + target : Table + The auto-populated table this jobs table manages. + pending : QueryExpression + Query for jobs with ``status='pending'``. + reserved : QueryExpression + Query for jobs with ``status='reserved'``. + errors : QueryExpression + Query for jobs with ``status='error'``. + completed : QueryExpression + Query for jobs with ``status='success'``. + ignored : QueryExpression + Query for jobs with ``status='ignore'``. + + Examples + -------- + >>> MyTable.jobs.refresh() # Add new jobs, clean up stale ones + >>> MyTable.jobs.pending # Query pending jobs + >>> MyTable.jobs.errors # Query failed jobs + """ + + def __init__(self, target_table: Table) -> None: + """ + Initialize jobs table for an auto-populated table. + + Parameters + ---------- + target_table : Table + The Computed/Imported table instance this jobs table manages. + """ + self._target = target_table + self._connection = target_table.connection + self.database = target_table.database + + # Compute table name: ~~base_name + target_name = target_table.table_name + base_name = target_name.lstrip("_") + self._table_name = f"~~{base_name}" + + # Generate definition from target's FK-derived primary key + self._definition = self._generate_definition() + + # Initialize heading and support + self._heading = Heading( + table_info=dict( + conn=self._connection, + database=self.database, + table_name=self._table_name, + context=None, + ) + ) + self._support = [self.full_table_name] + + @property + def table_name(self): + return self._table_name + + @property + def definition(self): + return self._definition + + @property + def target(self): + """The auto-populated table this jobs table manages.""" + return self._target + + def _generate_definition(self) -> str: + """ + Generate jobs table definition from target's FK-derived primary key. + + Returns + ------- + str + DataJoint table definition string. + """ + pk_attrs = self._get_fk_derived_pk_attrs() + + if not pk_attrs: + raise DataJointError( + f"Cannot create jobs table for {self._target.full_table_name}: no FK-derived primary key attributes found." + ) + + pk_lines = "\n ".join(f"{name} : {dtype}" for name, dtype in pk_attrs) + + return f""" + # Job queue for {self._target.full_table_name} + {pk_lines} + --- + status : enum('pending', 'reserved', 'success', 'error', 'ignore') + priority : uint8 + created_time=CURRENT_TIMESTAMP(3) : datetime(3) + scheduled_time=CURRENT_TIMESTAMP(3) : datetime(3) + reserved_time=null : datetime(3) + completed_time=null : datetime(3) + duration=null : float64 + error_message="" : varchar({ERROR_MESSAGE_LENGTH}) + error_stack=null : + user="" : varchar(255) + host="" : varchar(255) + pid=0 : uint32 + connection_id=0 : uint64 + version="" : varchar(64) + """ + + def _get_fk_derived_pk_attrs(self) -> list[tuple[str, str]]: + """ + Extract FK-derived primary key attributes using the dependency graph. + + FK-derived attributes are those that come from primary FK references. + Uses connection.dependencies to identify FK relationships. + + Returns + ------- + list[tuple[str, str]] + List of (attribute_name, datatype) tuples in target PK order. + """ + heading = self._target.heading + target_pk = heading.primary_key + + # Load dependency graph if not already loaded + self._connection.dependencies.load() + + # Get primary FK parents and collect their attribute mappings + # parents(primary=True) returns FKs that contribute to primary key + parents = self._target.parents(primary=True, foreign_key_info=True) + fk_derived_attrs = set() + for _parent_name, props in parents: + # attr_map: child_attr -> parent_attr + fk_derived_attrs.update(props.get("attr_map", {}).keys()) + + fk_attrs = [] + for name in target_pk: + if name in fk_derived_attrs: + # FK-derived: comes from a primary FK parent + attr = heading[name] + fk_attrs.append((name, attr.type)) + else: + # Native PK attribute - not from FK + logger.warning( + f"Ignoring non-FK primary key attribute '{name}' in jobs table " + f"for {self._target.full_table_name}. Job granularity will be degraded." + ) + + return fk_attrs + + def _get_pk(self, key: dict) -> dict: + """ + Extract primary key values from a key dict. + + Parameters + ---------- + key : dict + Dictionary containing at least the primary key attributes. + + Returns + ------- + dict + Dictionary with only the primary key attributes. + """ + return {k: key[k] for k in self.primary_key if k in key} + + def delete(self) -> None: + """Delete all entries, bypassing interactive prompts and dependencies.""" + self.delete_quick() + + def drop(self) -> None: + """Drop the table, bypassing interactive prompts and dependencies.""" + self.drop_quick() + + # ------------------------------------------------------------------------- + # Status filter properties + # ------------------------------------------------------------------------- + + @property + def pending(self) -> "Job": + """ + Query for pending jobs awaiting processing. + + Returns + ------- + Job + Restricted query with ``status='pending'``. + """ + return self & 'status="pending"' + + @property + def reserved(self) -> "Job": + """ + Query for jobs currently being processed. + + Returns + ------- + Job + Restricted query with ``status='reserved'``. + """ + return self & 'status="reserved"' + + @property + def errors(self) -> "Job": + """ + Query for jobs that failed with errors. + + Returns + ------- + Job + Restricted query with ``status='error'``. + """ + return self & 'status="error"' + + @property + def ignored(self) -> "Job": + """ + Query for jobs marked to be skipped. + + Returns + ------- + Job + Restricted query with ``status='ignore'``. + """ + return self & 'status="ignore"' + + @property + def completed(self) -> "Job": + """ + Query for successfully completed jobs. + + Returns + ------- + Job + Restricted query with ``status='success'``. + """ + return self & 'status="success"' + + # ------------------------------------------------------------------------- + # Core job management methods + # ------------------------------------------------------------------------- + + def refresh( + self, + *restrictions, + delay: float = 0, + priority: int | None = None, + stale_timeout: float | None = None, + orphan_timeout: float | None = None, + ) -> dict: + """ + Refresh the jobs queue: add new jobs and clean up stale/orphaned jobs. + + Parameters + ---------- + *restrictions : any + Conditions to filter key_source (for adding new jobs). + delay : float, optional + Seconds from now until new jobs become available for processing. + Default 0 (immediately available). Uses database server time. + priority : int, optional + Priority for new jobs (lower = more urgent). + Default from ``config.jobs.default_priority``. + stale_timeout : float, optional + Seconds after which jobs are checked for staleness. + Jobs older than this are removed if key not in key_source. + Default from ``config.jobs.stale_timeout``. Set to 0 to skip. + orphan_timeout : float, optional + Seconds after which reserved jobs are considered orphaned. + Reserved jobs older than this are deleted and re-added as pending. + Default None (no orphan cleanup). + + Returns + ------- + dict + Status counts with keys: ``'added'``, ``'removed'``, + ``'orphaned'``, ``'re_pended'``. + + Notes + ----- + Operations performed: + + 1. Add new jobs: ``(key_source & restrictions) - target - jobs`` → insert as pending + 2. Re-pend success jobs: if ``keep_completed=True`` and key in key_source but not in target + 3. Remove stale jobs: jobs older than stale_timeout whose keys not in key_source + 4. Remove orphaned jobs: reserved jobs older than orphan_timeout (if specified) + """ + from .settings import config + + # Ensure jobs table exists + if not self.is_declared: + self.declare() + + # Get defaults from config + if priority is None: + priority = config.jobs.default_priority + if stale_timeout is None: + stale_timeout = config.jobs.stale_timeout + + result = {"added": 0, "removed": 0, "orphaned": 0, "re_pended": 0} + + # 1. Add new jobs + key_source = self._target.key_source + if restrictions: + key_source = key_source & AndList(restrictions) + + # Keys that need jobs: in key_source, not in target, not in jobs + # Disable semantic_check for Job table (self) because its attributes may not have matching lineage + from .condition import Not + + new_keys = (key_source - self._target).restrict(Not(self), semantic_check=False).proj() + new_key_list = new_keys.keys() + + if new_key_list: + # Always use MySQL server time for scheduling (NOW(3) matches datetime(3) precision) + scheduled_time = self.connection.query(f"SELECT NOW(3) + INTERVAL {delay} SECOND").fetchone()[0] + + for key in new_key_list: + job_entry = { + **key, + "status": "pending", + "priority": priority, + "scheduled_time": scheduled_time, + } + try: + self.insert1(job_entry, ignore_extra_fields=True) + result["added"] += 1 + except DuplicateError: + pass # Job already exists + + # 2. Re-pend success jobs if keep_completed=True + if config.jobs.keep_completed: + # Success jobs whose keys are in key_source but not in target + # Disable semantic_check for Job table operations + success_to_repend = self.completed.restrict(key_source, semantic_check=False) - self._target + repend_keys = success_to_repend.keys() + for key in repend_keys: + (self & key).delete_quick() + self.insert1({**key, "status": "pending", "priority": priority}) + result["re_pended"] += 1 + + # 3. Remove stale jobs (not ignore status) - use MySQL NOW() for consistent timing + if stale_timeout > 0: + old_jobs = self & f"created_time < NOW() - INTERVAL {stale_timeout} SECOND" & 'status != "ignore"' + + for key in old_jobs.keys(): + # Check if key still in key_source + if not (key_source & key): + (self & key).delete_quick() + result["removed"] += 1 + + # 4. Handle orphaned reserved jobs - use MySQL NOW() for consistent timing + if orphan_timeout is not None and orphan_timeout > 0: + orphaned_jobs = self.reserved & f"reserved_time < NOW() - INTERVAL {orphan_timeout} SECOND" + + for key in orphaned_jobs.keys(): + (self & key).delete_quick() + self.insert1({**key, "status": "pending", "priority": priority}) + result["orphaned"] += 1 + + return result + + def reserve(self, key: dict) -> bool: + """ + Attempt to reserve a pending job for processing. + + Updates status to ``'reserved'`` if currently ``'pending'`` and + ``scheduled_time <= now``. + + Parameters + ---------- + key : dict + Primary key dict of the job to reserve. + + Returns + ------- + bool + True if reservation successful, False if job not available. + """ + # Check if job is pending and scheduled (use NOW(3) to match CURRENT_TIMESTAMP(3) precision) + job = (self & key & 'status="pending"' & "scheduled_time <= NOW(3)").to_dicts() + + if not job: + return False + + # Get MySQL server time for reserved_time + server_now = self.connection.query("SELECT NOW()").fetchone()[0] + + # Build update row with primary key and new values + pk = self._get_pk(key) + update_row = { + **pk, + "status": "reserved", + "reserved_time": server_now, + "host": platform.node(), + "pid": os.getpid(), + "connection_id": self.connection.connection_id, + "user": self.connection.get_user(), + "version": _get_job_version(), + } + + try: + self.update1(update_row) + return True + except Exception: + return False + + def complete(self, key: dict, duration: float | None = None) -> None: + """ + Mark a job as successfully completed. + + Parameters + ---------- + key : dict + Primary key dict of the job. + duration : float, optional + Execution duration in seconds. + + Notes + ----- + Based on ``config.jobs.keep_completed``: + + - If True: updates status to ``'success'`` with completion time and duration + - If False: deletes the job entry + """ + from .settings import config + + if config.jobs.keep_completed: + # Use MySQL server time for completed_time + server_now = self.connection.query("SELECT NOW()").fetchone()[0] + pk = self._get_pk(key) + update_row = { + **pk, + "status": "success", + "completed_time": server_now, + } + if duration is not None: + update_row["duration"] = duration + self.update1(update_row) + else: + (self & key).delete_quick() + + def error(self, key: dict, error_message: str, error_stack: str | None = None) -> None: + """ + Mark a job as failed with error details. + + Parameters + ---------- + key : dict + Primary key dict of the job. + error_message : str + Error message (truncated to 2047 chars if longer). + error_stack : str, optional + Full stack trace. + """ + if len(error_message) > ERROR_MESSAGE_LENGTH: + error_message = error_message[: ERROR_MESSAGE_LENGTH - len(TRUNCATION_APPENDIX)] + TRUNCATION_APPENDIX + + # Use MySQL server time for completed_time + server_now = self.connection.query("SELECT NOW()").fetchone()[0] + + pk = self._get_pk(key) + update_row = { + **pk, + "status": "error", + "completed_time": server_now, + "error_message": error_message, + } + if error_stack is not None: + update_row["error_stack"] = error_stack + + self.update1(update_row) + + def ignore(self, key: dict) -> None: + """ + Mark a job to be ignored (skipped during populate). + + If the key doesn't exist in the jobs table, inserts it with + ``status='ignore'``. If it exists, updates the status to ``'ignore'``. + + Parameters + ---------- + key : dict + Primary key dict of the job. + """ + from .settings import config + + pk = self._get_pk(key) + if pk in self: + self.update1({**pk, "status": "ignore"}) + else: + priority = config.jobs.default_priority + self.insert1({**pk, "status": "ignore", "priority": priority}) + + def progress(self) -> dict: + """ + Return job status breakdown. + + Returns + ------- + dict + Counts by status with keys: ``'pending'``, ``'reserved'``, + ``'success'``, ``'error'``, ``'ignore'``, ``'total'``. + """ + if not self.is_declared: + return { + "pending": 0, + "reserved": 0, + "success": 0, + "error": 0, + "ignore": 0, + "total": 0, + } + + # Query status counts + result = self.connection.query(f"SELECT status, COUNT(*) as n FROM {self.full_table_name} GROUP BY status").fetchall() + + counts = { + "pending": 0, + "reserved": 0, + "success": 0, + "error": 0, + "ignore": 0, + } + + for row in result: + status, n = row + counts[status] = n + + counts["total"] = sum(counts.values()) + return counts diff --git a/src/datajoint/lineage.py b/src/datajoint/lineage.py new file mode 100644 index 000000000..d40ed8dd8 --- /dev/null +++ b/src/datajoint/lineage.py @@ -0,0 +1,340 @@ +""" +Lineage management for semantic matching in DataJoint. + +Lineage identifies the origin of an attribute - where it was first defined. +It is represented as a string in the format: schema_name.table_name.attribute_name + +Semantic matching is applied to all binary operations that match attributes by name: +- Join (A * B): matches on homologous namesakes +- Restriction (A & B, A - B): matches on homologous namesakes +- Aggregation (A.aggr(B, ...)): requires homologous namesakes for grouping +- Union (A + B): requires all namesakes to have matching lineage + +If namesake attributes have different lineages (including either being None), +a DataJointError is raised. + +If the ~lineage table doesn't exist for a schema, a warning is issued and +semantic checking is disabled for operations involving that schema. + +The ~lineage table stores lineage information for each schema, populated at table +declaration time. Use schema.rebuild_lineage() to restore lineage for legacy schemas. +""" + +import logging + +from .errors import DataJointError + +logger = logging.getLogger(__name__.split(".")[0]) + + +def ensure_lineage_table(connection, database): + """ + Create the ~lineage table in the schema if it doesn't exist. + + Parameters + ---------- + connection : Connection + A DataJoint connection object. + database : str + The schema/database name. + """ + connection.query( + """ + CREATE TABLE IF NOT EXISTS `{database}`.`~lineage` ( + table_name VARCHAR(64) NOT NULL COMMENT 'table name within the schema', + attribute_name VARCHAR(64) NOT NULL COMMENT 'attribute name', + lineage VARCHAR(255) NOT NULL COMMENT 'origin: schema.table.attribute', + PRIMARY KEY (table_name, attribute_name) + ) ENGINE=InnoDB + """.format(database=database) + ) + + +def lineage_table_exists(connection, database): + """ + Check if the ~lineage table exists in the schema. + + Parameters + ---------- + connection : Connection + A DataJoint connection object. + database : str + The schema/database name. + + Returns + ------- + bool + True if the table exists, False otherwise. + """ + result = connection.query( + """ + SELECT COUNT(*) FROM information_schema.tables + WHERE table_schema = %s AND table_name = '~lineage' + """, + args=(database,), + ).fetchone() + return result[0] > 0 + + +def get_lineage(connection, database, table_name, attribute_name): + """ + Get the lineage for an attribute from the ~lineage table. + + Parameters + ---------- + connection : Connection + A DataJoint connection object. + database : str + The schema/database name. + table_name : str + The table name. + attribute_name : str + The attribute name. + + Returns + ------- + str or None + The lineage string, or None if not found. + """ + if not lineage_table_exists(connection, database): + return None + + result = connection.query( + """ + SELECT lineage FROM `{database}`.`~lineage` + WHERE table_name = %s AND attribute_name = %s + """.format(database=database), + args=(table_name, attribute_name), + ).fetchone() + return result[0] if result else None + + +def get_table_lineages(connection, database, table_name): + """ + Get all lineages for a table from the ~lineage table. + + Parameters + ---------- + connection : Connection + A DataJoint connection object. + database : str + The schema/database name. + table_name : str + The table name. + + Returns + ------- + dict[str, str] + Dict mapping attribute names to lineage strings. + """ + if not lineage_table_exists(connection, database): + return {} + + results = connection.query( + """ + SELECT attribute_name, lineage FROM `{database}`.`~lineage` + WHERE table_name = %s + """.format(database=database), + args=(table_name,), + ).fetchall() + return {row[0]: row[1] for row in results} + + +def get_schema_lineages(connection, database): + """ + Get all lineages for a schema from the ~lineage table. + + Parameters + ---------- + connection : Connection + A DataJoint connection object. + database : str + The schema/database name. + + Returns + ------- + dict[str, str] + Dict mapping 'schema.table.attribute' to its lineage. + """ + if not lineage_table_exists(connection, database): + return {} + + results = connection.query( + """ + SELECT table_name, attribute_name, lineage FROM `{database}`.`~lineage` + """.format(database=database), + ).fetchall() + + return {f"{database}.{table}.{attr}": lineage for table, attr, lineage in results} + + +def insert_lineages(connection, database, entries): + """ + Insert multiple lineage entries in the ~lineage table as a single transaction. + + Parameters + ---------- + connection : Connection + A DataJoint connection object. + database : str + The schema/database name. + entries : list[tuple[str, str, str]] + List of (table_name, attribute_name, lineage) tuples. + """ + if not entries: + return + ensure_lineage_table(connection, database) + # Build a single INSERT statement with multiple values for atomicity + placeholders = ", ".join(["(%s, %s, %s)"] * len(entries)) + # Flatten the entries into a single args tuple + args = tuple(val for entry in entries for val in entry) + connection.query( + """ + INSERT INTO `{database}`.`~lineage` (table_name, attribute_name, lineage) + VALUES {placeholders} + ON DUPLICATE KEY UPDATE lineage = VALUES(lineage) + """.format(database=database, placeholders=placeholders), + args=args, + ) + + +def delete_table_lineages(connection, database, table_name): + """ + Delete all lineage entries for a table. + + Parameters + ---------- + connection : Connection + A DataJoint connection object. + database : str + The schema/database name. + table_name : str + The table name. + """ + if not lineage_table_exists(connection, database): + return + connection.query( + """ + DELETE FROM `{database}`.`~lineage` + WHERE table_name = %s + """.format(database=database), + args=(table_name,), + ) + + +def rebuild_schema_lineage(connection, database): + """ + Rebuild the ~lineage table for all tables in a schema. + + This utility recomputes lineage for all attributes in all tables + by querying FK relationships from the information_schema. Use this + to restore lineage after corruption or for schemas that predate + the lineage system. + + This function assumes that any upstream schemas (referenced via + cross-schema foreign keys) have already had their lineage rebuilt. + If a referenced attribute in another schema has no lineage entry, + a DataJointError is raised. + + Parameters + ---------- + connection : Connection + A DataJoint connection object. + database : str + The schema/database name. + + Raises + ------ + DataJointError + If a referenced attribute in another schema has no lineage entry. + """ + # Ensure the lineage table exists + ensure_lineage_table(connection, database) + + # Clear all existing lineage entries for this schema + connection.query(f"DELETE FROM `{database}`.`~lineage`") + + # Get all tables in the schema (excluding hidden tables) + tables_result = connection.query( + """ + SELECT TABLE_NAME FROM information_schema.tables + WHERE TABLE_SCHEMA = %s AND TABLE_NAME NOT LIKE '~%%' + """, + args=(database,), + ).fetchall() + all_tables = {row[0] for row in tables_result} + + if not all_tables: + return + + # Get all primary key columns for all tables + pk_result = connection.query( + """ + SELECT TABLE_NAME, COLUMN_NAME FROM information_schema.KEY_COLUMN_USAGE + WHERE TABLE_SCHEMA = %s AND CONSTRAINT_NAME = 'PRIMARY' + """, + args=(database,), + ).fetchall() + # table -> set of PK columns + pk_columns = {} + for table, col in pk_result: + pk_columns.setdefault(table, set()).add(col) + + # Get all FK relationships within and across schemas + fk_result = connection.query( + """ + SELECT TABLE_NAME, COLUMN_NAME, + REFERENCED_TABLE_SCHEMA, REFERENCED_TABLE_NAME, REFERENCED_COLUMN_NAME + FROM information_schema.KEY_COLUMN_USAGE + WHERE TABLE_SCHEMA = %s AND REFERENCED_TABLE_NAME IS NOT NULL + """, + args=(database,), + ).fetchall() + + # Build FK map: (table, column) -> (parent_schema, parent_table, parent_column) + fk_map = {(table, col): (ref_schema, ref_table, ref_col) for table, col, ref_schema, ref_table, ref_col in fk_result} + + # Lineage cache: (table, column) -> lineage string (for this schema) + lineage_cache = {} + + def resolve_lineage(table, col): + """Recursively resolve lineage for an attribute.""" + if (table, col) in lineage_cache: + return lineage_cache[(table, col)] + + if (table, col) in fk_map: + # FK attribute - get parent's lineage + parent_schema, parent_table, parent_col = fk_map[(table, col)] + if parent_schema == database: + # Same schema - recurse + lineage = resolve_lineage(parent_table, parent_col) + else: + # Cross-schema - query parent's lineage table + lineage = get_lineage(connection, parent_schema, parent_table, parent_col) + if not lineage: + raise DataJointError( + f"Cannot rebuild lineage for `{database}`.`{table}`: " + f"referenced attribute `{parent_schema}`.`{parent_table}`.`{parent_col}` " + f"has no lineage. Rebuild lineage for schema `{parent_schema}` first." + ) + else: + # Native PK attribute - lineage is self + lineage = f"{database}.{table}.{col}" + + lineage_cache[(table, col)] = lineage + return lineage + + # Resolve lineage for all PK and FK attributes + for table in all_tables: + table_pk = pk_columns.get(table, set()) + table_fk_cols = {col for (t, col) in fk_map if t == table} + + # Process all attributes that need lineage (PK and FK) + for col in table_pk | table_fk_cols: + if not col.startswith("_"): + resolve_lineage(table, col) + + # Insert all lineages in one batch + if lineage_cache: + entries = [(table, col, lineage) for (table, col), lineage in lineage_cache.items()] + insert_lineages(connection, database, entries) diff --git a/datajoint/logging.py b/src/datajoint/logging.py similarity index 100% rename from datajoint/logging.py rename to src/datajoint/logging.py diff --git a/src/datajoint/migrate.py b/src/datajoint/migrate.py new file mode 100644 index 000000000..3a2bf2ce6 --- /dev/null +++ b/src/datajoint/migrate.py @@ -0,0 +1,452 @@ +""" +Migration utilities for DataJoint schema updates. + +This module provides tools for migrating existing schemas to use the new +Codec system, particularly for upgrading blob columns to use +explicit `` type declarations. +""" + +from __future__ import annotations + +import logging +import re +from typing import TYPE_CHECKING + +from .errors import DataJointError + +if TYPE_CHECKING: + from .schemas import Schema + +logger = logging.getLogger(__name__.split(".")[0]) + +# Pattern to detect blob types +BLOB_TYPES = re.compile(r"^(tiny|small|medium|long|)blob$", re.I) + + +def analyze_blob_columns(schema: Schema) -> list[dict]: + """ + Analyze a schema to find blob columns that could be migrated to . + + This function identifies blob columns that: + + 1. Have a MySQL blob type (tinyblob, blob, mediumblob, longblob) + 2. Do NOT already have a codec/type specified in their comment + + All blob size variants are included in the analysis. + + Parameters + ---------- + schema : Schema + The DataJoint schema to analyze. + + Returns + ------- + list[dict] + List of dicts with keys: + + - table_name: Full table name (database.table) + - column_name: Name of the blob column + - column_type: MySQL column type (tinyblob, blob, mediumblob, longblob) + - current_comment: Current column comment + - needs_migration: True if column should be migrated + + Examples + -------- + >>> import datajoint as dj + >>> schema = dj.schema('my_database') + >>> columns = dj.migrate.analyze_blob_columns(schema) + >>> for col in columns: + ... if col['needs_migration']: + ... print(f"{col['table_name']}.{col['column_name']} ({col['column_type']})") + """ + results = [] + + connection = schema.connection + + # Get all tables in the schema + tables_query = """ + SELECT TABLE_NAME + FROM information_schema.TABLES + WHERE TABLE_SCHEMA = %s + AND TABLE_TYPE = 'BASE TABLE' + AND TABLE_NAME NOT LIKE '~%%' + """ + + tables = connection.query(tables_query, args=(schema.database,)).fetchall() + + for (table_name,) in tables: + # Get column information for each table + columns_query = """ + SELECT COLUMN_NAME, COLUMN_TYPE, COLUMN_COMMENT + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = %s + AND TABLE_NAME = %s + AND DATA_TYPE IN ('tinyblob', 'blob', 'mediumblob', 'longblob') + """ + + columns = connection.query(columns_query, args=(schema.database, table_name)).fetchall() + + for column_name, column_type, comment in columns: + # Check if comment already has a codec type (starts with :type:) + has_codec = comment and comment.startswith(":") + + results.append( + { + "table_name": f"{schema.database}.{table_name}", + "column_name": column_name, + "column_type": column_type, + "current_comment": comment or "", + "needs_migration": not has_codec, + } + ) + + return results + + +def generate_migration_sql( + schema: Schema, + target_type: str = "blob", + dry_run: bool = True, +) -> list[str]: + """ + Generate SQL statements to migrate blob columns to use . + + This generates ALTER TABLE statements that update column comments to + include the `::` prefix, marking them as using explicit + DataJoint blob serialization. + + Parameters + ---------- + schema : Schema + The DataJoint schema to migrate. + target_type : str, optional + The type name to migrate to. Default "blob". + dry_run : bool, optional + If True, only return SQL without executing. + + Returns + ------- + list[str] + List of SQL ALTER TABLE statements. + + Examples + -------- + >>> sql_statements = dj.migrate.generate_migration_sql(schema) + >>> for sql in sql_statements: + ... print(sql) + + Notes + ----- + This is a metadata-only migration. The actual blob data format + remains unchanged - only the column comments are updated to + indicate explicit type handling. + """ + columns = analyze_blob_columns(schema) + sql_statements = [] + + for col in columns: + if not col["needs_migration"]: + continue + + # Build new comment with type prefix + old_comment = col["current_comment"] + new_comment = f":<{target_type}>:{old_comment}" + + # Escape special characters for SQL + new_comment_escaped = new_comment.replace("\\", "\\\\").replace("'", "\\'") + + # Parse table name + db_name, table_name = col["table_name"].split(".") + + # Generate ALTER TABLE statement + sql = ( + f"ALTER TABLE `{db_name}`.`{table_name}` " + f"MODIFY COLUMN `{col['column_name']}` {col['column_type']} " + f"COMMENT '{new_comment_escaped}'" + ) + sql_statements.append(sql) + + return sql_statements + + +def migrate_blob_columns( + schema: Schema, + target_type: str = "blob", + dry_run: bool = True, +) -> dict: + """ + Migrate blob columns in a schema to use explicit type. + + This updates column comments in the database to include the type + declaration. The data format remains unchanged. + + Parameters + ---------- + schema : Schema + The DataJoint schema to migrate. + target_type : str, optional + The type name to migrate to. Default "blob". + dry_run : bool, optional + If True, only preview changes without applying. Default True. + + Returns + ------- + dict + Dict with keys: + + - analyzed: Number of blob columns analyzed + - needs_migration: Number of columns that need migration + - migrated: Number of columns migrated (0 if dry_run) + - sql_statements: List of SQL statements (executed or to be executed) + + Examples + -------- + >>> # Preview migration + >>> result = dj.migrate.migrate_blob_columns(schema, dry_run=True) + >>> print(f"Would migrate {result['needs_migration']} columns") + + >>> # Apply migration + >>> result = dj.migrate.migrate_blob_columns(schema, dry_run=False) + >>> print(f"Migrated {result['migrated']} columns") + + Warnings + -------- + After migration, table definitions should be updated to use + ```` instead of ``longblob`` for consistency. The migration + only updates database metadata; source code changes are manual. + """ + columns = analyze_blob_columns(schema) + sql_statements = generate_migration_sql(schema, target_type=target_type) + + result = { + "analyzed": len(columns), + "needs_migration": sum(1 for c in columns if c["needs_migration"]), + "migrated": 0, + "sql_statements": sql_statements, + } + + if dry_run: + logger.info(f"Dry run: would migrate {result['needs_migration']} columns") + for sql in sql_statements: + logger.info(f" {sql}") + return result + + # Execute migrations + connection = schema.connection + for sql in sql_statements: + try: + connection.query(sql) + result["migrated"] += 1 + logger.info(f"Executed: {sql}") + except Exception as e: + logger.error(f"Failed to execute: {sql}\nError: {e}") + raise DataJointError(f"Migration failed: {e}") from e + + logger.info(f"Successfully migrated {result['migrated']} columns") + return result + + +def check_migration_status(schema: Schema) -> dict: + """ + Check the migration status of blob columns in a schema. + + Parameters + ---------- + schema : Schema + The DataJoint schema to check. + + Returns + ------- + dict + Dict with keys: + + - total_blob_columns: Total number of blob columns + - migrated: Number of columns with explicit type + - pending: Number of columns using implicit serialization + - columns: List of column details + + Examples + -------- + >>> status = dj.migrate.check_migration_status(schema) + >>> print(f"Migration progress: {status['migrated']}/{status['total_blob_columns']}") + """ + columns = analyze_blob_columns(schema) + + return { + "total_blob_columns": len(columns), + "migrated": sum(1 for c in columns if not c["needs_migration"]), + "pending": sum(1 for c in columns if c["needs_migration"]), + "columns": columns, + } + + +# ============================================================================= +# Job Metadata Migration +# ============================================================================= + +# Hidden job metadata columns added by config.jobs.add_job_metadata +JOB_METADATA_COLUMNS = [ + ("_job_start_time", "datetime(3) DEFAULT NULL"), + ("_job_duration", "float DEFAULT NULL"), + ("_job_version", "varchar(64) DEFAULT ''"), +] + + +def _get_existing_columns(connection, database: str, table_name: str) -> set[str]: + """Get set of existing column names for a table.""" + result = connection.query( + """ + SELECT COLUMN_NAME + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = %s AND TABLE_NAME = %s + """, + args=(database, table_name), + ) + return {row[0] for row in result.fetchall()} + + +def _is_autopopulated_table(table_name: str) -> bool: + """Check if a table name indicates a Computed or Imported table.""" + # Computed tables start with __ (but not part tables which have __ in middle) + # Imported tables start with _ (but not __) + if table_name.startswith("__"): + # Computed table if no __ after the prefix + return "__" not in table_name[2:] + elif table_name.startswith("_"): + # Imported table + return True + return False + + +def add_job_metadata_columns(target, dry_run: bool = True) -> dict: + """ + Add hidden job metadata columns to existing Computed/Imported tables. + + This migration utility adds the hidden columns (_job_start_time, _job_duration, + _job_version) to tables that were created before config.jobs.add_job_metadata + was enabled. + + Parameters + ---------- + target : Table or Schema + Either a table class/instance (dj.Computed or dj.Imported) or + a Schema object. If a Schema, all Computed/Imported tables in + the schema will be processed. + dry_run : bool, optional + If True, only preview changes without applying. Default True. + + Returns + ------- + dict + Dict with keys: + + - tables_analyzed: Number of tables checked + - tables_modified: Number of tables that were/would be modified + - columns_added: Total columns added across all tables + - details: List of dicts with per-table information + + Examples + -------- + >>> import datajoint as dj + >>> from datajoint.migrate import add_job_metadata_columns + >>> + >>> # Preview migration for a single table + >>> result = add_job_metadata_columns(MyComputedTable, dry_run=True) + >>> print(f"Would add {result['columns_added']} columns") + >>> + >>> # Apply migration to all tables in a schema + >>> result = add_job_metadata_columns(schema, dry_run=False) + >>> print(f"Modified {result['tables_modified']} tables") + + Notes + ----- + - Only Computed and Imported tables are modified (not Manual, Lookup, or Part) + - Existing rows will have NULL values for _job_start_time and _job_duration + - Future populate() calls will fill in metadata for new rows + - This does NOT retroactively populate metadata for existing rows + """ + from .schemas import Schema + from .table import Table + + result = { + "tables_analyzed": 0, + "tables_modified": 0, + "columns_added": 0, + "details": [], + } + + # Determine tables to process + if isinstance(target, Schema): + schema = target + # Get all user tables in the schema + tables_query = """ + SELECT TABLE_NAME + FROM information_schema.TABLES + WHERE TABLE_SCHEMA = %s + AND TABLE_TYPE = 'BASE TABLE' + AND TABLE_NAME NOT LIKE '~%%' + """ + table_names = [row[0] for row in schema.connection.query(tables_query, args=(schema.database,)).fetchall()] + tables_to_process = [ + (schema.database, name, schema.connection) for name in table_names if _is_autopopulated_table(name) + ] + elif isinstance(target, type) and issubclass(target, Table): + # Table class + instance = target() + tables_to_process = [(instance.database, instance.table_name, instance.connection)] + elif isinstance(target, Table): + # Table instance + tables_to_process = [(target.database, target.table_name, target.connection)] + else: + raise DataJointError(f"target must be a Table class, Table instance, or Schema, got {type(target)}") + + for database, table_name, connection in tables_to_process: + result["tables_analyzed"] += 1 + + # Skip non-autopopulated tables + if not _is_autopopulated_table(table_name): + continue + + # Check which columns need to be added + existing_columns = _get_existing_columns(connection, database, table_name) + columns_to_add = [(name, definition) for name, definition in JOB_METADATA_COLUMNS if name not in existing_columns] + + if not columns_to_add: + result["details"].append( + { + "table": f"{database}.{table_name}", + "status": "already_migrated", + "columns_added": 0, + } + ) + continue + + # Generate and optionally execute ALTER statements + table_detail = { + "table": f"{database}.{table_name}", + "status": "migrated" if not dry_run else "pending", + "columns_added": len(columns_to_add), + "sql_statements": [], + } + + for col_name, col_definition in columns_to_add: + sql = f"ALTER TABLE `{database}`.`{table_name}` ADD COLUMN `{col_name}` {col_definition}" + table_detail["sql_statements"].append(sql) + + if not dry_run: + try: + connection.query(sql) + logger.info(f"Added column {col_name} to {database}.{table_name}") + except Exception as e: + logger.error(f"Failed to add column {col_name} to {database}.{table_name}: {e}") + table_detail["status"] = "error" + table_detail["error"] = str(e) + raise DataJointError(f"Migration failed: {e}") from e + else: + logger.info(f"Would add column {col_name} to {database}.{table_name}") + + result["tables_modified"] += 1 + result["columns_added"] += len(columns_to_add) + result["details"].append(table_detail) + + return result diff --git a/src/datajoint/objectref.py b/src/datajoint/objectref.py new file mode 100644 index 000000000..5d84fb96c --- /dev/null +++ b/src/datajoint/objectref.py @@ -0,0 +1,408 @@ +""" +ObjectRef class for handling fetched object type attributes. + +This module provides the ObjectRef class which represents a reference to a file +or folder stored in the pipeline's object storage backend. It provides metadata +access and direct fsspec-based file operations. +""" + +import json +from dataclasses import dataclass +from datetime import datetime +from pathlib import Path +from typing import IO, Iterator + +import fsspec + +from .errors import DataJointError +from .storage import StorageBackend + + +class IntegrityError(DataJointError): + """Raised when object integrity verification fails.""" + + pass + + +@dataclass +class ObjectRef: + """ + Handle to a file or folder stored in the pipeline's object storage backend. + + This class is returned when fetching object-type attributes. It provides + metadata access without I/O, and methods for reading content directly + from the storage backend. + + Attributes: + path: Relative path within the store (includes token) + url: Full URI to the object (e.g., 's3://bucket/path/to/object.dat') + store: Store name (None for default store) + size: Total size in bytes (sum for folders), or None if not computed. + For large hierarchical data like Zarr stores, size computation can + be expensive and is optional. + hash: Content hash with algorithm prefix, or None if not computed + ext: File extension as tooling hint (e.g., ".dat", ".zarr") or None. + This is a conventional suffix for tooling, not a content-type declaration. + is_dir: True if stored content is a directory/key-prefix (e.g., Zarr store) + timestamp: ISO 8601 upload timestamp + mime_type: MIME type (files only, auto-detected from extension) + item_count: Number of files (folders only), or None if not computed + """ + + path: str + size: int | None + hash: str | None + ext: str | None + is_dir: bool + timestamp: datetime + url: str | None = None + store: str | None = None + mime_type: str | None = None + item_count: int | None = None + _backend: StorageBackend | None = None + + @classmethod + def from_json(cls, json_data: dict | str, backend: StorageBackend | None = None) -> "ObjectRef": + """ + Create an ObjectRef from JSON metadata stored in the database. + + Parameters + ---------- + json_data : dict or str + JSON string or dict containing object metadata. + backend : StorageBackend, optional + StorageBackend instance for file operations. + + Returns + ------- + ObjectRef + ObjectRef instance. + """ + if isinstance(json_data, str): + data = json.loads(json_data) + else: + data = json_data + + timestamp = data.get("timestamp") + if isinstance(timestamp, str): + timestamp = datetime.fromisoformat(timestamp.replace("Z", "+00:00")) + + return cls( + path=data["path"], + url=data.get("url"), + store=data.get("store"), + size=data["size"], + hash=data.get("hash"), + ext=data.get("ext"), + is_dir=data.get("is_dir", False), + timestamp=timestamp, + mime_type=data.get("mime_type"), + item_count=data.get("item_count"), + _backend=backend, + ) + + def to_json(self) -> dict: + """ + Convert ObjectRef to JSON-serializable dict for database storage. + + Returns + ------- + dict + Dict suitable for JSON serialization. + """ + data = { + "path": self.path, + "size": self.size, + "hash": self.hash, + "ext": self.ext, + "is_dir": self.is_dir, + "timestamp": self.timestamp.isoformat() if self.timestamp else None, + } + if self.url: + data["url"] = self.url + if self.store: + data["store"] = self.store + if self.mime_type: + data["mime_type"] = self.mime_type + if self.item_count is not None: + data["item_count"] = self.item_count + return data + + def _ensure_backend(self): + """Ensure storage backend is available for I/O operations.""" + if self._backend is None: + raise DataJointError( + "ObjectRef has no storage backend configured. " + "This usually means the object was created without a connection context." + ) + + @property + def fs(self) -> fsspec.AbstractFileSystem: + """ + Return fsspec filesystem for direct access. + + This allows integration with libraries like Zarr and xarray that + work with fsspec filesystems. + """ + self._ensure_backend() + return self._backend.fs + + @property + def fsmap(self) -> fsspec.FSMap: + """ + Return FSMap suitable for Zarr/xarray. + + This provides a dict-like interface to the storage location, + compatible with zarr.open() and xarray.open_zarr(). + + Example: + >>> z = zarr.open(obj_ref.fsmap, mode='r') + """ + self._ensure_backend() + full_path = self._backend._full_path(self.path) + return fsspec.FSMap(full_path, self._backend.fs) + + @property + def full_path(self) -> str: + """ + Return full URI (e.g., 's3://bucket/path'). + + This is the complete path including protocol and bucket/location. + """ + self._ensure_backend() + protocol = self._backend.protocol + if protocol == "file": + return str(Path(self._backend.spec.get("location", "")) / self.path) + elif protocol == "s3": + bucket = self._backend.spec["bucket"] + return f"s3://{bucket}/{self.path}" + elif protocol == "gcs": + bucket = self._backend.spec["bucket"] + return f"gs://{bucket}/{self.path}" + elif protocol == "azure": + container = self._backend.spec["container"] + return f"az://{container}/{self.path}" + else: + return self.path + + def read(self) -> bytes: + """ + Read entire file content as bytes. + + Returns + ------- + bytes + File contents as bytes. + + Raises + ------ + DataJointError + If object is a directory. + """ + if self.is_dir: + raise DataJointError("Cannot read() a directory. Use listdir() or walk() instead.") + self._ensure_backend() + return self._backend.get_buffer(self.path) + + def open(self, subpath: str | None = None, mode: str = "rb") -> IO: + """ + Open file for reading. + + Parameters + ---------- + subpath : str, optional + Path within directory (for folder objects). + mode : str, optional + File mode ('rb' for binary read, 'r' for text). Default 'rb'. + + Returns + ------- + IO + File-like object. + """ + self._ensure_backend() + path = self.path + if subpath: + if not self.is_dir: + raise DataJointError("Cannot use subpath on a file object") + path = f"{self.path}/{subpath}" + return self._backend.open(path, mode) + + def listdir(self, subpath: str = "") -> list[str]: + """ + List contents of directory. + + Parameters + ---------- + subpath : str, optional + Subdirectory path. Default empty string (root). + + Returns + ------- + list[str] + List of filenames/directory names. + """ + if not self.is_dir: + raise DataJointError("Cannot listdir() on a file. Use read() or open() instead.") + self._ensure_backend() + path = f"{self.path}/{subpath}" if subpath else self.path + full_path = self._backend._full_path(path) + entries = self._backend.fs.ls(full_path, detail=False) + # Return just the basename of each entry + return [e.split("/")[-1] for e in entries] + + def walk(self) -> Iterator[tuple[str, list[str], list[str]]]: + """ + Walk directory tree, similar to os.walk(). + + Yields + ------ + tuple[str, list[str], list[str]] + Tuples of (dirpath, dirnames, filenames). + """ + if not self.is_dir: + raise DataJointError("Cannot walk() on a file.") + self._ensure_backend() + full_path = self._backend._full_path(self.path) + for root, dirs, files in self._backend.fs.walk(full_path): + # Make paths relative to the object root + rel_root = root[len(full_path) :].lstrip("/") + yield rel_root, dirs, files + + def download(self, destination: Path | str, subpath: str | None = None) -> Path: + """ + Download object to local filesystem. + + Parameters + ---------- + destination : Path or str + Local directory or file path. + subpath : str, optional + Path within directory (for folder objects). + + Returns + ------- + Path + Path to downloaded file/directory. + """ + self._ensure_backend() + destination = Path(destination) + + if subpath: + if not self.is_dir: + raise DataJointError("Cannot use subpath on a file object") + remote_path = f"{self.path}/{subpath}" + else: + remote_path = self.path + + if self.is_dir and not subpath: + # Download entire directory + destination.mkdir(parents=True, exist_ok=True) + full_path = self._backend._full_path(remote_path) + self._backend.fs.get(full_path, str(destination), recursive=True) + else: + # Download single file + if destination.is_dir(): + filename = remote_path.split("/")[-1] + destination = destination / filename + destination.parent.mkdir(parents=True, exist_ok=True) + self._backend.get_file(remote_path, destination) + + return destination + + def exists(self, subpath: str | None = None) -> bool: + """ + Check if object (or subpath within it) exists. + + Parameters + ---------- + subpath : str, optional + Path within directory. + + Returns + ------- + bool + True if exists. + """ + self._ensure_backend() + path = f"{self.path}/{subpath}" if subpath else self.path + return self._backend.exists(path) + + def verify(self) -> bool: + """ + Verify object integrity. + + For files: checks size matches, and hash if available. + For folders: validates manifest (all files exist with correct sizes). + + Returns + ------- + bool + True if valid. + + Raises + ------ + IntegrityError + If verification fails with details. + """ + self._ensure_backend() + + if self.is_dir: + return self._verify_folder() + else: + return self._verify_file() + + def _verify_file(self) -> bool: + """Verify a single file.""" + # Check existence + if not self._backend.exists(self.path): + raise IntegrityError(f"File does not exist: {self.path}") + + # Check size if available + if self.size is not None: + actual_size = self._backend.size(self.path) + if actual_size != self.size: + raise IntegrityError(f"Size mismatch for {self.path}: expected {self.size}, got {actual_size}") + + # Check hash if available + if self.hash: + # TODO: Implement hash verification + pass + + return True + + def _verify_folder(self) -> bool: + """Verify a folder using its manifest.""" + manifest_path = f"{self.path}.manifest.json" + + if not self._backend.exists(manifest_path): + raise IntegrityError(f"Manifest file missing: {manifest_path}") + + # Read manifest + manifest_data = self._backend.get_buffer(manifest_path) + manifest = json.loads(manifest_data) + + # Verify each file in manifest + errors = [] + for file_info in manifest.get("files", []): + file_path = f"{self.path}/{file_info['path']}" + expected_size = file_info["size"] + + if not self._backend.exists(file_path): + errors.append(f"Missing file: {file_info['path']}") + else: + actual_size = self._backend.size(file_path) + if actual_size != expected_size: + errors.append(f"Size mismatch for {file_info['path']}: expected {expected_size}, got {actual_size}") + + if errors: + raise IntegrityError("Folder verification failed:\n" + "\n".join(errors)) + + return True + + def __repr__(self) -> str: + type_str = "folder" if self.is_dir else "file" + return f"ObjectRef({type_str}: {self.path}, size={self.size})" + + def __str__(self) -> str: + return self.path diff --git a/src/datajoint/preview.py b/src/datajoint/preview.py new file mode 100644 index 000000000..5e2c92e5f --- /dev/null +++ b/src/datajoint/preview.py @@ -0,0 +1,230 @@ +"""methods for generating previews of query expression results in python command line and Jupyter""" + +import json + +from .settings import config + + +def _format_object_display(json_data): + """Format object metadata for display in query results.""" + if json_data is None: + return "=OBJ[null]=" + if isinstance(json_data, str): + try: + json_data = json.loads(json_data) + except (json.JSONDecodeError, TypeError): + return "=OBJ=?" + ext = json_data.get("ext") + is_dir = json_data.get("is_dir", False) + if ext: + return f"=OBJ[{ext}]=" + elif is_dir: + return "=OBJ[folder]=" + else: + return "=OBJ[file]=" + + +def preview(query_expression, limit, width): + heading = query_expression.heading + rel = query_expression.proj(*heading.non_blobs) + # Object fields use codecs - not specially handled in simplified model + object_fields = [] + if limit is None: + limit = config["display.limit"] + if width is None: + width = config["display.width"] + tuples = rel.to_arrays(limit=limit + 1) + has_more = len(tuples) > limit + tuples = tuples[:limit] + + # Fetch object field JSON data for display (raw JSON, not ObjectRef) + object_data_list = [] + if object_fields: + # Fetch primary key and object fields as dicts + obj_rel = query_expression.proj(*object_fields) + obj_tuples = obj_rel.to_arrays(limit=limit) + for obj_tup in obj_tuples: + obj_dict = {} + for field in object_fields: + if field in obj_tup.dtype.names: + obj_dict[field] = obj_tup[field] + object_data_list.append(obj_dict) + + columns = heading.names + + def get_placeholder(f): + if f in object_fields: + return "=OBJ[.xxx]=" + return "=BLOB=" + + widths = { + f: min( + max([len(f)] + [len(str(e)) for e in tuples[f]] if f in tuples.dtype.names else [len(get_placeholder(f))]) + 4, + width, + ) + for f in columns + } + templates = {f: "%%-%d.%ds" % (widths[f], widths[f]) for f in columns} + + def get_display_value(tup, f, idx): + if f in tup.dtype.names: + return tup[f] + elif f in object_fields and idx < len(object_data_list): + return _format_object_display(object_data_list[idx].get(f)) + else: + return "=BLOB=" + + return ( + " ".join([templates[f] % ("*" + f if f in rel.primary_key else f) for f in columns]) + + "\n" + + " ".join(["+" + "-" * (widths[column] - 2) + "+" for column in columns]) + + "\n" + + "\n".join(" ".join(templates[f] % get_display_value(tup, f, idx) for f in columns) for idx, tup in enumerate(tuples)) + + ("\n ...\n" if has_more else "\n") + + (" (Total: %d)\n" % len(rel) if config["display.show_tuple_count"] else "") + ) + + +def repr_html(query_expression): + heading = query_expression.heading + rel = query_expression.proj(*heading.non_blobs) + # Object fields use codecs - not specially handled in simplified model + object_fields = [] + info = heading.table_status + tuples = rel.to_arrays(limit=config["display.limit"] + 1) + has_more = len(tuples) > config["display.limit"] + tuples = tuples[0 : config["display.limit"]] + + # Fetch object field JSON data for display (raw JSON, not ObjectRef) + object_data_list = [] + if object_fields: + obj_rel = query_expression.proj(*object_fields) + obj_tuples = obj_rel.to_arrays(limit=config["display.limit"]) + for obj_tup in obj_tuples: + obj_dict = {} + for field in object_fields: + if field in obj_tup.dtype.names: + obj_dict[field] = obj_tup[field] + object_data_list.append(obj_dict) + + def get_html_display_value(tup, name, idx): + if name in tup.dtype.names: + return tup[name] + elif name in object_fields and idx < len(object_data_list): + return _format_object_display(object_data_list[idx].get(name)) + else: + return "=BLOB=" + + css = """ + + """ + head_template = """
+

{column}

+ {comment} +
""" + return """ + {css} + {title} +
+ + + {body} +
{head}
+ {ellipsis} + {count}
+ """.format( + css=css, + title="" if info is None else "%s" % info["comment"], + head="".join( + head_template.format( + column=c, + comment=heading.attributes[c].comment, + primary=("primary" if c in query_expression.primary_key else "nonprimary"), + ) + for c in heading.names + ), + ellipsis="

...

" if has_more else "", + body="".join( + [ + "\n".join(["%s" % get_html_display_value(tup, name, idx) for name in heading.names]) + for idx, tup in enumerate(tuples) + ] + ), + count=(("

Total: %d

" % len(rel)) if config["display.show_tuple_count"] else ""), + ) diff --git a/src/datajoint/schemas.py b/src/datajoint/schemas.py new file mode 100644 index 000000000..399ab1b9f --- /dev/null +++ b/src/datajoint/schemas.py @@ -0,0 +1,909 @@ +""" +Schema management for DataJoint. + +This module provides the Schema class for binding Python table classes to +database schemas, and utilities for schema introspection and management. +""" + +from __future__ import annotations + +import collections +import inspect +import itertools +import logging +import re +import types +import warnings +from typing import TYPE_CHECKING, Any + +from .connection import conn +from .errors import AccessError, DataJointError + +if TYPE_CHECKING: + from .connection import Connection +from .heading import Heading +from .jobs import Job +from .settings import config +from .table import FreeTable, lookup_class_name +from .user_tables import Computed, Imported, Lookup, Manual, Part, _get_tier +from .utils import to_camel_case, user_choice + +logger = logging.getLogger(__name__.split(".")[0]) + + +def ordered_dir(class_: type) -> list[str]: + """ + List class attributes respecting declaration order. + + Similar to the ``dir()`` built-in, but preserves attribute declaration + order as much as possible. + + Parameters + ---------- + class_ : type + Class to list members for. + + Returns + ------- + list[str] + Attributes declared in class_ and its superclasses. + """ + attr_list = list() + for c in reversed(class_.mro()): + attr_list.extend(e for e in c.__dict__ if e not in attr_list) + return attr_list + + +class Schema: + """ + Decorator that binds table classes to a database schema. + + Schema objects associate Python table classes with database schemas and + provide the namespace context for foreign key resolution. + + Parameters + ---------- + schema_name : str, optional + Database schema name. If omitted, call ``activate()`` later. + context : dict, optional + Namespace for foreign key lookup. None uses caller's context. + connection : Connection, optional + Database connection. Defaults to ``dj.conn()``. + create_schema : bool, optional + If False, raise error if schema doesn't exist. Default True. + create_tables : bool, optional + If False, raise error when accessing missing tables. Default True. + add_objects : dict, optional + Additional objects for the declaration context. + + Examples + -------- + >>> schema = dj.Schema('my_schema') + >>> @schema + ... class Session(dj.Manual): + ... definition = ''' + ... session_id : int + ... ''' + """ + + def __init__( + self, + schema_name: str | None = None, + context: dict[str, Any] | None = None, + *, + connection: Connection | None = None, + create_schema: bool = True, + create_tables: bool = True, + add_objects: dict[str, Any] | None = None, + ) -> None: + """ + Initialize the schema object. + + Parameters + ---------- + schema_name : str, optional + Database schema name. If omitted, call ``activate()`` later. + context : dict, optional + Namespace for foreign key lookup. None uses caller's context. + connection : Connection, optional + Database connection. Defaults to ``dj.conn()``. + create_schema : bool, optional + If False, raise error if schema doesn't exist. Default True. + create_tables : bool, optional + If False, raise error when accessing missing tables. Default True. + add_objects : dict, optional + Additional objects for the declaration context. + """ + self.connection = connection + self.database = None + self.context = context + self.create_schema = create_schema + self.create_tables = create_tables + self.add_objects = add_objects + self.declare_list = [] + if schema_name: + self.activate(schema_name) + + def is_activated(self) -> bool: + """Check if the schema has been activated.""" + return self.database is not None + + def activate( + self, + schema_name: str | None = None, + *, + connection: Connection | None = None, + create_schema: bool | None = None, + create_tables: bool | None = None, + add_objects: dict[str, Any] | None = None, + ) -> None: + """ + Associate with a database schema. + + If the schema does not exist, attempts to create it on the server. + + Parameters + ---------- + schema_name : str, optional + Database schema name. None asserts schema is already activated. + connection : Connection, optional + Database connection. Defaults to ``dj.conn()``. + create_schema : bool, optional + If False, raise error if schema doesn't exist. + create_tables : bool, optional + If False, raise error when accessing missing tables. + add_objects : dict, optional + Additional objects for the declaration context. + + Raises + ------ + DataJointError + If schema_name is None and schema not yet activated, or if + schema already activated for a different database. + """ + if schema_name is None: + if self.exists: + return + raise DataJointError("Please provide a schema_name to activate the schema.") + if self.database is not None and self.exists: + if self.database == schema_name: # already activated + return + raise DataJointError("The schema is already activated for schema {db}.".format(db=self.database)) + if connection is not None: + self.connection = connection + if self.connection is None: + self.connection = conn() + self.database = schema_name + if create_schema is not None: + self.create_schema = create_schema + if create_tables is not None: + self.create_tables = create_tables + if add_objects: + self.add_objects = add_objects + if not self.exists: + if not self.create_schema or not self.database: + raise DataJointError( + "Database `{name}` has not yet been declared. Set argument create_schema=True to create it.".format( + name=schema_name + ) + ) + # create database + logger.debug("Creating schema `{name}`.".format(name=schema_name)) + try: + self.connection.query("CREATE DATABASE `{name}`".format(name=schema_name)) + except AccessError: + raise DataJointError( + "Schema `{name}` does not exist and could not be created. Check permissions.".format(name=schema_name) + ) + self.connection.register(self) + + # decorate all tables already decorated + for cls, context in self.declare_list: + if self.add_objects: + context = dict(context, **self.add_objects) + self._decorate_master(cls, context) + + def _assert_exists(self, message=None): + if not self.exists: + raise DataJointError(message or "Schema `{db}` has not been created.".format(db=self.database)) + + def __call__(self, cls: type, *, context: dict[str, Any] | None = None) -> type: + """ + Bind a table class to this schema. Used as a decorator. + + Parameters + ---------- + cls : type + Table class to decorate. + context : dict, optional + Declaration context. Supplied by spawn_missing_classes. + + Returns + ------- + type + The decorated class. + + Raises + ------ + DataJointError + If applied to a Part table (use on master only). + """ + context = context or self.context or inspect.currentframe().f_back.f_locals + if issubclass(cls, Part): + raise DataJointError("The schema decorator should not be applied to Part tables.") + if self.is_activated(): + self._decorate_master(cls, context) + else: + self.declare_list.append((cls, context)) + return cls + + def _decorate_master(self, cls: type, context: dict[str, Any]) -> None: + """ + Process a master table class and its part tables. + + Parameters + ---------- + cls : type + Master table class to process. + context : dict + Declaration context for foreign key resolution. + """ + self._decorate_table(cls, context=dict(context, self=cls, **{cls.__name__: cls})) + # Process part tables + for part in ordered_dir(cls): + if part[0].isupper(): + part = getattr(cls, part) + if inspect.isclass(part) and issubclass(part, Part): + part._master = cls + # allow addressing master by name or keyword 'master' + self._decorate_table( + part, + context=dict(context, master=cls, self=part, **{cls.__name__: cls}), + ) + + def _decorate_table(self, table_class: type, context: dict[str, Any], assert_declared: bool = False) -> None: + """ + Assign schema properties to the table class and declare the table. + + Parameters + ---------- + table_class : type + Table class to decorate. + context : dict + Declaration context for foreign key resolution. + assert_declared : bool, optional + If True, assert table is already declared. Default False. + """ + table_class.database = self.database + table_class._connection = self.connection + table_class._heading = Heading( + table_info=dict( + conn=self.connection, + database=self.database, + table_name=table_class.table_name, + context=context, + ) + ) + table_class._support = [table_class.full_table_name] + table_class.declaration_context = context + + # instantiate the class, declare the table if not already + instance = table_class() + is_declared = instance.is_declared + if not is_declared and not assert_declared and self.create_tables: + instance.declare(context) + self.connection.dependencies.clear() + is_declared = is_declared or instance.is_declared + + # add table definition to the doc string + if isinstance(table_class.definition, str): + table_class.__doc__ = (table_class.__doc__ or "") + "\nTable definition:\n\n" + table_class.definition + + # fill values in Lookup tables from their contents property + if isinstance(instance, Lookup) and hasattr(instance, "contents") and is_declared: + contents = list(instance.contents) + if len(contents) > len(instance): + if instance.heading.has_autoincrement: + warnings.warn( + ("Contents has changed but cannot be inserted because {table} has autoincrement.").format( + table=instance.__class__.__name__ + ) + ) + else: + instance.insert(contents, skip_duplicates=True) + + def __repr__(self): + return "Schema `{name}`\n".format(name=self.database) + + @property + def size_on_disk(self) -> int: + """ + Return the total size of all tables in the schema. + + Returns + ------- + int + Size in bytes (data + indices). + """ + self._assert_exists() + return int( + self.connection.query( + """ + SELECT SUM(data_length + index_length) + FROM information_schema.tables WHERE table_schema='{db}' + """.format(db=self.database) + ).fetchone()[0] + ) + + def spawn_missing_classes(self, context: dict[str, Any] | None = None) -> None: + """ + Create Python table classes for tables without existing classes. + + Introspects the database schema and creates appropriate Python classes + (Lookup, Manual, Imported, Computed, Part) for tables that don't have + corresponding classes in the context. + + Parameters + ---------- + context : dict, optional + Namespace to place created classes into. Defaults to caller's + local namespace. + """ + self._assert_exists() + if context is None: + if self.context is not None: + context = self.context + else: + # if context is missing, use the calling namespace + frame = inspect.currentframe().f_back + context = frame.f_locals + del frame + tables = [ + row[0] + for row in self.connection.query("SHOW TABLES in `%s`" % self.database) + if lookup_class_name("`{db}`.`{tab}`".format(db=self.database, tab=row[0]), context, 0) is None + ] + master_classes = (Lookup, Manual, Imported, Computed) + part_tables = [] + for table_name in tables: + class_name = to_camel_case(table_name) + if class_name not in context: + try: + cls = next(cls for cls in master_classes if re.fullmatch(cls.tier_regexp, table_name)) + except StopIteration: + if re.fullmatch(Part.tier_regexp, table_name): + part_tables.append(table_name) + else: + # declare and decorate master table classes + context[class_name] = self(type(class_name, (cls,), dict()), context=context) + + # attach parts to masters + for table_name in part_tables: + groups = re.fullmatch(Part.tier_regexp, table_name).groupdict() + class_name = to_camel_case(groups["part"]) + try: + master_class = context[to_camel_case(groups["master"])] + except KeyError: + raise DataJointError("The table %s does not follow DataJoint naming conventions" % table_name) + part_class = type(class_name, (Part,), dict(definition=...)) + part_class._master = master_class + self._decorate_table(part_class, context=context, assert_declared=True) + setattr(master_class, class_name, part_class) + + def drop(self, prompt: bool | None = None) -> None: + """ + Drop the associated schema and all its tables. + + Parameters + ---------- + prompt : bool, optional + If True, show confirmation prompt before dropping. + If False, drop without confirmation. + If None (default), use ``dj.config['safemode']`` setting. + + Raises + ------ + AccessError + If insufficient permissions to drop the schema. + """ + prompt = config["safemode"] if prompt is None else prompt + + if not self.exists: + logger.info("Schema named `{database}` does not exist. Doing nothing.".format(database=self.database)) + elif not prompt or user_choice("Proceed to delete entire schema `%s`?" % self.database, default="no") == "yes": + logger.debug("Dropping `{database}`.".format(database=self.database)) + try: + self.connection.query("DROP DATABASE `{database}`".format(database=self.database)) + logger.debug("Schema `{database}` was dropped successfully.".format(database=self.database)) + except AccessError: + raise AccessError( + "An attempt to drop schema `{database}` has failed. Check permissions.".format(database=self.database) + ) + + @property + def exists(self) -> bool: + """ + Check if the associated schema exists on the server. + + Returns + ------- + bool + True if the schema exists. + + Raises + ------ + DataJointError + If schema has not been activated. + """ + if self.database is None: + raise DataJointError("Schema must be activated first.") + return bool( + self.connection.query( + "SELECT schema_name FROM information_schema.schemata WHERE schema_name = '{database}'".format( + database=self.database + ) + ).rowcount + ) + + @property + def lineage_table_exists(self) -> bool: + """ + Check if the ~lineage table exists in this schema. + + Returns + ------- + bool + True if the lineage table exists. + """ + from .lineage import lineage_table_exists + + self._assert_exists() + return lineage_table_exists(self.connection, self.database) + + @property + def lineage(self) -> dict[str, str]: + """ + Get all lineages for tables in this schema. + + Returns + ------- + dict[str, str] + Mapping of ``'schema.table.attribute'`` to its lineage origin. + """ + from .lineage import get_schema_lineages + + self._assert_exists() + return get_schema_lineages(self.connection, self.database) + + def rebuild_lineage(self) -> None: + """ + Rebuild the ~lineage table for all tables in this schema. + + Recomputes lineage for all attributes by querying FK relationships + from the information_schema. Use to restore lineage for schemas that + predate the lineage system or after corruption. + + Notes + ----- + After rebuilding, restart the Python kernel and reimport to pick up + the new lineage information. + + Upstream schemas (referenced via cross-schema foreign keys) must + have their lineage rebuilt first. + """ + from .lineage import rebuild_schema_lineage + + self._assert_exists() + rebuild_schema_lineage(self.connection, self.database) + + @property + def jobs(self) -> list[Job]: + """ + Return Job objects for auto-populated tables with job tables. + + Only returns Job objects when both the target table and its + ``~~table_name`` job table exist in the database. Job tables are + created lazily on first access to ``table.jobs`` or + ``populate(reserve_jobs=True)``. + + Returns + ------- + list[Job] + Job objects for existing job tables. + """ + self._assert_exists() + jobs_list = [] + + # Get all existing job tables (~~prefix) + # Note: %% escapes the % in pymysql + result = self.connection.query(f"SHOW TABLES IN `{self.database}` LIKE '~~%%'").fetchall() + existing_job_tables = {row[0] for row in result} + + # Iterate over auto-populated tables and check if their job table exists + for table_name in self.list_tables(): + table = FreeTable(self.connection, f"`{self.database}`.`{table_name}`") + tier = _get_tier(table.full_table_name) + if tier in (Computed, Imported): + # Compute expected job table name: ~~base_name + base_name = table_name.lstrip("_") + job_table_name = f"~~{base_name}" + if job_table_name in existing_job_tables: + jobs_list.append(Job(table)) + + return jobs_list + + @property + def code(self): + self._assert_exists() + return self.save() + + def save(self, python_filename: str | None = None) -> str: + """ + Generate Python code that recreates this schema. + + Parameters + ---------- + python_filename : str, optional + If provided, write the code to this file. + + Returns + ------- + str + Python module source code defining this schema. + + Notes + ----- + This method is in preparation for a future release and is not + officially supported. + """ + self.connection.dependencies.load() + self._assert_exists() + module_count = itertools.count() + # add virtual modules for referenced modules with names vmod0, vmod1, ... + module_lookup = collections.defaultdict(lambda: "vmod" + str(next(module_count))) + db = self.database + + def make_class_definition(table): + tier = _get_tier(table).__name__ + class_name = table.split(".")[1].strip("`") + indent = "" + if tier == "Part": + class_name = class_name.split("__")[-1] + indent += " " + class_name = to_camel_case(class_name) + + def replace(s): + d, tabs = s.group(1), s.group(2) + return ("" if d == db else (module_lookup[d] + ".")) + ".".join( + to_camel_case(tab) for tab in tabs.lstrip("__").split("__") + ) + + return ("" if tier == "Part" else "\n@schema\n") + ( + '{indent}class {class_name}(dj.{tier}):\n{indent} definition = """\n{indent} {defi}"""' + ).format( + class_name=class_name, + indent=indent, + tier=tier, + defi=re.sub( + r"`([^`]+)`.`([^`]+)`", + replace, + FreeTable(self.connection, table).describe(), + ).replace("\n", "\n " + indent), + ) + + tables = self.connection.dependencies.topo_sort() + body = "\n\n".join(make_class_definition(table) for table in tables) + python_code = "\n\n".join( + ( + '"""This module was auto-generated by datajoint from an existing schema"""', + "import datajoint as dj\n\nschema = dj.Schema('{db}')".format(db=db), + "\n".join( + "{module} = dj.VirtualModule('{module}', '{schema_name}')".format(module=v, schema_name=k) + for k, v in module_lookup.items() + ), + body, + ) + ) + if python_filename is None: + return python_code + with open(python_filename, "wt") as f: + f.write(python_code) + + def list_tables(self) -> list[str]: + """ + Return all user tables in the schema. + + Excludes hidden tables (starting with ``~``) such as ``~lineage`` + and job tables (``~~``). + + Returns + ------- + list[str] + Table names in topological order. + """ + self.connection.dependencies.load() + return [ + t + for d, t in (table_name.replace("`", "").split(".") for table_name in self.connection.dependencies.topo_sort()) + if d == self.database + ] + + def _find_table_name(self, name: str) -> str | None: + """ + Find the actual SQL table name for a given base name. + + Handles tier prefixes: Manual (none), Lookup (#), Imported (_), Computed (__). + + Parameters + ---------- + name : str + Base table name without tier prefix. + + Returns + ------- + str or None + The actual SQL table name, or None if not found. + """ + tables = self.list_tables() + # Check exact match first + if name in tables: + return name + # Check with tier prefixes + for prefix in ("", "#", "_", "__"): + candidate = f"{prefix}{name}" + if candidate in tables: + return candidate + return None + + def get_table(self, name: str) -> FreeTable: + """ + Get a table instance by name. + + Returns a FreeTable instance for the given table name. This is useful + for accessing tables when you don't have the Python class available. + + Parameters + ---------- + name : str + Table name (e.g., 'experiment', 'session__trial' for parts). + Can be snake_case (SQL name) or CamelCase (class name). + Tier prefixes are optional and will be auto-detected. + + Returns + ------- + FreeTable + A FreeTable instance for the table. + + Raises + ------ + DataJointError + If the table does not exist. + + Examples + -------- + >>> schema = dj.Schema('my_schema') + >>> experiment = schema.get_table('experiment') + >>> experiment.fetch() + """ + self._assert_exists() + # Convert CamelCase to snake_case if needed + if name[0].isupper(): + name = re.sub(r"(? FreeTable: + """ + Get a table instance by name using bracket notation. + + Parameters + ---------- + name : str + Table name (snake_case or CamelCase). + + Returns + ------- + FreeTable + A FreeTable instance for the table. + + Examples + -------- + >>> schema = dj.Schema('my_schema') + >>> schema['Experiment'].fetch() + >>> schema['session'].fetch() + """ + return self.get_table(name) + + def __iter__(self): + """ + Iterate over all tables in the schema. + + Yields FreeTable instances for each table in topological order. + + Yields + ------ + FreeTable + Table instances in dependency order. + + Examples + -------- + >>> for table in schema: + ... print(table.full_table_name, len(table)) + """ + self._assert_exists() + for table_name in self.list_tables(): + yield self.get_table(table_name) + + def __contains__(self, name: str) -> bool: + """ + Check if a table exists in the schema. + + Parameters + ---------- + name : str + Table name (snake_case or CamelCase). + Tier prefixes are optional and will be auto-detected. + + Returns + ------- + bool + True if the table exists. + + Examples + -------- + >>> 'Experiment' in schema + True + """ + if name[0].isupper(): + name = re.sub(r"(?>> lab = dj.VirtualModule('lab', 'my_lab_schema') + >>> lab.Subject.fetch() + """ + + def __init__( + self, + module_name: str, + schema_name: str, + *, + create_schema: bool = False, + create_tables: bool = False, + connection: Connection | None = None, + add_objects: dict[str, Any] | None = None, + ) -> None: + """ + Initialize the virtual module. + + Parameters + ---------- + module_name : str + Display name for the module. + schema_name : str + Database schema name. + create_schema : bool, optional + If True, create the schema if it doesn't exist. Default False. + create_tables : bool, optional + If True, allow declaring new tables. Default False. + connection : Connection, optional + Database connection. Defaults to ``dj.conn()``. + add_objects : dict, optional + Additional objects to add to the module namespace. + """ + super(VirtualModule, self).__init__(name=module_name) + _schema = Schema( + schema_name, + create_schema=create_schema, + create_tables=create_tables, + connection=connection, + ) + if add_objects: + self.__dict__.update(add_objects) + self.__dict__["schema"] = _schema + _schema.spawn_missing_classes(context=self.__dict__) + + +def list_schemas(connection: Connection | None = None) -> list[str]: + """ + List all accessible schemas on the server. + + Parameters + ---------- + connection : Connection, optional + Database connection. Defaults to ``dj.conn()``. + + Returns + ------- + list[str] + Names of all accessible schemas. + """ + return [ + r[0] + for r in (connection or conn()).query( + 'SELECT schema_name FROM information_schema.schemata WHERE schema_name <> "information_schema"' + ) + ] + + +def virtual_schema( + schema_name: str, + *, + connection: Connection | None = None, + create_schema: bool = False, + create_tables: bool = False, + add_objects: dict[str, Any] | None = None, +) -> VirtualModule: + """ + Create a virtual module for an existing database schema. + + This is the recommended way to access database schemas when you don't have + the Python source code that defined them. Returns a module-like object with + table classes as attributes. + + Parameters + ---------- + schema_name : str + Database schema name. + connection : Connection, optional + Database connection. Defaults to ``dj.conn()``. + create_schema : bool, optional + If True, create the schema if it doesn't exist. Default False. + create_tables : bool, optional + If True, allow declaring new tables. Default False. + add_objects : dict, optional + Additional objects to add to the module namespace. + + Returns + ------- + VirtualModule + A module-like object with table classes as attributes. + + Examples + -------- + >>> lab = dj.virtual_schema('my_lab') + >>> lab.Subject.fetch() + >>> lab.Session & 'subject_id="M001"' + + See Also + -------- + Schema : For defining new schemas with Python classes. + VirtualModule : The underlying class (prefer virtual_schema function). + """ + return VirtualModule( + schema_name, + schema_name, + connection=connection, + create_schema=create_schema, + create_tables=create_tables, + add_objects=add_objects, + ) diff --git a/src/datajoint/settings.py b/src/datajoint/settings.py new file mode 100644 index 000000000..5812f2257 --- /dev/null +++ b/src/datajoint/settings.py @@ -0,0 +1,976 @@ +""" +DataJoint configuration system using pydantic-settings. + +This module provides strongly-typed configuration with automatic loading +from environment variables, secrets directories, and JSON config files. + +Configuration sources (in priority order): + +1. Environment variables (``DJ_*``) +2. Secrets directories (``.secrets/`` in project, ``/run/secrets/datajoint/``) +3. Project config file (``datajoint.json``, searched recursively up to ``.git/.hg``) + +Examples +-------- +>>> import datajoint as dj +>>> dj.config.database.host +'localhost' +>>> with dj.config.override(safemode=False): +... # dangerous operations here +... pass + +Project structure:: + + myproject/ + ├── .git/ + ├── datajoint.json # Project config (commit this) + ├── .secrets/ # Local secrets (gitignore this) + │ ├── database.password + │ └── aws.secret_access_key + └── src/ + └── analysis.py # Config found via parent search +""" + +from __future__ import annotations + +import json +import logging +import os +import warnings +from contextlib import contextmanager +from copy import deepcopy +from enum import Enum +from pathlib import Path +from typing import Any, Iterator, Literal + +from pydantic import Field, SecretStr, field_validator +from pydantic_settings import BaseSettings, SettingsConfigDict + +from .errors import DataJointError + +CONFIG_FILENAME = "datajoint.json" +SECRETS_DIRNAME = ".secrets" +SYSTEM_SECRETS_DIR = Path("/run/secrets/datajoint") +DEFAULT_SUBFOLDING = (2, 2) + +# Mapping of config keys to environment variables +# Environment variables take precedence over config file values +ENV_VAR_MAPPING = { + "database.host": "DJ_HOST", + "database.user": "DJ_USER", + "database.password": "DJ_PASS", + "database.port": "DJ_PORT", + "external.aws_access_key_id": "DJ_AWS_ACCESS_KEY_ID", + "external.aws_secret_access_key": "DJ_AWS_SECRET_ACCESS_KEY", + "loglevel": "DJ_LOG_LEVEL", +} + +Role = Enum("Role", "manual lookup imported computed job") +role_to_prefix = { + Role.manual: "", + Role.lookup: "#", + Role.imported: "_", + Role.computed: "__", + Role.job: "~", +} +prefix_to_role = dict(zip(role_to_prefix.values(), role_to_prefix)) + +logger = logging.getLogger(__name__.split(".")[0]) + + +def find_config_file(start: Path | None = None) -> Path | None: + """ + Search for datajoint.json in current and parent directories. + + Searches upward from ``start`` until finding the config file or hitting + a project boundary (``.git``, ``.hg``) or filesystem root. + + Parameters + ---------- + start : Path, optional + Directory to start search from. Defaults to current working directory. + + Returns + ------- + Path or None + Path to config file if found, None otherwise. + """ + current = (start or Path.cwd()).resolve() + + while True: + config_path = current / CONFIG_FILENAME + if config_path.is_file(): + return config_path + + # Stop at project/repo root + if (current / ".git").exists() or (current / ".hg").exists(): + return None + + # Stop at filesystem root + if current == current.parent: + return None + + current = current.parent + + +def find_secrets_dir(config_path: Path | None = None) -> Path | None: + """ + Find the secrets directory. + + Priority: + + 1. ``.secrets/`` in same directory as datajoint.json (project secrets) + 2. ``/run/secrets/datajoint/`` (Docker/Kubernetes secrets) + + Parameters + ---------- + config_path : Path, optional + Path to datajoint.json if found. + + Returns + ------- + Path or None + Path to secrets directory if found, None otherwise. + """ + # Check project secrets directory (next to config file) + if config_path is not None: + project_secrets = config_path.parent / SECRETS_DIRNAME + if project_secrets.is_dir(): + return project_secrets + + # Check system secrets directory (Docker/Kubernetes) + if SYSTEM_SECRETS_DIR.is_dir(): + return SYSTEM_SECRETS_DIR + + return None + + +def read_secret_file(secrets_dir: Path | None, name: str) -> str | None: + """ + Read a secret value from a file in the secrets directory. + + Parameters + ---------- + secrets_dir : Path or None + Path to secrets directory. + name : str + Name of the secret file (e.g., ``'database.password'``). + + Returns + ------- + str or None + Secret value as string, or None if not found. + """ + if secrets_dir is None: + return None + + secret_path = secrets_dir / name + if secret_path.is_file(): + return secret_path.read_text().strip() + + return None + + +class DatabaseSettings(BaseSettings): + """Database connection settings.""" + + model_config = SettingsConfigDict( + env_prefix="DJ_", + case_sensitive=False, + extra="forbid", + validate_assignment=True, + ) + + host: str = Field(default="localhost", validation_alias="DJ_HOST") + user: str | None = Field(default=None, validation_alias="DJ_USER") + password: SecretStr | None = Field(default=None, validation_alias="DJ_PASS") + port: int = Field(default=3306, validation_alias="DJ_PORT") + reconnect: bool = True + use_tls: bool | None = None + + +class ConnectionSettings(BaseSettings): + """Connection behavior settings.""" + + model_config = SettingsConfigDict(extra="forbid", validate_assignment=True) + + init_function: str | None = None + charset: str = "" # pymysql uses '' as default + + +class DisplaySettings(BaseSettings): + """Display and preview settings.""" + + model_config = SettingsConfigDict(extra="forbid", validate_assignment=True) + + limit: int = 12 + width: int = 14 + show_tuple_count: bool = True + + +class ExternalSettings(BaseSettings): + """External storage credentials.""" + + model_config = SettingsConfigDict( + env_prefix="DJ_", + case_sensitive=False, + extra="forbid", + validate_assignment=True, + ) + + aws_access_key_id: str | None = Field(default=None, validation_alias="DJ_AWS_ACCESS_KEY_ID") + aws_secret_access_key: SecretStr | None = Field(default=None, validation_alias="DJ_AWS_SECRET_ACCESS_KEY") + + +class JobsSettings(BaseSettings): + """Job queue configuration for AutoPopulate 2.0.""" + + model_config = SettingsConfigDict( + env_prefix="DJ_JOBS_", + case_sensitive=False, + extra="forbid", + validate_assignment=True, + ) + + auto_refresh: bool = Field(default=True, description="Auto-refresh jobs queue on populate") + keep_completed: bool = Field(default=False, description="Keep success records in jobs table") + stale_timeout: int = Field(default=3600, ge=0, description="Seconds before pending job is checked for staleness") + default_priority: int = Field(default=5, ge=0, le=255, description="Default priority for new jobs (lower = more urgent)") + version_method: Literal["git", "none"] | None = Field( + default=None, description="Method to obtain version: 'git' (commit hash), 'none' (empty), or None (disabled)" + ) + allow_new_pk_fields_in_computed_tables: bool = Field( + default=False, + description="Allow native (non-FK) primary key fields in Computed/Imported tables. " + "When True, bypasses the FK-only PK validation. Job granularity will be degraded for such tables.", + ) + add_job_metadata: bool = Field( + default=False, + description="Add hidden job metadata attributes (_job_start_time, _job_duration, _job_version) " + "to Computed and Imported tables during declaration. Tables created without this setting " + "will not receive metadata updates during populate.", + ) + + +class ObjectStorageSettings(BaseSettings): + """Object storage configuration for the object type.""" + + model_config = SettingsConfigDict( + env_prefix="DJ_OBJECT_STORAGE_", + case_sensitive=False, + extra="forbid", + validate_assignment=True, + ) + + # Required settings + project_name: str | None = Field(default=None, description="Unique project identifier") + protocol: str | None = Field(default=None, description="Storage protocol: file, s3, gcs, azure") + location: str | None = Field(default=None, description="Base path or bucket prefix") + + # Cloud storage settings + bucket: str | None = Field(default=None, description="Bucket name (S3, GCS)") + container: str | None = Field(default=None, description="Container name (Azure)") + endpoint: str | None = Field(default=None, description="S3 endpoint URL") + access_key: str | None = Field(default=None, description="Access key") + secret_key: SecretStr | None = Field(default=None, description="Secret key") + secure: bool = Field(default=True, description="Use HTTPS") + + # Optional settings + default_store: str | None = Field(default=None, description="Default store name when not specified") + partition_pattern: str | None = Field(default=None, description="Path pattern with {attribute} placeholders") + token_length: int = Field(default=8, ge=4, le=16, description="Random suffix length for filenames") + + # Named stores configuration (object_storage.stores..*) + stores: dict[str, dict[str, Any]] = Field(default_factory=dict, description="Named object stores") + + +class Config(BaseSettings): + """ + Main DataJoint configuration. + + Settings are loaded from (in priority order): + + 1. Environment variables (``DJ_*``) + 2. Secrets directory (``.secrets/`` or ``/run/secrets/datajoint/``) + 3. Config file (``datajoint.json``, searched in parent directories) + 4. Default values + + Examples + -------- + Access settings via attributes: + + >>> config.database.host + >>> config.safemode + + Override temporarily with context manager: + + >>> with config.override(safemode=False): + ... pass + """ + + model_config = SettingsConfigDict( + env_prefix="DJ_", + case_sensitive=False, + extra="forbid", + validate_assignment=True, + ) + + # Nested settings groups + database: DatabaseSettings = Field(default_factory=DatabaseSettings) + connection: ConnectionSettings = Field(default_factory=ConnectionSettings) + display: DisplaySettings = Field(default_factory=DisplaySettings) + external: ExternalSettings = Field(default_factory=ExternalSettings) + jobs: JobsSettings = Field(default_factory=JobsSettings) + object_storage: ObjectStorageSettings = Field(default_factory=ObjectStorageSettings) + + # Top-level settings + loglevel: Literal["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"] = Field(default="INFO", validation_alias="DJ_LOG_LEVEL") + safemode: bool = True + enable_python_native_blobs: bool = True + filepath_checksum_size_limit: int | None = None + + # External stores configuration + stores: dict[str, dict[str, Any]] = Field(default_factory=dict) + + # Cache paths + cache: Path | None = None + query_cache: Path | None = None + + # Download path for attachments and filepaths + download_path: str = "." + + # Internal: track where config was loaded from + _config_path: Path | None = None + _secrets_dir: Path | None = None + + @field_validator("loglevel", mode="after") + @classmethod + def set_logger_level(cls, v: str) -> str: + """Update logger level when loglevel changes.""" + logger.setLevel(v) + return v + + @field_validator("cache", "query_cache", mode="before") + @classmethod + def convert_path(cls, v: Any) -> Path | None: + """Convert string paths to Path objects.""" + if v is None: + return None + return Path(v) if not isinstance(v, Path) else v + + def get_store_spec(self, store: str) -> dict[str, Any]: + """ + Get configuration for an external store. + + Parameters + ---------- + store : str + Name of the store to retrieve. + + Returns + ------- + dict[str, Any] + Store configuration dict with validated fields. + + Raises + ------ + DataJointError + If store is not configured or has invalid config. + """ + if store not in self.stores: + raise DataJointError(f"Storage '{store}' is requested but not configured") + + spec = dict(self.stores[store]) + spec.setdefault("subfolding", DEFAULT_SUBFOLDING) + + # Validate protocol + protocol = spec.get("protocol", "").lower() + supported_protocols = ("file", "s3", "gcs", "azure") + if protocol not in supported_protocols: + raise DataJointError( + f'Missing or invalid protocol in config.stores["{store}"]. ' + f"Supported protocols: {', '.join(supported_protocols)}" + ) + + # Define required and allowed keys by protocol + required_keys: dict[str, tuple[str, ...]] = { + "file": ("protocol", "location"), + "s3": ("protocol", "endpoint", "bucket", "access_key", "secret_key", "location"), + "gcs": ("protocol", "bucket", "location"), + "azure": ("protocol", "container", "location"), + } + allowed_keys: dict[str, tuple[str, ...]] = { + "file": ("protocol", "location", "subfolding", "stage"), + "s3": ( + "protocol", + "endpoint", + "bucket", + "access_key", + "secret_key", + "location", + "secure", + "subfolding", + "stage", + "proxy_server", + ), + "gcs": ( + "protocol", + "bucket", + "location", + "token", + "project", + "subfolding", + "stage", + ), + "azure": ( + "protocol", + "container", + "location", + "account_name", + "account_key", + "connection_string", + "subfolding", + "stage", + ), + } + + # Check required keys + missing = [k for k in required_keys[protocol] if k not in spec] + if missing: + raise DataJointError(f'config.stores["{store}"] is missing: {", ".join(missing)}') + + # Check for invalid keys + invalid = [k for k in spec if k not in allowed_keys[protocol]] + if invalid: + raise DataJointError(f'Invalid key(s) in config.stores["{store}"]: {", ".join(invalid)}') + + return spec + + def get_object_storage_spec(self) -> dict[str, Any]: + """ + Get validated object storage configuration. + + Returns + ------- + dict[str, Any] + Object storage configuration dict. + + Raises + ------ + DataJointError + If object storage is not configured or has invalid config. + """ + os_settings = self.object_storage + + # Check if object storage is configured + if not os_settings.protocol: + raise DataJointError( + "Object storage is not configured. Set object_storage.protocol in datajoint.json " + "or DJ_OBJECT_STORAGE_PROTOCOL environment variable." + ) + + if not os_settings.project_name: + raise DataJointError( + "Object storage project_name is required. Set object_storage.project_name in datajoint.json " + "or DJ_OBJECT_STORAGE_PROJECT_NAME environment variable." + ) + + protocol = os_settings.protocol.lower() + supported_protocols = ("file", "s3", "gcs", "azure") + if protocol not in supported_protocols: + raise DataJointError( + f"Invalid object_storage.protocol: {protocol}. Supported protocols: {', '.join(supported_protocols)}" + ) + + # Build spec dict + spec = { + "project_name": os_settings.project_name, + "protocol": protocol, + "location": os_settings.location or "", + "partition_pattern": os_settings.partition_pattern, + "token_length": os_settings.token_length, + } + + # Add protocol-specific settings + if protocol == "s3": + if not os_settings.endpoint or not os_settings.bucket: + raise DataJointError("object_storage.endpoint and object_storage.bucket are required for S3") + if not os_settings.access_key or not os_settings.secret_key: + raise DataJointError("object_storage.access_key and object_storage.secret_key are required for S3") + spec.update( + { + "endpoint": os_settings.endpoint, + "bucket": os_settings.bucket, + "access_key": os_settings.access_key, + "secret_key": os_settings.secret_key.get_secret_value() if os_settings.secret_key else None, + "secure": os_settings.secure, + } + ) + elif protocol == "gcs": + if not os_settings.bucket: + raise DataJointError("object_storage.bucket is required for GCS") + spec["bucket"] = os_settings.bucket + elif protocol == "azure": + if not os_settings.container: + raise DataJointError("object_storage.container is required for Azure") + spec["container"] = os_settings.container + + return spec + + def get_object_store_spec(self, store_name: str | None = None) -> dict[str, Any]: + """ + Get validated configuration for a specific object store. + + Parameters + ---------- + store_name : str, optional + Name of the store. None for default store. + + Returns + ------- + dict[str, Any] + Object store configuration dict. + + Raises + ------ + DataJointError + If store is not configured or has invalid config. + """ + if store_name is None: + # Return default store spec + return self.get_object_storage_spec() + + os_settings = self.object_storage + + # Check if named store exists + if store_name not in os_settings.stores: + raise DataJointError( + f"Object store '{store_name}' is not configured. " + f"Add object_storage.stores.{store_name}.* settings to datajoint.json" + ) + + store_config = os_settings.stores[store_name] + protocol = store_config.get("protocol", "").lower() + + supported_protocols = ("file", "s3", "gcs", "azure") + if protocol not in supported_protocols: + raise DataJointError( + f"Invalid protocol for store '{store_name}': {protocol}. Supported protocols: {', '.join(supported_protocols)}" + ) + + # Use project_name from default config if not specified in store + project_name = store_config.get("project_name") or os_settings.project_name + if not project_name: + raise DataJointError( + f"project_name is required for object store '{store_name}'. " + "Set object_storage.project_name or object_storage.stores.{store_name}.project_name" + ) + + # Build spec dict + spec = { + "project_name": project_name, + "protocol": protocol, + "location": store_config.get("location", ""), + "partition_pattern": store_config.get("partition_pattern") or os_settings.partition_pattern, + "token_length": store_config.get("token_length") or os_settings.token_length, + "store_name": store_name, + } + + # Add protocol-specific settings + if protocol == "s3": + endpoint = store_config.get("endpoint") + bucket = store_config.get("bucket") + if not endpoint or not bucket: + raise DataJointError(f"endpoint and bucket are required for S3 store '{store_name}'") + spec.update( + { + "endpoint": endpoint, + "bucket": bucket, + "access_key": store_config.get("access_key"), + "secret_key": store_config.get("secret_key"), + "secure": store_config.get("secure", True), + } + ) + elif protocol == "gcs": + bucket = store_config.get("bucket") + if not bucket: + raise DataJointError(f"bucket is required for GCS store '{store_name}'") + spec["bucket"] = bucket + elif protocol == "azure": + container = store_config.get("container") + if not container: + raise DataJointError(f"container is required for Azure store '{store_name}'") + spec["container"] = container + + return spec + + def load(self, filename: str | Path) -> None: + """ + Load settings from a JSON file. + + Parameters + ---------- + filename : str or Path + Path to load configuration from. + """ + filepath = Path(filename) + if not filepath.exists(): + raise FileNotFoundError(f"Config file not found: {filepath}") + + logger.info(f"Loading configuration from {filepath.absolute()}") + + with open(filepath) as f: + data = json.load(f) + + self._update_from_flat_dict(data) + self._config_path = filepath + + def _update_from_flat_dict(self, data: dict[str, Any]) -> None: + """ + Update settings from a dict (flat dot-notation or nested). + + Environment variables take precedence over config file values. + If an env var is set for a setting, the file value is skipped. + """ + for key, value in data.items(): + # Handle nested dicts by recursively updating + if isinstance(value, dict) and hasattr(self, key): + group_obj = getattr(self, key) + for nested_key, nested_value in value.items(): + if hasattr(group_obj, nested_key): + # Check if env var is set for this nested key + full_key = f"{key}.{nested_key}" + env_var = ENV_VAR_MAPPING.get(full_key) + if env_var and os.environ.get(env_var): + logger.debug(f"Skipping {full_key} from file (env var {env_var} takes precedence)") + continue + setattr(group_obj, nested_key, nested_value) + continue + + # Handle flat dot-notation keys + parts = key.split(".") + if len(parts) == 1: + if hasattr(self, key) and not key.startswith("_"): + # Check if env var is set for this key + env_var = ENV_VAR_MAPPING.get(key) + if env_var and os.environ.get(env_var): + logger.debug(f"Skipping {key} from file (env var {env_var} takes precedence)") + continue + setattr(self, key, value) + elif len(parts) == 2: + group, attr = parts + if hasattr(self, group): + group_obj = getattr(self, group) + if hasattr(group_obj, attr): + # Check if env var is set for this key + env_var = ENV_VAR_MAPPING.get(key) + if env_var and os.environ.get(env_var): + logger.debug(f"Skipping {key} from file (env var {env_var} takes precedence)") + continue + setattr(group_obj, attr, value) + elif len(parts) == 4: + # Handle object_storage.stores.. pattern + group, subgroup, store_name, attr = parts + if group == "object_storage" and subgroup == "stores": + if store_name not in self.object_storage.stores: + self.object_storage.stores[store_name] = {} + self.object_storage.stores[store_name][attr] = value + + def _load_secrets(self, secrets_dir: Path) -> None: + """Load secrets from a secrets directory.""" + self._secrets_dir = secrets_dir + + # Map of secret file names to config paths + secret_mappings = { + "database.password": ("database", "password"), + "database.user": ("database", "user"), + "aws.access_key_id": ("external", "aws_access_key_id"), + "aws.secret_access_key": ("external", "aws_secret_access_key"), + } + + for secret_name, (group, attr) in secret_mappings.items(): + value = read_secret_file(secrets_dir, secret_name) + if value is not None: + group_obj = getattr(self, group) + # Only set if not already set by env var + if getattr(group_obj, attr) is None: + setattr(group_obj, attr, value) + logger.debug(f"Loaded secret '{secret_name}' from {secrets_dir}") + + @contextmanager + def override(self, **kwargs: Any) -> Iterator["Config"]: + """ + Temporarily override configuration values. + + Parameters + ---------- + **kwargs : Any + Settings to override. Use double underscore for nested settings + (e.g., ``database__host="localhost"``). + + Yields + ------ + Config + The config instance with overridden values. + + Examples + -------- + >>> with config.override(safemode=False, database__host="test"): + ... # config.safemode is False here + ... pass + >>> # config.safemode is restored + """ + # Store original values + backup = {} + + # Convert double underscore to nested access + converted = {} + for key, value in kwargs.items(): + if "__" in key: + parts = key.split("__") + converted[tuple(parts)] = value + else: + converted[(key,)] = value + + try: + # Save originals and apply overrides + for key_parts, value in converted.items(): + if len(key_parts) == 1: + key = key_parts[0] + if hasattr(self, key): + backup[key_parts] = deepcopy(getattr(self, key)) + setattr(self, key, value) + elif len(key_parts) == 2: + group, attr = key_parts + if hasattr(self, group): + group_obj = getattr(self, group) + if hasattr(group_obj, attr): + backup[key_parts] = deepcopy(getattr(group_obj, attr)) + setattr(group_obj, attr, value) + + yield self + + finally: + # Restore original values + for key_parts, original in backup.items(): + if len(key_parts) == 1: + setattr(self, key_parts[0], original) + elif len(key_parts) == 2: + group, attr = key_parts + setattr(getattr(self, group), attr, original) + + @staticmethod + def save_template( + path: str | Path = "datajoint.json", + minimal: bool = True, + create_secrets_dir: bool = True, + ) -> Path: + """ + Create a template datajoint.json configuration file. + + Credentials should NOT be stored in datajoint.json. Instead, use either: + + - Environment variables (``DJ_USER``, ``DJ_PASS``, ``DJ_HOST``, etc.) + - The ``.secrets/`` directory (created alongside datajoint.json) + + Parameters + ---------- + path : str or Path, optional + Where to save the template. Default ``'datajoint.json'``. + minimal : bool, optional + If True (default), create minimal template with just database settings. + If False, create full template with all available settings. + create_secrets_dir : bool, optional + If True (default), also create a ``.secrets/`` directory with + template files for credentials. + + Returns + ------- + Path + Absolute path to the created config file. + + Raises + ------ + FileExistsError + If config file already exists (won't overwrite). + + Examples + -------- + >>> import datajoint as dj + >>> dj.config.save_template() # Creates minimal template + .secrets/ + >>> dj.config.save_template("full-config.json", minimal=False) + """ + filepath = Path(path) + if filepath.exists(): + raise FileExistsError(f"File already exists: {filepath}. Remove it first or choose a different path.") + + if minimal: + template = { + "database": { + "host": "localhost", + "port": 3306, + }, + } + else: + template = { + "database": { + "host": "localhost", + "port": 3306, + "reconnect": True, + "use_tls": None, + }, + "connection": { + "init_function": None, + "charset": "", + }, + "display": { + "limit": 12, + "width": 14, + "show_tuple_count": True, + }, + "object_storage": { + "project_name": None, + "protocol": None, + "location": None, + "bucket": None, + "endpoint": None, + "secure": True, + "partition_pattern": None, + "token_length": 8, + }, + "stores": {}, + "loglevel": "INFO", + "safemode": True, + "enable_python_native_blobs": True, + "cache": None, + "query_cache": None, + "download_path": ".", + } + + with open(filepath, "w") as f: + json.dump(template, f, indent=2) + f.write("\n") + + logger.info(f"Created template configuration at {filepath.absolute()}") + + # Create .secrets/ directory with template files + if create_secrets_dir: + secrets_dir = filepath.parent / SECRETS_DIRNAME + secrets_dir.mkdir(exist_ok=True) + + # Create placeholder secret files + secret_templates = { + "database.user": "your_username", + "database.password": "your_password", + } + for secret_name, placeholder in secret_templates.items(): + secret_file = secrets_dir / secret_name + if not secret_file.exists(): + secret_file.write_text(placeholder) + + # Create .gitignore to prevent committing secrets + gitignore_path = secrets_dir / ".gitignore" + if not gitignore_path.exists(): + gitignore_path.write_text("# Never commit secrets\n*\n!.gitignore\n") + + logger.info( + f"Created {SECRETS_DIRNAME}/ directory with credential templates. " + f"Edit the files in {secrets_dir.absolute()}/ to set your credentials." + ) + + return filepath.absolute() + + # Dict-like access for convenience + def __getitem__(self, key: str) -> Any: + """Get setting by dot-notation key (e.g., 'database.host').""" + parts = key.split(".") + obj: Any = self + for part in parts: + if hasattr(obj, part): + obj = getattr(obj, part) + elif isinstance(obj, dict): + obj = obj[part] + else: + raise KeyError(f"Setting '{key}' not found") + # Unwrap SecretStr for compatibility + if isinstance(obj, SecretStr): + return obj.get_secret_value() + return obj + + def __setitem__(self, key: str, value: Any) -> None: + """Set setting by dot-notation key (e.g., 'database.host').""" + parts = key.split(".") + if len(parts) == 1: + if hasattr(self, key): + setattr(self, key, value) + else: + raise KeyError(f"Setting '{key}' not found") + else: + obj: Any = self + for part in parts[:-1]: + obj = getattr(obj, part) + setattr(obj, parts[-1], value) + + def __delitem__(self, key: str) -> None: + """Reset setting to default by dot-notation key.""" + # Get the default value from the model fields (access from class, not instance) + parts = key.split(".") + if len(parts) == 1: + field_info = type(self).model_fields.get(key) + if field_info is not None: + default = field_info.default + if default is not None: + setattr(self, key, default) + elif field_info.default_factory is not None: + setattr(self, key, field_info.default_factory()) + else: + setattr(self, key, None) + else: + raise KeyError(f"Setting '{key}' not found") + else: + # For nested settings, reset to None or empty + obj: Any = self + for part in parts[:-1]: + obj = getattr(obj, part) + setattr(obj, parts[-1], None) + + def get(self, key: str, default: Any = None) -> Any: + """Get setting with optional default value.""" + try: + return self[key] + except KeyError: + return default + + +def _create_config() -> Config: + """Create and initialize the global config instance.""" + cfg = Config() + + # Find config file (recursive parent search) + config_path = find_config_file() + + if config_path is not None: + try: + cfg.load(config_path) + except Exception as e: + warnings.warn(f"Failed to load config from {config_path}: {e}") + else: + warnings.warn( + f"No {CONFIG_FILENAME} found. Using defaults and environment variables. " + f"Run `dj.config.save_template()` to create a template configuration.", + stacklevel=2, + ) + + # Find and load secrets + secrets_dir = find_secrets_dir(config_path) + if secrets_dir is not None: + cfg._load_secrets(secrets_dir) + + # Set initial log level + logger.setLevel(cfg.loglevel) + + return cfg + + +# Global config instance +config = _create_config() diff --git a/src/datajoint/staged_insert.py b/src/datajoint/staged_insert.py new file mode 100644 index 000000000..8f9c94d2c --- /dev/null +++ b/src/datajoint/staged_insert.py @@ -0,0 +1,312 @@ +""" +Staged insert context manager for direct object storage writes. + +This module provides the StagedInsert class which allows writing directly +to object storage before finalizing the database insert. +""" + +import json +import mimetypes +from contextlib import contextmanager +from datetime import datetime, timezone +from typing import IO, Any + +import fsspec + +from .errors import DataJointError +from .settings import config +from .storage import StorageBackend, build_object_path + + +class StagedInsert: + """ + Context manager for staged insert operations. + + Allows direct writes to object storage before finalizing the database insert. + Used for large objects like Zarr arrays where copying from local storage + is inefficient. + + Usage: + with table.staged_insert1 as staged: + staged.rec['subject_id'] = 123 + staged.rec['session_id'] = 45 + + # Create object storage directly + z = zarr.open(staged.store('raw_data', '.zarr'), mode='w', shape=(1000, 1000)) + z[:] = data + + # Assign to record + staged.rec['raw_data'] = z + + # On successful exit: metadata computed, record inserted + # On exception: storage cleaned up, no record inserted + """ + + def __init__(self, table): + """ + Initialize a staged insert. + + Args: + table: The Table instance to insert into + """ + self._table = table + self._rec: dict[str, Any] = {} + self._staged_objects: dict[str, dict] = {} # field -> {path, ext, token} + self._backend: StorageBackend | None = None + + @property + def rec(self) -> dict[str, Any]: + """Record dict for setting attribute values.""" + return self._rec + + @property + def fs(self) -> fsspec.AbstractFileSystem: + """Return fsspec filesystem for advanced operations.""" + self._ensure_backend() + return self._backend.fs + + def _ensure_backend(self): + """Ensure storage backend is initialized.""" + if self._backend is None: + try: + spec = config.get_object_storage_spec() + self._backend = StorageBackend(spec) + except DataJointError: + raise DataJointError( + "Object storage is not configured. Set object_storage settings in datajoint.json " + "or DJ_OBJECT_STORAGE_* environment variables." + ) + + def _get_storage_path(self, field: str, ext: str = "") -> str: + """ + Get or create the storage path for a field. + + Args: + field: Name of the object attribute + ext: Optional extension (e.g., ".zarr") + + Returns: + Full storage path + """ + self._ensure_backend() + + if field in self._staged_objects: + return self._staged_objects[field]["full_path"] + + # Validate field is an object attribute + if field not in self._table.heading: + raise DataJointError(f"Attribute '{field}' not found in table heading") + + attr = self._table.heading[field] + # Check if this is an object Codec (has codec with "object" as name) + if not (attr.codec and attr.codec.name == "object"): + raise DataJointError(f"Attribute '{field}' is not an type") + + # Extract primary key from rec + primary_key = {k: self._rec[k] for k in self._table.primary_key if k in self._rec} + if len(primary_key) != len(self._table.primary_key): + raise DataJointError( + "Primary key values must be set in staged.rec before calling store() or open(). " + f"Missing: {set(self._table.primary_key) - set(primary_key)}" + ) + + # Get storage spec + spec = config.get_object_storage_spec() + partition_pattern = spec.get("partition_pattern") + token_length = spec.get("token_length", 8) + + # Build storage path (relative - StorageBackend will add location prefix) + relative_path, token = build_object_path( + schema=self._table.database, + table=self._table.class_name, + field=field, + primary_key=primary_key, + ext=ext if ext else None, + partition_pattern=partition_pattern, + token_length=token_length, + ) + + # Store staged object info (all paths are relative, backend adds location) + self._staged_objects[field] = { + "relative_path": relative_path, + "ext": ext if ext else None, + "token": token, + } + + return relative_path + + def store(self, field: str, ext: str = "") -> fsspec.FSMap: + """ + Get an FSMap store for direct writes to an object field. + + Args: + field: Name of the object attribute + ext: Optional extension (e.g., ".zarr", ".hdf5") + + Returns: + fsspec.FSMap suitable for Zarr/xarray + """ + path = self._get_storage_path(field, ext) + return self._backend.get_fsmap(path) + + def open(self, field: str, ext: str = "", mode: str = "wb") -> IO: + """ + Open a file for direct writes to an object field. + + Args: + field: Name of the object attribute + ext: Optional extension (e.g., ".bin", ".dat") + mode: File mode (default: "wb") + + Returns: + File-like object for writing + """ + path = self._get_storage_path(field, ext) + return self._backend.open(path, mode) + + def _compute_metadata(self, field: str) -> dict: + """ + Compute metadata for a staged object after writing is complete. + + Args: + field: Name of the object attribute + + Returns: + JSON-serializable metadata dict + """ + info = self._staged_objects[field] + relative_path = info["relative_path"] + ext = info["ext"] + + # Check if it's a directory (multiple files) or single file + # _full_path adds the location prefix + full_remote_path = self._backend._full_path(relative_path) + + try: + is_dir = self._backend.fs.isdir(full_remote_path) + except Exception: + is_dir = False + + if is_dir: + # Calculate total size and file count + total_size = 0 + item_count = 0 + files = [] + + for root, dirs, filenames in self._backend.fs.walk(full_remote_path): + for filename in filenames: + file_path = f"{root}/{filename}" + try: + file_size = self._backend.fs.size(file_path) + rel_path = file_path[len(full_remote_path) :].lstrip("/") + files.append({"path": rel_path, "size": file_size}) + total_size += file_size + item_count += 1 + except Exception: + pass + + # Create manifest + manifest = { + "files": files, + "total_size": total_size, + "item_count": item_count, + "created": datetime.now(timezone.utc).isoformat(), + } + + # Write manifest alongside folder + manifest_path = f"{relative_path}.manifest.json" + self._backend.put_buffer(json.dumps(manifest, indent=2).encode(), manifest_path) + + metadata = { + "path": relative_path, + "size": total_size, + "hash": None, + "ext": ext, + "is_dir": True, + "timestamp": datetime.now(timezone.utc).isoformat(), + "item_count": item_count, + } + else: + # Single file + try: + size = self._backend.size(relative_path) + except Exception: + size = 0 + + metadata = { + "path": relative_path, + "size": size, + "hash": None, + "ext": ext, + "is_dir": False, + "timestamp": datetime.now(timezone.utc).isoformat(), + } + + # Add mime_type for files + if ext: + mime_type, _ = mimetypes.guess_type(f"file{ext}") + if mime_type: + metadata["mime_type"] = mime_type + + return metadata + + def _finalize(self): + """ + Finalize the staged insert by computing metadata and inserting the record. + """ + # Process each staged object + for field in list(self._staged_objects.keys()): + metadata = self._compute_metadata(field) + # Store metadata dict in the record (ObjectType.encode handles it) + self._rec[field] = metadata + + # Insert the record + self._table.insert1(self._rec) + + def _cleanup(self): + """ + Clean up staged objects on failure. + """ + if self._backend is None: + return + + for field, info in self._staged_objects.items(): + relative_path = info["relative_path"] + try: + # Check if it's a directory + full_remote_path = self._backend._full_path(relative_path) + if self._backend.fs.exists(full_remote_path): + if self._backend.fs.isdir(full_remote_path): + self._backend.remove_folder(relative_path) + else: + self._backend.remove(relative_path) + except Exception: + pass # Best effort cleanup + + +@contextmanager +def staged_insert1(table): + """ + Context manager for staged insert operations. + + Args: + table: The Table instance to insert into + + Yields: + StagedInsert instance for setting record values and getting storage handles + + Example: + with staged_insert1(Recording) as staged: + staged.rec['subject_id'] = 123 + staged.rec['session_id'] = 45 + z = zarr.open(staged.store('raw_data', '.zarr'), mode='w') + z[:] = data + staged.rec['raw_data'] = z + """ + staged = StagedInsert(table) + try: + yield staged + staged._finalize() + except Exception: + staged._cleanup() + raise diff --git a/src/datajoint/storage.py b/src/datajoint/storage.py new file mode 100644 index 000000000..846228137 --- /dev/null +++ b/src/datajoint/storage.py @@ -0,0 +1,1016 @@ +""" +Storage backend abstraction using fsspec for unified file operations. + +This module provides a unified interface for storage operations across different +backends (local filesystem, S3, GCS, Azure, etc.) using the fsspec library. +""" + +from __future__ import annotations + +import json +import logging +import secrets +import urllib.parse +from datetime import datetime, timezone +from pathlib import Path, PurePosixPath +from typing import Any + +import fsspec + +from . import errors + +logger = logging.getLogger(__name__.split(".")[0]) + +# Characters safe for use in filenames and URLs +TOKEN_ALPHABET = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_" + +# Supported URL protocols +URL_PROTOCOLS = ("file://", "s3://", "gs://", "gcs://", "az://", "abfs://", "http://", "https://") + + +def is_url(path: str) -> bool: + """ + Check if a path is a URL. + + Parameters + ---------- + path : str + Path string to check. + + Returns + ------- + bool + True if path starts with a supported URL protocol. + """ + return path.lower().startswith(URL_PROTOCOLS) + + +def normalize_to_url(path: str) -> str: + """ + Normalize a path to URL form. + + Converts local filesystem paths to file:// URLs. URLs are returned unchanged. + + Parameters + ---------- + path : str + Path string (local path or URL). + + Returns + ------- + str + URL form of the path. + + Examples + -------- + >>> normalize_to_url("/data/file.dat") + 'file:///data/file.dat' + >>> normalize_to_url("s3://bucket/key") + 's3://bucket/key' + >>> normalize_to_url("file:///already/url") + 'file:///already/url' + """ + if is_url(path): + return path + # Convert local path to file:// URL + # Ensure absolute path and proper format + abs_path = str(Path(path).resolve()) + # Handle Windows paths (C:\...) vs Unix paths (/...) + if abs_path.startswith("/"): + return f"file://{abs_path}" + else: + # Windows: file:///C:/path + return f"file:///{abs_path.replace(chr(92), '/')}" + + +def parse_url(url: str) -> tuple[str, str]: + """ + Parse a URL into protocol and path. + + Parameters + ---------- + url : str + URL (e.g., ``'s3://bucket/path/file.dat'`` or ``'file:///path/to/file'``). + + Returns + ------- + tuple[str, str] + ``(protocol, path)`` where protocol is fsspec-compatible. + + Raises + ------ + DataJointError + If URL protocol is not supported. + + Examples + -------- + >>> parse_url("s3://bucket/key/file.dat") + ('s3', 'bucket/key/file.dat') + >>> parse_url("file:///data/file.dat") + ('file', '/data/file.dat') + """ + url_lower = url.lower() + + # Map URL schemes to fsspec protocols + protocol_map = { + "file://": "file", + "s3://": "s3", + "gs://": "gcs", + "gcs://": "gcs", + "az://": "abfs", + "abfs://": "abfs", + "http://": "http", + "https://": "https", + } + + for prefix, protocol in protocol_map.items(): + if url_lower.startswith(prefix): + path = url[len(prefix) :] + return protocol, path + + raise errors.DataJointError(f"Unsupported URL protocol: {url}") + + +def generate_token(length: int = 8) -> str: + """ + Generate a random token for filename collision avoidance. + + Parameters + ---------- + length : int, optional + Token length, clamped to 4-16 characters. Default 8. + + Returns + ------- + str + Random URL-safe string. + """ + length = max(4, min(16, length)) + return "".join(secrets.choice(TOKEN_ALPHABET) for _ in range(length)) + + +def encode_pk_value(value: Any) -> str: + """ + Encode a primary key value for use in storage paths. + + Parameters + ---------- + value : any + Primary key value (int, str, date, datetime, etc.). + + Returns + ------- + str + Path-safe string representation. + """ + if isinstance(value, (int, float)): + return str(value) + if isinstance(value, datetime): + # Use ISO format with safe separators + return value.strftime("%Y-%m-%dT%H-%M-%S") + if hasattr(value, "isoformat"): + # Handle date objects + return value.isoformat() + + # String handling + s = str(value) + # Check if path-safe (no special characters) + unsafe_chars = '/\\:*?"<>|' + if any(c in s for c in unsafe_chars) or len(s) > 100: + # URL-encode unsafe strings or truncate long ones + if len(s) > 100: + # Truncate and add hash suffix for uniqueness + import hashlib + + hash_suffix = hashlib.md5(s.encode()).hexdigest()[:8] + s = s[:50] + "_" + hash_suffix + return urllib.parse.quote(s, safe="") + return s + + +def build_object_path( + schema: str, + table: str, + field: str, + primary_key: dict[str, Any], + ext: str | None, + partition_pattern: str | None = None, + token_length: int = 8, +) -> tuple[str, str]: + """ + Build the storage path for an object attribute. + + Parameters + ---------- + schema : str + Schema name. + table : str + Table name. + field : str + Field/attribute name. + primary_key : dict[str, Any] + Dict of primary key attribute names to values. + ext : str or None + File extension (e.g., ``".dat"``). + partition_pattern : str, optional + Partition pattern with ``{attr}`` placeholders. + token_length : int, optional + Length of random token suffix. Default 8. + + Returns + ------- + tuple[str, str] + ``(relative_path, token)``. + """ + token = generate_token(token_length) + + # Build filename: field_token.ext + filename = f"{field}_{token}" + if ext: + if not ext.startswith("."): + ext = "." + ext + filename += ext + + # Build primary key path components + pk_parts = [] + partition_attrs = set() + + # Extract partition attributes if pattern specified + if partition_pattern: + import re + + partition_attrs = set(re.findall(r"\{(\w+)\}", partition_pattern)) + + # Build partition prefix (attributes specified in partition pattern) + partition_parts = [] + for attr in partition_attrs: + if attr in primary_key: + partition_parts.append(f"{attr}={encode_pk_value(primary_key[attr])}") + + # Build remaining PK path (attributes not in partition) + for attr, value in primary_key.items(): + if attr not in partition_attrs: + pk_parts.append(f"{attr}={encode_pk_value(value)}") + + # Construct full path + # Pattern: {partition_attrs}/{schema}/{table}/objects/{remaining_pk}/{filename} + parts = [] + if partition_parts: + parts.extend(partition_parts) + parts.append(schema) + parts.append(table) + parts.append("objects") + if pk_parts: + parts.extend(pk_parts) + parts.append(filename) + + return "/".join(parts), token + + +class StorageBackend: + """ + Unified storage backend using fsspec. + + Provides a consistent interface for file operations across different storage + backends including local filesystem and cloud object storage (S3, GCS, Azure). + + Parameters + ---------- + spec : dict[str, Any] + Storage configuration dictionary. See ``__init__`` for details. + + Attributes + ---------- + spec : dict + Storage configuration dictionary. + protocol : str + Storage protocol (``'file'``, ``'s3'``, ``'gcs'``, ``'azure'``). + """ + + def __init__(self, spec: dict[str, Any]) -> None: + """ + Initialize storage backend from configuration spec. + + Parameters + ---------- + spec : dict[str, Any] + Storage configuration dictionary containing: + + - ``protocol``: Storage protocol (``'file'``, ``'s3'``, ``'gcs'``, ``'azure'``) + - ``location``: Base path or bucket prefix + - ``bucket``: Bucket name (for cloud storage) + - ``endpoint``: Endpoint URL (for S3-compatible storage) + - ``access_key``: Access key (for cloud storage) + - ``secret_key``: Secret key (for cloud storage) + - ``secure``: Use HTTPS (default True for cloud) + """ + self.spec = spec + self.protocol = spec.get("protocol", "file") + self._fs = None + self._validate_spec() + + def _validate_spec(self): + """Validate configuration spec for the protocol.""" + if self.protocol == "file": + location = self.spec.get("location") + if location and not Path(location).is_dir(): + raise FileNotFoundError(f"Inaccessible local directory {location}") + elif self.protocol == "s3": + required = ["endpoint", "bucket", "access_key", "secret_key"] + missing = [k for k in required if not self.spec.get(k)] + if missing: + raise errors.DataJointError(f"Missing S3 configuration: {', '.join(missing)}") + + @property + def fs(self) -> fsspec.AbstractFileSystem: + """Get or create the fsspec filesystem instance.""" + if self._fs is None: + self._fs = self._create_filesystem() + return self._fs + + def _create_filesystem(self) -> fsspec.AbstractFileSystem: + """Create fsspec filesystem based on protocol.""" + if self.protocol == "file": + return fsspec.filesystem("file", auto_mkdir=True) + + elif self.protocol == "s3": + # Build S3 configuration + endpoint = self.spec["endpoint"] + # Determine if endpoint includes protocol + if not endpoint.startswith(("http://", "https://")): + secure = self.spec.get("secure", False) + endpoint_url = f"{'https' if secure else 'http'}://{endpoint}" + else: + endpoint_url = endpoint + + return fsspec.filesystem( + "s3", + key=self.spec["access_key"], + secret=self.spec["secret_key"], + client_kwargs={"endpoint_url": endpoint_url}, + ) + + elif self.protocol == "gcs": + return fsspec.filesystem( + "gcs", + token=self.spec.get("token"), + project=self.spec.get("project"), + ) + + elif self.protocol == "azure": + return fsspec.filesystem( + "abfs", + account_name=self.spec.get("account_name"), + account_key=self.spec.get("account_key"), + connection_string=self.spec.get("connection_string"), + ) + + else: + raise errors.DataJointError(f"Unsupported storage protocol: {self.protocol}") + + def _full_path(self, path: str | PurePosixPath) -> str: + """ + Construct full path including location/bucket prefix. + + Parameters + ---------- + path : str or PurePosixPath + Relative path within the storage location. + + Returns + ------- + str + Full path suitable for fsspec operations. + """ + path = str(path) + if self.protocol == "s3": + bucket = self.spec["bucket"] + location = self.spec.get("location", "") + if location: + return f"{bucket}/{location}/{path}" + return f"{bucket}/{path}" + elif self.protocol in ("gcs", "azure"): + bucket = self.spec.get("bucket") or self.spec.get("container") + location = self.spec.get("location", "") + if location: + return f"{bucket}/{location}/{path}" + return f"{bucket}/{path}" + else: + # Local filesystem - prepend location if specified + location = self.spec.get("location", "") + if location: + return str(Path(location) / path) + return path + + def get_url(self, path: str | PurePosixPath) -> str: + """ + Get the full URL for a path in storage. + + Returns a consistent URL representation for any storage backend, + including file:// URLs for local filesystem. + + Parameters + ---------- + path : str or PurePosixPath + Relative path within the storage location. + + Returns + ------- + str + Full URL (e.g., 's3://bucket/path' or 'file:///data/path'). + + Examples + -------- + >>> backend = StorageBackend({"protocol": "file", "location": "/data"}) + >>> backend.get_url("schema/table/file.dat") + 'file:///data/schema/table/file.dat' + + >>> backend = StorageBackend({"protocol": "s3", "bucket": "mybucket", ...}) + >>> backend.get_url("schema/table/file.dat") + 's3://mybucket/schema/table/file.dat' + """ + full_path = self._full_path(path) + + if self.protocol == "file": + # Ensure absolute path for file:// URL + abs_path = str(Path(full_path).resolve()) + if abs_path.startswith("/"): + return f"file://{abs_path}" + else: + # Windows path + return f"file:///{abs_path.replace(chr(92), '/')}" + elif self.protocol == "s3": + return f"s3://{full_path}" + elif self.protocol == "gcs": + return f"gs://{full_path}" + elif self.protocol == "azure": + return f"az://{full_path}" + else: + # Fallback: use protocol prefix + return f"{self.protocol}://{full_path}" + + def put_file(self, local_path: str | Path, remote_path: str | PurePosixPath, metadata: dict | None = None) -> None: + """ + Upload a file from local filesystem to storage. + + Parameters + ---------- + local_path : str or Path + Path to local file. + remote_path : str or PurePosixPath + Destination path in storage. + metadata : dict, optional + Metadata to attach to the file (cloud storage only). + """ + full_path = self._full_path(remote_path) + logger.debug(f"put_file: {local_path} -> {self.protocol}:{full_path}") + + if self.protocol == "file": + # For local filesystem, use safe copy with atomic rename + from .utils import safe_copy + + Path(full_path).parent.mkdir(parents=True, exist_ok=True) + safe_copy(local_path, full_path, overwrite=True) + else: + # For cloud storage, use fsspec put + self.fs.put_file(str(local_path), full_path) + + def get_file(self, remote_path: str | PurePosixPath, local_path: str | Path) -> None: + """ + Download a file from storage to local filesystem. + + Parameters + ---------- + remote_path : str or PurePosixPath + Path in storage. + local_path : str or Path + Destination path on local filesystem. + """ + full_path = self._full_path(remote_path) + logger.debug(f"get_file: {self.protocol}:{full_path} -> {local_path}") + + local_path = Path(local_path) + local_path.parent.mkdir(parents=True, exist_ok=True) + + if self.protocol == "file": + from .utils import safe_copy + + safe_copy(full_path, local_path) + else: + self.fs.get_file(full_path, str(local_path)) + + def put_buffer(self, buffer: bytes, remote_path: str | PurePosixPath) -> None: + """ + Write bytes to storage. + + Parameters + ---------- + buffer : bytes + Bytes to write. + remote_path : str or PurePosixPath + Destination path in storage. + """ + full_path = self._full_path(remote_path) + logger.debug(f"put_buffer: {len(buffer)} bytes -> {self.protocol}:{full_path}") + + if self.protocol == "file": + from .utils import safe_write + + Path(full_path).parent.mkdir(parents=True, exist_ok=True) + safe_write(full_path, buffer) + else: + self.fs.pipe_file(full_path, buffer) + + def get_buffer(self, remote_path: str | PurePosixPath) -> bytes: + """ + Read bytes from storage. + + Parameters + ---------- + remote_path : str or PurePosixPath + Path in storage. + + Returns + ------- + bytes + File contents. + + Raises + ------ + MissingExternalFile + If the file does not exist. + """ + full_path = self._full_path(remote_path) + logger.debug(f"get_buffer: {self.protocol}:{full_path}") + + try: + if self.protocol == "file": + return Path(full_path).read_bytes() + else: + return self.fs.cat_file(full_path) + except FileNotFoundError: + raise errors.MissingExternalFile(f"Missing external file {full_path}") from None + + def exists(self, remote_path: str | PurePosixPath) -> bool: + """ + Check if a file exists in storage. + + Parameters + ---------- + remote_path : str or PurePosixPath + Path in storage. + + Returns + ------- + bool + True if file exists. + """ + full_path = self._full_path(remote_path) + logger.debug(f"exists: {self.protocol}:{full_path}") + + if self.protocol == "file": + return Path(full_path).is_file() + else: + return self.fs.exists(full_path) + + def remove(self, remote_path: str | PurePosixPath) -> None: + """ + Remove a file from storage. + + Parameters + ---------- + remote_path : str or PurePosixPath + Path in storage. + """ + full_path = self._full_path(remote_path) + logger.debug(f"remove: {self.protocol}:{full_path}") + + try: + if self.protocol == "file": + Path(full_path).unlink(missing_ok=True) + else: + self.fs.rm(full_path) + except FileNotFoundError: + pass # Already gone + + def size(self, remote_path: str | PurePosixPath) -> int: + """ + Get file size in bytes. + + Parameters + ---------- + remote_path : str or PurePosixPath + Path in storage. + + Returns + ------- + int + File size in bytes. + """ + full_path = self._full_path(remote_path) + + if self.protocol == "file": + return Path(full_path).stat().st_size + else: + return self.fs.size(full_path) + + def open(self, remote_path: str | PurePosixPath, mode: str = "rb"): + """ + Open a file in storage. + + Parameters + ---------- + remote_path : str or PurePosixPath + Path in storage. + mode : str, optional + File mode (``'rb'``, ``'wb'``, etc.). Default ``'rb'``. + + Returns + ------- + file-like + File-like object for reading or writing. + """ + full_path = self._full_path(remote_path) + + # For write modes on local filesystem, ensure parent directory exists + if self.protocol == "file" and "w" in mode: + Path(full_path).parent.mkdir(parents=True, exist_ok=True) + + return self.fs.open(full_path, mode) + + def put_folder(self, local_path: str | Path, remote_path: str | PurePosixPath) -> dict: + """ + Upload a folder to storage. + + Parameters + ---------- + local_path : str or Path + Path to local folder. + remote_path : str or PurePosixPath + Destination path in storage. + + Returns + ------- + dict + Manifest with keys ``'files'``, ``'total_size'``, ``'item_count'``, + ``'created'``. + """ + local_path = Path(local_path) + if not local_path.is_dir(): + raise errors.DataJointError(f"Not a directory: {local_path}") + + full_path = self._full_path(remote_path) + logger.debug(f"put_folder: {local_path} -> {self.protocol}:{full_path}") + + # Collect file info for manifest + files = [] + total_size = 0 + + # Use os.walk for Python 3.10 compatibility (Path.walk() requires 3.12+) + import os + + for root, dirs, filenames in os.walk(local_path): + root_path = Path(root) + for filename in filenames: + file_path = root_path / filename + rel_path = file_path.relative_to(local_path).as_posix() + file_size = file_path.stat().st_size + files.append({"path": rel_path, "size": file_size}) + total_size += file_size + + # Upload folder contents + if self.protocol == "file": + import shutil + + dest = Path(full_path) + dest.mkdir(parents=True, exist_ok=True) + for item in local_path.iterdir(): + if item.is_file(): + shutil.copy2(item, dest / item.name) + else: + shutil.copytree(item, dest / item.name, dirs_exist_ok=True) + else: + self.fs.put(str(local_path), full_path, recursive=True) + + # Build manifest + manifest = { + "files": files, + "total_size": total_size, + "item_count": len(files), + "created": datetime.now(timezone.utc).isoformat(), + } + + # Write manifest alongside folder + manifest_path = f"{remote_path}.manifest.json" + self.put_buffer(json.dumps(manifest, indent=2).encode(), manifest_path) + + return manifest + + def remove_folder(self, remote_path: str | PurePosixPath) -> None: + """ + Remove a folder and its manifest from storage. + + Parameters + ---------- + remote_path : str or PurePosixPath + Path to folder in storage. + """ + full_path = self._full_path(remote_path) + logger.debug(f"remove_folder: {self.protocol}:{full_path}") + + try: + if self.protocol == "file": + import shutil + + shutil.rmtree(full_path, ignore_errors=True) + else: + self.fs.rm(full_path, recursive=True) + except FileNotFoundError: + pass + + # Also remove manifest + manifest_path = f"{remote_path}.manifest.json" + self.remove(manifest_path) + + def get_fsmap(self, remote_path: str | PurePosixPath) -> fsspec.FSMap: + """ + Get an FSMap for a path (useful for Zarr/xarray). + + Parameters + ---------- + remote_path : str or PurePosixPath + Path in storage. + + Returns + ------- + fsspec.FSMap + Mapping interface for the storage path. + """ + full_path = self._full_path(remote_path) + return fsspec.FSMap(full_path, self.fs) + + def copy_from_url(self, source_url: str, dest_path: str | PurePosixPath) -> int: + """ + Copy a file from a remote URL to managed storage. + + Parameters + ---------- + source_url : str + Remote URL (``s3://``, ``gs://``, ``http://``, etc.). + dest_path : str or PurePosixPath + Destination path in managed storage. + + Returns + ------- + int + Size of copied file in bytes. + """ + protocol, source_path = parse_url(source_url) + full_dest = self._full_path(dest_path) + + logger.debug(f"copy_from_url: {protocol}://{source_path} -> {self.protocol}:{full_dest}") + + # Get source filesystem + source_fs = fsspec.filesystem(protocol) + + # Check if source is a directory + if source_fs.isdir(source_path): + return self._copy_folder_from_url(source_fs, source_path, dest_path) + + # Copy single file + if self.protocol == "file": + # Download to local destination + Path(full_dest).parent.mkdir(parents=True, exist_ok=True) + source_fs.get_file(source_path, full_dest) + return Path(full_dest).stat().st_size + else: + # Remote-to-remote copy via streaming + with source_fs.open(source_path, "rb") as src: + content = src.read() + self.fs.pipe_file(full_dest, content) + return len(content) + + def _copy_folder_from_url( + self, source_fs: fsspec.AbstractFileSystem, source_path: str, dest_path: str | PurePosixPath + ) -> dict: + """ + Copy a folder from a remote URL to managed storage. + + Parameters + ---------- + source_fs : fsspec.AbstractFileSystem + Source filesystem. + source_path : str + Path in source filesystem. + dest_path : str or PurePosixPath + Destination path in managed storage. + + Returns + ------- + dict + Manifest with keys ``'files'``, ``'total_size'``, ``'item_count'``, + ``'created'``. + """ + full_dest = self._full_path(dest_path) + logger.debug(f"copy_folder_from_url: {source_path} -> {self.protocol}:{full_dest}") + + # Collect file info for manifest + files = [] + total_size = 0 + + # Walk source directory + for root, dirs, filenames in source_fs.walk(source_path): + for filename in filenames: + src_file = f"{root}/{filename}" if root != source_path else f"{source_path}/{filename}" + rel_path = src_file[len(source_path) :].lstrip("/") + file_size = source_fs.size(src_file) + files.append({"path": rel_path, "size": file_size}) + total_size += file_size + + # Copy file + dest_file = f"{full_dest}/{rel_path}" + if self.protocol == "file": + Path(dest_file).parent.mkdir(parents=True, exist_ok=True) + source_fs.get_file(src_file, dest_file) + else: + with source_fs.open(src_file, "rb") as src: + content = src.read() + self.fs.pipe_file(dest_file, content) + + # Build manifest + manifest = { + "files": files, + "total_size": total_size, + "item_count": len(files), + "created": datetime.now(timezone.utc).isoformat(), + } + + # Write manifest alongside folder + manifest_path = f"{dest_path}.manifest.json" + self.put_buffer(json.dumps(manifest, indent=2).encode(), manifest_path) + + return manifest + + def source_is_directory(self, source: str) -> bool: + """ + Check if a source path (local or remote URL) is a directory. + + Parameters + ---------- + source : str + Local path or remote URL. + + Returns + ------- + bool + True if source is a directory. + """ + if is_url(source): + protocol, path = parse_url(source) + source_fs = fsspec.filesystem(protocol) + return source_fs.isdir(path) + else: + return Path(source).is_dir() + + def source_exists(self, source: str) -> bool: + """ + Check if a source path (local or remote URL) exists. + + Parameters + ---------- + source : str + Local path or remote URL. + + Returns + ------- + bool + True if source exists. + """ + if is_url(source): + protocol, path = parse_url(source) + source_fs = fsspec.filesystem(protocol) + return source_fs.exists(path) + else: + return Path(source).exists() + + def get_source_size(self, source: str) -> int | None: + """ + Get the size of a source file (local or remote URL). + + Parameters + ---------- + source : str + Local path or remote URL. + + Returns + ------- + int or None + Size in bytes, or None if directory or cannot determine. + """ + try: + if is_url(source): + protocol, path = parse_url(source) + source_fs = fsspec.filesystem(protocol) + if source_fs.isdir(path): + return None + return source_fs.size(path) + else: + p = Path(source) + if p.is_dir(): + return None + return p.stat().st_size + except Exception: + return None + + +STORE_METADATA_FILENAME = "datajoint_store.json" + + +def get_storage_backend(spec: dict[str, Any]) -> StorageBackend: + """ + Factory function to create a storage backend from configuration. + + Parameters + ---------- + spec : dict[str, Any] + Storage configuration dictionary. + + Returns + ------- + StorageBackend + Configured storage backend instance. + """ + return StorageBackend(spec) + + +def verify_or_create_store_metadata(backend: StorageBackend, spec: dict[str, Any]) -> dict: + """ + Verify or create the store metadata file at the storage root. + + On first use, creates the ``datajoint_store.json`` file with project info. + On subsequent uses, verifies the ``project_name`` matches. + + Parameters + ---------- + backend : StorageBackend + Storage backend instance. + spec : dict[str, Any] + Object storage configuration spec. + + Returns + ------- + dict + Store metadata dictionary. + + Raises + ------ + DataJointError + If ``project_name`` mismatch detected. + """ + from .version import __version__ as dj_version + + project_name = spec.get("project_name") + location = spec.get("location", "") + + # Metadata file path at storage root + metadata_path = f"{location}/{STORE_METADATA_FILENAME}" if location else STORE_METADATA_FILENAME + + try: + # Try to read existing metadata + if backend.exists(metadata_path): + metadata_content = backend.get_buffer(metadata_path) + metadata = json.loads(metadata_content) + + # Verify project_name matches + store_project = metadata.get("project_name") + if store_project and store_project != project_name: + raise errors.DataJointError( + f"Object store project name mismatch.\n" + f' Client configured: "{project_name}"\n' + f' Store metadata: "{store_project}"\n' + f"Ensure all clients use the same object_storage.project_name setting." + ) + + return metadata + else: + # Create new metadata + metadata = { + "project_name": project_name, + "created": datetime.now(timezone.utc).isoformat(), + "format_version": "1.0", + "datajoint_version": dj_version, + } + + # Optional database info - not enforced, just informational + # These would need to be passed in from the connection context + # For now, omit them + + backend.put_buffer(json.dumps(metadata, indent=2).encode(), metadata_path) + return metadata + + except errors.DataJointError: + raise + except Exception as e: + # Log warning but don't fail - metadata is informational + logger.warning(f"Could not verify/create store metadata: {e}") + return {"project_name": project_name} diff --git a/src/datajoint/table.py b/src/datajoint/table.py new file mode 100644 index 000000000..0040943c5 --- /dev/null +++ b/src/datajoint/table.py @@ -0,0 +1,1368 @@ +import collections +import csv +import inspect +import itertools +import json +import logging +import re +import uuid +import warnings +from dataclasses import dataclass, field +from pathlib import Path + +import numpy as np +import pandas + +from .condition import make_condition +from .declare import alter, declare +from .errors import ( + AccessError, + DataJointError, + DuplicateError, + IntegrityError, + UnknownAttributeError, +) +from .expression import QueryExpression +from .heading import Heading +from .settings import config +from .staged_insert import staged_insert1 as _staged_insert1 +from .utils import get_master, is_camel_case, user_choice + +logger = logging.getLogger(__name__.split(".")[0]) + +foreign_key_error_regexp = re.compile( + r"[\w\s:]*\((?P`[^`]+`.`[^`]+`), " + r"CONSTRAINT (?P`[^`]+`) " + r"(FOREIGN KEY \((?P[^)]+)\) " + r"REFERENCES (?P`[^`]+`(\.`[^`]+`)?) \((?P[^)]+)\)[\s\w]+\))?" +) + +constraint_info_query = " ".join( + """ + SELECT + COLUMN_NAME as fk_attrs, + CONCAT('`', REFERENCED_TABLE_SCHEMA, '`.`', REFERENCED_TABLE_NAME, '`') as parent, + REFERENCED_COLUMN_NAME as pk_attrs + FROM INFORMATION_SCHEMA.KEY_COLUMN_USAGE + WHERE + CONSTRAINT_NAME = %s AND TABLE_SCHEMA = %s AND TABLE_NAME = %s; + """.split() +) + + +class _RenameMap(tuple): + """for internal use""" + + pass + + +@dataclass +class ValidationResult: + """ + Result of table.validate() call. + + Attributes: + is_valid: True if all rows passed validation + errors: List of (row_index, field_name, error_message) tuples + rows_checked: Number of rows that were validated + """ + + is_valid: bool + errors: list = field(default_factory=list) # list of (row_index, field_name | None, message) + rows_checked: int = 0 + + def __bool__(self) -> bool: + """Allow using ValidationResult in boolean context.""" + return self.is_valid + + def raise_if_invalid(self): + """Raise DataJointError if validation failed.""" + if not self.is_valid: + raise DataJointError(self.summary()) + + def summary(self) -> str: + """Return formatted error summary.""" + if self.is_valid: + return f"Validation passed: {self.rows_checked} rows checked" + lines = [f"Validation failed: {len(self.errors)} error(s) in {self.rows_checked} rows"] + for row_idx, field_name, message in self.errors[:10]: # Show first 10 errors + field_str = f" in field '{field_name}'" if field_name else "" + lines.append(f" Row {row_idx}{field_str}: {message}") + if len(self.errors) > 10: + lines.append(f" ... and {len(self.errors) - 10} more errors") + return "\n".join(lines) + + +class Table(QueryExpression): + """ + Table is an abstract class that represents a table in the schema. + It implements insert and delete methods and inherits query functionality. + To make it a concrete class, override the abstract properties specifying the connection, + table name, database, and definition. + """ + + _table_name = None # must be defined in subclass + + # These properties must be set by the schema decorator (schemas.py) at class level + # or by FreeTable at instance level + database = None + declaration_context = None + + @property + def table_name(self): + # For UserTable subclasses, table_name is computed by the metaclass. + # Delegate to the class's table_name if _table_name is not set. + if self._table_name is None: + return type(self).table_name + return self._table_name + + @property + def class_name(self): + return self.__class__.__name__ + + # Base tier class names that should not raise errors when heading is None + _base_tier_classes = frozenset({"Table", "UserTable", "Lookup", "Manual", "Imported", "Computed", "Part"}) + + @property + def heading(self): + """ + Return the table's heading, or raise a helpful error if not configured. + + Overrides QueryExpression.heading to provide a clear error message + when the table is not properly associated with an activated schema. + For base tier classes (Lookup, Manual, etc.), returns None to support + introspection (e.g., help()). + """ + if self._heading is None: + # Don't raise error for base tier classes - they're used for introspection + if self.__class__.__name__ in self._base_tier_classes: + return None + raise DataJointError( + f"Table `{self.__class__.__name__}` is not properly configured. " + "Ensure the schema is activated before using the table. " + "Example: schema.activate('database_name') or schema = dj.Schema('database_name')" + ) + return self._heading + + @property + def definition(self): + raise NotImplementedError("Subclasses of Table must implement the `definition` property") + + def declare(self, context=None): + """ + Declare the table in the schema based on self.definition. + + :param context: the context for foreign key resolution. If None, foreign keys are + not allowed. + """ + if self.connection.in_transaction: + raise DataJointError("Cannot declare new tables inside a transaction, e.g. from inside a populate/make call") + # Enforce strict CamelCase #1150 + if not is_camel_case(self.class_name): + raise DataJointError( + "Table class name `{name}` is invalid. Please use CamelCase. ".format(name=self.class_name) + + "Classes defining tables should be formatted in strict CamelCase." + ) + sql, _external_stores, primary_key, fk_attribute_map = declare(self.full_table_name, self.definition, context) + + # Call declaration hook for validation (subclasses like AutoPopulate can override) + self._declare_check(primary_key, fk_attribute_map) + + sql = sql.format(database=self.database) + try: + self.connection.query(sql) + except AccessError: + # Only suppress if table already exists (idempotent declaration) + # Otherwise raise - user needs to know about permission issues + if self.is_declared: + return + raise AccessError( + f"Cannot declare table {self.full_table_name}. " + f"Check that you have CREATE privilege on schema `{self.database}` " + f"and REFERENCES privilege on any referenced parent tables." + ) from None + + # Populate lineage table for this table's attributes + self._populate_lineage(primary_key, fk_attribute_map) + + def _declare_check(self, primary_key, fk_attribute_map): + """ + Hook for declaration-time validation. Subclasses can override. + + Called before the table is created in the database. Override this method + to add validation logic (e.g., AutoPopulate validates FK-only primary keys). + + :param primary_key: list of primary key attribute names + :param fk_attribute_map: dict mapping child_attr -> (parent_table, parent_attr) + """ + pass # Default: no validation + + def _populate_lineage(self, primary_key, fk_attribute_map): + """ + Populate the ~lineage table with lineage information for this table's attributes. + + Lineage is stored for: + - All FK attributes (traced to their origin) + - Native primary key attributes (lineage = self) + + :param primary_key: list of primary key attribute names + :param fk_attribute_map: dict mapping child_attr -> (parent_table, parent_attr) + """ + from .lineage import ( + ensure_lineage_table, + get_lineage, + delete_table_lineages, + insert_lineages, + ) + + # Ensure the ~lineage table exists + ensure_lineage_table(self.connection, self.database) + + # Delete any existing lineage entries for this table (for idempotent re-declaration) + delete_table_lineages(self.connection, self.database, self.table_name) + + entries = [] + + # FK attributes: copy lineage from parent (whether in PK or not) + for attr, (parent_table, parent_attr) in fk_attribute_map.items(): + # Parse parent table name: `schema`.`table` -> (schema, table) + parent_clean = parent_table.replace("`", "") + if "." in parent_clean: + parent_db, parent_tbl = parent_clean.split(".", 1) + else: + parent_db = self.database + parent_tbl = parent_clean + + # Get parent's lineage for this attribute + parent_lineage = get_lineage(self.connection, parent_db, parent_tbl, parent_attr) + if parent_lineage: + # Copy parent's lineage + entries.append((self.table_name, attr, parent_lineage)) + else: + # Parent doesn't have lineage entry - use parent as origin + # This can happen for legacy/external schemas without lineage tracking + lineage = f"{parent_db}.{parent_tbl}.{parent_attr}" + entries.append((self.table_name, attr, lineage)) + logger.warning( + f"Lineage for `{parent_db}`.`{parent_tbl}`.`{parent_attr}` not found " + f"(parent schema's ~lineage table may be missing or incomplete). " + f"Using it as origin. Once the parent schema's lineage is rebuilt, " + f"run schema.rebuild_lineage() on this schema to correct the lineage." + ) + + # Native PK attributes (in PK but not FK): this table is the origin + for attr in primary_key: + if attr not in fk_attribute_map: + lineage = f"{self.database}.{self.table_name}.{attr}" + entries.append((self.table_name, attr, lineage)) + + if entries: + insert_lineages(self.connection, self.database, entries) + + def alter(self, prompt=True, context=None): + """ + Alter the table definition from self.definition + """ + if self.connection.in_transaction: + raise DataJointError("Cannot update table declaration inside a transaction, e.g. from inside a populate/make call") + if context is None: + frame = inspect.currentframe().f_back + context = dict(frame.f_globals, **frame.f_locals) + del frame + old_definition = self.describe(context=context) + sql, _external_stores = alter(self.definition, old_definition, context) + if not sql: + if prompt: + logger.warning("Nothing to alter.") + else: + sql = "ALTER TABLE {tab}\n\t".format(tab=self.full_table_name) + ",\n\t".join(sql) + if not prompt or user_choice(sql + "\n\nExecute?") == "yes": + try: + self.connection.query(sql) + except AccessError: + # skip if no create privilege + pass + else: + # reset heading + self.__class__._heading = Heading(table_info=self.heading.table_info) + if prompt: + logger.info("Table altered") + + def from_clause(self): + """ + :return: the FROM clause of SQL SELECT statements. + """ + return self.full_table_name + + def get_select_fields(self, select_fields=None): + """ + :return: the selected attributes from the SQL SELECT statement. + """ + return "*" if select_fields is None else self.heading.project(select_fields).as_sql + + def parents(self, primary=None, as_objects=False, foreign_key_info=False): + """ + + :param primary: if None, then all parents are returned. If True, then only foreign keys composed of + primary key attributes are considered. If False, return foreign keys including at least one + secondary attribute. + :param as_objects: if False, return table names. If True, return table objects. + :param foreign_key_info: if True, each element in result also includes foreign key info. + :return: list of parents as table names or table objects + with (optional) foreign key information. + """ + get_edge = self.connection.dependencies.parents + nodes = [ + next(iter(get_edge(name).items())) if name.isdigit() else (name, props) + for name, props in get_edge(self.full_table_name, primary).items() + ] + if as_objects: + nodes = [(FreeTable(self.connection, name), props) for name, props in nodes] + if not foreign_key_info: + nodes = [name for name, props in nodes] + return nodes + + def children(self, primary=None, as_objects=False, foreign_key_info=False): + """ + :param primary: if None, then all children are returned. If True, then only foreign keys composed of + primary key attributes are considered. If False, return foreign keys including at least one + secondary attribute. + :param as_objects: if False, return table names. If True, return table objects. + :param foreign_key_info: if True, each element in result also includes foreign key info. + :return: list of children as table names or table objects + with (optional) foreign key information. + """ + get_edge = self.connection.dependencies.children + nodes = [ + next(iter(get_edge(name).items())) if name.isdigit() else (name, props) + for name, props in get_edge(self.full_table_name, primary).items() + ] + if as_objects: + nodes = [(FreeTable(self.connection, name), props) for name, props in nodes] + if not foreign_key_info: + nodes = [name for name, props in nodes] + return nodes + + def descendants(self, as_objects=False): + """ + :param as_objects: False - a list of table names; True - a list of table objects. + :return: list of tables descendants in topological order. + """ + return [ + FreeTable(self.connection, node) if as_objects else node + for node in self.connection.dependencies.descendants(self.full_table_name) + if not node.isdigit() + ] + + def ancestors(self, as_objects=False): + """ + :param as_objects: False - a list of table names; True - a list of table objects. + :return: list of tables ancestors in topological order. + """ + return [ + FreeTable(self.connection, node) if as_objects else node + for node in self.connection.dependencies.ancestors(self.full_table_name) + if not node.isdigit() + ] + + def parts(self, as_objects=False): + """ + return part tables either as entries in a dict with foreign key information or a list of objects + + :param as_objects: if False (default), the output is a dict describing the foreign keys. If True, return table objects. + """ + self.connection.dependencies.load(force=False) + nodes = [ + node + for node in self.connection.dependencies.nodes + if not node.isdigit() and node.startswith(self.full_table_name[:-1] + "__") + ] + return [FreeTable(self.connection, c) for c in nodes] if as_objects else nodes + + @property + def is_declared(self): + """ + :return: True is the table is declared in the schema. + """ + return ( + self.connection.query( + 'SHOW TABLES in `{database}` LIKE "{table_name}"'.format(database=self.database, table_name=self.table_name) + ).rowcount + > 0 + ) + + @property + def full_table_name(self): + """ + :return: full table name in the schema + """ + if self.database is None or self.table_name is None: + raise DataJointError( + f"Class {self.__class__.__name__} is not associated with a schema. " + "Apply a schema decorator or use schema() to bind it." + ) + return r"`{0:s}`.`{1:s}`".format(self.database, self.table_name) + + def update1(self, row): + """ + ``update1`` updates one existing entry in the table. + Caution: In DataJoint the primary modes for data manipulation is to ``insert`` and + ``delete`` entire records since referential integrity works on the level of records, + not fields. Therefore, updates are reserved for corrective operations outside of main + workflow. Use UPDATE methods sparingly with full awareness of potential violations of + assumptions. + + :param row: a ``dict`` containing the primary key values and the attributes to update. + Setting an attribute value to None will reset it to the default value (if any). + + The primary key attributes must always be provided. + + Examples: + + >>> table.update1({'id': 1, 'value': 3}) # update value in record with id=1 + >>> table.update1({'id': 1, 'value': None}) # reset value to default + """ + # argument validations + if not isinstance(row, collections.abc.Mapping): + raise DataJointError("The argument of update1 must be dict-like.") + if not set(row).issuperset(self.primary_key): + raise DataJointError("The argument of update1 must supply all primary key values.") + try: + raise DataJointError("Attribute `%s` not found." % next(k for k in row if k not in self.heading.names)) + except StopIteration: + pass # ok + if len(self.restriction): + raise DataJointError("Update cannot be applied to a restricted table.") + key = {k: row[k] for k in self.primary_key} + if len(self & key) != 1: + raise DataJointError("Update can only be applied to one existing entry.") + # UPDATE query + row = [self.__make_placeholder(k, v) for k, v in row.items() if k not in self.primary_key] + query = "UPDATE {table} SET {assignments} WHERE {where}".format( + table=self.full_table_name, + assignments=",".join("`%s`=%s" % r[:2] for r in row), + where=make_condition(self, key, set()), + ) + self.connection.query(query, args=list(r[2] for r in row if r[2] is not None)) + + def validate(self, rows, *, ignore_extra_fields=False) -> ValidationResult: + """ + Validate rows without inserting them. + + :param rows: Same format as insert() - iterable of dicts, tuples, numpy records, + or a pandas DataFrame. + :param ignore_extra_fields: If True, ignore fields not in the table heading. + :return: ValidationResult with is_valid, errors list, and rows_checked count. + + Validates: + - Field existence (all fields must be in table heading) + - Row format (correct number of attributes for positional inserts) + - Codec validation (type checking via codec.validate()) + - NULL constraints (non-nullable fields must have values) + - Primary key completeness (all PK fields must be present) + - UUID format and JSON serializability + + Cannot validate (database-enforced): + - Foreign key constraints + - Unique constraints (other than PK) + - Custom MySQL constraints + + Example:: + + result = table.validate(rows) + if result: + table.insert(rows) + else: + print(result.summary()) + """ + errors = [] + + # Convert DataFrame to records + if isinstance(rows, pandas.DataFrame): + rows = rows.reset_index(drop=len(rows.index.names) == 1 and not rows.index.names[0]).to_records(index=False) + + # Convert Path (CSV) to list of dicts + if isinstance(rows, Path): + with open(rows, newline="") as data_file: + rows = list(csv.DictReader(data_file, delimiter=",")) + + rows = list(rows) # Materialize iterator + row_count = len(rows) + + for row_idx, row in enumerate(rows): + # Validate row format and fields + row_dict = None + try: + if isinstance(row, np.void): # numpy record + fields = list(row.dtype.fields.keys()) + row_dict = {name: row[name] for name in fields} + elif isinstance(row, collections.abc.Mapping): + fields = list(row.keys()) + row_dict = dict(row) + else: # positional tuple/list + if len(row) != len(self.heading): + errors.append( + ( + row_idx, + None, + f"Incorrect number of attributes: {len(row)} given, {len(self.heading)} expected", + ) + ) + continue + fields = list(self.heading.names) + row_dict = dict(zip(fields, row)) + except TypeError: + errors.append((row_idx, None, f"Invalid row type: {type(row).__name__}")) + continue + + # Check for unknown fields + if not ignore_extra_fields: + for field_name in fields: + if field_name not in self.heading: + errors.append((row_idx, field_name, f"Field '{field_name}' not in table heading")) + + # Validate each field value + for name in self.heading.names: + if name not in row_dict: + # Check if field is required (non-nullable, no default, not autoincrement) + attr = self.heading[name] + if not attr.nullable and attr.default is None and not attr.autoincrement: + errors.append((row_idx, name, f"Required field '{name}' is missing")) + continue + + value = row_dict[name] + attr = self.heading[name] + + # Skip validation for None values on nullable columns + if value is None: + if not attr.nullable and attr.default is None: + errors.append((row_idx, name, f"NULL value not allowed for non-nullable field '{name}'")) + continue + + # Codec validation + if attr.codec: + try: + attr.codec.validate(value) + except (TypeError, ValueError) as e: + errors.append((row_idx, name, f"Codec validation failed: {e}")) + continue + + # UUID validation + if attr.uuid and not isinstance(value, uuid.UUID): + try: + uuid.UUID(value) + except (AttributeError, ValueError): + errors.append((row_idx, name, f"Invalid UUID format: {value}")) + continue + + # JSON serialization check + if attr.json: + try: + json.dumps(value) + except (TypeError, ValueError) as e: + errors.append((row_idx, name, f"Value not JSON serializable: {e}")) + continue + + # Numeric NaN check + if attr.numeric and value != "" and not isinstance(value, (bool, np.bool_)): + try: + if np.isnan(float(value)): + # NaN is allowed - will be converted to NULL + pass + except (TypeError, ValueError): + # Not a number that can be checked for NaN - let it pass + pass + + # Check primary key completeness + for pk_field in self.primary_key: + if pk_field not in row_dict or row_dict[pk_field] is None: + pk_attr = self.heading[pk_field] + if not pk_attr.autoincrement: + errors.append((row_idx, pk_field, f"Primary key field '{pk_field}' is missing or NULL")) + + return ValidationResult(is_valid=len(errors) == 0, errors=errors, rows_checked=row_count) + + def insert1(self, row, **kwargs): + """ + Insert one data record into the table. For ``kwargs``, see ``insert()``. + + :param row: a numpy record, a dict-like object, or an ordered sequence to be inserted + as one row. + """ + self.insert((row,), **kwargs) + + @property + def staged_insert1(self): + """ + Context manager for staged insert with direct object storage writes. + + Use this for large objects like Zarr arrays where copying from local storage + is inefficient. Allows writing directly to the destination storage before + finalizing the database insert. + + Example: + with table.staged_insert1 as staged: + staged.rec['subject_id'] = 123 + staged.rec['session_id'] = 45 + + # Create object storage directly + z = zarr.open(staged.store('raw_data', '.zarr'), mode='w', shape=(1000, 1000)) + z[:] = data + + # Assign to record + staged.rec['raw_data'] = z + + # On successful exit: metadata computed, record inserted + # On exception: storage cleaned up, no record inserted + + Yields: + StagedInsert: Context for setting record values and getting storage handles + """ + return _staged_insert1(self) + + def insert( + self, + rows, + replace=False, + skip_duplicates=False, + ignore_extra_fields=False, + allow_direct_insert=None, + chunk_size=None, + ): + """ + Insert a collection of rows. + + :param rows: Either (a) an iterable where an element is a numpy record, a + dict-like object, a pandas.DataFrame, a polars.DataFrame, a pyarrow.Table, + a sequence, or a query expression with the same heading as self, or + (b) a pathlib.Path object specifying a path relative to the current + directory with a CSV file, the contents of which will be inserted. + :param replace: If True, replaces the existing tuple. + :param skip_duplicates: If True, silently skip duplicate inserts. + :param ignore_extra_fields: If False, fields that are not in the heading raise error. + :param allow_direct_insert: Only applies in auto-populated tables. If False (default), + insert may only be called from inside the make callback. + :param chunk_size: If set, insert rows in batches of this size. Useful for very + large inserts to avoid memory issues. Each chunk is a separate transaction. + + Example: + + >>> Table.insert([ + >>> dict(subject_id=7, species="mouse", date_of_birth="2014-09-01"), + >>> dict(subject_id=8, species="mouse", date_of_birth="2014-09-02")]) + + # Large insert with chunking + >>> Table.insert(large_dataset, chunk_size=10000) + """ + if isinstance(rows, pandas.DataFrame): + # drop 'extra' synthetic index for 1-field index case - + # frames with more advanced indices should be prepared by user. + rows = rows.reset_index(drop=len(rows.index.names) == 1 and not rows.index.names[0]).to_records(index=False) + + # Polars DataFrame -> list of dicts (soft dependency, check by type name) + if type(rows).__module__.startswith("polars") and type(rows).__name__ == "DataFrame": + rows = rows.to_dicts() + + # PyArrow Table -> list of dicts (soft dependency, check by type name) + if type(rows).__module__.startswith("pyarrow") and type(rows).__name__ == "Table": + rows = rows.to_pylist() + + if isinstance(rows, Path): + with open(rows, newline="") as data_file: + rows = list(csv.DictReader(data_file, delimiter=",")) + + # prohibit direct inserts into auto-populated tables + if not allow_direct_insert and not getattr(self, "_allow_insert", True): + raise DataJointError( + "Inserts into an auto-populated table can only be done inside " + "its make method during a populate call." + " To override, set keyword argument allow_direct_insert=True." + ) + + if inspect.isclass(rows) and issubclass(rows, QueryExpression): + rows = rows() # instantiate if a class + if isinstance(rows, QueryExpression): + # insert from select - chunk_size not applicable + if chunk_size is not None: + raise DataJointError("chunk_size is not supported for QueryExpression inserts") + if not ignore_extra_fields: + try: + raise DataJointError( + "Attribute %s not found. To ignore extra attributes in insert, " + "set ignore_extra_fields=True." % next(name for name in rows.heading if name not in self.heading) + ) + except StopIteration: + pass + fields = list(name for name in rows.heading if name in self.heading) + query = "{command} INTO {table} ({fields}) {select}{duplicate}".format( + command="REPLACE" if replace else "INSERT", + fields="`" + "`,`".join(fields) + "`", + table=self.full_table_name, + select=rows.make_sql(fields), + duplicate=( + " ON DUPLICATE KEY UPDATE `{pk}`={table}.`{pk}`".format(table=self.full_table_name, pk=self.primary_key[0]) + if skip_duplicates + else "" + ), + ) + self.connection.query(query) + return + + # Chunked insert mode + if chunk_size is not None: + rows_iter = iter(rows) + while True: + chunk = list(itertools.islice(rows_iter, chunk_size)) + if not chunk: + break + self._insert_rows(chunk, replace, skip_duplicates, ignore_extra_fields) + return + + # Single batch insert (original behavior) + self._insert_rows(rows, replace, skip_duplicates, ignore_extra_fields) + + def _insert_rows(self, rows, replace, skip_duplicates, ignore_extra_fields): + """ + Internal helper to insert a batch of rows. + + :param rows: Iterable of rows to insert + :param replace: If True, use REPLACE instead of INSERT + :param skip_duplicates: If True, use ON DUPLICATE KEY UPDATE + :param ignore_extra_fields: If True, ignore unknown fields + """ + # collects the field list from first row (passed by reference) + field_list = [] + rows = list(self.__make_row_to_insert(row, field_list, ignore_extra_fields) for row in rows) + if rows: + try: + # Handle empty field_list (all-defaults insert) + fields_clause = f"(`{'`,`'.join(field_list)}`)" if field_list else "()" + query = "{command} INTO {destination}{fields} VALUES {placeholders}{duplicate}".format( + command="REPLACE" if replace else "INSERT", + destination=self.from_clause(), + fields=fields_clause, + placeholders=",".join("(" + ",".join(row["placeholders"]) + ")" for row in rows), + duplicate=( + " ON DUPLICATE KEY UPDATE `{pk}`=`{pk}`".format(pk=self.primary_key[0]) if skip_duplicates else "" + ), + ) + self.connection.query( + query, + args=list(itertools.chain.from_iterable((v for v in r["values"] if v is not None) for r in rows)), + ) + except UnknownAttributeError as err: + raise err.suggest("To ignore extra fields in insert, set ignore_extra_fields=True") + except DuplicateError as err: + raise err.suggest("To ignore duplicate entries in insert, set skip_duplicates=True") + + def insert_dataframe(self, df, index_as_pk=None, **insert_kwargs): + """ + Insert DataFrame with explicit index handling. + + This method provides symmetry with to_pandas(): data fetched with to_pandas() + (which sets primary key as index) can be modified and re-inserted using + insert_dataframe() without manual index manipulation. + + :param df: pandas DataFrame to insert + :param index_as_pk: How to handle DataFrame index: + - None (default): Auto-detect. Use index as primary key if index names + match primary_key columns. Drop if unnamed RangeIndex. + - True: Treat index as primary key columns. Raises if index names don't + match table primary key. + - False: Ignore index entirely (drop it). + :param **insert_kwargs: Passed to insert() - replace, skip_duplicates, + ignore_extra_fields, allow_direct_insert, chunk_size + + Example:: + + # Round-trip with to_pandas() + df = table.to_pandas() # PK becomes index + df['value'] = df['value'] * 2 # Modify data + table.insert_dataframe(df) # Auto-detects index as PK + + # Explicit control + table.insert_dataframe(df, index_as_pk=True) # Use index + table.insert_dataframe(df, index_as_pk=False) # Ignore index + """ + if not isinstance(df, pandas.DataFrame): + raise DataJointError("insert_dataframe requires a pandas DataFrame") + + # Auto-detect if index should be used as PK + if index_as_pk is None: + index_as_pk = self._should_index_be_pk(df) + + # Validate index if using as PK + if index_as_pk: + self._validate_index_columns(df) + + # Prepare rows + if index_as_pk: + rows = df.reset_index(drop=False).to_records(index=False) + else: + rows = df.reset_index(drop=True).to_records(index=False) + + self.insert(rows, **insert_kwargs) + + def _should_index_be_pk(self, df) -> bool: + """ + Auto-detect if DataFrame index should map to primary key. + + Returns True if: + - Index has named columns that exactly match the table's primary key + Returns False if: + - Index is unnamed RangeIndex (synthetic index) + - Index names don't match primary key + """ + # RangeIndex with no name -> False (synthetic index) + if df.index.names == [None]: + return False + # Check if index names match PK columns + index_names = set(n for n in df.index.names if n is not None) + return index_names == set(self.primary_key) + + def _validate_index_columns(self, df): + """Validate that index columns match the table's primary key.""" + index_names = [n for n in df.index.names if n is not None] + if set(index_names) != set(self.primary_key): + raise DataJointError( + f"DataFrame index columns {index_names} do not match " + f"table primary key {list(self.primary_key)}. " + f"Use index_as_pk=False to ignore index, or reset_index() first." + ) + + def delete_quick(self, get_count=False): + """ + Deletes the table without cascading and without user prompt. + If this table has populated dependent tables, this will fail. + """ + query = "DELETE FROM " + self.full_table_name + self.where_clause() + self.connection.query(query) + count = self.connection.query("SELECT ROW_COUNT()").fetchone()[0] if get_count else None + return count + + def delete( + self, + transaction: bool = True, + prompt: bool | None = None, + part_integrity: str = "enforce", + ) -> int: + """ + Deletes the contents of the table and its dependent tables, recursively. + + Args: + transaction: If `True`, use of the entire delete becomes an atomic transaction. + This is the default and recommended behavior. Set to `False` if this delete is + nested within another transaction. + prompt: If `True`, show what will be deleted and ask for confirmation. + If `False`, delete without confirmation. Default is `dj.config['safemode']`. + part_integrity: Policy for master-part integrity. One of: + - ``"enforce"`` (default): Error if parts would be deleted without masters. + - ``"ignore"``: Allow deleting parts without masters (breaks integrity). + - ``"cascade"``: Also delete masters when parts are deleted (maintains integrity). + + Returns: + Number of deleted rows (excluding those from dependent tables). + + Raises: + DataJointError: Delete exceeds maximum number of delete attempts. + DataJointError: When deleting within an existing transaction. + DataJointError: Deleting a part table before its master (when part_integrity="enforce"). + ValueError: Invalid part_integrity value. + """ + if part_integrity not in ("enforce", "ignore", "cascade"): + raise ValueError(f"part_integrity must be 'enforce', 'ignore', or 'cascade', got {part_integrity!r}") + deleted = set() + visited_masters = set() + + def cascade(table): + """service function to perform cascading deletes recursively.""" + max_attempts = 50 + for _ in range(max_attempts): + try: + delete_count = table.delete_quick(get_count=True) + except IntegrityError as error: + match = foreign_key_error_regexp.match(error.args[0]) + if match is None: + raise DataJointError( + "Cascading deletes failed because the error message is missing foreign key information." + "Make sure you have REFERENCES privilege to all dependent tables." + ) from None + match = match.groupdict() + # if schema name missing, use table + if "`.`" not in match["child"]: + match["child"] = "{}.{}".format(table.full_table_name.split(".")[0], match["child"]) + if match["pk_attrs"] is not None: # fully matched, adjusting the keys + match["fk_attrs"] = [k.strip("`") for k in match["fk_attrs"].split(",")] + match["pk_attrs"] = [k.strip("`") for k in match["pk_attrs"].split(",")] + else: # only partially matched, querying with constraint to determine keys + match["fk_attrs"], match["parent"], match["pk_attrs"] = list( + map( + list, + zip( + *table.connection.query( + constraint_info_query, + args=( + match["name"].strip("`"), + *[_.strip("`") for _ in match["child"].split("`.`")], + ), + ).fetchall() + ), + ) + ) + match["parent"] = match["parent"][0] + + # Restrict child by table if + # 1. if table's restriction attributes are not in child's primary key + # 2. if child renames any attributes + # Otherwise restrict child by table's restriction. + child = FreeTable(table.connection, match["child"]) + if set(table.restriction_attributes) <= set(child.primary_key) and match["fk_attrs"] == match["pk_attrs"]: + child._restriction = table._restriction + child._restriction_attributes = table.restriction_attributes + elif match["fk_attrs"] != match["pk_attrs"]: + child &= table.proj(**dict(zip(match["fk_attrs"], match["pk_attrs"]))) + else: + child &= table.proj() + + master_name = get_master(child.full_table_name) + if ( + part_integrity == "cascade" + and master_name + and master_name != table.full_table_name + and master_name not in visited_masters + ): + master = FreeTable(table.connection, master_name) + master._restriction_attributes = set() + master._restriction = [ + make_condition( # &= may cause in target tables in subquery + master, + (master.proj() & child.proj()).to_arrays(), + master._restriction_attributes, + ) + ] + visited_masters.add(master_name) + cascade(master) + else: + cascade(child) + else: + deleted.add(table.full_table_name) + logger.info("Deleting {count} rows from {table}".format(count=delete_count, table=table.full_table_name)) + break + else: + raise DataJointError("Exceeded maximum number of delete attempts.") + return delete_count + + prompt = config["safemode"] if prompt is None else prompt + + # Start transaction + if transaction: + if not self.connection.in_transaction: + self.connection.start_transaction() + else: + if not prompt: + transaction = False + else: + raise DataJointError( + "Delete cannot use a transaction within an ongoing transaction. Set transaction=False or prompt=False." + ) + + # Cascading delete + try: + delete_count = cascade(self) + except: + if transaction: + self.connection.cancel_transaction() + raise + + if part_integrity == "enforce": + # Avoid deleting from part before master (See issue #151) + for part in deleted: + master = get_master(part) + if master and master not in deleted: + if transaction: + self.connection.cancel_transaction() + raise DataJointError( + "Attempt to delete part table {part} before deleting from its master {master} first. " + "Use part_integrity='ignore' to allow, or part_integrity='cascade' to also delete master.".format( + part=part, master=master + ) + ) + + # Confirm and commit + if delete_count == 0: + if prompt: + logger.warning("Nothing to delete.") + if transaction: + self.connection.cancel_transaction() + elif not transaction: + logger.info("Delete completed") + else: + if not prompt or user_choice("Commit deletes?", default="no") == "yes": + if transaction: + self.connection.commit_transaction() + if prompt: + logger.info("Delete committed.") + else: + if transaction: + self.connection.cancel_transaction() + if prompt: + logger.warning("Delete cancelled") + delete_count = 0 # Reset count when delete is cancelled + return delete_count + + def drop_quick(self): + """ + Drops the table without cascading to dependent tables and without user prompt. + """ + if self.is_declared: + # Clean up lineage entries for this table + from .lineage import delete_table_lineages + + delete_table_lineages(self.connection, self.database, self.table_name) + + query = "DROP TABLE %s" % self.full_table_name + self.connection.query(query) + logger.info("Dropped table %s" % self.full_table_name) + else: + logger.info("Nothing to drop: table %s is not declared" % self.full_table_name) + + def drop(self, prompt: bool | None = None): + """ + Drop the table and all tables that reference it, recursively. + + Args: + prompt: If `True`, show what will be dropped and ask for confirmation. + If `False`, drop without confirmation. Default is `dj.config['safemode']`. + """ + if self.restriction: + raise DataJointError( + "A table with an applied restriction cannot be dropped. Call drop() on the unrestricted Table." + ) + prompt = config["safemode"] if prompt is None else prompt + + self.connection.dependencies.load() + do_drop = True + tables = [table for table in self.connection.dependencies.descendants(self.full_table_name) if not table.isdigit()] + + # avoid dropping part tables without their masters: See issue #374 + for part in tables: + master = get_master(part) + if master and master not in tables: + raise DataJointError( + "Attempt to drop part table {part} before dropping its master. Drop {master} first.".format( + part=part, master=master + ) + ) + + if prompt: + for table in tables: + logger.info(table + " (%d tuples)" % len(FreeTable(self.connection, table))) + do_drop = user_choice("Proceed?", default="no") == "yes" + if do_drop: + for table in reversed(tables): + FreeTable(self.connection, table).drop_quick() + logger.info("Tables dropped. Restart kernel.") + + @property + def size_on_disk(self): + """ + :return: size of data and indices in bytes on the storage device + """ + ret = self.connection.query( + 'SHOW TABLE STATUS FROM `{database}` WHERE NAME="{table}"'.format(database=self.database, table=self.table_name), + as_dict=True, + ).fetchone() + return ret["Data_length"] + ret["Index_length"] + + def describe(self, context=None, printout=False): + """ + :return: the definition string for the query using DataJoint DDL. + """ + if context is None: + frame = inspect.currentframe().f_back + context = dict(frame.f_globals, **frame.f_locals) + del frame + if self.full_table_name not in self.connection.dependencies: + self.connection.dependencies.load() + parents = self.parents(foreign_key_info=True) + in_key = True + definition = "# " + self.heading.table_status["comment"] + "\n" if self.heading.table_status["comment"] else "" + attributes_thus_far = set() + attributes_declared = set() + indexes = self.heading.indexes.copy() + for attr in self.heading.attributes.values(): + if in_key and not attr.in_key: + definition += "---\n" + in_key = False + attributes_thus_far.add(attr.name) + do_include = True + for parent_name, fk_props in parents: + if attr.name in fk_props["attr_map"]: + do_include = False + if attributes_thus_far.issuperset(fk_props["attr_map"]): + # foreign key properties + try: + index_props = indexes.pop(tuple(fk_props["attr_map"])) + except KeyError: + index_props = "" + else: + index_props = [k for k, v in index_props.items() if v] + index_props = " [{}]".format(", ".join(index_props)) if index_props else "" + + if not fk_props["aliased"]: + # simple foreign key + definition += "->{props} {class_name}\n".format( + props=index_props, + class_name=lookup_class_name(parent_name, context) or parent_name, + ) + else: + # projected foreign key + definition += "->{props} {class_name}.proj({proj_list})\n".format( + props=index_props, + class_name=lookup_class_name(parent_name, context) or parent_name, + proj_list=",".join( + '{}="{}"'.format(attr, ref) for attr, ref in fk_props["attr_map"].items() if ref != attr + ), + ) + attributes_declared.update(fk_props["attr_map"]) + if do_include: + attributes_declared.add(attr.name) + # Use original_type (core type alias) if available, otherwise use type + display_type = attr.original_type or attr.type + definition += "%-20s : %-28s %s\n" % ( + (attr.name if attr.default is None else "%s=%s" % (attr.name, attr.default)), + "%s%s" % (display_type, " auto_increment" if attr.autoincrement else ""), + "# " + attr.comment if attr.comment else "", + ) + # add remaining indexes + for k, v in indexes.items(): + definition += "{unique}INDEX ({attrs})\n".format(unique="UNIQUE " if v["unique"] else "", attrs=", ".join(k)) + if printout: + logger.info("\n" + definition) + return definition + + # --- private helper functions ---- + def __make_placeholder(self, name, value, ignore_extra_fields=False, row=None): + """ + For a given attribute `name` with `value`, return its processed value or value placeholder + as a string to be included in the query and the value, if any, to be submitted for + processing by mysql API. + + In the simplified type system: + - Codecs handle all custom encoding via type chains + - UUID values are converted to bytes + - JSON values are serialized + - Blob values pass through as bytes + - Numeric values are stringified + + :param name: name of attribute to be inserted + :param value: value of attribute to be inserted + :param ignore_extra_fields: if True, return None for unknown fields + :param row: the full row dict (unused in simplified model) + """ + if ignore_extra_fields and name not in self.heading: + return None + attr = self.heading[name] + + # Apply adapter encoding with type chain support + if attr.codec: + from .codecs import resolve_dtype + + # Skip validation and encoding for None values (nullable columns) + if value is None: + return name, "DEFAULT", None + + attr.codec.validate(value) + + # Resolve full type chain + _, type_chain, resolved_store = resolve_dtype(f"<{attr.codec.name}>", store_name=attr.store) + + # Apply encoders from outermost to innermost + for attr_type in type_chain: + # Pass store_name to encoders that support it (check via introspection) + import inspect + + sig = inspect.signature(attr_type.encode) + if "store_name" in sig.parameters: + value = attr_type.encode(value, key=None, store_name=resolved_store) + else: + value = attr_type.encode(value, key=None) + + # Handle NULL values + if value is None or (attr.numeric and (value == "" or np.isnan(float(value)))): + placeholder, value = "DEFAULT", None + else: + placeholder = "%s" + # UUID - convert to bytes + if attr.uuid: + if not isinstance(value, uuid.UUID): + try: + value = uuid.UUID(value) + except (AttributeError, ValueError): + raise DataJointError(f"badly formed UUID value {value} for attribute `{name}`") + value = value.bytes + # JSON - serialize to string + elif attr.json: + value = json.dumps(value) + # Numeric - convert to string + elif attr.numeric: + value = str(int(value) if isinstance(value, (bool, np.bool_)) else value) + # Blob - pass through as bytes (use for automatic serialization) + + return name, placeholder, value + + def __make_row_to_insert(self, row, field_list, ignore_extra_fields): + """ + Helper function for insert and update + + :param row: A tuple to insert + :return: a dict with fields 'names', 'placeholders', 'values' + """ + + def check_fields(fields): + """ + Validates that all items in `fields` are valid attributes in the heading + + :param fields: field names of a tuple + """ + if not field_list: + if not ignore_extra_fields: + for field in fields: + if field not in self.heading: + raise KeyError("`{0:s}` is not in the table heading".format(field)) + elif set(field_list) != set(fields).intersection(self.heading.names): + raise DataJointError("Attempt to insert rows with different fields.") + + # Convert row to dict for object attribute processing + row_dict = None + if isinstance(row, np.void): # np.array + check_fields(row.dtype.fields) + row_dict = {name: row[name] for name in row.dtype.fields} + attributes = [ + self.__make_placeholder(name, row[name], ignore_extra_fields, row=row_dict) + for name in self.heading + if name in row.dtype.fields + ] + elif isinstance(row, collections.abc.Mapping): # dict-based + check_fields(row) + row_dict = dict(row) + attributes = [ + self.__make_placeholder(name, row[name], ignore_extra_fields, row=row_dict) + for name in self.heading + if name in row + ] + else: # positional + warnings.warn( + "Positional inserts (tuples/lists) are deprecated and will be removed in a future version. " + "Use dict with explicit field names instead: table.insert1({'field': value, ...})", + DeprecationWarning, + stacklevel=4, # Point to user's insert()/insert1() call + ) + try: + if len(row) != len(self.heading): + raise DataJointError( + "Invalid insert argument. Incorrect number of attributes: {given} given; {expected} expected".format( + given=len(row), expected=len(self.heading) + ) + ) + except TypeError: + raise DataJointError("Datatype %s cannot be inserted" % type(row)) + else: + row_dict = dict(zip(self.heading.names, row)) + attributes = [ + self.__make_placeholder(name, value, ignore_extra_fields, row=row_dict) + for name, value in zip(self.heading, row) + ] + if ignore_extra_fields: + attributes = [a for a in attributes if a is not None] + + if not attributes: + # Check if empty insert is allowed (all attributes have defaults) + required_attrs = [ + attr.name + for attr in self.heading.attributes.values() + if not (attr.autoincrement or attr.nullable or attr.default is not None) + ] + if required_attrs: + raise DataJointError(f"Cannot insert empty row. The following attributes require values: {required_attrs}") + # All attributes have defaults - allow empty insert + row_to_insert = {"names": (), "placeholders": (), "values": ()} + else: + row_to_insert = dict(zip(("names", "placeholders", "values"), zip(*attributes))) + if not field_list: + # first row sets the composition of the field list + field_list.extend(row_to_insert["names"]) + else: + # reorder attributes in row_to_insert to match field_list + order = list(row_to_insert["names"].index(field) for field in field_list) + row_to_insert["names"] = list(row_to_insert["names"][i] for i in order) + row_to_insert["placeholders"] = list(row_to_insert["placeholders"][i] for i in order) + row_to_insert["values"] = list(row_to_insert["values"][i] for i in order) + return row_to_insert + + +def lookup_class_name(name, context, depth=3): + """ + given a table name in the form `schema_name`.`table_name`, find its class in the context. + + :param name: `schema_name`.`table_name` + :param context: dictionary representing the namespace + :param depth: search depth into imported modules, helps avoid infinite recursion. + :return: class name found in the context or None if not found + """ + # breadth-first search + nodes = [dict(context=context, context_name="", depth=depth)] + while nodes: + node = nodes.pop(0) + for member_name, member in node["context"].items(): + # skip IPython's implicit variables + if not member_name.startswith("_"): + if inspect.isclass(member) and issubclass(member, Table): + if member.full_table_name == name: # found it! + return ".".join([node["context_name"], member_name]).lstrip(".") + try: # look for part tables + parts = member.__dict__ + except AttributeError: + pass # not a UserTable -- cannot have part tables. + else: + for part in (getattr(member, p) for p in parts if p[0].isupper() and hasattr(member, p)): + if inspect.isclass(part) and issubclass(part, Table) and part.full_table_name == name: + return ".".join([node["context_name"], member_name, part.__name__]).lstrip(".") + elif node["depth"] > 0 and inspect.ismodule(member) and member.__name__ != "datajoint": + try: + nodes.append( + dict( + context=dict(inspect.getmembers(member)), + context_name=node["context_name"] + "." + member_name, + depth=node["depth"] - 1, + ) + ) + except (ImportError, TypeError): + pass # could not inspect module members, skip + return None + + +class FreeTable(Table): + """ + A base table without a dedicated class. Each instance is associated with a table + specified by full_table_name. + + :param conn: a dj.Connection object + :param full_table_name: in format `database`.`table_name` + """ + + def __init__(self, conn, full_table_name): + self.database, self._table_name = (s.strip("`") for s in full_table_name.split(".")) + self._connection = conn + self._support = [full_table_name] + self._heading = Heading( + table_info=dict( + conn=conn, + database=self.database, + table_name=self.table_name, + context=None, + ) + ) + + def __repr__(self): + return "FreeTable(`%s`.`%s`)\n" % (self.database, self._table_name) + super().__repr__() diff --git a/src/datajoint/types.py b/src/datajoint/types.py new file mode 100644 index 000000000..72cefee3c --- /dev/null +++ b/src/datajoint/types.py @@ -0,0 +1,60 @@ +""" +Type definitions for DataJoint. + +This module defines type aliases used throughout the DataJoint codebase +to improve code clarity and enable better static type checking. + +Python 3.10+ is required. +""" + +from __future__ import annotations + +from typing import Any, TypeAlias + +# Primary key types +PrimaryKey: TypeAlias = dict[str, Any] +"""A dictionary mapping attribute names to values that uniquely identify an entity.""" + +PrimaryKeyList: TypeAlias = list[dict[str, Any]] +"""A list of primary key dictionaries.""" + +# Row/record types +Row: TypeAlias = dict[str, Any] +"""A single row/record as a dictionary mapping attribute names to values.""" + +RowList: TypeAlias = list[dict[str, Any]] +"""A list of rows/records.""" + +# Attribute types +AttributeName: TypeAlias = str +"""Name of a table attribute/column.""" + +AttributeNames: TypeAlias = list[str] +"""List of attribute/column names.""" + +# Table and schema names +TableName: TypeAlias = str +"""Simple table name (e.g., 'session').""" + +FullTableName: TypeAlias = str +"""Fully qualified table name (e.g., '`schema`.`table`').""" + +SchemaName: TypeAlias = str +"""Database schema name.""" + +# Foreign key mapping +ForeignKeyMap: TypeAlias = dict[str, tuple[str, str]] +"""Mapping of child_attr -> (parent_table, parent_attr) for foreign keys.""" + +# Restriction types +Restriction: TypeAlias = str | dict[str, Any] | bool | "QueryExpression" | list | None +"""Valid restriction types for query operations.""" + +# Fetch result types +FetchResult: TypeAlias = list[dict[str, Any]] +"""Result of a fetch operation as list of dictionaries.""" + + +# For avoiding circular imports +if False: # TYPE_CHECKING equivalent that's always False + from .expression import QueryExpression diff --git a/datajoint/user_tables.py b/src/datajoint/user_tables.py similarity index 57% rename from datajoint/user_tables.py rename to src/datajoint/user_tables.py index 9c2e79d34..942179685 100644 --- a/datajoint/user_tables.py +++ b/src/datajoint/user_tables.py @@ -7,7 +7,7 @@ from .autopopulate import AutoPopulate from .errors import DataJointError from .table import Table -from .utils import ClassProperty, from_camel_case +from .utils import from_camel_case _base_regexp = r"[a-z][a-z0-9]*(_[a-z][a-z0-9]*)*" @@ -25,7 +25,13 @@ "proj", "aggr", "join", - "fetch", + "extend", + "to_dicts", + "to_pandas", + "to_polars", + "to_arrow", + "to_arrays", + "keys", "fetch1", "head", "tail", @@ -36,27 +42,26 @@ "children", "insert", "insert1", + "insert_dataframe", "update1", + "validate", "drop", "drop_quick", "delete", "delete_quick", + "staged_insert1", } class TableMeta(type): """ TableMeta subclasses allow applying some instance methods and properties directly - at class level. For example, this allows Table.fetch() instead of Table().fetch(). + at class level. For example, this allows Table.to_dicts() instead of Table().to_dicts(). """ def __getattribute__(cls, name): # trigger instantiation for supported class attrs - return ( - cls().__getattribute__(name) - if name in supported_class_attrs - else super().__getattribute__(name) - ) + return cls().__getattribute__(name) if name in supported_class_attrs else super().__getattribute__(name) def __and__(cls, arg): return cls() & arg @@ -82,6 +87,26 @@ def __add__(cls, arg): def __iter__(cls): return iter(cls()) + # Class properties - defined on metaclass to work at class level + @property + def connection(cls): + """The database connection for this table.""" + return cls._connection + + @property + def table_name(cls): + """The table name formatted for MySQL.""" + if cls._prefix is None: + raise AttributeError("Class prefix is not defined!") + return cls._prefix + from_camel_case(cls.__name__) + + @property + def full_table_name(cls): + """The fully qualified table name (`database`.`table`).""" + if cls.database is None: + return None + return r"`{0:s}`.`{1:s}`".format(cls.database, cls.table_name) + class UserTable(Table, metaclass=TableMeta): """ @@ -103,33 +128,7 @@ def definition(self): """ :return: a string containing the table definition using the DataJoint DDL. """ - raise NotImplementedError( - 'Subclasses of Table must implement the property "definition"' - ) - - @ClassProperty - def connection(cls): - return cls._connection - - @ClassProperty - def table_name(cls): - """ - :return: the table name of the table formatted for mysql. - """ - if cls._prefix is None: - raise AttributeError("Class prefix is not defined!") - return cls._prefix + from_camel_case(cls.__name__) - - @ClassProperty - def full_table_name(cls): - if cls not in {Manual, Imported, Lookup, Computed, Part, UserTable}: - # for derived classes only - if cls.database is None: - raise DataJointError( - "Class %s is not properly declared (schema decorator not applied?)" - % cls.__name__ - ) - return r"`{0:s}`.`{1:s}`".format(cls.database, cls.table_name) + raise NotImplementedError('Subclasses of Table must implement the property "definition"') class Manual(UserTable): @@ -149,9 +148,7 @@ class Lookup(UserTable): """ _prefix = "#" - tier_regexp = ( - r"(?P" + _prefix + _base_regexp.replace("TIER", "lookup") + ")" - ) + tier_regexp = r"(?P" + _prefix + _base_regexp.replace("TIER", "lookup") + ")" class Imported(UserTable, AutoPopulate): @@ -174,7 +171,28 @@ class Computed(UserTable, AutoPopulate): tier_regexp = r"(?P" + _prefix + _base_regexp + ")" -class Part(UserTable): +class PartMeta(TableMeta): + """Metaclass for Part tables with overridden class properties.""" + + @property + def table_name(cls): + """The table name for a Part is derived from its master table.""" + return None if cls.master is None else cls.master.table_name + "__" + from_camel_case(cls.__name__) + + @property + def full_table_name(cls): + """The fully qualified table name (`database`.`table`).""" + if cls.database is None or cls.table_name is None: + return None + return r"`{0:s}`.`{1:s}`".format(cls.database, cls.table_name) + + @property + def master(cls): + """The master table for this Part table.""" + return cls._master + + +class Part(UserTable, metaclass=PartMeta): """ Inherit from this class if the table's values are details of an entry in another table and if this table is populated by the other table. For example, the entries inheriting from @@ -195,51 +213,48 @@ class Part(UserTable): + ")" ) - @ClassProperty - def connection(cls): - return cls._connection - - @ClassProperty - def full_table_name(cls): - return ( - None - if cls.database is None or cls.table_name is None - else r"`{0:s}`.`{1:s}`".format(cls.database, cls.table_name) - ) - - @ClassProperty - def master(cls): - return cls._master - - @ClassProperty - def table_name(cls): - return ( - None - if cls.master is None - else cls.master.table_name + "__" + from_camel_case(cls.__name__) - ) - - def delete(self, force=False): + def delete(self, part_integrity: str = "enforce", **kwargs): """ - unless force is True, prohibits direct deletes from parts. + Delete from a Part table. + + Args: + part_integrity: Policy for master-part integrity. One of: + - ``"enforce"`` (default): Error - delete from master instead. + - ``"ignore"``: Allow direct deletion (breaks master-part integrity). + - ``"cascade"``: Delete parts AND cascade up to delete master. + **kwargs: Additional arguments passed to Table.delete() + (transaction, prompt) + + Raises: + DataJointError: If part_integrity="enforce" (direct Part deletes prohibited) """ - if force: - super().delete(force_parts=True) - else: + if part_integrity == "enforce": raise DataJointError( - "Cannot delete from a Part directly. Delete from master instead" + "Cannot delete from a Part directly. Delete from master instead, " + "or use part_integrity='ignore' to break integrity, " + "or part_integrity='cascade' to also delete master." ) + super().delete(part_integrity=part_integrity, **kwargs) - def drop(self, force=False): + def drop(self, part_integrity: str = "enforce"): """ - unless force is True, prohibits direct deletes from parts. + Drop a Part table. + + Args: + part_integrity: Policy for master-part integrity. One of: + - ``"enforce"`` (default): Error - drop master instead. + - ``"ignore"``: Allow direct drop (breaks master-part structure). + Note: ``"cascade"`` is not supported for drop (too destructive). + + Raises: + DataJointError: If part_integrity="enforce" (direct Part drops prohibited) """ - if force: + if part_integrity == "ignore": super().drop() + elif part_integrity == "enforce": + raise DataJointError("Cannot drop a Part directly. Drop master instead, or use part_integrity='ignore' to force.") else: - raise DataJointError( - "Cannot drop a Part directly. Delete from master instead" - ) + raise ValueError(f"part_integrity for drop must be 'enforce' or 'ignore', got {part_integrity!r}") def alter(self, prompt=True, context=None): # without context, use declaration context which maps master keyword to master table @@ -263,10 +278,6 @@ def _get_tier(table_name): return _AliasNode else: try: - return next( - tier - for tier in user_table_classes - if re.fullmatch(tier.tier_regexp, table_name.split("`")[-2]) - ) + return next(tier for tier in user_table_classes if re.fullmatch(tier.tier_regexp, table_name.split("`")[-2])) except StopIteration: return None diff --git a/datajoint/utils.py b/src/datajoint/utils.py similarity index 91% rename from datajoint/utils.py rename to src/datajoint/utils.py index c34536685..e8303a993 100644 --- a/datajoint/utils.py +++ b/src/datajoint/utils.py @@ -7,14 +7,6 @@ from .errors import DataJointError -class ClassProperty: - def __init__(self, f): - self.f = f - - def __get__(self, obj, owner): - return self.f(owner) - - def user_choice(prompt, choices=("yes", "no"), default=None): """ Prompts the user for confirmation. The default value, if any, is capitalized. @@ -25,9 +17,7 @@ def user_choice(prompt, choices=("yes", "no"), default=None): :return: the user's choice """ assert default is None or default in choices - choice_list = ", ".join( - (choice.title() if choice == default else choice for choice in choices) - ) + choice_list = ", ".join((choice.title() if choice == default else choice for choice in choices)) response = None while response not in choices: response = input(prompt + " [" + choice_list + "]: ") @@ -97,9 +87,7 @@ def convert(match): return ("_" if match.groups()[0] else "") + match.group(0).lower() if not is_camel_case(s): - raise DataJointError( - "ClassName must be alphanumeric in CamelCase, begin with a capital letter" - ) + raise DataJointError("ClassName must be alphanumeric in CamelCase, begin with a capital letter") return re.sub(r"(\B[A-Z])|(\b[A-Z])", convert, s) diff --git a/datajoint/version.py b/src/datajoint/version.py similarity index 64% rename from datajoint/version.py rename to src/datajoint/version.py index 5fb608cef..31f651ea6 100644 --- a/datajoint/version.py +++ b/src/datajoint/version.py @@ -1,6 +1,4 @@ # version bump auto managed by Github Actions: # label_prs.yaml(prep), release.yaml(bump), post_release.yaml(edit) # manually set this version will be eventually overwritten by the above actions -__version__ = "0.14.6" - -assert len(__version__) <= 10 # The log table limits version to the 10 characters +__version__ = "2.0.0a16" diff --git a/tests/conftest.py b/tests/conftest.py index 88d55e32f..6d03dece7 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,136 +1,297 @@ -import json +""" +Pytest configuration for DataJoint tests. + +Tests are organized by their dependencies: +- Unit tests: No external dependencies, run with `pytest -m "not requires_mysql"` +- Integration tests: Require MySQL/MinIO, marked with @pytest.mark.requires_mysql + +Containers are automatically started via testcontainers when needed. +Just run: pytest tests/ + +To use external containers instead (e.g., docker-compose), set: + DJ_USE_EXTERNAL_CONTAINERS=1 + DJ_HOST=localhost DJ_PORT=3306 S3_ENDPOINT=localhost:9000 pytest + +To run only unit tests (no Docker required): + pytest -m "not requires_mysql" +""" + +import logging import os -import shutil -from os import environ, remove -from pathlib import Path +from os import remove from typing import Dict, List import certifi -import minio -import networkx as nx import pytest import urllib3 -from packaging import version import datajoint as dj -from datajoint import errors -from datajoint.errors import ( - ADAPTED_TYPE_SWITCH, - FILEPATH_FEATURE_SWITCH, - DataJointError, -) - -from . import schema, schema_adapted, schema_advanced, schema_external, schema_simple +from datajoint.errors import DataJointError + +from . import schema, schema_advanced, schema_external, schema_object, schema_simple from . import schema_uuid as schema_uuid_module +from . import schema_type_aliases as schema_type_aliases_module + +logger = logging.getLogger(__name__) + + +# ============================================================================= +# Pytest Hooks +# ============================================================================= + + +def pytest_collection_modifyitems(config, items): + """Auto-mark integration tests based on their fixtures.""" + # Tests that use these fixtures require MySQL + mysql_fixtures = { + "connection_root", + "connection_root_bare", + "connection_test", + "schema_any", + "schema_any_fresh", + "schema_simp", + "schema_adv", + "schema_ext", + "schema_uuid", + "schema_type_aliases", + "schema_obj", + "db_creds_root", + "db_creds_test", + } + # Tests that use these fixtures require MinIO + minio_fixtures = { + "minio_client", + "s3fs_client", + "s3_creds", + "stores_config", + "mock_stores", + } + for item in items: + # Get all fixtures this test uses (directly or indirectly) + try: + fixturenames = set(item.fixturenames) + except AttributeError: + continue -@pytest.fixture(scope="session") -def prefix(): - return os.environ.get("DJ_TEST_DB_PREFIX", "djtest") + # Auto-add marks based on fixture usage + if fixturenames & mysql_fixtures: + item.add_marker(pytest.mark.requires_mysql) + if fixturenames & minio_fixtures: + item.add_marker(pytest.mark.requires_minio) -@pytest.fixture(scope="session") -def monkeysession(): - with pytest.MonkeyPatch.context() as mp: - yield mp +# ============================================================================= +# Container Fixtures - Auto-start MySQL and MinIO via testcontainers +# ============================================================================= +# Check if we should use external containers (for CI or manual docker-compose) +USE_EXTERNAL_CONTAINERS = os.environ.get("DJ_USE_EXTERNAL_CONTAINERS", "").lower() in ("1", "true", "yes") -@pytest.fixture(scope="module") -def monkeymodule(): - with pytest.MonkeyPatch.context() as mp: - yield mp +@pytest.fixture(scope="session") +def mysql_container(): + """Start MySQL container for the test session (or use external).""" + if USE_EXTERNAL_CONTAINERS: + # Use external container - return None, credentials come from env + logger.info("Using external MySQL container") + yield None + return -@pytest.fixture -def enable_adapted_types(monkeypatch): - monkeypatch.setenv(ADAPTED_TYPE_SWITCH, "TRUE") - yield - monkeypatch.delenv(ADAPTED_TYPE_SWITCH, raising=True) + from testcontainers.mysql import MySqlContainer + container = MySqlContainer( + image="mysql:8.0", + username="root", + password="password", + dbname="test", + ) + container.start() -@pytest.fixture -def enable_filepath_feature(monkeypatch): - monkeypatch.setenv(FILEPATH_FEATURE_SWITCH, "TRUE") - yield - monkeypatch.delenv(FILEPATH_FEATURE_SWITCH, raising=True) + host = container.get_container_host_ip() + port = container.get_exposed_port(3306) + logger.info(f"MySQL container started at {host}:{port}") + yield container -@pytest.fixture(scope="session") -def db_creds_test() -> Dict: - return dict( - host=os.getenv("DJ_TEST_HOST", "db"), - user=os.getenv("DJ_TEST_USER", "datajoint"), - password=os.getenv("DJ_TEST_PASSWORD", "datajoint"), - ) + container.stop() + logger.info("MySQL container stopped") @pytest.fixture(scope="session") -def db_creds_root() -> Dict: - return dict( - host=os.getenv("DJ_HOST", "db"), - user=os.getenv("DJ_USER", "root"), - password=os.getenv("DJ_PASS", "password"), +def minio_container(): + """Start MinIO container for the test session (or use external).""" + if USE_EXTERNAL_CONTAINERS: + # Use external container - return None, credentials come from env + logger.info("Using external MinIO container") + yield None + return + + from testcontainers.minio import MinioContainer + + container = MinioContainer( + image="minio/minio:latest", + access_key="datajoint", + secret_key="datajoint", ) + container.start() + + host = container.get_container_host_ip() + port = container.get_exposed_port(9000) + logger.info(f"MinIO container started at {host}:{port}") + + yield container + + container.stop() + logger.info("MinIO container stopped") + + +# ============================================================================= +# Credential Fixtures - Derived from containers or environment +# ============================================================================= @pytest.fixture(scope="session") -def connection_root_bare(db_creds_root): - connection = dj.Connection(**db_creds_root) - yield connection +def prefix(): + return os.environ.get("DJ_TEST_DB_PREFIX", "djtest") @pytest.fixture(scope="session") -def connection_root(connection_root_bare, prefix): - """Root user database connection.""" - dj.config["safemode"] = False - conn_root = connection_root_bare - # Create MySQL users - if version.parse( - conn_root.query("select @@version;").fetchone()[0] - ) >= version.parse("8.0.0"): - # create user if necessary on mysql8 - conn_root.query( - """ - CREATE USER IF NOT EXISTS 'datajoint'@'%%' - IDENTIFIED BY 'datajoint'; - """ +def db_creds_root(mysql_container) -> Dict: + """Root database credentials from container or environment.""" + if mysql_container is not None: + # From testcontainer + host = mysql_container.get_container_host_ip() + port = mysql_container.get_exposed_port(3306) + return dict( + host=f"{host}:{port}", + user="root", + password="password", ) - conn_root.query( - """ - CREATE USER IF NOT EXISTS 'djview'@'%%' - IDENTIFIED BY 'djview'; - """ + else: + # From environment (external container) + host = os.environ.get("DJ_HOST", "localhost") + port = os.environ.get("DJ_PORT", "3306") + return dict( + host=f"{host}:{port}" if port else host, + user=os.environ.get("DJ_USER", "root"), + password=os.environ.get("DJ_PASS", "password"), ) - conn_root.query( - """ - CREATE USER IF NOT EXISTS 'djssl'@'%%' - IDENTIFIED BY 'djssl' - REQUIRE SSL; - """ + + +@pytest.fixture(scope="session") +def db_creds_test(mysql_container) -> Dict: + """Test user database credentials from container or environment.""" + if mysql_container is not None: + # From testcontainer + host = mysql_container.get_container_host_ip() + port = mysql_container.get_exposed_port(3306) + return dict( + host=f"{host}:{port}", + user="datajoint", + password="datajoint", ) - conn_root.query("GRANT ALL PRIVILEGES ON `djtest%%`.* TO 'datajoint'@'%%';") - conn_root.query("GRANT SELECT ON `djtest%%`.* TO 'djview'@'%%';") - conn_root.query("GRANT SELECT ON `djtest%%`.* TO 'djssl'@'%%';") else: - # grant permissions. For MySQL 5.7 this also automatically creates user - # if not exists - conn_root.query( - """ - GRANT ALL PRIVILEGES ON `djtest%%`.* TO 'datajoint'@'%%' - IDENTIFIED BY 'datajoint'; - """ + # From environment (external container) + host = os.environ.get("DJ_HOST", "localhost") + port = os.environ.get("DJ_PORT", "3306") + return dict( + host=f"{host}:{port}" if port else host, + user=os.environ.get("DJ_TEST_USER", "datajoint"), + password=os.environ.get("DJ_TEST_PASSWORD", "datajoint"), ) - conn_root.query( - "GRANT SELECT ON `djtest%%`.* TO 'djview'@'%%' IDENTIFIED BY 'djview';" + + +@pytest.fixture(scope="session") +def s3_creds(minio_container) -> Dict: + """S3/MinIO credentials from container or environment.""" + if minio_container is not None: + # From testcontainer + host = minio_container.get_container_host_ip() + port = minio_container.get_exposed_port(9000) + return dict( + endpoint=f"{host}:{port}", + access_key="datajoint", + secret_key="datajoint", + bucket="datajoint.test", ) - conn_root.query( - """ - GRANT SELECT ON `djtest%%`.* TO 'djssl'@'%%' - IDENTIFIED BY 'djssl' - REQUIRE SSL; - """ + else: + # From environment (external container) + return dict( + endpoint=os.environ.get("S3_ENDPOINT", "localhost:9000"), + access_key=os.environ.get("S3_ACCESS_KEY", "datajoint"), + secret_key=os.environ.get("S3_SECRET_KEY", "datajoint"), + bucket=os.environ.get("S3_BUCKET", "datajoint.test"), ) + +# ============================================================================= +# DataJoint Configuration +# ============================================================================= + + +@pytest.fixture(scope="session") +def configure_datajoint(db_creds_root): + """Configure DataJoint to use test database. + + This fixture is NOT autouse - it only runs when a test requests + a fixture that depends on it (e.g., connection_root_bare). + """ + # Parse host:port from credentials + host_port = db_creds_root["host"] + if ":" in host_port: + host, port = host_port.rsplit(":", 1) + else: + host, port = host_port, "3306" + + dj.config["database.host"] = host + dj.config["database.port"] = int(port) + dj.config["safemode"] = False + + logger.info(f"Configured DataJoint to use MySQL at {host}:{port}") + + +# ============================================================================= +# Connection Fixtures +# ============================================================================= + + +@pytest.fixture(scope="session") +def connection_root_bare(db_creds_root, configure_datajoint): + """Bare root connection without user setup.""" + connection = dj.Connection(**db_creds_root) + yield connection + + +@pytest.fixture(scope="session") +def connection_root(connection_root_bare, prefix): + """Root database connection with test users created.""" + conn_root = connection_root_bare + + # Create MySQL users (MySQL 8.0+ syntax - we only support 8.0+) + conn_root.query( + """ + CREATE USER IF NOT EXISTS 'datajoint'@'%%' + IDENTIFIED BY 'datajoint'; + """ + ) + conn_root.query( + """ + CREATE USER IF NOT EXISTS 'djview'@'%%' + IDENTIFIED BY 'djview'; + """ + ) + conn_root.query( + """ + CREATE USER IF NOT EXISTS 'djssl'@'%%' + IDENTIFIED BY 'djssl' + REQUIRE SSL; + """ + ) + conn_root.query("GRANT ALL PRIVILEGES ON `djtest%%`.* TO 'datajoint'@'%%';") + conn_root.query("GRANT SELECT ON `djtest%%`.* TO 'djview'@'%%';") + conn_root.query("GRANT SELECT ON `djtest%%`.* TO 'djssl'@'%%';") + yield conn_root # Teardown @@ -142,7 +303,6 @@ def connection_root(connection_root_bare, prefix): if os.path.exists("dj_local_conf.json"): remove("dj_local_conf.json") - # Remove created users conn_root.query("DROP USER IF EXISTS `datajoint`") conn_root.query("DROP USER IF EXISTS `djview`") conn_root.query("DROP USER IF EXISTS `djssl`") @@ -155,33 +315,19 @@ def connection_test(connection_root, prefix, db_creds_test): database = f"{prefix}%%" permission = "ALL PRIVILEGES" - # Create MySQL users - if version.parse( - connection_root.query("select @@version;").fetchone()[0] - ) >= version.parse("8.0.0"): - # create user if necessary on mysql8 - connection_root.query( - f""" - CREATE USER IF NOT EXISTS '{db_creds_test["user"]}'@'%%' - IDENTIFIED BY '{db_creds_test["password"]}'; - """ - ) - connection_root.query( - f""" - GRANT {permission} ON `{database}`.* - TO '{db_creds_test["user"]}'@'%%'; - """ - ) - else: - # grant permissions. For MySQL 5.7 this also automatically creates user - # if not exists - connection_root.query( - f""" - GRANT {permission} ON `{database}`.* - TO '{db_creds_test["user"]}'@'%%' - IDENTIFIED BY '{db_creds_test["password"]}'; - """ - ) + # MySQL 8.0+ syntax + connection_root.query( + f""" + CREATE USER IF NOT EXISTS '{db_creds_test["user"]}'@'%%' + IDENTIFIED BY '{db_creds_test["password"]}'; + """ + ) + connection_root.query( + f""" + GRANT {permission} ON `{database}`.* + TO '{db_creds_test["user"]}'@'%%'; + """ + ) connection = dj.Connection(**db_creds_test) yield connection @@ -189,50 +335,63 @@ def connection_test(connection_root, prefix, db_creds_test): connection.close() -@pytest.fixture(scope="session") -def s3_creds() -> Dict: - return dict( - endpoint=os.environ.get("S3_ENDPOINT", "minio:9000"), - access_key=os.environ.get("S3_ACCESS_KEY", "datajoint"), - secret_key=os.environ.get("S3_SECRET_KEY", "datajoint"), - bucket=os.environ.get("S3_BUCKET", "datajoint.test"), - ) +# ============================================================================= +# S3/MinIO Fixtures +# ============================================================================= @pytest.fixture(scope="session") def stores_config(s3_creds, tmpdir_factory): - stores_config = { - "raw": dict(protocol="file", location=tmpdir_factory.mktemp("raw")), + """Configure object storage stores for tests.""" + return { + "raw": dict(protocol="file", location=str(tmpdir_factory.mktemp("raw"))), "repo": dict( - stage=tmpdir_factory.mktemp("repo"), + stage=str(tmpdir_factory.mktemp("repo")), protocol="file", - location=tmpdir_factory.mktemp("repo"), + location=str(tmpdir_factory.mktemp("repo")), ), "repo-s3": dict( - s3_creds, protocol="s3", + endpoint=s3_creds["endpoint"], + access_key=s3_creds["access_key"], + secret_key=s3_creds["secret_key"], + bucket=s3_creds.get("bucket", "datajoint-test"), location="dj/repo", - stage=tmpdir_factory.mktemp("repo-s3"), - ), - "local": dict( - protocol="file", location=tmpdir_factory.mktemp("local"), subfolding=(1, 1) + stage=str(tmpdir_factory.mktemp("repo-s3")), + secure=False, # MinIO runs without SSL in tests ), + "local": dict(protocol="file", location=str(tmpdir_factory.mktemp("local"))), "share": dict( - s3_creds, protocol="s3", location="dj/store/repo", subfolding=(2, 4) + protocol="s3", + endpoint=s3_creds["endpoint"], + access_key=s3_creds["access_key"], + secret_key=s3_creds["secret_key"], + bucket=s3_creds.get("bucket", "datajoint-test"), + location="dj/store/repo", + secure=False, # MinIO runs without SSL in tests ), } - return stores_config @pytest.fixture def mock_stores(stores_config): - og_stores_config = dj.config.get("stores") - dj.config["stores"] = stores_config + """Configure object storage stores for tests using new object_storage system.""" + # Save original configuration + og_project_name = dj.config.object_storage.project_name + og_stores = dict(dj.config.object_storage.stores) + + # Set test configuration + dj.config.object_storage.project_name = "djtest" + dj.config.object_storage.stores.clear() + for name, config in stores_config.items(): + dj.config.object_storage.stores[name] = config + yield - if og_stores_config is None: - del dj.config["stores"] - else: - dj.config["stores"] = og_stores_config + + # Restore original configuration + dj.config.object_storage.project_name = og_project_name + dj.config.object_storage.stores.clear() + dj.config.object_storage.stores.update(og_stores) @pytest.fixture @@ -246,16 +405,111 @@ def mock_cache(tmpdir_factory): dj.config["cache"] = og_cache -@pytest.fixture -def schema_any(connection_test, prefix): - schema_any = dj.Schema( - prefix + "_test1", schema.LOCALS_ANY, connection=connection_test +@pytest.fixture(scope="session") +def http_client(): + client = urllib3.PoolManager( + timeout=30, + cert_reqs="CERT_REQUIRED", + ca_certs=certifi.where(), + retries=urllib3.Retry(total=3, backoff_factor=0.2, status_forcelist=[500, 502, 503, 504]), ) - assert schema.LOCALS_ANY, "LOCALS_ANY is empty" + yield client + + +@pytest.fixture(scope="session") +def s3fs_client(s3_creds): + """Initialize s3fs filesystem for MinIO.""" + import s3fs + + return s3fs.S3FileSystem( + endpoint_url=f"http://{s3_creds['endpoint']}", + key=s3_creds["access_key"], + secret=s3_creds["secret_key"], + ) + + +@pytest.fixture(scope="session") +def minio_client(s3_creds, s3fs_client, teardown=False): + """S3 filesystem with test bucket created (legacy name for compatibility).""" + bucket = s3_creds["bucket"] + + # Create bucket if it doesn't exist + try: + s3fs_client.mkdir(bucket) + except Exception: + # Bucket may already exist + pass + + yield s3fs_client + + if not teardown: + return + # Clean up objects and bucket try: - schema_any.jobs.delete() - except DataJointError: + files = s3fs_client.ls(bucket, detail=False) + for f in files: + s3fs_client.rm(f) + s3fs_client.rmdir(bucket) + except Exception: pass + + +# ============================================================================= +# Cleanup Fixtures +# ============================================================================= + + +@pytest.fixture +def clean_autopopulate(experiment, trial, ephys): + """Cleanup fixture for autopopulate tests.""" + yield + ephys.delete() + trial.delete() + experiment.delete() + + +@pytest.fixture +def clean_jobs(schema_any): + """Cleanup fixture for jobs tests.""" + # schema.jobs returns a list of Job objects for existing job tables + for job in schema_any.jobs: + try: + job.delete() + except DataJointError: + pass + yield + + +@pytest.fixture +def clean_test_tables(test, test_extra, test_no_extra): + """Cleanup fixture for relation tests.""" + if not test: + test.insert(test.contents, skip_duplicates=True) + yield + test.delete() + test.insert(test.contents, skip_duplicates=True) + test_extra.delete() + test_no_extra.delete() + + +# ============================================================================= +# Schema Fixtures +# ============================================================================= + + +@pytest.fixture(scope="module") +def schema_any(connection_test, prefix): + schema_any = dj.Schema(prefix + "_test1", schema.LOCALS_ANY, connection=connection_test) + assert schema.LOCALS_ANY, "LOCALS_ANY is empty" + # Clean up any existing job tables (schema.jobs returns a list) + for job in schema_any.jobs: + try: + job.delete() + except DataJointError: + pass + # Allow native PK fields for legacy test tables (Experiment, Trial) + original_value = dj.config.jobs.allow_new_pk_fields_in_computed_tables + dj.config.jobs.allow_new_pk_fields_in_computed_tables = True schema_any(schema.TTest) schema_any(schema.TTest2) schema_any(schema.TTest3) @@ -294,11 +548,81 @@ def schema_any(connection_test, prefix): schema_any(schema.SessionDateA) schema_any(schema.Stimulus) schema_any(schema.Longblob) + # Restore original config value after all tables are declared + dj.config.jobs.allow_new_pk_fields_in_computed_tables = original_value yield schema_any - try: - schema_any.jobs.delete() - except DataJointError: - pass + # Clean up job tables before dropping schema (if schema still exists) + if schema_any.exists: + for job in schema_any.jobs: + try: + job.delete() + except DataJointError: + pass + schema_any.drop() + + +@pytest.fixture +def schema_any_fresh(connection_test, prefix): + """Function-scoped schema_any for tests that need fresh schema state.""" + schema_any = dj.Schema(prefix + "_test1_fresh", schema.LOCALS_ANY, connection=connection_test) + assert schema.LOCALS_ANY, "LOCALS_ANY is empty" + # Clean up any existing job tables + for job in schema_any.jobs: + try: + job.delete() + except DataJointError: + pass + # Allow native PK fields for legacy test tables (Experiment, Trial) + original_value = dj.config.jobs.allow_new_pk_fields_in_computed_tables + dj.config.jobs.allow_new_pk_fields_in_computed_tables = True + schema_any(schema.TTest) + schema_any(schema.TTest2) + schema_any(schema.TTest3) + schema_any(schema.NullableNumbers) + schema_any(schema.TTestExtra) + schema_any(schema.TTestNoExtra) + schema_any(schema.Auto) + schema_any(schema.User) + schema_any(schema.Subject) + schema_any(schema.Language) + schema_any(schema.Experiment) + schema_any(schema.Trial) + schema_any(schema.Ephys) + schema_any(schema.Image) + schema_any(schema.UberTrash) + schema_any(schema.UnterTrash) + schema_any(schema.SimpleSource) + schema_any(schema.SigIntTable) + schema_any(schema.SigTermTable) + schema_any(schema.DjExceptionName) + schema_any(schema.ErrorClass) + schema_any(schema.DecimalPrimaryKey) + schema_any(schema.IndexRich) + schema_any(schema.ThingA) + schema_any(schema.ThingB) + schema_any(schema.ThingC) + schema_any(schema.ThingD) + schema_any(schema.ThingE) + schema_any(schema.Parent) + schema_any(schema.Child) + schema_any(schema.ComplexParent) + schema_any(schema.ComplexChild) + schema_any(schema.SubjectA) + schema_any(schema.SessionA) + schema_any(schema.SessionStatusA) + schema_any(schema.SessionDateA) + schema_any(schema.Stimulus) + schema_any(schema.Longblob) + # Restore original config value after all tables are declared + dj.config.jobs.allow_new_pk_fields_in_computed_tables = original_value + yield schema_any + # Clean up job tables before dropping schema (if schema still exists) + if schema_any.exists: + for job in schema_any.jobs: + try: + job.delete() + except DataJointError: + pass schema_any.drop() @@ -310,7 +634,6 @@ def thing_tables(schema_any): d = schema.ThingD() e = schema.ThingE() - # clear previous contents if any. c.delete_quick() b.delete_quick() a.delete_quick() @@ -322,11 +645,9 @@ def thing_tables(schema_any): yield a, b, c, d, e -@pytest.fixture +@pytest.fixture(scope="module") def schema_simp(connection_test, prefix): - schema = dj.Schema( - prefix + "_relational", schema_simple.LOCALS_SIMPLE, connection=connection_test - ) + schema = dj.Schema(prefix + "_relational", schema_simple.LOCALS_SIMPLE, connection=connection_test) schema(schema_simple.SelectPK) schema(schema_simple.KeyPK) schema(schema_simple.IJ) @@ -352,7 +673,7 @@ def schema_simp(connection_test, prefix): schema.drop() -@pytest.fixture +@pytest.fixture(scope="module") def schema_adv(connection_test, prefix): schema = dj.Schema( prefix + "_advanced", @@ -373,9 +694,7 @@ def schema_adv(connection_test, prefix): @pytest.fixture -def schema_ext( - connection_test, enable_filepath_feature, mock_stores, mock_cache, prefix -): +def schema_ext(connection_test, mock_stores, mock_cache, prefix): schema = dj.Schema( prefix + "_extern", context=schema_external.LOCALS_EXTERNAL, @@ -393,7 +712,7 @@ def schema_ext( schema.drop() -@pytest.fixture +@pytest.fixture(scope="module") def schema_uuid(connection_test, prefix): schema = dj.Schema( prefix + "_test1", @@ -407,56 +726,24 @@ def schema_uuid(connection_test, prefix): schema.drop() -@pytest.fixture(scope="session") -def http_client(): - # Initialize httpClient with relevant timeout. - client = urllib3.PoolManager( - timeout=30, - cert_reqs="CERT_REQUIRED", - ca_certs=certifi.where(), - retries=urllib3.Retry( - total=3, backoff_factor=0.2, status_forcelist=[500, 502, 503, 504] - ), - ) - yield client - - -@pytest.fixture(scope="session") -def minio_client_bare(s3_creds): - """Initialize MinIO with an endpoint and access/secret keys.""" - client = minio.Minio( - endpoint=s3_creds["endpoint"], - access_key=s3_creds["access_key"], - secret_key=s3_creds["secret_key"], - secure=False, +@pytest.fixture(scope="module") +def schema_type_aliases(connection_test, prefix): + """Schema for testing numeric type aliases.""" + schema = dj.Schema( + prefix + "_type_aliases", + context=schema_type_aliases_module.LOCALS_TYPE_ALIASES, + connection=connection_test, ) - return client - - -@pytest.fixture(scope="session") -def minio_client(s3_creds, minio_client_bare, teardown=False): - """Initialize a MinIO client and create buckets for testing session.""" - # Setup MinIO bucket - aws_region = "us-east-1" - try: - minio_client_bare.make_bucket(s3_creds["bucket"], location=aws_region) - except minio.error.S3Error as e: - if e.code != "BucketAlreadyOwnedByYou": - raise e + schema(schema_type_aliases_module.TypeAliasTable) + schema(schema_type_aliases_module.TypeAliasPrimaryKey) + schema(schema_type_aliases_module.TypeAliasNullable) + yield schema + schema.drop() - yield minio_client_bare - if not teardown: - return - # Teardown S3 - objs = list(minio_client_bare.list_objects(s3_creds["bucket"], recursive=True)) - objs = [ - minio_client_bare.remove_object( - s3_creds["bucket"], o.object_name.encode("utf-8") - ) - for o in objs - ] - minio_client_bare.remove_bucket(s3_creds["bucket"]) +# ============================================================================= +# Table Fixtures +# ============================================================================= @pytest.fixture @@ -530,3 +817,71 @@ def channel(schema_any): @pytest.fixture def trash(schema_any): return schema.UberTrash() + + +# ============================================================================= +# Object Storage Fixtures +# ============================================================================= + + +@pytest.fixture +def object_storage_config(tmpdir_factory): + """Create object storage configuration for testing.""" + location = str(tmpdir_factory.mktemp("object_storage")) + return { + "project_name": "test_project", + "protocol": "file", + "location": location, + "token_length": 8, + } + + +@pytest.fixture +def mock_object_storage(object_storage_config): + """Mock object storage configuration in datajoint config.""" + # Save original values + original = { + "project_name": dj.config.object_storage.project_name, + "protocol": dj.config.object_storage.protocol, + "location": dj.config.object_storage.location, + "token_length": dj.config.object_storage.token_length, + "stores": dict(dj.config.object_storage.stores), + } + + # Set test values + dj.config.object_storage.project_name = object_storage_config["project_name"] + dj.config.object_storage.protocol = object_storage_config["protocol"] + dj.config.object_storage.location = object_storage_config["location"] + dj.config.object_storage.token_length = object_storage_config.get("token_length", 8) + + # Configure 'local' store using same location + dj.config.object_storage.stores["local"] = { + "protocol": "file", + "location": object_storage_config["location"], + } + + yield object_storage_config + + # Restore original values + dj.config.object_storage.project_name = original["project_name"] + dj.config.object_storage.protocol = original["protocol"] + dj.config.object_storage.location = original["location"] + dj.config.object_storage.token_length = original["token_length"] + dj.config.object_storage.stores.clear() + dj.config.object_storage.stores.update(original["stores"]) + + +@pytest.fixture +def schema_obj(connection_test, prefix, mock_object_storage): + """Schema for object type tests.""" + schema = dj.Schema( + prefix + "_object", + context=schema_object.LOCALS_OBJECT, + connection=connection_test, + ) + schema(schema_object.ObjectFile) + schema(schema_object.ObjectFolder) + schema(schema_object.ObjectMultiple) + schema(schema_object.ObjectWithOther) + yield schema + schema.drop() diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/data/Course.csv b/tests/integration/data/Course.csv similarity index 100% rename from tests/data/Course.csv rename to tests/integration/data/Course.csv diff --git a/tests/data/CurrentTerm.csv b/tests/integration/data/CurrentTerm.csv similarity index 100% rename from tests/data/CurrentTerm.csv rename to tests/integration/data/CurrentTerm.csv diff --git a/tests/data/Department.csv b/tests/integration/data/Department.csv similarity index 100% rename from tests/data/Department.csv rename to tests/integration/data/Department.csv diff --git a/tests/data/Enroll.csv b/tests/integration/data/Enroll.csv similarity index 100% rename from tests/data/Enroll.csv rename to tests/integration/data/Enroll.csv diff --git a/tests/data/Grade.csv b/tests/integration/data/Grade.csv similarity index 100% rename from tests/data/Grade.csv rename to tests/integration/data/Grade.csv diff --git a/tests/data/Section.csv b/tests/integration/data/Section.csv similarity index 100% rename from tests/data/Section.csv rename to tests/integration/data/Section.csv diff --git a/tests/data/Student.csv b/tests/integration/data/Student.csv similarity index 100% rename from tests/data/Student.csv rename to tests/integration/data/Student.csv diff --git a/tests/data/StudentMajor.csv b/tests/integration/data/StudentMajor.csv similarity index 100% rename from tests/data/StudentMajor.csv rename to tests/integration/data/StudentMajor.csv diff --git a/tests/data/Term.csv b/tests/integration/data/Term.csv similarity index 100% rename from tests/data/Term.csv rename to tests/integration/data/Term.csv diff --git a/tests/integration/test_aggr_regressions.py b/tests/integration/test_aggr_regressions.py new file mode 100644 index 000000000..cf4f920b0 --- /dev/null +++ b/tests/integration/test_aggr_regressions.py @@ -0,0 +1,249 @@ +""" +Regression tests for issues 386, 449, 484, and 558 — all related to processing complex aggregations and projections. +""" + +import pytest + +import datajoint as dj + +from tests.schema_aggr_regress import LOCALS_AGGR_REGRESS, A, B, Q, R, S, X +from tests.schema_uuid import Item, Topic + + +@pytest.fixture(scope="function") +def schema_aggr_reg(connection_test, prefix): + schema = dj.Schema( + prefix + "_aggr_regress", + context=LOCALS_AGGR_REGRESS, + connection=connection_test, + ) + schema(R) + schema(Q) + schema(S) + yield schema + schema.drop() + + +@pytest.fixture(scope="function") +def schema_aggr_reg_with_abx(connection_test, prefix): + schema = dj.Schema( + prefix + "_aggr_regress_with_abx", + context=LOCALS_AGGR_REGRESS, + connection=connection_test, + ) + schema(R) + schema(Q) + schema(S) + schema(A) + schema(B) + schema(X) + yield schema + schema.drop() + + +def test_issue386(schema_aggr_reg): + """ + --------------- ISSUE 386 ------------------- + Issue 386 resulted from the loss of aggregated attributes when the aggregation was used as the restrictor + Q & (R.aggr(S, n='count(*)') & 'n=2') + Error: Unknown column 'n' in HAVING + """ + result = R.aggr(S, n="count(*)") & "n=10" + result = Q & result + result.to_dicts() + + +def test_issue449(schema_aggr_reg): + """ + ---------------- ISSUE 449 ------------------ + Issue 449 arises from incorrect group by attributes after joining with a dj.U() + Note: dj.U() * table pattern is no longer supported in 2.0, use dj.U() & table instead + """ + result = dj.U("n") & R.aggr(S, n="max(s)") + result.to_dicts() + + +def test_issue484(schema_aggr_reg): + """ + ---------------- ISSUE 484 ----------------- + Issue 484 + """ + q = dj.U().aggr(S, n="max(s)") + q.to_arrays("n") + q.fetch1("n") + q = dj.U().aggr(S, n="avg(s)") + result = dj.U().aggr(q, m="max(n)") + result.to_dicts() + + +def test_union_join(schema_aggr_reg_with_abx): + """ + This test fails if it runs after TestIssue558. + + https://github.com/datajoint/datajoint-python/issues/930 + """ + A.insert(zip([100, 200, 300, 400, 500, 600])) + B.insert([(100, 11), (200, 22), (300, 33), (400, 44)]) + q1 = B & "id < 300" + q2 = B & "id > 300" + + expected_data = [ + {"id": 0, "id2": 5}, + {"id": 1, "id2": 6}, + {"id": 2, "id2": 7}, + {"id": 3, "id2": 8}, + {"id": 4, "id2": 9}, + {"id": 100, "id2": 11}, + {"id": 200, "id2": 22}, + {"id": 400, "id2": 44}, + ] + + assert ((q1 + q2) * A).to_dicts() == expected_data + + +class TestIssue558: + """ + --------------- ISSUE 558 ------------------ + Issue 558 resulted from the fact that DataJoint saves subqueries and often combines a restriction followed + by a projection into a single SELECT statement, which in several unusual cases produces unexpected results. + """ + + def test_issue558_part1(self, schema_aggr_reg_with_abx): + q = (A - B).proj(id2="3") + assert len(A - B) == len(q) + + def test_issue558_part2(self, schema_aggr_reg_with_abx): + d = dict(id=3, id2=5) + assert len(X & d) == len((X & d).proj(id2="3")) + + +def test_left_join_invalid_raises_error(schema_uuid): + """Left join requires A → B. Topic ↛ Item, so this should raise an error.""" + from datajoint.errors import DataJointError + + # Clean up from previous tests + Item().delete_quick() + Topic().delete_quick() + + Topic().add("jeff") + Item.populate() + with pytest.raises(DataJointError) as exc_info: + Topic.join(Item, left=True) + assert "left operand to determine" in str(exc_info.value).lower() + + +def test_left_join_valid(schema_uuid): + """Left join where A → B: Item → Topic (topic_id is in Item).""" + # Clean up from previous tests + Item().delete_quick() + Topic().delete_quick() + + Topic().add("jeff") + Item.populate() + Topic().add("jeff2") # Topic without Items + # Item.join(Topic, left=True) is valid because Item → Topic + q = Item.join(Topic, left=True) + qf = q.to_arrays() + assert len(q) == len(qf) + # All Items should have matching Topics since they were populated from Topics + assert len(q) == len(Item()) + + +def test_extend_valid(schema_uuid): + """extend() is an alias for join(left=True) when A → B.""" + # Clean up from previous tests + Item().delete_quick() + Topic().delete_quick() + + Topic().add("alice") + Item.populate() + # Item → Topic (topic_id is in Item), so extend is valid + q_extend = Item.extend(Topic) + q_left_join = Item.join(Topic, left=True) + # Should produce identical results + assert len(q_extend) == len(q_left_join) + assert set(q_extend.heading.names) == set(q_left_join.heading.names) + assert q_extend.primary_key == q_left_join.primary_key + + +def test_extend_invalid_raises_error(schema_uuid): + """extend() requires A → B. Topic ↛ Item, so this should raise an error.""" + from datajoint.errors import DataJointError + + # Clean up from previous tests + Item().delete_quick() + Topic().delete_quick() + + Topic().add("bob") + Item.populate() + # Topic ↛ Item (item_id not in Topic), so extend should fail + with pytest.raises(DataJointError) as exc_info: + Topic.extend(Item) + assert "left operand to determine" in str(exc_info.value).lower() + + +class TestBoolMethod: + """ + Tests for __bool__ method on Aggregation and Union (issue #1234). + + bool(query) should return True if query has rows, False if empty. + """ + + def test_aggregation_bool_with_results(self, schema_aggr_reg_with_abx): + """Aggregation with results should be truthy.""" + A.insert([(1,), (2,), (3,)]) + B.insert([(1, 10), (1, 20), (2, 30)]) + aggr = A.aggr(B, count="count(id2)") + assert bool(aggr) is True + assert len(aggr) > 0 + + def test_aggregation_bool_empty(self, schema_aggr_reg_with_abx): + """Aggregation with no results should be falsy.""" + A.insert([(1,), (2,), (3,)]) + B.insert([(1, 10), (1, 20), (2, 30)]) + # Restrict to non-existent entry + aggr = (A & "id=999").aggr(B, count="count(id2)") + assert bool(aggr) is False + assert len(aggr) == 0 + + def test_aggregation_bool_matches_len(self, schema_aggr_reg_with_abx): + """bool(aggr) should equal len(aggr) > 0.""" + A.insert([(10,), (20,)]) + B.insert([(10, 100)]) + # With results + aggr_has = A.aggr(B, count="count(id2)") + assert bool(aggr_has) == (len(aggr_has) > 0) + # Without results + aggr_empty = (A & "id=999").aggr(B, count="count(id2)") + assert bool(aggr_empty) == (len(aggr_empty) > 0) + + def test_union_bool_with_results(self, schema_aggr_reg_with_abx): + """Union with results should be truthy.""" + A.insert([(100,), (200,)]) + B.insert([(100, 1), (200, 2)]) + q1 = B & "id=100" + q2 = B & "id=200" + union = q1 + q2 + assert bool(union) is True + assert len(union) > 0 + + def test_union_bool_empty(self, schema_aggr_reg_with_abx): + """Union with no results should be falsy.""" + A.insert([(100,), (200,)]) + B.insert([(100, 1), (200, 2)]) + q1 = B & "id=999" + q2 = B & "id=998" + union = q1 + q2 + assert bool(union) is False + assert len(union) == 0 + + def test_union_bool_matches_len(self, schema_aggr_reg_with_abx): + """bool(union) should equal len(union) > 0.""" + A.insert([(100,), (200,)]) + B.insert([(100, 1)]) + # With results + union_has = (B & "id=100") + (B & "id=100") + assert bool(union_has) == (len(union_has) > 0) + # Without results + union_empty = (B & "id=999") + (B & "id=998") + assert bool(union_empty) == (len(union_empty) > 0) diff --git a/tests/integration/test_alter.py b/tests/integration/test_alter.py new file mode 100644 index 000000000..fbf074332 --- /dev/null +++ b/tests/integration/test_alter.py @@ -0,0 +1,54 @@ +import re + +import pytest + + +from tests import schema as schema_any_module +from tests.schema_alter import LOCALS_ALTER, Experiment, Parent + +COMBINED_CONTEXT = { + **schema_any_module.LOCALS_ANY, + **LOCALS_ALTER, +} + + +@pytest.fixture +def schema_alter(connection_test, schema_any_fresh): + # Overwrite Experiment and Parent nodes using fresh schema + schema_any_fresh(Experiment, context=LOCALS_ALTER) + schema_any_fresh(Parent, context=LOCALS_ALTER) + yield schema_any_fresh + schema_any_fresh.drop() + + +class TestAlter: + def verify_alter(self, schema_alter, table, attribute_sql): + definition_original = schema_alter.connection.query(f"SHOW CREATE TABLE {table.full_table_name}").fetchone()[1] + table.definition = table.definition_new + table.alter(prompt=False) + definition_new = schema_alter.connection.query(f"SHOW CREATE TABLE {table.full_table_name}").fetchone()[1] + assert re.sub(f"{attribute_sql},\n ", "", definition_new) == definition_original + + def test_alter(self, schema_alter): + original = schema_alter.connection.query("SHOW CREATE TABLE " + Experiment.full_table_name).fetchone()[1] + Experiment.definition = Experiment.definition1 + Experiment.alter(prompt=False, context=COMBINED_CONTEXT) + altered = schema_alter.connection.query("SHOW CREATE TABLE " + Experiment.full_table_name).fetchone()[1] + assert original != altered + Experiment.definition = Experiment.original_definition + Experiment().alter(prompt=False, context=COMBINED_CONTEXT) + restored = schema_alter.connection.query("SHOW CREATE TABLE " + Experiment.full_table_name).fetchone()[1] + assert altered != restored + assert original == restored + + def test_alter_part(self, schema_alter): + """ + https://github.com/datajoint/datajoint-python/issues/936 + """ + # Regex includes optional COMMENT for type annotations + self.verify_alter(schema_alter, table=Parent.Child, attribute_sql=r"`child_id` .* DEFAULT NULL[^,]*") + self.verify_alter( + schema_alter, + table=Parent.Grandchild, + attribute_sql=r"`grandchild_id` .* DEFAULT NULL[^,]*", + ) diff --git a/tests/test_attach.py b/tests/integration/test_attach.py similarity index 56% rename from tests/test_attach.py rename to tests/integration/test_attach.py index 362db6933..f7ad953fe 100644 --- a/tests/test_attach.py +++ b/tests/integration/test_attach.py @@ -1,13 +1,14 @@ import os from pathlib import Path -import pytest -from .schema_external import Attach +from tests.schema_external import Attach def test_attach_attributes(schema_ext, minio_client, tmpdir_factory): """Test saving files in attachments""" + import datajoint as dj + # create a mock file table = Attach() source_folder = tmpdir_factory.mktemp("source") @@ -23,29 +24,32 @@ def test_attach_attributes(schema_ext, minio_client, tmpdir_factory): table.insert1(dict(attach=i, img=attach1, txt=attach2)) download_folder = Path(tmpdir_factory.mktemp("download")) - keys, path1, path2 = table.fetch( - "KEY", "img", "txt", download_path=download_folder, order_by="KEY" - ) + keys = table.keys(order_by="KEY") + + with dj.config.override(download_path=str(download_folder)): + path1, path2 = table.to_arrays("img", "txt", order_by="KEY") - # verify that different attachment are renamed if their filenames collide - assert path1[0] != path2[0] - assert path1[0] != path1[1] - assert Path(path1[0]).parent == download_folder - with Path(path1[-1]).open("rb") as f: - check1 = f.read() - with Path(path2[-1]).open("rb") as f: - check2 = f.read() - assert data1 == check1 - assert data2 == check2 + # verify that different attachment are renamed if their filenames collide + assert path1[0] != path2[0] + assert path1[0] != path1[1] + assert Path(path1[0]).parent == download_folder + with Path(path1[-1]).open("rb") as f: + check1 = f.read() + with Path(path2[-1]).open("rb") as f: + check2 = f.read() + assert data1 == check1 + assert data2 == check2 - # verify that existing files are not duplicated if their filename matches issue #592 - p1, p2 = (Attach & keys[0]).fetch1("img", "txt", download_path=download_folder) - assert p1 == path1[0] - assert p2 == path2[0] + # verify that existing files are not duplicated if their filename matches issue #592 + p1, p2 = (Attach & keys[0]).fetch1("img", "txt") + assert p1 == path1[0] + assert p2 == path2[0] def test_return_string(schema_ext, minio_client, tmpdir_factory): """Test returning string on fetch""" + import datajoint as dj + # create a mock file table = Attach() source_folder = tmpdir_factory.mktemp("source") @@ -61,8 +65,7 @@ def test_return_string(schema_ext, minio_client, tmpdir_factory): table.insert1(dict(attach=2, img=attach1, txt=attach2)) download_folder = Path(tmpdir_factory.mktemp("download")) - keys, path1, path2 = table.fetch( - "KEY", "img", "txt", download_path=download_folder, order_by="KEY" - ) + with dj.config.override(download_path=str(download_folder)): + path1, path2 = table.to_arrays("img", "txt", order_by="KEY") assert isinstance(path1[0], str) diff --git a/tests/test_autopopulate.py b/tests/integration/test_autopopulate.py similarity index 62% rename from tests/test_autopopulate.py rename to tests/integration/test_autopopulate.py index 899d90d9e..6afa6d10b 100644 --- a/tests/test_autopopulate.py +++ b/tests/integration/test_autopopulate.py @@ -1,13 +1,11 @@ -import pymysql +import platform import pytest import datajoint as dj from datajoint import DataJointError -from . import schema - -def test_populate(trial, subject, experiment, ephys, channel): +def test_populate(clean_autopopulate, trial, subject, experiment, ephys, channel): # test simple populate assert subject, "root tables are empty" assert not experiment, "table already filled?" @@ -16,7 +14,7 @@ def test_populate(trial, subject, experiment, ephys, channel): # test restricted populate assert not trial, "table already filled?" - restriction = subject.proj(animal="subject_id").fetch("KEY")[0] + restriction = subject.proj(animal="subject_id").keys()[0] d = trial.connection.dependencies d.load() trial.populate(restriction) @@ -33,7 +31,7 @@ def test_populate(trial, subject, experiment, ephys, channel): assert channel -def test_populate_with_success_count(subject, experiment, trial): +def test_populate_with_success_count(clean_autopopulate, subject, experiment, trial): # test simple populate assert subject, "root tables are empty" assert not experiment, "table already filled?" @@ -43,7 +41,7 @@ def test_populate_with_success_count(subject, experiment, trial): # test restricted populate assert not trial, "table already filled?" - restriction = subject.proj(animal="subject_id").fetch("KEY")[0] + restriction = subject.proj(animal="subject_id").keys()[0] d = trial.connection.dependencies d.load() ret = trial.populate(restriction, suppress_errors=True) @@ -51,61 +49,70 @@ def test_populate_with_success_count(subject, experiment, trial): assert len(trial.key_source & trial) == success_count -def test_populate_key_list(subject, experiment, trial): - # test simple populate +def test_populate_max_calls(clean_autopopulate, subject, experiment, trial): + # test populate with max_calls limit assert subject, "root tables are empty" assert not experiment, "table already filled?" - keys = experiment.key_source.fetch("KEY", order_by="KEY") n = 3 - assert len(keys) > n - keys = keys[:n] - ret = experiment.populate(keys=keys) + total_keys = len(experiment.key_source) + assert total_keys > n + ret = experiment.populate(max_calls=n) assert n == ret["success_count"] -def test_populate_exclude_error_and_ignore_jobs(schema_any, subject, experiment): - # test simple populate +def test_populate_exclude_error_and_ignore_jobs(clean_autopopulate, subject, experiment): + # test that error and ignore jobs are excluded from populate assert subject, "root tables are empty" assert not experiment, "table already filled?" - keys = experiment.key_source.fetch("KEY", limit=2) + # Refresh jobs to create pending entries + experiment.jobs.refresh() + + keys = experiment.jobs.pending.keys(limit=2) for idx, key in enumerate(keys): if idx == 0: - schema_any.jobs.ignore(experiment.table_name, key) + experiment.jobs.ignore(key) else: - schema_any.jobs.error(experiment.table_name, key, "") + # Create an error job by first reserving then setting error + experiment.jobs.reserve(key) + experiment.jobs.error(key, "test error") - experiment.populate(reserve_jobs=True) + # Populate should skip error and ignore jobs + experiment.populate(reserve_jobs=True, refresh=False) assert len(experiment.key_source & experiment) == len(experiment.key_source) - 2 -def test_allow_direct_insert(subject, experiment): +def test_allow_direct_insert(clean_autopopulate, subject, experiment): assert subject, "root tables are empty" - key = subject.fetch("KEY", limit=1)[0] + key = subject.keys(limit=1)[0] key["experiment_id"] = 1000 key["experiment_date"] = "2018-10-30" experiment.insert1(key, allow_direct_insert=True) +@pytest.mark.skipif( + platform.system() == "Darwin", + reason="multiprocessing with spawn method (macOS default) cannot pickle thread locks", +) @pytest.mark.parametrize("processes", [None, 2]) -def test_multi_processing(subject, experiment, processes): +def test_multi_processing(clean_autopopulate, subject, experiment, processes): assert subject, "root tables are empty" assert not experiment, "table already filled?" - experiment.populate(processes=None) + experiment.populate(processes=processes) assert len(experiment) == len(subject) * experiment.fake_experiments_per_subject -def test_allow_insert(subject, experiment): +def test_allow_insert(clean_autopopulate, subject, experiment): assert subject, "root tables are empty" - key = subject.fetch("KEY")[0] + key = subject.keys()[0] key["experiment_id"] = 1001 key["experiment_date"] = "2018-10-30" with pytest.raises(DataJointError): experiment.insert1(key) -def test_load_dependencies(prefix): - schema = dj.Schema(f"{prefix}_load_dependencies_populate") +def test_load_dependencies(prefix, connection_test): + schema = dj.Schema(f"{prefix}_load_dependencies_populate", connection=connection_test) @schema class ImageSource(dj.Lookup): @@ -119,7 +126,7 @@ class Image(dj.Imported): definition = """ -> ImageSource --- - image_data: longblob + image_data: """ def make(self, key): @@ -132,7 +139,7 @@ class Crop(dj.Computed): definition = """ -> Image --- - crop_image: longblob + crop_image: """ def make(self, key): diff --git a/tests/test_blob.py b/tests/integration/test_blob.py similarity index 90% rename from tests/test_blob.py rename to tests/integration/test_blob.py index 7c790db75..d2d047aab 100644 --- a/tests/test_blob.py +++ b/tests/integration/test_blob.py @@ -11,7 +11,7 @@ import datajoint as dj from datajoint.blob import pack, unpack -from .schema import Longblob +from tests.schema import Longblob @pytest.fixture @@ -61,27 +61,19 @@ def test_pack(): x = -255 y = unpack(pack(x)) - assert ( - x == y and isinstance(y, int) and not isinstance(y, np.ndarray) - ), "Scalar int did not match" + assert x == y and isinstance(y, int) and not isinstance(y, np.ndarray), "Scalar int did not match" x = -25523987234234287910987234987098245697129798713407812347 y = unpack(pack(x)) - assert ( - x == y and isinstance(y, int) and not isinstance(y, np.ndarray) - ), "Unbounded int did not match" + assert x == y and isinstance(y, int) and not isinstance(y, np.ndarray), "Unbounded int did not match" x = 7.0 y = unpack(pack(x)) - assert ( - x == y and isinstance(y, float) and not isinstance(y, np.ndarray) - ), "Scalar float did not match" + assert x == y and isinstance(y, float) and not isinstance(y, np.ndarray), "Scalar float did not match" x = 7j y = unpack(pack(x)) - assert ( - x == y and isinstance(y, complex) and not isinstance(y, np.ndarray) - ), "Complex scalar did not match" + assert x == y and isinstance(y, complex) and not isinstance(y, np.ndarray), "Complex scalar did not match" x = True assert unpack(pack(x)) is True, "Scalar bool did not match" @@ -98,9 +90,7 @@ def test_pack(): } y = unpack(pack(x)) assert x == y, "Dict do not match!" - assert not isinstance( - ["range"][0], np.ndarray - ), "Scalar int was coerced into array." + assert not isinstance(["range"][0], np.ndarray), "Scalar int was coerced into array." x = uuid.uuid4() assert x == unpack(pack(x)), "UUID did not match" @@ -142,9 +132,7 @@ def test_pack(): assert x == unpack(pack(x)), "String object did not pack/unpack correctly" x = np.array(["yes"]) - assert x == unpack( - pack(x) - ), "Numpy string array object did not pack/unpack correctly" + assert x == unpack(pack(x)), "Numpy string array object did not pack/unpack correctly" x = np.datetime64("1998").astype("datetime64[us]") assert x == unpack(pack(x)) @@ -202,7 +190,7 @@ def test_insert_longblob_32bit(schema_any, enable_feature_32bit_dims): "0023000000410200000001000000070000000400000000000000640064006400640064006400640025" "00000041020000000100000008000000040000000000000053007400610067006500200031003000')" ) - dj.conn().query(query_32_blob).fetchall() + schema_any.connection.query(query_32_blob).fetchall() fetched = (Longblob & "id=1").fetch1() expected = { "id": 1, diff --git a/tests/test_blob_matlab.py b/tests/integration/test_blob_matlab.py similarity index 72% rename from tests/test_blob_matlab.py rename to tests/integration/test_blob_matlab.py index 081841fb4..b7b05a0cb 100644 --- a/tests/test_blob_matlab.py +++ b/tests/integration/test_blob_matlab.py @@ -11,7 +11,7 @@ class Blob(dj.Manual): id : int ----- comment : varchar(255) - blob : longblob + blob : """ @@ -36,17 +36,15 @@ def insert_blobs(schema): schema.connection.query( """ INSERT INTO {table_name} (`id`, `comment`, `blob`) VALUES - (1,'simple string',0x6D596D00410200000000000000010000000000000010000000000000000400000000000000630068006100720061006300740065007200200073007400720069006E006700), - (2,'1D vector',0x6D596D0041020000000000000001000000000000000C000000000000000600000000000000000000000000F03F00000000000030400000000000003F4000000000000047400000000000804E4000000000000053400000000000C056400000000000805A400000000000405E4000000000000061400000000000E062400000000000C06440), - (3,'string array',0x6D596D00430200000000000000010000000000000002000000000000002F0000000000000041020000000000000001000000000000000700000000000000040000000000000073007400720069006E00670031002F0000000000000041020000000000000001000000000000000700000000000000040000000000000073007400720069006E0067003200), - (4,'struct array',0x6D596D005302000000000000000100000000000000020000000000000002000000610062002900000000000000410200000000000000010000000000000001000000000000000600000000000000000000000000F03F9000000000000000530200000000000000010000000000000001000000000000000100000063006900000000000000410200000000000000030000000000000003000000000000000600000000000000000000000000204000000000000008400000000000001040000000000000F03F0000000000001440000000000000224000000000000018400000000000001C40000000000000004029000000000000004102000000000000000100000000000000010000000000000006000000000000000000000000000040100100000000000053020000000000000001000000000000000100000000000000010000004300E9000000000000004102000000000000000500000000000000050000000000000006000000000000000000000000003140000000000000374000000000000010400000000000002440000000000000264000000000000038400000000000001440000000000000184000000000000028400000000000003240000000000000F03F0000000000001C400000000000002A400000000000003340000000000000394000000000000020400000000000002C400000000000003440000000000000354000000000000000400000000000002E400000000000003040000000000000364000000000000008400000000000002240), - (5,'3D double array',0x6D596D004103000000000000000200000000000000030000000000000004000000000000000600000000000000000000000000F03F000000000000004000000000000008400000000000001040000000000000144000000000000018400000000000001C40000000000000204000000000000022400000000000002440000000000000264000000000000028400000000000002A400000000000002C400000000000002E40000000000000304000000000000031400000000000003240000000000000334000000000000034400000000000003540000000000000364000000000000037400000000000003840), - (6,'3D uint8 array',0x6D596D0041030000000000000002000000000000000300000000000000040000000000000009000000000000000102030405060708090A0B0C0D0E0F101112131415161718), - (7,'3D complex array',0x6D596D0041030000000000000002000000000000000300000000000000040000000000000006000000010000000000000000C0724000000000000028C000000000000038C0000000000000000000000000000038C0000000000000000000000000000052C00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000052C00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000052C00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000AA4C58E87AB62B400000000000000000AA4C58E87AB62BC0000000000000008000000000000052400000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000080000000000000008000000000000052C000000000000000800000000000000080000000000000008000000000000000800000000000000080 + (1,'simple string',0x6D596D00410200000000000000010000000000000010000000000000000400000000000000630068006100720061006300740065007200200073007400720069006E006700), # noqa: E501 + (2,'1D vector',0x6D596D0041020000000000000001000000000000000C000000000000000600000000000000000000000000F03F00000000000030400000000000003F4000000000000047400000000000804E4000000000000053400000000000C056400000000000805A400000000000405E4000000000000061400000000000E062400000000000C06440), # noqa: E501 + (3,'string array',0x6D596D00430200000000000000010000000000000002000000000000002F0000000000000041020000000000000001000000000000000700000000000000040000000000000073007400720069006E00670031002F0000000000000041020000000000000001000000000000000700000000000000040000000000000073007400720069006E0067003200), # noqa: E501 + (4,'struct array',0x6D596D005302000000000000000100000000000000020000000000000002000000610062002900000000000000410200000000000000010000000000000001000000000000000600000000000000000000000000F03F9000000000000000530200000000000000010000000000000001000000000000000100000063006900000000000000410200000000000000030000000000000003000000000000000600000000000000000000000000204000000000000008400000000000001040000000000000F03F0000000000001440000000000000224000000000000018400000000000001C40000000000000004029000000000000004102000000000000000100000000000000010000000000000006000000000000000000000000000040100100000000000053020000000000000001000000000000000100000000000000010000004300E9000000000000004102000000000000000500000000000000050000000000000006000000000000000000000000003140000000000000374000000000000010400000000000002440000000000000264000000000000038400000000000001440000000000000184000000000000028400000000000003240000000000000F03F0000000000001C400000000000002A400000000000003340000000000000394000000000000020400000000000002C400000000000003440000000000000354000000000000000400000000000002E400000000000003040000000000000364000000000000008400000000000002240), # noqa: E501 + (5,'3D double array',0x6D596D004103000000000000000200000000000000030000000000000004000000000000000600000000000000000000000000F03F000000000000004000000000000008400000000000001040000000000000144000000000000018400000000000001C40000000000000204000000000000022400000000000002440000000000000264000000000000028400000000000002A400000000000002C400000000000002E40000000000000304000000000000031400000000000003240000000000000334000000000000034400000000000003540000000000000364000000000000037400000000000003840), # noqa: E501 + (6,'3D uint8 array',0x6D596D0041030000000000000002000000000000000300000000000000040000000000000009000000000000000102030405060708090A0B0C0D0E0F101112131415161718), # noqa: E501 + (7,'3D complex array',0x6D596D0041030000000000000002000000000000000300000000000000040000000000000006000000010000000000000000C0724000000000000028C000000000000038C0000000000000000000000000000038C0000000000000000000000000000052C00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000052C00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000052C00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000AA4C58E87AB62B400000000000000000AA4C58E87AB62BC0000000000000008000000000000052400000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000080000000000000008000000000000052C000000000000000800000000000000080000000000000008000000000000000800000000000000080 # noqa: E501 ); - """.format( - table_name=Blob.full_table_name - ) + """.format(table_name=Blob.full_table_name) ) @@ -70,7 +68,7 @@ def test_complex_matlab_blobs(schema_blob_pop): """ test correct de-serialization of various blob types """ - blobs = Blob().fetch("blob", order_by="KEY") + blobs = Blob().to_arrays("blob", order_by="KEY") blob = blobs[0] # 'simple string' 'character string' assert blob[0] == "character string" @@ -84,9 +82,7 @@ def test_complex_matlab_blobs(schema_blob_pop): assert_array_equal(blob, np.array([["string1", "string2"]])) assert_array_equal(blob, unpack(pack(blob))) - blob = blobs[ - 3 - ] # 'struct array' struct('a', {1,2}, 'b', {struct('c', magic(3)), struct('C', magic(5))}) + blob = blobs[3] # 'struct array' struct('a', {1,2}, 'b', {struct('c', magic(3)), struct('C', magic(5))}) assert isinstance(blob, dj.MatStruct) assert tuple(blob.dtype.names) == ("a", "b") assert_array_equal(blob.a[0, 0], np.array([[1.0]])) @@ -117,17 +113,13 @@ def test_complex_matlab_squeeze(schema_blob_pop): """ test correct de-serialization of various blob types """ - blob = (Blob & "id=1").fetch1( - "blob", squeeze=True - ) # 'simple string' 'character string' + blob = (Blob & "id=1").fetch1("blob", squeeze=True) # 'simple string' 'character string' assert blob == "character string" blob = (Blob & "id=2").fetch1("blob", squeeze=True) # '1D vector' 1:15:180 assert_array_equal(blob, np.r_[1:180:15]) - blob = (Blob & "id=3").fetch1( - "blob", squeeze=True - ) # 'string array' {'string1' 'string2'} + blob = (Blob & "id=3").fetch1("blob", squeeze=True) # 'string array' {'string1' 'string2'} assert isinstance(blob, dj.MatCell) assert_array_equal(blob, np.array(["string1", "string2"])) @@ -148,9 +140,7 @@ def test_complex_matlab_squeeze(schema_blob_pop): assert isinstance(blob[1].b, dj.MatStruct) assert tuple(blob[1].b.C.item().shape) == (5, 5) - blob = (Blob & "id=5").fetch1( - "blob", squeeze=True - ) # '3D double array' reshape(1:24, [2,3,4]) + blob = (Blob & "id=5").fetch1("blob", squeeze=True) # '3D double array' reshape(1:24, [2,3,4]) assert np.array_equal(blob, np.r_[1:25].reshape((2, 3, 4), order="F")) assert blob.dtype == "float64" @@ -170,3 +160,71 @@ def test_iter(schema_blob_pop): from_iter = {d["id"]: d for d in Blob()} assert len(from_iter) == len(Blob()) assert from_iter[1]["blob"] == "character string" + + +def test_cell_array_with_nested_arrays(): + """ + Test unpacking MATLAB cell arrays containing arrays of different sizes. + Regression test for issue #1098. + """ + # Create a cell array with nested arrays of different sizes (ragged) + cell = np.empty(2, dtype=object) + cell[0] = np.array([1, 2, 3]) + cell[1] = np.array([4, 5, 6, 7, 8]) + cell = cell.reshape((1, 2)).view(dj.MatCell) + + # Pack and unpack + packed = pack(cell) + unpacked = unpack(packed) + + # Should preserve structure + assert isinstance(unpacked, dj.MatCell) + assert unpacked.shape == (1, 2) + assert_array_equal(unpacked[0, 0], np.array([1, 2, 3])) + assert_array_equal(unpacked[0, 1], np.array([4, 5, 6, 7, 8])) + + +def test_cell_array_with_empty_elements(): + """ + Test unpacking MATLAB cell arrays containing empty arrays. + Regression test for issue #1056. + """ + # Create a cell array with empty elements: {[], [], []} + cell = np.empty(3, dtype=object) + cell[0] = np.array([]) + cell[1] = np.array([]) + cell[2] = np.array([]) + cell = cell.reshape((3, 1)).view(dj.MatCell) + + # Pack and unpack + packed = pack(cell) + unpacked = unpack(packed) + + # Should preserve structure + assert isinstance(unpacked, dj.MatCell) + assert unpacked.shape == (3, 1) + for i in range(3): + assert unpacked[i, 0].size == 0 + + +def test_cell_array_mixed_empty_nonempty(): + """ + Test unpacking MATLAB cell arrays with mixed empty and non-empty elements. + """ + # Create a cell array: {[1,2], [], [3,4,5]} + cell = np.empty(3, dtype=object) + cell[0] = np.array([1, 2]) + cell[1] = np.array([]) + cell[2] = np.array([3, 4, 5]) + cell = cell.reshape((3, 1)).view(dj.MatCell) + + # Pack and unpack + packed = pack(cell) + unpacked = unpack(packed) + + # Should preserve structure + assert isinstance(unpacked, dj.MatCell) + assert unpacked.shape == (3, 1) + assert_array_equal(unpacked[0, 0], np.array([1, 2])) + assert unpacked[1, 0].size == 0 + assert_array_equal(unpacked[2, 0], np.array([3, 4, 5])) diff --git a/tests/test_cascading_delete.py b/tests/integration/test_cascading_delete.py similarity index 75% rename from tests/test_cascading_delete.py rename to tests/integration/test_cascading_delete.py index 71216fcb2..28f175bea 100644 --- a/tests/test_cascading_delete.py +++ b/tests/integration/test_cascading_delete.py @@ -2,12 +2,23 @@ import datajoint as dj -from .schema import ComplexChild, ComplexParent -from .schema_simple import A, B, D, E, G, L, Profile, Website +from tests.schema import ComplexChild, ComplexParent +from tests.schema_simple import A, B, D, E, G, L, Profile, Website @pytest.fixture def schema_simp_pop(schema_simp): + # Clean up tables first to ensure fresh state with module-scoped schema + # Delete in reverse dependency order + Profile().delete() + Website().delete() + G().delete() + E().delete() + D().delete() + B().delete() + L().delete() + A().delete() + A().insert(A.contents, skip_duplicates=True) L().insert(L.contents, skip_duplicates=True) B().populate() @@ -19,9 +30,7 @@ def schema_simp_pop(schema_simp): def test_delete_tree(schema_simp_pop): assert not dj.config["safemode"], "safemode must be off for testing" - assert ( - L() and A() and B() and B.C() and D() and E() and E.F() - ), "schema is not populated" + assert L() and A() and B() and B.C() and D() and E() and E.F(), "schema is not populated" A().delete() assert not A() or B() or B.C() or D() or E() or E.F(), "incomplete delete" @@ -29,19 +38,15 @@ def test_delete_tree(schema_simp_pop): def test_stepwise_delete(schema_simp_pop): assert not dj.config["safemode"], "safemode must be off for testing" assert L() and A() and B() and B.C(), "schema population failed" - B.C().delete(force=True) + B.C().delete(part_integrity="ignore") assert not B.C(), "failed to delete child tables" B().delete() - assert ( - not B() - ), "failed to delete from the parent table following child table deletion" + assert not B(), "failed to delete from the parent table following child table deletion" def test_delete_tree_restricted(schema_simp_pop): assert not dj.config["safemode"], "safemode must be off for testing" - assert ( - L() and A() and B() and B.C() and D() and E() and E.F() - ), "schema is not populated" + assert L() and A() and B() and B.C() and D() and E() and E.F(), "schema is not populated" cond = "cond_in_a" rel = A() & cond rest = dict( @@ -53,9 +58,7 @@ def test_delete_tree_restricted(schema_simp_pop): F=len(E.F() - rel), ) rel.delete() - assert not ( - rel or B() & rel or B.C() & rel or D() & rel or E() & rel or (E.F() & rel) - ), "incomplete delete" + assert not (rel or B() & rel or B.C() & rel or D() & rel or E() & rel or (E.F() & rel)), "incomplete delete" assert len(A()) == rest["A"], "invalid delete restriction" assert len(B()) == rest["B"], "invalid delete restriction" assert len(B.C()) == rest["C"], "invalid delete restriction" @@ -66,9 +69,7 @@ def test_delete_tree_restricted(schema_simp_pop): def test_delete_lookup(schema_simp_pop): assert not dj.config["safemode"], "safemode must be off for testing" - assert bool( - L() and A() and B() and B.C() and D() and E() and E.F() - ), "schema is not populated" + assert bool(L() and A() and B() and B.C() and D() and E() and E.F()), "schema is not populated" L().delete() assert not bool(L() or D() or E() or E.F()), "incomplete delete" A().delete() # delete all is necessary because delete L deletes from subtables. @@ -76,9 +77,7 @@ def test_delete_lookup(schema_simp_pop): def test_delete_lookup_restricted(schema_simp_pop): assert not dj.config["safemode"], "safemode must be off for testing" - assert ( - L() and A() and B() and B.C() and D() and E() and E.F() - ), "schema is not populated" + assert L() and A() and B() and B.C() and D() and E() and E.F(), "schema is not populated" rel = L() & "cond_in_l" original_count = len(L()) deleted_count = len(rel) @@ -96,10 +95,7 @@ def test_delete_complex_keys(schema_any): child_key_count = 1 restriction = dict( {"parent_id_{}".format(i + 1): i for i in range(parent_key_count)}, - **{ - "child_id_{}".format(i + 1): (i + parent_key_count) - for i in range(child_key_count) - }, + **{"child_id_{}".format(i + 1): (i + parent_key_count) for i in range(child_key_count)}, ) assert len(ComplexParent & restriction) == 1, "Parent record missing" assert len(ComplexChild & restriction) == 1, "Child record missing" @@ -117,19 +113,19 @@ def test_delete_parts_error(schema_simp_pop): """test issue #151""" with pytest.raises(dj.DataJointError): Profile().populate_random() - Website().delete(force_masters=False) + Website().delete(part_integrity="enforce") def test_delete_parts(schema_simp_pop): """test issue #151""" Profile().populate_random() - Website().delete(force_masters=True) + Website().delete(part_integrity="cascade") def test_delete_parts_complex(schema_simp_pop): """test issue #151 with complex master/part. PR #1158.""" prev_len = len(G()) - (A() & "id_a=1").delete(force_masters=True) + (A() & "id_a=1").delete(part_integrity="cascade") assert prev_len - len(G()) == 16, "Failed to delete parts" diff --git a/tests/test_cli.py b/tests/integration/test_cli.py similarity index 71% rename from tests/test_cli.py rename to tests/integration/test_cli.py index be0faf64d..35230ea4e 100644 --- a/tests/test_cli.py +++ b/tests/integration/test_cli.py @@ -2,7 +2,6 @@ Collection of test cases to test the dj cli """ -import json import subprocess import pytest @@ -13,7 +12,7 @@ def test_cli_version(capsys): with pytest.raises(SystemExit) as pytest_wrapped_e: dj.cli(args=["-V"]) - assert pytest_wrapped_e.type == SystemExit + assert pytest_wrapped_e.type is SystemExit assert pytest_wrapped_e.value.code == 0 captured_output = capsys.readouterr().out @@ -23,7 +22,7 @@ def test_cli_version(capsys): def test_cli_help(capsys): with pytest.raises(SystemExit) as pytest_wrapped_e: dj.cli(args=["--help"]) - assert pytest_wrapped_e.type == SystemExit + assert pytest_wrapped_e.type is SystemExit assert pytest_wrapped_e.value.code == 0 captured_output = capsys.readouterr().out @@ -44,13 +43,14 @@ def test_cli_config(): stdout, stderr = process.communicate() cleaned = stdout.strip(" >\t\n\r") - for key in ("database.user", "database.password", "database.host"): + # Config now uses pydantic format: Config(database=DatabaseSettings(host=..., user=..., ...)) + for key in ("host=", "user=", "password="): assert key in cleaned, f"Key {key} not found in config from stdout: {cleaned}" def test_cli_args(): process = subprocess.Popen( - ["dj", "-utest_user", "-ptest_pass", "-htest_host"], + ["dj", "-u", "test_user", "-p", "test_pass", "--host", "test_host"], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, @@ -63,12 +63,12 @@ def test_cli_args(): process.stdin.flush() stdout, stderr = process.communicate() - assert "test_user" == stdout[5:14] - assert "test_pass" == stdout[21:30] - assert "test_host" == stdout[37:46] + assert "test_user" in stdout + assert "test_pass" in stdout + assert "test_host" in stdout -def test_cli_schemas(prefix, connection_root): +def test_cli_schemas(prefix, connection_root, db_creds_root): schema = dj.Schema(prefix + "_cli", locals(), connection=connection_root) @schema @@ -79,8 +79,19 @@ class IJ(dj.Lookup): """ contents = list(dict(i=i, j=j + 2) for i in range(3) for j in range(3)) + # Pass credentials via CLI args to avoid prompting for username process = subprocess.Popen( - ["dj", "-s", "djtest_cli:test_schema"], + [ + "dj", + "-u", + db_creds_root["user"], + "-p", + db_creds_root["password"], + "--host", + db_creds_root["host"], + "-s", + f"{prefix}_cli:test_schema", + ], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, @@ -89,7 +100,7 @@ class IJ(dj.Lookup): process.stdin.write("test_schema.__dict__['__name__']\n") process.stdin.write("test_schema.__dict__['schema']\n") - process.stdin.write("test_schema.IJ.fetch(as_dict=True)\n") + process.stdin.write("test_schema.IJ.to_dicts()\n") process.stdin.flush() stdout, stderr = process.communicate() @@ -108,6 +119,6 @@ class IJ(dj.Lookup): cleaned = stdout.strip(" >\t\n\r") for key in ( "test_schema", - "Schema `djtest_cli`", + f"Schema `{prefix}_cli`", ): - assert key in cleaned, f"Key {key} not found in config from stdout: {cleaned}" + assert key in cleaned, f"Key {key} not found in stdout: {cleaned}" diff --git a/tests/integration/test_codec_chaining.py b/tests/integration/test_codec_chaining.py new file mode 100644 index 000000000..defbd428f --- /dev/null +++ b/tests/integration/test_codec_chaining.py @@ -0,0 +1,368 @@ +""" +Tests for codec chaining (composition). + +This tests the → → json composition pattern +and similar codec chains. +""" + +from datajoint.codecs import ( + Codec, + _codec_registry, + resolve_dtype, +) + + +class TestCodecChainResolution: + """Tests for resolving codec chains.""" + + def setup_method(self): + """Clear test codecs from registry before each test.""" + for name in list(_codec_registry.keys()): + if name.startswith("test_"): + del _codec_registry[name] + + def teardown_method(self): + """Clean up test codecs after each test.""" + for name in list(_codec_registry.keys()): + if name.startswith("test_"): + del _codec_registry[name] + + def test_single_codec_chain(self): + """Test resolving a single-codec chain.""" + + class TestSingle(Codec): + name = "test_single" + + def get_dtype(self, is_external: bool) -> str: + return "varchar(100)" + + def encode(self, value, *, key=None, store_name=None): + return str(value) + + def decode(self, stored, *, key=None): + return stored + + final_dtype, chain, store = resolve_dtype("") + + assert final_dtype == "varchar(100)" + assert len(chain) == 1 + assert chain[0].name == "test_single" + assert store is None + + def test_two_codec_chain(self): + """Test resolving a two-codec chain.""" + + class TestInner(Codec): + name = "test_inner" + + def get_dtype(self, is_external: bool) -> str: + return "bytes" + + def encode(self, value, *, key=None, store_name=None): + return value + + def decode(self, stored, *, key=None): + return stored + + class TestOuter(Codec): + name = "test_outer" + + def get_dtype(self, is_external: bool) -> str: + return "" + + def encode(self, value, *, key=None, store_name=None): + return value + + def decode(self, stored, *, key=None): + return stored + + final_dtype, chain, store = resolve_dtype("") + + assert final_dtype == "bytes" + assert len(chain) == 2 + assert chain[0].name == "test_outer" + assert chain[1].name == "test_inner" + + def test_three_codec_chain(self): + """Test resolving a three-codec chain.""" + + class TestBase(Codec): + name = "test_base" + + def get_dtype(self, is_external: bool) -> str: + return "json" + + def encode(self, value, *, key=None, store_name=None): + return value + + def decode(self, stored, *, key=None): + return stored + + class TestMiddle(Codec): + name = "test_middle" + + def get_dtype(self, is_external: bool) -> str: + return "" + + def encode(self, value, *, key=None, store_name=None): + return value + + def decode(self, stored, *, key=None): + return stored + + class TestTop(Codec): + name = "test_top" + + def get_dtype(self, is_external: bool) -> str: + return "" + + def encode(self, value, *, key=None, store_name=None): + return value + + def decode(self, stored, *, key=None): + return stored + + final_dtype, chain, store = resolve_dtype("") + + assert final_dtype == "json" + assert len(chain) == 3 + assert chain[0].name == "test_top" + assert chain[1].name == "test_middle" + assert chain[2].name == "test_base" + + +class TestCodecChainEncodeDecode: + """Tests for encode/decode through codec chains.""" + + def setup_method(self): + """Clear test codecs from registry before each test.""" + for name in list(_codec_registry.keys()): + if name.startswith("test_"): + del _codec_registry[name] + + def teardown_method(self): + """Clean up test codecs after each test.""" + for name in list(_codec_registry.keys()): + if name.startswith("test_"): + del _codec_registry[name] + + def test_encode_order(self): + """Test that encode is applied outer → inner.""" + encode_order = [] + + class TestInnerEnc(Codec): + name = "test_inner_enc" + + def get_dtype(self, is_external: bool) -> str: + return "bytes" + + def encode(self, value, *, key=None, store_name=None): + encode_order.append("inner") + return value + b"_inner" + + def decode(self, stored, *, key=None): + return stored + + class TestOuterEnc(Codec): + name = "test_outer_enc" + + def get_dtype(self, is_external: bool) -> str: + return "" + + def encode(self, value, *, key=None, store_name=None): + encode_order.append("outer") + return value + b"_outer" + + def decode(self, stored, *, key=None): + return stored + + _, chain, _ = resolve_dtype("") + + # Apply encode in order: outer first, then inner + value = b"start" + for codec in chain: + value = codec.encode(value) + + assert encode_order == ["outer", "inner"] + assert value == b"start_outer_inner" + + def test_decode_order(self): + """Test that decode is applied inner → outer (reverse of encode).""" + decode_order = [] + + class TestInnerDec(Codec): + name = "test_inner_dec" + + def get_dtype(self, is_external: bool) -> str: + return "bytes" + + def encode(self, value, *, key=None, store_name=None): + return value + + def decode(self, stored, *, key=None): + decode_order.append("inner") + return stored.replace(b"_inner", b"") + + class TestOuterDec(Codec): + name = "test_outer_dec" + + def get_dtype(self, is_external: bool) -> str: + return "" + + def encode(self, value, *, key=None, store_name=None): + return value + + def decode(self, stored, *, key=None): + decode_order.append("outer") + return stored.replace(b"_outer", b"") + + _, chain, _ = resolve_dtype("") + + # Apply decode in reverse order: inner first, then outer + value = b"start_outer_inner" + for codec in reversed(chain): + value = codec.decode(value) + + assert decode_order == ["inner", "outer"] + assert value == b"start" + + def test_roundtrip(self): + """Test encode/decode roundtrip through a codec chain.""" + + class TestInnerRt(Codec): + name = "test_inner_rt" + + def get_dtype(self, is_external: bool) -> str: + return "bytes" + + def encode(self, value, *, key=None, store_name=None): + # Compress (just add prefix for testing) + return b"COMPRESSED:" + value + + def decode(self, stored, *, key=None): + # Decompress + return stored.replace(b"COMPRESSED:", b"") + + class TestOuterRt(Codec): + name = "test_outer_rt" + + def get_dtype(self, is_external: bool) -> str: + return "" + + def encode(self, value, *, key=None, store_name=None): + # Serialize (just encode string for testing) + return str(value).encode("utf-8") + + def decode(self, stored, *, key=None): + # Deserialize + return stored.decode("utf-8") + + _, chain, _ = resolve_dtype("") + + # Original value + original = "test data" + + # Encode: outer → inner + encoded = original + for codec in chain: + encoded = codec.encode(encoded) + + assert encoded == b"COMPRESSED:test data" + + # Decode: inner → outer (reversed) + decoded = encoded + for codec in reversed(chain): + decoded = codec.decode(decoded) + + assert decoded == original + + +class TestBuiltinCodecChains: + """Tests for built-in codec chains.""" + + def test_blob_internal_resolves_to_bytes(self): + """Test that (internal) → bytes.""" + final_dtype, chain, _ = resolve_dtype("") + + assert final_dtype == "bytes" + assert len(chain) == 1 + assert chain[0].name == "blob" + + def test_blob_external_resolves_to_json(self): + """Test that → → json.""" + final_dtype, chain, store = resolve_dtype("") + + assert final_dtype == "json" + assert len(chain) == 2 + assert chain[0].name == "blob" + assert chain[1].name == "hash" + assert store == "store" + + def test_attach_internal_resolves_to_bytes(self): + """Test that (internal) → bytes.""" + final_dtype, chain, _ = resolve_dtype("") + + assert final_dtype == "bytes" + assert len(chain) == 1 + assert chain[0].name == "attach" + + def test_attach_external_resolves_to_json(self): + """Test that → → json.""" + final_dtype, chain, store = resolve_dtype("") + + assert final_dtype == "json" + assert len(chain) == 2 + assert chain[0].name == "attach" + assert chain[1].name == "hash" + assert store == "store" + + def test_hash_external_resolves_to_json(self): + """Test that → json (external only).""" + final_dtype, chain, store = resolve_dtype("") + + assert final_dtype == "json" + assert len(chain) == 1 + assert chain[0].name == "hash" + assert store == "store" + + def test_object_external_resolves_to_json(self): + """Test that → json (external only).""" + final_dtype, chain, store = resolve_dtype("") + + assert final_dtype == "json" + assert len(chain) == 1 + assert chain[0].name == "object" + assert store == "store" + + def test_filepath_external_resolves_to_json(self): + """Test that → json (external only).""" + final_dtype, chain, store = resolve_dtype("") + + assert final_dtype == "json" + assert len(chain) == 1 + assert chain[0].name == "filepath" + assert store == "store" + + +class TestStoreNameParsing: + """Tests for store name parsing in codec specs.""" + + def test_codec_with_store(self): + """Test parsing codec with store name.""" + final_dtype, chain, store = resolve_dtype("") + + assert final_dtype == "json" + assert store == "mystore" + + def test_codec_without_store(self): + """Test parsing codec without store name.""" + final_dtype, chain, store = resolve_dtype("") + + assert store is None + + def test_filepath_with_store(self): + """Test parsing filepath with store name.""" + final_dtype, chain, store = resolve_dtype("") + + assert final_dtype == "json" + assert store == "s3store" diff --git a/tests/test_adapted_attributes.py b/tests/integration/test_codecs.py similarity index 61% rename from tests/test_adapted_attributes.py rename to tests/integration/test_codecs.py index ffd137795..6d160e5b5 100644 --- a/tests/test_adapted_attributes.py +++ b/tests/integration/test_codecs.py @@ -1,5 +1,9 @@ -import os -import tempfile +""" +Tests for custom codecs. + +These tests verify the Codec system for custom data types. +""" + from itertools import zip_longest import networkx as nx @@ -7,66 +11,51 @@ import datajoint as dj -from . import schema_adapted -from .schema_adapted import Connectivity, Layout +from tests import schema_codecs +from tests.schema_codecs import Connectivity, Layout @pytest.fixture def schema_name(prefix): - return prefix + "_test_custom_datatype" - - -@pytest.fixture -def adapted_graph_instance(): - yield schema_adapted.GraphAdapter() + return prefix + "_test_codecs" @pytest.fixture -def schema_ad( +def schema_codec( connection_test, - adapted_graph_instance, - enable_adapted_types, - enable_filepath_feature, s3_creds, tmpdir, schema_name, ): - dj.config["stores"] = { - "repo-s3": dict( - s3_creds, protocol="s3", location="adapted/repo", stage=str(tmpdir) - ) - } - context = { - **schema_adapted.LOCALS_ADAPTED, - "graph": adapted_graph_instance, - "layout_to_filepath": schema_adapted.LayoutToFilepath(), - } + dj.config["stores"] = {"repo-s3": dict(s3_creds, protocol="s3", location="codecs/repo", stage=str(tmpdir))} + # Codecs are auto-registered via __init_subclass__ in schema_codecs + context = {**schema_codecs.LOCALS_CODECS} schema = dj.schema(schema_name, context=context, connection=connection_test) - schema(schema_adapted.Connectivity) - schema(schema_adapted.Layout) + schema(schema_codecs.Connectivity) + schema(schema_codecs.Layout) yield schema schema.drop() @pytest.fixture -def local_schema(schema_ad, schema_name): +def local_schema(schema_codec, schema_name): """Fixture for testing spawned classes""" - local_schema = dj.Schema(schema_name) + local_schema = dj.Schema(schema_name, connection=schema_codec.connection) local_schema.spawn_missing_classes() yield local_schema - local_schema.drop() + # Don't drop - schema_codec fixture handles cleanup @pytest.fixture -def schema_virtual_module(schema_ad, adapted_graph_instance, schema_name): +def schema_virtual_module(schema_codec, schema_name): """Fixture for testing virtual modules""" - schema_virtual_module = dj.VirtualModule( - "virtual_module", schema_name, add_objects={"graph": adapted_graph_instance} - ) + # Codecs are registered globally, no need to add_objects + schema_virtual_module = dj.VirtualModule("virtual_module", schema_name, connection=schema_codec.connection) return schema_virtual_module -def test_adapted_type(schema_ad): +def test_codec_graph(schema_codec): + """Test basic codec encode/decode with graph type.""" c = Connectivity() graphs = [ nx.lollipop_graph(4, 2), @@ -75,7 +64,7 @@ def test_adapted_type(schema_ad): nx.cycle_graph(5), ] c.insert((i, g) for i, g in enumerate(graphs)) - returned_graphs = c.fetch("conn_graph", order_by="connid") + returned_graphs = c.to_arrays("conn_graph", order_by="connid") for g1, g2 in zip(graphs, returned_graphs): assert isinstance(g2, nx.Graph) assert len(g1.edges) == len(g2.edges) @@ -83,8 +72,8 @@ def test_adapted_type(schema_ad): c.delete() -def test_adapted_filepath_type(schema_ad, minio_client): - """https://github.com/datajoint/datajoint-python/issues/684""" +def test_codec_chained(schema_codec, minio_client): + """Test codec chaining (layout -> blob).""" c = Connectivity() c.delete() c.insert1((0, nx.lollipop_graph(4, 2))) @@ -100,7 +89,8 @@ def test_adapted_filepath_type(schema_ad, minio_client): c.delete() -def test_adapted_spawned(local_schema, enable_adapted_types): +def test_codec_spawned(local_schema): + """Test codecs work with spawned classes.""" c = Connectivity() # a spawned class graphs = [ nx.lollipop_graph(4, 2), @@ -109,7 +99,7 @@ def test_adapted_spawned(local_schema, enable_adapted_types): nx.cycle_graph(5), ] c.insert((i, g) for i, g in enumerate(graphs)) - returned_graphs = c.fetch("conn_graph", order_by="connid") + returned_graphs = c.to_arrays("conn_graph", order_by="connid") for g1, g2 in zip(graphs, returned_graphs): assert isinstance(g2, nx.Graph) assert len(g1.edges) == len(g2.edges) @@ -117,7 +107,8 @@ def test_adapted_spawned(local_schema, enable_adapted_types): c.delete() -def test_adapted_virtual(schema_virtual_module): +def test_codec_virtual_module(schema_virtual_module): + """Test codecs work with virtual modules.""" c = schema_virtual_module.Connectivity() graphs = [ nx.lollipop_graph(4, 2), @@ -127,7 +118,7 @@ def test_adapted_virtual(schema_virtual_module): ] c.insert((i, g) for i, g in enumerate(graphs)) c.insert1({"connid": 100}) # test work with NULLs - returned_graphs = c.fetch("conn_graph", order_by="connid") + returned_graphs = c.to_arrays("conn_graph", order_by="connid") for g1, g2 in zip_longest(graphs, returned_graphs): if g1 is None: assert g2 is None diff --git a/tests/test_connection.py b/tests/integration/test_connection.py similarity index 65% rename from tests/test_connection.py rename to tests/integration/test_connection.py index db301d9af..ff3940587 100644 --- a/tests/test_connection.py +++ b/tests/integration/test_connection.py @@ -46,6 +46,36 @@ def test_dj_connection_class(connection_test): assert connection_test.is_connected +def test_connection_context_manager(db_creds_test): + """ + Connection should support context manager protocol for automatic cleanup. + """ + # Test basic context manager usage + with dj.Connection(**db_creds_test) as conn: + assert conn.is_connected + # Verify we can use the connection + result = conn.query("SELECT 1").fetchone() + assert result[0] == 1 + + # Connection should be closed after exiting context + assert not conn.is_connected + + +def test_connection_context_manager_exception(db_creds_test): + """ + Connection should close even when exception is raised inside context. + """ + conn = None + with pytest.raises(ValueError): + with dj.Connection(**db_creds_test) as conn: + assert conn.is_connected + raise ValueError("Test exception") + + # Connection should still be closed after exception + assert conn is not None + assert not conn.is_connected + + def test_persistent_dj_conn(db_creds_root): """ conn() method should provide persistent connection across calls. @@ -88,13 +118,9 @@ def test_transaction_rollback(schema_tx, connection_test): raise DataJointError("Testing rollback") except DataJointError: pass - assert ( - len(Subjects()) == 1 - ), "Length is not 1. Expected because rollback should have happened." + assert len(Subjects()) == 1, "Length is not 1. Expected because rollback should have happened." - assert ( - len(Subjects & "subject_id = 2") == 0 - ), "Length is not 0. Expected because rollback should have happened." + assert len(Subjects & "subject_id = 2") == 0, "Length is not 0. Expected because rollback should have happened." def test_cancel(schema_tx, connection_test): @@ -108,9 +134,5 @@ def test_cancel(schema_tx, connection_test): connection_test.start_transaction() Subjects.insert1(tmp[1]) connection_test.cancel_transaction() - assert ( - len(Subjects()) == 1 - ), "Length is not 1. Expected because rollback should have happened." - assert ( - len(Subjects & "subject_id = 2") == 0 - ), "Length is not 0. Expected because rollback should have happened." + assert len(Subjects()) == 1, "Length is not 1. Expected because rollback should have happened." + assert len(Subjects & "subject_id = 2") == 0, "Length is not 0. Expected because rollback should have happened." diff --git a/tests/integration/test_content_storage.py b/tests/integration/test_content_storage.py new file mode 100644 index 000000000..e6d0f14cc --- /dev/null +++ b/tests/integration/test_content_storage.py @@ -0,0 +1,231 @@ +""" +Tests for content-addressed storage (content_registry.py). +""" + +import hashlib +from unittest.mock import MagicMock, patch + +import pytest + +from datajoint.content_registry import ( + build_content_path, + compute_content_hash, + content_exists, + delete_content, + get_content, + get_content_size, + put_content, +) +from datajoint.errors import DataJointError + + +class TestComputeContentHash: + """Tests for compute_content_hash function.""" + + def test_computes_sha256(self): + """Test that SHA256 hash is computed correctly.""" + data = b"Hello, World!" + result = compute_content_hash(data) + + # Verify against known SHA256 hash + expected = hashlib.sha256(data).hexdigest() + assert result == expected + assert len(result) == 64 # SHA256 produces 64 hex chars + + def test_empty_bytes(self): + """Test hashing empty bytes.""" + result = compute_content_hash(b"") + expected = hashlib.sha256(b"").hexdigest() + assert result == expected + + def test_different_content_different_hash(self): + """Test that different content produces different hashes.""" + hash1 = compute_content_hash(b"content1") + hash2 = compute_content_hash(b"content2") + assert hash1 != hash2 + + def test_same_content_same_hash(self): + """Test that same content produces same hash.""" + data = b"identical content" + hash1 = compute_content_hash(data) + hash2 = compute_content_hash(data) + assert hash1 == hash2 + + +class TestBuildContentPath: + """Tests for build_content_path function.""" + + def test_builds_hierarchical_path(self): + """Test that path is built with proper hierarchy.""" + # Example hash: abcdef... + test_hash = "abcdef0123456789" * 4 # 64 chars + result = build_content_path(test_hash) + + # Path should be _content/{hash[:2]}/{hash[2:4]}/{hash} + assert result == f"_content/ab/cd/{test_hash}" + + def test_rejects_invalid_hash_length(self): + """Test that invalid hash length raises error.""" + with pytest.raises(DataJointError, match="Invalid content hash length"): + build_content_path("tooshort") + + with pytest.raises(DataJointError, match="Invalid content hash length"): + build_content_path("a" * 65) # Too long + + def test_real_hash_path(self): + """Test path building with a real computed hash.""" + data = b"test content" + content_hash = compute_content_hash(data) + path = build_content_path(content_hash) + + # Verify structure + parts = path.split("/") + assert parts[0] == "_content" + assert len(parts[1]) == 2 + assert len(parts[2]) == 2 + assert len(parts[3]) == 64 + assert parts[1] == content_hash[:2] + assert parts[2] == content_hash[2:4] + assert parts[3] == content_hash + + +class TestPutContent: + """Tests for put_content function.""" + + @patch("datajoint.content_registry.get_store_backend") + def test_stores_new_content(self, mock_get_backend): + """Test storing new content.""" + mock_backend = MagicMock() + mock_backend.exists.return_value = False + mock_get_backend.return_value = mock_backend + + data = b"new content" + result = put_content(data, store_name="test_store") + + # Verify return value + assert "hash" in result + assert result["hash"] == compute_content_hash(data) + assert result["store"] == "test_store" + assert result["size"] == len(data) + + # Verify backend was called + mock_backend.put_buffer.assert_called_once() + + @patch("datajoint.content_registry.get_store_backend") + def test_deduplicates_existing_content(self, mock_get_backend): + """Test that existing content is not re-uploaded.""" + mock_backend = MagicMock() + mock_backend.exists.return_value = True # Content already exists + mock_get_backend.return_value = mock_backend + + data = b"existing content" + result = put_content(data, store_name="test_store") + + # Verify return value is still correct + assert result["hash"] == compute_content_hash(data) + assert result["size"] == len(data) + + # Verify put_buffer was NOT called (deduplication) + mock_backend.put_buffer.assert_not_called() + + +class TestGetContent: + """Tests for get_content function.""" + + @patch("datajoint.content_registry.get_store_backend") + def test_retrieves_content(self, mock_get_backend): + """Test retrieving content by hash.""" + data = b"stored content" + content_hash = compute_content_hash(data) + + mock_backend = MagicMock() + mock_backend.get_buffer.return_value = data + mock_get_backend.return_value = mock_backend + + result = get_content(content_hash, store_name="test_store") + + assert result == data + + @patch("datajoint.content_registry.get_store_backend") + def test_verifies_hash(self, mock_get_backend): + """Test that hash is verified on retrieval.""" + data = b"original content" + content_hash = compute_content_hash(data) + + # Return corrupted data + mock_backend = MagicMock() + mock_backend.get_buffer.return_value = b"corrupted content" + mock_get_backend.return_value = mock_backend + + with pytest.raises(DataJointError, match="Content hash mismatch"): + get_content(content_hash, store_name="test_store") + + +class TestContentExists: + """Tests for content_exists function.""" + + @patch("datajoint.content_registry.get_store_backend") + def test_returns_true_when_exists(self, mock_get_backend): + """Test that True is returned when content exists.""" + mock_backend = MagicMock() + mock_backend.exists.return_value = True + mock_get_backend.return_value = mock_backend + + content_hash = "a" * 64 + assert content_exists(content_hash, store_name="test_store") is True + + @patch("datajoint.content_registry.get_store_backend") + def test_returns_false_when_not_exists(self, mock_get_backend): + """Test that False is returned when content doesn't exist.""" + mock_backend = MagicMock() + mock_backend.exists.return_value = False + mock_get_backend.return_value = mock_backend + + content_hash = "a" * 64 + assert content_exists(content_hash, store_name="test_store") is False + + +class TestDeleteContent: + """Tests for delete_content function.""" + + @patch("datajoint.content_registry.get_store_backend") + def test_deletes_existing_content(self, mock_get_backend): + """Test deleting existing content.""" + mock_backend = MagicMock() + mock_backend.exists.return_value = True + mock_get_backend.return_value = mock_backend + + content_hash = "a" * 64 + result = delete_content(content_hash, store_name="test_store") + + assert result is True + mock_backend.remove.assert_called_once() + + @patch("datajoint.content_registry.get_store_backend") + def test_returns_false_for_nonexistent(self, mock_get_backend): + """Test that False is returned when content doesn't exist.""" + mock_backend = MagicMock() + mock_backend.exists.return_value = False + mock_get_backend.return_value = mock_backend + + content_hash = "a" * 64 + result = delete_content(content_hash, store_name="test_store") + + assert result is False + mock_backend.remove.assert_not_called() + + +class TestGetContentSize: + """Tests for get_content_size function.""" + + @patch("datajoint.content_registry.get_store_backend") + def test_returns_size(self, mock_get_backend): + """Test getting content size.""" + mock_backend = MagicMock() + mock_backend.size.return_value = 1024 + mock_get_backend.return_value = mock_backend + + content_hash = "a" * 64 + result = get_content_size(content_hash, store_name="test_store") + + assert result == 1024 diff --git a/tests/test_declare.py b/tests/integration/test_declare.py similarity index 70% rename from tests/test_declare.py rename to tests/integration/test_declare.py index 828021939..3097a9457 100644 --- a/tests/test_declare.py +++ b/tests/integration/test_declare.py @@ -4,27 +4,21 @@ import datajoint as dj from datajoint.declare import declare -from datajoint.settings import config -from .schema import * - - -@pytest.fixture(scope="function") -def enable_add_hidden_timestamp(): - orig_config_val = config.get("add_hidden_timestamp") - config["add_hidden_timestamp"] = True - yield - if orig_config_val is not None: - config["add_hidden_timestamp"] = orig_config_val - - -@pytest.fixture(scope="function") -def disable_add_hidden_timestamp(): - orig_config_val = config.get("add_hidden_timestamp") - config["add_hidden_timestamp"] = False - yield - if orig_config_val is not None: - config["add_hidden_timestamp"] = orig_config_val +from tests.schema import ( + Auto, + Ephys, + Experiment, + IndexRich, + Subject, + TTest, + TTest2, + ThingA, # noqa: F401 - needed in globals for foreign key resolution + ThingB, # noqa: F401 - needed in globals for foreign key resolution + ThingC, + Trial, + User, +) def test_schema_decorator(schema_any): @@ -77,7 +71,7 @@ def test_part(schema_any): """ Lookup and part with the same name. See issue #365 """ - local_schema = dj.Schema(schema_any.database) + local_schema = dj.Schema(schema_any.database, connection=schema_any.connection) @local_schema class Type(dj.Lookup): @@ -101,10 +95,9 @@ class Type(dj.Part): def test_attributes(schema_any): """ - Test autoincrement declaration + Test attribute declarations """ auto = Auto() - auto.fill() subject = Subject() experiment = Experiment() trial = Trial() @@ -112,7 +105,7 @@ def test_attributes(schema_any): channel = Ephys.Channel() assert auto.heading.names == ["id", "name"] - assert auto.heading.attributes["id"].autoincrement + assert auto.heading.attributes["id"].numeric # test attribute declarations assert subject.heading.names == [ @@ -178,48 +171,30 @@ def test_dependencies(schema_any): assert set(experiment.parents(primary=False)) == {user.full_table_name} assert experiment.full_table_name in user.children(primary=False) assert set(experiment.parents(primary=False)) == {user.full_table_name} - assert set( - s.full_table_name for s in experiment.parents(primary=False, as_objects=True) - ) == {user.full_table_name} + assert set(s.full_table_name for s in experiment.parents(primary=False, as_objects=True)) == {user.full_table_name} assert experiment.full_table_name in subject.descendants() - assert experiment.full_table_name in { - s.full_table_name for s in subject.descendants(as_objects=True) - } + assert experiment.full_table_name in {s.full_table_name for s in subject.descendants(as_objects=True)} assert subject.full_table_name in experiment.ancestors() - assert subject.full_table_name in { - s.full_table_name for s in experiment.ancestors(as_objects=True) - } + assert subject.full_table_name in {s.full_table_name for s in experiment.ancestors(as_objects=True)} assert trial.full_table_name in experiment.descendants() - assert trial.full_table_name in { - s.full_table_name for s in experiment.descendants(as_objects=True) - } + assert trial.full_table_name in {s.full_table_name for s in experiment.descendants(as_objects=True)} assert experiment.full_table_name in trial.ancestors() - assert experiment.full_table_name in { - s.full_table_name for s in trial.ancestors(as_objects=True) - } + assert experiment.full_table_name in {s.full_table_name for s in trial.ancestors(as_objects=True)} assert set(trial.children(primary=True)) == { ephys.full_table_name, trial.Condition.full_table_name, } assert set(trial.parts()) == {trial.Condition.full_table_name} - assert set(s.full_table_name for s in trial.parts(as_objects=True)) == { - trial.Condition.full_table_name - } + assert set(s.full_table_name for s in trial.parts(as_objects=True)) == {trial.Condition.full_table_name} assert set(ephys.parents(primary=True)) == {trial.full_table_name} - assert set( - s.full_table_name for s in ephys.parents(primary=True, as_objects=True) - ) == {trial.full_table_name} + assert set(s.full_table_name for s in ephys.parents(primary=True, as_objects=True)) == {trial.full_table_name} assert set(ephys.children(primary=True)) == {channel.full_table_name} - assert set( - s.full_table_name for s in ephys.children(primary=True, as_objects=True) - ) == {channel.full_table_name} + assert set(s.full_table_name for s in ephys.children(primary=True, as_objects=True)) == {channel.full_table_name} assert set(channel.parents(primary=True)) == {ephys.full_table_name} - assert set( - s.full_table_name for s in channel.parents(primary=True, as_objects=True) - ) == {ephys.full_table_name} + assert set(s.full_table_name for s in channel.parents(primary=True, as_objects=True)) == {ephys.full_table_name} def test_descendants_only_contain_part_table(schema_any): @@ -268,7 +243,7 @@ class BadName(dj.Manual): schema_any(BadName) -def test_bad_fk_rename(schema_any): +def test_bad_fk_rename(schema_any_fresh): """issue #381""" class A(dj.Manual): @@ -281,9 +256,9 @@ class B(dj.Manual): b -> A # invalid, the new syntax is (b) -> A """ - schema_any(A) + schema_any_fresh(A) with pytest.raises(dj.DataJointError): - schema_any(B) + schema_any_fresh(B) def test_primary_nullable_foreign_key(schema_any): @@ -313,7 +288,7 @@ class Q(dj.Manual): definition = """ experiment : int --- - description : text + description : completely_invalid_type_xyz """ with pytest.raises(dj.DataJointError): @@ -388,42 +363,5 @@ class Table_With_Underscores(dj.Manual): """ schema_any(TableNoUnderscores) - with pytest.raises( - dj.DataJointError, match="must be alphanumeric in CamelCase" - ) as e: + with pytest.raises(dj.DataJointError, match="must be alphanumeric in CamelCase"): schema_any(Table_With_Underscores) - - -def test_add_hidden_timestamp_default_value(): - config_val = config.get("add_hidden_timestamp") - assert ( - config_val is not None and not config_val - ), "Default value for add_hidden_timestamp is not False" - - -def test_add_hidden_timestamp_enabled(enable_add_hidden_timestamp, schema_any): - assert config["add_hidden_timestamp"], "add_hidden_timestamp is not enabled" - msg = f"{Experiment().heading._attributes=}" - assert any( - a.name.endswith("_timestamp") for a in Experiment().heading._attributes.values() - ), msg - assert any( - a.name.startswith("_") for a in Experiment().heading._attributes.values() - ), msg - assert any(a.is_hidden for a in Experiment().heading._attributes.values()), msg - assert not any(a.is_hidden for a in Experiment().heading.attributes.values()), msg - - -def test_add_hidden_timestamp_disabled(disable_add_hidden_timestamp, schema_any): - assert not config[ - "add_hidden_timestamp" - ], "expected add_hidden_timestamp to be False" - msg = f"{Experiment().heading._attributes=}" - assert not any( - a.name.endswith("_timestamp") for a in Experiment().heading._attributes.values() - ), msg - assert not any( - a.name.startswith("_") for a in Experiment().heading._attributes.values() - ), msg - assert not any(a.is_hidden for a in Experiment().heading._attributes.values()), msg - assert not any(a.is_hidden for a in Experiment().heading.attributes.values()), msg diff --git a/tests/test_dependencies.py b/tests/integration/test_dependencies.py similarity index 96% rename from tests/test_dependencies.py rename to tests/integration/test_dependencies.py index 3be4a21dc..7d9c5dd6e 100644 --- a/tests/test_dependencies.py +++ b/tests/integration/test_dependencies.py @@ -17,7 +17,7 @@ def test_nullable_dependency(thing_tables): c.insert1(dict(a=3, b1=1, b2=1)) c.insert1(dict(a=4, b1=1, b2=2)) - assert len(c) == len(c.fetch()) == 5 + assert len(c) == len(c.to_arrays()) == 5 def test_topo_sort(): diff --git a/tests/test_erd.py b/tests/integration/test_erd.py similarity index 80% rename from tests/test_erd.py rename to tests/integration/test_erd.py index e2344cf8a..1fbad394b 100644 --- a/tests/test_erd.py +++ b/tests/integration/test_erd.py @@ -1,7 +1,6 @@ import datajoint as dj -from .schema_advanced import * -from .schema_simple import LOCALS_SIMPLE, A, B, D, E, G, L, OutfitLaunch +from tests.schema_simple import LOCALS_SIMPLE, A, B, D, E, G, L def test_decorator(schema_simp): @@ -20,9 +19,7 @@ def test_dependencies(schema_simp): assert set(A().children()) == set([B.full_table_name, D.full_table_name]) assert set(D().parents(primary=True)) == set([A.full_table_name]) assert set(D().parents(primary=False)) == set([L.full_table_name]) - assert set(deps.descendants(L.full_table_name)).issubset( - cls.full_table_name for cls in (L, D, E, E.F, E.G, E.H, E.M, G) - ) + assert set(deps.descendants(L.full_table_name)).issubset(cls.full_table_name for cls in (L, D, E, E.F, E.G, E.H, E.M, G)) def test_erd(schema_simp): @@ -39,14 +36,10 @@ def test_erd_algebra(schema_simp): erd3 = erd1 * erd2 erd4 = (erd0 + E).add_parts() - B - E assert erd0.nodes_to_show == set(cls.full_table_name for cls in [B]) - assert erd1.nodes_to_show == set( - cls.full_table_name for cls in (B, B.C, E, E.F, E.G, E.H, E.M, G) - ) + assert erd1.nodes_to_show == set(cls.full_table_name for cls in (B, B.C, E, E.F, E.G, E.H, E.M, G)) assert erd2.nodes_to_show == set(cls.full_table_name for cls in (A, B, D, E, L)) assert erd3.nodes_to_show == set(cls.full_table_name for cls in (B, E)) - assert erd4.nodes_to_show == set( - cls.full_table_name for cls in (B.C, E.F, E.G, E.H, E.M) - ) + assert erd4.nodes_to_show == set(cls.full_table_name for cls in (B.C, E.F, E.G, E.H, E.M)) def test_repr_svg(schema_adv): diff --git a/tests/integration/test_fetch.py b/tests/integration/test_fetch.py new file mode 100644 index 000000000..8cde34deb --- /dev/null +++ b/tests/integration/test_fetch.py @@ -0,0 +1,399 @@ +"""Tests for the modern fetch API: to_dicts, to_pandas, to_arrays, keys, fetch1""" + +import decimal +import itertools +import os +import shutil +from operator import itemgetter + +import numpy as np +import pandas +import pytest + +import datajoint as dj + +from tests import schema + + +def test_getattribute(subject): + """Testing fetch with attributes using new API""" + list1 = sorted(subject.proj().to_dicts(), key=itemgetter("subject_id")) + list2 = sorted(subject.keys(), key=itemgetter("subject_id")) + for l1, l2 in zip(list1, list2): + assert l1 == l2, "Primary key is not returned correctly" + + tmp = subject.to_arrays(order_by="subject_id") + + subject_notes, real_id = subject.to_arrays("subject_notes", "real_id") + + np.testing.assert_array_equal(sorted(subject_notes), sorted(tmp["subject_notes"])) + np.testing.assert_array_equal(sorted(real_id), sorted(tmp["real_id"])) + + +def test_getattribute_for_fetch1(subject): + """Testing Fetch1.__call__ with attributes""" + assert (subject & "subject_id=10").fetch1("subject_id") == 10 + assert (subject & "subject_id=10").fetch1("subject_id", "species") == ( + 10, + "monkey", + ) + + +def test_order_by(lang, languages): + """Tests order_by sorting order""" + for ord_name, ord_lang in itertools.product(*2 * [["ASC", "DESC"]]): + cur = lang.to_arrays(order_by=("name " + ord_name, "language " + ord_lang)) + languages.sort(key=itemgetter(1), reverse=ord_lang == "DESC") + languages.sort(key=itemgetter(0), reverse=ord_name == "DESC") + for c, l in zip(cur, languages): # noqa: E741 + assert np.all(cc == ll for cc, ll in zip(c, l)), "Sorting order is different" + + +def test_order_by_default(lang, languages): + """Tests order_by sorting order with defaults""" + cur = lang.to_arrays(order_by=("language", "name DESC")) + languages.sort(key=itemgetter(0), reverse=True) + languages.sort(key=itemgetter(1), reverse=False) + for c, l in zip(cur, languages): # noqa: E741 + assert np.all([cc == ll for cc, ll in zip(c, l)]), "Sorting order is different" + + +def test_limit(lang): + """Test the limit kwarg""" + limit = 4 + cur = lang.to_arrays(limit=limit) + assert len(cur) == limit, "Length is not correct" + + +def test_order_by_limit(lang, languages): + """Test the combination of order by and limit kwargs""" + cur = lang.to_arrays(limit=4, order_by=["language", "name DESC"]) + languages.sort(key=itemgetter(0), reverse=True) + languages.sort(key=itemgetter(1), reverse=False) + assert len(cur) == 4, "Length is not correct" + for c, l in list(zip(cur, languages))[:4]: # noqa: E741 + assert np.all([cc == ll for cc, ll in zip(c, l)]), "Sorting order is different" + + +def test_head_tail(schema_any): + """Test head() and tail() convenience methods""" + query = schema.User * schema.Language + n = 5 + # head and tail now return list of dicts + head_result = query.head(n) + assert isinstance(head_result, list) + assert len(head_result) == n + assert all(isinstance(row, dict) for row in head_result) + + n = 4 + tail_result = query.tail(n) + assert isinstance(tail_result, list) + assert len(tail_result) == n + assert all(isinstance(row, dict) for row in tail_result) + + +def test_limit_offset(lang, languages): + """Test the limit and offset kwargs together""" + cur = lang.to_arrays(offset=2, limit=4, order_by=["language", "name DESC"]) + languages.sort(key=itemgetter(0), reverse=True) + languages.sort(key=itemgetter(1), reverse=False) + assert len(cur) == 4, "Length is not correct" + for c, l in list(zip(cur, languages[2:6])): # noqa: E741 + assert np.all([cc == ll for cc, ll in zip(c, l)]), "Sorting order is different" + + +def test_iter(lang, languages): + """Test iterator - now lazy streaming""" + languages_copy = languages.copy() + languages_copy.sort(key=itemgetter(0), reverse=True) + languages_copy.sort(key=itemgetter(1), reverse=False) + + # Iteration now yields dicts directly + result = list(lang.to_dicts(order_by=["language", "name DESC"])) + for row, (tname, tlang) in list(zip(result, languages_copy)): + assert row["name"] == tname and row["language"] == tlang, "Values are not the same" + + +def test_keys(lang, languages): + """test key fetch""" + languages_copy = languages.copy() + languages_copy.sort(key=itemgetter(0), reverse=True) + languages_copy.sort(key=itemgetter(1), reverse=False) + + # Use to_arrays for attribute fetch + cur = lang.to_arrays("name", "language", order_by=("language", "name DESC")) + # Use keys() for primary key fetch + cur2 = list(lang.keys(order_by=["language", "name DESC"])) + + for c, c2 in zip(zip(*cur), cur2): + assert c == tuple(c2.values()), "Values are not the same" + + +def test_fetch1_step1(lang, languages): + assert ( + lang.contents + == languages + == [ + ("Fabian", "English"), + ("Edgar", "English"), + ("Dimitri", "English"), + ("Dimitri", "Ukrainian"), + ("Fabian", "German"), + ("Edgar", "Japanese"), + ] + ), "Unexpected contents in Language table" + key = {"name": "Edgar", "language": "Japanese"} + true = languages[-1] + dat = (lang & key).fetch1() + for k, (ke, c) in zip(true, dat.items()): + assert k == c == (lang & key).fetch1(ke), "Values are not the same" + + +def test_misspelled_attribute(schema_any): + """Test that misspelled attributes raise error""" + with pytest.raises(dj.DataJointError): + (schema.Language & 'lang = "ENGLISH"').to_dicts() + + +def test_to_dicts(lang): + """Test to_dicts returns list of dictionaries""" + d = lang.to_dicts() + for dd in d: + assert isinstance(dd, dict) + + +def test_offset(lang, languages): + """Tests offset""" + cur = lang.to_arrays(limit=4, offset=1, order_by=["language", "name DESC"]) + + languages.sort(key=itemgetter(0), reverse=True) + languages.sort(key=itemgetter(1), reverse=False) + assert len(cur) == 4, "Length is not correct" + for c, l in list(zip(cur, languages[1:]))[:4]: # noqa: E741 + assert np.all([cc == ll for cc, ll in zip(c, l)]), "Sorting order is different" + + +def test_len(lang): + """Tests __len__""" + assert len(lang.to_arrays()) == len(lang), "__len__ is not behaving properly" + + +def test_fetch1_step2(lang): + """Tests whether fetch1 raises error for multiple rows""" + with pytest.raises(dj.DataJointError): + lang.fetch1() + + +def test_fetch1_step3(lang): + """Tests whether fetch1 raises error for multiple rows with attribute""" + with pytest.raises(dj.DataJointError): + lang.fetch1("name") + + +def test_decimal(schema_any): + """Tests that decimal fields are correctly fetched and used in restrictions, see issue #334""" + rel = schema.DecimalPrimaryKey() + assert len(rel.to_arrays()), "Table DecimalPrimaryKey contents are empty" + rel.insert1([decimal.Decimal("3.1415926")]) + keys = rel.to_arrays() + assert len(keys) > 0 + assert len(rel & keys[0]) == 1 + keys = rel.keys() + assert len(keys) >= 2 + assert len(rel & keys[1]) == 1 + + +def test_nullable_numbers(schema_any): + """test mixture of values and nulls in numeric attributes""" + table = schema.NullableNumbers() + table.insert( + ( + ( + k, + np.random.randn(), + np.random.randint(-1000, 1000), + np.random.randn(), + ) + for k in range(10) + ) + ) + table.insert1((100, None, None, None)) + f, d, i = table.to_arrays("fvalue", "dvalue", "ivalue") + # Check for None in integer column + assert None in i + # Check for None or nan in float columns (None may be returned for nullable fields) + assert any(v is None or (isinstance(v, float) and np.isnan(v)) for v in d) + assert any(v is None or (isinstance(v, float) and np.isnan(v)) for v in f) + + +def test_to_pandas(subject): + """Test to_pandas returns DataFrame with primary key as index""" + df = subject.to_pandas(order_by="subject_id") + assert isinstance(df, pandas.DataFrame) + assert df.index.names == subject.primary_key + + +def test_to_polars(subject): + """Test to_polars returns polars DataFrame""" + polars = pytest.importorskip("polars") + df = subject.to_polars() + assert isinstance(df, polars.DataFrame) + + +def test_to_arrow(subject): + """Test to_arrow returns PyArrow Table""" + pyarrow = pytest.importorskip("pyarrow") + table = subject.to_arrow() + assert isinstance(table, pyarrow.Table) + + +def test_same_secondary_attribute(schema_any): + children = (schema.Child * schema.Parent().proj()).to_arrays()["name"] + assert len(children) == 1 + assert children[0] == "Dan" + + +def test_query_caching(schema_any): + """Test query caching with to_arrays""" + # initialize cache directory + os.makedirs(os.path.expanduser("~/dj_query_cache"), exist_ok=True) + + with dj.config.override(query_cache=os.path.expanduser("~/dj_query_cache")): + conn = schema.TTest3.connection + # insert sample data and load cache + schema.TTest3.insert([dict(key=100 + i, value=200 + i) for i in range(2)]) + conn.set_query_cache(query_cache="main") + cached_res = schema.TTest3().to_arrays() + # attempt to insert while caching enabled + try: + schema.TTest3.insert([dict(key=200 + i, value=400 + i) for i in range(2)]) + assert False, "Insert allowed while query caching enabled" + except dj.DataJointError: + conn.set_query_cache() + # insert new data + schema.TTest3.insert([dict(key=600 + i, value=800 + i) for i in range(2)]) + # re-enable cache to access old results + conn.set_query_cache(query_cache="main") + previous_cache = schema.TTest3().to_arrays() + # verify properly cached and how to refresh results + assert all([c == p for c, p in zip(cached_res, previous_cache)]) + conn.set_query_cache() + uncached_res = schema.TTest3().to_arrays() + assert len(uncached_res) > len(cached_res) + # purge query cache + conn.purge_query_cache() + + # reset cache directory state + shutil.rmtree(os.path.expanduser("~/dj_query_cache"), ignore_errors=True) + + +def test_fetch_group_by(schema_any): + """ + https://github.com/datajoint/datajoint-python/issues/914 + """ + assert schema.Parent().keys(order_by="name") == [{"parent_id": 1}] + + +def test_dj_u_distinct(schema_any): + """ + Test developed to see if removing DISTINCT from the select statement + generation breaks the dj.U universal set implementation + """ + + # Contents to be inserted + contents = [(1, 2, 3), (2, 2, 3), (3, 3, 2), (4, 5, 5)] + schema.Stimulus.insert(contents) + + # Query the whole table + test_query = schema.Stimulus() + + # Use dj.U to create a list of unique contrast and brightness combinations + result = dj.U("contrast", "brightness") & test_query + expected_result = [ + {"contrast": 2, "brightness": 3}, + {"contrast": 3, "brightness": 2}, + {"contrast": 5, "brightness": 5}, + ] + + fetched_result = result.to_dicts(order_by=("contrast", "brightness")) + schema.Stimulus.delete_quick() + assert fetched_result == expected_result + + +def test_backslash(schema_any): + """ + https://github.com/datajoint/datajoint-python/issues/999 + """ + expected = "She\\Hulk" + schema.Parent.insert([(2, expected)]) + q = schema.Parent & dict(name=expected) + assert q.fetch1("name") == expected + q.delete() + + +def test_lazy_iteration(lang, languages): + """Test that iteration is lazy (uses generator)""" + # The new iteration is a generator + iter_obj = iter(lang) + # Should be a generator + import types + + assert isinstance(iter_obj, types.GeneratorType) + + # Each item should be a dict + first = next(iter_obj) + assert isinstance(first, dict) + assert "name" in first and "language" in first + + +def test_to_arrays_include_key(lang, languages): + """Test to_arrays with include_key=True returns keys as list of dicts""" + # Fetch with include_key=True + keys, names, langs = lang.to_arrays("name", "language", include_key=True, order_by="KEY") + + # keys should be a list of dicts with primary key columns + assert isinstance(keys, list) + assert all(isinstance(k, dict) for k in keys) + assert all(set(k.keys()) == {"name", "language"} for k in keys) + + # names and langs should be numpy arrays + assert isinstance(names, np.ndarray) + assert isinstance(langs, np.ndarray) + + # Length should match + assert len(keys) == len(names) == len(langs) == len(languages) + + # Keys should match the data + for key, name, language in zip(keys, names, langs): + assert key["name"] == name + assert key["language"] == language + + # Keys should be usable for restrictions + first_key = keys[0] + restricted = lang & first_key + assert len(restricted) == 1 + assert restricted.fetch1("name") == first_key["name"] + + +def test_to_arrays_include_key_single_attr(subject): + """Test to_arrays include_key with single attribute""" + keys, species = subject.to_arrays("species", include_key=True) + + assert isinstance(keys, list) + assert isinstance(species, np.ndarray) + assert len(keys) == len(species) + + # Verify keys have only primary key columns + assert all("subject_id" in k for k in keys) + + +def test_to_arrays_without_include_key(lang): + """Test that to_arrays without include_key doesn't return keys""" + result = lang.to_arrays("name", "language") + + # Should return tuple of arrays, not (keys, ...) + assert isinstance(result, tuple) + assert len(result) == 2 + names, langs = result + assert isinstance(names, np.ndarray) + assert isinstance(langs, np.ndarray) diff --git a/tests/test_fetch_same.py b/tests/integration/test_fetch_same.py similarity index 85% rename from tests/test_fetch_same.py rename to tests/integration/test_fetch_same.py index 0c136b097..05c971836 100644 --- a/tests/test_fetch_same.py +++ b/tests/integration/test_fetch_same.py @@ -10,7 +10,7 @@ class ProjData(dj.Manual): --- resp : float sim : float - big : longblob + big : blah : varchar(10) """ @@ -47,23 +47,23 @@ def schema_fetch_same(connection_test, prefix): def test_object_conversion_one(schema_fetch_same): - new = ProjData().proj(sub="resp").fetch("sub") + new = ProjData().proj(sub="resp").to_arrays("sub") assert new.dtype == np.float64 def test_object_conversion_two(schema_fetch_same): - [sub, add] = ProjData().proj(sub="resp", add="sim").fetch("sub", "add") + [sub, add] = ProjData().proj(sub="resp", add="sim").to_arrays("sub", "add") assert sub.dtype == np.float64 assert add.dtype == np.float64 def test_object_conversion_all(schema_fetch_same): - new = ProjData().proj(sub="resp", add="sim").fetch() + new = ProjData().proj(sub="resp", add="sim").to_arrays() assert new["sub"].dtype == np.float64 assert new["add"].dtype == np.float64 def test_object_no_convert(schema_fetch_same): - new = ProjData().fetch() + new = ProjData().to_arrays() assert new["big"].dtype == "object" assert new["blah"].dtype == "object" diff --git a/tests/test_foreign_keys.py b/tests/integration/test_foreign_keys.py similarity index 80% rename from tests/test_foreign_keys.py rename to tests/integration/test_foreign_keys.py index b271c6c1f..014340898 100644 --- a/tests/test_foreign_keys.py +++ b/tests/integration/test_foreign_keys.py @@ -1,6 +1,12 @@ from datajoint.declare import declare -from .schema_advanced import * +from tests.schema_advanced import ( + Cell, # noqa: F401 - needed in globals for foreign key resolution + GlobalSynapse, + LocalSynapse, + Parent, + Person, +) def test_aliased_fk(schema_adv): @@ -16,7 +22,7 @@ def test_aliased_fk(schema_adv): link = person.proj(parent_name="full_name", parent="person_id") parents = person * parent * link parents &= dict(full_name="May K. Hall") - assert set(parents.fetch("parent_name")) == {"Hanna R. Walters", "Russel S. James"} + assert set(parents.to_arrays("parent_name")) == {"Hanna R. Walters", "Russel S. James"} delete_count = person.delete() assert delete_count == 16 @@ -25,9 +31,7 @@ def test_describe(schema_adv): """real_definition should match original definition""" for rel in (LocalSynapse, GlobalSynapse): describe = rel.describe() - s1 = declare(rel.full_table_name, rel.definition, schema_adv.context)[0].split( - "\n" - ) + s1 = declare(rel.full_table_name, rel.definition, schema_adv.context)[0].split("\n") s2 = declare(rel.full_table_name, describe, globals())[0].split("\n") for c1, c2 in zip(s1, s2): assert c1 == c2 diff --git a/tests/integration/test_gc.py b/tests/integration/test_gc.py new file mode 100644 index 000000000..e0c5fafca --- /dev/null +++ b/tests/integration/test_gc.py @@ -0,0 +1,341 @@ +""" +Tests for garbage collection (gc.py). +""" + +from unittest.mock import MagicMock, patch + +import pytest + +from datajoint import gc +from datajoint.errors import DataJointError + + +class TestUsesContentStorage: + """Tests for _uses_content_storage helper function.""" + + def test_returns_false_for_no_adapter(self): + """Test that False is returned when attribute has no codec.""" + attr = MagicMock() + attr.codec = None + + assert gc._uses_content_storage(attr) is False + + def test_returns_true_for_hash_type(self): + """Test that True is returned for type.""" + attr = MagicMock() + attr.codec = MagicMock() + attr.codec.name = "hash" + attr.store = "mystore" + + assert gc._uses_content_storage(attr) is True + + def test_returns_true_for_blob_external(self): + """Test that True is returned for type (external).""" + attr = MagicMock() + attr.codec = MagicMock() + attr.codec.name = "blob" + attr.store = "mystore" + + assert gc._uses_content_storage(attr) is True + + def test_returns_true_for_attach_external(self): + """Test that True is returned for type (external).""" + attr = MagicMock() + attr.codec = MagicMock() + attr.codec.name = "attach" + attr.store = "mystore" + + assert gc._uses_content_storage(attr) is True + + def test_returns_false_for_blob_internal(self): + """Test that False is returned for internal storage.""" + attr = MagicMock() + attr.codec = MagicMock() + attr.codec.name = "blob" + attr.store = None + + assert gc._uses_content_storage(attr) is False + + +class TestExtractContentRefs: + """Tests for _extract_content_refs helper function.""" + + def test_returns_empty_for_none(self): + """Test that empty list is returned for None value.""" + assert gc._extract_content_refs(None) == [] + + def test_parses_json_string(self): + """Test parsing JSON string with hash.""" + value = '{"hash": "abc123", "store": "mystore"}' + refs = gc._extract_content_refs(value) + + assert len(refs) == 1 + assert refs[0] == ("abc123", "mystore") + + def test_parses_dict_directly(self): + """Test parsing dict with hash.""" + value = {"hash": "def456", "store": None} + refs = gc._extract_content_refs(value) + + assert len(refs) == 1 + assert refs[0] == ("def456", None) + + def test_returns_empty_for_invalid_json(self): + """Test that empty list is returned for invalid JSON.""" + assert gc._extract_content_refs("not json") == [] + + def test_returns_empty_for_dict_without_hash(self): + """Test that empty list is returned for dict without hash key.""" + assert gc._extract_content_refs({"other": "data"}) == [] + + +class TestUsesObjectStorage: + """Tests for _uses_object_storage helper function.""" + + def test_returns_false_for_no_adapter(self): + """Test that False is returned when attribute has no codec.""" + attr = MagicMock() + attr.codec = None + + assert gc._uses_object_storage(attr) is False + + def test_returns_true_for_object_type(self): + """Test that True is returned for type.""" + attr = MagicMock() + attr.codec = MagicMock() + attr.codec.name = "object" + + assert gc._uses_object_storage(attr) is True + + def test_returns_false_for_other_types(self): + """Test that False is returned for non-object types.""" + attr = MagicMock() + attr.codec = MagicMock() + attr.codec.name = "blob" + + assert gc._uses_object_storage(attr) is False + + +class TestExtractObjectRefs: + """Tests for _extract_object_refs helper function.""" + + def test_returns_empty_for_none(self): + """Test that empty list is returned for None value.""" + assert gc._extract_object_refs(None) == [] + + def test_parses_json_string(self): + """Test parsing JSON string with path.""" + value = '{"path": "schema/table/objects/pk/field_abc123", "store": "mystore"}' + refs = gc._extract_object_refs(value) + + assert len(refs) == 1 + assert refs[0] == ("schema/table/objects/pk/field_abc123", "mystore") + + def test_parses_dict_directly(self): + """Test parsing dict with path.""" + value = {"path": "test/path", "store": None} + refs = gc._extract_object_refs(value) + + assert len(refs) == 1 + assert refs[0] == ("test/path", None) + + def test_returns_empty_for_dict_without_path(self): + """Test that empty list is returned for dict without path key.""" + assert gc._extract_object_refs({"other": "data"}) == [] + + +class TestScan: + """Tests for scan function.""" + + def test_requires_at_least_one_schema(self): + """Test that at least one schema is required.""" + with pytest.raises(DataJointError, match="At least one schema must be provided"): + gc.scan() + + @patch("datajoint.gc.scan_object_references") + @patch("datajoint.gc.list_stored_objects") + @patch("datajoint.gc.scan_references") + @patch("datajoint.gc.list_stored_content") + def test_returns_stats(self, mock_list_content, mock_scan_refs, mock_list_objects, mock_scan_objects): + """Test that scan returns proper statistics.""" + # Mock content-addressed storage + mock_scan_refs.return_value = {"hash1", "hash2"} + mock_list_content.return_value = { + "hash1": 100, + "hash3": 200, # orphaned + } + + # Mock path-addressed storage + mock_scan_objects.return_value = {"path/to/obj1"} + mock_list_objects.return_value = { + "path/to/obj1": 500, + "path/to/obj2": 300, # orphaned + } + + mock_schema = MagicMock() + stats = gc.scan(mock_schema, store_name="test_store") + + # Content stats + assert stats["content_referenced"] == 2 + assert stats["content_stored"] == 2 + assert stats["content_orphaned"] == 1 + assert "hash3" in stats["orphaned_hashes"] + + # Object stats + assert stats["object_referenced"] == 1 + assert stats["object_stored"] == 2 + assert stats["object_orphaned"] == 1 + assert "path/to/obj2" in stats["orphaned_paths"] + + # Combined totals + assert stats["referenced"] == 3 + assert stats["stored"] == 4 + assert stats["orphaned"] == 2 + assert stats["orphaned_bytes"] == 500 # 200 content + 300 object + + +class TestCollect: + """Tests for collect function.""" + + @patch("datajoint.gc.scan") + def test_dry_run_does_not_delete(self, mock_scan): + """Test that dry_run=True doesn't delete anything.""" + mock_scan.return_value = { + "referenced": 1, + "stored": 2, + "orphaned": 1, + "orphaned_bytes": 100, + "orphaned_hashes": ["orphan_hash"], + "orphaned_paths": [], + "content_orphaned": 1, + "object_orphaned": 0, + } + + mock_schema = MagicMock() + stats = gc.collect(mock_schema, store_name="test_store", dry_run=True) + + assert stats["deleted"] == 0 + assert stats["bytes_freed"] == 0 + assert stats["dry_run"] is True + + @patch("datajoint.gc.delete_content") + @patch("datajoint.gc.list_stored_content") + @patch("datajoint.gc.scan") + def test_deletes_orphaned_content(self, mock_scan, mock_list_stored, mock_delete): + """Test that orphaned content is deleted when dry_run=False.""" + mock_scan.return_value = { + "referenced": 1, + "stored": 2, + "orphaned": 1, + "orphaned_bytes": 100, + "orphaned_hashes": ["orphan_hash"], + "orphaned_paths": [], + "content_orphaned": 1, + "object_orphaned": 0, + } + mock_list_stored.return_value = {"orphan_hash": 100} + mock_delete.return_value = True + + mock_schema = MagicMock() + stats = gc.collect(mock_schema, store_name="test_store", dry_run=False) + + assert stats["deleted"] == 1 + assert stats["content_deleted"] == 1 + assert stats["bytes_freed"] == 100 + assert stats["dry_run"] is False + mock_delete.assert_called_once_with("orphan_hash", "test_store") + + @patch("datajoint.gc.delete_object") + @patch("datajoint.gc.list_stored_objects") + @patch("datajoint.gc.scan") + def test_deletes_orphaned_objects(self, mock_scan, mock_list_objects, mock_delete): + """Test that orphaned objects are deleted when dry_run=False.""" + mock_scan.return_value = { + "referenced": 1, + "stored": 2, + "orphaned": 1, + "orphaned_bytes": 500, + "orphaned_hashes": [], + "orphaned_paths": ["path/to/orphan"], + "content_orphaned": 0, + "object_orphaned": 1, + } + mock_list_objects.return_value = {"path/to/orphan": 500} + mock_delete.return_value = True + + mock_schema = MagicMock() + stats = gc.collect(mock_schema, store_name="test_store", dry_run=False) + + assert stats["deleted"] == 1 + assert stats["object_deleted"] == 1 + assert stats["bytes_freed"] == 500 + assert stats["dry_run"] is False + mock_delete.assert_called_once_with("path/to/orphan", "test_store") + + +class TestFormatStats: + """Tests for format_stats function.""" + + def test_formats_scan_stats(self): + """Test formatting scan statistics.""" + stats = { + "referenced": 10, + "stored": 15, + "orphaned": 5, + "orphaned_bytes": 1024 * 1024, # 1 MB + "content_referenced": 6, + "content_stored": 8, + "content_orphaned": 2, + "content_orphaned_bytes": 512 * 1024, + "object_referenced": 4, + "object_stored": 7, + "object_orphaned": 3, + "object_orphaned_bytes": 512 * 1024, + } + + result = gc.format_stats(stats) + + assert "Referenced in database: 10" in result + assert "Stored in backend: 15" in result + assert "Orphaned (unreferenced): 5" in result + assert "1.00 MB" in result + # Check for detailed sections + assert "Content-Addressed Storage" in result + assert "Path-Addressed Storage" in result + + def test_formats_collect_stats_dry_run(self): + """Test formatting collect statistics with dry_run.""" + stats = { + "referenced": 10, + "stored": 15, + "orphaned": 5, + "deleted": 0, + "bytes_freed": 0, + "dry_run": True, + } + + result = gc.format_stats(stats) + + assert "DRY RUN" in result + + def test_formats_collect_stats_actual(self): + """Test formatting collect statistics after actual deletion.""" + stats = { + "referenced": 10, + "stored": 15, + "orphaned": 5, + "deleted": 3, + "content_deleted": 2, + "object_deleted": 1, + "bytes_freed": 2 * 1024 * 1024, # 2 MB + "errors": 2, + "dry_run": False, + } + + result = gc.format_stats(stats) + + assert "Deleted: 3" in result + assert "Content: 2" in result + assert "Objects: 1" in result + assert "2.00 MB" in result + assert "Errors: 2" in result diff --git a/tests/test_groupby.py b/tests/integration/test_groupby.py similarity index 93% rename from tests/test_groupby.py rename to tests/integration/test_groupby.py index 109972760..8e13f5b64 100644 --- a/tests/test_groupby.py +++ b/tests/integration/test_groupby.py @@ -1,4 +1,4 @@ -from .schema_simple import A, D +from tests.schema_simple import A, D def test_aggr_with_proj(schema_simp): diff --git a/tests/integration/test_hidden_job_metadata.py b/tests/integration/test_hidden_job_metadata.py new file mode 100644 index 000000000..c86f32d9d --- /dev/null +++ b/tests/integration/test_hidden_job_metadata.py @@ -0,0 +1,273 @@ +"""Tests for hidden job metadata in computed tables.""" + +import time + +import pytest + +import datajoint as dj + + +@pytest.fixture +def schema_job_metadata(connection_test, prefix): + """Create a schema with job metadata enabled.""" + # Enable job metadata for this test + original_setting = dj.config.jobs.add_job_metadata + dj.config.jobs.add_job_metadata = True + + schema = dj.Schema(prefix + "_job_metadata", connection=connection_test) + + class Source(dj.Lookup): + definition = """ + source_id : uint8 + --- + value : float32 + """ + contents = [(1, 1.0), (2, 2.0), (3, 3.0)] + + class ComputedWithMetadata(dj.Computed): + definition = """ + -> Source + --- + result : float32 + """ + + def make(self, key): + time.sleep(0.01) # Small delay to ensure non-zero duration + source = (Source & key).fetch1() + self.insert1({**key, "result": source["value"] * 2}) + + class ImportedWithMetadata(dj.Imported): + definition = """ + -> Source + --- + imported_value : float32 + """ + + def make(self, key): + source = (Source & key).fetch1() + self.insert1({**key, "imported_value": source["value"] + 10}) + + class ManualTable(dj.Manual): + definition = """ + manual_id : uint8 + --- + data : float32 + """ + + class ComputedWithPart(dj.Computed): + definition = """ + -> Source + --- + total : float32 + """ + + class Detail(dj.Part): + definition = """ + -> master + detail_idx : uint8 + --- + detail_value : float32 + """ + + def make(self, key): + source = (Source & key).fetch1() + self.insert1({**key, "total": source["value"] * 3}) + self.Detail.insert1({**key, "detail_idx": 0, "detail_value": source["value"]}) + + context = { + "Source": Source, + "ComputedWithMetadata": ComputedWithMetadata, + "ImportedWithMetadata": ImportedWithMetadata, + "ManualTable": ManualTable, + "ComputedWithPart": ComputedWithPart, + } + + schema(Source, context=context) + schema(ComputedWithMetadata, context=context) + schema(ImportedWithMetadata, context=context) + schema(ManualTable, context=context) + schema(ComputedWithPart, context=context) + + yield { + "schema": schema, + "Source": Source, + "ComputedWithMetadata": ComputedWithMetadata, + "ImportedWithMetadata": ImportedWithMetadata, + "ManualTable": ManualTable, + "ComputedWithPart": ComputedWithPart, + } + + # Cleanup + schema.drop() + dj.config.jobs.add_job_metadata = original_setting + + +class TestHiddenJobMetadataDeclaration: + """Test that hidden job metadata columns are added during declaration.""" + + def test_computed_table_has_hidden_metadata(self, schema_job_metadata): + """Computed tables should have hidden job metadata columns.""" + table = schema_job_metadata["ComputedWithMetadata"] + # Force heading to load from database + _ = table.heading.attributes + # Check _attributes (includes hidden) + all_attrs = table.heading._attributes + assert all_attrs is not None, "heading._attributes should not be None after loading" + assert "_job_start_time" in all_attrs + assert "_job_duration" in all_attrs + assert "_job_version" in all_attrs + # Check that they're hidden + assert all_attrs["_job_start_time"].is_hidden + assert all_attrs["_job_duration"].is_hidden + assert all_attrs["_job_version"].is_hidden + + def test_imported_table_has_hidden_metadata(self, schema_job_metadata): + """Imported tables should have hidden job metadata columns.""" + table = schema_job_metadata["ImportedWithMetadata"] + _ = table.heading.attributes # Force load + all_attrs = table.heading._attributes + assert "_job_start_time" in all_attrs + assert "_job_duration" in all_attrs + assert "_job_version" in all_attrs + + def test_manual_table_no_hidden_metadata(self, schema_job_metadata): + """Manual tables should NOT have hidden job metadata columns.""" + table = schema_job_metadata["ManualTable"] + _ = table.heading.attributes # Force load + all_attrs = table.heading._attributes + assert "_job_start_time" not in all_attrs + assert "_job_duration" not in all_attrs + assert "_job_version" not in all_attrs + + def test_lookup_table_no_hidden_metadata(self, schema_job_metadata): + """Lookup tables should NOT have hidden job metadata columns.""" + table = schema_job_metadata["Source"] + _ = table.heading.attributes # Force load + all_attrs = table.heading._attributes + assert "_job_start_time" not in all_attrs + assert "_job_duration" not in all_attrs + assert "_job_version" not in all_attrs + + def test_part_table_no_hidden_metadata(self, schema_job_metadata): + """Part tables should NOT have hidden job metadata columns.""" + master = schema_job_metadata["ComputedWithPart"] + part = master.Detail + _ = part.heading.attributes # Force load + all_attrs = part.heading._attributes + assert "_job_start_time" not in all_attrs + assert "_job_duration" not in all_attrs + assert "_job_version" not in all_attrs + + +class TestHiddenJobMetadataPopulation: + """Test that job metadata is populated during make().""" + + def test_metadata_populated_after_make(self, schema_job_metadata): + """Job metadata should be populated after make() completes.""" + table = schema_job_metadata["ComputedWithMetadata"] + table.populate() + + # Fetch hidden attributes using raw SQL since fetch() filters them + conn = table.connection + result = conn.query(f"SELECT _job_start_time, _job_duration, _job_version FROM {table.full_table_name}").fetchall() + assert len(result) == 3 + + for row in result: + start_time, duration, version = row + assert start_time is not None + assert duration is not None + assert duration >= 0 + # Version may be empty string if git not available + assert version is not None + + def test_metadata_not_in_default_fetch(self, schema_job_metadata): + """Hidden metadata should not appear in default fetch().""" + table = schema_job_metadata["ComputedWithMetadata"] + table.populate() + + result = table.to_dicts() + for row in result: + assert "_job_start_time" not in row + assert "_job_duration" not in row + assert "_job_version" not in row + + def test_hidden_attrs_not_in_heading_names(self, schema_job_metadata): + """Hidden attributes should not appear in heading.names.""" + table = schema_job_metadata["ComputedWithMetadata"] + _ = table.heading.attributes # Force load + names = table.heading.names + assert "_job_start_time" not in names + assert "_job_duration" not in names + assert "_job_version" not in names + + +class TestHiddenAttributesExcludedFromJoins: + """Test that hidden attributes are excluded from join operations.""" + + def test_hidden_attrs_excluded_from_join(self, schema_job_metadata): + """Hidden attributes should not participate in join matching.""" + computed = schema_job_metadata["ComputedWithMetadata"] + imported = schema_job_metadata["ImportedWithMetadata"] + + # Populate both tables + computed.populate() + imported.populate() + + # Both have _job_start_time, _job_duration, _job_version + # But these should NOT be used for joining + joined = computed * imported + # Should join on source_id only + assert len(joined) == 3 + + # The result heading should not have hidden attributes + assert "_job_start_time" not in joined.heading.names + assert "_job_duration" not in joined.heading.names + + +class TestConfigDisabled: + """Test behavior when add_job_metadata is disabled.""" + + def test_no_metadata_when_disabled(self, connection_test, prefix): + """Tables should not have metadata columns when config is disabled.""" + # Ensure disabled + original_setting = dj.config.jobs.add_job_metadata + dj.config.jobs.add_job_metadata = False + + schema = dj.Schema(prefix + "_no_metadata", connection=connection_test) + + class Source(dj.Lookup): + definition = """ + source_id : uint8 + """ + contents = [(1,), (2,)] + + class ComputedNoMetadata(dj.Computed): + definition = """ + -> Source + --- + result : float32 + """ + + def make(self, key): + self.insert1({**key, "result": 1.0}) + + context = {"Source": Source, "ComputedNoMetadata": ComputedNoMetadata} + schema(Source, context=context) + schema(ComputedNoMetadata, context=context) + + try: + # Force heading to load from database + _ = ComputedNoMetadata.heading.attributes + # Check no hidden metadata columns + all_attrs = ComputedNoMetadata.heading._attributes + assert all_attrs is not None + assert "_job_start_time" not in all_attrs + assert "_job_duration" not in all_attrs + assert "_job_version" not in all_attrs + + # Populate should still work + ComputedNoMetadata.populate() + assert len(ComputedNoMetadata()) == 2 + finally: + schema.drop() + dj.config.jobs.add_job_metadata = original_setting diff --git a/tests/integration/test_insert.py b/tests/integration/test_insert.py new file mode 100644 index 000000000..de22e5565 --- /dev/null +++ b/tests/integration/test_insert.py @@ -0,0 +1,509 @@ +"""Tests for insert API improvements: validate(), chunk_size, insert_dataframe(), deprecation warnings.""" + +import warnings + +import numpy as np +import pandas +import pytest + +import datajoint as dj + + +class SimpleTable(dj.Manual): + definition = """ + id : int32 + --- + value : varchar(100) + score=null : float64 + """ + + +class AutoIncrementTable(dj.Manual): + definition = """ + # auto_increment requires native int type + id : int auto_increment + --- + value : varchar(100) + """ + + +@pytest.fixture +def schema_insert(connection_test, prefix): + schema = dj.Schema( + prefix + "_insert_test", + context=dict(SimpleTable=SimpleTable, AutoIncrementTable=AutoIncrementTable), + connection=connection_test, + ) + schema(SimpleTable) + schema(AutoIncrementTable) + yield schema + schema.drop() + + +class TestValidate: + """Tests for the validate() method.""" + + def test_validate_valid_rows(self, schema_insert): + """Test that valid rows pass validation.""" + table = SimpleTable() + rows = [ + {"id": 1, "value": "one", "score": 1.0}, + {"id": 2, "value": "two", "score": 2.0}, + ] + result = table.validate(rows) + assert result.is_valid + assert len(result.errors) == 0 + assert result.rows_checked == 2 + assert bool(result) is True + + def test_validate_missing_required_field(self, schema_insert): + """Test that missing required fields are detected.""" + table = SimpleTable() + rows = [{"value": "one"}] # Missing 'id' which is PK + result = table.validate(rows) + assert not result.is_valid + assert len(result.errors) > 0 + assert "id" in result.errors[0][2] # Error message mentions 'id' + + def test_validate_unknown_field(self, schema_insert): + """Test that unknown fields are detected.""" + table = SimpleTable() + rows = [{"id": 1, "value": "one", "unknown_field": "test"}] + result = table.validate(rows) + assert not result.is_valid + assert any("unknown_field" in err[2] for err in result.errors) + + def test_validate_ignore_extra_fields(self, schema_insert): + """Test that ignore_extra_fields works.""" + table = SimpleTable() + rows = [{"id": 1, "value": "one", "unknown_field": "test"}] + result = table.validate(rows, ignore_extra_fields=True) + assert result.is_valid + + def test_validate_wrong_tuple_length(self, schema_insert): + """Test that wrong tuple length is detected.""" + table = SimpleTable() + rows = [(1, "one")] # Missing score + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + result = table.validate(rows) + assert not result.is_valid + assert "Incorrect number of attributes" in result.errors[0][2] + + def test_validate_nullable_field(self, schema_insert): + """Test that nullable fields can be omitted.""" + table = SimpleTable() + rows = [{"id": 1, "value": "one"}] # score is nullable, can be omitted + result = table.validate(rows) + assert result.is_valid + + def test_validate_result_summary(self, schema_insert): + """Test that summary() produces readable output.""" + table = SimpleTable() + rows = [{"id": 1, "value": "one"}] + result = table.validate(rows) + summary = result.summary() + assert "Validation passed" in summary + + rows = [{"value": "one"}] # Missing id + result = table.validate(rows) + summary = result.summary() + assert "Validation failed" in summary + + def test_validate_raise_if_invalid(self, schema_insert): + """Test that raise_if_invalid() raises for invalid rows.""" + table = SimpleTable() + rows = [{"value": "one"}] # Missing id + result = table.validate(rows) + with pytest.raises(dj.DataJointError): + result.raise_if_invalid() + + def test_validate_dataframe(self, schema_insert): + """Test validating a DataFrame.""" + table = SimpleTable() + df = pandas.DataFrame({"id": [1, 2], "value": ["one", "two"], "score": [1.0, 2.0]}) + result = table.validate(df) + assert result.is_valid + + def test_validate_autoincrement_pk(self, schema_insert): + """Test that autoincrement PK doesn't require value.""" + table = AutoIncrementTable() + rows = [{"value": "one"}] # id is auto_increment, can be omitted + result = table.validate(rows) + assert result.is_valid + + +class TestChunkedInsert: + """Tests for chunk_size parameter in insert().""" + + def test_chunked_insert(self, schema_insert): + """Test inserting with chunk_size.""" + table = SimpleTable() + rows = [{"id": i, "value": f"val{i}", "score": float(i)} for i in range(100)] + table.insert(rows, chunk_size=10) + assert len(table) == 100 + + def test_chunked_insert_single_chunk(self, schema_insert): + """Test chunked insert where data fits in one chunk.""" + table = SimpleTable() + rows = [{"id": i, "value": f"val{i}"} for i in range(5)] + table.insert(rows, chunk_size=100) # chunk_size larger than data + assert len(table) == 5 + + def test_chunked_insert_exact_chunks(self, schema_insert): + """Test chunked insert where data divides evenly.""" + table = SimpleTable() + rows = [{"id": i, "value": f"val{i}"} for i in range(20)] + table.insert(rows, chunk_size=5) # 4 chunks of 5 + assert len(table) == 20 + + def test_chunked_insert_with_skip_duplicates(self, schema_insert): + """Test chunked insert with skip_duplicates.""" + table = SimpleTable() + rows = [{"id": i, "value": f"val{i}"} for i in range(10)] + table.insert(rows) + # Insert again with duplicates + more_rows = [{"id": i, "value": f"val{i}"} for i in range(15)] + table.insert(more_rows, chunk_size=5, skip_duplicates=True) + assert len(table) == 15 + + def test_chunked_insert_query_expression_error(self, schema_insert): + """Test that chunk_size raises error for QueryExpression inserts.""" + table = SimpleTable() + with pytest.raises(dj.DataJointError, match="chunk_size is not supported"): + table.insert(table.proj(), chunk_size=10) + + +class TestInsertDataFrame: + """Tests for insert_dataframe() method.""" + + def test_insert_dataframe_basic(self, schema_insert): + """Test basic DataFrame insert.""" + table = SimpleTable() + df = pandas.DataFrame({"id": [1, 2, 3], "value": ["a", "b", "c"], "score": [1.0, 2.0, 3.0]}) + table.insert_dataframe(df) + assert len(table) == 3 + + def test_insert_dataframe_index_as_pk_auto(self, schema_insert): + """Test auto-detection of index as PK.""" + table = SimpleTable() + # Create DataFrame with PK as index + df = pandas.DataFrame({"value": ["a", "b"], "score": [1.0, 2.0]}) + df.index = pandas.Index([1, 2], name="id") + table.insert_dataframe(df) # Auto-detects index as PK + assert len(table) == 2 + assert set(table.to_arrays("id")) == {1, 2} + + def test_insert_dataframe_index_as_pk_true(self, schema_insert): + """Test explicit index_as_pk=True.""" + table = SimpleTable() + df = pandas.DataFrame({"value": ["a", "b"], "score": [1.0, 2.0]}) + df.index = pandas.Index([1, 2], name="id") + table.insert_dataframe(df, index_as_pk=True) + assert len(table) == 2 + + def test_insert_dataframe_index_as_pk_false(self, schema_insert): + """Test explicit index_as_pk=False.""" + table = SimpleTable() + df = pandas.DataFrame({"id": [1, 2], "value": ["a", "b"], "score": [1.0, 2.0]}) + df = df.set_index("id") # Set id as index + # With index_as_pk=False, index is dropped and we need id as column + df = df.reset_index() # Put id back as column + table.insert_dataframe(df, index_as_pk=False) + assert len(table) == 2 + + def test_insert_dataframe_rangeindex_dropped(self, schema_insert): + """Test that RangeIndex is automatically dropped.""" + table = SimpleTable() + df = pandas.DataFrame({"id": [1, 2], "value": ["a", "b"], "score": [1.0, 2.0]}) + # df has default RangeIndex which should be dropped + table.insert_dataframe(df) + assert len(table) == 2 + + def test_insert_dataframe_index_mismatch_error(self, schema_insert): + """Test error when index doesn't match PK.""" + table = SimpleTable() + df = pandas.DataFrame({"value": ["a", "b"], "score": [1.0, 2.0]}) + df.index = pandas.Index([1, 2], name="wrong_name") + with pytest.raises(dj.DataJointError, match="do not match"): + table.insert_dataframe(df, index_as_pk=True) + + def test_insert_dataframe_not_dataframe_error(self, schema_insert): + """Test error when not a DataFrame.""" + table = SimpleTable() + with pytest.raises(dj.DataJointError, match="requires a pandas DataFrame"): + table.insert_dataframe([{"id": 1, "value": "a"}]) + + def test_insert_dataframe_roundtrip(self, schema_insert): + """Test roundtrip: to_pandas() -> modify -> insert_dataframe().""" + table = SimpleTable() + # Insert initial data + table.insert([{"id": i, "value": f"val{i}", "score": float(i)} for i in range(3)]) + + # Fetch as DataFrame + df = table.to_pandas() + + # Clear table and re-insert + with dj.config.override(safemode=False): + table.delete() + + table.insert_dataframe(df) + assert len(table) == 3 + + def test_insert_dataframe_with_chunk_size(self, schema_insert): + """Test insert_dataframe with chunk_size.""" + table = SimpleTable() + df = pandas.DataFrame({"id": range(100), "value": [f"v{i}" for i in range(100)], "score": np.arange(100.0)}) + table.insert_dataframe(df, chunk_size=25) + assert len(table) == 100 + + +try: + import polars + + HAS_POLARS = True +except ImportError: + HAS_POLARS = False + +try: + import pyarrow + + HAS_PYARROW = True +except ImportError: + HAS_PYARROW = False + + +@pytest.mark.skipif(not HAS_POLARS, reason="polars not installed") +class TestPolarsInsert: + """Tests for Polars DataFrame insert support.""" + + def test_insert_polars_basic(self, schema_insert): + """Test inserting a Polars DataFrame.""" + table = SimpleTable() + df = polars.DataFrame({"id": [1, 2, 3], "value": ["a", "b", "c"], "score": [1.0, 2.0, 3.0]}) + table.insert(df) + assert len(table) == 3 + assert set(table.to_arrays("id")) == {1, 2, 3} + + def test_insert_polars_with_options(self, schema_insert): + """Test Polars insert with skip_duplicates and chunk_size.""" + table = SimpleTable() + df = polars.DataFrame({"id": [1, 2], "value": ["a", "b"], "score": [1.0, 2.0]}) + table.insert(df) + + # Insert more with duplicates + df2 = polars.DataFrame({"id": [2, 3, 4], "value": ["b", "c", "d"], "score": [2.0, 3.0, 4.0]}) + table.insert(df2, skip_duplicates=True) + assert len(table) == 4 + + def test_insert_polars_chunk_size(self, schema_insert): + """Test Polars insert with chunk_size.""" + table = SimpleTable() + df = polars.DataFrame( + {"id": list(range(50)), "value": [f"v{i}" for i in range(50)], "score": [float(i) for i in range(50)]} + ) + table.insert(df, chunk_size=10) + assert len(table) == 50 + + def test_insert_polars_roundtrip(self, schema_insert): + """Test roundtrip: to_polars() -> insert().""" + table = SimpleTable() + table.insert([{"id": i, "value": f"val{i}", "score": float(i)} for i in range(3)]) + + # Fetch as Polars + df = table.to_polars() + assert isinstance(df, polars.DataFrame) + + # Clear and re-insert + with dj.config.override(safemode=False): + table.delete() + + table.insert(df) + assert len(table) == 3 + + +@pytest.mark.skipif(not HAS_PYARROW, reason="pyarrow not installed") +class TestArrowInsert: + """Tests for PyArrow Table insert support.""" + + def test_insert_arrow_basic(self, schema_insert): + """Test inserting a PyArrow Table.""" + table = SimpleTable() + arrow_table = pyarrow.table({"id": [1, 2, 3], "value": ["a", "b", "c"], "score": [1.0, 2.0, 3.0]}) + table.insert(arrow_table) + assert len(table) == 3 + assert set(table.to_arrays("id")) == {1, 2, 3} + + def test_insert_arrow_with_options(self, schema_insert): + """Test Arrow insert with skip_duplicates.""" + table = SimpleTable() + arrow_table = pyarrow.table({"id": [1, 2], "value": ["a", "b"], "score": [1.0, 2.0]}) + table.insert(arrow_table) + + # Insert more with duplicates + arrow_table2 = pyarrow.table({"id": [2, 3, 4], "value": ["b", "c", "d"], "score": [2.0, 3.0, 4.0]}) + table.insert(arrow_table2, skip_duplicates=True) + assert len(table) == 4 + + def test_insert_arrow_chunk_size(self, schema_insert): + """Test Arrow insert with chunk_size.""" + table = SimpleTable() + arrow_table = pyarrow.table( + {"id": list(range(50)), "value": [f"v{i}" for i in range(50)], "score": [float(i) for i in range(50)]} + ) + table.insert(arrow_table, chunk_size=10) + assert len(table) == 50 + + def test_insert_arrow_roundtrip(self, schema_insert): + """Test roundtrip: to_arrow() -> insert().""" + table = SimpleTable() + table.insert([{"id": i, "value": f"val{i}", "score": float(i)} for i in range(3)]) + + # Fetch as Arrow + arrow_table = table.to_arrow() + assert isinstance(arrow_table, pyarrow.Table) + + # Clear and re-insert + with dj.config.override(safemode=False): + table.delete() + + table.insert(arrow_table) + assert len(table) == 3 + + +class TestDeprecationWarning: + """Tests for positional insert deprecation warning.""" + + def test_positional_insert_warning(self, schema_insert): + """Test that positional inserts emit deprecation warning.""" + table = SimpleTable() + with pytest.warns(DeprecationWarning, match="Positional inserts"): + table.insert1((1, "value1", 1.0)) + + def test_positional_insert_multiple_warning(self, schema_insert): + """Test that positional inserts in insert() emit warning.""" + table = SimpleTable() + with pytest.warns(DeprecationWarning, match="Positional inserts"): + table.insert([(2, "value2", 2.0)]) + + def test_dict_insert_no_warning(self, schema_insert): + """Test that dict inserts don't emit warning.""" + table = SimpleTable() + with warnings.catch_warnings(): + warnings.simplefilter("error", DeprecationWarning) + # Should not raise DeprecationWarning + table.insert1({"id": 3, "value": "value3", "score": 3.0}) + + def test_numpy_record_no_warning(self, schema_insert): + """Test that numpy record inserts don't emit warning.""" + table = SimpleTable() + # Create numpy record + dtype = [("id", int), ("value", "U100"), ("score", float)] + record = np.array([(4, "value4", 4.0)], dtype=dtype)[0] + with warnings.catch_warnings(): + warnings.simplefilter("error", DeprecationWarning) + # Should not raise DeprecationWarning + table.insert1(record) + + +class TestValidationResult: + """Tests for ValidationResult class.""" + + def test_validation_result_bool(self, schema_insert): + """Test ValidationResult boolean behavior.""" + valid = dj.ValidationResult(is_valid=True, errors=[], rows_checked=1) + invalid = dj.ValidationResult(is_valid=False, errors=[(0, "field", "error")], rows_checked=1) + assert bool(valid) is True + assert bool(invalid) is False + + def test_validation_result_summary_valid(self, schema_insert): + """Test ValidationResult summary for valid result.""" + result = dj.ValidationResult(is_valid=True, errors=[], rows_checked=5) + assert "Validation passed" in result.summary() + assert "5 rows checked" in result.summary() + + def test_validation_result_summary_invalid(self, schema_insert): + """Test ValidationResult summary for invalid result.""" + errors = [(0, "field1", "error1"), (1, "field2", "error2")] + result = dj.ValidationResult(is_valid=False, errors=errors, rows_checked=2) + summary = result.summary() + assert "Validation failed" in summary + assert "2 error(s)" in summary + assert "Row 0" in summary + assert "Row 1" in summary + + def test_validation_result_summary_truncated(self, schema_insert): + """Test that summary truncates long error lists.""" + errors = [(i, f"field{i}", f"error{i}") for i in range(20)] + result = dj.ValidationResult(is_valid=False, errors=errors, rows_checked=20) + summary = result.summary() + assert "and 10 more errors" in summary + + +class AllDefaultsTable(dj.Manual): + """Table where all attributes have defaults.""" + + definition = """ + id : int auto_increment + --- + timestamp=CURRENT_TIMESTAMP : datetime + notes=null : varchar(200) + """ + + +class TestEmptyInsert: + """Tests for inserting empty dicts (GitHub issue #1280).""" + + @pytest.fixture + def schema_empty_insert(self, connection_test, prefix): + schema = dj.Schema( + prefix + "_empty_insert_test", + context=dict(AllDefaultsTable=AllDefaultsTable, SimpleTable=SimpleTable), + connection=connection_test, + ) + schema(AllDefaultsTable) + schema(SimpleTable) + yield schema + schema.drop() + + def test_empty_insert_all_defaults(self, schema_empty_insert): + """Test that empty insert succeeds when all attributes have defaults.""" + table = AllDefaultsTable() + assert len(table) == 0 + + # Insert empty dict - should use all defaults + table.insert1({}) + assert len(table) == 1 + + # Check that values were populated with defaults + row = table.fetch1() + assert row["id"] == 1 # auto_increment starts at 1 + assert row["timestamp"] is not None # CURRENT_TIMESTAMP + assert row["notes"] is None # nullable defaults to NULL + + def test_empty_insert_multiple(self, schema_empty_insert): + """Test inserting multiple empty dicts.""" + table = AllDefaultsTable() + + # Insert multiple empty dicts + table.insert([{}, {}, {}]) + assert len(table) == 3 + + # Each should have unique auto_increment id + ids = set(table.to_arrays("id")) + assert ids == {1, 2, 3} + + def test_empty_insert_required_fields_error(self, schema_empty_insert): + """Test that empty insert raises clear error when fields are required.""" + table = SimpleTable() + + # SimpleTable has required fields (id, value) + with pytest.raises(dj.DataJointError) as exc_info: + table.insert1({}) + + error_msg = str(exc_info.value) + assert "Cannot insert empty row" in error_msg + assert "require values" in error_msg + # Should list the required attributes + assert "id" in error_msg + assert "value" in error_msg diff --git a/tests/integration/test_jobs.py b/tests/integration/test_jobs.py new file mode 100644 index 000000000..bc00cf0f8 --- /dev/null +++ b/tests/integration/test_jobs.py @@ -0,0 +1,160 @@ +"""Tests for per-table Job management (AutoPopulate 2.0).""" + +import random +import string + +import datajoint as dj +from datajoint.jobs import ERROR_MESSAGE_LENGTH, TRUNCATION_APPENDIX + +from tests import schema + + +def test_reserve_job(clean_jobs, subject, experiment): + """Test job reservation, completion, and error workflows.""" + assert subject + + # Refresh jobs to create pending entries + experiment.jobs.refresh() + pending_count = len(experiment.jobs.pending) + assert pending_count > 0, "no pending jobs created" + + # Reserve all pending jobs + keys = experiment.jobs.pending.keys() + for key in keys: + assert experiment.jobs.reserve(key), "failed to reserve a job" + + # Try to reserve already-reserved jobs - should fail + for key in keys: + assert not experiment.jobs.reserve(key), "failed to respect reservation" + + # Complete jobs + for key in keys: + experiment.jobs.complete(key) + + # Check jobs are completed (or deleted if keep_completed=False) + if dj.config.jobs.keep_completed: + assert len(experiment.jobs.completed) == len(keys) + else: + assert len(experiment.jobs) == 0, "failed to free jobs" + + # Refresh again to create new pending jobs + experiment.jobs.refresh() + keys = experiment.jobs.pending.keys() + + # Reserve and mark as error + for key in keys: + experiment.jobs.reserve(key) + experiment.jobs.error(key, "error message") + + # Try to reserve error jobs - should fail + for key in keys: + assert not experiment.jobs.reserve(key), "failed to ignore error jobs" + + # Clear error jobs + experiment.jobs.errors.delete() + assert len(experiment.jobs) == 0, "failed to clear error jobs" + + +def test_job_status_filters(clean_jobs, subject, experiment): + """Test job status filter properties.""" + # Refresh to create pending jobs + experiment.jobs.refresh() + + # All should be pending + total = len(experiment.jobs) + assert total > 0 + assert len(experiment.jobs.pending) == total + assert len(experiment.jobs.reserved) == 0 + assert len(experiment.jobs.errors) == 0 + + # Reserve some jobs + keys = experiment.jobs.pending.keys(limit=2) + for key in keys: + experiment.jobs.reserve(key) + + assert len(experiment.jobs.reserved) == 2 + + # Mark one as error + experiment.jobs.error(keys[0], "test error") + assert len(experiment.jobs.errors) == 1 + + +def test_sigint(clean_jobs, schema_any): + """Test that KeyboardInterrupt is recorded as error.""" + sig_int_table = schema.SigIntTable() + try: + sig_int_table.populate(reserve_jobs=True) + except KeyboardInterrupt: + pass + + assert len(sig_int_table.jobs.errors) > 0, "SigInt job error not recorded" + status, error_message = sig_int_table.jobs.errors.fetch1("status", "error_message") + assert status == "error" + assert "KeyboardInterrupt" in error_message + + +def test_sigterm(clean_jobs, schema_any): + """Test that SystemExit is recorded as error.""" + sig_term_table = schema.SigTermTable() + try: + sig_term_table.populate(reserve_jobs=True) + except SystemExit: + pass + + assert len(sig_term_table.jobs.errors) > 0, "SigTerm job error not recorded" + status, error_message = sig_term_table.jobs.errors.fetch1("status", "error_message") + assert status == "error" + assert "SIGTERM" in error_message or "SystemExit" in error_message + + +def test_suppress_dj_errors(clean_jobs, schema_any): + """Test that DataJoint errors are suppressible without native py blobs.""" + error_class = schema.ErrorClass() + with dj.config.override(enable_python_native_blobs=False): + error_class.populate(reserve_jobs=True, suppress_errors=True) + assert len(schema.DjExceptionName()) == len(error_class.jobs.errors) > 0 + + +def test_long_error_message(clean_jobs, subject, experiment): + """Test that long error messages are truncated.""" + # Create long and short error messages + long_error_message = "".join(random.choice(string.ascii_letters) for _ in range(ERROR_MESSAGE_LENGTH + 100)) + short_error_message = "".join(random.choice(string.ascii_letters) for _ in range(ERROR_MESSAGE_LENGTH // 2)) + + # Refresh to create pending jobs + experiment.jobs.refresh() + key = experiment.jobs.pending.keys(limit=1)[0] + + # Test long error message truncation + experiment.jobs.reserve(key) + experiment.jobs.error(key, long_error_message) + error_message = experiment.jobs.errors.fetch1("error_message") + assert len(error_message) == ERROR_MESSAGE_LENGTH, "error message is longer than max allowed" + assert error_message.endswith(TRUNCATION_APPENDIX), "appropriate ending missing for truncated error message" + experiment.jobs.delete() + + # Refresh and test short error message (not truncated) + experiment.jobs.refresh() + key = experiment.jobs.pending.keys(limit=1)[0] + experiment.jobs.reserve(key) + experiment.jobs.error(key, short_error_message) + error_message = experiment.jobs.errors.fetch1("error_message") + assert error_message == short_error_message, "error messages do not agree" + assert not error_message.endswith(TRUNCATION_APPENDIX), "error message should not be truncated" + + +def test_long_error_stack(clean_jobs, subject, experiment): + """Test that long error stacks are stored correctly.""" + # Create long error stack + STACK_SIZE = 89942 # Does not fit into small blob (should be 64k, but found to be higher) + long_error_stack = "".join(random.choice(string.ascii_letters) for _ in range(STACK_SIZE)) + + # Refresh to create pending jobs + experiment.jobs.refresh() + key = experiment.jobs.pending.keys(limit=1)[0] + + # Test long error stack + experiment.jobs.reserve(key) + experiment.jobs.error(key, "error message", long_error_stack) + error_stack = experiment.jobs.errors.fetch1("error_stack") + assert error_stack == long_error_stack, "error stacks do not agree" diff --git a/tests/test_json.py b/tests/integration/test_json.py similarity index 70% rename from tests/test_json.py rename to tests/integration/test_json.py index 0a819b99e..40c8074de 100644 --- a/tests/test_json.py +++ b/tests/integration/test_json.py @@ -7,8 +7,18 @@ import datajoint as dj from datajoint.declare import declare -if Version(dj.conn().query("select @@version;").fetchone()[0]) < Version("8.0.0"): - pytest.skip("These tests require MySQL >= v8.0.0", allow_module_level=True) + +def mysql_version_check(connection): + """Check if MySQL version is >= 8.0.0""" + version_str = connection.query("select @@version;").fetchone()[0] + if Version(version_str) < Version("8.0.0"): + pytest.skip("These tests require MySQL >= v8.0.0") + + +@pytest.fixture(scope="module", autouse=True) +def check_mysql_version(connection_root): + """Automatically check MySQL version for all tests in this module""" + mysql_version_check(connection_root) class Team(dj.Lookup): @@ -67,9 +77,7 @@ class Team(dj.Lookup): @pytest.fixture def schema_json(connection_test, prefix): - schema = dj.Schema( - prefix + "_json", context=dict(Team=Team), connection=connection_test - ) + schema = dj.Schema(prefix + "_json", context=dict(Team=Team), connection=connection_test) schema(Team) yield schema schema.drop() @@ -131,13 +139,10 @@ def test_restrict(schema_json): assert (Team & {"car.safety_inspected": "false"}).fetch1("name") == "business" - assert (Team & {"car.safety_inspected:unsigned": False}).fetch1( - "name" - ) == "business" + assert (Team & {"car.safety_inspected:unsigned": False}).fetch1("name") == "business" - assert (Team & {"car.headlights[0].hyper_white": None}).fetch( - "name", order_by="name", as_dict=True - ) == [ + # to_dicts returns all columns, use proj to select only name + assert (Team & {"car.headlights[0].hyper_white": None}).proj("name").to_dicts(order_by="name") == [ {"name": "engineering"}, {"name": "marketing"}, ] # if entire record missing, JSON key is missing, or value set to JSON null @@ -146,79 +151,62 @@ def test_restrict(schema_json): assert (Team & {"car.tire_pressure": [34, 30, 27, 32]}).fetch1("name") == "business" - assert ( - Team & {"car.headlights[1]": {"side": "right", "hyper_white": True}} - ).fetch1("name") == "business" + assert (Team & {"car.headlights[1]": {"side": "right", "hyper_white": True}}).fetch1("name") == "business" # sql operators - assert (Team & "`car`->>'$.name' LIKE '%ching%'").fetch1( - "name" - ) == "business", "Missing substring" + assert (Team & "`car`->>'$.name' LIKE '%ching%'").fetch1("name") == "business", "Missing substring" assert (Team & "`car`->>'$.length' > 30").fetch1("name") == "business", "<= 30" - assert ( - Team & "JSON_VALUE(`car`, '$.safety_inspected' RETURNING UNSIGNED) = 0" - ).fetch1("name") == "business", "Has `safety_inspected` set to `true`" + assert (Team & "JSON_VALUE(`car`, '$.safety_inspected' RETURNING UNSIGNED) = 0").fetch1( + "name" + ) == "business", "Has `safety_inspected` set to `true`" assert (Team & "`car`->>'$.headlights[0].hyper_white' = 'null'").fetch1( "name" ) == "engineering", "Has 1st `headlight` with `hyper_white` not set to `null`" - assert (Team & "`car`->>'$.inspected' IS NOT NULL").fetch1( - "name" - ) == "engineering", "Missing `inspected` key" + assert (Team & "`car`->>'$.inspected' IS NOT NULL").fetch1("name") == "engineering", "Missing `inspected` key" assert (Team & "`car`->>'$.tire_pressure' = '[34, 30, 27, 32]'").fetch1( "name" ) == "business", "`tire_pressure` array did not match" - assert ( - Team - & """`car`->>'$.headlights[1]' = '{"side": "right", "hyper_white": true}'""" - ).fetch1("name") == "business", "2nd `headlight` object did not match" + assert (Team & """`car`->>'$.headlights[1]' = '{"side": "right", "hyper_white": true}'""").fetch1( + "name" + ) == "business", "2nd `headlight` object did not match" def test_proj(schema_json): # proj necessary since we need to rename indexed value into a proper attribute name - assert Team.proj(car_length="car.length").fetch( - as_dict=True, order_by="car_length" - ) == [ + assert Team.proj(car_length="car.length").to_dicts(order_by="car_length") == [ {"name": "marketing", "car_length": None}, {"name": "business", "car_length": "100"}, {"name": "engineering", "car_length": "20.5"}, ] - assert Team.proj(car_length="car.length:decimal(4, 1)").fetch( - as_dict=True, order_by="car_length" - ) == [ + assert Team.proj(car_length="car.length:decimal(4, 1)").to_dicts(order_by="car_length") == [ {"name": "marketing", "car_length": None}, {"name": "engineering", "car_length": 20.5}, {"name": "business", "car_length": 100.0}, ] - assert Team.proj( - car_width="JSON_VALUE(`car`, '$.length' RETURNING float) - 15" - ).fetch(as_dict=True, order_by="car_width") == [ + assert Team.proj(car_width="JSON_VALUE(`car`, '$.length' RETURNING float) - 15").to_dicts(order_by="car_width") == [ {"name": "marketing", "car_width": None}, {"name": "engineering", "car_width": 5.5}, {"name": "business", "car_width": 85.0}, ] - assert ( - (Team & {"name": "engineering"}).proj(car_tire_pressure="car.tire_pressure") - ).fetch1("car_tire_pressure") == "[32, 31, 33, 34]" + assert ((Team & {"name": "engineering"}).proj(car_tire_pressure="car.tire_pressure")).fetch1( + "car_tire_pressure" + ) == "[32, 31, 33, 34]" assert np.array_equal( - Team.proj(car_inspected="car.inspected").fetch( - "car_inspected", order_by="name" - ), + Team.proj(car_inspected="car.inspected").to_arrays("car_inspected", order_by="name"), np.array([None, "true", None]), ) assert np.array_equal( - Team.proj(car_inspected="car.inspected:unsigned").fetch( - "car_inspected", order_by="name" - ), + Team.proj(car_inspected="car.inspected:unsigned").to_arrays("car_inspected", order_by="name"), np.array([None, 1, None]), ) diff --git a/tests/test_nan.py b/tests/integration/test_nan.py similarity index 58% rename from tests/test_nan.py rename to tests/integration/test_nan.py index 25e4e332b..17ec988b4 100644 --- a/tests/test_nan.py +++ b/tests/integration/test_nan.py @@ -14,9 +14,7 @@ class NanTest(dj.Manual): @pytest.fixture def schema_nan(connection_test, prefix): - schema = dj.Schema( - prefix + "_nantest", context=dict(NanTest=NanTest), connection=connection_test - ) + schema = dj.Schema(prefix + "_nantest", context=dict(NanTest=NanTest), connection=connection_test) schema(NanTest) yield schema schema.drop() @@ -30,7 +28,7 @@ def arr_a(): @pytest.fixture def schema_nan_pop(schema_nan, arr_a): rel = NanTest() - with dj.config(safemode=False): + with dj.config.override(safemode=False): rel.delete() rel.insert(((i, value) for i, value in enumerate(arr_a))) return schema_nan @@ -38,15 +36,15 @@ def schema_nan_pop(schema_nan, arr_a): def test_insert_nan(schema_nan_pop, arr_a): """Test fetching of null values""" - b = NanTest().fetch("value", order_by="id") - assert (np.isnan(arr_a) == np.isnan(b)).all(), "incorrect handling of Nans" + b = NanTest().to_arrays("value", order_by="id") + # Convert None to np.nan for comparison + b_float = np.array([np.nan if v is None else v for v in b], dtype=float) + assert (np.isnan(arr_a) == np.isnan(b_float)).all(), "incorrect handling of Nans" assert np.allclose( - arr_a[np.logical_not(np.isnan(arr_a))], b[np.logical_not(np.isnan(b))] + arr_a[np.logical_not(np.isnan(arr_a))], b_float[np.logical_not(np.isnan(b_float))] ), "incorrect storage of floats" def test_nulls_do_not_affect_primary_keys(schema_nan_pop, arr_a): """Test against a case that previously caused a bug when skipping existing entries.""" - NanTest().insert( - ((i, value) for i, value in enumerate(arr_a)), skip_duplicates=True - ) + NanTest().insert(((i, value) for i, value in enumerate(arr_a)), skip_duplicates=True) diff --git a/tests/integration/test_object.py b/tests/integration/test_object.py new file mode 100644 index 000000000..d4d42a461 --- /dev/null +++ b/tests/integration/test_object.py @@ -0,0 +1,761 @@ +""" +Tests for the object column type. + +Tests cover: +- Storage path generation +- Insert with file, folder, and stream +- Fetch returning ObjectRef +- ObjectRef methods (read, open, download, listdir, walk, verify) +- Staged insert +- Error cases +""" + +import io +import json +import os +from pathlib import Path + +import pytest + +import datajoint as dj +from datajoint.objectref import ObjectRef +from datajoint.storage import build_object_path, generate_token, encode_pk_value + +from tests.schema_object import ObjectFile, ObjectFolder, ObjectMultiple, ObjectWithOther + + +class TestStoragePathGeneration: + """Tests for storage path generation utilities.""" + + def test_generate_token_default_length(self): + """Test token generation with default length.""" + token = generate_token() + assert len(token) == 8 + # All characters should be URL-safe + safe_chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_" + assert all(c in safe_chars for c in token) + + def test_generate_token_custom_length(self): + """Test token generation with custom length.""" + token = generate_token(12) + assert len(token) == 12 + + def test_generate_token_minimum_length(self): + """Test token generation respects minimum length.""" + token = generate_token(2) # Below minimum + assert len(token) == 4 # Should be clamped to minimum + + def test_generate_token_maximum_length(self): + """Test token generation respects maximum length.""" + token = generate_token(20) # Above maximum + assert len(token) == 16 # Should be clamped to maximum + + def test_generate_token_uniqueness(self): + """Test that generated tokens are unique.""" + tokens = [generate_token() for _ in range(100)] + assert len(set(tokens)) == 100 + + def test_encode_pk_value_integer(self): + """Test encoding integer primary key values.""" + assert encode_pk_value(123) == "123" + assert encode_pk_value(0) == "0" + assert encode_pk_value(-5) == "-5" + + def test_encode_pk_value_string(self): + """Test encoding string primary key values.""" + assert encode_pk_value("simple") == "simple" + assert encode_pk_value("test_value") == "test_value" + + def test_encode_pk_value_unsafe_chars(self): + """Test encoding strings with unsafe characters.""" + # Slash should be URL-encoded + result = encode_pk_value("path/to/file") + assert "/" not in result or result == "path%2Fto%2Ffile" + + def test_build_object_path_basic(self): + """Test basic object path building.""" + path, token = build_object_path( + schema="myschema", + table="MyTable", + field="data_file", + primary_key={"id": 123}, + ext=".dat", + ) + assert "myschema" in path + assert "MyTable" in path + assert "objects" in path + assert "id=123" in path + assert "data_file_" in path + assert path.endswith(".dat") + assert len(token) == 8 + + def test_build_object_path_no_extension(self): + """Test object path building without extension.""" + path, token = build_object_path( + schema="myschema", + table="MyTable", + field="data_folder", + primary_key={"id": 456}, + ext=None, + ) + assert not path.endswith(".") + assert "data_folder_" in path + + def test_build_object_path_multiple_pk(self): + """Test object path with multiple primary key attributes.""" + path, token = build_object_path( + schema="myschema", + table="MyTable", + field="raw_data", + primary_key={"subject_id": 1, "session_id": 2}, + ext=".zarr", + ) + assert "subject_id=1" in path + assert "session_id=2" in path + + def test_build_object_path_with_partition(self): + """Test object path with partition pattern.""" + path, token = build_object_path( + schema="myschema", + table="MyTable", + field="data", + primary_key={"subject_id": 1, "session_id": 2}, + ext=".dat", + partition_pattern="{subject_id}", + ) + # subject_id should be at the beginning due to partition + assert path.startswith("subject_id=1") + + +class TestObjectRef: + """Tests for ObjectRef class.""" + + def test_from_json_string(self): + """Test creating ObjectRef from JSON string.""" + json_str = json.dumps( + { + "path": "schema/Table/objects/id=1/data_abc123.dat", + "size": 1024, + "hash": None, + "ext": ".dat", + "is_dir": False, + "timestamp": "2025-01-15T10:30:00+00:00", + } + ) + obj = ObjectRef.from_json(json_str) + assert obj.path == "schema/Table/objects/id=1/data_abc123.dat" + assert obj.size == 1024 + assert obj.hash is None + assert obj.ext == ".dat" + assert obj.is_dir is False + + def test_from_json_dict(self): + """Test creating ObjectRef from dict.""" + data = { + "path": "schema/Table/objects/id=1/data_abc123.zarr", + "size": 5678, + "hash": None, + "ext": ".zarr", + "is_dir": True, + "timestamp": "2025-01-15T10:30:00+00:00", + "item_count": 42, + } + obj = ObjectRef.from_json(data) + assert obj.path == "schema/Table/objects/id=1/data_abc123.zarr" + assert obj.size == 5678 + assert obj.is_dir is True + assert obj.item_count == 42 + + def test_from_json_zarr_style(self): + """Test creating ObjectRef from Zarr-style JSON with null size.""" + data = { + "path": "schema/Recording/objects/id=1/neural_data_abc123.zarr", + "size": None, + "hash": None, + "ext": ".zarr", + "is_dir": True, + "timestamp": "2025-01-15T10:30:00+00:00", + } + obj = ObjectRef.from_json(data) + assert obj.path == "schema/Recording/objects/id=1/neural_data_abc123.zarr" + assert obj.size is None + assert obj.hash is None + assert obj.ext == ".zarr" + assert obj.is_dir is True + assert obj.item_count is None + + def test_to_json(self): + """Test converting ObjectRef to JSON dict.""" + from datetime import datetime, timezone + + obj = ObjectRef( + path="schema/Table/objects/id=1/data.dat", + size=1024, + hash=None, + ext=".dat", + is_dir=False, + timestamp=datetime(2025, 1, 15, 10, 30, tzinfo=timezone.utc), + ) + data = obj.to_json() + assert data["path"] == "schema/Table/objects/id=1/data.dat" + assert data["size"] == 1024 + assert data["is_dir"] is False + + def test_repr_file(self): + """Test string representation for file.""" + from datetime import datetime, timezone + + obj = ObjectRef( + path="test/path.dat", + size=1024, + hash=None, + ext=".dat", + is_dir=False, + timestamp=datetime.now(timezone.utc), + ) + assert "file" in repr(obj) + assert "test/path.dat" in repr(obj) + + def test_repr_folder(self): + """Test string representation for folder.""" + from datetime import datetime, timezone + + obj = ObjectRef( + path="test/folder.zarr", + size=5678, + hash=None, + ext=".zarr", + is_dir=True, + timestamp=datetime.now(timezone.utc), + ) + assert "folder" in repr(obj) + + def test_str(self): + """Test str() returns path.""" + from datetime import datetime, timezone + + obj = ObjectRef( + path="my/path/to/data.dat", + size=100, + hash=None, + ext=".dat", + is_dir=False, + timestamp=datetime.now(timezone.utc), + ) + assert str(obj) == "my/path/to/data.dat" + + +class TestObjectInsertFile: + """Tests for inserting files with object type.""" + + def test_insert_file(self, schema_obj, mock_object_storage, tmpdir_factory): + """Test inserting a file.""" + table = ObjectFile() + + # Create a test file + source_folder = tmpdir_factory.mktemp("source") + test_file = Path(source_folder, "test_data.dat") + data = os.urandom(1024) + with test_file.open("wb") as f: + f.write(data) + + # Insert the file + table.insert1({"file_id": 1, "data_file": str(test_file)}) + + # Verify record was inserted + assert len(table) == 1 + + # Cleanup + table.delete() + + def test_insert_file_with_extension(self, schema_obj, mock_object_storage, tmpdir_factory): + """Test that file extension is preserved.""" + table = ObjectFile() + + source_folder = tmpdir_factory.mktemp("source") + test_file = Path(source_folder, "data.csv") + test_file.write_text("a,b,c\n1,2,3\n") + + table.insert1({"file_id": 2, "data_file": str(test_file)}) + + # Fetch and check extension in metadata + record = table.fetch1() + obj = record["data_file"] + assert obj.ext == ".csv" + + table.delete() + + def test_insert_file_nonexistent(self, schema_obj, mock_object_storage): + """Test that inserting nonexistent file raises error.""" + table = ObjectFile() + + with pytest.raises(dj.DataJointError, match="not found"): + table.insert1({"file_id": 3, "data_file": "/nonexistent/path/file.dat"}) + + +class TestObjectInsertFolder: + """Tests for inserting folders with object type.""" + + def test_insert_folder(self, schema_obj, mock_object_storage, tmpdir_factory): + """Test inserting a folder.""" + table = ObjectFolder() + + # Create a test folder with files + source_folder = tmpdir_factory.mktemp("source") + data_folder = Path(source_folder, "data_folder") + data_folder.mkdir() + + # Add some files + (data_folder / "file1.txt").write_text("content1") + (data_folder / "file2.txt").write_text("content2") + subdir = data_folder / "subdir" + subdir.mkdir() + (subdir / "file3.txt").write_text("content3") + + # Insert the folder + table.insert1({"folder_id": 1, "data_folder": str(data_folder)}) + + assert len(table) == 1 + + # Fetch and verify + record = table.fetch1() + obj = record["data_folder"] + assert obj.is_dir is True + assert obj.item_count == 3 # 3 files + + table.delete() + + +class TestObjectInsertStream: + """Tests for inserting from streams with object type.""" + + def test_insert_stream(self, schema_obj, mock_object_storage): + """Test inserting from a stream.""" + table = ObjectFile() + + # Create a BytesIO stream + data = b"This is test data from a stream" + stream = io.BytesIO(data) + + # Insert with extension and stream tuple + table.insert1({"file_id": 10, "data_file": (".txt", stream)}) + + assert len(table) == 1 + + # Fetch and verify extension + record = table.fetch1() + obj = record["data_file"] + assert obj.ext == ".txt" + assert obj.size == len(data) + + table.delete() + + +class TestObjectFetch: + """Tests for fetching object type attributes.""" + + def test_fetch_returns_objectref(self, schema_obj, mock_object_storage, tmpdir_factory): + """Test that fetch returns ObjectRef.""" + table = ObjectFile() + + source_folder = tmpdir_factory.mktemp("source") + test_file = Path(source_folder, "test.dat") + test_file.write_bytes(os.urandom(512)) + + table.insert1({"file_id": 20, "data_file": str(test_file)}) + + record = table.fetch1() + obj = record["data_file"] + + assert isinstance(obj, ObjectRef) + assert obj.size == 512 + assert obj.is_dir is False + + table.delete() + + def test_fetch_metadata_no_io(self, schema_obj, mock_object_storage, tmpdir_factory): + """Test that accessing metadata does not perform I/O.""" + table = ObjectFile() + + source_folder = tmpdir_factory.mktemp("source") + test_file = Path(source_folder, "test.dat") + test_file.write_bytes(os.urandom(256)) + + table.insert1({"file_id": 21, "data_file": str(test_file)}) + + record = table.fetch1() + obj = record["data_file"] + + # These should all work without I/O + assert obj.path is not None + assert obj.size == 256 + assert obj.ext == ".dat" + assert obj.is_dir is False + assert obj.timestamp is not None + + table.delete() + + +class TestObjectRefOperations: + """Tests for ObjectRef file operations.""" + + def test_read_file(self, schema_obj, mock_object_storage, tmpdir_factory): + """Test reading file content via ObjectRef.""" + table = ObjectFile() + + source_folder = tmpdir_factory.mktemp("source") + test_file = Path(source_folder, "readable.dat") + original_data = os.urandom(128) + test_file.write_bytes(original_data) + + table.insert1({"file_id": 30, "data_file": str(test_file)}) + + record = table.fetch1() + obj = record["data_file"] + + # Read content + content = obj.read() + assert content == original_data + + table.delete() + + def test_open_file(self, schema_obj, mock_object_storage, tmpdir_factory): + """Test opening file via ObjectRef.""" + table = ObjectFile() + + source_folder = tmpdir_factory.mktemp("source") + test_file = Path(source_folder, "openable.txt") + test_file.write_text("Hello, World!") + + table.insert1({"file_id": 31, "data_file": str(test_file)}) + + record = table.fetch1() + obj = record["data_file"] + + # Open and read + with obj.open(mode="rb") as f: + content = f.read() + assert content == b"Hello, World!" + + table.delete() + + def test_download_file(self, schema_obj, mock_object_storage, tmpdir_factory): + """Test downloading file via ObjectRef.""" + table = ObjectFile() + + source_folder = tmpdir_factory.mktemp("source") + test_file = Path(source_folder, "downloadable.dat") + original_data = os.urandom(256) + test_file.write_bytes(original_data) + + table.insert1({"file_id": 32, "data_file": str(test_file)}) + + record = table.fetch1() + obj = record["data_file"] + + # Download to new location + download_folder = tmpdir_factory.mktemp("download") + local_path = obj.download(download_folder) + + assert Path(local_path).exists() + assert Path(local_path).read_bytes() == original_data + + table.delete() + + def test_exists(self, schema_obj, mock_object_storage, tmpdir_factory): + """Test exists() method.""" + table = ObjectFile() + + source_folder = tmpdir_factory.mktemp("source") + test_file = Path(source_folder, "exists.dat") + test_file.write_bytes(b"data") + + table.insert1({"file_id": 33, "data_file": str(test_file)}) + + record = table.fetch1() + obj = record["data_file"] + + assert obj.exists() is True + + table.delete() + + +class TestObjectRefFolderOperations: + """Tests for ObjectRef folder operations.""" + + def test_listdir(self, schema_obj, mock_object_storage, tmpdir_factory): + """Test listing folder contents.""" + table = ObjectFolder() + + source_folder = tmpdir_factory.mktemp("source") + data_folder = Path(source_folder, "listable") + data_folder.mkdir() + (data_folder / "a.txt").write_text("a") + (data_folder / "b.txt").write_text("b") + (data_folder / "c.txt").write_text("c") + + table.insert1({"folder_id": 40, "data_folder": str(data_folder)}) + + record = table.fetch1() + obj = record["data_folder"] + + contents = obj.listdir() + assert len(contents) == 3 + assert "a.txt" in contents + assert "b.txt" in contents + assert "c.txt" in contents + + table.delete() + + def test_walk(self, schema_obj, mock_object_storage, tmpdir_factory): + """Test walking folder tree.""" + table = ObjectFolder() + + source_folder = tmpdir_factory.mktemp("source") + data_folder = Path(source_folder, "walkable") + data_folder.mkdir() + (data_folder / "root.txt").write_text("root") + subdir = data_folder / "subdir" + subdir.mkdir() + (subdir / "nested.txt").write_text("nested") + + table.insert1({"folder_id": 41, "data_folder": str(data_folder)}) + + record = table.fetch1() + obj = record["data_folder"] + + # Collect walk results + walk_results = list(obj.walk()) + assert len(walk_results) >= 1 + + table.delete() + + def test_open_subpath(self, schema_obj, mock_object_storage, tmpdir_factory): + """Test opening file within folder using subpath.""" + table = ObjectFolder() + + source_folder = tmpdir_factory.mktemp("source") + data_folder = Path(source_folder, "subpathable") + data_folder.mkdir() + (data_folder / "inner.txt").write_text("inner content") + + table.insert1({"folder_id": 42, "data_folder": str(data_folder)}) + + record = table.fetch1() + obj = record["data_folder"] + + with obj.open("inner.txt", mode="rb") as f: + content = f.read() + assert content == b"inner content" + + table.delete() + + def test_read_on_folder_raises(self, schema_obj, mock_object_storage, tmpdir_factory): + """Test that read() on folder raises error.""" + table = ObjectFolder() + + source_folder = tmpdir_factory.mktemp("source") + data_folder = Path(source_folder, "folder") + data_folder.mkdir() + (data_folder / "file.txt").write_text("content") + + table.insert1({"folder_id": 43, "data_folder": str(data_folder)}) + + record = table.fetch1() + obj = record["data_folder"] + + with pytest.raises(dj.DataJointError, match="Cannot read"): + obj.read() + + table.delete() + + def test_listdir_on_file_raises(self, schema_obj, mock_object_storage, tmpdir_factory): + """Test that listdir() on file raises error.""" + table = ObjectFile() + + source_folder = tmpdir_factory.mktemp("source") + test_file = Path(source_folder, "file.dat") + test_file.write_bytes(b"data") + + table.insert1({"file_id": 44, "data_file": str(test_file)}) + + record = table.fetch1() + obj = record["data_file"] + + with pytest.raises(dj.DataJointError, match="Cannot listdir"): + obj.listdir() + + table.delete() + + +class TestObjectMultiple: + """Tests for tables with multiple object attributes.""" + + def test_multiple_objects(self, schema_obj, mock_object_storage, tmpdir_factory): + """Test inserting multiple object attributes.""" + table = ObjectMultiple() + + source_folder = tmpdir_factory.mktemp("source") + raw_file = Path(source_folder, "raw.dat") + raw_file.write_bytes(os.urandom(100)) + processed_file = Path(source_folder, "processed.dat") + processed_file.write_bytes(os.urandom(200)) + + table.insert1( + { + "record_id": 1, + "raw_data": str(raw_file), + "processed": str(processed_file), + } + ) + + record = table.fetch1() + raw_obj = record["raw_data"] + processed_obj = record["processed"] + + assert raw_obj.size == 100 + assert processed_obj.size == 200 + assert raw_obj.path != processed_obj.path + + table.delete() + + +class TestObjectWithOtherAttributes: + """Tests for object type mixed with other attributes.""" + + def test_object_with_other(self, schema_obj, mock_object_storage, tmpdir_factory): + """Test table with object and other attribute types.""" + table = ObjectWithOther() + + source_folder = tmpdir_factory.mktemp("source") + test_file = Path(source_folder, "data.bin") + test_file.write_bytes(os.urandom(64)) + + table.insert1( + { + "subject_id": 1, + "session_id": 1, + "name": "Test Session", + "data_file": str(test_file), + "notes": "Some notes here", + } + ) + + record = table.fetch1() + assert record["name"] == "Test Session" + assert record["notes"] == "Some notes here" + assert isinstance(record["data_file"], ObjectRef) + assert record["data_file"].size == 64 + + table.delete() + + +class TestObjectVerify: + """Tests for ObjectRef verification.""" + + def test_verify_file(self, schema_obj, mock_object_storage, tmpdir_factory): + """Test verifying file integrity.""" + table = ObjectFile() + + source_folder = tmpdir_factory.mktemp("source") + test_file = Path(source_folder, "verifiable.dat") + test_file.write_bytes(os.urandom(128)) + + table.insert1({"file_id": 50, "data_file": str(test_file)}) + + record = table.fetch1() + obj = record["data_file"] + + # Should not raise + assert obj.verify() is True + + table.delete() + + +class TestStagedInsert: + """Tests for staged insert operations.""" + + def test_staged_insert_basic(self, schema_obj, mock_object_storage): + """Test basic staged insert.""" + table = ObjectFile() + + with table.staged_insert1 as staged: + staged.rec["file_id"] = 60 + + # Write directly to storage + with staged.open("data_file", ".dat") as f: + f.write(b"staged data content") + + # No need to assign - metadata computed on exit + + # Verify record was inserted + assert len(table) == 1 + record = table.fetch1() + obj = record["data_file"] + assert obj.ext == ".dat" + + table.delete() + + def test_staged_insert_exception_cleanup(self, schema_obj, mock_object_storage): + """Test that staged insert cleans up on exception.""" + table = ObjectFile() + + try: + with table.staged_insert1 as staged: + staged.rec["file_id"] = 61 + + with staged.open("data_file", ".dat") as f: + f.write(b"will be cleaned up") + + raise ValueError("Simulated error") + except ValueError: + pass + + # No record should be inserted + assert len(table) == 0 + + def test_staged_insert_store_method(self, schema_obj, mock_object_storage): + """Test staged insert store() method returns FSMap.""" + import fsspec + + table = ObjectFile() + + with table.staged_insert1 as staged: + staged.rec["file_id"] = 62 + + store = staged.store("data_file", ".zarr") + assert isinstance(store, fsspec.FSMap) + + # Write some data + store["test_key"] = b"test_value" + + assert len(table) == 1 + + table.delete() + + def test_staged_insert_fs_property(self, schema_obj, mock_object_storage): + """Test staged insert fs property returns filesystem.""" + import fsspec + + table = ObjectFile() + + with table.staged_insert1 as staged: + staged.rec["file_id"] = 63 + + fs = staged.fs + assert isinstance(fs, fsspec.AbstractFileSystem) + + # Just open and write to test fs works + with staged.open("data_file", ".txt") as f: + f.write(b"test") + + table.delete() + + def test_staged_insert_missing_pk_raises(self, schema_obj, mock_object_storage): + """Test that staged insert raises if PK not set before store().""" + table = ObjectFile() + + with pytest.raises(dj.DataJointError, match="Primary key"): + with table.staged_insert1 as staged: + # Don't set primary key + staged.store("data_file", ".dat") diff --git a/tests/test_privileges.py b/tests/integration/test_privileges.py similarity index 69% rename from tests/test_privileges.py rename to tests/integration/test_privileges.py index 2bf67a386..0939823a0 100644 --- a/tests/test_privileges.py +++ b/tests/integration/test_privileges.py @@ -1,10 +1,8 @@ -import os - import pytest import datajoint as dj -from . import schema, schema_privileges +from tests import schema, schema_privileges namespace = locals() @@ -79,42 +77,36 @@ class TestUnprivileged: def test_fail_create_schema(self, connection_djview): """creating a schema with no CREATE privilege""" with pytest.raises(dj.DataJointError): - return dj.Schema( - "forbidden_schema", namespace, connection=connection_djview - ) + return dj.Schema("forbidden_schema", namespace, connection=connection_djview) def test_insert_failure(self, connection_djview, schema_any): - unprivileged = dj.Schema( - schema_any.database, namespace, connection=connection_djview - ) + unprivileged = dj.Schema(schema_any.database, namespace, connection=connection_djview) unprivileged.spawn_missing_classes() - assert issubclass(Language, dj.Lookup) and len(Language()) == len( + UnprivilegedLanguage = namespace["Language"] + assert issubclass(UnprivilegedLanguage, dj.Lookup) and len(UnprivilegedLanguage()) == len( schema.Language() ), "failed to spawn missing classes" with pytest.raises(dj.DataJointError): - Language().insert1(("Socrates", "Greek")) + UnprivilegedLanguage().insert1(("Socrates", "Greek")) def test_failure_to_create_table(self, connection_djview, schema_any): - unprivileged = dj.Schema( - schema_any.database, namespace, connection=connection_djview - ) - - @unprivileged - class Try(dj.Manual): - definition = """ # should not matter really - id : int - --- - value : float - """ + """Table declaration should raise AccessError when user lacks CREATE privilege.""" + unprivileged = dj.Schema(schema_any.database, namespace, connection=connection_djview) - with pytest.raises(dj.DataJointError): - Try().insert1((1, 1.5)) + # Should raise AccessError at declaration time, not silently fail + with pytest.raises(dj.errors.AccessError): + + @unprivileged + class Try(dj.Manual): + definition = """ # should not matter really + id : int + --- + value : float + """ class TestSubset: def test_populate_activate(self, connection_djsubset, schema_priv, prefix): - schema_priv.activate( - f"{prefix}_schema_privileges", create_schema=True, create_tables=False - ) + schema_priv.activate(f"{prefix}_schema_privileges", create_schema=True, create_tables=False) schema_privileges.Child.populate() assert schema_privileges.Child.progress(display=False)[0] == 0 diff --git a/tests/test_reconnection.py b/tests/integration/test_reconnection.py similarity index 100% rename from tests/test_reconnection.py rename to tests/integration/test_reconnection.py diff --git a/tests/test_relation.py b/tests/integration/test_relation.py similarity index 84% rename from tests/test_relation.py rename to tests/integration/test_relation.py index 565e1eafa..ea7d79d54 100644 --- a/tests/test_relation.py +++ b/tests/integration/test_relation.py @@ -9,7 +9,7 @@ import datajoint as dj from datajoint.table import Table -from . import schema +from tests import schema def test_contents(user, subject): @@ -19,13 +19,13 @@ def test_contents(user, subject): # test contents assert user assert len(user) == len(user.contents) - u = user.fetch(order_by=["username"]) + u = user.to_arrays(order_by=["username"]) assert list(u["username"]) == sorted([s[0] for s in user.contents]) # test prepare assert subject assert len(subject) == len(subject.contents) - u = subject.fetch(order_by=["subject_id"]) + u = subject.to_arrays(order_by=["subject_id"]) assert list(u["subject_id"]) == sorted([s[0] for s in subject.contents]) @@ -80,7 +80,7 @@ def test_wrong_insert_type(user): user.insert1(3) -def test_insert_select(subject, test, test2): +def test_insert_select(clean_test_tables, subject, test, test2): test2.delete() test2.insert(test) assert len(test2) == len(test) @@ -98,19 +98,19 @@ def test_insert_select(subject, test, test2): assert len(subject) == 2 * original_length -def test_insert_pandas_roundtrip(test, test2): +def test_insert_pandas_roundtrip(clean_test_tables, test, test2): """ensure fetched frames can be inserted""" test2.delete() n = len(test) assert n > 0 - df = test.fetch(format="frame") + df = test.to_pandas() assert isinstance(df, pandas.DataFrame) assert len(df) == n test2.insert(df) assert len(test2) == n -def test_insert_pandas_userframe(test, test2): +def test_insert_pandas_userframe(clean_test_tables, test, test2): """ ensure simple user-created frames (1 field, non-custom index) can be inserted without extra index adjustment @@ -118,50 +118,50 @@ def test_insert_pandas_userframe(test, test2): test2.delete() n = len(test) assert n > 0 - df = pandas.DataFrame(test.fetch()) + df = pandas.DataFrame(test.to_arrays()) assert isinstance(df, pandas.DataFrame) assert len(df) == n test2.insert(df) assert len(test2) == n -def test_insert_select_ignore_extra_fields0(test, test_extra): +def test_insert_select_ignore_extra_fields0(clean_test_tables, test, test_extra): """need ignore extra fields for insert select""" - test_extra.insert1((test.fetch("key").max() + 1, 0, 0)) + test_extra.insert1((test.to_arrays("key").max() + 1, 0, 0)) with pytest.raises(dj.DataJointError): test.insert(test_extra) -def test_insert_select_ignore_extra_fields1(test, test_extra): +def test_insert_select_ignore_extra_fields1(clean_test_tables, test, test_extra): """make sure extra fields works in insert select""" test_extra.delete() - keyno = test.fetch("key").max() + 1 + keyno = test.to_arrays("key").max() + 1 test_extra.insert1((keyno, 0, 0)) test.insert(test_extra, ignore_extra_fields=True) - assert keyno in test.fetch("key") + assert keyno in test.to_arrays("key") -def test_insert_select_ignore_extra_fields2(test_no_extra, test): +def test_insert_select_ignore_extra_fields2(clean_test_tables, test_no_extra, test): """make sure insert select still works when ignoring extra fields when there are none""" test_no_extra.delete() test_no_extra.insert(test, ignore_extra_fields=True) -def test_insert_select_ignore_extra_fields3(test, test_no_extra, test_extra): +def test_insert_select_ignore_extra_fields3(clean_test_tables, test, test_no_extra, test_extra): """make sure insert select works for from query result""" # Recreate table state from previous tests - keyno = test.fetch("key").max() + 1 + keyno = test.to_arrays("key").max() + 1 test_extra.insert1((keyno, 0, 0)) test.insert(test_extra, ignore_extra_fields=True) - assert len(test_extra.fetch("key")), "test_extra is empty" + assert len(test_extra.to_arrays("key")), "test_extra is empty" test_no_extra.delete() - assert len(test_extra.fetch("key")), "test_extra is empty" - keystr = str(test_extra.fetch("key").max()) + assert len(test_extra.to_arrays("key")), "test_extra is empty" + keystr = str(test_extra.to_arrays("key").max()) test_no_extra.insert((test_extra & "`key`=" + keystr), ignore_extra_fields=True) -def test_skip_duplicates(test_no_extra, test): +def test_skip_duplicates(clean_test_tables, test_no_extra, test): """test that skip_duplicates works when inserting from another table""" test_no_extra.delete() test_no_extra.insert(test, ignore_extra_fields=True, skip_duplicates=True) @@ -182,9 +182,7 @@ def test_replace(subject): skip_duplicates=True, ) assert date != str((subject & key).fetch1("date_of_birth")), "inappropriate replace" - subject.insert1( - dict(key, real_id=7, date_of_birth=date, subject_notes=""), replace=True - ) + subject.insert1(dict(key, real_id=7, date_of_birth=date, subject_notes=""), replace=True) assert date == str((subject & key).fetch1("date_of_birth")), "replace failed" @@ -248,7 +246,7 @@ def test_blob_insert(img): """Tests inserting and retrieving blobs.""" X = np.random.randn(20, 10) img.insert1((1, X)) - Y = img.fetch()[0]["img"] + Y = img.to_arrays()[0]["img"] assert np.all(X == Y), "Inserted and retrieved image are not identical" @@ -258,7 +256,7 @@ def test_drop(trash): with patch.object(dj.utils, "input", create=True, return_value="yes"): trash.drop() try: - trash.fetch() + trash.to_arrays() raise Exception("Fetched after table dropped.") except dj.DataJointError: pass @@ -277,9 +275,7 @@ def relation_selector(attr): tiers = [dj.Imported, dj.Manual, dj.Lookup, dj.Computed] for name, rel in getmembers(schema, relation_selector): - assert re.match( - rel.tier_regexp, rel.table_name - ), "Regular expression does not match for {name}".format(name=name) + assert re.match(rel.tier_regexp, rel.table_name), "Regular expression does not match for {name}".format(name=name) for tier in tiers: assert issubclass(rel, tier) or not re.match( tier.tier_regexp, rel.table_name diff --git a/tests/integration/test_relation_u.py b/tests/integration/test_relation_u.py new file mode 100644 index 000000000..f3f7a6cbb --- /dev/null +++ b/tests/integration/test_relation_u.py @@ -0,0 +1,81 @@ +from pytest import raises + +import datajoint as dj + +from tests.schema import Language, TTest +from tests.schema_simple import ArgmaxTest + + +def test_restriction(lang, languages, trial): + """Test dj.U restriction semantics.""" + language_set = {s[1] for s in languages} + rel = dj.U("language") & lang + assert list(rel.heading.names) == ["language"] + assert len(rel) == len(language_set) + assert set(rel.to_arrays("language")) == language_set + # dj.U & table promotes attributes to PK + assert list((dj.U("start_time") & trial).primary_key) == ["start_time"] + + +def test_invalid_restriction(schema_any): + with raises(dj.DataJointError): + dj.U("color") & dict(color="red") + + +def test_ineffective_restriction(lang): + rel = lang & dj.U("language") + assert rel.make_sql() == lang.make_sql() + + +def test_join_with_u_removed(experiment): + """Test that table * dj.U(...) raises an error (removed in 2.0).""" + with raises(dj.DataJointError): + experiment * dj.U("experiment_date") + + with raises(dj.DataJointError): + dj.U("experiment_date") * experiment + + +def test_invalid_join(schema_any): + """Test that dj.U * non-QueryExpression raises an error.""" + with raises(dj.DataJointError): + dj.U("language") * dict(language="English") + + +def test_repr_without_attrs(schema_any): + """test dj.U() display""" + query = dj.U().aggr(Language, n="count(*)") + repr(query) + + +def test_aggregations(schema_any): + lang = Language() + # test total aggregation on expression object + n1 = dj.U().aggr(lang, n="count(*)").fetch1("n") + assert n1 == len(lang.to_arrays()) + # test total aggregation on expression class + n2 = dj.U().aggr(Language, n="count(*)").fetch1("n") + assert n1 == n2 + rel = dj.U("language").aggr(Language, number_of_speakers="count(*)") + assert len(rel) == len(set(lang[1] for lang in Language.contents)) + assert (rel & 'language="English"').fetch1("number_of_speakers") == 3 + + +def test_argmax(schema_any): + """Test argmax pattern using aggregation and restriction.""" + rel = TTest() + # Get the maximum value using aggregation + max_val = dj.U().aggr(rel, mx="max(value)").fetch1("mx") + # Get tuples with that value + mx = rel & f"value={max_val}" + assert mx.to_arrays("value")[0] == max(rel.to_arrays("value")) + + +def test_aggr(schema_any, schema_simp): + """Test aggregation with dj.U - the old * pattern is removed.""" + rel = ArgmaxTest() + # The old pattern using dj.U("val") * rel is no longer supported + # Use aggregation directly instead + agg = dj.U("secondary_key").aggr(rel, min_val="min(val)") + # Verify aggregation works + assert len(agg) > 0 diff --git a/tests/test_relational_operand.py b/tests/integration/test_relational_operand.py similarity index 61% rename from tests/test_relational_operand.py rename to tests/integration/test_relational_operand.py index 2dbea672e..eea53288e 100644 --- a/tests/test_relational_operand.py +++ b/tests/integration/test_relational_operand.py @@ -9,8 +9,24 @@ import datajoint as dj from datajoint.errors import DataJointError -from .schema import * -from .schema_simple import * +from tests.schema import Child, Ephys, Experiment, Parent, SessionA, SessionDateA, SessionStatusA, SubjectA, TTest3, Trial +from tests.schema_simple import ( + F, + IJ, + JI, + L, + A, + B, + D, + E, + DataA, + DataB, + KeyPK, + OutfitLaunch, + ReservedWord, + SelectPK, + TTestUpdate, +) @pytest.fixture @@ -61,17 +77,13 @@ def test_rename(schema_simp_pop): # test renaming x = B().proj(i="id_a") & "i in (1,2,3,4)" lenx = len(x) - assert len(x) == len( - B() & "id_a in (1,2,3,4)" - ), "incorrect restriction of renamed attributes" + assert len(x) == len(B() & "id_a in (1,2,3,4)"), "incorrect restriction of renamed attributes" assert len(x & "id_b in (1,2)") == len( B() & "id_b in (1,2) and id_a in (1,2,3,4)" ), "incorrect restriction of renamed restriction" assert len(x) == lenx, "restriction modified original" y = x.proj(j="i") - assert len(y) == len( - B() & "id_a in (1,2,3,4)" - ), "incorrect projection of restriction" + assert len(y) == len(B() & "id_a in (1,2,3,4)"), "incorrect projection of restriction" z = y & "j in (3, 4, 5, 6)" assert len(z) == len(B() & "id_a in (3,4)"), "incorrect nested subqueries" @@ -92,24 +104,16 @@ def test_join(schema_simp_pop): y = L() rel = x * y assert len(rel) == len(x) * len(y), "incorrect join" - assert set(x.heading.names).union(y.heading.names) == set( - rel.heading.names - ), "incorrect join heading" - assert set(x.primary_key).union(y.primary_key) == set( - rel.primary_key - ), "incorrect join primary_key" + assert set(x.heading.names).union(y.heading.names) == set(rel.heading.names), "incorrect join heading" + assert set(x.primary_key).union(y.primary_key) == set(rel.primary_key), "incorrect join primary_key" # Test cartesian product of restricted relations x = A() & "cond_in_a=1" y = L() & "cond_in_l=1" rel = x * y assert len(rel) == len(x) * len(y), "incorrect join" - assert set(x.heading.names).union(y.heading.names) == set( - rel.heading.names - ), "incorrect join heading" - assert set(x.primary_key).union(y.primary_key) == set( - rel.primary_key - ), "incorrect join primary_key" + assert set(x.heading.names).union(y.heading.names) == set(rel.heading.names), "incorrect join heading" + assert set(x.primary_key).union(y.primary_key) == set(rel.primary_key), "incorrect join primary_key" # Test join with common attributes cond = A() & "cond_in_a=1" @@ -118,36 +122,22 @@ def test_join(schema_simp_pop): rel = x * y assert len(rel) >= len(x) and len(rel) >= len(y), "incorrect join" assert not rel - cond, "incorrect join, restriction, or antijoin" - assert set(x.heading.names).union(y.heading.names) == set( - rel.heading.names - ), "incorrect join heading" - assert set(x.primary_key).union(y.primary_key) == set( - rel.primary_key - ), "incorrect join primary_key" + assert set(x.heading.names).union(y.heading.names) == set(rel.heading.names), "incorrect join heading" + assert set(x.primary_key).union(y.primary_key) == set(rel.primary_key), "incorrect join primary_key" # test renamed join - x = B().proj( - i="id_a" - ) # rename the common attribute to achieve full cartesian product + x = B().proj(i="id_a") # rename the common attribute to achieve full cartesian product y = D() rel = x * y assert len(rel) == len(x) * len(y), "incorrect join" - assert set(x.heading.names).union(y.heading.names) == set( - rel.heading.names - ), "incorrect join heading" - assert set(x.primary_key).union(y.primary_key) == set( - rel.primary_key - ), "incorrect join primary_key" + assert set(x.heading.names).union(y.heading.names) == set(rel.heading.names), "incorrect join heading" + assert set(x.primary_key).union(y.primary_key) == set(rel.primary_key), "incorrect join primary_key" x = B().proj(a="id_a") y = D() rel = x * y assert len(rel) == len(x) * len(y), "incorrect join" - assert set(x.heading.names).union(y.heading.names) == set( - rel.heading.names - ), "incorrect join heading" - assert set(x.primary_key).union(y.primary_key) == set( - rel.primary_key - ), "incorrect join primary_key" + assert set(x.heading.names).union(y.heading.names) == set(rel.heading.names), "incorrect join heading" + assert set(x.primary_key).union(y.primary_key) == set(rel.primary_key), "incorrect join primary_key" # test pairing # Approach 1: join then restrict @@ -175,7 +165,7 @@ def test_issue_376(schema_any_pop): def test_issue_463(schema_simp_pop): - assert ((A & B) * B).fetch().size == len(A * B) + assert ((A & B) * B).to_arrays().size == len(A * B) def test_project(schema_simp_pop): @@ -187,53 +177,45 @@ def test_project(schema_simp_pop): # projection after restriction cond = L() & "cond_in_l" assert len(D() & cond) + len(D() - cond) == len(D()), "failed semijoin or antijoin" - assert len((D() & cond).proj()) == len((D() & cond)), ( - "projection failed: altered its argument" "s cardinality" - ) + assert len((D() & cond).proj()) == len((D() & cond)), "projection failed: altered its arguments cardinality" -def test_rename_non_dj_attribute( - connection_test, schema_simp_pop, schema_any_pop, prefix -): +def test_rename_non_dj_attribute(connection_test, schema_simp_pop, schema_any_pop, prefix): schema = prefix + "_test1" - connection_test.query( - f"CREATE TABLE {schema}.test_table (oldID int PRIMARY KEY)" - ).fetchall() - mySchema = dj.VirtualModule(schema, schema) + connection_test.query(f"CREATE TABLE {schema}.test_table (oldID int PRIMARY KEY)").fetchall() + mySchema = dj.VirtualModule(schema, schema, connection=connection_test) assert ( - "oldID" - not in mySchema.TestTable.proj(new_name="oldID").heading.attributes.keys() + "oldID" not in mySchema.TestTable.proj(new_name="oldID").heading.attributes.keys() ), "Failed to rename attribute correctly" connection_test.query(f"DROP TABLE {schema}.test_table") def test_union(schema_simp_pop): - x = set(zip(*IJ.fetch("i", "j"))) - y = set(zip(*JI.fetch("i", "j"))) - assert ( - len(x) > 0 and len(y) > 0 and len(IJ() * JI()) < len(x) - ) # ensure the IJ and JI are non-trivial - z = set(zip(*(IJ + JI).fetch("i", "j"))) # union + x = set(zip(*IJ.to_arrays("i", "j"))) + y = set(zip(*JI.to_arrays("i", "j"))) + # IJ and JI have attributes i,j from different origins, so use semantic_check=False + assert len(x) > 0 and len(y) > 0 and len(IJ().join(JI(), semantic_check=False)) < len(x) + z = set(zip(*(IJ + JI).to_arrays("i", "j"))) # union assert x.union(y) == z assert len(IJ + JI) == len(z) -def test_outer_union_fail(schema_simp_pop): +def test_outer_union_fail_1(schema_simp_pop): """Union of two tables with different primary keys raises an error.""" with pytest.raises(dj.DataJointError): A() + B() -def test_outer_union_fail(schema_any_pop): +def test_outer_union_fail_2(schema_any_pop): """Union of two tables with different primary keys raises an error.""" t = Trial + Ephys - t.fetch() + t.to_arrays() assert set(t.heading.names) == set(Trial.heading.names) | set(Ephys.heading.names) len(t) def test_preview(schema_simp_pop): - with dj.config(display__limit=7): + with dj.config.override(display__limit=7): x = A().proj(a="id_a") s = x.preview() assert len(s.split("\n")) == len(x) + 2 @@ -242,25 +224,21 @@ def test_preview(schema_simp_pop): def test_heading_repr(schema_simp_pop): x = A * D s = repr(x.heading) - assert len( - list( - 1 - for g in s.split("\n") - if g.strip() and not g.strip().startswith(("-", "#")) - ) - ) == len(x.heading.attributes) + assert len(list(1 for g in s.split("\n") if g.strip() and not g.strip().startswith(("-", "#")))) == len( + x.heading.attributes + ) def test_aggregate(schema_simp_pop): - x = B().aggregate(B.C()) + # With exclude_nonmatching=True, only rows with matches are kept (INNER JOIN) + x = B().aggregate(B.C(), exclude_nonmatching=True) assert len(x) == len(B() & B.C()) - x = B().aggregate(B.C(), keep_all_rows=True) + # Default behavior now keeps all rows (LEFT JOIN) + x = B().aggregate(B.C()) assert len(x) == len(B()) # test LEFT join - assert len((x & "id_b=0").fetch()) == len( - B() & "id_b=0" - ) # test restricted aggregation + assert len((x & "id_b=0").to_arrays()) == len(B() & "id_b=0") # test restricted aggregation x = B().aggregate( B.C(), @@ -268,37 +246,32 @@ def test_aggregate(schema_simp_pop): count="count(id_c)", mean="avg(value)", max="max(value)", - keep_all_rows=True, ) assert len(x) == len(B()) y = x & "mean>0" # restricted aggregation assert len(y) > 0 - assert all(y.fetch("mean") > 0) - for n, count, mean, max_, key in zip(*x.fetch("n", "count", "mean", "max", dj.key)): + assert all(y.to_arrays("mean") > 0) + for n, count, mean, max_, key in zip(*x.to_arrays("n", "count", "mean", "max"), x.keys()): assert n == count, "aggregation failed (count)" - values = (B.C() & key).fetch("value") + values = (B.C() & key).to_arrays("value") assert bool(len(values)) == bool(n), "aggregation failed (restriction)" if n: - assert np.isclose( - mean, values.mean(), rtol=1e-4, atol=1e-5 - ), "aggregation failed (mean)" - assert np.isclose( - max_, values.max(), rtol=1e-4, atol=1e-5 - ), "aggregation failed (max)" + assert np.isclose(mean, values.mean(), rtol=1e-4, atol=1e-5), "aggregation failed (mean)" + assert np.isclose(max_, values.max(), rtol=1e-4, atol=1e-5), "aggregation failed (max)" def test_aggr(schema_simp_pop): - x = B.aggr(B.C) + # With exclude_nonmatching=True, only rows with matches are kept (INNER JOIN) + x = B.aggr(B.C, exclude_nonmatching=True) l1 = len(x) l2 = len(B & B.C) assert l1 == l2 - x = B().aggr(B.C(), keep_all_rows=True) + # Default behavior now keeps all rows (LEFT JOIN) + x = B().aggr(B.C()) assert len(x) == len(B()) # test LEFT join - assert len((x & "id_b=0").fetch()) == len( - B() & "id_b=0" - ) # test restricted aggregation + assert len((x & "id_b=0").to_arrays()) == len(B() & "id_b=0") # test restricted aggregation x = B().aggr( B.C(), @@ -306,23 +279,18 @@ def test_aggr(schema_simp_pop): count="count(id_c)", mean="avg(value)", max="max(value)", - keep_all_rows=True, ) assert len(x) == len(B()) y = x & "mean>0" # restricted aggregation assert len(y) > 0 - assert all(y.fetch("mean") > 0) - for n, count, mean, max_, key in zip(*x.fetch("n", "count", "mean", "max", dj.key)): + assert all(y.to_arrays("mean") > 0) + for n, count, mean, max_, key in zip(*x.to_arrays("n", "count", "mean", "max"), x.keys()): assert n == count, "aggregation failed (count)" - values = (B.C() & key).fetch("value") + values = (B.C() & key).to_arrays("value") assert bool(len(values)) == bool(n), "aggregation failed (restriction)" if n: - assert np.isclose( - mean, values.mean(), rtol=1e-4, atol=1e-5 - ), "aggregation failed (mean)" - assert np.isclose( - max_, values.max(), rtol=1e-4, atol=1e-5 - ), "aggregation failed (max)" + assert np.isclose(mean, values.mean(), rtol=1e-4, atol=1e-5), "aggregation failed (mean)" + assert np.isclose(max_, values.max(), rtol=1e-4, atol=1e-5), "aggregation failed (max)" def test_semijoin(schema_simp_pop): @@ -331,21 +299,22 @@ def test_semijoin(schema_simp_pop): """ x = IJ() y = JI() - n = len(x & y.fetch(as_dict=True)) - m = len(x - y.fetch(as_dict=True)) + # IJ and JI have i,j from different origins - use semantic_check=False + n = len(x & y.to_dicts()) + m = len(x - y.to_dicts()) assert n > 0 and m > 0 assert len(x) == m + n - assert len(x & y.fetch()) == n - assert len(x - y.fetch()) == m - semi = x & y - anti = x - y + assert len(x & y.to_arrays()) == n + assert len(x - y.to_arrays()) == m + semi = x.restrict(y, semantic_check=False) + anti = x.restrict(dj.Not(y), semantic_check=False) assert len(semi) == n assert len(anti) == m def test_pandas_fetch_and_restriction(schema_simp_pop): q = L & "cond_in_l = 0" - df = q.fetch(format="frame") # pandas dataframe + df = q.to_pandas() # pandas dataframe assert isinstance(df, pandas.DataFrame) assert len(E & q) == len(E & df) @@ -383,19 +352,14 @@ def test_restrictions_by_lists(schema_simp_pop): assert len(x - set()) == lenx, "incorrect restriction by an empty set" assert len(x & {}) == lenx, "incorrect restriction by a tuple with no attributes" assert len(x - {}) == 0, "incorrect restriction by a tuple with no attributes" - assert ( - len(x & {"foo": 0}) == lenx - ), "incorrect restriction by a tuple with no matching attributes" - assert ( - len(x - {"foo": 0}) == 0 - ), "incorrect restriction by a tuple with no matching attributes" - assert len(x & y) == len(x & y.fetch()), "incorrect restriction by a list" - assert len(x - y) == len(x - y.fetch()), "incorrect restriction by a list" + assert len(x & {"foo": 0}) == lenx, "incorrect restriction by a tuple with no matching attributes" + assert len(x - {"foo": 0}) == 0, "incorrect restriction by a tuple with no matching attributes" + assert len(x & y) == len(x & y.to_arrays()), "incorrect restriction by a list" + assert len(x - y) == len(x - y.to_arrays()), "incorrect restriction by a list" w = A() assert len(w) > 0, "incorrect test setup: w is empty" assert ( - bool(set(w.heading.names) & set(y.heading.names)) - != "incorrect test setup: w and y should have no common attributes" + bool(set(w.heading.names) & set(y.heading.names)) != "incorrect test setup: w and y should have no common attributes" ) assert len(w) == len(w & y), "incorrect restriction without common attributes" assert len(w - y) == 0, "incorrect restriction without common attributes" @@ -403,7 +367,7 @@ def test_restrictions_by_lists(schema_simp_pop): def test_datetime(schema_any_pop): """Test date retrieval""" - date = Experiment().fetch("experiment_date")[0] + date = Experiment().to_arrays("experiment_date")[0] e1 = Experiment() & dict(experiment_date=str(date)) e2 = Experiment() & dict(experiment_date=date) assert len(e1) == len(e2) > 0, "Two date restriction do not yield the same result" @@ -423,28 +387,26 @@ def test_date(schema_simp_pop): assert (F & "id=2").fetch1("date") == new_value F.update1(dict((F & "id=2").fetch1("KEY"), date=None)) - assert (F & "id=2").fetch1("date") == None + assert (F & "id=2").fetch1("date") is None def test_join_project(schema_simp_pop): """Test join of projected relations with matching non-primary key""" - q = DataA.proj() * DataB.proj() - assert ( - len(q) == len(DataA()) == len(DataB()) - ), "Join of projected relations does not work" + # DataA and DataB have 'idx' from different origins, so use semantic_check=False + q = DataA.proj().join(DataB.proj(), semantic_check=False) + assert len(q) == len(DataA()) == len(DataB()), "Join of projected relations does not work" def test_ellipsis(schema_any_pop): - r = Experiment.proj(..., "- data_path").head(1, as_dict=True) + # head() now returns list of dicts by default + r = Experiment.proj(..., "- data_path").head(1) assert set(Experiment.heading).difference(r[0]) == {"data_path"} def test_update_single_key(schema_simp_pop): """Test that only one row can be updated""" with pytest.raises(dj.DataJointError): - TTestUpdate.update1( - dict(TTestUpdate.fetch1("KEY"), string_attr="my new string") - ) + TTestUpdate.update1(dict(TTestUpdate.fetch1("KEY"), string_attr="my new string")) def test_update_no_primary(schema_simp_pop): @@ -462,9 +424,7 @@ def test_update_missing_attribute(schema_simp_pop): def test_update_string_attribute(schema_simp_pop): """Test replacing a string value""" rel = TTestUpdate() & dict(primary_key=0) - s = "".join( - random.choice(string.ascii_uppercase + string.digits) for _ in range(10) - ) + s = "".join(random.choice(string.ascii_uppercase + string.digits) for _ in range(10)) TTestUpdate.update1(dict(rel.fetch1("KEY"), string_attr=s)) assert s == rel.fetch1("string_attr"), "Updated string does not match" @@ -476,7 +436,8 @@ def test_update_numeric_attribute(schema_simp_pop): TTestUpdate.update1(dict(rel.fetch1("KEY"), num_attr=s)) assert s == rel.fetch1("num_attr"), "Updated integer does not match" TTestUpdate.update1(dict(rel.fetch1("KEY"), num_attr=None)) - assert np.isnan(rel.fetch1("num_attr")), "Numeric value is not NaN" + # NULL values are returned as None + assert rel.fetch1("num_attr") is None, "Numeric value is not None/NULL" def test_update_blob_attribute(schema_simp_pop): @@ -490,9 +451,7 @@ def test_update_blob_attribute(schema_simp_pop): def test_reserved_words(schema_simp_pop): """Test the user of SQL reserved words as attributes""" rel = ReservedWord() - rel.insert1( - {"key": 1, "in": "ouch", "from": "bummer", "int": 3, "select": "major pain"} - ) + rel.insert1({"key": 1, "in": "ouch", "from": "bummer", "int": 3, "select": "major pain"}) assert (rel & {"key": 1, "in": "ouch", "from": "bummer"}).fetch1("int") == 3 assert (rel.proj("int", double="from") & {"double": "bummer"}).fetch1("int") == 3 (rel & {"key": 1}).delete() @@ -501,23 +460,21 @@ def test_reserved_words(schema_simp_pop): def test_reserved_words2(schema_simp_pop): """Test the user of SQL reserved words as attributes""" rel = ReservedWord() - rel.insert1( - {"key": 1, "in": "ouch", "from": "bummer", "int": 3, "select": "major pain"} - ) + rel.insert1({"key": 1, "in": "ouch", "from": "bummer", "int": 3, "select": "major pain"}) with pytest.raises(dj.DataJointError): - (rel & "key=1").fetch( - "in" - ) # error because reserved word `key` is not in backquotes. See issue #249 + (rel & "key=1").to_arrays("in") # error because reserved word `key` is not in backquotes. See issue #249 def test_permissive_join_basic(schema_any_pop): - """Verify join compatibility check is skipped for join""" - Child @ Parent + """Verify join compatibility check can be skipped with semantic_check=False""" + # The @ operator has been removed in 2.0, use .join(semantic_check=False) instead + Child().join(Parent(), semantic_check=False) def test_permissive_restriction_basic(schema_any_pop): - """Verify join compatibility check is skipped for restriction""" - Child ^ Parent + """Verify restriction compatibility check can be skipped with semantic_check=False""" + # The ^ operator has been removed in 2.0, use .restrict(semantic_check=False) instead + Child().restrict(Parent(), semantic_check=False) def test_complex_date_restriction(schema_simp_pop): @@ -560,9 +517,7 @@ def test_joins_with_aggregation(schema_any_pop): SessionA * SessionStatusA & 'status="trained_1a" or status="trained_1b"', date_trained="min(date(session_start_time))", ) - session_dates = ( - SessionDateA * (subj_query & 'date_trained<"2020-12-21"') - ) & "session_date result[1]["id_l"] > result[2]["id_l"] + assert [r["id_l"] for r in result] == [29, 28, 27] + + def test_top_merge_identical_order(self, schema_simp_pop): + """Test that Tops with identical order_by are merged.""" + # Both Tops specify same ordering - should merge + query = L() & dj.Top(10, "id_l desc") & dj.Top(5, "id_l desc") + result = query.to_dicts() + # Merged limit is min(10, 5) = 5 + assert len(result) == 5 + assert [r["id_l"] for r in result] == [29, 28, 27, 26, 25] + + def test_top_merge_offsets_add(self, schema_simp_pop): + """Test that offsets are added when merging Tops.""" + # First Top: offset 2, second Top: offset 3, inherited order + query = L() & dj.Top(10, "id_l desc", offset=2) & dj.Top(3, order_by=None, offset=3) + result = query.to_dicts() + # Total offset = 2 + 3 = 5, so starts at 6th element (id_l=24) + assert len(result) == 3 + assert [r["id_l"] for r in result] == [24, 23, 22] + + def test_preview_respects_order(self, schema_simp_pop): + """Test that preview (to_arrays with limit) respects Top ordering (issue #1242).""" + # Apply descending order with no limit (None = unlimited) + query = L() & dj.Top(None, order_by="id_l desc") + # Preview should respect the ordering (single attr returns array directly) + id_l = query.to_arrays("id_l", limit=5) + assert list(id_l) == [29, 28, 27, 26, 25] + + def test_top_different_order_subquery(self, schema_simp_pop): + """Test that different orderings create subquery.""" + # First Top: descending, second Top: ascending - cannot merge + query = L() & dj.Top(10, "id_l desc") & dj.Top(3, "id_l asc") + result = query.to_dicts() + # Second Top reorders the result of first Top + # First Top gives ids 29-20, second Top takes lowest 3 of those + assert len(result) == 3 + assert [r["id_l"] for r in result] == [20, 21, 22] diff --git a/tests/test_schema.py b/tests/integration/test_schema.py similarity index 75% rename from tests/test_schema.py rename to tests/integration/test_schema.py index fb3cfa752..8cf231bf5 100644 --- a/tests/test_schema.py +++ b/tests/integration/test_schema.py @@ -6,12 +6,8 @@ import datajoint as dj -from . import schema - - -class Ephys(dj.Imported): - definition = """ # This is already declare in ./schema.py - """ +from tests import schema +from tests.schema import Ephys def relation_selector(attr): @@ -52,14 +48,12 @@ def schema_empty_module(schema_any, schema_empty): @pytest.fixture def schema_empty(connection_test, schema_any, prefix): context = {**schema.LOCALS_ANY, "Ephys": Ephys} - schema_empty = dj.Schema( - prefix + "_test1", context=context, connection=connection_test - ) + schema_empty = dj.Schema(prefix + "_test1", context=context, connection=connection_test) schema_empty(Ephys) # load the rest of the classes schema_empty.spawn_missing_classes(context=context) yield schema_empty - schema_empty.drop() + # Don't drop the schema since schema_any still needs it def test_schema_size_on_disk(schema_any): @@ -72,8 +66,10 @@ def test_schema_list(schema_any): assert schema_any.database in schemas -def test_drop_unauthorized(): - info_schema = dj.schema("information_schema") +@pytest.mark.requires_mysql +def test_drop_unauthorized(connection_test): + """Test that dropping information_schema raises AccessError.""" + info_schema = dj.schema("information_schema", connection=connection_test) with pytest.raises(dj.errors.AccessError): info_schema.drop() @@ -93,18 +89,12 @@ def test_namespace_population(schema_empty_module): setattr(schema_empty_module, k, v) for name, rel in getmembers(schema, relation_selector): - assert hasattr( - schema_empty_module, name - ), "{name} not found in schema_empty".format(name=name) - assert ( - rel.__base__ is getattr(schema_empty_module, name).__base__ - ), "Wrong tier for {name}".format(name=name) + assert hasattr(schema_empty_module, name), "{name} not found in schema_empty".format(name=name) + assert rel.__base__ is getattr(schema_empty_module, name).__base__, "Wrong tier for {name}".format(name=name) for name_part in dir(rel): if name_part[0].isupper() and part_selector(getattr(rel, name_part)): - assert ( - getattr(rel, name_part).__base__ is dj.Part - ), "Wrong tier for {name}".format(name=name_part) + assert getattr(rel, name_part).__base__ is dj.Part, "Wrong tier for {name}".format(name=name_part) def test_undecorated_table(): @@ -120,6 +110,32 @@ class UndecoratedClass(dj.Manual): print(a.full_table_name) +def test_non_activated_schema_heading_error(): + """ + Tables from non-activated schemas should raise informative errors. + Regression test for issue #1039. + """ + # Create schema without activating (no database name) + schema = dj.Schema() + + @schema + class TableA(dj.Manual): + definition = """ + id : int + --- + value : float + """ + + # Accessing heading should raise a helpful error + instance = TableA() + with pytest.raises(dj.DataJointError, match="not properly configured"): + _ = instance.heading + + # Operations that use heading should also raise helpful errors + with pytest.raises(dj.DataJointError, match="not properly configured"): + _ = instance.primary_key # Uses heading.primary_key + + def test_reject_decorated_part(schema_any): """ Decorating a dj.Part table should raise an informative exception. @@ -141,15 +157,11 @@ def test_unauthorized_database(db_creds_test): an attempt to create a database to which user has no privileges should raise an informative exception. """ with pytest.raises(dj.DataJointError): - dj.Schema( - "unauthorized_schema", connection=dj.conn(reset=True, **db_creds_test) - ) + dj.Schema("unauthorized_schema", connection=dj.conn(reset=True, **db_creds_test)) def test_drop_database(db_creds_test, prefix): - schema = dj.Schema( - prefix + "_drop_test", connection=dj.conn(reset=True, **db_creds_test) - ) + schema = dj.Schema(prefix + "_drop_test", connection=dj.conn(reset=True, **db_creds_test)) assert schema.exists schema.drop() assert not schema.exists @@ -242,7 +254,7 @@ class Subject(dj.Manual): name: varchar(32) """ - Schema_A = dj.VirtualModule("Schema_A", "Schema_A") + Schema_A = dj.VirtualModule("Schema_A", "Schema_A") # noqa: F841 schema2 = dj.Schema("schema_b") diff --git a/tests/test_schema_keywords.py b/tests/integration/test_schema_keywords.py similarity index 100% rename from tests/test_schema_keywords.py rename to tests/integration/test_schema_keywords.py diff --git a/tests/integration/test_semantic_matching.py b/tests/integration/test_semantic_matching.py new file mode 100644 index 000000000..d8dff27fa --- /dev/null +++ b/tests/integration/test_semantic_matching.py @@ -0,0 +1,342 @@ +""" +Tests for semantic matching in joins. + +These tests verify the lineage-based semantic matching system +that prevents incorrect joins on attributes with the same name +but different origins. +""" + +import pytest + +import datajoint as dj +from datajoint.errors import DataJointError + + +# Schema definitions for semantic matching tests +LOCALS_SEMANTIC = {} + + +class Student(dj.Manual): + definition = """ + student_id : int + --- + name : varchar(100) + """ + + +class Course(dj.Manual): + definition = """ + course_id : int + --- + title : varchar(100) + """ + + +class Enrollment(dj.Manual): + definition = """ + -> Student + -> Course + --- + grade : varchar(2) + """ + + +class Session(dj.Manual): + definition = """ + session_id : int + --- + date : date + """ + + +class Trial(dj.Manual): + definition = """ + -> Session + trial_num : int + --- + stimulus : varchar(100) + """ + + +class Response(dj.Computed): + definition = """ + -> Trial + --- + response_time : float + """ + + +# Tables with generic 'id' attribute for collision testing +class TableWithId1(dj.Manual): + definition = """ + id : int + --- + value1 : int + """ + + +class TableWithId2(dj.Manual): + definition = """ + id : int + --- + value2 : int + """ + + +# Register all classes in LOCALS_SEMANTIC +for cls in [ + Student, + Course, + Enrollment, + Session, + Trial, + Response, + TableWithId1, + TableWithId2, +]: + LOCALS_SEMANTIC[cls.__name__] = cls + + +@pytest.fixture(scope="module") +def schema_semantic(connection_test, prefix): + """Schema for semantic matching tests.""" + schema = dj.Schema( + prefix + "_semantic", + context=LOCALS_SEMANTIC, + connection=connection_test, + ) + # Declare tables + schema(Student) + schema(Course) + schema(Enrollment) + schema(Session) + schema(Trial) + # Skip Response for now - it's a computed table + schema(TableWithId1) + schema(TableWithId2) + + yield schema + schema.drop() + + +class TestLineageComputation: + """Tests for lineage computation from dependency graph.""" + + def test_native_primary_key_has_lineage(self, schema_semantic): + """Native primary key attributes should have lineage pointing to their table.""" + student = Student() + lineage = student.heading["student_id"].lineage + assert lineage is not None + assert "student_id" in lineage + # The lineage should include schema and table name + assert "student" in lineage.lower() + + def test_inherited_attribute_traces_to_origin(self, schema_semantic): + """FK-inherited attributes should trace lineage to the original table.""" + enrollment = Enrollment() + # student_id is inherited from Student + student_lineage = enrollment.heading["student_id"].lineage + assert student_lineage is not None + assert "student" in student_lineage.lower() + + # course_id is inherited from Course + course_lineage = enrollment.heading["course_id"].lineage + assert course_lineage is not None + assert "course" in course_lineage.lower() + + def test_secondary_attribute_no_lineage(self, schema_semantic): + """Native secondary attributes should have no lineage.""" + student = Student() + name_lineage = student.heading["name"].lineage + assert name_lineage is None + + def test_multi_hop_inheritance(self, schema_semantic): + """Lineage should trace through multiple FK hops.""" + trial = Trial() + # session_id in Trial is inherited from Session + session_lineage = trial.heading["session_id"].lineage + assert session_lineage is not None + assert "session" in session_lineage.lower() + + +class TestJoinCompatibility: + """Tests for join compatibility checking.""" + + def test_join_on_shared_lineage_works(self, schema_semantic): + """Joining tables with shared lineage should work.""" + student = Student() + enrollment = Enrollment() + + # This should work - student_id has same lineage in both + result = student * enrollment + assert "student_id" in result.heading.names + + def test_join_different_lineage_default_fails(self, schema_semantic): + """By default (semantic_check=True), non-homologous namesakes cause an error.""" + table1 = TableWithId1() + table2 = TableWithId2() + + # Default is semantic_check=True, this should fail + with pytest.raises(DataJointError) as exc_info: + table1 * table2 + + assert "lineage" in str(exc_info.value).lower() + assert "id" in str(exc_info.value) + + def test_join_different_lineage_semantic_check_false_works(self, schema_semantic): + """With semantic_check=False, no lineage checking - natural join proceeds.""" + table1 = TableWithId1() + table2 = TableWithId2() + + # With semantic_check=False, no error even with different lineages + result = table1.join(table2, semantic_check=False) + assert "id" in result.heading.names + + +class TestRestrictCompatibility: + """Tests for restriction compatibility checking.""" + + def test_restrict_shared_lineage_works(self, schema_semantic): + """Restricting with shared lineage should work.""" + student = Student() + enrollment = Enrollment() + + # This should work - student_id has same lineage + result = student & enrollment + assert "student_id" in result.heading.names + + def test_restrict_different_lineage_default_fails(self, schema_semantic): + """By default (semantic_check=True), non-homologous namesakes cause an error.""" + table1 = TableWithId1() + table2 = TableWithId2() + + # Default is semantic_check=True, this should fail + with pytest.raises(DataJointError) as exc_info: + table1 & table2 + + assert "lineage" in str(exc_info.value).lower() + + def test_restrict_different_lineage_semantic_check_false_works(self, schema_semantic): + """With semantic_check=False, no lineage checking - restriction proceeds.""" + table1 = TableWithId1() + table2 = TableWithId2() + + # With semantic_check=False, no error even with different lineages + result = table1.restrict(table2, semantic_check=False) + assert "id" in result.heading.names + + +class TestProjectionLineage: + """Tests for lineage preservation in projections.""" + + def test_projection_preserves_lineage(self, schema_semantic): + """Projected attributes should preserve their lineage.""" + enrollment = Enrollment() + + projected = enrollment.proj("grade") + # Primary key attributes should still have lineage + assert projected.heading["student_id"].lineage is not None + + def test_renamed_attribute_preserves_lineage(self, schema_semantic): + """Renamed attributes should preserve their original lineage.""" + student = Student() + + renamed = student.proj(sid="student_id") + # The renamed attribute should have the same lineage as original + original_lineage = student.heading["student_id"].lineage + renamed_lineage = renamed.heading["sid"].lineage + assert renamed_lineage == original_lineage + + def test_computed_attribute_no_lineage(self, schema_semantic): + """Computed attributes should have no lineage.""" + student = Student() + + computed = student.proj(doubled="student_id * 2") + # Computed attributes have no lineage + assert computed.heading["doubled"].lineage is None + + +class TestRemovedOperators: + """Tests for removed operators.""" + + def test_matmul_operator_removed(self, schema_semantic): + """The @ operator should raise an error.""" + student = Student() + course = Course() + + with pytest.raises(DataJointError) as exc_info: + student @ course + + assert "@" in str(exc_info.value) or "matmul" in str(exc_info.value).lower() + assert "removed" in str(exc_info.value).lower() + + def test_xor_operator_removed(self, schema_semantic): + """The ^ operator should raise an error.""" + student = Student() + course = Course() + + with pytest.raises(DataJointError) as exc_info: + student ^ course + + assert "^" in str(exc_info.value) or "removed" in str(exc_info.value).lower() + + +class TestUniversalSetOperators: + """Tests for dj.U operations.""" + + def test_u_mul_raises_error(self, schema_semantic): + """dj.U * table should raise an error.""" + student = Student() + + with pytest.raises(DataJointError) as exc_info: + dj.U("student_id") * student + + assert "no longer supported" in str(exc_info.value).lower() + + def test_table_mul_u_raises_error(self, schema_semantic): + """table * dj.U should raise an error.""" + student = Student() + + with pytest.raises(DataJointError) as exc_info: + student * dj.U("student_id") + + assert "no longer supported" in str(exc_info.value).lower() + + def test_u_sub_raises_error(self, schema_semantic): + """dj.U - table should raise an error (infinite set).""" + student = Student() + + with pytest.raises(DataJointError) as exc_info: + dj.U("student_id") - student + + assert "infinite" in str(exc_info.value).lower() + + def test_u_and_works(self, schema_semantic): + """dj.U & table should work for restriction.""" + student = Student() + student.insert([{"student_id": 1, "name": "Alice"}, {"student_id": 2, "name": "Bob"}]) + + result = dj.U("student_id") & student + assert len(result) == 2 + + +class TestRebuildLineageUtility: + """Tests for the lineage rebuild utility.""" + + def test_rebuild_lineage_method_exists(self): + """The rebuild_lineage method should exist on Schema.""" + assert hasattr(dj.Schema, "rebuild_lineage") + + def test_rebuild_lineage_populates_table(self, schema_semantic): + """schema.rebuild_lineage() should populate the ~lineage table.""" + from datajoint.lineage import get_table_lineages + + # Run rebuild using Schema method + schema_semantic.rebuild_lineage() + + # Check that ~lineage table was created + assert schema_semantic.lineage_table_exists + + # Check that lineages were populated for Student table + lineages = get_table_lineages(schema_semantic.connection, schema_semantic.database, "student") + assert "student_id" in lineages diff --git a/tests/test_tls.py b/tests/integration/test_tls.py similarity index 66% rename from tests/test_tls.py rename to tests/integration/test_tls.py index 6c2effc43..e46825227 100644 --- a/tests/test_tls.py +++ b/tests/integration/test_tls.py @@ -5,20 +5,12 @@ def test_secure_connection(db_creds_test, connection_test): - result = ( - dj.conn(reset=True, **db_creds_test) - .query("SHOW STATUS LIKE 'Ssl_cipher';") - .fetchone()[1] - ) + result = dj.conn(reset=True, **db_creds_test).query("SHOW STATUS LIKE 'Ssl_cipher';").fetchone()[1] assert len(result) > 0 def test_insecure_connection(db_creds_test, connection_test): - result = ( - dj.conn(use_tls=False, reset=True, **db_creds_test) - .query("SHOW STATUS LIKE 'Ssl_cipher';") - .fetchone()[1] - ) + result = dj.conn(use_tls=False, reset=True, **db_creds_test).query("SHOW STATUS LIKE 'Ssl_cipher';").fetchone()[1] assert result == "" diff --git a/tests/integration/test_type_aliases.py b/tests/integration/test_type_aliases.py new file mode 100644 index 000000000..9aae0a8a9 --- /dev/null +++ b/tests/integration/test_type_aliases.py @@ -0,0 +1,184 @@ +""" +Tests for numeric type aliases (float32, float64, int8, int16, int32, int64, etc.) +""" + +import pytest + +from datajoint.declare import CORE_TYPE_SQL, SPECIAL_TYPES, match_type + +from tests.schema_type_aliases import TypeAliasTable, TypeAliasPrimaryKey, TypeAliasNullable + + +class TestTypeAliasPatterns: + """Test that type alias patterns are correctly defined and matched.""" + + @pytest.mark.parametrize( + "alias,expected_category", + [ + ("float32", "FLOAT32"), + ("float64", "FLOAT64"), + ("int64", "INT64"), + ("uint64", "UINT64"), + ("int32", "INT32"), + ("uint32", "UINT32"), + ("int16", "INT16"), + ("uint16", "UINT16"), + ("int8", "INT8"), + ("uint8", "UINT8"), + ("bool", "BOOL"), + ], + ) + def test_type_alias_pattern_matching(self, alias, expected_category): + """Test that type aliases are matched to correct categories.""" + category = match_type(alias) + assert category == expected_category + assert category in SPECIAL_TYPES + assert category.lower() in CORE_TYPE_SQL # CORE_TYPE_SQL uses lowercase keys + + @pytest.mark.parametrize( + "alias,expected_mysql_type", + [ + ("float32", "float"), + ("float64", "double"), + ("int64", "bigint"), + ("uint64", "bigint unsigned"), + ("int32", "int"), + ("uint32", "int unsigned"), + ("int16", "smallint"), + ("uint16", "smallint unsigned"), + ("int8", "tinyint"), + ("uint8", "tinyint unsigned"), + ("bool", "tinyint"), + ], + ) + def test_type_alias_mysql_mapping(self, alias, expected_mysql_type): + """Test that type aliases map to correct MySQL types.""" + category = match_type(alias) + mysql_type = CORE_TYPE_SQL[category.lower()] # CORE_TYPE_SQL uses lowercase keys + assert mysql_type == expected_mysql_type + + @pytest.mark.parametrize( + "native_type,expected_category", + [ + ("int", "INTEGER"), + ("bigint", "INTEGER"), + ("smallint", "INTEGER"), + ("tinyint", "INTEGER"), + ("float", "FLOAT"), + ("double", "FLOAT"), + ], + ) + def test_native_types_still_work(self, native_type, expected_category): + """Test that native MySQL types still match correctly.""" + category = match_type(native_type) + assert category == expected_category + + +class TestTypeAliasTableCreation: + """Test table creation with type aliases.""" + + def test_create_table_with_all_aliases(self, schema_type_aliases): + """Test that tables with all type aliases can be created.""" + assert TypeAliasTable().full_table_name is not None + + def test_create_table_with_alias_primary_key(self, schema_type_aliases): + """Test that tables with type aliases in primary key can be created.""" + assert TypeAliasPrimaryKey().full_table_name is not None + + def test_create_table_with_nullable_aliases(self, schema_type_aliases): + """Test that tables with nullable type alias columns can be created.""" + assert TypeAliasNullable().full_table_name is not None + + +class TestTypeAliasHeading: + """Test that headings correctly preserve type alias information.""" + + def test_heading_preserves_type_aliases(self, schema_type_aliases): + """Test that heading shows original type aliases.""" + heading = TypeAliasTable().heading + heading_str = repr(heading) + + # Check that type aliases appear in the heading representation + assert "float32" in heading_str + assert "float64" in heading_str + assert "int64" in heading_str + assert "uint64" in heading_str + assert "int32" in heading_str + assert "uint32" in heading_str + assert "int16" in heading_str + assert "uint16" in heading_str + assert "int8" in heading_str + assert "uint8" in heading_str + assert "bool" in heading_str + + +class TestTypeAliasInsertFetch: + """Test inserting and fetching data with type aliases.""" + + def test_insert_and_fetch(self, schema_type_aliases): + """Test inserting and fetching values with type aliases.""" + table = TypeAliasTable() + table.delete() + + test_data = dict( + id=1, + val_float32=3.14, + val_float64=2.718281828, + val_int64=9223372036854775807, # max int64 + val_uint64=18446744073709551615, # max uint64 + val_int32=2147483647, # max int32 + val_uint32=4294967295, # max uint32 + val_int16=32767, # max int16 + val_uint16=65535, # max uint16 + val_int8=127, # max int8 + val_uint8=255, # max uint8 + val_bool=1, # boolean true + ) + + table.insert1(test_data) + fetched = table.fetch1() + + assert fetched["id"] == test_data["id"] + assert abs(fetched["val_float32"] - test_data["val_float32"]) < 0.001 + assert abs(fetched["val_float64"] - test_data["val_float64"]) < 1e-9 + assert fetched["val_int64"] == test_data["val_int64"] + assert fetched["val_uint64"] == test_data["val_uint64"] + assert fetched["val_int32"] == test_data["val_int32"] + assert fetched["val_uint32"] == test_data["val_uint32"] + assert fetched["val_int16"] == test_data["val_int16"] + assert fetched["val_uint16"] == test_data["val_uint16"] + assert fetched["val_int8"] == test_data["val_int8"] + assert fetched["val_uint8"] == test_data["val_uint8"] + assert fetched["val_bool"] == test_data["val_bool"] + + def test_insert_primary_key_with_aliases(self, schema_type_aliases): + """Test using type aliases in primary key.""" + table = TypeAliasPrimaryKey() + table.delete() + + table.insert1(dict(pk_int32=100, pk_uint16=200, value="test")) + fetched = (table & dict(pk_int32=100, pk_uint16=200)).fetch1() + + assert fetched["pk_int32"] == 100 + assert fetched["pk_uint16"] == 200 + assert fetched["value"] == "test" + + def test_nullable_type_aliases(self, schema_type_aliases): + """Test nullable columns with type aliases.""" + table = TypeAliasNullable() + table.delete() + + # Insert with NULL values + table.insert1(dict(id=1, nullable_float32=None, nullable_int64=None)) + fetched = table.fetch1() + + assert fetched["id"] == 1 + assert fetched["nullable_float32"] is None + assert fetched["nullable_int64"] is None + + # Insert with actual values + table.insert1(dict(id=2, nullable_float32=1.5, nullable_int64=999)) + fetched = (table & dict(id=2)).fetch1() + + assert fetched["nullable_float32"] == 1.5 + assert fetched["nullable_int64"] == 999 diff --git a/tests/test_university.py b/tests/integration/test_university.py similarity index 70% rename from tests/test_university.py rename to tests/integration/test_university.py index 24f01dd4c..d30b9f3e0 100644 --- a/tests/test_university.py +++ b/tests/integration/test_university.py @@ -6,13 +6,24 @@ import datajoint as dj from datajoint import DataJointError -from . import schema_university -from .schema_university import * +from tests import schema_university +from tests.schema_university import ( + Student, + Department, + StudentMajor, + Course, + Term, + Section, + CurrentTerm, + Enroll, + LetterGrade, + Grade, +) def _hash4(table): """Hash of table contents""" - data = table.fetch(order_by="KEY", as_dict=True) + data = table.to_dicts(order_by="KEY") blob = dj.blob.pack(data, compress=False) return hashlib.md5(blob).digest().hex()[:4] @@ -37,9 +48,7 @@ def schema_uni_inactive(): @pytest.fixture def schema_uni(db_creds_test, schema_uni_inactive, connection_test, prefix): # Deferred activation - schema_uni_inactive.activate( - prefix + "_university", connection=dj.conn(**db_creds_test) - ) + schema_uni_inactive.activate(prefix + "_university", connection=dj.conn(**db_creds_test)) # --------------- Fill University ------------------- test_data_dir = Path(__file__).parent / "data" for table in ( @@ -62,9 +71,7 @@ def schema_uni(db_creds_test, schema_uni_inactive, connection_test, prefix): def test_activate_unauthorized(schema_uni_inactive, db_creds_test, connection_test): with pytest.raises(DataJointError): - schema_uni_inactive.activate( - "unauthorized", connection=dj.conn(**db_creds_test) - ) + schema_uni_inactive.activate("unauthorized", connection=dj.conn(**db_creds_test)) def test_fill(schema_uni): @@ -84,25 +91,17 @@ def test_restrict(schema_uni): """ utahns1 = Student & {"home_state": "UT"} utahns2 = Student & 'home_state="UT"' - assert len(utahns1) == len(utahns2.fetch("KEY")) == 7 + assert len(utahns1) == len(utahns2.keys()) == 7 # male nonutahns - sex1, state1 = ((Student & 'sex="M"') - {"home_state": "UT"}).fetch( - "sex", "home_state", order_by="student_id" - ) - sex2, state2 = ((Student & 'sex="M"') - {"home_state": "UT"}).fetch( - "sex", "home_state", order_by="student_id" - ) + sex1, state1 = ((Student & 'sex="M"') - {"home_state": "UT"}).to_arrays("sex", "home_state", order_by="student_id") + sex2, state2 = ((Student & 'sex="M"') - {"home_state": "UT"}).to_arrays("sex", "home_state", order_by="student_id") assert len(set(state1)) == len(set(state2)) == 44 assert set(sex1).pop() == set(sex2).pop() == "M" # students from OK, NM, TX - s1 = (Student & [{"home_state": s} for s in ("OK", "NM", "TX")]).fetch( - "KEY", order_by="student_id" - ) - s2 = (Student & 'home_state in ("OK", "NM", "TX")').fetch( - "KEY", order_by="student_id" - ) + s1 = (Student & [{"home_state": s} for s in ("OK", "NM", "TX")]).keys(order_by="student_id") + s2 = (Student & 'home_state in ("OK", "NM", "TX")').keys(order_by="student_id") assert len(s1) == 11 assert s1 == s2 @@ -139,34 +138,30 @@ def test_union(schema_uni): def test_aggr(schema_uni): - avg_grade_per_course = Course.aggr( - Grade * LetterGrade, avg_grade="round(avg(points), 2)" - ) + # Default: keeps all courses (some may have NULL avg_grade if no grades) + avg_grade_per_course = Course.aggr(Grade * LetterGrade, avg_grade="round(avg(points), 2)") assert len(avg_grade_per_course) == 45 - # GPA + # GPA - use exclude_nonmatching=True to only include students with grades student_gpa = Student.aggr( - Course * Grade * LetterGrade, gpa="round(sum(points*credits)/sum(credits), 2)" + Course * Grade * LetterGrade, + gpa="round(sum(points*credits)/sum(credits), 2)", + exclude_nonmatching=True, ) - gpa = student_gpa.fetch("gpa") - assert len(gpa) == 261 + gpa = student_gpa.to_arrays("gpa") + assert len(gpa) == 261 # only students with grades assert 2 < gpa.mean() < 3 # Sections in biology department with zero students in them - section = (Section & {"dept": "BIOL"}).aggr( - Enroll, n="count(student_id)", keep_all_rows=True - ) & "n=0" - assert len(set(section.fetch("dept"))) == 1 + # aggr now keeps all rows by default (like proj), so sections with 0 enrollments are included + section = (Section & {"dept": "BIOL"}).aggr(Enroll, n="count(student_id)") & "n=0" + assert len(set(section.to_arrays("dept"))) == 1 assert len(section) == 17 assert bool(section) # Test correct use of ellipses in a similar query - section = (Section & {"dept": "BIOL"}).aggr( - Grade, ..., n="count(student_id)", keep_all_rows=True - ) & "n>1" - assert not any( - name in section.heading.names for name in Grade.heading.secondary_attributes - ) - assert len(set(section.fetch("dept"))) == 1 + section = (Section & {"dept": "BIOL"}).aggr(Grade, ..., n="count(student_id)") & "n>1" + assert not any(name in section.heading.names for name in Grade.heading.secondary_attributes) + assert len(set(section.to_arrays("dept"))) == 1 assert len(section) == 168 assert bool(section) diff --git a/tests/test_update1.py b/tests/integration/test_update1.py similarity index 55% rename from tests/test_update1.py rename to tests/integration/test_update1.py index ff53466d4..ef6255bcc 100644 --- a/tests/test_update1.py +++ b/tests/integration/test_update1.py @@ -1,5 +1,4 @@ import os -import tempfile from pathlib import Path import numpy as np @@ -15,44 +14,50 @@ class Thing(dj.Manual): --- number=0 : int frac : float - picture = null : attach@update_store - params = null : longblob - img_file = null: filepath@update_repo + picture = null : + params = null : + img_file = null: timestamp = CURRENT_TIMESTAMP : datetime """ @pytest.fixture(scope="module") def mock_stores_update(tmpdir_factory): - og_stores_config = dj.config.get("stores") - if "stores" not in dj.config: - dj.config["stores"] = {} - dj.config["stores"]["update_store"] = dict( - protocol="file", location=tmpdir_factory.mktemp("store") + """Configure object storage stores for update tests.""" + og_project_name = dj.config.object_storage.project_name + og_stores = dict(dj.config.object_storage.stores) + + # Configure stores + dj.config.object_storage.project_name = "djtest" + store_location = str(tmpdir_factory.mktemp("store")) + repo_stage = str(tmpdir_factory.mktemp("repo_stage")) + repo_location = str(tmpdir_factory.mktemp("repo_loc")) + dj.config.object_storage.stores["update_store"] = dict( + protocol="file", + location=store_location, ) - dj.config["stores"]["update_repo"] = dict( - stage=tmpdir_factory.mktemp("repo_stage"), + dj.config.object_storage.stores["update_repo"] = dict( + stage=repo_stage, protocol="file", - location=tmpdir_factory.mktemp("repo_loc"), + location=repo_location, ) - yield - if og_stores_config is None: - del dj.config["stores"] - else: - dj.config["stores"] = og_stores_config + yield {"update_store": {"location": store_location}, "update_repo": {"stage": repo_stage, "location": repo_location}} + + # Restore original + dj.config.object_storage.project_name = og_project_name + dj.config.object_storage.stores.clear() + dj.config.object_storage.stores.update(og_stores) @pytest.fixture def schema_update1(connection_test, prefix): - schema = dj.Schema( - prefix + "_update1", context=dict(Thing=Thing), connection=connection_test - ) + schema = dj.Schema(prefix + "_update1", context=dict(Thing=Thing), connection=connection_test) schema(Thing) yield schema schema.drop() -def test_update1(tmpdir, enable_filepath_feature, schema_update1, mock_stores_update): +def test_update1(tmpdir, schema_update1, mock_stores_update): """Test normal updates""" # CHECK 1 -- initial insert key = dict(thing=1) @@ -70,21 +75,23 @@ def test_update1(tmpdir, enable_filepath_feature, schema_update1, mock_stores_up attach_file.unlink() assert not attach_file.is_file() - # filepath - stage_path = dj.config["stores"]["update_repo"]["stage"] + # filepath - note: stores a reference, doesn't move the file + store_location = mock_stores_update["update_repo"]["location"] relpath, filename = "one/two/three", "picture.dat" - managed_file = Path(stage_path, relpath, filename) + managed_file = Path(store_location, relpath, filename) managed_file.parent.mkdir(parents=True, exist_ok=True) original_file_data = os.urandom(3000) with managed_file.open("wb") as f: f.write(original_file_data) - Thing.update1(dict(key, img_file=managed_file)) - managed_file.unlink() - assert not managed_file.is_file() + # Insert the relative path within the store + Thing.update1(dict(key, img_file=f"{relpath}/{filename}")) - check2 = Thing.fetch1(download_path=tmpdir) + with dj.config.override(download_path=str(tmpdir)): + check2 = Thing.fetch1() buffer2 = Path(check2["picture"]).read_bytes() # read attachment - final_file_data = managed_file.read_bytes() # read filepath + # For filepath, fetch returns ObjectRef - read the file through it + filepath_ref = check2["img_file"] + final_file_data = filepath_ref.read() if filepath_ref else managed_file.read_bytes() # CHECK 3 -- reset to default values using None Thing.update1( @@ -99,9 +106,7 @@ def test_update1(tmpdir, enable_filepath_feature, schema_update1, mock_stores_up ) check3 = Thing.fetch1() - assert ( - check1["number"] == 0 and check1["picture"] is None and check1["params"] is None - ) + assert check1["number"] == 0 and check1["picture"] is None and check1["params"] is None assert ( check2["number"] == 3 @@ -124,23 +129,19 @@ def test_update1(tmpdir, enable_filepath_feature, schema_update1, mock_stores_up assert original_file_data == final_file_data -def test_update1_nonexistent( - enable_filepath_feature, schema_update1, mock_stores_update -): +def test_update1_nonexistent(schema_update1, mock_stores_update): with pytest.raises(DataJointError): # updating a non-existent entry Thing.update1(dict(thing=100, frac=0.5)) -def test_update1_noprimary(enable_filepath_feature, schema_update1, mock_stores_update): +def test_update1_noprimary(schema_update1, mock_stores_update): with pytest.raises(DataJointError): # missing primary key Thing.update1(dict(number=None)) -def test_update1_misspelled_attribute( - enable_filepath_feature, schema_update1, mock_stores_update -): +def test_update1_misspelled_attribute(schema_update1, mock_stores_update): key = dict(thing=17) Thing.insert1(dict(key, frac=1.5)) with pytest.raises(DataJointError): diff --git a/tests/test_utils.py b/tests/integration/test_utils.py similarity index 100% rename from tests/test_utils.py rename to tests/integration/test_utils.py diff --git a/tests/test_uuid.py b/tests/integration/test_uuid.py similarity index 95% rename from tests/test_uuid.py rename to tests/integration/test_uuid.py index 4392e4769..14095df7e 100644 --- a/tests/test_uuid.py +++ b/tests/integration/test_uuid.py @@ -5,7 +5,7 @@ from datajoint import DataJointError -from .schema_uuid import Basic, Item, Topic +from tests.schema_uuid import Basic, Item, Topic def test_uuid(schema_uuid): @@ -49,7 +49,7 @@ def test_invalid_uuid_restrict1(schema_uuid): k, m = (Basic & {"item": u}).fetch1("KEY", "number") -def test_invalid_uuid_restrict1(schema_uuid): +def test_invalid_uuid_restrict2(schema_uuid): """test that only UUID objects are accepted when inserting UUID fields""" u = "abc" with pytest.raises(DataJointError): diff --git a/tests/integration/test_virtual_module.py b/tests/integration/test_virtual_module.py new file mode 100644 index 000000000..a8e953273 --- /dev/null +++ b/tests/integration/test_virtual_module.py @@ -0,0 +1,107 @@ +"""Tests for virtual schema infrastructure.""" + +import pytest + +import datajoint as dj +from datajoint.table import FreeTable +from datajoint.user_tables import UserTable + + +class TestVirtualModule: + """Tests for VirtualModule class.""" + + def test_virtual_module_creates_table_classes(self, schema_any, connection_test): + """VirtualModule creates table classes from database schema.""" + module = dj.VirtualModule("module", schema_any.database, connection=connection_test) + assert issubclass(module.Experiment, UserTable) + + def test_virtual_module_has_schema_attribute(self, schema_any, connection_test): + """VirtualModule has schema attribute.""" + module = dj.VirtualModule("module", schema_any.database, connection=connection_test) + assert hasattr(module, "schema") + assert module.schema.database == schema_any.database + + +class TestVirtualSchema: + """Tests for dj.virtual_schema() function.""" + + def test_virtual_schema_creates_module(self, schema_any, connection_test): + """virtual_schema creates a VirtualModule.""" + lab = dj.virtual_schema(schema_any.database, connection=connection_test) + assert isinstance(lab, dj.VirtualModule) + + def test_virtual_schema_has_table_classes(self, schema_any, connection_test): + """virtual_schema module has table classes as attributes.""" + lab = dj.virtual_schema(schema_any.database, connection=connection_test) + assert issubclass(lab.Experiment, UserTable) + + def test_virtual_schema_tables_are_queryable(self, schema_any, connection_test): + """Tables from virtual_schema can be queried.""" + lab = dj.virtual_schema(schema_any.database, connection=connection_test) + # Should not raise + lab.Experiment().to_dicts() + + +class TestSchemaGetTable: + """Tests for Schema.get_table() method.""" + + def test_get_table_by_snake_case(self, schema_any): + """get_table works with snake_case table names.""" + table = schema_any.get_table("experiment") + assert isinstance(table, FreeTable) + assert "experiment" in table.full_table_name + + def test_get_table_by_camel_case(self, schema_any): + """get_table works with CamelCase table names.""" + table = schema_any.get_table("Experiment") + assert isinstance(table, FreeTable) + assert "experiment" in table.full_table_name + + def test_get_table_nonexistent_raises(self, schema_any): + """get_table raises DataJointError for nonexistent tables.""" + with pytest.raises(dj.DataJointError, match="does not exist"): + schema_any.get_table("NonexistentTable") + + +class TestSchemaGetItem: + """Tests for Schema.__getitem__() method.""" + + def test_getitem_by_name(self, schema_any): + """Schema['TableName'] returns table instance.""" + table = schema_any["Experiment"] + assert isinstance(table, FreeTable) + + def test_getitem_is_queryable(self, schema_any): + """Table from __getitem__ can be queried.""" + table = schema_any["Experiment"] + # Should not raise + table.to_dicts() + + +class TestSchemaIteration: + """Tests for Schema.__iter__() method.""" + + def test_iter_yields_tables(self, schema_any): + """Iterating over schema yields FreeTable instances.""" + tables = list(schema_any) + assert len(tables) > 0 + assert all(isinstance(t, FreeTable) for t in tables) + + def test_iter_in_dependency_order(self, schema_any): + """Iteration order respects dependencies.""" + table_names = [t.table_name for t in schema_any] + # Tables should be in topological order + assert len(table_names) == len(set(table_names)) # no duplicates + + +class TestSchemaContains: + """Tests for Schema.__contains__() method.""" + + def test_contains_existing_table(self, schema_any): + """'TableName' in schema returns True for existing tables.""" + assert "Experiment" in schema_any + assert "experiment" in schema_any + + def test_contains_nonexistent_table(self, schema_any): + """'TableName' in schema returns False for nonexistent tables.""" + assert "NonexistentTable" not in schema_any diff --git a/tests/schema.py b/tests/schema.py index 2b7977465..4035e1211 100644 --- a/tests/schema.py +++ b/tests/schema.py @@ -16,24 +16,24 @@ class TTest(dj.Lookup): """ definition = """ - key : int # key + key : int32 # key --- - value : int # value + value : int32 # value """ contents = [(k, 2 * k) for k in range(10)] class TTest2(dj.Manual): definition = """ - key : int # key + key : int32 # key --- - value : int # value + value : int32 # value """ class TTest3(dj.Manual): definition = """ - key : int + key : int32 --- value : varchar(300) """ @@ -41,11 +41,11 @@ class TTest3(dj.Manual): class NullableNumbers(dj.Manual): definition = """ - key : int + key : int32 --- - fvalue = null : float - dvalue = null : double - ivalue = null : int + fvalue = null : float32 + dvalue = null : float64 + ivalue = null : int32 """ @@ -54,7 +54,7 @@ class TTestExtra(dj.Manual): clone of Test but with an extra field """ - definition = TTest.definition + "\nextra : int # extra int\n" + definition = TTest.definition + "\nextra : int32 # extra int\n" class TTestNoExtra(dj.Manual): @@ -67,14 +67,11 @@ class TTestNoExtra(dj.Manual): class Auto(dj.Lookup): definition = """ - id :int auto_increment + id : uint8 --- name :varchar(12) """ - - def fill(self): - if not self: - self.insert([dict(name="Godel"), dict(name="Escher"), dict(name="Bach")]) + contents = [(1, "Godel"), (2, "Escher"), (3, "Bach")] class User(dj.Lookup): @@ -94,7 +91,7 @@ class User(dj.Lookup): class Subject(dj.Lookup): definition = """ # Basic information about animal subjects used in experiments - subject_id :int # unique subject id + subject_id :int32 # unique subject id --- real_id :varchar(40) # real-world name. Omit if the same as subject_id species = "mouse" :enum('mouse', 'monkey', 'human') @@ -131,13 +128,13 @@ class Language(dj.Lookup): class Experiment(dj.Imported): definition = """ # information about experiments -> Subject - experiment_id :smallint # experiment number for this subject + experiment_id :int16 # experiment number for this subject --- experiment_date :date # date when experiment was started -> [nullable] User data_path="" :varchar(255) # file path to recorded data notes="" :varchar(2048) # e.g. purpose of experiment - entry_time=CURRENT_TIMESTAMP :timestamp # automatic timestamp + entry_time=CURRENT_TIMESTAMP :datetime # automatic timestamp """ fake_experiments_per_subject = 5 @@ -148,15 +145,13 @@ def make(self, key): """ from datetime import date, timedelta - users = [None, None] + list(User().fetch()["username"]) + users = [None, None] + list(User().to_arrays()["username"]) random.seed("Amazing Seed4") self.insert( dict( key, experiment_id=experiment_id, - experiment_date=( - date.today() - timedelta(random.expovariate(1 / 30)) - ).isoformat(), + experiment_date=(date.today() - timedelta(random.expovariate(1 / 30))).isoformat(), username=random.choice(users), ) for experiment_id in range(self.fake_experiments_per_subject) @@ -166,17 +161,17 @@ def make(self, key): class Trial(dj.Imported): definition = """ # a trial within an experiment -> Experiment.proj(animal='subject_id') - trial_id :smallint # trial number + trial_id :int16 # trial number --- - start_time :double # (s) + start_time :float64 # (s) """ class Condition(dj.Part): definition = """ # trial conditions -> Trial - cond_idx : smallint # condition number + cond_idx : int16 # condition number ---- - orientation : float # degrees + orientation : float32 # degrees """ def make(self, key): @@ -186,27 +181,24 @@ def make(self, key): for trial_id in range(10): key["trial_id"] = trial_id self.insert1(dict(key, start_time=random.random() * 1e9)) - trial.insert( - dict(key, cond_idx=cond_idx, orientation=random.random() * 360) - for cond_idx in range(30) - ) + trial.insert(dict(key, cond_idx=cond_idx, orientation=random.random() * 360) for cond_idx in range(30)) class Ephys(dj.Imported): definition = """ # some kind of electrophysiological recording -> Trial ---- - sampling_frequency :double # (Hz) + sampling_frequency :float64 # (Hz) duration :decimal(7,3) # (s) """ class Channel(dj.Part): definition = """ # subtable containing individual channels -> master - channel :tinyint unsigned # channel number within Ephys + channel :uint8 # channel number within Ephys ---- - voltage : longblob - current = null : longblob # optional current to test null handling + voltage : + current = null : # optional current to test null handling """ def _make_tuples(self, key): @@ -214,9 +206,7 @@ def _make_tuples(self, key): populate with random data """ random.seed(str(key)) - row = dict( - key, sampling_frequency=6000, duration=np.minimum(2, random.expovariate(1)) - ) + row = dict(key, sampling_frequency=6000, duration=np.minimum(2, random.expovariate(1))) self.insert1(row) number_samples = int(row["duration"] * row["sampling_frequency"] + 0.5) sub = self.Channel() @@ -233,15 +223,15 @@ def _make_tuples(self, key): class Image(dj.Manual): definition = """ # table for testing blob inserts - id : int # image identifier + id : int32 # image identifier --- - img : longblob # image + img : # image """ class UberTrash(dj.Lookup): definition = """ - id : int + id : int32 --- """ contents = [(1,)] @@ -250,7 +240,7 @@ class UberTrash(dj.Lookup): class UnterTrash(dj.Lookup): definition = """ -> UberTrash - my_id : int + my_id : int32 --- """ contents = [(1, 1), (1, 2)] @@ -258,7 +248,7 @@ class UnterTrash(dj.Lookup): class SimpleSource(dj.Lookup): definition = """ - id : int # id + id : int32 # id """ contents = [(x,) for x in range(10)] @@ -318,7 +308,7 @@ class IndexRich(dj.Manual): --- -> [unique, nullable] User.proj(first="username") first_date : date - value : int + value : int32 index (first_date, value) """ @@ -326,16 +316,16 @@ class IndexRich(dj.Manual): # Schema for issue 656 class ThingA(dj.Manual): definition = """ - a: int + a: int32 """ class ThingB(dj.Manual): definition = """ - b1: int - b2: int + b1: int32 + b2: int32 --- - b3: int + b3: int32 """ @@ -350,7 +340,7 @@ class ThingC(dj.Manual): # Additional tables for #1159 class ThingD(dj.Manual): definition = """ - d: int + d: int32 --- -> ThingC """ @@ -364,7 +354,7 @@ class ThingE(dj.Manual): class Parent(dj.Lookup): definition = """ - parent_id: int + parent_id: int32 --- name: varchar(30) """ @@ -374,7 +364,7 @@ class Parent(dj.Lookup): class Child(dj.Lookup): definition = """ -> Parent - child_id: int + child_id: int32 --- name: varchar(30) """ @@ -383,14 +373,12 @@ class Child(dj.Lookup): # Related to issue #886 (8), #883 (5) class ComplexParent(dj.Lookup): - definition = "\n".join(["parent_id_{}: int".format(i + 1) for i in range(8)]) + definition = "\n".join(["parent_id_{}: int32".format(i + 1) for i in range(8)]) contents = [tuple(i for i in range(8))] class ComplexChild(dj.Lookup): - definition = "\n".join( - ["-> ComplexParent"] + ["child_id_{}: int".format(i + 1) for i in range(1)] - ) + definition = "\n".join(["-> ComplexParent"] + ["child_id_{}: int32".format(i + 1) for i in range(1)]) contents = [tuple(i for i in range(9))] @@ -452,18 +440,18 @@ class SessionDateA(dj.Lookup): class Stimulus(dj.Lookup): definition = """ - id: int + id: int32 --- - contrast: int - brightness: int + contrast: int32 + brightness: int32 """ class Longblob(dj.Manual): definition = """ - id: int + id: int32 --- - data: longblob + data: """ diff --git a/tests/schema_adapted.py b/tests/schema_adapted.py deleted file mode 100644 index 06e28c3d1..000000000 --- a/tests/schema_adapted.py +++ /dev/null @@ -1,64 +0,0 @@ -import inspect -import json -import tempfile -from pathlib import Path - -import networkx as nx - -import datajoint as dj - - -class GraphAdapter(dj.AttributeAdapter): - attribute_type = "longblob" # this is how the attribute will be declared - - @staticmethod - def get(obj): - # convert edge list into a graph - return nx.Graph(obj) - - @staticmethod - def put(obj): - # convert graph object into an edge list - assert isinstance(obj, nx.Graph) - return list(obj.edges) - - -class LayoutToFilepath(dj.AttributeAdapter): - """ - An adapted data type that saves a graph layout into fixed filepath - """ - - attribute_type = "filepath@repo-s3" - - @staticmethod - def get(path): - with open(path, "r") as f: - return json.load(f) - - @staticmethod - def put(layout): - path = Path(dj.config["stores"]["repo-s3"]["stage"], "layout.json") - with open(str(path), "w") as f: - json.dump(layout, f) - return path - - -class Connectivity(dj.Manual): - definition = """ - connid : int - --- - conn_graph = null : - """ - - -class Layout(dj.Manual): - definition = """ - # stores graph layout - -> Connectivity - --- - layout: - """ - - -LOCALS_ADAPTED = {k: v for k, v in locals().items() if inspect.isclass(v)} -__all__ = list(LOCALS_ADAPTED) diff --git a/tests/schema_alter.py b/tests/schema_alter.py index b86f6c7ec..936d9cc12 100644 --- a/tests/schema_alter.py +++ b/tests/schema_alter.py @@ -6,30 +6,30 @@ class Experiment(dj.Imported): original_definition = """ # information about experiments -> Subject - experiment_id :smallint # experiment number for this subject + experiment_id :int16 # experiment number for this subject --- experiment_date :date # date when experiment was started -> [nullable] User data_path="" :varchar(255) # file path to recorded data notes="" :varchar(2048) # e.g. purpose of experiment - entry_time=CURRENT_TIMESTAMP :timestamp # automatic timestamp + entry_time=CURRENT_TIMESTAMP :datetime # automatic timestamp """ definition1 = """ # Experiment -> Subject - experiment_id :smallint # experiment number for this subject + experiment_id :int16 # experiment number for this subject --- - data_path : int # some number - extra=null : longblob # just testing + data_path : int32 # some number + extra=null : # just testing -> [nullable] User subject_notes=null :varchar(2048) # {notes} e.g. purpose of experiment - entry_time=CURRENT_TIMESTAMP :timestamp # automatic timestamp + entry_time=CURRENT_TIMESTAMP :datetime # automatic timestamp """ class Parent(dj.Manual): definition = """ - parent_id: int + parent_id: int32 """ class Child(dj.Part): @@ -39,7 +39,7 @@ class Child(dj.Part): definition_new = """ -> master --- - child_id=null: int + child_id=null: int32 """ class Grandchild(dj.Part): @@ -49,7 +49,7 @@ class Grandchild(dj.Part): definition_new = """ -> master.Child --- - grandchild_id=null: int + grandchild_id=null: int32 """ diff --git a/tests/schema_codecs.py b/tests/schema_codecs.py new file mode 100644 index 000000000..6a8d478d4 --- /dev/null +++ b/tests/schema_codecs.py @@ -0,0 +1,63 @@ +import inspect + +import networkx as nx + +import datajoint as dj + + +class GraphCodec(dj.Codec): + """Custom codec for storing NetworkX graphs as edge lists.""" + + name = "graph" + + def get_dtype(self, is_external: bool) -> str: + """Chain to blob for serialization.""" + return "" + + def encode(self, obj, *, key=None, store_name=None): + """Convert graph object into an edge list.""" + assert isinstance(obj, nx.Graph) + return list(obj.edges) + + def decode(self, stored, *, key=None): + """Convert edge list into a graph.""" + return nx.Graph(stored) + + +class LayoutCodec(dj.Codec): + """Custom codec that saves a graph layout as serialized blob.""" + + name = "layout" + + def get_dtype(self, is_external: bool) -> str: + """Chain to blob for serialization.""" + return "" + + def encode(self, layout, *, key=None, store_name=None): + """Serialize layout dict.""" + return layout # blob handles serialization + + def decode(self, stored, *, key=None): + """Deserialize layout dict.""" + return stored # blob handles deserialization + + +class Connectivity(dj.Manual): + definition = """ + connid : int + --- + conn_graph = null : + """ + + +class Layout(dj.Manual): + definition = """ + # stores graph layout + -> Connectivity + --- + layout: + """ + + +LOCALS_CODECS = {k: v for k, v in locals().items() if inspect.isclass(v)} +__all__ = list(LOCALS_CODECS) diff --git a/tests/schema_external.py b/tests/schema_external.py index a9e86964f..ae1803f5e 100644 --- a/tests/schema_external.py +++ b/tests/schema_external.py @@ -3,7 +3,6 @@ """ import inspect -import tempfile import numpy as np @@ -14,7 +13,7 @@ class Simple(dj.Manual): definition = """ simple : int --- - item : blob@local + item : """ @@ -22,7 +21,7 @@ class SimpleRemote(dj.Manual): definition = """ simple : int --- - item : blob@share + item : """ @@ -37,7 +36,7 @@ class Dimension(dj.Lookup): definition = """ dim : int --- - dimensions : blob + dimensions : """ contents = ([0, [100, 50]], [1, [3, 4, 8, 6]]) @@ -48,8 +47,8 @@ class Image(dj.Computed): -> Seed -> Dimension ---- - img : blob@share # objects are stored as specified by dj.config['stores']['share'] - neg : blob@local # objects are stored as specified by dj.config['stores']['local'] + img : # objects are stored as specified by dj.config['stores']['share'] + neg : # objects are stored as specified by dj.config['stores']['local'] """ def make(self, key): @@ -63,8 +62,8 @@ class Attach(dj.Manual): # table for storing attachments attach : int ---- - img : attach@share # attachments are stored as specified by: dj.config['stores']['raw'] - txt : attach # attachments are stored directly in the database + img : # attachments are stored as specified by: dj.config['stores']['share'] + txt : # attachments are stored directly in the database """ @@ -73,7 +72,7 @@ class Filepath(dj.Manual): # table for file management fnum : int # test comment containing : --- - img : filepath@repo # managed files + img : # managed files """ @@ -82,7 +81,7 @@ class FilepathS3(dj.Manual): # table for file management fnum : int --- - img : filepath@repo-s3 # managed files + img : # managed files """ diff --git a/tests/schema_object.py b/tests/schema_object.py new file mode 100644 index 000000000..ef1d957dc --- /dev/null +++ b/tests/schema_object.py @@ -0,0 +1,51 @@ +""" +Schema definitions for object type tests. +""" + +import datajoint as dj + +LOCALS_OBJECT = locals() + + +class ObjectFile(dj.Manual): + """Table for testing object type with files.""" + + definition = """ + file_id : int + --- + data_file : # stored file + """ + + +class ObjectFolder(dj.Manual): + """Table for testing object type with folders.""" + + definition = """ + folder_id : int + --- + data_folder : # stored folder + """ + + +class ObjectMultiple(dj.Manual): + """Table for testing multiple object attributes.""" + + definition = """ + record_id : int + --- + raw_data : # raw data file + processed : # processed data file + """ + + +class ObjectWithOther(dj.Manual): + """Table for testing object type with other attributes.""" + + definition = """ + subject_id : int + session_id : int + --- + name : varchar(100) + data_file : + notes : varchar(255) + """ diff --git a/tests/schema_simple.py b/tests/schema_simple.py index 5e5137db5..f0e768d1f 100644 --- a/tests/schema_simple.py +++ b/tests/schema_simple.py @@ -83,10 +83,7 @@ def make(self, key): sigma = random.lognormvariate(0, 4) n = random.randint(0, 10) self.insert1(dict(key, mu=mu, sigma=sigma, n=n)) - sub.insert( - dict(key, id_c=j, value=random.normalvariate(mu, sigma)) - for j in range(n) - ) + sub.insert(dict(key, id_c=j, value=random.normalvariate(mu, sigma)) for j in range(n)) class L(dj.Lookup): @@ -109,7 +106,7 @@ class D(dj.Computed): def _make_tuples(self, key): # make reference to a random tuple from L random.seed(str(key)) - lookup = list(L().fetch("KEY")) + lookup = list(L().keys()) self.insert(dict(key, id_d=i, **random.choice(lookup)) for i in range(4)) @@ -144,26 +141,22 @@ class H(dj.Part): """ class M(dj.Part): - definition = """ # test force_masters revisit + definition = """ # test part_integrity cascade -> E - id_m :int + id_m : uint16 --- -> E.H """ def make(self, key): random.seed(str(key)) - l_contents = list(L().fetch("KEY")) + l_contents = list(L().keys()) part_f, part_g, part_h, part_m = E.F(), E.G(), E.H(), E.M() - bc_references = list((B.C() & key).fetch("KEY")) + bc_references = list((B.C() & key).keys()) random.shuffle(bc_references) self.insert1(dict(key, **random.choice(l_contents))) - part_f.insert( - dict(key, id_f=i, **ref) - for i, ref in enumerate(bc_references) - if random.getrandbits(1) - ) + part_f.insert(dict(key, id_f=i, **ref) for i, ref in enumerate(bc_references) if random.getrandbits(1)) g_inserts = [dict(key, id_g=i, **ref) for i, ref in enumerate(l_contents)] part_g.insert(g_inserts) h_inserts = [dict(key, id_h=i) for i in range(4)] @@ -248,9 +241,7 @@ def populate_random(self, n=10): with self.connection.transaction: self.insert1(profile, ignore_extra_fields=True) for url in profile["website"]: - self.Website().insert1( - dict(ssn=profile["ssn"], url_hash=Website().insert1_url(url)) - ) + self.Website().insert1(dict(ssn=profile["ssn"], url_hash=Website().insert1_url(url))) class TTestUpdate(dj.Lookup): @@ -259,7 +250,7 @@ class TTestUpdate(dj.Lookup): --- string_attr : varchar(255) num_attr=null : float - blob_attr : longblob + blob_attr : """ contents = [ diff --git a/tests/schema_type_aliases.py b/tests/schema_type_aliases.py new file mode 100644 index 000000000..eb586de5d --- /dev/null +++ b/tests/schema_type_aliases.py @@ -0,0 +1,50 @@ +""" +Schema for testing numeric type aliases. +""" + +import inspect + +import datajoint as dj + + +class TypeAliasTable(dj.Manual): + definition = """ + # Table with all numeric type aliases + id : int + --- + val_float32 : float32 # 32-bit float + val_float64 : float64 # 64-bit float + val_int64 : int64 # 64-bit signed integer + val_uint64 : uint64 # 64-bit unsigned integer + val_int32 : int32 # 32-bit signed integer + val_uint32 : uint32 # 32-bit unsigned integer + val_int16 : int16 # 16-bit signed integer + val_uint16 : uint16 # 16-bit unsigned integer + val_int8 : int8 # 8-bit signed integer + val_uint8 : uint8 # 8-bit unsigned integer + val_bool : bool # boolean value + """ + + +class TypeAliasPrimaryKey(dj.Manual): + definition = """ + # Table with type alias in primary key + pk_int32 : int32 + pk_uint16 : uint16 + --- + value : varchar(100) + """ + + +class TypeAliasNullable(dj.Manual): + definition = """ + # Table with nullable type alias columns + id : int + --- + nullable_float32 = null : float32 + nullable_int64 = null : int64 + """ + + +LOCALS_TYPE_ALIASES = {k: v for k, v in locals().items() if inspect.isclass(v)} +__all__ = list(LOCALS_TYPE_ALIASES) diff --git a/tests/schema_uuid.py b/tests/schema_uuid.py index 4e295bc86..75b9cd373 100644 --- a/tests/schema_uuid.py +++ b/tests/schema_uuid.py @@ -24,9 +24,7 @@ class Topic(dj.Manual): def add(self, topic): """add a new topic with a its UUID""" - self.insert1( - dict(topic_id=uuid.uuid5(top_level_namespace_id, topic), topic=topic) - ) + self.insert1(dict(topic_id=uuid.uuid5(top_level_namespace_id, topic), topic=topic)) class Item(dj.Computed): @@ -41,9 +39,7 @@ class Item(dj.Computed): def make(self, key): for word in ("Habenula", "Hippocampus", "Hypothalamus", "Hypophysis"): - self.insert1( - dict(key, word=word, item_id=uuid.uuid5(key["topic_id"], word)) - ) + self.insert1(dict(key, word=word, item_id=uuid.uuid5(key["topic_id"], word))) LOCALS_UUID = {k: v for k, v in locals().items() if inspect.isclass(v)} diff --git a/tests/test_admin.py b/tests/test_admin.py deleted file mode 100644 index b7fa15a33..000000000 --- a/tests/test_admin.py +++ /dev/null @@ -1,152 +0,0 @@ -""" -Collection of test cases to test admin module. -""" - -import os - -import pymysql -import pytest - -import datajoint as dj - - -@pytest.fixture() -def user_alice(db_creds_root) -> dict: - # set up - reset config, log in as root, and create a new user alice - # reset dj.config manually because its state may be changed by these tests - if os.path.exists(dj.settings.LOCALCONFIG): - os.remove(dj.settings.LOCALCONFIG) - dj.config["database.password"] = os.getenv("DJ_PASS") - root_conn = dj.conn(**db_creds_root, reset=True) - new_credentials = dict( - host=db_creds_root["host"], - user="alice", - password="oldpass", - ) - root_conn.query(f"DROP USER IF EXISTS '{new_credentials['user']}'@'%%';") - root_conn.query( - f"CREATE USER '{new_credentials['user']}'@'%%' " - f"IDENTIFIED BY '{new_credentials['password']}';" - ) - - # test the connection - dj.Connection(**new_credentials) - - # return alice's credentials - yield new_credentials - - # tear down - delete the user and the local config file - root_conn.query(f"DROP USER '{new_credentials['user']}'@'%%';") - if os.path.exists(dj.settings.LOCALCONFIG): - os.remove(dj.settings.LOCALCONFIG) - - -def test_set_password_prompt_match(monkeypatch, user_alice: dict): - """ - Should be able to change the password using user prompt - """ - # reset the connection to use alice's credentials - dj.conn(**user_alice, reset=True) - - # prompts: new password / confirm password - password_resp = iter(["newpass", "newpass"]) - # NOTE: because getpass.getpass is imported in datajoint.admin and used as - # getpass in that module, we need to patch datajoint.admin.getpass - # instead of getpass.getpass - monkeypatch.setattr("datajoint.admin.getpass", lambda _: next(password_resp)) - - # respond no to prompt to update local config - monkeypatch.setattr("builtins.input", lambda _: "no") - - # reset password of user of current connection (alice) - dj.set_password() - - # should not be able to connect with old credentials - with pytest.raises(pymysql.err.OperationalError): - dj.Connection(**user_alice) - - # should be able to connect with new credentials - dj.Connection(host=user_alice["host"], user=user_alice["user"], password="newpass") - - # check that local config is not updated - assert dj.config["database.password"] == os.getenv("DJ_PASS") - assert not os.path.exists(dj.settings.LOCALCONFIG) - - -def test_set_password_prompt_mismatch(monkeypatch, user_alice: dict): - """ - Should not be able to change the password when passwords do not match - """ - # reset the connection to use alice's credentials - dj.conn(**user_alice, reset=True) - - # prompts: new password / confirm password - password_resp = iter(["newpass", "wrong"]) - # NOTE: because getpass.getpass is imported in datajoint.admin and used as - # getpass in that module, we need to patch datajoint.admin.getpass - # instead of getpass.getpass - monkeypatch.setattr("datajoint.admin.getpass", lambda _: next(password_resp)) - - # reset password of user of current connection (alice) - # should be nop - dj.set_password() - - # should be able to connect with old credentials - dj.Connection(**user_alice) - - -def test_set_password_args(user_alice: dict): - """ - Should be able to change the password with an argument - """ - # reset the connection to use alice's credentials - dj.conn(**user_alice, reset=True) - - # reset password of user of current connection (alice) - dj.set_password(new_password="newpass", update_config=False) - - # should be able to connect with new credentials - dj.Connection(host=user_alice["host"], user=user_alice["user"], password="newpass") - - -def test_set_password_update_config(monkeypatch, user_alice: dict): - """ - Should be able to change the password and update local config - """ - # reset the connection to use alice's credentials - dj.conn(**user_alice, reset=True) - - # respond yes to prompt to update local config - monkeypatch.setattr("builtins.input", lambda _: "yes") - - # reset password of user of current connection (alice) - dj.set_password(new_password="newpass") - - # should be able to connect with new credentials - dj.Connection(host=user_alice["host"], user=user_alice["user"], password="newpass") - - # check that local config is updated - # NOTE: the global config state is changed unless dj modules are reloaded - # NOTE: this test is a bit unrealistic because the config user does not match - # the user whose password is being updated, so the config credentials - # will be invalid after update... - assert dj.config["database.password"] == "newpass" - assert os.path.exists(dj.settings.LOCALCONFIG) - - -def test_set_password_conn(user_alice: dict): - """ - Should be able to change the password using a given connection - """ - # create a connection with alice's credentials - conn_alice = dj.Connection(**user_alice) - - # reset password of user of alice's connection (alice) and do not update config - dj.set_password(new_password="newpass", connection=conn_alice, update_config=False) - - # should be able to connect with new credentials - dj.Connection(host=user_alice["host"], user=user_alice["user"], password="newpass") - - # check that local config is not updated - assert dj.config["database.password"] == os.getenv("DJ_PASS") - assert not os.path.exists(dj.settings.LOCALCONFIG) diff --git a/tests/test_aggr_regressions.py b/tests/test_aggr_regressions.py deleted file mode 100644 index ea740cd39..000000000 --- a/tests/test_aggr_regressions.py +++ /dev/null @@ -1,130 +0,0 @@ -""" -Regression tests for issues 386, 449, 484, and 558 — all related to processing complex aggregations and projections. -""" - -import uuid - -import pytest - -import datajoint as dj - -from .schema_aggr_regress import LOCALS_AGGR_REGRESS, A, B, Q, R, S, X -from .schema_uuid import Item, Topic, top_level_namespace_id - - -@pytest.fixture(scope="function") -def schema_aggr_reg(connection_test, prefix): - schema = dj.Schema( - prefix + "_aggr_regress", - context=LOCALS_AGGR_REGRESS, - connection=connection_test, - ) - schema(R) - schema(Q) - schema(S) - yield schema - schema.drop() - - -@pytest.fixture(scope="function") -def schema_aggr_reg_with_abx(connection_test, prefix): - schema = dj.Schema( - prefix + "_aggr_regress_with_abx", - context=LOCALS_AGGR_REGRESS, - connection=connection_test, - ) - schema(R) - schema(Q) - schema(S) - schema(A) - schema(B) - schema(X) - yield schema - schema.drop() - - -def test_issue386(schema_aggr_reg): - """ - --------------- ISSUE 386 ------------------- - Issue 386 resulted from the loss of aggregated attributes when the aggregation was used as the restrictor - Q & (R.aggr(S, n='count(*)') & 'n=2') - Error: Unknown column 'n' in HAVING - """ - result = R.aggr(S, n="count(*)") & "n=10" - result = Q & result - result.fetch() - - -def test_issue449(schema_aggr_reg): - """ - ---------------- ISSUE 449 ------------------ - Issue 449 arises from incorrect group by attributes after joining with a dj.U() - """ - result = dj.U("n") * R.aggr(S, n="max(s)") - result.fetch() - - -def test_issue484(schema_aggr_reg): - """ - ---------------- ISSUE 484 ----------------- - Issue 484 - """ - q = dj.U().aggr(S, n="max(s)") - n = q.fetch("n") - n = q.fetch1("n") - q = dj.U().aggr(S, n="avg(s)") - result = dj.U().aggr(q, m="max(n)") - result.fetch() - - -def test_union_join(schema_aggr_reg_with_abx): - """ - This test fails if it runs after TestIssue558. - - https://github.com/datajoint/datajoint-python/issues/930 - """ - A.insert(zip([100, 200, 300, 400, 500, 600])) - B.insert([(100, 11), (200, 22), (300, 33), (400, 44)]) - q1 = B & "id < 300" - q2 = B & "id > 300" - - expected_data = [ - {"id": 0, "id2": 5}, - {"id": 1, "id2": 6}, - {"id": 2, "id2": 7}, - {"id": 3, "id2": 8}, - {"id": 4, "id2": 9}, - {"id": 100, "id2": 11}, - {"id": 200, "id2": 22}, - {"id": 400, "id2": 44}, - ] - - assert ((q1 + q2) * A).fetch(as_dict=True) == expected_data - - -class TestIssue558: - """ - --------------- ISSUE 558 ------------------ - Issue 558 resulted from the fact that DataJoint saves subqueries and often combines a restriction followed - by a projection into a single SELECT statement, which in several unusual cases produces unexpected results. - """ - - def test_issue558_part1(self, schema_aggr_reg_with_abx): - q = (A - B).proj(id2="3") - assert len(A - B) == len(q) - - def test_issue558_part2(self, schema_aggr_reg_with_abx): - d = dict(id=3, id2=5) - assert len(X & d) == len((X & d).proj(id2="3")) - - -def test_left_join_len(schema_uuid): - Topic().add("jeff") - Item.populate() - Topic().add("jeff2") - Topic().add("jeff3") - q = Topic.join( - Item - dict(topic_id=uuid.uuid5(top_level_namespace_id, "jeff")), left=True - ) - qf = q.fetch() - assert len(q) == len(qf) diff --git a/tests/test_alter.py b/tests/test_alter.py deleted file mode 100644 index 375d31d55..000000000 --- a/tests/test_alter.py +++ /dev/null @@ -1,68 +0,0 @@ -import re - -import pytest - -import datajoint as dj - -from . import schema as schema_any_module -from .schema_alter import LOCALS_ALTER, Experiment, Parent - -COMBINED_CONTEXT = { - **schema_any_module.LOCALS_ANY, - **LOCALS_ALTER, -} - - -@pytest.fixture -def schema_alter(connection_test, schema_any): - # Overwrite Experiment and Parent nodes - schema_any(Experiment, context=LOCALS_ALTER) - schema_any(Parent, context=LOCALS_ALTER) - yield schema_any - schema_any.drop() - - -class TestAlter: - def verify_alter(self, schema_alter, table, attribute_sql): - definition_original = schema_alter.connection.query( - f"SHOW CREATE TABLE {table.full_table_name}" - ).fetchone()[1] - table.definition = table.definition_new - table.alter(prompt=False) - definition_new = schema_alter.connection.query( - f"SHOW CREATE TABLE {table.full_table_name}" - ).fetchone()[1] - assert ( - re.sub(f"{attribute_sql},\n ", "", definition_new) == definition_original - ) - - def test_alter(self, schema_alter): - original = schema_alter.connection.query( - "SHOW CREATE TABLE " + Experiment.full_table_name - ).fetchone()[1] - Experiment.definition = Experiment.definition1 - Experiment.alter(prompt=False, context=COMBINED_CONTEXT) - altered = schema_alter.connection.query( - "SHOW CREATE TABLE " + Experiment.full_table_name - ).fetchone()[1] - assert original != altered - Experiment.definition = Experiment.original_definition - Experiment().alter(prompt=False, context=COMBINED_CONTEXT) - restored = schema_alter.connection.query( - "SHOW CREATE TABLE " + Experiment.full_table_name - ).fetchone()[1] - assert altered != restored - assert original == restored - - def test_alter_part(self, schema_alter): - """ - https://github.com/datajoint/datajoint-python/issues/936 - """ - self.verify_alter( - schema_alter, table=Parent.Child, attribute_sql="`child_id` .* DEFAULT NULL" - ) - self.verify_alter( - schema_alter, - table=Parent.Grandchild, - attribute_sql="`grandchild_id` .* DEFAULT NULL", - ) diff --git a/tests/test_bypass_serialization.py b/tests/test_bypass_serialization.py deleted file mode 100644 index da7f0b0e3..000000000 --- a/tests/test_bypass_serialization.py +++ /dev/null @@ -1,57 +0,0 @@ -import numpy as np -import pytest -from numpy.testing import assert_array_equal - -import datajoint as dj - -test_blob = np.array([1, 2, 3]) - - -class Input(dj.Lookup): - definition = """ - id: int - --- - data: blob - """ - contents = [(0, test_blob)] - - -class Output(dj.Manual): - definition = """ - id: int - --- - data: blob - """ - - -@pytest.fixture -def schema_in(connection_test, prefix): - schema = dj.Schema( - prefix + "_test_bypass_serialization_in", - context=dict(Input=Input), - connection=connection_test, - ) - schema(Input) - yield schema - schema.drop() - - -@pytest.fixture -def schema_out(connection_test, prefix): - schema = dj.Schema( - prefix + "_test_blob_bypass_serialization_out", - context=dict(Output=Output), - connection=connection_test, - ) - schema(Output) - yield schema - schema.drop() - - -def test_bypass_serialization(schema_in, schema_out): - dj.blob.bypass_serialization = True - contents = Input.fetch(as_dict=True) - assert isinstance(contents[0]["data"], bytes) - Output.insert(contents) - dj.blob.bypass_serialization = False - assert_array_equal(Input.fetch1("data"), Output.fetch1("data")) diff --git a/tests/test_external.py b/tests/test_external.py deleted file mode 100644 index 10021c0aa..000000000 --- a/tests/test_external.py +++ /dev/null @@ -1,134 +0,0 @@ -import os - -import numpy as np -from numpy.testing import assert_array_equal - -import datajoint as dj -from datajoint.blob import pack, unpack -from datajoint.external import ExternalTable - -from .schema_external import Simple, SimpleRemote - - -def test_external_put(schema_ext, mock_stores, mock_cache): - """ - external storage put and get and remove - """ - ext = ExternalTable( - schema_ext.connection, store="raw", database=schema_ext.database - ) - initial_length = len(ext) - input_ = np.random.randn(3, 7, 8) - count = 7 - extra = 3 - for i in range(count): - hash1 = ext.put(pack(input_)) - for i in range(extra): - hash2 = ext.put(pack(np.random.randn(4, 3, 2))) - - fetched_hashes = ext.fetch("hash") - assert all(hash in fetched_hashes for hash in (hash1, hash2)) - assert len(ext) == initial_length + 1 + extra - - output_ = unpack(ext.get(hash1)) - assert_array_equal(input_, output_) - - -class TestLeadingSlash: - def test_s3_leading_slash(self, schema_ext, mock_stores, mock_cache, minio_client): - """ - s3 external storage configured with leading slash - """ - self._leading_slash(schema_ext, index=100, store="share") - - def test_file_leading_slash( - self, schema_ext, mock_stores, mock_cache, minio_client - ): - """ - File external storage configured with leading slash - """ - self._leading_slash(schema_ext, index=200, store="local") - - def _leading_slash(self, schema_ext, index, store): - oldConfig = dj.config["stores"][store]["location"] - value = np.array([1, 2, 3]) - - id = index - dj.config["stores"][store]["location"] = "leading/slash/test" - SimpleRemote.insert([{"simple": id, "item": value}]) - assert np.array_equal( - value, (SimpleRemote & "simple={}".format(id)).fetch1("item") - ) - - id = index + 1 - dj.config["stores"][store]["location"] = "/leading/slash/test" - SimpleRemote.insert([{"simple": id, "item": value}]) - assert np.array_equal( - value, (SimpleRemote & "simple={}".format(id)).fetch1("item") - ) - - id = index + 2 - dj.config["stores"][store]["location"] = "leading\\slash\\test" - SimpleRemote.insert([{"simple": id, "item": value}]) - assert np.array_equal( - value, (SimpleRemote & "simple={}".format(id)).fetch1("item") - ) - - id = index + 3 - dj.config["stores"][store]["location"] = "f:\\leading\\slash\\test" - SimpleRemote.insert([{"simple": id, "item": value}]) - assert np.array_equal( - value, (SimpleRemote & "simple={}".format(id)).fetch1("item") - ) - - id = index + 4 - dj.config["stores"][store]["location"] = "f:\\leading/slash\\test" - SimpleRemote.insert([{"simple": id, "item": value}]) - assert np.array_equal( - value, (SimpleRemote & "simple={}".format(id)).fetch1("item") - ) - - id = index + 5 - dj.config["stores"][store]["location"] = "/" - SimpleRemote.insert([{"simple": id, "item": value}]) - assert np.array_equal( - value, (SimpleRemote & "simple={}".format(id)).fetch1("item") - ) - - id = index + 6 - dj.config["stores"][store]["location"] = "C:\\" - SimpleRemote.insert([{"simple": id, "item": value}]) - assert np.array_equal( - value, (SimpleRemote & "simple={}".format(id)).fetch1("item") - ) - - id = index + 7 - dj.config["stores"][store]["location"] = "" - SimpleRemote.insert([{"simple": id, "item": value}]) - assert np.array_equal( - value, (SimpleRemote & "simple={}".format(id)).fetch1("item") - ) - - dj.config["stores"][store]["location"] = oldConfig - - -def test_remove_fail(schema_ext, mock_stores, mock_cache, minio_client): - """ - https://github.com/datajoint/datajoint-python/issues/953 - """ - assert dj.config["stores"]["local"]["location"] - - data = dict(simple=2, item=[1, 2, 3]) - Simple.insert1(data) - path1 = dj.config["stores"]["local"]["location"] + "/djtest_extern/4/c/" - currentMode = int(oct(os.stat(path1).st_mode), 8) - os.chmod(path1, 0o40555) - (Simple & "simple=2").delete() - listOfErrors = schema_ext.external["local"].delete(delete_external_files=True) - - assert ( - len(schema_ext.external["local"] & dict(hash=listOfErrors[0][0])) == 1 - ), "unexpected number of rows in external table" - # ---------------------CLEAN UP-------------------- - os.chmod(path1, currentMode) - listOfErrors = schema_ext.external["local"].delete(delete_external_files=True) diff --git a/tests/test_external_class.py b/tests/test_external_class.py deleted file mode 100644 index 84597e52f..000000000 --- a/tests/test_external_class.py +++ /dev/null @@ -1,52 +0,0 @@ -from numpy.testing import assert_almost_equal - -import datajoint as dj - -from . import schema_external - - -def test_heading(schema_ext, mock_stores): - heading = schema_external.Simple().heading - assert "item" in heading - assert heading["item"].is_external - - -def test_insert_and_fetch(schema_ext, mock_stores, mock_cache): - original_list = [1, 3, 8] - schema_external.Simple().insert1(dict(simple=1, item=original_list)) - # test fetch - q = (schema_external.Simple() & {"simple": 1}).fetch("item")[0] - assert list(q) == original_list - # test fetch1 as a tuple - q = (schema_external.Simple() & {"simple": 1}).fetch1("item") - assert list(q) == original_list - # test fetch1 as a dict - q = (schema_external.Simple() & {"simple": 1}).fetch1() - assert list(q["item"]) == original_list - # test without cache - previous_cache = dj.config["cache"] - dj.config["cache"] = None - q = (schema_external.Simple() & {"simple": 1}).fetch1() - assert list(q["item"]) == original_list - # test with cache - dj.config["cache"] = previous_cache - q = (schema_external.Simple() & {"simple": 1}).fetch1() - assert list(q["item"]) == original_list - - -def test_populate(schema_ext, mock_stores): - image = schema_external.Image() - image.populate() - remaining, total = image.progress() - assert ( - total == len(schema_external.Dimension() * schema_external.Seed()) - and remaining == 0 - ) - for img, neg, dimensions in zip( - *(image * schema_external.Dimension()).fetch("img", "neg", "dimensions") - ): - assert list(img.shape) == list(dimensions) - assert_almost_equal(img, -neg) - image.delete() - for external_table in image.external.values(): - external_table.delete(display_progress=False, delete_external_files=True) diff --git a/tests/test_fetch.py b/tests/test_fetch.py deleted file mode 100644 index 7df767028..000000000 --- a/tests/test_fetch.py +++ /dev/null @@ -1,374 +0,0 @@ -import decimal -import io -import itertools -import logging -import os -import warnings -from operator import itemgetter -from typing import List - -import numpy as np -import pandas -import pytest - -import datajoint as dj - -from . import schema - - -def test_getattribute(subject): - """Testing Fetch.__call__ with attributes""" - list1 = sorted(subject.proj().fetch(as_dict=True), key=itemgetter("subject_id")) - list2 = sorted(subject.fetch(dj.key), key=itemgetter("subject_id")) - for l1, l2 in zip(list1, list2): - assert l1 == l2, "Primary key is not returned correctly" - - tmp = subject.fetch(order_by="subject_id") - - subject_notes, key, real_id = subject.fetch("subject_notes", dj.key, "real_id") - - np.testing.assert_array_equal(sorted(subject_notes), sorted(tmp["subject_notes"])) - np.testing.assert_array_equal(sorted(real_id), sorted(tmp["real_id"])) - list1 = sorted(key, key=itemgetter("subject_id")) - for l1, l2 in zip(list1, list2): - assert l1 == l2, "Primary key is not returned correctly" - - -def test_getattribute_for_fetch1(subject): - """Testing Fetch1.__call__ with attributes""" - assert (subject & "subject_id=10").fetch1("subject_id") == 10 - assert (subject & "subject_id=10").fetch1("subject_id", "species") == ( - 10, - "monkey", - ) - - -def test_order_by(lang, languages): - """Tests order_by sorting order""" - for ord_name, ord_lang in itertools.product(*2 * [["ASC", "DESC"]]): - cur = lang.fetch(order_by=("name " + ord_name, "language " + ord_lang)) - languages.sort(key=itemgetter(1), reverse=ord_lang == "DESC") - languages.sort(key=itemgetter(0), reverse=ord_name == "DESC") - for c, l in zip(cur, languages): - assert np.all( - cc == ll for cc, ll in zip(c, l) - ), "Sorting order is different" - - -def test_order_by_default(lang, languages): - """Tests order_by sorting order with defaults""" - cur = lang.fetch(order_by=("language", "name DESC")) - languages.sort(key=itemgetter(0), reverse=True) - languages.sort(key=itemgetter(1), reverse=False) - for c, l in zip(cur, languages): - assert np.all([cc == ll for cc, ll in zip(c, l)]), "Sorting order is different" - - -def test_limit(lang): - """Test the limit kwarg""" - limit = 4 - cur = lang.fetch(limit=limit) - assert len(cur) == limit, "Length is not correct" - - -def test_order_by_limit(lang, languages): - """Test the combination of order by and limit kwargs""" - cur = lang.fetch(limit=4, order_by=["language", "name DESC"]) - languages.sort(key=itemgetter(0), reverse=True) - languages.sort(key=itemgetter(1), reverse=False) - assert len(cur) == 4, "Length is not correct" - for c, l in list(zip(cur, languages))[:4]: - assert np.all([cc == ll for cc, ll in zip(c, l)]), "Sorting order is different" - - -def test_head_tail(schema_any): - query = schema.User * schema.Language - n = 5 - frame = query.head(n, format="frame") - assert isinstance(frame, pandas.DataFrame) - array = query.head(n, format="array") - assert array.size == n - assert len(frame) == n - assert query.primary_key == frame.index.names - - n = 4 - frame = query.tail(n, format="frame") - array = query.tail(n, format="array") - assert array.size == n - assert len(frame) == n - assert query.primary_key == frame.index.names - - -def test_limit_offset(lang, languages): - """Test the limit and offset kwargs together""" - cur = lang.fetch(offset=2, limit=4, order_by=["language", "name DESC"]) - languages.sort(key=itemgetter(0), reverse=True) - languages.sort(key=itemgetter(1), reverse=False) - assert len(cur) == 4, "Length is not correct" - for c, l in list(zip(cur, languages[2:6])): - assert np.all([cc == ll for cc, ll in zip(c, l)]), "Sorting order is different" - - -def test_iter(lang, languages): - """Test iterator""" - cur = lang.fetch(order_by=["language", "name DESC"]) - languages.sort(key=itemgetter(0), reverse=True) - languages.sort(key=itemgetter(1), reverse=False) - for (name, lang_val), (tname, tlang) in list(zip(cur, languages)): - assert name == tname and lang_val == tlang, "Values are not the same" - # now as dict - cur = lang.fetch(as_dict=True, order_by=("language", "name DESC")) - for row, (tname, tlang) in list(zip(cur, languages)): - assert ( - row["name"] == tname and row["language"] == tlang - ), "Values are not the same" - - -def test_keys(lang, languages): - """test key fetch""" - languages.sort(key=itemgetter(0), reverse=True) - languages.sort(key=itemgetter(1), reverse=False) - - lang = schema.Language() - cur = lang.fetch("name", "language", order_by=("language", "name DESC")) - cur2 = list(lang.fetch("KEY", order_by=["language", "name DESC"])) - - for c, c2 in zip(zip(*cur), cur2): - assert c == tuple(c2.values()), "Values are not the same" - - -def test_attributes_as_dict(subject): - """ - Issue #595 - """ - attrs = ("species", "date_of_birth") - result = subject.fetch(*attrs, as_dict=True) - assert bool(result) and len(result) == len(subject) - assert set(result[0]) == set(attrs) - - -def test_fetch1_step1(lang, languages): - assert ( - lang.contents - == languages - == [ - ("Fabian", "English"), - ("Edgar", "English"), - ("Dimitri", "English"), - ("Dimitri", "Ukrainian"), - ("Fabian", "German"), - ("Edgar", "Japanese"), - ] - ), "Unexpected contents in Language table" - key = {"name": "Edgar", "language": "Japanese"} - true = languages[-1] - dat = (lang & key).fetch1() - for k, (ke, c) in zip(true, dat.items()): - assert k == c == (lang & key).fetch1(ke), "Values are not the same" - - -def test_misspelled_attribute(schema_any): - with pytest.raises(dj.DataJointError): - f = (schema.Language & 'lang = "ENGLISH"').fetch() - - -def test_repr(subject): - """Test string representation of fetch, returning table preview""" - repr = subject.fetch.__repr__() - n = len(repr.strip().split("\n")) - limit = dj.config["display.limit"] - # 3 lines are used for headers (2) and summary statement (1) - assert n - 3 <= limit - - -def test_fetch_none(lang): - """Test preparing attributes for getitem""" - with pytest.raises(dj.DataJointError): - lang.fetch(None) - - -def test_asdict(lang): - """Test returns as dictionaries""" - d = lang.fetch(as_dict=True) - for dd in d: - assert isinstance(dd, dict) - - -def test_offset(lang, languages): - """Tests offset""" - cur = lang.fetch(limit=4, offset=1, order_by=["language", "name DESC"]) - - languages.sort(key=itemgetter(0), reverse=True) - languages.sort(key=itemgetter(1), reverse=False) - assert len(cur) == 4, "Length is not correct" - for c, l in list(zip(cur, languages[1:]))[:4]: - assert np.all([cc == ll for cc, ll in zip(c, l)]), "Sorting order is different" - - -def test_len(lang): - """Tests __len__""" - assert len(lang.fetch()) == len(lang), "__len__ is not behaving properly" - - -def test_fetch1_step2(lang): - """Tests whether fetch1 raises error""" - with pytest.raises(dj.DataJointError): - lang.fetch1() - - -def test_fetch1_step3(lang): - """Tests whether fetch1 raises error""" - with pytest.raises(dj.DataJointError): - lang.fetch1("name") - - -def test_decimal(schema_any): - """Tests that decimal fields are correctly fetched and used in restrictions, see issue #334""" - rel = schema.DecimalPrimaryKey() - assert len(rel.fetch()), "Table DecimalPrimaryKey contents are empty" - rel.insert1([decimal.Decimal("3.1415926")]) - keys = rel.fetch() - assert len(keys) > 0 - assert len(rel & keys[0]) == 1 - keys = rel.fetch(dj.key) - assert len(keys) >= 2 - assert len(rel & keys[1]) == 1 - - -def test_nullable_numbers(schema_any): - """test mixture of values and nulls in numeric attributes""" - table = schema.NullableNumbers() - table.insert( - ( - ( - k, - np.random.randn(), - np.random.randint(-1000, 1000), - np.random.randn(), - ) - for k in range(10) - ) - ) - table.insert1((100, None, None, None)) - f, d, i = table.fetch("fvalue", "dvalue", "ivalue") - assert None in i - assert any(np.isnan(d)) - assert any(np.isnan(f)) - - -def test_fetch_format(subject): - """test fetch_format='frame'""" - with dj.config(fetch_format="frame"): - # test if lists are both dicts - list1 = sorted(subject.proj().fetch(as_dict=True), key=itemgetter("subject_id")) - list2 = sorted(subject.fetch(dj.key), key=itemgetter("subject_id")) - for l1, l2 in zip(list1, list2): - assert l1 == l2, "Primary key is not returned correctly" - - # tests if pandas dataframe - tmp = subject.fetch(order_by="subject_id") - assert isinstance(tmp, pandas.DataFrame) - tmp = tmp.to_records() - - subject_notes, key, real_id = subject.fetch("subject_notes", dj.key, "real_id") - - np.testing.assert_array_equal( - sorted(subject_notes), sorted(tmp["subject_notes"]) - ) - np.testing.assert_array_equal(sorted(real_id), sorted(tmp["real_id"])) - list1 = sorted(key, key=itemgetter("subject_id")) - for l1, l2 in zip(list1, list2): - assert l1 == l2, "Primary key is not returned correctly" - - -def test_key_fetch1(subject): - """test KEY fetch1 - issue #976""" - with dj.config(fetch_format="array"): - k1 = (subject & "subject_id=10").fetch1("KEY") - with dj.config(fetch_format="frame"): - k2 = (subject & "subject_id=10").fetch1("KEY") - assert k1 == k2 - - -def test_same_secondary_attribute(schema_any): - children = (schema.Child * schema.Parent().proj()).fetch()["name"] - assert len(children) == 1 - assert children[0] == "Dan" - - -def test_query_caching(schema_any): - # initialize cache directory - os.mkdir(os.path.expanduser("~/dj_query_cache")) - - with dj.config(query_cache=os.path.expanduser("~/dj_query_cache")): - conn = schema.TTest3.connection - # insert sample data and load cache - schema.TTest3.insert([dict(key=100 + i, value=200 + i) for i in range(2)]) - conn.set_query_cache(query_cache="main") - cached_res = schema.TTest3().fetch() - # attempt to insert while caching enabled - try: - schema.TTest3.insert([dict(key=200 + i, value=400 + i) for i in range(2)]) - assert False, "Insert allowed while query caching enabled" - except dj.DataJointError: - conn.set_query_cache() - # insert new data - schema.TTest3.insert([dict(key=600 + i, value=800 + i) for i in range(2)]) - # re-enable cache to access old results - conn.set_query_cache(query_cache="main") - previous_cache = schema.TTest3().fetch() - # verify properly cached and how to refresh results - assert all([c == p for c, p in zip(cached_res, previous_cache)]) - conn.set_query_cache() - uncached_res = schema.TTest3().fetch() - assert len(uncached_res) > len(cached_res) - # purge query cache - conn.purge_query_cache() - - # reset cache directory state (will fail if purge was unsuccessful) - os.rmdir(os.path.expanduser("~/dj_query_cache")) - - -def test_fetch_group_by(schema_any): - """ - https://github.com/datajoint/datajoint-python/issues/914 - """ - - assert schema.Parent().fetch("KEY", order_by="name") == [{"parent_id": 1}] - - -def test_dj_u_distinct(schema_any): - """ - Test developed to see if removing DISTINCT from the select statement - generation breaks the dj.U universal set implementation - """ - - # Contents to be inserted - contents = [(1, 2, 3), (2, 2, 3), (3, 3, 2), (4, 5, 5)] - schema.Stimulus.insert(contents) - - # Query the whole table - test_query = schema.Stimulus() - - # Use dj.U to create a list of unique contrast and brightness combinations - result = dj.U("contrast", "brightness") & test_query - expected_result = [ - {"contrast": 2, "brightness": 3}, - {"contrast": 3, "brightness": 2}, - {"contrast": 5, "brightness": 5}, - ] - - fetched_result = result.fetch(as_dict=True, order_by=("contrast", "brightness")) - schema.Stimulus.delete_quick() - assert fetched_result == expected_result - - -def test_backslash(schema_any): - """ - https://github.com/datajoint/datajoint-python/issues/999 - """ - expected = "She\\Hulk" - schema.Parent.insert([(2, expected)]) - q = schema.Parent & dict(name=expected) - assert q.fetch1("name") == expected - q.delete() diff --git a/tests/test_filepath.py b/tests/test_filepath.py deleted file mode 100644 index cc3db2cc2..000000000 --- a/tests/test_filepath.py +++ /dev/null @@ -1,268 +0,0 @@ -import io -import logging -import os -import random -from pathlib import Path - -import pytest - -import datajoint as dj - -from .schema_external import Filepath, FilepathS3 - - -def test_path_match(schema_ext, enable_filepath_feature, minio_client, store="repo"): - """test file path matches and empty file""" - ext = schema_ext.external[store] - stage_path = dj.config["stores"][store]["stage"] - - # create a mock file - relpath = "path/to/films" - managed_file = Path(stage_path, relpath, "vid.mov") - managed_file.parent.mkdir(parents=True, exist_ok=True) - open(str(managed_file), "a").close() - - # put the file - uuid = ext.upload_filepath(str(managed_file)) - - # remove - managed_file.unlink() - assert not managed_file.exists() - - # check filepath - assert (ext & {"hash": uuid}).fetch1("filepath") == str( - managed_file.relative_to(stage_path).as_posix() - ) - - # # Download the file and check its contents. - restored_path, checksum = ext.download_filepath(uuid) - assert restored_path == str(managed_file) - assert checksum == dj.hash.uuid_from_file(str(managed_file)) - - # cleanup - ext.delete(delete_external_files=True) - - -@pytest.mark.parametrize("store", ("repo", "repo-s3")) -def test_filepath(enable_filepath_feature, schema_ext, store): - """test file management""" - ext = schema_ext.external[store] - stage_path = dj.config["stores"][store]["stage"] - filename = "picture.dat" - - # create a mock file - relpath = "one/two/three" - managed_file = Path(stage_path, relpath, filename) - managed_file.parent.mkdir(parents=True, exist_ok=True) - data = os.urandom(3000) - with managed_file.open("wb") as f: - f.write(data) - - # put the same file twice to ensure storing once - uuid1 = ext.upload_filepath(str(managed_file)) - # no duplication should arise if file is the same - uuid2 = ext.upload_filepath(str(managed_file)) - assert uuid1 == uuid2 - - # remove to ensure downloading - managed_file.unlink() - assert not managed_file.exists() - - # Download the file and check its contents. Repeat causes no download from remote - for _ in 1, 2: - restored_path, checksum = ext.download_filepath(uuid1) - assert restored_path == str(managed_file) - assert checksum == dj.hash.uuid_from_file(str(managed_file)) - - # verify same data - with managed_file.open("rb") as f: - synced_data = f.read() - assert data == synced_data - - # cleanup - ext.delete(delete_external_files=True) - assert not ext.exists(ext._make_external_filepath(str(Path(relpath, filename)))) - - -@pytest.mark.parametrize("store", ("repo", "repo-s3")) -def test_duplicate_upload(schema_ext, store): - ext = schema_ext.external[store] - stage_path = dj.config["stores"][store]["stage"] - relpath = "one/two/three" - managed_file = Path(stage_path, relpath, "plot.dat") - managed_file.parent.mkdir(parents=True, exist_ok=True) - with managed_file.open("wb") as f: - f.write(os.urandom(300)) - ext.upload_filepath(str(managed_file)) - ext.upload_filepath(str(managed_file)) # this is fine because the file is the same - - -@pytest.mark.parametrize("store", ("repo", "repo-s3")) -def test_duplicate_error(schema_ext, store): - """syncing duplicate non-matching file should fail""" - ext = schema_ext.external[store] - stage_path = dj.config["stores"][store]["stage"] - relpath = "one/two/three" - managed_file = Path(stage_path, relpath, "thesis.dat") - managed_file.parent.mkdir(parents=True, exist_ok=True) - with managed_file.open("wb") as f: - f.write(os.urandom(300)) - ext.upload_filepath(str(managed_file)) - with managed_file.open("wb") as f: - f.write(os.urandom(300)) - # this should raise exception because the file has changed - with pytest.raises(dj.DataJointError): - ext.upload_filepath(str(managed_file)) - - -class TestFilepath: - def _test_filepath_class( - self, table=Filepath(), store="repo", verify_checksum=True - ): - if not verify_checksum: - dj.config["filepath_checksum_size_limit"] = 0 - stage_path = dj.config["stores"][store]["stage"] - # create a mock file - relative_path = "one/two/three" - managed_file = Path(stage_path, relative_path, "attachment.dat") - managed_file.parent.mkdir(parents=True, exist_ok=True) - data = os.urandom(3000) - with managed_file.open("wb") as f: - f.write(data) - with managed_file.open("rb") as f: - contents = f.read() - assert data == contents - - # upload file into shared repo - table.insert1((1, str(managed_file))) - - # remove file locally - managed_file.unlink() - assert not managed_file.is_file() - - # fetch file from remote - filepath = (table & {"fnum": 1}).fetch1("img") - assert filepath == str(managed_file) - - # verify original contents - with managed_file.open("rb") as f: - contents = f.read() - assert data == contents - - # delete from table - table.delete() - assert table.external[store] - - # delete from external table - table.external[store].delete(delete_external_files=True) - dj.config["filepath_checksum_size_limit"] = None - - @pytest.mark.parametrize( - "table, store, n_repeats", - ( - (Filepath(), "repo", 2), - (FilepathS3(), "repo-s3", 2), - ), - ) - def test_filepath_class( - self, - schema_ext, - table, - store, - n_repeats, - minio_client, - enable_filepath_feature, - verify_checksum=True, - ): - for _ in range(n_repeats): - self._test_filepath_class(table, store, verify_checksum) - - def test_filepath_class_no_checksum(self, schema_ext, enable_filepath_feature): - logger = logging.getLogger("datajoint") - log_capture = io.StringIO() - stream_handler = logging.StreamHandler(log_capture) - log_format = logging.Formatter( - "[%(asctime)s][%(funcName)s][%(levelname)s]: %(message)s" - ) - stream_handler.setFormatter(log_format) - stream_handler.set_name("test_limit_warning") - logger.addHandler(stream_handler) - self._test_filepath_class(table=Filepath(), store="repo", verify_checksum=False) - log_contents = log_capture.getvalue() - log_capture.close() - for handler in logger.handlers: # Clean up handler - if handler.name == "test_limit_warning": - logger.removeHandler(handler) - assert "Skipped checksum for file with hash:" in log_contents - - -@pytest.mark.parametrize( - "table, store", - ( - (Filepath(), "repo"), - (FilepathS3(), "repo-s3"), - ), -) -def test_filepath_cleanup(table, store, schema_ext, enable_filepath_feature): - """test deletion of filepath entries from external table""" - stage_path = dj.config["stores"][store]["stage"] - n = 20 - contents = os.urandom(345) - for i in range(n): - relative_path = Path(*random.sample(("one", "two", "three", "four"), k=3)) - managed_file = Path(stage_path, relative_path, "file.dat") - managed_file.parent.mkdir(parents=True, exist_ok=True) - with managed_file.open("wb") as f: - f.write(contents) # same in all files - table.insert1((i, str(managed_file))) - assert len(table) == n - - ext = schema_ext.external[store] - - assert len(table) == n - assert 0 < len(ext) < n - - (table & "fnum in (1, 2, 3, 4, 5, 6)").delete() - m = n - len(table) # number deleted - assert m == 6 - - ext.delete(delete_external_files=True) # delete unused entries - assert 0 < len(ext) <= n - m - - -def test_delete_without_files( - schema_ext, - enable_filepath_feature, - store="repo", -): - """test deletion of filepath entries from external table without removing files""" - # do not delete unused entries - schema_ext.external[store].delete(delete_external_files=False) - - -def test_return_string( - schema_ext, enable_filepath_feature, table=Filepath(), store="repo" -): - """test returning string on fetch""" - stage_path = dj.config["stores"][store]["stage"] - # create a mock file - relative_path = "this/is/a/test" - managed_file = Path(stage_path, relative_path, "string.dat") - managed_file.parent.mkdir(parents=True, exist_ok=True) - data = os.urandom(3000) - with managed_file.open("wb") as f: - f.write(data) - with managed_file.open("rb") as f: - contents = f.read() - assert data == contents - - # upload file into shared repo - table.insert1((138, str(managed_file))) - - # remove file locally - managed_file.unlink() - assert not managed_file.is_file() - - # fetch file from remote - filepath = (table & {"fnum": 138}).fetch1("img") - assert isinstance(filepath, str) diff --git a/tests/test_jobs.py b/tests/test_jobs.py deleted file mode 100644 index dc363076d..000000000 --- a/tests/test_jobs.py +++ /dev/null @@ -1,149 +0,0 @@ -import random -import string - -import pytest - -import datajoint as dj -from datajoint.jobs import ERROR_MESSAGE_LENGTH, TRUNCATION_APPENDIX - -from . import schema - - -def test_reserve_job(subject, schema_any): - assert subject - table_name = "fake_table" - - # reserve jobs - for key in subject.fetch("KEY"): - assert schema_any.jobs.reserve(table_name, key), "failed to reserve a job" - - # refuse jobs - for key in subject.fetch("KEY"): - assert not schema_any.jobs.reserve( - table_name, key - ), "failed to respect reservation" - - # complete jobs - for key in subject.fetch("KEY"): - schema_any.jobs.complete(table_name, key) - assert not schema_any.jobs, "failed to free jobs" - - # reserve jobs again - for key in subject.fetch("KEY"): - assert schema_any.jobs.reserve(table_name, key), "failed to reserve new jobs" - - # finish with error - for key in subject.fetch("KEY"): - schema_any.jobs.error(table_name, key, "error message") - - # refuse jobs with errors - for key in subject.fetch("KEY"): - assert not schema_any.jobs.reserve( - table_name, key - ), "failed to ignore error jobs" - - # clear error jobs - (schema_any.jobs & dict(status="error")).delete() - assert not schema_any.jobs, "failed to clear error jobs" - - -def test_restrictions(schema_any): - jobs = schema_any.jobs - jobs.delete() - jobs.reserve("a", {"key": "a1"}) - jobs.reserve("a", {"key": "a2"}) - jobs.reserve("b", {"key": "b1"}) - jobs.error("a", {"key": "a2"}, "error") - jobs.error("b", {"key": "b1"}, "error") - - assert len(jobs & {"table_name": "a"}) == 2 - assert len(jobs & {"status": "error"}) == 2 - assert len(jobs & {"table_name": "a", "status": "error"}) == 1 - jobs.delete() - - -def test_sigint(schema_any): - try: - schema.SigIntTable().populate(reserve_jobs=True) - except KeyboardInterrupt: - pass - - assert len(schema_any.jobs.fetch()), "SigInt jobs table is empty" - status, error_message = schema_any.jobs.fetch1("status", "error_message") - assert status == "error" - assert error_message == "KeyboardInterrupt" - - -def test_sigterm(schema_any): - try: - schema.SigTermTable().populate(reserve_jobs=True) - except SystemExit: - pass - - assert len(schema_any.jobs.fetch()), "SigTerm jobs table is empty" - status, error_message = schema_any.jobs.fetch1("status", "error_message") - assert status == "error" - assert error_message == "SystemExit: SIGTERM received" - - -def test_suppress_dj_errors(schema_any): - """test_suppress_dj_errors: dj errors suppressible w/o native py blobs""" - with dj.config(enable_python_native_blobs=False): - schema.ErrorClass.populate(reserve_jobs=True, suppress_errors=True) - assert len(schema.DjExceptionName()) == len(schema_any.jobs) > 0 - - -def test_long_error_message(subject, schema_any): - # create long error message - long_error_message = "".join( - random.choice(string.ascii_letters) for _ in range(ERROR_MESSAGE_LENGTH + 100) - ) - short_error_message = "".join( - random.choice(string.ascii_letters) for _ in range(ERROR_MESSAGE_LENGTH // 2) - ) - assert subject - table_name = "fake_table" - - key = subject.fetch("KEY", limit=1)[0] - - # test long error message - schema_any.jobs.reserve(table_name, key) - schema_any.jobs.error(table_name, key, long_error_message) - error_message = schema_any.jobs.fetch1("error_message") - assert ( - len(error_message) == ERROR_MESSAGE_LENGTH - ), "error message is longer than max allowed" - assert error_message.endswith( - TRUNCATION_APPENDIX - ), "appropriate ending missing for truncated error message" - schema_any.jobs.delete() - - # test long error message - schema_any.jobs.reserve(table_name, key) - schema_any.jobs.error(table_name, key, short_error_message) - error_message = schema_any.jobs.fetch1("error_message") - assert error_message == short_error_message, "error messages do not agree" - assert not error_message.endswith( - TRUNCATION_APPENDIX - ), "error message should not be truncated" - schema_any.jobs.delete() - - -def test_long_error_stack(subject, schema_any): - # create long error stack - STACK_SIZE = ( - 89942 # Does not fit into small blob (should be 64k, but found to be higher) - ) - long_error_stack = "".join( - random.choice(string.ascii_letters) for _ in range(STACK_SIZE) - ) - assert subject - table_name = "fake_table" - - key = subject.fetch("KEY", limit=1)[0] - - # test long error stack - schema_any.jobs.reserve(table_name, key) - schema_any.jobs.error(table_name, key, "error message", long_error_stack) - error_stack = schema_any.jobs.fetch1("error_stack") - assert error_stack == long_error_stack, "error stacks do not agree" diff --git a/tests/test_log.py b/tests/test_log.py deleted file mode 100644 index 4b6e64613..000000000 --- a/tests/test_log.py +++ /dev/null @@ -1,5 +0,0 @@ -def test_log(schema_any): - ts, events = (schema_any.log & 'event like "Declared%%"').fetch( - "timestamp", "event" - ) - assert len(ts) >= 2 diff --git a/tests/test_relation_u.py b/tests/test_relation_u.py deleted file mode 100644 index 59cee0249..000000000 --- a/tests/test_relation_u.py +++ /dev/null @@ -1,80 +0,0 @@ -import pytest -from pytest import raises - -import datajoint as dj - -from .schema import * -from .schema_simple import * - - -def test_restriction(lang, languages, trial): - language_set = {s[1] for s in languages} - rel = dj.U("language") & lang - assert list(rel.heading.names) == ["language"] - assert len(rel) == len(language_set) - assert set(rel.fetch("language")) == language_set - # Test for issue #342 - rel = trial * dj.U("start_time") - assert list(rel.primary_key) == trial.primary_key + ["start_time"] - assert list(rel.primary_key) == list((rel & "trial_id>3").primary_key) - assert list((dj.U("start_time") & trial).primary_key) == ["start_time"] - - -def test_invalid_restriction(schema_any): - with raises(dj.DataJointError): - result = dj.U("color") & dict(color="red") - - -def test_ineffective_restriction(lang): - rel = lang & dj.U("language") - assert rel.make_sql() == lang.make_sql() - - -def test_join(experiment): - rel = experiment * dj.U("experiment_date") - assert experiment.primary_key == ["subject_id", "experiment_id"] - assert rel.primary_key == experiment.primary_key + ["experiment_date"] - - rel = dj.U("experiment_date") * experiment - assert experiment.primary_key == ["subject_id", "experiment_id"] - assert rel.primary_key == experiment.primary_key + ["experiment_date"] - - -def test_invalid_join(schema_any): - with raises(dj.DataJointError): - rel = dj.U("language") * dict(language="English") - - -def test_repr_without_attrs(schema_any): - """test dj.U() display""" - query = dj.U().aggr(Language, n="count(*)") - repr(query) - - -def test_aggregations(schema_any): - lang = Language() - # test total aggregation on expression object - n1 = dj.U().aggr(lang, n="count(*)").fetch1("n") - assert n1 == len(lang.fetch()) - # test total aggregation on expression class - n2 = dj.U().aggr(Language, n="count(*)").fetch1("n") - assert n1 == n2 - rel = dj.U("language").aggr(Language, number_of_speakers="count(*)") - assert len(rel) == len(set(l[1] for l in Language.contents)) - assert (rel & 'language="English"').fetch1("number_of_speakers") == 3 - - -def test_argmax(schema_any): - rel = TTest() - # get the tuples corresponding to the maximum value - mx = (rel * dj.U().aggr(rel, mx="max(value)")) & "mx=value" - assert mx.fetch("value")[0] == max(rel.fetch("value")) - - -def test_aggr(schema_any, schema_simp): - rel = ArgmaxTest() - amax1 = (dj.U("val") * rel) & dj.U("secondary_key").aggr(rel, val="min(val)") - amax2 = (dj.U("val") * rel) * dj.U("secondary_key").aggr(rel, val="min(val)") - assert ( - len(amax1) == len(amax2) == rel.n - ), "Aggregated argmax with join and restriction does not yield the same length." diff --git a/tests/test_s3.py b/tests/test_s3.py deleted file mode 100644 index 970310ca8..000000000 --- a/tests/test_s3.py +++ /dev/null @@ -1,50 +0,0 @@ -import pytest -from minio import Minio - -from datajoint.blob import pack -from datajoint.errors import DataJointError -from datajoint.hash import uuid_from_buffer - -from .schema_external import SimpleRemote - - -def test_connection(http_client, minio_client, s3_creds): - assert minio_client.bucket_exists(s3_creds["bucket"]) - - -def test_connection_secure(minio_client, s3_creds): - assert minio_client.bucket_exists(s3_creds["bucket"]) - - -def test_remove_object_exception(schema_ext, s3_creds): - # https://github.com/datajoint/datajoint-python/issues/952 - - # Insert some test data and remove it so that the external table is populated - test = [1, [1, 2, 3]] - SimpleRemote.insert1(test) - SimpleRemote.delete() - - # Save the old external table minio client - old_client = schema_ext.external["share"].s3.client - - # Apply our new minio client which has a user that does not exist - schema_ext.external["share"].s3.client = Minio( - s3_creds["endpoint"], - access_key="jeffjeff", - secret_key="jeffjeff", - secure=False, - ) - - # This method returns a list of errors - error_list = schema_ext.external["share"].delete( - delete_external_files=True, errors_as_string=False - ) - - # Teardown - schema_ext.external["share"].s3.client = old_client - schema_ext.external["share"].delete(delete_external_files=True) - - with pytest.raises(DataJointError): - # Raise the error we want if the error matches the expected uuid - if str(error_list[0][0]) == str(uuid_from_buffer(pack(test[1]))): - raise error_list[0][2] diff --git a/tests/test_settings.py b/tests/test_settings.py deleted file mode 100644 index 4eb8be539..000000000 --- a/tests/test_settings.py +++ /dev/null @@ -1,107 +0,0 @@ -import os -import pprint -import random -import string - -import pytest - -import datajoint as dj -from datajoint import DataJointError, settings - -__author__ = "Fabian Sinz" - - -def test_load_save(): - """Testing load and save""" - dj.config.save("tmp.json") - conf = settings.Config() - conf.load("tmp.json") - assert conf == dj.config - os.remove("tmp.json") - - -def test_singleton(): - """Testing singleton property""" - dj.config.save("tmp.json") - conf = settings.Config() - conf.load("tmp.json") - conf["dummy.val"] = 2 - - assert conf == dj.config - os.remove("tmp.json") - - -def test_singleton2(): - """Testing singleton property""" - conf = settings.Config() - conf["dummy.val"] = 2 - _ = settings.Config() # a new instance should not delete dummy.val - assert conf["dummy.val"] == 2 - - -def test_validator(): - """Testing validator""" - with pytest.raises(DataJointError): - dj.config["database.port"] = "harbor" - - -def test_del(): - """Testing del""" - dj.config["peter"] = 2 - assert "peter" in dj.config - del dj.config["peter"] - assert "peter" not in dj.config - - -def test_len(): - """Testing len""" - len(dj.config) == len(dj.config._conf) - - -def test_str(): - """Testing str""" - str(dj.config) == pprint.pformat(dj.config._conf, indent=4) - - -def test_repr(): - """Testing repr""" - repr(dj.config) == pprint.pformat(dj.config._conf, indent=4) - - -def test_save(): - """Testing save of config""" - tmpfile = "".join( - random.choice(string.ascii_uppercase + string.digits) for _ in range(20) - ) - moved = False - if os.path.isfile(settings.LOCALCONFIG): - os.rename(settings.LOCALCONFIG, tmpfile) - moved = True - dj.config.save_local() - assert os.path.isfile(settings.LOCALCONFIG) - if moved: - os.rename(tmpfile, settings.LOCALCONFIG) - - -def test_load_save(): - """Testing load and save of config""" - filename_old = dj.settings.LOCALCONFIG - filename = ( - "".join( - random.choice(string.ascii_uppercase + string.digits) for _ in range(50) - ) - + ".json" - ) - dj.settings.LOCALCONFIG = filename - dj.config.save_local() - dj.config.load(filename=filename) - dj.settings.LOCALCONFIG = filename_old - os.remove(filename) - - -def test_contextmanager(): - """Testing context manager""" - dj.config["arbitrary.stuff"] = 7 - with dj.config(arbitrary__stuff=10): - assert dj.config["arbitrary.stuff"] == 10 - assert dj.config["arbitrary.stuff"] == 7 diff --git a/tests/test_virtual_module.py b/tests/test_virtual_module.py deleted file mode 100644 index bd8a0c754..000000000 --- a/tests/test_virtual_module.py +++ /dev/null @@ -1,7 +0,0 @@ -import datajoint as dj -from datajoint.user_tables import UserTable - - -def test_virtual_module(schema_any, connection_test): - module = dj.VirtualModule("module", schema_any.database, connection=connection_test) - assert issubclass(module.Experiment, UserTable) diff --git a/tests/unit/__init__.py b/tests/unit/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/unit/test_codecs.py b/tests/unit/test_codecs.py new file mode 100644 index 000000000..ada626748 --- /dev/null +++ b/tests/unit/test_codecs.py @@ -0,0 +1,429 @@ +""" +Tests for the Codec system. +""" + +import pytest + +import datajoint as dj +from datajoint.codecs import ( + Codec, + _codec_registry, + get_codec, + is_codec_registered, + list_codecs, + resolve_dtype, + unregister_codec, +) +from datajoint.errors import DataJointError + + +class TestCodecRegistry: + """Tests for the codec registry functionality.""" + + def setup_method(self): + """Clear any test codecs from registry before each test.""" + for name in list(_codec_registry.keys()): + if name.startswith("test_"): + del _codec_registry[name] + + def teardown_method(self): + """Clean up test codecs after each test.""" + for name in list(_codec_registry.keys()): + if name.startswith("test_"): + del _codec_registry[name] + + def test_register_codec_auto(self): + """Test auto-registration via __init_subclass__.""" + + class TestCodec(Codec): + name = "test_decorator" + + def get_dtype(self, is_external: bool) -> str: + return "bytes" + + def encode(self, value, *, key=None, store_name=None): + return value + + def decode(self, stored, *, key=None): + return stored + + assert is_codec_registered("test_decorator") + assert get_codec("test_decorator").name == "test_decorator" + + def test_register_codec_skip(self): + """Test skipping registration with register=False.""" + + class TestCodec(Codec, register=False): + name = "test_skip" + + def get_dtype(self, is_external: bool) -> str: + return "varchar(255)" + + def encode(self, value, *, key=None, store_name=None): + return str(value) + + def decode(self, stored, *, key=None): + return stored + + assert not is_codec_registered("test_skip") + + def test_register_codec_idempotent(self): + """Test that defining the same codec class twice is idempotent.""" + + class TestCodec(Codec): + name = "test_idempotent" + + def get_dtype(self, is_external: bool) -> str: + return "int32" + + def encode(self, value, *, key=None, store_name=None): + return value + + def decode(self, stored, *, key=None): + return stored + + # Redefine the same name should not raise (same class) + assert is_codec_registered("test_idempotent") + + def test_register_duplicate_name_different_class(self): + """Test that registering different classes with same name raises error.""" + + class TestCodec1(Codec): + name = "test_duplicate" + + def get_dtype(self, is_external: bool) -> str: + return "int32" + + def encode(self, value, *, key=None, store_name=None): + return value + + def decode(self, stored, *, key=None): + return stored + + with pytest.raises(DataJointError, match="already registered"): + + class TestCodec2(Codec): + name = "test_duplicate" + + def get_dtype(self, is_external: bool) -> str: + return "varchar(100)" + + def encode(self, value, *, key=None, store_name=None): + return str(value) + + def decode(self, stored, *, key=None): + return stored + + def test_unregister_codec(self): + """Test unregistering a codec.""" + + class TestCodec(Codec): + name = "test_unregister" + + def get_dtype(self, is_external: bool) -> str: + return "int32" + + def encode(self, value, *, key=None, store_name=None): + return value + + def decode(self, stored, *, key=None): + return stored + + assert is_codec_registered("test_unregister") + unregister_codec("test_unregister") + assert not is_codec_registered("test_unregister") + + def test_get_codec_not_found(self): + """Test that getting an unregistered codec raises error.""" + with pytest.raises(DataJointError, match="Unknown codec"): + get_codec("nonexistent_codec") + + def test_list_codecs(self): + """Test listing registered codecs.""" + + class TestCodec(Codec): + name = "test_list" + + def get_dtype(self, is_external: bool) -> str: + return "int32" + + def encode(self, value, *, key=None, store_name=None): + return value + + def decode(self, stored, *, key=None): + return stored + + codecs = list_codecs() + assert "test_list" in codecs + assert codecs == sorted(codecs) # Should be sorted + + def test_get_codec_strips_brackets(self): + """Test that get_codec accepts names with or without angle brackets.""" + + class TestCodec(Codec): + name = "test_brackets" + + def get_dtype(self, is_external: bool) -> str: + return "int32" + + def encode(self, value, *, key=None, store_name=None): + return value + + def decode(self, stored, *, key=None): + return stored + + assert get_codec("test_brackets") is get_codec("") + + +class TestCodecValidation: + """Tests for the validate method.""" + + def setup_method(self): + for name in list(_codec_registry.keys()): + if name.startswith("test_"): + del _codec_registry[name] + + def teardown_method(self): + for name in list(_codec_registry.keys()): + if name.startswith("test_"): + del _codec_registry[name] + + def test_validate_called_default(self): + """Test that default validate accepts any value.""" + + class TestCodec(Codec): + name = "test_validate_default" + + def get_dtype(self, is_external: bool) -> str: + return "bytes" + + def encode(self, value, *, key=None, store_name=None): + return value + + def decode(self, stored, *, key=None): + return stored + + t = get_codec("test_validate_default") + # Default validate should not raise for any value + t.validate(None) + t.validate(42) + t.validate("string") + t.validate([1, 2, 3]) + + def test_validate_custom(self): + """Test custom validation logic.""" + + class PositiveIntCodec(Codec): + name = "test_positive_int" + + def get_dtype(self, is_external: bool) -> str: + return "int32" + + def encode(self, value, *, key=None, store_name=None): + return value + + def decode(self, stored, *, key=None): + return stored + + def validate(self, value): + if not isinstance(value, int): + raise TypeError(f"Expected int, got {type(value).__name__}") + if value < 0: + raise ValueError("Value must be positive") + + t = get_codec("test_positive_int") + t.validate(42) # Should pass + + with pytest.raises(TypeError): + t.validate("not an int") + + with pytest.raises(ValueError): + t.validate(-1) + + +class TestCodecChaining: + """Tests for codec chaining (dtype referencing another codec).""" + + def setup_method(self): + for name in list(_codec_registry.keys()): + if name.startswith("test_"): + del _codec_registry[name] + + def teardown_method(self): + for name in list(_codec_registry.keys()): + if name.startswith("test_"): + del _codec_registry[name] + + def test_resolve_native_dtype(self): + """Test resolving a native dtype.""" + final_dtype, chain, store = resolve_dtype("bytes") + assert final_dtype == "bytes" + assert chain == [] + assert store is None + + def test_resolve_custom_dtype(self): + """Test resolving a custom dtype.""" + + class TestCodec(Codec): + name = "test_resolve" + + def get_dtype(self, is_external: bool) -> str: + return "varchar(100)" + + def encode(self, value, *, key=None, store_name=None): + return value + + def decode(self, stored, *, key=None): + return stored + + final_dtype, chain, store = resolve_dtype("") + assert final_dtype == "varchar(100)" + assert len(chain) == 1 + assert chain[0].name == "test_resolve" + assert store is None + + def test_resolve_chained_dtype(self): + """Test resolving a chained dtype.""" + + class InnerCodec(Codec): + name = "test_inner" + + def get_dtype(self, is_external: bool) -> str: + return "bytes" + + def encode(self, value, *, key=None, store_name=None): + return value + + def decode(self, stored, *, key=None): + return stored + + class OuterCodec(Codec): + name = "test_outer" + + def get_dtype(self, is_external: bool) -> str: + return "" + + def encode(self, value, *, key=None, store_name=None): + return value + + def decode(self, stored, *, key=None): + return stored + + final_dtype, chain, store = resolve_dtype("") + assert final_dtype == "bytes" + assert len(chain) == 2 + assert chain[0].name == "test_outer" + assert chain[1].name == "test_inner" + assert store is None + + def test_circular_reference_detection(self): + """Test that circular codec references are detected.""" + + class CodecA(Codec): + name = "test_circular_a" + + def get_dtype(self, is_external: bool) -> str: + return "" + + def encode(self, value, *, key=None, store_name=None): + return value + + def decode(self, stored, *, key=None): + return stored + + class CodecB(Codec): + name = "test_circular_b" + + def get_dtype(self, is_external: bool) -> str: + return "" + + def encode(self, value, *, key=None, store_name=None): + return value + + def decode(self, stored, *, key=None): + return stored + + with pytest.raises(DataJointError, match="Circular codec reference"): + resolve_dtype("") + + +class TestExportsAndAPI: + """Test that the public API is properly exported.""" + + def test_exports_from_datajoint(self): + """Test that Codec and helpers are exported from datajoint.""" + assert hasattr(dj, "Codec") + assert hasattr(dj, "get_codec") + assert hasattr(dj, "list_codecs") + + +class TestBlobCodec: + """Tests for the built-in BlobCodec.""" + + def test_blob_is_registered(self): + """Test that blob is automatically registered.""" + assert is_codec_registered("blob") + + def test_blob_properties(self): + """Test BlobCodec properties.""" + blob_codec = get_codec("blob") + assert blob_codec.name == "blob" + assert blob_codec.get_dtype(is_external=False) == "bytes" + assert blob_codec.get_dtype(is_external=True) == "" + + def test_blob_encode_decode_roundtrip(self): + """Test that encode/decode is a proper roundtrip.""" + import numpy as np + + blob_codec = get_codec("blob") + + # Test with various data types + test_data = [ + {"key": "value", "number": 42}, + [1, 2, 3, 4, 5], + np.array([1.0, 2.0, 3.0]), + "simple string", + (1, 2, 3), + None, + ] + + for original in test_data: + encoded = blob_codec.encode(original) + assert isinstance(encoded, bytes) + decoded = blob_codec.decode(encoded) + if isinstance(original, np.ndarray): + np.testing.assert_array_equal(decoded, original) + else: + assert decoded == original + + def test_blob_encode_produces_valid_blob_format(self): + """Test that encoded data has valid blob protocol header.""" + blob_codec = get_codec("blob") + encoded = blob_codec.encode({"test": "data"}) + + # Should start with compression prefix or protocol header + valid_prefixes = (b"ZL123\0", b"mYm\0", b"dj0\0") + assert any(encoded.startswith(p) for p in valid_prefixes) + + def test_blob_in_list_codecs(self): + """Test that blob appears in list_codecs.""" + codecs = list_codecs() + assert "blob" in codecs + + def test_blob_handles_serialization(self): + """Test that BlobCodec handles serialization internally. + + With the new design: + - Plain bytes columns store/return raw bytes (no serialization) + - handles pack/unpack in encode/decode + """ + blob_codec = get_codec("blob") + + # BlobCodec.encode() should produce packed bytes + data = {"key": "value"} + encoded = blob_codec.encode(data) + assert isinstance(encoded, bytes) + + # BlobCodec.decode() should unpack back to original + decoded = blob_codec.decode(encoded) + assert decoded == data diff --git a/tests/unit/test_condition.py b/tests/unit/test_condition.py new file mode 100644 index 000000000..3200e34c4 --- /dev/null +++ b/tests/unit/test_condition.py @@ -0,0 +1,95 @@ +"""Unit tests for condition.py - Top class and merge logic.""" + +import pytest +from datajoint.condition import Top + + +class TestTopMerge: + """Tests for Top.merge() method.""" + + def test_merge_inherits_order(self): + """When other.order_by is None, ordering is inherited.""" + top1 = Top(limit=10, order_by="score desc") + top2 = Top(limit=5, order_by=None) + merged = top1.merge(top2) + assert merged.order_by == ["score desc"] + assert merged.limit == 5 + assert merged.offset == 0 + + def test_merge_limits_take_min(self): + """Merged limit is minimum of both.""" + top1 = Top(limit=10, order_by="id") + top2 = Top(limit=3, order_by=None) + merged = top1.merge(top2) + assert merged.limit == 3 + + # Reverse order + top1 = Top(limit=3, order_by="id") + top2 = Top(limit=10, order_by=None) + merged = top1.merge(top2) + assert merged.limit == 3 + + def test_merge_none_limit_preserved(self): + """None limit (unlimited) is handled correctly.""" + top1 = Top(limit=None, order_by="id") + top2 = Top(limit=5, order_by=None) + merged = top1.merge(top2) + assert merged.limit == 5 + + top1 = Top(limit=5, order_by="id") + top2 = Top(limit=None, order_by=None) + merged = top1.merge(top2) + assert merged.limit == 5 + + top1 = Top(limit=None, order_by="id") + top2 = Top(limit=None, order_by=None) + merged = top1.merge(top2) + assert merged.limit is None + + def test_merge_offsets_add(self): + """Offsets are added together.""" + top1 = Top(limit=10, order_by="id", offset=5) + top2 = Top(limit=3, order_by=None, offset=2) + merged = top1.merge(top2) + assert merged.offset == 7 + + def test_merge_preserves_existing_order(self): + """Merged Top preserves first Top's ordering.""" + top1 = Top(limit=10, order_by=["col1 desc", "col2 asc"]) + top2 = Top(limit=5, order_by=None) + merged = top1.merge(top2) + assert merged.order_by == ["col1 desc", "col2 asc"] + + +class TestTopValidation: + """Tests for Top validation.""" + + def test_order_by_none_allowed(self): + """order_by=None is valid (means inherit).""" + top = Top(limit=5, order_by=None) + assert top.order_by is None + + def test_order_by_string_converted_to_list(self): + """Single string order_by is converted to list.""" + top = Top(order_by="id desc") + assert top.order_by == ["id desc"] + + def test_order_by_list_preserved(self): + """List order_by is preserved.""" + top = Top(order_by=["col1", "col2 desc"]) + assert top.order_by == ["col1", "col2 desc"] + + def test_invalid_limit_type_raises(self): + """Non-integer limit raises TypeError.""" + with pytest.raises(TypeError): + Top(limit="5") + + def test_invalid_order_by_type_raises(self): + """Non-string order_by raises TypeError.""" + with pytest.raises(TypeError): + Top(order_by=123) + + def test_invalid_offset_type_raises(self): + """Non-integer offset raises TypeError.""" + with pytest.raises(TypeError): + Top(offset="1") diff --git a/tests/test_hash.py b/tests/unit/test_hash.py similarity index 100% rename from tests/test_hash.py rename to tests/unit/test_hash.py diff --git a/tests/unit/test_lazy_imports.py b/tests/unit/test_lazy_imports.py new file mode 100644 index 000000000..7c1dc4c9e --- /dev/null +++ b/tests/unit/test_lazy_imports.py @@ -0,0 +1,121 @@ +""" +Tests for lazy import behavior. + +These tests verify that heavy dependencies (networkx, matplotlib, click) +are not loaded until their associated features are accessed. +""" + +import sys + + +def test_lazy_diagram_import(): + """Diagram module should not be loaded until dj.Diagram is accessed.""" + # Remove datajoint from sys.modules to get fresh import + modules_to_remove = [key for key in sys.modules if key.startswith("datajoint")] + for mod in modules_to_remove: + del sys.modules[mod] + + # Import datajoint + import datajoint as dj + + # Diagram module should not be loaded yet + assert "datajoint.diagram" not in sys.modules, "diagram module loaded eagerly" + + # Access Diagram - should trigger lazy load + Diagram = dj.Diagram + assert "datajoint.diagram" in sys.modules, "diagram module not loaded after access" + assert Diagram.__name__ == "Diagram" + + +def test_lazy_admin_import(): + """Admin module should not be loaded until dj.kill is accessed.""" + # Remove datajoint from sys.modules to get fresh import + modules_to_remove = [key for key in sys.modules if key.startswith("datajoint")] + for mod in modules_to_remove: + del sys.modules[mod] + + # Import datajoint + import datajoint as dj + + # Admin module should not be loaded yet + assert "datajoint.admin" not in sys.modules, "admin module loaded eagerly" + + # Access kill - should trigger lazy load + kill = dj.kill + assert "datajoint.admin" in sys.modules, "admin module not loaded after access" + assert callable(kill) + + +def test_lazy_cli_import(): + """CLI module should not be loaded until dj.cli is accessed.""" + # Remove datajoint from sys.modules to get fresh import + modules_to_remove = [key for key in sys.modules if key.startswith("datajoint")] + for mod in modules_to_remove: + del sys.modules[mod] + + # Import datajoint + import datajoint as dj + + # CLI module should not be loaded yet + assert "datajoint.cli" not in sys.modules, "cli module loaded eagerly" + + # Access cli - should trigger lazy load and return the function + cli_func = dj.cli + assert "datajoint.cli" in sys.modules, "cli module not loaded after access" + assert callable(cli_func), "dj.cli should be callable (the cli function)" + + +def test_diagram_module_access(): + """dj.diagram should return the diagram module for accessing module-level attrs.""" + # Remove datajoint from sys.modules to get fresh import + modules_to_remove = [key for key in sys.modules if key.startswith("datajoint")] + for mod in modules_to_remove: + del sys.modules[mod] + + import datajoint as dj + + # Access dj.diagram should return the module + diagram_module = dj.diagram + assert hasattr(diagram_module, "diagram_active"), "diagram module should have diagram_active" + assert hasattr(diagram_module, "Diagram"), "diagram module should have Diagram class" + + +def test_diagram_aliases(): + """Di and ERD should be aliases for Diagram.""" + # Remove datajoint from sys.modules to get fresh import + modules_to_remove = [key for key in sys.modules if key.startswith("datajoint")] + for mod in modules_to_remove: + del sys.modules[mod] + + import datajoint as dj + + # All aliases should resolve to the same class + assert dj.Diagram is dj.Di + assert dj.Diagram is dj.ERD + + +def test_core_imports_available(): + """Core functionality should be available immediately after import.""" + # Remove datajoint from sys.modules to get fresh import + modules_to_remove = [key for key in sys.modules if key.startswith("datajoint")] + for mod in modules_to_remove: + del sys.modules[mod] + + import datajoint as dj + + # Core classes should be available without triggering lazy loads + assert hasattr(dj, "Schema") + assert hasattr(dj, "Table") + assert hasattr(dj, "Manual") + assert hasattr(dj, "Lookup") + assert hasattr(dj, "Computed") + assert hasattr(dj, "Imported") + assert hasattr(dj, "Part") + assert hasattr(dj, "Connection") + assert hasattr(dj, "config") + assert hasattr(dj, "errors") + + # Heavy modules should still not be loaded + assert "datajoint.diagram" not in sys.modules + assert "datajoint.admin" not in sys.modules + assert "datajoint.cli" not in sys.modules diff --git a/tests/unit/test_pk_rules.py b/tests/unit/test_pk_rules.py new file mode 100644 index 000000000..2c554091b --- /dev/null +++ b/tests/unit/test_pk_rules.py @@ -0,0 +1,207 @@ +""" +Unit tests for primary key determination rules. + +These tests verify the functional dependency logic used to determine +primary keys in join operations. +""" + +from datajoint.heading import Heading + + +def make_heading(pk_attrs, secondary_attrs=None): + """Helper to create a Heading with specified PK and secondary attributes.""" + secondary_attrs = secondary_attrs or [] + attrs = [] + for name in pk_attrs: + attrs.append( + { + "name": name, + "type": "int", + "original_type": None, + "in_key": True, + "nullable": False, + "default": None, + "comment": "", + "autoincrement": False, + "numeric": True, + "string": False, + "uuid": False, + "json": False, + "is_blob": False, + "is_hidden": False, + "codec": None, + "store": None, + "unsupported": False, + "attribute_expression": None, + "dtype": object, + "lineage": None, + } + ) + for name in secondary_attrs: + attrs.append( + { + "name": name, + "type": "int", + "original_type": None, + "in_key": False, + "nullable": True, + "default": None, + "comment": "", + "autoincrement": False, + "numeric": True, + "string": False, + "uuid": False, + "json": False, + "is_blob": False, + "is_hidden": False, + "codec": None, + "store": None, + "unsupported": False, + "attribute_expression": None, + "dtype": object, + "lineage": None, + } + ) + return Heading(attrs) + + +class TestDetermines: + """Tests for Heading.determines() method.""" + + def test_a_determines_b_when_b_pk_subset_of_a(self): + """A → B when all of B's PK is in A.""" + a = make_heading(["x", "y"], ["z"]) + b = make_heading(["x"]) + assert a.determines(b) + + def test_a_determines_b_when_b_pk_in_a_secondary(self): + """A → B when B's PK attrs are in A's secondary.""" + a = make_heading(["x"], ["y", "z"]) + b = make_heading(["y"]) + assert a.determines(b) + + def test_a_not_determines_b_when_attr_missing(self): + """A ↛ B when B has PK attr not in A at all.""" + a = make_heading(["x", "y"]) + b = make_heading(["x", "z"]) + assert not a.determines(b) + + def test_both_determine_each_other(self): + """Both A → B and B → A can be true (bijection-like).""" + a = make_heading(["x", "y"], ["z"]) + b = make_heading(["y", "z"], ["x"]) + assert a.determines(b) + assert b.determines(a) + + def test_neither_determines(self): + """Neither direction when each has attrs not in the other.""" + a = make_heading(["x", "y"]) + b = make_heading(["y", "z"]) + assert not a.determines(b) + assert not b.determines(a) + + def test_empty_pk_always_determined(self): + """Empty PK is always determined by any heading.""" + a = make_heading(["x", "y"]) + b = make_heading([]) + assert a.determines(b) + + def test_session_trial_example(self): + """Classic FK example: Trial → Session (session_id in Trial's PK).""" + session = make_heading(["session_id"], ["date"]) + trial = make_heading(["session_id", "trial_num"], ["stimulus"]) + # Session → Trial? No (trial_num not in Session) + assert not session.determines(trial) + # Trial → Session? Yes (session_id in Trial) + assert trial.determines(session) + + +class TestJoinPrimaryKey: + """Tests for Heading.join() primary key determination.""" + + def test_join_a_determines_b(self): + """When A → B, result PK = PK(A).""" + a = make_heading(["x", "y"], ["z"]) + b = make_heading(["x"]) + result = a.join(b) + assert result.primary_key == ["x", "y"] + + def test_join_b_determines_a(self): + """When B → A (not A → B), result PK = PK(B), B's attrs first.""" + a = make_heading(["x", "y"]) + b = make_heading(["x", "z"], ["y"]) + # A → B? No (z not in A) + # B → A? Yes (y is secondary in B) + result = a.join(b) + assert result.primary_key == ["x", "z"] + # B's attributes should come first + assert result.names[0] == "x" + assert result.names[1] == "z" + + def test_join_both_determine(self): + """When both A → B and B → A, prefer A (left operand).""" + a = make_heading(["x", "y"], ["z"]) + b = make_heading(["y", "z"], ["x"]) + result = a.join(b) + assert result.primary_key == ["x", "y"] + + def test_join_neither_determines(self): + """When neither determines, result PK = union.""" + a = make_heading(["x", "y"]) + b = make_heading(["y", "z"]) + result = a.join(b) + # PK should be union: {x, y, z} + assert set(result.primary_key) == {"x", "y", "z"} + # A's PK first, then B's new PK attrs + assert result.primary_key == ["x", "y", "z"] + + def test_join_preserves_secondary_attrs(self): + """Secondary attributes should be preserved in join.""" + a = make_heading(["x"], ["a"]) + b = make_heading(["x"], ["b"]) + result = a.join(b) + assert "a" in result.secondary_attributes + assert "b" in result.secondary_attributes + + def test_join_session_trial(self): + """Session * Trial should have Trial's PK.""" + session = make_heading(["session_id"], ["date"]) + trial = make_heading(["session_id", "trial_num"], ["stimulus"]) + result = session.join(trial) + # B → A, so PK = PK(B) = {session_id, trial_num} + assert set(result.primary_key) == {"session_id", "trial_num"} + + def test_join_nullable_pk_forces_union(self): + """nullable_pk=True should force union PK.""" + a = make_heading(["x", "y"], ["z"]) + b = make_heading(["x"]) + # Normally A → B, so PK = PK(A) + normal_result = a.join(b) + assert normal_result.primary_key == ["x", "y"] + # With nullable_pk=True, force union + nullable_result = a.join(b, nullable_pk=True) + assert nullable_result.primary_key == ["x", "y"] # Still same since B's PK is subset + + +class TestJoinAttributeOrdering: + """Tests for attribute ordering in join results.""" + + def test_a_determines_b_ordering(self): + """When A → B, A's attributes come first.""" + a = make_heading(["x"], ["a"]) + b = make_heading(["x"], ["b"]) + result = a.join(b) + names = result.names + assert names.index("x") < names.index("a") + assert names.index("a") < names.index("b") + + def test_b_determines_a_ordering(self): + """When B → A, B's attributes come first.""" + a = make_heading(["x", "y"]) + b = make_heading(["x", "z"], ["y"]) + result = a.join(b) + names = result.names + # B's attrs first: x, z, then A's non-overlapping attrs + assert names.index("x") < names.index("z") + # y should be secondary (demoted from A's PK) + assert "y" in result.secondary_attributes diff --git a/tests/unit/test_settings.py b/tests/unit/test_settings.py new file mode 100644 index 000000000..66d817f0c --- /dev/null +++ b/tests/unit/test_settings.py @@ -0,0 +1,468 @@ +"""Tests for DataJoint settings module.""" + +from pathlib import Path + +import pytest +from pydantic import SecretStr, ValidationError + +import datajoint as dj +from datajoint import settings +from datajoint.errors import DataJointError +from datajoint.settings import ( + CONFIG_FILENAME, + SECRETS_DIRNAME, + find_config_file, + find_secrets_dir, + read_secret_file, +) + + +class TestConfigFileSearch: + """Test recursive config file search.""" + + def test_find_in_current_directory(self, tmp_path): + """Config file in current directory is found.""" + config_file = tmp_path / CONFIG_FILENAME + config_file.write_text("{}") + + found = find_config_file(tmp_path) + assert found == config_file + + def test_find_in_parent_directory(self, tmp_path): + """Config file in parent directory is found.""" + subdir = tmp_path / "src" / "pipeline" + subdir.mkdir(parents=True) + config_file = tmp_path / CONFIG_FILENAME + config_file.write_text("{}") + + found = find_config_file(subdir) + assert found == config_file + + def test_stop_at_git_boundary(self, tmp_path): + """Search stops at .git directory.""" + (tmp_path / ".git").mkdir() + subdir = tmp_path / "src" + subdir.mkdir() + # No config file - should return None, not search above .git + + found = find_config_file(subdir) + assert found is None + + def test_stop_at_hg_boundary(self, tmp_path): + """Search stops at .hg directory.""" + (tmp_path / ".hg").mkdir() + subdir = tmp_path / "src" + subdir.mkdir() + + found = find_config_file(subdir) + assert found is None + + def test_config_found_before_git(self, tmp_path): + """Config file found before reaching .git boundary.""" + (tmp_path / ".git").mkdir() + config_file = tmp_path / CONFIG_FILENAME + config_file.write_text("{}") + subdir = tmp_path / "src" + subdir.mkdir() + + found = find_config_file(subdir) + assert found == config_file + + def test_returns_none_when_not_found(self, tmp_path): + """Returns None when no config file exists.""" + (tmp_path / ".git").mkdir() # Create boundary + subdir = tmp_path / "src" + subdir.mkdir() + + found = find_config_file(subdir) + assert found is None + + +class TestSecretsDirectory: + """Test secrets directory detection and loading.""" + + def test_find_secrets_next_to_config(self, tmp_path): + """Finds .secrets/ directory next to config file.""" + config_file = tmp_path / CONFIG_FILENAME + config_file.write_text("{}") + secrets_dir = tmp_path / SECRETS_DIRNAME + secrets_dir.mkdir() + + found = find_secrets_dir(config_file) + assert found == secrets_dir + + def test_no_secrets_dir_returns_none(self, tmp_path): + """Returns None when no secrets directory exists.""" + config_file = tmp_path / CONFIG_FILENAME + config_file.write_text("{}") + + found = find_secrets_dir(config_file) + # May return system secrets dir if it exists, otherwise None + if found is not None: + assert found == settings.SYSTEM_SECRETS_DIR + + def test_read_secret_file(self, tmp_path): + """Reads secret value from file.""" + (tmp_path / "database.password").write_text("my_secret\n") + + value = read_secret_file(tmp_path, "database.password") + assert value == "my_secret" # Strips whitespace + + def test_read_missing_secret_returns_none(self, tmp_path): + """Returns None for missing secret file.""" + value = read_secret_file(tmp_path, "nonexistent") + assert value is None + + def test_read_secret_from_none_dir(self): + """Returns None when secrets_dir is None.""" + value = read_secret_file(None, "database.password") + assert value is None + + +class TestSecretStr: + """Test SecretStr handling for sensitive fields.""" + + def test_password_is_secret_str(self): + """Password field uses SecretStr type.""" + dj.config.database.password = "test_password" + assert isinstance(dj.config.database.password, SecretStr) + dj.config.database.password = None + + def test_secret_str_masked_in_repr(self): + """SecretStr values are masked in repr.""" + dj.config.database.password = "super_secret" + repr_str = repr(dj.config.database.password) + assert "super_secret" not in repr_str + assert "**" in repr_str + dj.config.database.password = None + + def test_dict_access_unwraps_secret(self): + """Dict-style access returns plain string for secrets.""" + dj.config.database.password = "unwrapped_secret" + value = dj.config["database.password"] + assert value == "unwrapped_secret" + assert isinstance(value, str) + assert not isinstance(value, SecretStr) + dj.config.database.password = None + + def test_aws_secret_key_is_secret_str(self): + """AWS secret key uses SecretStr type.""" + dj.config.external.aws_secret_access_key = "aws_secret" + assert isinstance(dj.config.external.aws_secret_access_key, SecretStr) + dj.config.external.aws_secret_access_key = None + + +class TestSettingsAccess: + """Test accessing settings via different methods.""" + + def test_attribute_access(self): + """Test accessing settings via attributes.""" + # Host can be localhost or db (docker), just verify it's a string + assert isinstance(dj.config.database.host, str) + assert len(dj.config.database.host) > 0 + # Port may be 3306 (default) or a random port (testcontainers) + assert isinstance(dj.config.database.port, int) + assert 1 <= dj.config.database.port <= 65535 + # safemode may be modified by conftest fixtures + assert isinstance(dj.config.safemode, bool) + + def test_dict_style_access(self): + """Test accessing settings via dict-style notation.""" + # Host can be localhost or db (docker), just verify it's a string + assert isinstance(dj.config["database.host"], str) + assert len(dj.config["database.host"]) > 0 + # Port may be 3306 (default) or a random port (testcontainers) + assert isinstance(dj.config["database.port"], int) + assert 1 <= dj.config["database.port"] <= 65535 + # safemode may be modified by conftest fixtures + assert isinstance(dj.config["safemode"], bool) + + def test_get_with_default(self): + """Test get() method with default values.""" + # Host can be localhost or db (docker), just verify it exists + assert dj.config.get("database.host") is not None + assert dj.config.get("nonexistent.key", "default") == "default" + assert dj.config.get("nonexistent.key") is None + + +class TestSettingsModification: + """Test modifying settings.""" + + def test_attribute_assignment(self): + """Test setting values via attribute assignment.""" + original = dj.config.database.host + try: + dj.config.database.host = "testhost" + assert dj.config.database.host == "testhost" + finally: + dj.config.database.host = original + + def test_dict_style_assignment(self): + """Test setting values via dict-style notation.""" + original = dj.config["database.host"] + try: + dj.config["database.host"] = "testhost2" + assert dj.config["database.host"] == "testhost2" + finally: + dj.config["database.host"] = original + + +class TestTypeValidation: + """Test pydantic type validation.""" + + def test_port_must_be_integer(self): + """Test that port must be an integer.""" + with pytest.raises(ValidationError): + dj.config.database.port = "not_an_integer" + + def test_loglevel_validation(self): + """Test that loglevel must be a valid level.""" + with pytest.raises(ValidationError): + dj.config.loglevel = "INVALID_LEVEL" + + def test_fetch_format_validation(self): + """Test that fetch_format must be array or frame.""" + with pytest.raises(ValidationError): + dj.config.fetch_format = "invalid" + + +class TestContextManager: + """Test the override context manager.""" + + def test_override_simple_value(self): + """Test overriding a simple value.""" + original = dj.config.safemode + with dj.config.override(safemode=False): + assert dj.config.safemode is False + assert dj.config.safemode == original + + def test_override_nested_value(self): + """Test overriding nested values with double underscore.""" + original = dj.config.database.host + with dj.config.override(database__host="override_host"): + assert dj.config.database.host == "override_host" + assert dj.config.database.host == original + + def test_override_restores_on_exception(self): + """Test that override restores values even when exception occurs.""" + original = dj.config.safemode + try: + with dj.config.override(safemode=False): + assert dj.config.safemode is False + raise ValueError("test exception") + except ValueError: + pass + assert dj.config.safemode == original + + +class TestLoad: + """Test loading configuration.""" + + def test_load_config_file(self, tmp_path, monkeypatch): + """Test loading configuration from file. + + Note: Environment variables take precedence over config file values. + We need to clear DJ_HOST to test file loading. + """ + filename = tmp_path / "test_config.json" + filename.write_text('{"database": {"host": "loaded_host"}}') + original_host = dj.config.database.host + + # Clear env var so file value takes effect + monkeypatch.delenv("DJ_HOST", raising=False) + + try: + dj.config.load(filename) + assert dj.config.database.host == "loaded_host" + finally: + dj.config.database.host = original_host + + def test_env_var_overrides_config_file(self, tmp_path, monkeypatch): + """Test that environment variables take precedence over config file. + + When DJ_HOST is set, loading a config file should NOT override the value. + The env var value should be preserved. + """ + filename = tmp_path / "test_config.json" + filename.write_text('{"database": {"host": "file_host"}}') + original_host = dj.config.database.host + + # Set env var - it should take precedence over file + monkeypatch.setenv("DJ_HOST", "env_host") + # Reset config to pick up new env var + dj.config.database.host = "env_host" + + try: + dj.config.load(filename) + # File value should be skipped because DJ_HOST is set + # The env var value should be preserved + assert dj.config.database.host == "env_host" + finally: + dj.config.database.host = original_host + + def test_load_nonexistent_file(self): + """Test loading nonexistent file raises FileNotFoundError.""" + with pytest.raises(FileNotFoundError): + dj.config.load("/nonexistent/path/config.json") + + +class TestStoreSpec: + """Test external store configuration.""" + + def test_get_store_spec_not_configured(self): + """Test getting unconfigured store raises error.""" + with pytest.raises(DataJointError, match="not configured"): + dj.config.get_store_spec("nonexistent_store") + + def test_get_store_spec_file_protocol(self): + """Test file protocol store spec validation.""" + original_stores = dj.config.stores.copy() + try: + dj.config.stores["test_file"] = { + "protocol": "file", + "location": "/tmp/test", + } + spec = dj.config.get_store_spec("test_file") + assert spec["protocol"] == "file" + assert spec["location"] == "/tmp/test" + assert spec["subfolding"] == settings.DEFAULT_SUBFOLDING + finally: + dj.config.stores = original_stores + + def test_get_store_spec_missing_required(self): + """Test missing required keys raises error.""" + original_stores = dj.config.stores.copy() + try: + dj.config.stores["bad_store"] = { + "protocol": "file", + # missing location + } + with pytest.raises(DataJointError, match="missing"): + dj.config.get_store_spec("bad_store") + finally: + dj.config.stores = original_stores + + +class TestDisplaySettings: + """Test display-related settings.""" + + def test_display_limit(self): + """Test display limit setting.""" + original = dj.config.display.limit + try: + dj.config.display.limit = 50 + assert dj.config.display.limit == 50 + finally: + dj.config.display.limit = original + + +class TestCachePaths: + """Test cache path settings.""" + + def test_cache_path_string(self): + """Test setting cache path as string.""" + original = dj.config.cache + try: + dj.config.cache = "/tmp/cache" + assert dj.config.cache == Path("/tmp/cache") + finally: + dj.config.cache = original + + def test_cache_path_none(self): + """Test cache path can be None.""" + original = dj.config.cache + try: + dj.config.cache = None + assert dj.config.cache is None + finally: + dj.config.cache = original + + +class TestSaveTemplate: + """Test save_template method for creating configuration templates.""" + + def test_save_minimal_template(self, tmp_path): + """Test creating a minimal template.""" + config_path = tmp_path / "datajoint.json" + result = dj.config.save_template(config_path, minimal=True, create_secrets_dir=False) + + assert result == config_path.absolute() + assert config_path.exists() + + import json + + with open(config_path) as f: + content = json.load(f) + + assert "database" in content + assert content["database"]["host"] == "localhost" + assert content["database"]["port"] == 3306 + # Minimal template should not have credentials + assert "password" not in content["database"] + assert "user" not in content["database"] + + def test_save_full_template(self, tmp_path): + """Test creating a full template.""" + config_path = tmp_path / "datajoint.json" + result = dj.config.save_template(config_path, minimal=False, create_secrets_dir=False) + + assert result == config_path.absolute() + assert config_path.exists() + + import json + + with open(config_path) as f: + content = json.load(f) + + # Full template should have all settings groups + assert "database" in content + assert "connection" in content + assert "display" in content + assert "object_storage" in content + assert "stores" in content + assert "loglevel" in content + assert "safemode" in content + # But still no credentials + assert "password" not in content["database"] + assert "user" not in content["database"] + + def test_save_template_creates_secrets_dir(self, tmp_path): + """Test that save_template creates .secrets/ directory.""" + config_path = tmp_path / "datajoint.json" + dj.config.save_template(config_path, create_secrets_dir=True) + + secrets_dir = tmp_path / SECRETS_DIRNAME + assert secrets_dir.exists() + assert secrets_dir.is_dir() + + # Check placeholder files created + assert (secrets_dir / "database.user").exists() + assert (secrets_dir / "database.password").exists() + + # Check .gitignore created + gitignore = secrets_dir / ".gitignore" + assert gitignore.exists() + assert "*" in gitignore.read_text() + + def test_save_template_refuses_overwrite(self, tmp_path): + """Test that save_template won't overwrite existing file.""" + config_path = tmp_path / "datajoint.json" + config_path.write_text("{}") + + with pytest.raises(FileExistsError, match="already exists"): + dj.config.save_template(config_path) + + def test_save_template_secrets_dir_idempotent(self, tmp_path): + """Test that creating secrets dir doesn't overwrite existing secrets.""" + config_path = tmp_path / "datajoint.json" + secrets_dir = tmp_path / SECRETS_DIRNAME + secrets_dir.mkdir() + + # Pre-populate a secret + password_file = secrets_dir / "database.password" + password_file.write_text("existing_password") + + dj.config.save_template(config_path, create_secrets_dir=True) + + # Original password should be preserved + assert password_file.read_text() == "existing_password" diff --git a/tests/unit/test_storage_urls.py b/tests/unit/test_storage_urls.py new file mode 100644 index 000000000..649d695b2 --- /dev/null +++ b/tests/unit/test_storage_urls.py @@ -0,0 +1,121 @@ +"""Unit tests for storage URL functions.""" + +import pytest + +from datajoint.storage import ( + URL_PROTOCOLS, + is_url, + normalize_to_url, + parse_url, +) + + +class TestURLProtocols: + """Test URL protocol constants.""" + + def test_url_protocols_includes_file(self): + """URL_PROTOCOLS should include file://.""" + assert "file://" in URL_PROTOCOLS + + def test_url_protocols_includes_s3(self): + """URL_PROTOCOLS should include s3://.""" + assert "s3://" in URL_PROTOCOLS + + def test_url_protocols_includes_cloud_providers(self): + """URL_PROTOCOLS should include major cloud providers.""" + assert "gs://" in URL_PROTOCOLS + assert "az://" in URL_PROTOCOLS + + +class TestIsUrl: + """Test is_url function.""" + + def test_s3_url(self): + assert is_url("s3://bucket/key") + + def test_gs_url(self): + assert is_url("gs://bucket/key") + + def test_file_url(self): + assert is_url("file:///path/to/file") + + def test_http_url(self): + assert is_url("http://example.com/file") + + def test_https_url(self): + assert is_url("https://example.com/file") + + def test_local_path_not_url(self): + assert not is_url("/path/to/file") + + def test_relative_path_not_url(self): + assert not is_url("relative/path/file.dat") + + def test_case_insensitive(self): + assert is_url("S3://bucket/key") + assert is_url("FILE:///path") + + +class TestNormalizeToUrl: + """Test normalize_to_url function.""" + + def test_local_path_to_file_url(self): + url = normalize_to_url("/data/file.dat") + assert url.startswith("file://") + assert "data/file.dat" in url + + def test_s3_url_unchanged(self): + url = "s3://bucket/key/file.dat" + assert normalize_to_url(url) == url + + def test_file_url_unchanged(self): + url = "file:///data/file.dat" + assert normalize_to_url(url) == url + + def test_relative_path_becomes_absolute(self): + url = normalize_to_url("relative/path.dat") + assert url.startswith("file://") + # Should be absolute (contain full path) + assert "/" in url[7:] # After "file://" + + +class TestParseUrl: + """Test parse_url function.""" + + def test_parse_s3(self): + protocol, path = parse_url("s3://bucket/key/file.dat") + assert protocol == "s3" + assert path == "bucket/key/file.dat" + + def test_parse_gs(self): + protocol, path = parse_url("gs://bucket/key") + assert protocol == "gcs" + assert path == "bucket/key" + + def test_parse_gcs(self): + protocol, path = parse_url("gcs://bucket/key") + assert protocol == "gcs" + assert path == "bucket/key" + + def test_parse_file(self): + protocol, path = parse_url("file:///data/file.dat") + assert protocol == "file" + assert path == "/data/file.dat" + + def test_parse_http(self): + protocol, path = parse_url("http://example.com/file") + assert protocol == "http" + assert path == "example.com/file" + + def test_parse_https(self): + protocol, path = parse_url("https://example.com/file") + assert protocol == "https" + assert path == "example.com/file" + + def test_unsupported_protocol_raises(self): + with pytest.raises(Exception, match="Unsupported URL protocol"): + parse_url("ftp://example.com/file") + + def test_local_path_raises(self): + with pytest.raises(Exception, match="Unsupported URL protocol"): + parse_url("/local/path")