diff --git a/CHANGES.md b/CHANGES.md index a565368..edb0c9f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,16 @@ +## Version 0.0.3 (in development) + +* **Bug fix** - `ExternalPythonOperator` does not need Airflow in external environment now. + +* Jupyter Lab can now be started in any conda/mamba environment via Gaiaflow. + +* Added user workflow diagram in the `Overview` page of the documentation. + +* Added a new subcommand `gaiaflow dev update-deps --help` to update + dependencies on the fly in Airflow containers for workflow tasks. + + + ## Version 0.0.2 * **Chore**: Update `pyproject.toml` to include `README.md` and necessary links. diff --git a/docs/index.md b/docs/index.md index 42f066e..b0b2abc 100644 --- a/docs/index.md +++ b/docs/index.md @@ -215,3 +215,64 @@ Quick rule of thumb: - Use `prod_local` for testing end-to-end workflows on production-like settngs. - Use `prod` for production pipelines in the real cluster. +## User workflow: +A typical user workflow could look like this: + +```mermaid +flowchart TD + A1[Start]--> A + A{Prerequisites installed except template?}-->|Yes| D{Which OS?} + A -->|No| STOP[Stop] + + D -->|Windows WSL2| E1[Install template via conda/mamba/miniforge prompt
then create env in WSL2 CLI + install gaiaflow] + D -->|Linux| E2[Install template + create env
then install gaiaflow] + E1 --> M + E2 --> M + + M[Start services in Dev Mode
`gaiaflow start dev -p .`] --> N[Experiment in JupyterLab if needed] + + N --> P[Refactor to production code
inside Python package and logging experiements to MLFlow and artifacts to S3 MinIO if needed] + P --> Q[Write tests in tests/ folder] + Q --> R[Create workflows with create_task mode=dev in Airflow] + R --> S[Run & monitor DAG in Airflow] + S --> T[Track experiments in MLflow & outputs in MinIO] + T --> V{Workflow ran successfully in dev mode?} + V --> |Yes| V1{Test your package in docker mode?} + + V1 --> |Yes| V2[Change create_task mode to dev_docker and run gaiaflow dev dockerize -p .] + + V2 --> V3{Docker mode ran successfully?} + V3 --> |Yes| V4{Move to production?} + + V4 --> |Yes| X1[Start Prod-Local infra
`gaiaflow prod-local start -p .`] + + V --> |No| V5[Fix bugs] + V1 --> |No| STOP[Stop] + V3 --> |No| V6[Fix bugs] + V4 --> |No| STOP[Stop] + + V5 --> R + V6 --> V3 + + X1 --> X2[Build Docker image
`gaiaflow prod-local dockerize -p .`] + X2 --> X3[Configure secrets if needed] + X3 --> Y{Prod-Local Success?} + Y2 --> Y + Y -->|Yes| Y1[Set create_task mode to prod - Local services are not needed anymore] + Y1 --> Z[Deploy to Production Cluster ] + + Y -->|No| Y2[Fix the bugs] + + class A,D,U,Y,V,V1,V3,V4 decision; + class E1,F1,G1,H1 windows; + class E2,G2,H2 linux; + class Z prod; + class STOP,STOP2 stop; + + classDef decision fill:#FFDD99,stroke:#E67E22,stroke-width:2px; + classDef windows fill:#AED6F1,stroke:#2874A6,stroke-width:2px; + classDef linux fill:#ABEBC6,stroke:#1E8449,stroke-width:2px; + classDef prod fill:#D7BDE2,stroke:#7D3C98,stroke-width:2px; + classDef stop fill:#F5B7B1,stroke:#922B21,stroke-width:2px; + +``` diff --git a/docs/start.md b/docs/start.md index 4a19ed3..3d5ed10 100644 --- a/docs/start.md +++ b/docs/start.md @@ -207,6 +207,8 @@ This means now you have successfully installed Docker. ### Gaiaflow Installation +In Windows, install `gaiaflow` using WSL2 terminal in a new mamba/conda environment. + You can install Gaiaflow directly via pip: `pip install gaiaflow` diff --git a/mkdocs.yml b/mkdocs.yml index 8031b5c..c68d348 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -76,8 +76,14 @@ markdown_extensions: - pymdownx.emoji: emoji_index: !!python/name:material.extensions.emoji.twemoji emoji_generator: !!python/name:material.extensions.emoji.to_svg + - pymdownx.superfences: + custom_fences: + - name: mermaid + class: mermaid + format: !!python/name:pymdownx.superfences.fence_code_format extra: + mermaid: true social: - icon: fontawesome/brands/github link: https://github.com/bcdev/gaiaflow diff --git a/pixi.lock b/pixi.lock index 0c2c03e..0cf2fe8 100644 --- a/pixi.lock +++ b/pixi.lock @@ -41,6 +41,8 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/arrow-1.3.0-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/asgiref-3.9.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/asn1crypto-1.5.1-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/asttokens-3.0.0-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/async-lru-2.0.5-pyh29332c3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/attrs-25.3.0-pyh71513ae_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/aws-c-auth-0.9.0-h0fbd49f_19.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/aws-c-cal-0.9.2-he7b75e1_1.conda @@ -68,6 +70,8 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/bcrypt-4.3.0-py312h680f630_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/beautifulsoup4-4.13.4-pyha770c72_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/billiard-4.2.1-py312h66e93f0_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/bleach-6.2.0-pyh29332c3_4.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/bleach-with-css-6.2.0-h82add2a_4.conda - conda: https://conda.anaconda.org/conda-forge/noarch/blinker-1.9.0-pyhff2d567_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/boto3-1.40.2-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/botocore-1.40.2-pyge310_1234567_0.conda @@ -77,6 +81,8 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/bzip2-1.0.8-h4bc722e_7.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/c-ares-1.34.5-hb9d3cd8_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025.8.3-hbd8a1cb_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/cached-property-1.5.2-hd8ed1ab_1.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/noarch/cached_property-1.5.2-pyha770c72_1.tar.bz2 - conda: https://conda.anaconda.org/conda-forge/noarch/cachelib-0.13.0-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/cachetools-5.5.2-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/cadwyn-5.4.4-pyhd8ed1ab_0.conda @@ -93,6 +99,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/colorama-0.4.6-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/colorlog-6.9.0-pyh707e725_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/colour-0.1.5-pyhd8ed1ab_2.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/comm-0.2.3-pyhe01879c_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/connexion-2.14.2-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/contourpy-1.3.3-py312hd9148b4_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/coverage-7.10.4-py312h8a5da7c_0.conda @@ -101,6 +108,9 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/cryptography-45.0.5-py312hda17c39_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/cycler-0.12.1-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/databricks-sdk-0.61.0-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/debugpy-1.8.16-py312h8285ef7_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/decorator-5.2.1-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/defusedxml-0.7.1-pyhd8ed1ab_0.tar.bz2 - conda: https://conda.anaconda.org/conda-forge/noarch/deprecated-1.2.18-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/dill-0.4.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/dnspython-2.7.0-pyhff2d567_1.conda @@ -111,6 +121,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/eval-type-backport-0.2.2-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/eval_type_backport-0.2.2-pyha770c72_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/exceptiongroup-1.3.0-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/executing-2.2.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/fastapi-0.116.1-h26c32bb_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/fastapi-cli-0.0.8-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/fastapi-core-0.116.1-pyhe01879c_1.conda @@ -124,6 +135,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/flask-sqlalchemy-2.5.1-pyhd8ed1ab_0.tar.bz2 - conda: https://conda.anaconda.org/conda-forge/noarch/flask-wtf-1.2.2-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/fonttools-4.59.0-py312h8a5da7c_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/fqdn-1.5.1-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/freetype-2.13.3-ha770c72_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/frozenlist-1.7.0-py312h447239a_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/fsspec-2025.7.0-pyhd8ed1ab_0.conda @@ -155,18 +167,40 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/inflection-0.5.1-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/iniconfig-2.0.0-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/intervals-0.9.2-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/ipykernel-6.30.1-pyh82676e8_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/ipython-9.4.0-pyhfa0c392_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/ipython_pygments_lexers-1.1.1-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/ipywidgets-8.1.7-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/isoduration-20.11.0-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/itsdangerous-2.2.0-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jedi-0.19.2-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/jinja2-3.1.6-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/jmespath-1.0.1-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/joblib-1.5.1-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/json5-0.12.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/jsonpath-ng-1.7.0-pyhe01879c_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/jsonpointer-3.0.0-py312h7900ff3_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/jsonschema-4.25.0-pyhe01879c_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/jsonschema-specifications-2025.4.1-pyh29332c3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jsonschema-with-format-nongpl-4.25.0-he01879c_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyter-1.1.1-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyter-lsp-2.2.6-pyhe01879c_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyter_client-8.6.3-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyter_console-6.6.3-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyter_core-5.8.1-pyh31011fe_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyter_events-0.12.0-pyh29332c3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyter_server-2.16.0-pyhe01879c_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyter_server_terminals-0.5.3-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyterlab-4.4.6-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyterlab_pygments-0.3.0-pyhd8ed1ab_2.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyterlab_server-2.27.3-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyterlab_widgets-3.0.15-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/keyutils-1.6.1-h166bdaf_0.tar.bz2 - conda: https://conda.anaconda.org/conda-forge/linux-64/kiwisolver-1.4.8-py312h68727a3_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/kombu-5.5.4-pyha770c72_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/krb5-1.21.3-h659f571_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/kubernetes_asyncio-32.3.2-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/lark-1.2.2-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/lazy-object-proxy-1.11.0-py312h66e93f0_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/lcms2-2.17-h717163a_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/ld_impl_linux-64-2.44-h1423503_1.conda @@ -241,9 +275,11 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/marshmallow-4.0.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/marshmallow-sqlalchemy-1.4.2-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/matplotlib-base-3.10.5-py312he3d6523_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/matplotlib-inline-0.1.7-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/mdurl-0.1.2-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/mergedeep-1.3.4-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/methodtools-0.4.7-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/mistune-3.1.3-pyh29332c3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/mkdocs-1.6.1-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/mkdocs-autorefs-1.4.2-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/mkdocs-get-deps-0.2.0-pyhd8ed1ab_1.conda @@ -258,8 +294,14 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/msgspec-0.19.0-py312h66e93f0_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/multidict-6.6.3-py312h178313f_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/munkres-1.1.4-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/nbclient-0.10.2-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/nbconvert-core-7.16.6-pyh29332c3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/nbformat-5.10.4-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/ncurses-6.5-h2d0b736_3.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/nest-asyncio-1.6.0-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/nlohmann_json-3.12.0-h3f2d84a_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/notebook-7.4.5-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/notebook-shim-0.2.4-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/numpy-2.3.2-py312h33ff503_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/oauthlib-3.3.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/openjpeg-2.5.3-h55fea9a_1.conda @@ -275,14 +317,19 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/orc-2.2.0-h1bc01a4_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/ordered-set-4.1.0-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/orderedmultidict-1.0.1-pyhd8ed1ab_2.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/overrides-7.7.0-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/packaging-25.0-pyh29332c3_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/paginate-0.5.7-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/pandas-2.2.3-py312hf9745cd_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/pandocfilters-1.5.0-pyhd8ed1ab_0.tar.bz2 - conda: https://conda.anaconda.org/conda-forge/noarch/paramiko-3.5.1-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/parso-0.8.4-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/passlib-1.7.4-pyhd8ed1ab_2.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pathspec-0.12.1-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/pendulum-3.1.0-py312h12e396e_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/pexpect-4.9.0-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/phonenumbers-9.0.10-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/pickleshare-0.7.5-pyhd8ed1ab_1004.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/pillow-11.3.0-py312h80c1187_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pixi-pycharm-0.0.8-unix_hf108a03_2.conda - conda: https://conda.anaconda.org/conda-forge/noarch/platformdirs-4.3.8-pyhe01879c_0.conda @@ -298,6 +345,8 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/protobuf-6.31.1-py312h7e1e8fc_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/psutil-7.0.0-py312h66e93f0_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/pthread-stubs-0.4-hb9d3cd8_1002.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/ptyprocess-0.7.0-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/pure_eval-0.2.3-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/pyarrow-20.0.0-py312h7900ff3_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/pyarrow-core-20.0.0-py312h01725c0_0_cpu.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pyasn1-0.6.1-pyhd8ed1ab_2.conda @@ -319,6 +368,8 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/python-daemon-3.1.2-pyhff2d567_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python-dateutil-2.9.0.post0-pyhe01879c_2.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python-dotenv-1.1.1-pyhe01879c_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python-fastjsonschema-2.21.2-pyhe01879c_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python-json-logger-2.0.7-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python-kubernetes-32.0.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python-multipart-0.0.20-pyhff2d567_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python-slugify-8.0.4-pyhd8ed1ab_1.conda @@ -329,6 +380,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/pywin32-on-windows-0.1.0-pyh1179c8e_3.tar.bz2 - conda: https://conda.anaconda.org/conda-forge/linux-64/pyyaml-6.0.2-py312h178313f_2.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pyyaml-env-tag-1.1-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/pyzmq-27.0.2-py312h6748674_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/qhull-2020.2-h434a139_5.conda - conda: https://conda.anaconda.org/conda-forge/noarch/querystring_parser-1.2.4-pyhd8ed1ab_2.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/re2-2025.07.22-h5a314c3_0.conda @@ -339,6 +391,9 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/requests-oauthlib-2.0.0-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/requests-toolbelt-1.0.0-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/retryhttp-1.3.3-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/rfc3339-validator-0.1.4-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/rfc3986-validator-0.1.1-pyh9f0ad1d_0.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/noarch/rfc3987-syntax-1.1.0-pyhe01879c_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/rich-13.9.4-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/rich-argparse-1.7.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/rich-toolkit-0.14.9-pyhe01879c_0.conda @@ -352,6 +407,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/scikit-learn-1.7.1-py312h4f0b9e3_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/scipy-1.16.0-py312hf734454_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/scramp-1.4.4-pyhd8ed1ab_0.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/noarch/send2trash-1.8.3-pyh0d859eb_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/setproctitle-1.3.6-py312h66e93f0_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/setuptools-80.9.0-pyhff2d567_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/shellingham-1.5.4-pyhd8ed1ab_1.conda @@ -375,18 +431,23 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/sqlalchemy-utils-timezone-0.41.2-hd8ed1ab_2.conda - conda: https://conda.anaconda.org/conda-forge/noarch/sqlalchemy-utils-url-0.41.2-hd8ed1ab_2.conda - conda: https://conda.anaconda.org/conda-forge/noarch/sqlparse-0.5.3-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/stack_data-0.6.3-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/starlette-0.47.2-pyh82d4cca_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/structlog-25.4.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/svcs-25.1.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/tabulate-0.9.0-pyhd8ed1ab_2.conda - conda: https://conda.anaconda.org/conda-forge/noarch/tenacity-9.1.2-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/termcolor-3.1.0-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/terminado-0.18.1-pyh0d859eb_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/text-unidecode-1.3-pyhd8ed1ab_2.conda - conda: https://conda.anaconda.org/conda-forge/noarch/threadpoolctl-3.6.0-pyhecae5ae_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/time-machine-2.16.0-py312h66e93f0_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/tinycss2-1.4.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/tk-8.6.13-noxft_hd72426e_102.conda - conda: https://conda.anaconda.org/conda-forge/noarch/toml-0.10.2-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/tomli-2.2.1-pyhe01879c_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/tornado-6.5.2-py312h4c3975b_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/traitlets-5.14.3-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/typer-0.16.0-pyh167b9f4_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/typer-slim-0.16.0-pyhe01879c_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/typer-slim-standard-0.16.0-hf964461_0.conda @@ -395,11 +456,13 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/typing-extensions-4.14.1-h4440ef1_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/typing-inspection-0.4.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/typing_extensions-4.14.1-pyhe01879c_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/typing_utils-0.1.0-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2025b-h78e105d_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/uc-micro-py-1.0.3-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/unicodedata2-16.0.0-py312h66e93f0_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/universal-pathlib-0.2.6-hd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/universal_pathlib-0.2.6-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/uri-template-1.3.0-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/urllib3-2.5.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/uuid6-2025.0.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/uvicorn-0.35.0-pyh31011fe_0.conda @@ -410,9 +473,12 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/watchfiles-1.1.0-py312h12e396e_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/watchtower-3.4.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/wcwidth-0.2.13-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/webcolors-24.11.1-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/webencodings-0.5.1-pyhd8ed1ab_3.conda - conda: https://conda.anaconda.org/conda-forge/noarch/websocket-client-1.8.0-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/websockets-15.0.1-py312h66e93f0_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/werkzeug-2.2.3-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/widgetsnbextension-4.0.14-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/wirerope-1.0.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/wrapt-1.17.2-py312h66e93f0_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/wtforms-3.2.1-pyhd8ed1ab_1.conda @@ -420,6 +486,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/xorg-libxdmcp-1.1.5-hb9d3cd8_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/yaml-0.2.5-h280c20c_3.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/yarl-1.20.1-py312h178313f_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/zeromq-4.3.5-h3b0a872_7.conda - conda: https://conda.anaconda.org/conda-forge/noarch/zipp-3.23.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/zlib-1.3.1-hb9d3cd8_2.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/zstandard-0.23.0-py312h66e93f0_2.conda @@ -968,6 +1035,32 @@ packages: - pkg:pypi/asn1crypto?source=hash-mapping size: 85881 timestamp: 1734342825337 +- conda: https://conda.anaconda.org/conda-forge/noarch/asttokens-3.0.0-pyhd8ed1ab_1.conda + sha256: 93b14414b3b3ed91e286e1cbe4e7a60c4e1b1c730b0814d1e452a8ac4b9af593 + md5: 8f587de4bcf981e26228f268df374a9b + depends: + - python >=3.9 + constrains: + - astroid >=2,<4 + license: Apache-2.0 + license_family: Apache + purls: + - pkg:pypi/asttokens?source=hash-mapping + size: 28206 + timestamp: 1733250564754 +- conda: https://conda.anaconda.org/conda-forge/noarch/async-lru-2.0.5-pyh29332c3_0.conda + sha256: 3b7233041e462d9eeb93ea1dfe7b18aca9c358832517072054bb8761df0c324b + md5: d9d0f99095a9bb7e3641bca8c6ad2ac7 + depends: + - python >=3.9 + - typing_extensions >=4.0.0 + - python + license: MIT + license_family: MIT + purls: + - pkg:pypi/async-lru?source=hash-mapping + size: 17335 + timestamp: 1742153708859 - conda: https://conda.anaconda.org/conda-forge/noarch/attrs-25.3.0-pyh71513ae_0.conda sha256: 99c53ffbcb5dc58084faf18587b215f9ac8ced36bbfb55fa807c00967e419019 md5: a10d11958cadc13fdb43df75f8b1903f @@ -1341,6 +1434,30 @@ packages: - pkg:pypi/billiard?source=hash-mapping size: 215186 timestamp: 1726941272577 +- conda: https://conda.anaconda.org/conda-forge/noarch/bleach-6.2.0-pyh29332c3_4.conda + sha256: a05971bb80cca50ce9977aad3f7fc053e54ea7d5321523efc7b9a6e12901d3cd + md5: f0b4c8e370446ef89797608d60a564b3 + depends: + - python >=3.9 + - webencodings + - python + constrains: + - tinycss >=1.1.0,<1.5 + license: Apache-2.0 AND MIT + purls: + - pkg:pypi/bleach?source=hash-mapping + size: 141405 + timestamp: 1737382993425 +- conda: https://conda.anaconda.org/conda-forge/noarch/bleach-with-css-6.2.0-h82add2a_4.conda + sha256: 0aba699344275b3972bd751f9403316edea2ceb942db12f9f493b63c74774a46 + md5: a30e9406c873940383555af4c873220d + depends: + - bleach ==6.2.0 pyh29332c3_4 + - tinycss2 + license: Apache-2.0 AND MIT + purls: [] + size: 4213 + timestamp: 1737382993425 - conda: https://conda.anaconda.org/conda-forge/noarch/blinker-1.9.0-pyhff2d567_0.conda sha256: f7efd22b5c15b400ed84a996d777b6327e5c402e79e3c534a7e086236f1eb2dc md5: 42834439227a4551b939beeeb8a4b085 @@ -1455,6 +1572,28 @@ packages: purls: [] size: 154402 timestamp: 1754210968730 +- conda: https://conda.anaconda.org/conda-forge/noarch/cached-property-1.5.2-hd8ed1ab_1.tar.bz2 + noarch: python + sha256: 561e6660f26c35d137ee150187d89767c988413c978e1b712d53f27ddf70ea17 + md5: 9b347a7ec10940d3f7941ff6c460b551 + depends: + - cached_property >=1.5.2,<1.5.3.0a0 + license: BSD-3-Clause + license_family: BSD + purls: [] + size: 4134 + timestamp: 1615209571450 +- conda: https://conda.anaconda.org/conda-forge/noarch/cached_property-1.5.2-pyha770c72_1.tar.bz2 + sha256: 6dbf7a5070cc43d90a1e4c2ec0c541c69d8e30a0e25f50ce9f6e4a432e42c5d7 + md5: 576d629e47797577ab0f1b351297ef4a + depends: + - python >=3.6 + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/cached-property?source=hash-mapping + size: 11065 + timestamp: 1615209567874 - conda: https://conda.anaconda.org/conda-forge/noarch/cachelib-0.13.0-pyhd8ed1ab_1.conda sha256: 42e3aaa2522066e4992b332aef3ce812e62cd73075432a41d2904c74fcd3cee6 md5: aa353e8215130ccadcc81cf5a45f9579 @@ -1660,6 +1799,18 @@ packages: - pkg:pypi/colour?source=hash-mapping size: 21815 timestamp: 1733900282006 +- conda: https://conda.anaconda.org/conda-forge/noarch/comm-0.2.3-pyhe01879c_0.conda + sha256: 576a44729314ad9e4e5ebe055fbf48beb8116b60e58f9070278985b2b634f212 + md5: 2da13f2b299d8e1995bafbbe9689a2f7 + depends: + - python >=3.9 + - python + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/comm?source=hash-mapping + size: 14690 + timestamp: 1753453984907 - conda: https://conda.anaconda.org/conda-forge/noarch/connexion-2.14.2-pyhd8ed1ab_1.conda sha256: 533529503cb8f3e89621429cd9d2defa755a932d9fcbcebb0ad7cf6a810a65b4 md5: d15744aeb29bbb38c8eebc20af21d788 @@ -1777,6 +1928,44 @@ packages: - pkg:pypi/databricks-sdk?source=hash-mapping size: 435514 timestamp: 1754090557730 +- conda: https://conda.anaconda.org/conda-forge/linux-64/debugpy-1.8.16-py312h8285ef7_0.conda + sha256: ad6193b4c2771a82a8df3408d9c6174016b487fd1f7501b1618fa034c5118534 + md5: 6205bf8723b4b79275dd52ef60cf6af1 + depends: + - python + - libgcc >=14 + - __glibc >=2.17,<3.0.a0 + - libstdcxx >=14 + - libgcc >=14 + - python_abi 3.12.* *_cp312 + license: MIT + license_family: MIT + purls: + - pkg:pypi/debugpy?source=compressed-mapping + size: 2856116 + timestamp: 1754523420446 +- conda: https://conda.anaconda.org/conda-forge/noarch/decorator-5.2.1-pyhd8ed1ab_0.conda + sha256: c17c6b9937c08ad63cb20a26f403a3234088e57d4455600974a0ce865cb14017 + md5: 9ce473d1d1be1cc3810856a48b3fab32 + depends: + - python >=3.9 + license: BSD-2-Clause + license_family: BSD + purls: + - pkg:pypi/decorator?source=hash-mapping + size: 14129 + timestamp: 1740385067843 +- conda: https://conda.anaconda.org/conda-forge/noarch/defusedxml-0.7.1-pyhd8ed1ab_0.tar.bz2 + sha256: 9717a059677553562a8f38ff07f3b9f61727bd614f505658b0a5ecbcf8df89be + md5: 961b3a227b437d82ad7054484cfa71b2 + depends: + - python >=3.6 + license: PSF-2.0 + license_family: PSF + purls: + - pkg:pypi/defusedxml?source=hash-mapping + size: 24062 + timestamp: 1615232388757 - conda: https://conda.anaconda.org/conda-forge/noarch/deprecated-1.2.18-pyhd8ed1ab_0.conda sha256: d614bcff10696f1efc714df07651b50bf3808401fcc03814309ecec242cc8870 md5: 0cef44b1754ae4d6924ac0eef6b9fdbe @@ -1904,6 +2093,17 @@ packages: - pkg:pypi/exceptiongroup?source=hash-mapping size: 21284 timestamp: 1746947398083 +- conda: https://conda.anaconda.org/conda-forge/noarch/executing-2.2.0-pyhd8ed1ab_0.conda + sha256: 7510dd93b9848c6257c43fdf9ad22adf62e7aa6da5f12a6a757aed83bcfedf05 + md5: 81d30c08f9a3e556e8ca9e124b044d14 + depends: + - python >=3.9 + license: MIT + license_family: MIT + purls: + - pkg:pypi/executing?source=hash-mapping + size: 29652 + timestamp: 1745502200340 - conda: https://conda.anaconda.org/conda-forge/noarch/fastapi-0.116.1-h26c32bb_1.conda sha256: 17c3c3cfe254932feefe6e56e528ec012afd6a95c048cef4c4231d9075c47f85 md5: 78152786477103630cd4b807569b8881 @@ -2120,6 +2320,18 @@ packages: - pkg:pypi/fonttools?source=hash-mapping size: 2854951 timestamp: 1752723143 +- conda: https://conda.anaconda.org/conda-forge/noarch/fqdn-1.5.1-pyhd8ed1ab_1.conda + sha256: 2509992ec2fd38ab27c7cdb42cf6cadc566a1cc0d1021a2673475d9fa87c6276 + md5: d3549fd50d450b6d9e7dddff25dd2110 + depends: + - cached-property >=1.3.0 + - python >=3.9,<4 + license: MPL-2.0 + license_family: MOZILLA + purls: + - pkg:pypi/fqdn?source=hash-mapping + size: 16705 + timestamp: 1733327494780 - conda: https://conda.anaconda.org/conda-forge/linux-64/freetype-2.13.3-ha770c72_1.conda sha256: 7ef7d477c43c12a5b4cddcf048a83277414512d1116aba62ebadfa7056a7d84f md5: 9ccd736d31e0c6e41f54e704e5312811 @@ -2170,8 +2382,21 @@ packages: timestamp: 1741505873840 - pypi: ./ name: gaiaflow - version: 0.0.1.dev0 - sha256: 16af4df6af9317d4f803f67b6dd69d3bb36594e7638b2c22f8d59a3fb1d661d8 + version: 0.0.3.dev0 + sha256: 64847b19a8e8c898e9c712054f2aecbdc022c3a2a8a6584fa7995cb5f7d69067 + requires_dist: + - typer>=0.16.0,<0.17 + - fsspec>=2025.7.0,<2026 + - ruamel-yaml>=0.18.14,<0.19 + - psutil>=7.0.0,<8 + - pyyaml>=6.0.2,<7 + - apache-airflow>=3.0.3,<4 + - apache-airflow-providers-cncf-kubernetes>=10.6.2,<11 + - apache-airflow-providers-amazon>=9.11.0,<10 + - apache-airflow-providers-fab>=2.3.1,<3 + - apache-airflow-providers-docker>=4.4.2,<5 + - mlflow>=3.2.0,<4 + - jupyter>=1.1.1,<2 requires_python: '>=3.11' editable: true - conda: https://conda.anaconda.org/conda-forge/linux-64/gflags-2.2.2-h5888daf_1005.conda @@ -2526,6 +2751,95 @@ packages: - pkg:pypi/intervals?source=hash-mapping size: 14020 timestamp: 1733902692838 +- conda: https://conda.anaconda.org/conda-forge/noarch/ipykernel-6.30.1-pyh82676e8_0.conda + sha256: cfc2c4e31dfedbb3d124d0055f55fda4694538fb790d52cd1b37af5312833e36 + md5: b0cc25825ce9212b8bee37829abad4d6 + depends: + - __linux + - comm >=0.1.1 + - debugpy >=1.6.5 + - ipython >=7.23.1 + - jupyter_client >=8.0.0 + - jupyter_core >=4.12,!=5.0.* + - matplotlib-inline >=0.1 + - nest-asyncio >=1.4 + - packaging >=22 + - psutil >=5.7 + - python >=3.9 + - pyzmq >=25 + - tornado >=6.2 + - traitlets >=5.4.0 + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/ipykernel?source=hash-mapping + size: 121367 + timestamp: 1754352984703 +- conda: https://conda.anaconda.org/conda-forge/noarch/ipython-9.4.0-pyhfa0c392_0.conda + sha256: ff5138bf6071ca01d84e1329f6baa96f0723df6fe183cfa1ab3ebc96240e6d8f + md5: cb7706b10f35e7507917cefa0978a66d + depends: + - __unix + - pexpect >4.3 + - decorator + - exceptiongroup + - ipython_pygments_lexers + - jedi >=0.16 + - matplotlib-inline + - pickleshare + - prompt-toolkit >=3.0.41,<3.1.0 + - pygments >=2.4.0 + - python >=3.11 + - stack_data + - traitlets >=5.13.0 + - typing_extensions >=4.6 + - python + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/ipython?source=hash-mapping + size: 628259 + timestamp: 1751465044469 +- conda: https://conda.anaconda.org/conda-forge/noarch/ipython_pygments_lexers-1.1.1-pyhd8ed1ab_0.conda + sha256: 894682a42a7d659ae12878dbcb274516a7031bbea9104e92f8e88c1f2765a104 + md5: bd80ba060603cc228d9d81c257093119 + depends: + - pygments + - python >=3.9 + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/ipython-pygments-lexers?source=hash-mapping + size: 13993 + timestamp: 1737123723464 +- conda: https://conda.anaconda.org/conda-forge/noarch/ipywidgets-8.1.7-pyhd8ed1ab_0.conda + sha256: fd496e7d48403246f534c5eec09fc1e63ac7beb1fa06541d6ba71f56b30cf29b + md5: 7c9449eac5975ef2d7753da262a72707 + depends: + - comm >=0.1.3 + - ipython >=6.1.0 + - jupyterlab_widgets >=3.0.15,<3.1.0 + - python >=3.9 + - traitlets >=4.3.1 + - widgetsnbextension >=4.0.14,<4.1.0 + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/ipywidgets?source=hash-mapping + size: 114557 + timestamp: 1746454722402 +- conda: https://conda.anaconda.org/conda-forge/noarch/isoduration-20.11.0-pyhd8ed1ab_1.conda + sha256: 08e838d29c134a7684bca0468401d26840f41c92267c4126d7b43a6b533b0aed + md5: 0b0154421989637d424ccf0f104be51a + depends: + - arrow >=0.15.0 + - python >=3.9 + license: MIT + license_family: MIT + purls: + - pkg:pypi/isoduration?source=hash-mapping + size: 19832 + timestamp: 1733493720346 - pypi: https://files.pythonhosted.org/packages/c1/11/114d0a5f4dabbdcedc1125dee0888514c3c3b16d3e9facad87ed96fad97c/isort-6.0.1-py3-none-any.whl name: isort version: 6.0.1 @@ -2545,6 +2859,17 @@ packages: - pkg:pypi/itsdangerous?source=hash-mapping size: 19180 timestamp: 1733308353037 +- conda: https://conda.anaconda.org/conda-forge/noarch/jedi-0.19.2-pyhd8ed1ab_1.conda + sha256: 92c4d217e2dc68983f724aa983cca5464dcb929c566627b26a2511159667dba8 + md5: a4f4c5dc9b80bc50e0d3dc4e6e8f1bd9 + depends: + - parso >=0.8.3,<0.9.0 + - python >=3.9 + license: Apache-2.0 AND MIT + purls: + - pkg:pypi/jedi?source=hash-mapping + size: 843646 + timestamp: 1733300981994 - conda: https://conda.anaconda.org/conda-forge/noarch/jinja2-3.1.6-pyhd8ed1ab_0.conda sha256: f1ac18b11637ddadc05642e8185a851c7fab5998c6f5470d716812fae943b2af md5: 446bd6c8cb26050d528881df495ce646 @@ -2580,6 +2905,17 @@ packages: - pkg:pypi/joblib?source=hash-mapping size: 224437 timestamp: 1748019237972 +- conda: https://conda.anaconda.org/conda-forge/noarch/json5-0.12.1-pyhd8ed1ab_0.conda + sha256: 4e08ccf9fa1103b617a4167a270768de736a36be795c6cd34c2761100d332f74 + md5: 0fc93f473c31a2f85c0bde213e7c63ca + depends: + - python >=3.9 + license: Apache-2.0 + license_family: APACHE + purls: + - pkg:pypi/json5?source=hash-mapping + size: 34191 + timestamp: 1755034963991 - conda: https://conda.anaconda.org/conda-forge/noarch/jsonpath-ng-1.7.0-pyhe01879c_0.conda sha256: de1bcebb37536d71efe302a640487557b68e324c12de974026759f54b259f6cc md5: cbabeb77ee9b2b3bc75eb09c39b3ba43 @@ -2593,6 +2929,18 @@ packages: - pkg:pypi/jsonpath-ng?source=hash-mapping size: 34863 timestamp: 1751583840995 +- conda: https://conda.anaconda.org/conda-forge/linux-64/jsonpointer-3.0.0-py312h7900ff3_1.conda + sha256: 76ccb7bffc7761d1d3133ffbe1f7f1710a0f0d9aaa9f7ea522652e799f3601f4 + md5: 6b51f7459ea4073eeb5057207e2e1e3d + depends: + - python >=3.12,<3.13.0a0 + - python_abi 3.12.* *_cp312 + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/jsonpointer?source=hash-mapping + size: 17277 + timestamp: 1725303032027 - conda: https://conda.anaconda.org/conda-forge/noarch/jsonschema-4.25.0-pyhe01879c_0.conda sha256: 87ba7cf3a65c8e8d1005368b9aee3f49e295115381b7a0b180e56f7b68b5975f md5: c6e3fd94e058dba67d917f38a11b50ab @@ -2622,6 +2970,242 @@ packages: - pkg:pypi/jsonschema-specifications?source=hash-mapping size: 19168 timestamp: 1745424244298 +- conda: https://conda.anaconda.org/conda-forge/noarch/jsonschema-with-format-nongpl-4.25.0-he01879c_0.conda + sha256: 72604d07afaddf2156e61d128256d686aee4a7bdc06e235d7be352955de7527a + md5: f4c7afaf838ab5bb1c4e73eb3095fb26 + depends: + - jsonschema >=4.25.0,<4.25.1.0a0 + - fqdn + - idna + - isoduration + - jsonpointer >1.13 + - rfc3339-validator + - rfc3986-validator >0.1.0 + - rfc3987-syntax >=1.1.0 + - uri-template + - webcolors >=24.6.0 + license: MIT + license_family: MIT + purls: [] + size: 4744 + timestamp: 1752925388185 +- conda: https://conda.anaconda.org/conda-forge/noarch/jupyter-1.1.1-pyhd8ed1ab_1.conda + sha256: b538e15067d05768d1c0532a6d9b0625922a1cce751dd6a2af04f7233a1a70e9 + md5: 9453512288d20847de4356327d0e1282 + depends: + - ipykernel + - ipywidgets + - jupyter_console + - jupyterlab + - nbconvert-core + - notebook + - python >=3.9 + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/jupyter?source=hash-mapping + size: 8891 + timestamp: 1733818677113 +- conda: https://conda.anaconda.org/conda-forge/noarch/jupyter-lsp-2.2.6-pyhe01879c_0.conda + sha256: 6f2d6c5983e013af68e7e1d7082cc46b11f55e28147bd0a72a44488972ed90a3 + md5: 7129ed52335cc7164baf4d6508a3f233 + depends: + - importlib-metadata >=4.8.3 + - jupyter_server >=1.1.2 + - python >=3.9 + - python + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/jupyter-lsp?source=hash-mapping + size: 58416 + timestamp: 1752935193718 +- conda: https://conda.anaconda.org/conda-forge/noarch/jupyter_client-8.6.3-pyhd8ed1ab_1.conda + sha256: 19d8bd5bb2fde910ec59e081eeb59529491995ce0d653a5209366611023a0b3a + md5: 4ebae00eae9705b0c3d6d1018a81d047 + depends: + - importlib-metadata >=4.8.3 + - jupyter_core >=4.12,!=5.0.* + - python >=3.9 + - python-dateutil >=2.8.2 + - pyzmq >=23.0 + - tornado >=6.2 + - traitlets >=5.3 + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/jupyter-client?source=hash-mapping + size: 106342 + timestamp: 1733441040958 +- conda: https://conda.anaconda.org/conda-forge/noarch/jupyter_console-6.6.3-pyhd8ed1ab_1.conda + sha256: aee0cdd0cb2b9321d28450aec4e0fd43566efcd79e862d70ce49a68bf0539bcd + md5: 801dbf535ec26508fac6d4b24adfb76e + depends: + - ipykernel >=6.14 + - ipython + - jupyter_client >=7.0.0 + - jupyter_core >=4.12,!=5.0.* + - prompt_toolkit >=3.0.30 + - pygments + - python >=3.9 + - pyzmq >=17 + - traitlets >=5.4 + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/jupyter-console?source=hash-mapping + size: 26874 + timestamp: 1733818130068 +- conda: https://conda.anaconda.org/conda-forge/noarch/jupyter_core-5.8.1-pyh31011fe_0.conda + sha256: 56a7a7e907f15cca8c4f9b0c99488276d4cb10821d2d15df9245662184872e81 + md5: b7d89d860ebcda28a5303526cdee68ab + depends: + - __unix + - platformdirs >=2.5 + - python >=3.8 + - traitlets >=5.3 + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/jupyter-core?source=hash-mapping + size: 59562 + timestamp: 1748333186063 +- conda: https://conda.anaconda.org/conda-forge/noarch/jupyter_events-0.12.0-pyh29332c3_0.conda + sha256: 37e6ac3ccf7afcc730c3b93cb91a13b9ae827fd306f35dd28f958a74a14878b5 + md5: f56000b36f09ab7533877e695e4e8cb0 + depends: + - jsonschema-with-format-nongpl >=4.18.0 + - packaging + - python >=3.9 + - python-json-logger >=2.0.4 + - pyyaml >=5.3 + - referencing + - rfc3339-validator + - rfc3986-validator >=0.1.1 + - traitlets >=5.3 + - python + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/jupyter-events?source=hash-mapping + size: 23647 + timestamp: 1738765986736 +- conda: https://conda.anaconda.org/conda-forge/noarch/jupyter_server-2.16.0-pyhe01879c_0.conda + sha256: 0082fb6f0afaf872affee4cde3b210f7f7497a5fb47f2944ab638fef0f0e2e77 + md5: f062e04d7cd585c937acbf194dceec36 + depends: + - anyio >=3.1.0 + - argon2-cffi >=21.1 + - jinja2 >=3.0.3 + - jupyter_client >=7.4.4 + - jupyter_core >=4.12,!=5.0.* + - jupyter_events >=0.11.0 + - jupyter_server_terminals >=0.4.4 + - nbconvert-core >=6.4.4 + - nbformat >=5.3.0 + - overrides >=5.0 + - packaging >=22.0 + - prometheus_client >=0.9 + - python >=3.9 + - pyzmq >=24 + - send2trash >=1.8.2 + - terminado >=0.8.3 + - tornado >=6.2.0 + - traitlets >=5.6.0 + - websocket-client >=1.7 + - python + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/jupyter-server?source=hash-mapping + size: 344376 + timestamp: 1747083217715 +- conda: https://conda.anaconda.org/conda-forge/noarch/jupyter_server_terminals-0.5.3-pyhd8ed1ab_1.conda + sha256: 0890fc79422191bc29edf17d7b42cff44ba254aa225d31eb30819f8772b775b8 + md5: 2d983ff1b82a1ccb6f2e9d8784bdd6bd + depends: + - python >=3.9 + - terminado >=0.8.3 + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/jupyter-server-terminals?source=hash-mapping + size: 19711 + timestamp: 1733428049134 +- conda: https://conda.anaconda.org/conda-forge/noarch/jupyterlab-4.4.6-pyhd8ed1ab_0.conda + sha256: c3558f1c2a5977799ce425f1f7c8d8d1cae3408da41ec4f5c3771a21e673d465 + md5: 70cb2903114eafc6ed5d70ca91ba6545 + depends: + - async-lru >=1.0.0 + - httpx >=0.25.0,<1 + - importlib-metadata >=4.8.3 + - ipykernel >=6.5.0,!=6.30.0 + - jinja2 >=3.0.3 + - jupyter-lsp >=2.0.0 + - jupyter_core + - jupyter_server >=2.4.0,<3 + - jupyterlab_server >=2.27.1,<3 + - notebook-shim >=0.2 + - packaging + - python >=3.9 + - setuptools >=41.1.0 + - tomli >=1.2.2 + - tornado >=6.2.0 + - traitlets + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/jupyterlab?source=compressed-mapping + size: 8408461 + timestamp: 1755263247917 +- conda: https://conda.anaconda.org/conda-forge/noarch/jupyterlab_pygments-0.3.0-pyhd8ed1ab_2.conda + sha256: dc24b900742fdaf1e077d9a3458fd865711de80bca95fe3c6d46610c532c6ef0 + md5: fd312693df06da3578383232528c468d + depends: + - pygments >=2.4.1,<3 + - python >=3.9 + constrains: + - jupyterlab >=4.0.8,<5.0.0 + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/jupyterlab-pygments?source=hash-mapping + size: 18711 + timestamp: 1733328194037 +- conda: https://conda.anaconda.org/conda-forge/noarch/jupyterlab_server-2.27.3-pyhd8ed1ab_1.conda + sha256: d03d0b7e23fa56d322993bc9786b3a43b88ccc26e58b77c756619a921ab30e86 + md5: 9dc4b2b0f41f0de41d27f3293e319357 + depends: + - babel >=2.10 + - importlib-metadata >=4.8.3 + - jinja2 >=3.0.3 + - json5 >=0.9.0 + - jsonschema >=4.18 + - jupyter_server >=1.21,<3 + - packaging >=21.3 + - python >=3.9 + - requests >=2.31 + constrains: + - openapi-core >=0.18.0,<0.19.0 + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/jupyterlab-server?source=hash-mapping + size: 49449 + timestamp: 1733599666357 +- conda: https://conda.anaconda.org/conda-forge/noarch/jupyterlab_widgets-3.0.15-pyhd8ed1ab_0.conda + sha256: 6214d345861b106076e7cb38b59761b24cd340c09e3f787e4e4992036ca3cd7e + md5: ad100d215fad890ab0ee10418f36876f + depends: + - python >=3.9 + constrains: + - jupyterlab >=3,<5 + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/jupyterlab-widgets?source=hash-mapping + size: 189133 + timestamp: 1746450926999 - conda: https://conda.anaconda.org/conda-forge/linux-64/keyutils-1.6.1-h166bdaf_0.tar.bz2 sha256: 150c05a6e538610ca7c43beb3a40d65c90537497a4f6a5f4d15ec0451b6f5ebb md5: 30186d27e2c9fa62b45fb1476b7200e3 @@ -2695,6 +3279,17 @@ packages: - pkg:pypi/kubernetes-asyncio?source=hash-mapping size: 498228 timestamp: 1746059841652 +- conda: https://conda.anaconda.org/conda-forge/noarch/lark-1.2.2-pyhd8ed1ab_1.conda + sha256: 637a9c32e15a4333f1f9c91e0a506dbab4a6dab7ee83e126951159c916c81c99 + md5: 3a8063b25e603999188ed4bbf3485404 + depends: + - python >=3.9 + license: MIT + license_family: MIT + purls: + - pkg:pypi/lark?source=hash-mapping + size: 92093 + timestamp: 1734709450256 - conda: https://conda.anaconda.org/conda-forge/linux-64/lazy-object-proxy-1.11.0-py312h66e93f0_0.conda sha256: 33c303b49cf7e4c236bee03d72efc3c1a5d04dfd3ac398dcf13f7113989726ab md5: 23451c809bb3aa647ffb36dca55fc358 @@ -3721,6 +4316,18 @@ packages: - pkg:pypi/matplotlib?source=compressed-mapping size: 8071030 timestamp: 1754005868258 +- conda: https://conda.anaconda.org/conda-forge/noarch/matplotlib-inline-0.1.7-pyhd8ed1ab_1.conda + sha256: 69b7dc7131703d3d60da9b0faa6dd8acbf6f6c396224cf6aef3e855b8c0c41c6 + md5: af6ab708897df59bd6e7283ceab1b56b + depends: + - python >=3.9 + - traitlets + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/matplotlib-inline?source=hash-mapping + size: 14467 + timestamp: 1733417051523 - conda: https://conda.anaconda.org/conda-forge/noarch/mdurl-0.1.2-pyhd8ed1ab_1.conda sha256: 78c1bbe1723449c52b7a9df1af2ee5f005209f67e40b6e1d3c7619127c43b1c7 md5: 592132998493b3ff25fd7479396e8351 @@ -3755,6 +4362,19 @@ packages: - pkg:pypi/methodtools?source=hash-mapping size: 9562 timestamp: 1733971720931 +- conda: https://conda.anaconda.org/conda-forge/noarch/mistune-3.1.3-pyh29332c3_0.conda + sha256: a67484d7dd11e815a81786580f18b6e4aa2392f292f29183631a6eccc8dc37b3 + md5: 7ec6576e328bc128f4982cd646eeba85 + depends: + - python >=3.9 + - typing_extensions + - python + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/mistune?source=hash-mapping + size: 72749 + timestamp: 1742402716323 - conda: https://conda.anaconda.org/conda-forge/noarch/mkdocs-1.6.1-pyhd8ed1ab_1.conda sha256: 902d2e251f9a7ffa7d86a3e62be5b2395e28614bd4dbe5f50acf921fd64a8c35 md5: 14661160be39d78f2b210f2cc2766059 @@ -4026,6 +4646,66 @@ packages: version: 1.1.0 sha256: 1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505 requires_python: '>=3.8' +- conda: https://conda.anaconda.org/conda-forge/noarch/nbclient-0.10.2-pyhd8ed1ab_0.conda + sha256: a20cff739d66c2f89f413e4ba4c6f6b59c50d5c30b5f0d840c13e8c9c2df9135 + md5: 6bb0d77277061742744176ab555b723c + depends: + - jupyter_client >=6.1.12 + - jupyter_core >=4.12,!=5.0.* + - nbformat >=5.1 + - python >=3.8 + - traitlets >=5.4 + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/nbclient?source=hash-mapping + size: 28045 + timestamp: 1734628936013 +- conda: https://conda.anaconda.org/conda-forge/noarch/nbconvert-core-7.16.6-pyh29332c3_0.conda + sha256: dcccb07c5a1acb7dc8be94330e62d54754c0e9c9cb2bb6865c8e3cfe44cf5a58 + md5: d24beda1d30748afcc87c429454ece1b + depends: + - beautifulsoup4 + - bleach-with-css !=5.0.0 + - defusedxml + - importlib-metadata >=3.6 + - jinja2 >=3.0 + - jupyter_core >=4.7 + - jupyterlab_pygments + - markupsafe >=2.0 + - mistune >=2.0.3,<4 + - nbclient >=0.5.0 + - nbformat >=5.7 + - packaging + - pandocfilters >=1.4.1 + - pygments >=2.4.1 + - python >=3.9 + - traitlets >=5.1 + - python + constrains: + - pandoc >=2.9.2,<4.0.0 + - nbconvert ==7.16.6 *_0 + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/nbconvert?source=hash-mapping + size: 200601 + timestamp: 1738067871724 +- conda: https://conda.anaconda.org/conda-forge/noarch/nbformat-5.10.4-pyhd8ed1ab_1.conda + sha256: 7a5bd30a2e7ddd7b85031a5e2e14f290898098dc85bea5b3a5bf147c25122838 + md5: bbe1963f1e47f594070ffe87cdf612ea + depends: + - jsonschema >=2.6 + - jupyter_core >=4.12,!=5.0.* + - python >=3.9 + - python-fastjsonschema >=2.15 + - traitlets >=5.1 + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/nbformat?source=hash-mapping + size: 100945 + timestamp: 1733402844974 - conda: https://conda.anaconda.org/conda-forge/linux-64/ncurses-6.5-h2d0b736_3.conda sha256: 3fde293232fa3fca98635e1167de6b7c7fda83caf24b9d6c91ec9eefb4f4d586 md5: 47e340acb35de30501a76c7c799c41d7 @@ -4036,6 +4716,17 @@ packages: purls: [] size: 891641 timestamp: 1738195959188 +- conda: https://conda.anaconda.org/conda-forge/noarch/nest-asyncio-1.6.0-pyhd8ed1ab_1.conda + sha256: bb7b21d7fd0445ddc0631f64e66d91a179de4ba920b8381f29b9d006a42788c0 + md5: 598fd7d4d0de2455fb74f56063969a97 + depends: + - python >=3.9 + license: BSD-2-Clause + license_family: BSD + purls: + - pkg:pypi/nest-asyncio?source=hash-mapping + size: 11543 + timestamp: 1733325673691 - conda: https://conda.anaconda.org/conda-forge/linux-64/nlohmann_json-3.12.0-h3f2d84a_0.conda sha256: e2fc624d6f9b2f1b695b6be6b905844613e813aa180520e73365062683fe7b49 md5: d76872d096d063e226482c99337209dc @@ -4044,6 +4735,34 @@ packages: purls: [] size: 135906 timestamp: 1744445169928 +- conda: https://conda.anaconda.org/conda-forge/noarch/notebook-7.4.5-pyhd8ed1ab_0.conda + sha256: ea9d7058d862530755abeb2ee8f0152453cf630b024c73906f689ca1c297cd79 + md5: 28062c17cdb444388c00903eaec1ba0e + depends: + - jupyter_server >=2.4.0,<3 + - jupyterlab >=4.4.5,<4.5 + - jupyterlab_server >=2.27.1,<3 + - notebook-shim >=0.2,<0.3 + - python >=3.9 + - tornado >=6.2.0 + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/notebook?source=hash-mapping + size: 10349114 + timestamp: 1754404047534 +- conda: https://conda.anaconda.org/conda-forge/noarch/notebook-shim-0.2.4-pyhd8ed1ab_1.conda + sha256: 7b920e46b9f7a2d2aa6434222e5c8d739021dbc5cc75f32d124a8191d86f9056 + md5: e7f89ea5f7ea9401642758ff50a2d9c1 + depends: + - jupyter_server >=1.8,<3 + - python >=3.9 + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/notebook-shim?source=hash-mapping + size: 16817 + timestamp: 1733408419340 - conda: https://conda.anaconda.org/conda-forge/linux-64/numpy-2.3.2-py312h33ff503_0.conda sha256: d54e52df67e0be7e5faa9e6f0efccea3d72f635a3159cc151c4668e5159f6ef3 md5: 3f6efbc40eb13f019c856c410fa921d2 @@ -4264,6 +4983,18 @@ packages: - pkg:pypi/orderedmultidict?source=hash-mapping size: 16278 timestamp: 1733900401804 +- conda: https://conda.anaconda.org/conda-forge/noarch/overrides-7.7.0-pyhd8ed1ab_1.conda + sha256: 1840bd90d25d4930d60f57b4f38d4e0ae3f5b8db2819638709c36098c6ba770c + md5: e51f1e4089cad105b6cac64bd8166587 + depends: + - python >=3.9 + - typing_utils + license: Apache-2.0 + license_family: APACHE + purls: + - pkg:pypi/overrides?source=hash-mapping + size: 30139 + timestamp: 1734587755455 - conda: https://conda.anaconda.org/conda-forge/noarch/packaging-25.0-pyh29332c3_1.conda sha256: 289861ed0c13a15d7bbb408796af4de72c2fe67e2bcb0de98f4c3fce259d7991 md5: 58335b26c38bf4a20f399384c33cbcf9 @@ -4307,6 +5038,17 @@ packages: - pkg:pypi/pandas?source=hash-mapping size: 15436913 timestamp: 1726879054912 +- conda: https://conda.anaconda.org/conda-forge/noarch/pandocfilters-1.5.0-pyhd8ed1ab_0.tar.bz2 + sha256: 2bb9ba9857f4774b85900c2562f7e711d08dd48e2add9bee4e1612fbee27e16f + md5: 457c2c8c08e54905d6954e79cb5b5db9 + depends: + - python !=3.0,!=3.1,!=3.2,!=3.3 + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/pandocfilters?source=hash-mapping + size: 11627 + timestamp: 1631603397334 - conda: https://conda.anaconda.org/conda-forge/noarch/paramiko-3.5.1-pyhd8ed1ab_0.conda sha256: 1499e558d31536707fdd7d0b569dbe29ae6e3aa8f2fdce9ea6f3df3ce4c1aaf1 md5: 4e6bea7eee94bb9d8a599385215719f9 @@ -4321,6 +5063,17 @@ packages: - pkg:pypi/paramiko?source=hash-mapping size: 161046 timestamp: 1738679235142 +- conda: https://conda.anaconda.org/conda-forge/noarch/parso-0.8.4-pyhd8ed1ab_1.conda + sha256: 17131120c10401a99205fc6fe436e7903c0fa092f1b3e80452927ab377239bcc + md5: 5c092057b6badd30f75b06244ecd01c9 + depends: + - python >=3.9 + license: MIT + license_family: MIT + purls: + - pkg:pypi/parso?source=hash-mapping + size: 75295 + timestamp: 1733271352153 - conda: https://conda.anaconda.org/conda-forge/noarch/passlib-1.7.4-pyhd8ed1ab_2.conda sha256: 2adfe01cdab93c39c4d8dfe3de74a31ae6fded21213f26925208ce6053cea93d md5: fba64c154edb7d7935af0d46d97ff536 @@ -4365,6 +5118,17 @@ packages: - pkg:pypi/pendulum?source=hash-mapping size: 405751 timestamp: 1745083011918 +- conda: https://conda.anaconda.org/conda-forge/noarch/pexpect-4.9.0-pyhd8ed1ab_1.conda + sha256: 202af1de83b585d36445dc1fda94266697341994d1a3328fabde4989e1b3d07a + md5: d0d408b1f18883a944376da5cf8101ea + depends: + - ptyprocess >=0.5 + - python >=3.9 + license: ISC + purls: + - pkg:pypi/pexpect?source=hash-mapping + size: 53561 + timestamp: 1733302019362 - conda: https://conda.anaconda.org/conda-forge/noarch/phonenumbers-9.0.10-pyhd8ed1ab_0.conda sha256: f43d9aa1bcb5e78f0af5f7ca9d73ec69417a69da0aadc9fe829da6604dbc50b3 md5: 4467b94f24e600e71ae4c32bc1760230 @@ -4376,6 +5140,17 @@ packages: - pkg:pypi/phonenumbers?source=hash-mapping size: 1465927 timestamp: 1752856987690 +- conda: https://conda.anaconda.org/conda-forge/noarch/pickleshare-0.7.5-pyhd8ed1ab_1004.conda + sha256: e2ac3d66c367dada209fc6da43e645672364b9fd5f9d28b9f016e24b81af475b + md5: 11a9d1d09a3615fc07c3faf79bc0b943 + depends: + - python >=3.9 + license: MIT + license_family: MIT + purls: + - pkg:pypi/pickleshare?source=hash-mapping + size: 11748 + timestamp: 1733327448200 - conda: https://conda.anaconda.org/conda-forge/linux-64/pillow-11.3.0-py312h80c1187_0.conda sha256: 7c9a8f65a200587bf7a0135ca476f9c472348177338ed8b825ddcc08773fde68 md5: 7911e727a6c24db662193a960b81b6b2 @@ -4578,6 +5353,27 @@ packages: purls: [] size: 8252 timestamp: 1726802366959 +- conda: https://conda.anaconda.org/conda-forge/noarch/ptyprocess-0.7.0-pyhd8ed1ab_1.conda + sha256: a7713dfe30faf17508ec359e0bc7e0983f5d94682492469bd462cdaae9c64d83 + md5: 7d9daffbb8d8e0af0f769dbbcd173a54 + depends: + - python >=3.9 + license: ISC + purls: + - pkg:pypi/ptyprocess?source=hash-mapping + size: 19457 + timestamp: 1733302371990 +- conda: https://conda.anaconda.org/conda-forge/noarch/pure_eval-0.2.3-pyhd8ed1ab_1.conda + sha256: 71bd24600d14bb171a6321d523486f6a06f855e75e547fa0cb2a0953b02047f0 + md5: 3bfdfb8dbcdc4af1ae3f9a8eb3948f04 + depends: + - python >=3.9 + license: MIT + license_family: MIT + purls: + - pkg:pypi/pure-eval?source=hash-mapping + size: 16668 + timestamp: 1733569518868 - conda: https://conda.anaconda.org/conda-forge/linux-64/pyarrow-20.0.0-py312h7900ff3_0.conda sha256: f7b08ff9ef4626e19a3cd08165ca1672675168fa9af9c2b0d2a5c104c71baf01 md5: 57b626b4232b77ee6410c7c03a99774d @@ -4888,6 +5684,29 @@ packages: - pkg:pypi/python-dotenv?source=hash-mapping size: 26031 timestamp: 1750789290754 +- conda: https://conda.anaconda.org/conda-forge/noarch/python-fastjsonschema-2.21.2-pyhe01879c_0.conda + sha256: df9aa74e9e28e8d1309274648aac08ec447a92512c33f61a8de0afa9ce32ebe8 + md5: 23029aae904a2ba587daba708208012f + depends: + - python >=3.9 + - python + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/fastjsonschema?source=hash-mapping + size: 244628 + timestamp: 1755304154927 +- conda: https://conda.anaconda.org/conda-forge/noarch/python-json-logger-2.0.7-pyhd8ed1ab_0.conda + sha256: 4790787fe1f4e8da616edca4acf6a4f8ed4e7c6967aa31b920208fc8f95efcca + md5: a61bf9ec79426938ff785eb69dbb1960 + depends: + - python >=3.6 + license: BSD-2-Clause + license_family: BSD + purls: + - pkg:pypi/python-json-logger?source=hash-mapping + size: 13383 + timestamp: 1677079727691 - conda: https://conda.anaconda.org/conda-forge/noarch/python-kubernetes-32.0.1-pyhd8ed1ab_0.conda sha256: 6d341f4c4fb5066dac6a8a522e12692cb93ec77132ac2265773d8684f59955e9 md5: d93b25519ea3ea3fb735060c7bbd9f14 @@ -5020,6 +5839,23 @@ packages: - pkg:pypi/pyyaml-env-tag?source=hash-mapping size: 11137 timestamp: 1747237061448 +- conda: https://conda.anaconda.org/conda-forge/linux-64/pyzmq-27.0.2-py312h6748674_0.conda + sha256: d697fb7e36427b085feffd63288365be543f7c2a779e35205cb1e52d1ca49957 + md5: e0770749ec419e8e68e71716507c1be4 + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=14 + - libsodium >=1.0.20,<1.0.21.0a0 + - libstdcxx >=14 + - python >=3.12,<3.13.0a0 + - python_abi 3.12.* *_cp312 + - zeromq >=4.3.5,<4.4.0a0 + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/pyzmq?source=compressed-mapping + size: 381481 + timestamp: 1755799909607 - conda: https://conda.anaconda.org/conda-forge/linux-64/qhull-2020.2-h434a139_5.conda sha256: 776363493bad83308ba30bcb88c2552632581b143e8ee25b1982c8c743e73abc md5: 353823361b1d27eb3960efb076dfcaf6 @@ -5157,6 +5993,42 @@ packages: - pkg:pypi/retryhttp?source=hash-mapping size: 19165 timestamp: 1746266529588 +- conda: https://conda.anaconda.org/conda-forge/noarch/rfc3339-validator-0.1.4-pyhd8ed1ab_1.conda + sha256: 2e4372f600490a6e0b3bac60717278448e323cab1c0fecd5f43f7c56535a99c5 + md5: 36de09a8d3e5d5e6f4ee63af49e59706 + depends: + - python >=3.9 + - six + license: MIT + license_family: MIT + purls: + - pkg:pypi/rfc3339-validator?source=hash-mapping + size: 10209 + timestamp: 1733600040800 +- conda: https://conda.anaconda.org/conda-forge/noarch/rfc3986-validator-0.1.1-pyh9f0ad1d_0.tar.bz2 + sha256: 2a5b495a1de0f60f24d8a74578ebc23b24aa53279b1ad583755f223097c41c37 + md5: 912a71cc01012ee38e6b90ddd561e36f + depends: + - python + license: MIT + license_family: MIT + purls: + - pkg:pypi/rfc3986-validator?source=hash-mapping + size: 7818 + timestamp: 1598024297745 +- conda: https://conda.anaconda.org/conda-forge/noarch/rfc3987-syntax-1.1.0-pyhe01879c_1.conda + sha256: 70001ac24ee62058557783d9c5a7bbcfd97bd4911ef5440e3f7a576f9e43bc92 + md5: 7234f99325263a5af6d4cd195035e8f2 + depends: + - python >=3.9 + - lark >=1.2.2 + - python + license: MIT + license_family: MIT + purls: + - pkg:pypi/rfc3987-syntax?source=hash-mapping + size: 22913 + timestamp: 1752876729969 - conda: https://conda.anaconda.org/conda-forge/noarch/rich-13.9.4-pyhd8ed1ab_1.conda sha256: 06a760c5ae572e72e865d5a87e9fe3cc171e1a9c996e63daf3db52ff1a0b4457 md5: 7aed65d4ff222bfb7335997aa40b7da5 @@ -5360,6 +6232,18 @@ packages: - pkg:pypi/scramp?source=hash-mapping size: 17332 timestamp: 1667412095353 +- conda: https://conda.anaconda.org/conda-forge/noarch/send2trash-1.8.3-pyh0d859eb_1.conda + sha256: 00926652bbb8924e265caefdb1db100f86a479e8f1066efe395d5552dde54d02 + md5: 938c8de6b9de091997145b3bf25cdbf9 + depends: + - __linux + - python >=3.9 + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/send2trash?source=hash-mapping + size: 22736 + timestamp: 1733322148326 - conda: https://conda.anaconda.org/conda-forge/linux-64/setproctitle-1.3.6-py312h66e93f0_0.conda sha256: 273562f3c2c5503e914804ea3ecbe07c344b6a5d355890bad2424d894b0ef6d9 md5: 0c9df8f9c424f4d3dea521dff3a6cfd6 @@ -5635,6 +6519,20 @@ packages: - pkg:pypi/sqlparse?source=hash-mapping size: 40443 timestamp: 1734007220139 +- conda: https://conda.anaconda.org/conda-forge/noarch/stack_data-0.6.3-pyhd8ed1ab_1.conda + sha256: 570da295d421661af487f1595045760526964f41471021056e993e73089e9c41 + md5: b1b505328da7a6b246787df4b5a49fbc + depends: + - asttokens + - executing + - pure_eval + - python >=3.9 + license: MIT + license_family: MIT + purls: + - pkg:pypi/stack-data?source=hash-mapping + size: 26988 + timestamp: 1733569565672 - conda: https://conda.anaconda.org/conda-forge/noarch/starlette-0.47.2-pyh82d4cca_0.conda sha256: 5112e37cb5fc739d5d386eae0a266f0687a6422b376d0078b41bcd8b0725b56f md5: e7456f20ee85cd9c13e36a7c7d7052a3 @@ -5705,6 +6603,20 @@ packages: - pkg:pypi/termcolor?source=hash-mapping size: 13131 timestamp: 1746039688416 +- conda: https://conda.anaconda.org/conda-forge/noarch/terminado-0.18.1-pyh0d859eb_0.conda + sha256: b300557c0382478cf661ddb520263508e4b3b5871b471410450ef2846e8c352c + md5: efba281bbdae5f6b0a1d53c6d4a97c93 + depends: + - __linux + - ptyprocess + - python >=3.8 + - tornado >=6.1.0 + license: BSD-2-Clause + license_family: BSD + purls: + - pkg:pypi/terminado?source=hash-mapping + size: 22452 + timestamp: 1710262728753 - conda: https://conda.anaconda.org/conda-forge/noarch/text-unidecode-1.3-pyhd8ed1ab_2.conda sha256: 4770807cc5a217638c9aea3f05ea55718a82c50f32462df196b5472aff02787f md5: 23b4ba5619c4752976eb7ba1f5acb7e8 @@ -5742,6 +6654,18 @@ packages: - pkg:pypi/time-machine?source=hash-mapping size: 40713 timestamp: 1728484516013 +- conda: https://conda.anaconda.org/conda-forge/noarch/tinycss2-1.4.0-pyhd8ed1ab_0.conda + sha256: cad582d6f978276522f84bd209a5ddac824742fe2d452af6acf900f8650a73a2 + md5: f1acf5fdefa8300de697982bcb1761c9 + depends: + - python >=3.5 + - webencodings >=0.4 + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/tinycss2?source=hash-mapping + size: 28285 + timestamp: 1729802975370 - conda: https://conda.anaconda.org/conda-forge/linux-64/tk-8.6.13-noxft_hd72426e_102.conda sha256: a84ff687119e6d8752346d1d408d5cf360dee0badd487a472aa8ddedfdc219e1 md5: a0116df4f4ed05c303811a837d5b39d8 @@ -5777,6 +6701,31 @@ packages: - pkg:pypi/tomli?source=compressed-mapping size: 21238 timestamp: 1753796677376 +- conda: https://conda.anaconda.org/conda-forge/linux-64/tornado-6.5.2-py312h4c3975b_0.conda + sha256: 891965f8e495ad5cef399db03a13df48df7add06ae131f4b77a88749c74b2060 + md5: 82dacd4832dcde0c2b7888248a3b3d7c + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=14 + - python >=3.12,<3.13.0a0 + - python_abi 3.12.* *_cp312 + license: Apache-2.0 + license_family: Apache + purls: + - pkg:pypi/tornado?source=compressed-mapping + size: 850503 + timestamp: 1754732194289 +- conda: https://conda.anaconda.org/conda-forge/noarch/traitlets-5.14.3-pyhd8ed1ab_1.conda + sha256: f39a5620c6e8e9e98357507262a7869de2ae8cc07da8b7f84e517c9fd6c2b959 + md5: 019a7385be9af33791c989871317e1ed + depends: + - python >=3.9 + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/traitlets?source=hash-mapping + size: 110051 + timestamp: 1733367480074 - conda: https://conda.anaconda.org/conda-forge/noarch/typer-0.16.0-pyh167b9f4_0.conda sha256: 1ca70f0c0188598f9425a947afb74914a068bee4b7c4586eabb1c3b02fbf669f md5: 985cc086b73bda52b2f8d66dcda460a1 @@ -5875,6 +6824,17 @@ packages: - pkg:pypi/typing-extensions?source=hash-mapping size: 51065 timestamp: 1751643513473 +- conda: https://conda.anaconda.org/conda-forge/noarch/typing_utils-0.1.0-pyhd8ed1ab_1.conda + sha256: 3088d5d873411a56bf988eee774559335749aed6f6c28e07bf933256afb9eb6c + md5: f6d7aa696c67756a650e91e15e88223c + depends: + - python >=3.9 + license: Apache-2.0 + license_family: APACHE + purls: + - pkg:pypi/typing-utils?source=hash-mapping + size: 15183 + timestamp: 1733331395943 - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2025b-h78e105d_0.conda sha256: 5aaa366385d716557e365f0a4e9c3fca43ba196872abbbe3d56bb610d131e192 md5: 4222072737ccff51314b5ece9c7d6f5a @@ -5929,6 +6889,17 @@ packages: - pkg:pypi/universal-pathlib?source=hash-mapping size: 45126 timestamp: 1744798402969 +- conda: https://conda.anaconda.org/conda-forge/noarch/uri-template-1.3.0-pyhd8ed1ab_1.conda + sha256: e0eb6c8daf892b3056f08416a96d68b0a358b7c46b99c8a50481b22631a4dfc0 + md5: e7cb0f5745e4c5035a460248334af7eb + depends: + - python >=3.9 + license: MIT + license_family: MIT + purls: + - pkg:pypi/uri-template?source=hash-mapping + size: 23990 + timestamp: 1733323714454 - conda: https://conda.anaconda.org/conda-forge/noarch/urllib3-2.5.0-pyhd8ed1ab_0.conda sha256: 4fb9789154bd666ca74e428d973df81087a697dbb987775bc3198d2215f240f8 md5: 436c165519e140cb08d246a4472a9d6a @@ -6065,6 +7036,28 @@ packages: - pkg:pypi/wcwidth?source=hash-mapping size: 32581 timestamp: 1733231433877 +- conda: https://conda.anaconda.org/conda-forge/noarch/webcolors-24.11.1-pyhd8ed1ab_0.conda + sha256: 08315dc2e61766a39219b2d82685fc25a56b2817acf84d5b390176080eaacf99 + md5: b49f7b291e15494aafb0a7d74806f337 + depends: + - python >=3.9 + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/webcolors?source=hash-mapping + size: 18431 + timestamp: 1733359823938 +- conda: https://conda.anaconda.org/conda-forge/noarch/webencodings-0.5.1-pyhd8ed1ab_3.conda + sha256: 19ff205e138bb056a46f9e3839935a2e60bd1cf01c8241a5e172a422fed4f9c6 + md5: 2841eb5bfc75ce15e9a0054b98dcd64d + depends: + - python >=3.9 + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/webencodings?source=hash-mapping + size: 15496 + timestamp: 1733236131358 - conda: https://conda.anaconda.org/conda-forge/noarch/websocket-client-1.8.0-pyhd8ed1ab_1.conda sha256: 1dd84764424ffc82030c19ad70607e6f9e3b9cb8e633970766d697185652053e md5: 84f8f77f0a9c6ef401ee96611745da8f @@ -6102,6 +7095,17 @@ packages: - pkg:pypi/werkzeug?source=hash-mapping size: 252561 timestamp: 1676412048120 +- conda: https://conda.anaconda.org/conda-forge/noarch/widgetsnbextension-4.0.14-pyhd8ed1ab_0.conda + sha256: 7df3620c88343f2d960a58a81b79d4e4aa86ab870249e7165db7c3e2971a2664 + md5: 2f1f99b13b9d2a03570705030a0b3e7c + depends: + - python >=3.9 + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/widgetsnbextension?source=hash-mapping + size: 889285 + timestamp: 1744291155057 - conda: https://conda.anaconda.org/conda-forge/noarch/wirerope-1.0.0-pyhd8ed1ab_0.conda sha256: ffdcf6cde6072d29750fbae70c9f5af98c48dbfd67827bad19b136f03490fefc md5: 873aede420c1d00afb523acb9161b3bd @@ -6190,6 +7194,20 @@ packages: - pkg:pypi/yarl?source=hash-mapping size: 149496 timestamp: 1749555225039 +- conda: https://conda.anaconda.org/conda-forge/linux-64/zeromq-4.3.5-h3b0a872_7.conda + sha256: a4dc72c96848f764bb5a5176aa93dd1e9b9e52804137b99daeebba277b31ea10 + md5: 3947a35e916fcc6b9825449affbf4214 + depends: + - __glibc >=2.17,<3.0.a0 + - krb5 >=1.21.3,<1.22.0a0 + - libgcc >=13 + - libsodium >=1.0.20,<1.0.21.0a0 + - libstdcxx >=13 + license: MPL-2.0 + license_family: MOZILLA + purls: [] + size: 335400 + timestamp: 1731585026517 - conda: https://conda.anaconda.org/conda-forge/noarch/zipp-3.23.0-pyhd8ed1ab_0.conda sha256: 7560d21e1b021fd40b65bfb72f67945a3fcb83d78ad7ccf37b8b3165ec3b68ad md5: df5e78d904988eb55042c0c97446079f diff --git a/pyproject.toml b/pyproject.toml index 71abd9f..100b002 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "gaiaflow" requires-python = ">= 3.11" -version = "0.0.2" +version = "0.0.3.dev0" description = "Local-first MLOps infrastructure python tool that simplifies the process of building, testing, and deploying ML workflows." authors = [{name = "Yogesh Kumar Baljeet Singh", email = "yogesh.baljeetsingh@brockmann-consult.de"}] dependencies = [ @@ -16,6 +16,7 @@ dependencies = [ "apache-airflow-providers-fab>=2.3.1,<3", "apache-airflow-providers-docker>=4.4.2,<5", "mlflow>=3.2.0,<4", + "jupyter>=1.1.1,<2", ] readme= "README.md" @@ -78,6 +79,8 @@ mkdocs-material = ">=9.6.17,<10" mkdocstrings = ">=0.30.0,<0.31" mkdocstrings-python = ">=1.17.0,<2" pytest-cov = ">=6.2.1,<7" +jupyter = ">=1.1.1,<2" +pymdown-extensions = ">=10.16.1,<11" [project.scripts] gaiaflow = "gaiaflow.cli.cli:app" diff --git a/src/docker_stuff/airflow/Dockerfile b/src/docker_stuff/airflow/Dockerfile index 1cbe2d8..1ab5342 100644 --- a/src/docker_stuff/airflow/Dockerfile +++ b/src/docker_stuff/airflow/Dockerfile @@ -19,7 +19,7 @@ RUN micromamba --version USER airflow COPY environment.yml . -RUN micromamba env create -f environment.yml -n default_user_env +RUN micromamba env create -f environment.yml -n default_user_env && rm environment.yml RUN micromamba env list RUN micromamba shell init -s bash diff --git a/src/gaiaflow/cli/commands/mlops.py b/src/gaiaflow/cli/commands/mlops.py index 902b2b0..a80447e 100644 --- a/src/gaiaflow/cli/commands/mlops.py +++ b/src/gaiaflow/cli/commands/mlops.py @@ -52,12 +52,16 @@ def start( jupyter_port: int = typer.Option( 8895, "--jupyter-port", "-j", help="Port for JupyterLab" ), - delete_volume: bool = typer.Option( - False, "--delete-volume", "-v", help="Delete volumes on shutdown" - ), docker_build: bool = typer.Option( False, "--docker-build", "-b", help="Force Docker image build" ), + user_env_name: str = typer.Option( + None, "--env", "-e", help="Provide conda/mamba environment name for " + "Jupyter Lab to run. If not set, it will use the name from your environment.yml file." + ), + env_tool: "str" = typer.Option( + "mamba", "--env-tool", "-t", help="Which tool to use for running your Jupyter lab. Options: mamba, conda", + ), ): imports = load_imports() typer.echo(f"Selected Gaiaflow services: {service}") @@ -84,8 +88,9 @@ def start( service=s, cache=cache, jupyter_port=jupyter_port, - delete_volume=delete_volume, docker_build=docker_build, + user_env_name=user_env_name, + env_tool=env_tool, ) else: typer.echo("Running start with all services") @@ -97,8 +102,9 @@ def start( service=Service.all, cache=cache, jupyter_port=jupyter_port, - delete_volume=delete_volume, docker_build=docker_build, + user_env_name=user_env_name, + env_tool=env_tool, ) @@ -265,8 +271,31 @@ def dockerize( image_name=image_name ) +@app.command(help="Update the dependencies for the Airflow tasks. This command " + "synchronizes the running container environments with the project's" + "`environment.yml`. Make sure you have updated " + "`environment.yml` before running" + "this, as the container environments are updated based on " + "its contents.") +def update_deps( + project_path: Path = typer.Option(..., "--path", "-p", help="Path to your project"), +): + imports = load_imports() + gaiaflow_path, user_project_path = imports.create_gaiaflow_context_path( + project_path + ) + gaiaflow_path_exists = imports.gaiaflow_path_exists_in_state(gaiaflow_path, True) + if not gaiaflow_path_exists: + imports.save_project_state(user_project_path, gaiaflow_path) + else: + typer.echo( + f"Gaiaflow project already exists at {gaiaflow_path}. Skipping " + f"saving to the state" + ) - -# TODO: To let the user update the current infra with new local packages or -# mounts as they want it. -# def update(): + typer.echo("Running update_deps") + imports.MlopsManager.run( + gaiaflow_path=gaiaflow_path, + user_project_path=user_project_path, + action=imports.ExtendedAction.UPDATE_DEPS, + ) diff --git a/src/gaiaflow/constants.py b/src/gaiaflow/constants.py index 75a3e3e..82c843a 100644 --- a/src/gaiaflow/constants.py +++ b/src/gaiaflow/constants.py @@ -19,6 +19,7 @@ class ExtendedAction: DOCKERIZE = Action("dockerize") CREATE_CONFIG = Action("create_config") CREATE_SECRET = Action("create_secret") + UPDATE_DEPS = Action("update_deps") GAIAFLOW_CONFIG_DIR = Path.home() / ".gaiaflow" diff --git a/src/gaiaflow/core/operators.py b/src/gaiaflow/core/operators.py index dfa4ff0..9b673a2 100644 --- a/src/gaiaflow/core/operators.py +++ b/src/gaiaflow/core/operators.py @@ -125,7 +125,7 @@ def create_task(self): from gaiaflow.core.runner import run args, kwargs = self.resolve_args_kwargs() - kwargs["params"] = self.params + kwargs["params"] = dict(self.params) op_kwargs = {"func_path": self.func_path, "args": args, "kwargs": kwargs} return ExternalPythonOperator( @@ -135,6 +135,8 @@ def create_task(self): op_kwargs=op_kwargs, do_xcom_push=True, retries=self.retries, + expect_airflow=False, + expect_pendulum=False, ) diff --git a/src/gaiaflow/core/runner.py b/src/gaiaflow/core/runner.py index dd3a094..e5f1b74 100644 --- a/src/gaiaflow/core/runner.py +++ b/src/gaiaflow/core/runner.py @@ -49,17 +49,13 @@ def run( print(f"Running {func_path} with args: {args} and kwargs :{kwargs}") result = func(*args, **kwargs) print("Function result:", result) - print("mode::::", mode, type(mode)) if mode == "prod" or mode == "prod_local": # This is needed when we use KubernetesPodOperator and want to # share information via XCOM. _write_xcom_result(result) if mode == "dev_docker": - print("inside dev_docker condition") with open("/tmp/script.out", "wb+") as tmp: pickle.dump(result, tmp) - # print("printing result now:::") - # print(json.dumps(result)) return result diff --git a/src/gaiaflow/managers/minikube_manager.py b/src/gaiaflow/managers/minikube_manager.py index f573fa6..58d2942 100644 --- a/src/gaiaflow/managers/minikube_manager.py +++ b/src/gaiaflow/managers/minikube_manager.py @@ -59,15 +59,13 @@ def __init__( ) def _get_valid_actions(self) -> Set[Action]: - return { - BaseAction.START, - BaseAction.STOP, - BaseAction.RESTART, - BaseAction.CLEANUP, + base_actions = super()._get_valid_actions() + extra_actions = { ExtendedAction.DOCKERIZE, ExtendedAction.CREATE_CONFIG, ExtendedAction.CREATE_SECRET, } + return base_actions | extra_actions @classmethod def run(cls, **kwargs): diff --git a/src/gaiaflow/managers/mlops_manager.py b/src/gaiaflow/managers/mlops_manager.py index 6cc4354..7c87ef1 100644 --- a/src/gaiaflow/managers/mlops_manager.py +++ b/src/gaiaflow/managers/mlops_manager.py @@ -5,20 +5,31 @@ import socket import subprocess from pathlib import Path +from typing import Set import fsspec import psutil +import yaml from ruamel.yaml import YAML -from gaiaflow.constants import (AIRFLOW_SERVICES, GAIAFLOW_STATE_FILE, - MINIO_SERVICES, MLFLOW_SERVICES, Action, - BaseAction, Service) +from gaiaflow.constants import ( + AIRFLOW_SERVICES, + GAIAFLOW_STATE_FILE, + MINIO_SERVICES, + MLFLOW_SERVICES, + Action, + BaseAction, + Service, + ExtendedAction, +) from gaiaflow.managers.base_manager import BaseGaiaflowManager from gaiaflow.managers.utils import (create_directory, delete_project_state, find_python_packages, gaiaflow_path_exists_in_state, handle_error, log_error, log_info, run, - save_project_state, set_permissions, convert_crlf_to_lf) + save_project_state, set_permissions, convert_crlf_to_lf, + env_exists, + update_micromamba_env_in_docker) _IMAGES = [ "docker-compose-airflow-apiserver:latest", @@ -32,6 +43,13 @@ "postgres:13", ] +_AIRFLOW_CONTAINERS = [ + "airflow-apiserver", + "airflow-scheduler", + "airflow-dag-processor", + "airflow-triggerer" +] + _VOLUMES = [ "docker-compose_postgres-db-volume-airflow", "docker-compose_postgres-db-volume-mlflow", @@ -54,10 +72,16 @@ def __init__( force_new: bool = False, prune: bool = False, prod_local: bool = False, + user_env_name: str | None = None, + env_tool: str = "mamba", **kwargs, ): if kwargs: raise TypeError(f"Unexpected keyword arguments: {list(kwargs.keys())}") + if env_tool not in ("mamba", "conda"): + raise ValueError( + f"Invalid env_tool: {env_tool}. Must be 'mamba' or 'conda'" + ) self.service = service self.cache = cache self.jupyter_port = jupyter_port @@ -67,6 +91,8 @@ def __init__( self.project_root = Path(__file__).resolve().parent self.fs = fsspec.filesystem("file") self.prod_local = prod_local + self.user_env_name = user_env_name + self.env_tool = env_tool super().__init__( gaiaflow_path=gaiaflow_path, @@ -89,6 +115,7 @@ def run(cls, **kwargs): BaseAction.STOP: manager.stop, BaseAction.RESTART: manager.restart, BaseAction.CLEANUP: manager.cleanup, + ExtendedAction.UPDATE_DEPS: manager.update_deps, } try: @@ -128,6 +155,8 @@ def start(self): "skipping creating new context." ) + self._copy_user_env_file() + if self.service == Service.jupyter or self.service == Service.all: self._check_port() @@ -137,7 +166,13 @@ def start(self): build_cmd.append("--no-cache") log_info("Building Docker images") - self._docker_compose_action(build_cmd, self.service) + + if self.service == Service.all: + self._docker_compose_action(build_cmd, service=None) + elif self.service == Service.jupyter: + pass + else: + self._docker_compose_action(build_cmd, self.service) if self.service == Service.all: self._start_jupyter() @@ -167,6 +202,12 @@ def stop(self): log_info("Stopped Gaiaflow services successfully") + @staticmethod + def update_deps(): + log_info("Running update_deps") + update_micromamba_env_in_docker(_AIRFLOW_CONTAINERS) + log_info("Finished running update_deps") + def _check_port(self): with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: if sock.connect_ex(("127.0.0.1", self.jupyter_port)) == 0: @@ -223,9 +264,25 @@ def _docker_compose_action(self, actions, service=None): log_info(f"Running: {' '.join(cmd)}") run(cmd, f"Error running docker compose {actions}") + def get_env_name(self): + if self.user_env_name: + return self.user_env_name + + ctx_path = Path(self.gaiaflow_path).resolve() + env_path = ctx_path / "environment.yml" + with open(env_path, "r") as f: + env_yml = yaml.safe_load(f) + return env_yml.get("name") + def _start_jupyter(self): - log_info("Starting Jupyter Lab...") - cmd = ["jupyter", "lab", "--ip=0.0.0.0", f"--port={self.jupyter_port}"] + env_name = self.get_env_name() + if not env_exists(env_name, env_tool=self.env_tool): + print( + f"Environment {env_name} not found. Run `mamba env create " + f"-f environment.yml`?" + ) + cmd = [self.env_tool, "run", "-n", env_name, "jupyter", "lab", "--ip=0.0.0.0", f"--port={self.jupyter_port}"] + log_info("Starting Jupyter Lab..." + " ".join(cmd)) subprocess.Popen(cmd) def _update_env_file_with_airflow_uid(self, env_path): @@ -256,6 +313,13 @@ def _update_env_file_with_airflow_uid(self, env_path): log_info(f"Set AIRFLOW_UID={uid} in {env_path}") + def _copy_user_env_file(self): + log_info("Copying user environment.yml file") + shutil.copy( + self.user_project_path / "environment.yml", + self.gaiaflow_path / "environment.yml", + ) + def _create_gaiaflow_context(self): self.fs.makedirs(self.gaiaflow_path, exist_ok=True) @@ -264,10 +328,7 @@ def _create_gaiaflow_context(self): # docker_dir = package_dir / "docker_stuff" shutil.copytree(docker_dir, self.gaiaflow_path / "docker_stuff", dirs_exist_ok=True) - shutil.copy( - self.user_project_path / "environment.yml", - self.gaiaflow_path / "environment.yml", - ) + self._copy_user_env_file() log_info(f"Gaiaflow context created at {self.gaiaflow_path}") def _update_files(self): @@ -319,6 +380,12 @@ def _update_files(self): pyproject_path = (self.user_project_path.resolve() / "pyproject.toml").as_posix() new_volumes.append(f"{pyproject_path}:/opt/airflow/pyproject.toml") + pyproject_path = ( + self.user_project_path.resolve() / "environment.yml" + ).as_posix() + + new_volumes.append(f"{pyproject_path}:/opt/airflow/environment.yml") + new_volumes.append("/var/run/docker.sock:/var/run/docker.sock") compose_data["x-airflow-common"]["volumes"] = new_volumes @@ -334,7 +401,8 @@ def _update_files(self): def cleanup(self): try: - log_info("Attempting deleting Gaiaflow context at {self.gaiaflow_path}") + log_info(f"Attempting deleting Gaiaflow context at { + self.gaiaflow_path}") shutil.rmtree(self.gaiaflow_path) except FileNotFoundError: log_error(f"Gaiaflow context not found at {self.gaiaflow_path}") @@ -370,3 +438,8 @@ def cleanup(self): f"Error removing volume {volume}", ) log_info("Gaiaflow cleanup complete!") + + def _get_valid_actions(self) -> Set[Action]: + base_actions = super()._get_valid_actions() + extra_actions = {ExtendedAction.UPDATE_DEPS} + return base_actions | extra_actions diff --git a/src/gaiaflow/managers/utils.py b/src/gaiaflow/managers/utils.py index f710f65..429ff3e 100644 --- a/src/gaiaflow/managers/utils.py +++ b/src/gaiaflow/managers/utils.py @@ -1,7 +1,10 @@ +import docker import json import subprocess import sys import tempfile +from concurrent.futures._base import as_completed +from concurrent.futures.thread import ThreadPoolExecutor from datetime import datetime from pathlib import Path @@ -259,4 +262,48 @@ def is_wsl() -> bool: with open("/proc/version", "r") as f: return "microsoft" in f.read().lower() except FileNotFoundError: - return False \ No newline at end of file + return False + +def env_exists(env_name, env_tool="mamba"): + result = subprocess.run( + [env_tool, "env", "list", "--json"], capture_output=True, text=True + ) + envs = json.loads(result.stdout).get("envs", []) + return any(env_name in env for env in envs) + +def update_micromamba_env_in_docker( + containers: list[str], + env_name: str = "default_user_env", + max_workers: int = 4, + ): + client = docker.from_env() + + def _update_one(cname: str): + try: + container = client.containers.get(cname) + except docker.errors.NotFound: + log_error(f"Container '{cname}' not found. Skipping.") + return + + cmd = f"micromamba install -y -n {env_name} -f /opt/airflow/environment.yml" + log_info(f"[{cname}] Running command: {cmd}") + exit_code, output = container.exec_run(cmd) + + if exit_code != 0: + log_error(f"[{cname}] micromamba failed: {output.decode()}") + return + + log_info(f"[{cname}] Updated successfully.") + log_info(output.decode()) + + with ThreadPoolExecutor(max_workers=max_workers) as executor: + futures = { + executor.submit(_update_one, cname): cname for cname in containers + } + + for future in as_completed(futures): + cname = futures[future] + try: + future.result() + except Exception as e: + log_error(f"[{cname}] Unexpected error: {e}") \ No newline at end of file