From df34a1a4cd23e89acc9c7ecd55851e54b9d0a91c Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Tue, 16 Jun 2026 15:10:14 +0200 Subject: [PATCH 1/2] feat(tutorials): ship tutorial data + downloadable example systems Notebook tutorial data was only reachable by cloning the repo or copying files out of GitHub, so users hit dead imports on a plain `pip install`. Add a public `fx.tutorials` API that makes every notebook standalone: - `get_data(name)` returns the synthetic datasets for notebooks 01-07, generated from numpy/pandas with no files or network. - `load_example(name)` downloads a pre-built FlowSystem for notebooks 08-09 from the project's GitHub releases (cached + hash-verified via pooch), so the heavy demandlib/pvlib/holidays generation no longer runs at user runtime. Gated behind the new `flixopt[tutorials]` extra. The dataset/example names live once each in the `DataName`/`ExampleName` Literals; lists, validation and builder dispatch all derive from them. Add `scripts/build_tutorial_datasets.py` and a manual `tutorial-data` workflow to build and upload the example artefacts to the data release, and migrate all notebooks to the new API. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/tutorial-data.yaml | 73 ++++++++++++ docs/notebooks/02-heat-system.ipynb | 4 +- .../03-investment-optimization.ipynb | 4 +- .../04-operational-constraints.ipynb | 4 +- docs/notebooks/05-multi-carrier-system.ipynb | 4 +- .../06a-time-varying-parameters.ipynb | 4 +- docs/notebooks/07-scenarios-and-periods.ipynb | 4 +- docs/notebooks/08a-aggregation.ipynb | 4 +- docs/notebooks/08b-rolling-horizon.ipynb | 4 +- docs/notebooks/08c-clustering.ipynb | 4 +- .../08c2-clustering-storage-modes.ipynb | 4 +- .../08d-clustering-multiperiod.ipynb | 4 +- docs/notebooks/08e-clustering-internals.ipynb | 4 +- .../08f-clustering-segmentation.ipynb | 8 +- .../09-plotting-and-data-access.ipynb | 8 +- flixopt/__init__.py | 3 +- flixopt/tutorials/__init__.py | 24 ++++ flixopt/tutorials/_examples.py | 108 ++++++++++++++++++ .../tutorials/_tutorial_data.py | 63 ++++++++-- pyproject.toml | 7 ++ scripts/build_tutorial_datasets.py | 72 ++++++++++++ tests/test_tutorials.py | 41 +++++++ 22 files changed, 395 insertions(+), 60 deletions(-) create mode 100644 .github/workflows/tutorial-data.yaml create mode 100644 flixopt/tutorials/__init__.py create mode 100644 flixopt/tutorials/_examples.py rename docs/notebooks/data/tutorial_data.py => flixopt/tutorials/_tutorial_data.py (79%) create mode 100644 scripts/build_tutorial_datasets.py create mode 100644 tests/test_tutorials.py diff --git a/.github/workflows/tutorial-data.yaml b/.github/workflows/tutorial-data.yaml new file mode 100644 index 000000000..bfaacaa52 --- /dev/null +++ b/.github/workflows/tutorial-data.yaml @@ -0,0 +1,73 @@ +name: Tutorial data + +# Builds the pre-built example FlowSystems and uploads them (plus registry.txt) as +# assets to the GitHub release that `flixopt.tutorials.load_example` downloads from. +# Run manually whenever the example systems change. The release tag must match +# `flixopt.tutorials._examples.DATA_RELEASE` (default: tutorial-data-v1). + +on: + workflow_dispatch: + inputs: + release_tag: + description: "Release tag to (re)upload the data assets to (must match DATA_RELEASE)." + required: true + default: "tutorial-data-v1" + +env: + PYTHON_VERSION: "3.11" + +jobs: + build-and-upload: + name: Build and upload example systems + runs-on: ubuntu-24.04 + permissions: + contents: write + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - uses: astral-sh/setup-uv@v7 + with: + version: "0.10.9" + enable-cache: true + + - uses: actions/setup-python@v6 + with: + python-version: ${{ env.PYTHON_VERSION }} + + # docs extra provides demandlib/pvlib/holidays used by the example generators. + - name: Install build dependencies + run: uv pip install --system -e ".[docs,full]" + + - name: Verify DATA_RELEASE matches the requested tag + run: | + EXPECTED=$(python -c "from flixopt.tutorials._examples import DATA_RELEASE; print(DATA_RELEASE)") + if [[ "$EXPECTED" != "$TAG" ]]; then + echo "::error::DATA_RELEASE is '$EXPECTED' but the workflow was asked to upload to '$TAG'." + echo "Update flixopt/tutorials/_examples.py or pass the matching tag." + exit 1 + fi + env: + TAG: ${{ inputs.release_tag }} + + - name: Build example systems + run: python scripts/build_tutorial_datasets.py --out-dir dist/tutorial_datasets + + - name: Create the data release if it does not exist + run: | + if ! gh release view "$TAG" >/dev/null 2>&1; then + gh release create "$TAG" \ + --title "Tutorial data ($TAG)" \ + --notes "Pre-built example FlowSystems downloaded by flixopt.tutorials.load_example." \ + --prerelease + fi + env: + GH_TOKEN: ${{ github.token }} + TAG: ${{ inputs.release_tag }} + + - name: Upload data assets + run: gh release upload "$TAG" dist/tutorial_datasets/* --clobber + env: + GH_TOKEN: ${{ github.token }} + TAG: ${{ inputs.release_tag }} diff --git a/docs/notebooks/02-heat-system.ipynb b/docs/notebooks/02-heat-system.ipynb index f36a1b7a9..829cb769c 100644 --- a/docs/notebooks/02-heat-system.ipynb +++ b/docs/notebooks/02-heat-system.ipynb @@ -57,9 +57,7 @@ "metadata": {}, "outputs": [], "source": [ - "from data.tutorial_data import get_heat_system_data\n", - "\n", - "data = get_heat_system_data()\n", + "data = fx.tutorials.get_data('heat_system')\n", "timesteps = data['timesteps']\n", "heat_demand = data['heat_demand']\n", "gas_price = data['gas_price']" diff --git a/docs/notebooks/03-investment-optimization.ipynb b/docs/notebooks/03-investment-optimization.ipynb index 9cfa0afee..03c27cdf5 100644 --- a/docs/notebooks/03-investment-optimization.ipynb +++ b/docs/notebooks/03-investment-optimization.ipynb @@ -82,9 +82,7 @@ "metadata": {}, "outputs": [], "source": [ - "from data.tutorial_data import get_investment_data\n", - "\n", - "data = get_investment_data()\n", + "data = fx.tutorials.get_data('investment')\n", "timesteps = data['timesteps']\n", "solar_profile = data['solar_profile']\n", "pool_demand = data['pool_demand']\n", diff --git a/docs/notebooks/04-operational-constraints.ipynb b/docs/notebooks/04-operational-constraints.ipynb index 401f99393..3210540aa 100644 --- a/docs/notebooks/04-operational-constraints.ipynb +++ b/docs/notebooks/04-operational-constraints.ipynb @@ -71,9 +71,7 @@ "metadata": {}, "outputs": [], "source": [ - "from data.tutorial_data import get_constraints_data\n", - "\n", - "data = get_constraints_data()\n", + "data = fx.tutorials.get_data('constraints')\n", "timesteps = data['timesteps']\n", "steam_demand = data['steam_demand']" ] diff --git a/docs/notebooks/05-multi-carrier-system.ipynb b/docs/notebooks/05-multi-carrier-system.ipynb index 3727227f4..f2eedb880 100644 --- a/docs/notebooks/05-multi-carrier-system.ipynb +++ b/docs/notebooks/05-multi-carrier-system.ipynb @@ -83,9 +83,7 @@ "metadata": {}, "outputs": [], "source": [ - "from data.tutorial_data import get_multicarrier_data\n", - "\n", - "data = get_multicarrier_data()\n", + "data = fx.tutorials.get_data('multicarrier')\n", "timesteps = data['timesteps']\n", "electricity_demand = data['electricity_demand']\n", "heat_demand = data['heat_demand']\n", diff --git a/docs/notebooks/06a-time-varying-parameters.ipynb b/docs/notebooks/06a-time-varying-parameters.ipynb index 5e1efa331..61b827932 100644 --- a/docs/notebooks/06a-time-varying-parameters.ipynb +++ b/docs/notebooks/06a-time-varying-parameters.ipynb @@ -77,9 +77,7 @@ "metadata": {}, "outputs": [], "source": [ - "from data.tutorial_data import get_time_varying_data\n", - "\n", - "data = get_time_varying_data()\n", + "data = fx.tutorials.get_data('time_varying')\n", "timesteps = data['timesteps']\n", "outdoor_temp = data['outdoor_temp']\n", "heat_demand = data['heat_demand']\n", diff --git a/docs/notebooks/07-scenarios-and-periods.ipynb b/docs/notebooks/07-scenarios-and-periods.ipynb index 1aae7660b..1b678ec91 100644 --- a/docs/notebooks/07-scenarios-and-periods.ipynb +++ b/docs/notebooks/07-scenarios-and-periods.ipynb @@ -71,9 +71,7 @@ "metadata": {}, "outputs": [], "source": [ - "from data.tutorial_data import get_scenarios_data\n", - "\n", - "data = get_scenarios_data()\n", + "data = fx.tutorials.get_data('scenarios')\n", "timesteps = data['timesteps']\n", "periods = data['periods']\n", "scenarios = data['scenarios']\n", diff --git a/docs/notebooks/08a-aggregation.ipynb b/docs/notebooks/08a-aggregation.ipynb index fc1748d86..747d09553 100644 --- a/docs/notebooks/08a-aggregation.ipynb +++ b/docs/notebooks/08a-aggregation.ipynb @@ -59,9 +59,7 @@ "metadata": {}, "outputs": [], "source": [ - "from data.generate_example_systems import create_district_heating_system\n", - "\n", - "flow_system = create_district_heating_system()\n", + "flow_system = fx.tutorials.load_example('district_heating')\n", "flow_system.connect_and_transform() # Align all data as xarray\n", "\n", "timesteps = flow_system.timesteps\n", diff --git a/docs/notebooks/08b-rolling-horizon.ipynb b/docs/notebooks/08b-rolling-horizon.ipynb index bc40aa01a..d405f5df0 100644 --- a/docs/notebooks/08b-rolling-horizon.ipynb +++ b/docs/notebooks/08b-rolling-horizon.ipynb @@ -63,9 +63,7 @@ "metadata": {}, "outputs": [], "source": [ - "from data.generate_example_systems import create_operational_system\n", - "\n", - "flow_system = create_operational_system().transform.resample('1h')\n", + "flow_system = fx.tutorials.load_example('operational').transform.resample('1h')\n", "flow_system.connect_and_transform() # Align all data as xarray\n", "\n", "timesteps = flow_system.timesteps\n", diff --git a/docs/notebooks/08c-clustering.ipynb b/docs/notebooks/08c-clustering.ipynb index 602468cba..2bed082b4 100644 --- a/docs/notebooks/08c-clustering.ipynb +++ b/docs/notebooks/08c-clustering.ipynb @@ -54,9 +54,7 @@ "metadata": {}, "outputs": [], "source": [ - "from data.generate_example_systems import create_district_heating_system\n", - "\n", - "flow_system = create_district_heating_system()\n", + "flow_system = fx.tutorials.load_example('district_heating')\n", "flow_system.connect_and_transform()\n", "\n", "timesteps = flow_system.timesteps\n", diff --git a/docs/notebooks/08c2-clustering-storage-modes.ipynb b/docs/notebooks/08c2-clustering-storage-modes.ipynb index 925adc82a..bc923e85c 100644 --- a/docs/notebooks/08c2-clustering-storage-modes.ipynb +++ b/docs/notebooks/08c2-clustering-storage-modes.ipynb @@ -63,9 +63,7 @@ "metadata": {}, "outputs": [], "source": [ - "from data.generate_example_systems import create_seasonal_storage_system\n", - "\n", - "flow_system = create_seasonal_storage_system()\n", + "flow_system = fx.tutorials.load_example('seasonal_storage')\n", "flow_system.connect_and_transform() # Align all data as xarray\n", "\n", "timesteps = flow_system.timesteps\n", diff --git a/docs/notebooks/08d-clustering-multiperiod.ipynb b/docs/notebooks/08d-clustering-multiperiod.ipynb index 82da05c6f..0ee0ef772 100644 --- a/docs/notebooks/08d-clustering-multiperiod.ipynb +++ b/docs/notebooks/08d-clustering-multiperiod.ipynb @@ -62,9 +62,7 @@ "metadata": {}, "outputs": [], "source": [ - "from data.generate_example_systems import create_multiperiod_system\n", - "\n", - "flow_system = create_multiperiod_system()\n", + "flow_system = fx.tutorials.load_example('multiperiod')\n", "\n", "print(f'Timesteps: {len(flow_system.timesteps)} ({len(flow_system.timesteps) // 24} days)')\n", "print(f'Periods: {list(flow_system.periods.values)}')\n", diff --git a/docs/notebooks/08e-clustering-internals.ipynb b/docs/notebooks/08e-clustering-internals.ipynb index 2d099ff34..4408b53b6 100644 --- a/docs/notebooks/08e-clustering-internals.ipynb +++ b/docs/notebooks/08e-clustering-internals.ipynb @@ -31,13 +31,11 @@ "metadata": {}, "outputs": [], "source": [ - "from data.generate_example_systems import create_district_heating_system\n", - "\n", "import flixopt as fx\n", "\n", "fx.CONFIG.notebook()\n", "\n", - "flow_system = create_district_heating_system()\n", + "flow_system = fx.tutorials.load_example('district_heating')\n", "flow_system.connect_and_transform()" ] }, diff --git a/docs/notebooks/08f-clustering-segmentation.ipynb b/docs/notebooks/08f-clustering-segmentation.ipynb index bc1915de4..f600cca8c 100644 --- a/docs/notebooks/08f-clustering-segmentation.ipynb +++ b/docs/notebooks/08f-clustering-segmentation.ipynb @@ -80,9 +80,7 @@ "metadata": {}, "outputs": [], "source": [ - "from data.generate_example_systems import create_district_heating_system\n", - "\n", - "flow_system = create_district_heating_system()\n", + "flow_system = fx.tutorials.load_example('district_heating')\n", "flow_system.connect_and_transform()\n", "\n", "print(f'Timesteps: {len(flow_system.timesteps)}')\n", @@ -466,9 +464,7 @@ "metadata": {}, "outputs": [], "source": [ - "from data.generate_example_systems import create_multiperiod_system\n", - "\n", - "fs_multi = create_multiperiod_system()\n", + "fs_multi = fx.tutorials.load_example('multiperiod')\n", "# Use first week only for faster demo\n", "fs_multi = fs_multi.transform.isel(time=slice(0, 168))\n", "\n", diff --git a/docs/notebooks/09-plotting-and-data-access.ipynb b/docs/notebooks/09-plotting-and-data-access.ipynb index a375fd641..449fb4446 100644 --- a/docs/notebooks/09-plotting-and-data-access.ipynb +++ b/docs/notebooks/09-plotting-and-data-access.ipynb @@ -35,8 +35,6 @@ "metadata": {}, "outputs": [], "source": [ - "from data.generate_example_systems import create_complex_system, create_multiperiod_system, create_simple_system\n", - "\n", "import flixopt as fx\n", "\n", "fx.CONFIG.notebook()" @@ -62,14 +60,14 @@ "# Create and optimize the example systems\n", "solver = fx.solvers.HighsSolver(mip_gap=0.01, log_to_console=False)\n", "\n", - "simple = create_simple_system()\n", + "simple = fx.tutorials.load_example('simple')\n", "\n", "simple.optimize(solver)\n", "\n", - "complex_sys = create_complex_system()\n", + "complex_sys = fx.tutorials.load_example('complex')\n", "complex_sys.optimize(solver)\n", "\n", - "multiperiod = create_multiperiod_system()\n", + "multiperiod = fx.tutorials.load_example('multiperiod')\n", "multiperiod.optimize(solver)\n", "\n", "print('Created systems:')\n", diff --git a/flixopt/__init__.py b/flixopt/__init__.py index bcf5f3ca9..d488225d3 100644 --- a/flixopt/__init__.py +++ b/flixopt/__init__.py @@ -17,7 +17,7 @@ # - xr.Dataset.fxstats (from stats_accessor) import xarray_plotly as _xpx # noqa: F401 -from . import clustering, linear_converters, plotting, results, solvers +from . import clustering, linear_converters, plotting, results, solvers, tutorials from . import stats_accessor as _fxstats # noqa: F401 from .carrier import Carrier, CarrierContainer from .comparison import Comparison @@ -69,6 +69,7 @@ 'results', 'linear_converters', 'solvers', + 'tutorials', ] # Initialize logger with default configuration (silent: WARNING level, NullHandler). diff --git a/flixopt/tutorials/__init__.py b/flixopt/tutorials/__init__.py new file mode 100644 index 000000000..09dfbc4bb --- /dev/null +++ b/flixopt/tutorials/__init__.py @@ -0,0 +1,24 @@ +"""Datasets and example systems for the flixopt tutorials and notebooks. + +Two tiers, so every notebook is standalone after ``pip install flixopt`` - no need +to clone the repository or copy files out of GitHub: + +* **Synthetic data** (notebooks 01-07) - generated on the fly from numpy/pandas, + no files and no network. Access by name with :func:`get_data`; see :func:`list_data`. + +* **Pre-built example systems** (notebooks 08-09) - downloaded (and cached) from the + project's GitHub releases with :func:`load_example`; see :func:`list_examples`. + Needs ``pooch`` (``pip install flixopt[tutorials]``). +""" + +from ._examples import ExampleName, list_examples, load_example +from ._tutorial_data import DataName, get_data, list_data + +__all__ = [ + 'DataName', + 'get_data', + 'list_data', + 'ExampleName', + 'load_example', + 'list_examples', +] diff --git a/flixopt/tutorials/_examples.py b/flixopt/tutorials/_examples.py new file mode 100644 index 000000000..5d672f1ef --- /dev/null +++ b/flixopt/tutorials/_examples.py @@ -0,0 +1,108 @@ +"""Download pre-built example FlowSystems for the advanced notebooks (08-09). + +Unlike the synthetic :mod:`._tutorial_data` helpers, these example systems are +built from realistic profiles (BDEW load profiles via ``demandlib``, weather via +``pvlib``) and real input time series. Rather than regenerating them - which would +pull in those heavy dependencies and the raw input data - we build them once, +serialise them with :meth:`flixopt.FlowSystem.to_netcdf`, host the artefacts on the +project's GitHub releases, and download them on demand. + +The download is cached on disk (via ``pooch``), so the network is only touched the +first time a given example is requested. + +Usage:: + + import flixopt as fx + + fs = fx.tutorials.load_example('district_heating') + fx.tutorials.list_examples() # -> available names +""" + +from __future__ import annotations + +import os +from typing import TYPE_CHECKING, Literal, get_args + +if TYPE_CHECKING: + from flixopt.flow_system import FlowSystem + +# GitHub release holding the example artefacts. Versioned independently of the +# package; bump it (and re-upload the assets) when the example systems change. +DATA_RELEASE = 'tutorial-data-v1' + +_BASE_URL_ENV = 'FLIXOPT_DATA_BASE_URL' # override for testing / self-hosting +_DEFAULT_BASE_URL = f'https://github.com/flixOpt/flixopt/releases/download/{DATA_RELEASE}/' +_REGISTRY_FILENAME = 'registry.txt' + +#: The available example systems - the single source of truth for their names. +#: Each is hosted as ``.nc`` and built by ``create__system`` in +#: ``docs/notebooks/data/generate_example_systems.py``. +ExampleName = Literal[ + 'simple', + 'complex', + 'district_heating', + 'operational', + 'seasonal_storage', + 'multiperiod', +] + +_INSTALL_HINT = ( + "Downloading example systems needs the 'pooch' package. Install it with " + '`pip install flixopt[tutorials]` (or `pip install pooch`).' +) + + +def list_examples() -> list[str]: + """Return the names of the example systems available via :func:`load_example`.""" + return list(get_args(ExampleName)) + + +def _base_url() -> str: + url = os.environ.get(_BASE_URL_ENV, _DEFAULT_BASE_URL) + return url if url.endswith('/') else url + '/' + + +def _make_pooch(): + try: + import pooch + except ModuleNotFoundError as e: + raise ModuleNotFoundError(_INSTALL_HINT) from e + + # Hashes are loaded from the hosted registry.txt so they never drift out of + # sync with the uploaded artefacts. + odie = pooch.create(path=pooch.os_cache('flixopt'), base_url=_base_url(), registry=None) + registry_path = pooch.retrieve( + url=_base_url() + _REGISTRY_FILENAME, + known_hash=None, + path=odie.path, + fname=_REGISTRY_FILENAME, + ) + odie.load_registry(registry_path) + return odie + + +def load_example(name: ExampleName) -> FlowSystem: + """Download (and cache) a pre-built example FlowSystem and return it. + + Args: + name: One of :func:`list_examples` (e.g. ``'district_heating'``). + + Returns: + The deserialised :class:`flixopt.FlowSystem`. + + Raises: + ValueError: If ``name`` is not a known example. + ModuleNotFoundError: If ``pooch`` is not installed. + + Note: + The first call for a given example downloads the artefact from the project's + GitHub releases; subsequent calls read it from the local cache. + """ + if name not in get_args(ExampleName): + raise ValueError(f'Unknown example {name!r}. Available: {", ".join(list_examples())}.') + + from flixopt.flow_system import FlowSystem + + odie = _make_pooch() + path = odie.fetch(f'{name}.nc') + return FlowSystem.from_netcdf(path) diff --git a/docs/notebooks/data/tutorial_data.py b/flixopt/tutorials/_tutorial_data.py similarity index 79% rename from docs/notebooks/data/tutorial_data.py rename to flixopt/tutorials/_tutorial_data.py index 3b4997e0a..ba2955d6d 100644 --- a/docs/notebooks/data/tutorial_data.py +++ b/flixopt/tutorials/_tutorial_data.py @@ -1,18 +1,23 @@ -"""Generate tutorial data for notebooks 01-07. +"""Synthetic tutorial data for the introductory notebooks (01-07). -These functions return data (timesteps, profiles, prices) rather than full FlowSystems, -so notebooks can demonstrate building systems step by step. +These functions return raw data (timesteps, profiles, prices) rather than full +FlowSystems, so the notebooks can demonstrate building systems step by step. -Usage: - from data.tutorial_data import get_quickstart_data, get_heat_system_data, ... +The data is generated purely from numpy/pandas - no files and no network access +are needed, so these helpers work straight out of a plain ``pip install flixopt``. + +These functions are private; use :func:`flixopt.tutorials.get_data` to access them +by name. """ +from typing import Literal, get_args + import numpy as np import pandas as pd import xarray as xr -def get_quickstart_data() -> dict: +def _get_quickstart_data() -> dict: """Data for 01-quickstart: minimal 4-hour example. Returns: @@ -31,7 +36,7 @@ def get_quickstart_data() -> dict: } -def get_heat_system_data() -> dict: +def _get_heat_system_data() -> dict: """Data for 02-heat-system: one week with storage. Returns: @@ -59,7 +64,7 @@ def get_heat_system_data() -> dict: } -def get_investment_data() -> dict: +def _get_investment_data() -> dict: """Data for 03-investment-optimization: solar pool heating. Returns: @@ -88,7 +93,7 @@ def get_investment_data() -> dict: } -def get_constraints_data() -> dict: +def _get_constraints_data() -> dict: """Data for 04-operational-constraints: factory steam demand. Returns: @@ -118,7 +123,7 @@ def get_constraints_data() -> dict: } -def get_multicarrier_data() -> dict: +def _get_multicarrier_data() -> dict: """Data for 05-multi-carrier-system: hospital CHP. Returns: @@ -164,7 +169,7 @@ def get_multicarrier_data() -> dict: } -def get_time_varying_data() -> dict: +def _get_time_varying_data() -> dict: """Data for 06a-time-varying-parameters: heat pump with variable COP. Returns: @@ -199,7 +204,7 @@ def get_time_varying_data() -> dict: } -def get_scenarios_data() -> dict: +def _get_scenarios_data() -> dict: """Data for 07-scenarios-and-periods: multi-year planning. Returns: @@ -244,3 +249,37 @@ def get_scenarios_data() -> dict: 'gas_prices': np.array([0.06, 0.08, 0.10]), 'elec_prices': np.array([0.28, 0.34, 0.43]), } + + +#: The available synthetic datasets - the single source of truth for their names. +#: Each maps to the private ``_get__data`` builder above. +DataName = Literal[ + 'quickstart', + 'heat_system', + 'investment', + 'constraints', + 'multicarrier', + 'time_varying', + 'scenarios', +] + + +def list_data() -> list[str]: + """Return the names accepted by :func:`get_data`.""" + return list(get_args(DataName)) + + +def get_data(name: DataName) -> dict: + """Return the synthetic tutorial dataset ``name`` as a dict of arrays. + + Generated from numpy/pandas, so this works offline with no extra dependencies. + + Args: + name: One of :func:`list_data` (e.g. ``'heat_system'``). + + Raises: + ValueError: If ``name`` is not a known dataset. + """ + if name not in get_args(DataName): + raise ValueError(f'Unknown dataset {name!r}. Available: {", ".join(list_data())}.') + return globals()[f'_get_{name}_data']() diff --git a/pyproject.toml b/pyproject.toml index 7cf3e2203..f9e292724 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -63,6 +63,11 @@ network_viz = [ "flask >= 3.0.0, < 4", # Explicit Flask cap to prevent transitive major bumps ] +# Downloading the pre-built example systems used by the advanced notebooks (08-09) +tutorials = [ + "pooch >= 1.8.0, < 2", # Download + cache example FlowSystems from GitHub releases +] + # Full feature set (everything except dev tools) full = [ "tsam >= 3.1.2, < 4", # Time series aggregation for clustering (3.0.0 and 3.1.0 yanked) @@ -75,6 +80,7 @@ full = [ "networkx >= 3.0.0, < 4", # Visualizing FlowSystem Network as app "werkzeug >= 3.0.0, < 4", # Visualizing FlowSystem Network as app "flask >= 3.0.0, < 4", # Explicit Flask cap to prevent transitive major bumps + "pooch >= 1.8.0, < 2", # Download + cache example FlowSystems for the notebooks ] # Development tools and testing @@ -95,6 +101,7 @@ dev = [ "dash-daq==0.6.0", "networkx==3.0.0", "werkzeug==3.1.8", + "pooch==1.8.2", ] # Documentation building diff --git a/scripts/build_tutorial_datasets.py b/scripts/build_tutorial_datasets.py new file mode 100644 index 000000000..fe14cf729 --- /dev/null +++ b/scripts/build_tutorial_datasets.py @@ -0,0 +1,72 @@ +"""Build the pre-built example FlowSystems hosted for the advanced notebooks (08-09). + +This regenerates the realistic example systems (which need ``demandlib``/``pvlib`` and +the raw input CSVs under ``docs/notebooks/data``), serialises each one with +``FlowSystem.to_netcdf`` and writes a ``registry.txt`` with sha256 hashes. + +The resulting ``*.nc`` files **and** ``registry.txt`` are uploaded as assets to the +GitHub release tagged ``flixopt.tutorials._examples.DATA_RELEASE``; at runtime +``flixopt.tutorials.load_example`` downloads them from there. Run this whenever the +example systems change, then re-upload the assets (the CI workflow does this on demand). + +Usage: + python scripts/build_tutorial_datasets.py [--out-dir dist/tutorial_datasets] +""" + +from __future__ import annotations + +import argparse +import hashlib +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent +DATA_DIR = REPO_ROOT / 'docs' / 'notebooks' / 'data' + + +def _sha256(path: Path) -> str: + h = hashlib.sha256() + with open(path, 'rb') as f: + for chunk in iter(lambda: f.read(1 << 20), b''): + h.update(chunk) + return h.hexdigest() + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + '--out-dir', + type=Path, + default=REPO_ROOT / 'dist' / 'tutorial_datasets', + help='Directory to write the *.nc artefacts and registry.txt into.', + ) + args = parser.parse_args() + + sys.path.insert(0, str(DATA_DIR)) + import generate_example_systems as ges # noqa: E402 + + out_dir: Path = args.out_dir + out_dir.mkdir(parents=True, exist_ok=True) + + from flixopt.tutorials import list_examples # noqa: E402 + + registry_lines = [] + for name in list_examples(): + func_name = f'create_{name}_system' + print(f'Building {name} via {func_name}() ...', flush=True) + fs = getattr(ges, func_name)() + path = out_dir / f'{name}.nc' + fs.to_netcdf(path) + digest = _sha256(path) + registry_lines.append(f'{name}.nc sha256:{digest}') + print(f' -> {path.name} ({path.stat().st_size:,} bytes) sha256:{digest}') + + registry_path = out_dir / 'registry.txt' + registry_path.write_text('\n'.join(registry_lines) + '\n') + print(f'\nWrote {registry_path} with {len(registry_lines)} entries.') + print('Upload every *.nc and registry.txt as assets to the GitHub release.') + return 0 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/tests/test_tutorials.py b/tests/test_tutorials.py new file mode 100644 index 000000000..b3c205599 --- /dev/null +++ b/tests/test_tutorials.py @@ -0,0 +1,41 @@ +"""Tests for the flixopt.tutorials tutorial-data helpers.""" + +from typing import get_args + +import pandas as pd +import pytest + +import flixopt as fx +from flixopt import tutorials + + +def test_tutorials_exposed_on_package(): + assert fx.tutorials is tutorials + + +@pytest.mark.parametrize('name', tutorials.list_data()) +def test_get_data_returns_timesteps(name): + """Synthetic tutorial data must work offline and expose a DatetimeIndex.""" + data = tutorials.get_data(name) + assert isinstance(data, dict) + assert isinstance(data['timesteps'], pd.DatetimeIndex) + assert len(data['timesteps']) > 0 + + +def test_list_data_matches_literal(): + assert tutorials.list_data() == list(get_args(tutorials.DataName)) + + +def test_get_data_rejects_unknown_name(): + with pytest.raises(ValueError, match='Unknown dataset'): + tutorials.get_data('does-not-exist') + + +def test_list_examples_matches_literal(): + assert tutorials.list_examples() == list(get_args(tutorials.ExampleName)) + + +def test_load_example_rejects_unknown_name(): + """Validation happens before any network access.""" + with pytest.raises(ValueError, match='Unknown example'): + tutorials.load_example('does-not-exist') From f95c86b7dca2410e277aa5f6999a48ad128f6892 Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Tue, 16 Jun 2026 15:41:27 +0200 Subject: [PATCH 2/2] ci(tutorials): disable checkout credential persistence The tutorial-data job is `contents: write` and authenticates its release steps via an explicit GH_TOKEN, so the checkout action's persisted git credentials are unnecessary. Set persist-credentials: false to reduce token exposure (per CodeRabbit review on #706). Action version tags left as-is to match the repo-wide convention. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/tutorial-data.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/tutorial-data.yaml b/.github/workflows/tutorial-data.yaml index bfaacaa52..ab05ed639 100644 --- a/.github/workflows/tutorial-data.yaml +++ b/.github/workflows/tutorial-data.yaml @@ -26,6 +26,7 @@ jobs: - uses: actions/checkout@v6 with: fetch-depth: 0 + persist-credentials: false - uses: astral-sh/setup-uv@v7 with: