Skip to content

Commit 30139a8

Browse files
Add tests for same clusters as input data in same_cluster_as_input_data.py Update CI (#173)
* Add tests for same clusters as input data in `same_cluster_as_input_data.py` Update CI * Update ci * Add max python version * dummy commit * increment version number * Refactor tests for same clusters to use parameterized inputs and improve readability * Enhance warning filters in pyproject.toml and update test_assert_raises to use pytest for warning assertions * Refactor filterwarnings in pyproject.toml for improved clarity and consistency * Normalize deprecated lowercase day alias in duration parsing
1 parent a21bd3b commit 30139a8

11 files changed

+165
-24
lines changed

.github/workflows/test_on_push.yml

Lines changed: 15 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,14 @@ on:
44
branches: ["**"]
55

66
jobs:
7-
TestPushCondaForgeDev:
8-
name: Test conda-forge development version on ${{ matrix.runner_tag }}
7+
extract-python-versions:
8+
uses: FZJ-IEK3-VSA/.github/.github/workflows/_extract_env_versions.yml@main
9+
with:
10+
library_name: python
11+
env_file: environment.yml
12+
13+
TestPullCondaForgeDev:
14+
name: Test development version on ${{ matrix.runner_tag }}
915
strategy:
1016
fail-fast: false
1117
matrix:
@@ -17,17 +23,17 @@ jobs:
1723
examples_to_execute: "docs/source/examples_notebooks/quickstart.ipynb docs/source/examples_notebooks/clustering_methods.ipynb docs/source/examples_notebooks/optimization_input.ipynb docs/source/examples_notebooks/representations.ipynb docs/source/examples_notebooks/segmentation.ipynb docs/source/examples_notebooks/k_maxoids.ipynb docs/source/examples_notebooks/clustering_transfer.ipynb"
1824
multiprocessing_pytest_string: "-n auto"
1925
multiprocessing_example_string: "-n auto"
20-
TestPushPyPIDev:
21-
name: Test PyPi development version on ${{ matrix.runner_tag }}
26+
TestPullPyPIDev:
27+
name: Test PyPI development version on ${{ matrix.runner_tag }}
28+
needs: extract-python-versions
2229
strategy:
2330
fail-fast: false
2431
matrix:
2532
runner_tag: ["self-hosted"]
26-
uses: FZJ-IEK3-VSA/.github/.github/workflows/_run_single_pypi_test.yml@main
33+
uses: FZJ-IEK3-VSA/.github/.github/workflows/_run_single_pypi_test_uv.yml@main
2734
with:
2835
runner_tag: ${{ matrix.runner_tag }}
29-
python_version: "3.13"
30-
optional_dependency_PyPI_tag: "[develop]"
3136
examples_to_execute: "docs/source/examples_notebooks/quickstart.ipynb docs/source/examples_notebooks/clustering_methods.ipynb docs/source/examples_notebooks/optimization_input.ipynb docs/source/examples_notebooks/representations.ipynb docs/source/examples_notebooks/segmentation.ipynb docs/source/examples_notebooks/k_maxoids.ipynb docs/source/examples_notebooks/clustering_transfer.ipynb"
32-
multiprocessing_pytest_string: "-n auto"
33-
multiprocessing_example_string: "-n auto"
37+
python_version: ${{ needs.extract-python-versions.outputs.max_version }}
38+
optional_dependency_PyPI_tag: "[develop]"
39+

.github/workflows/test_pull_dev.yml

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,10 +8,13 @@ jobs:
88
uses: FZJ-IEK3-VSA/.github/.github/workflows/_extract_env_versions.yml@main
99
with:
1010
library_name: python
11+
env_file: environment.yml
12+
1113

1214
TestPullCondaForgeDev:
1315
name: Test development version on ${{ matrix.runner_tag }}
1416
needs: extract-python-versions
17+
if: github.actor != 'renovate[bot]'
1518
strategy:
1619
fail-fast: false
1720
matrix:
@@ -30,25 +33,27 @@ jobs:
3033
TestPullPyPIDev:
3134
name: Test PyPI development version on ${{ matrix.runner_tag }}
3235
needs: extract-python-versions
36+
if: github.actor != 'renovate[bot]'
3337
strategy:
3438
fail-fast: false
3539
matrix:
3640
runner_tag: ["ubuntu-latest", "macos-latest", "windows-latest"]
37-
uses: FZJ-IEK3-VSA/.github/.github/workflows/_run_single_pypi_test.yml@main
41+
uses: FZJ-IEK3-VSA/.github/.github/workflows/_run_single_pypi_test_uv.yml@main
3842
with:
3943
runner_tag: ${{ matrix.runner_tag }}
4044
examples_to_execute: "docs/source/examples_notebooks/quickstart.ipynb docs/source/examples_notebooks/clustering_methods.ipynb docs/source/examples_notebooks/optimization_input.ipynb docs/source/examples_notebooks/representations.ipynb docs/source/examples_notebooks/segmentation.ipynb docs/source/examples_notebooks/k_maxoids.ipynb docs/source/examples_notebooks/clustering_transfer.ipynb"
4145
python_version: ${{ needs.extract-python-versions.outputs.max_version }}
4246
optional_dependency_PyPI_tag: "[develop]"
43-
additional_conda_forge_dependencies: "glpk"
4447

4548
extract-min-versions:
49+
if: github.actor != 'renovate[bot]'
4650
uses: FZJ-IEK3-VSA/.github/.github/workflows/_extract_env_versions_matrix.yml@main
4751
with:
4852
libraries: "python,scikit-learn,pandas,numpy,pyomo,networkx,tqdm,highspy"
4953
version_type: min
5054

5155
TestMinDependencies:
56+
if: github.actor != 'renovate[bot]'
5257
needs: extract-min-versions
5358
strategy:
5459
fail-fast: false

.github/workflows/test_pull_request_master_min_max_versions.yml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ jobs:
99
with:
1010
libraries: "scikit-learn,pandas,numpy,pyomo,networkx,tqdm,highspy"
1111
version_type: both
12+
env_file: environment.yml
1213

1314
TestDependencyVersions:
1415
name: Test ${{ matrix.dependencies.library_name }} ${{ matrix.dependencies.version }} (${{ matrix.dependencies.version_type }})

.github/workflows/test_pull_request_master_python_versions.yml

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ jobs:
88
uses: FZJ-IEK3-VSA/.github/.github/workflows/_extract_env_versions.yml@main
99
with:
1010
library_name: python
11+
env_file: environment.yml
1112

1213
TestPushCondaForgeDev:
1314
name: Test conda-forge development version on ${{ matrix.runner_tag }}
@@ -44,7 +45,7 @@ jobs:
4445
"windows-2022",
4546
]
4647
python_version: ${{ fromJSON(needs.extract-python-versions.outputs.versions) }}
47-
uses: FZJ-IEK3-VSA/.github/.github/workflows/_run_single_pypi_test.yml@main
48+
uses: FZJ-IEK3-VSA/.github/.github/workflows/_run_single_pypi_test_uv.yml@main
4849
with:
4950
runner_tag: ${{ matrix.runner_tag }}
5051
python_version: ${{ matrix.python_version }}
Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
on:
2+
pull_request:
3+
branches: ["develop"]
4+
5+
jobs:
6+
detect-changes:
7+
name: Detect changed dependencies
8+
if: startsWith(github.head_ref, 'renovate/')
9+
uses: FZJ-IEK3-VSA/.github/.github/workflows/_detect_renovate_changed_libraries.yml@main
10+
with:
11+
files: "environment.yml pyproject.toml"
12+
13+
extract-versions:
14+
name: Extract max versions of changed libraries
15+
needs: detect-changes
16+
if: startsWith(github.head_ref, 'renovate/') && needs.detect-changes.outputs.libraries != ''
17+
uses: FZJ-IEK3-VSA/.github/.github/workflows/_extract_env_versions_matrix.yml@main
18+
with:
19+
libraries: ${{ needs.detect-changes.outputs.libraries }}
20+
version_type: max
21+
env_file: environment.yml
22+
23+
TestRenovateMaxVersions:
24+
name: Test ${{ matrix.dependencies.library_name }} ${{ matrix.dependencies.version }} (max)
25+
needs: extract-versions
26+
if: startsWith(github.head_ref, 'renovate/')
27+
strategy:
28+
fail-fast: false
29+
matrix:
30+
runner_tag: ["self-hosted"]
31+
dependencies: ${{ fromJSON(needs.extract-versions.outputs.matrix) }}
32+
uses: FZJ-IEK3-VSA/.github/.github/workflows/_run_single_conda_forge_test.yml@main
33+
with:
34+
runner_tag: ${{ matrix.runner_tag }}
35+
requirements_file_name: environment.yml
36+
examples_to_execute: "docs/source/examples_notebooks/quickstart.ipynb docs/source/examples_notebooks/clustering_methods.ipynb docs/source/examples_notebooks/optimization_input.ipynb docs/source/examples_notebooks/representations.ipynb docs/source/examples_notebooks/segmentation.ipynb docs/source/examples_notebooks/k_maxoids.ipynb docs/source/examples_notebooks/clustering_transfer.ipynb"
37+
library_name: ${{ matrix.dependencies.library_name }}
38+
library_version: ${{ matrix.dependencies.version }}
39+
dependency_position_env_file: ${{ matrix.dependencies.yaml_position }}
40+
multiprocessing_pytest_string: "-n auto"
41+
multiprocessing_example_string: "-n auto"

README.md

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,6 @@ import pandas as pd
8080
import tsam
8181
```
8282

83-
8483
Read in the time series data set with pandas
8584
```python
8685
raw = pd.read_csv('testdata.csv', index_col=0, parse_dates=True)

environment.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ name: tsam_env
22
channels:
33
- conda-forge
44
dependencies:
5-
- python>=3.10,<3.15
5+
- python>=3.10,<=3.14.3
66
- pip
77
# Core dependencies
88
- scikit-learn >=1.3.0,<=1.8.0

pyproject.toml

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta"
55

66
[project]
77
name = "tsam"
8-
version = "3.1.1"
8+
version = "3.1.2"
99
description = "Time series aggregation module (tsam) to create typical periods"
1010
authors = [
1111
{ name = "Leander Kotzur", email = "leander.kotzur@googlemail.com" },
@@ -79,7 +79,17 @@ pythonpath = [
7979
"test",
8080
] # Sets the path which should be prepended to pythonpath relative to the root folder
8181
console_output_style = "count"
82-
filterwarnings = ["ignore::tsam.exceptions.LegacyAPIWarning"]
82+
filterwarnings = [
83+
"ignore::tsam.exceptions.LegacyAPIWarning",
84+
# Third-party library warnings outside of tsam's control
85+
"ignore::RuntimeWarning:threadpoolctl",
86+
"ignore:KMeans is known to have a memory leak:UserWarning:sklearn",
87+
"ignore::sklearn.exceptions.ConvergenceWarning",
88+
# Expected tsam warnings raised during edge-case tests
89+
"ignore:The cluster is too small:UserWarning:tsam",
90+
"ignore:Segmentation is turned off:UserWarning:tsam",
91+
"ignore:Max iteration number reached:UserWarning:tsam",
92+
]
8393

8494
[tool.ruff]
8595
target-version = "py310"
@@ -115,7 +125,6 @@ ignore = [
115125
"RUF012", # mutable class attributes - these are constants in this codebase
116126
"RUF002", # ambiguous unicode characters in docstrings
117127
"RUF059", # Unpacked variable is never used
118-
"UP038", # use X | Y in isinstance (performance regression, see ruff issue)
119128
]
120129

121130
[tool.ruff.lint.isort]

src/tsam/api.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
from __future__ import annotations
44

5+
import re
56
import warnings
67
from typing import cast
78

@@ -37,6 +38,8 @@ def _parse_duration_hours(value: int | float | str, param_name: str) -> float:
3738
return float(value)
3839
if isinstance(value, str):
3940
try:
41+
# Normalize deprecated lowercase day alias: '1d' → '1D' (pandas 4+)
42+
value = re.sub(r"(?<=[0-9])d(?![a-z])", "D", value)
4043
td = pd.Timedelta(value)
4144
return td.total_seconds() / 3600
4245
except ValueError as e:

test/same_cluster_as_input_data.py

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
import itertools
2+
3+
import numpy as np
4+
import pandas as pd
5+
import pytest
6+
7+
import tsam
8+
from tsam import ClusterConfig
9+
10+
# All clustering methods (excluding "averaging" which does not cluster into n_clusters)
11+
_METHODS = ["kmeans", "kmedoids", "kmaxoids", "hierarchical", "contiguous"]
12+
13+
# All representation methods
14+
_REPRESENTATIONS = ["mean", "medoid", "maxoid", "distribution", "distribution_minmax"]
15+
16+
# Use duration curves when clustering by value distribution
17+
_DISTRIBUTION_REPS = {"distribution", "distribution_minmax"}
18+
19+
_PARAMS = [
20+
pytest.param(
21+
method,
22+
rep,
23+
rep in _DISTRIBUTION_REPS,
24+
id=f"{method}_{rep}",
25+
)
26+
for method, rep in itertools.product(_METHODS, _REPRESENTATIONS)
27+
]
28+
29+
30+
@pytest.fixture(scope="module")
31+
def input_data() -> pd.DataFrame:
32+
costs = pd.DataFrame(
33+
[
34+
np.array([0.05, 0.0, 0.1, 0.051]),
35+
np.array([0.0, 0.0, 0.0, 0.0]),
36+
],
37+
index=["ElectrolyzerLocation", "IndustryLocation"],
38+
).T
39+
revenues = pd.DataFrame(
40+
[
41+
np.array([0.0, 0.01, 0.0, 0.0]),
42+
np.array([0.0, 0.0, 0.0, 0.0]),
43+
],
44+
index=["ElectrolyzerLocationRevenue", "IndustryLocationRevenue"],
45+
).T
46+
47+
timeSeriesData = pd.concat([costs, revenues], axis=1)
48+
timeSeriesData.index = pd.date_range(
49+
"2050-01-01 00:30:00",
50+
periods=4,
51+
freq="1h",
52+
tz="Europe/Berlin",
53+
)
54+
return timeSeriesData
55+
56+
57+
@pytest.mark.parametrize("method,representation,use_duration_curves", _PARAMS)
58+
def test_same_cluster_as_input_data(
59+
input_data: pd.DataFrame,
60+
method: str,
61+
representation: str,
62+
use_duration_curves: bool,
63+
) -> None:
64+
"""When n_clusters equals the number of input periods, reconstruction must
65+
be identical to the original time series for every method/representation."""
66+
results = tsam.aggregate(
67+
input_data,
68+
n_clusters=4,
69+
period_duration=1,
70+
cluster=ClusterConfig(
71+
method=method,
72+
representation=representation,
73+
use_duration_curves=use_duration_curves,
74+
),
75+
)
76+
pd.testing.assert_frame_equal(results.reconstructed, input_data)

0 commit comments

Comments
 (0)