Skip to content

Commit a203837

Browse files
J535D165J535D165
authored
Change GitHub Actions to run workflow job per service (#83)
1 parent eb9f98f commit a203837

File tree

6 files changed

+205
-96
lines changed

6 files changed

+205
-96
lines changed

‎.github/workflows/python-package.yml‎

Lines changed: 34 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2,15 +2,44 @@
22
# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
33

44
name: Test repositories
5-
on: [push, pull_request]
5+
on:
6+
push:
7+
branches:
8+
- main
9+
pull_request:
10+
branches:
11+
- main
612

713
jobs:
14+
test-api:
15+
16+
runs-on: ubuntu-latest
17+
strategy:
18+
fail-fast: false
19+
matrix:
20+
python-version: ["3.8", "3.12"]
21+
22+
steps:
23+
- uses: actions/checkout@v3
24+
- name: Set up Python ${{ matrix.python-version }}
25+
uses: actions/setup-python@v3
26+
with:
27+
python-version: ${{ matrix.python-version }}
28+
- name: Install dependencies
29+
run: |
30+
python -m pip install --upgrade pip
31+
python -m pip install ".[test]"
32+
- name: Test with pytest
33+
run: |
34+
pytest -n 4 --ignore=tests/test_repositories_plus.py --ignore=tests/test_repositories.py
35+
836
test-repositories:
937

1038
runs-on: ubuntu-latest
1139
strategy:
1240
fail-fast: false
1341
matrix:
42+
service: ["zenodo", "dataverse", "figshare", "djehuty", "dryad", "osf", "mendeley", "dataone", "dspace", "pangaea", "github"]
1443
python-version: ["3.8", "3.12"]
1544

1645
steps:
@@ -22,11 +51,10 @@ jobs:
2251
- name: Install dependencies
2352
run: |
2453
python -m pip install --upgrade pip
25-
python -m pip install .
26-
python -m pip install pytest pytest-xdist
54+
python -m pip install ".[test]"
2755
- name: Test with pytest
2856
run: |
29-
pytest -n 4 --ignore=tests/test_repositories_plus.py
57+
pytest tests/test_repositories.py --service ${{ matrix.service }}
3058
3159
test-repositories-plus:
3260

@@ -45,8 +73,8 @@ jobs:
4573
- name: Install dependencies
4674
run: |
4775
python -m pip install --upgrade pip
48-
python -m pip install .[all]
49-
python -m pip install pytest pytest-xdist
76+
python -m pip install ".[all]"
77+
python -m pip install ".[test]"
5078
- name: Test with pytest
5179
run: |
5280
pytest -n 4 tests/test_repositories_plus.py

‎pyproject.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ datahugger = "datahugger.__main__:main"
2626
all = ["datasets"]
2727
benchmark = ["pandas", "tabulate"]
2828
lint = ["ruff"]
29-
test = ["pytest"]
29+
test = ["pytest", "tomli; python_version < '3.11'", "pytest-xdist"]
3030
docs = ["mkdocs-material"]
3131

3232
[build-system]

‎tests/conftest.py‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
from pathlib import Path
2+
3+
try:
4+
import tomllib
5+
except ImportError:
6+
import tomli as tomllib
7+
8+
9+
def _parse_repositories(test_config):
10+
return (
11+
test_config["location"],
12+
test_config.get("files", []),
13+
test_config.get("ignored_files", []),
14+
test_config.get("options", {}),
15+
)
16+
17+
18+
def pytest_addoption(parser):
19+
parser.addoption("--service", action="store", default=None, help="Service to test")
20+
21+
22+
def pytest_generate_tests(metafunc):
23+
with open(Path("tests", "test_repositories.toml"), "rb") as f:
24+
test_repos = tomllib.load(f)
25+
26+
if "location" in metafunc.fixturenames:
27+
if service_value := metafunc.config.option.service:
28+
metafunc.parametrize(
29+
"location,files,ignored_files,dh_kwargs",
30+
map(_parse_repositories, test_repos[service_value]),
31+
)
32+
else:
33+
if service_value and service_value not in test_repos:
34+
raise ValueError(f"Unknown service {service_value}")
35+
36+
metafunc.parametrize(
37+
"location,files,ignored_files,dh_kwargs",
38+
map(
39+
_parse_repositories,
40+
[v for _, service_v in test_repos.items() for v in service_v],
41+
),
42+
)

‎tests/test_repositories.py‎

Lines changed: 11 additions & 89 deletions
Original file line numberDiff line numberDiff line change
@@ -1,100 +1,22 @@
1-
from pathlib import Path
1+
from pathlib import PosixPath
22

33
import pytest
44

55
import datahugger
6-
from datahugger.utils import _is_url
76

8-
TESTS_URLS = [
9-
# Zenodo
10-
("10.5281/zenodo.6625880", "README.md"),
11-
# Dataverse
12-
(
13-
"https://dataverse.harvard.edu/dataset.xhtml?persistentId=doi:10.7910/DVN/KBHLOD",
14-
"tutorial1.py",
15-
),
16-
("https://doi.org/10.7910/DVN/KBHLOD", "tutorial1.py"),
17-
("https://hdl.handle.net/10622/NHJZUD", "ERRHS_7_01_data_1795.tab"),
18-
# Dataverse single file
19-
("10.7910/DVN/HZBYG7/RQ26H2", "Table 2.do"),
20-
# Figshare
21-
("https://doi.org/10.6084/m9.figshare.8851784.v1", "cross_year_data2.csv"),
22-
(
23-
"https://figshare.com/articles/dataset/Long-term_behavioral_repeatability_in_wild_adult_and_captive_juvenile_turtles_implications_for_personality_development/8851784",
24-
"cross_year_data2.csv",
25-
),
26-
(
27-
"https://doi.org/10.15131/shef.data.22010159.v2",
28-
"ScHARR QUIT evaluation statistical and health economic analysis plan.pdf",
29-
),
30-
# Djehuty
31-
("https://doi.org/10.4121/21989216.v1", "README.txt"),
32-
# Dryad
33-
(
34-
"https://datadryad.org/stash/dataset/doi:10.5061/dryad.31zcrjdm5",
35-
"ReadmeFile.txt",
36-
),
37-
("https://doi.org/10.5061/dryad.31zcrjdm5", "ReadmeFile.txt"),
38-
# OSF
39-
("https://osf.io/ews27/", "Cross-comparison/amarlt1"),
40-
("https://osf.io/kq573/", "nest_area_data.xlsx"),
41-
("https://doi.org/10.17605/OSF.IO/KQ573", "nest_area_data.xlsx"),
42-
("https://doi.org/10.17605/OSF.IO/9X6CA", "jobs.sh"),
43-
# Mendeley
44-
(
45-
"https://doi.org/10.17632/p6wmtv6t5g.2",
46-
"READMI Stranding Sea Turtle records.pdf",
47-
),
48-
("https://doi.org/10.17632/p6wmtv6t5g", "READMI Stranding Sea Turtle records.pdf"),
49-
# Dataone
50-
("https://doi.org/10.18739/A2KH0DZ42", "2012F_Temperature_Data.csv"),
51-
# DSpace
52-
("https://uhra.herts.ac.uk/handle/2299/26087", "pdf.pdf"),
53-
(
54-
"https://repositorioinstitucional.ceu.es/handle/10637/2741",
55-
"Aquaporin_1_JAMartin_et_al_MedSport_2009.pdf",
56-
),
57-
# Pangaea
58-
("https://doi.org/10.1594/PANGAEA.954547", "Gubbio_age.tab"),
59-
("https://doi.pangaea.de/10.1594/PANGAEA.954543", "AA_age.tab"),
60-
]
617

8+
def test_get_repositories(location, files, ignored_files, dh_kwargs, tmpdir):
9+
datahugger.get(location, tmpdir, **dh_kwargs)
6210

63-
def test_url_checker():
64-
assert not _is_url("82949")
65-
assert _is_url("https://doi.org/10.5281/zenodo.6614829")
11+
if files:
12+
assert PosixPath(tmpdir, files).exists()
6613

14+
if ignored_files:
15+
assert not PosixPath(tmpdir, ignored_files).exists()
6716

68-
@pytest.mark.parametrize("url_or_id,output_file", TESTS_URLS)
69-
def test_load_dyno(url_or_id, output_file, tmpdir):
70-
"""Load repository with the generic loader."""
71-
datahugger.get(url_or_id, tmpdir)
7217

73-
assert Path(tmpdir, output_file).exists()
18+
@pytest.mark.skip("Not implemented")
19+
def test_info(location, files, ignored_files, dh_kwargs, tmpdir):
20+
dh_info = datahugger.info(location)
7421

75-
76-
@pytest.mark.parametrize(
77-
"url_or_id",
78-
[
79-
("10.5281/zenodo.6614829"),
80-
("https://zenodo.org/record/6614829"),
81-
("https://doi.org/10.5281/zenodo.6614829"),
82-
],
83-
)
84-
def test_load_zenodo_6614829(url_or_id, tmpdir):
85-
datahugger.get(url_or_id, tmpdir, max_file_size=1e6)
86-
87-
assert Path(tmpdir, "quasiperiod.m").exists()
88-
assert not Path(tmpdir, "quasiperiod.txt.gz").exists()
89-
90-
91-
def test_load_github_cbsodata(tmpdir):
92-
datahugger.get("https://github.com/j535d165/cbsodata", tmpdir)
93-
94-
95-
def test_info_without_loading(tmpdir):
96-
dh_get = datahugger.get("https://osf.io/wdzh5/", output_folder=".", print_only=True)
97-
98-
dh_info = datahugger.info("https://osf.io/wdzh5/")
99-
100-
assert dh_get.dataset.files == dh_info.files
22+
assert files in dh_info.files

‎tests/test_repositories.toml‎

Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,111 @@
1+
[[zenodo]]
2+
location = "10.5281/zenodo.6625880"
3+
files = "README.md"
4+
5+
[[zenodo]]
6+
location = "10.5281/zenodo.6614829"
7+
files = "quasiperiod.m"
8+
ignored_files="quasiperiod.txt.gz"
9+
10+
[zenodo.options]
11+
max_file_size=1000000
12+
13+
[[zenodo]]
14+
location = "https://zenodo.org/record/6614829"
15+
files = "quasiperiod.m"
16+
ignored_files="quasiperiod.txt.gz"
17+
18+
[zenodo.options]
19+
max_file_size=1000000
20+
21+
[[zenodo]]
22+
location = "https://doi.org/10.5281/zenodo.6614829"
23+
files = "quasiperiod.m"
24+
ignored_files="quasiperiod.txt.gz"
25+
26+
[zenodo.options]
27+
max_file_size=1000000
28+
29+
[[dataverse]]
30+
location = "https://dataverse.harvard.edu/dataset.xhtml?persistentId=doi:10.7910/DVN/KBHLOD"
31+
files = "tutorial1.py"
32+
33+
[[dataverse]]
34+
location = "https://doi.org/10.7910/DVN/KBHLOD"
35+
files = "tutorial1.py"
36+
37+
[[dataverse]]
38+
location = "https://hdl.handle.net/10622/NHJZUD"
39+
files = "ERRHS_7_01_data_1795.tab"
40+
41+
[[figshare]]
42+
location = "https://doi.org/10.6084/m9.figshare.8851784.v1"
43+
files = "cross_year_data2.csv"
44+
45+
[[figshare]]
46+
location = "https://figshare.com/articles/dataset/Long-term_behavioral_repeatability_in_wild_adult_and_captive_juvenile_turtles_implications_for_personality_development/8851784"
47+
files = "cross_year_data2.csv"
48+
49+
[[figshare]]
50+
location = "https://doi.org/10.15131/shef.data.22010159.v2"
51+
files = "ScHARR QUIT evaluation statistical and health economic analysis plan.pdf"
52+
53+
[[djehuty]]
54+
location = "https://doi.org/10.4121/21989216.v1"
55+
files = "README.txt"
56+
57+
[[dryad]]
58+
location = "https://datadryad.org/stash/dataset/doi:10.5061/dryad.31zcrjdm5"
59+
files = "ReadmeFile.txt"
60+
61+
[[dryad]]
62+
location = "https://doi.org/10.5061/dryad.31zcrjdm5"
63+
files = "ReadmeFile.txt"
64+
65+
[[osf]]
66+
location = "https://osf.io/ews27/"
67+
files = "Cross-comparison/amarlt1"
68+
69+
[[osf]]
70+
location = "https://osf.io/kq573/"
71+
files = "nest_area_data.xlsx"
72+
73+
[[osf]]
74+
location = "https://doi.org/10.17605/OSF.IO/KQ573"
75+
files = "nest_area_data.xlsx"
76+
77+
[[osf]]
78+
location = "https://doi.org/10.17605/OSF.IO/9X6CA"
79+
files = "jobs.sh"
80+
81+
[[mendeley]]
82+
location = "https://doi.org/10.17632/p6wmtv6t5g.2"
83+
files = "READMI Stranding Sea Turtle records.pdf"
84+
85+
[[mendeley]]
86+
location = "https://doi.org/10.17632/p6wmtv6t5g"
87+
files = "READMI Stranding Sea Turtle records.pdf"
88+
89+
[[dataone]]
90+
location = "https://doi.org/10.18739/A2KH0DZ42"
91+
files = "2012F_Temperature_Data.csv"
92+
93+
[[dspace]]
94+
location = "https://uhra.herts.ac.uk/handle/2299/26087"
95+
files = "pdf.pdf"
96+
97+
[[dspace]]
98+
location = "https://repositorioinstitucional.ceu.es/handle/10637/2741"
99+
files = "Aquaporin_1_JAMartin_et_al_MedSport_2009.pdf"
100+
101+
[[pangaea]]
102+
location = "https://doi.org/10.1594/PANGAEA.954547"
103+
files = "Gubbio_age.tab"
104+
105+
[[pangaea]]
106+
location = "https://doi.pangaea.de/10.1594/PANGAEA.954543"
107+
files = "AA_age.tab"
108+
109+
[[github]]
110+
location = "https://github.com/j535d165/cbsodata"
111+
files = "cbsodata-main/README.md"

‎tests/test_utils.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,11 @@
1+
from datahugger.utils import _is_url
12
from datahugger.utils import get_re3data_repositories
23

34

5+
def test_url_checker():
6+
assert not _is_url("82949")
7+
assert _is_url("https://doi.org/10.5281/zenodo.6614829")
8+
9+
410
def test_re3data_repos():
511
get_re3data_repositories()

0 commit comments

Comments
 (0)