Skip to content

Commit 7a303aa

Browse files
jamesgwenjamesgwen
authored
Merge pull request #459 from Dana-Farber-AIOS/dev
Adjusting the TQDM dependency. Updated all test YAML and environment YAML files. Pin specific versions of coverage and numba to ensure smooth testing on Python 3.10. Updated how the tests access the publication page to avoid security error.
2 parents 002653c + 2b68881 commit 7a303aa

File tree

11 files changed

+202
-209
lines changed

11 files changed

+202
-209
lines changed

‎.github/workflows/tests-linux.yml‎

Lines changed: 107 additions & 159 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
name: Tests linux
1+
name: Tests Linux (Low Memory)
22

33
on:
44
workflow_dispatch:
@@ -15,165 +15,113 @@ jobs:
1515
build-linux:
1616
runs-on: ubuntu-latest
1717
strategy:
18-
max-parallel: 5
18+
# Limit concurrency to reduce memory pressure
19+
max-parallel: 2
1920
matrix:
2021
python-version: ["3.9", "3.10"]
21-
timeout-minutes: 60 # add a timeout
22+
timeout-minutes: 60
2223

2324
steps:
24-
- uses: actions/checkout@v2
25-
- name: Set up Python ${{ matrix.python-version }}
26-
uses: actions/setup-python@v2
27-
with:
28-
python-version: ${{ matrix.python-version }}
29-
- name: Display Python version
30-
run: python -c "import sys; print(sys.version)"
31-
- name: Install dependencies
32-
shell: bash -l {0}
33-
run: |
34-
sudo apt-get update
35-
sudo apt-get install openslide-tools
36-
sudo apt-get install pandoc
37-
- name: Setup Micromamba
38-
uses: mamba-org/setup-micromamba@v1
39-
with:
40-
micromamba-version: '1.5.6-0'
41-
environment-name: pathml
42-
environment-file: requirements/environment_test.yml
43-
init-shell: >-
44-
bash
45-
create-args: >-
46-
python=${{ matrix.python-version }}
47-
- name: Debugging
48-
run: |
49-
echo "Printing the environment.yml file..."
50-
cat requirements/environment_test.yml
51-
echo "Checking the status of micro mamba..."
52-
micromamba config
53-
echo "Checking the available disk space..."
54-
df -h
55-
56-
# - name: Install dependencies with micromamba
57-
# shell: bash -l {0}
58-
# run: micromamba env update --file requirements/environment_test.yml --name pathml
59-
60-
- name: Micromamba info
61-
shell: bash -l {0}
62-
run: |
63-
micromamba info
64-
micromamba config
65-
micromamba list
66-
67-
- name: Set default Temurin JDK 17
68-
run: |
69-
sudo update-java-alternatives --set temurin-17-jdk-amd64 || true
70-
java -version
71-
72-
- name: Install PathML
73-
shell: bash -l {0}
74-
run: pip install -e .
75-
76-
- name: Install torchvision
77-
shell: bash -l {0}
78-
run: pip install torchvision==0.14.1
79-
80-
- name: disk usage
81-
shell: bash -l {0}
82-
run: |
83-
sudo df -h
84-
sudo du -h
85-
86-
# - name: free disk space
87-
# run: |
88-
# sudo swapoff -a
89-
# sudo rm -f /swapfile
90-
# sudo apt clean
91-
# docker rmi $(docker image ls -aq)
92-
# sudo df -h
93-
# sudo du -h
94-
95-
- name: free disk space
96-
run: |
97-
sudo swapoff -a
98-
sudo rm -f /swapfile
99-
sudo apt-get clean
100-
if [ "$(docker image ls -aq)" ]; then
101-
docker rmi $(docker image ls -aq)
102-
else
103-
echo "No Docker images to remove."
104-
fi
105-
sudo df -h
106-
sudo du -h
107-
108-
# - name: Check Coverage Command
109-
# run: |
110-
# which coverage
111-
# coverage --version
112-
113-
# - name: Test with pytest for tile_stitcher
114-
# run: |
115-
# java -version
116-
# python -m pytest tests/preprocessing_tests/test_tilestitcher.py
117-
118-
# - name: Test with pytest and generate coverage report
119-
# shell: bash -l {0}
120-
# run: |
121-
# coverage run -m pytest -m "not slow and not exclude"
122-
# coverage xml
123-
# - name: Upload coverage to Codecov
124-
# uses: codecov/codecov-action@v2
125-
# with:
126-
# token: ${{ secrets.CODECOV_TOKEN }}
127-
# env_vars: OS,PYTHON
128-
# fail_ci_if_error: true
129-
# files: ./coverage.xml
130-
# name: codecov-umbrella
131-
# verbose: true
132-
133-
- name: Test other modules with pytest and generate coverage
134-
shell: bash -l {0}
135-
run: |
136-
COVERAGE_FILE=.coverage_others coverage run -m pytest -m "not slow and not exclude"
137-
138-
- name: Test tile_stitcher with pytest and generate coverage
139-
shell: bash -l {0}
140-
run: |
141-
COVERAGE_FILE=.coverage_tilestitcher coverage run -m pytest tests/preprocessing_tests/test_tilestitcher.py
142-
143-
- name: List Files in Directory
144-
shell: bash -l {0}
145-
run: |
146-
ls -la
147-
148-
- name: Combine Coverage Data
149-
shell: bash -l {0}
150-
run: |
151-
coverage combine .coverage_tilestitcher .coverage_others
152-
153-
- name: Generate Combined Coverage Report
154-
shell: bash -l {0}
155-
run: |
156-
coverage xml -o combined_coverage.xml
157-
158-
# - name: Combine coverage data
159-
# shell: bash -l {0}
160-
# run: |
161-
# coverage combine coverage_tilestitcher.xml coverage_others.xml
162-
# coverage xml -o coverage_combined.xml
163-
164-
- name: Upload combined coverage to Codecov
165-
uses: codecov/codecov-action@v2
166-
with:
167-
token: ${{ secrets.CODECOV_TOKEN }}
168-
env_vars: OS,PYTHON
169-
fail_ci_if_error: true
170-
files: ./combined_coverage.xml
171-
name: codecov-umbrella
172-
verbose: true
173-
174-
- name: Compile docs
175-
shell: bash -l {0}
176-
run: |
177-
cd docs
178-
pip install -r readthedocs-requirements.txt
179-
make html
25+
- uses: actions/checkout@v4
26+
27+
- name: Set up Python ${{ matrix.python-version }}
28+
uses: actions/setup-python@v5
29+
with:
30+
python-version: ${{ matrix.python-version }}
31+
32+
- name: Display Python version
33+
run: python -c "import sys; print(sys.version)"
34+
35+
- name: Install minimal system dependencies
36+
run: |
37+
sudo apt-get update -qq
38+
sudo apt-get install -y --no-install-recommends openslide-tools pandoc
39+
sudo apt-get clean
40+
sudo rm -rf /var/lib/apt/lists/*
41+
42+
- name: Set up Micromamba (cached)
43+
uses: mamba-org/setup-micromamba@v2
44+
with:
45+
micromamba-version: "1.5.6-0"
46+
environment-name: pathml
47+
environment-file: requirements/environment_test.yml
48+
init-shell: bash
49+
cache-environment: true
50+
create-args: >-
51+
python=${{ matrix.python-version }}
52+
env:
53+
MAMBA_NO_BANNER: 1
54+
55+
- name: Show environment info
56+
run: |
57+
micromamba info
58+
micromamba list | head -n 20
59+
df -h
60+
61+
- name: Set default Temurin JDK 17
62+
run: |
63+
sudo update-java-alternatives --set temurin-17-jdk-amd64 || true
64+
java -version
65+
66+
- name: Install PathML (editable)
67+
shell: bash -l {0}
68+
run: pip install -e .
69+
70+
- name: Install lightweight torchvision
71+
shell: bash -l {0}
72+
run: pip install torchvision==0.14.1 --no-cache-dir
73+
74+
- name: Aggressive disk cleanup
75+
run: |
76+
echo "Freeing disk space..."
77+
sudo swapoff -a || true
78+
sudo rm -f /swapfile || true
79+
docker system prune -af || true
80+
sudo apt-get clean
81+
sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc || true
82+
df -h
83+
84+
- name: Fix coverage-numba incompatibility
85+
shell: bash -l {0}
86+
run: |
87+
micromamba activate pathml
88+
pip install "numpy==1.23.5" "coverage<7.3" "numba==0.59.1" --force-reinstall --no-cache-dir
89+
python -m coverage --version
90+
python -c "import numba; print('Numba version:', numba.__version__)"
91+
92+
- name: Test other modules with pytest and generate coverage
93+
shell: bash -l {0}
94+
run: |
95+
COVERAGE_FILE=.coverage_others coverage run -m pytest -m "not slow and not exclude"
96+
97+
- name: Test tile_stitcher with pytest and generate coverage
98+
shell: bash -l {0}
99+
run: |
100+
COVERAGE_FILE=.coverage_tilestitcher coverage run -m pytest tests/preprocessing_tests/test_tilestitcher.py
101+
102+
- name: Combine Coverage Data
103+
shell: bash -l {0}
104+
run: |
105+
coverage combine .coverage_tilestitcher .coverage_others
106+
107+
- name: Generate Combined Coverage Report
108+
shell: bash -l {0}
109+
run: |
110+
coverage xml -o combined_coverage.xml
111+
112+
- name: Upload combined coverage to Codecov
113+
uses: codecov/codecov-action@v2
114+
with:
115+
token: ${{ secrets.CODECOV_TOKEN }}
116+
env_vars: OS,PYTHON
117+
fail_ci_if_error: true
118+
files: ./combined_coverage.xml
119+
name: codecov-umbrella
120+
verbose: true
121+
122+
- name: Compile docs
123+
shell: bash -l {0}
124+
run: |
125+
cd docs
126+
pip install -r readthedocs-requirements.txt
127+
make html

‎.github/workflows/tests-macos.yml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,14 @@ jobs:
5858
- name: Install torchvision
5959
shell: bash -l {0}
6060
run: pip install torchvision==0.14.1
61+
62+
- name: Fix coverage-numba incompatibility
63+
shell: bash -l {0}
64+
run: |
65+
micromamba activate pathml
66+
pip install "numpy==1.23.5" "coverage<7.3" "numba==0.59.1" --force-reinstall
67+
python -m coverage --version
68+
python -c "import numba; print('Numba version:', numba.__version__)"
6169
6270
- name: Test other modules with pytest and generate coverage
6371
shell: bash -l {0}

‎.github/workflows/tests-windows.yml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -107,6 +107,14 @@ jobs:
107107
run: echo "OPENSLIDE_PATH=D:\a\pathml\pathml\openslide-binaries\openslide-win64-20231011\bin" >> $GITHUB_ENV
108108
shell: pwsh
109109

110+
- name: Fix coverage-numba incompatibility
111+
shell: bash -l {0}
112+
run: |
113+
micromamba activate pathml
114+
pip install "numpy==1.23.5" "coverage<7.3" "numba==0.59.1" --force-reinstall
115+
python -m coverage --version
116+
python -c "import numba; print('Numba version:', numba.__version__)"
117+
110118
- name: Test other modules with pytest and generate coverage
111119
shell: bash -l {0}
112120
run: |

‎environment.yml‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,12 +15,14 @@ dependencies:
1515
- h5py==3.10.0
1616
- dask<=2023.12.1
1717
- pydicom==2.4.4
18-
- pytest==7.4.3
18+
- pytest==7.4.3
1919
- pre-commit<=3.6.0
20-
- coverage==7.3.4
20+
- coverage==7.2.7
2121
- networkx<=3.2.1
2222
- pip:
2323
- -r requirements/requirements_torch.txt
24+
# - coverage==7.2.7
25+
- numba>=0.59
2426
- python-bioformats==4.0.7
2527
- python-javabridge==4.0.3
2628
- protobuf==3.20.3
@@ -30,7 +32,7 @@ dependencies:
3032
- openslide-python==1.3.1
3133
- scanpy==1.9.6
3234
- anndata<=0.10.3
33-
- tqdm==4.66.1
35+
- tqdm==4.66.3
3436
- loguru==0.7.2
3537
- pandas<=2.1.4
3638
- torch-geometric==2.3.1

‎pathml/_version.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,4 +3,4 @@
33
License: GNU GPL 2.0
44
"""
55

6-
__version__ = "3.0.3"
6+
__version__ = "3.0.4"

‎requirements/environment_docker.yml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,9 +15,10 @@ dependencies:
1515
- h5py==3.10.0
1616
- dask<=2023.12.1
1717
- pydicom==2.4.4
18+
- coverage<7.3
1819
- pytest==7.4.3
1920
- pre-commit<=3.6.0
20-
- coverage==7.3.4
21+
- coverage==7.2.7
2122
- networkx<=3.2.1
2223
- pip:
2324
- torch==2.6.0

‎requirements/environment_mac.yml‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,12 +18,14 @@ dependencies:
1818
- pydicom==2.4.4
1919
- pytest==7.4.3
2020
- pre-commit<=3.6.0
21-
- coverage==7.3.4
21+
- coverage==7.2.7
2222
- networkx<=3.2.1
2323
- pip:
24+
# - coverage==7.2.7
25+
- numba>=0.59
2426
# - setuptools==58.2.0
2527
# - numpy==1.23.5
26-
- torch==1.13.1
28+
- torch==2.8.0
2729
- python-bioformats==4.0.7
2830
# - python-javabridge==4.0.3
2931
- protobuf==3.20.3
@@ -32,7 +34,7 @@ dependencies:
3234
- opencv-contrib-python==4.8.1.78
3335
- scanpy==1.9.6
3436
- anndata<=0.10.3
35-
- tqdm==4.66.1
37+
- tqdm==4.66.3
3638
- loguru==0.7.2
3739
- pandas<=2.1.4
3840
- torch-geometric==2.3.1

0 commit comments

Comments
 (0)