Skip to content

Commit b7d874a

Browse files
jamesgwenjamesgwen
committed
adjusting memory handling
1 parent 83dffd7 commit b7d874a

File tree

1 file changed

+106
-167
lines changed

1 file changed

+106
-167
lines changed

‎.github/workflows/tests-linux.yml‎

Lines changed: 106 additions & 167 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
name: Tests linux
1+
name: Tests Linux (Low Memory)
22

33
on:
44
workflow_dispatch:
@@ -15,173 +15,112 @@ jobs:
1515
build-linux:
1616
runs-on: ubuntu-latest
1717
strategy:
18-
max-parallel: 5
18+
# Limit concurrency to reduce memory pressure
19+
max-parallel: 2
1920
matrix:
2021
python-version: ["3.9", "3.10"]
21-
timeout-minutes: 60 # add a timeout
22+
timeout-minutes: 60
2223

2324
steps:
24-
- uses: actions/checkout@v2
25-
- name: Set up Python ${{ matrix.python-version }}
26-
uses: actions/setup-python@v2
27-
with:
28-
python-version: ${{ matrix.python-version }}
29-
- name: Display Python version
30-
run: python -c "import sys; print(sys.version)"
31-
- name: Install dependencies
32-
shell: bash -l {0}
33-
run: |
34-
sudo apt-get update
35-
sudo apt-get install openslide-tools
36-
sudo apt-get install pandoc
37-
- name: Setup Micromamba
38-
uses: mamba-org/setup-micromamba@v1
39-
with:
40-
micromamba-version: '1.5.6-0'
41-
environment-name: pathml
42-
environment-file: requirements/environment_test.yml
43-
init-shell: >-
44-
bash
45-
create-args: >-
46-
python=${{ matrix.python-version }}
47-
- name: Debugging
48-
run: |
49-
echo "Printing the environment.yml file..."
50-
cat requirements/environment_test.yml
51-
echo "Checking the status of micro mamba..."
52-
micromamba config
53-
echo "Checking the available disk space..."
54-
df -h
55-
56-
# - name: Install dependencies with micromamba
57-
# shell: bash -l {0}
58-
# run: micromamba env update --file requirements/environment_test.yml --name pathml
59-
60-
- name: Micromamba info
61-
shell: bash -l {0}
62-
run: |
63-
micromamba info
64-
micromamba config
65-
micromamba list
66-
67-
- name: Set default Temurin JDK 17
68-
run: |
69-
sudo update-java-alternatives --set temurin-17-jdk-amd64 || true
70-
java -version
71-
72-
- name: Install PathML
73-
shell: bash -l {0}
74-
run: pip install -e .
75-
76-
- name: Install torchvision
77-
shell: bash -l {0}
78-
run: pip install torchvision==0.14.1
79-
80-
- name: disk usage
81-
shell: bash -l {0}
82-
run: |
83-
sudo df -h
84-
sudo du -h
85-
86-
# - name: free disk space
87-
# run: |
88-
# sudo swapoff -a
89-
# sudo rm -f /swapfile
90-
# sudo apt clean
91-
# docker rmi $(docker image ls -aq)
92-
# sudo df -h
93-
# sudo du -h
94-
95-
- name: free disk space
96-
run: |
97-
sudo swapoff -a
98-
sudo rm -f /swapfile
99-
sudo apt-get clean
100-
if [ "$(docker image ls -aq)" ]; then
101-
docker rmi $(docker image ls -aq)
102-
else
103-
echo "No Docker images to remove."
104-
fi
105-
sudo df -h
106-
sudo du -h
107-
108-
# - name: Check Coverage Command
109-
# run: |
110-
# which coverage
111-
# coverage --version
112-
113-
# - name: Test with pytest for tile_stitcher
114-
# run: |
115-
# java -version
116-
# python -m pytest tests/preprocessing_tests/test_tilestitcher.py
117-
118-
# - name: Test with pytest and generate coverage report
119-
# shell: bash -l {0}
120-
# run: |
121-
# coverage run -m pytest -m "not slow and not exclude"
122-
# coverage xml
123-
# - name: Upload coverage to Codecov
124-
# uses: codecov/codecov-action@v2
125-
# with:
126-
# token: ${{ secrets.CODECOV_TOKEN }}
127-
# env_vars: OS,PYTHON
128-
# fail_ci_if_error: true
129-
# files: ./coverage.xml
130-
# name: codecov-umbrella
131-
# verbose: true
132-
133-
- name: Fix coverage-numba incompatibility
134-
shell: bash -l {0}
135-
run: |
136-
micromamba activate pathml
137-
pip install "numpy==1.23.5" "coverage<7.3" "numba==0.59.1" --force-reinstall
138-
python -m coverage --version
139-
python -c "import numba; print('Numba version:', numba.__version__)"
140-
141-
- name: Test other modules with pytest and generate coverage
142-
shell: bash -l {0}
143-
run: |
144-
COVERAGE_FILE=.coverage_others coverage run -m pytest -m "not slow and not exclude"
145-
146-
- name: Test tile_stitcher with pytest and generate coverage
147-
shell: bash -l {0}
148-
run: |
149-
COVERAGE_FILE=.coverage_tilestitcher coverage run -m pytest tests/preprocessing_tests/test_tilestitcher.py
150-
151-
- name: List Files in Directory
152-
shell: bash -l {0}
153-
run: |
154-
ls -la
155-
156-
- name: Combine Coverage Data
157-
shell: bash -l {0}
158-
run: |
159-
coverage combine .coverage_tilestitcher .coverage_others
160-
161-
- name: Generate Combined Coverage Report
162-
shell: bash -l {0}
163-
run: |
164-
coverage xml -o combined_coverage.xml
165-
166-
# - name: Combine coverage data
167-
# shell: bash -l {0}
168-
# run: |
169-
# coverage combine coverage_tilestitcher.xml coverage_others.xml
170-
# coverage xml -o coverage_combined.xml
171-
172-
- name: Upload combined coverage to Codecov
173-
uses: codecov/codecov-action@v2
174-
with:
175-
token: ${{ secrets.CODECOV_TOKEN }}
176-
env_vars: OS,PYTHON
177-
fail_ci_if_error: true
178-
files: ./combined_coverage.xml
179-
name: codecov-umbrella
180-
verbose: true
181-
182-
- name: Compile docs
183-
shell: bash -l {0}
184-
run: |
185-
cd docs
186-
pip install -r readthedocs-requirements.txt
187-
make html
25+
- uses: actions/checkout@v4
26+
27+
- name: Set up Python ${{ matrix.python-version }}
28+
uses: actions/setup-python@v5
29+
with:
30+
python-version: ${{ matrix.python-version }}
31+
32+
- name: Display Python version
33+
run: python -c "import sys; print(sys.version)"
34+
35+
- name: Install minimal system dependencies
36+
run: |
37+
sudo apt-get update -qq
38+
sudo apt-get install -y --no-install-recommends openslide-tools pandoc
39+
sudo apt-get clean
40+
sudo rm -rf /var/lib/apt/lists/*
41+
42+
- name: Set up Micromamba (cached)
43+
uses: mamba-org/setup-micromamba@v2
44+
with:
45+
micromamba-version: "1.5.6-0"
46+
environment-name: pathml
47+
environment-file: requirements/environment_test.yml
48+
init-shell: bash
49+
cache-environment: true
50+
create-args: >-
51+
python=${{ matrix.python-version }}
52+
env:
53+
MAMBA_NO_BANNER: 1
54+
55+
- name: Show environment info
56+
run: |
57+
micromamba info
58+
micromamba list | head -n 20
59+
df -h
60+
61+
- name: Set default Temurin JDK 17
62+
run: |
63+
sudo update-java-alternatives --set temurin-17-jdk-amd64 || true
64+
java -version
65+
66+
- name: Install PathML (editable)
67+
shell: bash -l {0}
68+
run: pip install -e .
69+
70+
- name: Install lightweight torchvision
71+
shell: bash -l {0}
72+
run: pip install torchvision==0.14.1 --no-cache-dir
73+
74+
- name: Aggressive disk cleanup
75+
run: |
76+
echo "Freeing disk space..."
77+
sudo swapoff -a || true
78+
sudo rm -f /swapfile || true
79+
docker system prune -af || true
80+
sudo apt-get clean
81+
sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc || true
82+
df -h
83+
84+
- name: Fix coverage-numba incompatibility
85+
shell: bash -l {0}
86+
run: |
87+
micromamba activate pathml
88+
pip install "numpy==1.23.5" "coverage<7.3" "numba==0.59.1" --force-reinstall --no-cache-dir
89+
python -m coverage --version
90+
python -c "import numba; print('Numba version:', numba.__version__)"
91+
92+
- name: Run tests (non-tilestitcher)
93+
shell: bash -l {0}
94+
env:
95+
COVERAGE_FILE: .coverage_others
96+
run: |
97+
pytest -m "not slow and not exclude" --maxfail=2 --disable-warnings -q --cov --cov-append
98+
99+
- name: Run tests (tile_stitcher)
100+
shell: bash -l {0}
101+
env:
102+
COVERAGE_FILE: .coverage_tilestitcher
103+
run: |
104+
pytest tests/preprocessing_tests/test_tilestitcher.py --disable-warnings -q --cov --cov-append
105+
106+
- name: Combine coverage
107+
shell: bash -l {0}
108+
run: |
109+
coverage combine || true
110+
coverage xml -o combined_coverage.xml || true
111+
112+
- name: Upload coverage to Codecov
113+
uses: codecov/codecov-action@v4
114+
with:
115+
token: ${{ secrets.CODECOV_TOKEN }}
116+
files: ./combined_coverage.xml
117+
fail_ci_if_error: false
118+
name: codecov-umbrella
119+
verbose: false
120+
121+
- name: Build docs (lightweight)
122+
shell: bash -l {0}
123+
run: |
124+
cd docs
125+
pip install -r readthedocs-requirements.txt --no-cache-dir
126+
make html -j 1

0 commit comments

Comments
 (0)