Skip to content

Commit 2920bde

Browse files
beat-buesserbeat-buesser
committed
Cache test datasets
Signed-off-by: Beat Buesser <[email protected]>
1 parent 2da0770 commit 2920bde

12 files changed

+497
-0
lines changed

‎.github/workflows/ci-deepspeech-v3.yml

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,8 +29,48 @@ jobs:
2929
steps:
3030
- name: Checkout Repo
3131
uses: actions/checkout@v4
32+
33+
- name: Cache CIFAR-10 dataset
34+
uses: actions/cache@v3
35+
with:
36+
path: ~/.art/data/cifar-10-batches-py
37+
key: cifar10-dataset-cache-v1
38+
39+
- name: Cache MNIST dataset
40+
uses: actions/cache@v3
41+
with:
42+
path: ~/.art/data/mnist.npz
43+
key: mnist-dataset-cache-v1
44+
45+
- name: Download and extract CIFAR-10 if not cached
46+
run: |
47+
mkdir -p ~/.art/data
48+
if [ ! -d ~/.art/data/cifar-10-batches-py ]; then
49+
echo "Downloading CIFAR-10 dataset..."
50+
if curl -L -f -o cifar-10-python.tar.gz https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz; then
51+
echo "Downloaded from www.cs.toronto.edu"
52+
else
53+
echo "Primary download failed, trying Dropbox mirror..."
54+
curl -L -o cifar-10-python.tar.gz "https://www.dropbox.com/scl/fi/6mhiq4rtfszqrosasp8a9/cifar-10-batches-py.tar.gz?rlkey=g6v114un4je233j52tu5tdsbe&st=gxo2e50q&dl=1"
55+
fi
56+
tar -xzf cifar-10-python.tar.gz -C ~/.art/data/
57+
else
58+
echo "CIFAR-10 already cached."
59+
fi
60+
61+
- name: Download MNIST if not cached
62+
run: |
63+
mkdir -p ~/.art/data
64+
if [ ! -f ~/.art/data/mnist.npz ]; then
65+
echo "Downloading MNIST dataset..."
66+
curl -L -o ~/.art/data/mnist.npz https://s3.amazonaws.com/img-datasets/mnist.npz
67+
else
68+
echo "MNIST already cached."
69+
fi
70+
3271
- name: Run Test Action
3372
uses: ./.github/actions/deepspeech-v3
73+
3474
- name: Upload coverage to Codecov
3575
uses: codecov/codecov-action@v5
3676
with:

‎.github/workflows/ci-espresso.yml

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,8 +29,48 @@ jobs:
2929
steps:
3030
- name: Checkout Repo
3131
uses: actions/checkout@v4
32+
33+
- name: Cache CIFAR-10 dataset
34+
uses: actions/cache@v3
35+
with:
36+
path: ~/.art/data/cifar-10-batches-py
37+
key: cifar10-dataset-cache-v1
38+
39+
- name: Cache MNIST dataset
40+
uses: actions/cache@v3
41+
with:
42+
path: ~/.art/data/mnist.npz
43+
key: mnist-dataset-cache-v1
44+
45+
- name: Download and extract CIFAR-10 if not cached
46+
run: |
47+
mkdir -p ~/.art/data
48+
if [ ! -d ~/.art/data/cifar-10-batches-py ]; then
49+
echo "Downloading CIFAR-10 dataset..."
50+
if curl -L -f -o cifar-10-python.tar.gz https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz; then
51+
echo "Downloaded from www.cs.toronto.edu"
52+
else
53+
echo "Primary download failed, trying Dropbox mirror..."
54+
curl -L -o cifar-10-python.tar.gz "https://www.dropbox.com/scl/fi/6mhiq4rtfszqrosasp8a9/cifar-10-batches-py.tar.gz?rlkey=g6v114un4je233j52tu5tdsbe&st=gxo2e50q&dl=1"
55+
fi
56+
tar -xzf cifar-10-python.tar.gz -C ~/.art/data/
57+
else
58+
echo "CIFAR-10 already cached."
59+
fi
60+
61+
- name: Download MNIST if not cached
62+
run: |
63+
mkdir -p ~/.art/data
64+
if [ ! -f ~/.art/data/mnist.npz ]; then
65+
echo "Downloading MNIST dataset..."
66+
curl -L -o ~/.art/data/mnist.npz https://s3.amazonaws.com/img-datasets/mnist.npz
67+
else
68+
echo "MNIST already cached."
69+
fi
70+
3271
- name: Run Test Action
3372
uses: ./.github/actions/espresso
73+
3474
- name: Upload coverage to Codecov
3575
uses: codecov/codecov-action@v5
3676
with:

‎.github/workflows/ci-goturn.yml

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,8 +29,48 @@ jobs:
2929
steps:
3030
- name: Checkout Repo
3131
uses: actions/checkout@v4
32+
33+
- name: Cache CIFAR-10 dataset
34+
uses: actions/cache@v3
35+
with:
36+
path: ~/.art/data/cifar-10-batches-py
37+
key: cifar10-dataset-cache-v1
38+
39+
- name: Cache MNIST dataset
40+
uses: actions/cache@v3
41+
with:
42+
path: ~/.art/data/mnist.npz
43+
key: mnist-dataset-cache-v1
44+
45+
- name: Download and extract CIFAR-10 if not cached
46+
run: |
47+
mkdir -p ~/.art/data
48+
if [ ! -d ~/.art/data/cifar-10-batches-py ]; then
49+
echo "Downloading CIFAR-10 dataset..."
50+
if curl -L -f -o cifar-10-python.tar.gz https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz; then
51+
echo "Downloaded from www.cs.toronto.edu"
52+
else
53+
echo "Primary download failed, trying Dropbox mirror..."
54+
curl -L -o cifar-10-python.tar.gz "https://www.dropbox.com/scl/fi/6mhiq4rtfszqrosasp8a9/cifar-10-batches-py.tar.gz?rlkey=g6v114un4je233j52tu5tdsbe&st=gxo2e50q&dl=1"
55+
fi
56+
tar -xzf cifar-10-python.tar.gz -C ~/.art/data/
57+
else
58+
echo "CIFAR-10 already cached."
59+
fi
60+
61+
- name: Download MNIST if not cached
62+
run: |
63+
mkdir -p ~/.art/data
64+
if [ ! -f ~/.art/data/mnist.npz ]; then
65+
echo "Downloading MNIST dataset..."
66+
curl -L -o ~/.art/data/mnist.npz https://s3.amazonaws.com/img-datasets/mnist.npz
67+
else
68+
echo "MNIST already cached."
69+
fi
70+
3271
- name: Run Test Action
3372
uses: ./.github/actions/goturn
73+
3474
- name: Upload coverage to Codecov
3575
uses: codecov/codecov-action@v5
3676
env:

‎.github/workflows/ci-huggingface.yml

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,10 +40,12 @@ jobs:
4040
steps:
4141
- name: Checkout Repo
4242
uses: actions/checkout@v4
43+
4344
- name: Setup Python
4445
uses: actions/setup-python@v5
4546
with:
4647
python-version: ${{ matrix.python }}
48+
4749
- name: Install Dependencies
4850
run: |
4951
sudo apt-get update
@@ -57,8 +59,48 @@ jobs:
5759
pip install torchaudio==${{ matrix.torchaudio }} --index-url https://download.pytorch.org/whl/cpu
5860
pip install transformers==${{ matrix.transformers }}
5961
pip list
62+
63+
- name: Cache CIFAR-10 dataset
64+
uses: actions/cache@v3
65+
with:
66+
path: ~/.art/data/cifar-10-batches-py
67+
key: cifar10-dataset-cache-v1
68+
69+
- name: Cache MNIST dataset
70+
uses: actions/cache@v3
71+
with:
72+
path: ~/.art/data/mnist.npz
73+
key: mnist-dataset-cache-v1
74+
75+
- name: Download and extract CIFAR-10 if not cached
76+
run: |
77+
mkdir -p ~/.art/data
78+
if [ ! -d ~/.art/data/cifar-10-batches-py ]; then
79+
echo "Downloading CIFAR-10 dataset..."
80+
if curl -L -f -o cifar-10-python.tar.gz https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz; then
81+
echo "Downloaded from www.cs.toronto.edu"
82+
else
83+
echo "Primary download failed, trying Dropbox mirror..."
84+
curl -L -o cifar-10-python.tar.gz "https://www.dropbox.com/scl/fi/6mhiq4rtfszqrosasp8a9/cifar-10-batches-py.tar.gz?rlkey=g6v114un4je233j52tu5tdsbe&st=gxo2e50q&dl=1"
85+
fi
86+
tar -xzf cifar-10-python.tar.gz -C ~/.art/data/
87+
else
88+
echo "CIFAR-10 already cached."
89+
fi
90+
91+
- name: Download MNIST if not cached
92+
run: |
93+
mkdir -p ~/.art/data
94+
if [ ! -f ~/.art/data/mnist.npz ]; then
95+
echo "Downloading MNIST dataset..."
96+
curl -L -o ~/.art/data/mnist.npz https://s3.amazonaws.com/img-datasets/mnist.npz
97+
else
98+
echo "MNIST already cached."
99+
fi
100+
60101
- name: Run Tests
61102
run: ./run_tests.sh ${{ matrix.framework }}
103+
62104
- name: Upload coverage to Codecov
63105
uses: codecov/codecov-action@v5
64106
with:

‎.github/workflows/ci-keras.yml

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,10 +51,12 @@ jobs:
5151
steps:
5252
- name: Checkout Repo
5353
uses: actions/checkout@v4
54+
5455
- name: Setup Python
5556
uses: actions/setup-python@v5
5657
with:
5758
python-version: ${{ matrix.python }}
59+
5860
- name: Install Dependencies
5961
run: |
6062
sudo apt-get update
@@ -65,8 +67,48 @@ jobs:
6567
pip install keras==${{ matrix.keras }}
6668
pip install tensorflow-addons==${{ matrix.tf_addons }}
6769
pip list
70+
71+
- name: Cache CIFAR-10 dataset
72+
uses: actions/cache@v3
73+
with:
74+
path: ~/.art/data/cifar-10-batches-py
75+
key: cifar10-dataset-cache-v1
76+
77+
- name: Cache MNIST dataset
78+
uses: actions/cache@v3
79+
with:
80+
path: ~/.art/data/mnist.npz
81+
key: mnist-dataset-cache-v1
82+
83+
- name: Download and extract CIFAR-10 if not cached
84+
run: |
85+
mkdir -p ~/.art/data
86+
if [ ! -d ~/.art/data/cifar-10-batches-py ]; then
87+
echo "Downloading CIFAR-10 dataset..."
88+
if curl -L -f -o cifar-10-python.tar.gz https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz; then
89+
echo "Downloaded from www.cs.toronto.edu"
90+
else
91+
echo "Primary download failed, trying Dropbox mirror..."
92+
curl -L -o cifar-10-python.tar.gz "https://www.dropbox.com/scl/fi/6mhiq4rtfszqrosasp8a9/cifar-10-batches-py.tar.gz?rlkey=g6v114un4je233j52tu5tdsbe&st=gxo2e50q&dl=1"
93+
fi
94+
tar -xzf cifar-10-python.tar.gz -C ~/.art/data/
95+
else
96+
echo "CIFAR-10 already cached."
97+
fi
98+
99+
- name: Download MNIST if not cached
100+
run: |
101+
mkdir -p ~/.art/data
102+
if [ ! -f ~/.art/data/mnist.npz ]; then
103+
echo "Downloading MNIST dataset..."
104+
curl -L -o ~/.art/data/mnist.npz https://s3.amazonaws.com/img-datasets/mnist.npz
105+
else
106+
echo "MNIST already cached."
107+
fi
108+
68109
- name: Run Tests
69110
run: ./run_tests.sh ${{ matrix.framework }}
111+
70112
- name: Upload coverage to Codecov
71113
uses: codecov/codecov-action@v5
72114
with:

‎.github/workflows/ci-legacy.yml

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,10 +43,12 @@ jobs:
4343
steps:
4444
- name: Checkout Repo
4545
uses: actions/checkout@v4
46+
4647
- name: Setup Python
4748
uses: actions/setup-python@v5
4849
with:
4950
python-version: ${{ matrix.python }}
51+
5052
- name: Install Dependencies
5153
run: |
5254
sudo apt-get update
@@ -60,8 +62,48 @@ jobs:
6062
pip install torchvision==${{ matrix.torchvision }} --index-url https://download.pytorch.org/whl/cpu
6163
pip install torchaudio==${{ matrix.torchaudio }} --index-url https://download.pytorch.org/whl/cpu
6264
pip list
65+
66+
- name: Cache CIFAR-10 dataset
67+
uses: actions/cache@v3
68+
with:
69+
path: ~/.art/data/cifar-10-batches-py
70+
key: cifar10-dataset-cache-v1
71+
72+
- name: Cache MNIST dataset
73+
uses: actions/cache@v3
74+
with:
75+
path: ~/.art/data/mnist.npz
76+
key: mnist-dataset-cache-v1
77+
78+
- name: Download and extract CIFAR-10 if not cached
79+
run: |
80+
mkdir -p ~/.art/data
81+
if [ ! -d ~/.art/data/cifar-10-batches-py ]; then
82+
echo "Downloading CIFAR-10 dataset..."
83+
if curl -L -f -o cifar-10-python.tar.gz https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz; then
84+
echo "Downloaded from www.cs.toronto.edu"
85+
else
86+
echo "Primary download failed, trying Dropbox mirror..."
87+
curl -L -o cifar-10-python.tar.gz "https://www.dropbox.com/scl/fi/6mhiq4rtfszqrosasp8a9/cifar-10-batches-py.tar.gz?rlkey=g6v114un4je233j52tu5tdsbe&st=gxo2e50q&dl=1"
88+
fi
89+
tar -xzf cifar-10-python.tar.gz -C ~/.art/data/
90+
else
91+
echo "CIFAR-10 already cached."
92+
fi
93+
94+
- name: Download MNIST if not cached
95+
run: |
96+
mkdir -p ~/.art/data
97+
if [ ! -f ~/.art/data/mnist.npz ]; then
98+
echo "Downloading MNIST dataset..."
99+
curl -L -o ~/.art/data/mnist.npz https://s3.amazonaws.com/img-datasets/mnist.npz
100+
else
101+
echo "MNIST already cached."
102+
fi
103+
63104
- name: Run ${{ matrix.name }} ${{ matrix.module }} Tests
64105
run: ./run_tests.sh ${{ matrix.framework }} ${{ matrix.module }}
106+
65107
- name: Upload coverage to Codecov
66108
uses: codecov/codecov-action@v5
67109
with:

0 commit comments

Comments
 (0)