Skip to content

Commit e620b2d

Browse files
beat-buesserbeat-buesser
committed
Cache test datasets
Signed-off-by: Beat Buesser <[email protected]>
1 parent 3508b86 commit e620b2d

12 files changed

+497
-0
lines changed

‎.github/workflows/ci-deepspeech-v3.yml

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,8 +29,48 @@ jobs:
2929
steps:
3030
- name: Checkout Repo
3131
uses: actions/checkout@v4
32+
33+
- name: Cache CIFAR-10 dataset
34+
uses: actions/cache@v3
35+
with:
36+
path: ~/.art/data/cifar-10-batches-py
37+
key: cifar10-dataset-cache-v1
38+
39+
- name: Cache MNIST dataset
40+
uses: actions/cache@v3
41+
with:
42+
path: ~/.art/data/mnist.npz
43+
key: mnist-dataset-cache-v1
44+
45+
- name: Download and extract CIFAR-10 if not cached
46+
run: |
47+
mkdir -p ~/.art/data
48+
if [ ! -d ~/.art/data/cifar-10-batches-py ]; then
49+
echo "Downloading CIFAR-10 dataset..."
50+
if curl -L -f -o cifar-10-python.tar.gz https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz; then
51+
echo "Downloaded from www.cs.toronto.edu"
52+
else
53+
echo "Primary download failed, trying Dropbox mirror..."
54+
curl -L -o cifar-10-python.tar.gz "https://www.dropbox.com/scl/fi/6mhiq4rtfszqrosasp8a9/cifar-10-batches-py.tar.gz?rlkey=g6v114un4je233j52tu5tdsbe&st=gxo2e50q&dl=1"
55+
fi
56+
tar -xzf cifar-10-python.tar.gz -C ~/.art/data/
57+
else
58+
echo "CIFAR-10 already cached."
59+
fi
60+
61+
- name: Download MNIST if not cached
62+
run: |
63+
mkdir -p ~/.art/data
64+
if [ ! -f ~/.art/data/mnist.npz ]; then
65+
echo "Downloading MNIST dataset..."
66+
curl -L -o ~/.art/data/mnist.npz https://s3.amazonaws.com/img-datasets/mnist.npz
67+
else
68+
echo "MNIST already cached."
69+
fi
70+
3271
- name: Run Test Action
3372
uses: ./.github/actions/deepspeech-v3
73+
3474
- name: Upload coverage to Codecov
3575
uses: codecov/codecov-action@v5
3676
with:

‎.github/workflows/ci-espresso.yml

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,8 +29,48 @@ jobs:
2929
steps:
3030
- name: Checkout Repo
3131
uses: actions/checkout@v4
32+
33+
- name: Cache CIFAR-10 dataset
34+
uses: actions/cache@v3
35+
with:
36+
path: ~/.art/data/cifar-10-batches-py
37+
key: cifar10-dataset-cache-v1
38+
39+
- name: Cache MNIST dataset
40+
uses: actions/cache@v3
41+
with:
42+
path: ~/.art/data/mnist.npz
43+
key: mnist-dataset-cache-v1
44+
45+
- name: Download and extract CIFAR-10 if not cached
46+
run: |
47+
mkdir -p ~/.art/data
48+
if [ ! -d ~/.art/data/cifar-10-batches-py ]; then
49+
echo "Downloading CIFAR-10 dataset..."
50+
if curl -L -f -o cifar-10-python.tar.gz https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz; then
51+
echo "Downloaded from www.cs.toronto.edu"
52+
else
53+
echo "Primary download failed, trying Dropbox mirror..."
54+
curl -L -o cifar-10-python.tar.gz "https://www.dropbox.com/scl/fi/6mhiq4rtfszqrosasp8a9/cifar-10-batches-py.tar.gz?rlkey=g6v114un4je233j52tu5tdsbe&st=gxo2e50q&dl=1"
55+
fi
56+
tar -xzf cifar-10-python.tar.gz -C ~/.art/data/
57+
else
58+
echo "CIFAR-10 already cached."
59+
fi
60+
61+
- name: Download MNIST if not cached
62+
run: |
63+
mkdir -p ~/.art/data
64+
if [ ! -f ~/.art/data/mnist.npz ]; then
65+
echo "Downloading MNIST dataset..."
66+
curl -L -o ~/.art/data/mnist.npz https://s3.amazonaws.com/img-datasets/mnist.npz
67+
else
68+
echo "MNIST already cached."
69+
fi
70+
3271
- name: Run Test Action
3372
uses: ./.github/actions/espresso
73+
3474
- name: Upload coverage to Codecov
3575
uses: codecov/codecov-action@v5
3676
with:

‎.github/workflows/ci-goturn.yml

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,8 +29,48 @@ jobs:
2929
steps:
3030
- name: Checkout Repo
3131
uses: actions/checkout@v4
32+
33+
- name: Cache CIFAR-10 dataset
34+
uses: actions/cache@v3
35+
with:
36+
path: ~/.art/data/cifar-10-batches-py
37+
key: cifar10-dataset-cache-v1
38+
39+
- name: Cache MNIST dataset
40+
uses: actions/cache@v3
41+
with:
42+
path: ~/.art/data/mnist.npz
43+
key: mnist-dataset-cache-v1
44+
45+
- name: Download and extract CIFAR-10 if not cached
46+
run: |
47+
mkdir -p ~/.art/data
48+
if [ ! -d ~/.art/data/cifar-10-batches-py ]; then
49+
echo "Downloading CIFAR-10 dataset..."
50+
if curl -L -f -o cifar-10-python.tar.gz https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz; then
51+
echo "Downloaded from www.cs.toronto.edu"
52+
else
53+
echo "Primary download failed, trying Dropbox mirror..."
54+
curl -L -o cifar-10-python.tar.gz "https://www.dropbox.com/scl/fi/6mhiq4rtfszqrosasp8a9/cifar-10-batches-py.tar.gz?rlkey=g6v114un4je233j52tu5tdsbe&st=gxo2e50q&dl=1"
55+
fi
56+
tar -xzf cifar-10-python.tar.gz -C ~/.art/data/
57+
else
58+
echo "CIFAR-10 already cached."
59+
fi
60+
61+
- name: Download MNIST if not cached
62+
run: |
63+
mkdir -p ~/.art/data
64+
if [ ! -f ~/.art/data/mnist.npz ]; then
65+
echo "Downloading MNIST dataset..."
66+
curl -L -o ~/.art/data/mnist.npz https://s3.amazonaws.com/img-datasets/mnist.npz
67+
else
68+
echo "MNIST already cached."
69+
fi
70+
3271
- name: Run Test Action
3372
uses: ./.github/actions/goturn
73+
3474
- name: Upload coverage to Codecov
3575
uses: codecov/codecov-action@v5
3676
env:

‎.github/workflows/ci-huggingface.yml

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,10 +40,12 @@ jobs:
4040
steps:
4141
- name: Checkout Repo
4242
uses: actions/checkout@v4
43+
4344
- name: Setup Python
4445
uses: actions/setup-python@v5
4546
with:
4647
python-version: ${{ matrix.python }}
48+
4749
- name: Install Dependencies
4850
run: |
4951
sudo apt-get update
@@ -57,8 +59,48 @@ jobs:
5759
pip install torchaudio==${{ matrix.torchaudio }} --index-url https://download.pytorch.org/whl/cpu
5860
pip install transformers==${{ matrix.transformers }}
5961
pip list
62+
63+
- name: Cache CIFAR-10 dataset
64+
uses: actions/cache@v3
65+
with:
66+
path: ~/.art/data/cifar-10-batches-py
67+
key: cifar10-dataset-cache-v1
68+
69+
- name: Cache MNIST dataset
70+
uses: actions/cache@v3
71+
with:
72+
path: ~/.art/data/mnist.npz
73+
key: mnist-dataset-cache-v1
74+
75+
- name: Download and extract CIFAR-10 if not cached
76+
run: |
77+
mkdir -p ~/.art/data
78+
if [ ! -d ~/.art/data/cifar-10-batches-py ]; then
79+
echo "Downloading CIFAR-10 dataset..."
80+
if curl -L -f -o cifar-10-python.tar.gz https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz; then
81+
echo "Downloaded from www.cs.toronto.edu"
82+
else
83+
echo "Primary download failed, trying Dropbox mirror..."
84+
curl -L -o cifar-10-python.tar.gz "https://www.dropbox.com/scl/fi/6mhiq4rtfszqrosasp8a9/cifar-10-batches-py.tar.gz?rlkey=g6v114un4je233j52tu5tdsbe&st=gxo2e50q&dl=1"
85+
fi
86+
tar -xzf cifar-10-python.tar.gz -C ~/.art/data/
87+
else
88+
echo "CIFAR-10 already cached."
89+
fi
90+
91+
- name: Download MNIST if not cached
92+
run: |
93+
mkdir -p ~/.art/data
94+
if [ ! -f ~/.art/data/mnist.npz ]; then
95+
echo "Downloading MNIST dataset..."
96+
curl -L -o ~/.art/data/mnist.npz https://s3.amazonaws.com/img-datasets/mnist.npz
97+
else
98+
echo "MNIST already cached."
99+
fi
100+
60101
- name: Run Tests
61102
run: ./run_tests.sh ${{ matrix.framework }}
103+
62104
- name: Upload coverage to Codecov
63105
uses: codecov/codecov-action@v5
64106
with:

‎.github/workflows/ci-keras.yml

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,10 +38,12 @@ jobs:
3838
steps:
3939
- name: Checkout Repo
4040
uses: actions/checkout@v4
41+
4142
- name: Setup Python
4243
uses: actions/setup-python@v5
4344
with:
4445
python-version: ${{ matrix.python }}
46+
4547
- name: Install Dependencies
4648
run: |
4749
sudo apt-get update
@@ -51,8 +53,48 @@ jobs:
5153
pip install tensorflow==${{ matrix.tensorflow }}
5254
pip install keras==${{ matrix.keras }}
5355
pip list
56+
57+
- name: Cache CIFAR-10 dataset
58+
uses: actions/cache@v3
59+
with:
60+
path: ~/.art/data/cifar-10-batches-py
61+
key: cifar10-dataset-cache-v1
62+
63+
- name: Cache MNIST dataset
64+
uses: actions/cache@v3
65+
with:
66+
path: ~/.art/data/mnist.npz
67+
key: mnist-dataset-cache-v1
68+
69+
- name: Download and extract CIFAR-10 if not cached
70+
run: |
71+
mkdir -p ~/.art/data
72+
if [ ! -d ~/.art/data/cifar-10-batches-py ]; then
73+
echo "Downloading CIFAR-10 dataset..."
74+
if curl -L -f -o cifar-10-python.tar.gz https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz; then
75+
echo "Downloaded from www.cs.toronto.edu"
76+
else
77+
echo "Primary download failed, trying Dropbox mirror..."
78+
curl -L -o cifar-10-python.tar.gz "https://www.dropbox.com/scl/fi/6mhiq4rtfszqrosasp8a9/cifar-10-batches-py.tar.gz?rlkey=g6v114un4je233j52tu5tdsbe&st=gxo2e50q&dl=1"
79+
fi
80+
tar -xzf cifar-10-python.tar.gz -C ~/.art/data/
81+
else
82+
echo "CIFAR-10 already cached."
83+
fi
84+
85+
- name: Download MNIST if not cached
86+
run: |
87+
mkdir -p ~/.art/data
88+
if [ ! -f ~/.art/data/mnist.npz ]; then
89+
echo "Downloading MNIST dataset..."
90+
curl -L -o ~/.art/data/mnist.npz https://s3.amazonaws.com/img-datasets/mnist.npz
91+
else
92+
echo "MNIST already cached."
93+
fi
94+
5495
- name: Run Tests
5596
run: ./run_tests.sh ${{ matrix.framework }}
97+
5698
- name: Upload coverage to Codecov
5799
uses: codecov/codecov-action@v5
58100
with:

‎.github/workflows/ci-legacy.yml

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,10 +43,12 @@ jobs:
4343
steps:
4444
- name: Checkout Repo
4545
uses: actions/checkout@v4
46+
4647
- name: Setup Python
4748
uses: actions/setup-python@v5
4849
with:
4950
python-version: ${{ matrix.python }}
51+
5052
- name: Install Dependencies
5153
run: |
5254
sudo apt-get update
@@ -60,8 +62,48 @@ jobs:
6062
pip install torchvision==${{ matrix.torchvision }} --index-url https://download.pytorch.org/whl/cpu
6163
pip install torchaudio==${{ matrix.torchaudio }} --index-url https://download.pytorch.org/whl/cpu
6264
pip list
65+
66+
- name: Cache CIFAR-10 dataset
67+
uses: actions/cache@v3
68+
with:
69+
path: ~/.art/data/cifar-10-batches-py
70+
key: cifar10-dataset-cache-v1
71+
72+
- name: Cache MNIST dataset
73+
uses: actions/cache@v3
74+
with:
75+
path: ~/.art/data/mnist.npz
76+
key: mnist-dataset-cache-v1
77+
78+
- name: Download and extract CIFAR-10 if not cached
79+
run: |
80+
mkdir -p ~/.art/data
81+
if [ ! -d ~/.art/data/cifar-10-batches-py ]; then
82+
echo "Downloading CIFAR-10 dataset..."
83+
if curl -L -f -o cifar-10-python.tar.gz https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz; then
84+
echo "Downloaded from www.cs.toronto.edu"
85+
else
86+
echo "Primary download failed, trying Dropbox mirror..."
87+
curl -L -o cifar-10-python.tar.gz "https://www.dropbox.com/scl/fi/6mhiq4rtfszqrosasp8a9/cifar-10-batches-py.tar.gz?rlkey=g6v114un4je233j52tu5tdsbe&st=gxo2e50q&dl=1"
88+
fi
89+
tar -xzf cifar-10-python.tar.gz -C ~/.art/data/
90+
else
91+
echo "CIFAR-10 already cached."
92+
fi
93+
94+
- name: Download MNIST if not cached
95+
run: |
96+
mkdir -p ~/.art/data
97+
if [ ! -f ~/.art/data/mnist.npz ]; then
98+
echo "Downloading MNIST dataset..."
99+
curl -L -o ~/.art/data/mnist.npz https://s3.amazonaws.com/img-datasets/mnist.npz
100+
else
101+
echo "MNIST already cached."
102+
fi
103+
63104
- name: Run ${{ matrix.name }} ${{ matrix.module }} Tests
64105
run: ./run_tests.sh ${{ matrix.framework }} ${{ matrix.module }}
106+
65107
- name: Upload coverage to Codecov
66108
uses: codecov/codecov-action@v5
67109
with:

0 commit comments

Comments
 (0)