Skip to content

Commit 4b73ea4

Browse files
committed
CI: Compute Heatmaps
- add `heatmap-benchmark.sh`, which creates multiple heatmaps using the `feed-forward.py` example script, which I personally used to benchmark heatmaps - add the github action `benchmark.yml`, which computes VGG16 heatmaps and outputs them as an image montage - provide clearer names for the actions
1 parent b570fa2 commit 4b73ea4

File tree

4 files changed

+327
-2
lines changed

4 files changed

+327
-2
lines changed

.github/workflows/benchmark.yml

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
# This workflow will generate heatmaps to benchmark whether the current implementation is correct.
2+
name: Heatmap Benchmark
3+
4+
# This workflow runs when commits are pushed to the main/develop branch or when a pull request for the main branch is opened or pushed to
5+
on:
6+
push:
7+
branches:
8+
- main
9+
pull_request:
10+
branches:
11+
- main
12+
13+
# This workflow computes VGG16 heatmaps and outputs them as an image montage.
14+
jobs:
15+
benchmark:
16+
name: Create Heatmap Overview
17+
runs-on: ubuntu-latest
18+
steps:
19+
- name: Checkout Repository
20+
uses: actions/checkout@v4
21+
with:
22+
fetch-depth: 0
23+
- name: Install ImageMagick
24+
run: sudo apt-get update -y && sudo apt-get install -y libpng-dev libjpeg-dev libtiff-dev imagemagick
25+
- name: Install uv
26+
uses: astral-sh/setup-uv@v5
27+
with:
28+
version: 0.6.14
29+
- name: Install Python
30+
run: uv python install 3.13.3
31+
- name: Install Zennit and its Dependencies
32+
run: uv sync --dev
33+
- name: Prepare directories
34+
run: mkdir data params results
35+
- name: Download lighthouse data
36+
run: bash share/scripts/download-lighthouses.sh --output data/lighthouses
37+
- name: Download VGG16 model parameters
38+
run: curl -o params/vgg16-397923af.pth 'https://download.pytorch.org/models/vgg16-397923af.pth'
39+
- name: Run heatmap benchmark
40+
run: |
41+
bash share/scripts/heatmap-benchmark.sh \
42+
--python .venv/bin/python \
43+
--script share/example/feed_forward.py \
44+
--output results
45+
- name: Upload heatmap overview artifact
46+
id: artifact-upload-step
47+
uses: actions/upload-artifact@v4
48+
with:
49+
path: results/full_montage_vgg16.webp
50+
- name: Heatmap Overview
51+
run: echo "Heatmaps available <a href=\"${{steps.artifact-upload-step.outputs.artifact-url}}\">here</a> " >> "$GITHUB_STEP_SUMMARY"

.github/workflows/deploy.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11

22
# This workflow will build the Zennit project and publish the artifacts to PyPI when a GitHub release is created
3-
name: Zennit Continuous Deployment
3+
name: Deployment
44

55
# This workflow will run when a new release is created on GitHub, the process works like this:
66
# 1) For each milestone multiple issues are created

.github/workflows/tests.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
# This workflow will run the unit tests, the linters, the static type checker, the spell checker, and will build the documentation
2-
name: Zennit Continuous Integration
2+
name: Tests, Linter, Docs
33

44
# This workflow runs when commits are pushed to the main/develop branch or when a pull request for the main/develop branch is opened or pushed to
55
on:

share/scripts/heatmap-benchmark.sh

Lines changed: 274 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,274 @@
1+
#!/usr/bin/env bash
2+
3+
4+
COMPOSITES=(
5+
epsilon_gamma_box
6+
epsilon_plus
7+
epsilon_plus_flat
8+
epsilon_alpha2_beta1
9+
epsilon_alpha2_beta1_flat
10+
)
11+
12+
ABSCOMPOSITES=(
13+
guided_backprop
14+
excitation_backprop
15+
deconvnet
16+
)
17+
18+
ATTRIBUTORS=(
19+
smoothgrad
20+
integrads
21+
gradient
22+
)
23+
24+
EXTRA=(
25+
occlusion
26+
)
27+
28+
ALLNAMES=(
29+
input
30+
"${COMPOSITES[@]}"
31+
"${ABSCOMPOSITES[@]}"
32+
"${ATTRIBUTORS[@]}"
33+
"${EXTRA[@]}"
34+
)
35+
36+
37+
UNSIGNED_CMAPS=(
38+
hot
39+
cold
40+
wred
41+
wblue
42+
gray
43+
)
44+
45+
SIGNED_CMAPS=(
46+
coldnhot
47+
bwr
48+
)
49+
50+
SUFFIX=''
51+
52+
die() {
53+
echo >&2 -e "${1-}"
54+
exit "${2-1}"
55+
}
56+
57+
58+
usage() {
59+
cat <<EOF
60+
Usage: $(basename "${BASH_SOURCE[0]}") [-h] [-v] [-o output]
61+
62+
Compute heatmaps for builtin attribution methods.
63+
64+
Available options:
65+
66+
-o, --output Path to save heatmaps (default = 'results')
67+
-p, --python Path to python binay (default = '.venv/bin/python')
68+
-s, --script Path to feed_forward.py script (default = 'feed_forward.py')
69+
-d, --dataset Path to dataset (default = 'data/lighthouses')
70+
-c, --s-cmap Color-map for signed attributions (default = 'coldnhot')
71+
-u, --u-cmap Color-map for unsigned attributions (default = 'hot')
72+
-h, --help Print this help and exit
73+
-v, --verbose Print script debug info
74+
--pdb Run debugger
75+
EOF
76+
exit
77+
}
78+
79+
80+
parse_params() {
81+
output=""
82+
python=".venv/bin/python"
83+
script="feed_forward.py"
84+
dataset="data/lighthouses"
85+
python_args=()
86+
unsigned_cmap='hot'
87+
signed_cmap='coldnhot'
88+
attributions=()
89+
90+
MODEL='vgg16'
91+
PARAMS='params/vgg16-397923af.pth'
92+
93+
args=()
94+
95+
while (($#)); do
96+
case "${1-}" in
97+
-h | --help) usage ;;
98+
-v | --verbose) set -x ;;
99+
-o | --output)
100+
output="${2-}"
101+
shift
102+
;;
103+
-p | --python)
104+
python="${2-}"
105+
shift
106+
;;
107+
-s | --script)
108+
script="${2-}"
109+
shift
110+
;;
111+
-d | --dataset)
112+
dataset="${2-}"
113+
shift
114+
;;
115+
-m | --model)
116+
case "${2-}" in
117+
vgg16 | vgg16_bn | resnet50 | resnet18)
118+
MODEL="${2-}"
119+
PARAMS="$(find params -name "${MODEL}-*.pth" | head -n 1)"
120+
[[ -z "$PARAMS" ]] && die "No parameters found for model '${MODEL}'"
121+
;;
122+
*) die "Unknown model: ${2-}" ;;
123+
esac
124+
shift
125+
;;
126+
-a | --attribution)
127+
mapfile -td, -O "${#attributions[@]}" attributions < <(echo -n "${2}")
128+
shift
129+
;;
130+
-c | --signed-cmap)
131+
signed_cmap="${2-}"
132+
shift
133+
;;
134+
-u | --unsigned-cmap)
135+
unsigned_cmap="${2-}"
136+
shift
137+
;;
138+
--pdb) python_args+=('-m' 'pdb') ;;
139+
--) args+=("${@:2}"); break ;;
140+
-?*) die "Unknown option: ${1}" ;;
141+
*) args+=("${1-}") ;;
142+
# *) break ;;
143+
esac
144+
shift
145+
done
146+
147+
# args=("${@}")
148+
149+
[[ -z "${output}" ]] && output="$(uv pip show zennit | awk 'NR==2{print $2}')"
150+
# (( ${#args[@]} )) && die "Too many positional arguments"
151+
(( ${#args[@]} )) || args=('attribution' 'montage')
152+
(( ${#attributions[@]} )) || attributions=("${ALLNAMES[@]}")
153+
154+
return 0
155+
}
156+
157+
158+
attribution(){
159+
"$python" "${python_args[@]}" "$script" \
160+
"${dataset}" \
161+
"${output}/${MODEL}_${1}_{sample:02d}.png" \
162+
--model "$MODEL" \
163+
--parameters "$PARAMS" \
164+
"${@:2}"
165+
}
166+
167+
print_filenames(){
168+
for method in "${@}"; do
169+
printf "${output}/${MODEL}_${method}_%02d.png\n" 0 1 2 3 4 5 6 7
170+
done
171+
}
172+
173+
print_caption_filenames(){
174+
for method in "${@}"; do
175+
echo -ne "caption:${method//_/\ }\n"
176+
printf "${output}/${MODEL}_${method}_%02d.png\n" 0 1 2 3 4 5 6 7
177+
done
178+
}
179+
180+
all_attributions(){
181+
for composite in "${COMPOSITES[@]}"; do
182+
attribution \
183+
"${composite}" \
184+
--composite "${composite}" \
185+
--relevance-norm symmetric \
186+
--cmap "${signed_cmap}"
187+
done
188+
189+
for composite in "${ABSCOMPOSITES[@]}"; do
190+
attribution \
191+
"${composite}" \
192+
--composite "${composite}" \
193+
--relevance-norm absolute \
194+
--cmap "${unsigned_cmap}"
195+
done
196+
197+
for attributor in "${ATTRIBUTORS[@]}"; do
198+
attribution \
199+
"${attributor}" \
200+
--attributor "${attributor}" \
201+
--relevance-norm absolute \
202+
--cmap "${unsigned_cmap}"
203+
done
204+
205+
attribution \
206+
"occlusion" \
207+
--inputs "${output}/${MODEL}_input_{sample:02d}.png" \
208+
--attributor "occlusion" \
209+
--relevance-norm unaligned \
210+
--cmap "${unsigned_cmap}"
211+
}
212+
213+
full_montage(){
214+
mapfile -t filenames < <(print_caption_filenames "${@}")
215+
montage \
216+
-size 96x96 \
217+
-gravity center \
218+
-pointsize 16 \
219+
-background '#77f' \
220+
"${filenames[@]}" \
221+
-geometry 96x96+1+1 \
222+
-tile "9x${#}" \
223+
-define webp:lossless=true \
224+
"${output}/full_montage_${MODEL}${SUFFIX}.webp"
225+
}
226+
227+
color_montage(){
228+
s_names=(
229+
"${ABSCOMPOSITES[@]}"
230+
"${ATTRIBUTORS[@]}"
231+
"${EXTRA[@]}"
232+
)
233+
mapfile -t filenames_ < <(print_filenames "${s_names[@]}")
234+
for cmap in "${SIGNED_CMAPS[@]}"; do
235+
"$python" palette_swap.py --cmap "${cmap}" "${filenames_[@]}"
236+
SUFFIX="_${cmap}"
237+
full_montage input "${s_names[@]}"
238+
done
239+
"$python" palette_swap.py --cmap "${signed_cmap}" "${filenames_[@]}"
240+
241+
u_names=(
242+
"${COMPOSITES[@]}"
243+
)
244+
mapfile -t filenames_ < <(print_filenames "${u_names[@]}")
245+
for cmap in "${UNSIGNED_CMAPS[@]}"; do
246+
"$python" palette_swap.py --cmap "${cmap}" "${filenames_[@]}"
247+
SUFFIX="_${cmap}"
248+
full_montage input "${u_names[@]}"
249+
done
250+
"$python" palette_swap.py --cmap "${unsigned_cmap}" "${filenames_[@]}"
251+
252+
SUFFIX=''
253+
}
254+
255+
parse_params "$@"
256+
257+
mkdir -p "${output}"
258+
259+
for action in "${args[@]}"; do
260+
case "${action}" in
261+
attribution)
262+
all_attributions
263+
;;
264+
montage)
265+
full_montage "${attributions[@]}"
266+
;;
267+
color-montage)
268+
color_montage
269+
;;
270+
*)
271+
die "No such action: '${action}'"
272+
;;
273+
esac
274+
done

0 commit comments

Comments
 (0)