Refactor jupyter notebook cleaning script

Author: MisterArdavan

notebooks/.gitattributes

@@ -1,4 +1 @@
-*.ipynb filter=strip-notebook-output
-# keep the output of the following notebooks when committing
-**/SlurmGPU.ipynb !filter=strip-notebook-output
-**/Requested and Used VRAM.ipynb !filter=strip-notebook-output
+*.ipynb filter=strip-notebook-output

scripts/clean_notebook.sh

@@ -16,33 +16,47 @@ tmpnotebook=$(mktemp)
 # Ensure temp files are cleaned up on exit or error
 trap 'rm -f "$tmpjson" "$tmpnotebook"' EXIT
 
-# Check if the notebook is in the exclude list
+# Always strip metadata. Only strip outputs if notebook NOT in exclude list.
+
+# Check if input file is empty
+if [ ! -s "$fname" ]; then
+    echo "Error: Input file $fname is empty" >&2
+    exit 1
+fi
+
+# Convert notebook to JSON and assign sequential cell IDs with jq (array-safe)
+if ! cat "$fname" | \
+    jq 'if .cells then .cells |= [ range(0; length) as $i | .[$i] as $cell | ($cell | (if has("execution_count") then .execution_count = null else . end) + {id: ($i|tostring)}) ] else . end' > "$tmpjson"; then
+    echo "Error: jq failed for $fname" >&2
+    exit 2
+fi
+
+# Validate JSON
+if [ ! -s "$tmpjson" ] || ! jq empty "$tmpjson" 2>/dev/null; then
+    echo "Error: $tmpjson is not valid JSON after jq for $fname" >&2
+    exit 3
+fi
+
+BASE_NBCONVERT_ARGS=(--to=notebook --stdin --stdout --log-level=ERROR)
+
 if grep -Fxq "$(basename "$fname")" "$EXCLUDE_FILE" 2>/dev/null; then
-    cat "$fname"
-else
-    # Check if input file is empty
-    if [ ! -s "$fname" ]; then
-        echo "Error: Input file $fname is empty" >&2
-        exit 1
-    fi
-    # Convert notebook to JSON and assign sequential cell IDs with jq (array-safe)
-    if ! cat "$fname" | \
-        jq 'if .cells then .cells |= [ range(0; length) as $i | .[$i] as $cell | ($cell + {id: ($i|tostring)}) ] else . end' > "$tmpjson"; then
-        echo "Error: jq failed for $fname" >&2
-        exit 2
-    fi
-    # Check if tmpjson is valid and not empty
-    if [ ! -s "$tmpjson" ] || ! jq empty "$tmpjson" 2>/dev/null; then
-        echo "Error: $tmpjson is not valid JSON after jq for $fname" >&2
-        exit 3
+    # Excluded: keep outputs, strip metadata only
+    if ! jupyter nbconvert "${BASE_NBCONVERT_ARGS[@]}" < "$tmpjson" | \
+        jupyter nbconvert --ClearMetadataPreprocessor.enabled=True \
+            --TagRemovePreprocessor.enabled=True --ClearMetadataPreprocessor.preserve_cell_metadata_mask='[("tags")]' \
+            "${BASE_NBCONVERT_ARGS[@]}" > "$tmpnotebook"; then
+        echo "Error: nbconvert (metadata-only) failed for $fname" >&2
+        exit 4
     fi
-    # Convert JSON back to notebook and strip outputs/metadata
-    if ! jupyter nbconvert --to=notebook --stdin --stdout --log-level=ERROR < "$tmpjson" | \
+else
+    # Not excluded: strip outputs and metadata
+    if ! jupyter nbconvert "${BASE_NBCONVERT_ARGS[@]}" < "$tmpjson" | \
         jupyter nbconvert --ClearOutputPreprocessor.enabled=True --ClearMetadataPreprocessor.enabled=True \
             --TagRemovePreprocessor.enabled=True --ClearMetadataPreprocessor.preserve_cell_metadata_mask='[("tags")]' \
-            --to=notebook --stdin --stdout --log-level=ERROR > "$tmpnotebook"; then
-        echo "Error: nbconvert failed for $fname" >&2
+            "${BASE_NBCONVERT_ARGS[@]}" > "$tmpnotebook"; then
+        echo "Error: nbconvert (outputs+metadata) failed for $fname" >&2
         exit 4
     fi
-    cat "$tmpnotebook"
 fi
+
+cat "$tmpnotebook"