Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 19 additions & 0 deletions .github/workflows/test_models.yml
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,25 @@ jobs:
pip install git+https://github.com/pytorch-labs/tokenizers
fi
pip list
- name: Check pre-clean disk usage
run: df -h
- name: Turn off swap
run: |
sudo swapoff -a
sudo rm -f /swapfile
df -h
- name: Clean packages
run: |
sudo apt clean
df -h
- name: Docker clean
run: |
docker system prune -af
df -h
- name: Clean /tmp
run: |
sudo rm -rf /tmp/*
df -h
- name: Run tests
run: |
RUN_SLOW=1 pytest tests/models/test_modeling_${{ matrix.test-modeling }}.py -s -vvvv --durations=0 --log-cli-level=INFO
55 changes: 55 additions & 0 deletions optimum/executorch/modeling.py
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,61 @@ def __del__(self):
"""Clean up temporary files when the model instance is destroyed."""
self._cleanup_temp_resources()

print("df -dh:")
import psutil

for part in psutil.disk_partitions():
usage = psutil.disk_usage(part.mountpoint)
print(f"{part.device} mounted on {part.mountpoint}")
print(f" Total: {usage.total / 2**30:.2f} GB")
print(f" Used: {usage.used / 2**30:.2f} GB")
print(f" Free: {usage.free / 2**30:.2f} GB")
print(f" Percent Used: {usage.percent}%\n")
print()


print("Walking through all files:")
import os
from pathlib import Path

def get_dir_size(path):
total = 0
for dirpath, dirnames, filenames in os.walk(path, onerror=lambda e: None):
for f in filenames:
try:
fp = os.path.join(dirpath, f)
total += os.path.getsize(fp)
except (FileNotFoundError, PermissionError):
continue
return total

# Compute top 40 largest directories under /
dirs = []
root = Path("/")

for p in root.iterdir():
try:
size = get_dir_size(p)
dirs.append((size, str(p)))
except PermissionError:
continue

# Sort and print largest 40
dirs.sort(reverse=True)
for size, path in dirs[:40]:
print(f"{size / 1e9:.2f} GB\t{path}")
print()

print("Memory used:")
import psutil

mem = psutil.virtual_memory()

print(f"Total: {mem.total / (1024 ** 3):.2f} GB")
print(f"Available: {mem.available / (1024 ** 3):.2f} GB")
print(f"Used: {mem.used / (1024 ** 3):.2f} GB")
print(f"Percent: {mem.percent}%")

def _cleanup_temp_resources(self):
"""Clean up temporary directory and files."""
if hasattr(self, "_temp_dir") and self._temp_dir is not None:
Expand Down
Loading