Skip to content

Commit 5b412a8

Browse files
liavweissLiav Weiss
liavweiss
and
Liav Weiss
authored
Move model storage to /mnt directory to prevent disk space issues (#792)
Signed-off-by: Liav Weiss <[email protected]> Co-authored-by: Liav Weiss <[email protected]>
1 parent c911705 commit 5b412a8

File tree

5 files changed

+130
-23
lines changed

5 files changed

+130
-23
lines changed

‎.github/workflows/integration-test-docker.yml‎

Lines changed: 36 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -30,16 +30,43 @@ jobs:
3030
- name: Check out the repo
3131
uses: actions/checkout@v4
3232

33-
- name: Free up disk space
33+
- name: Setup model storage on /mnt
3434
run: |
35-
echo "Disk space before cleanup:"
36-
df -h
37-
sudo rm -rf /usr/share/dotnet
38-
sudo rm -rf /opt/ghc
39-
sudo rm -rf /usr/local/share/boost
40-
sudo rm -rf "$AGENT_TOOLSDIRECTORY"
41-
echo "Disk space after cleanup:"
42-
df -h
35+
# Use /mnt for model storage (has ~75GB vs ~14GB on root)
36+
# This helps prevent "no space left on device" errors
37+
echo "Disk space before setup:"
38+
df -h / && df -h /mnt
39+
40+
# Create /mnt/models directory if it doesn't exist
41+
sudo mkdir -p /mnt/models
42+
sudo chown -R $USER:$USER /mnt/models
43+
44+
# If models directory already exists in workspace, move it to /mnt
45+
if [ -d "models" ] && [ ! -L "models" ]; then
46+
echo "Moving existing models directory to /mnt/models..."
47+
# Move contents if /mnt/models is not empty, otherwise just move the directory
48+
if [ "$(ls -A /mnt/models 2>/dev/null)" ]; then
49+
echo "Warning: /mnt/models already has content, merging..."
50+
sudo cp -r models/* /mnt/models/ || true
51+
rm -rf models
52+
else
53+
sudo mv models /mnt/models
54+
fi
55+
fi
56+
57+
# Create symlink from models/ to /mnt/models/ so existing code continues to work
58+
if [ ! -e "models" ]; then
59+
ln -s /mnt/models models
60+
echo "Created symlink: models -> /mnt/models"
61+
elif [ -L "models" ]; then
62+
echo "Symlink already exists: models -> $(readlink models)"
63+
else
64+
echo "Warning: models exists but is not a symlink"
65+
fi
66+
67+
echo "Disk space after setup:"
68+
df -h / && df -h /mnt
69+
echo "Models directory setup complete. Models will be stored in /mnt/models"
4370
4471
- name: Set up Python
4572
uses: actions/setup-python@v5

‎.github/workflows/integration-test-k8s.yml‎

Lines changed: 1 addition & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -77,16 +77,6 @@ jobs:
7777
run: |
7878
make build-e2e
7979
80-
- name: Free up disk space
81-
run: |
82-
# Remove unnecessary toolchains to free ~25GB disk space
83-
# This helps prevent "no space left on device" errors
84-
echo "Disk before cleanup:"
85-
df -h /
86-
# Note: Do NOT remove $AGENT_TOOLSDIRECTORY - it contains Go/Rust from setup actions
87-
sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/share/boost /usr/local/lib/android /opt/hostedtoolcache/CodeQL || true
88-
echo "Disk after cleanup:"
89-
df -h /
9080
9181
- name: Run Integration E2E tests (${{ matrix.profile }})
9282
id: e2e-test
@@ -97,6 +87,7 @@ jobs:
9787
echo "test_exit_code=${TEST_EXIT_CODE}" >> $GITHUB_OUTPUT
9888
exit ${TEST_EXIT_CODE}
9989
90+
10091
- name: Upload test reports
10192
if: always()
10293
uses: actions/upload-artifact@v4

‎.github/workflows/test-and-build.yml‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,44 @@ jobs:
8080
restore-keys: |
8181
${{ runner.os }}-go-
8282
83+
- name: Setup model storage on /mnt
84+
run: |
85+
# Use /mnt for model storage (has ~75GB vs ~14GB on root)
86+
# This helps prevent "no space left on device" errors
87+
echo "Disk space before setup:"
88+
df -h / && df -h /mnt
89+
90+
# Create /mnt/models directory if it doesn't exist
91+
sudo mkdir -p /mnt/models
92+
sudo chown -R $USER:$USER /mnt/models
93+
94+
# If models directory already exists in workspace, move it to /mnt
95+
if [ -d "models" ] && [ ! -L "models" ]; then
96+
echo "Moving existing models directory to /mnt/models..."
97+
# Move contents if /mnt/models is not empty, otherwise just move the directory
98+
if [ "$(ls -A /mnt/models 2>/dev/null)" ]; then
99+
echo "Warning: /mnt/models already has content, merging..."
100+
sudo cp -r models/* /mnt/models/ || true
101+
rm -rf models
102+
else
103+
sudo mv models /mnt/models
104+
fi
105+
fi
106+
107+
# Create symlink from models/ to /mnt/models/ so existing code continues to work
108+
if [ ! -e "models" ]; then
109+
ln -s /mnt/models models
110+
echo "Created symlink: models -> /mnt/models"
111+
elif [ -L "models" ]; then
112+
echo "Symlink already exists: models -> $(readlink models)"
113+
else
114+
echo "Warning: models exists but is not a symlink"
115+
fi
116+
117+
echo "Disk space after setup:"
118+
df -h / && df -h /mnt
119+
echo "Models directory setup complete. Models will be stored in /mnt/models"
120+
83121
- name: Cache Models
84122
uses: actions/cache@v4
85123
with:

‎e2e/pkg/cluster/kind.go‎

Lines changed: 47 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -38,8 +38,37 @@ func (k *KindCluster) Create(ctx context.Context) error {
3838
return nil
3939
}
4040

41-
// Create cluster
42-
cmd := exec.CommandContext(ctx, "kind", "create", "cluster", "--name", k.Name)
41+
// Mount /mnt from host into Kind node so storage provisioner can use it (more disk space)
42+
configContent := fmt.Sprintf(`kind: Cluster
43+
apiVersion: kind.x-k8s.io/v1alpha4
44+
name: %s
45+
nodes:
46+
- role: control-plane
47+
extraMounts:
48+
- hostPath: /mnt
49+
containerPath: /mnt
50+
- role: worker
51+
extraMounts:
52+
- hostPath: /mnt
53+
containerPath: /mnt
54+
`, k.Name)
55+
56+
configFile, err := os.CreateTemp("", "kind-config-*.yaml")
57+
if err != nil {
58+
return fmt.Errorf("failed to create temp config file: %w", err)
59+
}
60+
defer os.Remove(configFile.Name())
61+
62+
if _, err := configFile.WriteString(configContent); err != nil {
63+
configFile.Close()
64+
return fmt.Errorf("failed to write config file: %w", err)
65+
}
66+
configFile.Close()
67+
68+
k.log("Using Kind config with /mnt mount for storage")
69+
70+
// Create cluster with config file
71+
cmd := exec.CommandContext(ctx, "kind", "create", "cluster", "--name", k.Name, "--config", configFile.Name())
4372
if k.Verbose {
4473
cmd.Stdout = os.Stdout
4574
cmd.Stderr = os.Stderr
@@ -55,6 +84,22 @@ func (k *KindCluster) Create(ctx context.Context) error {
5584
return fmt.Errorf("cluster failed to become ready: %w", err)
5685
}
5786

87+
// Configure storage provisioner to use /mnt (75GB) instead of /tmp (limited space)
88+
kubeConfig, err := k.GetKubeConfig(ctx)
89+
if err != nil {
90+
return fmt.Errorf("failed to get kubeconfig: %w", err)
91+
}
92+
defer os.Remove(kubeConfig)
93+
94+
// Simple one-liner: update ConfigMap and restart provisioner
95+
// Models downloaded in pods will be stored in /mnt via PVCs
96+
exec.CommandContext(ctx, "kubectl", "--kubeconfig", kubeConfig,
97+
"patch", "configmap", "local-path-config", "-n", "local-path-storage",
98+
"--type", "merge",
99+
"-p", `{"data":{"config.json":"{\"nodePathMap\":[{\"node\":\"DEFAULT_PATH_FOR_NON_LISTED_NODES\",\"paths\":[\"/mnt/local-path-provisioner\"]}]}"}}`).Run()
100+
exec.CommandContext(ctx, "kubectl", "--kubeconfig", kubeConfig,
101+
"rollout", "restart", "deployment/local-path-provisioner", "-n", "local-path-storage").Run()
102+
58103
k.log("Cluster %s created successfully", k.Name)
59104
return nil
60105
}

‎src/semantic-router/pkg/classification/model_discovery.go‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -62,8 +62,14 @@ func AutoDiscoverModels(modelsDir string) (*ModelPaths, error) {
6262
modelsDir = "./models"
6363
}
6464

65+
// Resolve symlinks to handle cases where models directory is a symlink (e.g., CI uses /mnt/models)
66+
resolved, err := filepath.EvalSymlinks(modelsDir)
67+
if err == nil && resolved != "" {
68+
modelsDir = resolved
69+
}
70+
6571
// Check if models directory exists
66-
if _, err := os.Stat(modelsDir); os.IsNotExist(err) {
72+
if _, statErr := os.Stat(modelsDir); os.IsNotExist(statErr) {
6773
return nil, fmt.Errorf("models directory does not exist: %s", modelsDir)
6874
}
6975

@@ -78,7 +84,7 @@ func AutoDiscoverModels(modelsDir string) (*ModelPaths, error) {
7884
legacyPaths := &ModelPaths{}
7985

8086
// Walk through the models directory to collect all models
81-
err := filepath.Walk(modelsDir, func(path string, info os.FileInfo, err error) error {
87+
err = filepath.Walk(modelsDir, func(path string, info os.FileInfo, err error) error {
8288
if err != nil {
8389
return err
8490
}

0 commit comments

Comments
 (0)