k8s: avoid non-empty clone dir by cloning code to /app/src; mount PVC at /data; use absolute /data paths

Author: mo

k8s-prepare-job.yaml

@@ -15,7 +15,7 @@ spec:
           - |
             set -eu
             apt-get update && apt-get install -y git
-            mkdir -p /app/repo && cd /app/repo
+            mkdir -p /app/src && cd /app/src
             git clone https://github.com/david-thrower/cerebros-core-algorithm-alpha.git .
             git fetch origin 208-refactor-nlp-example-to-tokenize-first
             git checkout 208-refactor-nlp-example-to-tokenize-first
@@ -24,7 +24,7 @@ spec:
             PIP_BREAK_SYSTEM_PACKAGES=1 python -m pip install --no-cache-dir --break-system-packages -r cicd-requirements.txt
             PIP_BREAK_SYSTEM_PACKAGES=1 python -m pip install --no-cache-dir --break-system-packages mlflow transformers
             python tokenize_first_runner.py --mode prepare \
-              --out data/train_tokens.npz --max_len 128 \
+              --out /data/train_tokens.npz --max_len 128 \
               --tokenizer_checkpoint HuggingFaceTB/SmolLM3-3B
         env:
         - name: MLFLOW_TRACKING_URI
@@ -35,7 +35,7 @@ spec:
           value: "1"
         volumeMounts:
         - name: data-storage
-          mountPath: /app/repo/data
+          mountPath: /data
         resources:
           requests:
             memory: "4Gi"

k8s-train-job.yaml

@@ -15,7 +15,7 @@ spec:
           - |
             set -eu
             apt-get update && apt-get install -y git
-            mkdir -p /app/repo && cd /app/repo
+            mkdir -p /app/src && cd /app/src
             git clone https://github.com/david-thrower/cerebros-core-algorithm-alpha.git .
             git fetch origin 208-refactor-nlp-example-to-tokenize-first
             git checkout 208-refactor-nlp-example-to-tokenize-first
@@ -24,7 +24,7 @@ spec:
             PIP_BREAK_SYSTEM_PACKAGES=1 python -m pip install --no-cache-dir --break-system-packages -r cicd-requirements.txt
             PIP_BREAK_SYSTEM_PACKAGES=1 python -m pip install --no-cache-dir --break-system-packages mlflow transformers
             python tokenize_first_runner.py --mode train \
-              --cache data/train_tokens.npz --epochs 1 --batch 8 --print-score-only
+              --cache /data/train_tokens.npz --epochs 1 --batch 8 --print-score-only
         env:
         - name: MLFLOW_TRACKING_URI
           value: "http://mlflow-service:5000"
@@ -34,7 +34,7 @@ spec:
           value: "1"
         volumeMounts:
         - name: data-storage
-          mountPath: /app/repo/data
+          mountPath: /data
         resources:
           requests:
             memory: "6Gi"