add timeouts

Signed-off-by: Brian Gallagher <[email protected]>

Author: briangallagher

.tekton/universal-image-py312-cuda128-torch280-pull-request.yaml

@@ -18,6 +18,8 @@ metadata:
   name: universal-image-py312-cuda128-torch280-on-pull-request
   namespace: open-data-hub-tenant
 spec:
+  timeouts:
+    pipeline: 9h  
   params:
   - name: git-url
     value: '{{source_url}}'

.tekton/universal-image-py312-cuda128-torch280-push.yaml

@@ -17,6 +17,8 @@ metadata:
   name: universal-image-py312-cuda128-torch280-on-push
   namespace: open-data-hub-tenant
 spec:
+  timeouts:
+    pipeline: 9h  
   params:
   - name: git-url
     value: '{{source_url}}'

images/universal/training/py312-cuda128-torch280/Dockerfile

@@ -19,7 +19,8 @@ LABEL name="universal:py312-cuda128-torch280" \
       io.k8s.display-name="Universal CUDA 12.8 Python 3.12 (Workbench + Runtime)" \
       io.k8s.description="Universal image: Jupyter workbench by default; runtime when command provided. Includes RDMA/IB libs, Torch 2.8.0 cu128, FlashAttention 2.8.3."
 
-COPY ./images/universal/training/py312-cuda128-torch280/LICENSE.md /licenses/cuda-license.md
+## TODO: Add license file
+# COPY LICENSE.md /licenses/cuda-license.md
 
 # For OS installs we need elevated privileges; base may default to 1001
 USER 0
@@ -108,10 +109,10 @@ RUN pip install --retries 5 --timeout 300 --no-cache-dir numpy==2.3.3
 
 # Install build backend for VCS package and the SDK itself (no build isolation so backend is visible)
 RUN pip install --retries 5 --timeout 300 --no-cache-dir hatchling hatch-vcs
-RUN pip install --retries 5 --timeout 300 --no-cache-dir --no-build-isolation "git+https://github.com/briangallagher/sdk@training-hub"
+RUN pip install --retries 5 --timeout 300 --no-cache-dir --no-build-isolation "git+https://github.com/briangallagher/sdk@add-training-hub"
 
 # Provide ninja via pip (RHEL/UBI repo ninja-build may be unavailable)
-RUN pip install --retries 5 --timeout 300 --no-cache-dir ninja
+# RUN pip install --retries 5 --timeout 300 --no-cache-dir ninja
 
 # Install remaining runtime packages (resolved from default PyPI), including FlashAttention
 # Note: We intentionally do not use a Pipfile/lock here to avoid mixing resolvers with the base (uv lock),
@@ -125,6 +126,8 @@ RUN pip install --retries 5 --timeout 300 --no-cache-dir \
     datasets==4.0.0 \
     pydantic>=2.11.7 \
     aiofiles==24.1.0 \
+    deprecated==1.2.18 \
+    typer==0.19.2 \
     "protobuf>=5.28.0,<6.0.0" \
     "simpleeval>=0.9.13,<1.0" \
     safetensors==0.6.2 \
@@ -140,9 +143,6 @@ RUN pip install --retries 5 --timeout 300 --no-cache-dir \
     kernels==0.10.3 \
     "sentencepiece>=0.1.99,<0.3" \
     tokenizers==0.22.1 \
-    instructlab-training==0.12.1 \
-    rhai-innovation-mini-trainer==0.3.0 \
-    training-hub==0.3.0 \
     trl==0.21.0 \
     deepspeed>=0.14.3 \
     async-timeout==4.0.3 \
@@ -151,12 +151,22 @@ RUN pip install --retries 5 --timeout 300 --no-cache-dir \
     huggingface-hub==0.34.4 \
     mlflow==3.4.0 \
     psutil==7.0.0 \
+    training-hub==0.3.0 \
+    instructlab-training==0.12.1 \
+    rhai-innovation-mini-trainer==0.3.0 \
  && chmod -R g+w /opt/app-root/lib/python3.12/site-packages \
  && fix-permissions /opt/app-root -P
 
-# Deterministic 2-step: sub-dep first, then parent without deps (align with runtime)
-RUN pip install --retries 5 --timeout 300 --no-cache-dir --no-build-isolation causal-conv1d==1.5.3.post1 && \
-    pip install --retries 5 --timeout 300 --no-cache-dir --no-build-isolation mamba-ssm==2.2.6.post3 --no-deps && \
+
+# Build helpers to compile PyTorch extensions
+RUN pip install -U pip setuptools wheel ninja cmake
+
+# Optional: set GPU archs if you hit arch issues
+# ENV TORCH_CUDA_ARCH_LIST="8.0;8.6;8.9;9.0"
+
+# Deterministic 2-step: sub-dep first, then parent without deps
+RUN pip install --no-build-isolation --no-cache-dir causal-conv1d==1.5.3.post1 && \
+    pip install --no-build-isolation --no-cache-dir mamba-ssm==2.2.6.post3 --no-deps && \
     fix-permissions /opt/app-root -P
 
 # Provide a POSIX entrypoint wrapper to choose behavior based on invocation