add timeouts

Signed-off-by: Brian Gallagher <[email protected]>

Author: briangallagher

.tekton/universal-image-py312-cuda128-torch280-pull-request.yaml

@@ -18,6 +18,8 @@ metadata:
   name: universal-image-py312-cuda128-torch280-on-pull-request
   namespace: open-data-hub-tenant
 spec:
+  timeouts:
+    pipeline: 9h  
   params:
   - name: git-url
     value: '{{source_url}}'

.tekton/universal-image-py312-cuda128-torch280-push.yaml

@@ -17,6 +17,8 @@ metadata:
   name: universal-image-py312-cuda128-torch280-on-push
   namespace: open-data-hub-tenant
 spec:
+  timeouts:
+    pipeline: 9h  
   params:
   - name: git-url
     value: '{{source_url}}'

images/universal/training/py312-cuda128-torch280/Dockerfile

@@ -154,9 +154,30 @@ RUN pip install --retries 5 --timeout 300 --no-cache-dir \
  && chmod -R g+w /opt/app-root/lib/python3.12/site-packages \
  && fix-permissions /opt/app-root -P
 
+# Provide CUDA user-space libraries via pip, aligning with runtime for extension builds
+RUN pip install --retries 5 --timeout 300 --no-cache-dir \
+    nvidia-nccl-cu12==2.27.3 \
+    nvidia-cublas-cu12==12.8.4.1 \
+    nvidia-cuda-cupti-cu12==12.8.90 \
+    nvidia-cuda-nvrtc-cu12==12.8.93 \
+    nvidia-cuda-runtime-cu12==12.8.90 \
+    nvidia-cudnn-cu12==9.10.2.21 \
+    nvidia-cufft-cu12==11.3.3.83 \
+    nvidia-cufile-cu12==1.13.1.3 \
+    nvidia-curand-cu12==10.3.9.90 \
+    nvidia-cusolver-cu12==11.7.3.90 \
+    nvidia-cusparse-cu12==12.5.8.93 \
+    nvidia-cusparselt-cu12==0.7.1 \
+    nvidia-nvjitlink-cu12==12.8.93 \
+    nvidia-nvtx-cu12==12.8.90 \
+ && fix-permissions /opt/app-root -P
+
+# Ensure cuDNN from pip is discoverable during source builds
+ENV LD_LIBRARY_PATH="/opt/app-root/lib/python3.12/site-packages/nvidia/cudnn/lib:${LD_LIBRARY_PATH}"
+
 # Deterministic 2-step: sub-dep first, then parent without deps (align with runtime)
-RUN pip install --retries 5 --timeout 300 --no-cache-dir --no-build-isolation causal-conv1d==1.5.3.post1 && \
-    pip install --retries 5 --timeout 300 --no-cache-dir --no-build-isolation mamba-ssm==2.2.6.post3 --no-deps && \
+RUN pip install --no-cache-dir --no-build-isolation causal-conv1d==1.5.3.post1 && \
+    pip install --no-cache-dir --no-build-isolation mamba-ssm==2.2.6.post3 --no-deps && \
     fix-permissions /opt/app-root -P
 
 # Provide a POSIX entrypoint wrapper to choose behavior based on invocation