backout switching gpu workers to docker + new worker image (#1120)

* backout switching gpu workers to docker + new worker image

* upgrade to taskgraph 14.2.1

This picks up taskcluster/taskgraph#691, which is needed to revert back to the old GPU worker image.

Author: bhearsum

.taskcluster.yml

@@ -227,7 +227,7 @@ tasks:
                                   taskclusterProxy: true
                                   chainOfTrust: true
 
-                              image: mozillareleases/taskgraph:decision-v14.1.1@sha256:a3f03dbaf6b52733e16e52cdf6fc547193839ed825e1c3740d2b0515f65c7d73
+                              image: mozillareleases/taskgraph:decision-v14.2.1@sha256:f4e3a22df9ec0017a2534b3a7b4cd9b60318f86619e0c2156c12c1ec1a0e32cb
                               maxRunTime: 1800
                               onExitStatus:
                                 retry:

docs/training/task-cluster.md

@@ -136,16 +136,24 @@ To start an interactive task, follow these steps:
 
 5. Reduce the maxRunTime to a best guess at how long you'll need the task and worker running for. (We pay for every minute a worker runs - so they should not be kept running, eg: overnight.)
 
-6. Adjust the payload to simply run bash and sleep (instead of a full pipeline step):
+6. Adjust the payload to simply run bash and sleep (instead of a full pipeline step). For docker-worker tasks use something like:
 ```
      command:
     - bash
     - '-c'
     - 'sleep 7200'
 ```
 
-7. Click "Create Task"
+For generic-worker tasks (those needing a GPU), use:
+```
+     command:
+    - - bash
+      - '-c'
+      - 'sleep 7200'
+```
+
+(docker-worker tasks have an `image` section in the payload)
 
-After a few minutes you should be able to get a shell (a link will show up in the tab when it's ready). This shell should drop you inside of docker container as root, running the same image as the task you started this process with. Most tasks drop privileges to the `worker` user before doing any work, so you may want to run `su - worker` before doing anything of note.
+7. Click "Create Task"
 
-When you are done with the worker you can use "Cancel" from the three dots menu to immediately shut it down. (This should happen within a few minutes of closing your last shell to the worker, but it's good practice to do it yourself to minimize costs.)
+After a few minutes you should be able to get a shell (a link will show up in the tab when it's ready).

pipeline/bicleaner/bicleaner.sh

@@ -13,6 +13,10 @@ echo "###### Bicleaner filtering"
 test -v SRC
 test -v TRG
 test -v CUDA_DIR
+test -v CUDNN_DIR
+
+# cuda and cudnn libs
+export LD_LIBRARY_PATH=${CUDA_DIR}/lib64:${CUDNN_DIR}:${LD_LIBRARY_PATH:+LD_LIBRARY_PATH:}
 
 corpus_prefix=$1
 output_prefix=$2

pipeline/translate/requirements/translate-ctranslate2.in

@@ -1,4 +1,3 @@
 ctranslate2==4.3.1
 sentencepiece==0.2.0
 gpustat==1.1.1
-requests==2.32.3

pipeline/translate/requirements/translate-ctranslate2.txt

@@ -44,7 +44,7 @@ PyGithub="2.4.0"
 pyperclip="1.9.0"
 ruamel-yaml = "^0.18.6"
 taskcluster = "^56.0.3"
-taskcluster-taskgraph = "^14.1.1"
+taskcluster-taskgraph = "^14.2.1"
 kinto-http="11.7.1"
 # Use an outdated version of pydantic due to dependency requirements conflict.
 pydantic="1.10.19"
@@ -69,7 +69,7 @@ requests-mock = "^1.11.0"
 sh = "^2.0.6"
 zstandard = "^0.22.0"
 translations_parser = {path="./tracking/", develop=true}
-taskcluster-taskgraph = "^14.1.1"
+taskcluster-taskgraph = "^14.2.1"
 translations_taskgraph = {path="./taskcluster/", develop=true}
 sacremoses = "0.1.1"
 hanzidentifier = "1.2.0"

poetry.lock

@@ -94,39 +94,39 @@ workers:
             worker-type: 'b-linux-large-gcp-1tb-64-512-std-d2g'
         b-linux-v100-gpu:
             provisioner: '{trust-domain}-{level}'
-            implementation: docker-worker
+            implementation: generic-worker
             os: linux
-            worker-type: 'b-linux-v100-gpu-d2g'
+            worker-type: '{alias}'
         b-linux-v100-gpu-4:
             provisioner: '{trust-domain}-{level}'
-            implementation: docker-worker
+            implementation: generic-worker
             os: linux
-            worker-type: 'b-linux-v100-gpu-d2g-4'
+            worker-type: '{alias}'
         b-linux-v100-gpu-4-300gb:
             provisioner: '{trust-domain}-{level}'
-            implementation: docker-worker
+            implementation: generic-worker
             os: linux
-            worker-type: 'b-linux-v100-gpu-d2g-4-300gb'
+            worker-type: '{alias}'
         b-linux-v100-gpu-4-300gb-standard:
             provisioner: '{trust-domain}-{level}'
-            implementation: docker-worker
+            implementation: generic-worker
             os: linux
-            worker-type: 'b-linux-v100-gpu-d2g-4-300gb-standard'
+            worker-type: '{alias}'
         b-linux-v100-gpu-4-1tb:
             provisioner: '{trust-domain}-{level}'
-            implementation: docker-worker
+            implementation: generic-worker
             os: linux
-            worker-type: 'b-linux-v100-gpu-d2g-4-1tb'
+            worker-type: '{alias}'
         b-linux-v100-gpu-4-2tb:
             provisioner: '{trust-domain}-{level}'
-            implementation: docker-worker
+            implementation: generic-worker
             os: linux
-            worker-type: 'b-linux-v100-gpu-d2g-4-2tb'
+            worker-type: '{alias}'
         b-linux-v100-gpu-4-1tb-standard:
             provisioner: '{trust-domain}-{level}'
-            implementation: docker-worker
+            implementation: generic-worker
             os: linux
-            worker-type: 'b-linux-v100-gpu-d2g-4-1tb-standard'
+            worker-type: '{alias}'
         images:
             provisioner: '{trust-domain}-{level}'
             implementation: docker-worker

pyproject.toml

@@ -59,6 +59,6 @@ ENV SHELL=/bin/bash \
     TERM=hterm-256color
 
 VOLUME /builds/worker/checkouts
-VOLUME /builds/worker/.task-cache/pip
+VOLUME /builds/worker/.cache
 
 USER root

taskcluster/config.yml

@@ -68,6 +68,6 @@ ENV SHELL=/bin/bash \
     PATH="/builds/worker/.local/bin:$PATH"
 
 VOLUME /builds/worker/checkouts
-VOLUME /builds/worker/.task-cache/pip
+VOLUME /builds/worker/.cache
 
 USER root