fix: 🐛 ensure all the workers have the same access to the disk (#811)

Author: severo

chart/Chart.yaml

@@ -18,7 +18,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 1.1.0
+version: 1.1.1
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

chart/templates/worker/dataset-info/_container.tpl

@@ -13,12 +13,16 @@
   {{ include "envQueue" . | nindent 2 }}
   {{ include "envCommon" . | nindent 2 }}
   {{ include "envWorker" . | nindent 2 }}
+  {{ include "envDatasetsBased" . | nindent 2 }}
+  - name: DATASETS_BASED_HF_DATASETS_CACHE
+    value: {{ printf "%s/datasets-info/datasets" .Values.cacheDirectory | quote }}
   - name: QUEUE_MAX_JOBS_PER_NAMESPACE
     # value: {{ .Values.queue.maxJobsPerNamespace | quote }}
     # overridden
     value: {{ .Values.datasetInfo.queue.maxJobsPerNamespace | quote }}
   volumeMounts:
   {{ include "volumeMountAssetsRW" . | nindent 2 }}
+  {{ include "volumeMountCache" . | nindent 2 }}
   securityContext:
     allowPrivilegeEscalation: false
   resources: {{ toYaml .Values.datasetInfo.resources | nindent 4 }}

chart/templates/worker/dataset-info/deployment.yaml

@@ -22,6 +22,7 @@ spec:
       {{- include "image.imagePullSecrets" . | nindent 6 }}
       initContainers:
         {{ include "initContainerAssets" . | nindent 8 }}
+        {{ include "initContainerCache" . | nindent 8 }}
       containers: {{ include "containerWorkerDatasetInfo" . | nindent 8 }}
       nodeSelector: {{ toYaml .Values.datasetInfo.nodeSelector | nindent 8 }}
       tolerations: {{ toYaml .Values.datasetInfo.tolerations | nindent 8 }}

chart/templates/worker/parquet/_container.tpl

@@ -13,12 +13,16 @@
   {{ include "envQueue" . | nindent 2 }}
   {{ include "envCommon" . | nindent 2 }}
   {{ include "envWorker" . | nindent 2 }}
+  {{ include "envDatasetsBased" . | nindent 2 }}
+  - name: DATASETS_BASED_HF_DATASETS_CACHE
+    value: {{ printf "%s/parquet/datasets" .Values.cacheDirectory | quote }}
   - name: QUEUE_MAX_JOBS_PER_NAMESPACE
     # value: {{ .Values.queue.maxJobsPerNamespace | quote }}
     # overridden
     value: {{ .Values.parquet.queue.maxJobsPerNamespace | quote }}
   volumeMounts:
   {{ include "volumeMountAssetsRW" . | nindent 2 }}
+  {{ include "volumeMountCache" . | nindent 2 }}
   securityContext:
     allowPrivilegeEscalation: false
   resources: {{ toYaml .Values.parquet.resources | nindent 4 }}

chart/templates/worker/parquet/deployment.yaml

@@ -22,6 +22,7 @@ spec:
       {{- include "image.imagePullSecrets" . | nindent 6 }}
       initContainers:
         {{ include "initContainerAssets" . | nindent 8 }}
+        {{ include "initContainerCache" . | nindent 8 }}
       containers: {{ include "containerWorkerParquet" . | nindent 8 }}
       nodeSelector: {{ toYaml .Values.parquet.nodeSelector | nindent 8 }}
       tolerations: {{ toYaml .Values.parquet.tolerations | nindent 8 }}

chart/templates/worker/sizes/_container.tpl

@@ -13,12 +13,16 @@
   {{ include "envQueue" . | nindent 2 }}
   {{ include "envCommon" . | nindent 2 }}
   {{ include "envWorker" . | nindent 2 }}
+  {{ include "envDatasetsBased" . | nindent 2 }}
+  - name: DATASETS_BASED_HF_DATASETS_CACHE
+    value: {{ printf "%s/sizes/datasets" .Values.cacheDirectory | quote }}
   - name: QUEUE_MAX_JOBS_PER_NAMESPACE
     # value: {{ .Values.queue.maxJobsPerNamespace | quote }}
     # overridden
     value: {{ .Values.sizes.queue.maxJobsPerNamespace | quote }}
   volumeMounts:
   {{ include "volumeMountAssetsRW" . | nindent 2 }}
+  {{ include "volumeMountCache" . | nindent 2 }}
   securityContext:
     allowPrivilegeEscalation: false
   resources: {{ toYaml .Values.sizes.resources | nindent 4 }}

chart/templates/worker/sizes/deployment.yaml

@@ -22,6 +22,7 @@ spec:
       {{- include "image.imagePullSecrets" . | nindent 6 }}
       initContainers:
         {{ include "initContainerAssets" . | nindent 8 }}
+        {{ include "initContainerCache" . | nindent 8 }}
       containers: {{ include "containerWorkerSizes" . | nindent 8 }}
       nodeSelector: {{ toYaml .Values.sizes.nodeSelector | nindent 8 }}
       tolerations: {{ toYaml .Values.sizes.tolerations | nindent 8 }}