Rename obsolete mentions to datasets_based (#805)

- rename prefix WORKER_LOOP_ to WORKER_
- rename DATASETS_BASED_ENDPOINT to WORKER_ENDPOINT
- rename DATASETS_BASED_CONTENT_MAX_BYTES to WORKER_CONTENT_MAX_BYTES
- ensure WORKER_STORAGE_PATHS is always used in Helm

Author: severo

DEVELOPER_GUIDE.md

@@ -155,7 +155,7 @@ GITHUB_TOKEN=xxx
 
 ## Mac OS
 
-To install the [datasets based worker](./services/worker) on Mac OS, you can follow the next steps.
+To install the [worker](./services/worker) on Mac OS, you can follow the next steps.
 
 ### First: as an administrator
 

chart/env/dev.yaml

@@ -239,7 +239,3 @@ sizes:
     limits:
       cpu: 1
       memory: "4Gi"
-
-# --- datasets_based ---
-datasetsBased:
-  contentMaxBytes: "10_000_000"

chart/env/prod.yaml

@@ -309,7 +309,3 @@ sizes:
     limits:
       cpu: 2
       memory: "1Gi"
-
-# --- datasets_based ---
-datasetsBased:
-  contentMaxBytes: "10_000_000"

chart/templates/_envDatasetsBased.tpl

@@ -8,7 +8,5 @@
   value: "/tmp/modules-cache"
 - name: NUMBA_CACHE_DIR
   value: "/tmp/numba-cache"
-- name: CONTENT_MAX_BYTES
-  value: {{ .Values.datasetsBased.contentMaxBytes}}
 {{- end -}}
 

chart/templates/_envWorker.tpl

@@ -0,0 +1,19 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright 2022 The HuggingFace Authors.
+
+{{- define "envWorker" -}}
+- name: WORKER_CONTENT_MAX_BYTES
+  value: {{ .Values.worker.contentMaxBytes | quote}}
+  # WORKER_ENDPOINT is not defined here, it's hard-coded in the template
+- name: WORKER_MAX_DISK_USAGE_PCT
+  value: {{ .Values.worker.maxDiskUsagePct | quote }}
+- name: WORKER_MAX_LOAD_PCT
+  value: {{ .Values.worker.maxLoadPct | quote }}
+- name: WORKER_MAX_MEMORY_PCT
+  value: {{ .Values.worker.maxMemoryPct | quote }}
+- name: WORKER_SLEEP_SECONDS
+  value: {{ .Values.worker.sleepSeconds | quote }}
+- name: WORKER_STORAGE_PATHS
+  value: {{ .Values.assets.storageDirectory | quote }}
+  # ^ note: for datasets_based workers, the datasets cache is automatically added, so no need to add it here
+{{- end -}}

chart/templates/_envWorkerLoop.tpl

@@ -82,7 +82,7 @@ imagePullSecrets:
 {{ include "datasetsServer.images.image" (dict "imageRoot" .Values.images.services.api "global" .Values.global.huggingface) }}
 {{- end -}}
 
-{{- define "workers.datasetsBased.image" -}}
+{{- define "services.worker.image" -}}
 {{ include "datasetsServer.images.image" (dict "imageRoot" .Values.images.services.worker "global" .Values.global.huggingface) }}
 {{- end -}}
 
@@ -263,4 +263,4 @@ Return the HUB url
 {{- $hubName := ((list $.Release.Name "hub") | join "-") | trunc 63 | trimSuffix "-" -}}
 http://{{ $hubName }}
 {{- end -}}
-{{- end -}}
+{{- end -}}

chart/templates/_helpers.tpl

@@ -3,21 +3,19 @@
 
 {{- define "containerWorkerConfigNames" -}}
 - name: "{{ include "name" . }}-worker-config-names"
-  image: {{ include "workers.datasetsBased.image" . }}
+  image: {{ include "services.worker.image" . }}
   imagePullPolicy: {{ .Values.images.pullPolicy }}
   env:
-  - name: DATASETS_BASED_ENDPOINT
+  - name: WORKER_ENDPOINT
     value: "/config-names"
     # ^ hard-coded
   {{ include "envCache" . | nindent 2 }}
   {{ include "envQueue" . | nindent 2 }}
   {{ include "envCommon" . | nindent 2 }}
-  {{ include "envWorkerLoop" . | nindent 2 }}
+  {{ include "envWorker" . | nindent 2 }}
   {{ include "envDatasetsBased" . | nindent 2 }}
   - name: DATASETS_BASED_HF_DATASETS_CACHE
     value: {{ printf "%s/config-names/datasets" .Values.cacheDirectory | quote }}
-  - name: DATASETS_BASED_CONTENT_MAX_BYTES
-    value: {{ .Values.datasetsBased.contentMaxBytes | quote}}
   - name: QUEUE_MAX_JOBS_PER_NAMESPACE
     # value: {{ .Values.queue.maxJobsPerNamespace | quote }}
     # overridden

chart/templates/worker/config-names/_container.tpl

@@ -3,18 +3,16 @@
 
 {{- define "containerWorkerDatasetInfo" -}}
 - name: "{{ include "name" . }}-worker-dataset-info"
-  image: {{ include "workers.datasetsBased.image" . }}
+  image: {{ include "services.worker.image" . }}
   imagePullPolicy: {{ .Values.images.pullPolicy }}
   env:
-  - name: DATASETS_BASED_ENDPOINT
+  - name: WORKER_ENDPOINT
     value: "/dataset-info"
     # ^ hard-coded
   {{ include "envCache" . | nindent 2 }}
   {{ include "envQueue" . | nindent 2 }}
   {{ include "envCommon" . | nindent 2 }}
-  {{ include "envWorkerLoop" . | nindent 2 }}
-  - name: DATASETS_BASED_CONTENT_MAX_BYTES
-    value: {{ .Values.datasetsBased.contentMaxBytes | quote}}
+  {{ include "envWorker" . | nindent 2 }}
   - name: QUEUE_MAX_JOBS_PER_NAMESPACE
     # value: {{ .Values.queue.maxJobsPerNamespace | quote }}
     # overridden

chart/templates/worker/dataset-info/_container.tpl

@@ -3,25 +3,20 @@
 
 {{- define "containerWorkerFirstRows" -}}
 - name: "{{ include "name" . }}-worker-first-rows"
-  image: {{ include "workers.datasetsBased.image" . }}
+  image: {{ include "services.worker.image" . }}
   imagePullPolicy: {{ .Values.images.pullPolicy }}
   env:
-  - name: DATASETS_BASED_ENDPOINT
+  - name: WORKER_ENDPOINT
     value: "/first-rows"
     # ^ hard-coded
   {{ include "envAssets" . | nindent 2 }}
   {{ include "envCache" . | nindent 2 }}
   {{ include "envQueue" . | nindent 2 }}
   {{ include "envCommon" . | nindent 2 }}
-  {{ include "envWorkerLoop" . | nindent 2 }}
-  - name: WORKER_LOOP_STORAGE_PATHS
-    value: {{ .Values.assets.storageDirectory | quote }}
-    # ^ note: the datasets cache is automatically added, so no need to add it here
+  {{ include "envWorker" . | nindent 2 }}
   {{ include "envDatasetsBased" . | nindent 2 }}
   - name: DATASETS_BASED_HF_DATASETS_CACHE
     value: {{ printf "%s/first-rows/datasets" .Values.cacheDirectory | quote }}
-  - name: DATASETS_BASED_CONTENT_MAX_BYTES
-    value: {{ .Values.datasetsBased.contentMaxBytes | quote}}
   - name: QUEUE_MAX_JOBS_PER_NAMESPACE
     # value: {{ .Values.queue.maxJobsPerNamespace | quote }}
     # overridden