diff --git a/README.md b/README.md index c5d0463f..0fe9df42 100644 --- a/README.md +++ b/README.md @@ -86,7 +86,7 @@ from prefect.blocks.system import Secret # S3 bucket name and subfolder that the Jupyter user, # Prefect and EOPF dask workers will have access to. -BUCKET_NAME="rs-cluster-temp" +BUCKET_NAME="rs-dev-cluster-temp" BUCKET_FOLDER="prefect-share" # See: https://docs.prefect.io/integrations/prefect-aws/index diff --git a/notebooks/resources/dask_utils.py b/notebooks/resources/dask_utils.py index 86b75d6e..8af564f5 100644 --- a/notebooks/resources/dask_utils.py +++ b/notebooks/resources/dask_utils.py @@ -81,6 +81,7 @@ def init_dask_cluster( cluster_tag: str = "", worker_cores: int = 1, worker_memory: float = 2.0, + worker_extra_pod_config: dict = {}, scheduler_memory_limit: int = 2, namespace="dask-gateway", ) -> tuple[Gateway, GatewayCluster, DaskClient]: @@ -95,6 +96,7 @@ def init_dask_cluster( cluster_tag: cluster name: "dask-staging" or "dask-eopf" worker_cores: number of worker cores worker_memory: worker memory in GB + worker_extra_pod_config: Node affinities and tolerations namespace: dask gateway namespace """ @@ -149,6 +151,7 @@ def init_dask_cluster( scheduler_memory_limit=scheduler_memory_limit, namespace=namespace, image=image, + worker_extra_pod_config=worker_extra_pod_config, cluster_name=cluster_tag, scheduler_extra_pod_labels={"cluster_name": cluster_tag}, ) @@ -179,7 +182,8 @@ def init_dask_cluster( def init_dask_cluster_staging( scale: int, - image: str = "ghcr.io/rs-python/rs-infra-core-dask-staging:latest", + image: str = "ghcr.io/rs-python/rs-infra-core-dask-staging:feat-rspy625-update-jupyter", + worker_extra_pod_config: dict = {'affinity': {'nodeAffinity': {'requiredDuringSchedulingIgnoredDuringExecution': {'nodeSelectorTerms': [{'matchExpressions': [{'key': 'node-role.kubernetes.io/access_csc', 'operator': 'Exists'}]}]}}}, 'tolerations': [{'key': 'role', 'operator': 'Equal', 'value': 'access_csc', 'effect': 'NoSchedule'}]}, *args, **kwargs, ): @@ -198,6 +202,7 @@ def init_dask_cluster_staging( ), scale, image=image, + worker_extra_pod_config=worker_extra_pod_config, cluster_tag="dask-staging", *args, **kwargs, @@ -206,7 +211,8 @@ def init_dask_cluster_staging( def init_dask_cluster_eopf( scale: int, - image: str = "ghcr.io/rs-python/rs-infra-core-dask-eopf:latest", + image: str = "ghcr.io/rs-python/rs-infra-core-dask-eopf:feat-rspy625-update-jupyter", + worker_extra_pod_config: dict = {'affinity': {'nodeAffinity': {'requiredDuringSchedulingIgnoredDuringExecution': {'nodeSelectorTerms': [{'matchExpressions': [{'key': 'node-role.kubernetes.io/processing', 'operator': 'Exists'}]}]}}}}, *args, **kwargs, ): @@ -225,6 +231,7 @@ def init_dask_cluster_eopf( ), scale, image=image, + worker_extra_pod_config=worker_extra_pod_config, cluster_tag="dask-eopf", *args, **kwargs, diff --git a/notebooks/sprints/sprint21/l0/config/s1/iw_joborder.short.yaml b/notebooks/sprints/sprint21/l0/config/s1/iw_joborder.short.yaml index e30b2a1a..c441a8cc 100644 --- a/notebooks/sprints/sprint21/l0/config/s1/iw_joborder.short.yaml +++ b/notebooks/sprints/sprint21/l0/config/s1/iw_joborder.short.yaml @@ -27,7 +27,7 @@ I/O: input_products: - id: S1ACADU - path: s3://rs-cluster-temp/stations/CADIP/S1A_20240410083700053369.short + path: s3://rs-dev-cluster-temp/stations/CADIP/S1A_20240410083700053369.short store_type: cadu store_params: storage_options: diff --git a/notebooks/sprints/sprint21/l0/config/s1/iw_joborder.yaml b/notebooks/sprints/sprint21/l0/config/s1/iw_joborder.yaml index 216a6fcf..7d2f9e66 100644 --- a/notebooks/sprints/sprint21/l0/config/s1/iw_joborder.yaml +++ b/notebooks/sprints/sprint21/l0/config/s1/iw_joborder.yaml @@ -27,7 +27,7 @@ I/O: input_products: - id: S1ACADU - path: s3://rs-cluster-temp/stations/CADIP/S1A_20240410083700053369 + path: s3://rs-dev-cluster-temp/stations/CADIP/S1A_20240410083700053369 store_type: cadu store_params: storage_options: diff --git a/notebooks/sprints/sprint21/l0/config/s3/s3_dordop_payload.yaml b/notebooks/sprints/sprint21/l0/config/s3/s3_dordop_payload.yaml index e05a2c58..c79cedbf 100644 --- a/notebooks/sprints/sprint21/l0/config/s3/s3_dordop_payload.yaml +++ b/notebooks/sprints/sprint21/l0/config/s3/s3_dordop_payload.yaml @@ -32,7 +32,7 @@ I/O: # input products descriptions and access parameters input_products: - id: S3ACADUS - path: s3://rs-cluster-temp/stations/CADIP/S3B_20240411142312031054 + path: s3://rs-dev-cluster-temp/stations/CADIP/S3B_20240411142312031054 store_type: cadu store_params: storage_options: