feat(workflows-local-runner-unifiedstorage): update start container script to reslove symlink and use its target for mount (#709)

Co-authored-by: Tejas Kashinath <[email protected]>
Co-authored-by: Tejas Kashinath <[email protected]>

Author: 3 people

template/v2/dirs/etc/sagemaker-ui/workflows/docker-compose.yaml

@@ -43,7 +43,7 @@ x-airflow-common: &airflow-common
     - /home/sagemaker-user/.workflows_setup/plugins:/usr/local/airflow/plugins
     - /home/sagemaker-user/.workflows_setup/requirements:/usr/local/airflow/requirements
     - /home/sagemaker-user/.workflows_setup/startup:/usr/local/airflow/startup
-    - /home/sagemaker-user/src:/home/sagemaker-user/src:rw
+    - ${MOUNT_DIR}:/home/sagemaker-user/src:rw
     - /home/sagemaker-user/jobs:/home/sagemaker-user/jobs:rw
   depends_on: &airflow-common-depends-on
     postgres:

template/v2/dirs/etc/sagemaker-ui/workflows/start-workflows-container.sh

@@ -3,6 +3,11 @@ set -eu
 
 # Get project directory based on storage type
 PROJECT_DIR=${SMUS_PROJECT_DIR:-"$HOME/src"}
+if [ -z "$SMUS_PROJECT_DIR" ]; then
+    MOUNT_DIR=$PROJECT_DIR
+else
+    MOUNT_DIR=$(readlink -f "$PROJECT_DIR")  # get the symlink source
+fi
 
 # Datazone project metadata
 RESOURCE_METADATA_FILE=/opt/ml/metadata/resource-metadata.json
@@ -178,6 +183,7 @@ cp -n "/etc/sagemaker-ui/workflows/sample_dag.py" "${WORKFLOW_DAG_PATH}/"
 aws ecr get-login-password --region ${AWS_REGION} | docker login --username AWS --password-stdin ${ECR_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com
 
 PROJECT_DIR=$(basename $PROJECT_DIR)  \
+MOUNT_DIR=$MOUNT_DIR \
 ECR_ACCOUNT_ID=$ECR_ACCOUNT_ID \
 ACCOUNT_ID=$AWS_ACCOUNT_ID \
 DZ_DOMAIN_ID=$DZ_DOMAIN_ID \

template/v3/dirs/etc/sagemaker-ui/workflows/docker-compose.yaml

@@ -43,7 +43,7 @@ x-airflow-common: &airflow-common
     - /home/sagemaker-user/.workflows_setup/plugins:/usr/local/airflow/plugins
     - /home/sagemaker-user/.workflows_setup/requirements:/usr/local/airflow/requirements
     - /home/sagemaker-user/.workflows_setup/startup:/usr/local/airflow/startup
-    - /home/sagemaker-user/src:/home/sagemaker-user/src:rw
+    - ${MOUNT_DIR}:/home/sagemaker-user/src:rw
     - /home/sagemaker-user/jobs:/home/sagemaker-user/jobs:rw
   depends_on: &airflow-common-depends-on
     postgres:

template/v3/dirs/etc/sagemaker-ui/workflows/start-workflows-container.sh

@@ -3,6 +3,11 @@ set -eu
 
 # Get project directory based on storage type
 PROJECT_DIR=${SMUS_PROJECT_DIR:-"$HOME/src"}
+if [ -z "$SMUS_PROJECT_DIR" ]; then
+    MOUNT_DIR=$PROJECT_DIR
+else
+    MOUNT_DIR=$(readlink -f "$PROJECT_DIR")  # get the symlink source
+fi
 
 # Datazone project metadata
 RESOURCE_METADATA_FILE=/opt/ml/metadata/resource-metadata.json
@@ -178,6 +183,7 @@ cp -n "/etc/sagemaker-ui/workflows/sample_dag.py" "${WORKFLOW_DAG_PATH}/"
 aws ecr get-login-password --region ${AWS_REGION} | docker login --username AWS --password-stdin ${ECR_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com
 
 PROJECT_DIR=$(basename $PROJECT_DIR)  \
+MOUNT_DIR=$MOUNT_DIR \
 ECR_ACCOUNT_ID=$ECR_ACCOUNT_ID \
 ACCOUNT_ID=$AWS_ACCOUNT_ID \
 DZ_DOMAIN_ID=$DZ_DOMAIN_ID \