Merge branch 'main' into release-3.1.1

Author: github-actions[bot]

build_artifacts/v2/v2.7/v2.7.0/dirs/etc/sagemaker-ui/sagemaker_ui_post_startup.sh

@@ -89,6 +89,10 @@ echo "Successfully removed the ~/.aws/config file"
 aws configure set credential_source EcsContainer
 echo "Successfully configured default profile"
 
+# Add region configuration using REGION_NAME environment variable
+aws configure set region "${REGION_NAME}"
+echo "Successfully configured region to ${REGION_NAME}"
+
 # add SparkMonitor and Connection Magic entrypoint
 NB_USER=sagemaker-user
 

build_artifacts/v3/v3.2/v3.2.0/dirs/etc/sagemaker-ui/sagemaker_ui_post_startup.sh

@@ -65,6 +65,10 @@ echo "Successfully removed the ~/.aws/config file"
 aws configure set credential_source EcsContainer
 echo "Successfully configured default profile"
 
+# Add region configuration using REGION_NAME environment variable
+aws configure set region "${REGION_NAME}"
+echo "Successfully configured region to ${REGION_NAME}"
+
 # add SparkMonitor and Connection Magic entrypoint
 NB_USER=sagemaker-user
 

template/v2/dirs/etc/sagemaker-ui/libmgmt/install-lib.sh

@@ -1,9 +1,9 @@
 #!/bin/bash
 set -eux
-
+PROJECT_DIR=${SMUS_PROJECT_DIR:-"$HOME/src"}
 # Check if the .libs.json file exists
-if [ -e $1/.libs.json ]; then
-  lib_config_json=`cat $1/.libs.json`
+if [ -e $PROJECT_DIR/.libs.json ]; then
+  lib_config_json=`cat $PROJECT_DIR/.libs.json`
 
   apply_change_to_space=`echo $lib_config_json | jq -r '.ApplyChangeToSpace'`
   # Extract conda channels from the config, add `-c ` before each channel and join the strings

template/v2/dirs/etc/sagemaker-ui/sagemaker_ui_post_startup.sh

@@ -62,6 +62,10 @@ echo "Successfully removed the ~/.aws/config file"
 aws configure set credential_source EcsContainer
 echo "Successfully configured default profile"
 
+# Add region configuration using REGION_NAME environment variable
+aws configure set region "${REGION_NAME}"
+echo "Successfully configured region to ${REGION_NAME}"
+
 # add SparkMonitor and Connection Magic entrypoint
 NB_USER=sagemaker-user
 
@@ -134,8 +138,8 @@ case "$auth_mode" in
         ;;
 esac
 
-# Checks if the project is using Git or Non-Git storage
-is_non_git_storage() {
+# Checks if the project is using Git or S3 storage
+is_s3_storage() {
   getProjectDefaultEnvResponse=$(sagemaker-studio project get-project-default-environment --domain-id "$dataZoneDomainId" --project-id "$dataZoneProjectId" --profile DomainExecutionRoleCreds)
   gitConnectionArn=$(echo "$getProjectDefaultEnvResponse" | jq -r '.provisionedResources[] | select(.name=="gitConnectionArn") | .value')
   codeRepositoryName=$(echo "$getProjectDefaultEnvResponse" | jq -r '.provisionedResources[] | select(.name=="codeRepositoryName") | .value')
@@ -149,7 +153,26 @@ is_non_git_storage() {
 
 echo "Checking Project Storage Type"
 
-if ! is_non_git_storage; then
+# Execute once to store the result
+is_s3_storage
+is_s3_storage_flag=$?  # 0 if S3 storage, 1 if Git
+
+if [ "$is_s3_storage_flag" -eq 0 ]; then
+    export SMUS_PROJECT_DIR="$HOME/shared"
+    echo "Project is using S3 storage, project directory set to: $SMUS_PROJECT_DIR"
+else
+    export SMUS_PROJECT_DIR="$HOME/src"
+    echo "Project is using Git storage, project directory set to: $SMUS_PROJECT_DIR"
+fi
+
+if grep -q "^SMUS_PROJECT_DIR=" ~/.bashrc; then
+  echo "SMUS_PROJECT_DIR is defined in the env"
+else
+  echo SMUS_PROJECT_DIR="$SMUS_PROJECT_DIR" >> ~/.bashrc
+  echo readonly SMUS_PROJECT_DIR >> ~/.bashrc
+fi
+
+if [ $is_s3_storage_flag -ne 0 ]; then
   # Creating a directory where the repository will be cloned
   mkdir -p "$HOME/src"
 
@@ -190,7 +213,7 @@ if [ "${SAGEMAKER_APP_TYPE_LOWERCASE}" = "jupyterlab" ]; then
     trap 'write_status_to_file "error" "An unexpected error occurred. Please stop and restart your space to retry."' ERR
 
     # Install conda and pip dependencies if lib mgmt config existing
-    bash /etc/sagemaker-ui/libmgmt/install-lib.sh $HOME/src
+    bash /etc/sagemaker-ui/libmgmt/install-lib.sh
 
     # Install sm-spark-cli
     bash /etc/sagemaker-ui/workflows/sm-spark-cli-install.sh

template/v2/dirs/etc/sagemaker-ui/workflows/docker-compose.yaml

@@ -39,11 +39,11 @@ x-airflow-common: &airflow-common
     DataZoneDomainRegion: ${DZ_DOMAIN_REGION}
     ProjectS3Path: ${DZ_PROJECT_S3PATH}
   volumes:
-    - /home/sagemaker-user/${HOME_FOLDER_NAME}/workflows/dags:/usr/local/airflow/dags
+    - /home/sagemaker-user/${PROJECT_DIR}/workflows/dags:/usr/local/airflow/dags
     - /home/sagemaker-user/.workflows_setup/plugins:/usr/local/airflow/plugins
     - /home/sagemaker-user/.workflows_setup/requirements:/usr/local/airflow/requirements
     - /home/sagemaker-user/.workflows_setup/startup:/usr/local/airflow/startup
-    - /home/sagemaker-user/src:/home/sagemaker-user/src:rw
+    - ${MOUNT_DIR}:/home/sagemaker-user/src:rw
     - /home/sagemaker-user/jobs:/home/sagemaker-user/jobs:rw
   depends_on: &airflow-common-depends-on
     postgres:

template/v2/dirs/etc/sagemaker-ui/workflows/plugins/amzn_SagemakerWorkflowsOperator-1.0-py3-none-any.whl

@@ -1,6 +1,43 @@
 #!/bin/bash
 set -eu
 
+# Get project directory based on storage type
+PROJECT_DIR=${SMUS_PROJECT_DIR:-"$HOME/src"}
+if [ -z "$SMUS_PROJECT_DIR" ]; then
+    MOUNT_DIR=$PROJECT_DIR
+else
+    MOUNT_DIR=$(readlink -f "$PROJECT_DIR")  # get the symlink source
+fi
+
+# Datazone project metadata
+RESOURCE_METADATA_FILE=/opt/ml/metadata/resource-metadata.json
+SM_DOMAIN_ID=$(jq -r ".DomainId" < $RESOURCE_METADATA_FILE)
+AWS_ACCOUNT_ID=$(jq -r '.ExecutionRoleArn | split(":")[4]' < $RESOURCE_METADATA_FILE)
+ECR_ACCOUNT_ID=058264401727
+DZ_DOMAIN_ID=$(jq -r '.AdditionalMetadata.DataZoneDomainId' < $RESOURCE_METADATA_FILE)
+DZ_PROJECT_ID=$(jq -r '.AdditionalMetadata.DataZoneProjectId' < $RESOURCE_METADATA_FILE)
+DZ_ENV_ID=$(jq -r '.AdditionalMetadata.DataZoneEnvironmentId' < $RESOURCE_METADATA_FILE)
+DZ_DOMAIN_REGION=$(jq -r '.AdditionalMetadata.DataZoneDomainRegion' < $RESOURCE_METADATA_FILE)
+DZ_ENDPOINT=$(jq -r '.AdditionalMetadata.DataZoneEndpoint' < $RESOURCE_METADATA_FILE)
+DZ_PROJECT_S3PATH=$(jq -r '.AdditionalMetadata.ProjectS3Path' < $RESOURCE_METADATA_FILE)
+
+# Workflows paths in JL
+WORKFLOW_DAG_PATH="${PROJECT_DIR}/workflows/dags"
+WORKFLOW_CONFIG_PATH="${PROJECT_DIR}/workflows/config"
+WORKFLOW_DB_DATA_PATH="$HOME/.workflows_setup/db-data"
+WORKFLOW_REQUIREMENTS_PATH="$HOME/.workflows_setup/requirements/"
+WORKFLOW_PLUGINS_PATH="$HOME/.workflows_setup/plugins"
+WORKFLOW_STARTUP_PATH="$HOME/.workflows_setup/startup/"
+WORKFLOW_ARTIFACTS_SOURCE_DIR="/etc/sagemaker-ui/workflows"
+WORKFLOW_PLUGINS_SOURCE_PATH="${WORKFLOW_ARTIFACTS_SOURCE_DIR}/plugins/*.whl"
+WORKFLOW_REQUIREMENTS_SOURCE_PATH="${WORKFLOW_ARTIFACTS_SOURCE_DIR}/requirements/requirements.txt"
+WORKFLOW_AIRFLOW_REQUIREMENTS_SOURCE_PATH="/etc/sagemaker-ui/workflows/requirements/requirements.txt"
+WORKFLOW_OUTPUT_PATH="$HOME/jobs"
+USER_REQUIREMENTS_FILE="${WORKFLOW_CONFIG_PATH}/requirements.txt"
+USER_PLUGINS_FOLDER="${WORKFLOW_CONFIG_PATH}/plugins"
+USER_STARTUP_FILE="${WORKFLOW_CONFIG_PATH}/startup.sh"
+
+
 handle_workflows_startup_error() {
     local step=$1
     local detailed_status=""
@@ -33,34 +70,8 @@ handle_workflows_startup_error() {
     exit 1
 }
 
-RESOURCE_METADATA_FILE=/opt/ml/metadata/resource-metadata.json
-SM_DOMAIN_ID=$(jq -r ".DomainId" < $RESOURCE_METADATA_FILE)
-HOME_FOLDER_NAME=src
-AWS_ACCOUNT_ID=$(jq -r '.ExecutionRoleArn | split(":")[4]' < $RESOURCE_METADATA_FILE)
-ECR_ACCOUNT_ID=058264401727
-DZ_DOMAIN_ID=$(jq -r '.AdditionalMetadata.DataZoneDomainId' < $RESOURCE_METADATA_FILE)
-DZ_PROJECT_ID=$(jq -r '.AdditionalMetadata.DataZoneProjectId' < $RESOURCE_METADATA_FILE)
-DZ_ENV_ID=$(jq -r '.AdditionalMetadata.DataZoneEnvironmentId' < $RESOURCE_METADATA_FILE)
-DZ_DOMAIN_REGION=$(jq -r '.AdditionalMetadata.DataZoneDomainRegion' < $RESOURCE_METADATA_FILE)
-DZ_ENDPOINT=$(jq -r '.AdditionalMetadata.DataZoneEndpoint' < $RESOURCE_METADATA_FILE)
-DZ_PROJECT_S3PATH=$(jq -r '.AdditionalMetadata.ProjectS3Path' < $RESOURCE_METADATA_FILE)
-WORKFLOW_DAG_PATH="/home/sagemaker-user/${HOME_FOLDER_NAME}/workflows/dags"
-WORKFLOW_CONFIG_PATH="/home/sagemaker-user/${HOME_FOLDER_NAME}/workflows/config"
-WORKFLOW_DB_DATA_PATH="/home/sagemaker-user/.workflows_setup/db-data"
-WORKFLOW_REQUIREMENTS_PATH="/home/sagemaker-user/.workflows_setup/requirements/"
-WORKFLOW_PLUGINS_PATH="/home/sagemaker-user/.workflows_setup/plugins"
-WORKFLOW_STARTUP_PATH="/home/sagemaker-user/.workflows_setup/startup/"
-WORKFLOW_ARTIFACTS_SOURCE_DIR="/etc/sagemaker-ui/workflows"
-WORKFLOW_PLUGINS_SOURCE_PATH="${WORKFLOW_ARTIFACTS_SOURCE_DIR}/plugins/*.whl"
-WORKFLOW_REQUIREMENTS_SOURCE_PATH="${WORKFLOW_ARTIFACTS_SOURCE_DIR}/requirements/requirements.txt"
-WORKFLOW_AIRFLOW_REQUIREMENTS_SOURCE_PATH="/etc/sagemaker-ui/workflows/requirements/requirements.txt"
-WORKFLOW_OUTPUT_PATH="/home/sagemaker-user/jobs"
-USER_REQUIREMENTS_FILE="${WORKFLOW_CONFIG_PATH}/requirements.txt"
-USER_PLUGINS_FOLDER="${WORKFLOW_CONFIG_PATH}/plugins"
-USER_STARTUP_FILE="${WORKFLOW_CONFIG_PATH}/startup.sh"
-
 # Create status log file if it doesn't exist
-WORKFLOW_HEALTH_PATH="/home/sagemaker-user/.workflows_setup/health"
+WORKFLOW_HEALTH_PATH="$HOME/.workflows_setup/health"
 mkdir -p $WORKFLOW_HEALTH_PATH
 if [ ! -f "${WORKFLOW_HEALTH_PATH}/status.json" ]; then
     echo "Creating status file"
@@ -171,7 +182,8 @@ cp -n "/etc/sagemaker-ui/workflows/sample_dag.py" "${WORKFLOW_DAG_PATH}/"
 # Log into ECR repository
 aws ecr get-login-password --region ${AWS_REGION} | docker login --username AWS --password-stdin ${ECR_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com
 
-HOME_FOLDER_NAME=$HOME_FOLDER_NAME \
+PROJECT_DIR=$(basename $PROJECT_DIR)  \
+MOUNT_DIR=$MOUNT_DIR \
 ECR_ACCOUNT_ID=$ECR_ACCOUNT_ID \
 ACCOUNT_ID=$AWS_ACCOUNT_ID \
 DZ_DOMAIN_ID=$DZ_DOMAIN_ID \

template/v2/dirs/etc/sagemaker-ui/workflows/plugins/amzn_sagemaker_studio-1.0.9-py3-none-any.whl renamed to template/v2/dirs/etc/sagemaker-ui/workflows/plugins/amzn_sagemaker_studio-1.0.15-py3-none-any.whl

@@ -1,9 +1,9 @@
 #!/bin/bash
 set -eux
-
+PROJECT_DIR=${SMUS_PROJECT_DIR:-"$HOME/src"}
 # Check if the .libs.json file exists
-if [ -e $1/.libs.json ]; then
-  lib_config_json=`cat $1/.libs.json`
+if [ -e $PROJECT_DIR/.libs.json ]; then
+  lib_config_json=`cat $PROJECT_DIR/.libs.json`
 
   apply_change_to_space=`echo $lib_config_json | jq -r '.ApplyChangeToSpace'`
   # Extract conda channels from the config, add `-c ` before each channel and join the strings

template/v2/dirs/etc/sagemaker-ui/workflows/start-workflows-container.sh

@@ -62,6 +62,10 @@ echo "Successfully removed the ~/.aws/config file"
 aws configure set credential_source EcsContainer
 echo "Successfully configured default profile"
 
+# Add region configuration using REGION_NAME environment variable
+aws configure set region "${REGION_NAME}"
+echo "Successfully configured region to ${REGION_NAME}"
+
 # add SparkMonitor and Connection Magic entrypoint
 NB_USER=sagemaker-user
 
@@ -101,7 +105,6 @@ else
         echo "Successfully configured DomainExecutionRoleCreds profile"
 fi
 
-
 # Run AWS CLI command to get the username from DataZone User Profile.
 if [ ! -z "$dataZoneEndPoint" ]; then
     response=$( aws datazone get-user-profile --endpoint-url "$dataZoneEndPoint" --domain-identifier "$dataZoneDomainId" --user-identifier "$dataZoneUserId" --region "$dataZoneDomainRegion" )
@@ -135,8 +138,8 @@ case "$auth_mode" in
         ;;
 esac
 
-# Checks if the project is using Git or Non-Git storage
-is_non_git_storage() {
+# Checks if the project is using Git or S3 storage
+is_s3_storage() {
   getProjectDefaultEnvResponse=$(sagemaker-studio project get-project-default-environment --domain-id "$dataZoneDomainId" --project-id "$dataZoneProjectId" --profile DomainExecutionRoleCreds)
   gitConnectionArn=$(echo "$getProjectDefaultEnvResponse" | jq -r '.provisionedResources[] | select(.name=="gitConnectionArn") | .value')
   codeRepositoryName=$(echo "$getProjectDefaultEnvResponse" | jq -r '.provisionedResources[] | select(.name=="codeRepositoryName") | .value')
@@ -150,7 +153,26 @@ is_non_git_storage() {
 
 echo "Checking Project Storage Type"
 
-if ! is_non_git_storage; then
+# Execute once to store the result
+is_s3_storage
+is_s3_storage_flag=$?  # 0 if S3 storage, 1 if Git
+
+if [ "$is_s3_storage_flag" -eq 0 ]; then
+    export SMUS_PROJECT_DIR="$HOME/shared"
+    echo "Project is using S3 storage, project directory set to: $SMUS_PROJECT_DIR"
+else
+    export SMUS_PROJECT_DIR="$HOME/src"
+    echo "Project is using Git storage, project directory set to: $SMUS_PROJECT_DIR"
+fi
+
+if grep -q "^SMUS_PROJECT_DIR=" ~/.bashrc; then
+  echo "SMUS_PROJECT_DIR is defined in the env"
+else
+  echo SMUS_PROJECT_DIR="$SMUS_PROJECT_DIR" >> ~/.bashrc
+  echo readonly SMUS_PROJECT_DIR >> ~/.bashrc
+fi
+
+if [ $is_s3_storage_flag -ne 0 ]; then
   # Creating a directory where the repository will be cloned
   mkdir -p "$HOME/src"
 
@@ -191,7 +213,7 @@ if [ "${SAGEMAKER_APP_TYPE_LOWERCASE}" = "jupyterlab" ]; then
     trap 'write_status_to_file "error" "An unexpected error occurred. Please stop and restart your space to retry."' ERR
 
     # Install conda and pip dependencies if lib mgmt config existing
-    bash /etc/sagemaker-ui/libmgmt/install-lib.sh $HOME/src
+    bash /etc/sagemaker-ui/libmgmt/install-lib.sh
 
     # Install sm-spark-cli
     bash /etc/sagemaker-ui/workflows/sm-spark-cli-install.sh