From 40cdc862879b3f8e062fed974cfcb3270dccca11 Mon Sep 17 00:00:00 2001 From: Holt Skinner Date: Fri, 30 May 2025 15:15:06 -0400 Subject: [PATCH 1/8] Update Script to use `--async` --- .cloud-build/executor/test_notebooks.sh | 39 +++++++++++++------------ 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/.cloud-build/executor/test_notebooks.sh b/.cloud-build/executor/test_notebooks.sh index ef445c3c3b6..7175452c7d6 100755 --- a/.cloud-build/executor/test_notebooks.sh +++ b/.cloud-build/executor/test_notebooks.sh @@ -21,12 +21,14 @@ total_count=0 successful_notebooks=() successful_count=0 +declare -A notebook_ops + for x in $TARGET; do total_count=$((total_count + 1)) # Use the full path from the repository for display name DISPLAY_NAME="${x##generative-ai/}" DISPLAY_NAME="${DISPLAY_NAME%.ipynb}-$current_date-$current_time" - echo "Starting execution for ${x}" + echo "Launching async execution for ${x}" # Execute and get the operation ID OPERATION_ID=$(gcloud colab executions create \ @@ -37,37 +39,36 @@ for x in $TARGET; do --project="$PROJECT_ID" \ --region="$REGION" \ --service-account="$SA" \ - --verbosity=debug \ --execution-timeout="1h30m" \ - --format="value(name)") + --format="value(name)" \ + --async) echo "Operation ID: $OPERATION_ID" TRUNCATED_OPERATION_ID=$(echo "$OPERATION_ID" | cut -c 67-85) + notebook_ops["$TRUNCATED_OPERATION_ID"]="$x" +done - # check job status - echo "Waiting for execution to complete..." - if ! EXECUTION_DETAILS=$(gcloud colab executions describe "$TRUNCATED_OPERATION_ID" --region="$REGION"); then - echo "Error describing execution for ${x}. See logs for details." - failed_count=$((failed_count + 1)) - failed_notebooks+=("${x}") - continue - else - echo "Execution completed for ${x}" - fi +# Wait and collect results +for op_id in "${!notebook_ops[@]}"; do + notebook="${notebook_ops[$op_id]}" + echo "Checking status for $notebook (operation ID: $op_id)" + + until EXECUTION_DETAILS=$(gcloud colab executions describe "$op_id" --region="$REGION" 2>/dev/null); do + echo "Waiting for operation $op_id..." + sleep 10 + done # Check the jobState JOB_STATE=$(echo "$EXECUTION_DETAILS" | grep "jobState:" | awk '{print $2}') if [[ "$JOB_STATE" == "JOB_STATE_SUCCEEDED" ]]; then - echo "Notebook execution succeeded." + echo "Notebook $notebook succeeded." successful_count=$((successful_count + 1)) - successful_notebooks+=("${x}") + successful_notebooks+=("$notebook") else - echo "Notebook execution failed. Job state: $JOB_STATE. Please use id $TRUNCATED_OPERATION_ID to troubleshoot notebook ${x}. See log for details." + echo "Notebook $notebook failed. Job state: $JOB_STATE" failed_count=$((failed_count + 1)) - failed_notebooks+=("${x}") - continue + failed_notebooks+=("$notebook") fi - done # Print the final list of failed notebooks From b7a686558cd6d6d6390b0bcf4f75fab6218f43ee Mon Sep 17 00:00:00 2001 From: Holt Skinner Date: Fri, 30 May 2025 15:15:43 -0400 Subject: [PATCH 2/8] Re-add debug verbosity --- .cloud-build/executor/test_notebooks.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.cloud-build/executor/test_notebooks.sh b/.cloud-build/executor/test_notebooks.sh index 7175452c7d6..2da535d450e 100755 --- a/.cloud-build/executor/test_notebooks.sh +++ b/.cloud-build/executor/test_notebooks.sh @@ -39,6 +39,7 @@ for x in $TARGET; do --project="$PROJECT_ID" \ --region="$REGION" \ --service-account="$SA" \ + --verbosity=debug \ --execution-timeout="1h30m" \ --format="value(name)" \ --async) From 93b6a5d532387ff5799ea444c7af826ff18111b4 Mon Sep 17 00:00:00 2001 From: Holt Skinner Date: Fri, 30 May 2025 15:25:44 -0400 Subject: [PATCH 3/8] Change checks to use job status --- .cloud-build/executor/test_notebooks.sh | 112 ++++++++++++++++-------- 1 file changed, 77 insertions(+), 35 deletions(-) diff --git a/.cloud-build/executor/test_notebooks.sh b/.cloud-build/executor/test_notebooks.sh index 2da535d450e..c0c812dce1a 100755 --- a/.cloud-build/executor/test_notebooks.sh +++ b/.cloud-build/executor/test_notebooks.sh @@ -21,17 +21,20 @@ total_count=0 successful_notebooks=() successful_count=0 -declare -A notebook_ops +# Array to store "OPERATION_ID:NOTEBOOK_PATH" for polling +launched_operation_details=() + +echo "Starting to launch notebook executions..." for x in $TARGET; do total_count=$((total_count + 1)) # Use the full path from the repository for display name DISPLAY_NAME="${x##generative-ai/}" DISPLAY_NAME="${DISPLAY_NAME%.ipynb}-$current_date-$current_time" - echo "Launching async execution for ${x}" + echo "Launching execution for ${x}" - # Execute and get the operation ID - OPERATION_ID=$(gcloud colab executions create \ + # Execute asynchronously and get the full operation name + OPERATION_ID_FULL=$(gcloud colab executions create \ --display-name="$DISPLAY_NAME" \ --notebook-runtime-template="$NOTEBOOK_RUNTIME_TEMPLATE" \ --direct-content="$x" \ @@ -41,37 +44,83 @@ for x in $TARGET; do --service-account="$SA" \ --verbosity=debug \ --execution-timeout="1h30m" \ - --format="value(name)" \ - --async) + --async \ + --format="value(name)") - echo "Operation ID: $OPERATION_ID" - TRUNCATED_OPERATION_ID=$(echo "$OPERATION_ID" | cut -c 67-85) - notebook_ops["$TRUNCATED_OPERATION_ID"]="$x" -done + if [[ -z "$OPERATION_ID_FULL" ]]; then + echo "Error: Failed to create execution for ${x}. No operation ID returned." + failed_count=$((failed_count + 1)) + failed_notebooks+=("${x} (failed to launch)") + continue + fi -# Wait and collect results -for op_id in "${!notebook_ops[@]}"; do - notebook="${notebook_ops[$op_id]}" - echo "Checking status for $notebook (operation ID: $op_id)" + # Extract the execution ID (last part of the full operation name) + TRUNCATED_OPERATION_ID=$(basename "$OPERATION_ID_FULL") - until EXECUTION_DETAILS=$(gcloud colab executions describe "$op_id" --region="$REGION" 2>/dev/null); do - echo "Waiting for operation $op_id..." - sleep 10 + echo "Launched execution for ${x}, Operation ID: $TRUNCATED_OPERATION_ID (Full Name: $OPERATION_ID_FULL)" + launched_operation_details+=("$TRUNCATED_OPERATION_ID:${x}") +done + +echo "All notebook executions launched. Now waiting for completion..." + +pending_operations=("${launched_operation_details[@]}") + +while [[ ${#pending_operations[@]} -gt 0 ]]; do + echo "Number of pending operations: ${#pending_operations[@]}" + still_pending_next_round=() # Operations that are still PENDING/RUNNING for the next iteration + + for op_info in "${pending_operations[@]}"; do + TRUNCATED_OPERATION_ID="${op_info%%:*}" + NOTEBOOK_PATH="${op_info#*:}" + + echo "Checking status for operation: $TRUNCATED_OPERATION_ID (Notebook: $NOTEBOOK_PATH)" + + EXECUTION_DETAILS="" + # Describe the execution. Added --project for explicitness. + if ! EXECUTION_DETAILS=$(gcloud colab executions describe "$TRUNCATED_OPERATION_ID" --project="$PROJECT_ID" --region="$REGION" 2>&1); then + echo "Error describing execution for $TRUNCATED_OPERATION_ID (Notebook: $NOTEBOOK_PATH). Marking as failed." + echo "Describe error details: $EXECUTION_DETAILS" + failed_count=$((failed_count + 1)) + failed_notebooks+=("${NOTEBOOK_PATH} (describe error)") + continue # Handled (as failed), move to the next operation in this polling round + fi + + # Parse the top-level 'state:' field from the describe output + EXECUTION_STATE=$(echo "$EXECUTION_DETAILS" | grep -E "^state:" | awk '{print $2}') + + echo "Notebook: $NOTEBOOK_PATH, Operation: $TRUNCATED_OPERATION_ID, Execution State: $EXECUTION_STATE" + + if [[ "$EXECUTION_STATE" == "SUCCEEDED" ]]; then + echo "Notebook execution succeeded for $NOTEBOOK_PATH." + successful_count=$((successful_count + 1)) + successful_notebooks+=("${NOTEBOOK_PATH}") + elif [[ "$EXECUTION_STATE" == "FAILED" || "$EXECUTION_STATE" == "CANCELLED" || "$EXECUTION_STATE" == "EXPIRED" ]]; then + echo "Notebook execution failed, was cancelled, or expired for $NOTEBOOK_PATH. State: $EXECUTION_STATE. Please use id $TRUNCATED_OPERATION_ID to troubleshoot." + failed_count=$((failed_count + 1)) + failed_notebooks+=("${NOTEBOOK_PATH}") + elif [[ "$EXECUTION_STATE" == "PENDING" || "$EXECUTION_STATE" == "RUNNING" || "$EXECUTION_STATE" == "INITIALIZING" ]]; then + echo "Notebook execution for $NOTEBOOK_PATH is still in progress (State: $EXECUTION_STATE). Will check again." + still_pending_next_round+=("$op_info") + else + # Handle unknown or unexpected states + echo "Unknown execution state '$EXECUTION_STATE' for $NOTEBOOK_PATH (Operation: $TRUNCATED_OPERATION_ID). Marking as failed." + echo "Full describe output for unknown state: $EXECUTION_DETAILS" + failed_count=$((failed_count + 1)) + failed_notebooks+=("${NOTEBOOK_PATH} (unknown state: $EXECUTION_STATE)") + fi done - # Check the jobState - JOB_STATE=$(echo "$EXECUTION_DETAILS" | grep "jobState:" | awk '{print $2}') - if [[ "$JOB_STATE" == "JOB_STATE_SUCCEEDED" ]]; then - echo "Notebook $notebook succeeded." - successful_count=$((successful_count + 1)) - successful_notebooks+=("$notebook") - else - echo "Notebook $notebook failed. Job state: $JOB_STATE" - failed_count=$((failed_count + 1)) - failed_notebooks+=("$notebook") + pending_operations=("${still_pending_next_round[@]}") + + if [[ ${#pending_operations[@]} -gt 0 ]]; then + SLEEP_DURATION=30 # seconds + echo "Waiting for $SLEEP_DURATION seconds before next check for ${#pending_operations[@]} pending operations..." + sleep $SLEEP_DURATION fi done +echo "All executions have completed processing." + # Print the final list of failed notebooks if [[ ${#failed_notebooks[@]} -gt 0 ]]; then echo "Failed Notebooks:" @@ -94,12 +143,6 @@ failed_notebooks_str=$( echo "${failed_notebooks[*]}" ) -# prep notebook name for pub/sub message -failed_notebooks_str=$( - IFS=', ' - echo "${failed_notebooks[*]}" -) - if [[ -n "$failed_notebooks_str" ]]; then IFS=',' read -ra failed_notebooks_array <<<"$failed_notebooks_str" trimmed_notebooks=() @@ -140,7 +183,6 @@ message_data="{\"total_count\":$((total_count + 0)),\"failed_count\":$((failed_c echo "$(date) - INFO - Publishing to Pub/Sub topic: $PUBSUB_TOPIC" if ! gcloud pubsub topics publish "$PUBSUB_TOPIC" --message="$message_data" --project="$PROJECT_ID"; then echo "$(date) - ERROR - Failed to publish to Pub/Sub topic $PUBSUB_TOPIC. Check permissions and topic configuration." - #exit 1 fi -echo "All notebook executions completed." +echo "All notebook test processing and reporting completed." From 7f288243fdd6f62db7b7abe28900c875ff62ca58 Mon Sep 17 00:00:00 2001 From: Holt Skinner Date: Fri, 30 May 2025 15:46:56 -0400 Subject: [PATCH 4/8] Add back notebook list --- .cloud-build/Notebooks.txt | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/.cloud-build/Notebooks.txt b/.cloud-build/Notebooks.txt index de971ff1399..a32e01436c4 100644 --- a/.cloud-build/Notebooks.txt +++ b/.cloud-build/Notebooks.txt @@ -1,2 +1,33 @@ +generative-ai/audio/speech/getting-started/get_started_with_chirp_3_hd_voices.ipynb +generative-ai/embeddings/intro_multimodal_embeddings.ipynb +generative-ai/embeddings/task-type-embedding.ipynb generative-ai/gemini/getting-started/intro_gemini_2_0_flash.ipynb generative-ai/gemini/getting-started/intro_gemini_2_0_flash_lite.ipynb +generative-ai/gemini/getting-started/intro_gemini_2_0_image_gen.ipynb +generative-ai/gemini/getting-started/intro_gemini_2_5_flash.ipynb +generative-ai/gemini/getting-started/intro_gemini_2_5_pro.ipynb +generative-ai/gemini/getting-started/intro_gemini_chat.ipynb +generative-ai/gemini/getting-started/intro_gemini_curl.ipynb +generative-ai/gemini/chat-completions/intro_chat_completions_api.ipynb +generative-ai/gemini/code-execution/intro_code_execution.ipynb +generative-ai/gemini/context-caching/intro_context_caching.ipynb +generative-ai/gemini/controlled-generation/intro_controlled_generation.ipynb +generative-ai/gemini/function-calling/intro_function_calling.ipynb +generative-ai/gemini/global-endpoint/intro_global_endpoint.ipynb +generative-ai/gemini/model-optimizer/intro_model_optimizer.ipynb +generative-ai/gemini/multimodal-live-api/intro_multimodal_live_api_genai_sdk.ipynb +generative-ai/gemini/orchestration/intro_langgraph_gemini.ipynb +generative-ai/gemini/prompts/intro_prompt_design.ipynb +generative-ai/gemini/prompts/examples/chain_of_thought_react.ipynb +generative-ai/gemini/prompts/examples/ideation.ipynb +generative-ai/gemini/prompts/examples/question_answering.ipynb +generative-ai/gemini/prompts/examples/text_classification.ipynb +generative-ai/gemini/prompts/examples/text_extraction.ipynb +generative-ai/gemini/prompts/examples/text_summarization.ipynb +generative-ai/gemini/responsible-ai/gemini_prompt_attacks_mitigation_examples.ipynb +generative-ai/gemini/responsible-ai/gemini_safety_ratings.ipynb +generative-ai/gemini/use-cases/document-processing/document_processing.ipynb +generative-ai/gemini/use-cases/education/use_cases_for_education.ipynb +generative-ai/gemini/use-cases/spatial-understanding/spatial_understanding.ipynb +generative-ai/gemini/use-cases/intro_multimodal_use_cases.ipynb +generative-ai/vision/getting-started/imagen3_image_generation.ipynb From 5c7c0e92c6b7125e95637d9e41aec047c3de9cf8 Mon Sep 17 00:00:00 2001 From: Holt Skinner Date: Fri, 30 May 2025 15:48:07 -0400 Subject: [PATCH 5/8] Change Imagen3 to imagen4 --- .cloud-build/Notebooks.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cloud-build/Notebooks.txt b/.cloud-build/Notebooks.txt index a32e01436c4..c2e9e4f9b80 100644 --- a/.cloud-build/Notebooks.txt +++ b/.cloud-build/Notebooks.txt @@ -30,4 +30,4 @@ generative-ai/gemini/use-cases/document-processing/document_processing.ipynb generative-ai/gemini/use-cases/education/use_cases_for_education.ipynb generative-ai/gemini/use-cases/spatial-understanding/spatial_understanding.ipynb generative-ai/gemini/use-cases/intro_multimodal_use_cases.ipynb -generative-ai/vision/getting-started/imagen3_image_generation.ipynb +generative-ai/vision/getting-started/imagen4_image_generation.ipynb From 78ea9c3e7860ab2d0247aa298bb1dd2640772330 Mon Sep 17 00:00:00 2001 From: Holt Skinner Date: Fri, 30 May 2025 15:56:32 -0400 Subject: [PATCH 6/8] Change back to short list of notebooks --- .cloud-build/Notebooks.txt | 31 ------------------------------- 1 file changed, 31 deletions(-) diff --git a/.cloud-build/Notebooks.txt b/.cloud-build/Notebooks.txt index c2e9e4f9b80..de971ff1399 100644 --- a/.cloud-build/Notebooks.txt +++ b/.cloud-build/Notebooks.txt @@ -1,33 +1,2 @@ -generative-ai/audio/speech/getting-started/get_started_with_chirp_3_hd_voices.ipynb -generative-ai/embeddings/intro_multimodal_embeddings.ipynb -generative-ai/embeddings/task-type-embedding.ipynb generative-ai/gemini/getting-started/intro_gemini_2_0_flash.ipynb generative-ai/gemini/getting-started/intro_gemini_2_0_flash_lite.ipynb -generative-ai/gemini/getting-started/intro_gemini_2_0_image_gen.ipynb -generative-ai/gemini/getting-started/intro_gemini_2_5_flash.ipynb -generative-ai/gemini/getting-started/intro_gemini_2_5_pro.ipynb -generative-ai/gemini/getting-started/intro_gemini_chat.ipynb -generative-ai/gemini/getting-started/intro_gemini_curl.ipynb -generative-ai/gemini/chat-completions/intro_chat_completions_api.ipynb -generative-ai/gemini/code-execution/intro_code_execution.ipynb -generative-ai/gemini/context-caching/intro_context_caching.ipynb -generative-ai/gemini/controlled-generation/intro_controlled_generation.ipynb -generative-ai/gemini/function-calling/intro_function_calling.ipynb -generative-ai/gemini/global-endpoint/intro_global_endpoint.ipynb -generative-ai/gemini/model-optimizer/intro_model_optimizer.ipynb -generative-ai/gemini/multimodal-live-api/intro_multimodal_live_api_genai_sdk.ipynb -generative-ai/gemini/orchestration/intro_langgraph_gemini.ipynb -generative-ai/gemini/prompts/intro_prompt_design.ipynb -generative-ai/gemini/prompts/examples/chain_of_thought_react.ipynb -generative-ai/gemini/prompts/examples/ideation.ipynb -generative-ai/gemini/prompts/examples/question_answering.ipynb -generative-ai/gemini/prompts/examples/text_classification.ipynb -generative-ai/gemini/prompts/examples/text_extraction.ipynb -generative-ai/gemini/prompts/examples/text_summarization.ipynb -generative-ai/gemini/responsible-ai/gemini_prompt_attacks_mitigation_examples.ipynb -generative-ai/gemini/responsible-ai/gemini_safety_ratings.ipynb -generative-ai/gemini/use-cases/document-processing/document_processing.ipynb -generative-ai/gemini/use-cases/education/use_cases_for_education.ipynb -generative-ai/gemini/use-cases/spatial-understanding/spatial_understanding.ipynb -generative-ai/gemini/use-cases/intro_multimodal_use_cases.ipynb -generative-ai/vision/getting-started/imagen4_image_generation.ipynb From 715d12dd1f0e1e0822dd301fb02e7370442f18ad Mon Sep 17 00:00:00 2001 From: Holt Skinner Date: Fri, 30 May 2025 16:02:43 -0400 Subject: [PATCH 7/8] Change truncated Operation ID back --- .cloud-build/executor/test_notebooks.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.cloud-build/executor/test_notebooks.sh b/.cloud-build/executor/test_notebooks.sh index c0c812dce1a..55b5ac9c55b 100755 --- a/.cloud-build/executor/test_notebooks.sh +++ b/.cloud-build/executor/test_notebooks.sh @@ -54,8 +54,7 @@ for x in $TARGET; do continue fi - # Extract the execution ID (last part of the full operation name) - TRUNCATED_OPERATION_ID=$(basename "$OPERATION_ID_FULL") + TRUNCATED_OPERATION_ID=$(echo "$OPERATION_ID" | cut -c 67-85) echo "Launched execution for ${x}, Operation ID: $TRUNCATED_OPERATION_ID (Full Name: $OPERATION_ID_FULL)" launched_operation_details+=("$TRUNCATED_OPERATION_ID:${x}") From 0ad05a7eb17c579cb920f3d01dd76855b26f35dd Mon Sep 17 00:00:00 2001 From: Holt Skinner Date: Fri, 30 May 2025 16:10:02 -0400 Subject: [PATCH 8/8] Fix variable name --- .cloud-build/executor/test_notebooks.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cloud-build/executor/test_notebooks.sh b/.cloud-build/executor/test_notebooks.sh index 55b5ac9c55b..00c2d80969c 100755 --- a/.cloud-build/executor/test_notebooks.sh +++ b/.cloud-build/executor/test_notebooks.sh @@ -54,7 +54,7 @@ for x in $TARGET; do continue fi - TRUNCATED_OPERATION_ID=$(echo "$OPERATION_ID" | cut -c 67-85) + TRUNCATED_OPERATION_ID=$(echo "$OPERATION_ID_FULL" | cut -c 67-85) echo "Launched execution for ${x}, Operation ID: $TRUNCATED_OPERATION_ID (Full Name: $OPERATION_ID_FULL)" launched_operation_details+=("$TRUNCATED_OPERATION_ID:${x}")