Making use of CloudSQL PRoxy V2 binaries and maitaining backward compatibility (#1355)

* Update the incompatibility notice to alert users not to use CloudSQL Proxy (V1) with CloudSQL MySQL 8.4.

* Making use of CloudSQL PRoxy V2 binaries and maitaining backward compatibility

* Update cloud-sql-proxy/cloud-sql-proxy.sh

Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>

---------

Co-authored-by: Dilip Godhia <[email protected]>
Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>

Author: 3 people

cloud-sql-proxy/README.md

@@ -10,19 +10,15 @@ metadata on a given Cloud SQL instance.
 
 **⚠️ IMPORTANT COMPATIBILITY NOTICE ⚠️**
 
-**Cloud SQL Proxy V1 (the version that's currently the default in this initialization action) is not fully compatible with newer database versions like MySQL 8.4 and some newer Cloud SQL features.**
+**Cloud SQL Proxy V2 script has been upgraded to use the latest binary client that is compatible with MySQL 8.4 and some newer Cloud SQL features.**
 
 Specifically:
-* **MySQL 8.4's updated security model is incompatible with Cloud SQL Proxy V1.**
-* **Enabling "Shared CA"** or **"Customer-managed CA"** features for any Cloud SQL database (including older MySQL versions and PostgreSQL) will break connectivity with Cloud SQL Proxy V1.
-
+* **MySQL 8.4 has updated its security model that were incompatible with the earlier version of this CloudSQL Proxy script.**
 
 To avoid connectivity issues, we highly recommend:
 * **Always testing your Cloud SQL Proxy configuration thoroughly** before adopting new Cloud SQL database versions or enabling advanced security features like Shared CA or Customer-managed CA.
 
-* **Consider upgrading to Cloud SQL Proxy V2** as soon as it's available. The Dataproc team is actively working on updating the underlying Cloud SQL clients in Dataproc images to use Cloud SQL Proxy V2.
-
-* This documentation will be updated once the Cloud SQL Proxy V2 is available along with the Dataproc versions where the fix is available. See [Dataproc release notes](https://cloud.google.com/dataproc/docs/release-notes) for upcoming updates.
+* The Dataproc team has updated the underlying Cloud SQL clients in Dataproc images to make use of Cloud SQL Proxy V2. The dataproc versions that are compatible with CloudSQL MySQL 8.4 are Dataproc versions 2.0.147, 2.1.96, 2.2.64 and 2.3.10 released on August 29, 2025.  See [Dataproc release notes](https://cloud.google.com/dataproc/docs/release-notes) for any new updates.
 
 ## Using this initialization action
 
@@ -71,12 +67,24 @@ shared hive metastore.
         --properties hive:hive.metastore.warehouse.dir=gs://${HIVE_DATA_BUCKET}/hive-warehouse \
         --metadata "hive-metastore-instance=${PROJECT_ID}:${REGION}:${INSTANCE_NAME}"
     ```
+    To use an earlier version of the CloudSQL Binary, add the following property to your Dataproc Cluster Creation
+    ```bash
+        --metadata "cloud_sql_proxy_version=1"
+    ```
+    Or for a specific version of the CloudSQL Binary Client, use:
+    ```bash
+        --metadata "cloud_sql_proxy_version_number=v2.17.1"
+    ```
+    You can obtain the list of all CloudSQL Proxy Client version by running the following command
+    ```
+    curl -s "https://api.github.com/repos/GoogleCloudPlatform/cloud-sql-proxy/releases"|jq -r '.[].tag_name'|grep 'v2'
+    ```
+
 
-    a. Optionally add other instances, paired with distict TCP ports for further
-    I/O.
+    a. Optionally add other instances, paired with distinct TCP ports for further I/O.
 
     ```bash
-    --metadata "additional-cloud-sql-instances=<PROJECT_ID>:<REGION>:<ANOTHER_INSTANCE_NAME>=tcp<PORT_#>[,...]"
+        --metadata "additional-cloud-sql-instances=<PROJECT_ID>:<REGION>:<ANOTHER_INSTANCE_NAME>=tcp<PORT_#>[,...]"
     ```
 
 1.  Submit pyspark_metastore_test.py to the cluster to validate the metatstore
@@ -87,7 +95,7 @@ shared hive metastore.
     ```
 
     a. You can test connections to your other instance(s) using the url
-    `"jdbc:mysql//localhost:<PORT_#>?user=root"`
+    `"jdbc:mysql//localhost:<PORT_#>?allowPublicKeyRetrieval=true&user=root"`
 
 1.  Create another dataproc cluster with the same Cloud SQL metastore.
 
@@ -177,7 +185,7 @@ additional setup.
 
     ```bash
     gcloud sql instances create <INSTANCE_NAME> \
-        --database-version="MYSQL_5_7" \
+        --database-version="MYSQL_8_0" \
         --activation-policy=ALWAYS \
         --zone <ZONE>
     ```

cloud-sql-proxy/cloud-sql-proxy.sh

@@ -144,6 +144,15 @@ readonly METASTORE_INSTANCE
 ADDITIONAL_INSTANCES="$(/usr/share/google/get_metadata_value ${ADDITIONAL_INSTANCES_KEY} || echo '')"
 readonly ADDITIONAL_INSTANCES
 
+PROXY_VERSION="$(get_metadata_attribute cloud_sql_proxy_version '2')"
+readonly PROXY_VERSION
+
+# CURRENT_PROXY_VERSION=$(curl -s "https://api.github.com/repos/GoogleCloudPlatform/cloud-sql-proxy/releases"|jq -r '.[].tag_name'|grep 'v2'| head -n 1)
+
+CURRENT_PROXY_VERSION="v2.18.1"
+OVERRIDE_PROXY_VERSION_NUMBER="$(get_metadata_attribute cloud_sql_proxy_version_number '' | tr '[:upper:]' '[:lower:]' || echo '')"
+readonly OVERRIDE_PROXY_VERSION_NUMBER
+
 function repair_old_backports {
   if ! is_debuntu ; then return ; fi
   # This script uses 'apt-get update' and is therefore potentially dependent on
@@ -209,8 +218,12 @@ fi
 readonly CLOUDSQL_INSTANCE_TYPE
 
 METASTORE_PROXY_PORT="$(/usr/share/google/get_metadata_value attributes/metastore-proxy-port || echo '')"
-if [[ "${METASTORE_INSTANCE}" =~ =tcp:[0-9]+$ ]]; then
-  METASTORE_PROXY_PORT="${METASTORE_INSTANCE##*:}"
+if [[ "${METASTORE_INSTANCE}" =~ tcp:[0-9]+$ ]]; then
+  if [[ ${PROXY_VERSION} == "1" ]]; then
+    METASTORE_PROXY_PORT="${METASTORE_INSTANCE##=*:}"
+  else
+    METASTORE_PROXY_PORT="${METASTORE_INSTANCE##?*:}"
+  fi
 else
   METASTORE_PROXY_PORT=${DEFAULT_DB_PORT["${CLOUDSQL_INSTANCE_TYPE}"]}
 fi
@@ -355,7 +368,11 @@ function run_with_retries() {
 function get_metastore_instance() {
   local metastore_instance="${METASTORE_INSTANCE}"
   if ! [[ "${metastore_instance}" =~ =tcp:[0-9]+$ ]]; then
-    metastore_instance+="=tcp:${METASTORE_PROXY_PORT}"
+    if [[ ${PROXY_VERSION} == "1" ]]; then
+      metastore_instance+="=tcp:${METASTORE_PROXY_PORT}"
+    else
+      metastore_instance+="?port=${METASTORE_PROXY_PORT}"
+    fi
   fi
   echo "${metastore_instance}"
 }
@@ -364,17 +381,29 @@ function get_proxy_flags() {
   local proxy_instances_flags=''
   # If a Cloud SQL instance has both public and private IP, use private IP.
   if [[ ${USE_CLOUD_SQL_PRIVATE_IP} == "true" ]]; then
-    proxy_instances_flags+=" --ip_address_types=PRIVATE"
+    if [[ ${PROXY_VERSION} == "1" ]]; then
+      proxy_instances_flags+=" --ip_address_types=PRIVATE"
+    else
+      proxy_instances_flags+=" --private-ip"
+    fi
   fi
   if [[ ${ENABLE_CLOUD_SQL_METASTORE} == "true" ]]; then
     local metastore_instance
     metastore_instance=$(get_metastore_instance)
-    proxy_instances_flags+=" -instances=${metastore_instance}"
+    if [[ ${PROXY_VERSION} == "1" ]]; then
+      proxy_instances_flags+=" -instances=${metastore_instance}"
+    else
+      proxy_instances_flags+=" ${metastore_instance}"
+    fi
   fi
 
   if [[ -n "${ADDITIONAL_INSTANCES}" ]]; then
     # Pass additional instances straight to the proxy.
-    proxy_instances_flags+=" -instances_metadata=instance/${ADDITIONAL_INSTANCES_KEY}"
+    if [[ ${PROXY_VERSION} == "1" ]]; then
+      proxy_instances_flags+=" -instances_metadata=instance/${ADDITIONAL_INSTANCES_KEY}"
+    else
+      proxy_instances_flags+=" instances_metadata=instance/${ADDITIONAL_INSTANCES_KEY}"
+    fi
   fi
 
   echo "${proxy_instances_flags}"
@@ -383,9 +412,24 @@ function get_proxy_flags() {
 function install_cloud_sql_proxy() {
   echo 'Installing Cloud SQL Proxy ...' >&2
   # Install proxy.
-  wget -nv --timeout=30 --tries=5 --retry-connrefused \
-    https://dl.google.com/cloudsql/cloud_sql_proxy.linux.amd64
-  mv cloud_sql_proxy.linux.amd64 ${PROXY_BIN}
+  local proxy_version
+
+  if [[ ${PROXY_VERSION} == "1" ]]; then
+    wget -nv --timeout=30 --tries=5 --retry-connrefused \
+      https://dl.google.com/cloudsql/cloud_sql_proxy.linux.amd64 -O /tmp/cloud_sql_proxy.linux.amd64 
+    echo "Using an OUTDATED version of CloudSQL Proxy V1"
+  else
+    if [[ -n ${OVERRIDE_PROXY_VERSION_NUMBER} ]]; then
+      proxy_version="${OVERRIDE_PROXY_VERSION_NUMBER}"
+    else
+      proxy_version="${CURRENT_PROXY_VERSION}"
+    fi
+    wget -nv --timeout=30 --tries=5 --retry-connrefused \
+      https://storage.googleapis.com/cloud-sql-connectors/cloud-sql-proxy/"${proxy_version}"/cloud-sql-proxy.linux.amd64 -O /tmp/cloud_sql_proxy.linux.amd64
+    echo "Using CloudSQL Proxy V2 - Version ${proxy_version}"
+  fi
+  mv /tmp/cloud_sql_proxy.linux.amd64 ${PROXY_BIN}
+
   chmod +x ${PROXY_BIN}
 
   mkdir -p ${PROXY_DIR}
@@ -401,6 +445,11 @@ function install_cloud_sql_proxy() {
   db_hive_password_xml_escaped=${db_hive_password_xml_escaped//>/>}
   db_hive_password_xml_escaped=${db_hive_password_xml_escaped//'"'/"}
 
+  dir_options=""
+  if [[ ${PROXY_VERSION} == "1" ]]; then
+    dir_options="-dir=${PROXY_DIR}"
+  fi
+
   # Install proxy as systemd service for reboot tolerance.
   cat <<EOF >${INIT_SCRIPT}
 [Unit]
@@ -411,8 +460,7 @@ Before=shutdown.target
 
 [Service]
 Type=simple
-ExecStart=/bin/sh -c '${PROXY_BIN} \
-  -dir=${PROXY_DIR} \
+ExecStart=/bin/sh -c '${PROXY_BIN} ${dir_options} \
   ${proxy_flags} >> /var/log/cloud-sql-proxy/cloud-sql-proxy.log 2>&1'
 
 [Install]
@@ -421,7 +469,7 @@ EOF
   chmod a+rw ${INIT_SCRIPT}
 
   if [[ $ENABLE_CLOUD_SQL_METASTORE == "true" ]]; then
-    local db_url=jdbc:${DEFAULT_DB_PROTO["${CLOUDSQL_INSTANCE_TYPE}"]}://localhost:${METASTORE_PROXY_PORT}/${METASTORE_DB}
+    local db_url=jdbc:${DEFAULT_DB_PROTO["${CLOUDSQL_INSTANCE_TYPE}"]}://localhost:${METASTORE_PROXY_PORT}/${METASTORE_DB}?allowPublicKeyRetrieval=true
     local db_driver=${DEFAULT_DB_DRIVER["${CLOUDSQL_INSTANCE_TYPE}"]}
 
     # Update hive-site.xml
@@ -469,13 +517,13 @@ function initialize_mysql_metastore_db() {
   fi
 
   # Check if metastore is initialized.
-  if ! mysql -h 127.0.0.1 -P "${METASTORE_PROXY_PORT}" -u "${DB_HIVE_USER}" "${db_hive_password_param}" -e ''; then
-    mysql -h 127.0.0.1 -P "${METASTORE_PROXY_PORT}" -u "${DB_ADMIN_USER}" "${db_password_param}" -e \
+  if ! mysql -h 127.0.0.1 -P "${METASTORE_PROXY_PORT}" -u "${DB_HIVE_USER}" "${db_hive_password_param}" --get-server-public-key -e ''; then
+    mysql -h 127.0.0.1 -P "${METASTORE_PROXY_PORT}" -u "${DB_ADMIN_USER}" "${db_password_param}" --get-server-public-key -e \
       "CREATE USER '${DB_HIVE_USER}' IDENTIFIED BY '${DB_HIVE_PASSWORD}';"
   fi
-  if ! mysql -h 127.0.0.1 -P "${METASTORE_PROXY_PORT}" -u "${DB_HIVE_USER}" "${db_hive_password_param}" -e "use ${METASTORE_DB}"; then
+  if ! mysql -h 127.0.0.1 -P "${METASTORE_PROXY_PORT}" -u "${DB_HIVE_USER}" "${db_hive_password_param}" --get-server-public-key -e "use ${METASTORE_DB}"; then
     # Initialize a Hive metastore DB
-    mysql -h 127.0.0.1 -P "${METASTORE_PROXY_PORT}" -u "${DB_ADMIN_USER}" "${db_password_param}" -e \
+    mysql -h 127.0.0.1 -P "${METASTORE_PROXY_PORT}" -u "${DB_ADMIN_USER}" "${db_password_param}" --get-server-public-key -e \
       "CREATE DATABASE ${METASTORE_DB};
        GRANT ALL PRIVILEGES ON ${METASTORE_DB}.* TO '${DB_HIVE_USER}';"
     /usr/lib/hive/bin/schematool -dbType mysql -initSchema ||