Update to Procfile, Artifact Registry, dedicated service account

Author: glasnt

run/django/Dockerfile

@@ -0,0 +1,5 @@
+# Default entrypoint: run Django
+web: gunicorn --bind 0.0.0.0:$PORT --workers 1 --threads 8 --timeout 0 mysite.wsgi:application
+
+# Apply database migrations
+migrate: python manage.py migrate && python manage.py collectstatic --verbosity 2 --no-input

run/django/Procfile

@@ -30,7 +30,6 @@
 SAMPLE_VERSION = os.environ.get("SAMPLE_VERSION", None)
 GOOGLE_CLOUD_PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"]
 REGION = "us-central1"
-PLATFORM = "managed"
 
 SERVICE = f"polls-{SUFFIX}"
 
@@ -99,7 +98,6 @@ def run_shell_cmd(args: list) -> subprocess.CompletedProcess:
 def deployed_service() -> str:
     substitutions = [
         f"_SERVICE={SERVICE},"
-        f"_PLATFORM={PLATFORM},"
         f"_REGION={REGION},"
         f"_STORAGE_BUCKET={CLOUD_STORAGE_BUCKET},"
         f"_DB_NAME={POSTGRES_DATABASE},"
@@ -134,7 +132,6 @@ def deployed_service() -> str:
 
     substitutions = [
         f"_SERVICE={SERVICE},"
-        f"_PLATFORM={PLATFORM},"
         f"_REGION={REGION},"
         f"_DB_USER={POSTGRES_USER},"
         f"_DB_NAME={POSTGRES_DATABASE},"
@@ -172,8 +169,6 @@ def service_url_auth_token(deployed_service: str) -> Iterator[tuple[str, str]]:
                 "services",
                 "describe",
                 deployed_service,
-                "--platform",
-                "managed",
                 "--region",
                 REGION,
                 "--format",

run/django/e2e_test.py

@@ -28,26 +28,34 @@ steps:
         ./retry.sh "gsutil ls gs://${_STORAGE_BUCKET}" \
           "gsutil -m rm -r gs://${_STORAGE_BUCKET}"
 
-        ./retry.sh "gcloud container images describe gcr.io/${PROJECT_ID}/${_SERVICE}:${_VERSION}" \
-          "gcloud container images delete gcr.io/${PROJECT_ID}/${_SERVICE}:${_VERSION} --quiet"
+        ./retry.sh "gcloud artifacts docker images describe ${_IMAGE_NAME}" \
+          "gcloud artifacts docker images delete ${_IMAGE_NAME} --quiet"
 
-        ./retry.sh "gcloud run services describe ${_SERVICE} --region ${_REGION} --platform ${_PLATFORM}" \
-          "gcloud run services delete ${_SERVICE} --region ${_REGION} --platform ${_PLATFORM} --async --quiet"
+        ./retry.sh "gcloud run jobs describe ${_CLOUD_RUN_JOB_NAME} --region ${_REGION}" \
+          "gcloud run jobs delete ${_CLOUD_RUN_JOB_NAME} --region ${_REGION} --async --quiet"
+
+        ./retry.sh "gcloud run services describe ${_SERVICE} --region ${_REGION}" \
+          "gcloud run services delete ${_SERVICE} --region ${_REGION} --async --quiet"
 
         WAIT=30 ./retry.sh "gcloud sql databases describe ${_DB_NAME} --instance ${_DB_INSTANCE} --project $PROJECT_ID" \
           "gcloud sql databases delete ${_DB_NAME} --instance ${_DB_INSTANCE} --quiet --project $PROJECT_ID"
 
         ./retry.sh "gcloud sql users list --filter \"name=${_DB_USER}\" --instance ${_DB_INSTANCE}" \
           "gcloud sql users delete ${_DB_USER} --instance ${_DB_INSTANCE} --quiet --project $PROJECT_ID"
 
+options:
+  dynamicSubstitutions: true
+
 substitutions:
   _SERVICE: django
   _VERSION: manual
   _REGION: us-central1
-  _PLATFORM: managed
-  _DB_USER: django
-  _DB_NAME: django
+  _ARTIFACT_REGISTRY: cloud-run-source-deploy
+  _IMAGE_NAME: ${_REGION}-docker.pkg.dev/${PROJECT_ID}/${_ARTIFACT_REGISTRY}/django-${_VERSION}
+  _STORAGE_BUCKET: ${PROJECT_ID}-bucket-${_VERSION}
+  _CLOUD_RUN_JOB_NAME: migrate-${_VERSION}
+  _DB_USER: django-${_VERSION}
+  _DB_NAME: django-${_VERSION}
   _DB_INSTANCE: django-instance
-  _SECRET_SETTINGS_NAME: django_settings
-  _SECRET_PASSWORD_NAME: admin_password
-  _STORAGE_BUCKET: ${PROJECT_ID}-bucket
+  _SECRET_SETTINGS_NAME: django_settings-${_VERSION}
+  _SECRET_PASSWORD_NAME: admin_password-${_VERSION}

run/django/e2e_test_cleanup.yaml

@@ -14,7 +14,7 @@
 
 steps:
   - id: "Create a dedicated database"
-    name: "gcr.io/cloud-builders/gcloud"
+    name: "gcr.io/google.com/cloudsdktool/cloud-sdk"
     entrypoint: "/bin/bash"
     args:
       - "-c"
@@ -24,7 +24,7 @@ steps:
           --project ${PROJECT_ID}"
 
   - id: "Create a dedicated database user"
-    name: "gcr.io/cloud-builders/gcloud"
+    name: "gcr.io/google.com/cloudsdktool/cloud-sdk"
     entrypoint: "/bin/bash"
     args:
       - "-c"
@@ -37,7 +37,7 @@ steps:
         rm db_password
 
   - id: "Create a dedicated storage bucket"
-    name: "gcr.io/cloud-builders/gcloud"
+    name: "gcr.io/google.com/cloudsdktool/cloud-sdk"
     entrypoint: "/bin/bash"
     args:
       - "-c"
@@ -47,8 +47,8 @@ steps:
           -p ${PROJECT_ID} \
           gs://${_STORAGE_BUCKET}"
 
-  - id: "Add Django secrets to Secret Manager"
-    name: "gcr.io/cloud-builders/gcloud"
+  - id: "IAM and Secrets"
+    name: "gcr.io/google.com/cloudsdktool/cloud-sdk"
     entrypoint: "/bin/bash"
     args:
       - "-c"
@@ -59,112 +59,98 @@ steps:
         SECRET_KEY=$(cat /dev/urandom | LC_ALL=C tr -dc '[:alpha:]' | fold -w 30 | head -n1)
         PASSWORD_NAME=${_SECRET_PASSWORD_NAME}" > ${_SECRET_SETTINGS_NAME}
 
+        ./retry.sh "gcloud iam service-accounts create ${_SERVICE_ACCOUNT}"
+
         ./retry.sh "gcloud secrets create ${_SECRET_SETTINGS_NAME} \
           --project $PROJECT_ID \
           --data-file=${_SECRET_SETTINGS_NAME}"
 
-        gcloud secrets add-iam-policy-binding ${_SECRET_SETTINGS_NAME} \
-        --member serviceAccount:$(gcloud projects list --filter "name=${PROJECT_ID}" --format "value(projectNumber)")@cloudbuild.gserviceaccount.com \
-        --role roles/secretmanager.secretAccessor \
-        --project ${PROJECT_ID}
-
-        rm ${_SECRET_SETTINGS_NAME}
-
         echo -n "${_SECRET_PASSWORD_VALUE}" > ${_SECRET_PASSWORD_NAME}
 
         ./retry.sh "gcloud secrets create ${_SECRET_PASSWORD_NAME} \
           --project $PROJECT_ID \
           --data-file=${_SECRET_PASSWORD_NAME}"
 
-        gcloud secrets add-iam-policy-binding ${_SECRET_PASSWORD_NAME} \
-          --member serviceAccount:$(gcloud projects list --filter "name=${PROJECT_ID}" --format "value(projectNumber)")@cloudbuild.gserviceaccount.com \
+        ./retry.sh "gcloud secrets add-iam-policy-binding ${_SECRET_SETTINGS_NAME} \
+        --member serviceAccount:${_SERVICE_ACCOUNT_EMAIL} \
+        --role roles/secretmanager.secretAccessor \
+        --project ${PROJECT_ID}"
+
+        ./retry.sh "gcloud secrets add-iam-policy-binding ${_SECRET_PASSWORD_NAME} \
+          --member serviceAccount:${_SERVICE_ACCOUNT_EMAIL} \
           --role roles/secretmanager.secretAccessor \
-          --project ${PROJECT_ID}
+          --project ${PROJECT_ID}"
+
+        ./retry.sh "gcloud projects add-iam-policy-binding ${PROJECT_ID} \
+          --member serviceAccount:${_SERVICE_ACCOUNT_EMAIL} \
+          --role roles/cloudsql.client \
+          --project ${PROJECT_ID}"
 
-        rm ${_SECRET_PASSWORD_NAME}
 
   - id: "Build Container Image"
-    name: "gcr.io/cloud-builders/docker"
-    entrypoint: "/bin/bash"
-    args:
-      - "-c"
-      - |
-        ./retry.sh "docker build -t gcr.io/${PROJECT_ID}/${_SERVICE}:${_VERSION} ."
+    name: gcr.io/k8s-skaffold/pack
+    args: ["build", "${_IMAGE_NAME}", "--builder=gcr.io/buildpacks/builder"]
 
   - id: "Push Container Image"
     name: "gcr.io/cloud-builders/docker"
     entrypoint: "/bin/bash"
     args:
       - "-c"
       - |
-        ./retry.sh "docker push gcr.io/${PROJECT_ID}/${_SERVICE}:${_VERSION}"
+        ./retry.sh "docker push  ${_IMAGE_NAME}"
 
   - id: "Migrate database"
-    name: "gcr.io/google-appengine/exec-wrapper"
-    args:
-      [
-        "-i",
-        "gcr.io/$PROJECT_ID/${_SERVICE}:${_VERSION}",
-        "-s",
-        "${_CLOUD_SQL_CONNECTION_NAME}",
-        "-e",
-        "SETTINGS_NAME=${_SECRET_SETTINGS_NAME}",
-        "-e",
-        "PASSWORD_NAME=${_SECRET_PASSWORD_NAME}",
-        "--",
-        "python",
-        "manage.py",
-        "migrate",
-      ]
-
-  - id: "Collect static"
-    name: "gcr.io/google-appengine/exec-wrapper"
+    name: "gcr.io/google.com/cloudsdktool/cloud-sdk"
+    entrypoint: /bin/bash
     args:
-      [
-        "-i",
-        "gcr.io/$PROJECT_ID/${_SERVICE}:${_VERSION}",
-        "-s",
-        "${_CLOUD_SQL_CONNECTION_NAME}",
-        "-e",
-        "SETTINGS_NAME=${_SECRET_SETTINGS_NAME}",
-        "--",
-        "python",
-        "manage.py",
-        "collectstatic",
-        "--verbosity",
-        "2",
-        "--no-input",
-      ]
+      - "-c"
+      - |
+        ./retry.sh "gcloud run jobs create ${_CLOUD_RUN_JOB_NAME} \
+          --region ${_REGION} \
+          --service-account ${_SERVICE_ACCOUNT_EMAIL} \
+          --image ${_IMAGE_NAME} \
+          --set-cloudsql-instances ${_CLOUD_SQL_CONNECTION_NAME} \
+          --set-env-vars SETTINGS_NAME=${_SECRET_SETTINGS_NAME} \
+          --command migrate \
+          --execute-now"
 
   - id: "Deploy to Cloud Run"
-    name: "gcr.io/cloud-builders/gcloud:latest"
+    name: "gcr.io/google.com/cloudsdktool/cloud-sdk:latest"
     entrypoint: /bin/bash
     args:
       - "-c"
       - |
         ./retry.sh "gcloud run deploy ${_SERVICE} \
           --project $PROJECT_ID \
-          --image gcr.io/${PROJECT_ID}/${_SERVICE}:${_VERSION} \
+          --image ${_IMAGE_NAME} \
           --no-allow-unauthenticated \
           --region ${_REGION} \
-          --platform ${_PLATFORM} \
-          --add-cloudsql-instances ${_CLOUD_SQL_CONNECTION_NAME} \
-          --update-env-vars SETTINGS_NAME=${_SECRET_SETTINGS_NAME}"
+          --service-account ${_SERVICE_ACCOUNT_EMAIL} \
+          --set-cloudsql-instances ${_CLOUD_SQL_CONNECTION_NAME} \
+          --set-env-vars SETTINGS_NAME=${_SECRET_SETTINGS_NAME}"
 
 images:
-  - gcr.io/${PROJECT_ID}/${_SERVICE}:${_VERSION}
+  - ${_IMAGE_NAME}
+
+options:
+  dynamicSubstitutions: true
+  logging: CLOUD_LOGGING_ONLY
 
 substitutions:
   _SERVICE: django
   _VERSION: manual
   _REGION: us-central1
-  _PLATFORM: managed
-  _STORAGE_BUCKET: ${PROJECT_ID}-bucket
-  _DB_INSTANCE: django-instance
-  _CLOUD_SQL_CONNECTION_NAME: $PROJECT_ID:us-central1:django-instance
-  _DB_NAME: postgres
-  _DB_USER: postgres
+  _ARTIFACT_REGISTRY: cloud-run-source-deploy
+  _IMAGE_NAME: ${_REGION}-docker.pkg.dev/${PROJECT_ID}/${_ARTIFACT_REGISTRY}/django-${_VERSION}
+  _STORAGE_BUCKET: ${PROJECT_ID}-bucket-${_VERSION}
+  _CLOUD_RUN_JOB_NAME: migrate-${_VERSION}
+  _SERVICE_ACCOUNT: django-sa-${_VERSION}
+  _SERVICE_ACCOUNT_EMAIL: ${_SERVICE_ACCOUNT}@${PROJECT_ID}.iam.gserviceaccount.com
+  _DB_INSTANCE: django-instance-${_VERSION}
+  _CLOUD_SQL_CONNECTION_NAME: ${PROJECT_ID}:${_REGION}:${_DB_INSTANCE}
+  _DB_NAME: postgres-${_VERSION}
+  _DB_USER: postgres-${_VERSION}
   _DB_PASS: password1234
-  _SECRET_SETTINGS_NAME: django_settings
-  _SECRET_PASSWORD_NAME: admin_password
+  _SECRET_SETTINGS_NAME: django_settings-${_VERSION}
+  _SECRET_PASSWORD_NAME: admin_password-${_VERSION}
   _SECRET_PASSWORD_VALUE: password