Streamline deployment configs

build/ci/cloudbuild.deploy_mixer.yaml

@@ -63,7 +63,7 @@ steps:
           --region=us-central1 \
           --skaffold-file=skaffold.yaml \
           --images="gcr.io/datcom-ci/datacommons-mixer=us-docker.pkg.dev/datcom-ci/gcr.io/datacommons-mixer:${mixer_githash}" \
-          --deploy-parameters="mixer.githash=${mixer_githash},MIXER_GITHASH=${mixer_githash}" \
+          --deploy-parameters="MIXER_GITHASH=${mixer_githash}" \
           --project=datcom-ci
 
         echo "Successfully created release ${release_name}."

build/ci/cloudbuild.deploy_website_mixer.yaml

@@ -89,7 +89,7 @@ steps:
           --region=us-central1 \
           --skaffold-file=skaffold.yaml \
           --images="gcr.io/datcom-ci/datacommons-website=us-docker.pkg.dev/datcom-ci/gcr.io/datacommons-website:$${website_githash},gcr.io/datcom-ci/datacommons-mixer=us-docker.pkg.dev/datcom-ci/gcr.io/datacommons-mixer:$${mixer_githash},gcr.io/datcom-ci/datacommons-nodejs=us-docker.pkg.dev/datcom-ci/gcr.io/datacommons-nodejs:$${website_githash},gcr.io/datcom-ci/datacommons-nl=us-docker.pkg.dev/datcom-ci/gcr.io/datacommons-nl:$${website_githash}" \
-          --deploy-parameters="mixer.githash=$${mixer_githash},MIXER_GITHASH=$${mixer_githash},website.githash=$${website_githash}" \
+          --deploy-parameters="MIXER_GITHASH=$${mixer_githash},WEBSITE_GITHASH=$${website_githash}" \
           --project=datcom-ci
 
         echo "Successfully created release $${release_name}."

deploy/helm_charts/dc_website/values.yaml

@@ -78,6 +78,9 @@ kgStoreConfig:
   baseBigtableInfo:
   customBigtableInfo: ""
 
+# NOTE: These serviceGroups configs are used for ROUTING ONLY (generating ingress-config).
+# To configure workload settings (replicas, resources, etc.) for the mixer deployments,
+# please modify the respective `website_mixer_{env}.yaml` overrides file.
 serviceGroups:
   svg:
     urlPaths:
@@ -86,37 +89,15 @@ serviceGroups:
       - "/v1/variable/*"
       - "/v1/bulk/info/variable-group"
       - "/v1/bulk/info/variable"
-    replicas: 1
-    resources:
-      memoryRequest: "8G"
-      memoryLimit: "8G"
-    cacheSVG: true
-    nodePool:
   observation:
     urlPaths:
       - "/v2/observation"
-    replicas: 1
-    resources:
-      memoryRequest: "4G"
-      memoryLimit: "4G"
-    nodePool:
   node:
     urlPaths:
       - "/v2/node"
-    replicas: 1
-    resources:
-      memoryRequest: "8G"
-      memoryLimit: "8G"
-    cacheSVG: true
-    nodePool:
   default:
     urlPaths:
       - "/*"
-    replicas: 1
-    resources:
-      memoryRequest: "2G"
-      memoryLimit: "2G"
-    nodePool:
 
 ###############################################################################
 # Config for NL

deploy/helm_charts/envs/autopush.yaml

@@ -39,22 +39,6 @@ website:
         }
       }
 
-mixer:
-  serviceName: "website-esp.endpoints.datcom-website-autopush.cloud.goog"
-  hostProject: "datcom-website-autopush"
-  cacheSVFormula: true
-  redis:
-    enabled: true
-    configFile: |
-      instances:
-        - region: "us-central1"
-          host: "10.13.112.19"
-          port: "6379"
-  enableOtlp: true
-
-serviceAccount:
-  name: website-ksa
-
 ingress:
   enabled: true
 
@@ -92,17 +76,6 @@ nl:
         prediction_endpoint_id: "284894457873039360"
     enable_reranking: true
 
-serviceGroups:
-  recon: null
-  svg:
-    replicas: 5
-  node:
-    replicas: 12
-  observation:
-    replicas: 12
-  default:
-    replicas: 12
-
 nodejs:
   enabled: true
   apiRoot: "http://website-service:8080"

deploy/helm_charts/envs/dev.yaml

@@ -29,35 +29,10 @@ website:
   redis:
     enabled: false
 
-mixer:
-  serviceName: "website-esp.endpoints.datcom-website-dev.cloud.goog"
-  hostProject: "datcom-website-dev"
-  redis:
-    enabled: true
-    configFile: |
-      instances:
-        - region: "us-central1"
-          host: "10.60.165.235"
-          port: "6379"
-  enableOtlp: true
-
-serviceAccount:
-  name: website-ksa
-
+# Enable ingress for website
 ingress:
   enabled: true
 
-serviceGroups:
-  recon: null
-  svg:
-    replicas: 1
-  observation:
-    replicas: 3
-  node:
-    replicas: 3
-  default:
-    replicas: 3
-
 nl:
   enabled: true
   image:

deploy/helm_charts/envs/prod.yaml

@@ -40,12 +40,14 @@ website:
         }
       }
 
-mixer:
-  serviceName: "website-esp.endpoints.datcom-website-prod.cloud.goog"
-  hostProject: "datcom-website-prod"
 
-serviceAccount:
-  name: website-ksa
+
+# Ingress is disabled because we use Multi Cluster Ingress (MCI) and Multi Cluster Service (MCS)
+# defined in gke/ which manage networking independently of this Helm chart.
+# Enabling ingress here would mean using templates/single_cluster.yaml to set up a standard
+# Ingress and Service, which is not what we want for this multi-cluster environment.
+ingress:
+  enabled: false
 
 nl:
   enabled: true
@@ -64,16 +66,5 @@ nl:
         prediction_endpoint_id: "430892009855647744"
     enable_reranking: false
 
-serviceGroups:
-  recon: null
-  svg:
-    replicas: 10
-  observation:
-    replicas: 100
-  node:
-    replicas: 100
-  default:
-    replicas: 50
-
 svg:
   blocklistFile: ["dc/g/Uncategorized", "oecd/g/OECD"]

deploy/helm_charts/envs/staging.yaml

@@ -37,19 +37,14 @@ website:
         }
       }
 
-mixer:
-  serviceName: "website-esp.endpoints.datcom-website-staging.cloud.goog"
-  hostProject: "datcom-website-staging"
-  redis:
-    enabled: true
-  enableOtlp: true
 
+# Ingress is disabled because we use Multi Cluster Ingress (MCI) and Multi Cluster Service (MCS)
+# defined in gke/ which manage networking independently of this Helm chart.
+# Enabling ingress here would mean using templates/single_cluster.yaml to set up a standard
+# Ingress and Service, which is not what we want for this environment.
 ingress:
   enabled: false
 
-serviceAccount:
-  name: website-ksa
-
 nl:
   enabled: true
   image:
@@ -69,17 +64,6 @@ nl:
         prediction_endpoint_id: "8435477397553283072"
     enable_reranking: false
 
-serviceGroups:
-  recon: null
-  svg:
-    replicas: 2
-  node:
-    replicas: 4
-  observation:
-    replicas: 4
-  default:
-    replicas: 4
-
 nodejs:
   enabled: true
   apiRoot: "http://website-service:8080"

deploy/helm_charts/envs/website_mixer_autopush.yaml

@@ -0,0 +1,33 @@
+
+
+mixer:
+  serviceName: "website-esp.endpoints.datcom-website-autopush.cloud.goog"
+  hostProject: "datcom-website-autopush"
+  cacheSVFormula: true
+  useRedis: true
+  redis:
+    enabled: true
+    configFile: |
+      instances:
+        - region: "us-central1"
+          host: "10.13.112.19"
+          port: "6379"
+  enableOtlp: true
+
+serviceAccount:
+  name: website-ksa
+
+# Disable ingress for mixer in datcom-website-autopush
+ingress:
+  enabled: false
+
+serviceGroups:
+  recon: null
+  svg:
+    replicas: 5
+  node:
+    replicas: 12
+  observation:
+    replicas: 12
+  default:
+    replicas: 12

deploy/helm_charts/envs/website_mixer_dev.yaml

@@ -0,0 +1,36 @@
+# Overrides for datcom-website-dev's mixer deployments
+# Default values are defined by 
+#  priority: /mixer/deploy/helm_charts/envs/mixer_dev.yaml
+#  fallback: /mixer/deploy/helm_charts/values.yaml
+
+project: "datcom-website-dev"
+
+mixer:
+  hostProject: "datcom-website-dev"
+  serviceName: "website-esp.endpoints.datcom-website-dev.cloud.goog"
+  redis:
+    enabled: true
+    configFile: |
+      instances:
+        - region: "us-central1"
+          host: "10.60.165.235"
+          port: "6379"
+  enableOtlp: true
+
+serviceAccount:
+  name: website-ksa
+
+# Disable ingress for mixer in datcom-website-dev
+ingress:
+  enabled: false
+
+serviceGroups:
+  recon: null
+  svg:
+    replicas: 1
+  observation:
+    replicas: 3
+  node:
+    replicas: 3
+  default:
+    replicas: 3

deploy/helm_charts/envs/website_mixer_prod.yaml

@@ -0,0 +1,32 @@
+
+project: "datcom-website-prod"
+mixer:
+  hostProject: "datcom-website-prod"
+  serviceName: "website-esp.endpoints.datcom-website-prod.cloud.goog"
+  useRedis: false
+  redis:
+    enabled: true
+    configFile: |
+      instances:
+        - region: "us-central1"
+          host: "10.212.232.163"
+          port: "6379"
+  enableOtlp: true
+
+serviceAccount:
+  name: website-ksa
+
+# Disable ingress for mixer in datcom-website-prod (Sidecar only)
+ingress:
+  enabled: false
+
+serviceGroups:
+  recon: null
+  svg:
+    replicas: 10
+  observation:
+    replicas: 100
+  node:
+    replicas: 100
+  default:
+    replicas: 50

deploy/helm_charts/envs/website_mixer_staging.yaml

@@ -0,0 +1,32 @@
+
+project: "datcom-website-staging"
+mixer:
+  hostProject: "datcom-website-staging"
+  serviceName: "website-esp.endpoints.datcom-website-staging.cloud.goog"
+  useRedis: true
+  redis:
+    enabled: true
+    configFile: |
+      instances:
+        - region: "us-central1"
+          host: "10.38.211.19"
+          port: "6379"
+  enableOtlp: true
+
+serviceAccount:
+  name: website-ksa
+ingress:
+  enabled: false
+
+serviceGroups:
+  recon: null
+  svg:
+    replicas: 2
+    nodePool: pool-1
+  node:
+    replicas: 4
+  observation:
+    replicas: 4
+  default:
+    replicas: 4
+

scripts/deploy_mixer_cloud_deploy.sh

@@ -30,5 +30,5 @@ gcloud deploy releases create "dev-manual-$TIMESTAMP" \
 --region=us-central1 \
 --skaffold-file=skaffold.yaml \
 --images="gcr.io/datcom-ci/datacommons-mixer=us-docker.pkg.dev/datcom-ci/gcr.io/datacommons-mixer:$MIXER_GITHASH" \
---deploy-parameters="mixer.githash=$MIXER_GITHASH,MIXER_GITHASH=$MIXER_GITHASH" \
+--deploy-parameters="MIXER_GITHASH=$MIXER_GITHASH" \
 --project=datcom-ci

scripts/deploy_website_cloud_deploy.sh

@@ -31,5 +31,5 @@ gcloud deploy releases create "dev-manual-$TIMESTAMP" \
 --region=us-central1 \
 --skaffold-file=skaffold.yaml \
 --images="gcr.io/datcom-ci/datacommons-website=us-docker.pkg.dev/datcom-ci/gcr.io/datacommons-website:$WEBSITE_GITHASH,gcr.io/datcom-ci/datacommons-mixer=us-docker.pkg.dev/datcom-ci/gcr.io/datacommons-mixer:$MIXER_GITHASH,gcr.io/datcom-ci/datacommons-nodejs=us-docker.pkg.dev/datcom-ci/gcr.io/datacommons-nodejs:$WEBSITE_GITHASH,gcr.io/datcom-ci/datacommons-nl=us-docker.pkg.dev/datcom-ci/gcr.io/datacommons-nl:$WEBSITE_GITHASH" \
---deploy-parameters="mixer.githash=$MIXER_GITHASH,MIXER_GITHASH=$MIXER_GITHASH,website.githash=$WEBSITE_GITHASH" \
---project=datcom-ci
+--deploy-parameters="MIXER_GITHASH=$MIXER_GITHASH,WEBSITE_GITHASH=$WEBSITE_GITHASH" \
+--project=datcom-ci