Make GKE the default in alpha and qa (#2624)

Author: jianglai

core/src/main/java/google/registry/request/auth/AuthModule.java

@@ -15,6 +15,7 @@
 package google.registry.request.auth;
 
 import static com.google.common.base.Preconditions.checkNotNull;
+import static com.google.common.base.Suppliers.memoizeWithExpiration;
 import static com.google.common.net.HttpHeaders.AUTHORIZATION;
 import static google.registry.util.RegistryEnvironment.UNITTEST;
 
@@ -37,8 +38,11 @@
 import google.registry.util.GoogleCredentialsBundle;
 import google.registry.util.RegistryEnvironment;
 import java.io.IOException;
+import java.time.Duration;
+import java.util.function.Supplier;
 import javax.annotation.Nullable;
 import javax.inject.Named;
+import javax.inject.Provider;
 import javax.inject.Qualifier;
 import javax.inject.Singleton;
 
@@ -87,13 +91,13 @@ ImmutableList<AuthenticationMechanism> provideApiAuthenticationMechanisms(
   TokenVerifier provideIapTokenVerifier(
       @Config("projectId") String projectId,
       @Config("projectIdNumber") long projectIdNumber,
-      @Named("backendServiceIdMap") ImmutableMap<String, Long> backendServiceIdMap) {
+      @Named("backendServiceIdMap") Supplier<ImmutableMap<String, Long>> backendServiceIdMap) {
     com.google.auth.oauth2.TokenVerifier.Builder tokenVerifierBuilder =
         com.google.auth.oauth2.TokenVerifier.newBuilder().setIssuer(IAP_ISSUER_URL);
     return (String service, String token) -> {
       String audience;
       if (RegistryEnvironment.isOnJetty()) {
-        Long backendServiceId = backendServiceIdMap.get(service);
+        Long backendServiceId = backendServiceIdMap.get().get(service);
         checkNotNull(
             backendServiceId,
             "Backend service ID not found for service: %s, available IDs are %s",
@@ -156,7 +160,6 @@ static BackendServicesClient provideBackendServicesClients(
   }
 
   @Provides
-  @Singleton
   @Named("backendServiceIdMap")
   static ImmutableMap<String, Long> provideBackendServiceList(
       Lazy<BackendServicesClient> client, @Config("projectId") String projectId) {
@@ -174,4 +177,15 @@ static ImmutableMap<String, Long> provideBackendServiceList(
     }
     return builder.build();
   }
+
+  // Use an expiring cache so that the backend service ID map can be refreshed without restarting
+  // the server. The map is very unlikely to change, except for when services are just deployed
+  // for the first time, because some pods might receive traffic before all services are deployed.
+  @Provides
+  @Singleton
+  @Named("backendServiceIdMap")
+  static Supplier<ImmutableMap<String, Long>> provideBackendServiceIdMapSupplier(
+      @Named("backendServiceIdMap") Provider<ImmutableMap<String, Long>> backendServiceIdMap) {
+    return memoizeWithExpiration(backendServiceIdMap::get, Duration.ofMinutes(15));
+  }
 }

core/src/main/java/google/registry/tools/RegistryCli.java

@@ -25,6 +25,7 @@
 import com.beust.jcommander.ParametersDelegate;
 import com.google.common.base.Throwables;
 import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Iterables;
 import google.registry.persistence.transaction.JpaTransactionManager;
 import google.registry.persistence.transaction.TransactionManagerFactory;
@@ -41,6 +42,9 @@
 @Parameters(separators = " =", commandDescription = "Command-line interface to the registry")
 final class RegistryCli implements CommandRunner {
 
+  private static final ImmutableSet<RegistryToolEnvironment> DEFAULT_GKE_ENVIRONMENTS =
+      ImmutableSet.of(RegistryToolEnvironment.ALPHA, RegistryToolEnvironment.QA);
+
   // The environment parameter is parsed twice: once here, and once with {@link
   // RegistryToolEnvironment#parseFromArgs} in the {@link RegistryTool#main} function.
   //
@@ -73,6 +77,9 @@ final class RegistryCli implements CommandRunner {
   @Parameter(names = "--gke", description = "Whether to use GKE runtime, instead of GAE")
   private boolean useGke = false;
 
+  @Parameter(names = "--gae", description = "Whether to use GAE runtime, instead of GKE")
+  private boolean useGae = false;
+
   @Parameter(names = "--canary", description = "Whether to connect to the canary instances")
   private boolean useCanary = false;
 
@@ -149,6 +156,13 @@ public void run(String[] args) throws Exception {
       }
       throw e;
     }
+
+    checkState(!useGke || !useGae, "Cannot specify both --gke and --gae");
+    // Special logic to set the default based on the environment if neither --gae nor --gke is set.
+    if (!useGke && !useGae) {
+      useGke = DEFAULT_GKE_ENVIRONMENTS.contains(environment);
+    }
+
     String parsedCommand = jcommander.getParsedCommand();
     // Show the list of all commands either if requested or if no subcommand name was specified
     // (which does not throw a ParameterException parse error above).

jetty/build.gradle

@@ -82,8 +82,36 @@ tasks.register('run', JavaExec) {
     dependsOn(tasks.named('stage'))
 }
 
+tasks.register('buildDeployer', Exec) {
+    workingDir("${rootDir}/release/builder/")
+    commandLine 'go', 'build', '-o', "${buildDir}/deployer", 'deployCloudSchedulerAndQueue.go'
+}
+
+// Once GKE is the only option, we can use the same task in the root project instaead.
+tasks.register('deployCloudSchedulerAndQueue') {
+    dependsOn(tasks.named('deployCloudScheduler'), tasks.named('deployQueue'))
+}
+
+tasks.register('deployCloudScheduler', Exec) {
+    dependsOn(tasks.named('buildDeployer'))
+    workingDir("$buildDir")
+    commandLine './deployer',
+        "${rootDir}/core/src/main/java/google/registry/config/files/nomulus-config-${rootProject.environment}.yaml",
+        "${rootDir}/core/src/main/java/google/registry/env/${rootProject.environment}/default/WEB-INF/cloud-scheduler-tasks.xml",
+        rootProject.gcpProject, '--gke'
+}
+
+tasks.register('deployQueue', Exec) {
+    dependsOn(tasks.named('buildDeployer'))
+    workingDir("$buildDir")
+    commandLine './deployer',
+        "${rootDir}/core/src/main/java/google/registry/config/files/nomulus-config-${rootProject.environment}.yaml",
+        "${rootDir}/core/src/main/java/google/registry/env/common/default/WEB-INF/cloud-tasks-queue.xml",
+        rootProject.gcpProject, '--gke'
+}
+
 tasks.register('deployNomulus', Exec) {
-    dependsOn('pushNomulusImage', ':proxy:pushProxyImage')
+    dependsOn('pushNomulusImage', 'deployCloudSchedulerAndQueue')
     configure verifyDeploymentConfig
     commandLine './deploy-nomulus-for-env.sh', "${rootProject.environment}", "${rootProject.baseDomain}"
 }

proxy/deploy-proxy-for-env.sh

@@ -32,13 +32,13 @@ do
   gcloud container clusters get-credentials "${parts[0]}" \
     --project "${project}" --zone "${parts[1]}"
   sed s/GCP_PROJECT/${project}/g "./kubernetes/proxy-deployment-${environment}.yaml" | \
-  kubectl replace -f -
-  kubectl replace -f "./kubernetes/proxy-service.yaml" --force
+  kubectl apply -f -
+  kubectl apply -f "./kubernetes/proxy-service.yaml" --force
   # Alpha does not have canary
   if [[ ${environment} != "alpha" ]]; then
     sed s/GCP_PROJECT/${project}/g "./kubernetes/proxy-deployment-${environment}-canary.yaml" | \
-    kubectl replace -f -
-    kubectl replace -f "./kubernetes/proxy-service-canary.yaml" --force
+    kubectl apply -f -
+    kubectl apply -f "./kubernetes/proxy-service-canary.yaml" --force
   fi
   # Kills all running pods, new pods created will be pulling the new image.
   kubectl delete pods --all

proxy/src/main/java/google/registry/proxy/ProxyConfig.java

@@ -41,7 +41,6 @@ enum Environment {
 
   public String projectId;
   public String oauthClientId;
-  public boolean canary;
   public List<String> gcpScopes;
   public int serverCertificateCacheSeconds;
   public Gcs gcs;

proxy/src/main/java/google/registry/proxy/ProxyModule.java

@@ -281,8 +281,8 @@ static Supplier<String> provideOidcToken(
   @Singleton
   @Provides
   @Named("canary")
-  static boolean provideIsCanary(ProxyConfig config) {
-    return config.canary;
+  boolean provideIsCanary(Environment env) {
+    return env.name().endsWith("_CANARY");
   }
 
   @Singleton

proxy/src/main/java/google/registry/proxy/config/default-config.yaml

@@ -8,9 +8,6 @@
 # GCP project ID
 projectId: your-gcp-project-id
 
-# Whether to connect to the canary (instead of regular) service.
-canary: false
-
 # OAuth client ID set as the audience of the OIDC token. This value must be the
 # same as the auth.oauthClientId value in Nomulus config file, which usually is
 # the IAP client ID, to allow the request to access IAP protected endpoints.