Add necessary changes to provision QA with Terraform (#2618)

Also programmatically determine backend service IDs.

Author: jianglai

core/build.gradle

@@ -166,6 +166,7 @@ dependencies {
   // gradleLint.ignore('unused-dependency') {
   implementation deps['com.google.gwt:gwt-user']
   // }
+  implementation deps['com.google.cloud:google-cloud-compute']
   implementation deps['com.google.cloud:google-cloud-core']
   implementation deps['com.google.cloud:google-cloud-storage']
   implementation deps['com.google.cloud:google-cloud-tasks']

core/gradle.lockfile

@@ -32,8 +32,8 @@ com.google.api-client:google-api-client-jackson2:2.0.1=compileClasspath,deploy_j
 com.google.api-client:google-api-client-jackson2:2.2.0=testRuntimeClasspath
 com.google.api-client:google-api-client-java6:2.1.4=compileClasspath,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 com.google.api-client:google-api-client-servlet:2.2.0=testRuntimeClasspath
-com.google.api-client:google-api-client-servlet:2.7.0=compileClasspath,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath
-com.google.api-client:google-api-client:2.7.0=compileClasspath,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.api-client:google-api-client-servlet:2.7.1=compileClasspath,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath
+com.google.api-client:google-api-client:2.7.1=compileClasspath,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 com.google.api.grpc:gapic-google-cloud-storage-v2:2.32.1-alpha=compileClasspath,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath
 com.google.api.grpc:gapic-google-cloud-storage-v2:2.45.0-beta=testCompileClasspath,testRuntimeClasspath
 com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:3.9.0=compileClasspath,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
@@ -54,6 +54,7 @@ com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.181.0=compileCl
 com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.181.0=compileClasspath,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 com.google.api.grpc:proto-google-cloud-bigtable-admin-v2:2.43.0=compileClasspath,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 com.google.api.grpc:proto-google-cloud-bigtable-v2:2.43.0=compileClasspath,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.api.grpc:proto-google-cloud-compute-v1:1.64.0=compileClasspath,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 com.google.api.grpc:proto-google-cloud-datastore-v1:0.112.2=compileClasspath,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 com.google.api.grpc:proto-google-cloud-firestore-v1:3.25.1=compileClasspath,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 com.google.api.grpc:proto-google-cloud-monitoring-v3:3.49.0=compileClasspath,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
@@ -94,7 +95,7 @@ com.google.apis:google-api-services-iam:v2-rev20240530-2.0.0=compileClasspath,de
 com.google.apis:google-api-services-iamcredentials:v1-rev20211203-2.0.0=compileClasspath,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 com.google.apis:google-api-services-monitoring:v3-rev20241017-2.0.0=compileClasspath,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 com.google.apis:google-api-services-pubsub:v1-rev20220904-2.0.0=compileClasspath,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-com.google.apis:google-api-services-sheets:v4-rev20241008-2.0.0=compileClasspath,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.apis:google-api-services-sheets:v4-rev20241203-2.0.0=compileClasspath,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 com.google.apis:google-api-services-sqladmin:v1beta4-rev20240925-2.0.0=compileClasspath,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 com.google.apis:google-api-services-storage:v1-rev20240706-2.0.0=compileClasspath,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath
 com.google.apis:google-api-services-storage:v1-rev20241008-2.0.0=testRuntimeClasspath
@@ -120,6 +121,7 @@ com.google.cloud.sql:jdbc-socket-factory-core:1.21.0=compileClasspath,deploy_jar
 com.google.cloud.sql:postgres-socket-factory:1.21.0=deploy_jar,runtimeClasspath,testRuntimeClasspath
 com.google.cloud:google-cloud-bigquerystorage:3.9.0=compileClasspath,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 com.google.cloud:google-cloud-bigtable:2.43.0=compileClasspath,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.cloud:google-cloud-compute:1.64.0=compileClasspath,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 com.google.cloud:google-cloud-core-grpc:2.42.0=compileClasspath,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath
 com.google.cloud:google-cloud-core-grpc:2.48.0=testCompileClasspath,testRuntimeClasspath
 com.google.cloud:google-cloud-core-http:2.31.0=compileClasspath,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath
@@ -153,7 +155,7 @@ com.google.devtools.ksp:symbol-processing-api:1.9.20-1.0.14=annotationProcessor,
 com.google.errorprone:error_prone_annotation:2.23.0=annotationProcessor,errorprone,nonprodAnnotationProcessor,testAnnotationProcessor
 com.google.errorprone:error_prone_annotations:2.20.0=soy
 com.google.errorprone:error_prone_annotations:2.23.0=annotationProcessor,errorprone,nonprodAnnotationProcessor,testAnnotationProcessor
-com.google.errorprone:error_prone_annotations:2.35.1=compileClasspath,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.errorprone:error_prone_annotations:2.36.0=compileClasspath,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 com.google.errorprone:error_prone_annotations:2.7.1=checkstyle
 com.google.errorprone:error_prone_check_api:2.23.0=annotationProcessor,errorprone,nonprodAnnotationProcessor,testAnnotationProcessor
 com.google.errorprone:error_prone_core:2.23.0=annotationProcessor,errorprone,nonprodAnnotationProcessor,testAnnotationProcessor
@@ -179,14 +181,14 @@ com.google.guava:guava:33.0.0-jre=annotationProcessor,testAnnotationProcessor
 com.google.guava:guava:33.3.1-jre=compileClasspath,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava=annotationProcessor,checkstyle,compileClasspath,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testAnnotationProcessor,testCompileClasspath,testRuntimeClasspath
 com.google.gwt:gwt-user:2.10.0=compileClasspath,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-com.google.http-client:google-http-client-apache-v2:1.45.0=compileClasspath,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.http-client:google-http-client-apache-v2:1.45.2=compileClasspath,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 com.google.http-client:google-http-client-appengine:1.43.3=compileClasspath,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath
 com.google.http-client:google-http-client-appengine:1.45.0=testCompileClasspath,testRuntimeClasspath
-com.google.http-client:google-http-client-gson:1.45.0=compileClasspath,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.http-client:google-http-client-gson:1.45.2=compileClasspath,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 com.google.http-client:google-http-client-jackson2:1.43.3=compileClasspath,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath
 com.google.http-client:google-http-client-jackson2:1.45.0=testCompileClasspath,testRuntimeClasspath
 com.google.http-client:google-http-client-protobuf:1.44.2=compileClasspath,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-com.google.http-client:google-http-client:1.45.0=compileClasspath,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.http-client:google-http-client:1.45.2=compileClasspath,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 com.google.inject:guice:5.1.0=annotationProcessor,errorprone,nonprodAnnotationProcessor,testAnnotationProcessor
 com.google.inject:guice:7.0.0=compileClasspath,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,soy,testCompileClasspath,testRuntimeClasspath
 com.google.j2objc:j2objc-annotations:1.3=checkstyle
@@ -262,10 +264,11 @@ io.github.eisop:dataflow-errorprone:3.34.0-eisop1=annotationProcessor,errorprone
 io.github.java-diff-utils:java-diff-utils:4.12=annotationProcessor,errorprone,nonprodAnnotationProcessor,testAnnotationProcessor
 io.github.java-diff-utils:java-diff-utils:4.15=deploy_jar,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 io.grpc:grpc-alts:1.68.1=compileClasspath,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-io.grpc:grpc-api:1.68.1=compileClasspath,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+io.grpc:grpc-api:1.68.1=compileClasspath,nonprodCompileClasspath,testCompileClasspath
+io.grpc:grpc-api:1.68.2=deploy_jar,nonprodRuntimeClasspath,runtimeClasspath,testRuntimeClasspath
 io.grpc:grpc-auth:1.68.1=compileClasspath,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 io.grpc:grpc-census:1.66.0=compileClasspath,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-io.grpc:grpc-context:1.68.1=compileClasspath,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+io.grpc:grpc-context:1.68.2=compileClasspath,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 io.grpc:grpc-core:1.68.1=compileClasspath,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 io.grpc:grpc-googleapis:1.68.1=deploy_jar,nonprodRuntimeClasspath,runtimeClasspath,testRuntimeClasspath
 io.grpc:grpc-grpclb:1.68.1=compileClasspath,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
@@ -484,7 +487,7 @@ org.jetbrains.kotlinx:kotlinx-serialization-core-jvm:1.0.1=deploy_jar,nonprodRun
 org.jetbrains.kotlinx:kotlinx-serialization-core:1.0.1=deploy_jar,nonprodRuntimeClasspath,runtimeClasspath,testRuntimeClasspath
 org.jetbrains:annotations:13.0=annotationProcessor,testAnnotationProcessor
 org.jetbrains:annotations:17.0.0=compileClasspath,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-org.jline:jline:3.27.1=compileClasspath,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+org.jline:jline:3.28.0=compileClasspath,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.joda:joda-money:2.0.0=compileClasspath,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.json:json:20230618=soy
 org.json:json:20240303=compileClasspath,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath

core/src/main/java/google/registry/config/RegistryConfig.java

@@ -46,7 +46,6 @@
 import java.lang.annotation.Retention;
 import java.net.URI;
 import java.net.URL;
-import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Optional;
 import java.util.function.Supplier;
@@ -118,12 +117,6 @@ public static long provideProjectIdNumber(RegistryConfigSettings config) {
       return config.gcpProject.projectIdNumber;
     }
 
-    @Provides
-    @Config("backendServiceIds")
-    public static Map<String, Long> provideBackendServiceIds(RegistryConfigSettings config) {
-      return config.gcpProject.backendServiceIds;
-    }
-
     @Provides
     @Config("baseDomain")
     public static String provideBaseDomain(RegistryConfigSettings config) {

core/src/main/java/google/registry/config/RegistryConfigSettings.java

@@ -56,7 +56,6 @@ public static class GcpProject {
     public String bsaServiceUrl;
     public String toolsServiceUrl;
     public String pubapiServiceUrl;
-    public Map<String, Long> backendServiceIds;
     public String baseDomain;
   }
 

core/src/main/java/google/registry/config/files/default-config.yaml

@@ -24,15 +24,6 @@ gcpProject:
   toolsServiceUrl: https://tools.example.com
   pubapiServiceUrl: https://pubapi.example.com
 
-  # The backend service IDs created when setting up GKE routes. They will be included in the
-  # audience field in the JWT that IAP creates.
-  # See: https://cloud.google.com/iap/docs/signed-headers-howto#verifying_the_jwt_payload
-  backendServiceIds:
-    frontend: 12345
-    backend: 12345
-    pubapi: 12345
-    console: 12345
-
   # The base domain name of the registry service. Services are reachable at [service].baseDomain.
   baseDomain: registry.test
 

core/src/main/java/google/registry/request/auth/AuthModule.java

@@ -14,19 +14,31 @@
 
 package google.registry.request.auth;
 
+import static com.google.common.base.Preconditions.checkNotNull;
 import static com.google.common.net.HttpHeaders.AUTHORIZATION;
+import static google.registry.util.RegistryEnvironment.UNITTEST;
 
+import com.google.cloud.compute.v1.BackendService;
+import com.google.cloud.compute.v1.BackendServicesClient;
+import com.google.cloud.compute.v1.BackendServicesSettings;
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.re2j.Matcher;
+import com.google.re2j.Pattern;
+import dagger.Lazy;
 import dagger.Module;
 import dagger.Provides;
+import google.registry.config.CredentialModule.ApplicationDefaultCredential;
 import google.registry.config.RegistryConfig.Config;
 import google.registry.request.auth.OidcTokenAuthenticationMechanism.IapOidcAuthenticationMechanism;
 import google.registry.request.auth.OidcTokenAuthenticationMechanism.RegularOidcAuthenticationMechanism;
 import google.registry.request.auth.OidcTokenAuthenticationMechanism.TokenExtractor;
 import google.registry.request.auth.OidcTokenAuthenticationMechanism.TokenVerifier;
+import google.registry.util.GoogleCredentialsBundle;
 import google.registry.util.RegistryEnvironment;
-import java.util.Map;
+import java.io.IOException;
 import javax.annotation.Nullable;
+import javax.inject.Named;
 import javax.inject.Qualifier;
 import javax.inject.Singleton;
 
@@ -44,6 +56,13 @@ public class AuthModule {
   private static final String IAP_GKE_AUDIENCE_FORMAT = "/projects/%d/global/backendServices/%d";
   private static final String IAP_ISSUER_URL = "https://cloud.google.com/iap";
   private static final String REGULAR_ISSUER_URL = "https://accounts.google.com";
+  // The backend service IDs created when setting up GKE routes. They will be included in the
+  // audience field in the JWT that IAP creates.
+  // See: https://cloud.google.com/iap/docs/signed-headers-howto#verifying_the_jwt_payload
+  // The automatically generated backend service ID has the following format:
+  // gkemcg1-default-console[-canary]-80-(some random string)
+  private static final Pattern BACKEND_END_PATTERN =
+      Pattern.compile(".*-default-((frontend|backend|console|pubapi)(-canary)?)-80-.*");
 
   /** Provides the custom authentication mechanisms. */
   @Provides
@@ -68,13 +87,18 @@ ImmutableList<AuthenticationMechanism> provideApiAuthenticationMechanisms(
   TokenVerifier provideIapTokenVerifier(
       @Config("projectId") String projectId,
       @Config("projectIdNumber") long projectIdNumber,
-      @Config("backendServiceIds") Map<String, Long> backendServiceIds) {
+      @Named("backendServiceIdMap") ImmutableMap<String, Long> backendServiceIdMap) {
     com.google.auth.oauth2.TokenVerifier.Builder tokenVerifierBuilder =
         com.google.auth.oauth2.TokenVerifier.newBuilder().setIssuer(IAP_ISSUER_URL);
     return (String service, String token) -> {
       String audience;
       if (RegistryEnvironment.isOnJetty()) {
-        long backendServiceId = backendServiceIds.get(service);
+        Long backendServiceId = backendServiceIdMap.get(service);
+        checkNotNull(
+            backendServiceId,
+            "Backend service ID not found for service: %s, available IDs are %s",
+            service,
+            backendServiceIdMap);
         audience = String.format(IAP_GKE_AUDIENCE_FORMAT, projectIdNumber, backendServiceId);
       } else {
         audience = String.format(IAP_GAE_AUDIENCE_FORMAT, projectIdNumber, projectId);
@@ -116,4 +140,38 @@ TokenExtractor provideRegularTokenExtractor() {
       return null;
     };
   }
+
+  @Provides
+  @Singleton
+  static BackendServicesClient provideBackendServicesClients(
+      @ApplicationDefaultCredential GoogleCredentialsBundle credentialsBundle) {
+    try {
+      return BackendServicesClient.create(
+          BackendServicesSettings.newBuilder()
+              .setCredentialsProvider(credentialsBundle::getGoogleCredentials)
+              .build());
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  @Provides
+  @Singleton
+  @Named("backendServiceIdMap")
+  static ImmutableMap<String, Long> provideBackendServiceList(
+      Lazy<BackendServicesClient> client, @Config("projectId") String projectId) {
+    if (RegistryEnvironment.isInTestServer() || RegistryEnvironment.get() == UNITTEST) {
+      return ImmutableMap.of();
+    }
+    ImmutableMap.Builder<String, Long> builder = ImmutableMap.builder();
+    for (BackendService service : client.get().list(projectId).iterateAll()) {
+      String name = service.getName();
+      Matcher matcher = BACKEND_END_PATTERN.matcher(name);
+      if (!matcher.matches()) {
+        continue;
+      }
+      builder.put(matcher.group(1), service.getId());
+    }
+    return builder.build();
+  }
 }

core/src/main/java/google/registry/request/auth/OidcTokenAuthenticationMechanism.java

@@ -87,6 +87,9 @@ public AuthResult authenticate(HttpServletRequest request) {
       if (RegistryEnvironment.isOnJetty()) {
         String hostname = request.getServerName();
         service = Splitter.on('.').split(hostname).iterator().next();
+        if (request.getHeader("canary") != null) {
+          service += "-canary";
+        }
       }
       token = tokenVerifier.verify(service, rawIdToken);
     } catch (Exception e) {