feat: Enable project id usage by performing lookups where necessary. (#223)

* fix: Modify path construction methods to only have valid Path objects and allow project id or number usage.

This enables called classes to determine more directly whether they need to call out to the project id to number mapping API.

* feat: Enable project id usage in java client library.

* fix: Revert samples changes until a new release happens.

* fix: Add project id transform to assign and commit streams.

* fix: Assorted fixes from PR.

* fix: Fix new usage of old path modifiers.

* fix: Add Serializable to path classes to allow use in beam tests.

Author: dpcollins-google

google-cloud-pubsublite/clirr-ignored-differences.xml

@@ -1,6 +1,28 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <!-- see http://www.mojohaus.org/clirr-maven-plugin/examples/ignored-differences.html -->
 <differences>
+  <!-- Path construction has been modified: remove on next release. -->
+  <difference>
+    <differenceType>7002</differenceType>
+    <className>com/google/cloud/pubsublite/*Path*</className>
+    <method>*</method>
+  </difference>
+  <difference>
+    <differenceType>7004</differenceType>
+    <className>com/google/cloud/pubsublite/*Path*</className>
+    <method>*</method>
+  </difference>
+  <difference>
+    <differenceType>7005</differenceType>
+    <className>com/google/cloud/pubsublite/*Path*</className>
+    <method>*</method>
+    <to>*</to>
+  </difference>
+  <difference>
+    <differenceType>7013</differenceType>
+    <className>com/google/cloud/pubsublite/*Path*</className>
+    <method>*</method>
+  </difference>
   <!-- Blanket ignored files -->
   <difference>
     <differenceType>6000</differenceType>

google-cloud-pubsublite/pom.xml

@@ -47,6 +47,11 @@
       <groupId>com.google.cloud</groupId>
       <artifactId>google-cloud-pubsub</artifactId>
     </dependency>
+    <dependency>
+      <groupId>com.google.cloud</groupId>
+      <artifactId>google-cloud-resourcemanager</artifactId>
+      <version>0.117.2-alpha</version>
+    </dependency>
     <dependency>
       <groupId>com.google.api.grpc</groupId>
       <artifactId>proto-google-cloud-pubsub-v1</artifactId>

google-cloud-pubsublite/src/main/java/com/google/cloud/pubsublite/LocationPath.java

@@ -16,24 +16,39 @@
 
 package com.google.cloud.pubsublite;
 
-import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.cloud.pubsublite.internal.Preconditions.checkArgument;
 
 import com.google.auto.value.AutoValue;
+import io.grpc.StatusException;
 import java.io.Serializable;
+import java.util.Arrays;
 
-/**
- * A string wrapper representing a project and location. Should be structured like:
- *
- * <p>projects/&lt;project number&gt;/locations/&lt;cloud zone&gt;
- */
+/** A string wrapper representing a project and location. */
 @AutoValue
 public abstract class LocationPath implements Serializable {
-  /** The string value of this location path. */
-  public abstract String value();
+  public abstract ProjectPath project();
+
+  public abstract CloudZone location();
+
+  @Override
+  public String toString() {
+    return project() + "/locations/" + location();
+  }
+
+  public static LocationPath of(ProjectPath project, CloudZone zone) {
+    return new AutoValue_LocationPath(project, zone);
+  }
 
-  /** Construct a LocationPath from its string value. */
-  public static LocationPath of(String value) {
-    checkArgument(!value.isEmpty());
-    return new AutoValue_LocationPath(value);
+  /**
+   * Parse a location path. Should be structured like:
+   *
+   * <p>projects/&lt;project number&gt;/locations/&lt;cloud zone&gt;
+   */
+  public static LocationPath parse(String path) throws StatusException {
+    String[] splits = path.split("/");
+    checkArgument(splits.length == 4);
+    checkArgument(splits[2].equals("locations"));
+    ProjectPath project = ProjectPath.parse(String.join("/", Arrays.copyOf(splits, 2)));
+    return LocationPath.of(project, CloudZone.parse(splits[3]));
   }
 }

google-cloud-pubsublite/src/main/java/com/google/cloud/pubsublite/LocationPaths.java

@@ -16,69 +16,29 @@
 
 package com.google.cloud.pubsublite;
 
-import static com.google.cloud.pubsublite.internal.ExtractStatus.toCanonical;
-import static com.google.cloud.pubsublite.internal.Preconditions.checkArgument;
-
 import com.google.auto.value.AutoValue;
-import io.grpc.StatusException;
 
 /** Helpers for constructing valid LocationPaths. */
 @AutoValue
 public abstract class LocationPaths {
-  abstract ProjectNumber projectNumber();
+  abstract ProjectIdOrNumber project();
 
-  abstract CloudZone zone();
+  abstract CloudZone location();
 
   /** Create a new LocationPath builder. */
   public static Builder newBuilder() {
     return new AutoValue_LocationPaths.Builder();
   }
 
   @AutoValue.Builder
-  public abstract static class Builder {
-    /** The project number. */
-    public abstract Builder setProjectNumber(ProjectNumber number);
-
-    /** The Google Cloud zone. */
-    public abstract Builder setZone(CloudZone zone);
+  public abstract static class Builder extends ProjectBuilderHelper<Builder> {
+    public abstract Builder setLocation(CloudZone zone);
 
     abstract LocationPaths autoBuild();
 
-    /** Build a new LocationPath. */
     public LocationPath build() {
       LocationPaths built = autoBuild();
-      return LocationPath.of(
-          String.format("projects/%s/locations/%s", built.projectNumber().value(), built.zone()));
+      return LocationPath.of(ProjectPath.of(built.project()), built.location());
     }
   }
-
-  private static void checkSplits(String[] splits) throws StatusException {
-    checkArgument(splits.length == 4);
-    checkArgument(splits[0].equals("projects"));
-    checkArgument(splits[2].equals("locations"));
-  }
-
-  /** Check that the provided LocationPath is valid. */
-  public static void check(LocationPath path) throws StatusException {
-    ProjectNumber unusedProjectNumber = getProjectNumber(path);
-    CloudZone unusedZone = getZone(path);
-  }
-
-  /** Get the ProjectNumber from a LocationPath. */
-  public static ProjectNumber getProjectNumber(LocationPath path) throws StatusException {
-    String[] splits = path.value().split("/");
-    checkSplits(splits);
-    try {
-      return ProjectNumber.of(Long.parseLong(splits[1]));
-    } catch (NumberFormatException e) {
-      throw toCanonical(e);
-    }
-  }
-
-  /** Get the CloudZone from a LocationPath. */
-  public static CloudZone getZone(LocationPath path) throws StatusException {
-    String[] splits = path.value().split("/");
-    checkSplits(splits);
-    return CloudZone.parse(splits[3]);
-  }
 }

google-cloud-pubsublite/src/main/java/com/google/cloud/pubsublite/PartitionLookupUtils.java

@@ -33,7 +33,7 @@ public static int numPartitions(TopicPath topic) throws StatusException {
     try (AdminClient client =
         AdminClient.create(
             AdminClientSettings.newBuilder()
-                .setRegion(TopicPaths.getZone(topic).region())
+                .setRegion(topic.location().location().region())
                 .build())) {;
       return numPartitions(topic, client);
     } catch (Exception e) {
@@ -67,7 +67,7 @@ public static int numPartitions(SubscriptionPath subscription) throws StatusExce
     try (AdminClient client =
         AdminClient.create(
             AdminClientSettings.newBuilder()
-                .setRegion(SubscriptionPaths.getZone(subscription).region())
+                .setRegion(subscription.location().location().region())
                 .build())) {
       return numPartitions(subscription, client);
     } catch (Exception e) {
@@ -83,7 +83,7 @@ public static int numPartitions(SubscriptionPath subscription, AdminClient clien
       throws StatusException {
     ApiFuture<Subscription> subscriptionFuture = client.getSubscription(subscription);
     try {
-      return numPartitions(TopicPath.of(subscriptionFuture.get().getTopic()), client);
+      return numPartitions(TopicPath.parse(subscriptionFuture.get().getTopic()), client);
     } catch (ExecutionException e) {
       throw ExtractStatus.toCanonical(e.getCause());
     } catch (Throwable t) {

google-cloud-pubsublite/src/main/java/com/google/cloud/pubsublite/ProjectBuilderHelper.java

@@ -0,0 +1,31 @@
+/*
+ * Copyright 2020 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.cloud.pubsublite;
+
+abstract class ProjectBuilderHelper<Builder extends ProjectBuilderHelper<Builder>> {
+  public abstract Builder setProject(ProjectIdOrNumber project);
+
+  public Builder setProject(ProjectId project) {
+    setProject(ProjectIdOrNumber.of(project));
+    return (Builder) this;
+  }
+
+  public Builder setProject(ProjectNumber project) {
+    setProject(ProjectIdOrNumber.of(project));
+    return (Builder) this;
+  }
+}

google-cloud-pubsublite/src/main/java/com/google/cloud/pubsublite/ProjectId.java

@@ -0,0 +1,38 @@
+/*
+ * Copyright 2020 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.cloud.pubsublite;
+
+import static com.google.cloud.pubsublite.internal.Preconditions.checkArgument;
+
+import com.google.auto.value.AutoValue;
+import io.grpc.StatusException;
+import java.io.Serializable;
+
+@AutoValue
+public abstract class ProjectId implements Serializable {
+  public abstract String value();
+
+  @Override
+  public String toString() {
+    return value();
+  }
+
+  public static ProjectId of(String value) throws StatusException {
+    checkArgument(!value.isEmpty());
+    return new AutoValue_ProjectId(value);
+  }
+}

google-cloud-pubsublite/src/main/java/com/google/cloud/pubsublite/ProjectIdOrNumber.java

@@ -0,0 +1,54 @@
+/*
+ * Copyright 2020 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.cloud.pubsublite;
+
+import com.google.auto.value.AutoOneOf;
+import java.io.Serializable;
+
+@AutoOneOf(ProjectIdOrNumber.Kind.class)
+public abstract class ProjectIdOrNumber implements Serializable {
+  enum Kind {
+    NAME,
+    NUMBER
+  }
+
+  public abstract Kind getKind();
+
+  public abstract ProjectId name();
+
+  public abstract ProjectNumber number();
+
+  public static ProjectIdOrNumber of(ProjectId name) {
+    return AutoOneOf_ProjectIdOrNumber.name(name);
+  }
+
+  public static ProjectIdOrNumber of(ProjectNumber number) {
+    return AutoOneOf_ProjectIdOrNumber.number(number);
+  }
+
+  @Override
+  public String toString() {
+    switch (getKind()) {
+      case NAME:
+        return name().toString();
+      case NUMBER:
+        return number().toString();
+      default:
+        throw new RuntimeException("Unknown case for ProjectIdOrNumber.");
+    }
+  }
+}

google-cloud-pubsublite/src/main/java/com/google/cloud/pubsublite/ProjectLookupUtils.java

@@ -0,0 +1,59 @@
+/*
+ * Copyright 2020 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.cloud.pubsublite;
+
+import com.google.cloud.pubsublite.ProjectIdOrNumber.Kind;
+import com.google.cloud.pubsublite.internal.ExtractStatus;
+import com.google.cloud.resourcemanager.Project;
+import com.google.cloud.resourcemanager.ResourceManager;
+import com.google.cloud.resourcemanager.ResourceManagerOptions;
+import io.grpc.StatusException;
+
+public class ProjectLookupUtils {
+  private static final ResourceManager resourceManager =
+      ResourceManagerOptions.getDefaultInstance().getService();
+
+  private static ProjectNumber getProjectNumber(ProjectId id) throws StatusException {
+    try {
+      Project project = resourceManager.get(id.toString());
+      return ProjectNumber.of(project.getProjectNumber());
+    } catch (Throwable t) {
+      throw ExtractStatus.toCanonical(t);
+    }
+  }
+
+  public static ProjectPath toCannonical(ProjectPath project) throws StatusException {
+    if (project.project().getKind() == Kind.NUMBER) return project;
+    return ProjectPath.of(ProjectIdOrNumber.of(getProjectNumber(project.project().name())));
+  }
+
+  public static LocationPath toCannonical(LocationPath location) throws StatusException {
+    ProjectPath canonicalProject = toCannonical(location.project());
+    return LocationPath.of(canonicalProject, location.location());
+  }
+
+  public static SubscriptionPath toCannonical(SubscriptionPath subscription)
+      throws StatusException {
+    LocationPath canonicalLocation = toCannonical(subscription.location());
+    return SubscriptionPath.of(canonicalLocation, subscription.name());
+  }
+
+  public static TopicPath toCannonical(TopicPath topic) throws StatusException {
+    LocationPath canonicalLocation = toCannonical(topic.location());
+    return TopicPath.of(canonicalLocation, topic.name());
+  }
+}