Gcs host configurable

application/src/ext-test/java/org/opentripplanner/ext/datastore/gs/GsConfigurationDocTest.java

@@ -0,0 +1,78 @@
+package org.opentripplanner.ext.datastore.gs;
+
+import static org.opentripplanner.framework.application.OtpFileNames.BUILD_CONFIG_FILENAME;
+import static org.opentripplanner.framework.io.FileUtils.assertFileEquals;
+import static org.opentripplanner.framework.io.FileUtils.readFile;
+import static org.opentripplanner.framework.io.FileUtils.writeFile;
+import static org.opentripplanner.generate.doc.framework.TemplateUtil.replaceSection;
+import static org.opentripplanner.utils.text.MarkdownFormatter.HEADER_4;
+
+import java.io.File;
+import org.junit.jupiter.api.Test;
+import org.opentripplanner.generate.doc.framework.DocBuilder;
+import org.opentripplanner.generate.doc.framework.DocsTestConstants;
+import org.opentripplanner.generate.doc.framework.GeneratesDocumentation;
+import org.opentripplanner.generate.doc.framework.ParameterDetailsList;
+import org.opentripplanner.generate.doc.framework.ParameterSummaryTable;
+import org.opentripplanner.generate.doc.framework.SkipNodes;
+import org.opentripplanner.generate.doc.framework.TemplateUtil;
+import org.opentripplanner.standalone.config.BuildConfig;
+import org.opentripplanner.standalone.config.framework.json.JsonSupport;
+import org.opentripplanner.standalone.config.framework.json.NodeAdapter;
+import org.opentripplanner.test.support.ResourceLoader;
+
+@GeneratesDocumentation
+public class GsConfigurationDocTest implements DocsTestConstants {
+
+  private static final File TEMPLATE = new File(TEMPLATE_PATH, "GoogleCloudStorage.md");
+  private static final File OUT_FILE = new File(SANDBOX_USER_DOC_PATH, "GoogleCloudStorage.md");
+  private static final SkipNodes SKIP_NODES = SkipNodes.of().build();
+
+  @Test
+  public void updateMapGcStorageConfigDoc() {
+    NodeAdapter node = readMapGcStorageConfigConfig();
+
+    String template = readFile(TEMPLATE);
+    String original = readFile(OUT_FILE);
+
+    template = replaceSection(template, "config", updaterDoc(node));
+
+    writeFile(OUT_FILE, template);
+    assertFileEquals(original, OUT_FILE);
+  }
+
+  private NodeAdapter readMapGcStorageConfigConfig() {
+    var buildConfigFile = ResourceLoader.of(GsConfigurationDocTest.class).extTestResourceFile(
+      BUILD_CONFIG_FILENAME
+    );
+
+    var json = JsonSupport.jsonNodeFromPath(buildConfigFile.toPath());
+    var conf = new BuildConfig(json, buildConfigFile.toString(), false);
+    return conf.asNodeAdapter();
+  }
+
+  private String updaterDoc(NodeAdapter node) {
+    DocBuilder buf = new DocBuilder();
+    addExample(buf, node);
+    addParameterSummaryTable(buf, node.child("gsConfig"));
+    addDetailsSection(buf, node.child("gsConfig"));
+    return buf.toString();
+  }
+
+  private void addParameterSummaryTable(DocBuilder buf, NodeAdapter node) {
+    buf
+      .header(3, "Overview", null)
+      .addSection(new ParameterSummaryTable(SKIP_NODES).createTable(node).toMarkdownTable());
+  }
+
+  private void addDetailsSection(DocBuilder buf, NodeAdapter node) {
+    buf
+      .header(3, "Details", null)
+      .addSection(ParameterDetailsList.listParametersWithDetails(node, SKIP_NODES, HEADER_4));
+  }
+
+  private void addExample(DocBuilder buf, NodeAdapter node) {
+    var root = TemplateUtil.jsonExampleBuilder(node.rawNode()).build();
+    buf.header(3, "Example configuration", null).addExample("build-config.json", root);
+  }
+}

application/src/ext-test/java/org/opentripplanner/ext/datastore/gs/GsDataSourceRepositoryTest.java

@@ -6,10 +6,13 @@
 import java.net.URI;
 import org.junit.jupiter.api.Test;
 import org.opentripplanner.datastore.api.FileType;
+import org.opentripplanner.datastore.api.GsParameters;
 
 public class GsDataSourceRepositoryTest {
 
-  private final GsDataSourceRepository subject = new GsDataSourceRepository(null);
+  private final GsDataSourceRepository subject = new GsDataSourceRepository(
+    GsParameters.defaultValues()
+  );
 
   @Test
   public void description() {

application/src/ext-test/java/org/opentripplanner/ext/datastore/gs/GsIntegrationTest.java

@@ -17,6 +17,8 @@
 import org.opentripplanner.datastore.api.CompositeDataSource;
 import org.opentripplanner.datastore.api.DataSource;
 import org.opentripplanner.datastore.api.FileType;
+import org.opentripplanner.datastore.api.GsParameters;
+import org.opentripplanner.ext.datastore.gs.config.GsConfig;
 
 /**
  * This is a manual integration test to test the Google Cloud Storage integration. To set up the
@@ -35,16 +37,18 @@ public class GsIntegrationTest {
 
   private static final String CREDENTIALS_FILE =
     "<Insert path to local Google Service Credential file here>";
+  private static final String HOST = "<Insert host here>";
   private static final String BUCKET_NAME = "<Insert bucket name here>";
   private static final URI GTFS_URI = toUri(BUCKET_NAME, "gtfs.zip");
   private static final String DATA = "{ \"key\" : \"data\" }";
 
+  private static final GsParameters GS_PARAMETERS = new GsConfig(HOST, CREDENTIALS_FILE);
   private GsDataSourceRepository repo;
 
   @BeforeEach
   public void setUp() {
     // Open a repository
-    repo = new GsDataSourceRepository(CREDENTIALS_FILE);
+    repo = new GsDataSourceRepository(GS_PARAMETERS);
     repo.open();
   }
 

application/src/ext-test/resources/org/opentripplanner/ext/datastore/gs/build-config.json

@@ -0,0 +1,25 @@
+{
+  "gsConfig": {
+    "cloudServiceHost": "http://fake-gcp:4443",
+    "credentialFile": "/path/to/file"
+  },
+
+  "graph": "gs://otp-test-bucket/a/b/graph.obj",
+  "buildReportDir": "gs://otp-test-bucket/a/b/np-report",
+  "osm": [
+    {
+      "source": "gs://otp-test-bucket/a/b/northpole.pbf"
+    }
+  ],
+  "dem": [
+    {
+      "source": "gs://otp-test-bucket/a/b/northpole.dem.tif"
+    }
+  ],
+  "transitFeeds": [
+    {
+      "type": "gtfs",
+      "source": "gs://otp-test-bucket/a/b/gtfs.zip"
+    }
+  ]
+}

application/src/ext/java/org/opentripplanner/ext/datastore/gs/GsDataSourceModule.java

@@ -25,6 +25,6 @@ DataSourceRepository provideGoogleStorageDataSourceRepository(OtpDataStoreConfig
       return null;
     }
     LOG.info("Google Cloud Store Repository enabled - GS resources detected.");
-    return new GsDataSourceRepository(config.gsCredentials());
+    return new GsDataSourceRepository(config.gsParameters());
   }
 }

application/src/ext/java/org/opentripplanner/ext/datastore/gs/GsDataSourceRepository.java

@@ -12,6 +12,7 @@
 import org.opentripplanner.datastore.api.CompositeDataSource;
 import org.opentripplanner.datastore.api.DataSource;
 import org.opentripplanner.datastore.api.FileType;
+import org.opentripplanner.datastore.api.GsParameters;
 import org.opentripplanner.datastore.base.DataSourceRepository;
 import org.opentripplanner.datastore.file.ZipStreamDataSourceDecorator;
 
@@ -21,10 +22,12 @@
 public class GsDataSourceRepository implements DataSourceRepository {
 
   private final String credentialsFilename;
+  private final String host;
   private Storage storage;
 
-  public GsDataSourceRepository(String credentialsFilename) {
-    this.credentialsFilename = credentialsFilename;
+  public GsDataSourceRepository(GsParameters gsParameters) {
+    this.credentialsFilename = gsParameters.credentialFile();
+    this.host = gsParameters.host();
   }
 
   @Override
@@ -90,8 +93,13 @@ private CompositeDataSource createCompositeSource(BlobId blobId, FileType type)
   }
 
   private Storage connectToStorage() {
+    StorageOptions.Builder builder;
     try {
-      StorageOptions.Builder builder = StorageOptions.getDefaultInstance().toBuilder();
+      if (host != null) {
+        builder = StorageOptions.newBuilder().setHost(host);
+      } else {
+        builder = StorageOptions.getDefaultInstance().toBuilder();
+      }
 
       if (credentialsFilename != null) {
         GoogleCredentials credentials = GoogleCredentials.fromStream(

application/src/ext/java/org/opentripplanner/ext/datastore/gs/config/GsConfig.java

@@ -0,0 +1,64 @@
+package org.opentripplanner.ext.datastore.gs.config;
+
+import org.opentripplanner.datastore.api.GsParameters;
+import org.opentripplanner.standalone.config.framework.json.NodeAdapter;
+import org.opentripplanner.standalone.config.framework.json.OtpVersion;
+
+public class GsConfig implements GsParameters {
+
+  private final String host;
+  private final String credentialFile;
+
+  public GsConfig(String host, String credentialFile) {
+    this.host = host;
+    this.credentialFile = credentialFile;
+  }
+
+  public static GsConfig fromConfig(NodeAdapter root, String parameterName) {
+    NodeAdapter gsRoot = root
+      .of(parameterName)
+      .since(OtpVersion.V2_8)
+      .summary("Configuration for Google Cloud Storage")
+      .asObject();
+
+    String host = gsRoot
+      .of("cloudServiceHost")
+      .since(OtpVersion.V2_8)
+      .summary("Host of the Google Cloud Storage Server")
+      .description(
+        """
+        Host of the Google Cloud Storage server. In case of a real GCS Bucket this parameter can be
+        omitted. When the host differs from the usual GCS host, for example when emulating GCS in a
+        docker container for testing purposes, the host has to be specified including the port.
+        Eg: http://localhost:4443"""
+      )
+      .asString(null);
+
+    String credentialFile = gsRoot
+      .of("credentialFile")
+      .since(OtpVersion.V2_8)
+      .summary("Local file system path to Google Cloud Platform service accounts credentials file.")
+      .description(
+        """
+        The credentials are used to access GCS URLs. When using GCS from outside of Google Cloud you
+        need to provide a path the the service credentials. Environment variables in the path are
+        resolved.
+
+        This is a path to a file on the local file system, not an URI.
+        """
+      )
+      .asString(null);
+
+    return new GsConfig(host, credentialFile);
+  }
+
+  @Override
+  public String host() {
+    return host;
+  }
+
+  @Override
+  public String credentialFile() {
+    return credentialFile;
+  }
+}

application/src/main/java/org/opentripplanner/datastore/api/GsParameters.java

@@ -0,0 +1,42 @@
+package org.opentripplanner.datastore.api;
+
+import javax.annotation.Nullable;
+
+public interface GsParameters {
+  /**
+   * Host of the Google Cloud Services, including the port.
+   * <p>
+   * Optional. May return {@code null}. If the host is not set, the connection to the Google Cloud
+   * Platform is done via the default host {@code https://storage.googleapis.com:4443}.
+   *
+   */
+  @Nullable
+  String host();
+
+  /**
+   * Local file system path to Google Cloud Platform service accounts credentials file. The
+   * credentials are used to access GCS urls. When using GCS from outside of the bucket cluster you
+   * need to provide a path to the service credentials.
+   * <p>
+   * This is a path to a file on the local file system, not an URI.
+   * <p>
+   * Optional. May return {@code null}. If the credentials are not set, the connection to the Google
+   * Cloud Platform is done without any authorization.
+   */
+  @Nullable
+  String credentialFile();
+
+  static GsParameters defaultValues() {
+    return new GsParameters() {
+      @Override
+      public String host() {
+        return null;
+      }
+
+      @Override
+      public String credentialFile() {
+        return null;
+      }
+    };
+  }
+}

application/src/main/java/org/opentripplanner/datastore/api/OtpDataStoreConfig.java

@@ -18,15 +18,9 @@ public interface OtpDataStoreConfig {
   URI reportDirectory();
 
   /**
-   * Local file system path to Google Cloud Platform service accounts credentials file. The
-   * credentials is used to access GCS urls. When using GCS from outside of the bucket cluster you
-   * need to provide a path the the service credentials.
-   * <p>
-   * This is a path to a file on the local file system, not an URI.
-   * <p>
-   * Optional. May return {@code null}.
+   * Parameters used to connect to Google Cloud Storage.
    */
-  String gsCredentials();
+  GsParameters gsParameters();
 
   /**
    * List of URIs to the open street map pbf files (the pbf format is the only one supported).

application/src/main/java/org/opentripplanner/standalone/config/BuildConfig.java

@@ -21,6 +21,7 @@
 import javax.annotation.Nullable;
 import org.opentripplanner.datastore.api.OtpDataStoreConfig;
 import org.opentripplanner.ext.dataoverlay.configuration.DataOverlayConfig;
+import org.opentripplanner.ext.datastore.gs.config.GsConfig;
 import org.opentripplanner.ext.emission.config.EmissionConfig;
 import org.opentripplanner.ext.emission.parameters.EmissionParameters;
 import org.opentripplanner.ext.fares.FaresConfiguration;
@@ -137,8 +138,6 @@ public class BuildConfig implements OtpDataStoreConfig {
 
   private final Pattern demLocalFilePattern;
 
-  private final String gsCredentials;
-
   private final URI streetGraph;
 
   private final URI graph;
@@ -187,6 +186,7 @@ public class BuildConfig implements OtpDataStoreConfig {
   public final ZoneId transitModelTimeZone;
   private final List<FeedScopedId> transitRouteToStationCentroid;
   public final URI stopConsolidation;
+  private final GsConfig gsConfig;
 
   /**
    * Set all parameters from the given Jackson JSON tree, applying defaults. Supplying
@@ -557,20 +557,6 @@ that we support remote input files (cloud storage or arbitrary URLs) not all dat
       )
       .asPattern(DEFAULT_DEM_PATTERN);
 
-    gsCredentials = root
-      .of("gsCredentials")
-      .since(V2_0)
-      .summary("Local file system path to Google Cloud Platform service accounts credentials file.")
-      .description(
-        """
-        The credentials is used to access GCS urls. When using GCS from outside of Google Cloud you
-        need to provide a path the the service credentials. Environment variables in the path are
-        resolved.
-
-        This is a path to a file on the local file system, not an URI.
-        """
-      )
-      .asString(null);
     graph = root
       .of("graph")
       .since(V2_0)
@@ -625,6 +611,8 @@ that we support remote input files (cloud storage or arbitrary URLs) not all dat
 
     transferRequests = TransferRequestConfig.map(root, "transferRequests");
 
+    gsConfig = GsConfig.fromConfig(root, "gsConfig");
+
     if (logUnusedParams && LOG.isWarnEnabled()) {
       root.logAllWarnings(LOG::warn);
     }
@@ -636,8 +624,8 @@ public URI reportDirectory() {
   }
 
   @Override
-  public String gsCredentials() {
-    return gsCredentials;
+  public GsConfig gsParameters() {
+    return gsConfig;
   }
 
   @Override

application/src/test/java/org/opentripplanner/generate/doc/BuildConfigurationDocTest.java

@@ -34,6 +34,7 @@ public class BuildConfigurationDocTest {
     .skip("dataOverlay", "sandbox/DataOverlay.md")
     .skip("emission", "sandbox/Emission.md")
     .skip("fares", "sandbox/Fares.md")
+    .skip("Google Cloud Storage", "sandbox/GoogleCloudStorage.md")
     .build();
 
   /**