add default project ID

Author: David Coe

csharp/src/Drivers/BigQuery/BigQueryConnection.cs

@@ -41,6 +41,7 @@ public class BigQueryConnection : AdbcConnection
         readonly IReadOnlyDictionary<string, string> properties;
         BigQueryClient? client;
         GoogleCredential? credential;
+        bool includePublicProjectIds = false;
 
         const string infoDriverName = "ADBC BigQuery Driver";
         const string infoDriverVersion = "1.0.0";
@@ -81,8 +82,15 @@ internal BigQueryClient Open()
 
             // TODO: handle token expiration
 
+            // if the caller doesn't specify a projectId, use the default
             if (!this.properties.TryGetValue(BigQueryParameters.ProjectId, out projectId))
-                throw new ArgumentException($"The {BigQueryParameters.ProjectId} parameter is not present");
+                projectId = BigQueryConstants.DetectProjectId;
+
+            if (this.properties.TryGetValue(BigQueryParameters.IncludePublicProjectIds, out string? result))
+            {
+                if (!string.IsNullOrEmpty(result))
+                    includePublicProjectIds = Convert.ToBoolean(result);
+            }
 
             if (this.properties.TryGetValue(BigQueryParameters.AuthenticationType, out string? newAuthenticationType))
             {
@@ -303,16 +311,26 @@ private IArrowArray[] GetCatalogs(
         {
             StringArray.Builder catalogNameBuilder = new StringArray.Builder();
             List<IArrowArray?> catalogDbSchemasValues = new List<IArrowArray?>();
+
             string catalogRegexp = PatternToRegEx(catalogPattern);
             PagedEnumerable<ProjectList, CloudProject>? catalogs = this.client?.ListProjects();
 
             if (catalogs != null)
             {
-                foreach (CloudProject catalog in catalogs)
+                List<string> projectIds = catalogs.Select(x => x.ProjectId).ToList();
+
+                string publicProjectId = "bigquery-public-data";
+
+                if (this.includePublicProjectIds && !projectIds.Contains(publicProjectId))
+                    projectIds.Add(publicProjectId);
+
+                projectIds.Sort();
+
+                foreach (string projectId in projectIds)
                 {
-                    if (Regex.IsMatch(catalog.ProjectId, catalogRegexp, RegexOptions.IgnoreCase))
+                    if (Regex.IsMatch(projectId, catalogRegexp, RegexOptions.IgnoreCase))
                     {
-                        catalogNameBuilder.Append(catalog.ProjectId);
+                        catalogNameBuilder.Append(projectId);
 
                         if (depth == GetObjectsDepth.Catalogs)
                         {
@@ -321,7 +339,7 @@ private IArrowArray[] GetCatalogs(
                         else
                         {
                             catalogDbSchemasValues.Add(GetDbSchemas(
-                                depth, catalog.ProjectId, dbSchemaPattern,
+                                depth, projectId, dbSchemaPattern,
                                 tableNamePattern, tableTypes, columnNamePattern));
                         }
                     }
@@ -333,6 +351,7 @@ private IArrowArray[] GetCatalogs(
                 catalogNameBuilder.Build(),
                 catalogDbSchemasValues.BuildListArrayForType(new StructType(StandardSchemas.DbSchemaSchema)),
             };
+
             StandardSchemas.GetObjectsSchema.Validate(dataArrays);
 
             return dataArrays;
@@ -994,7 +1013,7 @@ public override AdbcStatement CreateStatement()
         {
             if (this.credential == null)
             {
-                throw new InvalidOperationException();
+                throw new InvalidOperationException("A credential must be set");
             }
 
             if (this.client == null)

csharp/src/Drivers/BigQuery/BigQueryParameters.cs

@@ -36,6 +36,7 @@ public class BigQueryParameters
         public const string IncludeConstraintsWithGetObjects = "adbc.bigquery.include_constraints_getobjects";
         public const string GetQueryResultsOptionsTimeoutMinutes = "adbc.bigquery.get_query_results_options.timeout";
         public const string MaxFetchConcurrency = "adbc.bigquery.max_fetch_concurrency";
+        public const string IncludePublicProjectIds = "adbc.bigquery.include_public_project_ids";
     }
 
     /// <summary>
@@ -47,5 +48,8 @@ public class BigQueryConstants
         public const string ServiceAccountAuthenticationType = "service";
         public const string TokenEndpoint = "https://accounts.google.com/o/oauth2/token";
         public const string TreatLargeDecimalAsString = "true";
+
+        // default value per https://pkg.go.dev/cloud.google.com/go/bigquery#section-readme
+        public const string DetectProjectId = "*detect-project-id*";
     }
 }

csharp/src/Drivers/BigQuery/readme.md

@@ -64,7 +64,10 @@ https://cloud.google.com/dotnet/docs/reference/Google.Cloud.BigQuery.V2/latest/G
     Optional. Sets the [DestinationTable](https://cloud.google.com/dotnet/docs/reference/Google.Cloud.BigQuery.V2/latest/Google.Cloud.BigQuery.V2.QueryOptions#Google_Cloud_BigQuery_V2_QueryOptions_DestinationTable) value of the QueryOptions if configured. Expects the format to be `{projectId}.{datasetId}.{tableId}` to set the corresponding values in the [TableReference](https://github.com/googleapis/google-api-dotnet-client/blob/6c415c73788b848711e47c6dd33c2f93c76faf97/Src/Generated/Google.Apis.Bigquery.v2/Google.Apis.Bigquery.v2.cs#L9348) class.
 
 **adbc.bigquery.project_id**<br>
-&nbsp;&nbsp;&nbsp;&nbsp;The [Project ID](https://cloud.google.com/resource-manager/docs/creating-managing-projects) used for accessing BigQuery.
+&nbsp;&nbsp;&nbsp;&nbsp;The [Project ID](https://cloud.google.com/resource-manager/docs/creating-managing-projects) used for accessing BigQuery. If not specified, will default to detect the projectIds the credentials have access to.
+
+**adbc.bigquery.include_public_project_ids**<br>
+&nbsp;&nbsp;&nbsp;&nbsp;Include the `bigquery-public-data` project ID with the list of project IDs.
 
 **adbc.bigquery.refresh_token**<br>
 &nbsp;&nbsp;&nbsp;&nbsp;The refresh token used for when the generated OAuth token expires. Required for `user` authentication.

csharp/test/Drivers/BigQuery/BigQueryTestConfiguration.cs

@@ -31,7 +31,7 @@ public BigQueryTestConfiguration()
         }
 
         [JsonPropertyName("projectId")]
-        public string ProjectId { get; set; } = string.Empty;
+        public string? ProjectId { get; set; }
 
         [JsonPropertyName("clientId")]
         public string ClientId { get; set; } = string.Empty;
@@ -57,6 +57,9 @@ public BigQueryTestConfiguration()
         [JsonPropertyName("includeTableConstraints")]
         public bool IncludeTableConstraints { get; set; }
 
+        [JsonPropertyName("includePublicProjectIds")]
+        public bool IncludePublicProjectIds { get; set; } = false;
+
         [JsonPropertyName("timeoutMinutes")]
         public int? TimeoutMinutes { get; set; }
 

csharp/test/Drivers/BigQuery/BigQueryTestingUtils.cs

@@ -51,10 +51,12 @@ BigQueryTestConfiguration testConfiguration
         /// <returns></returns>
         internal static Dictionary<string, string> GetBigQueryParameters(BigQueryTestConfiguration testConfiguration)
         {
-            Dictionary<string, string> parameters = new Dictionary<string, string>
+            Dictionary<string, string> parameters = new Dictionary<string, string>{};
+
+            if (!string.IsNullOrEmpty(testConfiguration.ProjectId))
             {
-               { BigQueryParameters.ProjectId, testConfiguration.ProjectId },
-            };
+                parameters.Add(BigQueryParameters.ProjectId, testConfiguration.ProjectId!);
+            }
 
             if (!string.IsNullOrEmpty(testConfiguration.JsonCredential))
             {
@@ -81,6 +83,8 @@ internal static Dictionary<string, string> GetBigQueryParameters(BigQueryTestCon
 
             parameters.Add(BigQueryParameters.IncludeConstraintsWithGetObjects, testConfiguration.IncludeTableConstraints.ToString());
 
+            parameters.Add(BigQueryParameters.IncludePublicProjectIds, testConfiguration.IncludePublicProjectIds.ToString());
+
             if (!string.IsNullOrEmpty(testConfiguration.LargeResultsDestinationTable))
             {
                 parameters.Add(BigQueryParameters.LargeResultsDestinationTable, testConfiguration.LargeResultsDestinationTable);

csharp/test/Drivers/BigQuery/DriverTests.cs

@@ -23,6 +23,7 @@
 using Apache.Arrow.Adbc.Tests.Xunit;
 using Apache.Arrow.Ipc;
 using Xunit;
+using Xunit.Abstractions;
 
 namespace Apache.Arrow.Adbc.Tests.Drivers.BigQuery
 {
@@ -37,12 +38,14 @@ namespace Apache.Arrow.Adbc.Tests.Drivers.BigQuery
     public class DriverTests
     {
         BigQueryTestConfiguration _testConfiguration;
+        readonly ITestOutputHelper? outputHelper;
 
-        public DriverTests()
+        public DriverTests(ITestOutputHelper? outputHelper)
         {
+            this.outputHelper = outputHelper;
+
             Skip.IfNot(Utils.CanExecuteTestConfig(BigQueryTestingUtils.BIGQUERY_TEST_CONFIG_VARIABLE));
             _testConfiguration = Utils.LoadTestConfiguration<BigQueryTestConfiguration>(BigQueryTestingUtils.BIGQUERY_TEST_CONFIG_VARIABLE);
-
         }
 
         /// <summary>
@@ -98,6 +101,34 @@ public void CanGetInfo()
             }
         }
 
+        /// <summary>
+        /// Validates if the driver can call GetObjects.
+        /// </summary>
+        [SkippableFact, Order(3)]
+        public void CanGetObjectsAllCatalogs()
+        {
+            // need to add the database
+
+            AdbcConnection adbcConnection = BigQueryTestingUtils.GetBigQueryAdbcConnection(_testConfiguration);
+
+            IArrowArrayStream stream = adbcConnection.GetObjects(
+                    depth: AdbcConnection.GetObjectsDepth.Catalogs,
+                    catalogPattern: null,
+                    dbSchemaPattern: null,
+                    tableNamePattern: null,
+                    tableTypes: BigQueryTableTypes.TableTypes,
+                    columnNamePattern: null);
+
+            RecordBatch recordBatch = stream.ReadNextRecordBatchAsync().Result;
+
+            List<AdbcCatalog> catalogs = GetObjectsParser.ParseCatalog(recordBatch, schemaName);
+
+            foreach (AdbcCatalog ct in catalogs)
+            {
+                this.outputHelper?.WriteLine(ct.Name);
+            }
+        }
+
         /// <summary>
         /// Validates if the driver can call GetObjects.
         /// </summary>