Skip to content

Commit 80094ce

Browse files
davidhcoedavidhcoe
authored
feat(csharp/src/Drivers/BigQuery): choose the first project ID if not specified (apache#2541)
- The call to BigQueryClient.CreateQueryJob fails if a project ID is not present (even though the `*detect-project-id*` value is passed). This change locates the first project ID in the list of project IDs and uses it if no project ID is specified. - Adds support for multiple BigQuery test environments. Includes some refactoring for of Apache.Arrow.Adbc.Tests.Drivers.Interop.FlightSql to move common functionality to a shared library. --------- Co-authored-by: David Coe <>
1 parent dc0bfca commit 80094ce

File tree

14 files changed

+520
-335
lines changed

14 files changed

+520
-335
lines changed

‎csharp/src/Drivers/BigQuery/BigQueryStatement.cs‎

Lines changed: 44 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,8 @@ public BigQueryStatement(BigQueryClient client, GoogleCredential credential)
5151

5252
public override QueryResult ExecuteQuery()
5353
{
54-
QueryOptions? queryOptions = ValidateOptions();
54+
QueryOptions queryOptions = ValidateOptions();
55+
5556
BigQueryJob job = this.client.CreateQueryJob(SqlQuery, null, queryOptions);
5657

5758
GetQueryResultsOptions getQueryResultsOptions = new GetQueryResultsOptions();
@@ -122,7 +123,26 @@ public override QueryResult ExecuteQuery()
122123

123124
public override UpdateResult ExecuteUpdate()
124125
{
125-
BigQueryResults result = this.client.ExecuteQuery(SqlQuery, parameters: null);
126+
QueryOptions options = ValidateOptions();
127+
GetQueryResultsOptions getQueryResultsOptions = new GetQueryResultsOptions();
128+
129+
if (this.Options?.TryGetValue(BigQueryParameters.GetQueryResultsOptionsTimeoutMinutes, out string? timeoutMinutes) == true)
130+
{
131+
if (int.TryParse(timeoutMinutes, out int minutes))
132+
{
133+
if (minutes >= 0)
134+
{
135+
getQueryResultsOptions.Timeout = TimeSpan.FromMinutes(minutes);
136+
}
137+
}
138+
}
139+
140+
BigQueryResults result = this.client.ExecuteQuery(
141+
SqlQuery,
142+
parameters: null,
143+
queryOptions: options,
144+
resultsOptions: getQueryResultsOptions);
145+
126146
long updatedRows = result.NumDmlAffectedRows == null ? -1L : result.NumDmlAffectedRows.Value;
127147

128148
return new UpdateResult(updatedRows);
@@ -208,13 +228,31 @@ static IArrowReader ReadChunk(BigQueryReadClient readClient, string streamName)
208228
return new ArrowStreamReader(stream);
209229
}
210230

211-
private QueryOptions? ValidateOptions()
231+
private QueryOptions ValidateOptions()
212232
{
213-
if (this.Options == null || this.Options.Count == 0)
214-
return null;
215-
216233
QueryOptions options = new QueryOptions();
217234

235+
if (this.client.ProjectId == BigQueryConstants.DetectProjectId)
236+
{
237+
// An error occurs when calling CreateQueryJob without the ID set,
238+
// so use the first one that is found. This does not prevent from calling
239+
// to other 'project IDs' (catalogs) with a query.
240+
PagedEnumerable<ProjectList, CloudProject>? projects = this.client.ListProjects();
241+
242+
if (projects != null)
243+
{
244+
string? firstProjectId = projects.Select(x => x.ProjectId).FirstOrDefault();
245+
246+
if (firstProjectId != null)
247+
{
248+
options.ProjectId = firstProjectId;
249+
}
250+
}
251+
}
252+
253+
if (this.Options == null || this.Options.Count == 0)
254+
return options;
255+
218256
foreach (KeyValuePair<string, string> keyValuePair in this.Options)
219257
{
220258
if (keyValuePair.Key == BigQueryParameters.AllowLargeResults)

‎csharp/test/Apache.Arrow.Adbc.Tests/Apache.Arrow.Adbc.Tests.csproj‎

Lines changed: 3 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -26,18 +26,9 @@
2626
</ItemGroup>
2727

2828
<Target Name="CopyDuckDb" AfterTargets="Build" Condition="'$(PkgDuckDB_NET_Bindings_Full)' != ''">
29-
<Copy
30-
Condition="'$([System.Runtime.InteropServices.RuntimeInformation]::IsOSPlatform($([System.Runtime.InteropServices.OSPlatform]::Windows)))' == 'true'"
31-
SourceFiles="$(PkgDuckDB_NET_Bindings_Full)\runtimes\win-$(ProcessArchitecture)\native\duckdb.dll"
32-
DestinationFolder="$(OutputPath)" />
33-
<Copy
34-
Condition="'$([System.Runtime.InteropServices.RuntimeInformation]::IsOSPlatform($([System.Runtime.InteropServices.OSPlatform]::Linux)))' == 'true'"
35-
SourceFiles="$(PkgDuckDB_NET_Bindings_Full)\runtimes\linux-$(ProcessArchitecture)\native\libduckdb.so"
36-
DestinationFolder="$(OutputPath)" />
37-
<Copy
38-
Condition="'$([System.Runtime.InteropServices.RuntimeInformation]::IsOSPlatform($([System.Runtime.InteropServices.OSPlatform]::OSX)))' == 'true'"
39-
SourceFiles="$(PkgDuckDB_NET_Bindings_Full)\runtimes\osx\native\libduckdb.dylib"
40-
DestinationFolder="$(OutputPath)" />
29+
<Copy Condition="'$([System.Runtime.InteropServices.RuntimeInformation]::IsOSPlatform($([System.Runtime.InteropServices.OSPlatform]::Windows)))' == 'true'" SourceFiles="$(PkgDuckDB_NET_Bindings_Full)\runtimes\win-$(ProcessArchitecture)\native\duckdb.dll" DestinationFolder="$(OutputPath)" />
30+
<Copy Condition="'$([System.Runtime.InteropServices.RuntimeInformation]::IsOSPlatform($([System.Runtime.InteropServices.OSPlatform]::Linux)))' == 'true'" SourceFiles="$(PkgDuckDB_NET_Bindings_Full)\runtimes\linux-$(ProcessArchitecture)\native\libduckdb.so" DestinationFolder="$(OutputPath)" />
31+
<Copy Condition="'$([System.Runtime.InteropServices.RuntimeInformation]::IsOSPlatform($([System.Runtime.InteropServices.OSPlatform]::OSX)))' == 'true'" SourceFiles="$(PkgDuckDB_NET_Bindings_Full)\runtimes\osx\native\libduckdb.dylib" DestinationFolder="$(OutputPath)" />
4132
</Target>
4233

4334
</Project>

‎csharp/test/Apache.Arrow.Adbc.Tests/MultiEnvironmentTestConfiguration.cs‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
using System.Collections.Generic;
19+
using System.Text.Json.Serialization;
20+
21+
namespace Apache.Arrow.Adbc.Tests
22+
{
23+
public abstract class MultiEnvironmentTestConfiguration<T>
24+
where T : TestConfiguration
25+
{
26+
/// <summary>
27+
/// List of test environments that are used when running tests.
28+
/// </summary>
29+
/// <remarks>
30+
/// Multiple environments can be defined in the Environments dictionary. This is an array
31+
/// of testable environment names in the JSON file:
32+
/// "testEnvironments": [ "Environment_A", "Environment_B", "Environment_C" ]
33+
///
34+
/// These names match the keys in the Environments dictionary.
35+
/// </remarks>
36+
[JsonPropertyName("testEnvironments")]
37+
public List<string> TestEnvironmentNames { get; set; } = new List<string>();
38+
39+
/// <summary>
40+
/// Contains the configured environments where the key is the name of the environment
41+
/// and the value is the test configuration for that environment.
42+
/// </summary>
43+
[JsonPropertyName("environments")]
44+
public Dictionary<string, T> Environments { get; set; } = new Dictionary<string, T>();
45+
}
46+
}

‎csharp/test/Apache.Arrow.Adbc.Tests/MultiEnvironmentTestUtils.cs‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
using System;
19+
using System.Collections.Generic;
20+
using System.IO;
21+
using System.Text.Json;
22+
23+
namespace Apache.Arrow.Adbc.Tests
24+
{
25+
public static class MultiEnvironmentTestUtils
26+
{
27+
public static T LoadMultiEnvironmentTestConfiguration<T>(string environmentVariable)
28+
{
29+
T? testConfiguration = default(T);
30+
31+
if (!string.IsNullOrWhiteSpace(environmentVariable))
32+
{
33+
string? environmentValue = Environment.GetEnvironmentVariable(environmentVariable);
34+
35+
if (!string.IsNullOrWhiteSpace(environmentValue))
36+
{
37+
if (File.Exists(environmentValue))
38+
{
39+
// use a JSON file for the various settings
40+
string json = File.ReadAllText(environmentValue);
41+
42+
testConfiguration = JsonSerializer.Deserialize<T>(json)!;
43+
}
44+
}
45+
}
46+
47+
if (testConfiguration == null)
48+
throw new InvalidOperationException($"Cannot execute test configuration from environment variable `{environmentVariable}`");
49+
50+
return testConfiguration;
51+
}
52+
53+
public static List<TEnvironment> GetTestEnvironments<TEnvironment>(MultiEnvironmentTestConfiguration<TEnvironment> testConfiguration)
54+
where TEnvironment : TestConfiguration
55+
{
56+
if (testConfiguration == null)
57+
throw new ArgumentNullException(nameof(testConfiguration));
58+
59+
if (testConfiguration.Environments == null || testConfiguration.Environments.Count == 0)
60+
throw new InvalidOperationException("There are no environments configured");
61+
62+
List<TEnvironment> environments = new List<TEnvironment>();
63+
64+
foreach (string environmentName in GetEnvironmentNames(testConfiguration.TestEnvironmentNames))
65+
{
66+
if (testConfiguration.Environments.TryGetValue(environmentName, out TEnvironment? testEnvironment))
67+
{
68+
if (testEnvironment != null)
69+
{
70+
testEnvironment.Name = environmentName;
71+
environments.Add(testEnvironment);
72+
}
73+
}
74+
}
75+
76+
if (environments.Count == 0)
77+
throw new InvalidOperationException("Could not find a configured environment to execute the tests");
78+
79+
return environments;
80+
}
81+
82+
private static List<string> GetEnvironmentNames(List<string> names)
83+
{
84+
if (names == null)
85+
return new List<string>();
86+
87+
return names;
88+
}
89+
}
90+
}

‎csharp/test/Apache.Arrow.Adbc.Tests/TestConfiguration.cs‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,11 @@ namespace Apache.Arrow.Adbc.Tests
2525
/// </summary>
2626
public abstract class TestConfiguration : ICloneable
2727
{
28+
/// <summary>
29+
/// Optional. The name of the environment.
30+
/// </summary>
31+
public string? Name { get; set; }
32+
2833
/// <summary>
2934
/// The query to run.
3035
/// </summary>

‎csharp/test/Drivers/BigQuery/BigQueryTestConfiguration.cs‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,16 +15,24 @@
1515
* limitations under the License.
1616
*/
1717

18+
using System.Collections.Generic;
1819
using System.Text.Json.Serialization;
1920

2021
namespace Apache.Arrow.Adbc.Tests.Drivers.BigQuery
2122
{
2223
/// <summary>
2324
/// Configuration settings for working with BigQuery.
2425
/// </summary>
25-
internal class BigQueryTestConfiguration : TestConfiguration
26+
internal class BigQueryTestConfiguration : MultiEnvironmentTestConfiguration<BigQueryTestEnvironment>
2627
{
27-
public BigQueryTestConfiguration()
28+
}
29+
30+
/// <summary>
31+
/// Configuration environments for working with BigQuery.
32+
/// </summary>
33+
internal class BigQueryTestEnvironment : TestConfiguration
34+
{
35+
public BigQueryTestEnvironment()
2836
{
2937
AllowLargeResults = false;
3038
IncludeTableConstraints = true;

‎csharp/test/Drivers/BigQuery/BigQueryTestingUtils.cs‎

Lines changed: 28 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -28,16 +28,16 @@ internal class BigQueryTestingUtils
2828
internal const string BIGQUERY_TEST_CONFIG_VARIABLE = "BIGQUERY_TEST_CONFIG_FILE";
2929

3030
/// <summary>
31-
/// Gets a the BigQuery ADBC driver with settings from the <see cref="BigQueryTestConfiguration"/>.
31+
/// Gets a the BigQuery ADBC driver with settings from the <see cref="BigQueryTestEnvironment"/>.
3232
/// </summary>
33-
/// <param name="testConfiguration"><see cref="BigQueryTestConfiguration"/></param>
33+
/// <param name="testEnvironment"><see cref="BigQueryTestEnvironment"/></param>
3434
/// <param name="parameters"></param>
3535
/// <returns></returns>
3636
internal static AdbcConnection GetBigQueryAdbcConnection(
37-
BigQueryTestConfiguration testConfiguration
37+
BigQueryTestEnvironment testEnvironment
3838
)
3939
{
40-
Dictionary<string, string> parameters = GetBigQueryParameters(testConfiguration);
40+
Dictionary<string, string> parameters = GetBigQueryParameters(testEnvironment);
4141
AdbcDatabase database = new BigQueryDriver().Open(parameters);
4242
AdbcConnection connection = database.Connect(new Dictionary<string, string>());
4343

@@ -47,57 +47,57 @@ BigQueryTestConfiguration testConfiguration
4747
/// <summary>
4848
/// Gets the parameters for connecting to BigQuery.
4949
/// </summary>
50-
/// <param name="testConfiguration"><see cref="BigQueryTestConfiguration"/></param>
50+
/// <param name="testEnvironment"><see cref="BigQueryTestEnvironment"/></param>
5151
/// <returns></returns>
52-
internal static Dictionary<string, string> GetBigQueryParameters(BigQueryTestConfiguration testConfiguration)
52+
internal static Dictionary<string, string> GetBigQueryParameters(BigQueryTestEnvironment testEnvironment)
5353
{
5454
Dictionary<string, string> parameters = new Dictionary<string, string>{};
5555

56-
if (!string.IsNullOrEmpty(testConfiguration.ProjectId))
56+
if (!string.IsNullOrEmpty(testEnvironment.ProjectId))
5757
{
58-
parameters.Add(BigQueryParameters.ProjectId, testConfiguration.ProjectId!);
58+
parameters.Add(BigQueryParameters.ProjectId, testEnvironment.ProjectId!);
5959
}
6060

61-
if (!string.IsNullOrEmpty(testConfiguration.JsonCredential))
61+
if (!string.IsNullOrEmpty(testEnvironment.JsonCredential))
6262
{
6363
parameters.Add(BigQueryParameters.AuthenticationType, BigQueryConstants.ServiceAccountAuthenticationType);
64-
parameters.Add(BigQueryParameters.JsonCredential, testConfiguration.JsonCredential);
64+
parameters.Add(BigQueryParameters.JsonCredential, testEnvironment.JsonCredential);
6565
}
6666
else
6767
{
6868
parameters.Add(BigQueryParameters.AuthenticationType, BigQueryConstants.UserAuthenticationType);
69-
parameters.Add(BigQueryParameters.ClientId, testConfiguration.ClientId);
70-
parameters.Add(BigQueryParameters.ClientSecret, testConfiguration.ClientSecret);
71-
parameters.Add(BigQueryParameters.RefreshToken, testConfiguration.RefreshToken);
69+
parameters.Add(BigQueryParameters.ClientId, testEnvironment.ClientId);
70+
parameters.Add(BigQueryParameters.ClientSecret, testEnvironment.ClientSecret);
71+
parameters.Add(BigQueryParameters.RefreshToken, testEnvironment.RefreshToken);
7272
}
7373

74-
if (!string.IsNullOrEmpty(testConfiguration.Scopes))
74+
if (!string.IsNullOrEmpty(testEnvironment.Scopes))
7575
{
76-
parameters.Add(BigQueryParameters.Scopes, testConfiguration.Scopes);
76+
parameters.Add(BigQueryParameters.Scopes, testEnvironment.Scopes);
7777
}
7878

79-
if (testConfiguration.AllowLargeResults)
79+
if (testEnvironment.AllowLargeResults)
8080
{
81-
parameters.Add(BigQueryParameters.AllowLargeResults, testConfiguration.AllowLargeResults.ToString());
81+
parameters.Add(BigQueryParameters.AllowLargeResults, testEnvironment.AllowLargeResults.ToString());
8282
}
8383

84-
parameters.Add(BigQueryParameters.IncludeConstraintsWithGetObjects, testConfiguration.IncludeTableConstraints.ToString());
84+
parameters.Add(BigQueryParameters.IncludeConstraintsWithGetObjects, testEnvironment.IncludeTableConstraints.ToString());
8585

86-
parameters.Add(BigQueryParameters.IncludePublicProjectId, testConfiguration.IncludePublicProjectId.ToString());
86+
parameters.Add(BigQueryParameters.IncludePublicProjectId, testEnvironment.IncludePublicProjectId.ToString());
8787

88-
if (!string.IsNullOrEmpty(testConfiguration.LargeResultsDestinationTable))
88+
if (!string.IsNullOrEmpty(testEnvironment.LargeResultsDestinationTable))
8989
{
90-
parameters.Add(BigQueryParameters.LargeResultsDestinationTable, testConfiguration.LargeResultsDestinationTable);
90+
parameters.Add(BigQueryParameters.LargeResultsDestinationTable, testEnvironment.LargeResultsDestinationTable);
9191
}
9292

93-
if (testConfiguration.TimeoutMinutes.HasValue)
93+
if (testEnvironment.TimeoutMinutes.HasValue)
9494
{
95-
parameters.Add(BigQueryParameters.GetQueryResultsOptionsTimeoutMinutes, testConfiguration.TimeoutMinutes.Value.ToString());
95+
parameters.Add(BigQueryParameters.GetQueryResultsOptionsTimeoutMinutes, testEnvironment.TimeoutMinutes.Value.ToString());
9696
}
9797

98-
if (testConfiguration.MaxStreamCount.HasValue)
98+
if (testEnvironment.MaxStreamCount.HasValue)
9999
{
100-
parameters.Add(BigQueryParameters.MaxFetchConcurrency, testConfiguration.MaxStreamCount.Value.ToString());
100+
parameters.Add(BigQueryParameters.MaxFetchConcurrency, testEnvironment.MaxStreamCount.Value.ToString());
101101
}
102102

103103
return parameters;
@@ -106,8 +106,8 @@ internal static Dictionary<string, string> GetBigQueryParameters(BigQueryTestCon
106106
/// <summary>
107107
/// Parses the queries from resources/BigQueryData.sql
108108
/// </summary>
109-
/// <param name="testConfiguration"><see cref="BigQueryTestConfiguration"/></param>
110-
internal static string[] GetQueries(BigQueryTestConfiguration testConfiguration)
109+
/// <param name="testEnvironment"><see cref="BigQueryTestEnvironment"/></param>
110+
internal static string[] GetQueries(BigQueryTestEnvironment testEnvironment)
111111
{
112112
// get past the license header
113113
StringBuilder content = new StringBuilder();
@@ -122,7 +122,7 @@ internal static string[] GetQueries(BigQueryTestConfiguration testConfiguration)
122122
{
123123
if (line.Contains(placeholder))
124124
{
125-
string modifiedLine = line.Replace(placeholder, $"{testConfiguration.Metadata.Catalog}.{testConfiguration.Metadata.Schema}.{testConfiguration.Metadata.Table}");
125+
string modifiedLine = line.Replace(placeholder, $"{testEnvironment.Metadata.Catalog}.{testEnvironment.Metadata.Schema}.{testEnvironment.Metadata.Table}");
126126

127127
content.AppendLine(modifiedLine);
128128
}

0 commit comments

Comments
 (0)