Skip to content

Commit 6ed7976

Browse files
jsrinnnjsrinnn
authored
fix(bigquery): Job Library fixes for InsertJob api (#12746)
1 parent 7e6cc7b commit 6ed7976

File tree

10 files changed

+150
-72
lines changed

10 files changed

+150
-72
lines changed

‎google/cloud/bigquery/v2/minimal/internal/job_configuration_query.cc‎

Lines changed: 26 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -54,30 +54,31 @@ std::string JobConfigurationQuery::DebugString(absl::string_view name,
5454
}
5555

5656
void to_json(nlohmann::json& j, JobConfigurationQuery const& c) {
57-
j = nlohmann::json{{"query", c.query},
58-
{"createDisposition", c.create_disposition},
59-
{"writeDisposition", c.write_disposition},
60-
{"priority", c.priority},
61-
{"parameterMode", c.parameter_mode},
62-
{"preserveNulls", c.preserve_nulls},
63-
{"allowLargeResults", c.allow_large_results},
64-
{"useQueryCache", c.use_query_cache},
65-
{"flattenResults", c.flatten_results},
66-
{"useLegacySql", c.use_legacy_sql},
67-
{"createSession", c.create_session},
68-
{"maximumBytesBilled", c.maximum_bytes_billed},
69-
{"queryParameters", c.query_parameters},
70-
{"schemaUpdateOptions", c.schema_update_options},
71-
{"connectionProperties", c.connection_properties},
72-
{"defaultDataset", c.default_dataset},
73-
{"destinationTable", c.destination_table},
74-
{"timePartitioning", c.time_partitioning},
75-
{"rangePartitioning", c.range_partitioning},
76-
{"clustering", c.clustering},
77-
{"destinationEncryptionConfiguration",
78-
c.destination_encryption_configuration},
79-
{"scriptOptions", c.script_options},
80-
{"systemVariables", c.system_variables}};
57+
j = nlohmann::json{
58+
{"query", c.query},
59+
{"createDisposition", c.create_disposition},
60+
{"writeDisposition", c.write_disposition},
61+
{"priority", c.priority},
62+
{"parameterMode", c.parameter_mode},
63+
{"preserveNulls", c.preserve_nulls},
64+
{"allowLargeResults", c.allow_large_results},
65+
{"useQueryCache", c.use_query_cache},
66+
{"flattenResults", c.flatten_results},
67+
{"useLegacySql", c.use_legacy_sql},
68+
{"createSession", c.create_session},
69+
{"maximumBytesBilled", std::to_string(c.maximum_bytes_billed)},
70+
{"queryParameters", c.query_parameters},
71+
{"schemaUpdateOptions", c.schema_update_options},
72+
{"connectionProperties", c.connection_properties},
73+
{"defaultDataset", c.default_dataset},
74+
{"destinationTable", c.destination_table},
75+
{"timePartitioning", c.time_partitioning},
76+
{"rangePartitioning", c.range_partitioning},
77+
{"clustering", c.clustering},
78+
{"destinationEncryptionConfiguration",
79+
c.destination_encryption_configuration},
80+
{"scriptOptions", c.script_options},
81+
{"systemVariables", c.system_variables}};
8182
}
8283
void from_json(nlohmann::json const& j, JobConfigurationQuery& c) {
8384
SafeGetTo(c.query, j, "query");
@@ -91,7 +92,7 @@ void from_json(nlohmann::json const& j, JobConfigurationQuery& c) {
9192
SafeGetTo(c.flatten_results, j, "flattenResults");
9293
SafeGetTo(c.use_legacy_sql, j, "useLegacySql");
9394
SafeGetTo(c.create_session, j, "createSession");
94-
SafeGetTo(c.maximum_bytes_billed, j, "maximumBytesBilled");
95+
c.maximum_bytes_billed = GetNumberFromJson(j, "maximumBytesBilled");
9596
SafeGetTo(c.query_parameters, j, "queryParameters");
9697
SafeGetTo(c.schema_update_options, j, "schemaUpdateOptions");
9798
SafeGetTo(c.connection_properties, j, "connectionProperties");

‎google/cloud/bigquery/v2/minimal/internal/job_request.cc‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -227,7 +227,7 @@ StatusOr<rest_internal::RestRequest> BuildRestRequest(
227227

228228
auto const& job = json_payload.get<Job>();
229229

230-
if (job.configuration.job_type.empty() || job.id != r.job().id) {
230+
if (job.configuration.query.query.empty()) {
231231
return internal::InvalidArgumentError(
232232
"Invalid InsertJobRequest: Invalid Job object", GCP_ERROR_INFO());
233233
}

‎google/cloud/bigquery/v2/minimal/internal/job_request.h‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -227,6 +227,9 @@ class InsertJobRequest {
227227

228228
std::string const& project_id() const { return project_id_; }
229229
Job const& job() const { return job_; }
230+
std::vector<std::string> json_filter_keys() const {
231+
return json_filter_keys_;
232+
}
230233

231234
InsertJobRequest& set_project_id(std::string project_id) & {
232235
project_id_ = std::move(project_id);
@@ -244,13 +247,22 @@ class InsertJobRequest {
244247
return std::move(set_job(std::move(job)));
245248
}
246249

250+
InsertJobRequest& set_json_filter_keys(std::vector<std::string> keys) & {
251+
json_filter_keys_ = std::move(keys);
252+
return *this;
253+
}
254+
InsertJobRequest&& set_json_filter_keys(std::vector<std::string> keys) && {
255+
return std::move(set_json_filter_keys(std::move(keys)));
256+
}
257+
247258
std::string DebugString(absl::string_view name,
248259
TracingOptions const& options = {},
249260
int indent = 0) const;
250261

251262
private:
252263
std::string project_id_;
253264
Job job_;
265+
std::vector<std::string> json_filter_keys_;
254266
};
255267

256268
class CancelJobRequest {

‎google/cloud/bigquery/v2/minimal/internal/job_request_test.cc‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -314,7 +314,7 @@ TEST(InsertJobRequestTest, EmptyProjectId) {
314314
HasSubstr("Project Id is empty")));
315315
}
316316

317-
TEST(InsertJobRequestTest, Invalidjob) {
317+
TEST(InsertJobRequestTest, EmptyQuery) {
318318
InsertJobRequest request;
319319
request.set_project_id("1234");
320320

‎google/cloud/bigquery/v2/minimal/internal/job_response_test.cc‎

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -282,7 +282,7 @@ TEST(GetJobResponseTest, DebugString) {
282282
R"( create_disposition: "" write_disposition: "" priority: "")"
283283
R"( parameter_mode: "" preserve_nulls: false allow_large_results: false)"
284284
R"( use_query_cache: false flatten_results: false use_legacy_sql: false)"
285-
R"( create_session: false maximum_bytes_billed: 0)"
285+
R"( create_session: false maximum_bytes_billed: -1)"
286286
R"( default_dataset { project_id: "" dataset_id: "" } destination_table {)"
287287
R"( project_id: "" dataset_id: "" table_id: "" } time_partitioning {)"
288288
R"( type: "" expiration_time { "0" } field: "" } range_partitioning {)"
@@ -335,7 +335,7 @@ TEST(GetJobResponseTest, DebugString) {
335335
R"( create_disposition: "" write_disposition: "" priority: "")"
336336
R"( parameter_mode: "" preserve_nulls: false allow_large_results: false)"
337337
R"( use_query_cache: false flatten_results: false use_legacy_sql: false)"
338-
R"( create_session: false maximum_bytes_billed: 0)"
338+
R"( create_session: false maximum_bytes_billed: -1)"
339339
R"( default_dataset { project_id: "" dataset_id: "" } destination_table {)"
340340
R"( project_id: "" dataset_id: "" table_id: "" } time_partitioning { type: "")"
341341
R"( expiration_time { "0" } field: "" } range_partitioning { field: "" range {)"
@@ -408,7 +408,7 @@ TEST(GetJobResponseTest, DebugString) {
408408
flatten_results: false
409409
use_legacy_sql: false
410410
create_session: false
411-
maximum_bytes_billed: 0
411+
maximum_bytes_billed: -1
412412
default_dataset {
413413
project_id: ""
414414
dataset_id: ""
@@ -618,7 +618,7 @@ TEST(ListJobsResponseTest, DebugString) {
618618
R"( preserve_nulls: false allow_large_results: false)"
619619
R"( use_query_cache: false flatten_results: false)"
620620
R"( use_legacy_sql: false create_session: false)"
621-
R"( maximum_bytes_billed: 0 default_dataset { project_id: "")"
621+
R"( maximum_bytes_billed: -1 default_dataset { project_id: "")"
622622
R"( dataset_id: "" } destination_table { project_id: "")"
623623
R"( dataset_id: "" table_id: "" } time_partitioning { type: "")"
624624
R"( expiration_time { "0" } field: "" } range_partitioning {)"
@@ -672,7 +672,7 @@ TEST(ListJobsResponseTest, DebugString) {
672672
R"( write_disposition: "" priority: "" parameter_mode: "")"
673673
R"( preserve_nulls: false allow_large_results: false)"
674674
R"( use_query_cache: false flatten_results: false use_legacy_sql: false)"
675-
R"( create_session: false maximum_bytes_billed: 0)"
675+
R"( create_session: false maximum_bytes_billed: -1)"
676676
R"( default_dataset { project_id: "" dataset_id: "" } destination_table {)"
677677
R"( project_id: "" dataset_id: "" table_id: "" } time_partitioning { type: "")"
678678
R"( expiration_time { "0" } field: "" } range_partitioning { field: "" range {)"
@@ -740,7 +740,7 @@ TEST(ListJobsResponseTest, DebugString) {
740740
flatten_results: false
741741
use_legacy_sql: false
742742
create_session: false
743-
maximum_bytes_billed: 0
743+
maximum_bytes_billed: -1
744744
default_dataset {
745745
project_id: ""
746746
dataset_id: ""
@@ -942,7 +942,7 @@ TEST(InsertJobResponseTest, Success) {
942942
R"(,"defaultDataset":{"datasetId":"1","projectId":"2"})"
943943
R"(,"destinationEncryptionConfiguration":{"kmsKeyName":"encryption-key-name"})"
944944
R"(,"destinationTable":{"datasetId":"1","projectId":"2","tableId":"3"})"
945-
R"(,"flattenResults":true,"maximumBytesBilled":0,"parameterMode":"job-param-mode")"
945+
R"(,"flattenResults":true,"maximumBytesBilled":"0","parameterMode":"job-param-mode")"
946946
R"(,"preserveNulls":true,"priority":"job-priority","query":"select 1;")"
947947
R"(,"queryParameters":[{"name":"query-parameter-name")"
948948
R"(,"parameterType":{"arrayType":{"structTypes":[{"description":"array-struct-description")"
@@ -1110,7 +1110,7 @@ TEST(InsertJobResponseTest, DebugString) {
11101110
R"( create_disposition: "" write_disposition: "" priority: "")"
11111111
R"( parameter_mode: "" preserve_nulls: false allow_large_results: false)"
11121112
R"( use_query_cache: false flatten_results: false use_legacy_sql: false)"
1113-
R"( create_session: false maximum_bytes_billed: 0)"
1113+
R"( create_session: false maximum_bytes_billed: -1)"
11141114
R"( default_dataset { project_id: "" dataset_id: "" } destination_table {)"
11151115
R"( project_id: "" dataset_id: "" table_id: "" } time_partitioning {)"
11161116
R"( type: "" expiration_time { "0" } field: "" } range_partitioning {)"
@@ -1163,7 +1163,7 @@ TEST(InsertJobResponseTest, DebugString) {
11631163
R"( create_disposition: "" write_disposition: "" priority: "")"
11641164
R"( parameter_mode: "" preserve_nulls: false allow_large_results: false)"
11651165
R"( use_query_cache: false flatten_results: false use_legacy_sql: false)"
1166-
R"( create_session: false maximum_bytes_billed: 0)"
1166+
R"( create_session: false maximum_bytes_billed: -1)"
11671167
R"( default_dataset { project_id: "" dataset_id: "" } destination_table {)"
11681168
R"( project_id: "" dataset_id: "" table_id: "" } time_partitioning { type: "")"
11691169
R"( expiration_time { "0" } field: "" } range_partitioning { field: "" range {)"
@@ -1237,7 +1237,7 @@ TEST(InsertJobResponseTest, DebugString) {
12371237
flatten_results: false
12381238
use_legacy_sql: false
12391239
create_session: false
1240-
maximum_bytes_billed: 0
1240+
maximum_bytes_billed: -1
12411241
default_dataset {
12421242
project_id: ""
12431243
dataset_id: ""
@@ -1424,7 +1424,7 @@ TEST(CancelJobResponseTest, Success) {
14241424
R"(,"defaultDataset":{"datasetId":"1","projectId":"2"})"
14251425
R"(,"destinationEncryptionConfiguration":{"kmsKeyName":"encryption-key-name"})"
14261426
R"(,"destinationTable":{"datasetId":"1","projectId":"2","tableId":"3"})"
1427-
R"(,"flattenResults":true,"maximumBytesBilled":0,"parameterMode":"job-param-mode")"
1427+
R"(,"flattenResults":true,"maximumBytesBilled":"0","parameterMode":"job-param-mode")"
14281428
R"(,"preserveNulls":true,"priority":"job-priority","query":"select 1;")"
14291429
R"(,"queryParameters":[{"name":"query-parameter-name")"
14301430
R"(,"parameterType":{"arrayType":{"structTypes":[{"description":"array-struct-description")"
@@ -1611,7 +1611,7 @@ TEST(CancelJobResponseTest, DebugString) {
16111611
R"( create_disposition: "" write_disposition: "" priority: "")"
16121612
R"( parameter_mode: "" preserve_nulls: false allow_large_results: false)"
16131613
R"( use_query_cache: false flatten_results: false use_legacy_sql: false)"
1614-
R"( create_session: false maximum_bytes_billed: 0)"
1614+
R"( create_session: false maximum_bytes_billed: -1)"
16151615
R"( default_dataset { project_id: "" dataset_id: "" } destination_table {)"
16161616
R"( project_id: "" dataset_id: "" table_id: "" } time_partitioning {)"
16171617
R"( type: "" expiration_time { "0" } field: "" } range_partitioning {)"
@@ -1666,7 +1666,7 @@ TEST(CancelJobResponseTest, DebugString) {
16661666
R"( create_disposition: "" write_disposition: "" priority: "")"
16671667
R"( parameter_mode: "" preserve_nulls: false allow_large_results: false)"
16681668
R"( use_query_cache: false flatten_results: false use_legacy_sql: false)"
1669-
R"( create_session: false maximum_bytes_billed: 0)"
1669+
R"( create_session: false maximum_bytes_billed: -1)"
16701670
R"( default_dataset { project_id: "" dataset_id: "" } destination_table {)"
16711671
R"( project_id: "" dataset_id: "" table_id: "" } time_partitioning { type: "")"
16721672
R"( expiration_time { "0" } field: "" } range_partitioning { field: "" range {)"
@@ -1741,7 +1741,7 @@ TEST(CancelJobResponseTest, DebugString) {
17411741
flatten_results: false
17421742
use_legacy_sql: false
17431743
create_session: false
1744-
maximum_bytes_billed: 0
1744+
maximum_bytes_billed: -1
17451745
default_dataset {
17461746
project_id: ""
17471747
dataset_id: ""

‎google/cloud/bigquery/v2/minimal/internal/job_rest_stub.cc‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -64,11 +64,14 @@ StatusOr<InsertJobResponse> DefaultBigQueryJobRestStub::InsertJob(
6464
nlohmann::json json_payload;
6565
to_json(json_payload, request.job());
6666

67+
auto filtered_json = RemoveJsonKeysAndEmptyFields(json_payload.dump(),
68+
request.json_filter_keys());
69+
6770
// 4) Call the rest stub and parse the RestResponse.
6871
rest_internal::RestContext context;
6972
return ParseFromRestResponse<InsertJobResponse>(
7073
rest_stub_->Post(context, std::move(*rest_request),
71-
{absl::MakeConstSpan(json_payload.dump())}));
74+
{absl::MakeConstSpan(filtered_json.dump())}));
7275
}
7376

7477
StatusOr<CancelJobResponse> DefaultBigQueryJobRestStub::CancelJob(

‎google/cloud/bigquery/v2/minimal/internal/job_test.cc‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1305,7 +1305,7 @@ TEST(JobTest, JobToFromJson) {
13051305
R"(,"defaultDataset":{"datasetId":"1","projectId":"2"})"
13061306
R"(,"destinationEncryptionConfiguration":{"kmsKeyName":"encryption-key-name"})"
13071307
R"(,"destinationTable":{"datasetId":"1","projectId":"2","tableId":"3"})"
1308-
R"(,"flattenResults":true,"maximumBytesBilled":0,"parameterMode":"job-param-mode")"
1308+
R"(,"flattenResults":true,"maximumBytesBilled":"0","parameterMode":"job-param-mode")"
13091309
R"(,"preserveNulls":true,"priority":"job-priority","query":"select 1;")"
13101310
R"(,"queryParameters":[{"name":"query-parameter-name")"
13111311
R"(,"parameterType":{"arrayType":{"structTypes":[{"description":"array-struct-description")"
@@ -1430,7 +1430,7 @@ TEST(JobTest, ListFormatJobToFromJson) {
14301430
R"(,"createSession":true,"defaultDataset":{"datasetId":"1","projectId":"2"})"
14311431
R"(,"destinationEncryptionConfiguration":{"kmsKeyName":"encryption-key-name"})"
14321432
R"(,"destinationTable":{"datasetId":"1","projectId":"2","tableId":"3"})"
1433-
R"(,"flattenResults":true,"maximumBytesBilled":0,"parameterMode":"job-param-mode")"
1433+
R"(,"flattenResults":true,"maximumBytesBilled":"0","parameterMode":"job-param-mode")"
14341434
R"(,"preserveNulls":true,"priority":"job-priority","query":"select 1;")"
14351435
R"(,"queryParameters":[{"name":"query-parameter-name")"
14361436
R"(,"parameterType":{"arrayType":{"structTypes":[{"description":"array-struct-description")"

‎google/cloud/bigquery/v2/minimal/internal/json_utils.cc‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,30 @@ void ToJson(std::chrono::system_clock::time_point const& field,
8585
j[name] = std::to_string(m);
8686
}
8787

88+
nlohmann::json RemoveJsonKeysAndEmptyFields(
89+
std::string const& json_payload, std::vector<std::string> const& keys) {
90+
nlohmann::json::parser_callback_t remove_empty_call_back =
91+
[keys](int depth, nlohmann::json::parse_event_t event,
92+
nlohmann::json& parsed) {
93+
(void)depth;
94+
95+
if (event == nlohmann::json::parse_event_t::key) {
96+
auto const discard = std::any_of(
97+
keys.begin(), keys.end(),
98+
[&parsed](std::string const& key) { return parsed == key; });
99+
return !discard;
100+
}
101+
if (event == nlohmann::json::parse_event_t::object_end) {
102+
return parsed != nullptr && !parsed.empty();
103+
}
104+
if (event == nlohmann::json::parse_event_t::array_end) {
105+
return !parsed.empty();
106+
}
107+
return true;
108+
};
109+
return nlohmann::json::parse(json_payload, remove_empty_call_back, false);
110+
}
111+
88112
GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_END
89113
} // namespace bigquery_v2_minimal_internal
90114
} // namespace cloud

‎google/cloud/bigquery/v2/minimal/internal/json_utils.h‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,11 @@ void FromJson(std::chrono::hours& field, nlohmann::json const& j,
4444
void ToJson(std::chrono::hours const& field, nlohmann::json& j,
4545
char const* name);
4646

47+
// Removes not needed keys and empty arrays and objects from the json
48+
// payload.
49+
nlohmann::json RemoveJsonKeysAndEmptyFields(
50+
std::string const& json_payload, std::vector<std::string> const& keys = {});
51+
4752
// Suppress recursive clang-tidy warnings
4853
//
4954
// NOLINTBEGIN(misc-no-recursion)

0 commit comments

Comments
 (0)