Skip to content

Commit fdbaa14

Browse files
mpeddada1mpeddada1
authored
Merge branch 'main' into query-support-it
2 parents 5ab7b96 + 52439c1 commit fdbaa14

20 files changed

+633
-162
lines changed

‎ci/kokoro/windows/builds/cmake.ps1‎

Lines changed: 57 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -43,9 +43,54 @@ if ($missing.count -ge 1) {
4343
$project_root = (Get-Item -Path ".\" -Verbose).FullName -replace "\\", "/"
4444
$vcpkg_root = Install-Vcpkg "${project_root}" ""
4545
$binary_dir="cmake-out/${BuildName}"
46-
# Install all dependencies from the vcpkg.json manifest file.
47-
# This mirrors the behavior of our GHA builds.
48-
& "${vcpkg_root}/vcpkg.exe" install --triplet "${env:VCPKG_TRIPLET}"
46+
47+
# The VCPKG_ROOT env var is mismatched with the vcpkg we are running,
48+
# which causes vcpkg to ignore other env vars.
49+
# We'll unset them both and pass the overlay-triplets path manually.
50+
Remove-Item env:VCPKG_ROOT -ErrorAction SilentlyContinue
51+
Remove-Item env:VCPKG_OVERLAY_TRIPLETS -ErrorAction SilentlyContinue
52+
53+
# Define the correct path to our custom triplets.
54+
$overlay_triplets_path = "${project_root}/ci/kokoro/windows/triplets"
55+
56+
# Install all dependencies, now passing the overlay path on the command line
57+
& "${vcpkg_root}/vcpkg.exe" install --triplet "${env:VCPKG_TRIPLET}" --overlay-triplets="${overlay_triplets_path}"
58+
59+
# Manually check the exit code. vcpkg might not be throwing a terminating error.
60+
if ($LastExitCode -ne 0) {
61+
Write-Host -ForegroundColor Red "----------------------------------------------------------------"
62+
Write-Host -ForegroundColor Red "vcpkg install FAILED with exit code $LastExitCode."
63+
Write-Host -ForegroundColor Red "Dumping vcpkg buildtree logs for known failing packages..."
64+
Write-Host -ForegroundColor Red "----------------------------------------------------------------"
65+
66+
# We'll check logs for *all* known problematic packages
67+
$problem_packages = @("crc32c", "yaml-cpp")
68+
69+
foreach ($pkg in $problem_packages) {
70+
Write-Host -ForegroundColor Magenta "--- Checking logs for $pkg ---"
71+
$log1 = "${vcpkg_root}/buildtrees/${pkg}/config-x64-windows-static-out.log"
72+
$log2 = "${vcpkg_root}/buildtrees/${pkg}/config-x64-windows-static-dbg-CMakeCache.txt.log"
73+
$log3 = "${vcpkg_root}/buildtrees/${pkg}/config-x64-windows-static-rel-CMakeCache.txt.log"
74+
75+
foreach ($logFile in @($log1, $log2, $log3)) {
76+
if (Test-Path $logFile) {
77+
Write-Host -ForegroundColor Red "========= Contents of $logFile ========="
78+
Get-Content $logFile
79+
Write-Host -ForegroundColor Red "========= End of $logFile ========="
80+
} else {
81+
Write-Host -ForegroundColor Yellow "Log file not found, skipping: $logFile"
82+
}
83+
}
84+
}
85+
86+
Write-Host -ForegroundColor Red "----------------------------------------------------------------"
87+
Write-Host -ForegroundColor Red "Dumping complete. Forcing build failure."
88+
Write-Host -ForegroundColor Red "----------------------------------------------------------------"
89+
# Manually fail the build with the exit code from vcpkg
90+
exit $LastExitCode
91+
}
92+
93+
Write-Host -ForegroundColor Green "vcpkg install SUCCEEDED."
4994

5095
$cmake_args=@(
5196
"-G$env:GENERATOR",
@@ -54,15 +99,23 @@ $cmake_args=@(
5499
"-DCMAKE_TOOLCHAIN_FILE=`"${vcpkg_root}/scripts/buildsystems/vcpkg.cmake`""
55100
"-DCMAKE_BUILD_TYPE=${env:CONFIG}",
56101
"-DVCPKG_TARGET_TRIPLET=${env:VCPKG_TRIPLET}",
102+
"-DVCPKG_OVERLAY_TRIPLETS=${project_root}/ci/kokoro/windows/triplets",
57103
"-DCMAKE_C_COMPILER=cl.exe",
58104
"-DCMAKE_CXX_COMPILER=cl.exe",
59105
"-DGOOGLE_CLOUD_CPP_ENABLE_WERROR=ON",
60106
"-DGOOGLE_CLOUD_CPP_ENABLE_CTYPE_CORD_WORKAROUND=ON",
61-
"-DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreaded$<$<CONFIG:Debug>:Debug>"
107+
"-DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreaded$<$<CONFIG:Debug>:Debug>",
108+
"-DGOOGLE_CLOUD_CPP_ENABLE=universe_domain"
62109
)
63110

64111
# Configure CMake and create the build directory.
65112
Write-Host -ForegroundColor Yellow "`n$(Get-Date -Format o) Configuring CMake with $cmake_args"
113+
114+
# The VCPKG_ROOT env var is mismatched with the vcpkg we are running,
115+
# which causes vcpkg to ignore other env vars.
116+
# Unset the env var just before the cmake call, which also calls vcpkg.
117+
Remove-Item env:VCPKG_ROOT -ErrorAction SilentlyContinue
118+
66119
cmake $cmake_args
67120
if ($LastExitCode) {
68121
Write-Host -ForegroundColor Red "cmake config failed with exit code $LastExitCode"

‎ci/kokoro/windows/triplets/x64-windows-static.cmake‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,3 +4,5 @@ set(VCPKG_LIBRARY_LINKAGE static)
44
set(VCPKG_VISUAL_STUDIO_PATH
55
"C:\\Program Files (x86)\\Microsoft Visual Studio\\$ENV{MSVC_VERSION}\\BuildTools"
66
)
7+
set(VCPKG_CMAKE_CONFIGURE_OPTIONS
8+
"${VCPKG_CMAKE_CONFIGURE_OPTIONS};-DCMAKE_POLICY_VERSION_MINIMUM=3.5")

‎ci/lib/shard.sh‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,7 @@ function shard::cmake_features() {
9292
compute)
9393
echo "compute"
9494
echo "opentelemetry"
95+
echo "universe_domain"
9596
;;
9697
# If we decided to compile all libraries with clang-tidy we would need
9798
# to enable shards such as:

‎google/cloud/bigtable/bound_query.cc‎

Lines changed: 2 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -19,12 +19,8 @@ namespace cloud {
1919
namespace bigtable {
2020
GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_BEGIN
2121

22-
StatusOr<std::string> BoundQuery::prepared_query() const {
23-
return query_plan_->prepared_query();
24-
}
25-
26-
StatusOr<google::bigtable::v2::ResultSetMetadata> BoundQuery::metadata() const {
27-
return query_plan_->metadata();
22+
StatusOr<google::bigtable::v2::PrepareQueryResponse> BoundQuery::response() {
23+
return query_plan_->response();
2824
}
2925

3026
std::unordered_map<std::string, Value> const& BoundQuery::parameters() const {
@@ -36,10 +32,6 @@ InstanceResource const& BoundQuery::instance() const { return instance_; }
3632
google::bigtable::v2::ExecuteQueryRequest BoundQuery::ToRequestProto() const {
3733
google::bigtable::v2::ExecuteQueryRequest result;
3834
*result.mutable_instance_name() = instance_.FullName();
39-
auto prepared_query = query_plan_->prepared_query();
40-
if (prepared_query.ok()) {
41-
*result.mutable_prepared_query() = query_plan_->prepared_query().value();
42-
}
4335

4436
google::protobuf::Map<std::string, google::bigtable::v2::Value> parameters;
4537
for (auto const& kv : parameters_) {

‎google/cloud/bigtable/bound_query.h‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,12 @@
2626

2727
namespace google {
2828
namespace cloud {
29+
namespace bigtable_internal {
30+
GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_BEGIN
31+
class DataConnectionImpl;
32+
GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_END
33+
} // namespace bigtable_internal
34+
2935
namespace bigtable {
3036
GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_BEGIN
3137

@@ -42,15 +48,21 @@ class BoundQuery {
4248
BoundQuery& operator=(BoundQuery&&) = default;
4349

4450
// Accessors
45-
StatusOr<std::string> prepared_query() const;
46-
StatusOr<google::bigtable::v2::ResultSetMetadata> metadata() const;
51+
StatusOr<google::bigtable::v2::PrepareQueryResponse> response();
4752
std::unordered_map<std::string, Value> const& parameters() const;
4853
InstanceResource const& instance() const;
4954

5055
google::bigtable::v2::ExecuteQueryRequest ToRequestProto() const;
5156

57+
GOOGLE_CLOUD_CPP_DEPRECATED("use response()")
58+
StatusOr<std::string> prepared_query() const;
59+
GOOGLE_CLOUD_CPP_DEPRECATED("use response()")
60+
StatusOr<google::bigtable::v2::ResultSetMetadata> metadata() const;
61+
5262
private:
5363
friend class PreparedQuery;
64+
friend class bigtable_internal::DataConnectionImpl;
65+
5466
BoundQuery(InstanceResource instance,
5567
std::shared_ptr<bigtable_internal::QueryPlan> query_plan,
5668
std::unordered_map<std::string, Value> parameters)

‎google/cloud/bigtable/bound_query_test.cc‎

Lines changed: 21 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -35,9 +35,6 @@ TEST(BoundQuery, FromPreparedQuery) {
3535
"SELECT * FROM my_table WHERE col1 = @val1 and col2 = @val2;");
3636
SqlStatement sql_statement(statement_contents);
3737
PrepareQueryResponse response;
38-
std::unordered_map<std::string, Value> parameters = {{"val1", Value(true)},
39-
{"val2", Value(2.0)}};
40-
4138
// The following variables are only meant to confirm the metadata is correctly
4239
// passed down to the BoundQuery.
4340
auto metadata = std::make_unique<google::bigtable::v2::ResultSetMetadata>();
@@ -47,18 +44,24 @@ TEST(BoundQuery, FromPreparedQuery) {
4744
schema->mutable_columns()->Add(std::move(column));
4845
metadata->set_allocated_proto_schema(schema.release());
4946
response.set_allocated_metadata(metadata.release());
47+
auto refresh_fn = [&response]() {
48+
return make_ready_future(StatusOr<PrepareQueryResponse>(response));
49+
};
50+
auto query_plan = bigtable_internal::QueryPlan::Create(
51+
CompletionQueue(fake_cq_impl), std::move(response),
52+
std::move(refresh_fn));
53+
std::unordered_map<std::string, Value> parameters = {{"val1", Value(true)},
54+
{"val2", Value(2.0)}};
5055

51-
PreparedQuery pq(CompletionQueue(fake_cq_impl), instance, sql_statement,
52-
response);
56+
PreparedQuery pq(instance, sql_statement, query_plan);
5357
auto bq = pq.BindParameters(parameters);
5458
EXPECT_EQ(instance.FullName(), bq.instance().FullName());
55-
EXPECT_STATUS_OK(bq.prepared_query());
56-
EXPECT_EQ(statement_contents, bq.prepared_query().value());
5759
EXPECT_EQ(parameters, bq.parameters());
58-
EXPECT_STATUS_OK(bq.metadata());
59-
EXPECT_TRUE(bq.metadata().value().has_proto_schema());
60-
EXPECT_EQ(1, bq.metadata().value().proto_schema().columns_size());
61-
EXPECT_EQ("col1", bq.metadata().value().proto_schema().columns()[0].name());
60+
EXPECT_STATUS_OK(bq.response());
61+
EXPECT_TRUE(bq.response()->metadata().has_proto_schema());
62+
EXPECT_EQ(1, bq.response()->metadata().proto_schema().columns_size());
63+
EXPECT_EQ("col1",
64+
bq.response()->metadata().proto_schema().columns()[0].name());
6265

6366
// Cancel all pending operations, satisfying any remaining futures.
6467
fake_cq_impl->SimulateCompletion(false);
@@ -72,15 +75,19 @@ TEST(BoundQuery, ToRequestProto) {
7275
"SELECT * FROM my_table WHERE col1 = @val1 and col2 = @val2;");
7376
SqlStatement sql_statement(statement_contents);
7477
PrepareQueryResponse response;
78+
auto refresh_fn = []() {
79+
return make_ready_future(StatusOr<PrepareQueryResponse>{});
80+
};
81+
auto query_plan = bigtable_internal::QueryPlan::Create(
82+
CompletionQueue(fake_cq_impl), response, std::move(refresh_fn));
7583
std::unordered_map<std::string, Value> parameters = {{"val1", Value(true)},
7684
{"val2", Value(2.0)}};
7785

78-
PreparedQuery pq(CompletionQueue(fake_cq_impl), instance, sql_statement,
79-
response);
86+
PreparedQuery pq(instance, sql_statement, query_plan);
8087
auto bq = pq.BindParameters(parameters);
8188
google::bigtable::v2::ExecuteQueryRequest proto = bq.ToRequestProto();
8289
EXPECT_EQ(instance.FullName(), proto.instance_name());
83-
EXPECT_EQ(statement_contents, proto.prepared_query());
90+
EXPECT_EQ("", proto.prepared_query());
8491

8592
// Test param contents.
8693
EXPECT_EQ(parameters.size(), proto.mutable_params()->size());

‎google/cloud/bigtable/client_test.cc‎

Lines changed: 49 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
// limitations under the License.
1414

1515
#include "google/cloud/bigtable/client.h"
16+
#include "google/cloud/bigtable/internal/query_plan.h"
1617
#include "google/cloud/bigtable/mocks/mock_data_connection.h"
1718
#include "google/cloud/bigtable/mocks/mock_query_row.h"
1819
#include "google/cloud/bigtable/mocks/mock_row_reader.h"
@@ -44,9 +45,18 @@ TEST(Client, PrepareQuery) {
4445
EXPECT_EQ("projects/the-project/instances/the-instance",
4546
params.instance.FullName());
4647
EXPECT_EQ("SELECT * FROM the-table", params.sql_statement.sql());
47-
PreparedQuery q(CompletionQueue{fake_cq_impl}, params.instance,
48-
params.sql_statement, PrepareQueryResponse{});
49-
return q;
48+
49+
std::unordered_map<std::string, bigtable::Value> parameters;
50+
google::bigtable::v2::PrepareQueryResponse pq_response;
51+
auto refresh_fn = [&pq_response]() {
52+
return make_ready_future(
53+
StatusOr<google::bigtable::v2::PrepareQueryResponse>(
54+
pq_response));
55+
};
56+
auto query_plan = bigtable_internal::QueryPlan::Create(
57+
CompletionQueue(fake_cq_impl), std::move(pq_response),
58+
std::move(refresh_fn));
59+
return bigtable::PreparedQuery(instance, sql, std::move(query_plan));
5060
});
5161

5262
Client client(std::move(conn_mock));
@@ -67,9 +77,20 @@ TEST(Client, AsyncPrepareQuery) {
6777
EXPECT_EQ("projects/the-project/instances/the-instance",
6878
params.instance.FullName());
6979
EXPECT_EQ("SELECT * FROM the-table", params.sql_statement.sql());
70-
PreparedQuery q(CompletionQueue{fake_cq_impl}, params.instance,
71-
params.sql_statement, PrepareQueryResponse{});
72-
return make_ready_future(make_status_or(std::move(q)));
80+
81+
std::unordered_map<std::string, bigtable::Value> parameters;
82+
google::bigtable::v2::PrepareQueryResponse pq_response;
83+
auto refresh_fn = [&pq_response]() {
84+
return make_ready_future(
85+
StatusOr<google::bigtable::v2::PrepareQueryResponse>(
86+
pq_response));
87+
};
88+
auto query_plan = bigtable_internal::QueryPlan::Create(
89+
CompletionQueue(fake_cq_impl), std::move(pq_response),
90+
std::move(refresh_fn));
91+
StatusOr<bigtable::PreparedQuery> result =
92+
bigtable::PreparedQuery(instance, sql, std::move(query_plan));
93+
return make_ready_future(result);
7394
});
7495
Client client(std::move(conn_mock));
7596
auto prepared_query = client.AsyncPrepareQuery(instance, sql);
@@ -111,30 +132,28 @@ TEST(ClientTest, ExecuteQuery) {
111132
ASSERT_TRUE(google::protobuf::TextFormat::ParseFromString(
112133
kResultMetadataText, pq_response.mutable_metadata()));
113134
EXPECT_CALL(*conn_mock, ExecuteQuery)
114-
.WillOnce(
115-
[&](bigtable::ExecuteQueryParams const&) -> StatusOr<RowStream> {
116-
auto mock_source = std::make_unique<MockQueryRowSource>();
117-
EXPECT_CALL(*mock_source, Metadata)
118-
.WillRepeatedly(Return(pq_response.metadata()));
119-
120-
testing::InSequence s;
121-
EXPECT_CALL(*mock_source, NextRow)
122-
.WillOnce(Return(bigtable_mocks::MakeQueryRow(
123-
{{"key", bigtable::Value("r1")},
124-
{"val", bigtable::Value("v1")}})));
125-
EXPECT_CALL(*mock_source, NextRow)
126-
.WillOnce(Return(bigtable_mocks::MakeQueryRow(
127-
{{"key", bigtable::Value("r2")},
128-
{"val", bigtable::Value("v2")}})));
129-
EXPECT_CALL(*mock_source, NextRow)
130-
// Signal end of stream
131-
.WillOnce(
132-
Return(Status(StatusCode::kOutOfRange, "End of stream")));
133-
134-
// Create RowStream with the mock result source
135-
RowStream row_stream(std::move(mock_source));
136-
return StatusOr<RowStream>(std::move(row_stream));
137-
});
135+
.WillOnce([&](bigtable::ExecuteQueryParams) {
136+
auto mock_source = std::make_unique<MockQueryRowSource>();
137+
EXPECT_CALL(*mock_source, Metadata)
138+
.WillRepeatedly(Return(pq_response.metadata()));
139+
140+
testing::InSequence s;
141+
EXPECT_CALL(*mock_source, NextRow)
142+
.WillOnce(Return(bigtable_mocks::MakeQueryRow(
143+
{{"key", bigtable::Value("r1")},
144+
{"val", bigtable::Value("v1")}})));
145+
EXPECT_CALL(*mock_source, NextRow)
146+
.WillOnce(Return(bigtable_mocks::MakeQueryRow(
147+
{{"key", bigtable::Value("r2")},
148+
{"val", bigtable::Value("v2")}})));
149+
EXPECT_CALL(*mock_source, NextRow)
150+
// Signal end of stream
151+
.WillOnce(Return(Status(StatusCode::kOutOfRange, "End of stream")));
152+
153+
// Create RowStream with the mock result source
154+
RowStream row_stream(std::move(mock_source));
155+
return row_stream;
156+
});
138157

139158
Client client(conn_mock);
140159
InstanceResource instance(Project("test-project"), "test-instance");

‎google/cloud/bigtable/data_connection.cc‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
#include "google/cloud/bigtable/internal/data_tracing_connection.h"
1919
#include "google/cloud/bigtable/internal/defaults.h"
2020
#include "google/cloud/bigtable/internal/mutate_rows_limiter.h"
21+
#include "google/cloud/bigtable/internal/partial_result_set_source.h"
2122
#include "google/cloud/bigtable/internal/row_reader_impl.h"
2223
#include "google/cloud/bigtable/options.h"
2324
#include "google/cloud/bigtable/results.h"
@@ -167,9 +168,10 @@ future<StatusOr<bigtable::PreparedQuery>> DataConnection::AsyncPrepareQuery(
167168
Status(StatusCode::kUnimplemented, "not implemented"));
168169
}
169170

170-
StatusOr<bigtable::RowStream> DataConnection::ExecuteQuery(
171-
bigtable::ExecuteQueryParams const&) {
172-
return Status(StatusCode::kUnimplemented, "not implemented");
171+
bigtable::RowStream DataConnection::ExecuteQuery(bigtable::ExecuteQueryParams) {
172+
return RowStream(
173+
std::make_unique<bigtable_internal::StatusOnlyResultSetSource>(
174+
Status(StatusCode::kUnimplemented, "not implemented")));
173175
}
174176

175177
std::shared_ptr<DataConnection> MakeDataConnection(Options options) {

‎google/cloud/bigtable/data_connection.h‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -160,8 +160,7 @@ class DataConnection {
160160
bigtable::PrepareQueryParams const& p);
161161
virtual future<StatusOr<bigtable::PreparedQuery>> AsyncPrepareQuery(
162162
bigtable::PrepareQueryParams const& p);
163-
virtual StatusOr<bigtable::RowStream> ExecuteQuery(
164-
bigtable::ExecuteQueryParams const& p);
163+
virtual bigtable::RowStream ExecuteQuery(bigtable::ExecuteQueryParams p);
165164
};
166165

167166
/**

0 commit comments

Comments
 (0)