impl(bigtable): first group of query conformance tests

Author: scotthart

.typos.toml

@@ -60,6 +60,8 @@ extend-exclude = [
     "docfx/doxygen2toc_test.cc",
     # If we're using a post-download patch file for googleapis.
     "external/googleapis.patch",
+    # Mispelled test names in Bigtable conformance tests
+    "google/cloud/bigtable/ci/run_conformance_tests_proxy_bazel.sh",
 ]
 
 [default]

google/cloud/bigtable/ci/run_conformance_tests_proxy_bazel.sh

@@ -46,6 +46,43 @@ go test -v \
   -skip "Generic_CloseClient|Generic_DeadlineExceeded|NoRetry_OutOfOrderError_Reverse|Retry_LastScannedRow_Reverse|Retry_WithRetryInfo_OverallDedaline|TestExecuteQuery" \
   -proxy_addr=:9999
 exit_status=$?
+
+# Run all the ExecuteQuery tests that either work or we plan to skip such as
+# CloseClient
+go test -v \
+  -run "TestExecuteQuery" \
+  -skip "CloseClient|FailsOnEmptyMetadata|FailsOnExecuteQueryMetadata|FailsOnInvalidType|FailsOnNotEnoughData|FailsOnNotEnoughDataWithCompleteRows|FailsOnSuccesfulStreamWithNoToken|ChecksumMismatch|FailsOnTypeMismatch|FailsOnTypeMismatchWithinMap|FailsOnTypeMismatchWithinArray|FailsOnTypeMismatchWithinStruct|FailsOnStructMissingField|RetryTest_WithPlanRefresh|PlanRefresh" \
+  -proxy_addr=:9999
+exit_status=$?
+
+# These next four go test commands group the currently failing ExecuteQuery
+# tests into groups that exercise similar functionality and should be worked on
+# together.
+
+# Metadata tests b/461232934
+#go test -v \
+#  -run "FailsOnEmptyMetadata|FailsOnExecuteQueryMetadata|FailsOnInvalidType" \
+#  -proxy_addr=:9999
+#exit_status=$?
+
+# Stream reading tests b/461232110
+#go test -v \
+#  -run "FailsOnNotEnoughData|FailsOnNotEnoughDataWithCompleteRows|FailsOnSuccesfulStreamWithNoToken|ChecksumMismatch" \
+#  -proxy_addr=:9999
+#exit_status=$?
+
+# Response/Metadata mismatches b/461233335
+#go test -v \
+#  -run "FailsOnTypeMismatch|FailsOnTypeMismatchWithinMap|FailsOnTypeMismatchWithinArray|FailsOnTypeMismatchWithinStruct|FailsOnStructMissingField" \
+#  -proxy_addr=:9999
+#exit_status=$?
+
+# QueryPlan refresh tests b/461233613
+#go test -v \
+#  -run "RetryTest_WithPlanRefresh|PlanRefresh" \
+#  -proxy_addr=:9999
+#exit_status=$?
+
 # Remove the entire module cache, including unpacked source code of versioned
 # dependencies.
 go clean -modcache

google/cloud/bigtable/test_proxy/cbt_test_proxy.cc

@@ -14,6 +14,7 @@
 
 #include "google/cloud/bigtable/test_proxy/cbt_test_proxy.h"
 #include "google/cloud/bigtable/cell.h"
+#include "google/cloud/bigtable/client.h"
 #include "google/cloud/bigtable/idempotent_mutation_policy.h"
 #include "google/cloud/bigtable/mutations.h"
 #include "google/cloud/bigtable/table.h"
@@ -360,6 +361,70 @@ grpc::Status CbtTestProxy::ReadModifyWriteRow(
   return grpc::Status();
 }
 
+grpc::Status CbtTestProxy::ExecuteQuery(
+    grpc::ServerContext*,
+    google::bigtable::testproxy::ExecuteQueryRequest const* request,
+    google::bigtable::testproxy::ExecuteQueryResult* response) {
+  // Retrieve connection
+  auto const& conn = GetConnection(request->client_id());
+  if (!conn.ok()) return ToGrpcStatus(std::move(conn).status());
+  auto client = bigtable::Client(*conn);
+  auto const& request_proto = request->request();
+
+  // Call prepare query
+  auto instance = MakeInstanceResource(request_proto.instance_name());
+  // NOLINTNEXTLINE(deprecated-declarations)
+  bigtable::SqlStatement sql_statement{request_proto.query()};
+  auto prepared_query =
+      client.PrepareQuery(*std::move(instance), sql_statement);
+  if (!prepared_query.ok()) {
+    *response->mutable_status() = ToRpcStatus(prepared_query.status());
+    return ::grpc::Status();
+  }
+
+  // Bind parameters
+  std::unordered_map<std::string, Value> params;
+  for (auto const& param : request_proto.params()) {
+    auto value =
+        bigtable_internal::FromProto(param.second.type(), param.second);
+    params.insert(std::make_pair(param.first, std::move(value)));
+  }
+  auto bound_query = prepared_query->BindParameters(params);
+  auto bound_query_metadata = bound_query.response()->metadata();
+
+  RowStream result = client.ExecuteQuery(std::move(bound_query), {});
+
+  Status status;
+  std::vector<google::bigtable::testproxy::SqlRow> proxy_rows;
+  for (auto& row : result) {
+    if (!row.ok()) {
+      status = row.status();
+      break;
+    }
+    google::bigtable::testproxy::SqlRow proxy_row;
+    for (auto const& v : row->values()) {
+      *proxy_row.add_values() = bigtable_internal::ToProto(v).second;
+    }
+    proxy_rows.push_back(std::move(proxy_row));
+  }
+
+  if (status.ok()) {
+    for (auto& p : proxy_rows) {
+      *response->add_rows() = std::move(p);
+    }
+  }
+
+  // populate metadata
+  google::bigtable::testproxy::ResultSetMetadata metadata;
+  for (auto const& column : bound_query_metadata.proto_schema().columns()) {
+    *metadata.add_columns() = column;
+  }
+  *response->mutable_metadata() = metadata;
+
+  *response->mutable_status() = ToRpcStatus(status);
+  return grpc::Status();
+}
+
 StatusOr<std::shared_ptr<DataConnection>> CbtTestProxy::GetConnection(
     std::string const& client_id) {
   std::lock_guard<std::mutex> lk(mu_);

google/cloud/bigtable/test_proxy/cbt_test_proxy.h

@@ -91,6 +91,11 @@ class CbtTestProxy final
       ::google::bigtable::testproxy::ReadModifyWriteRowRequest const* request,
       ::google::bigtable::testproxy::RowResult* response) override;
 
+  grpc::Status ExecuteQuery(
+      ::grpc::ServerContext* context,
+      ::google::bigtable::testproxy::ExecuteQueryRequest const* request,
+      ::google::bigtable::testproxy::ExecuteQueryResult* response) override;
+
  private:
   StatusOr<std::shared_ptr<DataConnection>> GetConnection(
       std::string const& client_id);

protos/google/cloud/bigtable/test_proxy/test_proxy.proto

@@ -19,15 +19,47 @@ package google.bigtable.testproxy;
 import "google/api/client.proto";
 import "google/bigtable/v2/bigtable.proto";
 import "google/bigtable/v2/data.proto";
+import "google/protobuf/descriptor.proto";
 import "google/protobuf/duration.proto";
 import "google/rpc/status.proto";
 
-option go_package = "./testproxypb";
+option go_package = "cloud.google.com/go/bigtable/testproxy/testproxypb;testproxypb";
 option java_multiple_files = true;
 option java_package = "com.google.cloud.bigtable.testproxy";
 
+// A config flag that dictates how the optional features should be enabled
+// during the client creation. The optional features customize how the client
+// interacts with the server, and are defined in
+// https://github.com/googleapis/googleapis/blob/master/google/bigtable/v2/feature_flags.proto
+enum OptionalFeatureConfig {
+  OPTIONAL_FEATURE_CONFIG_DEFAULT = 0;
+
+  OPTIONAL_FEATURE_CONFIG_ENABLE_ALL = 1;
+}
+
 // Request to test proxy service to create a client object.
 message CreateClientRequest {
+  message SecurityOptions {
+    // Access token to use for client credentials. If empty, the client will not
+    // use any call credentials. Certain implementations may require `use_ssl`
+    // to be set when using this.
+    string access_token = 1;
+
+    // Whether to use SSL channel credentials when connecting to the data
+    // endpoint.
+    bool use_ssl = 2;
+
+    // If using SSL channel credentials, override the SSL endpoint to match the
+    // host that is specified in the backend's certificate. Also sets the
+    // client's authority header value.
+    string ssl_endpoint_override = 3;
+
+    // PEM encoding of the server root certificates. If not set, the default
+    // root certs will be used instead. The default can be overridden via the
+    // GRPC_DEFAULT_SSL_ROOTS_FILE_PATH env var.
+    string ssl_root_certs_pem = 4;
+  }
+
   // A unique ID associated with the client object to be created.
   string client_id = 1;
 
@@ -52,6 +84,21 @@ message CreateClientRequest {
   // the created client. Otherwise, the default timeout from the client library
   // will be used. Note that the override applies to all the methods.
   google.protobuf.Duration per_operation_timeout = 6;
+
+  // Optional config that dictates how the optional features should be enabled
+  // during the client creation. Please check the enum type's docstring above.
+  OptionalFeatureConfig optional_feature_config = 7;
+
+  // Options to allow connecting to backends with channel and/or call
+  // credentials. This is needed internally by Cloud Bigtable's own testing
+  // frameworks.It is not necessary to support these fields for client
+  // conformance testing.
+  //
+  // WARNING: this allows the proxy to connect to a real production
+  // CBT backend with the right options, however, the proxy itself is insecure
+  // so it is not recommended to use it with real credentials or outside testing
+  // contexts.
+  SecurityOptions security_options = 8;
 }
 
 // Response from test proxy service for CreateClientRequest.
@@ -203,6 +250,44 @@ message ReadModifyWriteRowRequest {
   google.bigtable.v2.ReadModifyWriteRowRequest request = 2;
 }
 
+// Request to test proxy service to execute a query.
+message ExecuteQueryRequest {
+  // The ID of the target client object.
+  string client_id = 1;
+
+  // The raw request to the Bigtable server.
+  google.bigtable.v2.ExecuteQueryRequest request = 2;
+
+  // A collection of proto descriptor files for deserializing PROTO/ENUM columns
+  // in the query result. The descriptor files must be provided in dependency
+  // order.
+  google.protobuf.FileDescriptorSet file_descriptor_set = 3;
+}
+
+// Response from test proxy service for ExecuteQueryRequest.
+message ExecuteQueryResult {
+  // The RPC status from the client binding.
+  google.rpc.Status status = 1;
+
+  // Name and type information for the query result.
+  ResultSetMetadata metadata = 4;
+
+  // Encoded version of the ResultSet. Should not contain type information.
+  repeated SqlRow rows = 3;
+}
+
+// Schema information for the query result.
+message ResultSetMetadata {
+  // Column metadata for each column inthe query result.
+  repeated google.bigtable.v2.ColumnMetadata columns = 1;
+}
+
+// Representation of a single row in the query result.
+message SqlRow {
+  // Columnar values returned by the query.
+  repeated google.bigtable.v2.Value values = 1;
+}
+
 // Note that all RPCs are unary, even when the equivalent client binding call
 // may be streaming. This is an intentional simplification.
 //
@@ -265,4 +350,7 @@ service CloudBigtableV2TestProxy {
 
   // Performs a read-modify-write operation with the client.
   rpc ReadModifyWriteRow(ReadModifyWriteRowRequest) returns (RowResult) {}
+
+  // Executes a BTQL query with the client.
+  rpc ExecuteQuery(ExecuteQueryRequest) returns (ExecuteQueryResult) {}
 }