Merge pull request #24 from scotthart/add_async_insert_job

impl: add async InsertJob

Author: scotthart

ci/cloudbuild/dockerfiles/fedora-latest-cmake.Dockerfile

@@ -67,15 +67,20 @@ RUN curl -fsSL https://distfiles.ariadne.space/pkgconf/pkgconf-2.2.0.tar.gz | \
 # set the search path.
 ENV PKG_CONFIG_PATH=/usr/local/lib64/pkgconfig:/usr/local/lib/pkgconfig:/usr/lib64/pkgconfig
 
+# In order to work around https://github.com/llvm/llvm-project/issues/102443 we set
+# these compiler env vars so that all our dependencies are built the same way.
+ENV CC="clang"
+ENV CXX="clang++"
+ENV CXXFLAGS="-fclang-abi-compat=17"
+
 # We disable the inline namespace because otherwise Abseil LTS updates break our
 # `check-api` build.
 WORKDIR /var/tmp/build
 RUN curl -fsSL https://github.com/abseil/abseil-cpp/archive/20240722.0.tar.gz | \
     tar -xzf - --strip-components=1 && \
-    sed -i 's/^#define ABSL_OPTION_USE_\(.*\) 2/#define ABSL_OPTION_USE_\1 0/' "absl/base/options.h" && \
-    sed -i 's/^#define ABSL_OPTION_USE_INLINE_NAMESPACE 1$/#define ABSL_OPTION_USE_INLINE_NAMESPACE 0/' "absl/base/options.h" && \
     cmake \
       -DCMAKE_BUILD_TYPE="Release" \
+      -DCMAKE_CXX_STANDARD=17 \
       -DABSL_BUILD_TESTING=OFF \
       -DBUILD_SHARED_LIBS=yes \
       -GNinja -S . -B cmake-out && \
@@ -129,10 +134,11 @@ RUN curl -fsSL https://github.com/nlohmann/json/archive/v3.11.3.tar.gz | \
     ldconfig && cd /var/tmp && rm -fr build
 
 WORKDIR /var/tmp/build/protobuf
-RUN curl -fsSL https://github.com/protocolbuffers/protobuf/archive/v29.0.tar.gz | \
+RUN curl -fsSL https://github.com/protocolbuffers/protobuf/archive/v29.3.tar.gz | \
     tar -xzf - --strip-components=1 && \
     cmake \
         -DCMAKE_BUILD_TYPE=Release \
+        -DCMAKE_CXX_STANDARD=17 \
         -DBUILD_SHARED_LIBS=yes \
         -Dprotobuf_BUILD_TESTS=OFF \
         -Dprotobuf_ABSL_PROVIDER=package \
@@ -144,7 +150,7 @@ WORKDIR /var/tmp/build/
 RUN curl -fsSL https://github.com/open-telemetry/opentelemetry-cpp/archive/v1.18.0.tar.gz | \
     tar -xzf - --strip-components=1 && \
     cmake \
-        -DCMAKE_CXX_STANDARD=14 \
+        -DCMAKE_CXX_STANDARD=17 \
         -DCMAKE_BUILD_TYPE=Release \
         -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE \
         -DBUILD_SHARED_LIBS=ON \
@@ -159,10 +165,11 @@ RUN curl -fsSL https://github.com/open-telemetry/opentelemetry-cpp/archive/v1.18
 
 WORKDIR /var/tmp/build/grpc
 RUN dnf makecache && dnf install -y c-ares-devel re2-devel
-RUN curl -fsSL https://github.com/grpc/grpc/archive/v1.67.0.tar.gz | \
+RUN curl -fsSL https://github.com/grpc/grpc/archive/v1.69.0.tar.gz | \
     tar -xzf - --strip-components=1 && \
     cmake \
       -DCMAKE_BUILD_TYPE=Release \
+      -DCMAKE_CXX_STANDARD=17 \
       -DBUILD_SHARED_LIBS=ON \
       -DgRPC_INSTALL=ON \
       -DgRPC_BUILD_TESTS=OFF \
@@ -184,10 +191,12 @@ RUN curl -fsSL https://github.com/grpc/grpc/archive/v1.67.0.tar.gz | \
 # files.
 ENV LD_LIBRARY_PATH=/usr/local/lib:/usr/local/lib64:${LD_LIBRARY_PATH}
 WORKDIR /var/tmp/build/google-cloud-cpp
-RUN curl -fsSL https://github.com/googleapis/google-cloud-cpp/archive/v2.31.0.tar.gz | \
+RUN curl -fsSL https://github.com/googleapis/google-cloud-cpp/archive/v2.33.0.tar.gz | \
     tar -xzf - --strip-components=1 && \
     cmake \
         -DCMAKE_BUILD_TYPE=Release \
+        -DCMAKE_CXX_STANDARD=17 \
+        -DGOOGLE_CLOUD_CPP_ENABLE_CLANG_ABI_COMPAT_17=ON \
         -DBUILD_SHARED_LIBS=yes \
         -DGOOGLE_CLOUD_CPP_ENABLE="bigquerycontrol,bigquery" \
       -GNinja -S . -B cmake-out && \
@@ -198,6 +207,7 @@ WORKDIR /var/tmp/build/arrow
 RUN curl -fsSL https://github.com/apache/arrow/archive/apache-arrow-18.1.0.tar.gz | \
     tar -xzf - --strip-components=1 && \
     cmake \
+      -DCMAKE_CXX_STANDARD=17 \
       -GNinja -S cpp -B cmake-out \
       --preset ninja-release-minimal \
       -DARROW_BUILD_STATIC=ON  && \

google/cloud/bigquery_unified/CMakeLists.txt

@@ -135,6 +135,7 @@ set(bigquery_unified_library_files
     connection.h
     idempotency_policy.cc
     idempotency_policy.h
+    internal/async_rest_long_running_operation_custom.h
     internal/connection_impl.cc
     internal/connection_impl.h
     internal/default_options.cc

google/cloud/bigquery_unified/client.cc

@@ -68,15 +68,13 @@ StreamRange<google::cloud::bigquery::v2::ListFormatJob> Client::ListJobs(
 }
 
 future<StatusOr<google::cloud::bigquery::v2::Job>> Client::InsertJob(
-    google::cloud::bigquery::v2::InsertJobRequest const& request,
-    Options opts) {
-  return make_ready_future<StatusOr<google::cloud::bigquery::v2::Job>>(
-      internal::UnimplementedError("not implemented"));
+    google::cloud::bigquery::v2::Job const& job, Options opts) {
+  return connection_->InsertJob(
+      job, internal::MergeOptions(std::move(opts), options_));
 }
 
 StatusOr<google::cloud::bigquery::v2::JobReference> Client::InsertJob(
-    google::cloud::NoAwaitTag,
-    google::cloud::bigquery::v2::InsertJobRequest const& request,
+    google::cloud::NoAwaitTag, google::cloud::bigquery::v2::Job const& job,
     Options opts) {
   return internal::UnimplementedError("not implemented");
 }

google/cloud/bigquery_unified/client.h

@@ -92,27 +92,25 @@ class Client {
       google::cloud::bigquery::v2::JobReference const& job_reference,
       Options opts = {});
 
+  // DeleteJob
+  Status DeleteJob(google::cloud::bigquery::v2::DeleteJobRequest const& request,
+                   Options opts = {});
+
   // GetJob
   StatusOr<google::cloud::bigquery::v2::Job> GetJob(
       google::cloud::bigquery::v2::GetJobRequest const& request,
       Options opts = {});
 
-  // DeleteJob
-  Status DeleteJob(google::cloud::bigquery::v2::DeleteJobRequest const& request,
-                   Options opts = {});
-
   // ListJobs
   StreamRange<google::cloud::bigquery::v2::ListFormatJob> ListJobs(
       google::cloud::bigquery::v2::ListJobsRequest request, Options opts = {});
 
   // InsertJob
   future<StatusOr<google::cloud::bigquery::v2::Job>> InsertJob(
-      google::cloud::bigquery::v2::InsertJobRequest const& request,
-      Options opts = {});
+      google::cloud::bigquery::v2::Job const& job, Options opts = {});
 
   StatusOr<google::cloud::bigquery::v2::JobReference> InsertJob(
-      google::cloud::NoAwaitTag,
-      google::cloud::bigquery::v2::InsertJobRequest const& request,
+      google::cloud::NoAwaitTag, google::cloud::bigquery::v2::Job const& job,
       Options opts = {});
 
   future<StatusOr<google::cloud::bigquery::v2::Job>> InsertJob(

google/cloud/bigquery_unified/connection.cc

@@ -71,16 +71,14 @@ StreamRange<google::cloud::bigquery::v2::ListFormatJob> Connection::ListJobs(
 
 // InsertJob
 future<StatusOr<google::cloud::bigquery::v2::Job>> Connection::InsertJob(
-    google::cloud::bigquery::v2::InsertJobRequest const& request,
-    Options opts) {
+    google::cloud::bigquery::v2::Job const& job, Options opts) {
   return google::cloud::make_ready_future<
       StatusOr<google::cloud::bigquery::v2::Job>>(
       Status(StatusCode::kUnimplemented, "not implemented"));
 }
 
 StatusOr<google::cloud::bigquery::v2::JobReference> Connection::InsertJob(
-    google::cloud::NoAwaitTag,
-    google::cloud::bigquery::v2::InsertJobRequest const& request,
+    google::cloud::NoAwaitTag, google::cloud::bigquery::v2::Job const& job,
     Options opts) {
   return internal::UnimplementedError("not implemented");
 }

google/cloud/bigquery_unified/connection.h

@@ -77,12 +77,10 @@ class Connection {
 
   // InsertJob
   virtual future<StatusOr<google::cloud::bigquery::v2::Job>> InsertJob(
-      google::cloud::bigquery::v2::InsertJobRequest const& request,
-      Options opts);
+      google::cloud::bigquery::v2::Job const& job, Options opts);
 
   virtual StatusOr<google::cloud::bigquery::v2::JobReference> InsertJob(
-      google::cloud::NoAwaitTag,
-      google::cloud::bigquery::v2::InsertJobRequest const& request,
+      google::cloud::NoAwaitTag, google::cloud::bigquery::v2::Job const& job,
       Options opts);
 
   virtual future<StatusOr<google::cloud::bigquery::v2::Job>> InsertJob(

google/cloud/bigquery_unified/google_cloud_cpp_bigquery_bigquery_unified.bzl

@@ -20,6 +20,7 @@ google_cloud_cpp_bigquery_bigquery_unified_hdrs = [
     "client.h",
     "connection.h",
     "idempotency_policy.h",
+    "internal/async_rest_long_running_operation_custom.h",
     "internal/connection_impl.h",
     "internal/default_options.h",
     "job_options.h",

google/cloud/bigquery_unified/integration_tests/job_integration_test.cc

@@ -13,6 +13,7 @@
 // limitations under the License.
 
 #include "google/cloud/bigquery_unified/client.h"
+#include "google/cloud/bigquery_unified/job_options.h"
 #include "google/cloud/bigquery_unified/testing_util/status_matchers.h"
 #include "google/cloud/bigquery_unified/version.h"
 #include "google/cloud/internal/getenv.h"
@@ -22,6 +23,7 @@ namespace google::cloud::bigquery_unified {
 GOOGLE_CLOUD_CPP_BIGQUERY_INLINE_NAMESPACE_BEGIN
 namespace {
 
+using ::google::cloud::bigquery_unified::testing_util::IsOk;
 using ::testing::Eq;
 
 class JobIntegrationTest : public ::testing::Test {
@@ -34,14 +36,43 @@ class JobIntegrationTest : public ::testing::Test {
   std::string project_id_;
 };
 
+TEST_F(JobIntegrationTest, InsertJob) {
+  namespace bigquery_proto = google::cloud::bigquery::v2;
+
+  bigquery_proto::JobConfigurationQuery query;
+  query.mutable_use_legacy_sql()->set_value(false);
+  query.set_query(
+      "SELECT name, state, year, sum(number) as total "
+      "FROM `bigquery-public-data.usa_names.usa_1910_2013` "
+      "WHERE year >= 2000 "
+      "GROUP BY name, state, year "
+      "LIMIT 100");
+
+  bigquery_proto::JobConfiguration config;
+  *config.mutable_query() = query;
+  config.mutable_labels()->insert({"test_suite", "job_integration_test"});
+  config.mutable_labels()->insert({"test_case", "insert_job"});
+
+  bigquery_proto::Job query_job_request;
+  *query_job_request.mutable_configuration() = config;
+  std::shared_ptr<Connection> connection =
+      google::cloud::bigquery_unified::MakeConnection();
+  auto client = google::cloud::bigquery_unified::Client(connection);
+
+  auto options =
+      google::cloud::Options{}.set<BillingProjectOption>(project_id_);
+  auto query_job = client.InsertJob(query_job_request, options).get();
+  EXPECT_THAT(query_job, IsOk());
+}
+
 TEST_F(JobIntegrationTest, GetJob) {
   namespace bigquery_proto = google::cloud::bigquery::v2;
   std::shared_ptr<Connection> connection =
       google::cloud::bigquery_unified::MakeConnection();
   auto client = google::cloud::bigquery_unified::Client(connection);
 
   // TODO: hard coding this id is brittle but currently necessary.
-  std::string const job_id = "job_TyRhPS6z-5_e9JSwtT8ieuwDOdLD";
+  std::string const job_id = "job_XORZAqWx6R3xcDQCL9K_-2peocI7";
   bigquery_proto::GetJobRequest get_request;
   get_request.set_project_id(project_id_);
   get_request.set_job_id(job_id);