BREAKING CHANGE: Change Value::Bytes to g::c::spanner::Bytes (#920)

Introduce `google::cloud::spanner::Bytes` as the representation for
the Spanner BYTES type ... a sequence of raw bytes (in contrast to the
Spanner STRING type, which is interpreted as Unicode characters).

Like the previous `Value::Bytes`, a `google::cloud::spanner::Bytes`
value can be converted to/from a base64-encoded US-ASCII `std::string`,
although those operations are now internal-only.

Unlike `Value::Bytes`, `google::cloud::spanner::Bytes` can be
converted to/from any ordered sequence of bytes, not just `std::string`.
That is, it can accept/produce data in any compatible representation
(for example, as a `std::vector<std::uint8_t>`).

Fixes #413.

Author: devbww

google/cloud/spanner/CMakeLists.txt

@@ -97,6 +97,8 @@ add_library(spanner_client
             ${CMAKE_CURRENT_BINARY_DIR}/internal/build_info.cc
             backoff_policy.h
             batch_dml_result.h
+            bytes.cc
+            bytes.h
             client.cc
             client.h
             commit_result.h
@@ -120,8 +122,6 @@ add_library(spanner_client
             instance_admin_connection.h
             internal/api_client_header.cc
             internal/api_client_header.h
-            internal/base64.cc
-            internal/base64.h
             internal/build_info.h
             internal/compiler_info.cc
             internal/compiler_info.h
@@ -299,7 +299,6 @@ function (spanner_client_define_tests)
         instance_admin_connection_test.cc
         instance_test.cc
         internal/api_client_header_test.cc
-        internal/base64_test.cc
         internal/build_info_test.cc
         internal/compiler_info_test.cc
         internal/connection_impl_test.cc
@@ -321,6 +320,7 @@ function (spanner_client_define_tests)
         internal/time_test.cc
         internal/transaction_impl_test.cc
         internal/tuple_utils_test.cc
+        bytes_test.cc
         keys_test.cc
         mutations_test.cc
         read_options_test.cc

google/cloud/spanner/internal/base64.cc renamed to google/cloud/spanner/bytes.cc

@@ -12,16 +12,16 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include "google/cloud/spanner/internal/base64.h"
+#include "google/cloud/spanner/bytes.h"
+#include "google/cloud/status.h"
 #include <array>
 #include <climits>
-#include <string>
 
 namespace google {
 namespace cloud {
 namespace spanner {
 inline namespace SPANNER_CLIENT_NS {
-namespace internal {
+
 namespace {
 
 constexpr char kPadding = '=';
@@ -51,83 +51,105 @@ constexpr std::array<unsigned char, UCHAR_MAX + 1> kCharToIndexExcessOne = {{
 // UCHAR_MAX is required to be at least 255, meaning std::string::value_type
 // can always hold an octet. If UCHAR_MAX > 255, however, we have no way to
 // base64 encode large values. So, we demand exactly 255.
-static_assert(UCHAR_MAX == 255, "required by Base64Encode()");
+static_assert(UCHAR_MAX == 255, "required by base64 encoder");
 
 }  // namespace
 
-std::string Base64Encode(std::string const& bytes) {
-  std::string encoded;
-  auto* p = reinterpret_cast<unsigned char const*>(bytes.data());
-  auto* const ep = p + bytes.size();
-  encoded.reserve((ep - p + 2) / 3 * 4);  // 3 octets to 4 sextets
-  while (ep - p >= 3) {
-    unsigned int const v = p[0] << 16 | p[1] << 8 | p[2];
-    encoded.push_back(kIndexToChar[v >> 18]);
-    encoded.push_back(kIndexToChar[v >> 12 & 0x3f]);
-    encoded.push_back(kIndexToChar[v >> 6 & 0x3f]);
-    encoded.push_back(kIndexToChar[v & 0x3f]);
-    p += 3;
-  }
-  switch (ep - p) {
+void Bytes::Encoder::Flush() {
+  unsigned int const v = buf_[0] << 16 | buf_[1] << 8 | buf_[2];
+  rep_.push_back(kIndexToChar[v >> 18]);
+  rep_.push_back(kIndexToChar[v >> 12 & 0x3f]);
+  rep_.push_back(kIndexToChar[v >> 6 & 0x3f]);
+  rep_.push_back(kIndexToChar[v & 0x3f]);
+  len_ = 0;
+}
+
+void Bytes::Encoder::FlushAndPad() {
+  switch (len_) {
     case 2: {
-      unsigned int const v = p[0] << 16 | p[1] << 8;
-      encoded.push_back(kIndexToChar[v >> 18]);
-      encoded.push_back(kIndexToChar[v >> 12 & 0x3f]);
-      encoded.push_back(kIndexToChar[v >> 6 & 0x3f]);
-      encoded.push_back(kPadding);
+      unsigned int const v = buf_[0] << 16 | buf_[1] << 8;
+      rep_.push_back(kIndexToChar[v >> 18]);
+      rep_.push_back(kIndexToChar[v >> 12 & 0x3f]);
+      rep_.push_back(kIndexToChar[v >> 6 & 0x3f]);
+      rep_.push_back(kPadding);
       break;
     }
     case 1: {
-      unsigned int const v = p[0] << 16;
-      encoded.push_back(kIndexToChar[v >> 18]);
-      encoded.push_back(kIndexToChar[v >> 12 & 0x3f]);
-      encoded.append(2, kPadding);
+      unsigned int const v = buf_[0] << 16;
+      rep_.push_back(kIndexToChar[v >> 18]);
+      rep_.push_back(kIndexToChar[v >> 12 & 0x3f]);
+      rep_.append(2, kPadding);
       break;
     }
   }
-  return encoded;
 }
 
-StatusOr<std::string> Base64Decode(std::string const& base64) {
-  std::string decoded;
-  auto* p = reinterpret_cast<unsigned char const*>(base64.data());
-  auto* ep = p + base64.size();
-  decoded.reserve((ep - p + 3) / 4 * 3);  // 4 sextets to 3 octets
+void Bytes::Decoder::iterator::Fill() {
+  if (pos_ != end_) {
+    unsigned char p0 = *pos_++;
+    unsigned char p1 = *pos_++;
+    unsigned char p2 = *pos_++;
+    unsigned char p3 = *pos_++;
+    auto i0 = kCharToIndexExcessOne[p0] - 1;
+    auto i1 = kCharToIndexExcessOne[p1] - 1;
+    if (p3 == kPadding) {
+      if (p2 == kPadding) {
+        buf_[++len_] = i0 << 2 | i1 >> 4;
+      } else {
+        auto i2 = kCharToIndexExcessOne[p2] - 1;
+        buf_[++len_] = i1 << 4 | i2 >> 2;
+        buf_[++len_] = i0 << 2 | i1 >> 4;
+      }
+    } else {
+      auto i2 = kCharToIndexExcessOne[p2] - 1;
+      auto i3 = kCharToIndexExcessOne[p3] - 1;
+      buf_[++len_] = i2 << 6 | i3;
+      buf_[++len_] = i1 << 4 | i2 >> 2;
+      buf_[++len_] = i0 << 2 | i1 >> 4;
+    }
+  }
+}
+
+namespace internal {
+
+// Construction from a base64-encoded US-ASCII `std::string`.
+StatusOr<Bytes> BytesFromBase64(std::string input) {
+  auto* p = reinterpret_cast<unsigned char const*>(input.data());
+  auto* ep = p + input.size();
   while (ep - p >= 4) {
     auto i0 = kCharToIndexExcessOne[p[0]];
     auto i1 = kCharToIndexExcessOne[p[1]];
     if (--i0 >= 64 || --i1 >= 64) break;
     if (p[3] == kPadding) {
       if (p[2] == kPadding) {
         if ((i1 & 0xf) != 0) break;
-        decoded.push_back(i0 << 2 | i1 >> 4);
       } else {
         auto i2 = kCharToIndexExcessOne[p[2]];
         if (--i2 >= 64 || (i2 & 0x3) != 0) break;
-        decoded.push_back(i0 << 2 | i1 >> 4);
-        decoded.push_back(i1 << 4 | i2 >> 2);
       }
       p += 4;
       break;
     }
     auto i2 = kCharToIndexExcessOne[p[2]];
     auto i3 = kCharToIndexExcessOne[p[3]];
     if (--i2 >= 64 || --i3 >= 64) break;
-    decoded.push_back(i0 << 2 | i1 >> 4);
-    decoded.push_back(i1 << 4 | i2 >> 2);
-    decoded.push_back(i2 << 6 | i3);
     p += 4;
   }
   if (p != ep) {
-    auto const offset = reinterpret_cast<char const*>(p) - base64.data();
-    auto const bad_chunk = base64.substr(offset, 4);
+    auto const offset = reinterpret_cast<char const*>(p) - input.data();
+    auto const bad_chunk = input.substr(offset, 4);
     auto message = "Invalid base64 chunk \"" + bad_chunk + "\"" +
                    " at offset " + std::to_string(offset);
     return Status(StatusCode::kInvalidArgument, std::move(message));
   }
-  return decoded;
+  Bytes bytes;
+  bytes.base64_rep_ = std::move(input);
+  return bytes;
 }
 
+// Conversion to a base64-encoded US-ASCII `std::string`.
+std::string BytesToBase64(Bytes const& b) { return b.base64_rep_; }
+
 }  // namespace internal
 }  // namespace SPANNER_CLIENT_NS
 }  // namespace spanner

google/cloud/spanner/bytes.h

@@ -0,0 +1,153 @@
+// Copyright 2019 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef GOOGLE_CLOUD_CPP_SPANNER_GOOGLE_CLOUD_SPANNER_BYTES_H_
+#define GOOGLE_CLOUD_CPP_SPANNER_GOOGLE_CLOUD_SPANNER_BYTES_H_
+
+#include "google/cloud/spanner/version.h"
+#include "google/cloud/status_or.h"
+#include <array>
+#include <cstddef>
+#include <iterator>
+#include <string>
+
+namespace google {
+namespace cloud {
+namespace spanner {
+inline namespace SPANNER_CLIENT_NS {
+
+class Bytes;  // defined below
+
+// Internal forward declarations to befriend.
+namespace internal {
+StatusOr<Bytes> BytesFromBase64(std::string input);
+std::string BytesToBase64(Bytes const& b);
+}  // namespace internal
+
+/**
+ * A representation of the Spanner BYTES type: variable-length binary data.
+ *
+ * A `Bytes` value can be constructed from, and converted to any sequence of
+ * octets. `Bytes` values can be compared for equality.
+ */
+class Bytes {
+ public:
+  /// An empty sequence.
+  Bytes() {}
+
+  /// Construction from a sequence of octets.
+  ///@{
+  template <typename InputIt>
+  Bytes(InputIt first, InputIt last) {
+    Encoder encoder(base64_rep_);
+    while (first != last) {
+      encoder.buf_[encoder.len_++] = *first++;
+      if (encoder.len_ == encoder.buf_.size()) encoder.Flush();
+    }
+    if (encoder.len_ != 0) encoder.FlushAndPad();
+  }
+  template <typename Container>
+  explicit Bytes(Container const& c) : Bytes(std::begin(c), std::end(c)) {}
+  ///@}
+
+  /// Conversion to a sequence of octets.  The `Container` must support
+  /// construction from a range specified as a pair of input iterators.
+  template <typename Container>
+  Container get() const {
+    Decoder decoder(base64_rep_);
+    return Container(decoder.begin(), decoder.end());
+  }
+
+  /// @name Relational operators
+  ///@{
+  friend bool operator==(Bytes const& a, Bytes const& b) {
+    return a.base64_rep_ == b.base64_rep_;
+  }
+  friend bool operator!=(Bytes const& a, Bytes const& b) { return !(a == b); }
+  ///@}
+
+ private:
+  friend StatusOr<Bytes> internal::BytesFromBase64(std::string input);
+  friend std::string internal::BytesToBase64(Bytes const& b);
+
+  struct Encoder {
+    Encoder(std::string& rep) : rep_(rep), len_(0) {}
+    void Flush();
+    void FlushAndPad();
+
+    std::string& rep_;  // encoded
+    std::size_t len_;   // buf_[0 .. len_-1] pending encode
+    std::array<unsigned char, 3> buf_;
+  };
+
+  struct Decoder {
+    class iterator {
+     public:
+      using iterator_category = std::input_iterator_tag;
+      using value_type = unsigned char;
+      using difference_type = std::ptrdiff_t;
+      using pointer = value_type*;
+      using reference = value_type&;
+
+      iterator(std::string::const_iterator begin,
+               std::string::const_iterator end)
+          : pos_(begin), end_(end), len_(0) {
+        Fill();
+      }
+
+      void Fill();
+
+      reference operator*() { return buf_[len_]; }
+      pointer operator->() { return &buf_[len_]; }
+
+      iterator& operator++() {
+        if (--len_ == 0) Fill();
+        return *this;
+      }
+      iterator operator++(int) {
+        auto const old = *this;
+        operator++();
+        return old;
+      }
+
+      friend bool operator==(iterator const& a, iterator const& b) {
+        return a.pos_ == b.pos_ && a.len_ == b.len_;
+      }
+      friend bool operator!=(iterator const& a, iterator const& b) {
+        return !(a == b);
+      }
+
+     private:
+      std::string::const_iterator pos_;  // [pos_ .. end_) pending decode
+      std::string::const_iterator end_;
+      std::size_t len_;  // buf_[len_ .. 1] decoded
+      std::array<value_type, 1 + 3> buf_;
+    };
+
+    Decoder(std::string const& rep) : rep_(rep) {}
+    iterator begin() { return iterator(rep_.begin(), rep_.end()); }
+    iterator end() { return iterator(rep_.end(), rep_.end()); }
+
+    std::string const& rep_;  // encoded
+  };
+
+  std::string base64_rep_;  // valid base64 representation
+};
+
+}  // namespace SPANNER_CLIENT_NS
+}  // namespace spanner
+}  // namespace cloud
+}  // namespace google
+
+#endif  // GOOGLE_CLOUD_CPP_SPANNER_GOOGLE_CLOUD_SPANNER_BYTES_H_