Add more big_endian/little_endian utilities

Abseil made their endian utilities private; create variants for tensorstore use.

PiperOrigin-RevId: 753055518
Change-Id: Icd0ed0137b3e62bc7fe4e95d185b1d75a471bc0f

Author: laramiel

tensorstore/internal/compression/BUILD

@@ -111,7 +111,7 @@ tensorstore_cc_library(
     srcs = ["neuroglancer_compressed_segmentation.cc"],
     hdrs = ["neuroglancer_compressed_segmentation.h"],
     deps = [
-        "@abseil-cpp//absl/base:endian",
+        "//tensorstore/util:endian",
         "@abseil-cpp//absl/container:flat_hash_map",
     ],
 )

tensorstore/internal/compression/neuroglancer_compressed_segmentation.cc

@@ -23,8 +23,8 @@
 #include <string_view>
 #include <vector>
 
-#include "absl/base/internal/endian.h"
 #include "absl/container/flat_hash_map.h"
+#include "tensorstore/util/endian.h"
 
 namespace tensorstore {
 namespace neuroglancer_compressed_segmentation {
@@ -34,10 +34,9 @@ constexpr size_t kBlockHeaderSize = 2;
 void WriteBlockHeader(size_t encoded_value_base_offset,
                       size_t table_base_offset, size_t encoding_bits,
                       void* output) {
-  absl::little_endian::Store32(output,
-                               table_base_offset | (encoding_bits << 24));
-  absl::little_endian::Store32(static_cast<char*>(output) + 4,
-                               encoded_value_base_offset);
+  little_endian::Store32(output, table_base_offset | (encoding_bits << 24));
+  little_endian::Store32(static_cast<char*>(output) + 4,
+                         encoded_value_base_offset);
 }
 
 template <typename Label>
@@ -136,9 +135,9 @@ void EncodeBlock(const Label* input, const ptrdiff_t input_shape[3],
     uint32_t index = seen_values.at(value);
     size_t output_offset = x + block_shape[2] * (y + block_shape[1] * z);
     void* cur_ptr = output_ptr + output_offset * encoded_bits / 32 * 4;
-    absl::little_endian::Store32(
-        cur_ptr, absl::little_endian::Load32(cur_ptr) |
-                     (index << (output_offset * encoded_bits % 32)));
+    little_endian::Store32(cur_ptr,
+                           little_endian::Load32(cur_ptr) |
+                               (index << (output_offset * encoded_bits % 32)));
   });
 
   // Write table
@@ -147,9 +146,8 @@ void EncodeBlock(const Label* input, const ptrdiff_t input_shape[3],
         output->data() + encoded_value_base_offset + encoded_size_32bits * 4;
     for (auto value : seen_values_inv) {
       for (size_t word_i = 0; word_i < num_32bit_words_per_label; ++word_i) {
-        absl::little_endian::Store32(
-            output_ptr + word_i * 4,
-            static_cast<uint32_t>(value >> (32 * word_i)));
+        little_endian::Store32(output_ptr + word_i * 4,
+                               static_cast<uint32_t>(value >> (32 * word_i)));
       }
       output_ptr += num_32bit_words_per_label * 4;
     }
@@ -206,8 +204,8 @@ void EncodeChannels(const Label* input, const ptrdiff_t input_shape[3 + 1],
   const size_t base_offset = output->size();
   output->resize(base_offset + input_shape[0] * 4);
   for (ptrdiff_t channel_i = 0; channel_i < input_shape[0]; ++channel_i) {
-    absl::little_endian::Store32(output->data() + base_offset + channel_i * 4,
-                                 (output->size() - base_offset) / 4);
+    little_endian::Store32(output->data() + base_offset + channel_i * 4,
+                           (output->size() - base_offset) / 4);
     EncodeChannel(
         reinterpret_cast<const Label*>(reinterpret_cast<const char*>(input) +
                                        input_byte_strides[0] * channel_i),
@@ -217,7 +215,7 @@ void EncodeChannels(const Label* input, const ptrdiff_t input_shape[3 + 1],
 
 void ReadBlockHeader(const void* header, size_t* encoded_value_base_offset,
                      size_t* table_base_offset, size_t* encoding_bits) {
-  auto h = absl::little_endian::Load64(header);
+  auto h = little_endian::Load64(header);
   *table_base_offset = h & 0xffffff;
   *encoding_bits = (h >> 24) & 0xff;
   *encoded_value_base_offset = (h >> 32) & 0xffffff;
@@ -258,9 +256,9 @@ bool DecodeBlock(size_t encoded_bits, const char* encoded_input,
   // Returns the label at the specified table index.
   const auto read_label = [&](size_t index) -> Label {
     if constexpr (sizeof(Label) == 4) {
-      return absl::little_endian::Load32(table_input + index * sizeof(Label));
+      return little_endian::Load32(table_input + index * sizeof(Label));
     } else {
-      return absl::little_endian::Load64(table_input + index * sizeof(Label));
+      return little_endian::Load64(table_input + index * sizeof(Label));
     }
   };
 
@@ -279,7 +277,7 @@ bool DecodeBlock(size_t encoded_bits, const char* encoded_input,
   return for_each_position([&](Label& output_label, ptrdiff_t z, ptrdiff_t y,
                                ptrdiff_t x) {
     size_t encoded_offset = x + block_shape[2] * (y + block_shape[1] * z);
-    auto index = absl::little_endian::Load32(
+    auto index = little_endian::Load32(
                      encoded_input + encoded_offset * encoded_bits / 32 * 4) >>
                      (encoded_offset * encoded_bits % 32) &
                  encoded_value_mask;
@@ -368,8 +366,7 @@ bool DecodeChannels(std::string_view input, const ptrdiff_t block_shape[3],
     return false;
   }
   for (ptrdiff_t channel_i = 0; channel_i < output_shape[0]; ++channel_i) {
-    const size_t offset =
-        absl::little_endian::Load32(input.data() + channel_i * 4);
+    const size_t offset = little_endian::Load32(input.data() + channel_i * 4);
     if (offset > input.size() / 4) {
       // channel offset is invalid
       return false;

tensorstore/kvstore/BUILD

@@ -81,8 +81,8 @@ tensorstore_cc_library(
     deps = [
         "//tensorstore/serialization",
         "//tensorstore/serialization:absl_time",
+        "//tensorstore/util:endian",
         "//tensorstore/util:quote_string",
-        "@abseil-cpp//absl/base:endian",
         "@abseil-cpp//absl/strings",
         "@abseil-cpp//absl/time",
     ],

tensorstore/kvstore/generation.cc

@@ -26,12 +26,12 @@
 #include <string_view>
 #include <utility>
 
-#include "absl/base/internal/endian.h"
 #include "absl/strings/str_cat.h"
 #include "absl/time/time.h"
 #include "tensorstore/serialization/absl_time.h"  // IWYU pragma: keep
 #include "tensorstore/serialization/fwd.h"
 #include "tensorstore/serialization/serialization.h"
+#include "tensorstore/util/endian.h"
 #include "tensorstore/util/quote_string.h"
 
 namespace tensorstore {
@@ -97,7 +97,7 @@ std::string StorageGeneration::DebugString() const {
         absl::StrAppend(&output, "|");
       }
       assert(i + 8 <= value.size());  // Already ensured by `Validate()`
-      uint64_t id = absl::little_endian::Load64(&value[i]);
+      uint64_t id = little_endian::Load64(&value[i]);
       i += 8;
       absl::StrAppend(&output, "M", id);
     }
@@ -179,23 +179,23 @@ StorageGeneration StorageGeneration::Clean(StorageGeneration generation) {
 
 bool StorageGeneration::LastMutatedBy(MutationId id) const {
   return value.size() >= 9 && (value[0] & kMutation) &&
-         absl::little_endian::Load64(&value[1]) == id;
+         little_endian::Load64(&value[1]) == id;
 }
 
 void StorageGeneration::MarkDirty(MutationId mutation_id) {
   if (value.empty()) {
     value.resize(9);
-    absl::little_endian::Store64(&value[1], mutation_id);
+    little_endian::Store64(&value[1], mutation_id);
     value[0] = kMutation;
   } else {
     if (value[0] & kMutation) {
       char buffer[9];
       buffer[0] = kMutation | kAdditionalTags;
-      absl::little_endian::Store64(&buffer[1], mutation_id);
+      little_endian::Store64(&buffer[1], mutation_id);
       value.insert(0, buffer, 9);
     } else {
       char buffer[8];
-      absl::little_endian::Store64(&buffer[0], mutation_id);
+      little_endian::Store64(&buffer[0], mutation_id);
       value.insert(1, buffer, 8);
       value[0] |= kMutation;
     }

tensorstore/kvstore/neuroglancer_uint64_sharded/BUILD

@@ -74,12 +74,12 @@ tensorstore_cc_library(
         "//tensorstore/internal:cord_util",
         "//tensorstore/internal/compression:zlib",
         "//tensorstore/kvstore:byte_range",
+        "//tensorstore/util:endian",
         "//tensorstore/util:result",
         "//tensorstore/util:span",
         "//tensorstore/util:status",
         "//tensorstore/util:str_cat",
         "@abseil-cpp//absl/algorithm:container",
-        "@abseil-cpp//absl/base:endian",
         "@abseil-cpp//absl/status",
         "@abseil-cpp//absl/strings:cord",
     ],
@@ -109,11 +109,11 @@ tensorstore_cc_library(
         "//tensorstore/internal:flat_cord_builder",
         "//tensorstore/internal/compression:zlib",
         "//tensorstore/kvstore:byte_range",
+        "//tensorstore/util:endian",
         "//tensorstore/util:result",
         "//tensorstore/util:span",
         "//tensorstore/util:status",
         "//tensorstore/util:str_cat",
-        "@abseil-cpp//absl/base:endian",
         "@abseil-cpp//absl/functional:function_ref",
         "@abseil-cpp//absl/status",
         "@abseil-cpp//absl/strings:cord",
@@ -176,7 +176,6 @@ tensorstore_cc_library(
         "//tensorstore/util/execution:sender_util",
         "//tensorstore/util/garbage_collection",
         "@abseil-cpp//absl/base:core_headers",
-        "@abseil-cpp//absl/base:endian",
         "@abseil-cpp//absl/status",
         "@abseil-cpp//absl/strings:cord",
         "@abseil-cpp//absl/synchronization",

tensorstore/kvstore/neuroglancer_uint64_sharded/neuroglancer_uint64_sharded.cc

@@ -27,7 +27,6 @@
 #include <utility>
 #include <vector>
 
-#include "absl/base/internal/endian.h"
 #include "absl/base/optimization.h"
 #include "absl/status/status.h"
 #include "absl/strings/cord.h"

tensorstore/kvstore/neuroglancer_uint64_sharded/uint64_sharded_decoder.cc

@@ -23,13 +23,13 @@
 #include <vector>
 
 #include "absl/algorithm/container.h"
-#include "absl/base/internal/endian.h"
 #include "absl/status/status.h"
 #include "absl/strings/cord.h"
 #include "tensorstore/internal/compression/zlib.h"
 #include "tensorstore/internal/cord_util.h"
 #include "tensorstore/kvstore/byte_range.h"
 #include "tensorstore/kvstore/neuroglancer_uint64_sharded/uint64_sharded.h"
+#include "tensorstore/util/endian.h"
 #include "tensorstore/util/result.h"
 #include "tensorstore/util/span.h"
 #include "tensorstore/util/status.h"
@@ -57,13 +57,13 @@ Result<std::vector<MinishardIndexEntry>> DecodeMinishardIndex(
   uint64_t byte_offset = 0;
   for (size_t i = 0; i < result.size(); ++i) {
     auto& entry = result[i];
-    chunk_id.value += absl::little_endian::Load64(decoded_flat.data() + i * 8);
+    chunk_id.value += little_endian::Load64(decoded_flat.data() + i * 8);
     entry.chunk_id = chunk_id;
-    byte_offset += absl::little_endian::Load64(decoded_flat.data() + i * 8 +
-                                               8 * result.size());
+    byte_offset +=
+        little_endian::Load64(decoded_flat.data() + i * 8 + 8 * result.size());
     entry.byte_range.inclusive_min = byte_offset;
-    byte_offset += absl::little_endian::Load64(decoded_flat.data() + i * 8 +
-                                               16 * result.size());
+    byte_offset +=
+        little_endian::Load64(decoded_flat.data() + i * 8 + 16 * result.size());
     entry.byte_range.exclusive_max = byte_offset;
     if (!entry.byte_range.SatisfiesInvariants()) {
       return absl::InvalidArgumentError(tensorstore::StrCat(
@@ -108,8 +108,8 @@ Result<ByteRange> DecodeShardIndexEntry(std::string_view input) {
         "Expected 16 bytes, but received: ", input.size(), " bytes"));
   }
   ByteRange r;
-  r.inclusive_min = absl::little_endian::Load64(input.data());
-  r.exclusive_max = absl::little_endian::Load64(input.data() + 8);
+  r.inclusive_min = little_endian::Load64(input.data());
+  r.exclusive_max = little_endian::Load64(input.data() + 8);
   if (!r.SatisfiesInvariants()) {
     return absl::FailedPreconditionError(
         tensorstore::StrCat("Shard index specified invalid byte range: ", r));

tensorstore/kvstore/neuroglancer_uint64_sharded/uint64_sharded_encoder.cc

@@ -21,14 +21,14 @@
 #include <string>
 #include <utility>
 
-#include "absl/base/internal/endian.h"
 #include "absl/functional/function_ref.h"
 #include "absl/status/status.h"
 #include "absl/strings/cord.h"
 #include "tensorstore/internal/compression/zlib.h"
 #include "tensorstore/internal/flat_cord_builder.h"
 #include "tensorstore/kvstore/byte_range.h"
 #include "tensorstore/kvstore/neuroglancer_uint64_sharded/uint64_sharded.h"
+#include "tensorstore/util/endian.h"
 #include "tensorstore/util/result.h"
 #include "tensorstore/util/span.h"
 #include "tensorstore/util/status.h"
@@ -44,12 +44,11 @@ absl::Cord EncodeMinishardIndex(
   int64_t prev_offset = 0;
   for (ptrdiff_t i = 0; i < minishard_index.size(); ++i) {
     const auto& e = minishard_index[i];
-    absl::little_endian::Store64(builder.data() + i * 8,
-                                 e.chunk_id.value - prev_chunk_id.value);
-    absl::little_endian::Store64(
-        builder.data() + minishard_index.size() * 8 + i * 8,
-        e.byte_range.inclusive_min - prev_offset);
-    absl::little_endian::Store64(
+    little_endian::Store64(builder.data() + i * 8,
+                           e.chunk_id.value - prev_chunk_id.value);
+    little_endian::Store64(builder.data() + minishard_index.size() * 8 + i * 8,
+                           e.byte_range.inclusive_min - prev_offset);
+    little_endian::Store64(
         builder.data() + minishard_index.size() * 16 + i * 8,
         e.byte_range.exclusive_max - e.byte_range.inclusive_min);
     prev_chunk_id = e.chunk_id;
@@ -62,8 +61,8 @@ absl::Cord EncodeShardIndex(span<const ShardIndexEntry> shard_index) {
   internal::FlatCordBuilder builder(shard_index.size() * 16);
   for (ptrdiff_t i = 0; i < shard_index.size(); ++i) {
     const auto& e = shard_index[i];
-    absl::little_endian::Store64(builder.data() + i * 16, e.inclusive_min);
-    absl::little_endian::Store64(builder.data() + i * 16 + 8, e.exclusive_max);
+    little_endian::Store64(builder.data() + i * 16, e.inclusive_min);
+    little_endian::Store64(builder.data() + i * 16 + 8, e.exclusive_max);
   }
   return std::move(builder).Build();
 }

tensorstore/kvstore/ocdbt/distributed/BUILD

@@ -154,7 +154,7 @@ tensorstore_cc_library(
     deps = [
         "//tensorstore/kvstore:key_range",
         "//tensorstore/kvstore/ocdbt/format",
-        "@abseil-cpp//absl/base:endian",
+        "//tensorstore/util:endian",
         "@blake3",
     ],
 )

tensorstore/kvstore/ocdbt/distributed/btree_node_identifier.cc

@@ -14,13 +14,15 @@
 
 #include "tensorstore/kvstore/ocdbt/distributed/btree_node_identifier.h"
 
+#include <stdint.h>
+
 #include <ostream>
 #include <string>
 #include <string_view>
 
-#include "absl/base/internal/endian.h"
 #include <blake3.h>
 #include "tensorstore/kvstore/key_range.h"
+#include "tensorstore/util/endian.h"
 
 namespace tensorstore {
 namespace internal_ocdbt {
@@ -38,7 +40,7 @@ std::string BtreeNodeIdentifier::GetKey(
                        database_identifier.size());
   char header[3];
   header[0] = static_cast<char>(range.full() ? 0 : height);
-  absl::little_endian::Store16(&header[1], range.inclusive_min.size());
+  little_endian::Store16(&header[1], range.inclusive_min.size());
   blake3_hasher_update(&hasher, header, 3);
   blake3_hasher_update(&hasher, range.inclusive_min.data(),
                        range.inclusive_min.size());