work

Author: rok

cpp/src/arrow/compute/kernels/scalar_string_test.cc

@@ -1225,8 +1225,8 @@ TYPED_TEST(TestStringKernels, Utf8Upper) {
   // test maximum buffer growth
   this->CheckUnary("utf8_upper", "[\"ɑɑɑɑ\"]", this->type(), "[\"ⱭⱭⱭⱭ\"]");
 
-  // Test invalid data
-  auto invalid_input = ArrayFromJSON(this->type(), "[\"ɑa\xFFɑ\", \"ɽ\xe1\xbdɽaa\"]");
+  // Test invalid data - use MakeArray since simdjson validates UTF-8 strictly
+  auto invalid_input = this->MakeArray({"ɑa\xFFɑ", "ɽ\xe1\xbdɽaa"});
   EXPECT_RAISES_WITH_MESSAGE_THAT(Invalid, testing::HasSubstr("Invalid UTF8 sequence"),
                                   CallFunction("utf8_upper", {invalid_input}));
 }
@@ -1247,8 +1247,8 @@ TYPED_TEST(TestStringKernels, Utf8Lower) {
   // test maximum buffer growth
   this->CheckUnary("utf8_lower", "[\"ȺȺȺȺ\"]", this->type(), "[\"ⱥⱥⱥⱥ\"]");
 
-  // Test invalid data
-  auto invalid_input = ArrayFromJSON(this->type(), "[\"Ⱥa\xFFⱭ\", \"Ɽ\xe1\xbdⱤaA\"]");
+  // Test invalid data - use MakeArray since simdjson validates UTF-8 strictly
+  auto invalid_input = this->MakeArray({"Ⱥa\xFFⱭ", "Ɽ\xe1\xbdⱤaA"});
   EXPECT_RAISES_WITH_MESSAGE_THAT(Invalid, testing::HasSubstr("Invalid UTF8 sequence"),
                                   CallFunction("utf8_lower", {invalid_input}));
 }
@@ -1267,8 +1267,8 @@ TYPED_TEST(TestStringKernels, Utf8SwapCase) {
   this->CheckUnary("utf8_swapcase", "[\"hEllO, WoRld!\", \"$. A35?\"]", this->type(),
                    "[\"HeLLo, wOrLD!\", \"$. a35?\"]");
 
-  // Test invalid data
-  auto invalid_input = ArrayFromJSON(this->type(), "[\"Ⱥa\xFFⱭ\", \"Ɽ\xe1\xbdⱤaA\"]");
+  // Test invalid data - use MakeArray since simdjson validates UTF-8 strictly
+  auto invalid_input = this->MakeArray({"Ⱥa\xFFⱭ", "Ɽ\xe1\xbdⱤaA"});
   EXPECT_RAISES_WITH_MESSAGE_THAT(Invalid, testing::HasSubstr("Invalid UTF8 sequence"),
                                   CallFunction("utf8_swapcase", {invalid_input}));
 }

cpp/src/arrow/dataset/file_json.cc

@@ -53,6 +53,18 @@ namespace {
 
 using ReaderPtr = std::shared_ptr<json::StreamingReader>;
 
+// Strip UTF-8 BOM from the beginning of data if present
+std::string_view StripBOM(std::string_view data) {
+  // UTF-8 BOM is 0xEF 0xBB 0xBF
+  if (data.size() >= 3) {
+    const auto* bytes = reinterpret_cast<const uint8_t*>(data.data());
+    if (bytes[0] == 0xEF && bytes[1] == 0xBB && bytes[2] == 0xBF) {
+      return data.substr(3);
+    }
+  }
+  return data;
+}
+
 struct JsonInspectedFragment : public InspectedFragment {
   JsonInspectedFragment() : InspectedFragment({}) {}
   JsonInspectedFragment(std::vector<std::string> column_names,
@@ -137,6 +149,8 @@ json::ParseOptions GetInitialParseOptions(json::ParseOptions options) {
 
 Result<std::shared_ptr<StructType>> ParseToStructType(
     std::string_view data, const json::ParseOptions& parse_options, MemoryPool* pool) {
+  // Strip UTF-8 BOM if present
+  data = StripBOM(data);
   auto full_buffer = std::make_shared<Buffer>(data);
   std::shared_ptr<Buffer> buffer, partial;
   auto chunker = json::MakeChunker(parse_options);