Test 1336 (#1347)

Co-authored-by: Noam Cohen (noamcohen97) <[email protected]>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: Duncan MacQuarrie <[email protected]>
Co-authored-by: 3dbrows <[email protected]>

Author: 5 people

DESCRIPTION.md

@@ -8,6 +8,10 @@ Source code is also available at: https://github.com/snowflakedb/snowflake-conne
 
 # Release Notes
 
+- v2.9.1(unreleased)
+
+  - Bumped pyarrow dependency from >=8.0.0,<8.1.0 to >=10.0.1,<10.1.0
+
 - v2.9.0(December 9, 2022)
 
   - Fixed a bug where the permission of the file downloaded via GET command is changed

pyproject.toml

@@ -6,7 +6,7 @@ requires = [
     "wheel",
     "cython",
     # Must be kept in sync with the `setup_requirements` in `setup.cfg`
-    "pyarrow>=8.0.0,<8.1.0",
+    "pyarrow>=10.0.1,<10.1.0",
 ]
 
 [tool.cibuildwheel]

setup.cfg

@@ -91,6 +91,6 @@ development =
     pytzdata
 pandas =
     pandas>=1.0.0,<1.6.0
-    pyarrow>=8.0.0,<8.1.0
+    pyarrow>=10.0.1,<10.1.0
 secure-local-storage =
     keyring!=16.1.0,<24.0.0

setup.py

@@ -71,23 +71,45 @@ class MyBuildExt(build_ext):
         # this list should be carefully examined when pyarrow lib is
         # upgraded
         arrow_libs_to_copy = {
-            "linux": ["libarrow.so.800", "libarrow_python.so.800", "libparquet.so.800"],
+            "linux": [
+                "libarrow.so.1000",
+                "libarrow_dataset.so.1000",
+                "libarrow_python.so.1000",
+                "libparquet.so.1000",
+            ],
             "darwin": [
-                "libarrow.800.dylib",
-                "libarrow_python.800.dylib",
-                "libparquet.800.dylib",
+                "libarrow.1000.dylib",
+                "libarrow_dataset.1000.dylib",
+                "libarrow_python.1000.dylib",
+                "libparquet.1000.dylib",
+            ],
+            "win32": [
+                "arrow.dll",
+                "arrow_dataset.dll",
+                "arrow_python.dll",
+                "parquet.dll",
             ],
-            "win32": ["arrow.dll", "arrow_python.dll", "parquet.dll"],
         }
 
         arrow_libs_to_link = {
-            "linux": ["libarrow.so.800", "libarrow_python.so.800", "libparquet.so.800"],
+            "linux": [
+                "libarrow.so.1000",
+                "libarrow_dataset.so.1000",
+                "libarrow_python.so.1000",
+                "libparquet.so.1000",
+            ],
             "darwin": [
-                "libarrow.800.dylib",
-                "libarrow_python.800.dylib",
-                "libparquet.800.dylib",
+                "libarrow.1000.dylib",
+                "libarrow_dataset.1000.dylib",
+                "libarrow_python.1000.dylib",
+                "libparquet.1000.dylib",
+            ],
+            "win32": [
+                "arrow.lib",
+                "arrow_dataset.lib",
+                "arrow_python.lib",
+                "parquet.lib",
             ],
-            "win32": ["arrow.lib", "arrow_python.lib", "parquet.lib"],
         }
 
         def build_extension(self, ext):
@@ -126,13 +148,15 @@ def build_extension(self, ext):
                 ext.include_dirs.append(LOGGING_SRC_DIR)
 
                 if sys.platform == "win32":
+                    if not any("/std" not in s for s in ext.extra_compile_args):
+                        ext.extra_compile_args.append("/std:c++17")
                     ext.include_dirs.append(pyarrow.get_include())
                     ext.include_dirs.append(numpy.get_include())
                 elif sys.platform == "linux" or sys.platform == "darwin":
                     ext.extra_compile_args.append("-isystem" + pyarrow.get_include())
                     ext.extra_compile_args.append("-isystem" + numpy.get_include())
                     if "std=" not in os.environ.get("CXXFLAGS", ""):
-                        ext.extra_compile_args.append("-std=c++11")
+                        ext.extra_compile_args.append("-std=c++17")
                         ext.extra_compile_args.append("-D_GLIBCXX_USE_CXX11_ABI=0")
 
                 ext.library_dirs.append(
@@ -160,9 +184,11 @@ def _get_arrow_lib_dir(self):
         def _copy_arrow_lib(self):
             libs_to_bundle = self.arrow_libs_to_copy[sys.platform]
 
+            build_dir = os.path.join(self.build_lib, "snowflake", "connector")
+            os.makedirs(build_dir, exist_ok=True)
+
             for lib in libs_to_bundle:
                 source = f"{self._get_arrow_lib_dir()}/{lib}"
-                build_dir = os.path.join(self.build_lib, "snowflake", "connector")
                 copy(source, build_dir)
 
         def _get_arrow_lib_as_linker_input(self):

src/snowflake/connector/cpp/ArrowIterator/BinaryConverter.cpp

@@ -18,7 +18,7 @@ PyObject* BinaryConverter::toPyObject(int64_t rowIndex) const
 {
   if (m_array->IsValid(rowIndex))
   {
-    arrow::util::string_view sv = m_array->GetView(rowIndex);
+    std::string_view sv = m_array->GetView(rowIndex);
     return PyByteArray_FromStringAndSize(sv.data(), sv.size());
   }
   else

src/snowflake/connector/cpp/ArrowIterator/StringConverter.cpp

@@ -18,7 +18,7 @@ PyObject* StringConverter::toPyObject(int64_t rowIndex) const
 {
   if (m_array->IsValid(rowIndex))
   {
-    arrow::util::string_view sv = m_array->GetView(rowIndex);
+    std::string_view sv = m_array->GetView(rowIndex);
     return PyUnicode_FromStringAndSize(sv.data(), sv.size());
   }
   else