ARROW-2034: [C++] Filesystem implementation for Azure Blob Storage

.github/workflows/cpp.yml

@@ -126,6 +126,7 @@ jobs:
     strategy:
       fail-fast: false
     env:
+      ARROW_AZURE: ON
       ARROW_BUILD_TESTS: ON
       ARROW_DATASET: ON
       ARROW_FLIGHT: ON
@@ -166,6 +167,9 @@ jobs:
             ci/scripts/install_minio.sh latest /usr/local
       - name: Install Google Cloud Storage Testbench
         run: ci/scripts/install_gcs_testbench.sh default
+      - name: Install Azurite Storage Emulator
+        shell: bash
+        run: ci/scripts/install_azurite.sh
       - name: Setup ccache
         run: |
           ci/scripts/ccache_setup.sh
@@ -296,6 +300,7 @@ jobs:
           - 32
           - 64
     env:
+      ARROW_AZURE: OFF
       ARROW_BUILD_SHARED: ON
       ARROW_BUILD_STATIC: OFF
       ARROW_BUILD_TESTS: ON

ci/appveyor-cpp-build.bat

@@ -56,6 +56,7 @@ pushd cpp\build
 @rem and enable runtime assertions.
 
 cmake -G "%GENERATOR%" %CMAKE_ARGS% ^
+      -DARROW_AZURE=OFF ^
       -DARROW_BOOST_USE_SHARED=ON ^
       -DARROW_BUILD_EXAMPLES=ON ^
       -DARROW_BUILD_STATIC=OFF ^

ci/docker/ubuntu-18.04-cpp.dockerfile

@@ -100,7 +100,8 @@ RUN apt-get update -y -q && \
 # - s3 tests would require boost-asio that is included since Boost 1.66.0
 # ARROW-17051: this build uses static Protobuf, so we must also use
 # static Arrow to run Flight/Flight SQL tests
-ENV ARROW_BUILD_STATIC=ON \
+ENV ARROW_AZURE=ON \
+    ARROW_BUILD_STATIC=ON \
     ARROW_BUILD_TESTS=ON \
     ARROW_DATASET=ON \
     ARROW_DEPENDENCY_SOURCE=SYSTEM \

ci/docker/ubuntu-20.04-cpp.dockerfile

@@ -94,6 +94,7 @@ RUN apt-get update -y -q && \
         make \
         ninja-build \
         nlohmann-json3-dev \
+        npm \
         pkg-config \
         protobuf-compiler \
         python3-dev \
@@ -113,6 +114,9 @@ RUN /arrow/ci/scripts/install_minio.sh latest /usr/local
 COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/
 RUN /arrow/ci/scripts/install_gcs_testbench.sh default
 
+COPY ci/scripts/install_azurite.sh /arrow/ci/scripts/
+RUN /arrow/ci/scripts/install_azurite.sh
+
 COPY ci/scripts/install_ceph.sh /arrow/ci/scripts/
 RUN /arrow/ci/scripts/install_ceph.sh
 
@@ -127,6 +131,7 @@ RUN /arrow/ci/scripts/install_ceph.sh
 # ARROW-17051: this build uses static Protobuf, so we must also use
 # static Arrow to run Flight/Flight SQL tests
 ENV absl_SOURCE=BUNDLED \
+    ARROW_AZURE=ON \
     ARROW_BUILD_STATIC=ON \
     ARROW_BUILD_TESTS=ON \
     ARROW_DEPENDENCY_SOURCE=SYSTEM \

ci/docker/ubuntu-22.04-cpp.dockerfile

@@ -93,6 +93,7 @@ RUN apt-get update -y -q && \
         make \
         ninja-build \
         nlohmann-json3-dev \
+        npm \
         pkg-config \
         protobuf-compiler \
         protobuf-compiler-grpc \
@@ -143,6 +144,9 @@ RUN /arrow/ci/scripts/install_minio.sh latest /usr/local
 COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/
 RUN /arrow/ci/scripts/install_gcs_testbench.sh default
 
+COPY ci/scripts/install_azurite.sh /arrow/ci/scripts/
+RUN /arrow/ci/scripts/install_azurite.sh
+
 # Prioritize system packages and local installation
 # The following dependencies will be downloaded due to missing/invalid packages
 # provided by the distribution:
@@ -154,6 +158,7 @@ RUN /arrow/ci/scripts/install_gcs_testbench.sh default
 # ARROW-17051: this build uses static Protobuf, so we must also use
 # static Arrow to run Flight/Flight SQL tests
 ENV absl_SOURCE=BUNDLED \
+    ARROW_AZURE=ON \
     ARROW_BUILD_STATIC=ON \
     ARROW_BUILD_TESTS=ON \
     ARROW_DEPENDENCY_SOURCE=SYSTEM \

ci/scripts/cpp_build.sh

@@ -70,6 +70,7 @@ pushd ${build_dir}
 
 cmake \
   -Dabsl_SOURCE=${absl_SOURCE:-} \
+  -DARROW_AZURE=${ARROW_AZURE:-OFF} \
   -DARROW_BOOST_USE_SHARED=${ARROW_BOOST_USE_SHARED:-ON} \
   -DARROW_BUILD_BENCHMARKS_REFERENCE=${ARROW_BUILD_BENCHMARKS:-OFF} \
   -DARROW_BUILD_BENCHMARKS=${ARROW_BUILD_BENCHMARKS:-OFF} \
@@ -142,7 +143,7 @@ cmake \
   -DCMAKE_VERBOSE_MAKEFILE=${CMAKE_VERBOSE_MAKEFILE:-OFF} \
   -DCMAKE_C_FLAGS="${CFLAGS:-}" \
   -DCMAKE_CXX_FLAGS="${CXXFLAGS:-}" \
-  -DCMAKE_CXX_STANDARD="${CMAKE_CXX_STANDARD:-11}" \
+  -DCMAKE_CXX_STANDARD="${CMAKE_CXX_STANDARD:-}" \
   -DCMAKE_INSTALL_LIBDIR=${CMAKE_INSTALL_LIBDIR:-lib} \
   -DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX:-${ARROW_HOME}} \
   -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD:-OFF} \

ci/scripts/install_azurite.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -e
+
+case "$(uname)" in
+  Darwin)
+    npm install -g azurite
+    which azurite
+    ;;
+  MINGW*)
+    choco install nodejs.install
+    npm install -g azurite
+    ;;
+  Linux)
+    npm install -g azurite
+    which azurite
+    ;;
+esac
+echo "node version = $(node --version)"
+echo "azurite version = $(azurite --version)"

cpp/Brewfile

@@ -31,6 +31,7 @@ brew "llvm"
 brew "llvm@12"
 brew "lz4"
 brew "ninja"
+brew "node"
 brew "numpy"
 brew "[email protected]"
 brew "protobuf"

cpp/CMakeLists.txt

@@ -835,6 +835,11 @@ if(ARROW_WITH_OPENTELEMETRY)
   list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS CURL::libcurl)
 endif()
 
+if(ARROW_AZURE)
+  list(APPEND ARROW_LINK_LIBS ${AZURESDK_LINK_LIBRARIES})
+  list(APPEND ARROW_STATIC_LINK_LIBS ${AZURESDK_LINK_LIBRARIES})
+endif()
+
 if(ARROW_WITH_UTF8PROC)
   list(APPEND ARROW_LINK_LIBS utf8proc::utf8proc)
   list(APPEND ARROW_STATIC_LINK_LIBS utf8proc::utf8proc)

cpp/cmake_modules/DefineOptions.cmake

@@ -226,6 +226,9 @@ if(ARROW_DEFINE_OPTIONS)
   #----------------------------------------------------------------------
   set_option_category("Project component")
 
+  define_option(ARROW_AZURE
+                "Build Arrow with Azure support (requires the Azure SDK for C++)" OFF)
+
   define_option(ARROW_BUILD_UTILITIES "Build Arrow commandline utilities" OFF)
 
   define_option(ARROW_COMPUTE "Build the Arrow Compute Modules" OFF)

cpp/cmake_modules/SetupCxxFlags.cmake

@@ -118,16 +118,20 @@ elseif(ARROW_CPU_FLAG STREQUAL "armv8")
 endif()
 
 # Support C11
-if(NOT DEFINED CMAKE_C_STANDARD)
+if("${CMAKE_C_STANDARD}" STREQUAL "")
   set(CMAKE_C_STANDARD 11)
 endif()
 
-# This ensures that things like c++11 get passed correctly
-if(NOT DEFINED CMAKE_CXX_STANDARD)
-  set(CMAKE_CXX_STANDARD 11)
+# This ensures that things like c++11/c++14 get passed correctly
+if("${CMAKE_CXX_STANDARD}" STREQUAL "")
+  if(ARROW_AZURE)
+    set(CMAKE_CXX_STANDARD 14)
+  else()
+    set(CMAKE_CXX_STANDARD 11)
+  endif()
 endif()
 
-# We require a C++11 compliant compiler
+# We require a C++11/14 compliant compiler
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 
 # ARROW-6848: Do not use GNU (or other CXX) extensions