Enable arrow tests in linux

Author: Haowei Yu

docker/manylinux2010/Dockerfile-x86_64_base

@@ -0,0 +1,19 @@
+# We use manylinux1 base image because pyarrow_manylinux2010 has a bug and wheel failed to be audited
+FROM quay.io/pypa/manylinux1_x86_64
+
+# This is to solve permission issue, read https://denibertovic.com/posts/handling-permissions-with-docker-volumes/
+RUN curl -o /usr/local/bin/gosu -SL "https://github.com/tianon/gosu/releases/download/1.11/gosu-amd64"
+RUN chmod +x /usr/local/bin/gosu
+
+COPY scripts/entrypoint.sh /usr/local/bin/entrypoint.sh
+RUN chmod +x /usr/local/bin/entrypoint.sh
+
+
+WORKDIR /home/user
+RUN chmod 777 /home/user
+RUN git clone https://github.com/matthew-brett/multibuild.git && cd /home/user/multibuild && git checkout 1a7f31be677185f2dface2643284846e14130c3f
+
+ADD scripts/build_virtualenvs.sh /home/user
+RUN /home/user/build_virtualenvs.sh
+
+ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]

docker/manylinux2010/scripts/build_virtualenvs.sh

@@ -0,0 +1,46 @@
+#!/bin/bash -e
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Build upon the scripts in https://github.com/matthew-brett/manylinux-builds
+# * Copyright (c) 2013-2016, Matt Terry and Matthew Brett (BSD 2-clause)
+
+PYTHON_VERSIONS="${PYTHON_VERSIONS:-35 36 37}"
+
+source /home/user/multibuild/manylinux_utils.sh
+
+for PYTHON in ${PYTHON_VERSIONS}; do
+    U_WIDTH=16
+    PYTHON_INTERPRETER="$(cpython_path $PYTHON {U_WIDTH})/bin/python"
+    PIP="$(cpython_path $PYTHON ${U_WIDTH})/bin/pip"
+    PATH="$PATH:$(cpython_path $PYTHON ${U_WIDTH})"
+
+    echo "=== (${PYTHON}, ${U_WIDTH}) Installing build dependencies ==="
+    $PIP install "virtualenv==16.3.0"
+
+    echo "=== (${PYTHON}, ${U_WIDTH}) Preparing virtualenv for build ==="
+    "$(cpython_path $PYTHON ${U_WIDTH})/bin/virtualenv" -p ${PYTHON_INTERPRETER} --no-download /home/user/venv-build-${PYTHON}
+    source /home/user/venv-build-${PYTHON}/bin/activate
+    pip install "cython==0.29.8" "setuptools" "flake8" "wheel" "pyarrow==0.14.1"
+    deactivate
+done
+
+# Remove pip cache again. It's useful during the virtualenv creation but we
+# don't want it persisted in the docker layer, ~264MiB
+rm -rf /root/.cache
+# Remove unused Python versions
+rm -rf /opt/_internal/cpython-3.4*

docker/manylinux2010/scripts/entrypoint.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+# Add local user
+# Either use the LOCAL_USER_ID if passed in at runtime or
+# fallback
+
+USER_ID=${LOCAL_USER_ID:-9001}
+
+echo "Starting with UID : $USER_ID"
+useradd --shell /bin/bash -u $USER_ID -o -c "" -m user
+export HOME=/home/user
+
+/usr/local/bin/gosu user "$@"

scripts/build_connector_linux.sh

@@ -0,0 +1,20 @@
+#!/bin/bash -e
+#
+# Install Snowflake Python Connector
+#
+set -o pipefail
+
+THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+cd $THIS_DIR/..
+PYTHON_VERSION=$1
+source "/home/user/venv-build-${PYTHON_VERSION}/bin/activate"
+rm -rf build/
+export ENABLE_EXT_MODULES=true
+rm -f generated_version.py
+CC=g++ python setup.py bdist_wheel -d $THIS_DIR/../dist/docker/$PYTHON_VERSION/
+unset ENABLE_EXT_MODULES
+
+mkdir -p $THIS_DIR/../dist/docker/repaired_wheels
+auditwheel repair --plat manylinux2010_x86_64 -L connector $THIS_DIR/../dist/docker/$PYTHON_VERSION/*.whl -w $THIS_DIR/../dist/docker/repaired_wheels
+rm $THIS_DIR/../dist/docker/repaired_wheels/*manylinux1_x86_64.whl || true

scripts/build_inside_docker.sh

@@ -0,0 +1,16 @@
+#!/bin/bash -e
+#
+# Install Snowflake Python Connector
+#
+set -o pipefail
+
+THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+cd $THIS_DIR/../docker/manylinux2010
+
+echo "[Info] Start building dokcer image"
+docker build -t manylinux:1.0 -f Dockerfile-x86_64_base .
+
+user_id=$(id -u $USER)
+docker run -e LOCAL_USER_ID=$user_id --mount type=bind,source="$THIS_DIR/..",target=/home/user/connector manylinux:1.0 \
+    /home/user/connector/scripts/build_connector_linux.sh $1

scripts/install.sh

@@ -4,6 +4,8 @@
 #
 set -o pipefail
 
+THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
 if [ "$TRAVIS_OS_NAME" == "osx" ]; then
     brew update
     brew install openssl readline sqlite3 xz zlib
@@ -33,10 +35,22 @@ source ./venv/bin/activate
 pip install pandas
 pip install numpy
 pip install pendulum
-pip install pyarrow
 pip install pytest pytest-cov pytest-rerunfailures
 if [[ "$TRAVIS_PYTHON_VERSION" == "2.7" ]] || [[ $PYTHON_VERSION == "2.7"* ]]; then
     pip install mock
 fi
-pip install .
+
+if [ "$TRAVIS_OS_NAME" == "osx" ]; then
+    pip install .
+else
+    if [[ "$TRAVIS_PYTHON_VERSION" == "2.7" ]]; then
+        pip install .
+    else
+        pv=${TRAVIS_PYTHON_VERSION/./}
+        $THIS_DIR/build_inside_docker.sh $pv 
+        CONNECTOR_WHL=$(ls $THIS_DIR/../dist/docker/repaired_wheels/snowflake_connector_python*cp${PYTHON_ENV}*.whl | sort -r | head -n 1)
+        pip install -U $CONNECTOR_WHL[pandas]
+        cd $THIS_DIR/..
+    fi
+fi
 pip list --format=columns