Skip to content

Commit 27e11fb

Browse files
sfc-gh-zyaosfc-gh-zyao
committed
SNOW-2483517 Adding anaconda ci build scripts for sproc CI/CD pipeline
1 parent db2a10f commit 27e11fb

File tree

10 files changed

+484
-32
lines changed

10 files changed

+484
-32
lines changed

‎.pre-commit-config.yaml‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,11 @@ repos:
2020
src/snowflake/connector/nanoarrow_cpp/ArrowIterator/flatcc/.*\.h|
2121
)$
2222
- id: check-yaml
23-
exclude: .github/repo_meta.yaml
23+
exclude: >
24+
(?x)^(
25+
.github/repo_meta.yaml|
26+
ci/anaconda/recipe/meta.yaml|
27+
)$
2428
- id: debug-statements
2529
- id: check-ast
2630
- repo: https://github.com/asottile/yesqa

‎ci/anaconda/bld.bat‎

Lines changed: 0 additions & 1 deletion
This file was deleted.

‎ci/anaconda/build.sh‎

Lines changed: 0 additions & 1 deletion
This file was deleted.

‎ci/anaconda/conda_build.sh‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
# Check https://snow-external.slack.com/archives/C02D68R4D0D/p1678899446863299 for context about using --numpy
2+
conda install conda-build
3+
conda install conda-verify
4+
conda install diffutils
5+
conda build ci/anaconda/recipe/ --python 3.9
6+
conda build ci/anaconda/recipe/ --python 3.10
7+
conda build ci/anaconda/recipe/ --python 3.11
8+
conda build ci/anaconda/recipe/ --python 3.12
9+
conda build ci/anaconda/recipe/ --python 3.13

‎ci/anaconda/meta.yaml‎

Lines changed: 0 additions & 29 deletions
This file was deleted.

‎ci/anaconda/package_builder.sh‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
#!/bin/bash
2+
3+
export SNOWFLAKE_CONNECTOR_PYTHON_DIR=/repo/snowflake-connector-python
4+
export CONDA_BLD_PATH=/repo/conda-bld
5+
6+
mkdir -p $CONDA_BLD_PATH
7+
cd "$SNOWFLAKE_CONNECTOR_PYTHON_DIR"
8+
conda config --set conda_build.pkg_format 1
9+
bash ./ci/anaconda/conda_build.sh
10+
# NOTE: the below is to output the build in .conda format.
11+
# To do so, we set conda_build.pkg_format = 2 and then
12+
# remove it later to go back to default behavior.
13+
conda config --set conda_build.pkg_format 2
14+
bash ./ci/anaconda/conda_build.sh
15+
conda config --remove-key conda_build.pkg_format
16+
conda build purge
17+
cd $CONDA_BLD_PATH
18+
conda index .
19+
chmod -R o+w,g+w $CONDA_BLD_PATH

‎ci/anaconda/recipe/meta.yaml‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
{% set name = "snowflake-connector-python" %}
2+
{% set version = os.environ.get('SNOWFLAKE_CONNECTOR_PYTHON_VERSION', 0) %}
3+
{% set build_number = os.environ.get('PUBLIC_CONNECTOR_BUILD_NUMBER', 0) %}
4+
5+
package:
6+
name: {{ name|lower }}
7+
version: {{ version }}
8+
9+
source:
10+
path: ../../..
11+
12+
build:
13+
number: {{ build_number }}
14+
string: "py{{ py }}_{{ build_number }}"
15+
script:
16+
- export SF_NO_COPY_ARROW_LIB=1 # [unix]
17+
- export SF_ARROW_LIBDIR="${PREFIX}/lib" # [unix]
18+
- export ENABLE_EXT_MODULES=true # [unix]
19+
- {{ PYTHON }} -m pip install . --no-use-pep517 --no-deps -vvv
20+
entry_points:
21+
- snowflake-dump-ocsp-response = snowflake.connector.tool.dump_ocsp_response:main
22+
- snowflake-dump-ocsp-response-cache = snowflake.connector.tool.dump_ocsp_response_cache:main
23+
- snowflake-dump-certs = snowflake.connector.tool.dump_certs:main
24+
25+
requirements:
26+
build:
27+
- {{ compiler("c") }}
28+
- {{ compiler("cxx") }}
29+
- libgcc-ng
30+
- libstdcxx-ng
31+
- patch # [not win]
32+
host:
33+
- setuptools >=40.6.0
34+
- wheel
35+
- cython
36+
- python {{ python }}
37+
run:
38+
{% if py == 39 %}
39+
- python >=3.9,<3.10.0a0
40+
{% elif py == 310 %}
41+
- python >=3.10,<3.11.0a0
42+
{% elif py == 311 %}
43+
- python >=3.11,<3.12.0a0
44+
{% elif py == 312 %}
45+
- python >=3.12,<3.13.0a0
46+
{% elif py == 313 %}
47+
- python >=3.13,<3.14.0a0
48+
{% else %}
49+
- python
50+
{% endif %}
51+
- asn1crypto >0.24.0,<2.0.0
52+
- cryptography >=44.0.1
53+
- pyOpenSSL >=24.0.0,<26.0.0
54+
- pyjwt >=2.10.1,<3.0.0
55+
- pytz
56+
- requests >=2.32.4,<3.0.0
57+
- packaging
58+
- charset-normalizer >=2,<4
59+
- idna >=3.7,<4
60+
- urllib3 >=1.26.5,<2.0.0 # [py<310]
61+
- certifi >=2024.7.4
62+
- typing_extensions >=4.3,<5
63+
- filelock >=3.5,<4
64+
- sortedcontainers >=2.4.0
65+
- platformdirs >=2.6.0,<5.0.0
66+
- tomlkit
67+
- boto3 >=1.24
68+
- botocore >=1.24
69+
test:
70+
requires:
71+
- pip
72+
imports:
73+
- snowflake
74+
- snowflake.connector
75+
- snowflake.connector.nanoarrow_arrow_iterator # [unix]
76+
commands:
77+
- pip check
78+
79+
about:
80+
home: https://github.com/snowflakedb/snowflake-connector-python
81+
summary: Snowflake Connector for Python

‎ci/anaconda/run.sh‎

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
2+
# Before manual running, do something similar to the following in command line.
3+
# export WORKSPACE=/home/jdoe/my_workspace;
4+
# export PUBLIC_CONNECTOR_BUILD_NUMBER=321;
5+
# export aarch64_base_image=<location_of_centos8_aarch64_base_image>;
6+
# export x86_base_image=<location_of_centos8_x86_base_image>;
7+
8+
# Here miniconda-install.sh is just a installer that I downloaded from Anaconda official site,
9+
# https://repo.anaconda.com/miniconda/
10+
11+
12+
if [[ -z $WORKSPACE ]]; then
13+
# Development on dev machine
14+
WORKSPACE=$HOME
15+
fi
16+
17+
# ===== Build docker image =====
18+
cd $WORKSPACE
19+
20+
# Validate dependency sync before building
21+
python3 $WORKSPACE/snowflake-connector-python/ci/anaconda/validate_deps_sync.py
22+
if [[ $? -ne 0 ]]; then
23+
echo "[FAILURE] setup.cfg and meta.yaml dependencies are not in sync"
24+
exit 1
25+
fi
26+
27+
docker build \
28+
--build-arg ARCH=$(uname -m) \
29+
--build-arg AARCH64_BASE_IMAGE="${aarch64_base_image}" \
30+
--build-arg X86_BASE_IMAGE="${x86_base_image}" \
31+
-t snowflake_connector_python_image \
32+
-f - . <<'DOCKERFILE'
33+
# Use different base images based on target platform
34+
35+
ARG ARCH
36+
ARG AARCH64_BASE_IMAGE=artifactory.int.snowflakecomputing.com/development-docker-virtual/arm64v8/centos:8
37+
ARG X86_BASE_IMAGE=artifactory.int.snowflakecomputing.com/development-docker-virtual/centos:8
38+
39+
FROM ${AARCH64_BASE_IMAGE} AS base-aarch64
40+
41+
FROM ${X86_BASE_IMAGE} AS base-x86_64
42+
43+
44+
45+
# Select the appropriate base image based on target architecture
46+
47+
FROM base-${ARCH} AS base
48+
49+
COPY miniconda-install.sh .
50+
51+
52+
53+
RUN chmod 0755 miniconda-install.sh
54+
55+
56+
57+
RUN mkdir -p /etc/miniconda && bash miniconda-install.sh -b -u -p /etc/miniconda/
58+
59+
60+
61+
RUN ln -s /etc/miniconda/bin/conda /usr/bin/conda && rm miniconda-install.sh
62+
DOCKERFILE
63+
64+
# Go back to the original directory
65+
cd $WORKSPACE
66+
67+
68+
# Check to make sure repos exist to build conda packages
69+
if [[ -d $WORKSPACE/snowflake-connector-python ]]; then
70+
echo "Check snowflake-connector-python repo exists - PASSED"
71+
else
72+
echo "[FAILURE] Please clone snowflake-connector-python repo at $WORKSPACE/snowflake-connector-python"
73+
fi
74+
75+
# Extract connector version if not provided
76+
if [[ -z "$SNOWFLAKE_CONNECTOR_PYTHON_VERSION" ]]; then
77+
VERSION_FILE="$WORKSPACE/snowflake-connector-python/src/snowflake/connector/version.py"
78+
if [[ -f "$VERSION_FILE" ]]; then
79+
SNOWFLAKE_CONNECTOR_PYTHON_VERSION=$( \
80+
grep -Eo 'VERSION\s*=\s*\([^)]*\)' "$VERSION_FILE" \
81+
| grep -Eo '[0-9]+' \
82+
| paste -sd '.' - \
83+
)
84+
export SNOWFLAKE_CONNECTOR_PYTHON_VERSION
85+
fi
86+
fi
87+
88+
# Run packager in docker image
89+
docker run \
90+
-v $WORKSPACE/snowflake-connector-python/:/repo/snowflake-connector-python \
91+
-v $WORKSPACE/conda-bld:/repo/conda-bld \
92+
-e SNOWFLAKE_CONNECTOR_PYTHON_VERSION=${SNOWFLAKE_CONNECTOR_PYTHON_VERSION} \
93+
-e PUBLIC_CONNECTOR_BUILD_NUMBER=${PUBLIC_CONNECTOR_BUILD_NUMBER} \
94+
snowflake_connector_python_image \
95+
/repo/snowflake-connector-python/ci/anaconda/package_builder.sh
96+
97+
# Cleanup image for disk space
98+
docker container prune -f
99+
docker rmi snowflake_connector_python_image

0 commit comments

Comments
 (0)