From e7a60f6fd4b3162615a8a340ff50e17c4eb0dfb7 Mon Sep 17 00:00:00 2001 From: Henry Finucane Date: Tue, 11 Oct 2022 18:18:41 +0000 Subject: [PATCH 1/9] Support Python3.8 --- Dockerfile.centos7 | 2 +- build_and_test_platform.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile.centos7 b/Dockerfile.centos7 index f90d536..4e96f25 100644 --- a/Dockerfile.centos7 +++ b/Dockerfile.centos7 @@ -6,7 +6,7 @@ ARG PYTHON_VERSION=3.8.5 ARG ASSET_VERSION=local_build ARG GREP_EXCLUDE='(ld.so|ld-linux-x86-64.so|libBrokenLocale.so|libSegFault.so|libanl.so|libc.so|libdl.so|libm.so|libmvec.so|libnss_compat.so|libnss_dns.so|libnss_files.so|libpthread.so|libresolv.so|librt.so|libthread_db.so|libutil.so|vdso.so)' -RUN yum update -y && yum groupinstall -y "Development Tools" && yum install -y curl bzip2-devel zlib-devel sqlite-devel openssl-devel readline-devel +RUN yum update -y && yum groupinstall -y "Development Tools" && yum install -y curl bzip2-devel zlib-devel sqlite-devel openssl-devel readline-devel libffi-devel xz-devel COPY scripts/pyenv-install pyenv-install RUN ./pyenv-install diff --git a/build_and_test_platform.sh b/build_and_test_platform.sh index 697f4d0..6829bf4 100755 --- a/build_and_test_platform.sh +++ b/build_and_test_platform.sh @@ -1,7 +1,7 @@ #!/bin/bash ignore_errors=0 -python_version=3.6.11 +python_version=3.8.14 asset_version=${TRAVIS_TAG:-local-build} asset_filename=sensu-python-runtime_${asset_version}_python-${python_version}_${platform}_linux_amd64.tar.gz asset_image=sensu-python-runtime-${python_version}-${platform}:${asset_version} From 6958bc0348d9fbfc47d7df298ddda4f9dbbfff82 Mon Sep 17 00:00:00 2001 From: Henry Finucane Date: Mon, 6 Feb 2023 22:32:19 +0000 Subject: [PATCH 2/9] Port to Python 3.11 Drop EOL'd distributions Update to a recent Alpine Add dependencies for Debian Do some extra tweaking of test.sh to generalize things --- Dockerfile.alpine | 2 +- Dockerfile.debian8 => Dockerfile.debian10 | 8 ++++---- build_and_test_platform.sh | 8 ++++---- build_platforms.sh | 13 +++---------- tests/test.sh | 2 +- 5 files changed, 13 insertions(+), 20 deletions(-) rename Dockerfile.debian8 => Dockerfile.debian10 (81%) diff --git a/Dockerfile.alpine b/Dockerfile.alpine index 19466cd..28202a8 100644 --- a/Dockerfile.alpine +++ b/Dockerfile.alpine @@ -1,4 +1,4 @@ -FROM alpine:3.8 +FROM alpine:3.16 ARG PYTHON_VERSION=3.8.5 ARG ASSET_VERSION=local_build diff --git a/Dockerfile.debian8 b/Dockerfile.debian10 similarity index 81% rename from Dockerfile.debian8 rename to Dockerfile.debian10 index 34c2911..3c2d80f 100644 --- a/Dockerfile.debian8 +++ b/Dockerfile.debian10 @@ -1,13 +1,13 @@ -FROM debian:8 +FROM debian:10 -LABEL name="sensu/sensu-python-runtime-debian8" +LABEL name="sensu/sensu-python-runtime-debian10" ARG PYTHON_VERSION=3.8.5 ARG ASSET_VERSION=local_build ARG GREP_EXCLUDE='(ld.so|ld-linux-x86-64.so|libBrokenLocale.so|libSegFault.so|libanl.so|libc.so|libdl.so|libm.so|libmvec.so|libnss_compat.so|libnss_dns.so|libnss_files.so|libpthread.so|libresolv.so|librt.so|libthread_db.so|libutil.so|vdso.so)' -RUN apt-get update && apt-get install -y build-essential zlib1g-dev libsqlite0-dev libbz2-dev libreadline-dev libssl-dev curl git +RUN apt-get update && apt-get install -y build-essential zlib1g-dev libsqlite0-dev libbz2-dev libreadline-dev libssl-dev curl git libsqlite3-dev liblzma-dev libffi-dev COPY scripts/pyenv-install pyenv-install RUN ./pyenv-install @@ -15,7 +15,7 @@ RUN LIBS=$(find /build/pyenv -type f -executable -exec ldd {} 2>/dev/null \;| g for f in $LIBS; do if [ -e $f ]; then echo "Copying Library: $f" && cp $f /build/lib/; fi; done RUN mkdir /assets/ && \ - export SENSU_ASSET="/assets/sensu-python-runtime_${ASSET_VERSION}_python-${PYTHON_VERSION}_debian8_$(uname -s | tr '[:upper:]' '[:lower:]')_amd64.tar.gz" && \ + export SENSU_ASSET="/assets/sensu-python-runtime_${ASSET_VERSION}_python-${PYTHON_VERSION}_debian10_$(uname -s | tr '[:upper:]' '[:lower:]')_amd64.tar.gz" && \ tar -czf $SENSU_ASSET -C /build/ . ENV PATH=$PATH:/build/bin diff --git a/build_and_test_platform.sh b/build_and_test_platform.sh index 6829bf4..ca4b836 100755 --- a/build_and_test_platform.sh +++ b/build_and_test_platform.sh @@ -1,7 +1,7 @@ #!/bin/bash ignore_errors=0 -python_version=3.8.14 +python_version=3.11.1 asset_version=${TRAVIS_TAG:-local-build} asset_filename=sensu-python-runtime_${asset_version}_python-${python_version}_${platform}_linux_amd64.tar.gz asset_image=sensu-python-runtime-${python_version}-${platform}:${asset_version} @@ -24,11 +24,12 @@ else echo "Building Docker Image: sensu-python-runtime:${python_version}-${platform}" docker build --build-arg "PYTHON_VERSION=$python_version" --build-arg "ASSET_VERSION=$asset_version" -t ${asset_image} -f Dockerfile.${platform} . echo "Making Asset: /assets/sensu-python-runtime_${asset_version}_python-${python_version}_${platform}_linux_amd64.tar.gz" - docker run -v "$PWD/dist:/dist" ${asset_image} cp /assets/${asset_filename} /dist/ + docker run --rm -v "$PWD/dist:/dist" ${asset_image} cp /assets/${asset_filename} /dist/ # #rm $PWD/test/* # #cp $PWD/dist/${asset_filename} $PWD/dist/${asset_filename} else echo "Image already exists!!!" + docker run --rm -v "$PWD/dist:/dist" ${asset_image} cp /assets/${asset_filename} /dist/ [ $ignore_errors -eq 0 ] && exit 1 fi fi @@ -37,13 +38,12 @@ fi test_arr=($test_platforms) for test_platform in "${test_arr[@]}"; do echo "Test: ${test_platform}" - docker run --name python_runtime_platform_test -e platform -e test_platform=${test_platform} -e asset_filename=${asset_filename} -v "$PWD/tests/:/tests" -v "$PWD/dist:/dist" ${test_platform} /tests/test.sh + docker run --rm --name python_runtime_platform_test -e platform -e test_platform=${test_platform} -e asset_filename=${asset_filename} -v "$PWD/tests/:/tests" -v "$PWD/dist:/dist" ${test_platform} /tests/test.sh retval=$? if [ $retval -ne 0 ]; then echo "!!! Error testing ${asset_filename} on ${test_platform}" exit $retval fi - docker rm python_runtime_platform_test done if [ -z "$TRAVIS_TAG" ]; then exit 0; fi diff --git a/build_platforms.sh b/build_platforms.sh index ad88eaf..4e24237 100755 --- a/build_platforms.sh +++ b/build_platforms.sh @@ -5,28 +5,21 @@ mkdir -p assets mkdir -p scripts # Alpine platform -platform="alpine" test_platforms="alpine:latest alpine:3 alpine:3.8" ./build_and_test_platform.sh +platform="alpine" test_platforms="alpine:latest alpine:3 alpine:3.16" ./build_and_test_platform.sh retval=$? if [[ retval -ne 0 ]]; then exit $retval fi # Debian8 platform -platform="debian8" test_platforms="debian:8 debian:9 debian:10 ubuntu:20.04 ubuntu:16.04 ubuntu:18.04 centos:7 centos:8" ./build_and_test_platform.sh +platform="debian10" test_platforms="debian:10 debian:11 ubuntu:20.04 ubuntu:18.04" ./build_and_test_platform.sh retval=$? if [[ retval -ne 0 ]]; then exit $retval fi # CentOS7 platform -platform="centos7" test_platforms="centos:8 centos:7 debian:8 debian:9 debian:10 ubuntu:16.04 ubuntu:18.04" ./build_and_test_platform.sh -retval=$? -if [[ retval -ne 0 ]]; then - exit $retval -fi - -# CentOS6 platform -platform="centos6" test_platforms="centos:6 centos:8 centos:7 debian:8 debian:9 debian:10 ubuntu:16.04 ubuntu:18.04" ./build_and_test_platform.sh +platform="centos7" test_platforms=" almalinux:8 centos:7 almalinux:9" ./build_and_test_platform.sh retval=$? if [[ retval -ne 0 ]]; then exit $retval diff --git a/tests/test.sh b/tests/test.sh index 5cdc839..cfb3f08 100755 --- a/tests/test.sh +++ b/tests/test.sh @@ -11,6 +11,6 @@ mkdir -p /build cd /build tar xzf /dist/$asset_filename #LD_LIBRARY_PATH="/build/lib:$LD_LIBRARY_PATH" /build/bin/python /tests/test_ssl_url.py -LD_LIBRARY_PATH="/build/lib:$LD_LIBRARY_PATH" [ "$(/build/bin/python --version)" = "Python 3.6.11" ] +[ "$(LD_LIBRARY_PATH="/build/lib:$LD_LIBRARY_PATH" /build/bin/python --version)" = "Python ${PYTHON_VERSION}" ] LD_LIBRARY_PATH="/build/lib:$LD_LIBRARY_PATH" /build/bin/python /tests/uuid_test.py LD_LIBRARY_PATH="/build/lib:$LD_LIBRARY_PATH" /build/bin/python /tests/module_search_path.py From 93d3d7e8cfb43a44a2489cf4722454cdd17d0419 Mon Sep 17 00:00:00 2001 From: Henry Finucane Date: Mon, 6 Feb 2023 22:46:24 +0000 Subject: [PATCH 3/9] Make sure Python builds cleanly on Alpine --- Dockerfile.alpine | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile.alpine b/Dockerfile.alpine index 28202a8..23f3639 100644 --- a/Dockerfile.alpine +++ b/Dockerfile.alpine @@ -6,7 +6,8 @@ LABEL name="sensu/sensu-python-runtime-alpine" RUN apk --no-cache add build-base gmp-dev zlib-dev bzip2-dev sqlite-dev gdbm-dev db-dev readline-dev libffi-dev coreutils yaml-dev linux-headers autoconf \ openssh-client openssl-dev libc6-compat \ - wget git sudo bash bash-doc bash-completion + wget git sudo bash bash-doc bash-completion \ + xz-dev COPY scripts/pyenv-install pyenv-install COPY scripts/binary_shim binary_shim From de9a2474f9bcd0f59ba967e21784036efebde361 Mon Sep 17 00:00:00 2001 From: Henry Finucane Date: Mon, 6 Feb 2023 22:46:37 +0000 Subject: [PATCH 4/9] Debian 10 can't build for Ubuntu 18.04, so drop it --- build_platforms.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_platforms.sh b/build_platforms.sh index 4e24237..827b320 100755 --- a/build_platforms.sh +++ b/build_platforms.sh @@ -12,7 +12,7 @@ if [[ retval -ne 0 ]]; then fi # Debian8 platform -platform="debian10" test_platforms="debian:10 debian:11 ubuntu:20.04 ubuntu:18.04" ./build_and_test_platform.sh +platform="debian10" test_platforms="debian:10 debian:11 ubuntu:20.04" ./build_and_test_platform.sh retval=$? if [[ retval -ne 0 ]]; then exit $retval From 57fdab54f27a441d105d773de331f9159222af28 Mon Sep 17 00:00:00 2001 From: Henry Finucane Date: Mon, 6 Feb 2023 23:54:22 +0000 Subject: [PATCH 5/9] Set errexit to avoid silent failures --- scripts/pyenv-install | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/pyenv-install b/scripts/pyenv-install index 5f7191a..41b0772 100755 --- a/scripts/pyenv-install +++ b/scripts/pyenv-install @@ -1,6 +1,7 @@ #!/bin/sh -git clone https://github.com/pyenv/pyenv.git ~/.pyenv && \ -export PYENV_ROOT="$HOME/.pyenv" && \ +set -e +git clone https://github.com/pyenv/pyenv.git ~/.pyenv +export PYENV_ROOT="$HOME/.pyenv" export PATH="$PYENV_ROOT/bin:$PATH" eval "$(pyenv init -)" pyenv install ${PYTHON_VERSION} @@ -10,4 +11,3 @@ cp -r $PYENV_ROOT build/pyenv cd build/bin ln -s ../pyenv/versions/${PYTHON_VERSION}/bin/python python ln -s ../pyenv/versions/${PYTHON_VERSION}/bin/pip pip - From e1cc7f859ed33a75071ef9f5e40cbd2ca68e211b Mon Sep 17 00:00:00 2001 From: Henry Finucane Date: Mon, 6 Feb 2023 23:55:22 +0000 Subject: [PATCH 6/9] Fall back to 3.9 to support CentOS 7 --- build_and_test_platform.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_and_test_platform.sh b/build_and_test_platform.sh index ca4b836..d7ab2da 100755 --- a/build_and_test_platform.sh +++ b/build_and_test_platform.sh @@ -1,7 +1,7 @@ #!/bin/bash ignore_errors=0 -python_version=3.11.1 +python_version=3.9.16 asset_version=${TRAVIS_TAG:-local-build} asset_filename=sensu-python-runtime_${asset_version}_python-${python_version}_${platform}_linux_amd64.tar.gz asset_image=sensu-python-runtime-${python_version}-${platform}:${asset_version} From ebbc9f195fe8fb03d4034d5d043cf8badbe27fc6 Mon Sep 17 00:00:00 2001 From: Henry Finucane Date: Tue, 7 Feb 2023 00:23:57 +0000 Subject: [PATCH 7/9] Use pyenv binaries directly instead of using the shell magic --- scripts/pyenv-install | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/pyenv-install b/scripts/pyenv-install index 41b0772..79f23dd 100755 --- a/scripts/pyenv-install +++ b/scripts/pyenv-install @@ -3,8 +3,7 @@ set -e git clone https://github.com/pyenv/pyenv.git ~/.pyenv export PYENV_ROOT="$HOME/.pyenv" export PATH="$PYENV_ROOT/bin:$PATH" -eval "$(pyenv init -)" -pyenv install ${PYTHON_VERSION} +~/.pyenv/bin/pyenv install ${PYTHON_VERSION} mkdir -p build/bin mkdir -p build/lib cp -r $PYENV_ROOT build/pyenv From a0892f8552ebfad482062fb5dc31d903b80aba29 Mon Sep 17 00:00:00 2001 From: Henry Finucane Date: Tue, 7 Feb 2023 00:28:29 +0000 Subject: [PATCH 8/9] Clean up platform list --- build_platforms.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_platforms.sh b/build_platforms.sh index 827b320..7b95ab9 100755 --- a/build_platforms.sh +++ b/build_platforms.sh @@ -19,7 +19,7 @@ if [[ retval -ne 0 ]]; then fi # CentOS7 platform -platform="centos7" test_platforms=" almalinux:8 centos:7 almalinux:9" ./build_and_test_platform.sh +platform="centos7" test_platforms="centos:7 almalinux:8 almalinux:9" ./build_and_test_platform.sh retval=$? if [[ retval -ne 0 ]]; then exit $retval From d712b0ddb56d022991997c26ee7479fee071650c Mon Sep 17 00:00:00 2001 From: Henry Finucane Date: Tue, 7 Feb 2023 22:09:57 +0000 Subject: [PATCH 9/9] More tests & make sure failures propagate The extra modules required by newer Python versions made me nervous, so I added some extra test plumbing --- build_and_test_platform.sh | 2 +- tests/lzma_test.py | 2 ++ tests/sqlite_test.py | 4 ++++ tests/ssl_test.py | 14 ++++++++++++++ tests/test.sh | 5 ++++- 5 files changed, 25 insertions(+), 2 deletions(-) create mode 100644 tests/lzma_test.py create mode 100644 tests/sqlite_test.py create mode 100644 tests/ssl_test.py diff --git a/build_and_test_platform.sh b/build_and_test_platform.sh index d7ab2da..e2ac219 100755 --- a/build_and_test_platform.sh +++ b/build_and_test_platform.sh @@ -38,7 +38,7 @@ fi test_arr=($test_platforms) for test_platform in "${test_arr[@]}"; do echo "Test: ${test_platform}" - docker run --rm --name python_runtime_platform_test -e platform -e test_platform=${test_platform} -e asset_filename=${asset_filename} -v "$PWD/tests/:/tests" -v "$PWD/dist:/dist" ${test_platform} /tests/test.sh + docker run --rm --name python_runtime_platform_test -e "PYTHON_VERSION=$python_version" -e platform -e test_platform=${test_platform} -e asset_filename=${asset_filename} -v "$PWD/tests/:/tests" -v "$PWD/dist:/dist" ${test_platform} /tests/test.sh retval=$? if [ $retval -ne 0 ]; then echo "!!! Error testing ${asset_filename} on ${test_platform}" diff --git a/tests/lzma_test.py b/tests/lzma_test.py new file mode 100644 index 0000000..4c00b8c --- /dev/null +++ b/tests/lzma_test.py @@ -0,0 +1,2 @@ +import lzma +print(lzma.compress(b"Hello World")) diff --git a/tests/sqlite_test.py b/tests/sqlite_test.py new file mode 100644 index 0000000..797a65f --- /dev/null +++ b/tests/sqlite_test.py @@ -0,0 +1,4 @@ +import sqlite3 +con = sqlite3.connect(":memory:") +cur = con.cursor() +cur.execute("CREATE TABLE movie(title, year, score)") diff --git a/tests/ssl_test.py b/tests/ssl_test.py new file mode 100644 index 0000000..949ecad --- /dev/null +++ b/tests/ssl_test.py @@ -0,0 +1,14 @@ +import socket +import ssl + +context = ssl.create_default_context() +context.check_hostname = False # Debian and Ubuntu containers ship with no certs +context.verify_mode = ssl.CERT_NONE + +hostname = 'www.google.com' + +print(ssl.get_default_verify_paths()) + +with socket.create_connection((hostname, 443)) as sock: + with context.wrap_socket(sock, server_hostname=hostname) as ssock: + print(ssock.version()) diff --git a/tests/test.sh b/tests/test.sh index cfb3f08..4c728d6 100755 --- a/tests/test.sh +++ b/tests/test.sh @@ -1,4 +1,5 @@ #!/bin/sh +set -e echo "Test Script:" echo " Asset Platform: ${platform}" echo " Target Platform: ${test_platform}" @@ -10,7 +11,9 @@ fi mkdir -p /build cd /build tar xzf /dist/$asset_filename -#LD_LIBRARY_PATH="/build/lib:$LD_LIBRARY_PATH" /build/bin/python /tests/test_ssl_url.py [ "$(LD_LIBRARY_PATH="/build/lib:$LD_LIBRARY_PATH" /build/bin/python --version)" = "Python ${PYTHON_VERSION}" ] LD_LIBRARY_PATH="/build/lib:$LD_LIBRARY_PATH" /build/bin/python /tests/uuid_test.py +LD_LIBRARY_PATH="/build/lib:$LD_LIBRARY_PATH" /build/bin/python /tests/ssl_test.py +LD_LIBRARY_PATH="/build/lib:$LD_LIBRARY_PATH" /build/bin/python /tests/lzma_test.py +LD_LIBRARY_PATH="/build/lib:$LD_LIBRARY_PATH" /build/bin/python /tests/sqlite_test.py LD_LIBRARY_PATH="/build/lib:$LD_LIBRARY_PATH" /build/bin/python /tests/module_search_path.py