Skip to content

Commit ab92daf

Browse files
TimPansinoTimPansino
authored
Containerized CI Pipeline (#836)
* Revert "Remove Python 2.7 and pypy2 testing (#835)" This reverts commit abb6405. * Containerize CI process * Publish new docker container for CI images * Rename github actions job * Copyright tag scripts * Drop debug line * Swap to new CI image * Move pip install to just main python * Remove libcurl special case from tox * Install special case packages into main image * Remove unused packages * Remove all other triggers besides manual * Add make run command * Cleanup small bugs
1 parent abb6405 commit ab92daf

File tree

11 files changed

+449
-230
lines changed

11 files changed

+449
-230
lines changed

‎.github/actions/setup-python-matrix/action.yml‎

Lines changed: 0 additions & 50 deletions
This file was deleted.

‎.github/containers/Dockerfile‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
2+
# Copyright 2010 New Relic, Inc.
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
FROM ubuntu:20.04
17+
18+
# Install OS packages
19+
RUN export DEBIAN_FRONTEND=noninteractive && \
20+
apt-get update && \
21+
apt-get install -y \
22+
bash \
23+
build-essential \
24+
curl \
25+
expat \
26+
gcc \
27+
git \
28+
libbz2-dev \
29+
libcurl4-openssl-dev \
30+
libffi-dev \
31+
libgmp-dev \
32+
liblzma-dev \
33+
libmpfr-dev \
34+
libncurses-dev \
35+
libpq-dev \
36+
libreadline-dev \
37+
libsqlite3-dev \
38+
libssl-dev \
39+
locales \
40+
make \
41+
openssl \
42+
python2-dev \
43+
python3-dev \
44+
python3-pip \
45+
odbc-postgresql \
46+
unzip \
47+
wget \
48+
zip \
49+
zlib1g \
50+
zlib1g-dev && \
51+
rm -rf /var/lib/apt/lists/*
52+
53+
# Setup ODBC config
54+
RUN sed -i 's/Driver=psqlodbca.so/Driver=\/usr\/lib\/x86_64-linux-gnu\/odbc\/psqlodbca.so/g' /etc/odbcinst.ini && \
55+
sed -i 's/Driver=psqlodbcw.so/Driver=\/usr\/lib\/x86_64-linux-gnu\/odbc\/psqlodbcw.so/g' /etc/odbcinst.ini && \
56+
sed -i 's/Setup=libodbcpsqlS.so/Setup=\/usr\/lib\/x86_64-linux-gnu\/odbc\/libodbcpsqlS.so/g' /etc/odbcinst.ini
57+
58+
# Set the locale
59+
RUN locale-gen --no-purge en_US.UTF-8
60+
ENV LANG=en_US.UTF-8 \ LANGUAGE=en_US:en \ LC_ALL=en_US.UTF-8
61+
62+
# Set user to non-root
63+
ENV HOME /home/github
64+
RUN groupadd -g 1000 github && \
65+
useradd -m -d "${HOME}" -s /bin/bash -u 1000 -g 1000 github
66+
USER 1000:1000
67+
WORKDIR "${HOME}"
68+
69+
# Install pyenv
70+
ENV PYENV_ROOT="${HOME}/.pyenv"
71+
RUN curl https://pyenv.run/ | /bin/bash
72+
ENV PATH="$PYENV_ROOT/bin:$PYENV_ROOT/shims:${PATH}"
73+
RUN echo 'eval "$(pyenv init -)"' >>$HOME/.bashrc && \
74+
pyenv update
75+
76+
# Install Python
77+
ARG PYTHON_VERSIONS="3.10 3.9 3.8 3.7 3.11 2.7 pypy2.7 pypy3.7"
78+
COPY --chown=1000:1000 --chmod=+x ./install-python.sh /tmp/install-python.sh
79+
COPY ./requirements.txt /requirements.txt
80+
RUN /tmp/install-python.sh && \
81+
rm /tmp/install-python.sh

‎.github/containers/Makefile‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
# Copyright 2010 New Relic, Inc.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
# Repository root for mounting into container.
16+
REPO_ROOT:=$(realpath $(dir $(realpath $(firstword $(MAKEFILE_LIST))))../../)
17+
18+
.PHONY: default
19+
default: test
20+
21+
.PHONY: build
22+
build:
23+
@# Perform a shortened build for testing
24+
@docker build --build-arg='PYTHON_VERSIONS=3.10 2.7' . -t ghcr.io/newrelic/python-agent-ci:local
25+
26+
.PHONY: test
27+
test: build
28+
@# Ensure python versions are usable
29+
@docker run --rm ghcr.io/newrelic/python-agent-ci:local /bin/bash -c '\
30+
python3.10 --version && \
31+
python2.7 --version && \
32+
touch tox.ini && tox --version && \
33+
echo "Success! Python versions installed."'
34+
35+
.PHONY: run
36+
run: build
37+
@docker run --rm -it \
38+
--mount type=bind,source="$(REPO_ROOT)",target=/home/github/python-agent \
39+
--workdir=/home/github/python-agent \
40+
-e NEW_RELIC_HOST="${NEW_RELIC_HOST}" \
41+
-e NEW_RELIC_LICENSE_KEY="${NEW_RELIC_LICENSE_KEY}" \
42+
-e NEW_RELIC_DEVELOPER_MODE="${NEW_RELIC_DEVELOPER_MODE}" \
43+
ghcr.io/newrelic/python-agent-ci:local /bin/bash

‎.github/containers/install-python.sh‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
#!/bin/bash
2+
# Copyright 2010 New Relic, Inc.
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
set -e
17+
18+
SED=$(which gsed || which sed)
19+
20+
SCRIPT_DIR=$(dirname "$0")
21+
PIP_REQUIREMENTS=$(cat /requirements.txt)
22+
23+
main() {
24+
# Coerce space separated string to array
25+
if [[ ${#PYTHON_VERSIONS[@]} -eq 1 ]]; then
26+
PYTHON_VERSIONS=($PYTHON_VERSIONS)
27+
fi
28+
29+
if [[ -z "${PYTHON_VERSIONS[@]}" ]]; then
30+
echo "No python versions specified. Make sure PYTHON_VERSIONS is set." 1>&2
31+
exit 1
32+
fi
33+
34+
# Find all latest pyenv supported versions for requested python versions
35+
PYENV_VERSIONS=()
36+
for v in "${PYTHON_VERSIONS[@]}"; do
37+
LATEST=$(pyenv latest -k "$v" || get_latest_patch_version "$v")
38+
if [[ -z "$LATEST" ]]; then
39+
echo "Latest version could not be found for ${v}." 1>&2
40+
exit 1
41+
fi
42+
PYENV_VERSIONS+=($LATEST)
43+
done
44+
45+
# Install each specific version
46+
for v in "${PYENV_VERSIONS[@]}"; do
47+
pyenv install "$v" &
48+
done
49+
wait
50+
51+
# Set all installed versions as globally accessible
52+
pyenv global ${PYENV_VERSIONS[@]}
53+
54+
# Install dependencies for main python installation
55+
pyenv exec pip install --upgrade $PIP_REQUIREMENTS
56+
}
57+
58+
get_latest_patch_version() {
59+
pyenv install --list | # Get all python versions
60+
$SED 's/^ *//g' | # Remove leading whitespace
61+
grep -E "^$1" | # Find specified version by matching start of line
62+
grep -v -- "-c-jit-latest" | # Filter out pypy JIT versions
63+
$SED -E '/(-[a-zA-Z]+$)|(a[0-9]+)|(b[0-9]+)|(rc[0-9]+)/!{s/$/_/}' | # Append trailing _ to any non development versions to place them lower when sorted
64+
sort -V | # Sort using version sorting
65+
$SED 's/_$//' | # Remove any added trailing underscores to correct version names
66+
tail -1 # Grab last result as latest version
67+
}
68+
69+
main

‎.github/containers/requirements.txt‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pip
2+
setuptools
3+
wheel
4+
virtualenv<20.22.1
5+
tox

‎.github/scripts/retry.sh‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,18 @@
11
#!/bin/bash
2+
# Copyright 2010 New Relic, Inc.
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
216

317
# Time in seconds to backoff after the initial attempt.
418
INITIAL_BACKOFF=10

‎.github/workflows/build-ci-image.yml‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
# Copyright 2010 New Relic, Inc.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
name: Build CI Image
16+
17+
on:
18+
workflow_dispatch: # Allow manual trigger
19+
20+
concurrency:
21+
group: ${{ github.ref || github.run_id }}
22+
cancel-in-progress: true
23+
24+
jobs:
25+
build:
26+
runs-on: ubuntu-latest
27+
28+
steps:
29+
- uses: actions/checkout@v3
30+
with:
31+
persist-credentials: false
32+
fetch-depth: 0
33+
34+
- name: Set up Docker Buildx
35+
id: buildx
36+
uses: docker/setup-buildx-action@v2
37+
38+
- name: Generate Docker Metadata (Tags and Labels)
39+
id: meta
40+
uses: docker/metadata-action@v4
41+
with:
42+
images: ghcr.io/${{ github.repository }}
43+
flavor: |
44+
prefix=
45+
suffix=
46+
latest=false
47+
tags: |
48+
type=raw,value=latest,enable={{is_default_branch}}
49+
type=schedule,pattern={{date 'YYYY-MM-DD'}}
50+
type=sha,format=short,prefix=sha-
51+
type=sha,format=long,prefix=sha-
52+
53+
- name: Login to GitHub Container Registry
54+
if: github.event_name != 'pull_request'
55+
uses: docker/login-action@v2
56+
with:
57+
registry: ghcr.io
58+
username: ${{ github.repository_owner }}
59+
password: ${{ secrets.GITHUB_TOKEN }}
60+
61+
- name: Build and Publish Image
62+
uses: docker/build-push-action@v3
63+
with:
64+
push: ${{ github.event_name != 'pull_request' }}
65+
context: .github/containers
66+
platforms: linux/amd64
67+
tags: ${{ steps.meta.outputs.tags }}
68+
labels: ${{ steps.meta.outputs.labels }}

‎.github/workflows/get-envs.py‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,18 @@
11
#!/usr/bin/env python3.8
2+
# Copyright 2010 New Relic, Inc.
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
216
import fileinput
317
import os
418

0 commit comments

Comments
 (0)