Skip to content

Commit 6905b28

Browse files
authored
Merge pull request #685 from gerrod3/3.12-update-ci
Update CI files
2 parents 8695c9b + a3c8595 commit 6905b28

File tree

8 files changed

+227
-97
lines changed

8 files changed

+227
-97
lines changed

.ci/ansible/Containerfile.j2

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ RUN pip3 install
2222
{{ " " }}-r ./{{ item.name }}/ci_requirements.txt
2323
{%- endif -%}
2424
{%- endfor %}
25+
{{ " " }}-c ./{{ plugins[0].name }}/.ci/assets/ci_constraints.txt
2526

2627
{% if pulp_env is defined and pulp_env %}
2728
{% for key, value in pulp_env.items() %}

.ci/assets/ci_constraints.txt

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
# Pulpcore versions without the openapi command do no longer work in the CI
2+
pulpcore>=3.21.30,!=3.23.*,!=3.24.*,!=3.25.*,!=3.26.*,!=3.27.*,!=3.29.*,!=3.30.*,!=3.31.*,!=3.32.*,!=3.33.*,!=3.34.*,!=3.35.*,!=3.36.*,!=3.37.*,!=3.38.*,!=3.40.*,!=3.41.*,!=3.42.*,!=3.43.*,!=3.44.*,!=3.45.*,!=3.46.*,!=3.47.*,!=3.48.*,!=3.50.*,!=3.51.*,!=3.52.*,!=3.53.*,!=3.54.*
3+
4+
5+
6+
tablib!=3.6.0
7+
# 3.6.0: This release introduced a regression removing the "html" optional dependency.

.github/template_gitref

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
2021.08.26-339-gf0d923e
1+
2021.08.26-345-g747bcf8

.github/workflows/build.yml

Lines changed: 69 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,18 +23,30 @@ jobs:
2323
with:
2424
fetch-depth: 1
2525
path: "pulp_python"
26+
- uses: "actions/checkout@v4"
27+
with:
28+
fetch-depth: 1
29+
repository: "pulp/pulp-openapi-generator"
30+
path: "pulp-openapi-generator"
2631
- uses: "actions/setup-python@v5"
2732
with:
2833
python-version: "3.11"
2934
- name: "Install python dependencies"
3035
run: |
3136
echo ::group::PYDEPS
32-
pip install packaging twine wheel
37+
pip install packaging twine wheel mkdocs jq
3338
echo ::endgroup::
3439
- name: "Build package"
3540
run: |
3641
python3 setup.py sdist bdist_wheel --python-tag py3
3742
twine check dist/*
43+
- name: "Install built packages"
44+
run: |
45+
pip install dist/pulp_python-*-py3-none-any.whl -c .ci/assets/ci_constraints.txt
46+
- name: "Generate api specs"
47+
run: |
48+
pulpcore-manager openapi --file "api.json"
49+
pulpcore-manager openapi --bindings --component "python" --file "python-api.json"
3850
- name: "Upload Package whl"
3951
uses: "actions/upload-artifact@v4"
4052
with:
@@ -43,3 +55,59 @@ jobs:
4355
if-no-files-found: "error"
4456
retention-days: 5
4557
overwrite: true
58+
- name: "Upload API specs"
59+
uses: "actions/upload-artifact@v4"
60+
with:
61+
name: "api_spec"
62+
path: |
63+
pulp_python/api.json
64+
pulp_python/python-api.json
65+
if-no-files-found: "error"
66+
retention-days: 5
67+
overwrite: true
68+
- name: "Build Python bindings packages"
69+
run: |
70+
.github/workflows/scripts/build_python_client.sh
71+
shell: "bash"
72+
env:
73+
PY_COLORS: "1"
74+
ANSIBLE_FORCE_COLOR: "1"
75+
GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
76+
GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}"
77+
- name: "Upload python client packages"
78+
uses: "actions/upload-artifact@v4"
79+
with:
80+
name: "python-client.tar"
81+
path: |
82+
pulp_python/python-python-client.tar
83+
if-no-files-found: "error"
84+
retention-days: 5
85+
overwrite: true
86+
- name: "Upload python client docs"
87+
uses: "actions/upload-artifact@v4"
88+
with:
89+
name: "python-client-docs.tar"
90+
path: |
91+
pulp_python/python-python-client-docs.tar
92+
if-no-files-found: "error"
93+
retention-days: 5
94+
overwrite: true
95+
- name: "Build Ruby bindings packages"
96+
run: |
97+
.github/workflows/scripts/build_ruby_client.sh
98+
shell: "bash"
99+
env:
100+
PY_COLORS: "1"
101+
ANSIBLE_FORCE_COLOR: "1"
102+
GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
103+
GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}"
104+
- name: "Upload Ruby client"
105+
uses: "actions/upload-artifact@v4"
106+
with:
107+
name: "ruby-client.tar"
108+
path: |
109+
pulp_python/python-ruby-client.tar
110+
if-no-files-found: "error"
111+
retention-days: 5
112+
overwrite: true
113+
...
Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
#!/bin/bash
2+
3+
# This script expects all <app_label>-api.json files to exist in the plugins root directory.
4+
# It produces a <app_label>-python-client.tar and <app_label>-python-client-docs.tar file in the plugins root directory.
5+
6+
# WARNING: DO NOT EDIT!
7+
#
8+
# This file was generated by plugin_template, and is managed by it. Please use
9+
# './plugin-template --github pulp_python' to update this file.
10+
#
11+
# For more info visit https://github.com/pulp/plugin_template
12+
13+
set -mveuo pipefail
14+
15+
# make sure this script runs at the repo root
16+
cd "$(dirname "$(realpath -e "$0")")"/../../..
17+
18+
pushd ../pulp-openapi-generator
19+
rm -rf "pulp_python-client"
20+
21+
# We need to copy that over to be visible in the container.
22+
cp "../pulp_python/python-api.json" .
23+
./gen-client.sh "python-api.json" "python" python "pulp_python"
24+
25+
pushd pulp_python-client
26+
python setup.py sdist bdist_wheel --python-tag py3
27+
28+
twine check "dist/pulp_python_client-"*"-py3-none-any.whl"
29+
twine check "dist/pulp_python-client-"*".tar.gz"
30+
31+
tar cvf "../../pulp_python/python-python-client.tar" ./dist
32+
33+
find ./docs/* -exec sed -i 's/Back to README/Back to HOME/g' {} \;
34+
find ./docs/* -exec sed -i 's/README//g' {} \;
35+
cp README.md docs/index.md
36+
sed -i 's/docs\///g' docs/index.md
37+
find ./docs/* -exec sed -i 's/\.md//g' {} \;
38+
39+
cat >> mkdocs.yml << DOCSYAML
40+
---
41+
site_name: PulpPython Client
42+
site_description: Python bindings
43+
site_author: Pulp Team
44+
site_url: https://docs.pulpproject.org/pulp_python_client/
45+
repo_name: pulp/pulp_python
46+
repo_url: https://github.com/pulp/pulp_python
47+
theme: readthedocs
48+
DOCSYAML
49+
50+
# Building the bindings docs
51+
mkdocs build
52+
53+
# Pack the built site.
54+
tar cvf ../../pulp_python/python-python-client-docs.tar ./site
55+
popd
56+
popd
Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
#!/bin/bash
2+
3+
# This script expects all <app_label>-api.json files to exist in the plugins root directory.
4+
# It produces a <app_label>-ruby-client.tar file in the plugins root directory.
5+
6+
# WARNING: DO NOT EDIT!
7+
#
8+
# This file was generated by plugin_template, and is managed by it. Please use
9+
# './plugin-template --github pulp_python' to update this file.
10+
#
11+
# For more info visit https://github.com/pulp/plugin_template
12+
13+
set -mveuo pipefail
14+
15+
# make sure this script runs at the repo root
16+
cd "$(dirname "$(realpath -e "$0")")"/../../..
17+
18+
pushd ../pulp-openapi-generator
19+
rm -rf "pulp_python-client"
20+
21+
# We need to copy that over to be visible in the container.
22+
#cp "../pulp_python/python-api.json" .
23+
#./gen-client.sh "python-api.json" "python" ruby "pulp_python"
24+
25+
# -------------
26+
# The generator still needs to have it called api.json at this time...
27+
cp "../pulp_python/api.json" .
28+
./gen-client.sh "api.json" "python" ruby "pulp_python"
29+
# -------------
30+
31+
pushd pulp_python-client
32+
gem build pulp_python_client
33+
tar cvf "../../pulp_python/python-ruby-client.tar" "./pulp_python_client-"*".gem"
34+
popd
35+
popd

.github/workflows/scripts/script.sh

Lines changed: 44 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -54,75 +54,62 @@ password password
5454
# Some commands like ansible-galaxy specifically require 600
5555
cmd_prefix bash -c "chmod 600 ~pulp/.netrc"
5656

57-
# Generate and install binding
57+
# Generate bindings
58+
###################
59+
60+
touch bindings_requirements.txt
5861
pushd ../pulp-openapi-generator
59-
if pulp debug has-plugin --name "core" --specifier ">=3.44.0.dev"
60-
then
6162
# Use app_label to generate api.json and package to produce the proper package name.
6263

63-
if [ "$(jq -r '.domain_enabled' <<<"$REPORTED_STATUS")" = "true" ]
64+
# Workaround: Domains are not supported by the published bindings.
65+
# Sadly: Different pulpcore-versions aren't either...
66+
# So we exclude the prebuilt ones only for domains disabled.
67+
if [ "$(jq -r '.domain_enabled' <<<"${REPORTED_STATUS}")" = "true" ] || [ "$(jq -r '.online_workers[0].pulp_href|startswith("/pulp/api/v3/")' <<< "${REPORTED_STATUS}")" = "false" ]
6468
then
65-
# Workaround: Domains are not supported by the published bindings.
66-
# Generate new bindings for all packages.
67-
for item in $(jq -r '.versions[] | tojson' <<<"$REPORTED_STATUS")
68-
do
69-
echo $item
70-
COMPONENT="$(jq -r '.component' <<<"$item")"
71-
VERSION="$(jq -r '.version' <<<"$item")"
72-
MODULE="$(jq -r '.module' <<<"$item")"
73-
PACKAGE="${MODULE%%.*}"
74-
curl --fail-with-body -k -o api.json "${PULP_URL}${PULP_API_ROOT}api/v3/docs/api.json?bindings&component=$COMPONENT"
75-
USE_LOCAL_API_JSON=1 ./generate.sh "${PACKAGE}" python "${VERSION}"
76-
cmd_prefix pip3 install "/root/pulp-openapi-generator/${PACKAGE}-client"
77-
sudo rm -rf "./${PACKAGE}-client"
78-
done
69+
BUILT_CLIENTS=""
7970
else
80-
# Sadly: Different pulpcore-versions aren't either...
81-
for item in $(jq -r '.versions[]| select(.component!="python")| tojson' <<<"$REPORTED_STATUS")
82-
do
83-
echo $item
84-
COMPONENT="$(jq -r '.component' <<<"$item")"
85-
VERSION="$(jq -r '.version' <<<"$item")"
86-
MODULE="$(jq -r '.module' <<<"$item")"
87-
PACKAGE="${MODULE%%.*}"
88-
curl --fail-with-body -k -o api.json "${PULP_URL}${PULP_API_ROOT}api/v3/docs/api.json?bindings&component=$COMPONENT"
89-
USE_LOCAL_API_JSON=1 ./generate.sh "${PACKAGE}" python "${VERSION}"
90-
cmd_prefix pip3 install "/root/pulp-openapi-generator/${PACKAGE}-client"
91-
sudo rm -rf "./${PACKAGE}-client"
92-
done
71+
BUILT_CLIENTS=" python "
9372
fi
94-
else
95-
# Infer the client name from the package name by replacing "-" with "_".
96-
# Use the component to infer the package name on older versions of pulpcore.
9773

98-
if [ "$(echo "$REPORTED_STATUS" | jq -r '.domain_enabled')" = "true" ]
99-
then
100-
# Workaround: Domains are not supported by the published bindings.
101-
# Generate new bindings for all packages.
102-
for item in $(echo "$REPORTED_STATUS" | jq -r '.versions[]|(.package // ("pulp_" + .component)|sub("pulp_core"; "pulpcore"))|sub("-"; "_")')
103-
do
104-
./generate.sh "${item}" python
105-
cmd_prefix pip3 install "/root/pulp-openapi-generator/${item}-client"
106-
sudo rm -rf "./${item}-client"
107-
done
108-
else
109-
# Sadly: Different pulpcore-versions aren't either...
110-
for item in $(echo "$REPORTED_STATUS" | jq -r '.versions[]|select(.component!="python")|(.package // ("pulp_" + .component)|sub("pulp_core"; "pulpcore"))|sub("-"; "_")')
111-
do
112-
./generate.sh "${item}" python
113-
cmd_prefix pip3 install "/root/pulp-openapi-generator/${item}-client"
114-
sudo rm -rf "./${item}-client"
115-
done
116-
fi
117-
fi
74+
for ITEM in $(jq -r '.versions[] | tojson' <<<"${REPORTED_STATUS}")
75+
do
76+
COMPONENT="$(jq -r '.component' <<<"${ITEM}")"
77+
VERSION="$(jq -r '.version' <<<"${ITEM}" | python3 -c "from packaging.version import Version; print(Version(input()))")"
78+
# On older status endpoints, the module was not provided, but the package should be accurate
79+
# there, because we did not merge plugins into pulpcore back then.
80+
MODULE="$(jq -r '.module // (.package|gsub("-"; "_"))' <<<"${ITEM}")"
81+
PACKAGE="${MODULE%%.*}"
82+
if [[ ! " ${BUILT_CLIENTS} " =~ "${COMPONENT}" ]]
83+
then
84+
rm -rf "./${PACKAGE}-client"
85+
cmd_prefix pulpcore-manager openapi --bindings --component "${COMPONENT}" > "${COMPONENT}-api.json"
86+
./gen-client.sh "${COMPONENT}-api.json" "${COMPONENT}" python "${PACKAGE}"
87+
pushd "${PACKAGE}-client"
88+
python setup.py sdist bdist_wheel --python-tag py3
89+
popd
90+
else
91+
if [ ! -f "${PACKAGE}-client/dist/${PACKAGE}_client-${VERSION}-py3-none-any.whl" ]
92+
then
93+
ls -lR "${PACKAGE}-client/"
94+
echo "Error: Client bindings for ${COMPONENT} not found."
95+
echo "File ${PACKAGE}-client/dist/${PACKAGE}_client-${VERSION}-py3-none-any.whl missing."
96+
exit 1
97+
fi
98+
fi
99+
echo "/root/pulp-openapi-generator/${PACKAGE}-client/dist/${PACKAGE}_client-${VERSION}-py3-none-any.whl" >> "../pulp_python/bindings_requirements.txt"
100+
done
118101
popd
119102

120-
# At this point, this is a safeguard only, so let's not make too much fuzz about the old status format.
121-
echo "$REPORTED_STATUS" | jq -r '.versions[]|select(.package)|(.package|sub("_"; "-")) + "-client==" + .version' > bindings_requirements.txt
103+
# Install test requirements
104+
###########################
105+
106+
# Add a safeguard to make sure the proper versions of the clients are installed.
107+
echo "$REPORTED_STATUS" | jq -r '.versions[]|select(.package)|(.package|sub("_"; "-")) + "-client==" + .version' > bindings_constraints.txt
122108
cmd_stdin_prefix bash -c "cat > /tmp/unittest_requirements.txt" < unittest_requirements.txt
123109
cmd_stdin_prefix bash -c "cat > /tmp/functest_requirements.txt" < functest_requirements.txt
124110
cmd_stdin_prefix bash -c "cat > /tmp/bindings_requirements.txt" < bindings_requirements.txt
125-
cmd_prefix pip3 install -r /tmp/unittest_requirements.txt -r /tmp/functest_requirements.txt -r /tmp/bindings_requirements.txt
111+
cmd_stdin_prefix bash -c "cat > /tmp/bindings_constraints.txt" < bindings_constraints.txt
112+
cmd_prefix pip3 install -r /tmp/unittest_requirements.txt -r /tmp/functest_requirements.txt -r /tmp/bindings_requirements.txt -c /tmp/bindings_constraints.txt
126113

127114
CERTIFI=$(cmd_prefix python3 -c 'import certifi; print(certifi.where())')
128115
cmd_prefix bash -c "cat /etc/pulp/certs/pulp_webserver.crt >> '$CERTIFI'"

0 commit comments

Comments
 (0)