Skip to content

Commit bc3e75f

Browse files
markgoddardmarkgoddard
authored
Merge branch 'stackhpc/yoga' into yoga-upstream-merge
2 parents e989f4f + 6b347d2 commit bc3e75f

File tree

11 files changed

+228
-35
lines changed

11 files changed

+228
-35
lines changed

‎.github/workflows/stackhpc-container-image-build.yml‎

Lines changed: 105 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,12 @@ on:
3838
type: boolean
3939
required: false
4040
default: true
41+
push-dirty:
42+
description: Push scanned images that have vulnerabilities?
43+
type: boolean
44+
required: false
45+
# NOTE(Alex-Welsh): This default should be flipped once we resolve existing failures
46+
default: true
4147

4248
env:
4349
ANSIBLE_FORCE_COLOR: True
@@ -109,7 +115,15 @@ jobs:
109115
- name: Install package dependencies
110116
run: |
111117
sudo apt update
112-
sudo apt install -y build-essential git unzip nodejs python3-wheel python3-pip python3-venv
118+
sudo apt install -y build-essential git unzip nodejs python3-wheel python3-pip python3-venv curl jq wget
119+
120+
- name: Install gh
121+
run: |
122+
sudo mkdir -p -m 755 /etc/apt/keyrings && wget -qO- https://cli.github.com/packages/githubcli-archive-keyring.gpg | sudo tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null
123+
sudo chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg
124+
echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null
125+
sudo apt update
126+
sudo apt install gh -y
113127
114128
- name: Checkout
115129
uses: actions/checkout@v4
@@ -127,6 +141,10 @@ jobs:
127141
run: |
128142
docker ps
129143
144+
- name: Install Trivy
145+
run: |
146+
curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sudo sh -s -- -b /usr/local/bin v0.49.0
147+
130148
- name: Install Kayobe
131149
run: |
132150
mkdir -p venvs &&
@@ -162,65 +180,124 @@ jobs:
162180
env:
163181
KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
164182

165-
- name: Build and push kolla overcloud images
183+
- name: Create build logs output directory
184+
run: mkdir image-build-logs
185+
186+
- name: Build kolla overcloud images
187+
id: build_overcloud_images
188+
continue-on-error: true
166189
run: |
167-
args="${{ github.event.inputs.regexes }}"
190+
args="${{ inputs.regexes }}"
168191
args="$args -e kolla_base_distro=${{ matrix.distro }}"
169192
args="$args -e kolla_tag=${{ needs.generate-tag.outputs.kolla_tag }}"
170193
args="$args -e stackhpc_repo_mirror_auth_proxy_enabled=true"
171-
if ${{ inputs.push }} == 'true'; then
172-
args="$args --push"
173-
fi
174194
source venvs/kayobe/bin/activate &&
175195
source src/kayobe-config/kayobe-env --environment ci-builder &&
176196
kayobe overcloud container image build $args
177197
env:
178198
KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
179-
if: github.event.inputs.overcloud == 'true'
199+
if: inputs.overcloud
200+
201+
- name: Copy overcloud container image build logs to output directory
202+
run: sudo mv /var/log/kolla-build.log image-build-logs/kolla-build-overcloud.log
203+
if: inputs.overcloud
180204

181-
- name: Build and push kolla seed images
205+
- name: Build kolla seed images
206+
id: build_seed_images
207+
continue-on-error: true
182208
run: |
183209
args="-e kolla_base_distro=${{ matrix.distro }}"
184210
args="$args -e kolla_tag=${{ needs.generate-tag.outputs.kolla_tag }}"
185211
args="$args -e stackhpc_repo_mirror_auth_proxy_enabled=true"
186-
if ${{ inputs.push }} == 'true'; then
187-
args="$args --push"
188-
fi
189212
source venvs/kayobe/bin/activate &&
190213
source src/kayobe-config/kayobe-env --environment ci-builder &&
191214
kayobe seed container image build $args
192215
env:
193216
KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
194-
if: github.event.inputs.seed == 'true'
217+
if: inputs.seed
218+
219+
- name: Copy seed container image build logs to output directory
220+
run: sudo mv /var/log/kolla-build.log image-build-logs/kolla-build-seed.log
221+
if: inputs.seed
195222

196223
- name: Get built container images
197-
run: |
198-
docker image ls --filter "reference=ark.stackhpc.com/stackhpc-dev/${{ matrix.distro }}-*:${{ needs.generate-tag.outputs.kolla_tag }}" > ${{ matrix.distro }}-container-images
224+
run: docker image ls --filter "reference=ark.stackhpc.com/stackhpc-dev/${{ matrix.distro }}-*:${{ needs.generate-tag.outputs.kolla_tag }}" > ${{ matrix.distro }}-container-images
199225

200226
- name: Fail if no images have been built
201227
run: if [ $(wc -l < ${{ matrix.distro }}-container-images) -le 1 ]; then exit 1; fi
202228

203-
- name: Upload container images artifact
229+
- name: Scan built container images
230+
run: src/kayobe-config/tools/scan-images.sh ${{ matrix.distro }} ${{ needs.generate-tag.outputs.kolla_tag }}
231+
232+
- name: Move image scan logs to output artifact
233+
run: mv image-scan-output image-build-logs/image-scan-output
234+
235+
- name: Fail if no images have passed scanning
236+
run: if [ $(wc -l < image-build-logs/image-scan-output/clean-images.txt) -le 0 ]; then exit 1; fi
237+
if: ${{ !inputs.push-dirty }}
238+
239+
- name: Copy clean images to push-attempt-images list
240+
run: cp image-build-logs/image-scan-output/clean-images.txt image-build-logs/push-attempt-images.txt
241+
if: inputs.push
242+
243+
- name: Append dirty images to push list
244+
run: |
245+
cat image-build-logs/image-scan-output/dirty-images.txt >> image-build-logs/push-attempt-images.txt
246+
if: ${{ inputs.push && inputs.push-dirty }}
247+
248+
- name: Push images
249+
run: |
250+
touch image-build-logs/push-failed-images.txt
251+
source venvs/kayobe/bin/activate &&
252+
source src/kayobe-config/kayobe-env --environment ci-builder &&
253+
kayobe playbook run ${KAYOBE_CONFIG_PATH}/ansible/docker-registry-login.yml &&
254+
255+
while read -r image; do
256+
# Retries!
257+
for i in {1..5}; do
258+
if docker push $image; then
259+
echo "Pushed $image"
260+
break
261+
elif $i == 5; then
262+
echo "Failed to push $image"
263+
echo $image >> image-build-logs/push-failed-images.txt
264+
else
265+
echo "Failed on retry $i"
266+
sleep 5
267+
fi;
268+
done
269+
done < image-build-logs/push-attempt-images.txt
270+
shell: bash
271+
env:
272+
KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
273+
if: inputs.push
274+
275+
- name: Upload output artifact
204276
uses: actions/upload-artifact@v4
205277
with:
206-
name: ${{ matrix.distro }} container images
207-
path: ${{ matrix.distro }}-container-images
278+
name: ${{ matrix.distro }}-logs
279+
path: image-build-logs
208280
retention-days: 7
281+
if: ${{ !cancelled() }}
282+
283+
- name: Fail when images failed to build
284+
run: echo "An image build failed. Check the workflow artifact for build logs" && exit 1
285+
if: ${{ steps.build_overcloud_images.outcome == 'failure' || steps.build_seed_images.outcome == 'failure' }}
286+
287+
- name: Fail when images failed to push
288+
run: if [ $(wc -l < image-build-logs/push-failed-images.txt) -gt 0 ]; then cat image-build-logs/push-failed-images.txt && exit 1; fi
289+
if: ${{ !cancelled() }}
290+
291+
- name: Fail when images failed scanning
292+
run: if [ $(wc -l < image-build-logs/dirty-images.txt) -gt 0 ]; then cat image-build-logs/dirty-images.txt && exit 1; fi
293+
if: ${{ !inputs.push-dirty && !cancelled() }}
209294

210-
sync-container-repositories:
211-
name: Trigger container image repository sync
212-
needs:
213-
- container-image-build
214-
if: github.repository == 'stackhpc/stackhpc-kayobe-config' && inputs.push
215-
runs-on: ubuntu-latest
216-
permissions: {}
217-
steps:
218295
# NOTE(mgoddard): Trigger another CI workflow in the
219296
# stackhpc-release-train repository.
220297
- name: Trigger container image repository sync
221298
run: |
222299
filter='${{ inputs.regexes }}'
223-
if [[ -n $filter ]] && [[ ${{ github.event.inputs.seed }} == 'true' ]]; then
300+
if [[ -n $filter ]] && [[ ${{ inputs.seed }} == 'true' ]]; then
224301
filter="$filter bifrost"
225302
fi
226303
gh workflow run \
@@ -231,7 +308,9 @@ jobs:
231308
-f sync-new-images=false
232309
env:
233310
GITHUB_TOKEN: ${{ secrets.STACKHPC_RELEASE_TRAIN_TOKEN }}
311+
if: ${{ github.repository == 'stackhpc/stackhpc-kayobe-config' && inputs.push && !cancelled() }}
234312

235313
- name: Display link to container image repository sync workflows
236314
run: |
237315
echo "::notice Container image repository sync workflows: https://github.com/stackhpc/stackhpc-release-train/actions/workflows/container-sync.yml"
316+
if: ${{ github.repository == 'stackhpc/stackhpc-kayobe-config' && inputs.push && !cancelled() }}

‎doc/source/operations/tempest.rst‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -277,7 +277,10 @@ command from the base of the ``kayobe-config`` directory:
277277

278278
.. code-block:: bash
279279
280-
sudo -E docker run --detach -it --rm --network host -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config -v $(pwd)/tempest-artifacts:/stack/tempest-artifacts -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY kayobe:latest /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/tempest.sh -e ansible_user=stack
280+
sudo -E docker run --name kayobe-automation --detach -it --rm --network host \
281+
-v $(pwd):/stack/kayobe-automation-env/src/kayobe-config -v $(pwd)/tempest-artifacts:/stack/tempest-artifacts \
282+
-e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY kayobe:latest \
283+
/stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/tempest.sh -e ansible_user=stack
281284
282285
By default, ``no_log`` is set to stop credentials from leaking. This can be
283286
disabled by adding ``-e rally_no_sensitive_log=false`` to the end.

‎etc/kayobe/ansible/docker-registry-login.yml‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
---
2+
- name: Login to docker registry
3+
gather_facts: false
4+
hosts: container-image-builders
5+
tasks:
6+
- name: Login to docker registry
7+
docker_login:
8+
registry_url: "{{ kolla_docker_registry or omit }}"
9+
username: "{{ kolla_docker_registry_username }}"
10+
password: "{{ kolla_docker_registry_password }}"
11+
reauthorize: yes

‎etc/kayobe/containers/pulp/post.yml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,3 +27,10 @@
2727
when:
2828
- stackhpc_pulp_sync_for_local_container_build | bool
2929
- pulp_settings.changed
30+
31+
- name: Login to docker registry
32+
docker_login:
33+
registry_url: "{{ kolla_docker_registry or omit }}"
34+
username: "{{ kolla_docker_registry_username }}"
35+
password: "{{ kolla_docker_registry_password }}"
36+
reauthorize: yes

‎etc/kayobe/environments/aufn-ceph/a-universe-from-nothing.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -87,7 +87,7 @@ kayobe seed vm provision
8787
kayobe seed host configure
8888

8989
# Deploy local pulp server as a container on the seed VM
90-
kayobe seed service deploy --tags seed-deploy-containers --kolla-tags none -e deploy_containers_registry_attempt_login=False
90+
kayobe seed service deploy --tags seed-deploy-containers --kolla-tags none
9191

9292
# Deploying the seed restarts networking interface, run configure-local-networking.sh again to re-add routes.
9393
$KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/configure-local-networking.sh

‎etc/kayobe/environments/aufn-ceph/tenks.yml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,3 +87,9 @@ bridge_type: linuxbridge
8787

8888
# No placement service.
8989
wait_for_placement: false
90+
91+
# NOTE(priteau): Disable libvirt_vm_trust_guest_rx_filters, which when enabled
92+
# triggers the following errors when booting baremetal instances with Tenks on
93+
# Libvirt 9: Cannot set interface flags on 'macvtap1': Value too large for
94+
# defined data type
95+
libvirt_vm_trust_guest_rx_filters: false

‎etc/kayobe/kolla/config/grafana/dashboards/ceph/ceph_overview.json‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1924,23 +1924,25 @@
19241924
}
19251925
],
19261926
"spaceLength": 10,
1927-
"stack": true,
1927+
"stack": false,
19281928
"steppedLine": false,
19291929
"targets": [
19301930
{
1931-
"expr": "ceph_cluster_total_objects",
1931+
"datasource": {
1932+
"uid": "$datasource"
1933+
},
1934+
"expr": "ceph_pool_objects * on(pool_id) group_left(instance,name) ceph_pool_metadata",
19321935
"format": "time_series",
19331936
"interval": "$interval",
19341937
"intervalFactor": 1,
1935-
"legendFormat": "Total",
1938+
"legendFormat": "{{name}}",
1939+
"range": true,
19361940
"refId": "A",
19371941
"step": 300
19381942
}
19391943
],
19401944
"thresholds": [],
1941-
"timeFrom": null,
19421945
"timeRegions": [],
1943-
"timeShift": null,
19441946
"title": "Objects in the Cluster",
19451947
"tooltip": {
19461948
"msResolution": false,

‎etc/kayobe/seed.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -154,7 +154,7 @@ seed_extra_containers: {}
154154

155155
# Whether to attempt a basic authentication login to a registry when
156156
# deploying seed containers
157-
#seed_deploy_containers_registry_attempt_login:
157+
seed_deploy_containers_registry_attempt_login: "{{ not seed_pulp_container_enabled | bool }}"
158158

159159
###############################################################################
160160
# Seed node firewalld configuration.

‎releasenotes/notes/container-image-scanning-e5adf2c6b540b502.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
---
2+
security:
3+
- |
4+
Kolla container images created using the
5+
``stackhpc-container-image-build.yml`` workflow are now automatically
6+
scanned for vulnerablilities.

‎requirements.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
11
kayobe@git+https://github.com/stackhpc/kayobe@stackhpc/yoga
22
ansible-modules-hashivault@git+https://github.com/stackhpc/ansible-modules-hashivault@stackhpc;python_version < "3.8"
3-
ansible-modules-hashivault@git+https://github.com/stackhpc/ansible-modules-hashivault@stackhpc-py39;python_version >= "3.8"
3+
ansible-modules-hashivault>=5.2.1;python_version >= "3.8"
44
jmespath

0 commit comments

Comments
 (0)