Skip to content

Commit b6b2c9b

Browse files
sd109sd109
authored
Merge pull request #10 from stackhpc/update/upstream-0.7.4
Update to upstream v0.7.4
2 parents 3b49098 + 8f22a5e commit b6b2c9b

File tree

617 files changed

+56137
-8899
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

617 files changed

+56137
-8899
lines changed

‎.github/actions/custom-build-and-push/action.yml‎

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
name: 'Build and Push Docker Image with Retry'
2+
description: 'Attempts to build and push a Docker image, with a retry on failure'
3+
inputs:
4+
context:
5+
description: 'Build context'
6+
required: true
7+
file:
8+
description: 'Dockerfile location'
9+
required: true
10+
platforms:
11+
description: 'Target platforms'
12+
required: true
13+
pull:
14+
description: 'Always attempt to pull a newer version of the image'
15+
required: false
16+
default: 'true'
17+
push:
18+
description: 'Push the image to registry'
19+
required: false
20+
default: 'true'
21+
load:
22+
description: 'Load the image into Docker daemon'
23+
required: false
24+
default: 'true'
25+
tags:
26+
description: 'Image tags'
27+
required: true
28+
cache-from:
29+
description: 'Cache sources'
30+
required: false
31+
cache-to:
32+
description: 'Cache destinations'
33+
required: false
34+
retry-wait-time:
35+
description: 'Time to wait before retry in seconds'
36+
required: false
37+
default: '5'
38+
39+
runs:
40+
using: "composite"
41+
steps:
42+
- name: Build and push Docker image (First Attempt)
43+
id: buildx1
44+
uses: docker/build-push-action@v5
45+
continue-on-error: true
46+
with:
47+
context: ${{ inputs.context }}
48+
file: ${{ inputs.file }}
49+
platforms: ${{ inputs.platforms }}
50+
pull: ${{ inputs.pull }}
51+
push: ${{ inputs.push }}
52+
load: ${{ inputs.load }}
53+
tags: ${{ inputs.tags }}
54+
cache-from: ${{ inputs.cache-from }}
55+
cache-to: ${{ inputs.cache-to }}
56+
57+
- name: Wait to retry
58+
if: steps.buildx1.outcome != 'success'
59+
run: |
60+
echo "First attempt failed. Waiting ${{ inputs.retry-wait-time }} seconds before retry..."
61+
sleep ${{ inputs.retry-wait-time }}
62+
shell: bash
63+
64+
- name: Build and push Docker image (Retry Attempt)
65+
if: steps.buildx1.outcome != 'success'
66+
uses: docker/build-push-action@v5
67+
with:
68+
context: ${{ inputs.context }}
69+
file: ${{ inputs.file }}
70+
platforms: ${{ inputs.platforms }}
71+
pull: ${{ inputs.pull }}
72+
push: ${{ inputs.push }}
73+
load: ${{ inputs.load }}
74+
tags: ${{ inputs.tags }}
75+
cache-from: ${{ inputs.cache-from }}
76+
cache-to: ${{ inputs.cache-to }}

‎.github/workflows/docker-build-push-backend-container-on-tag.yml‎

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,13 @@ on:
77

88
env:
99
REGISTRY_IMAGE: ghcr.io/stackhpc/danswer/danswer-backend
10+
LATEST_TAG: ${{ contains(github.ref_name, 'latest') }}
1011

1112
jobs:
1213
build-and-push:
14+
# TODO: investigate a matrix build like the web container
15+
# See https://runs-on.com/runners/linux/
16+
# NOTE(sd109): Can't use Danswer custom runners here
1317
runs-on: ubuntu-latest
1418

1519
steps:
@@ -35,21 +39,36 @@ jobs:
3539
type=raw,value=${{ github.ref_name }}
3640
type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }}
3741
42+
- name: Install build-essential
43+
run: |
44+
sudo apt-get update
45+
sudo apt-get install -y build-essential
46+
3847
- name: Backend Image Docker Build and Push
3948
uses: docker/build-push-action@v5
4049
with:
4150
context: ./backend
4251
file: ./backend/Dockerfile
4352
platforms: linux/amd64,linux/arm64
4453
push: true
45-
tags: ${{ steps.meta.outputs.tags }}
54+
tags: |
55+
${{ env.REGISTRY_IMAGE }}:${{ github.ref_name }}
56+
${{ env.LATEST_TAG == 'true' && format('{0}:latest', env.REGISTRY_IMAGE) || '' }}
4657
build-args: |
4758
DANSWER_VERSION=${{ github.ref_name }}
4859
cache-from: type=registry,ref=${{ env.REGISTRY_IMAGE }}:buildcache
4960
cache-to: type=registry,ref=${{ env.REGISTRY_IMAGE}}:buildcache,mode=max
5061

62+
# trivy has their own rate limiting issues causing this action to flake
63+
# we worked around it by hardcoding to different db repos in env
64+
# can re-enable when they figure it out
65+
# https://github.com/aquasecurity/trivy/discussions/7538
66+
# https://github.com/aquasecurity/trivy-action/issues/389
5167
- name: Run Trivy vulnerability scanner
5268
uses: aquasecurity/trivy-action@master
69+
env:
70+
TRIVY_DB_REPOSITORY: 'public.ecr.aws/aquasecurity/trivy-db:2'
71+
TRIVY_JAVA_DB_REPOSITORY: 'public.ecr.aws/aquasecurity/trivy-java-db:1'
5372
with:
5473
# To run locally: trivy image --severity HIGH,CRITICAL danswer/danswer-backend
5574
image-ref: ${{ env.REGISTRY_IMAGE }}:${{ github.ref_name }}

‎.github/workflows/docker-build-push-model-server-container-on-tag.yml‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,11 @@ on:
77

88
env:
99
REGISTRY_IMAGE: ghcr.io/stackhpc/danswer/danswer-model-server
10+
LATEST_TAG: ${{ contains(github.ref_name, 'latest') }}
1011

1112
jobs:
1213
build-and-push:
14+
# NOTE(sd109): Can't use Danswer custom runners here
1315
runs-on: ubuntu-latest
1416

1517
steps:
@@ -35,14 +37,22 @@ jobs:
3537
push: true
3638
tags: |
3739
${{ env.REGISTRY_IMAGE }}:${{ github.ref_name }}
38-
${{ env.REGISTRY_IMAGE }}:latest
40+
${{ env.LATEST_TAG == 'true' && format('{0}:latest', env.REGISTRY_IMAGE) || '' }}
3941
build-args: |
4042
DANSWER_VERSION=${{ github.ref_name }}
4143
cache-from: type=gha
4244
cache-to: type=gha,mode=max
4345

46+
# trivy has their own rate limiting issues causing this action to flake
47+
# we worked around it by hardcoding to different db repos in env
48+
# can re-enable when they figure it out
49+
# https://github.com/aquasecurity/trivy/discussions/7538
50+
# https://github.com/aquasecurity/trivy-action/issues/389
4451
- name: Run Trivy vulnerability scanner
4552
uses: aquasecurity/trivy-action@master
53+
env:
54+
TRIVY_DB_REPOSITORY: 'public.ecr.aws/aquasecurity/trivy-db:2'
55+
TRIVY_JAVA_DB_REPOSITORY: 'public.ecr.aws/aquasecurity/trivy-java-db:1'
4656
with:
4757
image-ref: ${{ env.REGISTRY_IMAGE }}:${{ github.ref_name }}
4858
severity: 'CRITICAL,HIGH'

‎.github/workflows/docker-build-push-web-container-on-tag.yml‎

Lines changed: 118 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -7,50 +7,126 @@ on:
77

88
env:
99
REGISTRY_IMAGE: ghcr.io/stackhpc/danswer/danswer-web-server
10+
LATEST_TAG: ${{ contains(github.ref_name, 'latest') }}
1011

1112
jobs:
12-
build-and-push:
13+
build:
14+
# NOTE(sd109): Can't use Danswer custom runners here
1315
runs-on: ubuntu-latest
16+
strategy:
17+
fail-fast: false
18+
matrix:
19+
platform:
20+
- linux/amd64
21+
# NOTE(sd109): Arm builds currently failing with error seen here:
22+
# https://github.com/stackhpc/danswer/actions/runs/11368042561/job/31622167035#step:7:366
23+
# - linux/arm64
1424

1525
steps:
16-
- name: Checkout code
17-
uses: actions/checkout@v4
18-
19-
- name: Set up Docker Buildx
20-
uses: docker/setup-buildx-action@v3
21-
22-
- name: Login to GitHub Container Registry
23-
uses: docker/login-action@v3
24-
with:
25-
registry: ghcr.io
26-
username: ${{ github.actor }}
27-
password: ${{ secrets.GITHUB_TOKEN }}
28-
29-
- name: Docker meta
30-
id: meta
31-
uses: docker/metadata-action@v5
32-
with:
33-
images: ${{ env.REGISTRY_IMAGE }}
34-
tags: |
35-
type=raw,value=${{ github.ref_name }}
36-
type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }}
37-
38-
- name: Web Image Docker Build and Push
39-
uses: docker/build-push-action@v5
40-
with:
41-
context: ./web
42-
file: ./web/Dockerfile
43-
platforms: linux/amd64 #,linux/arm64
44-
push: true
45-
tags: ${{ steps.meta.outputs.tags }}
46-
build-args: |
47-
DANSWER_VERSION=${{ github.ref_name }}
48-
cache-from: type=registry,ref=${{ env.REGISTRY_IMAGE }}:buildcache
49-
cache-to: type=registry,ref=${{ env.REGISTRY_IMAGE}}:buildcache,mode=max
50-
51-
- name: Run Trivy vulnerability scanner
52-
uses: aquasecurity/trivy-action@master
53-
with:
54-
# To run locally: trivy image --severity HIGH,CRITICAL danswer/danswer-web
55-
image-ref: ${{ env.REGISTRY_IMAGE }}:${{ github.ref_name }}
56-
severity: 'CRITICAL,HIGH'
26+
- name: Prepare
27+
run: |
28+
platform=${{ matrix.platform }}
29+
echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV
30+
31+
- name: Checkout
32+
uses: actions/checkout@v4
33+
34+
- name: Docker meta
35+
id: meta
36+
uses: docker/metadata-action@v5
37+
with:
38+
images: ${{ env.REGISTRY_IMAGE }}
39+
tags: |
40+
type=raw,value=${{ env.REGISTRY_IMAGE }}:${{ github.ref_name }}
41+
type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }}
42+
43+
- name: Set up Docker Buildx
44+
uses: docker/setup-buildx-action@v3
45+
46+
- name: Login to GitHub Container Registry
47+
uses: docker/login-action@v3
48+
with:
49+
registry: ghcr.io
50+
username: ${{ github.actor }}
51+
password: ${{ secrets.GITHUB_TOKEN }}
52+
53+
- name: Build and push by digest
54+
id: build
55+
uses: docker/build-push-action@v5
56+
with:
57+
context: ./web
58+
file: ./web/Dockerfile
59+
platforms: ${{ matrix.platform }}
60+
push: true
61+
build-args: |
62+
DANSWER_VERSION=${{ github.ref_name }}
63+
# needed due to weird interactions with the builds for different platforms
64+
no-cache: true
65+
labels: ${{ steps.meta.outputs.labels }}
66+
outputs: type=image,name=${{ env.REGISTRY_IMAGE }},push-by-digest=true,name-canonical=true,push=true
67+
68+
- name: Export digest
69+
run: |
70+
mkdir -p /tmp/digests
71+
digest="${{ steps.build.outputs.digest }}"
72+
touch "/tmp/digests/${digest#sha256:}"
73+
74+
- name: Upload digest
75+
uses: actions/upload-artifact@v4
76+
with:
77+
name: digests-${{ env.PLATFORM_PAIR }}
78+
path: /tmp/digests/*
79+
if-no-files-found: error
80+
retention-days: 1
81+
82+
merge:
83+
runs-on: ubuntu-latest
84+
needs:
85+
- build
86+
steps:
87+
- name: Download digests
88+
uses: actions/download-artifact@v4
89+
with:
90+
path: /tmp/digests
91+
pattern: digests-*
92+
merge-multiple: true
93+
94+
- name: Set up Docker Buildx
95+
uses: docker/setup-buildx-action@v3
96+
97+
- name: Docker meta
98+
id: meta
99+
uses: docker/metadata-action@v5
100+
with:
101+
images: ${{ env.REGISTRY_IMAGE }}
102+
103+
- name: Login to GitHub Container Registry
104+
uses: docker/login-action@v3
105+
with:
106+
registry: ghcr.io
107+
username: ${{ github.actor }}
108+
password: ${{ secrets.GITHUB_TOKEN }}
109+
110+
- name: Create manifest list and push
111+
working-directory: /tmp/digests
112+
run: |
113+
docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
114+
$(printf '${{ env.REGISTRY_IMAGE }}@sha256:%s ' *)
115+
116+
- name: Inspect image
117+
run: |
118+
docker buildx imagetools inspect ${{ env.REGISTRY_IMAGE }}:${{ steps.meta.outputs.version }}
119+
120+
# trivy has their own rate limiting issues causing this action to flake
121+
# we worked around it by hardcoding to different db repos in env
122+
# can re-enable when they figure it out
123+
# https://github.com/aquasecurity/trivy/discussions/7538
124+
# https://github.com/aquasecurity/trivy-action/issues/389
125+
- name: Run Trivy vulnerability scanner
126+
uses: aquasecurity/trivy-action@master
127+
env:
128+
TRIVY_DB_REPOSITORY: 'public.ecr.aws/aquasecurity/trivy-db:2'
129+
TRIVY_JAVA_DB_REPOSITORY: 'public.ecr.aws/aquasecurity/trivy-java-db:1'
130+
with:
131+
image-ref: ${{ env.REGISTRY_IMAGE }}:${{ github.ref_name }}
132+
severity: 'CRITICAL,HIGH'

‎.github/workflows/docker-tag-latest.yml‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,6 @@
1+
# This workflow is set up to be manually triggered via the GitHub Action tab.
2+
# Given a version, it will tag those backend and webserver images as "latest".
3+
14
name: Tag Latest Version
25

36
on:
@@ -9,7 +12,9 @@ on:
912

1013
jobs:
1114
tag:
12-
runs-on: ubuntu-latest
15+
# See https://runs-on.com/runners/linux/
16+
# use a lower powered instance since this just does i/o to docker hub
17+
runs-on: [runs-on,runner=2cpu-linux-x64,"run-id=${{ github.run_id }}"]
1318
steps:
1419
- name: Set up Docker Buildx
1520
uses: docker/setup-buildx-action@v1

0 commit comments

Comments
 (0)