Skip to content

Commit 5c53e82

Browse files
author
Scott Davidson
committed
Merge tag 'v0.7.4' into update/upstream-0.7.4
2 parents 3b49098 + a4f1bb9 commit 5c53e82

File tree

613 files changed

+56118
-8890
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

613 files changed

+56118
-8890
lines changed

‎.github/actions/custom-build-and-push/action.yml‎

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
name: 'Build and Push Docker Image with Retry'
2+
description: 'Attempts to build and push a Docker image, with a retry on failure'
3+
inputs:
4+
context:
5+
description: 'Build context'
6+
required: true
7+
file:
8+
description: 'Dockerfile location'
9+
required: true
10+
platforms:
11+
description: 'Target platforms'
12+
required: true
13+
pull:
14+
description: 'Always attempt to pull a newer version of the image'
15+
required: false
16+
default: 'true'
17+
push:
18+
description: 'Push the image to registry'
19+
required: false
20+
default: 'true'
21+
load:
22+
description: 'Load the image into Docker daemon'
23+
required: false
24+
default: 'true'
25+
tags:
26+
description: 'Image tags'
27+
required: true
28+
cache-from:
29+
description: 'Cache sources'
30+
required: false
31+
cache-to:
32+
description: 'Cache destinations'
33+
required: false
34+
retry-wait-time:
35+
description: 'Time to wait before retry in seconds'
36+
required: false
37+
default: '5'
38+
39+
runs:
40+
using: "composite"
41+
steps:
42+
- name: Build and push Docker image (First Attempt)
43+
id: buildx1
44+
uses: docker/build-push-action@v5
45+
continue-on-error: true
46+
with:
47+
context: ${{ inputs.context }}
48+
file: ${{ inputs.file }}
49+
platforms: ${{ inputs.platforms }}
50+
pull: ${{ inputs.pull }}
51+
push: ${{ inputs.push }}
52+
load: ${{ inputs.load }}
53+
tags: ${{ inputs.tags }}
54+
cache-from: ${{ inputs.cache-from }}
55+
cache-to: ${{ inputs.cache-to }}
56+
57+
- name: Wait to retry
58+
if: steps.buildx1.outcome != 'success'
59+
run: |
60+
echo "First attempt failed. Waiting ${{ inputs.retry-wait-time }} seconds before retry..."
61+
sleep ${{ inputs.retry-wait-time }}
62+
shell: bash
63+
64+
- name: Build and push Docker image (Retry Attempt)
65+
if: steps.buildx1.outcome != 'success'
66+
uses: docker/build-push-action@v5
67+
with:
68+
context: ${{ inputs.context }}
69+
file: ${{ inputs.file }}
70+
platforms: ${{ inputs.platforms }}
71+
pull: ${{ inputs.pull }}
72+
push: ${{ inputs.push }}
73+
load: ${{ inputs.load }}
74+
tags: ${{ inputs.tags }}
75+
cache-from: ${{ inputs.cache-from }}
76+
cache-to: ${{ inputs.cache-to }}

‎.github/workflows/docker-build-push-backend-container-on-tag.yml‎

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,13 @@ on:
77

88
env:
99
REGISTRY_IMAGE: ghcr.io/stackhpc/danswer/danswer-backend
10+
LATEST_TAG: ${{ contains(github.ref_name, 'latest') }}
1011

1112
jobs:
1213
build-and-push:
14+
# TODO: investigate a matrix build like the web container
15+
# See https://runs-on.com/runners/linux/
16+
# NOTE(sd109): Can't use Danswer custom runners here
1317
runs-on: ubuntu-latest
1418

1519
steps:
@@ -35,21 +39,36 @@ jobs:
3539
type=raw,value=${{ github.ref_name }}
3640
type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }}
3741
42+
- name: Install build-essential
43+
run: |
44+
sudo apt-get update
45+
sudo apt-get install -y build-essential
46+
3847
- name: Backend Image Docker Build and Push
3948
uses: docker/build-push-action@v5
4049
with:
4150
context: ./backend
4251
file: ./backend/Dockerfile
4352
platforms: linux/amd64,linux/arm64
4453
push: true
45-
tags: ${{ steps.meta.outputs.tags }}
54+
tags: |
55+
${{ env.REGISTRY_IMAGE }}:${{ github.ref_name }}
56+
${{ env.LATEST_TAG == 'true' && format('{0}:latest', env.REGISTRY_IMAGE) || '' }}
4657
build-args: |
4758
DANSWER_VERSION=${{ github.ref_name }}
4859
cache-from: type=registry,ref=${{ env.REGISTRY_IMAGE }}:buildcache
4960
cache-to: type=registry,ref=${{ env.REGISTRY_IMAGE}}:buildcache,mode=max
5061

62+
# trivy has their own rate limiting issues causing this action to flake
63+
# we worked around it by hardcoding to different db repos in env
64+
# can re-enable when they figure it out
65+
# https://github.com/aquasecurity/trivy/discussions/7538
66+
# https://github.com/aquasecurity/trivy-action/issues/389
5167
- name: Run Trivy vulnerability scanner
5268
uses: aquasecurity/trivy-action@master
69+
env:
70+
TRIVY_DB_REPOSITORY: 'public.ecr.aws/aquasecurity/trivy-db:2'
71+
TRIVY_JAVA_DB_REPOSITORY: 'public.ecr.aws/aquasecurity/trivy-java-db:1'
5372
with:
5473
# To run locally: trivy image --severity HIGH,CRITICAL danswer/danswer-backend
5574
image-ref: ${{ env.REGISTRY_IMAGE }}:${{ github.ref_name }}

‎.github/workflows/docker-build-push-model-server-container-on-tag.yml‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,11 @@ on:
77

88
env:
99
REGISTRY_IMAGE: ghcr.io/stackhpc/danswer/danswer-model-server
10+
LATEST_TAG: ${{ contains(github.ref_name, 'latest') }}
1011

1112
jobs:
1213
build-and-push:
14+
# NOTE(sd109): Can't use Danswer custom runners here
1315
runs-on: ubuntu-latest
1416

1517
steps:
@@ -35,14 +37,22 @@ jobs:
3537
push: true
3638
tags: |
3739
${{ env.REGISTRY_IMAGE }}:${{ github.ref_name }}
38-
${{ env.REGISTRY_IMAGE }}:latest
40+
${{ env.LATEST_TAG == 'true' && format('{0}:latest', env.REGISTRY_IMAGE) || '' }}
3941
build-args: |
4042
DANSWER_VERSION=${{ github.ref_name }}
4143
cache-from: type=gha
4244
cache-to: type=gha,mode=max
4345

46+
# trivy has their own rate limiting issues causing this action to flake
47+
# we worked around it by hardcoding to different db repos in env
48+
# can re-enable when they figure it out
49+
# https://github.com/aquasecurity/trivy/discussions/7538
50+
# https://github.com/aquasecurity/trivy-action/issues/389
4451
- name: Run Trivy vulnerability scanner
4552
uses: aquasecurity/trivy-action@master
53+
env:
54+
TRIVY_DB_REPOSITORY: 'public.ecr.aws/aquasecurity/trivy-db:2'
55+
TRIVY_JAVA_DB_REPOSITORY: 'public.ecr.aws/aquasecurity/trivy-java-db:1'
4656
with:
4757
image-ref: ${{ env.REGISTRY_IMAGE }}:${{ github.ref_name }}
4858
severity: 'CRITICAL,HIGH'

‎.github/workflows/docker-build-push-web-container-on-tag.yml‎

Lines changed: 115 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -7,50 +7,123 @@ on:
77

88
env:
99
REGISTRY_IMAGE: ghcr.io/stackhpc/danswer/danswer-web-server
10+
LATEST_TAG: ${{ contains(github.ref_name, 'latest') }}
1011

1112
jobs:
12-
build-and-push:
13+
build:
14+
# NOTE(sd109): Can't use Danswer custom runners here
1315
runs-on: ubuntu-latest
16+
strategy:
17+
fail-fast: false
18+
matrix:
19+
platform:
20+
- linux/amd64
21+
- linux/arm64
1422

1523
steps:
16-
- name: Checkout code
17-
uses: actions/checkout@v4
18-
19-
- name: Set up Docker Buildx
20-
uses: docker/setup-buildx-action@v3
21-
22-
- name: Login to GitHub Container Registry
23-
uses: docker/login-action@v3
24-
with:
25-
registry: ghcr.io
26-
username: ${{ github.actor }}
27-
password: ${{ secrets.GITHUB_TOKEN }}
28-
29-
- name: Docker meta
30-
id: meta
31-
uses: docker/metadata-action@v5
32-
with:
33-
images: ${{ env.REGISTRY_IMAGE }}
34-
tags: |
35-
type=raw,value=${{ github.ref_name }}
36-
type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }}
37-
38-
- name: Web Image Docker Build and Push
39-
uses: docker/build-push-action@v5
40-
with:
41-
context: ./web
42-
file: ./web/Dockerfile
43-
platforms: linux/amd64 #,linux/arm64
44-
push: true
45-
tags: ${{ steps.meta.outputs.tags }}
46-
build-args: |
47-
DANSWER_VERSION=${{ github.ref_name }}
48-
cache-from: type=registry,ref=${{ env.REGISTRY_IMAGE }}:buildcache
49-
cache-to: type=registry,ref=${{ env.REGISTRY_IMAGE}}:buildcache,mode=max
50-
51-
- name: Run Trivy vulnerability scanner
52-
uses: aquasecurity/trivy-action@master
53-
with:
54-
# To run locally: trivy image --severity HIGH,CRITICAL danswer/danswer-web
55-
image-ref: ${{ env.REGISTRY_IMAGE }}:${{ github.ref_name }}
56-
severity: 'CRITICAL,HIGH'
24+
- name: Prepare
25+
run: |
26+
platform=${{ matrix.platform }}
27+
echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV
28+
29+
- name: Checkout
30+
uses: actions/checkout@v4
31+
32+
- name: Docker meta
33+
id: meta
34+
uses: docker/metadata-action@v5
35+
with:
36+
images: ${{ env.REGISTRY_IMAGE }}
37+
tags: |
38+
type=raw,value=${{ env.REGISTRY_IMAGE }}:${{ github.ref_name }}
39+
type=raw,value=${{ env.LATEST_TAG == 'true' && format('{0}:latest', env.REGISTRY_IMAGE) || '' }}
40+
41+
- name: Set up Docker Buildx
42+
uses: docker/setup-buildx-action@v3
43+
44+
- name: Login to Docker Hub
45+
uses: docker/login-action@v3
46+
with:
47+
username: ${{ secrets.DOCKER_USERNAME }}
48+
password: ${{ secrets.DOCKER_TOKEN }}
49+
50+
- name: Build and push by digest
51+
id: build
52+
uses: docker/build-push-action@v5
53+
with:
54+
context: ./web
55+
file: ./web/Dockerfile
56+
platforms: ${{ matrix.platform }}
57+
push: true
58+
build-args: |
59+
DANSWER_VERSION=${{ github.ref_name }}
60+
# needed due to weird interactions with the builds for different platforms
61+
no-cache: true
62+
labels: ${{ steps.meta.outputs.labels }}
63+
outputs: type=image,name=${{ env.REGISTRY_IMAGE }},push-by-digest=true,name-canonical=true,push=true
64+
65+
- name: Export digest
66+
run: |
67+
mkdir -p /tmp/digests
68+
digest="${{ steps.build.outputs.digest }}"
69+
touch "/tmp/digests/${digest#sha256:}"
70+
71+
- name: Upload digest
72+
uses: actions/upload-artifact@v4
73+
with:
74+
name: digests-${{ env.PLATFORM_PAIR }}
75+
path: /tmp/digests/*
76+
if-no-files-found: error
77+
retention-days: 1
78+
79+
merge:
80+
runs-on: ubuntu-latest
81+
needs:
82+
- build
83+
steps:
84+
- name: Download digests
85+
uses: actions/download-artifact@v4
86+
with:
87+
path: /tmp/digests
88+
pattern: digests-*
89+
merge-multiple: true
90+
91+
- name: Set up Docker Buildx
92+
uses: docker/setup-buildx-action@v3
93+
94+
- name: Docker meta
95+
id: meta
96+
uses: docker/metadata-action@v5
97+
with:
98+
images: ${{ env.REGISTRY_IMAGE }}
99+
100+
- name: Login to GitHub Container Registry
101+
uses: docker/login-action@v3
102+
with:
103+
registry: ghcr.io
104+
username: ${{ github.actor }}
105+
password: ${{ secrets.GITHUB_TOKEN }}
106+
107+
- name: Create manifest list and push
108+
working-directory: /tmp/digests
109+
run: |
110+
docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
111+
$(printf '${{ env.REGISTRY_IMAGE }}@sha256:%s ' *)
112+
113+
- name: Inspect image
114+
run: |
115+
docker buildx imagetools inspect ${{ env.REGISTRY_IMAGE }}:${{ steps.meta.outputs.version }}
116+
117+
# trivy has their own rate limiting issues causing this action to flake
118+
# we worked around it by hardcoding to different db repos in env
119+
# can re-enable when they figure it out
120+
# https://github.com/aquasecurity/trivy/discussions/7538
121+
# https://github.com/aquasecurity/trivy-action/issues/389
122+
- name: Run Trivy vulnerability scanner
123+
uses: aquasecurity/trivy-action@master
124+
env:
125+
TRIVY_DB_REPOSITORY: 'public.ecr.aws/aquasecurity/trivy-db:2'
126+
TRIVY_JAVA_DB_REPOSITORY: 'public.ecr.aws/aquasecurity/trivy-java-db:1'
127+
with:
128+
image-ref: ${{ env.REGISTRY_IMAGE }}:${{ github.ref_name }}
129+
severity: 'CRITICAL,HIGH'

‎.github/workflows/docker-tag-latest.yml‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,6 @@
1+
# This workflow is set up to be manually triggered via the GitHub Action tab.
2+
# Given a version, it will tag those backend and webserver images as "latest".
3+
14
name: Tag Latest Version
25

36
on:
@@ -9,7 +12,9 @@ on:
912

1013
jobs:
1114
tag:
12-
runs-on: ubuntu-latest
15+
# See https://runs-on.com/runners/linux/
16+
# use a lower powered instance since this just does i/o to docker hub
17+
runs-on: [runs-on,runner=2cpu-linux-x64,"run-id=${{ github.run_id }}"]
1318
steps:
1419
- name: Set up Docker Buildx
1520
uses: docker/setup-buildx-action@v1

0 commit comments

Comments
 (0)