Skip to content

Commit b26787c

Browse files
jacjac
authored
Add Embeddings service (#283)
1 parent ba1053e commit b26787c

File tree

7 files changed

+232
-0
lines changed

7 files changed

+232
-0
lines changed

‎charts/sourcegraph/README.md‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,19 @@ In addition to the documented values, all services also support the following va
9090
| codeIntelDB.serviceAccount.create | bool | `false` | Enable creation of ServiceAccount for `codeintel-db` |
9191
| codeIntelDB.serviceAccount.name | string | `""` | Name of the ServiceAccount to be created or an existing ServiceAccount |
9292
| codeIntelDB.storageSize | string | `"200Gi"` | PVC Storage Request for `codeintel-db` data volume |
93+
| embeddings.containerSecurityContext | object | `{"allowPrivilegeEscalation":false,"readOnlyRootFilesystem":true,"runAsGroup":101,"runAsUser":100}` | Security context for the `worker` container, learn more from the [Kubernetes documentation](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container) |
94+
| embeddings.enabled | bool | `false` | Enable `embeddings` |
95+
| embeddings.env | object | `{}` | Environment variables for the `embeddings` container |
96+
| embeddings.extraVolumeMounts | object | `{}` | |
97+
| embeddings.extraVolumes | object | `{}` | |
98+
| embeddings.image.defaultTag | string | `"5.0.3@sha256:1d8f05ed57e361451d92eabbb3a0d90169c1108f48a274d556e43f541190e01a"` | Docker image tag for the `embeddings` image |
99+
| embeddings.image.name | string | `"embeddings"` | Docker image name for the `embeddings` image |
100+
| embeddings.name | string | `"embeddings"` | Name of the `embeddings` service |
101+
| embeddings.podSecurityContext | object | `{}` | Security context for the `embeddings` container, learn more from the [Kubernetes documentation](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container) |
102+
| embeddings.resources | object | `{"limits":{"cpu":"8","memory":"64G"},"requests":{"cpu":"4","memory":"32G"}}` | Resource requests & limits for the `worker` container, learn more from the [Kubernetes documentation](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/) |
103+
| embeddings.serviceAccount.annotations | object | `{}` | |
104+
| embeddings.serviceAccount.create | bool | `false` | Enable creation of ServiceAccount for `embeddings` |
105+
| embeddings.serviceAccount.name | string | `""` | Name of the ServiceAccount to be created or an existing ServiceAccount |
93106
| extraResources | list | `[]` | Additional resources to include in the rendered manifest. Templates are supported. |
94107
| frontend.containerSecurityContext | object | `{"allowPrivilegeEscalation":false,"readOnlyRootFilesystem":true,"runAsGroup":101,"runAsUser":100}` | Security context for the `frontend` container, learn more from the [Kubernetes documentation](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container) |
95108
| frontend.env | object | the chart will add some default environment values | Environment variables for the `frontend` container |
@@ -337,6 +350,7 @@ In addition to the documented values, all services also support the following va
337350
| syntectServer.serviceAccount.create | bool | `false` | Enable creation of ServiceAccount for `syntect-server` |
338351
| syntectServer.serviceAccount.name | string | `""` | Name of the ServiceAccount to be created or an existing ServiceAccount |
339352
| worker.containerSecurityContext | object | `{"allowPrivilegeEscalation":false,"readOnlyRootFilesystem":true,"runAsGroup":101,"runAsUser":100}` | Security context for the `worker` container, learn more from the [Kubernetes documentation](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container) |
353+
| worker.env | object | `{}` | Environment variables for the `worker` container |
340354
| worker.image.defaultTag | string | `"5.0.3@sha256:85016eadcb12a5d94b8ff47192899572bbd33cbc19769b5b1c7c3cc48ca5a914"` | Docker image tag for the `worker` image |
341355
| worker.image.name | string | `"worker"` | Docker image name for the `worker` image |
342356
| worker.name | string | `"worker"` | Name used by resources. Does not affect service names or PVCs. |

‎charts/sourcegraph/examples/external-object-storage-embeddings/override.yaml‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
# Override file demonstrating the use of external Object Storage services
2+
# Learn more from the tutorial below
3+
# - https://docs.sourcegraph.com/cody/explanations/code_graph_context#storing-embedding-indexes
4+
5+
# Disable deployment of the built-in object storage
6+
# Only disable if also using external storage for SCIP indexes
7+
blobstore:
8+
enabled: false
9+
10+
# we use YAML anchors and alias to keep override file clean
11+
objectStorageEnv: &objectStorageEnv
12+
EMBEDDINGS_UPLOAD_BACKEND:
13+
value: S3 # external object stoage type, one of "S3" or "GCS"
14+
EMBEDDINGS_UPLOAD_BUCKET:
15+
value: embedings-uploads # external object storage bucket name
16+
EMBEDDINGS_UPLOAD_AWS_ENDPOINT:
17+
value: https://s3.us-east-1.amazonaws.com
18+
EMBEDDINGS_UPLOAD_AWS_REGION:
19+
value: us-east-1
20+
EMBEDDINGS_UPLOAD_AWS_ACCESS_KEY_ID:
21+
secretKeyRef: # Pre-existing secret, not created by this chart
22+
name: sourcegraph-s3-credentials
23+
key: EMBEDDINGS_UPLOAD_AWS_ACCESS_KEY_ID
24+
EMBEDDINGS_UPLOAD_AWS_SECRET_ACCESS_KEY:
25+
secretKeyRef: # Pre-existing secret, not created by this chart
26+
name: sourcegraph-s3-credentials
27+
key: EMBEDDINGS_UPLOAD_AWS_SECRET_ACCESS_KEY
28+
29+
embeddings:
30+
enabled: true
31+
env:
32+
<<: *objectStorageEnv
33+
34+
worker:
35+
env:
36+
<<: *objectStorageEnv

‎charts/sourcegraph/templates/embeddings/embeddings.Deployment.yaml‎

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
{{- if .Values.embeddings.enabled -}}
2+
apiVersion: apps/v1
3+
kind: Deployment
4+
metadata:
5+
name: {{ .Values.embeddings.name }}
6+
annotations:
7+
description: Handles embeddings which enhance Cody's context fetching
8+
labels:
9+
{{- include "sourcegraph.labels" . | nindent 4 }}
10+
{{- if .Values.embeddings.labels }}
11+
{{- toYaml .Values.embeddings.labels | nindent 4 }}
12+
{{- end }}
13+
deploy: sourcegraph
14+
app.kubernetes.io/component: embeddings
15+
spec:
16+
minReadySeconds: 10
17+
replicas: 1
18+
revisionHistoryLimit: {{ .Values.sourcegraph.revisionHistoryLimit }}
19+
selector:
20+
matchLabels:
21+
{{- include "sourcegraph.selectorLabels" . | nindent 6 }}
22+
app: {{ .Values.embeddings.name }}
23+
strategy:
24+
rollingUpdate:
25+
maxSurge: 1
26+
maxUnavailable: 0
27+
type: RollingUpdate
28+
template:
29+
metadata:
30+
annotations:
31+
{{- if .Values.sourcegraph.podAnnotations }}
32+
{{- toYaml .Values.sourcegraph.podAnnotations | nindent 8 }}
33+
{{- end }}
34+
{{- if .Values.embeddings.podAnnotations }}
35+
{{- toYaml .Values.embeddings.podAnnotations | nindent 8 }}
36+
{{- end }}
37+
labels:
38+
app: {{ .Values.embeddings.name }}
39+
app.kubernetes.io/component: embeddings
40+
deploy: sourcegraph
41+
{{- include "sourcegraph.selectorLabels" . | nindent 8 }}
42+
{{- if .Values.sourcegraph.podLabels }}
43+
{{- toYaml .Values.sourcegraph.podLabels | nindent 8 }}
44+
{{- end }}
45+
{{- if .Values.embeddings.podLabels }}
46+
{{- toYaml .Values.embeddings.podLabels | nindent 8 }}
47+
{{- end }}
48+
spec:
49+
containers:
50+
- name: {{ .Values.embeddings.name }}
51+
image: {{ include "sourcegraph.image" (list . "embeddings") }}
52+
imagePullPolicy: {{ .Values.sourcegraph.image.pullPolicy }}
53+
env:
54+
{{- range $name, $item := .Values.embeddings.env }}
55+
- name: {{ $name }}
56+
{{- $item | toYaml | nindent 10 }}
57+
{{- end }}
58+
{{- if not .Values.embeddings.env.EMBEDDINGS_UPLOAD_BACKEND }}
59+
- name: EMBEDDINGS_UPLOAD_BACKEND
60+
value: blobstore
61+
{{- end }}
62+
{{- if not .Values.embeddings.env.EMBEDDINGS_UPLOAD_AWS_ENDPOINT }}
63+
- name: EMBEDDINGS_UPLOAD_AWS_ENDPOINT
64+
value: http://blobstore:9000
65+
{{- end }}
66+
ports:
67+
- containerPort: 9991
68+
name: http
69+
protocol: TCP
70+
volumeMounts:
71+
{{- if .Values.embeddings.extraVolumeMounts }}
72+
{{- toYaml .Values.embeddings.extraVolumeMounts | nindent 8 }}
73+
{{- end }}
74+
{{- if not .Values.sourcegraph.localDevMode}}
75+
resources:
76+
{{- toYaml .Values.embeddings.resources | nindent 10 }}
77+
{{- end }}
78+
securityContext:
79+
{{- toYaml .Values.embeddings.containerSecurityContext | nindent 10 }}
80+
securityContext:
81+
{{- toYaml .Values.embeddings.podSecurityContext | nindent 8 }}
82+
{{- include "sourcegraph.nodeSelector" (list . "embeddings" ) | trim | nindent 6 }}
83+
{{- include "sourcegraph.affinity" (list . "embeddings" ) | trim | nindent 6 }}
84+
{{- include "sourcegraph.tolerations" (list . "embeddings" ) | trim | nindent 6 }}
85+
{{- if .Values.embeddings.serviceAccount.create }}
86+
serviceAccountName: {{ .Values.embeddings.serviceAccount.name }}
87+
{{- end}}
88+
volumes:
89+
{{- if .Values.embeddings.extraVolumes }}
90+
{{- toYaml .Values.embeddings.extraVolumes | nindent 6 }}
91+
{{- end }}
92+
{{- end }}

‎charts/sourcegraph/templates/embeddings/embeddings.Service.yaml‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
{{- if .Values.embeddings.enabled -}}
2+
apiVersion: v1
3+
kind: Service
4+
metadata:
5+
annotations:
6+
prometheus.io/port: "6060"
7+
sourcegraph.prometheus/scrape: "true"
8+
{{- if .Values.frontend.serviceAnnotations }}
9+
{{- toYaml .Values.frontend.serviceAnnotations | nindent 4 }}
10+
{{- end }}
11+
labels:
12+
app: {{ .Values.embeddings.name }}
13+
deploy: sourcegraph
14+
app.kubernetes.io/component: embeddings
15+
{{- if .Values.sourcegraph.serviceLabels }}
16+
{{- toYaml .Values.sourcegraph.serviceLabels | nindent 4 }}
17+
{{- end }}
18+
name: {{ .Values.embeddings.name }}
19+
spec:
20+
ports:
21+
- name: http
22+
port: 9991
23+
protocol: TCP
24+
targetPort: http
25+
selector:
26+
{{- include "sourcegraph.selectorLabels" . | nindent 4 }}
27+
app: {{ .Values.embeddings.name }}
28+
type: {{ .Values.embeddings.serviceType | default "ClusterIP" }}
29+
{{- end }}

‎charts/sourcegraph/templates/embeddings/embeddings.ServiceAccount.yaml‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
{{- if and .Values.embeddings.enabled .Values.embeddings.serviceAccount.create -}}
2+
apiVersion: v1
3+
kind: ServiceAccount
4+
metadata:
5+
labels:
6+
category: rbac
7+
deploy: sourcegraph
8+
app.kubernetes.io/component: {{ .Values.embeddings.name }}
9+
{{- include "sourcegraph.serviceAccountAnnotations" (list . "embeddings") | trim | nindent 2 }}
10+
name: {{ include "sourcegraph.serviceAccountName" (list . "embeddings") }}
11+
{{- end }}

‎charts/sourcegraph/templates/worker/worker.Deployment.yaml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,14 @@ spec:
6060
- name: PRECISE_CODE_INTEL_UPLOAD_AWS_ENDPOINT
6161
value: http://blobstore:9000
6262
{{- end }}
63+
{{- if and .Values.embeddings.enabled (not .Values.worker.env.EMBEDDINGS_UPLOAD_BACKEND) }}
64+
- name: EMBEDDINGS_UPLOAD_BACKEND
65+
value: blobstore
66+
{{- end }}
67+
{{- if and .Values.embeddings.enabled (not .Values.worker.env.EMBEDDINGS_UPLOAD_AWS_ENDPOINT) }}
68+
- name: EMBEDDINGS_UPLOAD_AWS_ENDPOINT
69+
value: http://blobstore:9000
70+
{{- end }}
6371
- name: POD_NAME
6472
valueFrom:
6573
fieldRef:

‎charts/sourcegraph/values.yaml‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -283,6 +283,46 @@ codeIntelDB:
283283
# -- PVC Storage Request for `codeintel-db` data volume
284284
storageSize: 200Gi
285285

286+
embeddings:
287+
# -- Enable `embeddings`
288+
enabled: false
289+
# -- Name of the `embeddings` service
290+
name: embeddings
291+
image:
292+
# -- Docker image name for the `embeddings` image
293+
name: "embeddings"
294+
# -- Docker image tag for the `embeddings` image
295+
defaultTag: "5.0.3@sha256:1d8f05ed57e361451d92eabbb3a0d90169c1108f48a274d556e43f541190e01a"
296+
# -- Resource requests & limits for the `worker` container,
297+
# learn more from the [Kubernetes documentation](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/)
298+
resources:
299+
limits:
300+
cpu: "8"
301+
memory: 64G
302+
requests:
303+
cpu: "4"
304+
memory: 32G
305+
# -- Environment variables for the `embeddings` container
306+
env: {}
307+
# -- Security context for the `worker` container,
308+
# learn more from the [Kubernetes documentation](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container)
309+
containerSecurityContext:
310+
allowPrivilegeEscalation: false
311+
runAsUser: 100
312+
runAsGroup: 101
313+
readOnlyRootFilesystem: true
314+
# -- Security context for the `embeddings` container,
315+
# learn more from the [Kubernetes documentation](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container)
316+
podSecurityContext: {}
317+
serviceAccount:
318+
# -- Enable creation of ServiceAccount for `embeddings`
319+
create: false
320+
# -- Name of the ServiceAccount to be created or an existing ServiceAccount
321+
name: ""
322+
annotations: {}
323+
extraVolumeMounts: {}
324+
extraVolumes: {}
325+
286326
frontend:
287327
# -- Environment variables for the `frontend` container
288328
# @default -- the chart will add some default environment values
@@ -1240,6 +1280,8 @@ worker:
12401280
readOnlyRootFilesystem: true
12411281
# -- Name used by resources. Does not affect service names or PVCs.
12421282
name: "worker"
1283+
# -- Environment variables for the `worker` container
1284+
env: {}
12431285
# -- Security context for the `worker` pod,
12441286
# learn more from the [Kubernetes documentation](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod)
12451287
podSecurityContext: {}

0 commit comments

Comments
 (0)