Skip to content

Commit 9818984

Browse files
szabozoltan69szabozoltan69
authored
Merge pull request #2426 from IFRCGo/feature/solo-playwright
Use seperate container for running playwright
2 parents 0e9c6b4 + 4671474 commit 9818984

File tree

15 files changed

+162
-99
lines changed

15 files changed

+162
-99
lines changed

‎.github/docker-compose.yml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
services:
2+
serve:
3+
image: $DOCKER_IMAGE
4+
build: !reset null
5+
env_file: !reset null
6+
environment:
7+
CI: "true"
8+
DJANGO_SECRET_KEY: RANDOM-STRING-FOR-SECRET-KEYS

‎.github/workflows/build-publish-docker-helm.yml‎

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -80,10 +80,8 @@ jobs:
8080
PACKAGE_FILE=$(ls .helm-charts/*.tgz | head -n 1)
8181
echo "# Helm Chart" >> $GITHUB_STEP_SUMMARY
8282
echo "" >> $GITHUB_STEP_SUMMARY
83-
echo "Tagged Image: **$IMAGE**" >> $GITHUB_STEP_SUMMARY
84-
echo "" >> $GITHUB_STEP_SUMMARY
85-
echo "Helm push output" >> $GITHUB_STEP_SUMMARY
86-
echo "" >> $GITHUB_STEP_SUMMARY
87-
echo '```bash' >> $GITHUB_STEP_SUMMARY
83+
echo '```yaml' >> $GITHUB_STEP_SUMMARY
8884
helm push "$PACKAGE_FILE" $OCI_REPO 2>> $GITHUB_STEP_SUMMARY
8985
echo '```' >> $GITHUB_STEP_SUMMARY
86+
echo "> [!Important]" >> $GITHUB_STEP_SUMMARY
87+
echo "> NOTE: Tagged docker image: **$IMAGE**" >> $GITHUB_STEP_SUMMARY

‎.github/workflows/ci.yml‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@ on:
1919
pull_request:
2020
# NOTE: For develop & master, they are run through helm github action ./build-publish-docker-helm.yml
2121

22+
env:
23+
COMPOSE_FILE: docker-compose.yml:.github/docker-compose.yml
2224

2325
jobs:
2426
pre_commit_checks:
@@ -103,28 +105,38 @@ jobs:
103105
cache-to: type=gha,mode=max
104106

105107
- name: Docker config setup & Pull docker images
108+
env:
109+
DOCKER_IMAGE: ${{ steps.prep.outputs.tagged_image }}
106110
run: |
107-
cp .env-sample .env &&
111+
touch .env
108112
docker compose run --rm serve ls
109113
110114
- name: 🕮 Validate if there are no pending django migrations.
115+
env:
116+
DOCKER_IMAGE: ${{ steps.prep.outputs.tagged_image }}
111117
run: |
112118
docker compose run --rm serve ./manage.py makemigrations --check --dry-run || {
113119
echo 'There are some changes to be reflected in the migration. Make sure to run makemigrations';
114120
exit 1;
115121
}
116122
117123
- name: 🕮 Validate SentryMonitor config
124+
env:
125+
DOCKER_IMAGE: ${{ steps.prep.outputs.tagged_image }}
118126
run: |
119127
docker compose run --rm serve ./manage.py cron_job_monitor --validate-only || {
120128
echo 'There are some changes to be reflected in the SentryMonitor. Make sure to update SentryMonitor';
121129
exit 1;
122130
}
123131
124132
- name: Run django migrations
133+
env:
134+
DOCKER_IMAGE: ${{ steps.prep.outputs.tagged_image }}
125135
run: docker compose run --rm serve ./manage.py test --keepdb -v 2 --pattern="test_fake.py"
126136

127137
- name: 🤞 Run Test 🧪
138+
env:
139+
DOCKER_IMAGE: ${{ steps.prep.outputs.tagged_image }}
128140
run: docker compose run --rm serve pytest --reuse-db --durations=10
129141

130142
- name: 🐳 Docker push

‎Dockerfile‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -33,9 +33,6 @@ RUN --mount=type=cache,target=$UV_CACHE_DIR \
3333
--mount=type=bind,source=pyproject.toml,target=pyproject.toml \
3434
uv sync --frozen --no-install-project --all-groups
3535

36-
RUN playwright install \
37-
&& playwright install-deps
38-
3936
# To avoid some SyntaxWarnings ("is" with a literal), still needed on 20241024:
4037
ENV AZUREROOT=/usr/local/lib/python3.11/site-packages/azure/storage/
4138
RUN perl -pi -e 's/ is 0 / == 0 /' ${AZUREROOT}blob/_upload_chunking.py

‎api/tasks.py‎

Lines changed: 5 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@ def generate_url(url, export_id, user, title):
5555
export = Export.objects.get(id=export_id)
5656
user = User.objects.get(id=user)
5757
token = Token.objects.filter(user=user).last()
58+
logger.info(f"Starting export: {export.pk}")
5859

5960
footer_template = """
6061
<div class="footer" style="width: 100%;font-size: 8px;color: #FEFEFE; bottom: 10px; position: absolute;">
@@ -86,19 +87,7 @@ def generate_url(url, export_id, user, title):
8687
try:
8788
with tempfile.TemporaryDirectory() as tmp_dir:
8889
with sync_playwright() as p:
89-
browser = p.chromium.launch(
90-
headless=True,
91-
args=[
92-
# https://github.com/microsoft/playwright-python/issues/1453
93-
# Usually required when running AWS lambda
94-
# "--single-process",
95-
"--no-zygote",
96-
"--no-sandbox",
97-
"--disable-setuid-sandbox",
98-
"--disable-dev-shm-usage",
99-
],
100-
devtools=False,
101-
)
90+
browser = p.chromium.connect(settings.PLAYWRIGHT_SERVER_URL)
10291
storage_state = build_storage_state(
10392
tmp_dir,
10493
user,
@@ -108,7 +97,8 @@ def generate_url(url, export_id, user, title):
10897
page = context.new_page()
10998
if settings.DEBUG_PLAYWRIGHT:
11099
DebugPlaywright.debug(page)
111-
timeout = 300000
100+
# FIXME: Use of Timeout correct?
101+
timeout = 300_000 # 5 min
112102
page.goto(url, timeout=timeout)
113103
time.sleep(5)
114104
page.wait_for_selector("#pdf-preview-ready", state="attached", timeout=timeout)
@@ -143,3 +133,4 @@ def generate_url(url, export_id, user, title):
143133
)
144134
export.status = Export.ExportStatus.ERRORED
145135
export.save(update_fields=["status"])
136+
logger.info(f"End export: {export.pk}")

‎deploy/helm/ifrcgo-helm/templates/config/configmap.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,12 @@ data:
2222
AZURE_STORAGE_ENABLED: "true"
2323
{{- end }}
2424

25+
{{- if .Values.playwright.enabled }}
26+
PLAYWRIGHT_SERVER_URL: "ws://{{ template "ifrcgo-helm.fullname" . }}-playwright:{{ .Values.playwright.containerPort }}/"
27+
{{- else }}
28+
PLAYWRIGHT_SERVER_URL: {{ required "env.PLAYWRIGHT_SERVER_URL" .Values.env.PLAYWRIGHT_SERVER_URL | quote }}
29+
{{- end }}
30+
2531
CACHE_MIDDLEWARE_SECONDS: {{ .Values.env.CACHE_MIDDLEWARE_SECONDS | quote }}
2632
DJANGO_DEBUG: {{ .Values.env.DJANGO_DEBUG | quote }}
2733
ELASTIC_SEARCH_HOST: {{ default (printf "elasticsearch://%s-elasticsearch:9200" (include "ifrcgo-helm.fullname" .)) .Values.env.ELASTIC_SEARCH_HOST | quote }}

‎deploy/helm/ifrcgo-helm/templates/playwright/deployment.yaml‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
{{- if .Values.playwright.enabled }}
2+
3+
apiVersion: apps/v1
4+
kind: Deployment
5+
metadata:
6+
name: {{ template "ifrcgo-helm.fullname" . }}-playwright
7+
labels:
8+
component: playwright-deployment
9+
environment: {{ .Values.environment }}
10+
release: {{ .Release.Name }}
11+
spec:
12+
replicas: {{ .Values.playwright.replicaCount }}
13+
selector:
14+
matchLabels:
15+
app: {{ template "ifrcgo-helm.name" . }}
16+
release: {{ .Release.Name }}
17+
run: {{ .Release.Name }}-playwright
18+
template:
19+
metadata:
20+
labels:
21+
app: {{ template "ifrcgo-helm.name" . }}
22+
release: {{ .Release.Name }}
23+
run: {{ .Release.Name }}-playwright
24+
spec:
25+
containers:
26+
- name: playwright
27+
image: "{{ .Values.playwright.image.name }}:{{ .Values.playwright.image.tag }}"
28+
workingDir: /home/pwuser
29+
securityContext:
30+
runAsUser: 1001 # pwuser
31+
runAsNonRoot: true
32+
command:
33+
- "bash"
34+
- "-xc"
35+
- |
36+
PLAYWRIGHT_VERSION=$(cat /ms-playwright/.docker-info | grep -oP '"driverVersion": "\K[^"]+')
37+
npx -y playwright@$$PLAYWRIGHT_VERSION run-server --host 0.0.0.0 --port {{ .Values.playwright.containerPort }}
38+
ports:
39+
- name: http
40+
containerPort: {{ .Values.playwright.containerPort }}
41+
protocol: TCP
42+
# livenessProbe: # TODO:
43+
# httpGet:
44+
# path: /
45+
# port: {{ .Values.playwright.containerPort }}
46+
# initialDelaySeconds: 10180
47+
# periodSeconds: 5
48+
resources:
49+
requests:
50+
cpu: {{ .Values.playwright.resources.requests.cpu }}
51+
memory: {{ .Values.playwright.resources.requests.memory }}
52+
limits:
53+
cpu: {{ .Values.playwright.resources.limits.cpu }}
54+
memory: {{ .Values.playwright.resources.limits.memory }}
55+
56+
{{- end }}

‎deploy/helm/ifrcgo-helm/templates/playwright/service.yaml‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
{{- if .Values.playwright.enabled -}}
2+
3+
apiVersion: v1
4+
kind: Service
5+
metadata:
6+
name: {{ template "ifrcgo-helm.fullname" . }}-playwright
7+
labels:
8+
app: {{ template "ifrcgo-helm.name" . }}
9+
component: playwright-service
10+
environment: {{ .Values.environment }}
11+
release: {{ .Release.Name }}
12+
spec:
13+
type: ClusterIP
14+
ports:
15+
- protocol: TCP
16+
port: {{ .Values.playwright.containerPort }}
17+
targetPort: {{ .Values.playwright.containerPort }}
18+
nodePort: null
19+
selector:
20+
app: {{ template "ifrcgo-helm.name" . }}
21+
release: {{ .Release.Name }}
22+
run: {{ .Release.Name }}-playwright
23+
24+
{{- end }}

‎deploy/helm/ifrcgo-helm/values.yaml‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -149,6 +149,23 @@ postgresql:
149149
enabled: true
150150
size: 8Gi
151151

152+
playwright:
153+
enabled: rue
154+
replicaCount: 1
155+
containerPort: 3000
156+
image:
157+
# NOTE: Make sure this matches with pyproject playwright dependency and root docker-compose
158+
name: 'mcr.microsoft.com/playwright'
159+
tag: 'v1.50.0-noble'
160+
pullPolicy: 'IfNotPresent'
161+
resources:
162+
requests:
163+
cpu: "0.1"
164+
memory: 1Gi
165+
limits:
166+
cpu: "2"
167+
memory: 2Gi
168+
152169
api:
153170
domain: "go-staging.ifrc.org"
154171
tls:

‎docker-compose.yml‎

Lines changed: 19 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,10 @@
11
# NOTE: Used Only for local development
22

33
x-server: &base_server_setup
4-
image: ifrcgo/go-api:uv-latest
5-
build: .
4+
build:
5+
context: .
6+
tags:
7+
- ifrcgo/go-api:latest
68
# To attach to container with stdin `docker attach <container_name>`
79
# Used for python debugging.
810
stdin_open: true
@@ -19,6 +21,7 @@ x-server: &base_server_setup
1921
GO_ENVIRONMENT: ${GO_ENVIRONMENT:-development}
2022
API_FQDN: ${API_FQDN:-http://localhost:8000}
2123
FRONTEND_URL: ${FRONTEND_URL:-http://localhost:3000}
24+
PLAYWRIGHT_SERVER_URL: ${PLAYWRIGHT_SERVER_URL:-ws://playwright:3000/}
2225
GO_WEB_INTERNAL_URL: ${GO_WEB_INTERNAL_URL:-http://host.docker.internal:3000}
2326
DJANGO_ADDITIONAL_ALLOWED_HOSTS: ${DJANGO_ADDITIONAL_ALLOWED_HOSTS:-host.docker.internal}
2427
DEBUG_EMAIL: ${DEBUG_EMAIL:-true}
@@ -57,7 +60,7 @@ x-server: &base_server_setup
5760
- db
5861
- redis
5962
- elasticsearch
60-
63+
- playwright
6164

6265
services:
6366
db:
@@ -76,7 +79,19 @@ services:
7679
volumes:
7780
- redis-data:/data
7881

79-
# NOTE: Used Only for local development
82+
# NOTE: Make sure this matches with pyproject playwright dependency and helm
83+
playwright:
84+
image: mcr.microsoft.com/playwright:v1.50.0-noble
85+
working_dir: /home/pwuser
86+
user: pwuser
87+
command: >
88+
bash -xc "
89+
PLAYWRIGHT_VERSION=$(cat /ms-playwright/.docker-info | grep -oP '\"driverVersion\": \"\\K[^\"]+')
90+
npx -y playwright@$$PLAYWRIGHT_VERSION run-server --port 3000 --host 0.0.0.0
91+
"
92+
extra_hosts:
93+
- "host.docker.internal:host-gateway"
94+
8095
elasticsearch:
8196
image: docker.elastic.co/elasticsearch/elasticsearch:7.0.0
8297
container_name: elasticsearch
@@ -96,7 +111,6 @@ services:
96111
ports:
97112
- 9200:9200
98113

99-
# NOTE: Used Only for local development
100114
kibana:
101115
image: 'docker.elastic.co/kibana/kibana:7.0.0'
102116
container_name: kibana

0 commit comments

Comments
 (0)