Skip to content

Commit 307200f

Browse files
susilnemsusilnem
authored
Merge pull request #2623 from IFRCGo/feature/eap-export-pdf-generation
2 parents 8b87dc0 + e76a49b commit 307200f

File tree

16 files changed

+1075
-254
lines changed

16 files changed

+1075
-254
lines changed

‎api/playwright.py‎

Lines changed: 121 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,121 @@
1+
import json
2+
import pathlib
3+
import tempfile
4+
import time
5+
6+
from django.conf import settings
7+
from django.core.files.base import ContentFile
8+
from playwright.sync_api import sync_playwright
9+
10+
from .utils import DebugPlaywright
11+
12+
footer_template = """
13+
<div class="footer" style="width: 100%;font-size: 8px;color: #FEFEFE; bottom: 10px; position: absolute;">
14+
<div style="float: left; margin-top: 10px; margin-left: 40px;">
15+
Page <span class="pageNumber"></span> / <span class="totalPages"></span>
16+
</div>
17+
<div style="float: right; margin-right: 40px;">
18+
<svg
19+
xmlns="http://www.w3.org/2000/svg"
20+
viewBox="0 0 89.652 89.654"
21+
height="48"
22+
width="48"
23+
>
24+
<path
25+
d="M50.284 18.637a5.14 5.14 0 00-5.136-5.135 5.139 5.139 0 00-5.135 5.135 5.141 5.141 0 005.135 5.138 5.146 5.146 0 005.136-5.138M28.416 63.032a5.143 5.143 0 00-5.138 5.138 5.14 5.14 0 005.138 5.133 5.14 5.14 0 005.136-5.133 5.143 5.143 0 00-5.136-5.138M45.151 34.057a7.021 7.021 0 00-7.02 7.025 7.02 7.02 0 0014.04 0 7.021 7.021 0 00-7.02-7.025M61.883 63.032a5.143 5.143 0 00-5.135 5.138 5.138 5.138 0 005.135 5.133 5.14 5.14 0 005.136-5.133 5.143 5.143 0 00-5.136-5.138"
26+
class="st1"
27+
fill="#F5333F"
28+
/>
29+
<path
30+
d="M61.883 75.769c-4.19 0-7.601-3.41-7.601-7.602 0-2.32 1.05-4.4 2.696-5.794L49.726 50.26a10.205 10.205 0 01-4.575 1.085c-1.648 0-3.196-.397-4.577-1.085l-7.252 12.113a7.571 7.571 0 012.693 5.794c0 4.191-3.408 7.602-7.599 7.602-4.19 0-7.601-3.41-7.601-7.602 0-4.19 3.41-7.601 7.601-7.601.984 0 1.926.196 2.791.54l7.303-12.2a10.236 10.236 0 01-3.63-7.827c0-5.254 3.947-9.58 9.038-10.189v-4.762c-3.606-.59-6.368-3.72-6.368-7.49 0-4.192 3.41-7.602 7.601-7.602s7.599 3.41 7.599 7.601c0 3.77-2.762 6.9-6.366 7.49v4.763c5.093.611 9.038 4.935 9.038 10.19a10.23 10.23 0 01-3.633 7.826l7.306 12.2a7.544 7.544 0 012.791-.54c4.191 0 7.599 3.41 7.599 7.601s-3.41 7.602-7.602 7.602m-49.286-34.65c0-5.485 3.44-10.057 9.194-10.057 4.194 0 7.715 2.236 8.226 6.562h-3.281c-.32-2.524-2.524-3.818-4.945-3.818-4.117 0-5.834 3.627-5.834 7.313s1.717 7.313 5.834 7.313c3.44.056 5.32-2.016 5.376-5.268h-5.106v-2.556h8.173v10.11h-2.151l-.51-2.257c-1.803 2.043-3.44 2.715-5.78 2.715-5.754 0-9.196-4.57-9.196-10.057M44.826 0C20.07 0 0 20.069 0 44.828c0 24.755 20.071 44.826 44.826 44.826 24.757 0 44.826-20.071 44.826-44.826C89.652 20.068 69.582 0 44.826 0"
31+
class="st1"
32+
fill="#F5333F"
33+
/>
34+
</svg>
35+
</div>
36+
</div>
37+
""" # noqa
38+
39+
40+
def build_storage_state(tmp_dir, user, token, language="en"):
41+
temp_file = pathlib.Path(tmp_dir, "storage_state.json")
42+
temp_file.touch()
43+
44+
state = {
45+
"origins": [
46+
{
47+
"origin": settings.GO_WEB_INTERNAL_URL + "/",
48+
"localStorage": [
49+
{
50+
"name": "user",
51+
"value": json.dumps(
52+
{
53+
"id": user.id,
54+
"username": user.username,
55+
"firstName": user.first_name,
56+
"lastName": user.last_name,
57+
"token": token.key,
58+
}
59+
),
60+
},
61+
{"name": "language", "value": json.dumps(language)},
62+
],
63+
}
64+
]
65+
}
66+
with open(temp_file, "w") as f:
67+
json.dump(state, f)
68+
return temp_file
69+
70+
71+
def render_pdf_from_url(
72+
*,
73+
url: str,
74+
user,
75+
token,
76+
language: str = "en",
77+
timeout: int = 300_000,
78+
):
79+
"""
80+
Renders a URL to PDF using Playwright.
81+
Returns a Django ContentFile.
82+
"""
83+
with tempfile.TemporaryDirectory() as tmp_dir:
84+
storage_state = build_storage_state(
85+
tmp_dir=tmp_dir,
86+
user=user,
87+
token=token,
88+
language=language,
89+
)
90+
91+
with sync_playwright() as playwright:
92+
browser = playwright.chromium.connect(settings.PLAYWRIGHT_SERVER_URL)
93+
94+
try:
95+
context = browser.new_context(storage_state=storage_state)
96+
page = context.new_page()
97+
98+
if settings.DEBUG_PLAYWRIGHT:
99+
DebugPlaywright.debug(page)
100+
101+
page.goto(url, timeout=timeout)
102+
time.sleep(5)
103+
# NOTE: Use wait_for_load_state instead of sleep?
104+
# page.wait_for_load_state("networkidle", timeout=timeout)
105+
page.wait_for_selector(
106+
"#pdf-preview-ready",
107+
state="attached",
108+
timeout=timeout,
109+
)
110+
111+
pdf_bytes = page.pdf(
112+
display_header_footer=True,
113+
prefer_css_page_size=True,
114+
print_background=True,
115+
footer_template=footer_template,
116+
header_template="<p></p>",
117+
)
118+
finally:
119+
browser.close()
120+
121+
return ContentFile(pdf_bytes)

‎api/serializers.py‎

Lines changed: 17 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,8 @@
1111
from rest_framework import serializers
1212

1313
# from api.utils import pdf_exporter
14-
from api.tasks import generate_url
15-
from api.utils import CountryValidator, RegionValidator
14+
from api.tasks import generate_export_pdf
15+
from api.utils import CountryValidator, RegionValidator, generate_eap_export_url
1616
from deployments.models import EmergencyProject, Personnel, PersonnelDeployment
1717
from dref.models import Dref, DrefFinalReport, DrefOperationalUpdate
1818
from eap.models import EAPRegistration, FullEAP, SimplifiedEAP
@@ -2546,9 +2546,11 @@ class ExportSerializer(serializers.ModelSerializer):
25462546
is_pga = serializers.BooleanField(default=False, required=False, write_only=True)
25472547
# NOTE: diff is used to determine if the export is requested for diff view or not
25482548
# Currently only used for EAP exports
2549-
diff = serializers.BooleanField(default=False, required=False, write_only=True)
2549+
diff = serializers.BooleanField(default=False, required=False, write_only=True, help_text="Only applicable for EAP exports")
25502550
# NOTE: Version of a EAP export being requested, only applicable for full and simplified EAP exports
2551-
version = serializers.IntegerField(required=False, write_only=True)
2551+
version = serializers.IntegerField(required=False, write_only=True, help_text="Only applicable for EAP exports")
2552+
# NOTE: Only for FUll eap export
2553+
summary = serializers.BooleanField(default=False, required=False, write_only=True, help_text="Only applicable for FUll EAP")
25522554

25532555
class Meta:
25542556
model = Export
@@ -2560,11 +2562,12 @@ def validate_pdf_file(self, pdf_file):
25602562
return pdf_file
25612563

25622564
def create(self, validated_data):
2563-
language = django_get_language()
25642565
export_id = validated_data.get("export_id")
25652566
export_type = validated_data.get("export_type")
25662567
country_id = validated_data.get("per_country")
25672568
version = validated_data.pop("version", None)
2569+
diff = validated_data.pop("diff", False)
2570+
summary = validated_data.pop("summary", False)
25682571
if export_type == Export.ExportType.DREF:
25692572
title = Dref.objects.filter(id=export_id).first().title
25702573
elif export_type == Export.ExportType.OPS_UPDATE:
@@ -2623,19 +2626,18 @@ def create(self, validated_data):
26232626
Export.ExportType.SIMPLIFIED_EAP,
26242627
Export.ExportType.FULL_EAP,
26252628
]:
2626-
validated_data["url"] = f"{settings.GO_WEB_INTERNAL_URL}/eap/{export_id}/export/"
2627-
# NOTE: EAP exports with diff view only for EAPs exports
2628-
if version:
2629-
validated_data["url"] += f"?version={version}"
2630-
diff = validated_data.pop("diff")
2631-
if diff:
2632-
validated_data["url"] += "&diff=true" if version else "?diff=true"
2629+
validated_data["url"] = generate_eap_export_url(
2630+
registration_id=export_id,
2631+
version=version,
2632+
diff=diff,
2633+
summary=summary,
2634+
)
26332635

26342636
else:
26352637
validated_data["url"] = f"{settings.GO_WEB_INTERNAL_URL}/{export_type}/{export_id}/export/"
26362638

26372639
# Adding is_pga to the url
2638-
is_pga = validated_data.pop("is_pga")
2640+
is_pga = validated_data.pop("is_pga", False)
26392641
if is_pga:
26402642
validated_data["url"] += "?is_pga=true"
26412643
validated_data["requested_by"] = user
@@ -2645,7 +2647,8 @@ def create(self, validated_data):
26452647
export.requested_at = timezone.now()
26462648
export.save(update_fields=["status", "requested_at"])
26472649

2648-
transaction.on_commit(lambda: generate_url.delay(export.url, export.id, user.id, title, language))
2650+
language = django_get_language()
2651+
transaction.on_commit(lambda: generate_export_pdf.delay(export.id, title, language))
26492652
return export
26502653

26512654
def update(self, instance, validated_data):

‎api/tasks.py‎

Lines changed: 29 additions & 122 deletions
Original file line numberDiff line numberDiff line change
@@ -1,148 +1,55 @@
1-
import json
2-
import pathlib
3-
import tempfile
4-
import time
51
from datetime import datetime
62

73
from celery import shared_task
8-
from django.conf import settings
94
from django.contrib.auth.models import User
10-
from django.core.files.base import ContentFile
115
from django.utils import timezone
12-
from playwright.sync_api import sync_playwright
136
from rest_framework.authtoken.models import Token
147

8+
from api.playwright import render_pdf_from_url
159
from main.utils import logger_context
1610

1711
from .logger import logger
1812
from .models import Export
19-
from .utils import DebugPlaywright
2013

2114

22-
def build_storage_state(tmp_dir, user, token, language="en"):
23-
temp_file = pathlib.Path(tmp_dir, "storage_state.json")
24-
temp_file.touch()
15+
def build_export_filename(export: Export, title: str) -> str:
16+
timestamp = datetime.now().strftime("%Y-%m-%d %H-%M-%S")
2517

26-
state = {
27-
"origins": [
28-
{
29-
"origin": settings.GO_WEB_INTERNAL_URL + "/",
30-
"localStorage": [
31-
{
32-
"name": "user",
33-
"value": json.dumps(
34-
{
35-
"id": user.id,
36-
"username": user.username,
37-
"firstName": user.first_name,
38-
"lastName": user.last_name,
39-
"token": token.key,
40-
}
41-
),
42-
},
43-
{"name": "language", "value": json.dumps(language)},
44-
],
45-
}
46-
]
18+
prefix_map = {
19+
Export.ExportType.PER: "PER",
20+
Export.ExportType.SIMPLIFIED_EAP: "SIMPLIFIED EAP",
21+
Export.ExportType.FULL_EAP: "FULL EAP",
4722
}
48-
with open(temp_file, "w") as f:
49-
json.dump(state, f)
50-
return temp_file
23+
24+
prefix = prefix_map.get(export.export_type, "DREF")
25+
return f"{prefix} {title} ({timestamp}).pdf"
5126

5227

5328
@shared_task
54-
def generate_url(url, export_id, user, title, language):
29+
def generate_export_pdf(export_id, title, set_user_language="en"):
5530
export = Export.objects.get(id=export_id)
56-
user = User.objects.get(id=user)
31+
user = User.objects.get(id=export.requested_by.id)
5732
token = Token.objects.filter(user=user).last()
5833
logger.info(f"Starting export: {export.pk}")
5934

60-
footer_template = """
61-
<div class="footer" style="width: 100%;font-size: 8px;color: #FEFEFE; bottom: 10px; position: absolute;">
62-
<div style="float: left; margin-top: 10px; margin-left: 40px;">
63-
Page <span class="pageNumber"></span> / <span class="totalPages"></span>
64-
</div>
65-
<div style="float: right; margin-right: 40px;">
66-
<svg
67-
xmlns="http://www.w3.org/2000/svg"
68-
viewBox="0 0 89.652 89.654"
69-
height="48"
70-
width="48"
71-
>
72-
<path
73-
d="M50.284 18.637a5.14 5.14 0 00-5.136-5.135 5.139 5.139 0 00-5.135 5.135 5.141 5.141 0 005.135 5.138 5.146 5.146 0 005.136-5.138M28.416 63.032a5.143 5.143 0 00-5.138 5.138 5.14 5.14 0 005.138 5.133 5.14 5.14 0 005.136-5.133 5.143 5.143 0 00-5.136-5.138M45.151 34.057a7.021 7.021 0 00-7.02 7.025 7.02 7.02 0 0014.04 0 7.021 7.021 0 00-7.02-7.025M61.883 63.032a5.143 5.143 0 00-5.135 5.138 5.138 5.138 0 005.135 5.133 5.14 5.14 0 005.136-5.133 5.143 5.143 0 00-5.136-5.138"
74-
class="st1"
75-
fill="#F5333F"
76-
/>
77-
<path
78-
d="M61.883 75.769c-4.19 0-7.601-3.41-7.601-7.602 0-2.32 1.05-4.4 2.696-5.794L49.726 50.26a10.205 10.205 0 01-4.575 1.085c-1.648 0-3.196-.397-4.577-1.085l-7.252 12.113a7.571 7.571 0 012.693 5.794c0 4.191-3.408 7.602-7.599 7.602-4.19 0-7.601-3.41-7.601-7.602 0-4.19 3.41-7.601 7.601-7.601.984 0 1.926.196 2.791.54l7.303-12.2a10.236 10.236 0 01-3.63-7.827c0-5.254 3.947-9.58 9.038-10.189v-4.762c-3.606-.59-6.368-3.72-6.368-7.49 0-4.192 3.41-7.602 7.601-7.602s7.599 3.41 7.599 7.601c0 3.77-2.762 6.9-6.366 7.49v4.763c5.093.611 9.038 4.935 9.038 10.19a10.23 10.23 0 01-3.633 7.826l7.306 12.2a7.544 7.544 0 012.791-.54c4.191 0 7.599 3.41 7.599 7.601s-3.41 7.602-7.602 7.602m-49.286-34.65c0-5.485 3.44-10.057 9.194-10.057 4.194 0 7.715 2.236 8.226 6.562h-3.281c-.32-2.524-2.524-3.818-4.945-3.818-4.117 0-5.834 3.627-5.834 7.313s1.717 7.313 5.834 7.313c3.44.056 5.32-2.016 5.376-5.268h-5.106v-2.556h8.173v10.11h-2.151l-.51-2.257c-1.803 2.043-3.44 2.715-5.78 2.715-5.754 0-9.196-4.57-9.196-10.057M44.826 0C20.07 0 0 20.069 0 44.828c0 24.755 20.071 44.826 44.826 44.826 24.757 0 44.826-20.071 44.826-44.826C89.652 20.068 69.582 0 44.826 0"
79-
class="st1"
80-
fill="#F5333F"
81-
/>
82-
</svg>
83-
</div>
84-
</div>
85-
""" # noqa: E501
86-
8735
try:
88-
with tempfile.TemporaryDirectory() as tmp_dir:
89-
with sync_playwright() as p:
90-
browser = p.chromium.connect(settings.PLAYWRIGHT_SERVER_URL)
91-
# NOTE: DREF Export use the language from request
92-
if export.export_type in [
93-
Export.ExportType.DREF,
94-
Export.ExportType.OPS_UPDATE,
95-
Export.ExportType.FINAL_REPORT,
96-
]:
97-
storage_state = build_storage_state(
98-
tmp_dir,
99-
user,
100-
token,
101-
language,
102-
)
103-
else:
104-
# NOTE: Other Export types use default language (en)
105-
storage_state = build_storage_state(
106-
tmp_dir,
107-
user,
108-
token,
109-
)
110-
context = browser.new_context(storage_state=storage_state)
111-
page = context.new_page()
112-
if settings.DEBUG_PLAYWRIGHT:
113-
DebugPlaywright.debug(page)
114-
# FIXME: Use of Timeout correct?
115-
timeout = 300_000 # 5 min
116-
page.goto(url, timeout=timeout)
117-
time.sleep(5)
118-
page.wait_for_selector("#pdf-preview-ready", state="attached", timeout=timeout)
119-
if export.export_type == Export.ExportType.PER:
120-
file_name = f'PER {title} ({datetime.now().strftime("%Y-%m-%d %H-%M-%S")}).pdf'
121-
elif export.export_type == Export.ExportType.SIMPLIFIED_EAP:
122-
file_name = f'SIMPLIFIED EAP {title} ({datetime.now().strftime("%Y-%m-%d %H-%M-%S")}).pdf'
123-
elif export.export_type == Export.ExportType.FULL_EAP:
124-
file_name = f'FULL EAP {title} ({datetime.now().strftime("%Y-%m-%d %H-%M-%S")}).pdf'
125-
else:
126-
file_name = f'DREF {title} ({datetime.now().strftime("%Y-%m-%d %H-%M-%S")}).pdf'
127-
file = ContentFile(
128-
page.pdf(
129-
display_header_footer=True,
130-
prefer_css_page_size=True,
131-
print_background=True,
132-
footer_template=footer_template,
133-
header_template="<p></p>",
134-
)
135-
)
136-
browser.close()
137-
export.pdf_file.save(file_name, file)
138-
export.status = Export.ExportStatus.COMPLETED
139-
export.completed_at = timezone.now()
140-
export.save(
141-
update_fields=[
142-
"status",
143-
"completed_at",
144-
]
145-
)
36+
file = render_pdf_from_url(
37+
url=export.url,
38+
user=user,
39+
token=token,
40+
language=set_user_language,
41+
)
42+
43+
file_name = build_export_filename(export, title)
44+
export.pdf_file.save(file_name, file)
45+
export.status = Export.ExportStatus.COMPLETED
46+
export.completed_at = timezone.now()
47+
export.save(
48+
update_fields=[
49+
"status",
50+
"completed_at",
51+
]
52+
)
14653
except Exception:
14754
logger.error(
14855
f"Failed to export PDF: {export.export_type}",

0 commit comments

Comments
 (0)