Remove pypdf for generating pdfs

Aptfile

@@ -1,3 +1,2 @@
 libjpeg62
 libc6
-https://mirrors.edge.kernel.org/ubuntu/pool/main/o/openssl1.1/libssl1.1_1.1.1f-1ubuntu2_amd64.deb

requirements.txt

@@ -13,7 +13,7 @@ MarkupSafe==2.1.2
 phonenumbers==8.13.6
 pydantic[email]==1.9.2
 python-multipart==0.0.6
-python-pdf==0.39
+weasyprint==65.1
 requests==2.28.2
 starlette==0.14.2
 sentry-sdk==1.16.0

src/worker/email.py

@@ -14,8 +14,8 @@
 from httpx import ConnectError, ReadTimeout
 from itertools import chain
 from pathlib import Path
-from pydf import generate_pdf
 from typing import List, Optional
+from weasyprint import HTML
 
 from src.ext import ApiError
 from src.render import EmailInfo, MessageDef, render_email
@@ -36,6 +36,55 @@
 email_retrying = [5, 10, 60, 600, 1800, 3600, 12 * 3600]
 
 
+# Add a simple url_fetcher for debugging
+def logging_url_fetcher(url, timeout=10, ssl_context=None):
+    from weasyprint.urls import default_url_fetcher
+    main_logger.info(f"WeasyPrint is attempting to fetch URL: {url}")
+    try:
+        result = default_url_fetcher(url, timeout=timeout, ssl_context=ssl_context)
+        main_logger.info(f"Successfully fetched {url}, content type: {result.get('mime_type')}")
+        return result
+    except Exception as e:
+        main_logger.error(f"Failed to fetch {url}: {e}", exc_info=True)
+        # Propagate the error to see it in PDF generation or logs
+        raise
+
+
+def generate_pdf_from_html(
+    html: str, 
+    page_size: str = 'A4', 
+    zoom: str = '1.0', 
+    margin_left: str = '10mm', 
+    margin_right: str = '10mm',
+    base_url_for_html: Optional[str] = None 
+) -> bytes:
+    from weasyprint import CSS
+
+    page_css = f"""
+    @page {{
+        size: {page_size};
+        margin-left: {margin_left};
+        margin-right: {margin_right};
+    }}
+    body {{
+        zoom: {zoom};
+    }}
+    """
+
+    html_doc = HTML(
+        string=html, 
+        base_url=base_url_for_html, 
+        url_fetcher=logging_url_fetcher
+    )
+    css_doc = CSS(string=page_css)
+
+    pdf_bytes = html_doc.write_pdf(
+        stylesheets=[css_doc],
+        presentational_hints=True,
+    )
+    return pdf_bytes
+
+
 def utcnow():
     return datetime.utcnow().replace(tzinfo=timezone.utc)
 
@@ -201,13 +250,22 @@ async def _render_email(self, context, headers) -> Optional[EmailInfo]:
             await self._store_email_failed(MessageStatus.render_failed, f'Error rendering email: {e}')
 
     async def _generate_base64_pdf(self, pdf_attachments):
-        kwargs = dict(page_size='A4', zoom='1.25', margin_left='8mm', margin_right='8mm')
         for a in pdf_attachments:
             if a.html:
                 try:
-                    pdf_content = generate_pdf(a.html, **kwargs)
-                except RuntimeError as e:
-                    main_logger.warning('error generating pdf, data: %s', e)
+                    # Assuming URLs in a.html (like {{ bootstrap_url }}) are absolute,
+                    # base_url_for_html can remain None.
+                    # If they could be relative, a proper base_url would be needed here.
+                    pdf_content = generate_pdf_from_html(
+                        a.html, 
+                        page_size='A4', 
+                        zoom='1.25', 
+                        margin_left='8mm', 
+                        margin_right='8mm',
+                        base_url_for_html=None 
+                    )
+                except Exception as e:
+                    main_logger.warning('error generating pdf, data: %s', e, exc_info=True)
                 else:
                     yield dict(type='application/pdf', name=a.name, content=base64.b64encode(pdf_content).decode())
 

tests/test_email.py

@@ -475,26 +475,26 @@ def test_invalid_mustache_body(send_email, sync_db: SyncDb):
     assert m['body'] == 'Error rendering email: unclosed tag at line 1'
 
 
-# def test_send_with_pdf(send_email, tmpdir, sync_db: SyncDb):
-#     message_id = send_email(
-#         recipients=[
-#             {
-#                 'address': '[email protected]',
-#                 'pdf_attachments': [
-#                     {'name': 'testing.pdf', 'html': '<h1>testing</h1>', 'id': 123},
-#                     {'name': 'different.pdf', 'html': '<h1>different</h1>'},
-#                 ],
-#             }
-#         ]
-#     )
-#     assert len(tmpdir.listdir()) == 1
-#     msg_file = tmpdir.join(f'{message_id}.txt').read()
-#     assert 'testing.pdf' in msg_file
-#
-#     attachments = sync_db.fetchrow_b('select * from messages where :where', where=V('external_id') == message_id)[
-#         'attachments'
-#     ]
-#     assert set(attachments) == {'123::testing.pdf', '::different.pdf'}
+def test_send_with_pdf(send_email, tmpdir, sync_db: SyncDb):
+    message_id = send_email(
+        recipients=[
+            {
+                'address': '[email protected]',
+                'pdf_attachments': [
+                    {'name': 'testing.pdf', 'html': '<h1>testing</h1>', 'id': 123},
+                    {'name': 'different.pdf', 'html': '<h1>different</h1>'},
+                ],
+            }
+        ]
+    )
+    assert len(tmpdir.listdir()) == 1
+    msg_file = tmpdir.join(f'{message_id}.txt').read()
+    assert 'testing.pdf' in msg_file
+
+    attachments = sync_db.fetchrow_b('select * from messages where :where', where=V('external_id') == message_id)[
+        'attachments'
+    ]
+    assert set(attachments) == {'123::testing.pdf', '::different.pdf'}
 
 
 def test_send_with_other_attachment(send_email, tmpdir, sync_db: SyncDb):
@@ -541,15 +541,15 @@ def test_send_with_other_attachment_pdf(send_email, tmpdir, sync_db: SyncDb):
     assert set(attachments) == {'::test_pdf.pdf', '::test_pdf_encoded.pdf'}
 
 
-# def test_pdf_not_unicode(send_email, tmpdir, cli):
-#     message_id = send_email(
-#         recipients=[
-#             {'address': '[email protected]', 'pdf_attachments': [{'name': 'testing.pdf', 'html': '<h1>binary</h1>'}]}
-#         ]
-#     )
-#     assert len(tmpdir.listdir()) == 1
-#     msg_file = tmpdir.join(f'{message_id}.txt').read()
-#     assert 'testing.pdf' in msg_file
+def test_pdf_not_unicode(send_email, tmpdir, cli):
+    message_id = send_email(
+        recipients=[
+            {'address': '[email protected]', 'pdf_attachments': [{'name': 'testing.pdf', 'html': '<h1>binary</h1>'}]}
+        ]
+    )
+    assert len(tmpdir.listdir()) == 1
+    msg_file = tmpdir.join(f'{message_id}.txt').read()
+    assert 'testing.pdf' in msg_file
 
 
 def test_pdf_empty(send_email, tmpdir, dummy_server):