Download pretix data (#33)

First draft of downloading basic pretix data for later reporting – based on the pretalx data download implementation.

Co-authored-by: Mia Bajić <[email protected]>

Author: artcz

deploy/playbooks/04_cron.yml

@@ -2,9 +2,16 @@
   hosts: intbot_app
 
   tasks:
-    - name: "Download pretalx data every hour"
+    - name: "Download pretalx data once a day"
       ansible.builtin.cron:
-        name: "Download pretalx data every hour"
+        name: "Download pretalx data once a day at 05:05am"
         minute: "5"
-        hour: "8"     # run once a day at 08:05 am
+        hour: "5"
         job: "make prod/cron/pretalx"
+
+    - name: "Download pretix data once a day"
+      ansible.builtin.cron:
+        name: "Download pretix data once a day at 06:05am"
+        minute: "5"
+        hour: "6"
+        job: "make prod/cron/pretix"

deploy/templates/app/Makefile.app.j2

@@ -18,5 +18,9 @@ prod/manage:
 prod/cron/pretalx:
 	$(MAKE_APP) in-container/manage ARG="download_pretalx_data --event=europython-2025"
 
+
+prod/cron/pretix:
+	$(MAKE_APP) in-container/manage ARG="download_pretix_data --event=ep2025"
+
 logs:
 	docker compose logs -f

deploy/templates/app/intbot.env.example

@@ -58,3 +58,9 @@ ZAMMAD_GROUP_PROGRAMME="zammad-programme-group-name-goes-here"
 ZAMMAD_GROUP_FINAID="zammad-finaid-group-name-goes-here"
 ZAMMAD_GROUP_SPONSORS="zammad-sponsors-group-name-goes-here"
 ZAMMAD_GROUP_GRANTS="zammad-grants-group-name-goes-here"
+
+# Pretalx
+PRETALX_API_TOKEN="pretalx-api-token"
+
+# Pretix
+PRETIX_API_TOKEN="pretix-api-token"

intbot/core/admin.py

@@ -1,6 +1,6 @@
 import json
 
-from core.models import DiscordMessage, PretalxData, Webhook
+from core.models import DiscordMessage, PretalxData, PretixData, Webhook
 from django.contrib import admin
 from django.utils.html import format_html
 
@@ -93,6 +93,33 @@ def pretty_content(self, obj: PretalxData):
     pretty_content.short_description = "Content"
 
 
+class PretixDataAdmin(admin.ModelAdmin):
+    list_display = [
+        "uuid",
+        "resource",
+        "created_at",
+        "modified_at",
+    ]
+    list_filter = [
+        "created_at",
+        "resource",
+    ]
+    readonly_fields = fields = [
+        "uuid",
+        "resource",
+        "pretty_content",
+        "created_at",
+        "modified_at",
+        "processed_at",
+    ]
+
+    def pretty_content(self, obj: PretixData):
+        return format_html("<pre>{}</pre>", json.dumps(obj.content, indent=4))
+
+    pretty_content.short_description = "Content"
+
+
 admin.site.register(Webhook, WebhookAdmin)
 admin.site.register(DiscordMessage, DiscordMessageAdmin)
 admin.site.register(PretalxData, PretalxDataAdmin)
+admin.site.register(PretixData, PretixDataAdmin)

intbot/core/integrations/pretix.py

@@ -0,0 +1,100 @@
+import logging
+from typing import Any
+
+import httpx
+from core.models import PretixData
+from django.conf import settings
+
+logger = logging.getLogger(__name__)
+
+PRETIX_EVENTS = [
+    "2022",
+    "ep2023",
+    "ep2024",
+    "ep2025",
+]
+
+ENDPOINTS = {
+    PretixData.PretixResources.orders: "orders/",
+    PretixData.PretixResources.products: "items/",
+    PretixData.PretixResources.vouchers: "vouchers/",
+}
+
+
+JsonType = dict[str, Any]
+
+
+def get_event_url(event: str) -> str:
+    assert event in PRETIX_EVENTS
+
+    pretix_url = "https://tickets.europython.eu"
+    return f"{pretix_url}/api/v1/organizers/europython/events/{event}/"
+
+
+def fetch_pretix_data(
+    event: str, resource: PretixData.PretixResources
+) -> list[JsonType]:
+    headers = {
+        "Authorization": f"Token {settings.PRETIX_API_TOKEN}",
+        "Content-Type": "application/json",
+    }
+
+    base_url = get_event_url(event)
+    endpoint = ENDPOINTS[resource]
+    url = f"{base_url}{endpoint}"
+
+    # Pretix paginates the output, so we will need to do multiple requests and
+    # then merge multiple pages to one big dictionary
+    results = []
+    page = 0
+
+    # This takes advantage of the fact that url will contain a url to the
+    # next page, until there is more data to fetch. If this is the last page,
+    # then the url will be None (falsy), and thus stop the while loop.
+    while url:
+        page += 1
+        response = httpx.get(url, headers=headers)
+
+        if response.status_code != 200:
+            raise Exception(f"Error {response.status_code}: {response.text}")
+
+        logger.info("Fetching data from %s, page %s", url, page)
+
+        data = response.json()
+        results += data["results"]
+        url = data["next"]
+
+    return results
+
+
+def download_latest_orders(event: str) -> PretixData:
+    data = fetch_pretix_data(event, PretixData.PretixResources.orders)
+
+    pretix_data = PretixData.objects.create(
+        resource=PretixData.PretixResources.orders,
+        content=data,
+    )
+
+    return pretix_data
+
+
+def download_latest_products(event: str) -> PretixData:
+    data = fetch_pretix_data(event, PretixData.PretixResources.products)
+
+    pretix_data = PretixData.objects.create(
+        resource=PretixData.PretixResources.products,
+        content=data,
+    )
+
+    return pretix_data
+
+
+def download_latest_vouchers(event: str) -> PretixData:
+    data = fetch_pretix_data(event, PretixData.PretixResources.vouchers)
+
+    pretix_data = PretixData.objects.create(
+        resource=PretixData.PretixResources.vouchers,
+        content=data,
+    )
+
+    return pretix_data

intbot/core/management/commands/download_pretix_data.py

@@ -0,0 +1,32 @@
+from core.integrations.pretix import (
+    PRETIX_EVENTS,
+    download_latest_orders,
+    download_latest_products,
+    download_latest_vouchers,
+)
+from django.core.management.base import BaseCommand
+
+
+class Command(BaseCommand):
+    help = "Downloads latest pretix data"
+
+    def add_arguments(self, parser):
+        # Add keyword argument event
+        parser.add_argument(
+            "--event",
+            choices=PRETIX_EVENTS,
+            help="slug of the event (for example `ep2025`)",
+            required=True,
+        )
+
+    def handle(self, **kwargs):
+        event = kwargs["event"]
+
+        self.stdout.write(f"Downloading latest products from pretix... {event}")
+        download_latest_products(event)
+
+        self.stdout.write(f"Downloading latest vouchers from pretix... {event}")
+        download_latest_vouchers(event)
+
+        self.stdout.write(f"Downloading latest orders from pretix... {event}")
+        download_latest_orders(event)

intbot/core/migrations/0006_add_pretix_data_model.py

@@ -0,0 +1,43 @@
+# Generated by Django 5.1.4 on 2025-04-24 22:08
+
+import uuid
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("core", "0005_add_pretalx_data_model"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="PretixData",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("uuid", models.UUIDField(default=uuid.uuid4)),
+                (
+                    "resource",
+                    models.CharField(
+                        choices=[
+                            ("orders", "Orders"),
+                            ("products", "Products"),
+                            ("vouchers", "Vouchers"),
+                        ],
+                        max_length=255,
+                    ),
+                ),
+                ("content", models.JSONField()),
+                ("created_at", models.DateTimeField(auto_now_add=True)),
+                ("modified_at", models.DateTimeField(auto_now=True)),
+                ("processed_at", models.DateTimeField(blank=True, null=True)),
+            ],
+        ),
+    ]

intbot/core/models.py

@@ -87,9 +87,7 @@ class PretalxData(models.Model):
     """
     Table to store raw data download from pretalx for later parsing.
 
-    We first download data from pretalx to this table, and then fire a separate
-    background task that pulls data from this table and stores in separate
-    "business" tables, like "Proposal" or "Speaker".
+    We first download data from pretix, then we later parse the latest jsons.
     """
 
     class PretalxResources(models.TextChoices):
@@ -108,5 +106,38 @@ class PretalxResources(models.TextChoices):
     modified_at = models.DateTimeField(auto_now=True)
     processed_at = models.DateTimeField(blank=True, null=True)
 
+    class Meta:
+        verbose_name_plural = "Pretalx Data"
+
+    def __str__(self):
+        return f"{self.uuid}"
+
+
+class PretixData(models.Model):
+    """
+    Table to store raw data download from pretix for later parsing.
+
+    We first download data from pretix, then we later parse the latest jsons.
+    """
+
+    class PretixResources(models.TextChoices):
+        orders = "orders", "Orders"
+        products = "products", "Products"
+        vouchers = "vouchers", "Vouchers"
+
+    uuid = models.UUIDField(default=uuid.uuid4)
+    resource = models.CharField(
+        max_length=255,
+        choices=PretixResources.choices,
+    )
+    content = models.JSONField()
+
+    created_at = models.DateTimeField(auto_now_add=True)
+    modified_at = models.DateTimeField(auto_now=True)
+    processed_at = models.DateTimeField(blank=True, null=True)
+
+    class Meta:
+        verbose_name_plural = "Pretix Data"
+
     def __str__(self):
         return f"{self.uuid}"

intbot/intbot/settings.py

@@ -200,6 +200,9 @@ def get(name) -> str:
 # Pretalx
 PRETALX_API_TOKEN = get("PRETALX_API_TOKEN")
 
+# Pretix
+PRETIX_API_TOKEN = get("PRETIX_API_TOKEN")
+
 
 if DJANGO_ENV == "dev":
     DEBUG = True