parse Submissions data from pretalx

Example of the implementation, including pushing reports and basic
charts to both web and discord

Author: artcz

intbot/core/analysis/submissions.py

@@ -0,0 +1,119 @@
+"""
+Basic analysis of Pretalx Submissions data
+"""
+
+from datetime import datetime
+from typing import ClassVar, Iterable
+
+import plotly.express as px
+import polars as pl
+from core.models import PretalxData
+from pydantic import BaseModel, model_validator
+
+
+class LocalisedFieldsMixin:
+    # Marking as ClassVar here is important. It doens't work without it :)
+    _localised_fields: ClassVar[Iterable[str]] = ()
+
+    @model_validator(mode="before")
+    @classmethod
+    def extract(cls, values):
+        for field in cls._localised_fields:
+            if isinstance(values[field], dict) and "en" in values[field]:
+                values[field] = values[field]["en"]
+                continue
+
+        return values
+
+
+class Submission(LocalisedFieldsMixin, BaseModel):
+    code: str
+    title: str
+    submission_type: str | dict
+    track: str | None
+    state: str
+    abstract: str
+    duration: int
+    created: datetime
+    level: str = ""
+    outline: str | None = None
+    event: str = "ep2025"
+
+    _localised_fields = ["submission_type", "track"]
+
+    class Questions:
+        level = "Expected audience expertise"
+        outline = "Outline"
+
+    @model_validator(mode="before")
+    def extract_answers(cls, values):
+        # Some things are available as answers to questions and we can extract
+        # them here
+        # But using .get since this should be optional for creating objects
+        # manually
+        for answer in values.get("answers", ""):
+            if answer["submission"] is not None and cls.question_is(
+                answer, cls.Questions.level
+            ):
+                values["level"] = answer["answer"]
+
+            if answer["submission"] is not None and cls.question_is(
+                answer, cls.Questions.outline
+            ):
+                values["outline"] = answer["answer"]
+
+        return values
+
+    @staticmethod
+    def question_is(answer: dict, question: str) -> bool:
+        return answer.get("question", {}).get("question", {}).get("en") == question
+
+
+def get_latest_submissions_data() -> PretalxData:
+    qs = PretalxData.objects.filter(resource=PretalxData.PretalxResources.submissions)
+    return qs.latest("created_at")
+
+
+def parse_latest_submissions_to_objects(pretalx_data: PretalxData) -> list[Submission]:
+    data = pretalx_data.content
+    # NOTE: add event as context here
+    submissions = [Submission.model_validate(entry) for entry in data]
+    return submissions
+
+
+def flat_submissions_data(submissions: list[Submission]) -> pl.DataFrame:
+    """
+    Returns a polars data frame with flat description of Submissions
+
+    This functions mostly exists to make the API consistent between different
+    types of data
+    """
+    return pl.DataFrame(submissions)
+
+
+def latest_flat_submissions_data() -> pl.DataFrame:
+    """
+    Thin wrapper on getting latest information from the database, and
+    converting into a polars data frame
+    """
+    pretalx_data = get_latest_submissions_data()
+    submissions = parse_latest_submissions_to_objects(pretalx_data)
+    return flat_submissions_data(submissions)
+
+
+def group_submissions_by_state(submissions: pl.DataFrame) -> pl.DataFrame:
+    by_state = submissions.group_by("state").len().sort("len", descending=True)
+
+    return by_state
+
+
+def piechart_submissions_by_state(submissions_by_state: pl.DataFrame):
+    fig = px.pie(
+        submissions_by_state,
+        values="len",
+        names="state",
+        title="state of the submission",
+        color_discrete_sequence=px.colors.qualitative.Pastel,
+    )
+
+    return fig

intbot/core/bot/main.py

@@ -1,6 +1,13 @@
+import io
+
 import discord
 from asgiref.sync import sync_to_async
 from core.analysis.products import latest_flat_product_data
+from core.analysis.submissions import (
+    group_submissions_by_state,
+    latest_flat_submissions_data,
+    piechart_submissions_by_state,
+)
 from core.models import DiscordMessage, InboxItem
 from discord.ext import commands, tasks
 from django.conf import settings
@@ -220,6 +227,26 @@ async def products(ctx):
     await ctx.send(f"```{str(data)}```")
 
 
+@bot.command()
+async def submissions_status(ctx):
+    df = await sync_to_async(latest_flat_submissions_data)()
+    by_state = group_submissions_by_state(df)
+
+    await ctx.send(f"```{str(by_state)}```")
+
+
+@bot.command()
+async def submissions_status_pie_chart(ctx):
+    df = await sync_to_async(latest_flat_submissions_data)()
+    by_state = group_submissions_by_state(df)
+    piechart = piechart_submissions_by_state(by_state)
+
+    png_bytes = piechart.to_image(format="png")
+    file = discord.File(io.BytesIO(png_bytes), filename="submissions_by_state.png")
+
+    await ctx.send(file=file)
+
+
 def run_bot():
     bot_token = settings.DISCORD_BOT_TOKEN
     bot.run(bot_token)

intbot/core/views.py

@@ -1,8 +1,14 @@
 from core.analysis.products import latest_flat_product_data
+from core.analysis.submissions import (
+    group_submissions_by_state,
+    latest_flat_submissions_data,
+    piechart_submissions_by_state,
+)
 from django.conf import settings
 from django.contrib.auth.decorators import login_required
 from django.template.response import TemplateResponse
 from django.utils import timezone
+from django.utils.safestring import mark_safe
 
 
 def days_until(request):
@@ -37,3 +43,27 @@ def products(request):
             "rows": rows,
         },
     )
+
+
+@login_required
+def submissions(request):
+    """
+    Show some basic aggregation of submissions data
+    """
+
+    df = latest_flat_submissions_data()
+    by_state = group_submissions_by_state(df)
+    piechart = piechart_submissions_by_state(by_state)
+
+    return TemplateResponse(
+        request,
+        "submissions.html",
+        {
+            "piechart": mark_safe(
+                piechart.to_html(
+                    full_html=False,
+                    include_plotlyjs="cdn",
+                )
+            ),
+        },
+    )

intbot/intbot/urls.py

@@ -4,7 +4,7 @@
     internal_webhook_endpoint,
     zammad_webhook_endpoint,
 )
-from core.views import days_until, products
+from core.views import days_until, products, submissions
 from django.contrib import admin
 from django.urls import path
 
@@ -18,4 +18,5 @@
     # Public Pages
     path("days-until/", days_until),
     path("products/", products),
+    path("submissions/", submissions),
 ]

intbot/templates/submissions.html

@@ -0,0 +1,40 @@
+
+<html>
+    <body>
+        <style type="text/css">
+body { background-color: #ffe; color: #151f38; font-family: sans-serif;}
+h1 { font-size: 3em; text-align: center; margin-top: 3em; }
+thead { background: #aaa; }
+table { font-family: monospace; width: 100%}
+table td { padding: .5em 1em; }
+table tr:nth-child(2n) td { background: #eee; }
+table tr:hover td { background: #fee; }
+        </style>
+        <div style="margin: 3em">
+
+            <h1>Submissions</h1>
+
+            <div id="piechart">
+                {{ piechart }}
+            </div>
+            <p style="text-align: center">This is not particularly useful now, but it's here as a proof of concept :)</p>
+
+            <table>
+                <thead>
+                    {% for column in columns %}
+                    <td>{{ column }}</td>
+                    {% endfor %}
+                </thead>
+
+                {% for row in rows %}
+                <tr>
+                    {% for cell in row %}
+                    <td>{{ cell }}</td>
+                    {% endfor %}
+                </tr>
+                {% endfor %}
+            </table>
+
+        </div>
+    </body>
+</html>