Fix remaining Pretalx scripts (#2420)

Co-authored-by: Sam <[email protected]>

Author: mxsasha

docs/_scripts/pretalx2mc-info.py

@@ -10,7 +10,7 @@
 sys.path.insert(1, '../_ext')
 
 import os
-from collections import OrderedDict
+from collections import OrderedDict, defaultdict
 from utils import slugify
 import requests
 from ruamel import yaml
@@ -27,18 +27,19 @@ def convert_to_yaml(year, series, series_slug, yaml_output, pretalx_slug):
         print('Error: PRETALX_TOKEN not found in environment variables.')
         return
     http_headers = {'Authorization': 'Token ' + os.environ['PRETALX_TOKEN']}
-    submissions_url = f'https://pretalx.com/api/events/{pretalx_slug}/submissions/?state=confirmed'
+    submissions_url = f'https://pretalx.com/api/events/{pretalx_slug}/submissions/?state=confirmed&expand=speakers'
     print(f'Loading submissions from {submissions_url}...')
     submissions = requests.get(submissions_url, headers=http_headers)
     if submissions.status_code != 200:
         print(f'Error: submissions request failed: {submissions.status_code}: {submissions.text}')
         return
 
+    answers = get_answers(pretalx_slug, http_headers)
     for index, talk in enumerate(submissions.json()['results']):
         slug = slugify(talk['title'] + '-' + talk['speakers'][0]['name'])
         print(f'Processing talk {slug}...')
 
-        speaker_info = retrieve_speaker_info([s['code'] for s in talk['speakers']], http_headers, pretalx_slug)
+        speaker_info = retrieve_speaker_info([s['code'] for s in talk['speakers']], http_headers, pretalx_slug, answers)
         if not speaker_info:
             print(f'Error: failed to retrieve info for speaker s["code"]')
             return
@@ -57,7 +58,7 @@ def convert_to_yaml(year, series, series_slug, yaml_output, pretalx_slug):
     print('Completed!')
 
 
-def retrieve_speaker_info(speaker_codes, http_headers, pretalx_slug):
+def retrieve_speaker_info(speaker_codes, http_headers, pretalx_slug, answers):
     result = []
     for speaker_code in speaker_codes:
         speaker_url = f'https://pretalx.com/api/events/{pretalx_slug}/speakers/{speaker_code}/?questions=all'
@@ -68,12 +69,11 @@ def retrieve_speaker_info(speaker_codes, http_headers, pretalx_slug):
             return
 
         def search_answers(speaker_dict, search_string):
-            for answer in speaker_dict['answers']:
-                if search_string in answer['question']['question']['en'].lower():
-                    return answer['answer']
+            for question, answer in answers.get(speaker_code, []):
+                if search_string in question.lower():
+                    return answer
 
         speaker = speaker_response.json()
-        speaker_slug = slugify(speaker['name'])
 
         result.append(OrderedDict([
             ('name', speaker['name']),
@@ -88,6 +88,34 @@ def search_answers(speaker_dict, search_string):
         ]))
     return result
 
+
+def get_answers(pretalx_slug, http_headers):
+    url = f'https://pretalx.com/api/events/{pretalx_slug}/answers/?expand=question'
+    print(f'Loading answers from {url}...')
+    answers = load_pretalx_resource(url, http_headers)
+    result = defaultdict(list)
+    for answer in answers:
+        result[answer['person']].append((answer['question']['question']['en'], answer['answer']))
+    return result
+
+def load_pretalx_resource(url, http_headers):
+    results = []
+    while url:
+        response = requests.get(url, headers=http_headers)
+        # 403 may occur as a pretalx bug
+        if response.status_code not in [200, 403]:
+            print(f'Error: request failed: {response.status_code}: {response.text}')
+            return
+
+        results += response.json()['results']
+
+        if response.json()['next']:
+            url = response.json()['next']
+        else:
+            break
+    return results
+
+
 if __name__ == '__main__':
     convert_to_yaml(
         year='2025',

docs/_scripts/pretalx2wtd.py

@@ -10,7 +10,7 @@
 sys.path.insert(1, '../_ext')
 
 import os
-from collections import OrderedDict
+from collections import OrderedDict, defaultdict
 from utils import slugify
 import requests
 from ruamel import yaml
@@ -36,18 +36,19 @@ def convert_to_yaml(year, series, series_slug, yaml_output, pretalx_slug):
         print('Error: PRETALX_TOKEN not found in environment variables.')
         return
     http_headers = {'Authorization': 'Token ' + os.environ['PRETALX_TOKEN']}
-    submissions_url = f'https://pretalx.com/api/events/{pretalx_slug}/submissions/?state=confirmed'
+    submissions_url = f'https://pretalx.com/api/events/{pretalx_slug}/submissions/?state=confirmed&expand=speakers'
     print(f'Loading submissions from {submissions_url}...')
     submissions = requests.get(submissions_url, headers=http_headers)
     if submissions.status_code != 200:
         print(f'Error: submissions request failed: {submissions.status_code}: {submissions.text}')
         return
 
+    answers = get_answers(pretalx_slug, http_headers)
     for index, talk in enumerate(submissions.json()['results']):
         slug = slugify(talk['title'][:MAX_TITLE_LENGTH_FOR_SLUG] + '-' + talk['speakers'][0]['name'])
         print(f'Processing talk {slug}...')
 
-        speaker_info = retrieve_speaker_info([s['code'] for s in talk['speakers']], http_headers, pretalx_slug)
+        speaker_info = retrieve_speaker_info([s['code'] for s in talk['speakers']], http_headers, pretalx_slug, answers)
         if not speaker_info:
             print(f'Error: failed to retrieve info for speaker s["code"]')
             return
@@ -71,7 +72,7 @@ def convert_to_yaml(year, series, series_slug, yaml_output, pretalx_slug):
     print('Completed!')
 
 
-def retrieve_speaker_info(speaker_codes, http_headers, pretalx_slug):
+def retrieve_speaker_info(speaker_codes, http_headers, pretalx_slug, answers):
     result = []
     for speaker_code in speaker_codes:
         speaker_url = f'https://pretalx.com/api/events/{pretalx_slug}/speakers/{speaker_code}/'
@@ -82,15 +83,15 @@ def retrieve_speaker_info(speaker_codes, http_headers, pretalx_slug):
             return
 
         def search_answers(speaker_dict, search_string):
-            for answer in speaker_dict['answers']:
-                if search_string in answer['question']['question']['en'].lower():
-                    return answer['answer']
+            for question, answer in answers.get(speaker_code, []):
+                if search_string in question.lower():
+                    return answer
 
         speaker = speaker_response.json()
         speaker_slug = slugify(speaker['name'])
 
-        if speaker['avatar']:
-            image_response = requests.get(speaker['avatar'], stream=True)
+        if speaker['avatar_url']:
+            image_response = requests.get(speaker['avatar_url'], stream=True)
             avatar_path = SPEAKER_IMAGE_PATH + speaker_slug + '.' + CONTENT_TYPE_EXTENSIONS[image_response.headers['content-type']]
             if image_response.status_code != 200:
                 print(f'Error: speaker avatar request failed: {image_response.status_code}: {image_response.text}')
@@ -108,9 +109,36 @@ def search_answers(speaker_dict, search_string):
         ]))
     return result
 
+
+def get_answers(pretalx_slug, http_headers):
+    url = f'https://pretalx.com/api/events/{pretalx_slug}/answers/?expand=question'
+    print(f'Loading answers from {url}...')
+    answers = load_pretalx_resource(url, http_headers)
+    result = defaultdict(list)
+    for answer in answers:
+        result[answer['person']].append((answer['question']['question']['en'], answer['answer']))
+    return result
+
+def load_pretalx_resource(url, http_headers):
+    results = []
+    while url:
+        response = requests.get(url, headers=http_headers)
+        # 403 may occur as a pretalx bug
+        if response.status_code not in [200, 403]:
+            print(f'Error: request failed: {response.status_code}: {response.text}')
+            return
+
+        results += response.json()['results']
+
+        if response.json()['next']:
+            url = response.json()['next']
+        else:
+            break
+    return results
+
 if __name__ == '__main__':
     year='2025'
-    place="Portland"
+    place="Berlin"
     convert_to_yaml(
         year,
         series='Write the Docs '+place,