update meetup scripts with minor fixes, allow location overriding, support hybrid events going in two sections, remove stale meetup groups

Author: bdillo

tools/events/meetup-automation/country_code_to_continent.py

@@ -230,6 +230,7 @@
     'TZ': 'Africa',
     'UA': 'Europe',
     'UG': 'Africa',
+    'UK': 'Europe', # not an actual country code, but using it here
     'US': 'North America',
     'UY': 'South America',
     'UZ': 'Asia',

tools/events/meetup-automation/event.py

@@ -2,6 +2,9 @@
 import string
 from dataclasses import dataclass
 from datetime import datetime
+from typing import Optional
+
+from utils import LocationOverride
 
 
 logger = logging.getLogger(__name__)
@@ -37,6 +40,8 @@ def __post_init__(self):
       # looks like in GB meetup considers part of the post code as the "state", which is not a common way to write
       # locations in GB (or that's my understanding at least)
       self.state = None
+      # and "UK" is more accurate it seems, so replace "GB"
+      self.country = "UK"
 
 
   def fields_present(self) -> int:
@@ -76,6 +81,7 @@ class Event:
   virtual: bool
   organizer_name: str
   organizer_url: str
+  hybrid: bool
 
   def __post_init__(self):
     """ Normalize the event data here """
@@ -91,11 +97,17 @@ def to_dict(self) -> dict:
       "url": self.url,
       "virtual": self.virtual,
       "organizer_name": self.organizer_name,
-      "organizer_url": self.organizer_url
+      "organizer_url": self.organizer_url,
+      "hybrid": self.hybrid
     }
 
   def to_markdown_string(self) -> str:
-    location = f"Virtual ({self.location.to_str()})" if self.virtual else self.location.to_str()
+    if self.hybrid:
+      location = f"Hybrid ({self.location.to_str()})"
+    elif self.virtual:
+      location = f"Virtual ({self.location.to_str()})"
+    else:
+      location = self.location.to_str()
 
     return f'* {self.date.date()} | {location} | [{self.organizer_name}]({self.organizer_url})\n    * [**{self.name}**]({self.url})'
 
@@ -112,8 +124,6 @@ class RawGqlEvent:
   event_url_str: str
   venue_type: None | str
   event_location: Location
-  lat: float
-  long: float
 
   def __init__(self, **kwargs) -> None:
     logger.debug(f"Constructing RawGqlEvent from: {kwargs}")
@@ -130,14 +140,18 @@ def __init__(self, **kwargs) -> None:
 
     venue = node["venue"]
     self.venue_type = venue["venueType"]
-    # TODO: do we need these lat longs?
-    self.lat = venue["lat"]
-    self.long = venue["lng"]
     self.event_location = Location(venue["city"], venue["state"], venue["country"])
 
-  def to_event(self, group_url: str) -> Event:
+  def to_event(self, group_url: str, location_override: Optional[LocationOverride]) -> Event:
+    is_hybrid = False
     is_virtual = self.venue_type == "online"
 
+    if location_override:
+      if location_override == LocationOverride.VIRTUAL:
+        is_virtual = True
+      elif location_override == LocationOverride.HYBRID:
+        is_hybrid = True
+
     # this is a bit weird because we want a naive datetime object that just contains the year/month/day because we get
     # timestamps with tz info like "2025-01-16T19:00+01:00", just strip the time and tz info before parsing
     date = datetime.strptime(self.date_time_str.split('T')[0], '%Y-%m-%d')
@@ -155,6 +169,7 @@ def to_event(self, group_url: str) -> Event:
       url=self.event_url_str,
       virtual=is_virtual,
       organizer_name=self.group_name,
-      organizer_url=group_url
+      organizer_url=group_url,
+      hybrid=is_hybrid
     )
 

tools/events/meetup-automation/generate_events_meetup.py

@@ -4,8 +4,8 @@
 
 from jwt_auth import generate_signed_jwt
 from event import Event, RawGqlEvent
-from utils import MeetupGroupUrl
-from typing import List
+from utils import LocationOverride, MeetupGroupUrl
+from typing import List, Optional
 
 logger = logging.getLogger(__name__)
 
@@ -96,15 +96,27 @@ def _build_event_listing_gql_query(self, group_url_name: str) -> dict:
             }
         }
 
-    def _parse_event_listing_gql_response(self, response: dict) -> List[RawGqlEvent]:
+    def _parse_event_listing_gql_response(self, response: dict, location_override: Optional[LocationOverride]) -> List[RawGqlEvent]:
         edges = response["groupByUrlname"]["upcomingEvents"]["edges"]
 
         events = []
         # TODO: maybe move this validation somewhere else?
         for edge_kwargs in edges:
             if not edge_kwargs["node"]["venue"]:
-                logger.error(f"Event response missing venue: {edge_kwargs}")
-                continue
+                # for events where there's no venue in the response and we have a virtual override, inherit the location from the group here
+                # a bit gross but oh well
+                if location_override == LocationOverride.VIRTUAL:
+                    logger.info(f"Overriding location for event: {edge_kwargs}")
+                    venue = {}
+                    venue["city"] = edge_kwargs["node"]["group"]["city"]
+                    venue["state"] = edge_kwargs["node"]["group"]["state"]
+                    venue["country"] = edge_kwargs["node"]["group"]["country"]
+                    venue["venueType"] = "online"
+
+                    edge_kwargs["node"]["venue"] = venue
+                else:
+                    logger.error(f"Event response missing venue: {edge_kwargs}")
+                    continue
 
             events.append(RawGqlEvent(**edge_kwargs))
 
@@ -206,7 +218,7 @@ def get_raw_events_for_group(self, group: MeetupGroupUrl) -> List[RawGqlEvent]:
             "Content-Type": "application/json",
         }
 
-        logger.info(f"Fetching events for group {group}")
+        logger.debug(f"Fetching events for group {group}")
         query = self._build_event_listing_gql_query(group.url_name)
         response = requests.post(url=self.GQL_ENDPOINT, headers=headers, json=query)
         data = response.json()["data"]
@@ -216,5 +228,5 @@ def get_raw_events_for_group(self, group: MeetupGroupUrl) -> List[RawGqlEvent]:
             logger.error(f"Group {group} not valid, skipping")
             return []
 
-        return self._parse_event_listing_gql_response(data)
+        return self._parse_event_listing_gql_response(data, group.location_override)
 

tools/events/meetup-automation/main.py

@@ -35,7 +35,7 @@ def main():
     for group_url in group_urls:
         group_raw_events = meetup_client.get_raw_events_for_group(group_url)
 
-        events += [raw_event.to_event(group_url.url) for raw_event in group_raw_events]
+        events += [raw_event.to_event(group_url.url, group_url.location_override) for raw_event in group_raw_events]
 
     # Remove events outside of date range.
     events = date_window_filter(events, args.weeks)
@@ -74,7 +74,7 @@ def output_to_screen(event_list):
         if len(value) == 0:
             continue
         else:
-            print(f'### {key}:\n')
+            print(f'### {key}')
 
             # Output event details
             for event in value:
@@ -104,9 +104,13 @@ def group_virtual_continent(event_list):
     separated_event_list = {}
 
     for event in event_list:
-        # Separates Events by Virtual or by Continent
-        key = "Virtual" if event.virtual else country_code_to_continent(event.location.country)
-        separated_event_list.setdefault(key, []).append(event)
+        if event.hybrid:
+            keys = "Virtual", country_code_to_continent(event.location.country)
+            for key in keys:
+                separated_event_list.setdefault(key, []).append(event)
+        else:
+            key = "Virtual" if event.virtual else country_code_to_continent(event.location.country)
+            separated_event_list.setdefault(key, []).append(event)
 
     return separated_event_list
 

tools/events/meetup-automation/utils.py

@@ -1,16 +1,23 @@
 from dataclasses import dataclass
-from typing import List
+from enum import Enum
+from typing import List, Optional
 from urllib.parse import urlparse
 import json
 
+class LocationOverride(str, Enum):
+    HYBRID = "hybrid"
+    VIRTUAL = "virtual"
+
+
 @dataclass
 class MeetupGroupUrl:
     MEETUP_HOSTNAME = "www.meetup.com"
 
     url: str
     url_name: str
+    location_override: Optional[LocationOverride]
 
-    def __init__(self, url_str: str) -> None:
+    def __init__(self, url_str: str, location_override: Optional[str]) -> None:
         parsed = urlparse(url_str)
 
         if parsed.hostname != self.MEETUP_HOSTNAME:
@@ -23,11 +30,21 @@ def __init__(self, url_str: str) -> None:
 
         self.url = url_str
         self.url_name = path_split[1]
-        
+
+        if location_override:
+            self.location_override = LocationOverride(location_override)
+        else:
+            self.location_override = None
+
 
 def read_meetup_group_urls(meetups_json: str) -> List[MeetupGroupUrl]:
     with open(meetups_json, "r") as f:
         group_urls = json.loads(f.read())
+        parsed_groups = []
+
+        for url, metadata in group_urls.items():
+            location_override = metadata.get("location_override")
+            parsed = MeetupGroupUrl(url, location_override)
+            parsed_groups.append(parsed)
 
-        parsed_groups = [MeetupGroupUrl(url) for url in group_urls]
         return parsed_groups