From a5608b78398187e70dd232b3285541d0f6c6682c Mon Sep 17 00:00:00 2001 From: bdillo Date: Mon, 18 Nov 2024 16:57:22 -0800 Subject: [PATCH 01/18] copying over with small changes from existing pr --- tools/events-automation/README.md | 73 +++++ .../country_code_to_continent.py | 256 ++++++++++++++++ tools/events-automation/country_to_abbrev.py | 281 +++++++++++++++++ tools/events-automation/event.py | 47 +++ .../generate_events_meetup.py | 283 ++++++++++++++++++ tools/events-automation/jwt_auth.py | 85 ++++++ tools/events-automation/main.py | 96 ++++++ tools/events-automation/meetup_events.md | 75 +++++ tools/events-automation/requirements.txt | 1 + .../events-automation/rust_meetup_groups.csv | 118 ++++++++ .../state_territory_to_abbrev.py | 102 +++++++ tools/events-automation/test_events.py | 15 + 12 files changed, 1432 insertions(+) create mode 100644 tools/events-automation/README.md create mode 100644 tools/events-automation/country_code_to_continent.py create mode 100644 tools/events-automation/country_to_abbrev.py create mode 100644 tools/events-automation/event.py create mode 100644 tools/events-automation/generate_events_meetup.py create mode 100644 tools/events-automation/jwt_auth.py create mode 100644 tools/events-automation/main.py create mode 100644 tools/events-automation/meetup_events.md create mode 100644 tools/events-automation/requirements.txt create mode 100644 tools/events-automation/rust_meetup_groups.csv create mode 100644 tools/events-automation/state_territory_to_abbrev.py create mode 100644 tools/events-automation/test_events.py diff --git a/tools/events-automation/README.md b/tools/events-automation/README.md new file mode 100644 index 000000000..baa4a3441 --- /dev/null +++ b/tools/events-automation/README.md @@ -0,0 +1,73 @@ +# Event Sink +The maintainer of the Events section is faced with manual work; this process automates generation of a draft events list using various Event Source Modules that gather event information, and an Event Sink that sorts, groups and formats the events into a pre-determined markdown format. + +## Getting Started: +### Pre-Requisites: +- Event sink requires Python3 installation. +- For specific module requirements: `pip install -r requirements.txt` +- See https://geopy.readthedocs.io/en/stable/# for `geopy` module documentation. + +### Running: +Before running please check that all Event Source module function calls are included in `event_list` (function calls should concatenate into a single list of event objects). + +To run this code: + +```py +pip install -r requirements.txt +python3 main.py +``` + +### How to Add a New Event Source Module: +- To write a new event source module, it must be written in python. Event sources should be a function that passes no parameters and returns a list of events with required variables detailed above. If the event source has additional requirements, that should be added to `requirements.txt`. The event source should detail any specific run instructions in it's own documentation. Look at `test_events.py` for bare-minimum event list output. +- To run a new event source, import the new module from new event source and add the function call to `event_list`. + +## Architecture: +### Event Class: +**Required Variables** +- `name` (string): Title of the event. +- `location` (string): Location of the event either in full detail (e.g. `"111 test st, city, country, postcode"`) to be formatted by `geopy` module and `format_location()` in event sink, or in `"city, state/territory, country"` format with state/territory details included if the location is in Australia, Canada or United States. See code sample included below for an example of location formatting. + - Note: If location string is in `"city, state, country"` format, for both state/territory and country ISO alpha-2 codes must be used (e.g. AU for Australia, CA for California). +- `date` (date or datetime): Date of event in the locations local time (NOT the local time of where the program is being run). +- `url` (string): Unique URL for event page details. +- `virtual` (boolean): If event is online. +- `organizerName` (string): Name of event organiser. +- `organizerUrl` (string): URL for event organiser webpage. + +**Additional Variable(s) for Internal Use:** +- `duplicate` (boolean): Flag for potential event duplicate based on evaluation during event sink. + +**Code Sample for Creating an Event:** +```py +Event(name="Test Event", location="Melbourne, VIC, AU", date=date.today(), url="website3.com", virtual=True, organizerName="Test Organizer", organizerUrl="testorg.com") +``` + +### Within Scope of Event Sink: +- The event sink will take a list of event objects (see `test_events.py` for example data), format the date and location data (if not done already), filter out events that are outside of the pre-determined 'date window' for the current TWiR issue then sort the events by date and then location alphabetically. After this process the list is then split via virtual or continent, and any potential duplicate events within the list are flagged (through comparison of event data). Finally, the event sink will output the details of the finalised list of events in a pre-determined markdown format, complete with virtual/continent headers. +- Note that potential duplicate events will be flagged with a `** NOTE POTENTIAL DUPLICATE: **` warning immediately preceding the event information. + + +### Out of Scope: +- The purpose of the event sink is to cross-reference and curate data from various sources. It shouldn't be responsible for gathering or adding required fields of data into the Event class. Any edge cases should be managed by the event sources. + +### Expected Output: +Example Output from `test_events.py` data: +``` +### Virtual: + +* 2024-04-12 | Virtual (Dublin, IE) | [Test Organizer](testorg.com) + *[**Test Event 1**](website1.com) + +### North America: + +* 2024-04-03 | New York, NY, US | [Test Organizer](testorg.com) + *[**Test Event 7**](website7.com) +* 2024-04-04 | San Francisco, CA, US | [Test Organizer](testorg.com) + *[**Test Event 6**](website6.com) +* 2024-04-18 | Indianapolis, IN, US | [Test Organizer](testorg.com) + *[**Test Event 2**](website2.com) + +### Oceania: + +* 2024-04-04 | Sydney, NSW, AU | [Test Organizer](testorg.com) + *[**Test Event 4**](website4.com) +``` diff --git a/tools/events-automation/country_code_to_continent.py b/tools/events-automation/country_code_to_continent.py new file mode 100644 index 000000000..03be89754 --- /dev/null +++ b/tools/events-automation/country_code_to_continent.py @@ -0,0 +1,256 @@ +# Takes an ISO alpha-2 country code and returns the continent. +# Mapping of ISO alpha-2 Country Codes to Continent, from Wikipedia. + +COUNTRY_CODE_TO_CONTINENT = { + 'AB': 'Asia', + 'AD': 'Europe', + 'AE': 'Asia', + 'AF': 'Asia', + 'AG': 'North America', + 'AI': 'North America', + 'AL': 'Europe', + 'AM': 'Asia', + 'AO': 'Africa', + 'AR': 'South America', + 'AS': 'Oceania', + 'AT': 'Europe', + 'AU': 'Oceania', + 'AW': 'North America', + 'AX': 'Europe', + 'AZ': 'Asia', + 'BA': 'Europe', + 'BB': 'North America', + 'BD': 'Asia', + 'BE': 'Europe', + 'BF': 'Africa', + 'BG': 'Europe', + 'BH': 'Asia', + 'BI': 'Africa', + 'BJ': 'Africa', + 'BL': 'North America', + 'BM': 'North America', + 'BN': 'Asia', + 'BO': 'South America', + 'BQ': 'North America', + 'BR': 'South America', + 'BS': 'North America', + 'BT': 'Asia', + 'BV': 'Antarctica', + 'BW': 'Africa', + 'BY': 'Europe', + 'BZ': 'North America', + 'CA': 'North America', + 'CC': 'Asia', + 'CD': 'Africa', + 'CF': 'Africa', + 'CG': 'Africa', + 'CH': 'Europe', + 'CI': 'Africa', + 'CK': 'Oceania', + 'CL': 'South America', + 'CM': 'Africa', + 'CN': 'Asia', + 'CO': 'South America', + 'CR': 'North America', + 'CU': 'North America', + 'CV': 'Africa', + 'CW': 'North America', + 'CX': 'Asia', + 'CY': 'Asia', + 'CZ': 'Europe', + 'DE': 'Europe', + 'DJ': 'Africa', + 'DK': 'Europe', + 'DM': 'North America', + 'DO': 'North America', + 'DZ': 'Africa', + 'EC': 'South America', + 'EE': 'Europe', + 'EG': 'Africa', + 'ER': 'Africa', + 'ES': 'Europe', + 'ET': 'Africa', + 'FI': 'Europe', + 'FJ': 'Oceania', + 'FK': 'South America', + 'FM': 'Oceania', + 'FO': 'Europe', + 'FR': 'Europe', + 'GA': 'Africa', + 'GB': 'Europe', + 'GD': 'North America', + 'GE': 'Asia', + 'GF': 'South America', + 'GG': 'Europe', + 'GH': 'Africa', + 'GI': 'Europe', + 'GL': 'North America', + 'GM': 'Africa', + 'GN': 'Africa', + 'GP': 'North America', + 'GQ': 'Africa', + 'GR': 'Europe', + 'GS': 'South America', + 'GT': 'North America', + 'GU': 'Oceania', + 'GW': 'Africa', + 'GY': 'South America', + 'HK': 'Asia', + 'HM': 'Antarctica', + 'HN': 'North America', + 'HR': 'Europe', + 'HT': 'North America', + 'HU': 'Europe', + 'ID': 'Asia', + 'IE': 'Europe', + 'IL': 'Asia', + 'IM': 'Europe', + 'IN': 'Asia', + 'IO': 'Asia', + 'IQ': 'Asia', + 'IR': 'Asia', + 'IS': 'Europe', + 'IT': 'Europe', + 'JE': 'Europe', + 'JM': 'North America', + 'JO': 'Asia', + 'JP': 'Asia', + 'KE': 'Africa', + 'KG': 'Asia', + 'KH': 'Asia', + 'KI': 'Oceania', + 'KM': 'Africa', + 'KN': 'North America', + 'KP': 'Asia', + 'KR': 'Asia', + 'KW': 'Asia', + 'KY': 'North America', + 'KZ': 'Asia', + 'LA': 'Asia', + 'LB': 'Asia', + 'LC': 'North America', + 'LI': 'Europe', + 'LK': 'Asia', + 'LR': 'Africa', + 'LS': 'Africa', + 'LT': 'Europe', + 'LU': 'Europe', + 'LV': 'Europe', + 'LY': 'Africa', + 'MA': 'Africa', + 'MC': 'Europe', + 'MD': 'Europe', + 'ME': 'Europe', + 'MF': 'North America', + 'MG': 'Africa', + 'MH': 'Oceania', + 'MK': 'Europe', + 'ML': 'Africa', + 'MM': 'Asia', + 'MN': 'Asia', + 'MO': 'Asia', + 'MP': 'Oceania', + 'MQ': 'North America', + 'MR': 'Africa', + 'MS': 'North America', + 'MT': 'Europe', + 'MU': 'Africa', + 'MV': 'Asia', + 'MW': 'Africa', + 'MX': 'North America', + 'MY': 'Asia', + 'MZ': 'Africa', + 'NA': 'Africa', + 'NC': 'Oceania', + 'NE': 'Africa', + 'NF': 'Oceania', + 'NG': 'Africa', + 'NI': 'North America', + 'NL': 'Europe', + 'NO': 'Europe', + 'NP': 'Asia', + 'NR': 'Oceania', + 'NU': 'Oceania', + 'NZ': 'Oceania', + 'OM': 'Asia', + 'OS': 'Asia', + 'PA': 'North America', + 'PE': 'South America', + 'PF': 'Oceania', + 'PG': 'Oceania', + 'PH': 'Asia', + 'PK': 'Asia', + 'PL': 'Europe', + 'PM': 'North America', + 'PR': 'North America', + 'PS': 'Asia', + 'PT': 'Europe', + 'PW': 'Oceania', + 'PY': 'South America', + 'QA': 'Asia', + 'RE': 'Africa', + 'RO': 'Europe', + 'RS': 'Europe', + 'RU': 'Europe', + 'RW': 'Africa', + 'SA': 'Asia', + 'SB': 'Oceania', + 'SC': 'Africa', + 'SD': 'Africa', + 'SE': 'Europe', + 'SG': 'Asia', + 'SH': 'Africa', + 'SI': 'Europe', + 'SJ': 'Europe', + 'SK': 'Europe', + 'SL': 'Africa', + 'SM': 'Europe', + 'SN': 'Africa', + 'SO': 'Africa', + 'SR': 'South America', + 'SS': 'Africa', + 'ST': 'Africa', + 'SV': 'North America', + 'SY': 'Asia', + 'SZ': 'Africa', + 'TC': 'North America', + 'TD': 'Africa', + 'TG': 'Africa', + 'TH': 'Asia', + 'TJ': 'Asia', + 'TK': 'Oceania', + 'TM': 'Asia', + 'TN': 'Africa', + 'TO': 'Oceania', + 'TP': 'Asia', + 'TR': 'Asia', + 'TT': 'North America', + 'TV': 'Oceania', + 'TW': 'Asia', + 'TZ': 'Africa', + 'UA': 'Europe', + 'UG': 'Africa', + 'US': 'North America', + 'UY': 'South America', + 'UZ': 'Asia', + 'VC': 'North America', + 'VE': 'South America', + 'VG': 'North America', + 'VI': 'North America', + 'VN': 'Asia', + 'VU': 'Oceania', + 'WF': 'Oceania', + 'WS': 'Oceania', + 'XK': 'Europe', + 'YE': 'Asia', + 'YT': 'Africa', + 'ZA': 'Africa', + 'ZM': 'Africa', + 'ZW': 'Africa', +} + + +def country_code_to_continent(country_code): + # Returns the continent a country code belongs to. + return COUNTRY_CODE_TO_CONTINENT[country_code] + diff --git a/tools/events-automation/country_to_abbrev.py b/tools/events-automation/country_to_abbrev.py new file mode 100644 index 000000000..6168f18bf --- /dev/null +++ b/tools/events-automation/country_to_abbrev.py @@ -0,0 +1,281 @@ +# Takes a country and returns it's ISO alpha-2 code. + +COUNTRY_TO_ABBREV = { + 'Abkhazia': 'AB', + 'Afghanistan': 'AF', + 'Albania': 'AL', + 'Algeria': 'DZ', + 'American Samoa': 'AS', + 'Andorra': 'AD', + 'Angola': 'AO', + 'Anguilla': 'AI', + 'Antigua and Barbuda': 'AG', + 'Argentina': 'AR', + 'Armenia': 'AM', + 'Aruba': 'AW', + 'Australia': 'AU', + 'Austria': 'AT', + 'Azerbaijan': 'AZ', + 'Bahamas': 'BS', + 'Bahrain': 'BH', + 'Bangladesh': 'BD', + 'Barbados': 'BB', + 'Belarus': 'BY', + 'Belgium': 'BE', + 'Belize': 'BZ', + 'Benin': 'BJ', + 'Bermuda': 'BM', + 'Bhutan': 'BT', + 'Bolivia': 'BO', + 'Bonaire': 'BQ', + 'Bosnia and Herzegovina': 'BA', + 'Botswana': 'BW', + 'Bouvet Island': 'BV', + 'Brazil': 'BR', + 'British Indian Ocean Territory': 'IO', + 'British Virgin Islands': 'VG', + 'Virgin Islands, British': 'VG', + 'Brunei': 'BN', + 'Brunei Darussalam': 'BN', + 'Bulgaria': 'BG', + 'Burkina Faso': 'BF', + 'Burundi': 'BI', + 'Cambodia': 'KH', + 'Cameroon': 'CM', + 'Canada': 'CA', + 'Cape Verde': 'CV', + 'Cayman Islands': 'KY', + 'Central African Republic': 'CF', + 'Chad': 'TD', + 'Chile': 'CL', + 'China': 'CN', + 'Christmas Island': 'CX', + 'Cocos (Keeling) Islands': 'CC', + 'Colombia': 'CO', + 'Comoros': 'KM', + 'Congo': 'CG', + 'Congo, Republic of': 'CG', + 'Republic of the Congo': 'CG', + 'Cook Islands': 'CK', + 'Costa Rica': 'CR', + 'Croatia': 'HR', + 'Cuba': 'CU', + 'Curaçao': 'CW', + 'Cyprus': 'CY', + 'Czech Republic': 'CZ', + 'Congo, Democratic Republic of': 'CD', + 'Democratic Republic of the Congo': 'CD', + 'Denmark': 'DK', + 'Djibouti': 'DJ', + 'Dominica': 'DM', + 'Dominican Republic': 'DO', + 'East Timor': 'TP', + 'Ecuador': 'EC', + 'Egypt': 'EG', + 'El Salvador': 'SV', + 'Equatorial Guinea': 'GQ', + 'Eritrea': 'ER', + 'Estonia': 'EE', + 'Ethiopia': 'ET', + 'Falkland Islands': 'FK', + 'Faroe Islands': 'FO', + 'Fiji': 'FJ', + 'Finland': 'FI', + 'France': 'FR', + 'French Guiana': 'GF', + 'French Polynesia': 'PF', + 'Gabon': 'GA', + 'Gambia': 'GM', + 'Georgia': 'GE', + 'Germany': 'DE', + 'Ghana': 'GH', + 'Gibraltar': 'GI', + 'Greece': 'GR', + 'Greenland': 'GL', + 'Grenada': 'GD', + 'Guadeloupe': 'GP', + 'Great Britain': 'GB', + 'Guam': 'GU', + 'Guatemala': 'GT', + 'Guernsey': 'GG', + 'Guinea': 'GN', + 'Guinea-Bissau': 'GW', + 'Guyana': 'GY', + 'Haiti': 'HT', + 'Heard Island and McDonald Islands': 'HM', + 'Honduras': 'HN', + 'Hong Kong': 'HK', + 'Hungary': 'HU', + 'Iceland': 'IS', + 'India': 'IN', + 'Indonesia': 'ID', + 'Iran': 'IR', + 'Iraq': 'IQ', + 'Ireland': 'IE', + 'Isle of Man': 'IM', + 'Islamic Republic of Iran': 'IR', + 'Israel': 'IL', + 'Italy': 'IT', + 'Ivory Coast': 'CI', + 'Jamaica': 'JM', + 'Japan': 'JP', + 'Jersey': 'JE', + 'Jordan': 'JO', + 'Kazakhstan': 'KZ', + 'Kenya': 'KE', + "Korea, Democratic People's Republic of": 'KP', + 'Kiribati': 'KI', + 'Korea, Republic Of': 'KR', + 'Kosovo': 'XK', + 'Kuwait': 'KW', + 'Kyrgyzstan': 'KG', + 'Laos': 'LA', + "Lao People's Democratic Republic": 'LA', + 'Latvia': 'LV', + 'Lebanon': 'LB', + 'Lesotho': 'LS', + 'Liberia': 'LR', + 'Libya': 'LY', + 'Liechtenstein': 'LI', + 'Lithuania': 'LT', + 'Luxembourg': 'LU', + 'Macau': 'MO', + 'Macedonia': 'MK', + 'Macedonia, The Former Yugoslav Republic Of': 'MK', + 'Madagascar': 'MG', + 'Malawi': 'MW', + 'Malaysia': 'MY', + 'Maldives': 'MV', + 'Mali': 'ML', + 'Malta': 'MT', + 'Marshall Islands': 'MH', + 'Martinique': 'MQ', + 'Mauritania': 'MR', + 'Mauritius': 'MU', + 'Mayotte': 'YT', + 'Mexico': 'MX', + 'Micronesia': 'FM', + 'Micronesia, Federated States of': 'FM', + 'Moldova': 'MD', + 'Moldova, Republic Of': 'MD', + 'Monaco': 'MC', + 'Mongolia': 'MN', + 'Montenegro': 'ME', + 'Montserrat': 'MS', + 'Morocco': 'MA', + 'Mozambique': 'MZ', + 'Myanmar': 'MM', + 'Namibia': 'NA', + 'Nauru': 'NR', + 'Nepal': 'NP', + 'Netherlands': 'NL', + 'New Caledonia': 'NC', + 'New Zealand': 'NZ', + 'Nicaragua': 'NI', + 'Niger': 'NE', + 'Nigeria': 'NG', + 'Niue': 'NU', + 'Norfolk Island': 'NF', + 'North Korea': 'KP', + 'Northern Cyprus': 'CY', + 'Northern Mariana Islands': 'MP', + 'Norway': 'NO', + 'Oman': 'OM', + 'Pakistan': 'PK', + 'Palau': 'PW', + 'Palestine': 'PS', + 'Panama': 'PA', + 'Papua New Guinea': 'PG', + 'Paraguay': 'PY', + 'Peru': 'PE', + 'Philippines': 'PH', + 'Poland': 'PL', + 'Portugal': 'PT', + 'Puerto Rico': 'PR', + 'Qatar': 'QA', + 'Romania': 'RO', + 'Russia': 'RU', + 'Russian Federation': 'RU', + 'Rwanda': 'RW', + 'Réunion': 'RE', + 'Saba': 'BQ', + 'Saint Barthélemy': 'BL', + 'Saint Helena, Ascension and Tristan da Cunha': 'SH', + 'Saint Kitts and Nevis': 'KN', + 'St. Kitts and Nevis': 'KN', + 'Saint Lucia': 'LC', + 'St. Lucia': 'LC', + 'Saint Martin': 'MF', + 'St. Martin': 'MF', + 'Saint Pierre and Miquelon': 'PM', + 'St. Pierre and Miquelon': 'PM', + 'Saint Vincent and the Grenadines': 'VC', + 'St. Vincent and The Grenadines': 'VC', + 'Samoa': 'WS', + 'San Marino': 'SM', + 'Saudi Arabia': 'SA', + 'Senegal': 'SN', + 'Serbia': 'RS', + 'Seychelles': 'SC', + 'Sierra Leone': 'SL', + 'Singapore': 'SG', + 'Sint Eustatius': 'BQ', + 'Slovakia': 'SK', + 'Slovenia': 'SI', + 'Solomon Islands': 'SB', + 'Somalia': 'SO', + 'Somaliland': 'SO', + 'South Africa': 'ZA', + 'South Georgia and the South Sandwich Islands': 'GS', + 'South Korea': 'KR', + 'South Ossetia': 'OS', + 'South Sudan': 'SS', + 'Spain': 'ES', + 'Sri Lanka': 'LK', + 'Sudan': 'SD', + 'Suriname': 'SR', + 'Svalbard': 'SJ', + 'Swaziland': 'SZ', + 'Sweden': 'SE', + 'Switzerland': 'CH', + 'Syria': 'SY', + 'Syrian Arab Republic': 'SY', + 'São Tomé and Príncipe': 'ST', + 'Taiwan': 'TW', + 'Taiwan, Province of China': 'TW', + 'Tajikistan': 'TJ', + 'Tanzania': 'TZ', + 'Tanzania, United Republic Of': 'TZ', + 'Thailand': 'TH', + 'Togo': 'TG', + 'Tokelau': 'TK', + 'Tonga': 'TO', + 'Trinidad and Tobago': 'TT', + 'Tunisia': 'TN', + 'Turkey': 'TR', + 'Turkmenistan': 'TM', + 'Turks and Caicos Islands': 'TC', + 'Turks and Caicos': 'TC', + 'Tuvalu': 'TV', + 'Uganda': 'UG', + 'Ukraine': 'UA', + 'United Arab Emirates': 'AE', + 'United Kingdom': 'GB', + 'United States Virgin Islands': 'VI', + 'United States': 'US', + 'United States of America': 'US', + 'Uruguay': 'UY', + 'Uzbekistan': 'UZ', + 'Vanuatu': 'VU', + 'Venezuela': 'VE', + 'Vietnam': 'VN', + 'Wallis and Futuna': 'WF', + 'Yemen': 'YE', + 'Zambia': 'ZM', + 'Zimbabwe': 'ZW', + 'Åland Islands': 'AX', +} + + +def country_to_abbrev(country): + return COUNTRY_TO_ABBREV[country] diff --git a/tools/events-automation/event.py b/tools/events-automation/event.py new file mode 100644 index 000000000..3af3037dd --- /dev/null +++ b/tools/events-automation/event.py @@ -0,0 +1,47 @@ +import datetime +from geopy.geocoders import Nominatim +from state_territory_to_abbrev import au_state_territory_to_abbrev, us_state_to_abbrev, ca_state_territory_to_abbrev + +class Event(): + def __init__(self, name, location, date, url, virtual, organizerName, organizerUrl, duplicate=False) -> None: + self.name = name + self.location = location + self.date = date + self.url = url + self.virtual = virtual + self.organizerName = organizerName + self.organizerUrl = organizerUrl + self.duplicate = duplicate + + def to_markdown_string(self) -> str: + if self.virtual: + return f'* {self.date} | Virtual ({self.location}) | [{self.organizerName}]({self.organizerUrl})\n\t*[**{self.name}**]({self.url})' + else: + return f'* {self.date} | {self.location} | [{self.organizerName}]({self.organizerUrl})\n\t*[**{self.name}**]({self.url})' + + def format_date(self): + # Formats datetime data into date. + if isinstance(self.date, datetime.datetime): + self.date = self.date.date() + + def format_location(self): + # Formats location data into (city, +/-state, country). + geocoder = Nominatim(user_agent="TWiR", timeout=5) + locationData = geocoder.geocode(self.location, language="en", addressdetails=True).raw["address"] + + country_code, city = locationData["country_code"].upper(), locationData.get("city", locationData.get("town", locationData.get("village", "**NO CITY DATA**"))) + if country_code in ["AU", "CA", "US"]: + state = locationData.get("state", locationData.get("territory", "**NO STATE DATA**")) + if state == "**NO STATE DATA**": + state_abbrev = state + elif country_code == "AU": + state_abbrev = au_state_territory_to_abbrev(state) + elif country_code == "CA": + state_abbrev = ca_state_territory_to_abbrev(state) + elif country_code == "US": + state_abbrev = us_state_to_abbrev(state) + self.location = f'{city}, {state_abbrev}, {country_code}' + else: + self.location = f'{city}, {country_code}' + + diff --git a/tools/events-automation/generate_events_meetup.py b/tools/events-automation/generate_events_meetup.py new file mode 100644 index 000000000..8b375f44f --- /dev/null +++ b/tools/events-automation/generate_events_meetup.py @@ -0,0 +1,283 @@ +import requests +import datetime +import concurrent.futures +import csv + +from jwt_auth import generate_signed_jwt +from urllib.parse import urlsplit +from geopy.geocoders import Nominatim +from event import Event + +def authenticate(): + """ + Handles the OAuth 2.0 authentication process. + Returns obtaining access and refresh tokens from the Meetup API + """ + # API Configuration: + URL = "https://secure.meetup.com/oauth2/access" + headers = { + "Content-Type": "application/x-www-form-urlencoded" + } + body = { + "grant_type": "urn:ietf:params:oauth:grant-type:jwt-bearer", + "assertion": generate_signed_jwt() + } + + # Make a request for access and refresh tokens + response = requests.post(url=URL, headers=headers, data=body) + if response.status_code == 200: + access_token = response.json().get("access_token") + refresh_token = response.json().get("refresh_token") + return access_token, refresh_token + else: + print("Failed to obtain access token") + print("Status Code:", response.status_code) + print("Response:", response.text) + return None, None + +# Initialize variables for querying and formatting data: +#ACCESS_TOKEN, REFRESH_TOKEN = authenticate() +# initialize Nominatim API +GEOLOCATOR = Nominatim(user_agent="TWiR") + +def fetch_groups(endCursor=""): + """ + Returns the response from the API call, which includes data on groups matching the criteria specified in the GraphQL query. + :type endCursor: An optional string parameter used for pagination, indicating the starting point of the query for fetching subsequent pages of results + :rtype: requests.Response + """ + + # API Configuration: + # Sets the API endpoint and constructs headers using an access token for authentication. + URL = "https://api.meetup.com/gql" + access_token, refresh_token = ACCESS_TOKEN, REFRESH_TOKEN + + if not access_token: + print("Authentication failed, cannot proceed to fetch events.") + return + + # Sets the content type to application/json for the request body. + headers = { + "Authorization": f"Bearer {access_token}", + "Content-Type": "application/json", + } + + # GraphQL Query: + # Below is a GraphQL query that requests information about groups such as ID, name, link, URL name, latitude, and longitude. + data = { + "query": """ + query ( + $searchGroupInput: ConnectionInput!, + $searchGroupFilter: SearchConnectionFilter!, + $sortOrder: KeywordSort! + ) { + keywordSearch( + input: $searchGroupInput, + filter: $searchGroupFilter, + sort: $sortOrder + ) { + pageInfo { + hasNextPage + endCursor + } + edges { + node { + result { + ... on Group { + id + name + link + urlname + latitude + longitude + } + } + } + } + } + } + """, + # The query filters results based on the keyword "Rust" and sorts them by relevance + "variables": { + "searchGroupFilter": { + "query": "Rust", + "lat": 0.0, + "lon": 0.0, + "radius": 20000, + "source": "GROUPS" + }, + "searchGroupInput": { + "first": 200, + "after": endCursor + }, + "sortOrder":{ + "sortField": "RELEVANCE" + } + } + } + return requests.post(url=URL, headers=headers, json=data) + +def get_rush_groups() -> dict: + """ + Returns a dictionary where each key represents the unique ID of a group, and the corresponding value is another dictionary containing details about the group such as name, link, URL name, latitude, and longitude + :rtype: dict + """ + endCursor = None + groups = dict() + while True: + response = fetch_groups(endCursor).json() + data = response['data'] + edges = data['keywordSearch']['edges'] + pageInfo = data['keywordSearch']['pageInfo'] + for node in edges: + group = node["node"]["result"] + if not (group["id"] in groups): + groups[group["id"]] = group + if pageInfo['hasNextPage']: + endCursor = pageInfo['endCursor'] + else: + break + return groups + +def get_known_rush_groups(fileName="rust_meetup_groups.csv") -> dict: + """ + Returns a dictionary represents all groups from a specified CSV file + :type fileName: Name or Path of the CSV file that contains the URLs and locations of the groups. + """ + + # Reads the CSV file, specifically extracting data from the 'url' and 'location' columns + groups = dict() # main dictionary that stores all information of different groups + df = pd.read_csv(fileName, header=0, usecols=['url', 'location']) + + # Extracting the url name of known Rust groups + # Format of extracting the URL: + # [source](https://stackoverflow.com/questions/35616434/how-can-i-get-the-base-of-a-url-in-python) + # https://www.meetup.com/seattle-rust-user-group/ + # split_url.scheme "http" + # split_url.netloc "www.meetup.com" + # split_url.path "/seattle-rust-user-group/" + for index, row in df.iterrows(): + group = {} + group["link"] = row["url"] + split_url = urlsplit(group["link"]) + group["urlname"] = (split_url.path).replace("/", "") + group["location"] = row["location"] + groups[index] = group + return groups + +def get_20_events(groups) -> list[Event]: + """ + Returns a list where each element is an instance of the Event class, representing event data from the Meetup API + :type groups: A dictionary of groups where each entry contains the group's URL name to make an API request + :rtype: dict + """ + events = [] # main list to store data about each fetched event. + + # API Configuration: + URL = "https://api.meetup.com/gql" + access_token, refresh_token = ACCESS_TOKEN, REFRESH_TOKEN + + if not access_token: + print("Authentication failed, cannot proceed to fetch events.") + return + + headers = { + "Authorization": f"Bearer {access_token}", + "Content-Type": "application/json", + } + + # Constructs and sends a GraphQL query for each group to fetch up to 20 upcoming events from the Meetup API using the group's URL name + data = {} + for group in groups.values(): + urlName = group["urlname"] + data = { + "query": """ + query ($urlName: String!, $searchEventInput: ConnectionInput!) { + groupByUrlname(urlname: $urlName) { + upcomingEvents(input: $searchEventInput, sortOrder: ASC) { + pageInfo { + hasNextPage + endCursor + } + edges { + node { + id + title + dateTime + eventUrl + venue { + venueType + lat + lng + } + } + } + } + } + } + """, + "variables": { + "urlName": urlName, + "searchEventInput": { + "first": 20 + } + } + } + response = requests.post(url=URL, headers=headers, json=data) + data = response.json()["data"] + + # Constructs Event with attributes such as title, location, date, URL, and organizer details + if data: + searchGroupByUrlname = data["groupByUrlname"] + if searchGroupByUrlname: + edges = searchGroupByUrlname["upcomingEvents"]["edges"] + if edges: + for edge in edges: + node = edge["node"] + if node: + venue = node["venue"] + # TODO: Handle events don't have venue: + # 1. Flagging the events and they will have to be check manually, + # 2. Putting them in separate list to check + # (for now ignore those events) + if venue: + name = node["title"] + virtual = True + if venue["venueType"] != "online": + virtual = False + + # Convert obtained latitude and longitude of an event to formatted location + address = (GEOLOCATOR.reverse(str(venue["lat"]) +","+ str(venue["lng"]))).raw["address"] + location = format_location(address) + date = datetime.datetime.fromisoformat(node["dateTime"]).date() + url = node["eventUrl"] + organizerName = group.get("name", urlName) + organizerUrl = group["link"] + events.append(Event(name, location, date, url, virtual, organizerName, organizerUrl)) + return events + +def format_location(address) -> str: + """ + Helper method to format address of events with required components for a location + :rtype: string + """ + if not address: + return "No location" + + # All components for a location + components = ['road', 'city', 'state', 'postcode', 'country'] + + # Get available components, otherwise replace missing component with an empty string + location = [address.get(component, "") for component in components] + + + return ','.join(location) if location else "No location" + +def get_events() -> list[Event]: + """ + Returns a list of Event objects querying from known, and Meetup API Rust groups + :rtype: list[Event] + """ + # TODO: once the handling events without venue successful, get events_meetup_groups = get_20_events(get_rush_groups()) + events_known_groups = get_20_events(get_known_rush_groups()) + return events_known_groups diff --git a/tools/events-automation/jwt_auth.py b/tools/events-automation/jwt_auth.py new file mode 100644 index 000000000..c32b86122 --- /dev/null +++ b/tools/events-automation/jwt_auth.py @@ -0,0 +1,85 @@ +import jwt +import os +import datetime + +#from dotenv import load_dotenv +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.backends import default_backend + +# Automate loading environment variables in Python script, make them accessible to the project +#load_dotenv() + +def get_PEM_private_key(): + """ + Loads the PRIVATE_KEY in string from .env file. + Returns it in PEM-formatted bytes + """ + pem_bytes = (os.getenv('PRIVATE_KEY', "")).encode() + return pem_bytes + +def get_RSA_private_key(): + """ + Deserializes and sign the private key in PEM-formatted in bytes to an RSA private key object using cryptographic operations. + Returns the RSA private key object + """ + private_key = serialization.load_pem_private_key( + get_PEM_private_key(), password=None, backend=default_backend() + ) + return private_key + +def get_RSA_public_key(): + """ + Returns the corresponding RSA public key object from RSA private_key + """ + public_key = get_RSA_private_key().public_key() + return public_key + +def get_PEM_public_key(): + """ + Returns the public key in in PEM-formatted in bytes using RSA public key, to verify digital signatures + """ + pem_bytes = (get_RSA_public_key().public_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PublicFormat.SubjectPublicKeyInfo + )).decode() + return pem_bytes + +# This function is essential for authorize step when calling Meetup API +def generate_signed_jwt(): + """ + Generates a JWT: + Encodes and signs the payload using RS256 and the private RSA key, forming a base64-url encoded header, payload, and signature. + Then returns it. + """ + AUTHORIZED_MEMBER_ID = os.getenv('AUTHORIZED_MEMBER_ID', "") # the member id that owns the OAuth Client + CLIENT_KEY = os.getenv('CLIENT_KEY', "") + private_key = get_RSA_private_key() + payload = { + "sub": AUTHORIZED_MEMBER_ID, + "iss": CLIENT_KEY, + "aud": "api.meetup.com", + "exp": (datetime.datetime.utcnow() + datetime.timedelta(hours=24)).timestamp() + } + return jwt.encode( + payload=payload, + key=private_key, + algorithm="RS256" + ) + +def decode_and_validate_jwt(): + """ + Checks/Validates the signed jwt. + Returns a decoded jwt payload/claim + """ + token = generate_signed_jwt() + pem_public_key = get_PEM_public_key() + try: + payload = jwt.decode( + token, + key=pem_public_key, + algorithms="RS256", + audience="api.meetup.com" + ) + return payload + except ExpiredSignatureError as error: + print(f'Unable to decode the token, error: {error}') diff --git a/tools/events-automation/main.py b/tools/events-automation/main.py new file mode 100644 index 000000000..6f5c84295 --- /dev/null +++ b/tools/events-automation/main.py @@ -0,0 +1,96 @@ +# import all the event sources & event sink +# collect all the events from the event sources +# call event sink with our collected events +# print to console / output to file formatted markdown + +from test_events import get_test_events +from datetime import date, timedelta +from country_code_to_continent import country_code_to_continent +from generate_events_meetup import get_events as get_meetup_events + +# TODO: Flagged events list handling. + +def main(): + # Get Events list from Event Sources. + event_list = get_meetup_events() + + # Format date and location data. + format_data(event_list) + + # Remove events outside of date range. + date_window_filter(event_list) + + # Sort remaining events by date, then location. + event_list.sort(key=lambda event: (event.date, event.location)) + + # Flag potential duplicate events. + potential_duplicate(event_list) + + # Group by virtual or by continent. + event_list = group_virtual_continent(event_list) + + # Output Sorted Event List. + output_to_screen(event_list) + + +def output_to_screen(event_list): + # Prints sorted Event List to terminal screen. + for key, value in event_list.items(): + if len(value) == 0: + continue + else: + print(f'### {key}:\n') + + # Output event details + for event in value: + if event.duplicate: + print("** NOTE POTENTIAL DUPLICATE: **") + print(event.to_markdown_string()) + print() + + +def format_data(event_list): + # Formats date and location data into specified format. + for event in event_list: + event.format_date() + event.format_location() + + +def date_window_filter(event_list): + # Removes Events that are outside current date window. + # Date window = closest wednesday + 5 weeks. + start_date = date.today() + while start_date.weekday() != 2: + start_date = start_date + timedelta(days=1) + + for event in event_list: + if not (start_date <= event.date <= start_date + timedelta(weeks=5)): + event_list.remove(event) + + +def group_virtual_continent(event_list): + # Return dictionary of events separated in virtual and by continent. + separated_event_list = {} + + for event in event_list: + # Separates Events by Virtual or by Continent + key = "Virtual" if event.virtual else country_code_to_continent(event.location[-2:]) + separated_event_list.setdefault(key, []).append(event) + + return separated_event_list + + +def potential_duplicate(event_list): + # Identifies possible duplicate Events within Event List. + for i in range(len(event_list)): + for j in range(i+1, len(event_list)): + if event_list[i].date == event_list[j].date: + if event_list[i].url == event_list[j].url: + if event_list[i].name == event_list[j].name: + if event_list[i].organizerName == event_list[j].organizerName: + if event_list[i].location == event_list[j].location: + event_list[i].duplicate = True + + +if __name__ == "__main__": + main() diff --git a/tools/events-automation/meetup_events.md b/tools/events-automation/meetup_events.md new file mode 100644 index 000000000..0247c0a74 --- /dev/null +++ b/tools/events-automation/meetup_events.md @@ -0,0 +1,75 @@ +## Guide to Generating Events Using Meetup APIs + +### Introduction +This guide provides step-by-step instructions on how to use Meetup APIs to fetch event data related to Rust programming groups. It outlines the installation of necessary packages, setting up environment variables, and executing the script to obtain event data. + +### Prerequisites +Before you start, ensure you have the following: +- Python installed on your system (Python 3.8 or later recommended). +- `pip` for managing Python packages. +- Access to terminal or command line. + +### Tips for Managing Python Packages + +- **Virtual Environment**: Create a virtual environment using `venv` (built into Python 3) or `virtualenv`. Here's how to activate a virtual environment: + ```bash + # Creating a virtual environment (for Linux/macOS) + python3 -m venv myenv + # Activating the virtual environment (for Linux/macOS) + source myenv/bin/activate + + or + + # Creating a virtual environment (for Windows) + python -m venv myenv + # Activating the virtual environment (for Windows) + myenv\Scripts\activate + ``` +- **Upgrade `pip`**: Ensure your `pip` is up-to-date to avoid installation issues with newer packages: + ```bash + pip install --upgrade pip + ``` + +### Installation +1. **Open Terminal or Command Prompt**: + Navigate to the directory `.../tools/events-automation` where `requirements.txt` file is located. + +2. **Run the Installation Command**: + Execute the following command to install all the packages listed in `requirements.txt`: + ```bash + pip install -r requirements.txt + ``` + +3. **Set Up Environment Variables**: + - Create a `.env` file for project directory. + - Add the following environment variables with your actual values: + ``` + AUTHORIZED_MEMBER_ID= + CLIENT_KEY= + PRIVATE_KEY= + ``` + These values are used for authentication with the Meetup API and to generate JWT tokens securely. + +### Running the Script +To fetch events, run the following command from the project directory `.../tools/events-automation`: +```bash +python3 main.py +``` +This script performs the following operations: +- Authenticates with the Meetup API using JWT. +- Fetches data for known Rust groups from `.../tools/events-automation/rust_meetup_groups.csv` file and Meetup API. +- Filters and formats the event data into a standardized structure. +- Outputs the details of upcoming events. + +### Example Output Format +The script outputs event details in the following format: +``` +* 2025-05-08T19:00+02:00 | Virtual (,,Tuvalu) | [rust-noris](TODO: ORGANISER URL HERE) + *[**Rust Nürnberg online**](https://www.meetup.com/rust-noris/events/gmkpctyhchblb) +``` +This format includes the date and time of the event, the location (virtual in this case), and links to both the organizer's profile and the event itself. + +### Challenges and Considerations +- **Data Accuracy**: The script uses "Rust" as a keyword for searching groups and events. This can sometimes pull in events that merely mention Rust but aren't directly related to Rust programming in the event descriptions. +- **Event Data Completeness**: Not all events have complete venue information, especially for virtual events. The script currently ignores handling missing data, but the ideal is flagging such events for manual review or excluding them from the final output. +- **API Limitations**: The Meetup API has rate limits and other constraints that may affect how frequently you can fetch data. \ No newline at end of file diff --git a/tools/events-automation/requirements.txt b/tools/events-automation/requirements.txt new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/tools/events-automation/requirements.txt @@ -0,0 +1 @@ + diff --git a/tools/events-automation/rust_meetup_groups.csv b/tools/events-automation/rust_meetup_groups.csv new file mode 100644 index 000000000..d6785b2a7 --- /dev/null +++ b/tools/events-automation/rust_meetup_groups.csv @@ -0,0 +1,118 @@ +name,url,location,non-meetup.com? +Rust London User Group,https://www.meetup.com/rust-london-user-group/,"London, 17, United Kingdom",no. +Rust NYC,https://www.meetup.com/rust-nyc/,"New York, NY, USA",no. +Rust Bay Area,https://www.meetup.com/Rust-Bay-Area/,"San Francisco, CA, USA",no. +Rust India,https://www.meetup.com/rustox/,"Bangalore, India",no. +Rust Berlin,https://www.meetup.com/rust-berlin/,"Berlin, Germany",no. +Seattle Rust User Group,https://www.meetup.com/seattle-rust-user-group/,"Seattle, WA, USA",no. +Rust Moscow,https://www.meetup.com/Rust-%D0%B2-%D0%9C%D0%BE%D1%81%D0%BA%D0%B2%D0%B5/,"Moscow, Russia",no. +Rust Paris,https://www.meetup.com/Rust-Paris/,"Paris, France",no. +Rust Nederland,https://www.meetup.com/rust-nederland/,"Amsterdam, Netherlands",no. +Rust Munich,https://www.meetup.com/rust-munich/,"München, Germany",no. +Vancouver Rust,https://www.meetup.com/vancouver-rust/,"Vancouver, BC, Canada",no. +Rust Chennai,https://www.meetup.com/mad-rs/,"Chennai, India",no. +Rust Linz,https://www.meetup.com/de-DE/rust-linz/,"Linz, Austria",no. +Boston Rust Meetup,https://www.meetup.com/bostonrust/,"Boston, MA, USA",no. +Rust São Paulo Meetup,https://www.meetup.com/Rust-Sao-Paulo-Meetup/,"São Paulo, Brazil",no. +Rust Language Hyderabad,https://www.meetup.com/Rust-Hyderabad/,"Hyderabad, India",no. +Rust Dublin,https://www.meetup.com/rust-dublin/,"Dublin, Ireland",no. +Rust Zurich,https://www.meetup.com/rust-zurich/,"Zürich, Switzerland",no. +Rust Melbourne,https://www.meetup.com/Rust-Melbourne/,"Melbourne, Australia",no. +Rust Argentina,https://www.meetup.com/rust-argentina/,"Buenos Aires, Argentina",no. +Rust TLV,https://www.meetup.com/rust-tlv/,"Tel Aviv-Yafo, Israel",no. +Rust and C++ Cardiff,https://www.meetup.com/rust-and-c-plus-plus-in-cardiff/,"Cardiff, X5, United Kingdom",no. +Rust MX,https://www.meetup.com/Rust-MX/,"México City, Mexico",no. +Rust Denver,https://www.meetup.com/Rust-Boulder-Denver/,"Denver, CO, USA",no. +Rust Delhi,https://www.meetup.com/rustdelhi/,"Delhi, India",no. +Rust DC,https://www.meetup.com/RustDC/,"Washington, DC, USA",no. +Rust Sydney,https://www.meetup.com/Rust-Sydney/,"Sydney, Australia",no. +Rust Oslo,https://www.meetup.com/Rust-Oslo/,"Oslo, Norway",no. +Dallas Rust User Meetup,https://www.meetup.com/dallasrust/,"Dallas, TX, USA",no. +Rust Cologne,https://www.meetup.com/rustcologne/,"Köln, Germany",no. +Stockholm Rust,https://www.meetup.com/stockholm-rust/,"Espoo, Finland",no. +PDXRust,https://www.meetup.com/PDXRust/,"Portland, OR, USA",no. +Tokyo Rust Meetup,https://www.meetup.com/tokyo-rust-meetup/,"Tokyo, Japan",no. +Rust Sthlm,https://www.meetup.com/ruststhlm/,"Stockholm, Sweden",no. +BcnRust,https://www.meetup.com/bcnrust/,"Barcelona, Spain",no. +Rust Toronto,https://www.meetup.com/Rust-Toronto/,"Toronto, ON, Canada",no. +Rust Language Milano,https://www.meetup.com/Rust-lang-Milano/,"Milano, MI, Italy",no. +Rust Meetup Hamburg,https://www.meetup.com/Rust-Meetup-Hamburg/,"Hamburg, Germany",no. +Rust AKL,https://www.meetup.com/rust-akl/,"Auckland, New Zealand",no. +Columbus Rust Society,https://www.meetup.com/columbus-rs/,"Columbus, OH, USA",no. +Finland Rust-lang Group,https://www.meetup.com/Finland-Rust-Meetup/,"Helsinki, Finland",no. +Rust ATX,https://www.meetup.com/rust-atx/,"Austin, TX, USA",no. +Rust Warsaw,https://www.meetup.com/Rust-Warsaw/,"Warsaw, Poland",no. +Rust Nuremberg,https://www.meetup.com/rust-noris/,"Nürnberg, Germany",no. +Utah Rust,https://www.meetup.com/utah-rust/,"Lehi, UT, USA",no. +Rust Los Angeles,https://www.meetup.com/Rust-Los-Angeles/,"Los Angeles, CA, USA",no. +Copenhagen Rust Community,https://www.meetup.com/copenhagen-rust-community/,"Copenhagen, Denmark",no. +MadRust,https://www.meetup.com/MadRust/,"Madrid, Spain",no. +Rust Vienna,https://www.meetup.com/rust-vienna/,"Vienna, Austria",no. +San Diego Rust,https://www.meetup.com/San-Diego-Rust/,"San Diego, CA, USA",no. +Minneapolis Rust Meetup,https://www.meetup.com/minneapolis-rust-meetup/,"Minneapolis, MN, USA",no. +Rust-Saar,https://www.meetup.com/Rust-Saar/,"Saarbrücken, Germany",no. +San Francisco Rust Study Group,https://www.meetup.com/san-francisco-rust-study-group/,"San Francisco, CA, USA",no. +Charlottesville Rust Meetup,https://www.meetup.com/charlottesville-rust-meetup/,"Charlottesville, VA, USA",no. +Chicago Rust Meetup,https://www.meetup.com/Chicago-Rust-Meetup/,"Chicago, IL, USA",no. +Rust Brisbane,https://www.meetup.com/Rust-Brisbane/,"Brisbane, Australia",no. +Buffalo Rust Meetup,https://www.meetup.com/buffalo-rust-meetup/,"Buffalo, NY, USA",no. +Rust Roma,https://www.meetup.com/it-IT/Rust-Roma/,"Roma, RM, Italia",no. +Rust Meetup Uruguay,https://www.meetup.com/Rust-Uruguay/,"Montevideo, Uruguay",no. +Indy Rust,https://www.meetup.com/indyrs/,"Indianapolis, IN, USA",no. +Cambridge Rust meetup,https://www.meetup.com/Cambridge-Rust-Meetup/,"Cambridge, C3, United Kingdom",no. +Rust Atlanta,https://www.meetup.com/Rust-ATL/,"Atlanta, GA, USA",no. +Montpellier Rust Meetup,https://www.meetup.com/Montpellier-Rust-Meetup/,"Montpellier, France",no. +Rust KW,https://www.meetup.com/Rust-KW/,"Kitchener, ON, Canada",no. +Rust Wellington,https://www.meetup.com/Wellington-Rust-Meetup/,"Wellington, New Zealand",no. +Desert Rust,https://www.meetup.com/Desert-Rustaceans/,"Phoenix, AZ, USA",no. +Rust Prague,https://www.meetup.com/rust-prague/,"Prague, Czech Republic",no. +Rust Aarhus,https://www.meetup.com/rust-aarhus/,"Aarhus, Denmark",no. +Rust Nigeria,https://www.meetup.com/rust-meetup-group/,"Lagos, Nigeria",no. +Rust Lille,https://www.meetup.com/meetup-group-zgphbyet/,"Lille, France",no. +Rust Montréal,https://www.meetup.com/Rust-Montreal/,"Montréal, QC, Canada",no. +Rust Vilnius,https://www.meetup.com/Rust-in-Vilnius/,"Vilnius, Lithuania",no. +Rust Medellin,https://www.meetup.com/rust-medellin/,"Medellin, Colombia",no. +Rust Rhein-Main,https://www.meetup.com/rust-rhein-main/,"Darmstadt, Germany",no. +Belgium Rust user group,https://www.meetup.com/belgium-rust-user-group/,"Brussels, Belgium",no. +Rust Basel,https://www.meetup.com/rust-basel/,"Basel, Switzerland",no. +impl Zagreb for Rust,https://www.meetup.com/zagreb-rust-meetup/,"Zagreb, Croatia",no. +Cap Hill Rust Coding/Hacking/Learning,https://www.meetup.com/cap-hill-rust/,"Seattle, WA, USA",no. +Oxford Rust Meetup Group,https://www.meetup.com/oxford-rust-meetup-group/,"Oxford, K2, United Kingdom",no. +Mountain View Rust Meetup,https://www.meetup.com/mv-rust-meetup/,"Mountain View, CA, USA",no. +Rust Franken,https://www.meetup.com/rust-nerf/,"Erlangen, Germany",no. +Rust Bern,https://www.meetup.com/rust-bern/,"Bern, Switzerland",no. +Rust Hack & Learn Karlsruhe,https://www.meetup.com/Rust-Hack-Learn-Karlsruhe/,"Karlsruhe, Germany",no. +Deep Dish Rust,https://www.meetup.com/deep-dish-rust/,"Chicago, IL, USA",no. +Vilnius Rust and Go Meetup Group,https://www.meetup.com/vilnius-rust-go-meetup-group/,"Vilnius, Lithuania",no. +Rust Czech Republic,https://www.meetup.com/rust-czech-republic/,"Prague, Czech Republic",no. +Ottawa Rust Language Meetup,https://www.meetup.com/Ottawa-Rust-Language-Meetup/,"Ottawa, ON, Canada",no. +Rust Belfast Meetup,https://www.meetup.com/Rust-Belfast-Meetup/,"Belfast, R3, United Kingdom",no. +Pasadena Thursday Go / Rust,https://www.meetup.com/thursday-go/,"Pasadena, CA, USA",no. +Hack-away with Rust,https://www.meetup.com/hack-away-with-rust/,"Espoo, Finland",no. +Rust Colombia,https://www.meetup.com/rust-colombia/,"Medellín, Colombia",no. +Rust Gdansk,https://www.meetup.com/rust-gdansk/,"Gdansk, Poland",no. +RustSchool Rotterdam,https://www.meetup.com/RustSchool-Rotterdam/,"Rotterdam, Netherlands",no. +Rust Rio,https://www.meetup.com/Rust-Rio/,"Rio de Janeiro, Brazil",no. +Rust Lang Comunidad Mexico,https://www.meetup.com/rustlangmx/,"Guadalajara, Mexico",no. +Rust - Modern Systems Programming in Leipzig,https://www.meetup.com/rust-modern-systems-programming-in-leipzig/,"Leipzig, Germany",no. +Rust Perth Meetup Group,https://www.meetup.com/perth-rust-meetup-group/,"Perth, Australia",no. +Rust Trondheim,https://www.meetup.com/rust-trondheim/,"Trondheim, Norway",no. +Rust Tbilisi,https://www.meetup.com/tbilisi-rustaceans/,"Tbilisi, Georgia",no. +Rust Halifax,https://www.meetup.com/rust-tell-halifax/,"Halifax, NS, Canada",no. +Rust Seoul,https://www.meetup.com/Rust-Seoul/,"Seoul, South Korea",no. +Canberra Rust User Group,https://www.meetup.com/rust-canberra/,"Canberra, Australia",no. +Budapest Rust Meetup Group,https://www.meetup.com/budapest-rust-meetup-group/,"Budapest, Hungary",no. +Paessler Rust Camp 2024,https://www.meetup.com/paessler-rust-camp-2024/,"Nürnbert, Germany",no. +Rust Taiwan Community,https://github.com/rust-tw/meetup,"Taipei, Taiwan",x +Rust Bordeaux,https://www.meetup.com/bordeaux-rust/,"Bordeaux, France",no. +Rust Girona,https://www.meetup.com/rust-girona/,"Girona, Spain",no. +Rust Circle Kampala,https://www.eventbrite.com/o/rust-circle-kampala-65249289033,Online,x +Rust Würzburg Meetup Group,https://www.meetup.com/rust-wurzburg-meetup-group/,"Würzburg, Germany",no. +Spokane,https://www.meetup.com/spokane-rust/,"Spokane, WA, USA",no. +Calgary Rust,https://www.eventbrite.ca/o/rust-calgary-63449860593,"Calgary, AB, Canada",x +Rust Indonesia,https://github.com/rustid/meetup,Indonesia,no. +Chengdu,https://github.com/RPG-Alex/rust-chengdu,"Chengdu, China",no. +Rust Lyon,https://www.meetup.com/fr-FR/rust-lyon/,"Lyon, France",no. +Rust Meetup Augsburg,https://www.meetup.com/de-DE/rust-meetup-augsburg/,"Augsburg, Germany",no. + Bratislava Rust Meetup Group,https://www.meetup.com/bratislava-rust-meetup-group/,"Bratislava, Slovakia",no. +Music City Rust Developers,https://www.meetup.com/music-city-rust-developers/,"Nashville, TN, USA",no. \ No newline at end of file diff --git a/tools/events-automation/state_territory_to_abbrev.py b/tools/events-automation/state_territory_to_abbrev.py new file mode 100644 index 000000000..0d74642f8 --- /dev/null +++ b/tools/events-automation/state_territory_to_abbrev.py @@ -0,0 +1,102 @@ +# Returns the abbreviated version of state/territory name for AU, CA, and US. +# Information from Wikipedia. + +US_STATE_TO_ABBREV = { + "Alabama": "AL", + "Alaska": "AK", + "Arizona": "AZ", + "Arkansas": "AR", + "California": "CA", + "Colorado": "CO", + "Connecticut": "CT", + "Delaware": "DE", + "Florida": "FL", + "Georgia": "GA", + "Hawaii": "HI", + "Idaho": "ID", + "Illinois": "IL", + "Indiana": "IN", + "Iowa": "IA", + "Kansas": "KS", + "Kentucky": "KY", + "Louisiana": "LA", + "Maine": "ME", + "Maryland": "MD", + "Massachusetts": "MA", + "Michigan": "MI", + "Minnesota": "MN", + "Mississippi": "MS", + "Missouri": "MO", + "Montana": "MT", + "Nebraska": "NE", + "Nevada": "NV", + "New Hampshire": "NH", + "New Jersey": "NJ", + "New Mexico": "NM", + "New York": "NY", + "North Carolina": "NC", + "North Dakota": "ND", + "Ohio": "OH", + "Oklahoma": "OK", + "Oregon": "OR", + "Pennsylvania": "PA", + "Rhode Island": "RI", + "South Carolina": "SC", + "South Dakota": "SD", + "Tennessee": "TN", + "Texas": "TX", + "Utah": "UT", + "Vermont": "VT", + "Virginia": "VA", + "Washington": "WA", + "West Virginia": "WV", + "Wisconsin": "WI", + "Wyoming": "WY", + "District of Columbia": "DC", + "American Samoa": "AS", + "Guam": "GU", + "Northern Mariana Islands": "MP", + "Puerto Rico": "PR", + "United States Minor Outlying Islands": "UM", + "U.S. Virgin Islands": "VI", +} + +CA_STATE_TERRITORY_TO_ABBREV = { + "Alberta": "AB", + "British Columbia": "BC", + "Manitoba": "MB", + "New Brunswick": "NB", + "Newfoundland and Labrador": "NL", + "Northwest Territories": "NT", + "Nova Scotia": "NS", + "Nunavut": "NU", + "Ontario": "ON", + "Prince Edward Island": "PE", + "Quebec": "QC", + "Saskatchewan": "SK", + "Yukon": "YT", +} + +AU_STATE_TERRITORY_TO_ABBREV = { + "New South Wales": "NSW", + "Northern Territory": "NT", + "Queensland": "QLD", + "South Australia": "SA", + "Tasmania": "TAS", + "Victoria": "VIC", + "Western Australia": "WA", +} + +def us_state_to_abbrev(state): + # Returns the abbreviated alpha code for input state. + return US_STATE_TO_ABBREV[state] + + +def ca_state_territory_to_abbrev(state): + # Returns the abbreviated alpha code for input state/territory. + return CA_STATE_TERRITORY_TO_ABBREV[state] + + +def au_state_territory_to_abbrev(state): + # Returns the abbreviated alpha code for input state/territory. + return AU_STATE_TERRITORY_TO_ABBREV[state] diff --git a/tools/events-automation/test_events.py b/tools/events-automation/test_events.py new file mode 100644 index 000000000..bbff39d90 --- /dev/null +++ b/tools/events-automation/test_events.py @@ -0,0 +1,15 @@ +from event import Event +from typing import List +from datetime import date, timedelta + + +def get_test_events() -> List[Event]: + return [Event(name="Test Event 1", location="Dublin, IE", date=date.today() + timedelta(days=2), url="website1.com", virtual=True, organizerName="Test Organizer", organizerUrl="testorg.com"), + Event(name="Test Event 2", location="Indianapolis, IN, US", date=date.today() + timedelta(weeks=2), url="website2.com", virtual=False, organizerName="Test Organizer", organizerUrl="testorg.com"), + Event(name="Test Event 3", location="Melbourne, VIC, AU", date=date.today(), url="website3.com", virtual=True, organizerName="Test Organizer", organizerUrl="testorg.com"), + Event(name="Test Event 4", location="Sydney, NSW, AU", date=date.today(), url="website4.com", virtual=False, organizerName="Test Organizer", organizerUrl="testorg.com"), + Event(name="Test Event 5", location="Melbourne, VIC, AU", date=date.today(), url="website5.com", virtual=False, organizerName="Test Organizer", organizerUrl="testorg.com"), + Event(name="Test Event 6", location="San Francisco, CA, US", date=date.today(), url="website6.com", virtual=False, organizerName="Test Organizer", organizerUrl="testorg.com"), + Event(name="Test Event 7", location="New York, NY, US", date=date.today() - timedelta(days=1), url="website7.com", virtual=False, organizerName="Test Organizer", organizerUrl="testorg.com"), + Event(name="Test Event 7", location="New York, NY, US", date=date.today() - timedelta(days=1), url="website7.com", virtual=False, organizerName="Test Organizer", organizerUrl="testorg.com") + ] From a6fadff49d4a6c9f9d3a03aed388be42b73de9fb Mon Sep 17 00:00:00 2001 From: bdillo Date: Mon, 18 Nov 2024 17:00:52 -0800 Subject: [PATCH 02/18] add dependencies --- tools/events-automation/requirements.txt | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tools/events-automation/requirements.txt b/tools/events-automation/requirements.txt index 8b1378917..08fdb05fd 100644 --- a/tools/events-automation/requirements.txt +++ b/tools/events-automation/requirements.txt @@ -1 +1,11 @@ - +certifi==2024.8.30 +cffi==1.17.1 +charset-normalizer==3.4.0 +cryptography==43.0.3 +geographiclib==2.0 +geopy==2.4.1 +idna==3.10 +pycparser==2.22 +PyJWT==2.10.0 +requests==2.32.3 +urllib3==2.2.3 From 949afda2ed77aeaf8d74e4330954c76d941db3f4 Mon Sep 17 00:00:00 2001 From: bdillo Date: Mon, 18 Nov 2024 17:43:33 -0800 Subject: [PATCH 03/18] minor refactoring --- .../generate_events_meetup.py | 453 +++++++++--------- tools/events-automation/jwt_auth.py | 11 +- tools/events-automation/main.py | 7 +- 3 files changed, 224 insertions(+), 247 deletions(-) diff --git a/tools/events-automation/generate_events_meetup.py b/tools/events-automation/generate_events_meetup.py index 8b375f44f..a5228a0c4 100644 --- a/tools/events-automation/generate_events_meetup.py +++ b/tools/events-automation/generate_events_meetup.py @@ -1,6 +1,5 @@ import requests import datetime -import concurrent.futures import csv from jwt_auth import generate_signed_jwt @@ -8,276 +7,254 @@ from geopy.geocoders import Nominatim from event import Event -def authenticate(): - """ - Handles the OAuth 2.0 authentication process. - Returns obtaining access and refresh tokens from the Meetup API - """ - # API Configuration: - URL = "https://secure.meetup.com/oauth2/access" - headers = { - "Content-Type": "application/x-www-form-urlencoded" - } - body = { - "grant_type": "urn:ietf:params:oauth:grant-type:jwt-bearer", - "assertion": generate_signed_jwt() - } +class TwirMeetupClient: + AUTH_ENDPOINT = "https://secure.meetup.com/oauth2/access" + GQL_ENDPOINT = "https://api.meetup.com/gql" - # Make a request for access and refresh tokens - response = requests.post(url=URL, headers=headers, data=body) - if response.status_code == 200: - access_token = response.json().get("access_token") - refresh_token = response.json().get("refresh_token") - return access_token, refresh_token - else: - print("Failed to obtain access token") - print("Status Code:", response.status_code) - print("Response:", response.text) - return None, None + def __init__(self) -> None: + self._access_token = None + self._refresh_token = None + self._geolocator = Nominatim(user_agent="TWiR") -# Initialize variables for querying and formatting data: -#ACCESS_TOKEN, REFRESH_TOKEN = authenticate() -# initialize Nominatim API -GEOLOCATOR = Nominatim(user_agent="TWiR") - -def fetch_groups(endCursor=""): - """ - Returns the response from the API call, which includes data on groups matching the criteria specified in the GraphQL query. - :type endCursor: An optional string parameter used for pagination, indicating the starting point of the query for fetching subsequent pages of results - :rtype: requests.Response - """ + def authenticate(self): + """ + Handles the OAuth 2.0 authentication process. + Returns obtaining access and refresh tokens from the Meetup API + """ + headers = { + "Content-Type": "application/x-www-form-urlencoded" + } + body = { + "grant_type": "urn:ietf:params:oauth:grant-type:jwt-bearer", + "assertion": generate_signed_jwt() + } - # API Configuration: - # Sets the API endpoint and constructs headers using an access token for authentication. - URL = "https://api.meetup.com/gql" - access_token, refresh_token = ACCESS_TOKEN, REFRESH_TOKEN + # Make a request for access and refresh tokens + response = requests.post(url=self.AUTH_ENDPOINT, headers=headers, data=body) + if response.status_code == 200: + access_token = response.json().get("access_token") + refresh_token = response.json().get("refresh_token") + self._access_token = access_token + self._refresh_token = refresh_token + else: + response.raise_for_status() - if not access_token: - print("Authentication failed, cannot proceed to fetch events.") - return + def fetch_groups(self, endCursor=""): + """ + Returns the response from the API call, which includes data on groups matching the criteria specified in the GraphQL query. + :type endCursor: An optional string parameter used for pagination, indicating the starting point of the query for fetching subsequent pages of results + :rtype: requests.Response + """ - # Sets the content type to application/json for the request body. - headers = { - "Authorization": f"Bearer {access_token}", - "Content-Type": "application/json", - } + # Sets the content type to application/json for the request body. + headers = { + "Authorization": f"Bearer {self._access_token}", + "Content-Type": "application/json", + } - # GraphQL Query: - # Below is a GraphQL query that requests information about groups such as ID, name, link, URL name, latitude, and longitude. - data = { - "query": """ - query ( - $searchGroupInput: ConnectionInput!, - $searchGroupFilter: SearchConnectionFilter!, - $sortOrder: KeywordSort! - ) { - keywordSearch( - input: $searchGroupInput, - filter: $searchGroupFilter, - sort: $sortOrder + # GraphQL Query: + # Below is a GraphQL query that requests information about groups such as ID, name, link, URL name, latitude, and longitude. + data = { + "query": """ + query ( + $searchGroupInput: ConnectionInput!, + $searchGroupFilter: SearchConnectionFilter!, + $sortOrder: KeywordSort! ) { - pageInfo { - hasNextPage - endCursor - } - edges { - node { - result { - ... on Group { - id - name - link - urlname - latitude - longitude + keywordSearch( + input: $searchGroupInput, + filter: $searchGroupFilter, + sort: $sortOrder + ) { + pageInfo { + hasNextPage + endCursor + } + edges { + node { + result { + ... on Group { + id + name + link + urlname + latitude + longitude + } } } } } } - } - """, - # The query filters results based on the keyword "Rust" and sorts them by relevance - "variables": { - "searchGroupFilter": { - "query": "Rust", - "lat": 0.0, - "lon": 0.0, - "radius": 20000, - "source": "GROUPS" - }, - "searchGroupInput": { - "first": 200, - "after": endCursor - }, - "sortOrder":{ - "sortField": "RELEVANCE" + """, + # The query filters results based on the keyword "Rust" and sorts them by relevance + "variables": { + "searchGroupFilter": { + "query": "Rust", + "lat": 0.0, + "lon": 0.0, + "radius": 20000, + "source": "GROUPS" + }, + "searchGroupInput": { + "first": 200, + "after": endCursor + }, + "sortOrder":{ + "sortField": "RELEVANCE" + } } } - } - return requests.post(url=URL, headers=headers, json=data) + return requests.post(url=self.GQL_ENDPOINT, headers=headers, json=data) -def get_rush_groups() -> dict: - """ - Returns a dictionary where each key represents the unique ID of a group, and the corresponding value is another dictionary containing details about the group such as name, link, URL name, latitude, and longitude - :rtype: dict - """ - endCursor = None - groups = dict() - while True: - response = fetch_groups(endCursor).json() - data = response['data'] - edges = data['keywordSearch']['edges'] - pageInfo = data['keywordSearch']['pageInfo'] - for node in edges: - group = node["node"]["result"] - if not (group["id"] in groups): - groups[group["id"]] = group - if pageInfo['hasNextPage']: - endCursor = pageInfo['endCursor'] - else: - break - return groups - -def get_known_rush_groups(fileName="rust_meetup_groups.csv") -> dict: - """ - Returns a dictionary represents all groups from a specified CSV file - :type fileName: Name or Path of the CSV file that contains the URLs and locations of the groups. - """ + def get_rust_groups(self) -> dict: + """ + Returns a dictionary where each key represents the unique ID of a group, and the corresponding value is another dictionary containing details about the group such as name, link, URL name, latitude, and longitude + :rtype: dict + """ + endCursor = None + groups = dict() + while True: + response = self.fetch_groups(endCursor).json() + data = response['data'] + edges = data['keywordSearch']['edges'] + pageInfo = data['keywordSearch']['pageInfo'] + for node in edges: + group = node["node"]["result"] + if not (group["id"] in groups): + groups[group["id"]] = group + if pageInfo['hasNextPage']: + endCursor = pageInfo['endCursor'] + else: + break + return groups - # Reads the CSV file, specifically extracting data from the 'url' and 'location' columns - groups = dict() # main dictionary that stores all information of different groups - df = pd.read_csv(fileName, header=0, usecols=['url', 'location']) + def get_known_rust_groups(self, filename="rust_meetup_groups.csv") -> dict: + """ + Returns a dictionary represents all groups from a specified CSV file + :type fileName: Name or Path of the CSV file that contains the URLs and locations of the groups. + """ + # TODO: this whole method really needs to be cleaned up + groups = dict() # main dictionary that stores all information of different groups - # Extracting the url name of known Rust groups - # Format of extracting the URL: - # [source](https://stackoverflow.com/questions/35616434/how-can-i-get-the-base-of-a-url-in-python) - # https://www.meetup.com/seattle-rust-user-group/ - # split_url.scheme "http" - # split_url.netloc "www.meetup.com" - # split_url.path "/seattle-rust-user-group/" - for index, row in df.iterrows(): - group = {} - group["link"] = row["url"] - split_url = urlsplit(group["link"]) - group["urlname"] = (split_url.path).replace("/", "") - group["location"] = row["location"] - groups[index] = group - return groups + with open(filename, newline='') as csv_file: + csv_reader = csv.reader(csv_file) + for (i, row) in enumerate(csv_reader): + name, url, location, non_meetup = row + group = {} + group["link"] = url + split_url = urlsplit(group["link"]) + group["urlname"] = (split_url.path).replace("/", "") + group["location"] = location + groups[i] = group -def get_20_events(groups) -> list[Event]: - """ - Returns a list where each element is an instance of the Event class, representing event data from the Meetup API - :type groups: A dictionary of groups where each entry contains the group's URL name to make an API request - :rtype: dict - """ - events = [] # main list to store data about each fetched event. + return groups - # API Configuration: - URL = "https://api.meetup.com/gql" - access_token, refresh_token = ACCESS_TOKEN, REFRESH_TOKEN + def get_20_events(self, groups) -> list[Event]: + """ + Returns a list where each element is an instance of the Event class, representing event data from the Meetup API + :type groups: A dictionary of groups where each entry contains the group's URL name to make an API request + :rtype: dict + """ + events = [] # main list to store data about each fetched event. - if not access_token: - print("Authentication failed, cannot proceed to fetch events.") - return - - headers = { - "Authorization": f"Bearer {access_token}", - "Content-Type": "application/json", - } + headers = { + "Authorization": f"Bearer {self._access_token}", + "Content-Type": "application/json", + } - # Constructs and sends a GraphQL query for each group to fetch up to 20 upcoming events from the Meetup API using the group's URL name - data = {} - for group in groups.values(): - urlName = group["urlname"] - data = { - "query": """ - query ($urlName: String!, $searchEventInput: ConnectionInput!) { - groupByUrlname(urlname: $urlName) { - upcomingEvents(input: $searchEventInput, sortOrder: ASC) { - pageInfo { - hasNextPage - endCursor - } - edges { - node { - id - title - dateTime - eventUrl - venue { - venueType - lat - lng + # Constructs and sends a GraphQL query for each group to fetch up to 20 upcoming events from the Meetup API using the group's URL name + data = {} + for group in groups.values(): + urlName = group["urlname"] + data = { + "query": """ + query ($urlName: String!, $searchEventInput: ConnectionInput!) { + groupByUrlname(urlname: $urlName) { + upcomingEvents(input: $searchEventInput, sortOrder: ASC) { + pageInfo { + hasNextPage + endCursor + } + edges { + node { + id + title + dateTime + eventUrl + venue { + venueType + lat + lng + } } } } } } - } - """, - "variables": { - "urlName": urlName, - "searchEventInput": { - "first": 20 + """, + "variables": { + "urlName": urlName, + "searchEventInput": { + "first": 20 + } } } - } - response = requests.post(url=URL, headers=headers, json=data) - data = response.json()["data"] + response = requests.post(url=self.GQL_ENDPOINT, headers=headers, json=data) + data = response.json()["data"] - # Constructs Event with attributes such as title, location, date, URL, and organizer details - if data: - searchGroupByUrlname = data["groupByUrlname"] - if searchGroupByUrlname: - edges = searchGroupByUrlname["upcomingEvents"]["edges"] - if edges: - for edge in edges: - node = edge["node"] - if node: - venue = node["venue"] - # TODO: Handle events don't have venue: - # 1. Flagging the events and they will have to be check manually, - # 2. Putting them in separate list to check - # (for now ignore those events) - if venue: - name = node["title"] - virtual = True - if venue["venueType"] != "online": - virtual = False + # Constructs Event with attributes such as title, location, date, URL, and organizer details + if data: + searchGroupByUrlname = data["groupByUrlname"] + if searchGroupByUrlname: + edges = searchGroupByUrlname["upcomingEvents"]["edges"] + if edges: + for edge in edges: + node = edge["node"] + if node: + venue = node["venue"] + # TODO: Handle events don't have venue: + # 1. Flagging the events and they will have to be check manually, + # 2. Putting them in separate list to check + # (for now ignore those events) + if venue: + name = node["title"] + virtual = True + if venue["venueType"] != "online": + virtual = False - # Convert obtained latitude and longitude of an event to formatted location - address = (GEOLOCATOR.reverse(str(venue["lat"]) +","+ str(venue["lng"]))).raw["address"] - location = format_location(address) - date = datetime.datetime.fromisoformat(node["dateTime"]).date() - url = node["eventUrl"] - organizerName = group.get("name", urlName) - organizerUrl = group["link"] - events.append(Event(name, location, date, url, virtual, organizerName, organizerUrl)) - return events + # Convert obtained latitude and longitude of an event to formatted location + address = (self._geolocator.reverse(str(venue["lat"]) +","+ str(venue["lng"]))).raw["address"] + location = self.format_location(address) + date = datetime.datetime.fromisoformat(node["dateTime"]).date() + url = node["eventUrl"] + organizerName = group.get("name", urlName) + organizerUrl = group["link"] + events.append(Event(name, location, date, url, virtual, organizerName, organizerUrl)) + return events -def format_location(address) -> str: - """ - Helper method to format address of events with required components for a location - :rtype: string - """ - if not address: - return "No location" + def format_location(self, address) -> str: + """ + Helper method to format address of events with required components for a location + :rtype: string + """ + if not address: + return "No location" - # All components for a location - components = ['road', 'city', 'state', 'postcode', 'country'] + # All components for a location + components = ['road', 'city', 'state', 'postcode', 'country'] - # Get available components, otherwise replace missing component with an empty string - location = [address.get(component, "") for component in components] + # Get available components, otherwise replace missing component with an empty string + location = [address.get(component, "") for component in components] - return ','.join(location) if location else "No location" + return ','.join(location) if location else "No location" -def get_events() -> list[Event]: - """ - Returns a list of Event objects querying from known, and Meetup API Rust groups - :rtype: list[Event] - """ - # TODO: once the handling events without venue successful, get events_meetup_groups = get_20_events(get_rush_groups()) - events_known_groups = get_20_events(get_known_rush_groups()) - return events_known_groups + def get_events(self) -> list[Event]: + """ + Returns a list of Event objects querying from known, and Meetup API Rust groups + :rtype: list[Event] + """ + # TODO: once the handling events without venue successful, get events_meetup_groups = get_20_events(get_rush_groups()) + events_known_groups = self.get_20_events(self.get_known_rust_groups()) + return events_known_groups diff --git a/tools/events-automation/jwt_auth.py b/tools/events-automation/jwt_auth.py index c32b86122..2978c52b5 100644 --- a/tools/events-automation/jwt_auth.py +++ b/tools/events-automation/jwt_auth.py @@ -2,20 +2,17 @@ import os import datetime -#from dotenv import load_dotenv from cryptography.hazmat.primitives import serialization from cryptography.hazmat.backends import default_backend -# Automate loading environment variables in Python script, make them accessible to the project -#load_dotenv() def get_PEM_private_key(): """ Loads the PRIVATE_KEY in string from .env file. Returns it in PEM-formatted bytes """ - pem_bytes = (os.getenv('PRIVATE_KEY', "")).encode() - return pem_bytes + with open(os.getenv("PRIVATE_KEY"), 'rb') as pem_file: + return pem_file.read() def get_RSA_private_key(): """ @@ -51,8 +48,8 @@ def generate_signed_jwt(): Encodes and signs the payload using RS256 and the private RSA key, forming a base64-url encoded header, payload, and signature. Then returns it. """ - AUTHORIZED_MEMBER_ID = os.getenv('AUTHORIZED_MEMBER_ID', "") # the member id that owns the OAuth Client - CLIENT_KEY = os.getenv('CLIENT_KEY', "") + AUTHORIZED_MEMBER_ID = os.getenv('AUTHORIZED_MEMBER_ID') # the member id that owns the OAuth Client + CLIENT_KEY = os.getenv('CLIENT_KEY') private_key = get_RSA_private_key() payload = { "sub": AUTHORIZED_MEMBER_ID, diff --git a/tools/events-automation/main.py b/tools/events-automation/main.py index 6f5c84295..960d7c03f 100644 --- a/tools/events-automation/main.py +++ b/tools/events-automation/main.py @@ -6,13 +6,16 @@ from test_events import get_test_events from datetime import date, timedelta from country_code_to_continent import country_code_to_continent -from generate_events_meetup import get_events as get_meetup_events +from generate_events_meetup import TwirMeetupClient # TODO: Flagged events list handling. def main(): + meetup_client = TwirMeetupClient() + meetup_client.authenticate() + # Get Events list from Event Sources. - event_list = get_meetup_events() + event_list = meetup_client.get_events() # Format date and location data. format_data(event_list) From 305acf43a08c0196f73ff7308bb22f68eb06b08a Mon Sep 17 00:00:00 2001 From: bdillo Date: Wed, 20 Nov 2024 16:44:09 -0800 Subject: [PATCH 04/18] minor changes, start adding logging --- .../generate_events_meetup.py | 23 +++++++++++++++---- tools/events-automation/main.py | 8 ++++++- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/tools/events-automation/generate_events_meetup.py b/tools/events-automation/generate_events_meetup.py index a5228a0c4..43a764788 100644 --- a/tools/events-automation/generate_events_meetup.py +++ b/tools/events-automation/generate_events_meetup.py @@ -1,12 +1,16 @@ import requests import datetime import csv +import logging from jwt_auth import generate_signed_jwt from urllib.parse import urlsplit from geopy.geocoders import Nominatim from event import Event +logger = logging.getLogger(__name__) + + class TwirMeetupClient: AUTH_ENDPOINT = "https://secure.meetup.com/oauth2/access" GQL_ENDPOINT = "https://api.meetup.com/gql" @@ -16,11 +20,13 @@ def __init__(self) -> None: self._refresh_token = None self._geolocator = Nominatim(user_agent="TWiR") - def authenticate(self): + def _authenticate(self): """ Handles the OAuth 2.0 authentication process. - Returns obtaining access and refresh tokens from the Meetup API + Sets access and refresh tokens from the Meetup API """ + logger.info("Fetching auth tokens...") + headers = { "Content-Type": "application/x-www-form-urlencoded" } @@ -39,6 +45,14 @@ def authenticate(self): else: response.raise_for_status() + logger.info("Done fetching auth tokens!") + + def _get_access_token(self): + if not self._access_token: + self._authenticate() + + return self._get_access_token + def fetch_groups(self, endCursor=""): """ Returns the response from the API call, which includes data on groups matching the criteria specified in the GraphQL query. @@ -48,7 +62,7 @@ def fetch_groups(self, endCursor=""): # Sets the content type to application/json for the request body. headers = { - "Authorization": f"Bearer {self._access_token}", + "Authorization": f"Bearer {self._get_access_token}", "Content-Type": "application/json", } @@ -147,7 +161,6 @@ def get_known_rust_groups(self, filename="rust_meetup_groups.csv") -> dict: group["urlname"] = (split_url.path).replace("/", "") group["location"] = location groups[i] = group - return groups def get_20_events(self, groups) -> list[Event]: @@ -159,7 +172,7 @@ def get_20_events(self, groups) -> list[Event]: events = [] # main list to store data about each fetched event. headers = { - "Authorization": f"Bearer {self._access_token}", + "Authorization": f"Bearer {self._get_access_token()}", "Content-Type": "application/json", } diff --git a/tools/events-automation/main.py b/tools/events-automation/main.py index 960d7c03f..74a72b531 100644 --- a/tools/events-automation/main.py +++ b/tools/events-automation/main.py @@ -3,6 +3,8 @@ # call event sink with our collected events # print to console / output to file formatted markdown +import logging + from test_events import get_test_events from datetime import date, timedelta from country_code_to_continent import country_code_to_continent @@ -10,9 +12,13 @@ # TODO: Flagged events list handling. +logger = logging.getLogger(__name__) + def main(): + logging.basicConfig(level=logging.INFO) + logger.info("Starting...") + meetup_client = TwirMeetupClient() - meetup_client.authenticate() # Get Events list from Event Sources. event_list = meetup_client.get_events() From d53af886b8078ec3c83a9ed1bff24ab20b10c338 Mon Sep 17 00:00:00 2001 From: bdillo Date: Tue, 26 Nov 2024 18:05:58 -0800 Subject: [PATCH 05/18] remove meetup groups that no longer exist --- tools/events/meetups.json | 77 +-------------------------------------- 1 file changed, 1 insertion(+), 76 deletions(-) diff --git a/tools/events/meetups.json b/tools/events/meetups.json index 193ee98f9..643d82c6b 100644 --- a/tools/events/meetups.json +++ b/tools/events/meetups.json @@ -1,14 +1,11 @@ [ "https://www.meetup.com/8th-light-university/events/", "https://www.meetup.com/altow-academy/events/", - "https://www.meetup.com/ann-arbor-rust-meetup/events/", - "https://www.meetup.com/ardech-drom-dev/events/", "https://www.meetup.com/atx-rustaceans/events/", "https://www.meetup.com/aws-cologne/events/", "https://www.meetup.com/barcelona-free-software/events/", "https://www.meetup.com/bay-area-newsql-database-meetup/events/", "https://www.meetup.com/bcnrust/events/", - "https://www.meetup.com/beijing-infrastructure-meetup/events/", "https://www.meetup.com/belgium-rust-user-group/events/", "https://www.meetup.com/belgrade-rust-meetup-group/events/", "https://www.meetup.com/bellinghamcodes/events/", @@ -30,7 +27,6 @@ "https://www.meetup.com/buffalo-rust-meetup/events/", "https://www.meetup.com/c-programmer-meetup/events/", "https://www.meetup.com/cambridge-rust-meetup/events/", - "https://www.meetup.com/cap-hill-rust/events/", "https://www.meetup.com/charlottesville-rust-meetup/events/", "https://www.meetup.com/chicago-healthcare-tech-and-ai/events/", "https://www.meetup.com/chicago-rust-meetup/events/", @@ -41,27 +37,20 @@ "https://www.meetup.com/columbus-rs/events/", "https://www.meetup.com/conf42/events/", "https://www.meetup.com/copenhagen-rust-community/events/", - "https://www.meetup.com/copenhagen-rust-meetup-group/events/", "https://www.meetup.com/copenhagen-tech-polyglots/events/", - "https://www.meetup.com/dallas-rust/events/", "https://www.meetup.com/dallasrust/events/", "https://www.meetup.com/data-ai-online/events/", "https://www.meetup.com/data-science-on-aws/events/", "https://www.meetup.com/data-umbrella/events/", "https://www.meetup.com/data-umbrella-africa2/events/", - "https://www.meetup.com/dedotalk/events/", "https://www.meetup.com/deep-dish-rust/events/", "https://www.meetup.com/desert-rustaceans/events/", "https://www.meetup.com/detroitrust/events/", - "https://www.meetup.com/developing-embedded-systems-in-rhein-main/events/", "https://www.meetup.com/digital-craftsmanship-nordoberpfalz/events/", "https://www.meetup.com/dutch-rust-meetup/events/", "https://www.meetup.com/edmonton-r-user-group-yegrug/events/", "https://www.meetup.com/everyonecancontribute-cafe/events/", - "https://www.meetup.com/facebook-developer-circle-ruhr/events/", - "https://www.meetup.com/find/events/", "https://www.meetup.com/finland-rust-meetup/events/", - "https://www.meetup.com/floss-grenoble/events/", "https://www.meetup.com/fp-eug/events/", "https://www.meetup.com/freshminds-future-proof-software-development/events/", "https://www.meetup.com/frontend-developer-meetup-amsterdam/events/", @@ -69,16 +58,12 @@ "https://www.meetup.com/func-prog-sweden/events/", "https://www.meetup.com/functional-vilnius/events/", "https://www.meetup.com/gdg-columbus/events/", - "https://www.meetup.com/google-open-source/events/", "https://www.meetup.com/goto-nights-berlin/events/", "https://www.meetup.com/granadagdg/events/", - "https://www.meetup.com/graz-rust-meetup/events/", "https://www.meetup.com/guru-sp-grupo-de-usuarios-ruby-de-sao-paulo/events/", - "https://www.meetup.com/hackathons-denver/events/", "https://www.meetup.com/hackerdojo/events/", "https://www.meetup.com/hackschool-rhein-neckar/events/", "https://www.meetup.com/helsinki-rust-meetup-group/events/", - "https://www.meetup.com/hk-functional-programming/events/", "https://www.meetup.com/houston-functional-programming-users-group/events/", "https://www.meetup.com/hwswfree/events/", "https://www.meetup.com/indyrs/events/", @@ -86,11 +71,7 @@ "https://www.meetup.com/java-user-group-hessen-jugh/events/", "https://www.meetup.com/johannesburg-rust-meetup/events/", "https://www.meetup.com/join-srug/events/", - "https://www.meetup.com/just-code/events/", "https://www.meetup.com/kaibee/events/", - "https://www.meetup.com/kansai-rust/events/", - "https://www.meetup.com/kyoto-rust/events/", - "https://www.meetup.com/linuxing-in-london/events/", "https://www.meetup.com/longview-code-and-coffee/events/", "https://www.meetup.com/los-gatos-rust-reading-group/events/", "https://www.meetup.com/ludwigslust-rust-meetup/events/", @@ -98,9 +79,7 @@ "https://www.meetup.com/madrust/events/", "https://www.meetup.com/maibornwolff-software-engineering-netzwerk/events/", "https://www.meetup.com/meetup-group-boston-nosql-database-group/events/", - "https://www.meetup.com/meetup-group-bxuhnetv/events/", "https://www.meetup.com/meetup-group-zgphbyet/events/", - "https://www.meetup.com/meetup-paris-ecole-superieur-du-genie-informatique/events/", "https://www.meetup.com/michigan-python/events/", "https://www.meetup.com/microsoft-reactor-london/events/", "https://www.meetup.com/microsoft-reactor-new-york/events/", @@ -110,20 +89,13 @@ "https://www.meetup.com/microsoft-reactor-stockholm/events/", "https://www.meetup.com/microsoft-reactor-toronto/events/", "https://www.meetup.com/minneapolis-rust-meetup/events/", - "https://www.meetup.com/minneapolis-rust-meetup-group/events/", "https://www.meetup.com/mob-programming-on-open-source-software/events/", "https://www.meetup.com/monkey-tech-days/events/", "https://www.meetup.com/montpellier-rust-meetup/events/", - "https://www.meetup.com/montreal-rust-language-meetup/events/", - "https://www.meetup.com/mozilla-community-dresden/events/", "https://www.meetup.com/mozilla-meetup-switzerland/events/", - "https://www.meetup.com/mozilla-torino/events/", "https://www.meetup.com/mucplusplus/events/", "https://www.meetup.com/music-city-rust-developers/events/", "https://www.meetup.com/mv-rust-meetup/events/", - "https://www.meetup.com/n-languages-in-n-months-nyc/events/", - "https://www.meetup.com/native-developers-in-ua/events/", - "https://www.meetup.com/neighborhood-math-club/events/", "https://www.meetup.com/newspace-nyc/events/", "https://www.meetup.com/oc-rust/events/", "https://www.meetup.com/ocaml-paris/events/", @@ -133,7 +105,6 @@ "https://www.meetup.com/papers-we-love-vienna/events/", "https://www.meetup.com/paris-scala-user-group-psug/events/", "https://www.meetup.com/pdxrust/events/", - "https://www.meetup.com/peerlab-native-developers/events/", "https://www.meetup.com/perth-rust-meetup-group/events/", "https://www.meetup.com/phx-android/events/", "https://www.meetup.com/polkadot-india/events/", @@ -154,7 +125,6 @@ "https://www.meetup.com/rust-argentina/events/", "https://www.meetup.com/rust-atl/events/", "https://www.meetup.com/rust-atx/events/", - "https://www.meetup.com/rust-barcelona/events/", "https://www.meetup.com/rust-basel/events/", "https://www.meetup.com/rust-bay-area/events/", "https://www.meetup.com/rust-berlin/events/", @@ -163,54 +133,37 @@ "https://www.meetup.com/rust-breakfast-learn/events/", "https://www.meetup.com/rust-brisbane/events/", "https://www.meetup.com/rust-bristol/events/", - "https://www.meetup.com/rust-bulgaria/events/", "https://www.meetup.com/rust-canberra/events/", "https://www.meetup.com/rust-chinese-group/events/", "https://www.meetup.com/rust-cologne-bonn/events/", - "https://www.meetup.com/rust-community-stuttgart/events/", "https://www.meetup.com/rust-czech-republic/events/", - "https://www.meetup.com/rust-detroit/events/", - "https://www.meetup.com/rust-dev-in-mountain-view/events/", "https://www.meetup.com/rust-dublin/events/", "https://www.meetup.com/rust-edi/events/", "https://www.meetup.com/rust-frankfurt/events/", "https://www.meetup.com/rust-gdansk/events/", "https://www.meetup.com/rust-getting-started/events/", "https://www.meetup.com/rust-girona/events/", - "https://www.meetup.com/rust-gouda/events/", "https://www.meetup.com/rust-hack-learn-karlsruhe/events/", - "https://www.meetup.com/rust-hangzhou/events/", - "https://www.meetup.com/rust-hungary-meetup/events/", "https://www.meetup.com/rust-hyderabad/events/", - "https://www.meetup.com/rust-in-blockchain-berlin/events/", - "https://www.meetup.com/rust-in-blockchain-hangzhou/events/", "https://www.meetup.com/rust-in-blockchain-san-francisco/events/", - "https://www.meetup.com/rust-in-israel/events/", "https://www.meetup.com/rust-in-vilnius/events/", "https://www.meetup.com/rust-kw/events/", - "https://www.meetup.com/rust-lang-bucharest-meetup/events/", "https://www.meetup.com/rust-language-milano/events/", "https://www.meetup.com/rust-linz/events/", - "https://www.meetup.com/rust-lisbon/events/", "https://www.meetup.com/rust-london-user-group/events/", "https://www.meetup.com/rust-los-angeles/events/", "https://www.meetup.com/rust-lyon/events/", - "https://www.meetup.com/rust-madrid/events/", "https://www.meetup.com/rust-manchester/events/", "https://www.meetup.com/rust-medellin/events/", "https://www.meetup.com/rust-meetup-augsburg/events/", "https://www.meetup.com/rust-meetup-hamburg/events/", - "https://www.meetup.com/rust-meetup-sofia/events/", "https://www.meetup.com/rust-melbourne/events/", "https://www.meetup.com/rust-modern-systems-programming-in-leipzig/events/", "https://www.meetup.com/rust-montreal/events/", "https://www.meetup.com/rust-moravia/events/", "https://www.meetup.com/rust-munich/events/", "https://www.meetup.com/rust-mx/events/", - "https://www.meetup.com/rust-nairobi/events/", "https://www.meetup.com/rust-nederland/events/", - "https://www.meetup.com/rust-nerf/events/", - "https://www.meetup.com/rust-nijmegen/events/", "https://www.meetup.com/rust-noris/events/", "https://www.meetup.com/rust-nyc/events/", "https://www.meetup.com/rust-oslo/events/", @@ -220,52 +173,32 @@ "https://www.meetup.com/rust-rhein-main/events/", "https://www.meetup.com/rust-roma/events/", "https://www.meetup.com/rust-saar/events/", - "https://www.meetup.com/rust-sacramento/events/", - "https://www.meetup.com/rust-santiago-de-chile/events/", "https://www.meetup.com/rust-sao-paulo-meetup/events/", "https://www.meetup.com/rust-seoul-meetup/events/", "https://www.meetup.com/rust-singapore/events/", "https://www.meetup.com/rust-study-group/events/", "https://www.meetup.com/rust-sydney/events/", - "https://www.meetup.com/rust-tampa/events/", - "https://www.meetup.com/rust-tell-halifax/events/", "https://www.meetup.com/rust-tlv/events/", "https://www.meetup.com/rust-toronto/events/", "https://www.meetup.com/rust-trondheim/events/", - "https://www.meetup.com/rust-tunisia/events/", - "https://www.meetup.com/rust-tw/events/", "https://www.meetup.com/rust-uruguay/events/", - "https://www.meetup.com/rust-utrecht/events/", "https://www.meetup.com/rust-vienna/events/", "https://www.meetup.com/rust-warsaw/events/", "https://www.meetup.com/rust-wellington/events/", "https://www.meetup.com/rust-wroclaw/events/", - "https://www.meetup.com/rust-wurzburg-meetup-group/events/", "https://www.meetup.com/rust-zurich/events/", - "https://www.meetup.com/rustaceans-rva/events/", "https://www.meetup.com/rustcologne/events/", "https://www.meetup.com/rustdc/events/", "https://www.meetup.com/rustdelhi/events/", - "https://www.meetup.com/rustfloripa/events/", "https://www.meetup.com/rustgbg/events/", - "https://www.meetup.com/rustmn/events/", - "https://www.meetup.com/rustny/events/", "https://www.meetup.com/rustox/events/", - "https://www.meetup.com/rustphilly/events/", "https://www.meetup.com/ruststhlm/events/", "https://www.meetup.com/san-diego-rust/events/", "https://www.meetup.com/san-francisco-rust-study-group/events/", - "https://www.meetup.com/seattle-rust-meetup/events/", - "https://www.meetup.com/seattle-rust-user-group/events/", - "https://www.meetup.com/seoul-substrate-blockchain-meetup/events/", "https://www.meetup.com/sfugcgn/events/", - "https://www.meetup.com/shop-apotheke-europe/events/", "https://www.meetup.com/singapore-web3-blockchain-meetup/events/", - "https://www.meetup.com/slowtec/events/", "https://www.meetup.com/softwerkskammer-ruhrgebiet/events/", "https://www.meetup.com/solidstatedepot/events/", - "https://www.meetup.com/south-florida-rust-meetup/events/", - "https://www.meetup.com/spbrust/events/", "https://www.meetup.com/spokane-rust/events/", "https://www.meetup.com/stacja-it-krakow/events/", "https://www.meetup.com/stacja-it-trojmiasto/events/", @@ -274,24 +207,18 @@ "https://www.meetup.com/stockholm-google-developer-group/events/", "https://www.meetup.com/stockholm-rust/events/", "https://www.meetup.com/stockholmcpp/events/", - "https://www.meetup.com/t-rust-meetup/events/", "https://www.meetup.com/techatagoda/events/", "https://www.meetup.com/techceleration/events/", "https://www.meetup.com/techmeetupostrava/events/", - "https://www.meetup.com/technocracy/events/", "https://www.meetup.com/tel-aviv-data-science-odsc/events/", "https://www.meetup.com/the-karlsruhe-functional-programmers-meetup-group/events/", "https://www.meetup.com/the-south-padre-island-reading-group/events/", "https://www.meetup.com/thursday-go/events/", "https://www.meetup.com/tokyo-rust-meetup/events/", - "https://www.meetup.com/toulouse-rust-meetup/events/", "https://www.meetup.com/triangle-bitdevs/events/", "https://www.meetup.com/triangle-rust/events/", - "https://www.meetup.com/triangle-rustaceans/events/", - "https://www.meetup.com/turing-community/events/", "https://www.meetup.com/ucsc-extension-community/events/", "https://www.meetup.com/utah-rust/events/", - "https://www.meetup.com/utahrust/events/", "https://www.meetup.com/vancouver-postgres/events/", "https://www.meetup.com/vancouver-rust/events/", "https://www.meetup.com/vilnius-rust-go-meetup-group/events/", @@ -299,10 +226,8 @@ "https://www.meetup.com/wasmna/events/", "https://www.meetup.com/wasmsf/events/", "https://www.meetup.com/wearedevelopers-community/events/", - "https://www.meetup.com/web3devc/events/", "https://www.meetup.com/webassembly-and-wasmedge/events/", "https://www.meetup.com/wellington-rust-meetup/events/", "https://www.meetup.com/women-in-rust/events/", - "https://www.meetup.com/yeducation/events/", "https://www.meetup.com/zagreb-rust-meetup/events/" -] \ No newline at end of file +] From 03912a379a320e3a05f1141c2da453cc8c3aed36 Mon Sep 17 00:00:00 2001 From: bdillo Date: Sun, 15 Dec 2024 17:19:55 -0800 Subject: [PATCH 06/18] misc cleanup, refactoring --- .../country_code_to_continent.py | 4 +- tools/events-automation/event.py | 173 ++++++++++++---- .../generate_events_meetup.py | 186 ++++++------------ tools/events-automation/jwt_auth.py | 26 ++- tools/events-automation/main.py | 52 +++-- tools/events-automation/utils.py | 33 ++++ 6 files changed, 285 insertions(+), 189 deletions(-) create mode 100644 tools/events-automation/utils.py diff --git a/tools/events-automation/country_code_to_continent.py b/tools/events-automation/country_code_to_continent.py index 03be89754..1c2f482e2 100644 --- a/tools/events-automation/country_code_to_continent.py +++ b/tools/events-automation/country_code_to_continent.py @@ -250,7 +250,7 @@ } -def country_code_to_continent(country_code): +def country_code_to_continent(country_code: str): # Returns the continent a country code belongs to. - return COUNTRY_CODE_TO_CONTINENT[country_code] + return COUNTRY_CODE_TO_CONTINENT[country_code.upper()] diff --git a/tools/events-automation/event.py b/tools/events-automation/event.py index 3af3037dd..16899f97a 100644 --- a/tools/events-automation/event.py +++ b/tools/events-automation/event.py @@ -1,47 +1,144 @@ -import datetime +from dataclasses import dataclass +from datetime import datetime from geopy.geocoders import Nominatim from state_territory_to_abbrev import au_state_territory_to_abbrev, us_state_to_abbrev, ca_state_territory_to_abbrev -class Event(): - def __init__(self, name, location, date, url, virtual, organizerName, organizerUrl, duplicate=False) -> None: - self.name = name - self.location = location - self.date = date - self.url = url - self.virtual = virtual - self.organizerName = organizerName - self.organizerUrl = organizerUrl - self.duplicate = duplicate + +@dataclass +class Location: + city: None | str + state: None | str + country: None | str + + def fields_present(self) -> int: + """ Check how many fields are present, used to determine which Location has more information when comparing """ + c = 0 + + if self.city: + c += 1 + if self.state: + c += 1 + if self.country: + c += 1 + + return c + + def to_str(self) -> str: + s = '' + + if self.city: + s += self.city.lower().capitalize() + if self.state: + s += ', ' + s += self.state.upper() + if self.country: + s += ', ' + s += self.country.upper() + + return s + + +@dataclass +class Event: + name: str + location: Location + date: datetime + url: str + virtual: bool + organizer_name: str + organizer_url: str def to_markdown_string(self) -> str: - if self.virtual: - return f'* {self.date} | Virtual ({self.location}) | [{self.organizerName}]({self.organizerUrl})\n\t*[**{self.name}**]({self.url})' - else: - return f'* {self.date} | {self.location} | [{self.organizerName}]({self.organizerUrl})\n\t*[**{self.name}**]({self.url})' + date_str = self.date.date() + location = f"Virtual ({self.location.to_str()})" if self.virtual else self.location.to_str() + + return f'* {date_str} | {location} | [{self.organizer_name}]({self.organizer_url})\n * [**{self.name}**]({self.url})' + + +@dataclass +class RawGqlEvent: + """ + Dataclass for our GQL responses for upcomingEvents. Maps very closely to the raw API response with minimal field parsing + """ + title: str + group_name: str + group_location: Location + date_time_str: str + event_url_str: str + venue_type: None | str + event_location: Location + lat: float + long: float + + def __init__(self, **kwargs) -> None: + # TODO: add some validation here, these error messages will be not useful currently + node = kwargs["node"] + self.title = node["title"] + + group = node["group"] + self.group_name = group["name"] + self.group_location = Location(group["city"], group["state"], group["country"]) - def format_date(self): - # Formats datetime data into date. - if isinstance(self.date, datetime.datetime): - self.date = self.date.date() - - def format_location(self): - # Formats location data into (city, +/-state, country). - geocoder = Nominatim(user_agent="TWiR", timeout=5) - locationData = geocoder.geocode(self.location, language="en", addressdetails=True).raw["address"] - - country_code, city = locationData["country_code"].upper(), locationData.get("city", locationData.get("town", locationData.get("village", "**NO CITY DATA**"))) - if country_code in ["AU", "CA", "US"]: - state = locationData.get("state", locationData.get("territory", "**NO STATE DATA**")) - if state == "**NO STATE DATA**": - state_abbrev = state - elif country_code == "AU": - state_abbrev = au_state_territory_to_abbrev(state) - elif country_code == "CA": - state_abbrev = ca_state_territory_to_abbrev(state) - elif country_code == "US": - state_abbrev = us_state_to_abbrev(state) - self.location = f'{city}, {state_abbrev}, {country_code}' + self.date_time_str = node["dateTime"] + self.event_url_str = node["eventUrl"] + + venue = node["venue"] + self.venue_type = venue["venueType"] + # TODO: do we need these lat longs? + self.lat = venue["lat"] + self.long = venue["lng"] + self.event_location = Location(venue["city"], venue["state"], venue["country"]) + + def to_event(self, geolocator: Nominatim, group_url: str) -> Event: + # address = (geolocator.reverse(str(self.lat) +","+ str(self.long))).raw["address"] + # location = self._format_location(address) + + is_virtual = self.venue_type == "online" + date = datetime.fromisoformat(self.date_time_str) + + # prefer the event specific location, otherwise fall back to the group's location + if self.event_location.fields_present() > self.group_location.fields_present(): + location = self.event_location else: - self.location = f'{city}, {country_code}' + location = self.group_location + + return Event( + name=self.title, + location=location, + date=date, + url=self.event_url_str, + virtual=is_virtual, + organizer_name=self.group_name, + organizer_url=group_url + ) + + # @staticmethod + # def _format_location(address: dict) -> str: + # # TODO: look this over... + # if not address: + # return "No location" + + # # All components for a location + # components = ['road', 'city', 'state', 'postcode', 'country'] + + # # Get available components, otherwise replace missing component with an empty string + # location = [address.get(component, "") for component in components] + + # return ','.join(location) if location else "No location" + # # ???????????? + # country_code, city = locationData["country_code"].upper(), locationData.get("city", locationData.get("town", locationData.get("village", "**NO CITY DATA**"))) + # if country_code in ["AU", "CA", "US"]: + # state = locationData.get("state", locationData.get("territory", "**NO STATE DATA**")) + # if state == "**NO STATE DATA**": + # state_abbrev = state + # elif country_code == "AU": + # state_abbrev = au_state_territory_to_abbrev(state) + # elif country_code == "CA": + # state_abbrev = ca_state_territory_to_abbrev(state) + # elif country_code == "US": + # state_abbrev = us_state_to_abbrev(state) + # self.location = f'{city}, {state_abbrev}, {country_code}' + # else: + # self.location = f'{city}, {country_code}' diff --git a/tools/events-automation/generate_events_meetup.py b/tools/events-automation/generate_events_meetup.py index 43a764788..855514591 100644 --- a/tools/events-automation/generate_events_meetup.py +++ b/tools/events-automation/generate_events_meetup.py @@ -1,12 +1,12 @@ import requests import datetime -import csv import logging from jwt_auth import generate_signed_jwt -from urllib.parse import urlsplit from geopy.geocoders import Nominatim -from event import Event +from event import Event, RawGqlEvent +from utils import MeetupGroupUrl +from typing import List logger = logging.getLogger(__name__) @@ -51,7 +51,56 @@ def _get_access_token(self): if not self._access_token: self._authenticate() - return self._get_access_token + return self._access_token + + def _build_event_listing_gql_query(self, group_url_name: str) -> dict: + return { + "query": """ + query ($urlName: String!, $searchEventInput: ConnectionInput!) { + groupByUrlname(urlname: $urlName) { + upcomingEvents(input: $searchEventInput, sortOrder: ASC) { + pageInfo { + hasNextPage + endCursor + } + edges { + node { + id + group { + name + city + state + country + } + title + dateTime + eventUrl + venue { + city + state + country + venueType + lat + lng + } + } + } + } + } + } + """, + "variables": { + "urlName": group_url_name, + "searchEventInput": { + # TODO: see if we need this limit or not + "first": 20 + } + } + } + + def _parse_event_listing_gql_response(self, response: dict) -> List[RawGqlEvent]: + edges = response["groupByUrlname"]["upcomingEvents"]["edges"] + return [RawGqlEvent(**kwargs) for kwargs in edges] def fetch_groups(self, endCursor=""): """ @@ -62,7 +111,7 @@ def fetch_groups(self, endCursor=""): # Sets the content type to application/json for the request body. headers = { - "Authorization": f"Bearer {self._get_access_token}", + "Authorization": f"Bearer {self._get_access_token()}", "Content-Type": "application/json", } @@ -143,131 +192,16 @@ def get_rust_groups(self) -> dict: break return groups - def get_known_rust_groups(self, filename="rust_meetup_groups.csv") -> dict: - """ - Returns a dictionary represents all groups from a specified CSV file - :type fileName: Name or Path of the CSV file that contains the URLs and locations of the groups. - """ - # TODO: this whole method really needs to be cleaned up - groups = dict() # main dictionary that stores all information of different groups - - with open(filename, newline='') as csv_file: - csv_reader = csv.reader(csv_file) - for (i, row) in enumerate(csv_reader): - name, url, location, non_meetup = row - group = {} - group["link"] = url - split_url = urlsplit(group["link"]) - group["urlname"] = (split_url.path).replace("/", "") - group["location"] = location - groups[i] = group - return groups - - def get_20_events(self, groups) -> list[Event]: - """ - Returns a list where each element is an instance of the Event class, representing event data from the Meetup API - :type groups: A dictionary of groups where each entry contains the group's URL name to make an API request - :rtype: dict - """ - events = [] # main list to store data about each fetched event. - + def get_raw_events_for_group(self, group: MeetupGroupUrl) -> List[RawGqlEvent]: headers = { "Authorization": f"Bearer {self._get_access_token()}", "Content-Type": "application/json", } - # Constructs and sends a GraphQL query for each group to fetch up to 20 upcoming events from the Meetup API using the group's URL name - data = {} - for group in groups.values(): - urlName = group["urlname"] - data = { - "query": """ - query ($urlName: String!, $searchEventInput: ConnectionInput!) { - groupByUrlname(urlname: $urlName) { - upcomingEvents(input: $searchEventInput, sortOrder: ASC) { - pageInfo { - hasNextPage - endCursor - } - edges { - node { - id - title - dateTime - eventUrl - venue { - venueType - lat - lng - } - } - } - } - } - } - """, - "variables": { - "urlName": urlName, - "searchEventInput": { - "first": 20 - } - } - } - response = requests.post(url=self.GQL_ENDPOINT, headers=headers, json=data) - data = response.json()["data"] - - # Constructs Event with attributes such as title, location, date, URL, and organizer details - if data: - searchGroupByUrlname = data["groupByUrlname"] - if searchGroupByUrlname: - edges = searchGroupByUrlname["upcomingEvents"]["edges"] - if edges: - for edge in edges: - node = edge["node"] - if node: - venue = node["venue"] - # TODO: Handle events don't have venue: - # 1. Flagging the events and they will have to be check manually, - # 2. Putting them in separate list to check - # (for now ignore those events) - if venue: - name = node["title"] - virtual = True - if venue["venueType"] != "online": - virtual = False + query = self._build_event_listing_gql_query(group.url_name) + response = requests.post(url=self.GQL_ENDPOINT, headers=headers, json=query) + data = response.json()["data"] + logger.debug(data) - # Convert obtained latitude and longitude of an event to formatted location - address = (self._geolocator.reverse(str(venue["lat"]) +","+ str(venue["lng"]))).raw["address"] - location = self.format_location(address) - date = datetime.datetime.fromisoformat(node["dateTime"]).date() - url = node["eventUrl"] - organizerName = group.get("name", urlName) - organizerUrl = group["link"] - events.append(Event(name, location, date, url, virtual, organizerName, organizerUrl)) - return events - - def format_location(self, address) -> str: - """ - Helper method to format address of events with required components for a location - :rtype: string - """ - if not address: - return "No location" - - # All components for a location - components = ['road', 'city', 'state', 'postcode', 'country'] + return self._parse_event_listing_gql_response(data) - # Get available components, otherwise replace missing component with an empty string - location = [address.get(component, "") for component in components] - - - return ','.join(location) if location else "No location" - - def get_events(self) -> list[Event]: - """ - Returns a list of Event objects querying from known, and Meetup API Rust groups - :rtype: list[Event] - """ - # TODO: once the handling events without venue successful, get events_meetup_groups = get_20_events(get_rush_groups()) - events_known_groups = self.get_20_events(self.get_known_rust_groups()) - return events_known_groups diff --git a/tools/events-automation/jwt_auth.py b/tools/events-automation/jwt_auth.py index 2978c52b5..c7b177f3b 100644 --- a/tools/events-automation/jwt_auth.py +++ b/tools/events-automation/jwt_auth.py @@ -5,13 +5,21 @@ from cryptography.hazmat.primitives import serialization from cryptography.hazmat.backends import default_backend +MEETUP_PEM_ENV_VAR = "MEETUP_PRIVATE_KEY" +MEETUP_AUTHORIZED_MEMBER_ID_ENV_VAR = "MEETUP_AUTHORIZED_MEMBER_ID" +MEETUP_CLIENT_KEY_ENV_VAR = "MEETUP_CLIENT_KEY" + def get_PEM_private_key(): """ Loads the PRIVATE_KEY in string from .env file. Returns it in PEM-formatted bytes """ - with open(os.getenv("PRIVATE_KEY"), 'rb') as pem_file: + pem_env_var = os.getenv(MEETUP_PEM_ENV_VAR) + if not pem_env_var: + raise RuntimeError(f"Env var {MEETUP_PEM_ENV_VAR} not set!") + + with open(pem_env_var, "rb") as pem_file: return pem_file.read() def get_RSA_private_key(): @@ -48,12 +56,20 @@ def generate_signed_jwt(): Encodes and signs the payload using RS256 and the private RSA key, forming a base64-url encoded header, payload, and signature. Then returns it. """ - AUTHORIZED_MEMBER_ID = os.getenv('AUTHORIZED_MEMBER_ID') # the member id that owns the OAuth Client - CLIENT_KEY = os.getenv('CLIENT_KEY') + authorized_member_id = os.getenv(MEETUP_AUTHORIZED_MEMBER_ID_ENV_VAR) # the member id that owns the OAuth Client + client_key = os.getenv(MEETUP_CLIENT_KEY_ENV_VAR) + + # TODO: consolidate fetching + checking env vars into one place + if not authorized_member_id: + raise RuntimeError(f"Env var {MEETUP_AUTHORIZED_MEMBER_ID_ENV_VAR} not set!") + + if not client_key: + raise RuntimeError(f"Env var {MEETUP_CLIENT_KEY_ENV_VAR} not set!") + private_key = get_RSA_private_key() payload = { - "sub": AUTHORIZED_MEMBER_ID, - "iss": CLIENT_KEY, + "sub": authorized_member_id, + "iss": client_key, "aud": "api.meetup.com", "exp": (datetime.datetime.utcnow() + datetime.timedelta(hours=24)).timestamp() } diff --git a/tools/events-automation/main.py b/tools/events-automation/main.py index 74a72b531..9e171ef36 100644 --- a/tools/events-automation/main.py +++ b/tools/events-automation/main.py @@ -3,43 +3,65 @@ # call event sink with our collected events # print to console / output to file formatted markdown +import argparse import logging +from geopy.geocoders import Nominatim from test_events import get_test_events from datetime import date, timedelta from country_code_to_continent import country_code_to_continent from generate_events_meetup import TwirMeetupClient +from utils import read_meetup_group_urls # TODO: Flagged events list handling. logger = logging.getLogger(__name__) def main(): - logging.basicConfig(level=logging.INFO) + args = parse_args() + + log_level = logging.DEBUG if args.debug else logging.INFO + logging.basicConfig(level=log_level) + logger.info("Starting...") meetup_client = TwirMeetupClient() + geolocator = Nominatim(user_agent="TWiR") + + # get our known rust meetup groups + group_urls = read_meetup_group_urls(args.groups_file) - # Get Events list from Event Sources. - event_list = meetup_client.get_events() + events = [] + for group_url in group_urls: + group_raw_events = meetup_client.get_raw_events_for_group(group_url) - # Format date and location data. - format_data(event_list) + events += [raw_event.to_event(geolocator, group_url.url) for raw_event in group_raw_events] # Remove events outside of date range. - date_window_filter(event_list) + date_window_filter(events) # Sort remaining events by date, then location. - event_list.sort(key=lambda event: (event.date, event.location)) + events.sort(key=lambda event: (event.date, event.location)) + + for event in events: + print(event.to_markdown_string()) # Flag potential duplicate events. - potential_duplicate(event_list) + # potential_duplicate(events) # Group by virtual or by continent. - event_list = group_virtual_continent(event_list) + # events = group_virtual_continent(events) # Output Sorted Event List. - output_to_screen(event_list) + # output_to_screen(events) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description='Fetches meetup events for TWIR') + parser.add_argument("-d", "--debug", action="store_true", dest="debug", help="Enable debug logging") + parser.add_argument("-g", "--groups", action="store", type=str, dest="groups_file", required=True, help="File with a JSON array of meetup group URLS") + + return parser.parse_args() def output_to_screen(event_list): @@ -58,13 +80,6 @@ def output_to_screen(event_list): print() -def format_data(event_list): - # Formats date and location data into specified format. - for event in event_list: - event.format_date() - event.format_location() - - def date_window_filter(event_list): # Removes Events that are outside current date window. # Date window = closest wednesday + 5 weeks. @@ -73,7 +88,8 @@ def date_window_filter(event_list): start_date = start_date + timedelta(days=1) for event in event_list: - if not (start_date <= event.date <= start_date + timedelta(weeks=5)): + if not (start_date <= event.date.date() <= start_date + timedelta(weeks=5)): + logger.debug(f"Removed event outside of date range: {event}") event_list.remove(event) diff --git a/tools/events-automation/utils.py b/tools/events-automation/utils.py new file mode 100644 index 000000000..df53ae60e --- /dev/null +++ b/tools/events-automation/utils.py @@ -0,0 +1,33 @@ +from dataclasses import dataclass +from typing import List +from urllib.parse import urlparse +import json + +@dataclass +class MeetupGroupUrl: + MEETUP_HOSTNAME = "www.meetup.com" + + url: str + url_name: str + + def __init__(self, url_str: str) -> None: + parsed = urlparse(url_str) + + if parsed.hostname != self.MEETUP_HOSTNAME: + raise ValueError(f"Invalid hostname in URL {url_str}, expected {self.MEETUP_HOSTNAME}") + + path_split = parsed.path.split('/') + + if len(path_split) < 2: + raise ValueError(f"Unable to parse meetup group name from URL {url_str}") + + self.url = url_str + self.url_name = path_split[1] + + +def read_meetup_group_urls(meetups_json: str) -> List[MeetupGroupUrl]: + with open(meetups_json, "r") as f: + group_urls = json.loads(f.read()) + + parsed_groups = [MeetupGroupUrl(url) for url in group_urls] + return parsed_groups From 7c8383b8f972251d3973c382bb8d13f0904c8724 Mon Sep 17 00:00:00 2001 From: bdillo Date: Sun, 15 Dec 2024 17:46:06 -0800 Subject: [PATCH 07/18] fix some date issues --- tools/events-automation/event.py | 9 +++++---- tools/events-automation/main.py | 15 ++++++++++----- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/tools/events-automation/event.py b/tools/events-automation/event.py index 16899f97a..802854ead 100644 --- a/tools/events-automation/event.py +++ b/tools/events-automation/event.py @@ -1,7 +1,6 @@ from dataclasses import dataclass from datetime import datetime from geopy.geocoders import Nominatim -from state_territory_to_abbrev import au_state_territory_to_abbrev, us_state_to_abbrev, ca_state_territory_to_abbrev @dataclass @@ -49,10 +48,9 @@ class Event: organizer_url: str def to_markdown_string(self) -> str: - date_str = self.date.date() location = f"Virtual ({self.location.to_str()})" if self.virtual else self.location.to_str() - return f'* {date_str} | {location} | [{self.organizer_name}]({self.organizer_url})\n * [**{self.name}**]({self.url})' + return f'* {self.date.date()} | {location} | [{self.organizer_name}]({self.organizer_url})\n * [**{self.name}**]({self.url})' @dataclass @@ -94,7 +92,10 @@ def to_event(self, geolocator: Nominatim, group_url: str) -> Event: # location = self._format_location(address) is_virtual = self.venue_type == "online" - date = datetime.fromisoformat(self.date_time_str) + + # this is a bit weird because we want a naive datetime object that just contains the year/month/day because we get + # timestamps with tz info like "2025-01-16T19:00+01:00", just strip the time and tz info before parsing + date = datetime.strptime(self.date_time_str.split('T')[0], '%Y-%m-%d') # prefer the event specific location, otherwise fall back to the group's location if self.event_location.fields_present() > self.group_location.fields_present(): diff --git a/tools/events-automation/main.py b/tools/events-automation/main.py index 9e171ef36..ad6afe2ff 100644 --- a/tools/events-automation/main.py +++ b/tools/events-automation/main.py @@ -6,11 +6,12 @@ import argparse import logging from geopy.geocoders import Nominatim +from typing import List -from test_events import get_test_events from datetime import date, timedelta from country_code_to_continent import country_code_to_continent from generate_events_meetup import TwirMeetupClient +from event import Event from utils import read_meetup_group_urls # TODO: Flagged events list handling. @@ -38,7 +39,7 @@ def main(): events += [raw_event.to_event(geolocator, group_url.url) for raw_event in group_raw_events] # Remove events outside of date range. - date_window_filter(events) + events = date_window_filter(events) # Sort remaining events by date, then location. events.sort(key=lambda event: (event.date, event.location)) @@ -80,17 +81,21 @@ def output_to_screen(event_list): print() -def date_window_filter(event_list): +def date_window_filter(events: List[Event]) -> List[Event]: # Removes Events that are outside current date window. # Date window = closest wednesday + 5 weeks. start_date = date.today() while start_date.weekday() != 2: start_date = start_date + timedelta(days=1) - for event in event_list: + valid = [] + for event in events: if not (start_date <= event.date.date() <= start_date + timedelta(weeks=5)): logger.debug(f"Removed event outside of date range: {event}") - event_list.remove(event) + else: + valid.append(event) + + return valid def group_virtual_continent(event_list): From d9b8e821bbea88376714418240ac1fc455ca080a Mon Sep 17 00:00:00 2001 From: bdillo Date: Sun, 15 Dec 2024 18:01:41 -0800 Subject: [PATCH 08/18] get duplicate removal/output formatting working --- tools/events-automation/main.py | 46 +++++++++++++++++---------------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/tools/events-automation/main.py b/tools/events-automation/main.py index ad6afe2ff..3d0099ec8 100644 --- a/tools/events-automation/main.py +++ b/tools/events-automation/main.py @@ -39,28 +39,29 @@ def main(): events += [raw_event.to_event(geolocator, group_url.url) for raw_event in group_raw_events] # Remove events outside of date range. - events = date_window_filter(events) + events = date_window_filter(events, args.weeks) # Sort remaining events by date, then location. - events.sort(key=lambda event: (event.date, event.location)) + events.sort(key=lambda event: (event.date, event.location.to_str())) - for event in events: - print(event.to_markdown_string()) + # for event in events: + # print(event.to_markdown_string()) - # Flag potential duplicate events. - # potential_duplicate(events) + # Remove potential duplicate events. + events = remove_duplicate_events(events) # Group by virtual or by continent. - # events = group_virtual_continent(events) + events = group_virtual_continent(events) # Output Sorted Event List. - # output_to_screen(events) + output_to_screen(events) def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description='Fetches meetup events for TWIR') parser.add_argument("-d", "--debug", action="store_true", dest="debug", help="Enable debug logging") parser.add_argument("-g", "--groups", action="store", type=str, dest="groups_file", required=True, help="File with a JSON array of meetup group URLS") + parser.add_argument("-w", "--weeks", action="store", type=int, dest="weeks", default=5, help="Number of weeks to search for events from, starting next Wednesday") return parser.parse_args() @@ -75,13 +76,11 @@ def output_to_screen(event_list): # Output event details for event in value: - if event.duplicate: - print("** NOTE POTENTIAL DUPLICATE: **") print(event.to_markdown_string()) print() -def date_window_filter(events: List[Event]) -> List[Event]: +def date_window_filter(events: List[Event], weeks: int) -> List[Event]: # Removes Events that are outside current date window. # Date window = closest wednesday + 5 weeks. start_date = date.today() @@ -90,7 +89,7 @@ def date_window_filter(events: List[Event]) -> List[Event]: valid = [] for event in events: - if not (start_date <= event.date.date() <= start_date + timedelta(weeks=5)): + if not (start_date <= event.date.date() <= start_date + timedelta(weeks=weeks)): logger.debug(f"Removed event outside of date range: {event}") else: valid.append(event) @@ -104,23 +103,26 @@ def group_virtual_continent(event_list): for event in event_list: # Separates Events by Virtual or by Continent - key = "Virtual" if event.virtual else country_code_to_continent(event.location[-2:]) + key = "Virtual" if event.virtual else country_code_to_continent(event.location.country) separated_event_list.setdefault(key, []).append(event) return separated_event_list -def potential_duplicate(event_list): +def remove_duplicate_events(events: List[Event]) -> List[Event]: # Identifies possible duplicate Events within Event List. - for i in range(len(event_list)): - for j in range(i+1, len(event_list)): - if event_list[i].date == event_list[j].date: - if event_list[i].url == event_list[j].url: - if event_list[i].name == event_list[j].name: - if event_list[i].organizerName == event_list[j].organizerName: - if event_list[i].location == event_list[j].location: - event_list[i].duplicate = True + seen_event_urls = set() + checked = [] + for event in events: + if event.url in seen_event_urls: + logger.warning(f"Found duplicate event: {event}") + else: + seen_event_urls.add(event.url) + checked.append(event) + + return checked + if __name__ == "__main__": main() From f51d7a55f318ce7540a4c96a226c25f8727cc5c0 Mon Sep 17 00:00:00 2001 From: bdillo Date: Sun, 15 Dec 2024 18:29:07 -0800 Subject: [PATCH 09/18] fix a few edge cases in events --- tools/events-automation/event.py | 6 ++++++ .../events-automation/generate_events_meetup.py | 17 ++++++++++++++++- tools/events-automation/main.py | 6 ++---- 3 files changed, 24 insertions(+), 5 deletions(-) diff --git a/tools/events-automation/event.py b/tools/events-automation/event.py index 802854ead..40b6a1bb9 100644 --- a/tools/events-automation/event.py +++ b/tools/events-automation/event.py @@ -1,8 +1,12 @@ +import logging from dataclasses import dataclass from datetime import datetime from geopy.geocoders import Nominatim +logger = logging.getLogger(__name__) + + @dataclass class Location: city: None | str @@ -69,6 +73,7 @@ class RawGqlEvent: long: float def __init__(self, **kwargs) -> None: + logger.debug(f"Constructing RawGqlEvent from: {kwargs}") # TODO: add some validation here, these error messages will be not useful currently node = kwargs["node"] self.title = node["title"] @@ -113,6 +118,7 @@ def to_event(self, geolocator: Nominatim, group_url: str) -> Event: organizer_url=group_url ) + # not sure if this is needed anymore, looks like the meetup API has a gooda mount of location data # @staticmethod # def _format_location(address: dict) -> str: # # TODO: look this over... diff --git a/tools/events-automation/generate_events_meetup.py b/tools/events-automation/generate_events_meetup.py index 855514591..24a4ade9c 100644 --- a/tools/events-automation/generate_events_meetup.py +++ b/tools/events-automation/generate_events_meetup.py @@ -100,7 +100,17 @@ def _build_event_listing_gql_query(self, group_url_name: str) -> dict: def _parse_event_listing_gql_response(self, response: dict) -> List[RawGqlEvent]: edges = response["groupByUrlname"]["upcomingEvents"]["edges"] - return [RawGqlEvent(**kwargs) for kwargs in edges] + + events = [] + # TODO: maybe move this validation somewhere else? + for edge_kwargs in edges: + if not edge_kwargs["node"]["venue"]: + logger.error(f"Event response missing venue: {edge_kwargs}") + continue + + events.append(RawGqlEvent(**edge_kwargs)) + + return events def fetch_groups(self, endCursor=""): """ @@ -198,10 +208,15 @@ def get_raw_events_for_group(self, group: MeetupGroupUrl) -> List[RawGqlEvent]: "Content-Type": "application/json", } + logger.info(f"Fetching events for group {group}") query = self._build_event_listing_gql_query(group.url_name) response = requests.post(url=self.GQL_ENDPOINT, headers=headers, json=query) data = response.json()["data"] logger.debug(data) + if data["groupByUrlname"] is None: + logger.error(f"Group {group} not valid, skipping") + return [] + return self._parse_event_listing_gql_response(data) diff --git a/tools/events-automation/main.py b/tools/events-automation/main.py index 3d0099ec8..f588eaad2 100644 --- a/tools/events-automation/main.py +++ b/tools/events-automation/main.py @@ -14,10 +14,10 @@ from event import Event from utils import read_meetup_group_urls -# TODO: Flagged events list handling. logger = logging.getLogger(__name__) + def main(): args = parse_args() @@ -27,6 +27,7 @@ def main(): logger.info("Starting...") meetup_client = TwirMeetupClient() + # TODO: see if we actually need to use this geolocator = Nominatim(user_agent="TWiR") # get our known rust meetup groups @@ -44,9 +45,6 @@ def main(): # Sort remaining events by date, then location. events.sort(key=lambda event: (event.date, event.location.to_str())) - # for event in events: - # print(event.to_markdown_string()) - # Remove potential duplicate events. events = remove_duplicate_events(events) From 5a756def51d3e1b7554704fa942be69ab3df75ca Mon Sep 17 00:00:00 2001 From: bdillo Date: Sat, 4 Jan 2025 17:18:48 -0800 Subject: [PATCH 10/18] minor doc updates --- tools/events-automation/meetup_events.md | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/tools/events-automation/meetup_events.md b/tools/events-automation/meetup_events.md index 0247c0a74..0eed1b11f 100644 --- a/tools/events-automation/meetup_events.md +++ b/tools/events-automation/meetup_events.md @@ -41,23 +41,21 @@ Before you start, ensure you have the following: ``` 3. **Set Up Environment Variables**: - - Create a `.env` file for project directory. - - Add the following environment variables with your actual values: + - Set the following environment variables with your actual values: ``` - AUTHORIZED_MEMBER_ID= - CLIENT_KEY= - PRIVATE_KEY= + MEETUP_AUTHORIZED_MEMBER_ID= + MEETUP_CLIENT_KEY= + MEETUP_PRIVATE_KEY= ``` These values are used for authentication with the Meetup API and to generate JWT tokens securely. ### Running the Script To fetch events, run the following command from the project directory `.../tools/events-automation`: ```bash -python3 main.py +python3 main.py -g ``` This script performs the following operations: - Authenticates with the Meetup API using JWT. -- Fetches data for known Rust groups from `.../tools/events-automation/rust_meetup_groups.csv` file and Meetup API. - Filters and formats the event data into a standardized structure. - Outputs the details of upcoming events. @@ -72,4 +70,4 @@ This format includes the date and time of the event, the location (virtual in th ### Challenges and Considerations - **Data Accuracy**: The script uses "Rust" as a keyword for searching groups and events. This can sometimes pull in events that merely mention Rust but aren't directly related to Rust programming in the event descriptions. - **Event Data Completeness**: Not all events have complete venue information, especially for virtual events. The script currently ignores handling missing data, but the ideal is flagging such events for manual review or excluding them from the final output. -- **API Limitations**: The Meetup API has rate limits and other constraints that may affect how frequently you can fetch data. \ No newline at end of file +- **API Limitations**: The Meetup API has rate limits and other constraints that may affect how frequently you can fetch data. From ba8123ec481267bf5daef3ce69f3798bd3262ddb Mon Sep 17 00:00:00 2001 From: bdillo Date: Sat, 4 Jan 2025 17:27:38 -0800 Subject: [PATCH 11/18] remove files that are no longer used --- tools/events-automation/country_to_abbrev.py | 281 ------------------ .../events-automation/rust_meetup_groups.csv | 118 -------- .../state_territory_to_abbrev.py | 102 ------- 3 files changed, 501 deletions(-) delete mode 100644 tools/events-automation/country_to_abbrev.py delete mode 100644 tools/events-automation/rust_meetup_groups.csv delete mode 100644 tools/events-automation/state_territory_to_abbrev.py diff --git a/tools/events-automation/country_to_abbrev.py b/tools/events-automation/country_to_abbrev.py deleted file mode 100644 index 6168f18bf..000000000 --- a/tools/events-automation/country_to_abbrev.py +++ /dev/null @@ -1,281 +0,0 @@ -# Takes a country and returns it's ISO alpha-2 code. - -COUNTRY_TO_ABBREV = { - 'Abkhazia': 'AB', - 'Afghanistan': 'AF', - 'Albania': 'AL', - 'Algeria': 'DZ', - 'American Samoa': 'AS', - 'Andorra': 'AD', - 'Angola': 'AO', - 'Anguilla': 'AI', - 'Antigua and Barbuda': 'AG', - 'Argentina': 'AR', - 'Armenia': 'AM', - 'Aruba': 'AW', - 'Australia': 'AU', - 'Austria': 'AT', - 'Azerbaijan': 'AZ', - 'Bahamas': 'BS', - 'Bahrain': 'BH', - 'Bangladesh': 'BD', - 'Barbados': 'BB', - 'Belarus': 'BY', - 'Belgium': 'BE', - 'Belize': 'BZ', - 'Benin': 'BJ', - 'Bermuda': 'BM', - 'Bhutan': 'BT', - 'Bolivia': 'BO', - 'Bonaire': 'BQ', - 'Bosnia and Herzegovina': 'BA', - 'Botswana': 'BW', - 'Bouvet Island': 'BV', - 'Brazil': 'BR', - 'British Indian Ocean Territory': 'IO', - 'British Virgin Islands': 'VG', - 'Virgin Islands, British': 'VG', - 'Brunei': 'BN', - 'Brunei Darussalam': 'BN', - 'Bulgaria': 'BG', - 'Burkina Faso': 'BF', - 'Burundi': 'BI', - 'Cambodia': 'KH', - 'Cameroon': 'CM', - 'Canada': 'CA', - 'Cape Verde': 'CV', - 'Cayman Islands': 'KY', - 'Central African Republic': 'CF', - 'Chad': 'TD', - 'Chile': 'CL', - 'China': 'CN', - 'Christmas Island': 'CX', - 'Cocos (Keeling) Islands': 'CC', - 'Colombia': 'CO', - 'Comoros': 'KM', - 'Congo': 'CG', - 'Congo, Republic of': 'CG', - 'Republic of the Congo': 'CG', - 'Cook Islands': 'CK', - 'Costa Rica': 'CR', - 'Croatia': 'HR', - 'Cuba': 'CU', - 'Curaçao': 'CW', - 'Cyprus': 'CY', - 'Czech Republic': 'CZ', - 'Congo, Democratic Republic of': 'CD', - 'Democratic Republic of the Congo': 'CD', - 'Denmark': 'DK', - 'Djibouti': 'DJ', - 'Dominica': 'DM', - 'Dominican Republic': 'DO', - 'East Timor': 'TP', - 'Ecuador': 'EC', - 'Egypt': 'EG', - 'El Salvador': 'SV', - 'Equatorial Guinea': 'GQ', - 'Eritrea': 'ER', - 'Estonia': 'EE', - 'Ethiopia': 'ET', - 'Falkland Islands': 'FK', - 'Faroe Islands': 'FO', - 'Fiji': 'FJ', - 'Finland': 'FI', - 'France': 'FR', - 'French Guiana': 'GF', - 'French Polynesia': 'PF', - 'Gabon': 'GA', - 'Gambia': 'GM', - 'Georgia': 'GE', - 'Germany': 'DE', - 'Ghana': 'GH', - 'Gibraltar': 'GI', - 'Greece': 'GR', - 'Greenland': 'GL', - 'Grenada': 'GD', - 'Guadeloupe': 'GP', - 'Great Britain': 'GB', - 'Guam': 'GU', - 'Guatemala': 'GT', - 'Guernsey': 'GG', - 'Guinea': 'GN', - 'Guinea-Bissau': 'GW', - 'Guyana': 'GY', - 'Haiti': 'HT', - 'Heard Island and McDonald Islands': 'HM', - 'Honduras': 'HN', - 'Hong Kong': 'HK', - 'Hungary': 'HU', - 'Iceland': 'IS', - 'India': 'IN', - 'Indonesia': 'ID', - 'Iran': 'IR', - 'Iraq': 'IQ', - 'Ireland': 'IE', - 'Isle of Man': 'IM', - 'Islamic Republic of Iran': 'IR', - 'Israel': 'IL', - 'Italy': 'IT', - 'Ivory Coast': 'CI', - 'Jamaica': 'JM', - 'Japan': 'JP', - 'Jersey': 'JE', - 'Jordan': 'JO', - 'Kazakhstan': 'KZ', - 'Kenya': 'KE', - "Korea, Democratic People's Republic of": 'KP', - 'Kiribati': 'KI', - 'Korea, Republic Of': 'KR', - 'Kosovo': 'XK', - 'Kuwait': 'KW', - 'Kyrgyzstan': 'KG', - 'Laos': 'LA', - "Lao People's Democratic Republic": 'LA', - 'Latvia': 'LV', - 'Lebanon': 'LB', - 'Lesotho': 'LS', - 'Liberia': 'LR', - 'Libya': 'LY', - 'Liechtenstein': 'LI', - 'Lithuania': 'LT', - 'Luxembourg': 'LU', - 'Macau': 'MO', - 'Macedonia': 'MK', - 'Macedonia, The Former Yugoslav Republic Of': 'MK', - 'Madagascar': 'MG', - 'Malawi': 'MW', - 'Malaysia': 'MY', - 'Maldives': 'MV', - 'Mali': 'ML', - 'Malta': 'MT', - 'Marshall Islands': 'MH', - 'Martinique': 'MQ', - 'Mauritania': 'MR', - 'Mauritius': 'MU', - 'Mayotte': 'YT', - 'Mexico': 'MX', - 'Micronesia': 'FM', - 'Micronesia, Federated States of': 'FM', - 'Moldova': 'MD', - 'Moldova, Republic Of': 'MD', - 'Monaco': 'MC', - 'Mongolia': 'MN', - 'Montenegro': 'ME', - 'Montserrat': 'MS', - 'Morocco': 'MA', - 'Mozambique': 'MZ', - 'Myanmar': 'MM', - 'Namibia': 'NA', - 'Nauru': 'NR', - 'Nepal': 'NP', - 'Netherlands': 'NL', - 'New Caledonia': 'NC', - 'New Zealand': 'NZ', - 'Nicaragua': 'NI', - 'Niger': 'NE', - 'Nigeria': 'NG', - 'Niue': 'NU', - 'Norfolk Island': 'NF', - 'North Korea': 'KP', - 'Northern Cyprus': 'CY', - 'Northern Mariana Islands': 'MP', - 'Norway': 'NO', - 'Oman': 'OM', - 'Pakistan': 'PK', - 'Palau': 'PW', - 'Palestine': 'PS', - 'Panama': 'PA', - 'Papua New Guinea': 'PG', - 'Paraguay': 'PY', - 'Peru': 'PE', - 'Philippines': 'PH', - 'Poland': 'PL', - 'Portugal': 'PT', - 'Puerto Rico': 'PR', - 'Qatar': 'QA', - 'Romania': 'RO', - 'Russia': 'RU', - 'Russian Federation': 'RU', - 'Rwanda': 'RW', - 'Réunion': 'RE', - 'Saba': 'BQ', - 'Saint Barthélemy': 'BL', - 'Saint Helena, Ascension and Tristan da Cunha': 'SH', - 'Saint Kitts and Nevis': 'KN', - 'St. Kitts and Nevis': 'KN', - 'Saint Lucia': 'LC', - 'St. Lucia': 'LC', - 'Saint Martin': 'MF', - 'St. Martin': 'MF', - 'Saint Pierre and Miquelon': 'PM', - 'St. Pierre and Miquelon': 'PM', - 'Saint Vincent and the Grenadines': 'VC', - 'St. Vincent and The Grenadines': 'VC', - 'Samoa': 'WS', - 'San Marino': 'SM', - 'Saudi Arabia': 'SA', - 'Senegal': 'SN', - 'Serbia': 'RS', - 'Seychelles': 'SC', - 'Sierra Leone': 'SL', - 'Singapore': 'SG', - 'Sint Eustatius': 'BQ', - 'Slovakia': 'SK', - 'Slovenia': 'SI', - 'Solomon Islands': 'SB', - 'Somalia': 'SO', - 'Somaliland': 'SO', - 'South Africa': 'ZA', - 'South Georgia and the South Sandwich Islands': 'GS', - 'South Korea': 'KR', - 'South Ossetia': 'OS', - 'South Sudan': 'SS', - 'Spain': 'ES', - 'Sri Lanka': 'LK', - 'Sudan': 'SD', - 'Suriname': 'SR', - 'Svalbard': 'SJ', - 'Swaziland': 'SZ', - 'Sweden': 'SE', - 'Switzerland': 'CH', - 'Syria': 'SY', - 'Syrian Arab Republic': 'SY', - 'São Tomé and Príncipe': 'ST', - 'Taiwan': 'TW', - 'Taiwan, Province of China': 'TW', - 'Tajikistan': 'TJ', - 'Tanzania': 'TZ', - 'Tanzania, United Republic Of': 'TZ', - 'Thailand': 'TH', - 'Togo': 'TG', - 'Tokelau': 'TK', - 'Tonga': 'TO', - 'Trinidad and Tobago': 'TT', - 'Tunisia': 'TN', - 'Turkey': 'TR', - 'Turkmenistan': 'TM', - 'Turks and Caicos Islands': 'TC', - 'Turks and Caicos': 'TC', - 'Tuvalu': 'TV', - 'Uganda': 'UG', - 'Ukraine': 'UA', - 'United Arab Emirates': 'AE', - 'United Kingdom': 'GB', - 'United States Virgin Islands': 'VI', - 'United States': 'US', - 'United States of America': 'US', - 'Uruguay': 'UY', - 'Uzbekistan': 'UZ', - 'Vanuatu': 'VU', - 'Venezuela': 'VE', - 'Vietnam': 'VN', - 'Wallis and Futuna': 'WF', - 'Yemen': 'YE', - 'Zambia': 'ZM', - 'Zimbabwe': 'ZW', - 'Åland Islands': 'AX', -} - - -def country_to_abbrev(country): - return COUNTRY_TO_ABBREV[country] diff --git a/tools/events-automation/rust_meetup_groups.csv b/tools/events-automation/rust_meetup_groups.csv deleted file mode 100644 index d6785b2a7..000000000 --- a/tools/events-automation/rust_meetup_groups.csv +++ /dev/null @@ -1,118 +0,0 @@ -name,url,location,non-meetup.com? -Rust London User Group,https://www.meetup.com/rust-london-user-group/,"London, 17, United Kingdom",no. -Rust NYC,https://www.meetup.com/rust-nyc/,"New York, NY, USA",no. -Rust Bay Area,https://www.meetup.com/Rust-Bay-Area/,"San Francisco, CA, USA",no. -Rust India,https://www.meetup.com/rustox/,"Bangalore, India",no. -Rust Berlin,https://www.meetup.com/rust-berlin/,"Berlin, Germany",no. -Seattle Rust User Group,https://www.meetup.com/seattle-rust-user-group/,"Seattle, WA, USA",no. -Rust Moscow,https://www.meetup.com/Rust-%D0%B2-%D0%9C%D0%BE%D1%81%D0%BA%D0%B2%D0%B5/,"Moscow, Russia",no. -Rust Paris,https://www.meetup.com/Rust-Paris/,"Paris, France",no. -Rust Nederland,https://www.meetup.com/rust-nederland/,"Amsterdam, Netherlands",no. -Rust Munich,https://www.meetup.com/rust-munich/,"München, Germany",no. -Vancouver Rust,https://www.meetup.com/vancouver-rust/,"Vancouver, BC, Canada",no. -Rust Chennai,https://www.meetup.com/mad-rs/,"Chennai, India",no. -Rust Linz,https://www.meetup.com/de-DE/rust-linz/,"Linz, Austria",no. -Boston Rust Meetup,https://www.meetup.com/bostonrust/,"Boston, MA, USA",no. -Rust São Paulo Meetup,https://www.meetup.com/Rust-Sao-Paulo-Meetup/,"São Paulo, Brazil",no. -Rust Language Hyderabad,https://www.meetup.com/Rust-Hyderabad/,"Hyderabad, India",no. -Rust Dublin,https://www.meetup.com/rust-dublin/,"Dublin, Ireland",no. -Rust Zurich,https://www.meetup.com/rust-zurich/,"Zürich, Switzerland",no. -Rust Melbourne,https://www.meetup.com/Rust-Melbourne/,"Melbourne, Australia",no. -Rust Argentina,https://www.meetup.com/rust-argentina/,"Buenos Aires, Argentina",no. -Rust TLV,https://www.meetup.com/rust-tlv/,"Tel Aviv-Yafo, Israel",no. -Rust and C++ Cardiff,https://www.meetup.com/rust-and-c-plus-plus-in-cardiff/,"Cardiff, X5, United Kingdom",no. -Rust MX,https://www.meetup.com/Rust-MX/,"México City, Mexico",no. -Rust Denver,https://www.meetup.com/Rust-Boulder-Denver/,"Denver, CO, USA",no. -Rust Delhi,https://www.meetup.com/rustdelhi/,"Delhi, India",no. -Rust DC,https://www.meetup.com/RustDC/,"Washington, DC, USA",no. -Rust Sydney,https://www.meetup.com/Rust-Sydney/,"Sydney, Australia",no. -Rust Oslo,https://www.meetup.com/Rust-Oslo/,"Oslo, Norway",no. -Dallas Rust User Meetup,https://www.meetup.com/dallasrust/,"Dallas, TX, USA",no. -Rust Cologne,https://www.meetup.com/rustcologne/,"Köln, Germany",no. -Stockholm Rust,https://www.meetup.com/stockholm-rust/,"Espoo, Finland",no. -PDXRust,https://www.meetup.com/PDXRust/,"Portland, OR, USA",no. -Tokyo Rust Meetup,https://www.meetup.com/tokyo-rust-meetup/,"Tokyo, Japan",no. -Rust Sthlm,https://www.meetup.com/ruststhlm/,"Stockholm, Sweden",no. -BcnRust,https://www.meetup.com/bcnrust/,"Barcelona, Spain",no. -Rust Toronto,https://www.meetup.com/Rust-Toronto/,"Toronto, ON, Canada",no. -Rust Language Milano,https://www.meetup.com/Rust-lang-Milano/,"Milano, MI, Italy",no. -Rust Meetup Hamburg,https://www.meetup.com/Rust-Meetup-Hamburg/,"Hamburg, Germany",no. -Rust AKL,https://www.meetup.com/rust-akl/,"Auckland, New Zealand",no. -Columbus Rust Society,https://www.meetup.com/columbus-rs/,"Columbus, OH, USA",no. -Finland Rust-lang Group,https://www.meetup.com/Finland-Rust-Meetup/,"Helsinki, Finland",no. -Rust ATX,https://www.meetup.com/rust-atx/,"Austin, TX, USA",no. -Rust Warsaw,https://www.meetup.com/Rust-Warsaw/,"Warsaw, Poland",no. -Rust Nuremberg,https://www.meetup.com/rust-noris/,"Nürnberg, Germany",no. -Utah Rust,https://www.meetup.com/utah-rust/,"Lehi, UT, USA",no. -Rust Los Angeles,https://www.meetup.com/Rust-Los-Angeles/,"Los Angeles, CA, USA",no. -Copenhagen Rust Community,https://www.meetup.com/copenhagen-rust-community/,"Copenhagen, Denmark",no. -MadRust,https://www.meetup.com/MadRust/,"Madrid, Spain",no. -Rust Vienna,https://www.meetup.com/rust-vienna/,"Vienna, Austria",no. -San Diego Rust,https://www.meetup.com/San-Diego-Rust/,"San Diego, CA, USA",no. -Minneapolis Rust Meetup,https://www.meetup.com/minneapolis-rust-meetup/,"Minneapolis, MN, USA",no. -Rust-Saar,https://www.meetup.com/Rust-Saar/,"Saarbrücken, Germany",no. -San Francisco Rust Study Group,https://www.meetup.com/san-francisco-rust-study-group/,"San Francisco, CA, USA",no. -Charlottesville Rust Meetup,https://www.meetup.com/charlottesville-rust-meetup/,"Charlottesville, VA, USA",no. -Chicago Rust Meetup,https://www.meetup.com/Chicago-Rust-Meetup/,"Chicago, IL, USA",no. -Rust Brisbane,https://www.meetup.com/Rust-Brisbane/,"Brisbane, Australia",no. -Buffalo Rust Meetup,https://www.meetup.com/buffalo-rust-meetup/,"Buffalo, NY, USA",no. -Rust Roma,https://www.meetup.com/it-IT/Rust-Roma/,"Roma, RM, Italia",no. -Rust Meetup Uruguay,https://www.meetup.com/Rust-Uruguay/,"Montevideo, Uruguay",no. -Indy Rust,https://www.meetup.com/indyrs/,"Indianapolis, IN, USA",no. -Cambridge Rust meetup,https://www.meetup.com/Cambridge-Rust-Meetup/,"Cambridge, C3, United Kingdom",no. -Rust Atlanta,https://www.meetup.com/Rust-ATL/,"Atlanta, GA, USA",no. -Montpellier Rust Meetup,https://www.meetup.com/Montpellier-Rust-Meetup/,"Montpellier, France",no. -Rust KW,https://www.meetup.com/Rust-KW/,"Kitchener, ON, Canada",no. -Rust Wellington,https://www.meetup.com/Wellington-Rust-Meetup/,"Wellington, New Zealand",no. -Desert Rust,https://www.meetup.com/Desert-Rustaceans/,"Phoenix, AZ, USA",no. -Rust Prague,https://www.meetup.com/rust-prague/,"Prague, Czech Republic",no. -Rust Aarhus,https://www.meetup.com/rust-aarhus/,"Aarhus, Denmark",no. -Rust Nigeria,https://www.meetup.com/rust-meetup-group/,"Lagos, Nigeria",no. -Rust Lille,https://www.meetup.com/meetup-group-zgphbyet/,"Lille, France",no. -Rust Montréal,https://www.meetup.com/Rust-Montreal/,"Montréal, QC, Canada",no. -Rust Vilnius,https://www.meetup.com/Rust-in-Vilnius/,"Vilnius, Lithuania",no. -Rust Medellin,https://www.meetup.com/rust-medellin/,"Medellin, Colombia",no. -Rust Rhein-Main,https://www.meetup.com/rust-rhein-main/,"Darmstadt, Germany",no. -Belgium Rust user group,https://www.meetup.com/belgium-rust-user-group/,"Brussels, Belgium",no. -Rust Basel,https://www.meetup.com/rust-basel/,"Basel, Switzerland",no. -impl Zagreb for Rust,https://www.meetup.com/zagreb-rust-meetup/,"Zagreb, Croatia",no. -Cap Hill Rust Coding/Hacking/Learning,https://www.meetup.com/cap-hill-rust/,"Seattle, WA, USA",no. -Oxford Rust Meetup Group,https://www.meetup.com/oxford-rust-meetup-group/,"Oxford, K2, United Kingdom",no. -Mountain View Rust Meetup,https://www.meetup.com/mv-rust-meetup/,"Mountain View, CA, USA",no. -Rust Franken,https://www.meetup.com/rust-nerf/,"Erlangen, Germany",no. -Rust Bern,https://www.meetup.com/rust-bern/,"Bern, Switzerland",no. -Rust Hack & Learn Karlsruhe,https://www.meetup.com/Rust-Hack-Learn-Karlsruhe/,"Karlsruhe, Germany",no. -Deep Dish Rust,https://www.meetup.com/deep-dish-rust/,"Chicago, IL, USA",no. -Vilnius Rust and Go Meetup Group,https://www.meetup.com/vilnius-rust-go-meetup-group/,"Vilnius, Lithuania",no. -Rust Czech Republic,https://www.meetup.com/rust-czech-republic/,"Prague, Czech Republic",no. -Ottawa Rust Language Meetup,https://www.meetup.com/Ottawa-Rust-Language-Meetup/,"Ottawa, ON, Canada",no. -Rust Belfast Meetup,https://www.meetup.com/Rust-Belfast-Meetup/,"Belfast, R3, United Kingdom",no. -Pasadena Thursday Go / Rust,https://www.meetup.com/thursday-go/,"Pasadena, CA, USA",no. -Hack-away with Rust,https://www.meetup.com/hack-away-with-rust/,"Espoo, Finland",no. -Rust Colombia,https://www.meetup.com/rust-colombia/,"Medellín, Colombia",no. -Rust Gdansk,https://www.meetup.com/rust-gdansk/,"Gdansk, Poland",no. -RustSchool Rotterdam,https://www.meetup.com/RustSchool-Rotterdam/,"Rotterdam, Netherlands",no. -Rust Rio,https://www.meetup.com/Rust-Rio/,"Rio de Janeiro, Brazil",no. -Rust Lang Comunidad Mexico,https://www.meetup.com/rustlangmx/,"Guadalajara, Mexico",no. -Rust - Modern Systems Programming in Leipzig,https://www.meetup.com/rust-modern-systems-programming-in-leipzig/,"Leipzig, Germany",no. -Rust Perth Meetup Group,https://www.meetup.com/perth-rust-meetup-group/,"Perth, Australia",no. -Rust Trondheim,https://www.meetup.com/rust-trondheim/,"Trondheim, Norway",no. -Rust Tbilisi,https://www.meetup.com/tbilisi-rustaceans/,"Tbilisi, Georgia",no. -Rust Halifax,https://www.meetup.com/rust-tell-halifax/,"Halifax, NS, Canada",no. -Rust Seoul,https://www.meetup.com/Rust-Seoul/,"Seoul, South Korea",no. -Canberra Rust User Group,https://www.meetup.com/rust-canberra/,"Canberra, Australia",no. -Budapest Rust Meetup Group,https://www.meetup.com/budapest-rust-meetup-group/,"Budapest, Hungary",no. -Paessler Rust Camp 2024,https://www.meetup.com/paessler-rust-camp-2024/,"Nürnbert, Germany",no. -Rust Taiwan Community,https://github.com/rust-tw/meetup,"Taipei, Taiwan",x -Rust Bordeaux,https://www.meetup.com/bordeaux-rust/,"Bordeaux, France",no. -Rust Girona,https://www.meetup.com/rust-girona/,"Girona, Spain",no. -Rust Circle Kampala,https://www.eventbrite.com/o/rust-circle-kampala-65249289033,Online,x -Rust Würzburg Meetup Group,https://www.meetup.com/rust-wurzburg-meetup-group/,"Würzburg, Germany",no. -Spokane,https://www.meetup.com/spokane-rust/,"Spokane, WA, USA",no. -Calgary Rust,https://www.eventbrite.ca/o/rust-calgary-63449860593,"Calgary, AB, Canada",x -Rust Indonesia,https://github.com/rustid/meetup,Indonesia,no. -Chengdu,https://github.com/RPG-Alex/rust-chengdu,"Chengdu, China",no. -Rust Lyon,https://www.meetup.com/fr-FR/rust-lyon/,"Lyon, France",no. -Rust Meetup Augsburg,https://www.meetup.com/de-DE/rust-meetup-augsburg/,"Augsburg, Germany",no. - Bratislava Rust Meetup Group,https://www.meetup.com/bratislava-rust-meetup-group/,"Bratislava, Slovakia",no. -Music City Rust Developers,https://www.meetup.com/music-city-rust-developers/,"Nashville, TN, USA",no. \ No newline at end of file diff --git a/tools/events-automation/state_territory_to_abbrev.py b/tools/events-automation/state_territory_to_abbrev.py deleted file mode 100644 index 0d74642f8..000000000 --- a/tools/events-automation/state_territory_to_abbrev.py +++ /dev/null @@ -1,102 +0,0 @@ -# Returns the abbreviated version of state/territory name for AU, CA, and US. -# Information from Wikipedia. - -US_STATE_TO_ABBREV = { - "Alabama": "AL", - "Alaska": "AK", - "Arizona": "AZ", - "Arkansas": "AR", - "California": "CA", - "Colorado": "CO", - "Connecticut": "CT", - "Delaware": "DE", - "Florida": "FL", - "Georgia": "GA", - "Hawaii": "HI", - "Idaho": "ID", - "Illinois": "IL", - "Indiana": "IN", - "Iowa": "IA", - "Kansas": "KS", - "Kentucky": "KY", - "Louisiana": "LA", - "Maine": "ME", - "Maryland": "MD", - "Massachusetts": "MA", - "Michigan": "MI", - "Minnesota": "MN", - "Mississippi": "MS", - "Missouri": "MO", - "Montana": "MT", - "Nebraska": "NE", - "Nevada": "NV", - "New Hampshire": "NH", - "New Jersey": "NJ", - "New Mexico": "NM", - "New York": "NY", - "North Carolina": "NC", - "North Dakota": "ND", - "Ohio": "OH", - "Oklahoma": "OK", - "Oregon": "OR", - "Pennsylvania": "PA", - "Rhode Island": "RI", - "South Carolina": "SC", - "South Dakota": "SD", - "Tennessee": "TN", - "Texas": "TX", - "Utah": "UT", - "Vermont": "VT", - "Virginia": "VA", - "Washington": "WA", - "West Virginia": "WV", - "Wisconsin": "WI", - "Wyoming": "WY", - "District of Columbia": "DC", - "American Samoa": "AS", - "Guam": "GU", - "Northern Mariana Islands": "MP", - "Puerto Rico": "PR", - "United States Minor Outlying Islands": "UM", - "U.S. Virgin Islands": "VI", -} - -CA_STATE_TERRITORY_TO_ABBREV = { - "Alberta": "AB", - "British Columbia": "BC", - "Manitoba": "MB", - "New Brunswick": "NB", - "Newfoundland and Labrador": "NL", - "Northwest Territories": "NT", - "Nova Scotia": "NS", - "Nunavut": "NU", - "Ontario": "ON", - "Prince Edward Island": "PE", - "Quebec": "QC", - "Saskatchewan": "SK", - "Yukon": "YT", -} - -AU_STATE_TERRITORY_TO_ABBREV = { - "New South Wales": "NSW", - "Northern Territory": "NT", - "Queensland": "QLD", - "South Australia": "SA", - "Tasmania": "TAS", - "Victoria": "VIC", - "Western Australia": "WA", -} - -def us_state_to_abbrev(state): - # Returns the abbreviated alpha code for input state. - return US_STATE_TO_ABBREV[state] - - -def ca_state_territory_to_abbrev(state): - # Returns the abbreviated alpha code for input state/territory. - return CA_STATE_TERRITORY_TO_ABBREV[state] - - -def au_state_territory_to_abbrev(state): - # Returns the abbreviated alpha code for input state/territory. - return AU_STATE_TERRITORY_TO_ABBREV[state] From 93425a324cc534f5c2025bbf1cc60bd62f769fb4 Mon Sep 17 00:00:00 2001 From: bdillo Date: Sat, 4 Jan 2025 17:39:28 -0800 Subject: [PATCH 12/18] remove geolocator and dependencies, meetup api has the location info we need --- tools/events-automation/event.py | 37 +------------------ .../generate_events_meetup.py | 2 - tools/events-automation/main.py | 5 +-- tools/events-automation/requirements.txt | 2 - 4 files changed, 2 insertions(+), 44 deletions(-) diff --git a/tools/events-automation/event.py b/tools/events-automation/event.py index 40b6a1bb9..31387885e 100644 --- a/tools/events-automation/event.py +++ b/tools/events-automation/event.py @@ -1,7 +1,6 @@ import logging from dataclasses import dataclass from datetime import datetime -from geopy.geocoders import Nominatim logger = logging.getLogger(__name__) @@ -92,10 +91,7 @@ def __init__(self, **kwargs) -> None: self.long = venue["lng"] self.event_location = Location(venue["city"], venue["state"], venue["country"]) - def to_event(self, geolocator: Nominatim, group_url: str) -> Event: - # address = (geolocator.reverse(str(self.lat) +","+ str(self.long))).raw["address"] - # location = self._format_location(address) - + def to_event(self, group_url: str) -> Event: is_virtual = self.venue_type == "online" # this is a bit weird because we want a naive datetime object that just contains the year/month/day because we get @@ -118,34 +114,3 @@ def to_event(self, geolocator: Nominatim, group_url: str) -> Event: organizer_url=group_url ) - # not sure if this is needed anymore, looks like the meetup API has a gooda mount of location data - # @staticmethod - # def _format_location(address: dict) -> str: - # # TODO: look this over... - # if not address: - # return "No location" - - # # All components for a location - # components = ['road', 'city', 'state', 'postcode', 'country'] - - # # Get available components, otherwise replace missing component with an empty string - # location = [address.get(component, "") for component in components] - - # return ','.join(location) if location else "No location" - - # # ???????????? - # country_code, city = locationData["country_code"].upper(), locationData.get("city", locationData.get("town", locationData.get("village", "**NO CITY DATA**"))) - # if country_code in ["AU", "CA", "US"]: - # state = locationData.get("state", locationData.get("territory", "**NO STATE DATA**")) - # if state == "**NO STATE DATA**": - # state_abbrev = state - # elif country_code == "AU": - # state_abbrev = au_state_territory_to_abbrev(state) - # elif country_code == "CA": - # state_abbrev = ca_state_territory_to_abbrev(state) - # elif country_code == "US": - # state_abbrev = us_state_to_abbrev(state) - # self.location = f'{city}, {state_abbrev}, {country_code}' - # else: - # self.location = f'{city}, {country_code}' - diff --git a/tools/events-automation/generate_events_meetup.py b/tools/events-automation/generate_events_meetup.py index 24a4ade9c..c6b1f565b 100644 --- a/tools/events-automation/generate_events_meetup.py +++ b/tools/events-automation/generate_events_meetup.py @@ -3,7 +3,6 @@ import logging from jwt_auth import generate_signed_jwt -from geopy.geocoders import Nominatim from event import Event, RawGqlEvent from utils import MeetupGroupUrl from typing import List @@ -18,7 +17,6 @@ class TwirMeetupClient: def __init__(self) -> None: self._access_token = None self._refresh_token = None - self._geolocator = Nominatim(user_agent="TWiR") def _authenticate(self): """ diff --git a/tools/events-automation/main.py b/tools/events-automation/main.py index f588eaad2..536352fa9 100644 --- a/tools/events-automation/main.py +++ b/tools/events-automation/main.py @@ -5,7 +5,6 @@ import argparse import logging -from geopy.geocoders import Nominatim from typing import List from datetime import date, timedelta @@ -27,8 +26,6 @@ def main(): logger.info("Starting...") meetup_client = TwirMeetupClient() - # TODO: see if we actually need to use this - geolocator = Nominatim(user_agent="TWiR") # get our known rust meetup groups group_urls = read_meetup_group_urls(args.groups_file) @@ -37,7 +34,7 @@ def main(): for group_url in group_urls: group_raw_events = meetup_client.get_raw_events_for_group(group_url) - events += [raw_event.to_event(geolocator, group_url.url) for raw_event in group_raw_events] + events += [raw_event.to_event(group_url.url) for raw_event in group_raw_events] # Remove events outside of date range. events = date_window_filter(events, args.weeks) diff --git a/tools/events-automation/requirements.txt b/tools/events-automation/requirements.txt index 08fdb05fd..3715b70b1 100644 --- a/tools/events-automation/requirements.txt +++ b/tools/events-automation/requirements.txt @@ -2,8 +2,6 @@ certifi==2024.8.30 cffi==1.17.1 charset-normalizer==3.4.0 cryptography==43.0.3 -geographiclib==2.0 -geopy==2.4.1 idna==3.10 pycparser==2.22 PyJWT==2.10.0 From 82a4ced74908c516329fc2693e666d9ac96940da Mon Sep 17 00:00:00 2001 From: Marianne Goldin <23177337+mariannegoldin@users.noreply.github.com> Date: Wed, 8 Jan 2025 00:50:29 -0800 Subject: [PATCH 13/18] Add to README --- tools/events-automation/README.md | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/tools/events-automation/README.md b/tools/events-automation/README.md index baa4a3441..a459b94c6 100644 --- a/tools/events-automation/README.md +++ b/tools/events-automation/README.md @@ -7,14 +7,30 @@ The maintainer of the Events section is faced with manual work; this process aut - For specific module requirements: `pip install -r requirements.txt` - See https://geopy.readthedocs.io/en/stable/# for `geopy` module documentation. +### Setup the virutal environment: + +```bash +python3 -m venv events-venv && source events-venv/bin/activate +pip install -r requirements.txt +``` + +### Setup your environment variables: + +```bash +export MEETUP_PRIVATE_KEY="" +# e.g. export MEETUP_PRIVATE_KEY="$HOME/.ssh/meetup_signing_key.pem" +export MEETUP_AUTHORIZED_MEMBER_ID="" +export MEETUP_CLIENT_KEY="" +``` + ### Running: Before running please check that all Event Source module function calls are included in `event_list` (function calls should concatenate into a single list of event objects). To run this code: ```py -pip install -r requirements.txt -python3 main.py +python3 main.py -g +# e.g. python3 main.py -g ../events/meetups.json ``` ### How to Add a New Event Source Module: From 725cbe5cf401211c8df00e923cc2b44a52fdad7d Mon Sep 17 00:00:00 2001 From: bdillo Date: Sat, 21 Dec 2024 12:33:54 -0800 Subject: [PATCH 14/18] rename meetups.json to rust-meetups.json to show it's for rust specific meetup groups --- tools/events/{meetups.json => rust-meetups.json} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tools/events/{meetups.json => rust-meetups.json} (100%) diff --git a/tools/events/meetups.json b/tools/events/rust-meetups.json similarity index 100% rename from tools/events/meetups.json rename to tools/events/rust-meetups.json From 7cb23d1adb1e26a1499bda2884c7de18acdf3083 Mon Sep 17 00:00:00 2001 From: bdillo Date: Sat, 21 Dec 2024 12:41:16 -0800 Subject: [PATCH 15/18] Cherry pick changes from meetup-group-updates --- tools/events/maybe-rust-meetups.json | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 tools/events/maybe-rust-meetups.json diff --git a/tools/events/maybe-rust-meetups.json b/tools/events/maybe-rust-meetups.json new file mode 100644 index 000000000..236a229cb --- /dev/null +++ b/tools/events/maybe-rust-meetups.json @@ -0,0 +1,27 @@ +[ + "https://www.meetup.com/barcelona-free-software/events/", + "https://www.meetup.com/bellinghamcodes/events/", + "https://www.meetup.com/bergen-html-css-meetup-group/events/", + "https://www.meetup.com/bitdevsla/events/", + "https://www.meetup.com/blockchaincenter/events/", + "https://www.meetup.com/boulder-elixir/events/", + "https://www.meetup.com/code-mavens/events/", + "https://www.meetup.com/coderrange-endless-programming-languages/events/", + "https://www.meetup.com/conf42/events/", + "https://www.meetup.com/data-ai-online/events/", + "https://www.meetup.com/func-prog-sweden/events/", + "https://www.meetup.com/hackerdojo/events/", + "https://www.meetup.com/hackschool-rhein-neckar/events/", + "https://www.meetup.com/houston-functional-programming-users-group/events/", + "https://www.meetup.com/pydelhi/events/", + "https://www.meetup.com/stacja-it-krakow/events/", + "https://www.meetup.com/stacja-it-trojmiasto/events/", + "https://www.meetup.com/stacja-it-wroclaw/events/", + "https://www.meetup.com/techmeetupostrava/events/", + "https://www.meetup.com/the-karlsruhe-functional-programmers-meetup-group/events/", + "https://www.meetup.com/ucsc-extension-community/events/", + "https://www.meetup.com/vancouver-postgres/events/", + "https://www.meetup.com/vilnius-rust-go-meetup-group/events/", + "https://www.meetup.com/wasm-rust-meetup/events/", + "https://www.meetup.com/wearedevelopers-community/events/" +] From b2cfc43d6f1f2372db4c2d368c0b6a881a234ddf Mon Sep 17 00:00:00 2001 From: bdillo Date: Sun, 12 Jan 2025 11:04:16 -0800 Subject: [PATCH 16/18] move tools/events-automation dir to tools/events/meetup-automation to group event tooling together and match upstream dir layout --- tools/{events-automation => events/meetup-automation}/README.md | 0 .../meetup-automation}/country_code_to_continent.py | 0 tools/{events-automation => events/meetup-automation}/event.py | 0 .../meetup-automation}/generate_events_meetup.py | 0 tools/{events-automation => events/meetup-automation}/jwt_auth.py | 0 tools/{events-automation => events/meetup-automation}/main.py | 0 .../meetup-automation}/meetup_events.md | 0 .../meetup-automation}/requirements.txt | 0 .../meetup-automation}/test_events.py | 0 tools/{events-automation => events/meetup-automation}/utils.py | 0 10 files changed, 0 insertions(+), 0 deletions(-) rename tools/{events-automation => events/meetup-automation}/README.md (100%) rename tools/{events-automation => events/meetup-automation}/country_code_to_continent.py (100%) rename tools/{events-automation => events/meetup-automation}/event.py (100%) rename tools/{events-automation => events/meetup-automation}/generate_events_meetup.py (100%) rename tools/{events-automation => events/meetup-automation}/jwt_auth.py (100%) rename tools/{events-automation => events/meetup-automation}/main.py (100%) rename tools/{events-automation => events/meetup-automation}/meetup_events.md (100%) rename tools/{events-automation => events/meetup-automation}/requirements.txt (100%) rename tools/{events-automation => events/meetup-automation}/test_events.py (100%) rename tools/{events-automation => events/meetup-automation}/utils.py (100%) diff --git a/tools/events-automation/README.md b/tools/events/meetup-automation/README.md similarity index 100% rename from tools/events-automation/README.md rename to tools/events/meetup-automation/README.md diff --git a/tools/events-automation/country_code_to_continent.py b/tools/events/meetup-automation/country_code_to_continent.py similarity index 100% rename from tools/events-automation/country_code_to_continent.py rename to tools/events/meetup-automation/country_code_to_continent.py diff --git a/tools/events-automation/event.py b/tools/events/meetup-automation/event.py similarity index 100% rename from tools/events-automation/event.py rename to tools/events/meetup-automation/event.py diff --git a/tools/events-automation/generate_events_meetup.py b/tools/events/meetup-automation/generate_events_meetup.py similarity index 100% rename from tools/events-automation/generate_events_meetup.py rename to tools/events/meetup-automation/generate_events_meetup.py diff --git a/tools/events-automation/jwt_auth.py b/tools/events/meetup-automation/jwt_auth.py similarity index 100% rename from tools/events-automation/jwt_auth.py rename to tools/events/meetup-automation/jwt_auth.py diff --git a/tools/events-automation/main.py b/tools/events/meetup-automation/main.py similarity index 100% rename from tools/events-automation/main.py rename to tools/events/meetup-automation/main.py diff --git a/tools/events-automation/meetup_events.md b/tools/events/meetup-automation/meetup_events.md similarity index 100% rename from tools/events-automation/meetup_events.md rename to tools/events/meetup-automation/meetup_events.md diff --git a/tools/events-automation/requirements.txt b/tools/events/meetup-automation/requirements.txt similarity index 100% rename from tools/events-automation/requirements.txt rename to tools/events/meetup-automation/requirements.txt diff --git a/tools/events-automation/test_events.py b/tools/events/meetup-automation/test_events.py similarity index 100% rename from tools/events-automation/test_events.py rename to tools/events/meetup-automation/test_events.py diff --git a/tools/events-automation/utils.py b/tools/events/meetup-automation/utils.py similarity index 100% rename from tools/events-automation/utils.py rename to tools/events/meetup-automation/utils.py From 5ef4ba90759629469846a50f1b4e1c0d2945ceda Mon Sep 17 00:00:00 2001 From: bdillo Date: Tue, 21 Jan 2025 18:33:05 -0800 Subject: [PATCH 17/18] update meetup group list to match twir master --- tools/events/rust-meetups.json | 107 --------------------------------- 1 file changed, 107 deletions(-) diff --git a/tools/events/rust-meetups.json b/tools/events/rust-meetups.json index 643d82c6b..ee020f99f 100644 --- a/tools/events/rust-meetups.json +++ b/tools/events/rust-meetups.json @@ -1,120 +1,41 @@ [ - "https://www.meetup.com/8th-light-university/events/", - "https://www.meetup.com/altow-academy/events/", - "https://www.meetup.com/atx-rustaceans/events/", - "https://www.meetup.com/aws-cologne/events/", - "https://www.meetup.com/barcelona-free-software/events/", - "https://www.meetup.com/bay-area-newsql-database-meetup/events/", "https://www.meetup.com/bcnrust/events/", "https://www.meetup.com/belgium-rust-user-group/events/", "https://www.meetup.com/belgrade-rust-meetup-group/events/", - "https://www.meetup.com/bellinghamcodes/events/", - "https://www.meetup.com/bergen-html-css-meetup-group/events/", - "https://www.meetup.com/berlin-mozilla-meetup/events/", - "https://www.meetup.com/berlinawsug/events/", "https://www.meetup.com/bevy-game-development/events/", - "https://www.meetup.com/bitdevsla/events/", - "https://www.meetup.com/blockchaincenter/events/", "https://www.meetup.com/bordeaux-rust/events/", - "https://www.meetup.com/boston-rust-meetup-25317522anphwzdw/events/", "https://www.meetup.com/bostonrust/events/", - "https://www.meetup.com/boulder-elixir/events/", - "https://www.meetup.com/boulder-elixir-rust/events/", "https://www.meetup.com/boulder-rust-meetup/events/", "https://www.meetup.com/bratislava-rust-meetup-group/events/", - "https://www.meetup.com/brussels-hackerspace/events/", - "https://www.meetup.com/budapest-rust-meetup-group/events/", "https://www.meetup.com/buffalo-rust-meetup/events/", - "https://www.meetup.com/c-programmer-meetup/events/", "https://www.meetup.com/cambridge-rust-meetup/events/", "https://www.meetup.com/charlottesville-rust-meetup/events/", - "https://www.meetup.com/chicago-healthcare-tech-and-ai/events/", "https://www.meetup.com/chicago-rust-meetup/events/", "https://www.meetup.com/christchurch-rustlang-meetup-group/events/", - "https://www.meetup.com/cloud-native-new-york/events/", - "https://www.meetup.com/code-mavens/events/", - "https://www.meetup.com/coderrange-endless-programming-languages/events/", "https://www.meetup.com/columbus-rs/events/", - "https://www.meetup.com/conf42/events/", "https://www.meetup.com/copenhagen-rust-community/events/", - "https://www.meetup.com/copenhagen-tech-polyglots/events/", "https://www.meetup.com/dallasrust/events/", - "https://www.meetup.com/data-ai-online/events/", - "https://www.meetup.com/data-science-on-aws/events/", - "https://www.meetup.com/data-umbrella/events/", - "https://www.meetup.com/data-umbrella-africa2/events/", "https://www.meetup.com/deep-dish-rust/events/", "https://www.meetup.com/desert-rustaceans/events/", "https://www.meetup.com/detroitrust/events/", - "https://www.meetup.com/digital-craftsmanship-nordoberpfalz/events/", "https://www.meetup.com/dutch-rust-meetup/events/", - "https://www.meetup.com/edmonton-r-user-group-yegrug/events/", - "https://www.meetup.com/everyonecancontribute-cafe/events/", "https://www.meetup.com/finland-rust-meetup/events/", - "https://www.meetup.com/fp-eug/events/", - "https://www.meetup.com/freshminds-future-proof-software-development/events/", - "https://www.meetup.com/frontend-developer-meetup-amsterdam/events/", - "https://www.meetup.com/frontend_rm/events/", - "https://www.meetup.com/func-prog-sweden/events/", - "https://www.meetup.com/functional-vilnius/events/", - "https://www.meetup.com/gdg-columbus/events/", - "https://www.meetup.com/goto-nights-berlin/events/", - "https://www.meetup.com/granadagdg/events/", - "https://www.meetup.com/guru-sp-grupo-de-usuarios-ruby-de-sao-paulo/events/", - "https://www.meetup.com/hackerdojo/events/", - "https://www.meetup.com/hackschool-rhein-neckar/events/", "https://www.meetup.com/helsinki-rust-meetup-group/events/", - "https://www.meetup.com/houston-functional-programming-users-group/events/", - "https://www.meetup.com/hwswfree/events/", "https://www.meetup.com/indyrs/events/", - "https://www.meetup.com/itgilde-cooperatie-amsterdam-unix-linux-meetups/events/", - "https://www.meetup.com/java-user-group-hessen-jugh/events/", "https://www.meetup.com/johannesburg-rust-meetup/events/", "https://www.meetup.com/join-srug/events/", - "https://www.meetup.com/kaibee/events/", - "https://www.meetup.com/longview-code-and-coffee/events/", - "https://www.meetup.com/los-gatos-rust-reading-group/events/", - "https://www.meetup.com/ludwigslust-rust-meetup/events/", "https://www.meetup.com/mad-rs/events/", "https://www.meetup.com/madrust/events/", - "https://www.meetup.com/maibornwolff-software-engineering-netzwerk/events/", - "https://www.meetup.com/meetup-group-boston-nosql-database-group/events/", "https://www.meetup.com/meetup-group-zgphbyet/events/", - "https://www.meetup.com/michigan-python/events/", - "https://www.meetup.com/microsoft-reactor-london/events/", - "https://www.meetup.com/microsoft-reactor-new-york/events/", - "https://www.meetup.com/microsoft-reactor-redmond/events/", - "https://www.meetup.com/microsoft-reactor-san-francisco/events/", - "https://www.meetup.com/microsoft-reactor-sao-paulo/events/", - "https://www.meetup.com/microsoft-reactor-stockholm/events/", - "https://www.meetup.com/microsoft-reactor-toronto/events/", "https://www.meetup.com/minneapolis-rust-meetup/events/", - "https://www.meetup.com/mob-programming-on-open-source-software/events/", - "https://www.meetup.com/monkey-tech-days/events/", "https://www.meetup.com/montpellier-rust-meetup/events/", - "https://www.meetup.com/mozilla-meetup-switzerland/events/", - "https://www.meetup.com/mucplusplus/events/", "https://www.meetup.com/music-city-rust-developers/events/", "https://www.meetup.com/mv-rust-meetup/events/", - "https://www.meetup.com/newspace-nyc/events/", "https://www.meetup.com/oc-rust/events/", - "https://www.meetup.com/ocaml-paris/events/", - "https://www.meetup.com/opentechschool-berlin/events/", "https://www.meetup.com/oxford-rust-meetup-group/events/", "https://www.meetup.com/paessler-rust-camp-2024/events/", - "https://www.meetup.com/papers-we-love-vienna/events/", - "https://www.meetup.com/paris-scala-user-group-psug/events/", "https://www.meetup.com/pdxrust/events/", "https://www.meetup.com/perth-rust-meetup-group/events/", - "https://www.meetup.com/phx-android/events/", - "https://www.meetup.com/polkadot-india/events/", - "https://www.meetup.com/portland-solana-blockchain-meetup/events/", - "https://www.meetup.com/prague-containers-meetup/events/", - "https://www.meetup.com/prenzlauer-berg-software-engineers/events/", - "https://www.meetup.com/programming-languages-toronto-meetup/events/", - "https://www.meetup.com/pydelhi/events/", - "https://www.meetup.com/react-berlin-meetup/events/", - "https://www.meetup.com/reactive-application-programmers-in-delhi-ncr/events/", "https://www.meetup.com/reading-rust-workshop/events/", "https://www.meetup.com/rust-aarhus/events/", "https://www.meetup.com/rust-aarhus-organizers/events/", @@ -142,11 +63,8 @@ "https://www.meetup.com/rust-frankfurt/events/", "https://www.meetup.com/rust-gdansk/events/", "https://www.meetup.com/rust-getting-started/events/", - "https://www.meetup.com/rust-girona/events/", "https://www.meetup.com/rust-hack-learn-karlsruhe/events/", "https://www.meetup.com/rust-hyderabad/events/", - "https://www.meetup.com/rust-in-blockchain-san-francisco/events/", - "https://www.meetup.com/rust-in-vilnius/events/", "https://www.meetup.com/rust-kw/events/", "https://www.meetup.com/rust-language-milano/events/", "https://www.meetup.com/rust-linz/events/", @@ -195,38 +113,13 @@ "https://www.meetup.com/ruststhlm/events/", "https://www.meetup.com/san-diego-rust/events/", "https://www.meetup.com/san-francisco-rust-study-group/events/", - "https://www.meetup.com/sfugcgn/events/", - "https://www.meetup.com/singapore-web3-blockchain-meetup/events/", - "https://www.meetup.com/softwerkskammer-ruhrgebiet/events/", - "https://www.meetup.com/solidstatedepot/events/", "https://www.meetup.com/spokane-rust/events/", - "https://www.meetup.com/stacja-it-krakow/events/", - "https://www.meetup.com/stacja-it-trojmiasto/events/", - "https://www.meetup.com/stacja-it-wroclaw/events/", "https://www.meetup.com/stl-rust/events/", - "https://www.meetup.com/stockholm-google-developer-group/events/", "https://www.meetup.com/stockholm-rust/events/", - "https://www.meetup.com/stockholmcpp/events/", - "https://www.meetup.com/techatagoda/events/", - "https://www.meetup.com/techceleration/events/", - "https://www.meetup.com/techmeetupostrava/events/", - "https://www.meetup.com/tel-aviv-data-science-odsc/events/", - "https://www.meetup.com/the-karlsruhe-functional-programmers-meetup-group/events/", - "https://www.meetup.com/the-south-padre-island-reading-group/events/", - "https://www.meetup.com/thursday-go/events/", "https://www.meetup.com/tokyo-rust-meetup/events/", - "https://www.meetup.com/triangle-bitdevs/events/", "https://www.meetup.com/triangle-rust/events/", - "https://www.meetup.com/ucsc-extension-community/events/", "https://www.meetup.com/utah-rust/events/", - "https://www.meetup.com/vancouver-postgres/events/", "https://www.meetup.com/vancouver-rust/events/", - "https://www.meetup.com/vilnius-rust-go-meetup-group/events/", - "https://www.meetup.com/wasm-rust-meetup/events/", - "https://www.meetup.com/wasmna/events/", - "https://www.meetup.com/wasmsf/events/", - "https://www.meetup.com/wearedevelopers-community/events/", - "https://www.meetup.com/webassembly-and-wasmedge/events/", "https://www.meetup.com/wellington-rust-meetup/events/", "https://www.meetup.com/women-in-rust/events/", "https://www.meetup.com/zagreb-rust-meetup/events/" From bb13506f8f2f0189f19085b714efbf3869928f6a Mon Sep 17 00:00:00 2001 From: bdillo Date: Wed, 22 Jan 2025 16:04:30 -0800 Subject: [PATCH 18/18] add some new groups --- tools/events/maybe-rust-meetups.json | 1 + tools/events/rust-meetups.json | 1 + 2 files changed, 2 insertions(+) diff --git a/tools/events/maybe-rust-meetups.json b/tools/events/maybe-rust-meetups.json index 236a229cb..29120b238 100644 --- a/tools/events/maybe-rust-meetups.json +++ b/tools/events/maybe-rust-meetups.json @@ -13,6 +13,7 @@ "https://www.meetup.com/hackerdojo/events/", "https://www.meetup.com/hackschool-rhein-neckar/events/", "https://www.meetup.com/houston-functional-programming-users-group/events/", + "https://www.meetup.com/jersey-city-classy-curious-coders-club-cooperative/events/", "https://www.meetup.com/pydelhi/events/", "https://www.meetup.com/stacja-it-krakow/events/", "https://www.meetup.com/stacja-it-trojmiasto/events/", diff --git a/tools/events/rust-meetups.json b/tools/events/rust-meetups.json index ee020f99f..d098f135f 100644 --- a/tools/events/rust-meetups.json +++ b/tools/events/rust-meetups.json @@ -24,6 +24,7 @@ "https://www.meetup.com/indyrs/events/", "https://www.meetup.com/johannesburg-rust-meetup/events/", "https://www.meetup.com/join-srug/events/", + "https://www.meetup.com/london-rust-project-group/events/", "https://www.meetup.com/mad-rs/events/", "https://www.meetup.com/madrust/events/", "https://www.meetup.com/meetup-group-zgphbyet/events/",