Added multi-threading to the update script

scripts/add_new_contribution_to_yaml.py

@@ -1,7 +1,7 @@
 """
 given properties, add a new contribution to the contributions.yaml database file.
 """
-from datetime import datetime
+from datetime import datetime, UTC
 import json
 import pathlib
 from sys import argv
@@ -39,7 +39,7 @@
 
     # append new contribution with next index
     # add status, at top
-    datetime_today = datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%S%z')
+    datetime_today = datetime.now(UTC).strftime('%Y-%m-%dT%H:%M:%S%z')
     contribution = {
         'id': max_index + 1,
         'status': 'VALID',

scripts/fetch_updates.py

@@ -2,15 +2,16 @@
 Reads in the contributions.yaml file, and updates the entries by hitting the 'source' url.
 """
 import argparse
-from datetime import datetime
+from datetime import datetime, UTC
 import pathlib
 from ruamel.yaml import YAML
+from multiprocessing import Pool
 
 from parse_and_validate_properties_txt import read_properties_txt, parse_text, validate_existing
 
 
 def update_contribution(contribution, props):
-  datetime_today = datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%S%z')
+  datetime_today = datetime.now(UTC).strftime('%Y-%m-%dT%H:%M:%S%z')
   contribution['lastUpdated'] = datetime_today
   if 'previousVersions' not in contribution:
     contribution['previousVersions'] = []
@@ -29,6 +30,7 @@ def update_contribution(contribution, props):
 
   if 'download' not in contribution:
     contribution['download'] = contribution['source'][:contribution['source'].rfind('.')] + '.zip'
+
 
 def log_broken(contribution, msg):
   if contribution['status'] == 'VALID':
@@ -38,7 +40,7 @@ def log_broken(contribution, msg):
     contribution['log'].append(msg)
 
 def process_contribution(contribution):
-  date_today = datetime.utcnow().strftime('%Y-%m-%d')
+  date_today = datetime.now(UTC).strftime('%Y-%m-%d')
   this_version = '0'
 
   if contribution['status'] != 'DEPRECATED':
@@ -92,9 +94,16 @@ def process_contribution(contribution):
   contributions_list = data['contributions']
 
   if index == 'all':
-    # update all contributions
-    for contribution in contributions_list:
-      process_contribution(contribution)
+    total = len(contributions_list)
+    completed = 0
+    print(f"Starting processing of {total} contributions...")
+
+    with Pool(processes=256) as pool:
+        for _ in pool.imap_unordered(process_contribution, contributions_list):
+            completed += 1
+            print(f"Progress: {completed}/{total} ({(completed/total*100):.1f}%)")
+
+    print("All processing complete")
   else:
     # update only contribution with id==index
     contribution = next((x for x in contributions_list if x['id'] == int(index)), None)

scripts/parse_and_validate_properties_txt.py

@@ -64,7 +64,7 @@ def read_properties_txt(properties_url):
         'User-Agent': 'Mozilla/5.0',
         'Accept': 'text/html',
     }
-    r = requests.get(properties_url, headers=headers)
+    r = requests.get(properties_url, headers=headers, timeout=30)
 
     if r.status_code != 200:
         raise FileNotFoundError(f"status code {r.status_code} returned for url {r.url}")