edit API script with rate limitting param

Author: a5dur

sites-workflow/1getSitesURL.py

@@ -4,12 +4,90 @@
 import requests
 import pandas as pd
 import re
+import time
 from urllib.parse import urlparse
+from requests.adapters import HTTPAdapter
+from urllib3.util.retry import Retry
 
 class SimpleSiteURLExtractor:
-    def __init__(self):
+    def __init__(self, requests_per_second=1):
         self.base_url = "https://catalog.civicdataecosystem.org"
         self.api_base = f"{self.base_url}/api/3/action"
+        self.min_interval = 1.0 / requests_per_second
+        self.last_request_time = 0
+        
+        # Setup session with retry strategy
+        self.session = requests.Session()
+        
+        # Handle different urllib3 versions
+        try:
+            retry_strategy = Retry(
+                total=3,
+                status_forcelist=[429, 500, 502, 503, 504],
+                allowed_methods=["HEAD", "GET", "OPTIONS"],  # New parameter name
+                backoff_factor=2
+            )
+        except TypeError:
+            # Fallback for older urllib3 versions
+            retry_strategy = Retry(
+                total=3,
+                status_forcelist=[429, 500, 502, 503, 504],
+                method_whitelist=["HEAD", "GET", "OPTIONS"],  # Old parameter name
+                backoff_factor=2
+            )
+        adapter = HTTPAdapter(max_retries=retry_strategy)
+        self.session.mount("http://", adapter)
+        self.session.mount("https://", adapter)
+        
+        # Set proper headers to identify ourselves
+        self.session.headers.update({
+            'User-Agent': 'CKAN-Metadata-Workflow/1.0 (Educational/Research Purpose)',
+            'Accept': 'application/json',
+            'Accept-Encoding': 'gzip, deflate',
+            'Connection': 'keep-alive'
+        })
+        
+    def _rate_limit(self):
+        """Enforce rate limiting between requests"""
+        current_time = time.time()
+        time_since_last = current_time - self.last_request_time
+        if time_since_last < self.min_interval:
+            sleep_time = self.min_interval - time_since_last
+            print(f"Rate limiting: waiting {sleep_time:.2f}s")
+            time.sleep(sleep_time)
+        self.last_request_time = time.time()
+    
+    def _make_request(self, url, params=None, max_retries=3):
+        """Make a rate-limited request with error handling"""
+        self._rate_limit()
+        
+        for attempt in range(max_retries):
+            try:
+                response = self.session.get(url, params=params, timeout=30)
+                
+                if response.status_code == 403:
+                    wait_time = (attempt + 1) * 30
+                    print(f"HTTP 403 - Rate limited or forbidden. Waiting {wait_time}s (attempt {attempt + 1}/{max_retries})")
+                    time.sleep(wait_time)
+                    continue
+                    
+                elif response.status_code == 429:
+                    wait_time = (attempt + 1) * 60
+                    print(f"HTTP 429 - Rate limit exceeded. Waiting {wait_time}s (attempt {attempt + 1}/{max_retries})")
+                    time.sleep(wait_time)
+                    continue
+                
+                response.raise_for_status()
+                return response
+                
+            except requests.exceptions.RequestException as e:
+                if attempt == max_retries - 1:  # Last attempt
+                    print(f"Request failed after {max_retries} attempts: {e}")
+                    return None
+                print(f"Request failed (attempt {attempt + 1}/{max_retries}): {e}")
+                time.sleep((attempt + 1) * 10)
+        
+        return None
 
     def clean_url(self, url):
         """Clean and validate URL"""
@@ -81,12 +159,40 @@ def get_all_sites(self):
 
         all_packages = []
         start = 0
-        rows = 1000
+        rows = 100  # Reduced batch size to be more conservative
+        
+        # First, try to get a smaller test batch to verify API access
+        print("Testing API access with small batch...")
+        test_response = self._make_request(
+            f"{self.api_base}/package_search",
+            params={
+                'q': 'type:site',
+                'start': 0,
+                'rows': 10,
+                'include_private': False
+            }
+        )
 
-        while True:
+        if not test_response:
+            print("Failed to access API.")
+            return []
+        
+        test_data = test_response.json()
+        if not test_data.get('success'):
+            print("API test failed.")
+            return []
+        
+        total_count = test_data['result'].get('count', 0)
+        print(f"API test successful. Total sites available: {total_count}")
+        
+        # Add a reasonable limit to avoid overwhelming the API
+        max_sites = min(total_count, 1000)  # Limit to 1000 sites max
+        print(f"Will fetch up to {max_sites} sites")
+        
+        while len(all_packages) < max_sites:
             print(f"Fetching batch starting at {start}...")
 
-            response = requests.get(
+            response = self._make_request(
                 f"{self.api_base}/package_search",
                 params={
                     'q': 'type:site',
@@ -96,37 +202,49 @@ def get_all_sites(self):
                 }
             )
 
-            if response.status_code != 200:
-                print(f"API failed with status {response.status_code}")
+            if not response:
+                print("Failed to fetch batch, stopping...")
                 break
 
             data = response.json()
             if not data.get('success'):
-                print("API returned error")
+                print("API returned error, stopping...")
                 break
 
             result = data['result']
             batch_packages = result.get('results', [])
-            total_count = result.get('count', 0)
 
+            if not batch_packages:
+                print("No more results available")
+                break
+                
             all_packages.extend(batch_packages)
 
-            print(f"Fetched {len(all_packages)}/{total_count} sites")
+            print(f"Fetched {len(all_packages)}/{min(total_count, max_sites)} sites")
 
-            # Stop if we got fewer results than requested or reached the total
-            if len(batch_packages) < rows or len(all_packages) >= total_count:
+            # Stop if we got fewer results than requested
+            if len(batch_packages) < rows:
                 break
 
             start += rows
+            
+            # Extra delay between batches to be respectful
+            print("Waiting between batches...")
+            time.sleep(5)
 
         print(f"Total sites found: {len(all_packages)}")
-        
+        return self._process_packages(all_packages)
+    
+    def _process_packages(self, all_packages):
+        """Process packages to extract URLs"""
         results = []
 
         for i, pkg in enumerate(all_packages, 1):
             site_name = pkg.get('name', '')
             site_title = pkg.get('title', '')
-            print(f"Processing {i}/{len(all_packages)}: {site_name}")
+            
+            if i % 50 == 0:  # Progress update every 50 items
+                print(f"Processing {i}/{len(all_packages)}: {site_name}")
 
             # Extract the visit URL
             visit_url = self.extract_visit_url(pkg)
@@ -139,6 +257,11 @@ def get_all_sites(self):
                 'name': site_name,
                 'url': visit_url
             })
+            
+            # Small delay between processing items if extracting from notes
+            # to avoid overwhelming regex processing
+            if i % 100 == 0:
+                time.sleep(1)
 
         return results
 
@@ -147,7 +270,14 @@ def save_to_csv(self, results, filename):
         # Filter out empty URLs
         filtered_results = [r for r in results if r['url']]
 
-        df = pd.DataFrame(filtered_results)
+        if not filtered_results:
+            # If no URLs found, create empty CSV
+            print("No URLs found, creating empty CSV file...")
+            # Create empty DataFrame with proper columns
+            df = pd.DataFrame(columns=['name', 'url'])
+        else:
+            df = pd.DataFrame(filtered_results)
+        
         df.to_csv(filename, index=False, encoding='utf-8')
 
         # Print summary
@@ -157,26 +287,36 @@ def save_to_csv(self, results, filename):
         print(f"\nResults saved to {filename}")
         print(f"Total sites processed: {total}")
         print(f"Sites with URLs: {with_urls}")
-        print(f"Success rate: {with_urls/total*100:.1f}%")
 
-        # Show some examples
-        print(f"\nFirst 5 sites with URLs:")
-        for i, result in enumerate(filtered_results[:5], 1):
-            print(f"  {i}. {result['name']}: {result['url']}")
+        if total > 0 and with_urls > 0:
+            print(f"Success rate: {with_urls/total*100:.1f}%")
+        
+        # Show some examples if there are results
+        if with_urls > 0:
+            print(f"\nFirst 5 sites with URLs:")
+            for i, (_, row) in enumerate(df.head().iterrows(), 1):
+                print(f"  {i}. {row['name']}: {row['url']}")
+        else:
+            print("\nNo sites with URLs found.")
 
 def main():
     print("Simple CKAN Site URL Extractor")
     print("=" * 40)
 
     output_file = "site_urls.csv"
 
-    extractor = SimpleSiteURLExtractor()
+    # Use conservative rate limiting (1 request per second)
+    extractor = SimpleSiteURLExtractor(requests_per_second=1)
     results = extractor.get_all_sites()
 
     if results:
         extractor.save_to_csv(results, output_file)
     else:
         print("No results found!")
+        # Create empty CSV with proper headers
+        df = pd.DataFrame(columns=['name', 'url'])
+        df.to_csv(output_file, index=False, encoding='utf-8')
+        print(f"Created empty {output_file}")
 
 if __name__ == "__main__":
     main()