Fix crawl group a bit more, add readme

Author: ddxv

README.md

@@ -41,6 +41,8 @@ Scrapers:
 
 - `-u, --update-app-store-details` Scrape app stores for app details (e.g., downloads, ratings). Requires existing store IDs.
 
+- `--country-priority-group` Default is 1 for US. Other groups for crawling can be configured in db.
+
 - `--workers` Number of workers to use for updating app store details. Default: `1`.
 
 - `-n, --new-apps-check` This crawls app rank data and stores to S3. It is also the source of new apps. Crawl the iTunes and Play Store front pages to discover new apps. Checks top apps for each category and collection.

main.py

@@ -279,6 +279,7 @@ def setup_database_connection(self) -> None:
     def main(self) -> None:
         logger.info(f"Main starting with args: {self.args}")
         platform: str = self.args.platform or None
+        country_priority_group: int = self.args.country_priority_group or None
 
         store = STORES_MAP.get(platform) if platform else None
 
@@ -292,7 +293,7 @@ def main(self) -> None:
             self.scrape_new_apps_devs(store)
 
         if self.args.update_app_store_details:
-            self.update_app_details(store, self.args.country_priority_group)
+            self.update_app_details(store, country_priority_group)
 
         if self.args.app_ads_txt_scrape:
             self.crawl_app_ads()
@@ -366,7 +367,7 @@ def scrape_new_apps_devs(self, store: int) -> None:
     def update_app_details(self, store: int, country_priority_group: int) -> None:
         if not country_priority_group:
             logger.error(
-                "No country priority group provided, ie country-priority-group=1"
+                "No country priority group provided, ie --country-priority-group=1"
             )
             return
         update_app_details(