Prioritize root accounts and accounts with more tweets

Author: PyGeek03

use_botometer.py

@@ -2,6 +2,7 @@
 import os
 import json
 import argparse
+from collections import Counter
 
 from dotenv import load_dotenv
 
@@ -39,42 +40,60 @@
 parser.add_argument('--dry', dest='dry_run', action='store_true',
                     help='make a dry run (without using the API)')
 
-parser.add_argument('--max-calls', dest='max_calls', type=int, default=2000,
+parser.add_argument('--max', dest='max_calls', type=int, default=2000,
                         help=("maximum number of Botometer API calls "
-                              "(default: 2000, which is Botometer API's free daily limit)"))
+                              "(default: 2000, which is Botometer API's free daily limit); "
+                              "priority will be given to the original tweet's author, "
+                              "and then the most frequently appeared accounts"))
 
 args = parser.parse_args()
 
-accounts = set()
+all_accounts = Counter()
 for file in args.conversation_files:
     with open(file, 'r') as f:
         data = json.load(f)
     tweets = data["data"]
-    accounts |= set(tweet["author_id"] for tweet in tweets)  # Twitter API v2
+    tweets.sort(key=lambda t: t['id'])
+    orig_author = tweets[0]["author_id"]
+    all_accounts[orig_author] = float('inf')
+    all_accounts |= Counter(tweet["author_id"] for tweet in tweets)  # Twitter API v2
 
 output_file = "outputs/botometer_cache.json"
 
 with open(output_file, 'r') as f:
-    all_output = json.load(f)
-prev_accounts = set(all_output)
+    checked_accounts = json.load(f)
 
-new_accounts = accounts - prev_accounts
-if len(new_accounts) > args.max_calls:
-    new_accounts = set(list(new_accounts)[:args.max_calls])
+for acc in checked_accounts:
+    if acc in all_accounts:
+        all_accounts[acc] = 0
+
+unchecked_accounts = all_accounts.most_common(args.max_calls)
+new_accounts = list(k for k, v in unchecked_accounts if v > 0)
+#if len(new_accounts) > args.max_calls:
+#    new_accounts = set(list(new_accounts)[:args.max_calls])
 
 if args.dry_run:
-    print(new_accounts)
+    print(all_accounts.most_common(args.max_calls))
     print(len(new_accounts))
 else:
-    new_output = {screen_name: result for screen_name, result in bom.check_accounts_in(new_accounts)}
-    [print(f"{k}: {v}") for k, v in new_output.items()]
-    print(f"Num. of new accounts added: {len(new_accounts)}")
-    all_output.update(new_output)
-    serialized_json = json.dumps(all_output,
+    count = 0
+    try:
+        for screen_name, result in bom.check_accounts_in(new_accounts):
+            checked_accounts[screen_name] = result
+            count += 1
+            print(f"Checked {count} accounts\n")
+            print(f"{screen_name}: {result}\n")
+    except KeyboardInterrupt:  # gracefully exit if user presses Ctrl-C
+        print()
+
+    #[print(f"{k}: {v}\n") for k, v in new_output.items()]
+    print(f"Num. of new accounts added: {count}")
+
+    serialized_json = json.dumps(checked_accounts,
                                 indent=4,
                                 sort_keys=True)
 
-    print(f"Total num. of accounts checked: {len(all_output)}")
+    print(f"Total num. of accounts checked: {len(checked_accounts)}")
 
     with open(output_file, 'w') as f:
         f.writelines(serialized_json)