Use pushshift when searching comments, decode percent encoded URLs in comments

Author: pmdevita

core/parse.py

@@ -1,7 +1,9 @@
 import re
 import praw.exceptions
 from datetime import datetime
-
+from core.pushshift import get_original_comment_from_psaw, get_comment_from_psaw
+import urllib.parse
+import praw.models
 """
 Provides different methods to parse Reddit data
 """
@@ -130,12 +132,13 @@ def parse_comment(text):
     if matches:
         # Now check for a reddit link
         for i in matches:
-            match = REPatterns.reddit_detect.findall(i[1])
+            url = urllib.parse.unquote(i[1]).replace(" ", "")
+            match = REPatterns.reddit_detect.findall(url)
             if match:
-                return RedditURL(i[1].strip())
-            match = REPatterns.short_reddit_detect.findall(i[1])
+                return RedditURL(url.strip())
+            match = REPatterns.short_reddit_detect.findall(url)
             if match:
-                return RedditURL(i[1].strip())
+                return RedditURL(url.strip())
     # Search for just general URLs in the comment
     matches = REPatterns.reddit_detect.findall(text)
     if matches:
@@ -204,22 +207,55 @@ def only_reddit_url(text):
 def find_roo_recursive(comment, starting_depth, depth):
     if not depth:
         return None
-    url = parse_comment(comment.body)
+    if comment.body == "[deleted]" or comment.body == "[removed]":
+        print("Comment was deleted")
+        url = search_pushshift(comment)
+    else:
+        body = comment.body
+        url = parse_comment(body)
     if url.comment_id:
         url = RedditURL(f"https://reddit.com{comment.permalink}")
         url.params['context'] = starting_depth - depth
         return url
     else:
-        comment.refresh()
+        try:
+            comment.refresh()
+        except praw.exceptions.ClientException:
+            return None
         for reply in comment.replies:
             url = find_roo_recursive(reply, starting_depth, depth - 1)
             if url:
                 return url
     return None
 
 
+def find_roo_parent_recursive(comment, starting_depth, depth):
+    if not depth:
+        return None
+    if isinstance(comment, praw.models.Submission):
+        return None
+    url = parse_comment(comment.body)
+    if url.comment_id:
+        url = RedditURL(f"https://reddit.com{comment.permalink}")
+        # url.params['context'] = starting_depth - depth
+        return url
+    else:
+        try:
+            comment.refresh()
+        except praw.exceptions.ClientException:
+            return None
+        url = find_roo_parent_recursive(comment.parent(), starting_depth, depth - 1)
+        if url:
+            return url
+    return None
+
 def find_roo_comment(comment):
-    return find_roo_recursive(comment, 4, 4)
+    roo = find_roo_recursive(comment, 4, 4)
+    if roo:
+        return roo
+    roo = find_roo_parent_recursive(comment, 3, 3)
+    if roo:
+        return roo
 
 
 # If we have only responded to this in the past, then pretend we already have it in FixRequests
@@ -241,5 +277,36 @@ def has_responded_to_post(submission):
     return response
 
 
+def search_pushshift(comment, last_url=None):
+    if not last_url:
+        last_url = RedditURL(f"https://reddit.com{comment.permalink}")
+    print("Searching PushShift for", last_url.comment_id)
+    # psaw leaves a little to be desired in default functionality
+    ps_comment = get_comment_from_psaw(comment.parent_id[3:], last_url.comment_id)
+    if ps_comment:
+        ps_comment = parse_comment(ps_comment['body'])
+    pso_comment = get_original_comment_from_psaw(last_url.comment_id)
+    if pso_comment:
+        pso_comment = parse_comment(pso_comment['body'])
+    if ps_comment and pso_comment:
+        if ps_comment == pso_comment:
+            url = ps_comment
+        else:
+            print("Two versions of comment, which one to use? (1/2)")
+            print(pso_comment.to_link(comment._reddit), ps_comment.to_link(comment._reddit))
+            option = input()
+            if option == "1":
+                url = pso_comment
+            else:
+                url = ps_comment
+    elif ps_comment:
+        url = ps_comment
+    elif pso_comment:
+        url = pso_comment
+    else:
+        url = RedditURL("")
+    return url
+
+
 if __name__ == '__main__':
     print(parse_comment("""[](/kderpymeow-90-f) Ah, the ol' reddit [third-world-aroo!](https://old.reddit.com/r/interestingasfuck/comments/a9xt2g/you_may_be_cool_but_you_will_never_be_as_cool_as/ecnjvlf/?context=2)"""))

core/pushshift.py

@@ -0,0 +1,34 @@
+import requests
+import urllib.parse
+
+
+# Weird other way to get the data but it returns the edited version?
+def get_comment_from_psaw(parent_id, comment_id):
+    params = {'parent_id': f"t1_{parent_id}", "filter": "id,created_utc,edited,body"}
+    # Come on PushShift, percent coding is a standard
+    payload_str = urllib.parse.urlencode(params, safe=",")
+    r = requests.get("https://api.pushshift.io/reddit/comment/search/",
+                     params=payload_str)
+    try:
+        j = r.json()
+    except Exception as e:
+        print(r.status_code, r.text)
+        raise e
+    for i in j['data']:
+        if i['id'] == comment_id:
+            return i
+    return None
+
+
+def get_original_comment_from_psaw(comment_id):
+    params = {'ids': comment_id, "filter": "id,created_utc,body"}
+    # Come on PushShift, percent coding is a standard
+    payload_str = urllib.parse.urlencode(params, safe=",")
+    r = requests.get("https://api.pushshift.io/reddit/comment/search/",
+                     params=payload_str)
+    j = r.json()
+    if j.get('data', None):
+        if len(j['data']) > 0:
+            return j['data'][0]
+    return None
+

tools/db.py

@@ -27,6 +27,11 @@ def roo_id_to_submission(id):
     print(f"https://reddit.com{roo.submission.permalink}")
 
 
+def roo_id_to_comment(id):
+    roo = last_switcharoo.get_roo(id)
+    print(f"https://reddit.com{roo.comment.permalink}")
+
+
 def roo_id_to_issues(id):
     roo = last_switcharoo.get_roo(id)
     issues = last_switcharoo.get_issues(roo)

tracer.py

@@ -1,9 +1,8 @@
-import requests
-import urllib.parse
 import psaw
 import praw.exceptions
 import time
 import pendulum
+import webbrowser
 from datetime import datetime, timedelta
 
 import prawcore.exceptions
@@ -13,6 +12,7 @@
 from core import parse
 from core.history import SwitcharooLog
 from core.arguments import tracer as argparser
+from core.pushshift import get_comment_from_psaw, get_original_comment_from_psaw
 
 credentials = CredentialsLoader.get_credentials()['reddit']
 
@@ -30,6 +30,7 @@
 
 args = argparser.parse_args()
 
+
 def get_newest_id(subreddit, index=0):
     """Retrieves the newest post's id. Used for starting the last switcharoo history trackers"""
     return [i for i in subreddit.new(params={"limit": "1"})][index].url
@@ -55,33 +56,6 @@ def get_newest_id(subreddit, index=0):
 print("SwitcharooHelper Tracer v{} Ctrl+C to stop".format(consts.version))
 
 
-# Weird other way to get the data but it returns the edited version?
-def get_comment_from_psaw(parent_id, comment_id):
-    params = {'parent_id': f"t1_{parent_id}", "filter": "id,created_utc,edited,body"}
-    # Come on PushShift, percent coding is a standard
-    payload_str = urllib.parse.urlencode(params, safe=",")
-    r = requests.get("https://api.pushshift.io/reddit/comment/search/",
-                     params=payload_str)
-    j = r.json()
-    for i in j['data']:
-        if i['id'] == comment_id:
-            return i
-    return None
-
-
-def get_original_comment_from_psaw(comment_id):
-    params = {'ids': comment_id, "filter": "id,created_utc,body"}
-    # Come on PushShift, percent coding is a standard
-    payload_str =  urllib.parse.urlencode(params, safe=",")
-    r = requests.get("https://api.pushshift.io/reddit/comment/search/",
-                     params=payload_str)
-    j = r.json()
-    if j.get('data', None):
-        if len(j['data']) > 0:
-            return j['data'][0]
-    return None
-
-
 def unable_to_find_link(url: parse.RedditURL, last_url: parse.RedditURL):
     print("Unable to find a link in this roo.")
     print(last_url.to_link(reddit))
@@ -100,35 +74,6 @@ def unable_to_find_link(url: parse.RedditURL, last_url: parse.RedditURL):
     return parse.RedditURL(url)
 
 
-def search_pushshift(last_url):
-    print("Searching PushShift for", last_url.comment_id)
-    # psaw leaves a little to be desired in default functionality
-    ps_comment = get_comment_from_psaw(comment.parent_id[3:], last_url.comment_id)
-    if ps_comment:
-        ps_comment = parse.parse_comment(ps_comment['body'])
-    pso_comment = get_original_comment_from_psaw(last_url.comment_id)
-    if pso_comment:
-        pso_comment = parse.parse_comment(pso_comment['body'])
-    if ps_comment and pso_comment:
-        if ps_comment == pso_comment:
-            url = ps_comment
-        else:
-            print("Two versions of comment, which one to use? (1/2)")
-            print(pso_comment.to_link(reddit), ps_comment.to_link(reddit))
-            option = input()
-            if option == "1":
-                url = pso_comment
-            else:
-                url = ps_comment
-    elif ps_comment:
-        url = ps_comment
-    elif pso_comment:
-        url = pso_comment
-    else:
-        url = parse.RedditURL("")
-    return url
-
-
 def add_comment(url: parse.RedditURL, start_url: parse.RedditURL = None):
     # Double check it's not already there
     q = log.search(comment_id=url.comment_id)
@@ -142,7 +87,13 @@ def add_comment(url: parse.RedditURL, start_url: parse.RedditURL = None):
     if q:
         print("Adjusting roo time")
         comment_time = q.time - timedelta(seconds=1)
-    log.add_comment(url.thread_id, url.comment_id, url.params.get("context", 0), comment_time)
+    try:
+        context = int(url.params.get("context", 0))
+    except ValueError:
+        print(f"Got {url.params['context']} for url {url}, what should it be?")
+        context = int(input())
+
+    log.add_comment(url.thread_id, url.comment_id, context, comment_time)
 
 
 
@@ -171,9 +122,9 @@ def add_comment(url: parse.RedditURL, start_url: parse.RedditURL = None):
         roo_count += 1
 
         last_url = url
-        if comment.body == "[deleted]":
+        if comment.body == "[deleted]" or comment.body == "[removed]":
             print("Comment was deleted")
-            url = search_pushshift(last_url)
+            url = parse.search_pushshift(comment, last_url)
         else:
             url = parse.parse_comment(comment.body)
 
@@ -182,15 +133,31 @@ def add_comment(url: parse.RedditURL, start_url: parse.RedditURL = None):
             print("Roo linked incorrectly, searching thread for link")
             new_last_url = parse.find_roo_comment(comment)
             if new_last_url and last_url:
-                new_last_url.params['context'] = str(int(new_last_url.params.get('context', 0)) +
-                                                     int(last_url.params.get('context', 0)))
+                try:
+                    new_last_url.params['context'] = str(int(new_last_url.params.get('context', 0)) +
+                                                         int(last_url.params.get('context', 0)))
+                except ValueError:
+                    print(f"Got {new_last_url.params['context']} and {last_url.params['context']}, what should it be?")
+                    new_last_url.params['context'] = int(input())
             if new_last_url:
-                print(last_url.to_link(reddit), "should actually be", new_last_url.to_link(reddit))
-                last_url = new_last_url
+                if args.discover:
+                    print("Should", last_url.to_link(reddit), "actually be", new_last_url.to_link(reddit), "?")
+                    print("(y/n)")
+                    webbrowser.open(last_url.to_link(reddit))
+                    webbrowser.open(new_last_url.to_link(reddit))
+                    option = input()
+                    if option == "y":
+                        last_url = new_last_url
+                else:
+                    last_url = new_last_url
                 comment = reddit.comment(last_url.comment_id)
-                url = parse.parse_comment(comment.body)
+                if comment.body == "[deleted]" or comment.body == "[removed]":
+                    print("Comment was deleted")
+                    url = parse.search_pushshift(comment, last_url)
+                else:
+                    url = parse.parse_comment(comment.body)
             else:
-                url = search_pushshift(last_url)
+                url = parse.search_pushshift(comment, last_url)
 
         if args.discover:
             add_comment(last_url, start_url=start_url)