saving cursor to db on first full scan

Author: Bombg

utils/Database.py

@@ -61,6 +61,30 @@ def GetKickClipIdTitles(self):
         conn.close()
         return titleIdDict
 
+    def InsertClipCursor(self,cursor:str):
+        self.createKickClipsTable()
+        conn, cur = self.connectCursor()
+        rowVal = (cursor,"greenLight")
+        exeString = '''UPDATE kick_clips SET channel_slug=? WHERE clip_id=?'''
+        cur.execute(exeString,rowVal)
+        conn.commit()
+        cur.close()
+        conn.close()
+
+    def GetClipCursor(self):
+        self.createKickClipsTable()
+        conn, cur = self.connectCursor()
+        cursor = ''
+        rowVal = ("greenLight",)
+        exeString = '''SELECT channel_slug FROM kick_clips WHERE clip_id=?'''
+        cur.execute(exeString,rowVal)
+        fetch = cur.fetchall()
+        if fetch and fetch[0][0] != None:
+            cursor = fetch[0][0]
+        cur.close()
+        conn.close()
+        return cursor
+    
     def isClipRowCountZero(self):
         self.createKickClipsTable()
         conn, cur = self.connectCursor()

utils/KickDataGrabber.py

@@ -22,10 +22,12 @@
 async def CollectClipData(kickSlug:str, rest: hikari.impl.RESTClientImpl) -> None:
     db = Database()
     apiUrl = f"https://kick.com/api/v2/channels/{kickSlug}/clips"
-    if globals.kickClipCursor:
-        apiUrl = f"{apiUrl}?cursor={globals.kickClipCursor}"
     if db.isClipRowCountZero():
         db.addKickClipToTable("greenLight",None,None,None,None,None,None,None)
+    if not db.isClipsFullyScanned():
+        globals.kickClipCursor = db.GetClipCursor()
+    if globals.kickClipCursor:
+        apiUrl = f"{apiUrl}?cursor={globals.kickClipCursor}"
     browser = await ndb.GetBrowser(proxy=Constants.KICK_PROXY)
     try:
         if globals.kickClipCursor in globals.kickVisitedCursors:
@@ -73,6 +75,8 @@ async def CollectClipData(kickSlug:str, rest: hikari.impl.RESTClientImpl) -> Non
                     globals.kickClipCursor = ""
                 if viewIncrease > globals.kickClipMostViews:
                     AddMostViewedClipToGlobal(clip, viewIncrease)
+            if not db.isClipsFullyScanned():
+                db.InsertClipCursor(results['nextCursor'])
             if not globals.kickClipCursor:
                 await AnnounceWinnersHandleData(kickSlug, rest, db) 
     except Exception as e: