fixes and improvements

fix: luscious works again
fix: debug print in furbooru
feat: log if shit goes south
feat: database support for rule34 and furbooru
refactor: more safe filenames

Author: Official-Husko

.gitignore

@@ -4,14 +4,10 @@ __pycache__/
 dist/
 build/
 media/
-.nn-d/
 .env/
 testing_accounts.txt
-config.json.bak
 old_config.json
-testing_accounts.txt.bak
 db/
 outdated
-modules/updateManager.old
 runtime.log
-delete-exe.bat
+config.json.dev

Build Release.bat

@@ -1,5 +1 @@
-".\.env\Scripts\activate" && pyinstaller --onefile --icon "icon.ico" --console --name "NN-Downloader" --upx-dir "Z:\Projects\Python\### UPX ###" --add-data="Z:/Projects/Python/NN-Downloader/.env/Lib/site-packages/grapheme/data/*;grapheme/data/" main.py
-
-rmdir /s /q .\build
-rmdir /s /q .\__pycache__
-del ".\NN-Downloader.spec"
+".\.env\Scripts\activate" && pyinstaller --onefile --icon "icon.ico" --console --name "NN-Downloader" --upx-dir "Z:\Projects\Python\### UPX ###" --add-data=--add-data="./.env/Lib/site-packages/grapheme/data/*;grapheme/data/" main.py && rmdir /s /q .\build && rmdir /s /q .\__pycache__ && del ".\NN-Downloader.spec"

README.md

@@ -15,7 +15,7 @@ This project is unfinished and only works on the [listed][13] sites currently. M
 - [Furbooru][6] (API)
 - [Multporn][7]
 - [Yiffer][8]
-- [Luscious][16] ***(Currently Broken!)***
+- [Luscious][16]
 
 #### Planned:
 - [YiffGallery][9]

main.py

@@ -7,12 +7,13 @@
 from sys import exit
 import inquirer
 
-version = "1.4.1"
+version = "1.4.2"
 windll.kernel32.SetConsoleTitleW(f"NN-Downloader | v{version}")
 proxy_list = []
 header = {"User-Agent":f"nn-downloader/{version} (by Official Husko on GitHub)"}
 needed_folders = ["db", "media"]
-database_list = ["e621.db"]
+database_list = ["e621", "furbooru", "rule34"]
+unsafe_chars = ["/", "\\", ":", "*", "?", "\"", "<", ">", "|", "\0", "$", "#", "@", "&", "%", "!", "`", "^", "(", ")", "{", "}", "[", "]", "=", "+", "~", ",", ";"]
 
 if os.path.exists("outdated"):
     version_for_logo = colored(f"v{version}", "cyan", attrs=["blink"])
@@ -70,13 +71,13 @@ def main_startup():
 
         if oneTimeDownload == True:
             for database in database_list:
-                with open(f"db/{database}", "a") as db_creator:
+                with open(f"db/{database}.db", "a") as db_creator:
                     db_creator.close()
 
         print(colored("What site do you want to download from?", "green"))
         questions = [
             inquirer.List('selection',
-                          choices=['E621', 'E926', 'Furbooru', 'Multporn', 'Rule34', 'Yiffer']), #choices=['E621', 'E926', 'Furbooru', 'Luscious', 'Multporn', 'Rule34', 'Yiffer']),
+                          choices=['E621', 'E926', 'Furbooru', 'Luscious', 'Multporn', 'Rule34', 'Yiffer']),
         ]
         answers = inquirer.prompt(questions)
         print("")
@@ -108,7 +109,8 @@ def main_startup():
                 print(colored("Please add your Api Key into the config.json", "red"))
                 sleep(5)
             else:
-                E621.Fetcher(user_tags=user_tags, user_blacklist=config["blacklisted_tags"], proxy_list=proxy_list, max_sites=max_sites, user_proxies=config["proxies"], apiUser=apiUser, apiKey=apiKey, header=header, db=database)
+                output = E621.Fetcher(user_tags=user_tags, user_blacklist=config["blacklisted_tags"], proxy_list=proxy_list, max_sites=max_sites, user_proxies=config["proxies"], apiUser=apiUser, apiKey=apiKey, header=header, db=database)
+        
         elif site == "e926":
             apiUser = config["user_credentials"]["e926"]["apiUser"]
             apiKey = config["user_credentials"]["e926"]["apiKey"]
@@ -119,44 +121,76 @@ def main_startup():
                 print(colored("Please add your Api Key into the config.json", "red"))
                 sleep(5)
             else:
-                E926.Fetcher(user_tags=user_tags, user_blacklist=config["blacklisted_tags"], proxy_list=proxy_list, max_sites=max_sites, user_proxies=config["proxies"], apiUser=apiUser, apiKey=apiKey, header=header, db=database)
+                output = E926.Fetcher(user_tags=user_tags, user_blacklist=config["blacklisted_tags"], proxy_list=proxy_list, max_sites=max_sites, user_proxies=config["proxies"], apiUser=apiUser, apiKey=apiKey, header=header, db=database)
+        
         elif site == "rule34":
-            RULE34.Fetcher(user_tags=user_tags, user_blacklist=config["blacklisted_tags"], proxy_list=proxy_list, max_sites=max_sites, user_proxies=config["proxies"], header=header)
+            if oneTimeDownload == True:
+                with open("db/rule34.db", "r") as db_reader:
+                    database = db_reader.read().splitlines()
+            output = RULE34.Fetcher(user_tags=user_tags, user_blacklist=config["blacklisted_tags"], proxy_list=proxy_list, max_sites=max_sites, user_proxies=config["proxies"], header=header, db=database)
+        
         elif site == "furbooru":
             apiKey = config["user_credentials"]["furbooru"]["apiKey"]
+            if oneTimeDownload == True:
+                with open("db/furbooru.db", "r") as db_reader:
+                    database = db_reader.read().splitlines()
             if apiKey == "":
                 print(colored("Please add your Api Key into the config.json", "red"))
                 sleep(5)
             else:
-                FURBOORU.Fetcher(user_tags=user_tags, user_blacklist=config["blacklisted_tags"], proxy_list=proxy_list, max_sites=max_sites, user_proxies=config["proxies"], apiKey=apiKey, header=header)
+                output = FURBOORU.Fetcher(user_tags=user_tags, user_blacklist=config["blacklisted_tags"], proxy_list=proxy_list, max_sites=max_sites, user_proxies=config["proxies"], apiKey=apiKey, header=header, db=database)
+        
         elif site == "multporn":
             print(colored("Please enter the link. (e.g. https://multporn.net/comics/double_trouble_18)", "green"))
             URL = input(">> ")
             while URL == "":    
                 print(colored("Please enter a valid link.", "red"))
                 sleep(1.5)
                 URL = input(">> ")
-            Multporn.Fetcher(proxy_list=proxy_list, user_proxies=config["proxies"], header=header, URL=URL)
+            output = Multporn.Fetcher(proxy_list=proxy_list, user_proxies=config["proxies"], header=header, URL=URL)
+        
         elif site == "yiffer":
             print(colored("Please enter the link. (e.g. https://yiffer.xyz/Howl & Jasper)", "green"))
             URL = input(">> ")
             while URL == "":    
                 print(colored("Please enter a valid link.", "red"))
                 sleep(1.5)
                 URL = input(">> ")
-            Yiffer.Fetcher(proxy_list=proxy_list, user_proxies=config["proxies"], header=header, URL=URL)
+            output = Yiffer.Fetcher(proxy_list=proxy_list, user_proxies=config["proxies"], header=header, URL=URL)
+        
         elif site == "luscious":
             print(colored("Please enter the link. (e.g. https://www.luscious.net/albums/bifurcation-ongoing_437722)", "green"))
             URL = input(">> ")
             while URL == "":    
                 print(colored("Please enter a valid link.", "red"))
                 sleep(1.5)
                 URL = input(">> ")
-            Luscious.Fetcher(proxy_list=proxy_list, user_proxies=config["proxies"], header=header, URL=URL)
+            output = Luscious.Fetcher(proxy_list=proxy_list, user_proxies=config["proxies"], header=header, URL=URL)
+
+        else:
+            print(colored("Site not supported. Open a ticket to request support for that site!", "red"))
+            raise Exception(f"This shouldn't be possible! User tried to download from {site}.")
+            Main.main_startup()
+
+        status = output.get("status", "why no status man?")
+        uinput = output.get("uinput", "URL overdosed :(")
+        exception_str = output.get("exception", "Fuck me there was no exception.")
+        extra = output.get("extra", "")
+        
+        if status == "ok":
+            pass
 
+        elif status == "error":
+            print(f"{error} An error occured while downloading from {colored(site, 'yellow')}! Please report this. Exception: {colored(exception_str, 'red')}")
+            error_str = f"An error occured while downloading from {site}! Please report this. Exception: {exception_str}"
+            Logger.log_event(error_str, extra, uinput)
+            sleep(7)
 
         else:
-            print(colored("Site not supported. Open a ticket to request support for that site!", "red"))
+            print(f"{major_error} An unknown error occured while downloading from {colored(site, 'yellow')}! Please report this. Exception: {colored(exception_str, 'red')}")
+            error_str = f"An unknown error occured while downloading from {site}! Please report this. Exception: {exception_str}"
+            Logger.log_event(error_str, extra, uinput)
+            sleep(7)
 
         # Jump back to start
         Main.main_startup()
@@ -168,9 +202,3 @@ def main_startup():
         print("User Cancelled")
         sleep(3)
         exit(0)
-        
-        
-"""
-TODO: fix luscious being broken
-
-"""

modules/e621.py

@@ -7,85 +7,100 @@
 from datetime import datetime
 import os
 
+from main import unsafe_chars
 now = datetime.now()
 dt_now = now.strftime("%d-%m-%Y_%H-%M-%S")
 
 class E621():
     def Fetcher(user_tags, user_blacklist, proxy_list, max_sites, user_proxies, apiUser ,apiKey, header, db):
-        approved_list = []
-        page = 1
-        while True:
-            URL = f"https://e621.net/posts.json?tags={user_tags}&limit=320&page={page}"
-            if user_proxies == True:
-                proxy = random.choice(proxy_list)
-                req = requests.get(URL, headers=header, proxies=proxy, auth=HTTPBasicAuth(apiUser, apiKey)).json()
-            else:
-                req = requests.get(URL, headers=header, auth=HTTPBasicAuth(apiUser, apiKey)).json()
+        try:
+            approved_list = []
+            page = 1
+            while True:
+                URL = f"https://e621.net/posts.json?tags={user_tags}&limit=320&page={page}"
+                if user_proxies == True:
+                    proxy = random.choice(proxy_list)
+                    raw_req = requests.get(URL, headers=header, proxies=proxy, auth=HTTPBasicAuth(apiUser, apiKey))
+                else:
+                    raw_req = requests.get(URL, headers=header, auth=HTTPBasicAuth(apiUser, apiKey))
 
-            try:
-                if req["message"] == "You cannot go beyond page 750. Please narrow your search terms.": 
-                    print(colored(req["message"] + " (API limit)", "red"))
+                req = raw_req.json()
+
+                try:
+                    if req["message"] == "You cannot go beyond page 750. Please narrow your search terms.": 
+                        print(colored(req["message"] + " (API limit)", "red"))
+                        sleep(5)
+                        break
+                except:
+                    pass
+                
+                if req["posts"] == []:
+                    print(colored("No images found or all downloaded! Try different tags.", "yellow"))
                     sleep(5)
                     break
-            except:
-                pass
-            
-            if req["posts"] == []:
-                print(colored("No images found or all downloaded! Try different tags.", "yellow"))
-                sleep(5)
-                break
 
-            elif page == max_sites:
-                break
-            
-            else: 
-                for item in req["posts"]:
-                    image_id = item["id"]
-                    image_address = item["file"]["url"]
-                    post_tags1 = item["tags"]["general"]
-                    post_tags2 = item["tags"]["species"]
-                    post_tags3 = item["tags"]["character"]
-                    post_tags4 = item["tags"]["copyright"]
-                    post_tags5 = item["tags"]["artist"]
-                    post_tags = post_tags1 + post_tags2 + post_tags3 + post_tags4 + post_tags5
-                    image_format = item["file"]["ext"]
-                    user_blacklist_lenght = len(user_blacklist)
-                    passed = 0
+                elif page == max_sites:
+                    print(colored(f"Finished Downloading {max_sites} of {max_sites} pages.", "yellow"))
+                    sleep(5)
+                    break
+                
+                else: 
+                    for item in req["posts"]:
+                        image_id = item["id"]
+                        image_address = item["file"]["url"]
+                        post_tags1 = item["tags"]["general"]
+                        post_tags2 = item["tags"]["species"]
+                        post_tags3 = item["tags"]["character"]
+                        post_tags4 = item["tags"]["copyright"]
+                        post_tags5 = item["tags"]["artist"]
+                        post_tags = post_tags1 + post_tags2 + post_tags3 + post_tags4 + post_tags5
+                        image_format = item["file"]["ext"]
+                        user_blacklist_lenght = len(user_blacklist)
+                        passed = 0
 
-                    for blacklisted_tag in user_blacklist:
-                        if blacklisted_tag in post_tags:
-                            break
+                        for blacklisted_tag in user_blacklist:
+                            if blacklisted_tag in post_tags:
+                                break
+                            else:
+                                passed += 1
+                        if passed == user_blacklist_lenght and str(image_id) not in db and image_address != None:
+                            image_data = {"image_address": image_address, "image_format": image_format, "image_id": image_id}
+                            approved_list.append(image_data)
                         else:
-                            passed += 1
-                    if passed == user_blacklist_lenght and str(image_id) not in db and image_address != None:
-                        image_data = {"image_address": image_address, "image_format": image_format, "image_id": image_id}
-                        approved_list.append(image_data)
-                    else:
-                        pass
+                            pass
 
-            # Download Each file
-            with alive_bar(len(approved_list), calibrate=1, dual_line=True, title='Downloading') as bar:
-                for data in approved_list:
-                    image_address = data["image_address"]
-                    image_format = data["image_format"]
-                    image_id = data["image_id"]
-                    bar.text = f'-> Downloading: {image_id}, please wait...'
-                    if user_proxies == True:
-                        proxy = random.choice(proxy_list)
-                        img_data = requests.get(image_address, proxies=proxy).content
-                    else:
-                        sleep(1)
-                        img_data = requests.get(image_address).content
-                    safe_user_tags = user_tags.replace("\\", "").replace("/", "").replace(":", "").replace("*", "").replace("?", "").replace('"', "").replace("<", "").replace(">", "").replace("|", "").replace(" ", "_")
-                    if not os.path.exists("media/" + dt_now + " " + safe_user_tags):
-                        os.mkdir("media/" + dt_now + " " + safe_user_tags)
-                    with open("media/" + dt_now + " " + safe_user_tags + "/" + str(image_id) + "." + image_format, 'wb') as handler:
-                        handler.write(img_data)
-                    with open("db/e621.db", "a") as db_writer:
-                        db_writer.write(f"{str(image_id)}\n")
-                    bar()
+                # Download Each file
+                with alive_bar(len(approved_list), calibrate=1, dual_line=True, title='Downloading') as bar:
+                    for data in approved_list:
+                        image_address = data["image_address"]
+                        image_format = data["image_format"]
+                        image_id = data["image_id"]
+                        bar.text = f'-> Downloading: {image_id}, please wait...'
+                        if user_proxies == True:
+                            proxy = random.choice(proxy_list)
+                            img_data = requests.get(image_address, proxies=proxy).content
+                        else:
+                            sleep(1)
+                            img_data = requests.get(image_address).content
+
+                            safe_user_tags = user_tags.replace(" ", "_")
+                            for char in unsafe_chars:
+                                safe_user_tags = safe_user_tags.replace(char, "")
+
+                        if not os.path.exists(f"media/{dt_now}_{safe_user_tags}"):
+                            os.mkdir(f"media/{dt_now}_{safe_user_tags}")
+                        with open(f"media/{dt_now}_{safe_user_tags}/{str(image_id)}.{image_format}", 'wb') as handler:
+                            handler.write(img_data)
+                        with open("db/e621.db", "a") as db_writer:
+                            db_writer.write(f"{str(image_id)}\n")
+                        bar()
+
+                print(colored(f"Page {page} Completed", "green"))
+                approved_list.clear()
+                page += 1
+                sleep(5)
 
-            print(colored(f"Page {page} Completed", "green"))
-            approved_list.clear()
-            page += 1
-            sleep(5)
+            return {"status": "ok"}
+        
+        except Exception as e:
+            return {"status": "error", "uinput": user_tags, "exception": str(e), "extra": raw_req.content}