Merge pull request #15 from Oshan96/dev

Oshan96 · web-flow · commit 775654a541b2 · 2020-04-06T20:16:56.000+05:30
Dev Changes for v1.0.2
diff --git a/README.md b/README.md
@@ -13,8 +13,8 @@ If this project is helpful to you and love my work and feel like showing love/ap
 |---                                            |---                    |---                            |---                |---                            |---                                                                                                |        
 | [9Anime](https://9anime.to/)                  | Yes                   | Default only                  | No                | 500-600MB                     | Will always work, provided token                                                                  |
 | [4Anime](https://4anime.to/)                  | No                    | Default only                  | No                | Around 150MB                  | Upon failure, visit 4anime website and restart anime downloader                                   |
-| [AnimePahe](https://animepahe.com/)           | No                    | 720p, 1080p                   | No                | 720p: ~150MB, 1080p: ~200MB   | Anime Downloader v1.0.1 upwards(v1.0.0 no longer works). Also download speed is capped by host    |
-| [AnimeUltima](https://www.animeultima.to/)    | No                    | 240p, 360p, 480p, 720p, 1080p | Yes               | 1080p is around 1GB           | -                                                                                                 |
+| [AnimePahe](https://animepahe.com/)           | No                    | 720p, 1080p                   | No                | 720p: ~150MB, 1080p: ~200MB   | 2captcha API key is needed to download from AnimePahe. Also download speed is capped by host      |
+| [AnimeUltima](https://www.animeultima.to/)    | No                    | 240p, 360p, 480p, 720p, 1080p | Yes               | 1080p is around 1GB           | AnimeUltima is having issues in their end. Will be supported again once they are backup           |
 
 ## Download Anime Downloader [Windows]
 > Note : Currently only windows executable is provided (Linux, Mac users go to [Build from source](#Building-from-source))
diff --git a/anime_downloader/extractors/base_extractor.py b/anime_downloader/extractors/base_extractor.py
@@ -1,12 +1,13 @@
+import cloudscraper
 class BaseExtractor:
 
     def __init__(self, url, session):
         self.url = url
         self.session = session
 
     def extract_page_content(self):
-        video_page = self.session.get(self.url).content
-        return video_page.decode('utf-8')
+        video_page = self.session.get(self.url)
+        return video_page.text
 
     def extract_direct_url(self):
         raise NotImplementedError
diff --git a/anime_downloader/extractors/jwplayer_extractor.py b/anime_downloader/extractors/jwplayer_extractor.py
@@ -10,14 +10,19 @@ def __init__(self, url, session):
     def extract_sources(self):
         page_content = self.extract_page_content()
 
+        # print(page_content)
+
         link_sources = [match.group(1) for match in
                         re.finditer("{\s*file\s*:\s*[\"\']\s*([htps][^\"\']+)", page_content)]
 
         return link_sources
 
     def extract_direct_url(self):
+        print("extracting direct stream links")
         direct_links = self.extract_sources()
 
+        # print(direct_links)
+
         if len(direct_links) > 0:
             # return the first direct link
             return direct_links[0]
@@ -51,6 +56,7 @@ def get_resolution_link(self, master_url, resolution):
         return link
 
     def extract_stream_link(self, resolution="720"):
+        print("Extracting stream link")
         link = self.extract_direct_url()
 
         print("Master Link : " + link)
diff --git a/anime_downloader/gui/GUI.py b/anime_downloader/gui/GUI.py
@@ -1,4 +1,5 @@
 import queue
+import json
 import cloudscraper
 import PySimpleGUI as sg
 from threading import Thread
@@ -40,6 +41,27 @@ def download(anime_url, names_url, start_epi, end_epi, is_filler, is_titles, tok
 
         elif "animepahe.com" in anime_url:
             printer("INFO", "AnimePahe URL detected...", gui)
+            api_key = ""
+            try:
+                with open("settings.json") as (json_file):
+                    data = json.load(json_file)
+                    api_key = data["api_key"]
+            except:
+                api_key = ""
+
+            if api_key != "" and api_key != "insert_2captcha_api_key":
+                session = cloudscraper.create_scraper(
+                    recaptcha={
+                        'provider': '2captcha',
+                        'api_key': api_key
+                    }
+                )
+
+            else:
+                printer("ERROR", "You need 2captcha API key to download from AnimePahe!", gui)
+                printer("ERROR", "Set 2captcha API key in 'settings.json' file to download from AnimePahe!", gui)
+                return
+
             scraper = AnimePaheScraper(anime_url, start_epi, end_epi, session, gui, resolution, is_filler)
 
         else:
diff --git a/anime_downloader/scrapers/animepahe/animepahe_scraper.py b/anime_downloader/scrapers/animepahe/animepahe_scraper.py
@@ -1,4 +1,5 @@
 import re
+from bs4 import BeautifulSoup
 from util.Episode import Episode
 from scrapers.base_scraper import BaseScraper
 from util.Color import printer
@@ -16,9 +17,17 @@ def __init__(self, url, start_episode, end_episode, session, gui=None, resolutio
         self.end_page = 1
         self.extractor = KwikExtractor(session, gui)
 
+        self.__set_working_url()
         self.__set_anime_id()
         self.__set_start_end_page()
 
+    def __set_working_url(self):
+        page = self.session.get(self.url).content
+        soup_page = BeautifulSoup(page, "html.parser")
+        og_url = soup_page.find("meta", attrs={"property": "og:url"})
+        if og_url is not None:
+            self.url = og_url["content"]
+
     def __set_anime_id(self):
         page = self.session.get(self.url).text
         self.id = re.search("release&id=(.*)&l=", page).group(1)
diff --git a/anime_downloader/scrapers/animeultima/animeultima_scraper.py b/anime_downloader/scrapers/animeultima/animeultima_scraper.py
@@ -8,7 +8,7 @@ class AnimeUltimaScraper(BaseScraper):
 
     def __init__(self, url, start_episode, end_episode, session, gui=None, resolution="720", is_dub=False):
         super().__init__(url, start_episode, end_episode, session, gui)
-        self.is_dub = False
+        self.is_dub = is_dub
         self.resolution = resolution
         self.base_url = "https://www1.animeultima.to"
         self.extractor = JWPlayerExtractor(None, self.session)
@@ -37,6 +37,8 @@ def get_start_and_end_page(self, anime_id):
 
         data = self.session.get("https://www1.animeultima.to/api/episodeList?animeId=" + anime_id).json()
 
+        # print("start end data")
+        # print(data)
         last_page = data["last_page"]
         max_total_epis = last_page * 50
 
@@ -75,25 +77,30 @@ def collect_episodes(self, anime_id, start_page, end_page):
             url = base_url + str(page_counter)
 
             data = self.session.get(url).json()
+            # print("data")
+            # print(data)
+
             has_dub = data["anime"]["hasDub"]
             epis = data["episodes"]
 
             for epi in epis:
                 epi_no = int(epi["episode_num"])
+                # print(str(epi_no))
 
                 if epi_no < self.start_episode or epi_no > self.end_episode:
                     continue
 
                 title = epi["title"]
                 page_url = None
                 if not self.is_dub:
+                    # print("sub")
                     page_url = epi["urls"]["sub"]
                 elif has_dub:
                     page_url = epi["urls"]["dub"]
                 else:
                     print("Dubbed episodes not available")
 
-                if page_url:
+                if page_url is not None:
                     page_url = self.get_page_url(page_url)
 
                 episode = Episode(title, "Episode - " + str(epi_no))
diff --git a/anime_downloader/scrapers/nineanime/nineanime_scraper.py b/anime_downloader/scrapers/nineanime/nineanime_scraper.py
@@ -19,6 +19,8 @@ def __init__(self, url, start_episode, end_episode, session, gui=None, token=Non
         self.server_name = "Mp4upload"
         self.nine_anime_url = "https://9anime.to"
 
+        self.headers = {"origin": self.nine_anime_url, "referer": url, "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.162 Safari/537.36 Edg/80.0.361.109"}
+
         self.episodes_url = "https://9anime.to/ajax/film/servers/" + url.split(".")[2].split("/")[0]
 
         if not token:
@@ -45,10 +47,14 @@ def __verify(self):
             "g-recaptcha-response": self.token
         }
 
-        self.session.post("https://9anime.to/waf-verify", data=payload)
+        data = self.session.post("https://9anime.to/waf-verify", data=payload, headers=self.headers, allow_redirects=False)
+        self.headers["cookie"] = data.headers["set-cookie"]
 
     def __extract_page_urls(self):
-        if self.token is None :
+        d = self.session.get("https://9anime.to/waf-verify", headers=self.headers, allow_redirects=True)
+        self.headers["cookie"] = d.headers["set-cookie"]
+
+        if self.token is None:
             if self.api_key != "" and self.api_key != "insert_2captcha_api_key":
                 Color.printer("INFO", "Solving recaptcha...", self.gui)
 
@@ -60,21 +66,29 @@ def __extract_page_urls(self):
                     Color.printer("INFO", "Trying to continue ...", self.gui)
 
         if self.token:
+            # print(self.token)
             self.__verify()
         else:
             Color.printer("INFO", "No API key or token given, trying to continue...", self.gui)
 
         Color.printer("INFO", "Extracting page URLs...", self.gui)
 
-        anime_page = self.session.get(self.url).content
+        data = self.session.get(self.url, headers=self.headers)
+        anime_page = data.content
+
         soup_html = BeautifulSoup(anime_page, "html.parser")
 
-        try :
+        try:
             self.ts_no = soup_html.find("html")["data-ts"]
 
             eps_url = self.episodes_url + "?ts=" + self.ts_no
 
-            epi_data = self.session.get(eps_url).json()["html"]
+            self.headers["referer"] = eps_url
+
+            resp = self.session.get(eps_url, headers=self.headers, allow_redirects=False)
+            epi_data = resp.json()["html"]
+
+            # print(epi_data)
 
             soup = BeautifulSoup(epi_data, "html.parser")
 
@@ -119,7 +133,7 @@ def __extract_download_urls(self):
                 continue
 
             url = down_base + "ts=" + self.ts_no + "&id=" + episode.id + "&server=" + self.server_id
-            target = self.session.get(url).json()["target"]
+            target = self.session.get(url, headers=self.headers).json()["target"]
 
             episode.page_url = target
 
diff --git a/requirements.txt b/requirements.txt
@@ -1,5 +1,7 @@
-art==4.5
 requests==2.22.0
-cloudscraper==1.2.30
+art==4.5
+cloudscraper==1.2.33
 beautifulsoup4==4.8.2
-PySimpleGUI==4.16.0
+js2py==0.68
+PySimpleGUI==4.18.0
+polling==0.3.1
