Merge pull request #2043 from diamondpete/scraper-changes

Various Site Changes

Author: DirtyRacer1337

Contents/Code/PAdatabaseActors.py

@@ -1165,6 +1165,7 @@
         'Timea Bella': ['Luciana'],
         'Traci Lynn': ['Hallee'],
         'Tracy Gold': ['Noleta'],
+        'Tristan Summers': ['Tristan Taylor'],
         'Tyra Moon': ['Athina'],
         'Valerie Herrera': ['Valerie'],
         'Vanessa Monroe': ['Vanessa'],
@@ -1596,6 +1597,7 @@
     22: {  # Reality Kings
         'Agness Miller': ['Agness'],
         'Morgan Layne': ['Morgan'],
+        'Jessica Robbin': ['Jessica Rabbit'],
     },
     23: {  # WowGirls
         'Goldie Baby': ['Nikia'],
@@ -6109,13 +6111,23 @@
     69: {  # Adult Empire
         'Addison Vodka': ['Addison'],
         'Ariana Grand': ['Ariana'],
+        'Carol Foxxx': ['Carol Foxx'],
+        'Cherry Torn': ['Cherry Thorn'],
+        'Christina Stevens': ['Kristina'],
         'Christine DeShaffer': ['Christine De Shaffer'],
+        'Desiree Deluca': ['Desiree'],
         'Graycee Baybee': ['Grace Baybee'],
         'Ice La Fox': ['Ice LaFox'],
+        'Jana Malto': ['Jana'],
         'Jesus Reyes Vazquez': ['Jesus Reyes Vazqeuz'],
         'Red Julia': ['Judith'],
+        'Kalina Ryu': ['Lily Star'],
+        'Kay Carter': ['Kay Karter'],
+        'Kitti Skyway': ['Kitty Skyway'],
         'Kya Tropic': ['Kya Clover'],
+        'Eva Gomez': ['Kylina'],
         'Lutro Steel': ['Lutro Steele'],
+        'Mai Ly': ['Mai Lynn'],
         'Mandarine': ['Alexandrine'],
         'Mazzy Paige': ['Mazzie Paige', 'Mazzi'],
         'Maude Carolle': ['Aude Lecocq'],
@@ -6128,6 +6140,7 @@
         'Suzan Nielsen': ['Susan'],
         'Talon': ['Talon Valenti'],
         'Vanessa D\'Angely': ['Vanessa'],
+        'Will Ryder': ['Jeff Mullen'],
         'Zeus': ['Jesus "Zeus"Munoz'],
     },
     70: {  # Heavy on Hotties
@@ -6148,6 +6161,9 @@
     },
     71: {  # Brazzers
         'Alice Fabre': ['Alice']
+    },
+    72: {  # Kelly Madison Productions
+        'Ryan Madison': ['Madison']
     }
 }
 
@@ -6224,6 +6240,7 @@
     69: ['Adult Empire'],
     70: ['Heavy on Hotties'],
     71: ['Brazzers'],
+    72: ['Kelly Madison', 'TeenFidelity', 'PornFidelity'],
 }
 
 

Contents/Code/PAsiteList.py

@@ -373,9 +373,9 @@
     181: ('We Live Together', 'https://www.realitykings.com', 'https://site-api.project1service.com'),
     182: ('Wives in Pantyhose', 'https://www.realitykings.com', 'https://site-api.project1service.com'),
     183: ('21Naturals', 'https://www.21naturals.com', 'https://tsmkfa364q-dsn.algolia.net/1/indexes/*/queries'),
-    184: ('PornFidelity', 'https://www.pornfidelity.com', '/episodes/search/??site=2&page=1&search='),
-    185: ('TeenFidelity', 'https://www.pornfidelity.com', '/episodes/search/??site=3&page=1&search='),
-    186: ('Kelly Madison', 'https://www.pornfidelity.com', '/episodes/search/??site=1&page=1&search='),
+    184: ('PornFidelity', 'https://www.pornfidelity.com', '/search?q='),
+    185: ('TeenFidelity', 'https://www.pornfidelity.com', '/search?q='),
+    186: ('Kelly Madison', 'https://www.pornfidelity.com', '/search?q='),
     187: ('TeamSkeet', 'https://www.teamskeet.com', '/movies/'),
     188: ('Exxxtra Small', 'https://www.teamskeet.com', '/movies/'),
     189: ('Teen Pies', 'https://www.teamskeet.com', '/movies/'),
@@ -1385,39 +1385,39 @@
     1193: ('Over 40 Handjobs', 'https://www.over40handjobs.com', '/models/'),
     1194: ('Ebony Tugs', 'https://ebonytugs.com', '/models/'),
     1195: ('Teen Tugs', 'https://teentugs.com', '/models/'),
-    1196: ('Czech Sex Casting', 'https://www.czechsexcasting.com', '/en/search-results?value='),
-    1197: ('Sex With Muslims', 'https://www.sexwithmuslims.com', '/en/search-results?value='),
-    1198: ('Sex In Taxi', 'https://www.sexintaxi.com', '/en/search-results?value='),
-    1199: ('VR Porn CZ', 'https://www.vrporncz.com', '/en/search-results?value='),
-    1200: ('Fucking Street', 'https://www.fuckingstreet.com', '/en/search-results?value='),
-    1201: ('Hunter POV', 'https://www.hunterpov.com', '/en/search-results?value='),
-    1202: ('Czech Gypsies', 'https://www.czechgypsies.com', '/en/search-results?value='),
-    1203: ('Dick On Trip', 'https://www.dickontrip.com', '/en/search-results?value='),
-    1204: ('Czech Boobs', 'https://www.czechboobs.com', '/en/search-results?value='),
-    1205: ('Czech Deviant', 'https://www.czechdeviant.com', '/en/search-results?value='),
-    1206: ('Amateri Premium', 'https://www.amateripremium.com', '/en/search-results?value='),
-    1207: ('Fucking Office', 'https://www.fuckingoffice.com', '/en/search-results?value='),
-    1208: ('Czech Executor', 'https://www.czechexecutor.com', '/en/search-results?value='),
-    1209: ('Czech Hitchhikers', 'https://www.czechhitchhikers.com', '/en/search-results?value='),
-    1210: ('Girls Take Away', 'https://www.girlstakeaway.com', '/en/search-results?value='),
-    1211: ('Czech Escort Girls', 'https://www.czechescortgirls.com', '/en/search-results?value='),
-    1212: ('Horny Doctor', 'https://www.hornydoctor.com', '/en/search-results?value='),
-    1213: ('Lady Dee', 'https://www.ladydee.com', '/en/search-results?value='),
-    1214: ('Teen From Bohemia', 'https://www.teenfrombohemia.com', '/en/search-results?value='),
-    1215: ('Czech Real Dolls', 'https://www.czechrealdolls.com', '/en/search-results?value='),
-    1216: ('Amateur From Bohemia', 'https://www.amateursfrombohemia.com', '/en/search-results?value='),
-    1217: ('Czech Anal Sex', 'https://www.czechanalsex.com', '/en/search-results?value='),
-    1218: ('Dellia Twins', 'https://www.dellaitwins.com', '/en/search-results?value='),
-    1219: ('Chloe Lamour', 'https://www.chloelamour.com', '/en/search-results?value='),
-    1220: ('Public From Bohemia', 'https://www.publicfrombohemia.com', '/en/search-results?value='),
-    1221: ('Susan Ayn', 'https://www.susanayn.com', '/en/search-results?value='),
-    1222: ('Horny Girls CZ', 'https://www.hornygirlscz.com', '/en/search-results?value='),
-    1223: ('Czech Sex Party', 'https://www.czechsexparty.com', '/en/search-results?value='),
-    1224: ('Retro Porn CZ', 'https://www.retroporncz.com', '/en/search-results?value='),
-    1225: ('Boys Fuck MILFs', 'https://www.boysfuckmilfs.com', '/en/search-results?value='),
-    1226: ('Czech Bi Porn', 'https://www.czechbiporn.com', '/en/search-results?value='),
-    1227: ('Czech Shemale', 'https://www.czechshemale.com', '/en/search-results?value='),
-    1228: ('Czech Gay City', 'https://www.czechgaycity.com', '/en/search-results?value='),
+    1196: ('Czech Sex Casting', 'https://www.czechsexcasting.com', '/en/search?q='),
+    1197: ('Sex With Muslims', 'https://www.sexwithmuslims.com', '/en/search?q='),
+    1198: ('Sex In Taxi', 'https://www.sexintaxi.com', '/en/search?q='),
+    1199: ('VR Porn CZ', 'https://www.vrporncz.com', '/en/search?q='),
+    1200: ('Fucking Street', 'https://www.fuckingstreet.com', '/en/search?q='),
+    1201: ('Hunter POV', 'https://www.hunterpov.com', '/en/search?q='),
+    1202: ('Czech Gypsies', 'https://www.czechgypsies.com', '/en/search?q='),
+    1203: ('Dick On Trip', 'https://www.dickontrip.com', '/en/search?q='),
+    1204: ('Czech Boobs', 'https://www.czechboobs.com', '/en/search?q='),
+    1205: ('Czech Deviant', 'https://www.czechdeviant.com', '/en/search?q='),
+    1206: ('Amateri Premium', 'https://www.amateripremium.com', '/en/search?q='),
+    1207: ('Fucking Office', 'https://www.fuckingoffice.com', '/en/search?q='),
+    1208: ('Czech Executor', 'https://www.czechexecutor.com', '/en/search?q='),
+    1209: ('Czech Hitchhikers', 'https://www.czechhitchhikers.com', '/en/search?q='),
+    1210: ('Girls Take Away', 'https://www.girlstakeaway.com', '/en/search?q='),
+    1211: ('Czech Escort Girls', 'https://www.czechescortgirls.com', '/en/search?q='),
+    1212: ('Horny Doctor', 'https://www.hornydoctor.com', '/en/search?q='),
+    1213: ('Lady Dee', 'https://www.ladydee.com', '/en/search?q='),
+    1214: ('Teen From Bohemia', 'https://www.teenfrombohemia.com', '/en/search?q='),
+    1215: ('Czech Real Dolls', 'https://www.czechrealdolls.com', '/en/search?q='),
+    1216: ('Amateur From Bohemia', 'https://www.amateursfrombohemia.com', '/en/search?q='),
+    1217: ('Czech Anal Sex', 'https://www.czechanalsex.com', '/en/search?q='),
+    1218: ('Dellia Twins', 'https://www.dellaitwins.com', '/en/search?q='),
+    1219: ('Chloe Lamour', 'https://www.chloelamour.com', '/en/search?q='),
+    1220: ('Public From Bohemia', 'https://www.publicfrombohemia.com', '/en/search?q='),
+    1221: ('Susan Ayn', 'https://www.susanayn.com', '/en/search?q='),
+    1222: ('Horny Girls CZ', 'https://www.hornygirlscz.com', '/en/search?q='),
+    1223: ('Czech Sex Party', 'https://www.czechsexparty.com', '/en/search?q='),
+    1224: ('Retro Porn CZ', 'https://www.retroporncz.com', '/en/search?q='),
+    1225: ('Boys Fuck MILFs', 'https://www.boysfuckmilfs.com', '/en/search?q='),
+    1226: ('Czech Bi Porn', 'https://www.czechbiporn.com', '/en/search?q='),
+    1227: ('Czech Shemale', 'https://www.czechshemale.com', '/en/search?q='),
+    1228: ('Czech Gay City', 'https://www.czechgaycity.com', '/en/search?q='),
     1229: ('Top Web Models', 'https://tour.topwebmodels.com', '/scenes'),
     1230: ('Big Gulp Girls', 'https://tour.biggulpgirls.com', '/scenes'),
     1231: ('2 Girls 1 Camera', 'https://tour.2girls1camera.com', '/scenes'),
@@ -1819,7 +1819,7 @@
     1727: ('Teen Sneaks', 'https://adulttime.com', 'https://tsmkfa364q-dsn.algolia.net/1/indexes/*/queries'),
     1728: ('Modern Day Sins', 'https://adulttime.com', 'https://tsmkfa364q-dsn.algolia.net/1/indexes/*/queries'),
     1729: ('Accidental Gangbang', 'https://adulttime.com', 'https://tsmkfa364q-dsn.algolia.net/1/indexes/*/queries'),
-    1730: ('Aunt Judys XXX', 'https://auntjudysxxx.com', '/tour/search.php?query='),
+    1730: ('Aunt Judys XXX', 'https://auntjudys.com', '/tour/search.php?query='),
     1731: ('Aunt Judys', 'https://auntjudys.com', '/tour/search.php?query='),
     1732: ('Gilfed', 'https://gilfed.com', 'https://site-api.project1service.com'),
     1733: ('Dilfed', 'https://dilfed.com', 'https://site-api.project1service.com'),

Contents/Code/networkAdultEmpireCash.py

@@ -114,8 +114,9 @@ def update(metadata, lang, siteNum, movieGenres, movieActors, movieCollections,
 
     # Posters
     for poster in detailsPageElements.xpath('//div[@id="dv_frames"]//img/@src'):
-        img = poster.replace('/320/', '/1280/')
-        art.append(img)
+        img = poster.replace('/320/', '/3840/').replace('/10/', '/3840/').replace('_320c.jpg', '_10.jpg')
+        if img not in art:
+            art.append(img)
 
     Log('Artwork found: %d' % len(art))
     for idx, posterUrl in enumerate(art, 1):

Contents/Code/networkAdultPrime.py

@@ -80,10 +80,10 @@ def update(metadata, lang, siteNum, movieGenres, movieActors, movieCollections,
         metadata.summary = summary
 
     # Studio
-    metadata.studio = 'Adult Prime'
+    metadata.studio = detailsPageElements.xpath('//p[@class="update-info-line regular"][./b[contains(., "Studio")]]//a')[0].text_content().strip()
 
     # Tagline and Collection(s)
-    tagline = detailsPageElements.xpath('//p[@class="update-info-line regular"][./b[contains(., "Studio")]]//a')[0].text_content().strip()
+    tagline = detailsPageElements.xpath('//p[@class="update-info-line regular"][./b[contains(., "Series")]]//a')[1].text_content().strip()
     metadata.tagline = tagline
     movieCollections.addCollection(tagline)
 
@@ -135,7 +135,11 @@ def update(metadata, lang, siteNum, movieGenres, movieActors, movieCollections,
             art.append(img)
 
     Log('Artwork found: %d' % len(art))
+    postersClean = list()
     for idx, posterUrl in enumerate(art, 1):
+        # Remove Timestamp and Token from URL
+        cleanUrl = posterUrl.split('?')[0]
+        postersClean.append(cleanUrl)
         if not PAsearchSites.posterAlreadyExists(posterUrl, metadata):
             # Download image file for analysis
             try:
@@ -153,6 +157,8 @@ def update(metadata, lang, siteNum, movieGenres, movieActors, movieCollections,
             except:
                 pass
 
+    art.extend(postersClean)
+
     return metadata
 
 

Contents/Code/networkBang.py

@@ -48,13 +48,9 @@ def search(results, lang, siteNum, searchData):
             sceneURL = PAsearchSites.getSearchBaseURL(siteNum) + sceneURL
         curID = PAutils.Encode(sceneURL)
 
-        try:
-            date = searchResult.xpath('.//span[@class="hidden xs:inline-block truncate"]/text()')[0].strip()
-        except:
-            date = ''
-
+        date = searchResult.xpath('.//span[@class="hidden xs:inline-block truncate"]')
         if date:
-            releaseDate = datetime.strptime(date, '%b %d, %Y').strftime('%Y-%m-%d')
+            releaseDate = datetime.strptime(date[0].text_content().split('\xe2\x80\xa2')[-1].strip(), '%b %d, %Y').strftime('%Y-%m-%d')
         else:
             releaseDate = searchData.dateFormat() if searchData.date else ''
         displayDate = releaseDate if date else ''
@@ -175,6 +171,7 @@ def update(metadata, lang, siteNum, movieGenres, movieActors, movieCollections,
     for idx, posterUrl in enumerate(art, 1):
         if not PAsearchSites.posterAlreadyExists(posterUrl, metadata):
             # Download image file for analysis
+            posterUrl = posterUrl.split('?')[0]
             try:
                 image = PAutils.HTTPRequest(posterUrl)
                 im = StringIO(image.content)

Contents/Code/networkCherryPimps.py

@@ -8,7 +8,7 @@ def search(results, lang, siteNum, searchData):
         url = PAsearchSites.getSearchSearchURL(siteNum) + '%s&page=%d' % (searchData.encoded, searchPageNum)
         req = PAutils.HTTPRequest(url)
         searchResults = HTML.ElementFromString(req.text)
-        for searchResult in searchResults.xpath('//div[contains(@class, "video-thumb") or contains(@class, "item-video")]'):
+        for searchResult in searchResults.xpath('//div[@class="item-updates"]//div[contains(@class, "item-update")]'):
             titleNoFormatting = PAutils.parseTitle(searchResult.xpath('(.//p[@class="text-thumb"] | .//div[@class="item-title"])/a')[0].text_content().strip(), siteNum)
             curID = PAutils.Encode(searchResult.xpath('(.//p[@class="text-thumb"] | .//div[@class="item-title"])/a/@href')[0])
             subSite = searchResult.xpath('.//p[@class="text-thumb"]/a[@class="badge"] | .//div[@class="item-sitename"]/a')[0].text_content().strip()
@@ -117,8 +117,12 @@ def update(metadata, lang, siteNum, movieGenres, movieActors, movieCollections,
                 art.append(poster)
 
     Log('Artwork found: %d' % len(art))
+    postersClean = list()
     for idx, posterUrl in enumerate(art, 1):
-        if not PAsearchSites.posterAlreadyExists(posterUrl, metadata):
+        # Remove Timestamp and Token from URL
+        cleanUrl = posterUrl.split('?')[0]
+        postersClean.append(cleanUrl)
+        if not PAsearchSites.posterAlreadyExists(cleanUrl, metadata):
             # Download image file for analysis
             try:
                 image = PAutils.HTTPRequest(posterUrl)
@@ -128,11 +132,13 @@ def update(metadata, lang, siteNum, movieGenres, movieActors, movieCollections,
                 # Add the image proxy items to the collection
                 if width > 1:
                     # Item is a poster
-                    metadata.posters[posterUrl] = Proxy.Media(image.content, sort_order=idx)
+                    metadata.posters[cleanUrl] = Proxy.Media(image.content, sort_order=idx)
                 if width > 100:
                     # Item is an art item
-                    metadata.art[posterUrl] = Proxy.Media(image.content, sort_order=idx)
+                    metadata.art[cleanUrl] = Proxy.Media(image.content, sort_order=idx)
             except:
                 pass
 
+    art.extend(postersClean)
+
     return metadata

Contents/Code/networkDirtyFlix.py

@@ -75,9 +75,40 @@ def update(metadata, lang, siteNum, movieGenres, movieActors, movieCollections,
     sceneID = PAutils.Decode(metadata_id[0])
     sceneDate = metadata_id[2]
     searchPageURL = PAutils.Decode(metadata_id[3])
+    (siteKey, sitePages) = PAutils.getDictValuesFromKey(siteDB, PAsearchSites.getSearchSiteName(siteNum))
 
     req = PAutils.HTTPRequest(searchPageURL)
-    detailsPageElements = HTML.ElementFromString(req.text).xpath('//div[@class="movie-block"][.//*[contains(@src, "%s")]]' % sceneID)[0]
+    originalPageElements = HTML.ElementFromString(req.text).xpath('//div[@class="movie-block"][.//*[contains(@src, "%s")]]' % sceneID)
+
+    if originalPageElements:
+        detailsPageElements = originalPageElements[0]
+    else:
+        lastSearchPage = searchPageURL.split('/')[-1]
+        if lastSearchPage.isdigit():
+            startPage = int(lastSearchPage) + 1
+        else:
+            startPage = 2
+        searchPage = '%s%d' % (PAsearchSites.getSearchSearchURL(siteNum), startPage)
+        req = PAutils.HTTPRequest(searchPage)
+        searchResults = HTML.ElementFromString(req.text)
+
+        re_sceneid = re.compile(r'(?<=tour_thumbs/).*(?=\/)')
+        for idx in range(startPage, sitePages):
+            for searchResult in searchResults.xpath('//div[@class="movie-block"]'):
+                searchID = 0
+                m = re_sceneid.search(searchResult.xpath('.//li/img/@src')[0])
+                if m:
+                    searchID = m.group(0)
+
+                if searchID == sceneID:
+                    detailsPageElements = HTML.ElementFromString(req.text).xpath('//div[@class="movie-block"][.//*[contains(@src, "%s")]]' % sceneID)[0]
+                    break
+            else:
+                searchPage = '%s%d' % (PAsearchSites.getSearchSearchURL(siteNum), idx)
+                req = PAutils.HTTPRequest(searchPage)
+                searchResults = HTML.ElementFromString(req.text)
+                continue
+            break
 
     xPath = PAutils.getDictValuesFromKey(xPathDB, PAsearchSites.getSearchSiteName(siteNum))
 
@@ -158,7 +189,7 @@ def update(metadata, lang, siteNum, movieGenres, movieActors, movieCollections,
 siteDB = {
     'Trick Your GF': [7, 4],
     'Make Him Cuckold': [9, 5],
-    'She Is Nerdy': [10, 12],
+    'She Is Nerdy': [10, 15],
     'Tricky Agent': [11, 4],
 }
 
@@ -255,7 +286,7 @@ def update(metadata, lang, siteNum, movieGenres, movieActors, movieCollections,
     'Iris Kiss': ['snc165', 'wnc1637'],
     'Isabel Stern': ['wfc1075'],
     'Iva Zan': ['wrygf536', 'wtag558', 'wnc745'],
-    'Izi Ashley': ['wfc978', 'wtag980', 'wnc97'],
+    'Izi Ashley': ['wfc978', 'wtag980', 'wnc97', 'wnc976'],
     'Jane Fox': ['wtag1235'],
     'Jenny Fer': ['wnc1330'],
     'Jenny Love': ['wrygf634', 'wfc607', 'wtag601'],