add keyone autoloader urls, bbmobile support scraper

Author: thurask

bbarchivist/networkutils.py

@@ -721,18 +721,39 @@ def root_generator(folder, build, variant="common"):
     dtek50x = "bbSupport/DTEK50" if build[:3] == "AAF" else "bbfoundation/hashfiles_priv/dtek50"
     #DTEK60 specific
     dtek60x = dtek50x  # still uses dtek50 folder, for some reason
-    #KEYone specific
-    keyonex = "bbfoundation/hashfiles_priv/keyone"  # PLACEHOLDER
-    #Aurora specific
-    aurorax = "bbfoundation/hashfiles_priv/aurora"  # PLACEHOLDER
     #Pack it up
-    roots = {"Priv": privx, "DTEK50": dtek50x, "DTEK60": dtek60x, "KEYone": keyonex, "Aurora": aurorax}
+    roots = {"Priv": privx, "DTEK50": dtek50x, "DTEK60": dtek60x}
     return roots
 
 
-def make_droid_skeleton(method, build, device, variant="common"):
+def make_droid_skeleton_bbm(method, build, device, variant="common"):
     """
-    Make an Android autoloader/hash URL.
+    Make an Android autoloader/hash URL, on the BB Mobile site.
+
+    :param method: None for regular OS links, "sha256/512" for SHA256 or 512 hash.
+    :type method: str
+
+    :param build: Build to check, 3 letters + 3 numbers.
+    :type build: str
+
+    :param device: Device to check.
+    :type device: str
+
+    :param variant: Autoloader variant. Default is "common".
+    :type variant: str
+    """
+    devices = {"KEYone": "qc8953"}
+    base = "bbry_{2}_autoloader_user-{0}-{1}".format(variant, build.upper(), devices[device])
+    if method is None:
+        skel = "http://54.247.87.13/softwareupgrade/BBM/{0}.zip".format(base)
+    else:
+        skel = "http://54.247.87.13/softwareupgrade/BBM/{0}.{1}sum".format(base, method.lower())
+    return skel
+
+
+def make_droid_skeleton_og(method, build, device, variant="common"):
+    """
+    Make an Android autoloader/hash URL, on the original site.
 
     :param method: None for regular OS links, "sha256/512" for SHA256 or 512 hash.
     :type method: str
@@ -747,7 +768,7 @@ def make_droid_skeleton(method, build, device, variant="common"):
     :type variant: str
     """
     folder = {"vzw-vzw": "verizon", "na-att": "att", "na-tmo": "tmo", "common": "default"}
-    devices = {"Priv": "qc8992", "DTEK50": "qc8952_64_sfi", "DTEK60": "qc8996", "KEYone": "qc8953", "Aurora": "qc8917"}  # PLACEHOLDER
+    devices = {"Priv": "qc8992", "DTEK50": "qc8952_64_sfi", "DTEK60": "qc8996"}
     roots = root_generator(folder, build, variant)
     base = "bbry_{2}_autoloader_user-{0}-{1}".format(variant, build.upper(), devices[device])
     if method is None:
@@ -757,6 +778,32 @@ def make_droid_skeleton(method, build, device, variant="common"):
     return skel
 
 
+def make_droid_skeleton(method, build, device, variant="common"):
+    """
+    Make an Android autoloader/hash URL.
+
+    :param method: None for regular OS links, "sha256/512" for SHA256 or 512 hash.
+    :type method: str
+
+    :param build: Build to check, 3 letters + 3 numbers.
+    :type build: str
+
+    :param device: Device to check.
+    :type device: str
+
+    :param variant: Autoloader variant. Default is "common".
+    :type variant: str
+    """
+    # No Aurora
+    oglist = ("Priv", "DTEK50", "DTEK60")  # BlackBerry
+    bbmlist = ("KEYone")   # BB Mobile
+    if device in oglist:
+        skel = make_droid_skeleton_og(method, build, device, variant)
+    elif device in bbmlist:
+        skel = make_droid_skeleton_bbm(method, build, device, variant)
+    return skel
+
+
 def bulk_droid_skeletons(devs, build, method=None):
     """
     Prepare list of Android autoloader/hash URLs.
@@ -770,9 +817,9 @@ def bulk_droid_skeletons(devs, build, method=None):
     :param method: None for regular OS links, "sha256/512" for SHA256 or 512 hash.
     :type method: str
     """
-    carrier_variants = ("common", "vzw-vzw", "na-tmo", "na-att")  # device variants
+    carrier_variants = ("common", "vzw-vzw", "na-tmo", "na-att")  # TODO: add Sprint KEYone
     common_variants = ("common", )  # no Americans
-    carrier_devices = ("Priv", )  # may this list never expand in the future
+    carrier_devices = ("Priv", )  # TODO: add KEYone carrier loaders
     skels = []
     for dev in devs:
         varlist = carrier_variants if dev in carrier_devices else common_variants
@@ -872,21 +919,47 @@ def table_headers(pees):
 @pem_wrapper
 def loader_page_scraper(session=None):
     """
-    Return scraped autoloader page.
+    Return scraped autoloader pages.
 
     :param session: Requests session object, default is created on the fly.
     :type session: requests.Session()
     """
-    url = "http://ca.blackberry.com/support/smartphones/Android-OS-Reload.html"
     session = generic_session(session)
+    loader_page_scraper_og(session)
+    loader_page_scraper_bbm(session)
+
+
+def loader_page_scraper_og(session=None):
+    """
+    Return scraped autoloader page, original site.
+
+    :param session: Requests session object, default is created on the fly.
+    :type session: requests.Session()
+    """
+    url = "http://ca.blackberry.com/support/smartphones/Android-OS-Reload.html"
     soup = generic_soup_parser(url, session)
     tables = soup.find_all("table")
     headers = table_headers(soup.find_all("p"))
     for idx, table in enumerate(tables):
-        loader_page_chunker(idx, table, headers)
+        loader_page_chunker_og(idx, table, headers)
+
+
+def loader_page_scraper_bbm(session=None):
+    """
+    Return scraped autoloader page, new site.
+
+    :param session: Requests session object, default is created on the fly.
+    :type session: requests.Session()
+    """
+    url = "http://www.blackberrymobile.com/support/reload-software/"
+    soup = generic_soup_parser(url, session)
+    ulls = soup.find_all("ul", {"class": "list-two"})[1:]
+    print("~~~BlackBerry KEYone~~~")
+    for ull in ulls:
+        loader_page_chunker_bbm(ull)
 
 
-def loader_page_chunker(idx, table, headers):
+def loader_page_chunker_og(idx, table, headers):
     """
     Given a loader page table, chunk it into lists of table cells.
 
@@ -906,6 +979,18 @@ def loader_page_chunker(idx, table, headers):
     print(" ")
 
 
+def loader_page_chunker_bbm(ull):
+    """
+    Given a loader page list, chunk it into lists of list items.
+
+    :param ull: HTML unordered list tag.
+    :type ull: bs4.element.Tag
+    """
+    chunks = chunker(ull.find_all("li"), 3)
+    for chunk in chunks:
+        loader_page_printer(chunk)
+
+
 def loader_page_printer(chunk):
     """
     Print individual cell texts given a list of table cells.

bbarchivist/scripts/droidlookup.py

@@ -75,11 +75,14 @@ def grab_args():
         parser.set_defaults()
         if args.single:
             args.ceil = args.floor  # range(x, x+1) == x
+        famlist = jsonutils.load_json("droidfamilies")
+        cleanlist = famlist[:4]  # Priv/DTEK50/DTEK60/KEYone
         if args.device is None:
-            famlist = jsonutils.load_json("droidfamilies")
             if not args.alldevices:
-                famlist = famlist[:3]  # Priv/DTEKx0, but not Keyone/Aurora
+                famlist = cleanlist
             droidlookup_main(famlist, args.branch, args.floor, args.ceil, args.type)
+        elif args.device not in cleanlist:
+            print("Selected device {0} has unknown autoloader scheme!".format(args.device))
         else:
             droidlookup_main(args.device, args.branch, args.floor, args.ceil, args.type)
     else:

tests/test_networkutils.py

@@ -152,7 +152,7 @@ def ls_mock(url, request):
     """
     Mock for loader scraping.
     """
-    thebody = '<p><b>BlackBerry DTEK50</b></p><table><tbody><tr><td style="padding-right: 20px;">BlackBerry common SW for STH100-1 &amp; STH100-2 devices</td><td>AAG326</td><td><a href="https://bbapps.download.blackberry.com/Priv/bbry_qc8952_64_sfi_autoloader_user-common-AAG326.zip" target="_blank">Click Here</a></td></tr></tbody></table>'
+    thebody = '<p><b>BlackBerry DTEK50</b></p><table><tbody><tr><td style="padding-right: 20px;">BlackBerry common SW for STH100-1 &amp; STH100-2 devices</td><td>AAG326</td><td><a href="https://bbapps.download.blackberry.com/Priv/bbry_qc8952_64_sfi_autoloader_user-common-AAG326.zip" target="_blank">Click Here</a></td></tr></tbody></table>' if "blackberrymobile" not in request.url else '<ul class="list-two special-a"></ul><li><ul class="list-two special-b"><li class="list-two-a"><p>BlackBerry common SW for BBB100-1 devices use this if you bought your device direct from retailer or through a carrier other than those listed below.</p></li><li class="list-two-b"><p>AAK399</p></li><li class="list-two-c"><p><a href="http://54.247.87.13/softwareupgrade/BBM/bbry_qc8953_autoloader_user-common-AAK399%20-With%20SHA%20files.zip">Click here</a></p></li></ul>'
     return {'status_code': 200, 'content': thebody}
 
 
@@ -164,6 +164,22 @@ def pa_good_mock(url, request):
     return {'status_code': 200, 'content': thebody}
 
 
+def pa_bbm_mock(url, request):
+    """
+    Mock for Android autoloader lookup, new site.
+    """
+    thebody = "http://54.247.87.13/softwareupgrade/BBM/bbry_qc8953_autoloader_user-common-AAL093.zip"
+    return {'status_code': 200, 'content': thebody}
+
+
+def pa_bbm_hash_mock(url, request):
+    """
+    Mock for Android autoloader lookup, new site.
+    """
+    thebody = "http://54.247.87.13/softwareupgrade/BBM/bbry_qc8953_autoloader_user-common-AAL093.sha512sum"
+    return {'status_code': 200, 'content': thebody}
+
+
 def pa_bad_mock(url, request):
     """
     Mock for Android autoloader lookup, worst case.
@@ -628,6 +644,7 @@ def test_autoloader_scan_good(self):
             results = bn.droid_scanner("AAD250", "Priv")
             assert "user-common-AAD250" in results[0]
 
+
     def test_autoloader_scan_bad(self):
         """
         Test Android autoloader lookup, worst case.
@@ -636,6 +653,14 @@ def test_autoloader_scan_bad(self):
             results = bn.droid_scanner("AAD250", "Priv")
             assert results is None
 
+    def test_autoloader_scan_bbm(self):
+        """
+        Test Android autoloader lookup, new site.
+        """
+        with httmock.HTTMock(pa_bbm_mock):
+            results = bn.droid_scanner("AAL093", "KEYone")
+            assert "user-common-AAL093" in results[0]
+
     def test_autoloader_scan_hash(self):
         """
         Test Android autoloader hash lookup.
@@ -644,6 +669,14 @@ def test_autoloader_scan_hash(self):
             results = bn.droid_scanner("AAD250", "Priv", "sha256")
             assert "user-common-AAD250" in results[0]
 
+    def test_autoloader_scan_hash_bbm(self):
+        """
+        Test Android autoloader hash lookup, new site.
+        """
+        with httmock.HTTMock(pa_bbm_hash_mock):
+            results = bn.droid_scanner("AAL093", "KEYone", "sha512")
+            assert "user-common-AAL093" in results[0]
+
     def test_autoloader_scan_list(self):
         """
         Test Android autoloader lookup, list of devices.