Decompress ZIP files to calculate SS checksums.

Author: Wintermute0110

resources/scrap.py

@@ -30,6 +30,7 @@
 import time
 import urllib
 import urlparse
+import zipfile
 
 # --- AEL packages ---
 from .constants import *
@@ -3318,23 +3319,19 @@ def get_candidates(self, search_term, rom_FN, rom_checksums_FN, platform, status
             log_debug('ScreenScraper.get_candidates() Scraper disabled. Returning empty data.')
             return None
 
-        # Prepare data for scraping.
-        rombase = rom_FN.getBase()
-        rompath = rom_FN.getPath()
-        romchecksums_path = rom_checksums_FN.getPath()
-
         # --- Get candidates ---
         # ScreenScraper jeuInfos.php returns absolutely everything about a single ROM, including
         # metadata, artwork, etc. jeuInfos.php returns one game or nothing at all.
         # ScreenScraper returns only one game or nothing at all.
+        rompath = rom_FN.getPath()
+        romchecksums_path = rom_checksums_FN.getPath()
         scraper_platform = AEL_platform_to_ScreenScraper(platform)
-        log_debug('ScreenScraper.get_candidates() rombase      "{}"'.format(rombase))
         log_debug('ScreenScraper.get_candidates() rompath      "{}"'.format(rompath))
         log_debug('ScreenScraper.get_candidates() romchecksums "{}"'.format(romchecksums_path))
         log_debug('ScreenScraper.get_candidates() AEL platform "{}"'.format(platform))
         log_debug('ScreenScraper.get_candidates() SS platform  "{}"'.format(scraper_platform))
         candidate_list = self._search_candidates_jeuInfos(
-            rombase, rompath, romchecksums_path, platform, scraper_platform, status_dic)
+            rom_FN, rom_checksums_FN, platform, scraper_platform, status_dic)
         # _search_candidates_jeuRecherche() does not work for get_metadata() and get_assets()
         # because jeu_dic is not introduced in the internal cache.
         # candidate_list = self._search_candidates_jeuRecherche(
@@ -3559,8 +3556,7 @@ def debug_game_search(self, search_term, rombase_noext, platform, status_dic):
         self._dump_json_debug('ScreenScraper_gameSearch.json', json_data)
 
     # Call to ScreenScraper jeuInfos.php.
-    def _search_candidates_jeuInfos(self, rombase, rompath, romchecksums_path, platform,
-        scraper_platform, status_dic):
+    def _search_candidates_jeuInfos(self, rom_FN, rom_checksums_FN, platform, scraper_platform, status_dic):
         # --- Test data ---
         # * Example from ScreenScraper API info page.
         #   #crc=50ABC90A&systemeid=1&romtype=rom&romnom=Sonic%20The%20Hedgehog%202%20(World).zip&romtaille=749652
@@ -3585,17 +3581,16 @@ def _search_candidates_jeuInfos(self, rombase, rompath, romchecksums_path, platf
 
         # --- IMPORTANT ---
         # ScreenScraper requires all CRC, MD5 and SHA1 and the correct file size of the
-        # files scraped. Put these data in a SS checksums cache so it is calculated once for
-        # every file.
+        # files scraped.
         if self.debug_checksums_flag:
             # Use fake checksums when developing the scraper with fake 0-sized files.
             log_info('Using debug checksums and not computing real ones.')
             checksums = {
-                'crc'  : self.debug_crc, 'md5'  : self.debug_md5,
-                'sha1' : self.debug_sha1, 'size' : self.debug_size,
+                'crc'  : self.debug_crc, 'md5'  : self.debug_md5, 'sha1' : self.debug_sha1,
+                'size' : self.debug_size, 'rom_name' : rom_FN.getBase(),
             }
         else:
-            checksums = misc_calculate_checksums(romchecksums_path)
+            checksums = self._get_SS_checksum(rom_checksums_FN)
             if checksums is None:
                 status_dic['status'] = False
                 status_dic['msg'] = 'Error computing file checksums.'
@@ -3608,8 +3603,8 @@ def _search_candidates_jeuInfos(self, rombase, rompath, romchecksums_path, platf
         crc_str = checksums['crc']
         md5_str = checksums['md5']
         sha1_str = checksums['sha1']
-        # rom_name = urllib.quote(rombase)
-        rom_name = urllib.quote_plus(rombase)
+        # rom_name = urllib.quote(checksums['rom_name'])
+        rom_name = urllib.quote_plus(checksums['rom_name'])
         rom_size = checksums['size']
         # log_debug('ScreenScraper._search_candidates_jeuInfos() ssid       "{0}"'.format(self.ssid))
         # log_debug('ScreenScraper._search_candidates_jeuInfos() ssid       "{0}"'.format('***'))
@@ -3854,6 +3849,46 @@ def _retrieve_all_assets(self, jeu_dic, status_dic):
 
         return asset_list
 
+    # 1) If rom_checksums_FN is a ZIP file and contains one and only one file, then consider that
+    #    file the ROM, decompress in memory and calculate the checksums.
+    # 2) If rom_checksums_FN is a standard file or 1) fails then calculate the checksums of
+    #    the file.
+    # 3) Return a checksums dictionary if everything is OK. Return None in case of any error.
+    def _get_SS_checksum(self, rom_checksums_FN):
+        f_basename = rom_checksums_FN.getBase()
+        f_path = rom_checksums_FN.getPath()
+        log_debug('_get_SS_checksum() Processing "{}"'.format(f_path))
+        if f_basename.lower().endswith('.zip'):
+            log_debug('_get_SS_checksum() ZIP file detected.')
+            if not zipfile.is_zipfile(f_path):
+                log_error('zipfile.is_zipfile() returns False. Bad ZIP file.')
+                return None
+            else:
+                log_debug('_get_SS_checksum() ZIP file seems to be correct.')
+            zip = zipfile.ZipFile(f_path)
+            namelist = zip.namelist()
+            # log_variable('namelist', namelist)
+            if len(namelist) == 1:
+                log_debug('_get_SS_checksum() ZIP file has one file only.')
+                log_debug('_get_SS_checksum() Decompressing file "{}"'.format(namelist[0]))
+                file_bytes = zip.read(namelist[0])
+                log_debug('_get_SS_checksum() Decompressed size is {} bytes'.format(len(file_bytes)))
+                checksums = misc_calculate_stream_checksums(file_bytes)
+                checksums['rom_name'] = namelist[0]
+                log_debug('_get_SS_checksum() ROM name is "{}"'.format(checksums['rom_name']))
+                return checksums
+            else:
+                log_debug('_get_SS_checksum() ZIP file has {} files.'.format(len(namelist)))
+                log_debug('_get_SS_checksum() Computing checksum of whole ZIP file.')
+        else:
+            log_debug('_get_SS_checksum() File is not ZIP. Computing checksum of whole ZIP file.')
+        # Otherwise calculate checksums of the whole file
+        checksums = misc_calculate_file_checksums(f_path)
+        checksums['rom_name'] = f_basename
+        log_debug('_get_SS_checksum() ROM name is "{}"'.format(checksums['rom_name']))
+
+        return checksums
+
     # ScreenScraper URLs have the developer password and the user password.
     # Clean URLs for safe logging.
     def _clean_URL_for_log(self, url):

resources/utils.py

@@ -651,7 +651,7 @@ def misc_generate_random_SID():
 #
 # Lazy function (generator) to read a file piece by piece. Default chunk size: 8k.
 #
-def misc_read_in_chunks(file_object, chunk_size = 8192):
+def misc_read_file_in_chunks(file_object, chunk_size = 8192):
     while True:
         data = file_object.read(chunk_size)
         if not data: break
@@ -664,14 +664,14 @@ def misc_read_in_chunks(file_object, chunk_size = 8192):
 # https://stackoverflow.com/questions/519633/lazy-method-for-reading-big-file-in-python
 # https://stackoverflow.com/questions/1742866/compute-crc-of-file-in-python 
 #
-def misc_calculate_checksums(full_file_path):
+def misc_calculate_file_checksums(full_file_path):
     log_debug('Computing checksums "{}"'.format(full_file_path))
     try:
         f = open(full_file_path, 'rb')
         crc_prev = 0
         md5 = hashlib.md5()
         sha1 = hashlib.sha1()
-        for piece in misc_read_in_chunks(f):
+        for piece in misc_read_file_in_chunks(f):
             crc_prev = zlib.crc32(piece, crc_prev)
             md5.update(piece)
             sha1.update(piece)
@@ -680,7 +680,7 @@ def misc_calculate_checksums(full_file_path):
         sha1_digest = sha1.hexdigest()
         size = os.path.getsize(full_file_path)
     except:
-        log_debug('(Exception) In misc_calculate_checksums()')
+        log_debug('(Exception) In misc_calculate_file_checksums()')
         log_debug('Returning None')
         return None
     checksums = {
@@ -692,6 +692,44 @@ def misc_calculate_checksums(full_file_path):
 
     return checksums
 
+# This function not finished yet.
+def misc_read_bytes_in_chunks(file_bytes, chunk_size = 8192):
+    file_length = len(file_bytes)
+    block_number = 0
+    while True:
+        start_index = None
+        end_index = None
+        data = file_bytes[start_index:end_index]
+        yield data
+
+def misc_calculate_stream_checksums(file_bytes):
+    log_debug('Computing checksums of bytes stream...'.format(len(file_bytes)))
+    crc_prev = 0
+    md5 = hashlib.md5()
+    sha1 = hashlib.sha1()
+    # Process bytes stream block by block
+    # for piece in misc_read_bytes_in_chunks(file_bytes):
+    #     crc_prev = zlib.crc32(piece, crc_prev)
+    #     md5.update(piece)
+    #     sha1.update(piece)
+    # Process bytes in one go
+    crc_prev = zlib.crc32(file_bytes, crc_prev)
+    md5.update(file_bytes)
+    sha1.update(file_bytes)
+    crc_digest = '{:08X}'.format(crc_prev & 0xFFFFFFFF)
+    md5_digest = md5.hexdigest()
+    sha1_digest = sha1.hexdigest()
+    size = len(file_bytes)
+
+    checksums = {
+        'crc'  : crc_digest.upper(),
+        'md5'  : md5_digest.upper(),
+        'sha1' : sha1_digest.upper(),
+        'size' : size,
+    }
+
+    return checksums
+
 # -------------------------------------------------------------------------------------------------
 # Filesystem helper class
 # This class always takes and returns Unicode string paths. Decoding to UTF-8 must be done in