Audio table decoding: Detect the audio table copy more liberally

Most or all Ravensburger GMEs have a duplicated Audio table. Most of
them have an identical copy, but it seems not all. For those where the
table is 80% equal, treat it like a copy (and throw away the extra
information for now).

It remains future work to make sense of this extra table.

Author: nomeata

src/Commands.hs

@@ -117,7 +117,7 @@ dumpInfo conf file = do
         (fst (head st)) (fst (last st))
         (length (filter (isNothing . snd) st)) (length st)
     printf "Audio table entries: %d\n" (length ttAudioFiles)
-    when ttAudioFilesDoubles $ printf "Audio table repeated twice\n"
+    printf "Audio table copy: %s\n" (show ttAudioFilesDoubles)
     printf "Binary tables entries: %d/%d/%d\n"
         (length ttBinaries1)
         (length ttBinaries2)

src/GMEParser.hs

@@ -270,27 +270,34 @@ getBinaries = do
         binary <- getSegAt offset (BC.unpack desc) (getBS length)
         return (desc, binary)
 
-getAudios :: Word32 -> SGet ([B.ByteString], Bool, Word8)
+getAudios :: Word32 -> SGet ([B.ByteString], Similarity, Word8)
 getAudios rawXor = do
     until <- lookAhead getWord32
     x <- case () of
           () | rawXor == knownRawXOR -> return knownXOR
              | otherwise             -> lookAhead $ jumpTo until >> getXor
     offset <- bytesRead
     let n_entries = fromIntegral ((until - offset) `div` 8)
-    at_doubled <- lookAhead $ do
-        half1 <- getBS (n_entries * 8 `div` 2)
-        half2 <- getBS (n_entries * 8 `div` 2)
-        return $ half1 == half2
-    let n_entries' | at_doubled = n_entries `div` 2
-                   | otherwise  = n_entries
+    similarity <- determine_similiarity <$> lookAhead (getBS (n_entries * 8))
+    let n_entries' | Absent <- similarity = n_entries
+                   | otherwise = n_entries `div` 2
     decoded <- forM [0..n_entries'-1] $ \n -> do
         cypher x <$> indirectBS (show n)
-    -- Fix segment
-    when at_doubled $ lookAhead $ getSeg "Audio table copy" $
+    -- pretend we read the rest too
+    unless (similarity == Absent) $ lookAhead $ getSeg "Audio table copy" $
         replicateM_ (fromIntegral n_entries') (getWord32 >> getWord32)
 
-    return (decoded, at_doubled, x)
+    return (decoded, similarity, x)
+  where
+    -- Fuzzy comparison: More than 80% the same?
+    determine_similiarity bs
+        | a == b            = Equal
+        | 5 * same >= 4 * n = Similar
+        | otherwise         = Absent
+      where
+        (a,b) = B.splitAt (B.length bs `div` 2) bs
+        n = fromIntegral (B.length a)
+        same = length [ () | (x,y) <- B.zip a b, x == y ]
 
 getXor :: SGet Word8
 getXor = do
@@ -307,7 +314,7 @@ getXor = do
 getChecksum :: SGet Word32
 getChecksum = do
     l <- getLength
-    getSegAt (l-4) "Checksum" $ getWord32
+    getSegAt (l-4) "Checksum" getWord32
 
 calcChecksum :: SGet Word32
 calcChecksum = do

src/TipToiYaml.hs

@@ -884,7 +884,7 @@ ttYaml2tt no_date dir (TipToiYAML {..}) extCodes = do
         , ttGames = games
         , ttAudioFiles = files
         , ttAudioXor = knownXOR
-        , ttAudioFilesDoubles = False
+        , ttAudioFilesDoubles = Absent
         , ttMediaFlags = Nothing
         , ttChecksum = 0x00
         , ttChecksumCalc = 0x00

src/Types.hs

@@ -64,6 +64,7 @@ data Line r = Line Offset [Conditional r] [Command r] PlayList
 
 type ProductID = Word32
 
+data Similarity = Absent | Equal | Similar deriving (Show, Eq)
 
 data TipToiFile = TipToiFile
     { ttProductId :: ProductID
@@ -76,7 +77,7 @@ data TipToiFile = TipToiFile
     , ttScripts :: [(Word16, Maybe [Line ResReg])]
     , ttGames :: [Game]
     , ttAudioFiles :: [B.ByteString]
-    , ttAudioFilesDoubles :: Bool
+    , ttAudioFilesDoubles :: Similarity
     , ttAudioXor :: Word8
     , ttMediaFlags :: Maybe [Word16]
     , ttBinaries1 :: [(B.ByteString, B.ByteString)]

testsuite/expected/downloaded/Alle meine Tiere.gme.info.txt

@@ -8,7 +8,8 @@ Number of registers: 3
 Initial registers: [0,0,0]
 Initial sounds: [[18,25]]
 Scripts for OIDs from 4201 to 4282; 0/82 are disabled.
-Audio table entries: 1086
+Audio table entries: 543
+Audio table copy: Similar
 Binary tables entries: 0/0/0
 Single binary table entries: 1/1/1
 Special OIDs: 4210, 0

testsuite/expected/downloaded/Alle meine Tiere.gme.lint.txt

@@ -9,7 +9,7 @@ Initial registers: [1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
 Initial sounds: [[1447,317],[317]]
 Scripts for OIDs from 5670 to 9969; 3709/4300 are disabled.
 Audio table entries: 1448
-Audio table repeated twice
+Audio table copy: Equal
 Binary tables entries: 1/1/1
 Single binary table entries: 1/1/1
 Special OIDs: 9756, 0

testsuite/expected/downloaded/Dein Koerper und Du.gme.info.txt

@@ -8,7 +8,8 @@ Number of registers: 3
 Initial registers: [1,0,0]
 Initial sounds: [[29,44,108]]
 Scripts for OIDs from 4100 to 4300; 0/201 are disabled.
-Audio table entries: 4230
+Audio table entries: 2115
+Audio table copy: Similar
 Binary tables entries: 1/1/1
 Single binary table entries: 1/1/1
 Special OIDs: 4105, 0

testsuite/expected/downloaded/Duell-der-Superquizzer.gme.info.txt

@@ -9,7 +9,7 @@ Initial registers: [1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
 Initial sounds: [[907,908],[908]]
 Scripts for OIDs from 34 to 3583; 3018/3550 are disabled.
 Audio table entries: 909
-Audio table repeated twice
+Audio table copy: Equal
 Binary tables entries: 14/14/14
 Single binary table entries: 1/1/1
 Special OIDs: 3010, 3009