Add CRCs for duplicate detection

Author: francis-page

Analyzer/AnalyzerTool.cs

@@ -8,9 +8,9 @@ namespace UnityDataTools.Analyzer;
 
 public class AnalyzerTool
 {
-    public int Analyze(string path, string databaseName, string searchPattern, bool extractReferences)
+    public int Analyze(string path, string databaseName, string searchPattern, bool skipReferences)
     {
-        using SQLiteWriter writer = new (databaseName, extractReferences);
+        using SQLiteWriter writer = new (databaseName, skipReferences);
 
         try
         {
@@ -46,7 +46,7 @@ public int Analyze(string path, string databaseName, string searchPattern, bool
 
                     Console.Write($"\rProcessing {i * 100 / files.Length}% ({i}/{files.Length}) {file}");
 
-                    writer.WriteSerializedFile(serializedFileName, file);
+                    writer.WriteSerializedFile(serializedFileName, Path.GetDirectoryName(file) + Path.DirectorySeparatorChar);
                 }
 
                 if (archive != null)
@@ -65,7 +65,7 @@ public int Analyze(string path, string databaseName, string searchPattern, bool
                         {
                             if (node.Flags.HasFlag(ArchiveNodeFlags.SerializedFile))
                             {
-                                writer.WriteSerializedFile(node.Path, "/" + node.Path);
+                                writer.WriteSerializedFile(node.Path, "/");
                             }
                         }
                     }

…FileSystem/TypeTreeReaders/PPtrReader.cs Analyzer/PPtrAndCrcProcessor.csUnityFileSystem/TypeTreeReaders/PPtrReader.cs renamed to Analyzer/PPtrAndCrcProcessor.cs UnityFileSystem/TypeTreeReaders/PPtrReader.cs renamed to Analyzer/PPtrAndCrcProcessor.cs

@@ -1,50 +1,87 @@
 using System;
-using System.Collections;
 using System.Collections.Generic;
-using System.Linq;
 using System.Text;
+using UnityDataTools.FileSystem;
+using Force.Crc32;
 
-namespace UnityDataTools.FileSystem.TypeTreeReaders;
+namespace UnityDataTools.Analyzer;
 
 // This class is used to extract all the PPtrs in a serialized object. It executes a callback whenever a PPtr is found.
 // It provides a string representing the property path of the property (e.g. "m_MyObject.m_MyArray[2].m_PPtrProperty").
-public class PPtrReader
+public class PPtrAndCrcProcessor : IDisposable
 {
-    public delegate void CallbackDelegate(long objectId, int fileId, long pathId, string propertyPath, string propertyType);
+    public delegate int CallbackDelegate(long objectId, int fileId, long pathId, string propertyPath, string propertyType);
 
     private SerializedFile m_SerializedFile;
     private UnityFileReader m_Reader;
     private long m_Offset;
     private long m_ObjectId;
+    private uint m_Crc32;
+    private string m_Folder;
     private StringBuilder m_StringBuilder = new();
+    private byte[] m_pptrBytes = new byte[4];
 
     private CallbackDelegate m_Callback;
 
-    public PPtrReader(SerializedFile serializedFile, UnityFileReader reader,
+    private Dictionary<string, UnityFileReader> m_resourceReaders = new();
+
+    public PPtrAndCrcProcessor(SerializedFile serializedFile, UnityFileReader reader, string folder,
         CallbackDelegate callback)
     {
         m_SerializedFile = serializedFile;
         m_Reader = reader;
+        m_Folder = folder;
         m_Callback = callback;
     }
+    
+    public void Dispose()
+    {
+        foreach (var r in m_resourceReaders.Values)
+        {
+            r.Dispose();
+        }
+        
+        m_resourceReaders.Clear();
+    }
 
-    public void Process(long objectId, long offset, TypeTreeNode node)
+    private UnityFileReader GetResourceReader(string filename)
+    {
+        var slashPos = filename.LastIndexOf('/');
+        if (slashPos > 0)
+        {
+            filename = filename.Remove(0, slashPos + 1);
+        }
+        
+        if (!m_resourceReaders.TryGetValue(filename, out var reader))
+        {
+            reader = new UnityFileReader(m_Folder + filename, 4 * 1024 * 1024);
+            m_resourceReaders[filename] = reader;
+        }
+
+        return reader;
+    }
+
+    public uint Process(long objectId, long offset, TypeTreeNode node)
     {
         m_Offset = offset;
         m_ObjectId = objectId;
+        m_Crc32 = 0;
 
         foreach (var child in node.Children)
         {
             m_StringBuilder.Clear();
             m_StringBuilder.Append(child.Name);
             ProcessNode(child);
         }
+
+        return m_Crc32;
     }
 
     private void ProcessNode(TypeTreeNode node)
     {
         if (node.IsBasicType)
         {
+            m_Crc32 = m_Reader.ComputeCRC(m_Offset, node.Size, m_Crc32);
             m_Offset += node.Size;
         }
         else if (node.IsArray)
@@ -63,9 +100,57 @@ private void ProcessNode(TypeTreeNode node)
 
             ExtractPPtr(referencedType);
         }
+        else if (node.Type == "StreamingInfo")
+        {
+            if (node.Children.Count != 3)
+                throw new Exception("Invalid StreamingInfo");
+
+            var offset = node.Children[0].Size == 4 ? m_Reader.ReadInt32(m_Offset) : m_Reader.ReadInt64(m_Offset);
+            m_Offset += node.Children[0].Size;
+            
+            var size = m_Reader.ReadInt32(m_Offset);
+            m_Offset += 4;
+            
+            var stringSize = m_Reader.ReadInt32(m_Offset);
+            var filename = m_Reader.ReadString(m_Offset + 4, stringSize);
+            m_Offset += stringSize + 4;
+            m_Offset = (m_Offset + 3) & ~(3);
+
+            if (size > 0)
+            {
+                var resourceFile = GetResourceReader(filename);
+
+                m_Crc32 = resourceFile.ComputeCRC(offset, size, m_Crc32);
+            }
+        }
+        else if (node.Type == "StreamedResource")
+        {
+            if (node.Children.Count != 3)
+                throw new Exception("Invalid StreamedResource");
+            
+            var stringSize = m_Reader.ReadInt32(m_Offset);
+            var filename = m_Reader.ReadString(m_Offset + 4, stringSize);
+            m_Offset += stringSize + 4;
+            m_Offset = (m_Offset + 3) & ~(3);
+
+            var offset = m_Reader.ReadInt64(m_Offset);
+            m_Offset += 8;
+            
+            var size = (int)m_Reader.ReadInt64(m_Offset);
+            m_Offset += 8;
+
+            if (size > 0)
+            {
+                var resourceFile = GetResourceReader(filename);
+
+                m_Crc32 = resourceFile.ComputeCRC(offset, size, m_Crc32);
+            }
+        }
         else if (node.CSharpType == typeof(string))
         {
+            var prevOffset = m_Offset;
             m_Offset += m_Reader.ReadInt32(m_Offset) + 4;
+            m_Crc32 = m_Reader.ComputeCRC(prevOffset, (int)(m_Offset - prevOffset), m_Crc32);
         }
         else if (node.IsManagedReferenceRegistry)
         {
@@ -99,10 +184,12 @@ private void ProcessArray(TypeTreeNode node, bool isManagedReferenceRegistry = f
         if (dataNode.IsBasicType)
         {
             var arraySize = m_Reader.ReadInt32(m_Offset);
+            m_Crc32 = m_Reader.ComputeCRC(m_Offset, dataNode.Size * arraySize + 4, m_Crc32);
             m_Offset += dataNode.Size * arraySize + 4;
         }
         else
         {
+            m_Crc32 = m_Reader.ComputeCRC(m_Offset, 4, m_Crc32);
             var arraySize = m_Reader.ReadInt32(m_Offset);
             m_Offset += 4;
 
@@ -127,6 +214,7 @@ private void ProcessArray(TypeTreeNode node, bool isManagedReferenceRegistry = f
 
                     // First child is rid.
                     long rid = m_Reader.ReadInt64(m_Offset);
+                    m_Crc32 = m_Reader.ComputeCRC(m_Offset, 8, m_Crc32);
                     m_Offset += 8;
 
                     ProcessManagedReferenceData(dataNode.Children[1], dataNode.Children[2], rid);
@@ -142,6 +230,7 @@ private void ProcessManagedReferenceRegistry(TypeTreeNode node)
 
         // First child is version number.
         var version = m_Reader.ReadInt32(m_Offset);
+        m_Crc32 = m_Reader.ComputeCRC(m_Offset, node.Children[0].Size, m_Crc32);
         m_Offset += node.Children[0].Size;
 
         if (version == 1)
@@ -187,16 +276,19 @@ bool ProcessManagedReferenceData(TypeTreeNode refTypeNode, TypeTreeNode referenc
             throw new Exception("Invalid ReferencedManagedType");
 
         var stringSize = m_Reader.ReadInt32(m_Offset);
+        m_Crc32 = m_Reader.ComputeCRC(m_Offset, (int)(m_Offset + stringSize + 4), m_Crc32);
         var className = m_Reader.ReadString(m_Offset + 4, stringSize);
         m_Offset += stringSize + 4;
         m_Offset = (m_Offset + 3) & ~(3);
 
         stringSize = m_Reader.ReadInt32(m_Offset);
+        m_Crc32 = m_Reader.ComputeCRC(m_Offset, (int)(m_Offset + stringSize + 4), m_Crc32);
         var namespaceName = m_Reader.ReadString(m_Offset + 4, stringSize);
         m_Offset += stringSize + 4;
         m_Offset = (m_Offset + 3) & ~(3);
 
         stringSize = m_Reader.ReadInt32(m_Offset);
+        m_Crc32 = m_Reader.ComputeCRC(m_Offset, (int)(m_Offset + stringSize + 4), m_Crc32);
         var assemblyName = m_Reader.ReadString(m_Offset + 4, stringSize);
         m_Offset += stringSize + 4;
         m_Offset = (m_Offset + 3) & ~(3);
@@ -229,7 +321,12 @@ private void ExtractPPtr(string referencedType)
 
         if (fileId != 0 || pathId != 0)
         {
-            m_Callback(m_ObjectId, fileId, pathId, m_StringBuilder.ToString(), referencedType);
+            var refId = m_Callback(m_ObjectId, fileId, pathId, m_StringBuilder.ToString(), referencedType);
+            m_pptrBytes[0] = (byte)(refId >> 24);
+            m_pptrBytes[1] = (byte)(refId >> 16);
+            m_pptrBytes[2] = (byte)(refId >> 8);
+            m_pptrBytes[3] = (byte)(refId);
+            m_Crc32 = Crc32Algorithm.Append(m_Crc32, m_pptrBytes);
         }
     }
 }

Analyzer/README.md

@@ -22,11 +22,13 @@ but it can easily be adapter to use another writer type. It takes four parameter
   recursively.
 * databaseName (string): database filename, it will be overwritten if it already exists.
 * searchPattern (string): file search pattern (e.g. \*.bundle).
-* extractReferences (bool): determines if the references (PPtrs) must be extracted and saved in
-  the 'refs' table. Calling this method will create the SQLite output database and will recursively
-  process the files matching the search pattern in the provided path. It will add a row in
-  the 'objects' table for each serialized object. This table contain basic information such as the
-  size and the name of the object (if it has one).
+* skipReferences (bool): determines if the CRC calculation and references (PPtrs) extraction must
+  skipped. This is faster, but the refs table will be empty and the duplicate assets won't be
+  accurate.
+Calling this method will create the SQLite output database and will recursively
+process the files matching the search pattern in the provided path. It will add a row in
+the 'objects' table for each serialized object. This table contain basic information such as the
+size and the name of the object (if it has one).
 
 # How to use the database
 
@@ -62,10 +64,8 @@ case, Unity will include the asset in all the AssetBundles with a reference to i
 view_potential_duplicates provides the number of instances and the total size of the potentially
 duplicated assets. It also lists all the AssetBundles where the asset was found.
 
-It is important to understand that there is a lot of false positives in that view. All the objects
-having an identical name, size and type are reported as potential duplicates. For example, if
-several animated characters have a bone GameObject named "Hand_L", they will all be reported as
-potential duplicates even if they are not part of the same object.
+If the skipReferences option is used, there will be a lot of false positives in that view. Otherwise,
+it should be very accurate because CRCs are used to determine if objects are identical. 
 
 ## asset_view (AssetBundleProcessor)
 

Analyzer/Resources/Init.sql

@@ -49,7 +49,7 @@ CASE
     WHEN size >=  1024 AND size < (1024 * 1024) THEN printf('%!5.1f KB', size / 1024.0)
     WHEN size >= (1024 * 1024)  AND size < (1024 * 1024 * 1024) THEN printf('%!5.1f MB', size / 1024.0 / 1024)
     WHEN size >= (1024 * 1024 * 1024) THEN printf('%!5.1f GB', size / 1024.0 / 1024 / 1024)
-END AS pretty_size
+END AS pretty_size, o.crc32
 FROM objects o
 INNER JOIN types t ON o.type = t.id
 INNER JOIN serialized_files sf ON o.serialized_file = sf.id