1.10.0 - virtual filesystem, better dependency resolving, not breaking ContainerDict (#186)

* MeshRenderExporter - use m_Name instead of name

* Texture2DConverter - image - error if there is no image data

* Texture2D - load image data from resource lazily

* rework dependency loading and resolving

* SerializedFile - rework container handling

* Object - use new container handling and fix NodeHelper error with tuples

* add optional wrap argument for reading typetrees

* File - fix read_files

* implement custom filesystem handling

* Update README.md

* EndianBinaryReader - fix passing of LocalFileOpenener

* Readme.md - min python 3.6.0 -> 3.7.0

* setup.py - possible legacy egg fix

* pyproject.toml - remove 3.6 support, add 3.10 and 3.11 support

Author: K0lb3

README.md

@@ -37,11 +37,12 @@ if UnityPy.__version__ != '1.9.6':
 2. [Example](#example)
 3. [Important Classes](#important-classes)
 4. [Important Object Types](#important-object-types)
-5. [Credits](#credits)
+5. [Custom Fileystem](#custom-filesystem)
+6. [Credits](#credits)
 
 ## Installation
 
-**Python 3.6.0 or higher is required**
+**Python 3.7.0 or higher is required**
 
 via pypi
 
@@ -364,6 +365,22 @@ if mesh_renderer.m_GameObject:
 mesh_renderer.export(export_dir)
 ```
 
+## Custom-Filesystem
+
+UnityPy uses [fsspec](https://github.com/fsspec/filesystem_spec) under the hood to manage all filesystem interactions.
+This allows using various different types of filesystems without having to change UnityPy's code.
+It also means that you can use your own custom filesystem to e.g. handle indirection via catalog files, load assets on demand from a server, or decrypt files.
+
+Following methods of the filesystem have to be implemented for using it in UnityPy.
+
+-   sep (not a function, just the seperator as character)
+-   isfile(self, path: str) -> bool
+-   isdir(self, path: str) -> bool
+-   exists(self, path: str, \*\*kwargs) -> bool
+-   walk(self, path: str, \*\*kwargs) -> Iterable[List[str], List[str], List[str]]
+-   open(self, path: str, mode: str = "rb", \*\*kwargs) -> file ("rb" mode required, "wt" required for ModelExporter)
+-   makedirs(self, path: str, exist_ok: bool = False) -> bool
+
 ## Credits
 
 First of all,

UnityPy/__init__.py

@@ -1,11 +1,11 @@
-__version__ = "1.9.28"
+__version__ = "1.10.0"
 
 from .environment import Environment
 from .helpers.ArchiveStorageManager import set_assetbundle_decrypt_key
 
 
-def load(*args):
-    return Environment(*args)
+def load(*args, fs=None, **kwargs):
+    return Environment(*args, fs=fs, **kwargs)
 
 
 # backward compatibility

UnityPy/classes/Object.py

@@ -5,7 +5,6 @@
 from ..files import ObjectReader
 import types
 from ..exceptions import TypeTreeError as TypeTreeError
-from .. import classes
 
 
 class Object(object):
@@ -26,11 +25,7 @@ def __init__(self, reader: ObjectReader):
         if self.platform == BuildTarget.NoTarget:
             self._object_hide_flags = reader.read_u_int()
 
-        self.container = (
-            self.assets_file._container[self.path_id]
-            if self.path_id in self.assets_file._container
-            else None
-        )
+        self.container = self.assets_file.container.path_dict.get(self.path_id)
 
         self.reader.reset()
         if type(self) == Object:
@@ -46,10 +41,10 @@ def dump_typetree(self, nodes: list = None) -> str:
     def dump_typetree_structure(self) -> str:
         return self.reader.dump_typetree_structure()
 
-    def read_typetree(self, nodes: list = None) -> dict:
+    def read_typetree(self, nodes: list = None, wrap: bool = False) -> dict:
         tree = self.reader.read_typetree(nodes)
         self.type_tree = NodeHelper(tree, self.assets_file)
-        return tree
+        return self.type_tree if wrap else tree
 
     def save_typetree(self, nodes: list = None, writer: EndianBinaryWriter = None):
         def class_to_dict(value):
@@ -145,6 +140,8 @@ def __new__(cls, data, assets_file):
             return super(NodeHelper, cls).__new__(cls)
         elif isinstance(data, list):
             return [NodeHelper(x, assets_file) for x in data]
+        elif isinstance(data, tuple):
+            return tuple(NodeHelper(x, assets_file) for x in data)
         return data
 
     def __getitem__(self, item):

UnityPy/classes/PPtr.py

@@ -1,10 +1,6 @@
 from ..files import ObjectReader
 from ..streams import EndianBinaryWriter
-from ..helpers import ImportHelper
-from .. import files
-from ..enums import FileType, ClassIDType
-import os
-from .. import environment
+from ..enums import ClassIDType
 
 
 def save_ptr(obj, writer: EndianBinaryWriter):
@@ -33,7 +29,9 @@ def save(self, writer: EndianBinaryWriter):
     def get_obj(self):
         if self._obj != None:
             return self._obj
+
         manager = None
+
         if self.file_id == 0:
             manager = self.assets_file
 
@@ -43,27 +41,13 @@ def get_obj(self):
                 external_name = self.external_name
                 # try to find it in the already registered cabs
                 manager = environment.get_cab(external_name)
-
+                # not found, load all dependencies and try again
                 if not manager:
-                    # guess we have to try to find it as file then
-                    path = environment.path
-                    if path is not None:
-                        basename = os.path.basename(external_name)
-                        possible_names = [basename, basename.lower(), basename.upper()]
-                        for root, dirs, files in os.walk(path):
-                            for name in files:
-                                if name in possible_names:
-                                    manager = environment.load_file(
-                                        os.path.join(root, name)
-                                    )
-                                    environment.register_cab(name, manager)
-                                    break
-                            else:
-                                # else is reached if the previous loop didn't break
-                                continue
-                            break
-        if manager and self.path_id in manager.objects:
-            self._obj = manager.objects[self.path_id]
+                    self.assets_file.load_dependencies([external_name])
+                    manager = environment.get_cab(external_name)
+
+        if manager is not None:
+            self._obj = manager.objects.get(self.path_id)
         else:
             self._obj = None
             if self.external_name:

UnityPy/classes/Texture2D.py

@@ -47,6 +47,13 @@ def image(self, img):
 
     @property
     def image_data(self):
+        if not self._image_data and self.m_StreamData is not None:
+            self._image_data = get_resource_data(
+                self.m_StreamData.path,
+                self.assets_file,
+                self.m_StreamData.offset,
+                self.m_StreamData.size,
+            )
         return self._image_data
 
     def reset_streamdata(self):
@@ -166,13 +173,8 @@ def __init__(self, reader):
         if version >= (5, 3):  # 5.3 and up
             # always read the StreamingInfo for resaving
             self.m_StreamData = StreamingInfo(reader, version)
-            if image_data_size == 0 and self.m_StreamData.path:
-                self._image_data = get_resource_data(
-                    self.m_StreamData.path,
-                    self.assets_file,
-                    self.m_StreamData.offset,
-                    self.m_StreamData.size,
-                )
+            # don't read the data directly,
+            # as we don't want the parser break if the file is missing
 
     def save(self, writer: EndianBinaryWriter = None):
         if writer is None: