Merge pull request #105984 from bruvzg/sparse_pack

Android: Implement sparse bundle PCK support.

Author: akien-mga

core/config/project_settings.cpp

@@ -614,6 +614,8 @@ void ProjectSettings::_convert_to_last_version(int p_from_version) {
  *      appending '.pck' to the binary name (e.g. 'linux_game' -> 'linux_game.pck').
  *    o PCK with the same basename as the binary in the current working directory.
  *      Same as above for the two possible PCK file names.
+ *  - On Android, look for 'assets.sparsepck' and try loading it, if it doesn't work,
+ *    proceed to the next step.
  *  - On relevant platforms (Android/iOS), lookup project file in OS resource path.
  *    If found, load it or fail.
  *  - Lookup project file in passed `p_path` (--path passed by the user), i.e. we
@@ -701,6 +703,11 @@ Error ProjectSettings::_setup(const String &p_path, const String &p_main_pack, b
 		}
 	}
 
+#ifdef ANDROID_ENABLED
+	// Attempt to load sparse PCK assets.
+	_load_resource_pack("res://assets.sparsepck", false, 0, true);
+#endif
+
 	// Try to use the filesystem for files, according to OS.
 	// (Only Android -when reading from pck- and iOS use this.)
 

core/io/file_access.cpp

@@ -159,7 +159,7 @@ Ref<FileAccess> FileAccess::open(const String &p_path, int p_mode_flags, Error *
 	//try packed data first
 
 	Ref<FileAccess> ret;
-	if (!(p_mode_flags & WRITE) && PackedData::get_singleton() && !PackedData::get_singleton()->is_disabled()) {
+	if (!(p_mode_flags & WRITE) && !(p_mode_flags & SKIP_PACK) && PackedData::get_singleton() && !PackedData::get_singleton()->is_disabled()) {
 		ret = PackedData::get_singleton()->try_open_path(p_path);
 		if (ret.is_valid()) {
 			if (r_error) {
@@ -170,7 +170,7 @@ Ref<FileAccess> FileAccess::open(const String &p_path, int p_mode_flags, Error *
 	}
 
 	ret = create_for_path(p_path);
-	Error err = ret->open_internal(p_path, p_mode_flags);
+	Error err = ret->open_internal(p_path, p_mode_flags & ~SKIP_PACK);
 
 	if (r_error) {
 		*r_error = err;

core/io/file_access.h

@@ -58,6 +58,7 @@ class FileAccess : public RefCounted {
 		WRITE = 2,
 		READ_WRITE = 3,
 		WRITE_READ = 7,
+		SKIP_PACK = 16,
 	};
 
 	enum UnixPermissionFlags : int32_t {

core/io/file_access_pack.cpp

@@ -45,14 +45,15 @@ Error PackedData::add_pack(const String &p_path, bool p_replace_files, uint64_t
 	return ERR_FILE_UNRECOGNIZED;
 }
 
-void PackedData::add_path(const String &p_pkg_path, const String &p_path, uint64_t p_ofs, uint64_t p_size, const uint8_t *p_md5, PackSource *p_src, bool p_replace_files, bool p_encrypted) {
+void PackedData::add_path(const String &p_pkg_path, const String &p_path, uint64_t p_ofs, uint64_t p_size, const uint8_t *p_md5, PackSource *p_src, bool p_replace_files, bool p_encrypted, bool p_bundle) {
 	String simplified_path = p_path.simplify_path().trim_prefix("res://");
 	PathMD5 pmd5(simplified_path.md5_buffer());
 
 	bool exists = files.has(pmd5);
 
 	PackedFile pf;
 	pf.encrypted = p_encrypted;
+	pf.bundle = p_bundle;
 	pf.pack = p_pkg_path;
 	pf.offset = p_ofs;
 	pf.size = p_size;
@@ -268,6 +269,7 @@ bool PackedSourcePCK::try_open_pack(const String &p_path, bool p_replace_files,
 	uint32_t pack_flags = f->get_32();
 	bool enc_directory = (pack_flags & PACK_DIR_ENCRYPTED);
 	bool rel_filebase = (pack_flags & PACK_REL_FILEBASE); // Note: Always enabled for V3.
+	bool sparse_bundle = (pack_flags & PACK_SPARSE_BUNDLE);
 
 	uint64_t file_base = f->get_64();
 	if ((version == PACK_FORMAT_VERSION_V3) || (version == PACK_FORMAT_VERSION_V2 && rel_filebase)) {
@@ -320,7 +322,7 @@ bool PackedSourcePCK::try_open_pack(const String &p_path, bool p_replace_files,
 		if (flags & PACK_FILE_REMOVAL) { // The file was removed.
 			PackedData::get_singleton()->remove_path(path);
 		} else {
-			PackedData::get_singleton()->add_path(p_path, path, file_base + ofs, size, md5, this, p_replace_files, (flags & PACK_FILE_ENCRYPTED));
+			PackedData::get_singleton()->add_path(p_path, path, file_base + ofs, size, md5, this, p_replace_files, (flags & PACK_FILE_ENCRYPTED), sparse_bundle);
 		}
 	}
 
@@ -360,7 +362,7 @@ void PackedSourceDirectory::add_directory(const String &p_path, bool p_replace_f
 	for (const String &file_name : da->get_files()) {
 		String file_path = p_path.path_join(file_name);
 		uint8_t md5[16] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
-		PackedData::get_singleton()->add_path(p_path, file_path, 0, 0, md5, this, p_replace_files, false);
+		PackedData::get_singleton()->add_path(p_path, file_path, 0, 0, md5, this, p_replace_files, false, false);
 	}
 
 	for (const String &sub_dir_name : da->get_directories()) {
@@ -467,13 +469,19 @@ void FileAccessPack::close() {
 	f = Ref<FileAccess>();
 }
 
-FileAccessPack::FileAccessPack(const String &p_path, const PackedData::PackedFile &p_file) :
-		pf(p_file),
-		f(FileAccess::open(pf.pack, FileAccess::READ)) {
-	ERR_FAIL_COND_MSG(f.is_null(), vformat("Can't open pack-referenced file '%s'.", String(pf.pack)));
+FileAccessPack::FileAccessPack(const String &p_path, const PackedData::PackedFile &p_file) {
+	pf = p_file;
+	if (pf.bundle) {
+		String simplified_path = p_path.simplify_path();
+		f = FileAccess::open(simplified_path, FileAccess::READ | FileAccess::SKIP_PACK);
+		off = 0; // For the sparse pack offset is always zero.
+	} else {
+		f = FileAccess::open(pf.pack, FileAccess::READ);
+		f->seek(pf.offset);
+		off = pf.offset;
+	}
 
-	f->seek(pf.offset);
-	off = pf.offset;
+	ERR_FAIL_COND_MSG(f.is_null(), vformat("Can't open pack-referenced file '%s'.", String(pf.pack)));
 
 	if (pf.encrypted) {
 		Ref<FileAccessEncrypted> fae;

core/io/file_access_pack.h

@@ -48,6 +48,7 @@
 enum PackFlags {
 	PACK_DIR_ENCRYPTED = 1 << 0,
 	PACK_REL_FILEBASE = 1 << 1,
+	PACK_SPARSE_BUNDLE = 1 << 2,
 };
 
 enum PackFileFlags {
@@ -70,6 +71,7 @@ class PackedData {
 		uint8_t md5[16];
 		PackSource *src = nullptr;
 		bool encrypted;
+		bool bundle;
 	};
 
 private:
@@ -114,7 +116,7 @@ class PackedData {
 
 public:
 	void add_pack_source(PackSource *p_source);
-	void add_path(const String &p_pkg_path, const String &p_path, uint64_t p_ofs, uint64_t p_size, const uint8_t *p_md5, PackSource *p_src, bool p_replace_files, bool p_encrypted = false); // for PackSource
+	void add_path(const String &p_pkg_path, const String &p_path, uint64_t p_ofs, uint64_t p_size, const uint8_t *p_md5, PackSource *p_src, bool p_replace_files, bool p_encrypted = false, bool p_bundle = false); // for PackSource
 	void remove_path(const String &p_path);
 	uint8_t *get_file_hash(const String &p_path);
 	HashSet<String> get_file_paths() const;