feat(filesystem): Implement access to shadowed files inside archives

Author: xezon

Core/GameEngine/Include/Common/ArchiveFileSystem.h

@@ -84,26 +84,19 @@ class ArchivedDirectoryInfo;
 class DetailedArchivedDirectoryInfo;
 class ArchivedFileInfo;
 
-typedef std::map<AsciiString, DetailedArchivedDirectoryInfo> DetailedArchivedDirectoryInfoMap;
-typedef std::map<AsciiString, ArchivedDirectoryInfo> ArchivedDirectoryInfoMap;
-typedef std::map<AsciiString, ArchivedFileInfo> ArchivedFileInfoMap;
-typedef std::map<AsciiString, ArchiveFile *> ArchiveFileMap;
-typedef std::map<AsciiString, AsciiString> ArchivedFileLocationMap; // first string is the file name, second one is the archive filename.
+typedef std::map<AsciiString, DetailedArchivedDirectoryInfo> DetailedArchivedDirectoryInfoMap; // Archived directory name to detailed archived directory info
+typedef std::map<AsciiString, ArchivedDirectoryInfo> ArchivedDirectoryInfoMap; // Archived directory name to archived directory info
+typedef std::map<AsciiString, ArchivedFileInfo> ArchivedFileInfoMap; // Archived file name to archived file info
+typedef std::map<AsciiString, ArchiveFile *> ArchiveFileMap; // Archive file name to archive data
+typedef std::multimap<AsciiString, ArchiveFile *> ArchivedFileLocationMap; // Archived file name to archive data
 
 class ArchivedDirectoryInfo
 {
 public:
-	AsciiString								m_directoryName;
-	ArchivedDirectoryInfoMap	m_directories;
-	ArchivedFileLocationMap		m_files;
-
-	void clear()
-	{
-		m_directoryName.clear();
-		m_directories.clear();
-		m_files.clear();
-	}
-
+	AsciiString								m_path; // The full path to this directory
+	AsciiString								m_directoryName; // The current directory
+	ArchivedDirectoryInfoMap	m_directories; // Contained leaf directories
+	ArchivedFileLocationMap		m_files; // Contained files
 };
 
 class DetailedArchivedDirectoryInfo
@@ -112,13 +105,6 @@ class DetailedArchivedDirectoryInfo
 	AsciiString												m_directoryName;
 	DetailedArchivedDirectoryInfoMap	m_directories;
 	ArchivedFileInfoMap								m_files;
-
-	void clear()
-	{
-		m_directoryName.clear();
-		m_directories.clear();
-		m_files.clear();
-	}
 };
 
 class ArchivedFileInfo
@@ -130,23 +116,16 @@ class ArchivedFileInfo
 	UnsignedInt m_size;
 
 	ArchivedFileInfo()
+		: m_offset(0)
+		, m_size(0)
 	{
-		clear();
-	}
-
-	void clear()
-	{
-		m_filename.clear();
-		m_archiveFilename.clear();
-		m_offset = 0;
-		m_size = 0;
 	}
 };
 
 
 class ArchiveFileSystem : public SubsystemInterface
 {
-	public:
+public:
 	ArchiveFileSystem();
 	virtual ~ArchiveFileSystem();
 
@@ -158,24 +137,38 @@ class ArchiveFileSystem : public SubsystemInterface
 	// ArchiveFile operations
 	virtual ArchiveFile*	openArchiveFile( const Char *filename ) = 0;		///< Create new or return existing Archive file from file name
 	virtual void					closeArchiveFile( const Char *filename ) = 0;		///< Close the one specified big file.
-	virtual void					closeAllArchiveFiles( void ) = 0;								///< Close all Archivefiles currently open
+	virtual void					closeAllArchiveFiles( void ) = 0;								///< Close all Archive files currently open
 
 	// File operations
-	virtual File*					openFile( const Char *filename, Int access = 0);	///< Search Archive files for specified file name and open it if found
-	virtual void					closeAllFiles( void ) = 0;									///< Close all files associated with ArchiveFiles
-	virtual Bool					doesFileExist(const Char *filename) const;		///< return true if that file exists in an archive file somewhere.
+	virtual File*					openFile( const Char *filename, Int access = 0, UnsignedInt instance = 0);	///< Search Archive files for specified file name and open it if found
+	virtual void					closeAllFiles( void ) = 0;									///< Close all files associated with Archive files
+	virtual Bool					doesFileExist(const Char *filename, UnsignedInt instance = 0) const;	///< return true if that file exists in an archive file somewhere.
 
 	void					getFileListInDirectory(const AsciiString& currentDirectory, const AsciiString& originalDirectory, const AsciiString& searchName, FilenameList &filenameList, Bool searchSubdirectories) const; ///< search the given directory for files matching the searchName (egs. *.ini, *.rep).  Possibly search subdirectories.  Scans each Archive file.
-	Bool					getFileInfo(const AsciiString& filename, FileInfo *fileInfo) const; ///< see FileSystem.h
+	Bool					getFileInfo(const AsciiString& filename, FileInfo *fileInfo, UnsignedInt instance = 0) const; ///< see FileSystem.h
 
 	virtual Bool	loadBigFilesFromDirectory(AsciiString dir, AsciiString fileMask, Bool overwrite = FALSE) = 0;
 
 	// Unprotected this for copy-protection routines
-	AsciiString						getArchiveFilenameForFile(const AsciiString& filename) const;
+	ArchiveFile* getArchiveFile(const AsciiString& filename, UnsignedInt instance = 0) const;
+
 	void loadMods( void );
 
+	ArchivedDirectoryInfo* friend_getArchivedDirectoryInfo(const Char* directory);
+
 protected:
-	virtual void					loadIntoDirectoryTree(const ArchiveFile *archiveFile, const AsciiString& archiveFilename, Bool overwrite = FALSE );	///< load the archive file's header information and apply it to the global archive directory tree.
+	struct ArchivedDirectoryInfoResult
+	{
+		ArchivedDirectoryInfoResult() : dirInfo(NULL) {}
+		Bool valid() const { return dirInfo != NULL; }
+
+		ArchivedDirectoryInfo* dirInfo;
+		AsciiString lastToken; ///< Synonymous for file name if the search directory was a file path
+	};
+
+	ArchivedDirectoryInfoResult getArchivedDirectoryInfo(const Char* directory);
+
+	virtual void loadIntoDirectoryTree(ArchiveFile *archiveFile, Bool overwrite = FALSE);	///< load the archive file's header information and apply it to the global archive directory tree.
 
 	ArchiveFileMap m_archiveFileMap;
 	ArchivedDirectoryInfo m_rootDirectory;

Core/GameEngine/Include/Common/FileSystem.h

@@ -54,6 +54,8 @@
 #include "Common/STLTypedefs.h"
 #include "Common/SubsystemInterface.h"
 
+#include <map>
+
 //----------------------------------------------------------------------------
 //           Forward References
 //----------------------------------------------------------------------------
@@ -98,7 +100,16 @@ typedef FilenameList::iterator FilenameListIter;
 #define TEST_TGA_DIR_PATH "../TestArt/"		///< .tga texture files live here
 #endif
 
+#ifndef ENABLE_FILESYSTEM_LOGGING
+#define ENABLE_FILESYSTEM_LOGGING (0)
+#endif
+
+
 struct FileInfo {
+
+	Int64 size() const { return (Int64)sizeHigh << 32 | sizeLow; }
+	Int64 timestamp() const { return (Int64)timestampHigh << 32 | timestampLow; }
+
 	Int sizeHigh;
 	Int sizeLow;
 	Int timestampHigh;
@@ -111,12 +122,13 @@ struct FileInfo {
 /**
   * FileSystem is an interface class for creating specific FileSystem objects.
   *
-	* A FileSystem object's implemenation decides what derivative of File object needs to be
+	* A FileSystem object's implementation decides what derivative of File object needs to be
 	* created when FileSystem::Open() gets called.
 	*/
+// TheSuperHackers @feature xezon 23/08/2025 Implements file instance access.
+// Can be used to access different versions of files in different archives under the same name.
+// Instance 0 refers to the top file that shadows all other files under the same name.
 //===============================
-#include <map>
-
 class FileSystem : public SubsystemInterface
 {
   FileSystem(const FileSystem&);
@@ -130,22 +142,30 @@ class FileSystem : public SubsystemInterface
 	void reset();
 	void update();
 
-	File* openFile( const Char *filename, Int access = File::NONE, size_t bufferSize = File::BUFFERSIZE );		///< opens a File interface to the specified file
-	Bool doesFileExist(const Char *filename) const;								///< returns TRUE if the file exists.  filename should have no directory.
+	File* openFile( const Char *filename, Int access = File::NONE, size_t bufferSize = File::BUFFERSIZE, UnsignedInt instance = 0 ); ///< opens a File interface to the specified file
+	Bool doesFileExist(const Char *filename, UnsignedInt instance = 0) const; ///< returns TRUE if the file exists.  filename should have no directory.
 	void getFileListInDirectory(const AsciiString& directory, const AsciiString& searchName, FilenameList &filenameList, Bool searchSubdirectories) const; ///< search the given directory for files matching the searchName (egs. *.ini, *.rep).  Possibly search subdirectories.
-	Bool getFileInfo(const AsciiString& filename, FileInfo *fileInfo) const; ///< fills in the FileInfo struct for the file given. returns TRUE if successful.
+	Bool getFileInfo(const AsciiString& filename, FileInfo *fileInfo, UnsignedInt instance = 0) const; ///< fills in the FileInfo struct for the file given. returns TRUE if successful.
 
 	Bool createDirectory(AsciiString directory); ///< create a directory of the given name.
 
 	Bool areMusicFilesOnCD();
 	void loadMusicFilesFromCD();
 	void unloadMusicFilesFromCD();
-	AsciiString normalizePath(const AsciiString& path) const;	///< normalizes a file path. The path can refer to a directory. File path must be absolute, but does not need to exist. Returns an empty string on failure.
+
+	static AsciiString normalizePath(const AsciiString& path);	///< normalizes a file path. The path can refer to a directory. File path must be absolute, but does not need to exist. Returns an empty string on failure.
 	static Bool isPathInDirectory(const AsciiString& testPath, const AsciiString& basePath);	///< determines if a file path is within a base path. Both paths must be absolute, but do not need to exist.
 
 protected:
 #if ENABLE_FILESYSTEM_EXISTENCE_CACHE
-	mutable std::map<unsigned,bool> m_fileExist;
+	struct FileExistData
+	{
+		FileExistData() : instanceExists(0), instanceDoesNotExist(~0u) {}
+		UnsignedInt instanceExists;
+		UnsignedInt instanceDoesNotExist;
+	};
+	typedef std::map<unsigned, FileExistData> FileExistMap; ///< Filename key to data
+	mutable FileExistMap m_fileExist;
 #endif
 };
 

Core/GameEngine/Include/Common/STLUtils.h

@@ -21,10 +21,44 @@
 #include <Utility/CppMacros.h>
 #include <utility>
 #include <algorithm>
+#include <map>
 
 namespace stl
 {
 
+// Convenience struct to avoid the std::pair<iterator, iterator>
+template <typename Container>
+struct range
+{
+	typedef typename Container::iterator iterator;
+
+	range(iterator b, iterator e) : begin(b), end(e) {}
+
+	iterator get() const { return begin; }
+	bool valid() const { return begin != end; }
+	ptrdiff_t distance() const { return std::distance(begin, end); }
+
+	iterator begin;
+	iterator end;
+};
+
+template <typename Container>
+struct const_range
+{
+	typedef typename Container::const_iterator iterator;
+
+	const_range(iterator b, iterator e) : begin(b), end(e) {}
+	const_range(const range<Container>& other) : begin(other.begin), end(other.end) {}
+
+	iterator get() const { return begin; }
+	bool valid() const { return begin != end; }
+	ptrdiff_t distance() const { return std::distance(begin, end); }
+
+	iterator begin;
+	iterator end;
+};
+
+
 // Finds first matching element in vector-like container and erases it.
 template <typename Container>
 bool find_and_erase(Container& container, const typename Container::value_type& value)
@@ -73,4 +107,38 @@ bool push_back_unique(Container& container, const typename Container::value_type
 	return false;
 }
 
+
+template <typename Iter>
+Iter advance_in_range(Iter first, Iter last, std::ptrdiff_t n)
+{
+	if (n <= 0)
+		return first;
+
+	const std::ptrdiff_t count = std::distance(first, last);
+
+	if (n >= count)
+		return last;
+
+	std::advance(first, n);
+	return first;
+}
+
+template <typename Key, typename Val>
+range<std::multimap<Key, Val> > get_range(std::multimap<Key, Val>& mm, const Key& key, std::ptrdiff_t n = 0)
+{
+	typedef typename std::multimap<Key, Val>::iterator Iter;
+	const std::pair<Iter, Iter> pair = mm.equal_range(key);
+	const Iter it = advance_in_range(pair.first, pair.second, n);
+	return range<std::multimap<Key, Val> >(it, pair.second);
+}
+
+template <typename Key, typename Val>
+const_range<std::multimap<Key, Val> > get_range(const std::multimap<Key, Val>& mm, const Key& key, std::ptrdiff_t n = 0)
+{
+	typedef typename std::multimap<Key, Val>::const_iterator Iter;
+	const std::pair<Iter, Iter> pair = mm.equal_range(key);
+	const Iter it = advance_in_range(pair.first, pair.second, n);
+	return const_range<std::multimap<Key, Val> >(it, pair.second);
+}
+
 } // namespace stl

Core/GameEngine/Source/Common/System/ArchiveFile.cpp

@@ -90,50 +90,46 @@ ArchiveFile::~ArchiveFile()
 }
 
 ArchiveFile::ArchiveFile()
+	: m_file(NULL)
 {
-	m_rootDirectory.clear();
 }
 
 void ArchiveFile::addFile(const AsciiString& path, const ArchivedFileInfo *fileInfo)
 {
-	AsciiString temp;
-	temp = path;
-	temp.toLower();
-	AsciiString token;
-	AsciiString debugpath;
-
 	DetailedArchivedDirectoryInfo *dirInfo = &m_rootDirectory;
 
-	temp.nextToken(&token, "\\/");
+	AsciiString token;
+	AsciiString tokenizer = path;
+	tokenizer.toLower();
+	tokenizer.nextToken(&token, "\\/");
 
-	while (token.getLength() > 0) {
-		if (dirInfo->m_directories.find(token) == dirInfo->m_directories.end())
+	while (token.getLength() > 0)
+	{
+		DetailedArchivedDirectoryInfoMap::iterator tempiter = dirInfo->m_directories.find(token);
+		if (tempiter == dirInfo->m_directories.end())
+		{
+			dirInfo = &(dirInfo->m_directories[token]);
+			dirInfo->m_directoryName = token;
+		}
+		else
 		{
-			dirInfo->m_directories[token].clear();
-			dirInfo->m_directories[token].m_directoryName = token;
+			dirInfo = &tempiter->second;
 		}
 
-		debugpath.concat(token);
-		debugpath.concat('\\');
-		dirInfo = &(dirInfo->m_directories[token]);
-		temp.nextToken(&token, "\\/");
+		tokenizer.nextToken(&token, "\\/");
 	}
 
 	dirInfo->m_files[fileInfo->m_filename] = *fileInfo;
-	//path.concat(fileInfo->m_filename);
 }
 
 void ArchiveFile::getFileListInDirectory(const AsciiString& currentDirectory, const AsciiString& originalDirectory, const AsciiString& searchName, FilenameList &filenameList, Bool searchSubdirectories) const
 {
-
-	AsciiString searchDir;
 	const DetailedArchivedDirectoryInfo *dirInfo = &m_rootDirectory;
 
-	searchDir = originalDirectory;
-	searchDir.toLower();
 	AsciiString token;
-
-	searchDir.nextToken(&token, "\\/");
+	AsciiString tokenizer = originalDirectory;
+	tokenizer.toLower();
+	tokenizer.nextToken(&token, "\\/");
 
 	while (token.getLength() > 0) {
 
@@ -148,7 +144,7 @@ void ArchiveFile::getFileListInDirectory(const AsciiString& currentDirectory, co
 			return;
 		}
 
-		searchDir.nextToken(&token, "\\/");
+		tokenizer.nextToken(&token, "\\/");
 	}
 
 	getFileListInDirectory(dirInfo, originalDirectory, searchName, filenameList, searchSubdirectories);
@@ -198,17 +194,15 @@ void ArchiveFile::attachFile(File *file)
 
 const ArchivedFileInfo * ArchiveFile::getArchivedFileInfo(const AsciiString& filename) const
 {
-	AsciiString path;
-	path = filename;
-	path.toLower();
-	AsciiString token;
-
 	const DetailedArchivedDirectoryInfo *dirInfo = &m_rootDirectory;
 
-	path.nextToken(&token, "\\/");
-
-	while ((token.find('.') == NULL) || (path.find('.') != NULL)) {
+	AsciiString token;
+	AsciiString tokenizer = filename;
+	tokenizer.toLower();
+	tokenizer.nextToken(&token, "\\/");
 
+	while (!token.find('.') || tokenizer.find('.'))
+	{
 		DetailedArchivedDirectoryInfoMap::const_iterator it = dirInfo->m_directories.find(token);
 		if (it != dirInfo->m_directories.end())
 		{
@@ -219,7 +213,7 @@ const ArchivedFileInfo * ArchiveFile::getArchivedFileInfo(const AsciiString& fil
 			return NULL;
 		}
 
-		path.nextToken(&token, "\\/");
+		tokenizer.nextToken(&token, "\\/");
 	}
 
 	ArchivedFileInfoMap::const_iterator it = dirInfo->m_files.find(token);