Optimized FFileReader

Taking into account that most of read calls will be done for sequential reads of small data.

Author: gildor2

Unreal/UnCore.h

@@ -762,14 +762,9 @@ class FFileArchive : public FArchive
 	FFileArchive(const char *Filename, unsigned InOptions);
 	virtual ~FFileArchive();
 
-	virtual void Seek(int Pos);
-	virtual void Seek64(int64 Pos);
-	virtual int Tell() const;
-	virtual int64 Tell64() const;
 	virtual int GetFileSize() const;
 //	virtual int64 GetFileSize64() const; -- implemented in derived classes
 
-	virtual bool IsEof() const;
 	virtual bool IsOpen() const;
 	virtual void Close();
 
@@ -778,12 +773,10 @@ class FFileArchive : public FArchive
 	unsigned	Options;
 	const char	*FullName;		// allocated with appStrdup
 	const char	*ShortName;		// points to FullName[N]
-	int64		FileSize;
 
 	byte*		Buffer;
 	int			BufferSize;
 	int64		BufferPos;		// position of Buffer in file
-	int64		ArPos64;
 	int64		FilePos;		// where 'f' position points to (when reading, it usually equals to 'BufferPos + BufferSize')
 
 	bool OpenFile();
@@ -811,7 +804,18 @@ class FFileReader : public FFileArchive
 
 	virtual void Serialize(void *data, int size);
 	virtual bool Open();
+	virtual void Seek(int Pos);
+	virtual void Seek64(int64 Pos);
+	virtual int Tell() const;
+	virtual int64 Tell64() const;
 	virtual int64 GetFileSize64() const;
+	virtual bool IsEof() const;
+
+protected:
+	int64		SeekPos;
+	int64		FileSize;
+	int			BufferBytesLeft;
+	int			LocalReadPos;
 };
 
 
@@ -825,11 +829,19 @@ class FFileWriter : public FFileArchive
 	virtual void Serialize(void *data, int size);
 	virtual bool Open();
 	virtual void Close();
+	virtual void Seek(int Pos);
+	virtual void Seek64(int64 Pos);
+	virtual int Tell() const;
+	virtual int64 Tell64() const;
 	virtual int64 GetFileSize64() const;
+	virtual bool IsEof() const;
 
 	static void CleanupOnError();
 
 protected:
+	int64		FileSize;
+	int64		ArPos64;
+
 	void FlushBuffer();
 };
 
@@ -2338,7 +2350,7 @@ void appReadCompressedChunk(FArchive &Ar, byte *Buffer, int Size, int Compressio
 //#define BULKDATA_Unused				0x0020		// the same value as for UE3
 #define BULKDATA_ForceInlinePayload		0x0040		// bulk data stored immediately after header
 #define BULKDATA_PayloadInSeperateFile	0x0100		// data stored in .ubulk file near the asset (UE4.12+)
-#define BULKDATA_SerializeCompressedBitWindow 0x0200 // use platform-specific compression
+#define BULKDATA_SerializeCompressedBitWindow 0x0200 // use platform-specific compression (deprecated, seems not used)
 #define BULKDATA_OptionalPayload		0x0800		// same as BULKDATA_PayloadInSeperateFile, but stored with .uptnl extension (UE4.20+)
 #define BULKDATA_Size64Bit				0x2000		// 64-bit size fields, UE4.22+
 

Unreal/UnCoreSerialize.cpp

@@ -495,11 +495,9 @@ void FArchive::Printf(const char *fmt, ...)
 FFileArchive::FFileArchive(const char *Filename, unsigned InOptions)
 :	Options(InOptions)
 ,	f(NULL)
-,	FileSize(-1)
 ,	Buffer(NULL)
 ,	BufferPos(0)
 ,	BufferSize(0)
-,	ArPos64(0)
 {
 	// process the filename
 	FullName = appStrdup(Filename);
@@ -514,40 +512,13 @@ FFileArchive::~FFileArchive()
 	appFree(const_cast<char*>(FullName));
 }
 
-void FFileArchive::Seek(int Pos)
-{
-	ArPos64 = Pos;
-}
-
-void FFileArchive::Seek64(int64 Pos)
-{
-	ArPos64 = Pos;
-}
-
-int FFileArchive::Tell() const
-{
-	guard(FFileArchive::Tell());
-	return (int)ArPos64;
-	unguard;
-}
-
-int64 FFileArchive::Tell64() const
-{
-	return ArPos64;
-}
-
 int FFileArchive::GetFileSize() const
 {
 	int64 size = GetFileSize64();
 	if (size >= (1LL << 31)) appError("GetFileSize returns 0x%llX", size);
 	return (int)size;
 }
 
-bool FFileArchive::IsEof() const
-{
-	return ArPos64 >= GetFileSize64();
-}
-
 // this function is useful only for FRO_NoOpenError mode
 bool FFileArchive::IsOpen() const
 {
@@ -570,7 +541,7 @@ bool FFileArchive::OpenFile()
 	guard(FFileArchive::OpenFile);
 	assert(!IsOpen());
 
-	ArPos64 = FilePos = 0;
+	FilePos = 0;
 	Buffer = (byte*)appMalloc(FILE_BUFFER_SIZE);
 	BufferPos = 0;
 	BufferSize = 0;
@@ -597,6 +568,10 @@ bool FFileArchive::OpenFile()
 
 FFileReader::FFileReader(const char *Filename, unsigned InOptions)
 :	FFileArchive(Filename, InOptions)
+,	SeekPos(-1)
+,	FileSize(-1)
+,	BufferBytesLeft(0)
+,	LocalReadPos(0)
 {
 	guard(FFileReader::FFileReader);
 	IsLoading = true;
@@ -615,62 +590,79 @@ void FFileReader::Serialize(void *data, int size)
 
 	assert(data);
 
-	if (ArStopper > 0 && ArPos64 + size > ArStopper)
-		appError("Serializing behind stopper (%llX+%X > %X)", ArPos64, size, ArStopper);
+	if (ArStopper > 0 && LocalReadPos + size > ArStopper - BufferPos)
+		appError("Serializing behind stopper (%llX+%X > %X)", BufferPos + LocalReadPos, size, ArStopper);
 
+	// The function is optimized for calling frequently with reading data from buffer
 	while (size > 0)
 	{
-		int64 LocalPos64 = ArPos64 - BufferPos;
-		if (LocalPos64 < 0 || LocalPos64 >= BufferSize)
+		if (BufferBytesLeft > 0)
 		{
-			// seek to desired position if needed
-			if (ArPos64 != FilePos)
+			// Use the buffer
+			byte* BufferPtr = Buffer + LocalReadPos;
+			int CanCopy = size > BufferBytesLeft ? BufferBytesLeft : size;
+			// Copy data. If we're copying 1-2-4 bytes, "special" code works faster than the case with memcpy.
+			switch (CanCopy)
 			{
-				if (fseeko64(f, ArPos64, SEEK_SET) != 0)
-					appError("Error seeking to position 0x%llX", ArPos64);
-				FilePos = ArPos64;
+			case 1:
+				*(byte*)data = *(BufferPtr);
+				break;
+			case 2:
+				*(uint16*)data = *(uint16*)BufferPtr;
+				break;
+			case 4:
+				*(uint32*)data = *(uint32*)BufferPtr;
+				break;
+			default:
+				memcpy(data, BufferPtr, CanCopy);
 			}
-			// the requested data is not in buffer
-			if (size >= FILE_BUFFER_SIZE)
+			// Advance pointers
+			BufferBytesLeft -= CanCopy;
+			data = OffsetPointer(data, CanCopy);
+			size -= CanCopy;
+			LocalReadPos += CanCopy;
+		}
+		else
+		{
+			// Buffer is empty
+			if (SeekPos >= 0)
+			{
+				// Seek to desired position
+				if (SeekPos != FilePos)
+				{
+					if (fseeko64(f, SeekPos, SEEK_SET) != 0)
+						appError("Error seeking to position 0x%llX", SeekPos);
+					FilePos = SeekPos;
+				}
+				SeekPos = -1;
+			}
+			if (size >= FILE_BUFFER_SIZE / 2)
 			{
-				// large block, read directly from file
+				// Large block, read directly to destination skipping buffer
 				int res = fread(data, size, 1, f);
 				if (res != 1)
-					appError("Unable to read %d bytes at pos=0x%llX", size, ArPos64);
+					appError("Unable to read %d bytes at pos=0x%llX", size, FilePos);
 			#if PROFILE
 				GNumSerialize++;
 				GSerializeBytes += size;
 			#endif
-				ArPos64 += size;
 				FilePos += size;
 				return;
 			}
-			// fill buffer
+			// Fill buffer
 			int ReadBytes = fread(Buffer, 1, FILE_BUFFER_SIZE, f);
 			if (ReadBytes == 0)
-				appError("Unable to read %d bytes at pos=0x%llX", 1, ArPos64);
+				appError("Unable to read %d bytes at pos=0x%llX", 1, FilePos);
 		#if PROFILE
 			GNumSerialize++;
 			GSerializeBytes += ReadBytes;
 		#endif
 			BufferPos = FilePos;
 			BufferSize = ReadBytes;
 			FilePos += ReadBytes;
-			// update LocalPos
-			LocalPos64 = ArPos64 - BufferPos;
-			assert(LocalPos64 >= 0 && LocalPos64 < BufferSize);
+			BufferBytesLeft = ReadBytes;
+			LocalReadPos = 0;
 		}
-
-		// here we have 32-bit position in buffer
-		int LocalPos = (int)LocalPos64;
-
-		// have something in buffer
-		int CanCopy = BufferSize - LocalPos;
-		if (CanCopy > size) CanCopy = size;
-		memcpy(data, Buffer + LocalPos, CanCopy);
-		data = OffsetPointer(data, CanCopy);
-		size -= CanCopy;
-		ArPos64 += CanCopy;
 	}
 
 	unguardf("File=%s", ShortName);
@@ -681,6 +673,41 @@ bool FFileReader::Open()
 	return OpenFile();
 }
 
+void FFileReader::Seek(int Pos)
+{
+	Seek64(Pos);
+}
+
+void FFileReader::Seek64(int64 Pos)
+{
+	// Check for buffer validity
+	int64 LocalPos64 = Pos - BufferPos;
+	if (LocalPos64 < 0 || LocalPos64 >= BufferSize)
+	{
+		// Outside of the buffer
+		BufferBytesLeft = 0;
+		// SeekPos will be reset to -1 after actual seek
+		SeekPos = Pos;
+	}
+	else
+	{
+		// Inside of the buffer, recompute number of bytes to the end
+		LocalReadPos = (int)LocalPos64;
+		BufferBytesLeft = BufferSize - LocalReadPos;
+	}
+}
+
+int FFileReader::Tell() const
+{
+	assert((BufferPos >> 32) == 0);
+	return (int)BufferPos + LocalReadPos;
+}
+
+int64 FFileReader::Tell64() const
+{
+	return BufferPos + LocalReadPos;
+}
+
 int64 FFileReader::GetFileSize64() const
 {
 	// lazy file size computation
@@ -700,10 +727,17 @@ int64 FFileReader::GetFileSize64() const
 	return FileSize;
 }
 
+bool FFileReader::IsEof() const
+{
+	return (BufferBytesLeft == 0) && (FilePos == GetFileSize64());
+}
+
 static TArray<FFileWriter*> GFileWriters;
 
 FFileWriter::FFileWriter(const char *Filename, unsigned InOptions)
 :	FFileArchive(Filename, InOptions)
+,	FileSize(0)
+,	ArPos64(0)
 {
 	guard(FFileWriter::FFileWriter);
 	IsLoading = false;
@@ -798,6 +832,7 @@ bool FFileWriter::Open()
 	Buffer = (byte*)appMalloc(FILE_BUFFER_SIZE);
 	BufferPos = 0;
 	BufferSize = 0;
+	ArPos64 = 0;
 	return OpenFile();
 }
 
@@ -830,11 +865,36 @@ void FFileWriter::FlushBuffer()
 	}
 }
 
+void FFileWriter::Seek(int Pos)
+{
+	ArPos64 = Pos;
+}
+
+void FFileWriter::Seek64(int64 Pos)
+{
+	ArPos64 = Pos;
+}
+
+int FFileWriter::Tell() const
+{
+	return (int)ArPos64;
+}
+
+int64 FFileWriter::Tell64() const
+{
+	return ArPos64;
+}
+
 int64 FFileWriter::GetFileSize64() const
 {
 	return max(FileSize, FilePos + BufferSize);
 }
 
+bool FFileWriter::IsEof() const
+{
+	return ArPos64 >= GetFileSize64();
+}
+
 
 /*-----------------------------------------------------------------------------
 	Dummy archive class