diff --git a/UmodelTool/umodel.project b/UmodelTool/umodel.project index abea1ea1..85d70f2f 100644 --- a/UmodelTool/umodel.project +++ b/UmodelTool/umodel.project @@ -44,4 +44,4 @@ sources(MAIN) = { !message NO EXE_NAME !endif -target(executable, $EXE_NAME, MAIN + COMP_LIBS + UE4_LIBS + IMG_LIBS + NV_LIBS + MOBILE_LIBS, MAIN) +target(executable, $EXE_NAME, MAIN + COMP_LIBS + UE4_LIBS + IMG_LIBS + NV_LIBS + MOBILE_LIBS + MD5_LIBS, MAIN) diff --git a/Unreal/FileSystem/GameFileSystem.cpp b/Unreal/FileSystem/GameFileSystem.cpp index f77fbead..dcc7f3a4 100644 --- a/Unreal/FileSystem/GameFileSystem.cpp +++ b/Unreal/FileSystem/GameFileSystem.cpp @@ -1,6 +1,7 @@ #include "Core.h" #include "UnCore.h" #include "GameFileSystem.h" +#include "GameFileSystemSmite.h" #include "UnArchiveObb.h" #include "UnArchivePak.h" @@ -816,6 +817,20 @@ void appSetRootDirectory(const char *dir, bool recurse) } #endif // GEARS4 +#if SMITE + if(GForceGame == GAME_Smite) { + const CGameFileInfo* manifest = CGameFileInfo::Find("MergedFileIndexCache.bin"); + if (manifest) + { + LoadSmiteManifest(manifest); + } + else + { + appNotify("Smite: missing MergedFileIndexCache.bin file."); + } + } +#endif + appPrintf("Found %d game files (%d skipped) in %d folders at path \"%s\"\n", GameFiles.Num(), GNumForeignFiles, GameFolders.Num() ? GameFolders.Num()-1 : 0, dir); #if UNREAL4 diff --git a/Unreal/FileSystem/GameFileSystemSmite.cpp b/Unreal/FileSystem/GameFileSystemSmite.cpp new file mode 100644 index 00000000..bc6b23ec --- /dev/null +++ b/Unreal/FileSystem/GameFileSystemSmite.cpp @@ -0,0 +1,74 @@ +#include "Core.h" +#include "UnCore.h" +#include "GameFileSystem.h" +#include "GameFileSystemSmite.h" +#include + +#if SMITE +static FSmiteManifest* GSmiteManifest = NULL; + +void LoadSmiteManifest(const CGameFileInfo* info) { + guard(LoadSmiteManifest); + + appPrintf("Loading Smite manifest %s...\n", *info->GetRelativeName()); + + FArchive* loader = info->CreateReader(); + assert(loader); + loader->Game = GAME_Smite; + if(GSmiteManifest != nullptr) { + delete GSmiteManifest; + } + GSmiteManifest = new FSmiteManifest; + GSmiteManifest->Serialize(*loader); + delete loader; + + unguard; +} + + +FMemReader* GetSmiteBlob(const char* name, int name_len, int level, const char* ext) { + guard(GetSmiteBlob); + + if(GSmiteManifest == nullptr) { + return nullptr; + } + + MD5Context ctx; + md5Init(&ctx); + md5Update(&ctx, (unsigned char*)name, name_len); + md5Finalize(&ctx); + + TArray* item = GSmiteManifest->Files.Find(*reinterpret_cast(ctx.digest)); + if(item == nullptr) { + return nullptr; + } + + int i; + for(i = 0; i < item->Num(); i++) { + FSmiteFile* entry = item->GetData() + i; + if(entry->tier == level) { + FString* bulk = GSmiteManifest->BulkFiles.Find(entry->guid); + char filename[512]; + appSprintf(ARRAY_ARG(filename), "%s.%s", bulk->GetDataArray().GetData(), ext); + const CGameFileInfo* info = CGameFileInfo::Find(filename); + if(info == nullptr) { + appNotify("Smite: can't find tfc %s for %s", filename, name); + return nullptr; + } + + FArchive* Ar = info->CreateReader(); + Ar->Game = GAME_Smite; + Ar->Seek(entry->offset); + byte *data = (byte*)appMalloc(entry->blob_size); + Ar->Serialize(data, entry->blob_size); + delete Ar; + FMemReader* MemAr = new FMemReader(data, entry->blob_size); + MemAr->Game = GAME_Smite; + return MemAr; + } + } + unguard; + + return nullptr; +} +#endif // SMITE diff --git a/Unreal/FileSystem/GameFileSystemSmite.h b/Unreal/FileSystem/GameFileSystemSmite.h new file mode 100644 index 00000000..40b5a01a --- /dev/null +++ b/Unreal/FileSystem/GameFileSystemSmite.h @@ -0,0 +1,34 @@ +#include "Core.h" +#include "UnCore.h" + +#if SMITE +struct FSmiteFile { + int32 tier; + int32 size; + int32 offset; + int32 blob_size; + FGuid guid; + + friend FArchive& operator<<(FArchive &Ar, FSmiteFile &H) + { + return Ar << H.tier << H.size << H.offset << H.blob_size << H.guid; + } +}; + +struct FSmiteManifest +{ + TMap> Files; + TMap BulkFiles; + + void Serialize(FArchive& Ar) + { + Ar << Files; + Ar << BulkFiles; + } +}; + +void LoadSmiteManifest(const CGameFileInfo* info); + +FMemReader* GetSmiteBlob(const char* name, int name_len, int level, const char* ext); + +#endif // SMITE diff --git a/Unreal/UnCore.h b/Unreal/UnCore.h index f5b3c7f8..5be799cd 100644 --- a/Unreal/UnCore.h +++ b/Unreal/UnCore.h @@ -1027,6 +1027,13 @@ class FMemReader : public FArchive return DataSize; } + void Free() { + appFree((void*)DataPtr); + DataPtr = nullptr; + DataSize = 0; + ArPos = 0; + } + protected: const byte *DataPtr; int DataSize; @@ -2555,6 +2562,11 @@ void appReadCompressedChunk(FArchive &Ar, byte *Buffer, int Size, int Compressio #define BULKDATA_SeparateData 0x40 // unknown name - bulk stored in a different place in the same file #define BULKDATA_CompressedLzx 0x80 // unknown name +#if SMITE +#define BULKDATA_Unkn 0x100 // ??? on mip 1+ if in tfc +#define BULKDATA_CompressedOodle_SMITE 0x200 // oodle +#endif + #if BLADENSOUL #define BULKDATA_CompressedLzoEncr 0x100 // encrypted LZO #endif diff --git a/Unreal/UnrealMaterial/UnTexture3.cpp b/Unreal/UnrealMaterial/UnTexture3.cpp index f37080aa..4d845bac 100644 --- a/Unreal/UnrealMaterial/UnTexture3.cpp +++ b/Unreal/UnrealMaterial/UnTexture3.cpp @@ -643,6 +643,97 @@ static int GetRealTextureOffset_MH(const UTexture2D *Obj, int MipIndex) #endif // MARVEL_HEROES +#if SMITE +#include "../FileSystem/GameFileSystemSmite.h" +#include "../UnrealPackage/UnPackageUE3Reader.h" + +static bool LoadBulkTextureSMITE(const UTexture2D* texture, const TArray &MipsArray, int MipIndex, bool verbose) { + FMemReader* MemAr = nullptr; + const FTexture2DMipMap &Mip = MipsArray[MipIndex]; + + int i; + static char buf[2048]; + for(i = 0; i < 4; ++i) { + static char tmp[2048]; + texture->GetFullName(ARRAY_ARG(tmp), true, true, false); + switch(i) { + case 0: + appSprintf(ARRAY_ARG(buf), "%s", tmp); + break; + case 1: + if(texture->Package == nullptr) { + continue; + } + appSprintf(ARRAY_ARG(buf), "%s.%s", texture->Package->Name, tmp); + break; + case 2: + appSprintf(ARRAY_ARG(buf), "Textures.%s", tmp); + break; + case 3: + if(texture->Package == nullptr) { + continue; + } + appSprintf(ARRAY_ARG(buf), "%s.Textures.%s", texture->Package->Name, tmp); + break; + } + char *s = buf; + int len = 0; + if(verbose) { + appPrintf("Smite: Finding %s (Mip %d) in MergedFileIndexCache\n", buf, MipIndex); + } + while (*s) { + *s = toupper((unsigned char) *s); + len++; + s++; + } + + MemAr = GetSmiteBlob(buf, len, MipIndex, "tfc"); + if(MemAr != NULL) { + break; + } + } + + if(MemAr == NULL) { + appPrintf("Smite: unable to find %s (Mip %d) in MergedFileIndexCache\n", texture->Name, MipIndex); + return false; + } + + FCompressedChunkHeader H; + *MemAr << H; + TArray Chunks; + FCompressedChunk *Chunk = new (Chunks) FCompressedChunk; + Chunk->UncompressedOffset = 0; + Chunk->UncompressedSize = H.Sum.UncompressedSize; + Chunk->CompressedOffset = 0; + Chunk->CompressedSize = H.Sum.CompressedSize; + FByteBulkData *Bulk = const_cast(&Mip.Data); + int flags = COMPRESS_LZO; + if (Bulk->BulkDataFlags & BULKDATA_CompressedOodle_SMITE) flags = COMPRESS_OODLE; + else if (Bulk->BulkDataFlags & BULKDATA_CompressedZlib) flags = COMPRESS_ZLIB; + else if (Bulk->BulkDataFlags & BULKDATA_CompressedLzx) flags = COMPRESS_LZX; + + FUE3ArchiveReader* Ar = new FUE3ArchiveReader(MemAr, flags, Chunks); + Ar->IsFullyCompressed = true; + + if (verbose) + { + appPrintf("Reading %s mip level %d (%dx%d) from TFC\n", texture->Name, MipIndex, Mip.SizeX, Mip.SizeY); + } + + Bulk->BulkDataSizeOnDisk = H.Sum.UncompressedSize; + Bulk->ElementCount = H.Sum.UncompressedSize; + Bulk->BulkDataOffsetInFile = 0; + int backup = Bulk->BulkDataFlags; + Bulk->BulkDataFlags = 0; // wipe compression flags temporarily + Bulk->SerializeData(*Ar); + Bulk->BulkDataFlags = backup; + + MemAr->Free(); + delete Ar; + return true; +} +#endif // SMITE + bool UTexture2D::LoadBulkTexture(const TArray &MipsArray, int MipIndex, const char* tfcSuffix, bool verbose) const { @@ -656,6 +747,11 @@ bool UTexture2D::LoadBulkTexture(const TArray &MipsArray, int FStaticString bulkFileName; if (TextureFileCacheName != "None") { + #if SMITE + if(Package && Package->Game == GAME_Smite) { + return LoadBulkTextureSMITE(this, MipsArray, MipIndex, verbose); + } + #endif // TFC file is assigned bulkFileName = *TextureFileCacheName; @@ -1014,6 +1110,11 @@ bool UTexture2D::GetTextureData(CTextureData &TexData) const //?? Separate this function ? //!! * -notfc cmdline switch //!! * material viewer: support switching mip levels (for xbox decompression testing) + #if SMITE + if(Package && Package->Game == GAME_Smite) { + bulkFailed = false; + } else + #endif if (Bulk.BulkDataFlags & BULKDATA_Unused) continue; // mip level is stripped if (!(Bulk.BulkDataFlags & BULKDATA_StoreInSeparateFile)) continue; // equals to BULKDATA_PayloadAtEndOfFile for UE4 // some optimization in a case of missing bulk file diff --git a/common.project b/common.project index ab6c0981..23a14913 100644 --- a/common.project +++ b/common.project @@ -189,6 +189,10 @@ sources(UE4_LIBS) = { $R/libs/rijndael/*.c } +LIBINCLUDES += $R/libs/md5 +sources(MD5_LIBS) = { + $R/libs/md5/*.c +} #------------------------------------------------ # Project-specific options diff --git a/libs/md5/SOURCE b/libs/md5/SOURCE new file mode 100644 index 00000000..e4847cd9 --- /dev/null +++ b/libs/md5/SOURCE @@ -0,0 +1 @@ +https://github.com/Zunawe/md5-c \ No newline at end of file diff --git a/libs/md5/md5.c b/libs/md5/md5.c new file mode 100644 index 00000000..8eb6af11 --- /dev/null +++ b/libs/md5/md5.c @@ -0,0 +1,225 @@ +/* + * Derived from the RSA Data Security, Inc. MD5 Message-Digest Algorithm + * and modified slightly to be functionally identical but condensed into control structures. + */ + +#include "md5.h" + +/* + * Constants defined by the MD5 algorithm + */ +#define A 0x67452301 +#define B 0xefcdab89 +#define C 0x98badcfe +#define D 0x10325476 + +static uint32_t S[] = {7, 12, 17, 22, 7, 12, 17, 22, 7, 12, 17, 22, 7, 12, 17, 22, + 5, 9, 14, 20, 5, 9, 14, 20, 5, 9, 14, 20, 5, 9, 14, 20, + 4, 11, 16, 23, 4, 11, 16, 23, 4, 11, 16, 23, 4, 11, 16, 23, + 6, 10, 15, 21, 6, 10, 15, 21, 6, 10, 15, 21, 6, 10, 15, 21}; + +static uint32_t K[] = {0xd76aa478, 0xe8c7b756, 0x242070db, 0xc1bdceee, + 0xf57c0faf, 0x4787c62a, 0xa8304613, 0xfd469501, + 0x698098d8, 0x8b44f7af, 0xffff5bb1, 0x895cd7be, + 0x6b901122, 0xfd987193, 0xa679438e, 0x49b40821, + 0xf61e2562, 0xc040b340, 0x265e5a51, 0xe9b6c7aa, + 0xd62f105d, 0x02441453, 0xd8a1e681, 0xe7d3fbc8, + 0x21e1cde6, 0xc33707d6, 0xf4d50d87, 0x455a14ed, + 0xa9e3e905, 0xfcefa3f8, 0x676f02d9, 0x8d2a4c8a, + 0xfffa3942, 0x8771f681, 0x6d9d6122, 0xfde5380c, + 0xa4beea44, 0x4bdecfa9, 0xf6bb4b60, 0xbebfbc70, + 0x289b7ec6, 0xeaa127fa, 0xd4ef3085, 0x04881d05, + 0xd9d4d039, 0xe6db99e5, 0x1fa27cf8, 0xc4ac5665, + 0xf4292244, 0x432aff97, 0xab9423a7, 0xfc93a039, + 0x655b59c3, 0x8f0ccc92, 0xffeff47d, 0x85845dd1, + 0x6fa87e4f, 0xfe2ce6e0, 0xa3014314, 0x4e0811a1, + 0xf7537e82, 0xbd3af235, 0x2ad7d2bb, 0xeb86d391}; + +/* + * Bit-manipulation functions defined by the MD5 algorithm + */ +#define F(X, Y, Z) ((X & Y) | (~X & Z)) +#define G(X, Y, Z) ((X & Z) | (Y & ~Z)) +#define H(X, Y, Z) (X ^ Y ^ Z) +#define I(X, Y, Z) (Y ^ (X | ~Z)) + +/* + * Padding used to make the size (in bits) of the input congruent to 448 mod 512 + */ +static uint8_t PADDING[] = {0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; + +/* + * Initialize a context + */ +void md5Init(MD5Context *ctx){ + ctx->size = (uint64_t)0; + + ctx->buffer[0] = (uint32_t)A; + ctx->buffer[1] = (uint32_t)B; + ctx->buffer[2] = (uint32_t)C; + ctx->buffer[3] = (uint32_t)D; +} + +/* + * Add some amount of input to the context + * + * If the input fills out a block of 512 bits, apply the algorithm (md5Step) + * and save the result in the buffer. Also updates the overall size. + */ +void md5Update(MD5Context *ctx, uint8_t *input_buffer, size_t input_len){ + uint32_t input[16]; + unsigned int offset = ctx->size % 64; + ctx->size += (uint64_t)input_len; + + // Copy each byte in input_buffer into the next space in our context input + for(unsigned int i = 0; i < input_len; ++i){ + ctx->input[offset++] = (uint8_t)*(input_buffer + i); + + // If we've filled our context input, copy it into our local array input + // then reset the offset to 0 and fill in a new buffer. + // Every time we fill out a chunk, we run it through the algorithm + // to enable some back and forth between cpu and i/o + if(offset % 64 == 0){ + for(unsigned int j = 0; j < 16; ++j){ + // Convert to little-endian + // The local variable `input` our 512-bit chunk separated into 32-bit words + // we can use in calculations + input[j] = (uint32_t)(ctx->input[(j * 4) + 3]) << 24 | + (uint32_t)(ctx->input[(j * 4) + 2]) << 16 | + (uint32_t)(ctx->input[(j * 4) + 1]) << 8 | + (uint32_t)(ctx->input[(j * 4)]); + } + md5Step(ctx->buffer, input); + offset = 0; + } + } +} + +/* + * Pad the current input to get to 448 bytes, append the size in bits to the very end, + * and save the result of the final iteration into digest. + */ +void md5Finalize(MD5Context *ctx){ + uint32_t input[16]; + unsigned int offset = ctx->size % 64; + unsigned int padding_length = offset < 56 ? 56 - offset : (56 + 64) - offset; + + // Fill in the padding andndo the changes to size that resulted from the update + md5Update(ctx, PADDING, padding_length); + ctx->size -= (uint64_t)padding_length; + + // Do a final update (internal to this function) + // Last two 32-bit words are the two halves of the size (converted from bytes to bits) + for(unsigned int j = 0; j < 14; ++j){ + input[j] = (uint32_t)(ctx->input[(j * 4) + 3]) << 24 | + (uint32_t)(ctx->input[(j * 4) + 2]) << 16 | + (uint32_t)(ctx->input[(j * 4) + 1]) << 8 | + (uint32_t)(ctx->input[(j * 4)]); + } + input[14] = (uint32_t)(ctx->size * 8); + input[15] = (uint32_t)((ctx->size * 8) >> 32); + + md5Step(ctx->buffer, input); + + // Move the result into digest (convert from little-endian) + for(unsigned int i = 0; i < 4; ++i){ + ctx->digest[(i * 4) + 0] = (uint8_t)((ctx->buffer[i] & 0x000000FF)); + ctx->digest[(i * 4) + 1] = (uint8_t)((ctx->buffer[i] & 0x0000FF00) >> 8); + ctx->digest[(i * 4) + 2] = (uint8_t)((ctx->buffer[i] & 0x00FF0000) >> 16); + ctx->digest[(i * 4) + 3] = (uint8_t)((ctx->buffer[i] & 0xFF000000) >> 24); + } +} + +/* + * Step on 512 bits of input with the main MD5 algorithm. + */ +void md5Step(uint32_t *buffer, uint32_t *input){ + uint32_t AA = buffer[0]; + uint32_t BB = buffer[1]; + uint32_t CC = buffer[2]; + uint32_t DD = buffer[3]; + + uint32_t E; + + unsigned int j; + + for(unsigned int i = 0; i < 64; ++i){ + switch(i / 16){ + case 0: + E = F(BB, CC, DD); + j = i; + break; + case 1: + E = G(BB, CC, DD); + j = ((i * 5) + 1) % 16; + break; + case 2: + E = H(BB, CC, DD); + j = ((i * 3) + 5) % 16; + break; + default: + E = I(BB, CC, DD); + j = (i * 7) % 16; + break; + } + + uint32_t temp = DD; + DD = CC; + CC = BB; + BB = BB + rotateLeft(AA + E + K[i] + input[j], S[i]); + AA = temp; + } + + buffer[0] += AA; + buffer[1] += BB; + buffer[2] += CC; + buffer[3] += DD; +} + +/* + * Functions that will return a pointer to the hash of the provided input + */ +uint8_t* md5String(char *input){ + MD5Context ctx; + md5Init(&ctx); + md5Update(&ctx, (uint8_t *)input, strlen(input)); + md5Finalize(&ctx); + + uint8_t *result = malloc(16); + memcpy(result, ctx.digest, 16); + return result; +} + +uint8_t* md5File(FILE *file){ + char *input_buffer = malloc(1024); + size_t input_size = 0; + + MD5Context ctx; + md5Init(&ctx); + + while((input_size = fread(input_buffer, 1, 1024, file)) > 0){ + md5Update(&ctx, (uint8_t *)input_buffer, input_size); + } + + md5Finalize(&ctx); + + free(input_buffer); + + uint8_t *result = malloc(16); + memcpy(result, ctx.digest, 16); + return result; +} + +/* + * Rotates a 32-bit word left by n bits + */ +uint32_t rotateLeft(uint32_t x, uint32_t n){ + return (x << n) | (x >> (32 - n)); +} diff --git a/libs/md5/md5.h b/libs/md5/md5.h new file mode 100644 index 00000000..4589c098 --- /dev/null +++ b/libs/md5/md5.h @@ -0,0 +1,32 @@ +#include +#include +#include +#include + +typedef struct{ + uint64_t size; // Size of input in bytes + uint32_t buffer[4]; // Current accumulation of hash + uint8_t input[64]; // Input to be used in the next step + uint8_t digest[16]; // Result of algorithm +}MD5Context; + +#ifdef __cplusplus +extern "C" { +#endif +void md5Init(MD5Context *ctx); +void md5Update(MD5Context *ctx, uint8_t *input, size_t input_len); +void md5Finalize(MD5Context *ctx); +void md5Step(uint32_t *buffer, uint32_t *input); + +uint8_t* md5String(char *input); +uint8_t* md5File(FILE *file); + +uint32_t F(uint32_t X, uint32_t Y, uint32_t Z); +uint32_t G(uint32_t X, uint32_t Y, uint32_t Z); +uint32_t H(uint32_t X, uint32_t Y, uint32_t Z); +uint32_t I(uint32_t X, uint32_t Y, uint32_t Z); + +uint32_t rotateLeft(uint32_t x, uint32_t n); +#ifdef __cplusplus +} +#endif \ No newline at end of file