Download objects and load unihex from store

Author: atxi

polymer/CMakeLists.txt

@@ -34,6 +34,8 @@ if (UNIX)
   target_link_libraries(polymer PRIVATE glfw)
 elseif (WIN32)
   add_compile_definitions(WIN32_LEAN_AND_MEAN VK_USE_PLATFORM_WIN32_KHR)
+  
+  target_link_libraries(polymer PRIVATE ../vcpkg_installed/x64-windows-static/lib/tomcrypt)
 endif()
 
 target_include_directories(polymer PRIVATE ${CURL_INCLUDE_DIRS})

polymer/asset/asset_store.cpp

@@ -20,6 +20,7 @@ constexpr const char* kVersionDescriptorUrl =
     "https://piston-meta.mojang.com/v1/packages/715ccf3330885e75b205124f09f8712542cbe7e0/1.20.1.json";
 
 const HashSha1 kVersionDescriptorHash("715ccf3330885e75b205124f09f8712542cbe7e0");
+const String kResourceApi = POLY_STR("https://resources.download.minecraft.net/");
 
 static json_object_s* FindJsonObjectElement(json_object_s* obj, const char* name) {
   json_object_element_s* element = obj->start;
@@ -94,15 +95,20 @@ inline char* GetAbsolutePath(MemoryArena& arena, const String& path, const char*
   return buffer;
 }
 
-inline bool GetOrCreateFolder(Platform& platform, const char* path) {
-  if (!platform.FolderExists(path)) {
-    if (!platform.CreateFolder(path)) {
-      fprintf(stderr, "Failed to create folder '%s'\n", path);
-      return false;
-    }
-  }
+inline char* GetObjectUrl(MemoryArena& arena, HashSha1 hash) {
+  char hash_str[3] = {};
+  u8 hash_value = hash.hash[0];
+
+  sprintf(hash_str, "%02x", hash_value);
+
+  char full_hash[41];
+  hash.ToString(full_hash);
+
+  char* buffer = (char*)arena.Allocate(kResourceApi.size + sizeof(full_hash) + sizeof(hash_str) + 1);
 
-  return true;
+  sprintf(buffer, "%.*s%s/%s", (u32)kResourceApi.size, kResourceApi.data, hash_str, full_hash);
+
+  return buffer;
 }
 
 void AssetStore::Initialize() {
@@ -112,33 +118,23 @@ void AssetStore::Initialize() {
 
   // Check local store for version descriptor file
   if (HasAsset(version_info)) {
-    ProcessVersionDescriptor(version_info);
+    char* filename = GetAbsolutePath(trans_arena, path, "versions\\%s", kVersionDescriptor);
+
+    ProcessVersionDescriptor(filename);
   } else {
     net_queue.PushRequest(kVersionDescriptorUrl, this, [](NetworkRequest* request, NetworkResponse* response) {
       AssetStore* store = (AssetStore*)request->userp;
 
       char* filename = GetAbsolutePath(store->trans_arena, store->path, "versions\\%s", kVersionDescriptor);
 
       response->SaveToFile(filename);
-      AssetInfo info;
-
-      info.type = AssetType::VersionDescriptor;
-      info.exists = true;
-      strcpy(info.path, filename);
-
-      store->ProcessVersionDescriptor(info);
+      store->ProcessVersionDescriptor(filename);
     });
   }
 }
 
-void AssetStore::ProcessVersionDescriptor(const AssetInfo& info) {
-  //
-  // Load json and grab client/index
-
-  // If not client, net_queue request
-  // if not index, net_queue request
-
-  String contents = ReadEntireFile(info.path, trans_arena);
+void AssetStore::ProcessVersionDescriptor(const char* path) {
+  String contents = ReadEntireFile(path, trans_arena);
 
   json_value_s* root_value = json_parse(contents.data, contents.size);
   if (!root_value || root_value->type != json_type_object) {
@@ -215,86 +211,163 @@ void AssetStore::ProcessVersionDescriptor(const AssetInfo& info) {
         char* filename = GetAbsolutePath(store->trans_arena, store->path, "index\\%s", kVersionIndex);
 
         response->SaveToFile(filename);
+        store->ProcessIndex(filename);
+      });
+    } else {
+      char* filename = GetAbsolutePath(trans_arena, this->path, "index\\%s", kVersionIndex);
 
-        AssetInfo index_info = {};
+      ProcessIndex(filename);
+    }
+  }
+}
 
-        index_info.type = AssetType::Index;
-        strcpy(index_info.path, filename);
+void AssetStore::ProcessIndex(const char* filename) {
+  String contents = ReadEntireFile(filename, trans_arena);
 
-        store->ProcessIndex(index_info);
-      });
-    } else {
-      char* filename = GetAbsolutePath(trans_arena, path, "index\\%s", kVersionIndex);
+  json_value_s* root_value = json_parse(contents.data, contents.size);
+  if (!root_value || root_value->type != json_type_object) {
+    fprintf(stderr, "AssetStore: Failed to parse version index json.\n");
+    exit(1);
+  }
+
+  json_object_s* root = json_value_as_object(root_value);
+  json_object_s* objects = FindJsonObjectElement(root, "objects");
+
+  if (!objects) {
+    fprintf(stderr, "AssetStore: Invalid 'objects' element of version index. Expected object.\n");
+    exit(1);
+  }
+
+  json_object_element_s* object_element = objects->start;
+  while (object_element) {
+    String element_name(object_element->name->string, object_element->name->string_size);
+
+    json_object_element_s* current_obj_ele = object_element;
+
+    object_element = object_element->next;
+
+    // Skip over objects that aren't currently necessary.
+    if (poly_contains(element_name, POLY_STR("sound"))) continue;
+    if (poly_contains(element_name, POLY_STR("/lang/"))) continue;
+    if (poly_contains(element_name, POLY_STR("icons/"))) continue;
+    if (poly_contains(element_name, POLY_STR("/resourcepacks/"))) continue;
+
+    if (current_obj_ele->value->type == json_type_object) {
+      json_object_s* obj = json_value_as_object(current_obj_ele->value);
+
+      String obj_hash_str = FindJsonStringValue(obj, "hash");
+
+      if (obj_hash_str.size > 0) {
+        HashSha1 hash(obj_hash_str.data, obj_hash_str.size);
+
+        AssetInfo info = {};
+        info.type = AssetType::Object;
+        info.hash = hash;
+
+        asset_hash_map.Insert(element_name, hash);
+
+        // Check local store for item.
+        // If not found, request from server.
+        if (!HasAsset(info)) {
+          char* url = GetObjectUrl(trans_arena, hash);
 
-      strcpy(index_info.path, filename);
+          net_queue.PushRequest(url, this, [](NetworkRequest* request, NetworkResponse* response) {
+            AssetStore* store = (AssetStore*)request->userp;
 
-      ProcessIndex(index_info);
+            char* relative_name = request->url + kResourceApi.size;
+            char* filename = GetAbsolutePath(store->trans_arena, store->path, "objects\\%s", relative_name);
+
+            response->SaveToFile(filename);
+          });
+        }
+      }
     }
   }
 }
 
-void AssetStore::ProcessIndex(const AssetInfo& info) {
-  printf("TODO: Process index. (%s)\n", info.path);
+String AssetStore::LoadObject(MemoryArena& arena, String name) {
+  HashSha1* hash = asset_hash_map.Find(name);
+
+  if (hash) {
+    char hash_str[3] = {};
+    u8 hash_value = hash->hash[0];
+
+    sprintf(hash_str, "%02x", hash_value);
+
+    char* objects_folder = GetAbsolutePath(trans_arena, path, "objects");
+    if (platform.FolderExists(objects_folder)) {
+      char* hash_folder = GetAbsolutePath(trans_arena, path, "objects\\%s", hash_str);
+
+      if (platform.FolderExists(hash_folder)) {
+        char fullhash[41];
+
+        hash->ToString(fullhash);
+
+        char* filename = GetAbsolutePath(trans_arena, path, "objects\\%s\\%s", hash_str, fullhash);
+
+        return ReadEntireFile(filename, arena);
+      }
+    }
+  }
+
+  fprintf(stderr, "AssetStore::GetAsset failed to find asset with name %.*s\n", (u32)name.size, name.data);
+  return {};
 }
 
 bool AssetStore::HasAsset(AssetInfo& info) {
+  char* filename = nullptr;
+
+  ArenaSnapshot snapshot = trans_arena.GetSnapshot();
+
   switch (info.type) {
   case AssetType::Client: {
     char* versions_folder = GetAbsolutePath(trans_arena, path, "versions");
     if (!platform.FolderExists(versions_folder)) return false;
 
-    char* filename = GetAbsolutePath(trans_arena, path, "versions\\%s", kVersionJar);
-    HashSha1 existing_hash = GetFileSha1(trans_arena, filename);
-
-    return existing_hash == info.hash;
+    filename = GetAbsolutePath(trans_arena, path, "versions\\%s", kVersionJar);
   } break;
   case AssetType::VersionDescriptor: {
     char* index_folder = GetAbsolutePath(trans_arena, path, "versions");
     if (!platform.FolderExists(index_folder)) return false;
 
-    char* filename = GetAbsolutePath(trans_arena, path, "versions\\%s", kVersionDescriptor);
-    HashSha1 existing_hash = GetFileSha1(trans_arena, filename);
-
-    if (existing_hash != info.hash) {
-      return false;
-    }
-
-    strcpy(info.path, filename);
-    return true;
+    filename = GetAbsolutePath(trans_arena, path, "versions\\%s", kVersionDescriptor);
   } break;
   case AssetType::Index: {
     char* index_folder = GetAbsolutePath(trans_arena, path, "index");
     if (!platform.FolderExists(index_folder)) return false;
 
-    char* filename = GetAbsolutePath(trans_arena, path, "index\\%s", kVersionIndex);
-    HashSha1 existing_hash = GetFileSha1(trans_arena, filename);
-
-    return existing_hash == info.hash;
+    filename = GetAbsolutePath(trans_arena, path, "index\\%s", kVersionIndex);
   } break;
   case AssetType::Object: {
-    char hash_str[3] = {};
+    char minihash[3] = {};
     u8 hash_value = info.hash.hash[0];
 
-    sprintf(hash_str, "%02x", hash_value);
+    sprintf(minihash, "%02x", hash_value);
+    char fullhash[41];
+    info.hash.ToString(fullhash);
 
     char* objects_folder = GetAbsolutePath(trans_arena, path, "objects");
     if (!platform.FolderExists(objects_folder)) return false;
 
-    char* hash_folder = GetAbsolutePath(trans_arena, path, "objects\\%s", hash_str);
+    char* hash_folder = GetAbsolutePath(trans_arena, path, "objects\\%s", minihash);
     if (!platform.FolderExists(hash_folder)) return false;
 
-    char* filename =
-        GetAbsolutePath(trans_arena, path, "objects\\%s\\%.*s", hash_str, (u32)info.name.size, info.name.data);
-
-    HashSha1 existing_hash = GetFileSha1(trans_arena, filename);
-
-    return existing_hash == info.hash;
+    filename = GetAbsolutePath(trans_arena, path, "objects\\%s\\%s", minihash, fullhash);
   } break;
   default: {
   } break;
   }
 
-  return false;
+  bool exists = false;
+
+  if (filename) {
+    HashSha1 existing_hash = GetFileSha1(trans_arena, filename);
+    exists = existing_hash == info.hash;
+  }
+
+  trans_arena.Revert(snapshot);
+
+  return exists;
 }
 
 char* AssetStore::GetClientPath(MemoryArena& arena) {

polymer/asset/asset_store.h

@@ -5,6 +5,7 @@
 #include <polymer/platform/platform.h>
 #include <polymer/types.h>
 
+#include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 
@@ -39,18 +40,22 @@ struct HashSha1 {
   bool operator!=(const HashSha1& other) const {
     return !(*this == other);
   }
+
+  void ToString(char* out) {
+    for (size_t i = 0; i < sizeof(hash); ++i) {
+      sprintf(out + i * 2, "%02x", (int)hash[i]);
+    }
+
+    out[40] = 0;
+  }
 };
 
 enum class AssetType : u8 { VersionDescriptor, Index, Object, Client };
 
 struct AssetInfo {
   String name;
   HashSha1 hash;
-  char path[2048];
-  size_t size;
-
   AssetType type;
-  bool exists;
 };
 
 // Keeps a local asset store synchronized with the remote store by using the index.
@@ -64,24 +69,26 @@ struct AssetStore {
   MemoryArena& perm_arena;
   MemoryArena& trans_arena;
   NetworkQueue& net_queue;
-  HashMap<String, AssetInfo, MapStringHasher> asset_map;
+  HashMap<String, HashSha1, MapStringHasher> asset_hash_map;
   String path;
 
-  AssetStore(Platform& platform, MemoryArena& perm_arena, MemoryArena& trans_arena, NetworkQueue& net_queue,
-             String path)
-      : platform(platform), perm_arena(perm_arena), trans_arena(trans_arena), asset_map(perm_arena),
-        net_queue(net_queue), path(path) {}
+  AssetStore(Platform& platform, MemoryArena& perm_arena, MemoryArena& trans_arena, NetworkQueue& net_queue)
+      : platform(platform), perm_arena(perm_arena), trans_arena(trans_arena), asset_hash_map(perm_arena),
+        net_queue(net_queue) {
+    path = platform.GetAssetStorePath(trans_arena);
+  }
 
   void Initialize();
 
   bool HasAsset(AssetInfo& info);
-  bool StoreAsset(const AssetInfo& info);
 
   char* GetClientPath(MemoryArena& arena);
 
+  String LoadObject(MemoryArena& arena, String name);
+
 private:
-  void ProcessVersionDescriptor(const AssetInfo& info);
-  void ProcessIndex(const AssetInfo& info);
+  void ProcessVersionDescriptor(const char* path);
+  void ProcessIndex(const char* path);
 };
 
 } // namespace asset

polymer/asset/asset_system.cpp

@@ -88,7 +88,24 @@ bool AssetSystem::LoadFont(render::VulkanRenderer& renderer, MemoryArena& perm_a
 
   UnihexFont font(glyph_size_table, kGlyphPageWidth, kGlyphPageWidth, kGlyphPageCount);
 
-  if (!font.Load("unifont.hex", perm_arena, trans_arena)) {
+  String font_zip = asset_store->LoadObject(trans_arena, POLY_STR("minecraft/font/unifont.zip"));
+  ZipArchive zip = {};
+
+  if (!zip.OpenFromMemory(font_zip)) {
+    fprintf(stderr, "AssetSystem: Failed to open 'minecraft/font/unifont.zip' from memory.\n");
+    return false;
+  }
+
+  size_t unifont_size = 0;
+  char* unifont_data = zip.ReadFile(&trans_arena, "unifont_all_no_pua-15.0.06.hex", &unifont_size);
+
+  if (!unifont_data) {
+    fprintf(stderr, "AssetSystem: Failed to read 'unifont_all_no_pua-15.0.06.hex' in 'minecraft/font/unifont.zip'");
+    trans_arena.Revert(snapshot);
+    return false;
+  }
+
+  if (!font.Load(perm_arena, trans_arena, String(unifont_data, unifont_size))) {
     trans_arena.Revert(snapshot);
     return false;
   }

polymer/asset/asset_system.h

@@ -6,6 +6,7 @@
 #include <polymer/types.h>
 #include <polymer/world/block.h>
 
+#include <polymer/asset/asset_store.h>
 #include <polymer/asset/block_assets.h>
 
 namespace polymer {
@@ -23,6 +24,7 @@ struct AssetSystem {
   BlockAssets* block_assets = nullptr;
   render::TextureArray* glyph_page_texture = nullptr;
   u8* glyph_size_table = nullptr;
+  AssetStore* asset_store = nullptr;
 
   AssetSystem();