Config file tracking (#819)

* Update config files to track file changes

* Remove print statement

* Test fix

* Add missing properties

Author: GregHib

cache/src/main/kotlin/world/gregs/voidps/cache/FileCache.kt

@@ -36,7 +36,6 @@ class FileCache(
     }
 
     override fun data(index: Int, archive: Int, file: Int, xtea: IntArray?): ByteArray? {
-        println("DATA ${files.size}")
         if (index >= files.size) {
             return null
         }

engine/src/main/kotlin/world/gregs/voidps/engine/data/ConfigFiles.kt

@@ -2,43 +2,109 @@ package world.gregs.voidps.engine.data
 
 import it.unimi.dsi.fastutil.objects.Object2ObjectOpenHashMap
 import it.unimi.dsi.fastutil.objects.ObjectArrayList
+import world.gregs.voidps.buffer.read.ArrayReader
+import world.gregs.voidps.buffer.write.ArrayWriter
 import world.gregs.voidps.engine.timedLoad
+import java.io.File
 import java.nio.file.Files
 import java.nio.file.NoSuchFileException
 import java.nio.file.Path
+import kotlin.io.path.exists
+import kotlin.io.path.extension
 import kotlin.io.path.isDirectory
 import kotlin.io.path.name
 import kotlin.io.path.pathString
+import kotlin.io.path.readBytes
+import kotlin.io.path.writeBytes
 
-typealias ConfigFiles = Map<String, List<String>>
+data class ConfigFiles(
+    val map: Map<String, List<String>>,
+    val cacheUpdate: Boolean = false,
+    val extensions: Set<String> = emptySet(),
+) {
+    fun list(path: String): List<String> = map.getOrDefault(path, emptyList())
 
-fun ConfigFiles.list(path: String): List<String> = getOrDefault(path, emptyList())
+    fun getValue(key: String) = map.getValue(key)
 
-fun ConfigFiles.find(path: String, type: String = "toml"): String = getOrDefault(type, emptyList()).firstOrNull { it.endsWith(path) } ?: throw NoSuchFileException("Unable to find config file '$path' in /data/ directory.")
+    fun find(path: String, type: String = "toml"): String = map.getOrDefault(type, emptyList()).firstOrNull { it.endsWith(path) } ?: throw NoSuchFileException("Unable to find config file '$path' in /data/ directory.")
+}
+
+fun tempCache(cachePath: String = Settings["storage.cache.temp.path"]): File? {
+    if (!Settings["storage.cache.temp.active", false]) {
+        return null
+    }
+    val directory = File(cachePath)
+    if (!directory.exists()) {
+        directory.mkdirs()
+    }
+    return directory
+}
 
 fun configFiles(): ConfigFiles {
     val map = Object2ObjectOpenHashMap<String, MutableList<String>>()
+    val path = Path.of(Settings["storage.data"])
+    val modified = Path.of(Settings["storage.data.modified"])
+    val lastUpdated = loadLastUpdate(modified)
+    val extensions = mutableSetOf<String>()
     timedLoad("config file paths") {
-        walkPath(map, Path.of(Settings["storage.data"]))
+        walkPath(map, path, lastUpdated, extensions)
+        map.size
     }
-    return map
+    return ConfigFiles(map, cacheChanged(lastUpdated), extensions)
 }
 
-private fun walkPath(map: MutableMap<String, MutableList<String>>, dir: Path): Int {
-    // Exclusions
+fun updateModified() {
+    val modified = Path.of(Settings["storage.data.modified"])
+    val writer = ArrayWriter(8)
+    writer.writeLong(System.currentTimeMillis())
+    modified.writeBytes(writer.toArray())
+}
+
+private fun loadLastUpdate(path: Path): Long {
+    if (path.exists()) {
+        val reader = ArrayReader(path.readBytes())
+        return reader.readLong()
+    }
+    return 0
+}
+
+private fun walkPath(
+    map: MutableMap<String, MutableList<String>>,
+    dir: Path,
+    lastUpdated: Long,
+    invalidatedExtensions: MutableSet<String>,
+) {
     val name = dir.name
     if (name == "saves" || name == "players" || name == "cache") {
-        return 0
+        return
     }
-    var count = 0
     for (path in Files.newDirectoryStream(dir)) {
         if (path.isDirectory()) {
-            count += walkPath(map, path)
-        } else {
-            val extension = path.name.substringAfter('.')
-            count++
-            map.getOrPut(extension) { ObjectArrayList() }.add(path.pathString)
+            walkPath(map, path, lastUpdated, invalidatedExtensions)
+            continue
+        }
+        val extension = path.name.substringAfter('.')
+        map.getOrPut(extension) { ObjectArrayList() }.add(path.pathString)
+
+        // Check file-type hasn't been marked as invalidated before checking the last modified time for invalidation
+        if (!invalidatedExtensions.contains(extension) && Files.getLastModifiedTime(path).toMillis() > lastUpdated) {
+            invalidatedExtensions.add(extension)
         }
     }
-    return count
 }
+
+private fun cacheChanged(
+    lastUpdated: Long,
+    dir: Path = Path.of(Settings["storage.cache.path"]),
+): Boolean {
+    for (path in Files.newDirectoryStream(dir)) {
+        if (path.isDirectory() || (!path.extension.startsWith("dat") && !path.extension.startsWith("idx"))) {
+            continue
+        }
+        val lastModified = Files.getLastModifiedTime(path).toMillis()
+        if (lastModified > lastUpdated) {
+            return true
+        }
+    }
+    return false
+}

engine/src/main/kotlin/world/gregs/voidps/engine/data/definition/MapDefinitions.kt

@@ -10,7 +10,6 @@ import world.gregs.voidps.engine.client.ui.chat.plural
 import world.gregs.voidps.engine.client.update.batch.ZoneBatchUpdates
 import world.gregs.voidps.engine.data.Settings
 import world.gregs.voidps.engine.data.configFiles
-import world.gregs.voidps.engine.data.list
 import world.gregs.voidps.engine.entity.obj.GameObjects
 import world.gregs.voidps.engine.map.collision.CollisionDecoder
 import world.gregs.voidps.engine.map.collision.Collisions

engine/src/main/kotlin/world/gregs/voidps/engine/entity/World.kt

@@ -5,8 +5,8 @@ import org.koin.core.component.KoinComponent
 import world.gregs.voidps.engine.GameLoop
 import world.gregs.voidps.engine.client.variable.VariableStore
 import world.gregs.voidps.engine.client.variable.Variables
+import world.gregs.voidps.engine.data.ConfigFiles
 import world.gregs.voidps.engine.data.Settings
-import world.gregs.voidps.engine.data.list
 import world.gregs.voidps.engine.entity.character.npc.NPCs
 import world.gregs.voidps.engine.entity.character.npc.loadNpcSpawns
 import world.gregs.voidps.engine.entity.item.floor.FloorItems
@@ -32,7 +32,7 @@ object World : Entity, VariableStore, Runnable, KoinComponent {
     val members: Boolean
         get() = Settings["world.members", false]
 
-    fun start(files: Map<String, List<String>>) {
+    fun start(files: ConfigFiles) {
         loadItemSpawns(get<FloorItems>(), get<ItemSpawns>(), files.list(Settings["spawns.items"]), get())
         loadObjectSpawns(get<GameObjects>(), files.list(Settings["spawns.objects"]), get())
         loadNpcSpawns(get<NPCs>(), files.list(Settings["spawns.npcs"]), get())

engine/src/test/kotlin/world/gregs/voidps/engine/entity/SpawnTest.kt

@@ -9,6 +9,7 @@ import world.gregs.voidps.cache.definition.data.ObjectDefinition
 import world.gregs.voidps.engine.Caller
 import world.gregs.voidps.engine.Script
 import world.gregs.voidps.engine.ScriptTest
+import world.gregs.voidps.engine.data.ConfigFiles
 import world.gregs.voidps.engine.data.definition.ObjectDefinitions
 import world.gregs.voidps.engine.entity.character.npc.NPC
 import world.gregs.voidps.engine.entity.character.player.Player
@@ -135,7 +136,7 @@ class SpawnTest {
         }
 
         override fun invoke(args: List<String>) {
-            Spawn.world(mapOf())
+            Spawn.world(ConfigFiles(mapOf()))
         }
 
         override val apis = listOf(Spawn)

game/src/main/kotlin/content/activity/penguin_hide_and_seek/PenguinHideAndSeek.kt

@@ -19,7 +19,6 @@ import world.gregs.voidps.engine.data.Settings
 import world.gregs.voidps.engine.data.configFiles
 import world.gregs.voidps.engine.data.definition.AccountDefinitions
 import world.gregs.voidps.engine.data.definition.AreaDefinitions
-import world.gregs.voidps.engine.data.find
 import world.gregs.voidps.engine.entity.World
 import world.gregs.voidps.engine.entity.character.npc.NPC
 import world.gregs.voidps.engine.entity.character.npc.NPCs

game/src/main/kotlin/content/entity/player/command/ServerCommands.kt

@@ -34,8 +34,6 @@ import world.gregs.voidps.engine.data.definition.RenderEmoteDefinitions
 import world.gregs.voidps.engine.data.definition.SoundDefinitions
 import world.gregs.voidps.engine.data.definition.SpellDefinitions
 import world.gregs.voidps.engine.data.definition.VariableDefinitions
-import world.gregs.voidps.engine.data.find
-import world.gregs.voidps.engine.data.list
 import world.gregs.voidps.engine.entity.World
 import world.gregs.voidps.engine.entity.character.npc.NPCs
 import world.gregs.voidps.engine.entity.character.npc.loadNpcSpawns

game/src/main/resources/game.properties

@@ -233,6 +233,9 @@ storage.autoSave.minutes=5
 # The base directory for all game config data
 storage.data=./data/
 
+# File storing timestamp of when config files or cache last changed
+storage.data.modified=./data/modified.dat
+
 # The directory where player save files are stored
 storage.players.path=./data/saves/
 
@@ -251,6 +254,12 @@ storage.wildcards=./data/wildcards.txt
 # The directory where game cache data is stored
 storage.cache.path=./data/cache/
 
+# Whether to use more memory to store temporary files for faster loading
+storage.cache.temp.active=true
+
+# Directory where temporary cached files are stored
+storage.cache.temp.path=./data/cache/temp/
+
 # The type of file server backend (options: internal, external)
 storage.cache.server=internal
 

game/src/test/kotlin/content/entity/player/inv/DropTest.kt

@@ -9,7 +9,6 @@ import org.junit.jupiter.api.Assertions.assertTrue
 import org.junit.jupiter.api.Test
 import world.gregs.voidps.engine.data.Settings
 import world.gregs.voidps.engine.data.configFiles
-import world.gregs.voidps.engine.data.list
 import world.gregs.voidps.engine.entity.item.Item
 import world.gregs.voidps.engine.entity.item.floor.FloorItems
 import world.gregs.voidps.engine.entity.item.floor.ItemSpawns

game/src/test/kotlin/content/quest/free/cooks_assistant/CooksAssistantTest.kt

@@ -14,7 +14,6 @@ import world.gregs.voidps.engine.client.instruction.handle.interactFloorItem
 import world.gregs.voidps.engine.client.instruction.handle.interactObject
 import world.gregs.voidps.engine.client.ui.dialogue
 import world.gregs.voidps.engine.data.Settings
-import world.gregs.voidps.engine.data.list
 import world.gregs.voidps.engine.entity.character.move.tele
 import world.gregs.voidps.engine.entity.character.player.Player
 import world.gregs.voidps.engine.entity.item.floor.loadItemSpawns