Adding a limited persistence system for doing serde based on level_id.

Author: MistakeNot4892

code/controllers/subsystems/atoms.dm

@@ -11,6 +11,8 @@ SUBSYSTEM_DEF(atoms)
 	var/atom_init_stage = INITIALIZATION_INSSATOMS
 	var/old_init_stage
 
+	/// A linear list of atoms that were deserialized prior to flush.
+	var/list/deserialized_atoms = list()
 	/// A non-associative list of lists, with the format list(list(atom, list(Initialize arguments))).
 	var/list/created_atoms = list()
 	/// A non-associative list of lists, with the format list(list(atom, list(LateInitialize arguments))).
@@ -29,6 +31,13 @@ SUBSYSTEM_DEF(atoms)
 
 	atom_init_stage = INITIALIZATION_INNEW_MAPLOAD
 
+	// Preload any atoms that have deserialized during the initial load process prior to flush.
+	while(length(deserialized_atoms))
+		var/atom/A = deserialized_atoms[deserialized_atoms.len]
+		deserialized_atoms.len--
+		A.Preload()
+		CHECK_TICK
+
 	var/list/mapload_arg = list(TRUE)
 
 	var/index = 1

code/controllers/subsystems/initialization/persistence.dm

@@ -1,11 +1,13 @@
 SUBSYSTEM_DEF(persistence)
 	name = "Persistence"
 	init_order = SS_INIT_MISC_LATE
-	flags = SS_NO_FIRE | SS_NEEDS_SHUTDOWN
+	flags = SS_NEEDS_SHUTDOWN
+	wait = 15 MINUTES
 
-	var/elevator_fall_path = "data/elevator_falls_tracking.txt"
-	var/elevator_fall_shifts = -1 // This is snowflake, but oh well.
-	var/list/tracking_values = list()
+	VAR_PRIVATE/elevator_fall_path = "data/elevator_falls_tracking.txt"
+	VAR_PRIVATE/elevator_fall_shifts = -1 // This is snowflake, but oh well.
+	VAR_PRIVATE/list/tracking_values = list()
+	VAR_PRIVATE/_persistent_save_running = FALSE
 
 /datum/controller/subsystem/persistence/Initialize()
 	. = ..()
@@ -34,6 +36,30 @@ SUBSYSTEM_DEF(persistence)
 		fdel(elevator_fall_path)
 	text2file("[elevator_fall_shifts]", elevator_fall_path)
 
+	// Handle level data shutdown.
+	start_persistent_level_save()
+	while(_persistent_save_running)
+		sleep(1)
+
+/datum/controller/subsystem/persistence/fire(resumed)
+	. = ..()
+	start_persistent_level_save()
+
+/datum/controller/subsystem/persistence/proc/start_persistent_level_save()
+	// Is this actually desirable? People moving around or modifying
+	// atoms across save could result in inconsistent data.
+	set waitfor = FALSE
+	if(_persistent_save_running)
+		return // debounce
+	_persistent_save_running = TRUE // used to avoid shutting down mid-write
+	try
+		for(var/z = 1 to length(SSmapping.levels_by_z))
+			var/datum/level_data/level = SSmapping.levels_by_z[z]
+			level.save_persistent_data()
+	catch(var/exception/E)
+		error("Exception when running persistent level save: [EXCEPTION_TEXT(E)]")
+	_persistent_save_running = FALSE
+
 /datum/controller/subsystem/persistence/proc/track_value(var/atom/value, var/track_type)
 
 	var/turf/T = get_turf(value)

code/controllers/subsystems/mapping.dm

@@ -149,6 +149,27 @@ SUBSYSTEM_DEF(mapping)
 
 	setup_data_for_levels(min_z = old_maxz + 1)
 
+	// Now that levels are in place, preload any associated persistent data.
+	// This is to avoid dependencies on other atoms or any other weird ordering
+	// problems like we used to get with old DMMS and SSatoms.
+	var/list/preloaded_levels = list()
+	for(var/z = 1 to length(levels_by_z))
+		var/datum/level_data/level = levels_by_z[z]
+		if(level.preload_persistent_data())
+			preloaded_levels += level
+
+	// Now actually load the serde data into the map.
+	for(var/datum/level_data/level as anything in preloaded_levels)
+		level.load_persistent_data()
+
+	// Clear our reference data for GC
+	// This might not be needed but it saves refs floating around I guess.
+	for(var/key in level_persistence_ref_map)
+		var/list/stale_data = global.level_persistence_ref_map[key]
+		stale_data.Cut()
+
+	global.level_persistence_ref_map.Cut()
+
 	// Generate turbolifts last, since away sites may have elevators to generate too.
 	for(var/obj/abstract/turbolift_spawner/turbolift as anything in turbolifts_to_initialize)
 		turbolift.build_turbolift()

code/game/atoms_serde.dm

@@ -0,0 +1,38 @@
+/atom
+	// Var for holding serde information when this atom was loaded from a persistent source.
+	var/__init_deserialization_payload
+
+// Called when an atom is being preloaded with information from deserialization.
+/atom/proc/Preload()
+	SHOULD_CALL_PARENT(TRUE)
+	SHOULD_NOT_SLEEP(TRUE)
+	if(__init_deserialization_payload)
+		try
+			PreloadData()
+		catch(var/exception/E)
+			PRINT_STACK_TRACE("Exception when preloading [type]: [E]")
+			__init_deserialization_payload = null
+	else
+		PRINT_STACK_TRACE("[type] tried to preload with no deserialization payload.")
+
+/atom/proc/PreloadData()
+	SHOULD_CALL_PARENT(TRUE)
+	SHOULD_NOT_SLEEP(TRUE)
+	for(var/data_key in __init_deserialization_payload)
+		if(data_key in vars)
+			try
+				vars[data_key] = __init_deserialization_payload[data_key]
+			catch(var/exception/E)
+				error("Failed to write [data_key] to [type] vars: [E]")
+	__init_deserialization_payload = null
+
+// Used for saving atoms via the level persistence system.
+// Returns an assoc list of var name to var value.
+// Expected format is:
+// list("atom field" = "atom value", "so on" = "so forth"))
+// If serializing an atom reference, use "\ref[atom]" as a GUID.
+// Hardcoded/non-var keys:
+// - "type" = type (or a different serde type)
+// - "location" = list(loc.x, loc.y)
+/atom/proc/Serialize()
+	return

code/modules/multiz/level_data.dm

@@ -232,6 +232,7 @@
 
 ///Prepare level for being used. Setup borders, lateral z connections, ambient lighting, atmosphere, etc..
 /datum/level_data/proc/setup_level_data(var/skip_gen = FALSE)
+
 	if(_level_setup_completed)
 		log_debug("level_data for [src], on z [level_z], had setup_level_data called more than once!")
 		return //Since we can defer setup, make sure we only setup once

code/modules/multiz/level_persistence_handler.dm

@@ -0,0 +1,8 @@
+/decl/level_persistence_handler/proc/get_data_path(location, map, level)
+	return "[location]/[map]/[level]"
+
+/decl/level_persistence_handler/proc/save_data(datum/level_data/level_data, location, map, level)
+	return // Unimplemented, so return null to indicate a failure
+
+/decl/level_persistence_handler/proc/load_data(location, map, level)
+	return // Unimplemented, so return null to indicate a failure

code/modules/multiz/level_persistence_handler_json.dm

@@ -0,0 +1,40 @@
+/decl/level_persistence_handler/json/get_data_path(location, map, level)
+	return "[..()].json"
+
+/decl/level_persistence_handler/json/save_data(datum/level_data/level_data, location, map, level)
+	var/save_file = get_data_path(location, map, level)
+	try
+		var/list/save_data = level_data.get_persistent_data()
+		if(!length(save_data))
+			return
+
+		var/write_data = json_encode(save_data)
+		var/write_file = file(save_file)
+
+		// Do a backup (at the end to avoid overwriting then throwing an exception)
+		if(fexists(save_file))
+			var/backup_contents = file2text(save_file)
+			var/backup_file = file("[save_file].backup")
+			to_file(backup_file, backup_contents)
+			// Clear old file to avoid appending data.
+			fdel(save_file)
+
+		// Finally, write out our new json.
+		to_file(write_file, write_data)
+
+	catch(var/exception/E)
+		error("Exception when saving persistent level data to [save_file]: [EXCEPTION_TEXT(E)]")
+		return null
+	return 1 // Return a non-null value just to show we didn't throw an exception.
+
+/decl/level_persistence_handler/json/load_data(location, map, level)
+	var/load_file = get_data_path(location, map, level)
+	try
+		if(fexists(load_file)) // done separately to avoid generating an error for levels with no saved data
+			var/loaded_json = safe_file2text(load_file)
+			if(loaded_json)
+				return json_decode(loaded_json) // do not cache this giant blob pls
+	catch(var/exception/E)
+		error("Exception when loading persistent level data from [load_file]: [EXCEPTION_TEXT(E)]")
+		return null
+	return 1 // Return a non-null value just to show we didn't throw an exception.

code/modules/multiz/level_persistence_serialization.dm

@@ -0,0 +1,105 @@
+var/global/list/level_persistence_ref_map = list()
+/datum/level_data
+	/// String pointing to a base directory on the filesystem for use in serde.
+	/// If set, will automatically suffix map path and level name.
+	/// Leave null to opt out of any persistence for this level.
+	var/persistent_data_location
+	/// Decl handler, mostly forcing myself to keep this general so it can be
+	// optimized with a DB or something down the track.
+	var/persistence_handler = /decl/level_persistence_handler/json
+
+/datum/level_data/proc/get_persistent_data()
+	. = list()
+	var/list/atoms_to_save = get_persistent_atoms()
+	if(!length(atoms_to_save))
+		return
+	for(var/atom/thing as anything in get_persistent_atoms())
+		var/serialized_atom = thing.Serialize()
+		if(length(serialized_atom))
+			.["\ref[thing]"] = serialized_atom
+
+// Returns a linear list of atoms that we are interested in saving.
+/datum/level_data/proc/get_persistent_atoms()
+	return
+
+// First load all the raw data into memory so every reference is populated.
+/datum/level_data/proc/preload_persistent_data()
+
+	// Don't bother if we aren't configured for it at all.
+	if(!persistence_handler || !persistent_data_location)
+		return FALSE
+
+	// Basic sanity check.
+	if(!level_id)
+		persistent_data_location = null
+		PRINT_STACK_TRACE("Level data [type] tried to initialize persistent data but had no level_id.")
+		return FALSE
+
+	// Atoms on a map are expected to be returned as an associative list with some specific text keys.
+	var/decl/level_persistence_handler/load_handler = GET_DECL(persistence_handler)
+	var/list/loaded_data = load_handler?.load_data(persistent_data_location, global.using_map.path, ckey(level_id))
+	if(islist(loaded_data) && length(loaded_data))
+		var/list/atom_map = list()
+		global.level_persistence_ref_map[level_id] = atom_map
+		for(var/uid in loaded_data)
+			atom_map[uid] = loaded_data[uid]
+		return TRUE
+	return FALSE
+
+// Now create the atoms and register them in the global map. Note that levels with no level_id
+// or no persistence handling set will not reach this proc.
+/datum/level_data/proc/load_persistent_data()
+	var/load_count = 0
+	var/list/atom_map = global.level_persistence_ref_map[level_id]
+	for(var/uid in atom_map)
+
+		var/list/atom_data = atom_map[uid]
+
+		// TODO: datum handling? Or expect atoms to create datums internally during serde?
+		var/atom_type = text2path(atom_data["type"])
+		if(!ispath(atom_type, /atom))
+			error("[level_id]: attempted to load persistent atom with invalid or non-atom type.")
+			continue
+
+		// TODO: maybe we can hook DMMS for loading individual atoms?
+		// TODO: if we can write our serde out as .TGM we won't need bespoke handling at all.
+
+		var/atom/created_atom
+		var/list/atom_coords = atom_data["location"]
+		if(atom_coords)
+			if(!islist(atom_coords) || length(atom_coords) < 2)
+				error("[level_id]: attempted to load persistent atom with malformed coordinates.")
+				continue
+			var/turf/spawn_loc = locate(atom_coords[1], atom_coords[2], level_z)
+			if(!istype(spawn_loc))
+				error("[level_id]: attempted to load persistent atom but could not find spawn loc.")
+				continue
+			if(ispath(atom_type, /turf))
+				created_atom = spawn_loc.ChangeTurf(atom_type)
+			else if(ispath(atom_type, /area))
+				// VERY BAD, TODO
+				ChangeArea(spawn_loc, new atom_type)
+				created_atom = get_area(spawn_loc)
+			else
+				created_atom = new atom_type(spawn_loc)
+
+		else if(ispath(atom_type, /turf) || ispath(atom_type, /area))
+			error("[level_id]: attempted to load persistent turf or area with no corresponding coordinate.")
+			continue
+		else
+			created_atom = new atom_type
+
+		load_count++
+		created_atom.__init_deserialization_payload = atom_data["data"]
+		atom_map[uid] = created_atom
+		SSatoms.deserialized_atoms += created_atom
+
+	to_world_log("[level_id]/[name] loaded [load_count] persistent atom\s.")
+	return !!load_count
+
+// Write any data out if we need to.
+/datum/level_data/proc/save_persistent_data()
+	// TODO: block any changes to persistent data structures while save is running?
+	if(persistence_handler && persistent_data_location && level_id)
+		var/decl/level_persistence_handler/save_handler = GET_DECL(persistence_handler)
+		save_handler?.save_data(src, persistent_data_location, global.using_map.path, ckey(level_id))

maps/tradeship/tradeship_levels.dm

@@ -12,6 +12,7 @@
 
 /datum/level_data/main_level/tradeship
 	abstract_type = /datum/level_data/main_level/tradeship
+	persistent_data_location = "data/level_data"
 
 /datum/level_data/main_level/tradeship/basement
 	name = "Tradeship Basement Deck"

nebula.dme

@@ -756,6 +756,7 @@
 #include "code\game\atoms_movable_grabs.dm"
 #include "code\game\atoms_movable_interactions.dm"
 #include "code\game\atoms_movable_overlay.dm"
+#include "code\game\atoms_serde.dm"
 #include "code\game\atoms_temperature.dm"
 #include "code\game\base_turf.dm"
 #include "code\game\movietitles.dm"
@@ -3239,6 +3240,9 @@
 #include "code\modules\multiz\hoist.dm"
 #include "code\modules\multiz\ladder.dm"
 #include "code\modules\multiz\level_data.dm"
+#include "code\modules\multiz\level_persistence_handler.dm"
+#include "code\modules\multiz\level_persistence_handler_json.dm"
+#include "code\modules\multiz\level_persistence_serialization.dm"
 #include "code\modules\multiz\map_data.dm"
 #include "code\modules\multiz\mobile_ladder.dm"
 #include "code\modules\multiz\movement.dm"