Implementing z-level persistence.

Author: MistakeNot4892

code/__defines/misc.dm

@@ -87,27 +87,28 @@
 #define EVENT_LEVEL_MAJOR    3
 
 //Area flags, possibly more to come
-#define AREA_FLAG_RAD_SHIELDED         BITFLAG(1)  // Shielded from radiation, clearly.
-#define AREA_FLAG_EXTERNAL             BITFLAG(2)  // External as in exposed to space, not outside in a nice, green, forest.
-#define AREA_FLAG_ION_SHIELDED         BITFLAG(3)  // Shielded from ionospheric anomalies.
-#define AREA_FLAG_IS_NOT_PERSISTENT    BITFLAG(4)  // SSpersistence will not track values from this area.
-#define AREA_FLAG_IS_BACKGROUND        BITFLAG(5)  // Blueprints can create areas on top of these areas. Cannot edit the name of or delete these areas.
-#define AREA_FLAG_MAINTENANCE          BITFLAG(6)  // Area is a maintenance area.
-#define AREA_FLAG_SHUTTLE              BITFLAG(7)  // Area is a shuttle area.
-#define AREA_FLAG_HALLWAY              BITFLAG(8)  // Area is a public hallway suitable for event selection
-#define AREA_FLAG_PRISON               BITFLAG(9)  // Area is a prison for the purposes of brigging objectives.
-#define AREA_FLAG_HOLY                 BITFLAG(10) // Area is holy for the purposes of marking turfs as cult-resistant.
-#define AREA_FLAG_SECURITY             BITFLAG(11) // Area is security for the purposes of newscaster init.
-#define AREA_FLAG_HIDE_FROM_HOLOMAP    BITFLAG(12) // if we shouldn't be drawn on station holomaps
+#define AREA_FLAG_RAD_SHIELDED            BITFLAG(1)  // Shielded from radiation, clearly.
+#define AREA_FLAG_EXTERNAL                BITFLAG(2)  // External as in exposed to space, not outside in a nice, green, forest.
+#define AREA_FLAG_ION_SHIELDED            BITFLAG(3)  // Shielded from ionospheric anomalies.
+#define AREA_FLAG_IS_NOT_PERSISTENT       BITFLAG(4)  // SSpersistence will not track values from this area.
+#define AREA_FLAG_IS_BACKGROUND           BITFLAG(5)  // Blueprints can create areas on top of these areas. Cannot edit the name of or delete these areas.
+#define AREA_FLAG_MAINTENANCE             BITFLAG(6)  // Area is a maintenance area.
+#define AREA_FLAG_SHUTTLE                 BITFLAG(7)  // Area is a shuttle area.
+#define AREA_FLAG_HALLWAY                 BITFLAG(8)  // Area is a public hallway suitable for event selection
+#define AREA_FLAG_PRISON                  BITFLAG(9)  // Area is a prison for the purposes of brigging objectives.
+#define AREA_FLAG_HOLY                    BITFLAG(10) // Area is holy for the purposes of marking turfs as cult-resistant.
+#define AREA_FLAG_SECURITY                BITFLAG(11) // Area is security for the purposes of newscaster init.
+#define AREA_FLAG_HIDE_FROM_HOLOMAP       BITFLAG(12) // if we shouldn't be drawn on station holomaps
+#define AREA_FLAG_ALLOW_LEVEL_PERSISTENCE BITFLAG(13) // Whether or not this area should pass changed turfs to SSpersistence.
 
 //Map template flags
-#define TEMPLATE_FLAG_ALLOW_DUPLICATES   BITFLAG(0)  // Lets multiple copies of the template to be spawned
-#define TEMPLATE_FLAG_SPAWN_GUARANTEED   BITFLAG(1)  // Makes it ignore away site budget and just spawn (only for away sites)
-#define TEMPLATE_FLAG_CLEAR_CONTENTS     BITFLAG(2)  // if it should destroy objects it spawns on top of
-#define TEMPLATE_FLAG_NO_RUINS           BITFLAG(3)  // if it should forbid ruins from spawning on top of it
-#define TEMPLATE_FLAG_NO_RADS            BITFLAG(4)  // Removes all radiation from the template after spawning.
-#define TEMPLATE_FLAG_TEST_DUPLICATES    BITFLAG(5)  // Makes unit testing attempt to spawn mutliple copies of this template. Assumes unit testing is spawning at least one copy.
-#define TEMPLATE_FLAG_GENERIC_REPEATABLE BITFLAG(6) // Template can be picked repeatedly for the same level gen run.
+#define TEMPLATE_FLAG_ALLOW_DUPLICATES    BITFLAG(0)  // Lets multiple copies of the template to be spawned
+#define TEMPLATE_FLAG_SPAWN_GUARANTEED    BITFLAG(1)  // Makes it ignore away site budget and just spawn (only for away sites)
+#define TEMPLATE_FLAG_CLEAR_CONTENTS      BITFLAG(2)  // if it should destroy objects it spawns on top of
+#define TEMPLATE_FLAG_NO_RUINS            BITFLAG(3)  // if it should forbid ruins from spawning on top of it
+#define TEMPLATE_FLAG_NO_RADS             BITFLAG(4)  // Removes all radiation from the template after spawning.
+#define TEMPLATE_FLAG_TEST_DUPLICATES     BITFLAG(5)  // Makes unit testing attempt to spawn mutliple copies of this template. Assumes unit testing is spawning at least one copy.
+#define TEMPLATE_FLAG_GENERIC_REPEATABLE  BITFLAG(6) // Template can be picked repeatedly for the same level gen run.
 
 // Convoluted setup so defines can be supplied by Bay12 main server compile script.
 // Should still work fine for people jamming the icons into their repo.

code/__defines/persistence.dm

@@ -0,0 +1,13 @@
+#define SERDE_TYPE       "type"
+#define SERDE_LOC        "loc"
+
+#define SERDE_NAME       "name"
+#define SERDE_DESC       "desc"
+#define SERDE_COLOR      "color"
+#define SERDE_ICON_STATE "icon_state"
+
+// Handled elsewhere, do not let them load like vars.
+var/global/list/_forbid_field_load = list(
+	(SERDE_TYPE) = TRUE,
+	(SERDE_LOC)  = TRUE
+)

code/__defines/serde.dm

@@ -0,0 +1,2 @@
+#define SERDE_HINT_FINISHED 1
+#define SERDE_HINT_POSTINIT 2

code/controllers/subsystems/atoms.dm

@@ -11,6 +11,8 @@ SUBSYSTEM_DEF(atoms)
 	var/atom_init_stage = INITIALIZATION_INSSATOMS
 	var/old_init_stage
 
+	/// A linear list of atoms that were deserialized prior to flush.
+	var/list/deserialized_atoms = list()
 	/// A non-associative list of lists, with the format list(list(atom, list(Initialize arguments))).
 	var/list/created_atoms = list()
 	/// A non-associative list of lists, with the format list(list(atom, list(LateInitialize arguments))).
@@ -29,9 +31,21 @@ SUBSYSTEM_DEF(atoms)
 
 	atom_init_stage = INITIALIZATION_INNEW_MAPLOAD
 
-	var/list/mapload_arg = list(TRUE)
-
+	// Preload any atoms that have deserialized during the initial load process prior to flush.
 	var/index = 1
+	var/list/postinit_serde_atoms = list()
+	if(length(deserialized_atoms))
+		while(index <= length(deserialized_atoms))
+			var/atom/A = deserialized_atoms[index++]
+			var/res = A.Preload()
+			if(res == SERDE_HINT_POSTINIT)
+				postinit_serde_atoms += res
+			CHECK_TICK
+		deserialized_atoms.Cut()
+		report_progress("Deserialized [index-1] atom\s.")
+		index = 1
+
+	var/list/mapload_arg = list(TRUE)
 	// Things can add to the end of this list while we iterate, so we can't use a for loop.
 	while(index <= length(created_atoms))
 		// Don't remove from this list while we run, that's expensive.
@@ -49,10 +63,10 @@ SUBSYSTEM_DEF(atoms)
 			else
 				InitAtom(A, mapload_arg)
 			CHECK_TICK
-
-	report_progress("Initialized [index] atom\s")
 	created_atoms.Cut()
 
+	report_progress("Initialized [index-1] atom\s.")
+
 	atom_init_stage = INITIALIZATION_INNEW_REGULAR
 
 	if(length(late_loaders))
@@ -65,6 +79,14 @@ SUBSYSTEM_DEF(atoms)
 		report_progress("Late initialized [index] atom\s")
 		late_loaders.Cut()
 
+	if(length(postinit_serde_atoms))
+		index = 1
+		while(index <= length(postinit_serde_atoms))
+			var/atom/A = postinit_serde_atoms[index++]
+			A.PostInitLoad()
+			CHECK_TICK
+		postinit_serde_atoms.Cut()
+
 /datum/controller/subsystem/atoms/proc/InitAtom(atom/A, list/arguments)
 	var/the_type = A.type
 	if(QDELING(A))

code/controllers/subsystems/mapping.dm

@@ -149,6 +149,27 @@ SUBSYSTEM_DEF(mapping)
 
 	setup_data_for_levels(min_z = old_maxz + 1)
 
+	// Now that levels are in place, preload any associated persistent data.
+	// This is to avoid dependencies on other atoms or any other weird ordering
+	// problems like we used to get with old DMMS and SSatoms.
+	var/list/preloaded_levels = list()
+	for(var/z = 1 to length(levels_by_z))
+		var/datum/level_data/level = levels_by_z[z]
+		if(level.preload_persistent_data())
+			preloaded_levels += level
+
+	// Now actually load the serde data into the map.
+	for(var/datum/level_data/level as anything in preloaded_levels)
+		level.load_persistent_data()
+
+	// Clear our reference data for GC
+	// This might not be needed but it saves refs floating around I guess.
+	for(var/key in level_persistence_ref_map)
+		var/list/stale_data = global.level_persistence_ref_map[key]
+		stale_data.Cut()
+
+	global.level_persistence_ref_map.Cut()
+
 	// Generate turbolifts last, since away sites may have elevators to generate too.
 	for(var/obj/abstract/turbolift_spawner/turbolift as anything in turbolifts_to_initialize)
 		turbolift.build_turbolift()

…subsystems/initialization/persistence.dm …de/controllers/subsystems/persistence.dmcode/controllers/subsystems/initialization/persistence.dm renamed to code/controllers/subsystems/persistence.dm code/controllers/subsystems/initialization/persistence.dm renamed to code/controllers/subsystems/persistence.dm

@@ -1,19 +1,34 @@
+/datum/admins/proc/force_persistence_save_verb()
+	set name = "Force Early Level Save"
+	set category = "Admin"
+	set desc = "Forces an early level save run by SSpersistence."
+	if(!SSpersistence)
+		return
+	if(UNLINT(SSpersistence._persistent_save_running))
+		to_chat(usr, SPAN_WARNING("There is already a level save running. Please wait for it to finish."))
+		return
+	log_admin("[key_name(usr)] has started an early level save.")
+	message_admins("[key_name(usr)] has started an early level save.")
+	SSpersistence.start_persistent_level_save()
+
 SUBSYSTEM_DEF(persistence)
 	name = "Persistence"
 	init_order = SS_INIT_MISC_LATE
-	flags = SS_NO_FIRE | SS_NEEDS_SHUTDOWN
+	flags = SS_NEEDS_SHUTDOWN
+	wait = 60 MINUTES
 
-	var/elevator_fall_path = "data/elevator_falls_tracking.txt"
+	VAR_PRIVATE/const/ELEVATOR_FALL_PATH = "data/elevator_falls_tracking.txt"
 	var/elevator_fall_shifts = -1 // This is snowflake, but oh well.
 	var/list/tracking_values = list()
+	VAR_PRIVATE/_persistent_save_running = FALSE
 
 /datum/controller/subsystem/persistence/Initialize()
 	. = ..()
 
 	decls_repository.get_decls_of_subtype(/decl/persistence_handler) // Initialize()s persistence categories.
 
 	// Begin snowflake.
-	var/elevator_file = safe_file2text(elevator_fall_path, FALSE)
+	var/elevator_file = safe_file2text(ELEVATOR_FALL_PATH, FALSE)
 	if(elevator_file)
 		elevator_fall_shifts = text2num(elevator_file)
 	else
@@ -30,9 +45,37 @@ SUBSYSTEM_DEF(persistence)
 		P.Shutdown()
 
 	// Refer to snowflake above.
-	if(fexists(elevator_fall_path))
-		fdel(elevator_fall_path)
-	text2file("[elevator_fall_shifts]", elevator_fall_path)
+	if(fexists(ELEVATOR_FALL_PATH))
+		fdel(ELEVATOR_FALL_PATH)
+	text2file("[elevator_fall_shifts]", ELEVATOR_FALL_PATH)
+
+	// Handle level data shutdown.
+	start_persistent_level_save()
+	while(_persistent_save_running)
+		sleep(1)
+
+/datum/controller/subsystem/persistence/fire(resumed)
+	start_persistent_level_save()
+
+/datum/controller/subsystem/persistence/proc/start_persistent_level_save()
+	set waitfor = FALSE
+	if(_persistent_save_running)
+		return // debounce
+	_persistent_save_running = TRUE // used to avoid shutting down mid-write
+
+	var/started_run = world.time
+	report_progress("Starting persistent level save.")
+	// TODO: suspend all subsystems while the save is running
+	// TODO: prevent player input somehow?
+	try
+		for(var/z = 1 to length(SSmapping.levels_by_z))
+			var/datum/level_data/level = SSmapping.levels_by_z[z]
+			level.save_persistent_data()
+	catch(var/exception/E)
+		error("Exception when running persistent level save: [EXCEPTION_TEXT(E)]")
+	// TODO: re-enable all subsystems
+	report_progress("Persistent level save finished in [(world.time-started_run)/10] second\s.")
+	_persistent_save_running = FALSE
 
 /datum/controller/subsystem/persistence/proc/track_value(var/atom/value, var/track_type)
 
@@ -45,7 +88,7 @@ SUBSYSTEM_DEF(persistence)
 		return
 
 	var/datum/level_data/level = SSmapping.levels_by_z[T.z]
-	if(!istype(level) || !level.permit_persistence)
+	if(!istype(level) || !level.permit_legacy_persistence)
 		return
 
 	if(!tracking_values[track_type])

code/datums/datum.dm

@@ -9,6 +9,8 @@
 	var/list/active_timers
 	/// Used to avoid unnecessary refstring creation in Destroy().
 	var/tmp/has_state_machine = FALSE
+	/// Var for holding a unique-to-this-run identifier for a serialized datum.
+	VAR_PRIVATE/__run_uid
 
 #ifdef REFTRACKING_ENABLED
 	var/tmp/running_find_references
@@ -114,3 +116,22 @@
  */
 /datum/proc/PopulateClone(var/datum/clone)
 	return clone
+
+// Used for saving instances via the level persistence system.
+// Returns an assoc list of var name to var value.
+// Expected format is:
+// list("field" = "value", "so on" = "so forth"))
+// If serializing an instance reference, use get_run_uid() to get a UID.
+// Hardcoded/non-var keys:
+// - (SERDE_TYPE) = type (or a different serde type)
+// - (SERDE_LOC)  = list(loc.x, loc.y)
+/datum/proc/Serialize()
+	SHOULD_CALL_PARENT(TRUE)
+	. = list((SERDE_TYPE) = type)
+
+// Returns a UID for this instance, used for serde across rounds.
+// Probably-kind-of a GUID but only for this run.
+/datum/proc/get_run_uid()
+	if(isnull(__run_uid))
+		__run_uid = "\ref[src]-[sequential_id(type)]" // Staple seq_id on there in case of \ref reuse.
+	return __run_uid

code/datums/repositories/decls.dm

@@ -20,6 +20,17 @@
 
 var/global/repository/decls/decls_repository = new
 
+/proc/decls_are_equivalent(decl/first, decl/second)
+	first = RESOLVE_TO_DECL(first)
+	second = RESOLVE_TO_DECL(second)
+	return second == first
+
+/proc/resolve_decl_uid_list(list/decl_uids)
+	for(var/uid in decl_uids)
+		var/decl/decl = decls_repository.get_decl_by_id(uid)
+		if(istype(decl))
+			LAZYADD(., decl)
+
 /repository/decls
 	var/list/fetched_decls =                 list()
 	var/list/fetched_decl_ids =              list()

code/game/atoms_serde.dm

@@ -0,0 +1,47 @@
+/atom
+	/// Var for holding serde information when this atom was loaded from a persistent source.
+	var/__init_deserialization_payload
+
+// Called when an atom is being preloaded with information from deserialization.
+/atom/proc/Preload()
+	SHOULD_CALL_PARENT(TRUE)
+	SHOULD_NOT_SLEEP(TRUE)
+	if(__init_deserialization_payload)
+		try
+			. = Deserialize()
+		catch(var/exception/E)
+			PRINT_STACK_TRACE("Exception when deserializing [type]: [E]")
+		__init_deserialization_payload = null
+	else
+		PRINT_STACK_TRACE("[type] tried to preload with no deserialization payload.")
+
+/atom/proc/PreloadKey(data_key, payload)
+	return
+
+/atom/proc/Deserialize()
+	SHOULD_CALL_PARENT(TRUE)
+	SHOULD_NOT_SLEEP(TRUE)
+	for(var/data_key in __init_deserialization_payload)
+		if(data_key in vars)
+			try
+				if(!global._forbid_field_load[data_key] && (data_key in vars))
+					vars[data_key] = __init_deserialization_payload[data_key]
+				else
+					PreloadKey(data_key, __init_deserialization_payload[data_key])
+			catch(var/exception/E)
+				error("Failed to write [data_key] to [type] vars: [E]")
+	return SERDE_HINT_FINISHED
+
+// Called after Initialize()/LateInitialize() if an atom returns SERDE_HINT_POSTINIT to Deserialize().
+/atom/proc/PostInitLoad()
+	return
+
+/atom/movable/Serialize()
+	. = ..()
+	if(isturf(loc))
+		.[SERDE_LOC] = list(loc.x, loc.y)
+	// The below does not handle cases where the nested instance is not itself persistent.
+	// In this case, if the instance tried to serialize while inside a non-persistent instance, it would
+	// throw a runtime on subsequent loads due to having a UID as a loc that does not map to a loaded instance.
+	else if(isatom(loc))
+		.[SERDE_LOC] = loc.get_run_uid()

code/game/turfs/floors/_floor.dm

@@ -34,6 +34,11 @@
 
 	set_turf_materials(floor_material, skip_update = TRUE)
 
+	if(istext(_flooring))
+		_flooring = resolve_decl_uid_list(cached_json_decode(_flooring))
+		if(!length(_flooring))
+			_flooring = null
+
 	if(!floortype && (ispath(_flooring) || islist(_flooring)))
 		floortype = _flooring
 	else
@@ -49,7 +54,6 @@
 		if(ml) // We skipped the update above to avoid updating our neighbors, but we need to update ourselves.
 			lazy_update_icon()
 
-
 /turf/floor/ChangeTurf(turf/N, tell_universe, force_lighting_update, keep_air, update_open_turfs_above, keep_height)
 	if(is_processing)
 		STOP_PROCESSING(SSobj, src)