Adds a per-level persistence system.

Author: MistakeNot4892

code/__defines/misc.dm

@@ -87,27 +87,28 @@
 #define EVENT_LEVEL_MAJOR    3
 
 //Area flags, possibly more to come
-#define AREA_FLAG_RAD_SHIELDED         BITFLAG(1)  // Shielded from radiation, clearly.
-#define AREA_FLAG_EXTERNAL             BITFLAG(2)  // External as in exposed to space, not outside in a nice, green, forest.
-#define AREA_FLAG_ION_SHIELDED         BITFLAG(3)  // Shielded from ionospheric anomalies.
-#define AREA_FLAG_IS_NOT_PERSISTENT    BITFLAG(4)  // SSpersistence will not track values from this area.
-#define AREA_FLAG_IS_BACKGROUND        BITFLAG(5)  // Blueprints can create areas on top of these areas. Cannot edit the name of or delete these areas.
-#define AREA_FLAG_MAINTENANCE          BITFLAG(6)  // Area is a maintenance area.
-#define AREA_FLAG_SHUTTLE              BITFLAG(7)  // Area is a shuttle area.
-#define AREA_FLAG_HALLWAY              BITFLAG(8)  // Area is a public hallway suitable for event selection
-#define AREA_FLAG_PRISON               BITFLAG(9)  // Area is a prison for the purposes of brigging objectives.
-#define AREA_FLAG_HOLY                 BITFLAG(10) // Area is holy for the purposes of marking turfs as cult-resistant.
-#define AREA_FLAG_SECURITY             BITFLAG(11) // Area is security for the purposes of newscaster init.
-#define AREA_FLAG_HIDE_FROM_HOLOMAP    BITFLAG(12) // if we shouldn't be drawn on station holomaps
+#define AREA_FLAG_RAD_SHIELDED            BITFLAG(1)  // Shielded from radiation, clearly.
+#define AREA_FLAG_EXTERNAL                BITFLAG(2)  // External as in exposed to space, not outside in a nice, green, forest.
+#define AREA_FLAG_ION_SHIELDED            BITFLAG(3)  // Shielded from ionospheric anomalies.
+#define AREA_FLAG_NO_LEGACY_PERSISTENCE   BITFLAG(4)  // SSpersistence will not track values from this area.
+#define AREA_FLAG_IS_BACKGROUND           BITFLAG(5)  // Blueprints can create areas on top of these areas. Cannot edit the name of or delete these areas.
+#define AREA_FLAG_MAINTENANCE             BITFLAG(6)  // Area is a maintenance area.
+#define AREA_FLAG_SHUTTLE                 BITFLAG(7)  // Area is a shuttle area.
+#define AREA_FLAG_HALLWAY                 BITFLAG(8)  // Area is a public hallway suitable for event selection
+#define AREA_FLAG_PRISON                  BITFLAG(9)  // Area is a prison for the purposes of brigging objectives.
+#define AREA_FLAG_HOLY                    BITFLAG(10) // Area is holy for the purposes of marking turfs as cult-resistant.
+#define AREA_FLAG_SECURITY                BITFLAG(11) // Area is security for the purposes of newscaster init.
+#define AREA_FLAG_HIDE_FROM_HOLOMAP       BITFLAG(12) // if we shouldn't be drawn on station holomaps
+#define AREA_FLAG_ALLOW_LEVEL_PERSISTENCE BITFLAG(13) // Whether or not this area should pass changed turfs to SSpersistence.
 
 //Map template flags
-#define TEMPLATE_FLAG_ALLOW_DUPLICATES   BITFLAG(0)  // Lets multiple copies of the template to be spawned
-#define TEMPLATE_FLAG_SPAWN_GUARANTEED   BITFLAG(1)  // Makes it ignore away site budget and just spawn (only for away sites)
-#define TEMPLATE_FLAG_CLEAR_CONTENTS     BITFLAG(2)  // if it should destroy objects it spawns on top of
-#define TEMPLATE_FLAG_NO_RUINS           BITFLAG(3)  // if it should forbid ruins from spawning on top of it
-#define TEMPLATE_FLAG_NO_RADS            BITFLAG(4)  // Removes all radiation from the template after spawning.
-#define TEMPLATE_FLAG_TEST_DUPLICATES    BITFLAG(5)  // Makes unit testing attempt to spawn mutliple copies of this template. Assumes unit testing is spawning at least one copy.
-#define TEMPLATE_FLAG_GENERIC_REPEATABLE BITFLAG(6) // Template can be picked repeatedly for the same level gen run.
+#define TEMPLATE_FLAG_ALLOW_DUPLICATES    BITFLAG(0)  // Lets multiple copies of the template to be spawned
+#define TEMPLATE_FLAG_SPAWN_GUARANTEED    BITFLAG(1)  // Makes it ignore away site budget and just spawn (only for away sites)
+#define TEMPLATE_FLAG_CLEAR_CONTENTS      BITFLAG(2)  // if it should destroy objects it spawns on top of
+#define TEMPLATE_FLAG_NO_RUINS            BITFLAG(3)  // if it should forbid ruins from spawning on top of it
+#define TEMPLATE_FLAG_NO_RADS             BITFLAG(4)  // Removes all radiation from the template after spawning.
+#define TEMPLATE_FLAG_TEST_DUPLICATES     BITFLAG(5)  // Makes unit testing attempt to spawn mutliple copies of this template. Assumes unit testing is spawning at least one copy.
+#define TEMPLATE_FLAG_GENERIC_REPEATABLE  BITFLAG(6) // Template can be picked repeatedly for the same level gen run.
 
 // Convoluted setup so defines can be supplied by Bay12 main server compile script.
 // Should still work fine for people jamming the icons into their repo.

code/__defines/persistence.dm

@@ -0,0 +1,5 @@
+// Handled elsewhere, do not let them load like vars.
+var/global/list/_forbid_field_load = list(
+	(nameof(/datum::type)) = TRUE,
+	(nameof(/atom::loc))  = TRUE
+)

code/__defines/serde.dm

@@ -0,0 +1,63 @@
+#define SERDE_HINT_FINISHED 1
+#define SERDE_HINT_POSTINIT 2
+
+#define SERDE_REAGENT_LIST "_reagent_list"
+#define SERDE_REAGENT_VOLUME "_reagent_volume"
+
+#define SERIALIZE_VALUE(V, T, VAL) .[nameof(T::V)] = VAL;
+#define SERIALIZE(V, T) SERIALIZE_VALUE(V, T, V)
+#define SERIALIZE_IF_MODIFIED(V, T) if(V != initial(V)) { SERIALIZE_VALUE(V, T, V) }
+#define SERIALIZE_TYPE_IF_MODIFIED(V, T) if(V != initial(V)) { SERIALIZE_VALUE(V, T, "[V]") }
+#define SERIALIZE_DECL_IF_MODIFIED(V, T) if((isnull(V) && !isnull(initial(V))) || ((istext(V) || istype(V, /decl) || ispath(V, /decl)) && !DECLS_ARE_EQUIVALENT(V, initial(V)))) { var/decl/__D = RESOLVE_TO_DECL(V); SERIALIZE_VALUE(V, T, __D?.uid) }
+#define SERIALIZE_DECL_LIST(V, T) if(islist(V)) { var/list/__decl_uids = list(); for(var/decl/__decl in V) { __decl_uids += __decl.uid }; SERIALIZE_VALUE(V, T, __decl_uids) }
+#define SERIALIZE_REAGENTS(V, T, I) if(istype(V, /datum/reagents)) {                                               \
+	.[I + SERDE_REAGENT_VOLUME] = V.maximum_volume;                                                                \
+	if(V.total_volume) {                                                                                           \
+		var/list/__compiled_reagents = list();                                                                     \
+		for(var/decl/material/R in V.liquid_volumes) {                                                             \
+			__compiled_reagents[++__compiled_reagents.len] = list(R.uid, V.liquid_volumes[R], (MAT_PHASE_LIQUID)); \
+		}                                                                                                          \
+		for(var/decl/material/R in V.solid_volumes) {                                                              \
+			__compiled_reagents[++__compiled_reagents.len] = list(R.uid, V.solid_volumes[R], (MAT_PHASE_SOLID));   \
+		}                                                                                                          \
+		.[I + SERDE_REAGENT_LIST] = __compiled_reagents;                                                           \
+	} else {                                                                                                       \
+		.[I + SERDE_REAGENT_LIST] = list();                                                                        \
+	}                                                                                                              \
+} else {                                                                                                           \
+	.[I + SERDE_REAGENT_LIST]   = list();                                                                          \
+	.[I + SERDE_REAGENT_VOLUME] = 0;                                                                               \
+}
+
+#define DESERIALIZE_REAGENTS(V, I) if(((I + SERDE_REAGENT_LIST) in __deserialization_payload) && ((I + SERDE_REAGENT_VOLUME) in __deserialization_payload)) {         \
+	V = list((SERDE_REAGENT_VOLUME) = __deserialization_payload[I + SERDE_REAGENT_VOLUME], (SERDE_REAGENT_LIST) = __deserialization_payload[I + SERDE_REAGENT_LIST]); \
+}
+
+#define DESERIALIZE_DECL_TO_TYPE(V) if(istext(V) || ispath(V, /decl) || istype(V, /decl)) { var/decl/__D = RESOLVE_TO_DECL(V); V = __D?.type; } else { V = null; }
+#define DESERIALIZE_TYPE(V) if(istext(V)) { V = text2path(V); } else if(!ispath(V)) { V = null; }
+#define DESERIALIZE_DECL(V) if(istext(V) || ispath(V)) { V = RESOLVE_TO_DECL(V); } else { V = null; }
+
+// List cast is to avoid OpenDream complaining about V typically being typed as a reagents datum, but holding a list for serde.
+#define FINALIZE_REAGENTS_SERDE_BODY(V) try {                                              \
+	if((SERDE_REAGENT_LIST in V) && (SERDE_REAGENT_VOLUME in V)) {                         \
+		var/list/LV = V;                                                                   \
+		var/__serde_volume = LV[SERDE_REAGENT_VOLUME];                                     \
+		if(__serde_volume <= 0) {                                                          \
+			V = null;                                                                      \
+		} else {                                                                           \
+			var/list/__serde_reagents = LV[SERDE_REAGENT_LIST];                            \
+			V = new /datum/reagents(__serde_volume, src);                                  \
+			for(var/list/entry in __serde_reagents) {                                      \
+				V.add_reagent(RESOLVE_TO_DECL(entry[1]), entry[2], phase = entry[3]);      \
+			}                                                                              \
+		}                                                                                  \
+	} else {                                                                               \
+		V = null;                                                                          \
+	}                                                                                      \
+} catch(var/exception/E) {                                                                 \
+	log_error("Exception while finalizing reagents load for [type]: [EXCEPTION_TEXT(E)]"); \
+	V = null;                                                                              \
+}
+
+#define FINALIZE_REAGENTS_SERDE(V)            if(islist(V)) { FINALIZE_REAGENTS_SERDE_BODY(V); }
+#define FINALIZE_REAGENTS_SERDE_AND_RETURN(V) if(islist(V)) { FINALIZE_REAGENTS_SERDE_BODY(V); return; }

code/__defines/subsystems.dm

@@ -19,12 +19,13 @@
 // Subsystems shutdown in the reverse of the order they initialize in
 // The numbers just define the ordering, they are meaningless otherwise.
 
-#define SS_INIT_INPUT            22
-#define SS_INIT_EARLY            21
-#define SS_INIT_WEBHOOKS         20
-#define SS_INIT_MODPACKS         19
-#define SS_INIT_SECRETS          18
-#define SS_INIT_GARBAGE          17
+#define SS_INIT_INPUT            23
+#define SS_INIT_EARLY            22
+#define SS_INIT_WEBHOOKS         21
+#define SS_INIT_MODPACKS         20
+#define SS_INIT_SECRETS          19
+#define SS_INIT_GARBAGE          18
+#define SS_INIT_SERDE            17
 #define SS_INIT_MATERIALS        16
 #define SS_INIT_PLANTS           15
 #define SS_INIT_LORE             14

code/_helpers/serde.dm

@@ -0,0 +1,131 @@
+/proc/instantiate_serialized_data(load_z, requestor, list/instance_map, entries_decay_at, entry_decay_weight)
+
+	var/list/nested_instances = list()
+	var/list/instanced_areas  = list()
+	var/list/created_data     = list()
+
+	LAZYINITLIST(instance_map)
+
+	to_world_log("Finalising load of [length(instance_map)] instance\s for level '[requestor]'.")
+	for(var/uid in instance_map)
+
+		var/list/instance_data = instance_map[uid]
+		try
+
+			var/raw_load_path = instance_data[nameof(/datum::type)]
+			var/load_path = ispath(raw_load_path, /datum) ? raw_load_path : text2path(raw_load_path)
+			if(!ispath(load_path, /datum))
+				error("[requestor]: attempted to load persistent instance with invalid or non-/datum type '[raw_load_path]'")
+				continue
+
+			var/datum/created_instance
+
+			// Instance is a /datum.
+			// Just pass the data in and assume the datum type knows what to do with it.
+			if(!ispath(load_path, /atom) && ispath(load_path, /datum))
+				created_instance = new load_path(instance_data)
+				created_data += created_instance
+			else
+				var/list/spawn_data = instance_data[nameof(/atom/movable::loc)]
+				if(spawn_data)
+
+					if(isnull(spawn_data) || length(spawn_data) < 3)
+						error("[requestor]: attempted to load persistent instance with malformed loc.")
+						continue
+
+					// Instance has a world coordinate.
+					if(islist(spawn_data))
+						var/turf/spawn_loc = locate(spawn_data[1], spawn_data[2], isnull(load_z) ? spawn_data[3] : load_z)
+						if(!istype(spawn_loc))
+							error("[requestor]: attempted to load persistent instance but could not find spawn loc.")
+							continue
+						if(ispath(load_path, /turf))
+							if(spawn_loc.type == load_path)
+								created_instance = spawn_loc
+							else
+								created_instance = spawn_loc.ChangeTurf(load_path)
+
+						// TODO: Areas will need bespoke handling for non-subtype-related persistence (blueprint renaming etc).
+						else if(ispath(load_path, /area))
+							var/area/area = instanced_areas[load_path]
+							if(!area)
+								area = new load_path(null)
+								instanced_areas[load_path] = area
+							ChangeArea(spawn_loc, area)
+
+						else if(ispath(load_path, /atom))
+							created_instance = new load_path(spawn_loc)
+							spawn_loc._contents_were_modified = TRUE // ensure
+						else
+							error("[requestor]: attempted to instantiate unimplemented path '[load_path]'.")
+							continue
+
+					// Instance is inside another instance; implies/requires /atom/movable
+					else if(istext(spawn_data))
+						if(!ispath(load_path, /atom/movable))
+							error("[requestor]: tried to spawn non-movable [load_path] inside an instance.")
+							continue
+						created_instance = new load_path
+						nested_instances[created_instance] = spawn_data
+
+					else
+						error("[requestor]: attempted to load persistent instance with malformed loc.")
+						continue
+
+				else
+					// Should we just go ahead and do this to create atoms in nullspace?
+					// Would we ever want to track an atom in nullspace via level persistence?
+					error("[requestor]: attempted to load non-/datum persistent instance with no spawn loc.")
+
+			if(istype(created_instance))
+				LAZYSET(., uid, created_instance)
+				if(isatom(created_instance))
+					var/atom/atom = created_instance
+					atom.__deserialization_payload = instance_data
+					SSatoms.deserialized_atoms[uid] = atom
+				if(!isnull(entries_decay_at) && !isnull(entry_decay_weight))
+					created_instance.HandlePersistentDecay(entries_decay_at, entry_decay_weight)
+
+		catch(var/exception/E)
+			log_error("Exception during persistent instance load - [islist(instance_data) ? json_encode(instance_data) : "no instance data"]: [EXCEPTION_TEXT(E)]")
+
+	// Atoms use SSatoms for this, datums don't go through SSatoms so need to do it here.
+	for(var/datum/instance in created_data)
+		instance.DeserializePostInit(.)
+
+	// Resolve any loc references to instances.
+	for(var/atom/movable/atom as anything in nested_instances)
+		var/nested_atom_id = nested_instances[atom]
+		var/atom/nested_atom = .[nested_atom_id]
+		if(!istype(nested_atom))
+			error("[requestor]: could not resolve instance ref [nested_atom_id] to instance.")
+			continue
+		atom.forceMove(nested_atom)
+		nested_atom.contents_were_modified()
+
+	// Now that everything is loaded and placed, clear out anything that should not be present on the turfs we've loaded.
+	for(var/uid in SSatoms.deserialized_atoms)
+		var/turf/turf = SSatoms.deserialized_atoms[uid]
+		if(!istype(turf))
+			continue
+		for(var/atom/thing in turf)
+			if(!thing.simulated)
+				continue
+			if(!isnull(thing.__deserialization_payload))
+				continue
+			qdel(thing)
+
+	to_world_log("[requestor] loaded [length(.)] persistent instance\s.")
+
+/proc/apply_serde_message_decay(_message, _age, _decay_weight, _decay_at)
+	var/static/list/decayed_chars = list(".",",","-","'","\\","/","\"",":",";")
+	if(_age < _decay_at || isnull(_message))
+		return _message
+	. = ""
+	for(var/i = 1 to length(_message))
+		var/char = copytext(_message, i, i + 1)
+		if(prob(round(_age * _decay_weight)))
+			if(prob(99))
+				. += pick(decayed_chars)
+		else
+			. += char

code/controllers/subsystems/atoms.dm

@@ -11,6 +11,8 @@ SUBSYSTEM_DEF(atoms)
 	var/atom_init_stage = INITIALIZATION_INSSATOMS
 	var/old_init_stage
 
+	/// An associative list of UIDs to atoms that were deserialized prior to flush.
+	var/list/deserialized_atoms = list()
 	/// A non-associative list of lists, with the format list(list(atom, list(Initialize arguments))).
 	var/list/created_atoms = list()
 	/// A non-associative list of lists, with the format list(list(atom, list(LateInitialize arguments))).
@@ -29,9 +31,20 @@ SUBSYSTEM_DEF(atoms)
 
 	atom_init_stage = INITIALIZATION_INNEW_MAPLOAD
 
-	var/list/mapload_arg = list(TRUE)
-
+	// Preload any atoms that have deserialized during the initial load process prior to flush.
 	var/index = 1
+	var/list/postinit_serde_atoms = list()
+	if(length(deserialized_atoms))
+		while(index <= length(deserialized_atoms))
+			var/uid = deserialized_atoms[index++]
+			var/atom/instance = deserialized_atoms[uid]
+			if(instance.Preload(deserialized_atoms) == SERDE_HINT_POSTINIT)
+				postinit_serde_atoms += instance
+			CHECK_TICK
+		report_progress("Deserialized [index-1] atom\s.")
+		index = 1
+
+	var/list/mapload_arg = list(TRUE)
 	// Things can add to the end of this list while we iterate, so we can't use a for loop.
 	while(index <= length(created_atoms))
 		// Don't remove from this list while we run, that's expensive.
@@ -49,10 +62,10 @@ SUBSYSTEM_DEF(atoms)
 			else
 				InitAtom(A, mapload_arg)
 			CHECK_TICK
-
-	report_progress("Initialized [index] atom\s")
 	created_atoms.Cut()
 
+	report_progress("Initialized [index-1] atom\s.")
+
 	atom_init_stage = INITIALIZATION_INNEW_REGULAR
 
 	if(length(late_loaders))
@@ -65,6 +78,25 @@ SUBSYSTEM_DEF(atoms)
 		report_progress("Late initialized [index] atom\s")
 		late_loaders.Cut()
 
+	if(length(postinit_serde_atoms))
+		index = 1
+		while(index <= length(postinit_serde_atoms))
+			var/atom/instance = postinit_serde_atoms[index++]
+			instance.DeserializePostInit(deserialized_atoms)
+			CHECK_TICK
+		postinit_serde_atoms.Cut()
+
+	// Clear out the serde payloads now that everything should be tidied away.
+	if(length(deserialized_atoms))
+		index = 1
+		while(index <= length(deserialized_atoms))
+			var/uid = deserialized_atoms[index++]
+			var/atom/instance = deserialized_atoms[uid]
+			if(istype(instance))
+				instance.__deserialization_payload = null
+			CHECK_TICK
+		deserialized_atoms.Cut()
+
 /datum/controller/subsystem/atoms/proc/InitAtom(atom/A, list/arguments)
 	var/the_type = A.type
 	if(QDELING(A))

code/controllers/subsystems/mapping.dm

@@ -149,12 +149,36 @@ SUBSYSTEM_DEF(mapping)
 
 	setup_data_for_levels(min_z = old_maxz + 1)
 
+	// Now that levels are in place, preload any associated persistent data.
+	// This is to avoid dependencies on other atoms or any other weird ordering
+	// problems like we used to get with old DMMS and SSatoms.
+	var/list/preloaded_levels = list()
+	for(var/datum/level_data/level in levels_by_z)
+		if(level.preload_persistent_data())
+			preloaded_levels += level
+
+	// Now actually load the serde data into the map.
+	for(var/datum/level_data/level as anything in preloaded_levels)
+		level.load_persistent_data()
+
+	// Clear our reference data for GC
+	// This might not be needed but it saves refs floating around I guess.
+	for(var/key in level_persistence_ref_map)
+		var/list/stale_data = global.level_persistence_ref_map[key]
+		stale_data.Cut()
+
+	global.level_persistence_ref_map.Cut()
+
+	// With levels set up and serde complete (and levels flagged) we can do any remaining level generation.
+	for(var/datum/level_data/level in levels_by_z)
+		level.generate_level()
+
+	global.using_map.finalize_map_generation()
+
 	// Generate turbolifts last, since away sites may have elevators to generate too.
 	for(var/obj/abstract/turbolift_spawner/turbolift as anything in turbolifts_to_initialize)
 		turbolift.build_turbolift()
 
-	global.using_map.finalize_map_generation()
-
 	. = ..()
 
 /datum/controller/subsystem/mapping/proc/setup_data_for_levels(min_z = 1, max_z = world.maxz)