Don't retry mod load on error, but allow a new load if DB is reloaded

Author: jmquigs

MMManaged/MeshRelation.fs

@@ -267,8 +267,8 @@ module MeshRelation =
 
             modVertRels
 
-        
         let buildIt() = 
+            //log.Info "Starting build of meshrelation for mod: %A" md.Name
             modMesh <- Some ((verifyAndGet md.Name md.Mesh).Force())
             refMesh <- Some ((ref.Mesh.Force()))
 
@@ -280,7 +280,7 @@ module MeshRelation =
 
         let vertRels = lazy (buildIt())
 
-        member x.IsBuilt = vertRels.IsValueCreated
+        member x.IsBuilt = vertRels.IsValueCreated 
         member x.Build() = vertRels.Force()
 
         member x.DBMod = md

MMManaged/ModDB.fs

@@ -647,7 +647,7 @@ module ModDB =
                         | None -> newMeshRel()
                     | _ -> newMeshRel()
                 )
-            log().Info "MeshRelations: %d cached, %d built" nCached nBuilt
+            log().Info "MeshRelations: %d cached, %d new" nCached nBuilt
             meshRels
 
         // this code forces an immediate load of the data, maybe useful for debugging

MMManaged/ModDBInterop.fs

@@ -311,8 +311,16 @@ module ModDBInterop =
             List.item delIdx moddb.DeletionMods
         | n -> failwithf "invalid mod index: %A" i
 
-    // technically, because I use an exclusive lock below, this doesn't need to be a concurrent dictionary
-    let modLoadThread = new System.Collections.Concurrent.ConcurrentDictionary<string, System.Threading.Thread>()
+    /// Used to track the loading thread for a mod (see lodModData for more details).
+    type LoadInfo = {
+        Thread: System.Threading.Thread
+        /// When the database is reloaded this will change, allowing a new set of loads.  This is useful
+        /// if some of the mods had errors that have been corrected.
+        MeshRelationId: int
+        LastLoadHadError: option<bool>
+    }
+   
+    let modLoadThread = new System.Collections.Concurrent.ConcurrentDictionary<string, LoadInfo>()
 
     /// Load the mod data for mod i.  This is assumed to be an index into the mesh relation list.  
     /// If the mesh relation is already built, this does nothing (the mod is not reloaded).  
@@ -329,30 +337,56 @@ module ModDBInterop =
     /// to this function.  The native code should observe that the (new) mesh relation's data is still 
     /// unavailable and trigger a another call to this function on an ongoing basis until the old thread 
     /// finishes and a new one can be started.
-    let loadModData(i) = 
+    let loadModData(i) =
         let mutable loadState = 0 // not available
-        try     
+        try
             let moddb = State.Data.Moddb
-            if i < moddb.MeshRelations.Length then 
+            if i < moddb.MeshRelations.Length then
                 let meshrel = List.item i (moddb.MeshRelations)
-                
-                if meshrel.IsBuilt then 
+
+                if meshrel.IsBuilt then
                     loadState <- 1 // loaded
-                else  
+                else
                     let loadKey = i.ToString() + meshrel.DBMod.Name + meshrel.DBRef.Name
+                    let currentMeshRelId = meshrel.GetHashCode()
 
-                    Locking.write( fun _ -> 
-                        let ok,mlt = modLoadThread.TryGetValue loadKey
-                        if not ok || (ok && not mlt.IsAlive) then 
-                            let loadit() =                             
+                    let loadit () =
+                        let ok,currInfo = modLoadThread.TryGetValue loadKey 
+                        if ok then 
+                            try
+                                //log.Info "beginning new load for key %A with meshrel id %A (curr load info: %A)" loadKey currentMeshRelId currInfo
                                 meshrel.Build() |> ignore
-                            let t = new System.Threading.Thread( new System.Threading.ThreadStart(loadit)  )
-                            t.Start()
-                            modLoadThread.AddOrUpdate(loadKey, t, (fun _ (oldT) -> t)) |> ignore
-                            loadState <- 2 // load was started
-                        else 
-                            loadState <- 3 // a started load is still pending
-                    )                    
+                                let newInfo = { currInfo with LastLoadHadError = Some(false) }
+                                modLoadThread.TryUpdate(loadKey, newInfo, currInfo) |> ignore
+                            with
+                            | e ->
+                                // this is in a separate thread so log explicitly to make sure it goes to output log
+                                log.Error "Mesh relation build exception: %A" e
+                                let newInfo = { currInfo with LastLoadHadError = Some(true) }
+                                modLoadThread.TryUpdate(loadKey, newInfo, currInfo) |> ignore
+                        else
+                            log.Error "load info for mod %A not found" meshrel.DBMod.Name
+
+                    let startNewLoad() = 
+                        let t = new System.Threading.Thread(new System.Threading.ThreadStart(loadit))
+                        let newLoadInfo = { Thread = t; MeshRelationId = currentMeshRelId; LastLoadHadError = None }
+                        modLoadThread.AddOrUpdate(loadKey, newLoadInfo, (fun _ _ -> newLoadInfo)) |> ignore
+                        t.Start()
+                        loadState <- 2  // load was started
+
+                    Locking.write (fun _ ->
+                        match modLoadThread.TryGetValue(loadKey) with
+                        | (true, loadInfo) ->
+                            //log.Info "Found existing load entry: %A (curr meshrel is: %A)" loadInfo currentMeshRelId
+                            if loadInfo.MeshRelationId = currentMeshRelId && loadInfo.LastLoadHadError = Some true then
+                                loadState <- 4  // error encountered previously, do not start a new load
+                            elif loadInfo.Thread.IsAlive then
+                                loadState <- 3  // a started load is still pending
+                            else
+                                startNewLoad()
+                        | (false, _) ->
+                            startNewLoad()
+                    )
             loadState
         with
             | e ->

Native/mod_load/src/mod_load.rs

@@ -118,6 +118,7 @@ unsafe fn load_tex(dp:DevicePointer, texpath:&[u16]) -> Option<TexPtr> {
     }
 }
 
+const MAX_FILL_ATTEMPTS:u32 = 1;
 /// Create D3D resources for a mod using the data loaded by managed code. This usually consists of a
 /// vertex buffer, declaration and optionally one or more textures.  `midx` is the mod index
 /// into the current mod DB (and should be less than GetModCount()).
@@ -554,6 +555,7 @@ pub unsafe fn setup_mod_data(device: DevicePointer, callbacks: interop::ManagedC
             parent_mod_names: parent_mods,
             last_frame_render: 0,
             name: mod_name.to_owned(),
+            fill_attempts: 0,
         };
 
         // get mod key
@@ -746,6 +748,14 @@ pub unsafe fn load_deferred_mods(device: DevicePointer, callbacks: interop::Mana
             let mut nmod =
                 get_mod_by_name(&nmd, &mut GLOBAL_STATE.loaded_mods);
             if let Some(ref mut nmod) = nmod {
+                if nmod.fill_attempts > MAX_FILL_ATTEMPTS {
+                    if nmod.fill_attempts == MAX_FILL_ATTEMPTS + 1 {
+                        write_log_file(&format!("load_deferred_mods warning: mod exceeded max fill attempts, no more will be done: {}", nmod.name));
+                        // increment so that we don't log about it again
+                        nmod.fill_attempts += 1;
+                    }
+                    continue;
+                }
                 if let ModD3DState::Loaded(_) = nmod.d3d_data {
                     write_log_file(&format!("load_deferred_mods: mod already loaded: {}", nmod.name));
                     continue;
@@ -754,15 +764,19 @@ pub unsafe fn load_deferred_mods(device: DevicePointer, callbacks: interop::Mana
                     let mdat: *mut interop::ModData = (callbacks.GetModData)(nmod.midx);
                     if mdat != null_mut() {
                         let mut lres = 0;
-                        if !(*mdat).data_available {                        
+                        if !(*mdat).data_available {
                             lres = (callbacks.LoadModData)(nmod.midx);
                             match lres {
                                 0 => write_log_file(&format!("load_deferred_mods: mod data not yet available: {}", nmod.name)),
                                 1 => write_log_file(&format!("load_deferred_mods: mod data is now available: {}", nmod.name)),
                                 2 => write_log_file(&format!("load_deferred_mods: mod data load has started: {}", nmod.name)),
                                 3 => write_log_file(&format!("load_deferred_mods: mod data load is in progress: {}", nmod.name)),
+                                4 => write_log_file(&format!("load_deferred_mods: mod data load had error: {}", nmod.name)),
                                 n=> write_log_file(&format!("load_deferred_mods: unexpected return code: {} for mod: {}", n, nmod.name)),
                             }
+                            if lres == 2 || lres == 4 {
+                                nmod.fill_attempts += 1;
+                            }
 
                             if lres != 1 {
                                 continue;    

Native/types/src/native_mod.rs

@@ -40,6 +40,7 @@ pub struct NativeModData {
     pub parent_mod_names: Vec<String>,
     pub last_frame_render: u64,
     pub name: String,
+    pub fill_attempts: u32,
 }
 
 pub const MAX_RECENT_RENDER_USAGE_THRESH:u64 = 500;
@@ -55,6 +56,7 @@ impl NativeModData {
             parent_mod_names: vec![],
             last_frame_render: 0,
             name: "".to_owned(),
+            fill_attempts: 0,
         }
     }
     pub fn mod_key(vert_count: u32, prim_count: u32) -> u32 {