Fix SMP deadlock by avoiding nested monitors/processes_table locks

https://ampcode.com/threads/T-019ba1c2-2a7f-77c7-bd33-ce9f303152a2

Fix lock ordering inversion causing deadlocks on ESP32 SMP under heavy socket load.

The functions enif_demonitor_process and destroy_resource_monitors previously
acquired locks in the order: monitors -> processes_table. However,
enif_monitor_process and context_get_process_info (via resource_monitor_to_resource)
use the opposite order: processes_table -> monitors. This created an ABBA
deadlock under SMP when these code paths ran concurrently.

The fix uses a two-phase approach in both functions:
- Phase 1: Acquire monitors lock, find/remove entries, release monitors lock
- Phase 2: Acquire process lock, send signals, release process lock

This ensures the two locks are never held simultaneously, eliminating the
deadlock while preserving correct semantics. If the target process dies
between phases, the demonitor signal is simply skipped (harmless no-op).

Also fixes a pre-existing bug where enif_demonitor_process failed to unlock
the monitors list on the resource_type mismatch error path.

Signed-off-by: Peter M <petermm@gmail.com>

Author: petermm

src/libAtomVM/resources.c

@@ -476,54 +476,77 @@ int enif_demonitor_process(ErlNifEnv *env, void *obj, const ErlNifMonitor *mon)
         return -1;
     }
 
+    int32_t target_process_id = INVALID_PROCESS_ID;
+    uint64_t ref_ticks = 0;
+
+    // Phase 1: Find and remove from monitors list while holding monitors lock
     struct ListHead *monitors = synclist_wrlock(&resource_type->monitors);
     struct ListHead *item;
     LIST_FOR_EACH (item, monitors) {
         struct ResourceMonitor *monitor = GET_LIST_ENTRY(item, struct ResourceMonitor, resource_list_head);
         if (monitor->ref_ticks == mon->ref_ticks) {
             struct RefcBinary *resource = refc_binary_from_data(obj);
             if (resource->resource_type != mon->resource_type) {
+                synclist_unlock(&resource_type->monitors);
                 return -1;
             }
 
-            Context *target = globalcontext_get_process_lock(global, monitor->process_id);
-            if (target) {
-                mailbox_send_ref_signal(target, DemonitorSignal, monitor->ref_ticks);
-                globalcontext_get_process_unlock(global, target);
-            }
-
+            target_process_id = monitor->process_id;
+            ref_ticks = monitor->ref_ticks;
             list_remove(&monitor->resource_list_head);
             free(monitor);
-            synclist_unlock(&resource_type->monitors);
-            return 0;
+            break;
         }
     }
-
     synclist_unlock(&resource_type->monitors);
 
-    return -1;
+    if (target_process_id == INVALID_PROCESS_ID) {
+        return -1;
+    }
+
+    // Phase 2: Send demonitor signal without holding monitors lock
+    // This avoids lock order inversion with processes_table
+    Context *target = globalcontext_get_process_lock(global, target_process_id);
+    if (target) {
+        mailbox_send_ref_signal(target, DemonitorSignal, ref_ticks);
+        globalcontext_get_process_unlock(global, target);
+    }
+
+    return 0;
 }
 
 void destroy_resource_monitors(struct RefcBinary *resource, GlobalContext *global)
 {
     struct ResourceType *resource_type = resource->resource_type;
+
+    // Phase 1: Collect monitors to destroy while holding monitors lock
+    struct ListHead to_signal;
+    list_init(&to_signal);
+
     struct ListHead *monitors = synclist_wrlock(&resource_type->monitors);
     struct ListHead *item;
     struct ListHead *tmp;
     MUTABLE_LIST_FOR_EACH (item, tmp, monitors) {
         struct ResourceMonitor *monitor = GET_LIST_ENTRY(item, struct ResourceMonitor, resource_list_head);
         if (monitor->resource == resource) {
-            Context *target = globalcontext_get_process_lock(global, monitor->process_id);
-            if (target) {
-                mailbox_send_ref_signal(target, DemonitorSignal, monitor->ref_ticks);
-                globalcontext_get_process_unlock(global, target);
-            }
             list_remove(&monitor->resource_list_head);
-            free(monitor);
+            list_append(&to_signal, &monitor->resource_list_head);
         }
     }
-
     synclist_unlock(&resource_type->monitors);
+
+    // Phase 2: Send demonitor signals without holding monitors lock
+    // This avoids lock order inversion with processes_table
+    MUTABLE_LIST_FOR_EACH (item, tmp, &to_signal) {
+        struct ResourceMonitor *monitor = GET_LIST_ENTRY(item, struct ResourceMonitor, resource_list_head);
+        Context *target = globalcontext_get_process_lock(global, monitor->process_id);
+        if (target) {
+            mailbox_send_ref_signal(target, DemonitorSignal, monitor->ref_ticks);
+            globalcontext_get_process_unlock(global, target);
+        }
+        list_remove(&monitor->resource_list_head);
+        free(monitor);
+    }
 }
 
 int enif_compare_monitors(const ErlNifMonitor *monitor1, const ErlNifMonitor *monitor2)