diff --git a/com.unity.netcode.gameobjects/Runtime/Connection/NetworkConnectionManager.cs b/com.unity.netcode.gameobjects/Runtime/Connection/NetworkConnectionManager.cs
index f4f44301ae..a95df88a4e 100644
--- a/com.unity.netcode.gameobjects/Runtime/Connection/NetworkConnectionManager.cs
+++ b/com.unity.netcode.gameobjects/Runtime/Connection/NetworkConnectionManager.cs
@@ -1173,16 +1173,17 @@ internal void OnClientDisconnectFromServer(ulong clientId)
                     if (ownedObject)
                     {
                         // If destroying with owner, then always despawn and destroy (or defer destroying to prefab handler)
-                        if (!ownedObject.DontDestroyWithOwner)
+                        // Handle an object with no observers other than the current disconnecting client as destroying with owner
+                        if (!ownedObject.DontDestroyWithOwner || ownedObject.Observers.Count == 0 || (ownedObject.Observers.Count == 1 && ownedObject.Observers.Contains(clientId)))
                         {
-                            if (NetworkManager.PrefabHandler.ContainsHandler(clientOwnedObjects[i].GlobalObjectIdHash))
+                            if (NetworkManager.PrefabHandler.ContainsHandler(ownedObject.GlobalObjectIdHash))
                             {
                                 if (ownedObject.IsSpawned)
                                 {
                                     // Don't destroy (prefab handler will determine this, but always notify
                                     NetworkManager.SpawnManager.DespawnObject(ownedObject, false, true);
                                 }
-                                NetworkManager.PrefabHandler.HandleNetworkPrefabDestroy(clientOwnedObjects[i]);
+                                NetworkManager.PrefabHandler.HandleNetworkPrefabDestroy(ownedObject);
                             }
                             else
                             {
@@ -1240,6 +1241,12 @@ internal void OnClientDisconnectFromServer(ulong clientId)
                                     {
                                         continue;
                                     }
+
+                                    // Skip destroy with owner objects as they will be processed by the outer loop
+                                    if (!childObject.DontDestroyWithOwner || childObject.Observers.Count == 0 || (childObject.Observers.Count == 1 && childObject.Observers.Contains(clientId)))
+                                    {
+                                        continue;
+                                    }
                                     // If the client owner disconnected, it is ok to unlock this at this point in time.
                                     if (childObject.IsOwnershipLocked)
                                     {
diff --git a/com.unity.netcode.gameobjects/Runtime/Spawning/NetworkSpawnManager.cs b/com.unity.netcode.gameobjects/Runtime/Spawning/NetworkSpawnManager.cs
index d2319cb96d..7b390547a4 100644
--- a/com.unity.netcode.gameobjects/Runtime/Spawning/NetworkSpawnManager.cs
+++ b/com.unity.netcode.gameobjects/Runtime/Spawning/NetworkSpawnManager.cs
@@ -1975,8 +1975,8 @@ internal void DistributeNetworkObjects(ulong clientId)
                             // the owned distributable parent with the owned distributable children
                             foreach (var child in children)
                             {
-                                // Ignore the parent and any child that does not have the same owner or that is already owned by the currently targeted client
-                                if (child == ownerList.Value[i] || child.OwnerClientId != ownerList.Value[i].OwnerClientId || child.OwnerClientId == clientId)
+                                // Ignore any child that does not have the same owner, that is already owned by the currently targeted client, or that doesn't have the targeted client as an observer
+                                if (child == ownerList.Value[i] || child.OwnerClientId != ownerList.Value[i].OwnerClientId || child.OwnerClientId == clientId || !child.Observers.Contains(clientId))
                                 {
                                     continue;
                                 }