fix: remove observers from da host (#3526)

Removing observers in disconnected client check.


## Changelog

NA

## Testing and Documentation

- No tests have been added.
- No documentation changes or additions were necessary.

<!-- Uncomment and mark items off with a * if this PR deprecates any
API:
### Deprecated API
- [ ] An `[Obsolete]` attribute was added along with a `(RemovedAfter
yyyy-mm-dd)` entry.
- [ ] An [api updater] was added.
- [ ] Deprecation of the API is explained in the CHANGELOG.
- [ ] The users can understand why this API was removed and what they
should use instead.
-->

## Backport

<!-- If this is a backport:
 - Add the following to the PR title: "\[Backport\] ..." .
 - Link to the original PR.
If this needs a backport - state this here
If a backport is not needed please provide the reason why.
If the "Backports" section is not present it will lead to a CI test
failure.
-->

Author: NoelStephensUnity

com.unity.netcode.gameobjects/Runtime/Connection/NetworkConnectionManager.cs

@@ -1177,8 +1177,7 @@ internal void OnClientDisconnectFromServer(ulong clientId)
                     if (ownedObject)
                     {
                         // If destroying with owner, then always despawn and destroy (or defer destroying to prefab handler)
-                        // Handle an object with no observers other than the current disconnecting client as destroying with owner
-                        if (!ownedObject.DontDestroyWithOwner && (ownedObject.Observers.Count == 0 || (ownedObject.Observers.Count >= 1 && ownedObject.Observers.Contains(clientId))))
+                        if (!ownedObject.DontDestroyWithOwner)
                         {
                             if (NetworkManager.PrefabHandler.ContainsHandler(ownedObject.GlobalObjectIdHash))
                             {
@@ -1244,7 +1243,7 @@ internal void OnClientDisconnectFromServer(ulong clientId)
                                     }
 
                                     // Skip destroy with owner objects as they will be processed by the outer loop
-                                    if (!childObject.DontDestroyWithOwner && (childObject.Observers.Count == 0 || (childObject.Observers.Count == 1 && childObject.Observers.Contains(clientId))))
+                                    if (!childObject.DontDestroyWithOwner)
                                     {
                                         continue;
                                     }