diff --git a/packages/react-native-gesture-handler/android/src/main/java/com/swmansion/gesturehandler/react/RNGestureHandlerDetectorView.kt b/packages/react-native-gesture-handler/android/src/main/java/com/swmansion/gesturehandler/react/RNGestureHandlerDetectorView.kt
index b7d7b9d065..17424a455d 100644
--- a/packages/react-native-gesture-handler/android/src/main/java/com/swmansion/gesturehandler/react/RNGestureHandlerDetectorView.kt
+++ b/packages/react-native-gesture-handler/android/src/main/java/com/swmansion/gesturehandler/react/RNGestureHandlerDetectorView.kt
@@ -13,7 +13,7 @@ class RNGestureHandlerDetectorView(context: Context) : ReactViewGroup(context) {
   private val reactContext: ThemedReactContext
     get() = context as ThemedReactContext
   private var handlersToAttach: List<Int>? = null
-  private var nativeHandlersToAttach: MutableSet<Int> = mutableSetOf()
+  private var nativeHandlers: MutableSet<Int> = mutableSetOf()
   private var attachedHandlers: MutableSet<Int> = mutableSetOf()
   private var moduleId: Int = -1
 
@@ -48,7 +48,7 @@ class RNGestureHandlerDetectorView(context: Context) : ReactViewGroup(context) {
   override fun addView(child: View, index: Int, params: LayoutParams?) {
     super.addView(child, index, params)
 
-    tryAttachHandlerToChildView(child.id)
+    tryAttachNativeHandlersToChildView(child.id)
   }
 
   override fun removeViewAt(index: Int) {
@@ -78,7 +78,7 @@ class RNGestureHandlerDetectorView(context: Context) : ReactViewGroup(context) {
         // It might happen that `attachHandlers` will be called before children are added into view hierarchy. In that case we cannot
         // attach `NativeViewGestureHandlers` here and we have to do it in `addView` method.
         if (shouldAttachGestureToChildView(tag)) {
-          nativeHandlersToAttach.add(tag)
+          nativeHandlers.add(tag)
         } else {
           registry.attachHandlerToView(tag, this.id, GestureHandler.ACTION_TYPE_NATIVE_DETECTOR)
 
@@ -86,6 +86,7 @@ class RNGestureHandlerDetectorView(context: Context) : ReactViewGroup(context) {
         }
       } else if (entry.value == GestureHandlerMutation.Detach) {
         registry.detachHandler(tag)
+        nativeHandlers.remove(tag)
         attachedHandlers.remove(tag)
       }
     }
@@ -94,32 +95,28 @@ class RNGestureHandlerDetectorView(context: Context) : ReactViewGroup(context) {
     val child = getChildAt(0)
 
     if (child != null) {
-      tryAttachHandlerToChildView(child.id)
+      tryAttachNativeHandlersToChildView(child.id)
     }
   }
 
-  private fun tryAttachHandlerToChildView(childId: Int) {
+  private fun tryAttachNativeHandlersToChildView(childId: Int) {
     val registry = RNGestureHandlerModule.registries[moduleId]
       ?: throw Exception("Tried to access a non-existent registry")
 
-    for (tag in nativeHandlersToAttach) {
+    for (tag in nativeHandlers) {
       registry.attachHandlerToView(tag, childId, GestureHandler.ACTION_TYPE_NATIVE_DETECTOR)
 
       attachedHandlers.add(tag)
     }
-
-    nativeHandlersToAttach.clear()
   }
 
   private fun detachNativeGestureHandlers() {
     val registry = RNGestureHandlerModule.registries[moduleId]
       ?: throw Exception("Tried to access a non-existent registry")
 
-    for (tag in attachedHandlers) {
-      if (shouldAttachGestureToChildView(tag)) {
-        registry.detachHandler(tag)
-        attachedHandlers.remove(tag)
-      }
+    for (tag in nativeHandlers) {
+      registry.detachHandler(tag)
+      attachedHandlers.remove(tag)
     }
   }
 
diff --git a/packages/react-native-gesture-handler/apple/RNGestureHandlerDetector.h b/packages/react-native-gesture-handler/apple/RNGestureHandlerDetector.h
index b049eba7e4..9ee88d0eb2 100644
--- a/packages/react-native-gesture-handler/apple/RNGestureHandlerDetector.h
+++ b/packages/react-native-gesture-handler/apple/RNGestureHandlerDetector.h
@@ -20,7 +20,7 @@ NS_ASSUME_NONNULL_BEGIN
 
 - (void)dispatchTouchEvent:(RNGestureHandlerDetectorEventEmitter::OnGestureHandlerTouchEvent)event;
 
-- (void)tryAttachHandlerToChildView;
+- (void)tryAttachNativeHandlersToChildView;
 
 - (void)detachNativeGestureHandlers;
 
diff --git a/packages/react-native-gesture-handler/apple/RNGestureHandlerDetector.mm b/packages/react-native-gesture-handler/apple/RNGestureHandlerDetector.mm
index 4c7cfadb62..dbf8c076d0 100644
--- a/packages/react-native-gesture-handler/apple/RNGestureHandlerDetector.mm
+++ b/packages/react-native-gesture-handler/apple/RNGestureHandlerDetector.mm
@@ -13,7 +13,7 @@
 
 @interface RNGestureHandlerDetector () <RCTRNGestureHandlerDetectorViewProtocol>
 
-@property (nonatomic, nonnull) NSMutableSet *nativeHandlersToAttach;
+@property (nonatomic, nonnull) NSMutableSet *nativeHandlers;
 @property (nonatomic, nonnull) NSMutableSet *attachedHandlers;
 
 @end
@@ -41,7 +41,7 @@ - (instancetype)initWithFrame:(CGRect)frame
     static const auto defaultProps = std::make_shared<const RNGestureHandlerDetectorProps>();
     _props = defaultProps;
     _moduleId = -1;
-    _nativeHandlersToAttach = [NSMutableSet set];
+    _nativeHandlers = [NSMutableSet set];
     _attachedHandlers = [NSMutableSet set];
   }
 
@@ -94,11 +94,11 @@ - (BOOL)shouldAttachGestureToSubview:(NSNumber *)handlerTag
   return [[[handlerManager registry] handlerWithTag:handlerTag] wantsToAttachDirectlyToView];
 }
 
-- (void)addSubview:(UIView *)view
+- (void)didAddSubview:(UIView *)view
 {
-  [super addSubview:view];
+  [super didAddSubview:view];
 
-  [self tryAttachHandlerToChildView];
+  [self tryAttachNativeHandlersToChildView];
 }
 
 - (void)willRemoveSubview:(UIView *)subview
@@ -152,7 +152,7 @@ - (void)updateProps:(const Props::Shared &)propsBase oldProps:(const Props::Shar
 
     if (handlerChange.second == RNGestureHandlerMutationAttach) {
       if ([self shouldAttachGestureToSubview:handlerTag]) {
-        [_nativeHandlersToAttach addObject:handlerTag];
+        [_nativeHandlers addObject:handlerTag];
       } else {
         [handlerManager.registry attachHandlerWithTag:handlerTag
                                                toView:self
@@ -163,40 +163,38 @@ - (void)updateProps:(const Props::Shared &)propsBase oldProps:(const Props::Shar
     } else if (handlerChange.second == RNGestureHandlerMutationDetach) {
       [handlerManager.registry detachHandlerWithTag:handlerTag];
       [_attachedHandlers removeObject:handlerTag];
+      [_nativeHandlers removeObject:handlerTag];
     }
-
-    [self tryAttachHandlerToChildView];
   }
+  [self tryAttachNativeHandlersToChildView];
 
   [super updateProps:propsBase oldProps:oldPropsBase];
   // Override to force hittesting to work outside bounds
   self.clipsToBounds = NO;
 }
 
-- (void)tryAttachHandlerToChildView
+- (void)tryAttachNativeHandlersToChildView
 {
+  if (!self.subviews[0])
+    return;
   RNGestureHandlerManager *handlerManager = [RNGestureHandlerModule handlerManagerForModuleId:_moduleId];
 
-  for (NSNumber *handlerTag in _nativeHandlersToAttach) {
+  for (NSNumber *handlerTag in _nativeHandlers) {
     [handlerManager.registry attachHandlerWithTag:handlerTag
                                            toView:self.subviews[0]
                                    withActionType:RNGestureHandlerActionTypeNativeDetector];
 
     [_attachedHandlers addObject:handlerTag];
   }
-
-  [_nativeHandlersToAttach removeAllObjects];
 }
 
 - (void)detachNativeGestureHandlers
 {
   RNGestureHandlerManager *handlerManager = [RNGestureHandlerModule handlerManagerForModuleId:_moduleId];
 
-  for (NSNumber *handlerTag in _attachedHandlers) {
-    if ([self shouldAttachGestureToSubview:handlerTag]) {
-      [[handlerManager registry] detachHandlerWithTag:handlerTag];
-      [_attachedHandlers removeObject:handlerTag];
-    }
+  for (NSNumber *handlerTag in _nativeHandlers) {
+    [[handlerManager registry] detachHandlerWithTag:handlerTag];
+    [_attachedHandlers removeObject:handlerTag];
   }
 }