Handle pending connections losing context on frame navigation

Author: francisbeaudoin

lib/ferrum/network.rb

@@ -86,11 +86,11 @@ def idle?(connections = 0)
     end
 
     def total_connections
-      @traffic.size
+      exchange_connections.count
     end
 
     def finished_connections
-      @traffic.count(&:finished?)
+      exchange_connections.count(&:finished?)
     end
 
     def pending_connections
@@ -389,7 +389,10 @@ def subscribe_request_will_be_sent
         request.headers.merge!(Hash(exchange.request_extra_info&.dig("headers")))
         exchange.request = request
 
-        @exchange = exchange if exchange.navigation_request?(@page.main_frame.id)
+        if exchange.navigation_request?(@page.main_frame.id)
+          @exchange = exchange
+          mark_pending_exchanges_as_unknown(exchange)
+        end
       end
 
       @page.on("Network.requestWillBeSentExtraInfo") do |params|
@@ -400,6 +403,17 @@ def subscribe_request_will_be_sent
       end
     end
 
+    # When the main frame navigates Chrome doesn't send `Network.loadingFailed`
+    # for pending async requests. Therefore, we mark pending connections as unknown since
+    # they are not relevant to the current navigation.
+    def mark_pending_exchanges_as_unknown(navigation_exchange)
+      @traffic.each do |exchange|
+        break if exchange.id == navigation_exchange.id
+
+        exchange.unknown = true if exchange.pending?
+      end
+    end
+
     def subscribe_response_received
       @page.on("Network.responseReceived") do |params|
         exchange = select(params["requestId"]).last
@@ -415,6 +429,8 @@ def subscribe_loading_finished
         exchange = select(params["requestId"]).last
         next unless exchange
 
+        exchange.unknown = false
+
         if (response = exchange.response)
           response.loaded = true
           response.body_size = params["encodedDataLength"]
@@ -497,5 +513,16 @@ def blacklist?
     def whitelist?
       Array(@whitelist).any?
     end
+
+    def exchange_connections
+      @traffic.select { |e| exchange_connection?(e) }
+    end
+
+    def exchange_connection?(exchange)
+      return false if !@page.frames.any? { |f| f.id == exchange.request&.frame_id }
+      return false if exchange.request&.frame_id != @exchange.request&.frame_id
+
+      exchange.request&.loader_id == @exchange.request&.loader_id
+    end
   end
 end

lib/ferrum/network/exchange.rb

@@ -28,6 +28,12 @@ class Exchange
       # @return [Error, nil]
       attr_accessor :error
 
+      # Determines if the network exchange is unknown due to
+      # a lost of its context
+      #
+      # @return Boolean
+      attr_accessor :unknown
+
       # @api private
       attr_accessor :request_extra_info
 
@@ -44,6 +50,7 @@ def initialize(page, id)
         @intercepted_request = nil
         @request = @response = @error = nil
         @request_extra_info = nil
+        @unknown = false
       end
 
       #
@@ -78,12 +85,12 @@ def blocked?
 
       #
       # Determines if the request was blocked, a response was returned, or if an
-      # error occurred.
+      # error occurred or the exchange is unknown and cannot be inferred.
       #
       # @return [Boolean]
       #
       def finished?
-        blocked? || response&.loaded? || !error.nil? || ping? || blob?
+        blocked? || response&.loaded? || !error.nil? || ping? || blob? || unknown
       end
 
       #
@@ -169,7 +176,8 @@ def inspect
           "@intercepted_request=#{@intercepted_request.inspect} " \
           "@request=#{@request.inspect} " \
           "@response=#{@response.inspect} " \
-          "@error=#{@error.inspect}>"
+          "@error=#{@error.inspect}> " \
+          "@unknown=#{@unknown.inspect}>"
       end
     end
   end

lib/ferrum/network/request.rb

@@ -71,6 +71,15 @@ def frame_id
         @params["frameId"]
       end
 
+      #
+      # The loader ID of the request.
+      #
+      # @return [String]
+      #
+      def loader_id
+        @params["loaderId"]
+      end
+
       #
       # The request timestamp.
       #

lib/ferrum/page/frames.rb

@@ -177,12 +177,16 @@ def subscribe_execution_context_destroyed
           execution_id = params["executionContextId"]
           frame = frame_by(execution_id: execution_id)
           frame&.execution_id = nil
+          frame&.state = :stopped_loading
         end
       end
 
       def subscribe_execution_contexts_cleared
         on("Runtime.executionContextsCleared") do
-          @frames.each_value { |f| f.execution_id = nil }
+          @frames.each_value do |f|
+            f.execution_id = nil
+            f.state = :stopped_loading
+          end
         end
       end