Retry validation in more cases

Author: RogerPodacter

app/jobs/validation_job.rb

@@ -1,27 +1,25 @@
 class ValidationJob < ApplicationJob
   queue_as :validation
 
-  # Import TransientValidationError from BlockValidator
-  TransientValidationError = BlockValidator::TransientValidationError
-
-  # Only retry transient errors, not all StandardError
-  retry_on TransientValidationError,
+  # Retry all errors - any exception means we couldn't validate, not that validation failed
+  # StandardError catches all normal exceptions (network, RPC, API, etc.)
+  retry_on StandardError,
            wait: ENV.fetch('VALIDATION_RETRY_WAIT_SECONDS', 5).to_i.seconds,
-           attempts: ENV.fetch('VALIDATION_TRANSIENT_RETRIES', 5).to_i
+           attempts: ENV.fetch('VALIDATION_TRANSIENT_RETRIES', 1000).to_i
 
   def perform(l1_block_number, l2_block_hashes)
     start_time = Time.current
 
     # ValidationResult.validate_and_save will:
     # 1. Create ValidationResult with success: true (job succeeds)
     # 2. Create ValidationResult with success: false (job succeeds - real validation failure found)
-    # 3. Raise TransientValidationError (job retries via retry_on, then fails if exhausted)
+    # 3. Raise any exception (job retries via retry_on StandardError)
     ValidationResult.validate_and_save(l1_block_number, l2_block_hashes)
 
     elapsed_time = Time.current - start_time
     Rails.logger.info "ValidationJob: Block #{l1_block_number} validation completed in #{elapsed_time.round(3)}s"
-
-    # Job completes successfully for cases 1 & 2
-    # TransientValidationError will be handled by retry_on automatically
+  rescue => e
+    Rails.logger.error "ValidationJob failed for L1 #{l1_block_number}: #{e.class}: #{e.message}"
+    raise
   end
 end

app/models/validation_result.rb

@@ -62,70 +62,42 @@ def self.recent_failures(limit: 10)
   def self.validate_and_save(l1_block_number, l2_block_hashes)
     Rails.logger.info "ValidationResult: Validating L1 block #{l1_block_number}"
 
-    begin
-      # Create validator and validate (validator fetches its own API data)
-      validator = BlockValidator.new
-      start_time = Time.current
-      block_result = validator.validate_l1_block(l1_block_number, l2_block_hashes)
+    # Create validator and validate (validator fetches its own API data)
+    validator = BlockValidator.new
+    start_time = Time.current
+    block_result = validator.validate_l1_block(l1_block_number, l2_block_hashes)
+
+    # Find or initialize - idempotent for re-runs
+    validation_result = find_or_initialize_by(l1_block: l1_block_number)
+
+    validation_result.assign_attributes(
+      success: block_result.success,
+      error_details: block_result.errors,
+      validation_stats: {
+        # Basic stats
+        success: block_result.success,
+        l1_block: l1_block_number,
+        l2_blocks: l2_block_hashes,
 
-      # Find or initialize - idempotent for re-runs
-      validation_result = find_or_initialize_by(l1_block: l1_block_number)
+        # Detailed comparison data
+        validation_details: block_result.stats,
 
-      validation_result.assign_attributes(
-        success: block_result.success,
-        error_details: block_result.errors,
-        validation_stats: {
-          # Basic stats
-          success: block_result.success,
-          l1_block: l1_block_number,
-          l2_blocks: l2_block_hashes,
-
-          # Detailed comparison data
-          validation_details: block_result.stats,
-
-          # Store the raw data for debugging
-          raw_api_data: block_result.respond_to?(:api_data) ? block_result.api_data : nil,
-          raw_l2_events: block_result.respond_to?(:l2_events) ? block_result.l2_events : nil,
-
-          # Timing info
-          validation_duration_ms: ((Time.current - start_time) * 1000).round(2)
-        },
-        validated_at: Time.current
-      )
+        # Store the raw data for debugging
+        raw_api_data: block_result.respond_to?(:api_data) ? block_result.api_data : nil,
+        raw_l2_events: block_result.respond_to?(:l2_events) ? block_result.l2_events : nil,
 
-      validation_result.save!
-
-      # Log the result
-      validation_result.log_summary
-
-      validation_result
-    rescue BlockValidator::TransientValidationError => e
-      # Don't persist transient errors - let ValidationJob handle retries
-      Rails.logger.debug "ValidationResult: Transient error for block #{l1_block_number}: #{e.message}"
-      raise e
-    rescue => e
-      Rails.logger.error "ValidationResult: Exception validating block #{l1_block_number}: #{e.message}"
-
-      # Only persist non-transient validation errors - idempotent for re-runs
-      validation_result = find_or_initialize_by(l1_block: l1_block_number)
-
-      validation_result.assign_attributes(
-        success: false,
-        error_details: [e.message],
-        validation_stats: {
-          exception: true,
-          exception_class: e.class.name,
-          exception_message: e.message,
-          exception_backtrace: e.backtrace&.first(10)  # Store first 10 lines of backtrace
-        },
-        validated_at: Time.current
-      )
+        # Timing info
+        validation_duration_ms: ((Time.current - start_time) * 1000).round(2)
+      },
+      validated_at: Time.current
+    )
 
-      validation_result.save!
+    validation_result.save!
 
-      validation_result.log_summary
-      raise e
-    end
+    # Log the result
+    validation_result.log_summary
+
+    validation_result
   end
 
   # Instance methods

lib/block_validator.rb

@@ -2,6 +2,7 @@ class BlockValidator
   attr_reader :errors, :stats
 
   # Exception for transient errors that should trigger retries
+  # This is informational - all exceptions are treated as transient
   class TransientValidationError < StandardError; end
 
   def initialize
@@ -117,15 +118,10 @@ def load_genesis_transaction_hashes
 
   def fetch_expected_data(l1_block_number)
     EthscriptionsApiClient.fetch_block_data(l1_block_number)
-  rescue EthscriptionsApiClient::ApiUnavailableError => e
-    # API unavailable after exhausting all retries - this is an infrastructure issue
-    Rails.logger.warn "API unavailable for block #{l1_block_number}: #{e.message}"
-    raise TransientValidationError, e.message
   rescue => e
-    # Other unexpected errors - log and continue with empty data
-    message = "Unexpected error fetching API data: #{e.message}"
-    @errors << message
-    {creations: [], transfers: []}
+    # Treat any API client failure as transient to avoid false negatives
+    Rails.logger.warn "Transient API error for block #{l1_block_number}: #{e.class}: #{e.message}"
+    raise TransientValidationError, e.message
   end
 
   def aggregate_l2_events(block_hashes)
@@ -137,24 +133,20 @@ def aggregate_l2_events(block_hashes)
       begin
         receipts = EthRpcClient.l2.call('eth_getBlockReceipts', [block_hash])
         if receipts.nil?
+          # Treat missing receipts as transient infrastructure issue
           error_msg = "No receipts returned for L2 block #{block_hash}"
-          @errors << error_msg
-          # Treat missing receipts as potentially transient
+          Rails.logger.warn "Transient L2 error: #{error_msg}"
           raise TransientValidationError, error_msg
         end
 
         data = EventDecoder.decode_block_receipts(receipts)
         all_creations.concat(data[:creations])
         all_transfers.concat(data[:transfers])  # Ethscriptions protocol transfers
       rescue => e
+        # Treat any L2 RPC failure as transient to avoid false negatives
         error_msg = "Failed to get receipts for block #{block_hash}: #{e.message}"
-        @errors << error_msg
-        # Classify L2 receipt fetch errors - network issues are transient
-        if transient_error?(e)
-          raise TransientValidationError, error_msg
-        else
-          raise
-        end
+        Rails.logger.warn "Transient L2 error: #{error_msg}"
+        raise TransientValidationError, error_msg
       end
     end
 
@@ -348,14 +340,15 @@ def verify_ethscription_storage(creation, l1_block_num, block_tag)
     begin
       stored = StorageReader.get_ethscription_with_content(tx_hash, block_tag: block_tag)
     rescue => e
-      @errors << "Ethscription #{tx_hash} not found in contract storage: #{e.message}"
-      @storage_checks_performed.increment
-      return
+      # RPC/network error - treat as transient inability to validate
+      Rails.logger.warn "Transient storage error for #{tx_hash}: #{e.message}"
+      raise TransientValidationError, "Storage read failed for #{tx_hash}: #{e.message}"
     end
 
     @storage_checks_performed.increment
 
     if stored.nil?
+      # Ethscription genuinely doesn't exist in contract - this is a validation failure
       @errors << "Ethscription #{tx_hash} not found in contract storage"
       return
     end
@@ -509,18 +502,32 @@ def verify_transfer_ownership(transfers, block_tag)
     # Verify each token's final owner
     final_owners.each do |token_id, expected_owner|
       # First check if the ethscription exists in storage
-      ethscription = StorageReader.get_ethscription(token_id, block_tag: block_tag)
+      begin
+        ethscription = StorageReader.get_ethscription(token_id, block_tag: block_tag)
+      rescue => e
+        # RPC/network error - treat as transient inability to validate
+        Rails.logger.warn "Transient storage error for #{token_id}: #{e.message}"
+        raise TransientValidationError, "Storage read failed for #{token_id}: #{e.message}"
+      end
 
       if ethscription.nil?
-        # Token doesn't exist yet - treat as fatal divergence
+        # Token genuinely doesn't exist - this is a validation failure
         @errors << "Token #{token_id} not found in storage"
         next
       end
 
-      actual_owner = StorageReader.get_owner(token_id, block_tag: block_tag)
+      begin
+        actual_owner = StorageReader.get_owner(token_id, block_tag: block_tag)
+      rescue => e
+        # RPC/network error - treat as transient inability to validate
+        Rails.logger.warn "Transient owner read error for #{token_id}: #{e.message}"
+        raise TransientValidationError, "Owner read failed for #{token_id}: #{e.message}"
+      end
+
       @storage_checks_performed.increment
 
       if actual_owner.nil?
+        # Owner doesn't exist (shouldn't happen if ethscription exists) - validation failure
         @errors << "Could not verify owner of token #{token_id}"
         next
       end
@@ -570,20 +577,6 @@ def safe_content_preview(content, length: 50)
     preview + (content.length > length ? "..." : "")
   end
 
-  # Classify L2 RPC errors as transient (infrastructure) vs permanent (logic)
-  def transient_error?(error)
-    case error
-    # L2 RPC network errors
-    when SocketError, Errno::ECONNREFUSED, Errno::ECONNRESET,
-         Net::OpenTimeout, Net::ReadTimeout
-      true
-    # L2 RPC client errors that might be transient
-    when EthRpcClient::HttpError, EthRpcClient::ApiError
-      true
-    else
-      false
-    end
-  end
 
   # Sanitize data structures for JSON serialization
   def sanitize_for_json(data)
@@ -604,4 +597,3 @@ def sanitize_for_json(data)
     end
   end
 end
-

lib/storage_reader.rb

@@ -101,7 +101,11 @@ def get_ethscription_with_content(tx_hash, block_tag: 'latest')
 
       # Make the eth_call
       result = eth_call('0x' + calldata.unpack1('H*'), block_tag)
-      raise StandardError, "Ethscription not found: #{tx_hash}" if result.nil? || result == '0x' || result == '0x0'
+      # When contract returns 0x/0x0, the ethscription doesn't exist (not an error, just not found)
+      return nil if result == '0x' || result == '0x0'
+
+      # If result is nil, that's an RPC/network error
+      raise StandardError, "RPC call failed for ethscription #{tx_hash}" if result.nil?
 
       # Decode the tuple: (Ethscription, bytes)
       types = ['((bytes32,bytes32,string,string,string,bool),address,address,address,uint256,uint256,uint64,uint64,bytes32)', 'bytes']
@@ -152,7 +156,10 @@ def get_ethscription(tx_hash, block_tag: 'latest')
 
       # Make the eth_call
       result = eth_call('0x' + calldata.unpack1('H*'), block_tag)
-      return nil if result.nil? || result == '0x' || result == '0x0'
+      # Deterministic not-found from contract returns 0x/0x0
+      return nil if result == '0x' || result == '0x0'
+      # Nil indicates an RPC/network failure
+      raise StandardError, "RPC call failed for ethscription #{tx_hash}" if result.nil?
 
       # Decode using Eth::Abi
       # Updated types for nested struct: ContentInfo is a tuple within the main tuple
@@ -225,7 +232,10 @@ def get_owner(token_id, block_tag: 'latest')
 
       # Make the eth_call
       result = eth_call('0x' + calldata.unpack1('H*'), block_tag)
-      return nil if result.nil? || result == '0x'
+      # Some nodes return 0x when the call yields no data
+      return nil if result == '0x'
+      # Nil indicates an RPC/network failure
+      raise StandardError, "RPC call failed for ownerOf #{token_id}" if result.nil?
 
       # Decode the result - ownerOf returns a single address
       decoded = Eth::Abi.decode(['address'], result)