Improves performance

Author: cgranleese-r7

tools/dev/detect_dead_reference_links.rb

@@ -4,20 +4,20 @@
 #
 ##
 
-require 'net/http'
-require 'uri'
+require 'faraday'
 require 'json'
 require 'csv'
 require 'concurrent'
 require 'logger'
 require 'fileutils'
 require 'optparse'
 require 'benchmark'
+require 'oj'  # Optimized JSON
 
 class UrlChecker
   WAYBACK_MACHINE_API_URL = 'https://archive.org/wayback/available?url='
   MAX_REDIRECTS = 5
-  THREAD_POOL_SIZE = 5
+  THREAD_POOL_SIZE = 20  # Increased thread pool size for better concurrency
   CHECKED_URLS_FILE = 'checked_urls.jsonl'
   BATCH_SIZE = 1000
   MAX_RETRIES = 3
@@ -45,7 +45,7 @@ def check_urls
           result = check_url(url_with_path)
           @results << result
           @checked_urls << url_with_path[:url]
-          save_progress(result)
+          save_progress_batch([result])
 
           # Update the progress bar after each URL is processed
           update_progress
@@ -87,15 +87,17 @@ def check_url(url_with_path)
       return url_result
     end
 
-    uri = URI.parse(cleaned_url)
-    http = Net::HTTP.new(uri.host, uri.port)
-    http.use_ssl = uri.scheme == 'https'
+    # Use Faraday for HTTP requests (persistent connection with pooling)
+    conn = Faraday.new(url: cleaned_url) do |faraday|
+      faraday.adapter Faraday.default_adapter  # Net::HTTP
+      faraday.options.timeout = 10  # seconds
+      faraday.options.open_timeout = 5  # seconds
+    end
 
     start_time = Time.now
-
     begin
-      response = get_response(http, uri)
-      follow_redirects(http, uri, response)
+      response = get_response(conn, cleaned_url)
+      follow_redirects(conn, response)
     rescue StandardError => e
       handle_error(url_result, e)
     end
@@ -113,28 +115,27 @@ def valid_url?(url)
     URI.parse(url).is_a?(URI::HTTP) rescue false
   end
 
-  def get_response(http, uri)
-    http.get(uri.request_uri)
+  def get_response(conn, url)
+    conn.get(url)
   end
 
-  def follow_redirects(http, uri, response)
+  def follow_redirects(conn, response)
     redirect_count = 0
-    while response.is_a?(Net::HTTPRedirection) && redirect_count < MAX_REDIRECTS
-      location = response['location']
+    while response.status.to_i == 3 && redirect_count < MAX_REDIRECTS  # HTTP 3xx redirects
+      location = response.headers['Location']
       @logger.info("Redirecting to: #{location}")
-      uri = URI.parse(location)
-      response = http.get(uri.request_uri)
+      response = conn.get(location)
       redirect_count += 1
     end
   end
 
   def process_response(response, url_result)
     if response.nil?
       url_result[:status] = 'Error: No response received'
-    elsif response.is_a?(Net::HTTPSuccess) || response.is_a?(Net::HTTPRedirection)
+    elsif response.status.to_i.between?(200, 299) || response.status.to_i.between?(300, 399)
       url_result[:status] = 'Alive'
     else
-      url_result[:status] = "Not Alive (Status Code: #{response.code})"
+      url_result[:status] = "Not Alive (Status Code: #{response.status})"
       fetch_wayback_snapshot(url_result)
     end
   end
@@ -149,8 +150,7 @@ def fetch_wayback_snapshot(url_result)
     retries = 0
 
     begin
-      uri = URI.parse(wayback_url)
-      response = Net::HTTP.get_response(uri)
+      response = Faraday.get(wayback_url)
       handle_wayback_response(response, url_result)
     rescue StandardError => e
       retries += 1
@@ -165,8 +165,8 @@ def fetch_wayback_snapshot(url_result)
   end
 
   def handle_wayback_response(response, url_result)
-    if response.is_a?(Net::HTTPSuccess)
-      data = JSON.parse(response.body)
+    if response.status.to_i == 200
+      data = Oj.load(response.body)
       snapshot = data.dig('archived_snapshots', 'closest', 'url')
       url_result[:archived_snapshot] = snapshot || 'No archived version found'
     else
@@ -183,6 +183,12 @@ def save_progress(result)
     File.open(CHECKED_URLS_FILE, 'a') { |file| file.puts JSON.generate(result) }
   end
 
+  def save_progress_batch(results)
+    File.open(CHECKED_URLS_FILE, 'a') do |file|
+      results.each { |result| file.puts JSON.generate(result) }
+    end
+  end
+
   def load_checked_urls
     return [] unless File.exist?(CHECKED_URLS_FILE)
 
@@ -226,7 +232,7 @@ def update_progress
   log_level = options[:log_level] || 'INFO'
   log_level = Logger.const_get(log_level)
 
-  urls_with_paths = JSON.parse(File.read(options[:file]))
+  urls_with_paths = Oj.load(File.read(options[:file]))
 
   mapped_data = urls_with_paths.flat_map do |_path, metadata|
     metadata['references'].map { |ref| { 'path' => metadata['path'], 'ref' => ref } }