Final tidy up

Author: cgranleese-r7

tools/dev/detect_dead_reference_links.rb

@@ -25,28 +25,27 @@ class UrlChecker
   MAX_RETRIES = 3 # Maximum number of retries for failed requests to the Wayback Machine
   RETRY_DELAY = 5 # Delay in seconds between retries
 
-  # Initializes the URL checker with the given URLs and logging options
-  #
-  # @param urls_with_paths [Array<Hash>] An array of hashes containing URL and path information.
-  # @param log_level [Logger::Severity] The desired logging level (default: Logger::INFO)
+  # Initializes the UrlChecker instance with given URLs and configuration options
+  # @param [Array<Hash>] urls_with_paths A list of URLs with associated paths to check
+  # @param [Logger::Level] log_level The logging level (defaults to Logger::INFO)
   def initialize(urls_with_paths, log_level: Logger::INFO)
     @urls_with_paths = urls_with_paths
     @results = []
     @checked_urls = load_checked_urls
     @url_times = []
-    @logger = Logger.new(STDOUT)
+    @logger = Logger.new($stdout)
     @logger.level = log_level
     @total_urls = urls_with_paths.size
     @processed_urls = 0
   end
 
-  # Checks the provided URLs for status and fetches Wayback Machine snapshots if needed.
-  # URLs are processed in batches, and results are saved to a file.
+  # Starts the process of checking all URLs in batches, logging results and saving progress
+  # in a thread-safe manner.
   def check_urls
     pool = Concurrent::FixedThreadPool.new(THREAD_POOL_SIZE)
     at_exit { shutdown_thread_pool(pool) }
 
-    # Process URLs in batches to prevent excessive memory usage
+    # Process URLs in batches to avoid overwhelming the system
     @urls_with_paths.each_slice(BATCH_SIZE) do |batch|
       futures = batch.map do |url_with_path|
         Concurrent::Promises.future(pool) do
@@ -55,18 +54,17 @@ def check_urls
           @checked_urls << url_with_path[:url]
           save_progress(result)
 
-          # Update the progress bar after each URL is processed
           update_progress
         end
       end
 
-      # Wait for all futures in the batch to finish
+      # Wait for all futures in the current batch to finish before proceeding
       Concurrent::Promises.zip(*futures).wait!
-      # Sleep between batches to reduce resource consumption
+
+      # Sleep between batches to avoid overloading the server
       sleep 5
     end
 
-    # Save the results to a file after all URLs are processed
     save_results_to_file
   ensure
     pool.shutdown
@@ -76,21 +74,19 @@ def check_urls
 
   private
 
-  # Returns the URLs that haven't been checked yet
-  #
-  # @return [Array<Hash>] An array of URLs with paths that haven't been checked
+  # Filters out URLs that have already been checked.
+  # @return [Array<Hash>] List of URLs and paths that have not been checked yet
   def unchecked_urls
     @urls_with_paths.reject { |url_with_path| @checked_urls.include?(url_with_path[:url]) }
   end
 
-  # Checks the status of a single URL.
-  #
-  # @param url_with_path [Hash] The URL and associated path to check
-  # @return [Hash] A hash with the URL, path, status, and archived snapshot (if any)
+  # Checks a single URL and processes its response.
+  # @param [Hash] url_with_path The URL and its associated path to check
+  # @return [Hash] A result containing the URL, path, status, and archived snapshot (if available)
   def check_url(url_with_path)
     url_result = { url: url_with_path[:url], path: url_with_path[:path], status: nil, archived_snapshot: nil }
 
-    # Skip non-URL references or Wayback links
+    # Skip non-URL references and Wayback links
     if !url_with_path[:url].start_with?('URL-')
       url_result[:status] = 'Skipped (not a URL- reference)'
       return url_result
@@ -101,8 +97,10 @@ def check_url(url_with_path)
 
     # Clean the URL and validate it
     cleaned_url = url_with_path[:url].sub(/^URL-/, '')
+
+    # Check if the URL is valid
     if !valid_url?(cleaned_url)
-      url_result[:status] = "Invalid URL"
+      url_result[:status] = 'Invalid URL'
       return url_result
     end
 
@@ -114,12 +112,14 @@ def check_url(url_with_path)
     start_time = Time.now
 
     begin
+      # Get the HTTP response and handle redirects
       response = get_response(http, uri)
       follow_redirects(http, uri, response)
     rescue StandardError => e
       handle_error(url_result, e)
     end
 
+    # Process the response (check for success, failure, or error)
     process_response(response, url_result)
     elapsed_time = Time.now - start_time
     @url_times << elapsed_time
@@ -129,28 +129,27 @@ def check_url(url_with_path)
     save_progress(url_result)
   end
 
-  # Checks if a given URL is valid.
-  #
-  # @param url [String] The URL to check
+  # Validates if a URL is properly formatted
+  # @param [String] url The URL to check
   # @return [Boolean] True if the URL is valid, false otherwise
   def valid_url?(url)
-    URI.parse(url).is_a?(URI::HTTP) rescue false
+    URI.parse(url).is_a?(URI::HTTP)
+  rescue StandardError
+    false
   end
 
-  # Sends an HTTP GET request to the specified URI.
-  #
-  # @param http [Net::HTTP] The HTTP client to use
-  # @param uri [URI] The URI to send the GET request to
-  # @return [Net::HTTPResponse] The response from the HTTP request
+  # Sends an HTTP GET request to the specified URI
+  # @param [Net::HTTP] http The HTTP client
+  # @param [URI] uri The URI to send the GET request to
+  # @return [Net::HTTPResponse] The HTTP response
   def get_response(http, uri)
     http.get(uri.request_uri)
   end
 
-  # Follows HTTP redirects until the maximum redirect limit is reached.
-  #
-  # @param http [Net::HTTP] The HTTP client to use
-  # @param uri [URI] The current URI to check
-  # @param response [Net::HTTPResponse] The current response from the server
+  # Follows HTTP redirects up to a maximum limit (MAX_REDIRECTS)
+  # @param [Net::HTTP] http The HTTP client
+  # @param [URI] uri The original URI
+  # @param [Net::HTTPResponse] response The HTTP response to process
   def follow_redirects(http, uri, response)
     redirect_count = 0
     while response.is_a?(Net::HTTPRedirection) && redirect_count < MAX_REDIRECTS
@@ -162,10 +161,9 @@ def follow_redirects(http, uri, response)
     end
   end
 
-  # Processes the HTTP response and updates the URL result with its status.
-  #
-  # @param response [Net::HTTPResponse] The HTTP response to process
-  # @param url_result [Hash] The result hash for the URL
+  # Processes the HTTP response to determine the URL status
+  # @param [Net::HTTPResponse] response The HTTP response to process
+  # @param [Hash] url_result The result hash to update with the status
   def process_response(response, url_result)
     if response.nil?
       url_result[:status] = 'Error: No response received'
@@ -177,18 +175,16 @@ def process_response(response, url_result)
     end
   end
 
-  # Handles errors by logging them and setting the error status on the URL result.
-  #
-  # @param url_result [Hash] The result hash for the URL
-  # @param error [StandardError] The error encountered
+  # Handles errors encountered during URL checking (e.g., network errors)
+  # @param [Hash] url_result The result hash to update with error information
+  # @param [StandardError] error The error that was raised
   def handle_error(url_result, error)
     url_result[:status] = "Error: #{error.message}"
     url_result[:archived_snapshot] = nil
   end
 
-  # Fetches the archived snapshot of a URL from the Wayback Machine.
-  #
-  # @param url_result [Hash] The result hash for the URL
+  # Attempts to fetch the Wayback Machine snapshot for the URL
+  # @param [Hash] url_result The result hash to update with the Wayback snapshot information
   def fetch_wayback_snapshot(url_result)
     wayback_url = "#{WAYBACK_MACHINE_API_URL}#{url_result[:url]}"
     retries = 0
@@ -209,10 +205,9 @@ def fetch_wayback_snapshot(url_result)
     end
   end
 
-  # Handles the Wayback Machine response and updates the URL result with the archived snapshot.
-  #
-  # @param response [Net::HTTPResponse] The Wayback Machine response
-  # @param url_result [Hash] The result hash for the URL
+  # Processes the response from the Wayback Machine API
+  # @param [Net::HTTPResponse] response The response from the Wayback Machine
+  # @param [Hash] url_result The result hash to update with the archived snapshot URL
   def handle_wayback_response(response, url_result)
     if response.is_a?(Net::HTTPSuccess)
       data = JSON.parse(response.body)
@@ -223,38 +218,35 @@ def handle_wayback_response(response, url_result)
     end
   end
 
-  # Saves the results of the URL checks to a file.
+  # Saves the final results of the URL checks to a JSON file
   def save_results_to_file
     File.open('url_check_results.json', 'w') { |file| file.write(JSON.pretty_generate(@results)) }
     @logger.info('Results have been saved to "url_check_results.json".')
   end
 
-  # Appends the progress of a URL check to the checked URLs file.
-  #
-  # @param result [Hash] The result of the URL check to save
+  # Saves the progress of checked URLs to a file
+  # @param [Hash] result The result of a single URL check
   def save_progress(result)
     File.open(CHECKED_URLS_FILE, 'a') { |file| file.puts JSON.generate(result) }
   end
 
-  # Loads the URLs that have already been checked from the file.
-  #
-  # @return [Array<String>] An array of URLs that have been checked previously
+  # Loads the list of already checked URLs from the progress file
+  # @return [Array<String>] A list of checked URLs
   def load_checked_urls
     return [] unless File.exist?(CHECKED_URLS_FILE)
 
     File.readlines(CHECKED_URLS_FILE).map { |row| JSON.parse(row)['url'] }
   end
 
-  # Shuts down the thread pool and waits for all threads to finish.
-  #
-  # @param pool [Concurrent::FixedThreadPool] The thread pool to shut down
+  # Shuts down the thread pool after URL checking is complete
+  # @param [Concurrent::FixedThreadPool] pool The thread pool to shut down
   def shutdown_thread_pool(pool)
     pool.shutdown
     pool.wait_for_termination
     @logger.info('Thread pool shut down successfully.')
   end
 
-  # Updates the progress bar and prints the progress to the console.
+  # Updates the progress bar based on the number of URLs processed
   def update_progress
     @processed_urls += 1
     percentage = (@processed_urls.to_f / @total_urls * 100).round
@@ -264,27 +256,28 @@ def update_progress
   end
 end
 
-if __FILE__ == $0
+# Main entry point to run the URL checking process
+if __FILE__ == $PROGRAM_NAME
   options = {}
   OptionParser.new do |opts|
-    opts.banner = "Usage: ruby url_checker.rb [options]"
+    opts.banner = 'Usage: ruby url_checker.rb [options]'
 
-    opts.on("-f", "--file FILE", "JSON file containing URLs and paths") do |file|
+    opts.on('-f', '--file FILE', 'JSON file containing URLs and paths') do |file|
       options[:file] = file
     end
 
-    opts.on("-l", "--log-level LEVEL", "Log level (DEBUG, INFO, WARN, ERROR, FATAL, UNKNOWN)") do |log_level|
+    opts.on('-l', '--log-level LEVEL', 'Log level (DEBUG, INFO, WARN, ERROR, FATAL, UNKNOWN)') do |log_level|
       options[:log_level] = log_level.upcase.to_sym
     end
   end.parse!
 
   # Validate input file
   unless options[:file] && File.exist?(options[:file])
-    puts "Please provide a valid JSON file with URLs and paths."
+    puts 'Please provide a valid JSON file with URLs and paths.'
     exit 1
   end
 
-  # Set the desired log level
+  # Handling for log level
   log_level = options[:log_level] || 'INFO'
   log_level = Logger.const_get(log_level)
 
@@ -305,15 +298,15 @@ def update_progress
   # Create the final list of URLs and paths
   urls_with_paths_final = mapped_data.map { |entry| { url: entry['ref'], path: entry['path'] } }
 
-  # Record the start time for performance tracking
   start_time = Time.now
 
   # Create and run the UrlChecker instance
   url_checker = UrlChecker.new(urls_with_paths_final, log_level: log_level)
   url_checker.check_urls
 
-  # Calculate and display the total time taken
+
   end_time = Time.now
+  # Calculate and display the total time taken
   elapsed_time = end_time - start_time
   minutes = (elapsed_time / 60).to_i
   seconds = (elapsed_time % 60).to_i

tools/dev/find_and_replace_dead_reference_links.rb

@@ -10,32 +10,46 @@
 require 'json'
 require 'fileutils'
 
+# Loads JSON data from the specified file.
+# @param file_path [String] the path to the JSON file to load.
+# @return [Array] parsed JSON data.
+# @raise [Errno::ENOENT] if the file cannot be found.
+# @raise [JSON::ParserError] if the JSON is malformed.
 def load_json(file_path)
   JSON.parse(File.read(file_path))
 end
 
+# Replaces the original URLs with archived snapshots in the content of files.
+# This method processes each entry in the provided data, and if a valid
+# archived snapshot is available, it replaces the URL in the corresponding file.
+# @param data [Array] the array of data containing URL and archived_snapshot pairs.
+# @return [void]
 def replace_links_in_files(data)
   data.each_with_index do |entry, index|
     puts "Processing entry #{index + 1}: #{entry['url']} -> #{entry['archived_snapshot']}"
 
-    url = entry["url"].sub(/^URL-/, '')
-    path = entry["path"]
-    archived_snapshot = entry["archived_snapshot"]
+    url = entry['url'].sub(/^URL-/, '')
+    path = entry['path']
+    archived_snapshot = entry['archived_snapshot']
 
-    if archived_snapshot == "No archived version found" || archived_snapshot.nil?
-      puts "Skipping entry #{index + 1} because no archived version is available."
+    # Skip entries with no archived version or errors fetching the snapshot
+    if archived_snapshot == 'No archived version found' || archived_snapshot.nil? || archived_snapshot.start_with?('Error fetching Wayback')
+      puts "Skipping entry #{index + 1} because no archived version is available or there was an error fetching it."
       next
     end
 
+    # Construct full file path and check if file exists
     full_path = File.join(Dir.pwd, path)
 
     if File.exist?(full_path)
       file_content = File.read(full_path)
 
+      # Replace the original URL with the archived snapshot
       updated_content = file_content.gsub(url, archived_snapshot)
 
+      # Write changes back to the file if any replacements were made
       if file_content != updated_content
-        File.open(full_path, "w") { |file| file.write(updated_content) }
+        File.open(full_path, 'w') { |file| file.write(updated_content) }
         puts "Replaced URL in file: #{full_path}"
       else
         puts "No change needed for file: #{full_path}"
@@ -47,8 +61,12 @@ def replace_links_in_files(data)
 end
 
 begin
-  json_data = load_json('url_check_results.json') # Change this to the actual JSON file path
+  # Load the JSON data from the file 'url_check_results.json'
+  json_data = load_json('url_check_results.json')
+
+  # Replace the URLs in files based on the loaded data
   replace_links_in_files(json_data)
-rescue => e
+rescue StandardError => e
+  # Handle errors gracefully and provide meaningful feedback
   puts "An error occurred: #{e.message}"
 end