Add RichText link checker functionality and related specs

rsmithlal · rsmithlal · commit 44c6d458e851 · 2025-09-02T18:16:32.000-02:30
- Implement RichTextLinkIdentifier service to extract and persist links from ActionText::RichText records.
- Create ExternalLinkCheckerJob and InternalLinkCheckerJob for checking link validity.
- Update Rake tasks for identifying and checking links.
- Add specs for the new jobs and the RichTextLinkIdentifier service.
- Include WebMock for stubbing HTTP requests in tests.
- Add diagrams and documentation for the link checking process.
- Introduce WebMock gem for stubbing external HTTP requests in specs.
diff --git a/Gemfile b/Gemfile
@@ -98,6 +98,8 @@ group :test do
   # Capybara for integration testing
   gem 'capybara', '>= 2.15'
   gem 'capybara-screenshot'
+  # WebMock for stubbing external HTTP requests in specs
+  gem 'webmock'
   # Coveralls for test coverage reporting
   gem 'coveralls_reborn', require: false
   # Database cleaner for test database cleaning
diff --git a/Gemfile.lock b/Gemfile.lock
@@ -229,6 +229,9 @@ GEM
       term-ansicolor (~> 1.7)
       thor (~> 1.2)
       tins (~> 1.32)
+    crack (1.0.0)
+      bigdecimal
+      rexml
     crass (1.0.6)
     css_parser (1.21.1)
       addressable
@@ -362,6 +365,7 @@ GEM
       rake (>= 13)
     groupdate (6.7.0)
       activesupport (>= 7.1)
+    hashdiff (1.2.0)
     hashie (5.0.0)
     highline (3.1.2)
       reline
@@ -801,6 +805,10 @@ GEM
       activemodel (>= 6.0.0)
       bindex (>= 0.4.0)
       railties (>= 6.0.0)
+    webmock (3.25.1)
+      addressable (>= 2.8.0)
+      crack (>= 0.3.2)
+      hashdiff (>= 0.4.0, < 2.0.0)
     websocket (1.2.11)
     websocket-driver (0.8.0)
       base64
@@ -873,6 +881,7 @@ DEPENDENCIES
   storext!
   uglifier (>= 1.3.0)
   web-console (>= 3.3.0)
+  webmock
 
 RUBY VERSION
    ruby 3.4.4p34
diff --git a/app/jobs/better_together/metrics/external_link_checker_job.rb b/app/jobs/better_together/metrics/external_link_checker_job.rb
@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+require 'net/http'
+require 'uri'
+
+module BetterTogether
+  module Metrics
+    class ExternalLinkCheckerJob < ApplicationJob
+      queue_as :default
+
+      def perform(link_id)
+        link = BetterTogether::Content::Link.find(link_id)
+        uri = URI.parse(link.url)
+        response = http_head(uri)
+
+        link.update!(last_checked_at: Time.current, latest_status_code: response.code.to_s, valid_link: response.is_a?(Net::HTTPSuccess))
+      rescue StandardError => e
+        link.update!(last_checked_at: Time.current, latest_status_code: nil, valid_link: false, error_message: e.message)
+      end
+
+      private
+
+      def http_head(uri)
+        Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == 'https', open_timeout: 5, read_timeout: 5) do |http|
+          request = Net::HTTP::Head.new(uri.request_uri)
+          http.request(request)
+        end
+      end
+    end
+  end
+end
diff --git a/app/jobs/better_together/metrics/internal_link_checker_job.rb b/app/jobs/better_together/metrics/internal_link_checker_job.rb
@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+require 'net/http'
+require 'uri'
+
+module BetterTogether
+  module Metrics
+    class InternalLinkCheckerJob < ApplicationJob
+      queue_as :default
+
+      def perform(link_id)
+        link = BetterTogether::Content::Link.find(link_id)
+        uri = URI.parse(link.url)
+        response = http_head(uri)
+
+        link.update!(last_checked_at: Time.current, latest_status_code: response.code.to_s, valid_link: response.is_a?(Net::HTTPSuccess))
+      rescue StandardError => e
+        link.update!(last_checked_at: Time.current, latest_status_code: nil, valid_link: false, error_message: e.message)
+      end
+
+      private
+
+      def http_head(uri)
+        Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == 'https', open_timeout: 5, read_timeout: 5) do |http|
+          request = Net::HTTP::Head.new(uri.request_uri)
+          http.request(request)
+        end
+      end
+    end
+  end
+end
diff --git a/app/jobs/better_together/metrics/rich_text_external_link_checker_queue_job.rb b/app/jobs/better_together/metrics/rich_text_external_link_checker_queue_job.rb
@@ -11,6 +11,10 @@ class RichTextExternalLinkCheckerQueueJob < RichTextLinkCheckerQueueJob
       def model_collection
         super.where(link_type: 'external')
       end
+      
+      def child_job_class
+        BetterTogether::Metrics::ExternalLinkCheckerJob
+      end
     end
   end
 end
diff --git a/app/jobs/better_together/metrics/rich_text_internal_link_checker_queue_job.rb b/app/jobs/better_together/metrics/rich_text_internal_link_checker_queue_job.rb
@@ -15,6 +15,10 @@ def model_collection
       def queue_delay
         5.minutes
       end
+      
+      def child_job_class
+        BetterTogether::Metrics::InternalLinkCheckerJob
+      end
     end
   end
 end
diff --git a/app/services/better_together/metrics/rich_text_link_identifier.rb b/app/services/better_together/metrics/rich_text_link_identifier.rb
@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+module BetterTogether
+  module Metrics
+    # Service to scan ActionText::RichText records, extract links, and persist
+    # both the link metadata (BetterTogether::Content::Link) and the join
+    # records (BetterTogether::Metrics::RichTextLink).
+    #
+    # Usage:
+    #   BetterTogether::Metrics::RichTextLinkIdentifier.call
+    class RichTextLinkIdentifier
+      def self.call(rich_texts: nil)
+        new(rich_texts: rich_texts).call
+      end
+
+      def initialize(rich_texts: nil)
+        @rich_texts = rich_texts
+      end
+
+      def call
+        texts = rich_texts || ActionText::RichText.includes(:record).where.not(body: nil)
+        valid_count = 0
+        invalid_count = 0
+
+        texts.find_each do |rt|
+          links = extract_links(rt)
+          next if links.empty?
+
+          links.each_with_index do |link, index|
+            uri = parse_uri(link)
+            if uri.nil? || (uri.host.nil? && uri.scheme.nil?)
+              create_invalid(rt, index, link, 'undetermined')
+              invalid_count += 1
+              next
+            end
+
+            # Create or find the canonical Link record
+            bt_link = BetterTogether::Content::Link.find_or_initialize_by(url: link)
+            bt_link.host ||= uri.host
+            bt_link.scheme ||= uri.scheme
+            bt_link.external = (uri.host.present? && (rt_platform_host != uri.host))
+            bt_link.save! if bt_link.changed?
+
+            # Create or update the rich text link join record
+            attrs = {
+              link_id: bt_link.id,
+              rich_text_id: rt.id,
+              rich_text_record_id: rt.record_id,
+              rich_text_record_type: rt.record_type,
+              position: index,
+              locale: rt.locale
+            }
+
+            BetterTogether::Metrics::RichTextLink.find_or_create_by!(attrs)
+            valid_count += 1
+          rescue URI::InvalidURIError
+            create_invalid(rt, index, link, 'invalid_uri')
+            invalid_count += 1
+          end
+        end
+
+        { valid: valid_count, invalid: invalid_count }
+      end
+
+      private
+
+      attr_reader :rich_texts
+
+      def extract_links(rt)
+        # ActionText stores HTML; use the body helper to extract hrefs
+        rt.body.links.uniq
+      rescue StandardError
+        []
+      end
+
+      def parse_uri(link)
+        URI.parse(link)
+      end
+
+      def create_invalid(rt, index, link, invalid_type)
+        BetterTogether::Metrics::RichTextLink.create!(
+          rich_text_id: rt.id,
+          rich_text_record_id: rt.record_id,
+          rich_text_record_type: rt.record_type,
+          position: index,
+          locale: rt.locale,
+          link: BetterTogether::Content::Link.create!(url: link, valid_link: false, error_message: invalid_type)
+        )
+      end
+
+      def rt_platform_host
+        @rt_platform_host ||= begin
+          host_platform = BetterTogether::Platform.host.first
+          URI(host_platform.url).host
+        rescue StandardError
+          nil
+        end
+      end
+    end
+  end
+end
diff --git a/diagrams/source/rich_text_link_checker_flow.mmd b/diagrams/source/rich_text_link_checker_flow.mmd
@@ -0,0 +1,11 @@
+%% Mermaid source: RichText Link Checker Flow
+flowchart TD
+  A[ActionText::RichText records] --> B[RichTextLinkIdentifier]
+  B --> C[BetterTogether::Content::Link]
+  B --> D[BetterTogether::Metrics::RichTextLink]
+  E[Rake: check task] --> F[RichTextLinkCheckerQueueJob (internal)]
+  E --> G[RichTextLinkCheckerQueueJob (external)]
+  F --> H[InternalLinkCheckerJob]
+  G --> I[ExternalLinkCheckerJob]
+  H --> C
+  I --> C
diff --git a/docs/rich_text_link_checker.md b/docs/rich_text_link_checker.md
@@ -0,0 +1,29 @@
+# RichText Link Checker
+
+This document describes the pipeline that identifies links in ActionText rich content and checks them.
+
+Process overview:
+
+1. Identify: `BetterTogether::Metrics::RichTextLinkIdentifier` scans `ActionText::RichText` records and extracts links.
+2. Persist: For each link, create or find a `BetterTogether::Content::Link` and a `BetterTogether::Metrics::RichTextLink` join record.
+3. Queue: `rich_text:links:check` Rake task enqueues two queue jobs: internal and external checker queues.
+4. Distribute: `RichTextLinkCheckerQueueJob` groups links by host and schedules child check jobs over a time window to avoid bursts.
+5. Check: Child jobs (`InternalLinkCheckerJob` and `ExternalLinkCheckerJob`) perform HTTP HEAD requests and update Link metadata.
+
+Documentation files:
+- diagrams/source/rich_text_link_checker_flow.mmd (Mermaid source)
+- diagrams/exports/png/rich_text_link_checker_flow.png (export placeholder)
+
+Running locally:
+
+Use Docker wrapper for commands that need DB access (see repo README):
+
+```
+bin/dc-run rails runner "BetterTogether::Metrics::RichTextLinkIdentifier.call"
+bin/dc-run rake better_together:qa:rich_text:links:identify
+bin/dc-run rake better_together:qa:rich_text:links:check
+```
+
+Notes:
+- External HTTP checks are rate-limited by the queueing logic. Configure behavior in the queue job if needed.
+- Tests use WebMock to stub external HTTP calls.
diff --git a/lib/tasks/quality_assurance.rake b/lib/tasks/quality_assurance.rake
@@ -8,117 +8,15 @@ namespace :better_together do
       namespace :links do
         desc 'Generates report of status of RichText links'
         task identify: :environment do
-          require 'uri'
-
-          host_platform = BetterTogether::Platform.host.first
-          platform_uri = URI(host_platform.url)
-
-          rich_texts = ActionText::RichText.includes(:record).where.not(body: nil)
-          puts 'rich text count:', rich_texts.size
-
-          valid_rich_text_links = []
-          invalid_rich_text_links = []
-
-          rich_texts.each do |rt|
-            links = rt.body.links.uniq # Deduplicate links within the same rich text
-            next unless links.any?
-
-            links.each_with_index do |link, index|
-              uri = URI.parse(link)
-
-              internal_link = uri.host == platform_uri.host
-              link_type = determine_link_type(uri, internal_link)
-
-              if uri.host.nil? && uri.scheme.nil?
-                invalid_type = if uri.path
-                                 'path'
-                               elsif link.include?('mailto')
-                                 'email'
-                               elsif link.include?('tel')
-                                 'phone'
-                               else
-                                 'undetermined'
-                               end
-
-                invalid_rich_text_links << {
-                  rich_text_id: rt.id,
-                  rich_text_record_id: rt.record_id,
-                  rich_text_record_type: rt.record_type,
-                  locale: rt.locale,
-                  position: index, # Track the first position for clarity
-
-                  link_attributes: {
-                    url: link,
-                    link_type: "invalid:#{invalid_type}",
-                    valid_link: false,
-                    error_message: 'No host or scheme. Needs review.'
-                  }
-                }
-
-                next
-              end
-
-              valid_rich_text_links << {
-                rich_text_id: rt.id,
-                rich_text_record_id: rt.record_id,
-                rich_text_record_type: rt.record_type,
-                locale: rt.locale,
-                position: index, # Track the first position for clarity
-
-                link_attributes: {
-                  url: link,
-                  host: uri.host,
-                  link_type: link_type,
-                  valid_link: true,
-                  external: !internal_link
-                }
-              }
-            rescue URI::InvalidURIError => e
-              invalid_type = if link.include?('mailto')
-                               'email'
-                             elsif link.include?('tel')
-                               'phone'
-                             else
-                               'undetermined'
-                             end
-
-              invalid_rich_text_links << {
-                rich_text_id: rt.id,
-                rich_text_record_id: rt.record_id,
-                rich_text_record_type: rt.record_type,
-                locale: rt.locale,
-                position: index, # Track the first position for clarity
-
-                link_attributes: {
-                  url: link,
-                  link_type: "invalid:#{invalid_type}",
-                  valid_link: false,
-                  error_message: e.message
-                }
-              }
-            end
-          end
-
-          # Upsert valid and invalid links
-          if valid_rich_text_links.any?
-            BetterTogether::Metrics::RichTextLink.upsert_all(valid_rich_text_links,
-                                                             unique_by: %i[rich_text_id position
-                                                                           locale])
-          end
-          if invalid_rich_text_links.any?
-            BetterTogether::Metrics::RichTextLink.upsert_all(invalid_rich_text_links,
-                                                             unique_by: %i[rich_text_id position
-                                                                           locale])
-          end
-
-          puts "Valid links processed: #{valid_rich_text_links.size}"
-          puts "Invalid links processed: #{invalid_rich_text_links.size}"
+          result = BetterTogether::Metrics::RichTextLinkIdentifier.call
+          puts "Valid links processed: #{result[:valid]}"
+          puts "Invalid links processed: #{result[:invalid]}"
         end
 
         desc 'checks rich text links and returns their status code'
         task check: :environment do
-          BetterTogether::Metrics::RichTextInternalLinkCheckerQueueJob.new
-          BetterTogether::Metrics::RichTextExternalLinkCheckerQueueJob.new
+          BetterTogether::Metrics::RichTextInternalLinkCheckerQueueJob.perform_later
+          BetterTogether::Metrics::RichTextExternalLinkCheckerQueueJob.perform_later
         end
 
         def determine_link_type(uri, internal_link)
diff --git a/spec/factories/better_together/content/rich_texts.rb b/spec/factories/better_together/content/rich_texts.rb
diff --git a/spec/jobs/better_together/metrics/external_link_checker_job_spec.rb b/spec/jobs/better_together/metrics/external_link_checker_job_spec.rb
diff --git a/spec/jobs/better_together/metrics/internal_link_checker_job_spec.rb b/spec/jobs/better_together/metrics/internal_link_checker_job_spec.rb
diff --git a/spec/services/better_together/metrics/rich_text_link_identifier_spec.rb b/spec/services/better_together/metrics/rich_text_link_identifier_spec.rb
