diff --git a/Gemfile b/Gemfile index 0bd522e78..059dee34c 100644 --- a/Gemfile +++ b/Gemfile @@ -98,6 +98,8 @@ group :test do # Capybara for integration testing gem 'capybara', '>= 2.15' gem 'capybara-screenshot' + # WebMock for stubbing external HTTP requests in specs + gem 'webmock' # Coveralls for test coverage reporting gem 'coveralls_reborn', require: false # Database cleaner for test database cleaning diff --git a/Gemfile.lock b/Gemfile.lock index d2d99a042..a0a15c38e 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -57,6 +57,7 @@ PATH reform-rails (>= 0.2, < 0.4) rswag (>= 2.3.1, < 2.17.0) ruby-openai + sidekiq-scheduler simple_calendar sprockets-rails stackprof @@ -229,6 +230,9 @@ GEM term-ansicolor (~> 1.7) thor (~> 1.2) tins (~> 1.32) + crack (1.0.0) + bigdecimal + rexml crass (1.0.6) css_parser (1.21.1) addressable @@ -297,6 +301,8 @@ GEM multi_json erb (5.0.2) erubi (1.13.1) + et-orbi (1.3.0) + tzinfo event_stream_parser (1.0.0) excon (1.2.8) logger @@ -343,6 +349,9 @@ GEM friendly_id-mobility (1.0.4) friendly_id (>= 5.0.0, < 5.5) mobility (>= 1.0.1, < 2.0) + fugit (1.11.2) + et-orbi (~> 1, >= 1.2.11) + raabro (~> 1.4) fuubar (2.5.1) rspec-core (~> 3.0) ruby-progressbar (~> 1.4) @@ -362,6 +371,7 @@ GEM rake (>= 13) groupdate (6.7.0) activesupport (>= 7.1) + hashdiff (1.2.0) hashie (5.0.0) highline (3.1.2) reline @@ -521,6 +531,7 @@ GEM nio4r (~> 2.0) pundit (2.5.0) activesupport (>= 3.0.0) + raabro (1.4.0) racc (1.8.1) rack (3.1.16) rack-attack (6.7.0) @@ -699,6 +710,8 @@ GEM ffi (~> 1.12) logger rubyzip (3.0.1) + rufus-scheduler (3.9.2) + fugit (~> 1.1, >= 1.11.1) sass-embedded (1.86.3-aarch64-linux-gnu) google-protobuf (~> 4.30) sass-embedded (1.86.3-arm64-darwin) @@ -732,6 +745,9 @@ GEM logger (>= 1.6.2) rack (>= 3.1.0) redis-client (>= 0.23.2) + sidekiq-scheduler (6.0.1) + rufus-scheduler (~> 3.2) + sidekiq (>= 7.3, < 9) simple_calendar (3.1.0) rails (>= 6.1) simplecov (0.22.0) @@ -801,6 +817,10 @@ GEM activemodel (>= 6.0.0) bindex (>= 0.4.0) railties (>= 6.0.0) + webmock (3.25.1) + addressable (>= 2.8.0) + crack (>= 0.3.2) + hashdiff (>= 0.4.0, < 2.0.0) websocket (1.2.11) websocket-driver (0.8.0) base64 @@ -873,6 +893,7 @@ DEPENDENCIES storext! uglifier (>= 1.3.0) web-console (>= 3.3.0) + webmock RUBY VERSION ruby 3.4.4p34 diff --git a/app/controllers/better_together/metrics/link_checker_reports_controller.rb b/app/controllers/better_together/metrics/link_checker_reports_controller.rb new file mode 100644 index 000000000..8acc45317 --- /dev/null +++ b/app/controllers/better_together/metrics/link_checker_reports_controller.rb @@ -0,0 +1,109 @@ +# frozen_string_literal: true + +module BetterTogether + module Metrics + # Controller for creating and downloading Link Checker reports + class LinkCheckerReportsController < ApplicationController + before_action :set_report, only: %i[download] + + def index + @link_checker_reports = BetterTogether::Metrics::LinkCheckerReport.order(created_at: :desc) + + if request.headers['Turbo-Frame'].present? + render partial: 'better_together/metrics/link_checker_reports/index', + locals: { reports: @link_checker_reports }, + layout: false + else + render :index + end + end + + def new + @link_checker_report = BetterTogether::Metrics::LinkCheckerReport.new + end + + # rubocop:todo Metrics/AbcSize + # rubocop:todo Metrics/MethodLength + # rubocop:todo Metrics/BlockLength + def create + opts = { + from_date: params.dig(:metrics_link_checker_report, :filters, :from_date), + to_date: params.dig(:metrics_link_checker_report, :filters, :to_date), + file_format: params.dig(:metrics_link_checker_report, :file_format) || 'csv' + } + + @link_checker_report = BetterTogether::Metrics::LinkCheckerReport.create_and_generate!(**opts) + + respond_to do |format| # rubocop:todo Metrics/BlockLength + if @link_checker_report.persisted? + flash[:notice] = t('flash.generic.created', resource: t('resources.report')) + format.html { redirect_to metrics_link_checker_reports_path, notice: flash[:notice] } + format.turbo_stream do + render turbo_stream: [ + turbo_stream.prepend( + 'link_checker_reports_table_body', + partial: 'better_together/metrics/link_checker_reports/link_checker_report', + locals: { link_checker_report: @link_checker_report } + ), + turbo_stream.replace( + 'flash_messages', + partial: 'layouts/better_together/flash_messages', + locals: { flash: flash } + ), + turbo_stream.replace('new_link_checker_report', + '') + ] + end + else + flash.now[:alert] = t('flash.generic.error_create', resource: t('resources.report')) + format.html { render :new, status: :unprocessable_content } + format.turbo_stream do + render turbo_stream: [ + turbo_stream.update('form_errors', partial: 'layouts/errors', locals: { object: @link_checker_report }), + turbo_stream.replace( + 'flash_messages', + partial: 'layouts/better_together/flash_messages', + locals: { flash: flash } + ) + ] + end + end + end + end + # rubocop:enable Metrics/BlockLength + # rubocop:enable Metrics/MethodLength + # rubocop:enable Metrics/AbcSize + + # rubocop:todo Metrics/AbcSize + # rubocop:todo Metrics/MethodLength + def download + if @link_checker_report.report_file.attached? + BetterTogether::Metrics::TrackDownloadJob.perform_later( + @link_checker_report, + @link_checker_report.report_file.filename.to_s, + @link_checker_report.report_file.content_type, + @link_checker_report.report_file.byte_size, + I18n.locale.to_s + ) + + send_data @link_checker_report.report_file.download, + filename: @link_checker_report.report_file.filename.to_s, + type: @link_checker_report.report_file.content_type, + disposition: 'attachment' + + return + end + + redirect_to metrics_link_checker_reports_path, alert: t('resources.download_failed') + end + # rubocop:enable Metrics/MethodLength + # rubocop:enable Metrics/AbcSize + + private + + def set_report + @link_checker_report = BetterTogether::Metrics::LinkCheckerReport.find(params[:id]) + end + end + end +end diff --git a/app/controllers/better_together/metrics/reports_controller.rb b/app/controllers/better_together/metrics/reports_controller.rb index e21738471..fe7694c35 100644 --- a/app/controllers/better_together/metrics/reports_controller.rb +++ b/app/controllers/better_together/metrics/reports_controller.rb @@ -67,6 +67,12 @@ def index # rubocop:todo Metrics/AbcSize, Metrics/MethodLength } end } + + # Link Checker charts: aggregate data from stored links + links_scope = BetterTogether::Content::Link.all + @links_by_host = links_scope.group(:host).count + @invalid_by_host = links_scope.where(valid_link: false).group(:host).count + @failures_daily = links_scope.where(valid_link: false).group_by_day(:last_checked_at).count end # A helper method to generate a random color for each platform (this can be customized). diff --git a/app/javascript/controllers/better_together/metrics_charts_controller.js b/app/javascript/controllers/better_together/metrics_charts_controller.js index 1af07a83c..f25e85c9a 100644 --- a/app/javascript/controllers/better_together/metrics_charts_controller.js +++ b/app/javascript/controllers/better_together/metrics_charts_controller.js @@ -58,7 +58,7 @@ const platformBorderColors = { }; export default class extends Controller { - static targets = ["pageViewsChart", "dailyPageViewsChart", "linkClicksChart", "dailyLinkClicksChart", "downloadsChart", "sharesChart", "sharesPerUrlPerPlatformChart"] + static targets = ["pageViewsChart", "dailyPageViewsChart", "linkClicksChart", "dailyLinkClicksChart", "downloadsChart", "sharesChart", "sharesPerUrlPerPlatformChart", "linksByHostChart", "invalidByHostChart", "failuresDailyChart"] connect() { this.renderPageViewsChart() @@ -68,6 +68,9 @@ export default class extends Controller { this.renderDownloadsChart() this.renderSharesChart() this.renderSharesPerUrlPerPlatformChart() + this.renderLinksByHostChart() + this.renderInvalidByHostChart() + this.renderFailuresDailyChart() } renderPageViewsChart() { @@ -203,4 +206,58 @@ export default class extends Controller { }) }) } + + renderLinksByHostChart() { + const data = JSON.parse(this.linksByHostChartTarget.dataset.chartData) + new Chart(this.linksByHostChartTarget, { + type: 'bar', + data: { + labels: data.labels, + datasets: [{ + label: 'Links by Host', + data: data.values, + backgroundColor: 'rgba(99, 132, 255, 0.2)', + borderColor: 'rgba(99, 132, 255, 1)', + borderWidth: 1 + }] + }, + options: Object.assign({}, sharedChartOptions) + }) + } + + renderInvalidByHostChart() { + const data = JSON.parse(this.invalidByHostChartTarget.dataset.chartData) + new Chart(this.invalidByHostChartTarget, { + type: 'bar', + data: { + labels: data.labels, + datasets: [{ + label: 'Invalid Links by Host', + data: data.values, + backgroundColor: 'rgba(255, 159, 64, 0.2)', + borderColor: 'rgba(255, 159, 64, 1)', + borderWidth: 1 + }] + }, + options: Object.assign({}, sharedChartOptions) + }) + } + + renderFailuresDailyChart() { + const data = JSON.parse(this.failuresDailyChartTarget.dataset.chartData) + new Chart(this.failuresDailyChartTarget, { + type: 'line', + data: { + labels: data.labels, + datasets: [{ + label: 'Invalid Links Over Time', + data: data.values, + backgroundColor: 'rgba(255, 99, 132, 0.2)', + borderColor: 'rgba(255, 99, 132, 1)', + borderWidth: 1 + }] + }, + options: Object.assign({}, sharedChartOptions) + }) + } } diff --git a/app/jobs/better_together/event_reminder_scan_job.rb b/app/jobs/better_together/event_reminder_scan_job.rb new file mode 100644 index 000000000..45b7a95e8 --- /dev/null +++ b/app/jobs/better_together/event_reminder_scan_job.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module BetterTogether + # Scans upcoming events and schedules per-event reminders via + # BetterTogether::EventReminderSchedulerJob. This job is intended to be run + # periodically by the scheduler (sidekiq-scheduler / sidekiq-cron). + class EventReminderScanJob < ApplicationJob + queue_as :notifications + + # Keep lightweight: find events starting in the near future and enqueue the + # existing per-event scheduler job which handles cancellation/rescheduling. + # default: next 7 days + def perform(window_hours: 168) + cutoff = Time.current + window_hours.hours + + BetterTogether::Event.where('starts_at <= ? AND starts_at >= ?', cutoff, Time.current).find_each do |event| + # Use the id to avoid serializing AR objects into the job payload + BetterTogether::EventReminderSchedulerJob.perform_later(event.id) + rescue StandardError => e + Rails.logger.error "Failed to enqueue reminder scheduler for event #{event&.id}: #{e.message}" + end + end + end +end diff --git a/app/jobs/better_together/metrics/external_link_checker_job.rb b/app/jobs/better_together/metrics/external_link_checker_job.rb new file mode 100644 index 000000000..0daa30cf6 --- /dev/null +++ b/app/jobs/better_together/metrics/external_link_checker_job.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +require 'net/http' +require 'uri' + +module BetterTogether + module Metrics + # Background job that checks an external link's HTTP status and updates + # the corresponding BetterTogether::Content::Link record with the + # latest check timestamp, status code and validity flag. + class ExternalLinkCheckerJob < ApplicationJob + queue_as :default + + def perform(link_id) + link = BetterTogether::Content::Link.find(link_id) + checker = BetterTogether::Metrics::HttpLinkChecker.new(link.url) + result = checker.call + + update_link_from_result(link, result) + end + + private + + def update_link_from_result(link, result) + attrs = { + last_checked_at: Time.current, + latest_status_code: result.status_code, + valid_link: result.success + } + + attrs[:error_message] = result.error&.message unless result.success + link.update!(attrs) + end + end + end +end diff --git a/app/jobs/better_together/metrics/internal_link_checker_job.rb b/app/jobs/better_together/metrics/internal_link_checker_job.rb new file mode 100644 index 000000000..2c210ab85 --- /dev/null +++ b/app/jobs/better_together/metrics/internal_link_checker_job.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +require 'net/http' +require 'uri' + +module BetterTogether + module Metrics + # Background job that checks an internal link's HTTP status and updates + # the corresponding BetterTogether::Content::Link record with the + # latest check timestamp, status code and validity flag. + class InternalLinkCheckerJob < ApplicationJob + queue_as :default + + def perform(link_id) + link = BetterTogether::Content::Link.find(link_id) + checker = BetterTogether::Metrics::HttpLinkChecker.new(link.url) + result = checker.call + + update_link_from_result(link, result) + end + + private + + def update_link_from_result(link, result) + attrs = { + last_checked_at: Time.current, + latest_status_code: result.status_code, + valid_link: result.success + } + + attrs[:error_message] = result.error&.message unless result.success + link.update!(attrs) + end + end + end +end diff --git a/app/jobs/better_together/metrics/link_checker_report_scheduler_job.rb b/app/jobs/better_together/metrics/link_checker_report_scheduler_job.rb new file mode 100644 index 000000000..fac8cd373 --- /dev/null +++ b/app/jobs/better_together/metrics/link_checker_report_scheduler_job.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +module BetterTogether + module Metrics + # Scheduler job to create and send LinkChecker reports + class LinkCheckerReportSchedulerJob < MetricsJob + def perform(from_date: nil, to_date: nil, file_format: 'csv') + report = BetterTogether::Metrics::LinkCheckerReport.create_and_generate!( + from_date: from_date, + to_date: to_date, + file_format: file_format + ) + + return unless report.report_file.attached? + + BetterTogether::Metrics::ReportMailer.link_checker_report(report.id).deliver_later + end + end + end +end diff --git a/app/jobs/better_together/metrics/rich_text_external_link_checker_queue_job.rb b/app/jobs/better_together/metrics/rich_text_external_link_checker_queue_job.rb new file mode 100644 index 000000000..ee80c57db --- /dev/null +++ b/app/jobs/better_together/metrics/rich_text_external_link_checker_queue_job.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +module BetterTogether + module Metrics + # Queues jobs that check external links found inside ActionText rich content. + # Subclasses of RichTextLinkCheckerQueueJob should implement the specifics + # for how individual link check jobs are performed. + class RichTextExternalLinkCheckerQueueJob < RichTextLinkCheckerQueueJob + protected + + def model_collection + super.where(link_type: 'external') + end + + def child_job_class + BetterTogether::Metrics::ExternalLinkCheckerJob + end + end + end +end diff --git a/app/jobs/better_together/metrics/rich_text_internal_link_checker_queue_job.rb b/app/jobs/better_together/metrics/rich_text_internal_link_checker_queue_job.rb new file mode 100644 index 000000000..2dcfaabce --- /dev/null +++ b/app/jobs/better_together/metrics/rich_text_internal_link_checker_queue_job.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module BetterTogether + module Metrics + # Queues jobs that check internal links found inside ActionText rich content. + # This job narrows the collection to internal links and may delay processing + # to reduce immediate load on the application. + class RichTextInternalLinkCheckerQueueJob < RichTextLinkCheckerQueueJob + protected + + def model_collection + super.where(link_type: 'internal') + end + + def queue_delay + 5.minutes + end + + def child_job_class + BetterTogether::Metrics::InternalLinkCheckerJob + end + end + end +end diff --git a/app/jobs/better_together/metrics/rich_text_link_checker_queue_job.rb b/app/jobs/better_together/metrics/rich_text_link_checker_queue_job.rb new file mode 100644 index 000000000..f7b9de472 --- /dev/null +++ b/app/jobs/better_together/metrics/rich_text_link_checker_queue_job.rb @@ -0,0 +1,72 @@ +# frozen_string_literal: true + +module BetterTogether + module Metrics + # Base queueing job that distributes RichText link check work across hosts. + # It groups RichText links by host and schedules child jobs with delays to + # avoid overloading external hosts or the application. + class RichTextLinkCheckerQueueJob < MetricsJob + def perform + records_size = model_collection.size + return if records_size.zero? + + # Define the total time window for each host (e.g., 1 hour in seconds) + time_window = 3600 + + records_by_host.each do |host, link_count| + next if link_count.zero? + + delay_between_requests = time_window / link_count.to_f + queue_jobs_for_host(host, delay_between_requests) + end + end + + def records_by_host + model_collection.group(:host) + .order('count_all DESC') + .count + end + + protected + + # Operate on the content links table which holds last_checked_at and valid_link + # This avoids querying columns that exist only on BetterTogether::Content::Link + def model_class + BetterTogether::Content::Link + end + + def model_collection + # Limit to links that are referenced by rich_text_links; join through the + # metrics rich_text_links table if records exist, otherwise operate on + # the full Link set. + if BetterTogether::Metrics::RichTextLink.table_exists? + model_class.joins(:rich_text_links) + .where('last_checked_at IS NULL OR last_checked_at < ?', last_checked_lt) + else + model_class.where('last_checked_at IS NULL OR last_checked_at < ?', last_checked_lt) + end + end + + def queue_jobs_for_host(host, delay_between_requests) + links_for_host = model_collection.where(host: host) + links_for_host.each_with_index do |link, index| + schedule_time = Time.current + (delay_between_requests * index).seconds + child_job_class.set(wait_until: schedule_time).perform_later(link.id) + end + end + + def child_job_class + # Define this in subclasses (e.g., InternalLinkCheckerJob, ExternalLinkCheckerJob) + raise NotImplementedError, 'Subclasses must implement `child_job_class`' + end + + def last_checked_lt + Time.current - last_checked_threshold + end + + def last_checked_threshold + 14.days + end + end + end +end diff --git a/app/mailers/better_together/metrics/report_mailer.rb b/app/mailers/better_together/metrics/report_mailer.rb new file mode 100644 index 000000000..6ee6d3db2 --- /dev/null +++ b/app/mailers/better_together/metrics/report_mailer.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module BetterTogether + module Metrics + # Mailer for delivering metrics reports + class ReportMailer < BetterTogether::ApplicationMailer + # rubocop:todo Metrics/AbcSize + def link_checker_report(report_id) + @report = BetterTogether::Metrics::LinkCheckerReport.find(report_id) + return unless @report&.report_file&.attached? + + attachments[@report.report_file.filename.to_s] = { + mime_type: @report.report_file.content_type, + content: @report.report_file.download + } + + mail( + to: BetterTogether::ApplicationMailer.default[:from], + subject: "Link Checker Report: #{@report.created_at.strftime('%Y-%m-%d')}" + ) + end + # rubocop:enable Metrics/AbcSize + end + end +end diff --git a/app/models/better_together/content/link.rb b/app/models/better_together/content/link.rb new file mode 100644 index 000000000..4dae42871 --- /dev/null +++ b/app/models/better_together/content/link.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +module BetterTogether + module Content + # Represents a persisted link discovered in rich content. Stores metadata + # about the link (host, scheme, validity) and associates to RichText + # metrics records. + class Link < ApplicationRecord + has_many :rich_text_links, class_name: 'BetterTogether::Metrics::RichTextLink', inverse_of: :link + has_many :rich_texts, through: :rich_text_links + has_many :rich_text_records, through: :rich_text_links + + # Provide safe defaults for tests and ad-hoc creation so callers don't + # need to remember non-nullable columns. These mirror reasonable + # expectations for persisted links. + after_initialize do |record| + record.link_type = 'website' if record.link_type.blank? + record.valid_link = false if record.valid_link.nil? + end + end + end +end diff --git a/app/models/better_together/links.rb b/app/models/better_together/links.rb new file mode 100644 index 000000000..af9f84330 --- /dev/null +++ b/app/models/better_together/links.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +module BetterTogether + # Namespace helper for links-related tables. Ensures a consistent + # table name prefix for models placed under BetterTogether::Links. + module Links + def self.table_name_prefix + 'better_together_links_' + end + end +end diff --git a/app/models/better_together/metrics/link_checker_report.rb b/app/models/better_together/metrics/link_checker_report.rb new file mode 100644 index 000000000..bdae38c0b --- /dev/null +++ b/app/models/better_together/metrics/link_checker_report.rb @@ -0,0 +1,138 @@ +# frozen_string_literal: true + +module BetterTogether + module Metrics + # LinkCheckerReport maintains a generated report for link-checker results + # including counts by host and failures over time. + class LinkCheckerReport < ApplicationRecord + has_one_attached :report_file + + validates :file_format, presence: true + attribute :filters, :jsonb, default: {} + + before_create :generate_report! + after_create_commit :export_file_if_report_exists + after_destroy_commit :purge_report_file + + # rubocop:todo Metrics/AbcSize + # rubocop:todo Metrics/MethodLength + def generate_report! + from_date = filters['from_date'].present? ? Date.parse(filters['from_date']) : nil + to_date = filters['to_date'].present? ? Date.parse(filters['to_date']) : nil + + base_scope = BetterTogether::Content::Link.all + base_scope = base_scope.where('last_checked_at >= ?', from_date) if from_date + base_scope = base_scope.where('last_checked_at <= ?', to_date) if to_date + + by_host = base_scope.group(:host).count + invalid_by_host = base_scope.where(valid_link: false).group(:host).count + failures_daily = base_scope.where(valid_link: false).group_by_day(:last_checked_at).count + + self.report_data = { + by_host: by_host, + invalid_by_host: invalid_by_host, + failures_daily: failures_daily + } + end + # rubocop:enable Metrics/MethodLength + # rubocop:enable Metrics/AbcSize + + # rubocop:todo Metrics/MethodLength + def export_file! + file_path = if file_format == 'csv' + generate_csv_file + else + raise "Unsupported file format: #{file_format}" + end + + report_file.attach( + io: File.open(file_path), + filename: build_filename, + content_type: file_format == 'csv' ? 'text/csv' : 'application/octet-stream' + ) + ensure + File.delete(file_path) if file_path && File.exist?(file_path) + end + # rubocop:enable Metrics/MethodLength + + private + + def purge_report_file + report_file.purge_later if report_file.attached? + end + + def export_file_if_report_exists + export_file! if report_data.present? && !report_data.empty? + end + + # rubocop:todo Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity + def generate_csv_file + file_path = Rails.root.join('tmp', build_filename) + + CSV.open(file_path, 'w') do |csv| + csv << ['Host', 'Total Links', 'Invalid Links'] + + hosts = (report_data['by_host'] || report_data[:by_host] || {}).keys + hosts.each do |host| + total = (report_data['by_host'] || report_data[:by_host] || {})[host] || 0 + invalid = (report_data['invalid_by_host'] || report_data[:invalid_by_host] || {})[host] || 0 + csv << [host, total, invalid] + end + + csv << [] + csv << ['Date', 'Invalid Count'] + (report_data['failures_daily'] || report_data[:failures_daily] || {}).each do |date, count| + csv << [date.to_s, count] + end + end + + file_path + end + # rubocop:enable Metrics/PerceivedComplexity + # rubocop:enable Metrics/MethodLength + # rubocop:enable Metrics/CyclomaticComplexity + # rubocop:enable Metrics/AbcSize + + # rubocop:todo Metrics/AbcSize + # rubocop:todo Metrics/MethodLength + def build_filename + filters_summary = [] + + if filters['from_date'].present? + from_stamp = Date.parse(filters['from_date']).strftime('%Y-%m-%d') + filters_summary << "from_#{from_stamp}" + end + + if filters['to_date'].present? + to_stamp = Date.parse(filters['to_date']).strftime('%Y-%m-%d') + filters_summary << "to_#{to_stamp}" + end + + filters_summary = filters_summary.join('_') + filters_summary = 'all' if filters_summary.blank? + + timestamp = Time.current.strftime('%Y-%m-%d_%H%M%S') + + "LinkCheckerReport_#{timestamp}_#{filters_summary}.#{file_format}" + end + # rubocop:enable Metrics/MethodLength + # rubocop:enable Metrics/AbcSize + + class << self + def create_and_generate!(from_date: nil, to_date: nil, file_format: 'csv') + filters = {} + filters['from_date'] = from_date if from_date.present? + filters['to_date'] = to_date if to_date.present? + + create!(filters: filters, file_format: file_format) + end + + def export_existing!(id) + report = find(id) + report.export_file_if_report_exists + report + end + end + end + end +end diff --git a/app/models/better_together/metrics/rich_text_link.rb b/app/models/better_together/metrics/rich_text_link.rb new file mode 100644 index 000000000..362fee214 --- /dev/null +++ b/app/models/better_together/metrics/rich_text_link.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module BetterTogether + module Metrics + # Tracks occurrences of links found inside ActionText rich content and + # associates them with the original Link record and owning rich text. + class RichTextLink < ApplicationRecord + include Positioned + + belongs_to :link, class_name: 'BetterTogether::Content::Link' + belongs_to :rich_text, class_name: 'ActionText::RichText' + belongs_to :rich_text_record, polymorphic: true + + accepts_nested_attributes_for :link, reject_if: ->(attributes) { attributes['url'].blank? }, allow_destroy: false + end + end +end diff --git a/app/services/better_together/metrics/http_link_checker.rb b/app/services/better_together/metrics/http_link_checker.rb new file mode 100644 index 000000000..f61f15111 --- /dev/null +++ b/app/services/better_together/metrics/http_link_checker.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +require 'net/http' +require 'uri' + +module BetterTogether + module Metrics + # Service to perform HTTP checks for a given URI with simple retry logic. + # Returns a struct with :success, :status_code, :error + CheckResult = Struct.new(:success, :status_code, :error) + + # Performs a HEAD request with configurable timeouts and retries. The + # service returns a CheckResult struct indicating success, the HTTP + # status code (string) and any error encountered. + class HttpLinkChecker + DEFAULT_RETRIES = 2 + DEFAULT_OPEN_TIMEOUT = 5 + DEFAULT_READ_TIMEOUT = 5 + + def initialize(uri, retries: DEFAULT_RETRIES, + open_timeout: DEFAULT_OPEN_TIMEOUT, + read_timeout: DEFAULT_READ_TIMEOUT) + @uri = URI.parse(uri) + @retries = retries + @open_timeout = open_timeout + @read_timeout = read_timeout + end + + def call + attempts = 0 + begin + attempts += 1 + response = http_head(@uri) + CheckResult.new(response.is_a?(Net::HTTPSuccess), response.code.to_s, nil) + rescue StandardError => e + retry if attempts <= @retries + CheckResult.new(false, nil, e) + end + end + + private + + def http_head(uri) + Net::HTTP.start( + uri.host, + uri.port, + use_ssl: uri.scheme == 'https', + open_timeout: @open_timeout, + read_timeout: @read_timeout + ) do |http| + request = Net::HTTP::Head.new(uri.request_uri) + http.request(request) + end + end + end + end +end diff --git a/app/services/better_together/metrics/rich_text_link_identifier.rb b/app/services/better_together/metrics/rich_text_link_identifier.rb new file mode 100644 index 000000000..303c24115 --- /dev/null +++ b/app/services/better_together/metrics/rich_text_link_identifier.rb @@ -0,0 +1,186 @@ +# frozen_string_literal: true + +module BetterTogether + module Metrics + # Service to scan ActionText::RichText records, extract links, and persist + # both the link metadata (BetterTogether::Content::Link) and the join + # records (BetterTogether::Metrics::RichTextLink). + # + # Usage: + # BetterTogether::Metrics::RichTextLinkIdentifier.call + class RichTextLinkIdentifier # rubocop:disable Metrics/ClassLength + def self.call(rich_texts: nil) + new(rich_texts: rich_texts).call + end + + def initialize(rich_texts: nil) + @rich_texts = rich_texts + end + + # rubocop:disable Metrics/MethodLength + def call + texts = rich_texts || ActionText::RichText.includes(:record).where.not(body: nil) + valid_count = 0 + invalid_count = 0 + + if texts.respond_to?(:find_each) + texts.find_each do |rich_text| + v, i = process_rich_text(rich_text) + valid_count += v + invalid_count += i + end + else + Array(texts).each do |rich_text| + v, i = process_rich_text(rich_text) + valid_count += v + invalid_count += i + end + end + + { valid: valid_count, invalid: invalid_count } + end + # rubocop:enable Metrics/MethodLength + + private + + attr_reader :rich_texts + + def extract_links(rich_text) + # ActionText stores HTML; use the body helper to extract hrefs + rich_text.body.links.uniq + rescue StandardError + [] + end + + # rubocop:disable Metrics/MethodLength,Metrics/AbcSize,Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity + def process_rich_text(rich_text) + valid_count = 0 + invalid_count = 0 + links = extract_links(rich_text) + return [0, 0] if links.empty? + + links.each_with_index do |link, index| + uri_obj = parse_uri(link) + if uri_obj.nil? + create_invalid(rich_text, index, link, 'undetermined') + invalid_count += 1 + next + end + + canonical_host = uri_obj.host + if canonical_host.nil? && uri_obj.scheme.nil? + if link.start_with?('/') + canonical_host = rt_platform_host + else + create_invalid(rich_text, index, link, 'undetermined') + invalid_count += 1 + next + end + end + + persist_link_and_rich_text_link(rich_text, link, index, canonical_host, uri_obj) + valid_count += 1 + rescue URI::InvalidURIError + create_invalid(rich_text, index, link, 'invalid_uri') + invalid_count += 1 + end + + [valid_count, invalid_count] + end + # rubocop:enable Metrics/MethodLength,Metrics/AbcSize,Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity + + # rubocop:disable Metrics/MethodLength,Metrics/AbcSize,Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity + def persist_link_and_rich_text_link(rich_text, link, index, canonical_host, uri_obj) + bt_link = BetterTogether::Content::Link.find_or_initialize_by(url: link) + bt_link.host ||= canonical_host + bt_link.scheme ||= uri_obj.scheme + bt_link.external = (canonical_host.present? && (rt_platform_host != canonical_host)) + # Ensure link_type is set to a sensible default before persisting to avoid + # NOT NULL constraint violations (some callers create links directly). + bt_link.link_type ||= 'website' + bt_link.save! if bt_link.changed? + + # Persist the RichTextLink depending on the schema available. + model = BetterTogether::Metrics::RichTextLink + + if model.column_names.include?('link_id') + attrs = { + link_id: bt_link.id, + rich_text_id: rich_text.id, + rich_text_record_id: rich_text.record_id, + rich_text_record_type: rich_text.record_type, + position: index, + locale: rich_text.locale + } + + # Build optional metadata hash only for columns that exist on the table + optional_cols = %w[url link_type external valid_link host error_message] + optional_cols.each do |c| + next unless model.column_names.include?(c) + + attrs[c.to_sym] = case c + when 'url' then bt_link.url + when 'link_type' then bt_link.link_type || 'website' + when 'external' then bt_link.external || false + when 'valid_link' then bt_link.valid_link || false + when 'host' then bt_link.host + when 'error_message' then bt_link.error_message + end + end + + begin + model.create!(attrs) + rescue ActiveRecord::RecordNotUnique + # another process inserted concurrently; ignore + end + else + # Fallback schema: metrics rich_text_links store URL and metadata inline. + attrs = { + rich_text_id: rich_text.id, + url: link, + link_type: bt_link.link_type || 'website', + external: bt_link.external || false, + valid_link: bt_link.valid_link || false, + host: bt_link.host, + error_message: bt_link.error_message + } + + begin + model.create!(attrs.merge(position: index, locale: rich_text.locale)) + rescue ActiveRecord::RecordNotUnique + # ignore duplicate insertion races + end + end + end + # rubocop:enable Metrics/MethodLength,Metrics/AbcSize,Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity + + def parse_uri(link) + URI.parse(link) + end + + def create_invalid(rich_text, index, link, invalid_type) + # Create the content link with a default link_type to satisfy DB constraints + bt_link = BetterTogether::Content::Link.create!(url: link, valid_link: false, error_message: invalid_type, + link_type: 'website') + + BetterTogether::Metrics::RichTextLink.create!( + rich_text_id: rich_text.id, + rich_text_record_id: rich_text.record_id, + rich_text_record_type: rich_text.record_type, + position: index, + locale: rich_text.locale, + link: bt_link + ) + end + + def rt_platform_host + @rt_platform_host ||= begin + host_platform = BetterTogether::Platform.host.first + URI(host_platform.url).host + rescue StandardError + nil + end + end + end + end +end diff --git a/app/views/better_together/metrics/link_checker_reports/_index.html.erb b/app/views/better_together/metrics/link_checker_reports/_index.html.erb new file mode 100644 index 000000000..8e56fa362 --- /dev/null +++ b/app/views/better_together/metrics/link_checker_reports/_index.html.erb @@ -0,0 +1,27 @@ + +

<%= t('views.headers.link_checker_reports') %>

+ + + <%= link_to t('views.buttons.new', resource: t('resources.report')), new_metrics_link_checker_report_path, class: "btn btn-primary mb-3", data: { turbo_frame: "new_link_checker_report" } %> + + + + + +
+ + + + + + + + + + + + <%= render @link_checker_reports %> + +
<%= t('views.labels.id') %><%= t('views.labels.created_at') %><%= t('views.labels.filters') %><%= t('views.labels.file_format') %><%= t('views.labels.actions') %>
+
+ diff --git a/app/views/better_together/metrics/link_checker_reports/_link_checker_report.html.erb b/app/views/better_together/metrics/link_checker_reports/_link_checker_report.html.erb new file mode 100644 index 000000000..722383b99 --- /dev/null +++ b/app/views/better_together/metrics/link_checker_reports/_link_checker_report.html.erb @@ -0,0 +1,21 @@ + + <%= link_checker_report.id %> + <%= l(link_checker_report.created_at, format: :long) %> + + <% if link_checker_report.filters.present? %> + <%= link_checker_report.filters.to_json %> + <% else %> + <%= t('views.labels.all') %> + <% end %> + + <%= link_checker_report.file_format %> + + <% if link_checker_report.report_file.attached? %> + <%= link_to t('views.buttons.download'), download_metrics_link_checker_report_path(link_checker_report), + class: "btn btn-primary btn-sm", + data: { turbo: false } %> + <% else %> + <%= t('views.labels.no_file') %> + <% end %> + + diff --git a/app/views/better_together/metrics/link_checker_reports/index.html.erb b/app/views/better_together/metrics/link_checker_reports/index.html.erb new file mode 100644 index 000000000..b2b28d3dd --- /dev/null +++ b/app/views/better_together/metrics/link_checker_reports/index.html.erb @@ -0,0 +1,2 @@ +<%= render partial: 'better_together/metrics/link_checker_reports/index', + locals: { link_checker_reports: @link_checker_reports } %> \ No newline at end of file diff --git a/app/views/better_together/metrics/link_checker_reports/new.html.erb b/app/views/better_together/metrics/link_checker_reports/new.html.erb new file mode 100644 index 000000000..82b08b796 --- /dev/null +++ b/app/views/better_together/metrics/link_checker_reports/new.html.erb @@ -0,0 +1,79 @@ + +

<%= t('views.headers.new_link_checker_report') %>

+ + <%= form_with model: @link_checker_report, url: metrics_link_checker_reports_path, local: true, class: "form" do |f| %> + <%= turbo_frame_tag 'form_errors' %> +
+ <%= f.label :from_date, t('views.labels.from_date') %> + <%= f.date_field :from_date, name: "metrics_link_checker_report[filters][from_date]", class: "form-control", placeholder: "YYYY-MM-DD" %> +
+ +
+ <%= f.label :to_date, t('views.labels.to_date') %> + <%= f.date_field :to_date, name: "metrics_link_checker_report[filters][to_date]", class: "form-control", placeholder: "YYYY-MM-DD" %> +
+ +
+ <%= f.label :filter_internal, t('views.labels.internal_filter') %> + <%= f.select :filter_internal, + options_for_select([[t('views.labels.all'), ""], [t('views.labels.internal'), true], [t('views.labels.external'), false]], + @link_checker_report.filters["filter_internal"]), + { include_blank: false }, + name: "metrics_link_checker_report[filters][filter_internal]", + class: "form-control" %> +
+ +
+ <%= f.label :file_format, t('views.labels.file_format') %> + <%= f.select :file_format, options_for_select([["CSV", "csv"]], "csv"), {}, class: "form-control" %> +
+ +
+ <%= f.submit t('views.buttons.create_report'), class: "btn btn-primary" %> + <%= link_to t('views.buttons.back'), metrics_link_checker_reports_path, class: "btn btn-secondary" %> +
+ <% end %> + + + +
+

<%= t('views.headers.new_link_checker_report') %>

+ + <%= form_with model: @link_checker_report, url: metrics_link_checker_reports_path, local: true, class: "form" do |f| %> + <%= turbo_frame_tag 'form_errors' %> +
+ <%= f.label :from_date, t('views.labels.from_date') %> + <%= f.date_field :from_date, name: "metrics_link_checker_report[filters][from_date]", class: "form-control", placeholder: "YYYY-MM-DD" %> +
+ +
+ <%= f.label :to_date, t('views.labels.to_date') %> + <%= f.date_field :to_date, name: "metrics_link_checker_report[filters][to_date]", class: "form-control", placeholder: "YYYY-MM-DD" %> +
+ +
+ <%= f.label :filter_internal, t('views.labels.internal_filter') %> + <%= f.select :filter_internal, + options_for_select([[t('views.labels.all'), ""], [t('views.labels.internal'), true], [t('views.labels.external'), false]], + @link_checker_report.filters["filter_internal"]), + { include_blank: false }, + name: "metrics_link_checker_report[filters][filter_internal]", + class: "form-control" %> +
+ +
+ <%= f.check_box :sort_by_total_clicks, class: "form-check-input" %> + <%= f.label :sort_by_total_clicks, t('views.labels.sort_by_total_clicks'), class: "form-check-label" %> +
+ +
+ <%= f.label :file_format, t('views.labels.file_format') %> + <%= f.select :file_format, options_for_select([["CSV", "csv"]], "csv"), {}, class: "form-control" %> +
+ +
+ <%= f.submit t('views.buttons.create_report'), class: "btn btn-primary" %> + <%= link_to t('views.buttons.back'), metrics_link_checker_reports_path, class: "btn btn-secondary" %> +
+ <% end %> +
diff --git a/app/views/better_together/metrics/report_mailer/link_checker_report.html.erb b/app/views/better_together/metrics/report_mailer/link_checker_report.html.erb new file mode 100644 index 000000000..21041bbaa --- /dev/null +++ b/app/views/better_together/metrics/report_mailer/link_checker_report.html.erb @@ -0,0 +1,8 @@ +

Hello,

+ +

Attached is the Link Checker report for <%= @report.created_at.strftime('%Y-%m-%d') %>.

+ +

Report file: <%= @report.report_file.filename.to_s if @report&.report_file&.attached? %>

+ +

Regards,
+<%= BetterTogether::ApplicationMailer.default[:from] %>

diff --git a/app/views/better_together/metrics/report_mailer/link_checker_report.text.erb b/app/views/better_together/metrics/report_mailer/link_checker_report.text.erb new file mode 100644 index 000000000..903519b9a --- /dev/null +++ b/app/views/better_together/metrics/report_mailer/link_checker_report.text.erb @@ -0,0 +1,8 @@ +Hello, + +Attached is the Link Checker report for <%= @report.created_at.strftime('%Y-%m-%d') %>. + +Report file: <%= @report.report_file.filename.to_s if @report&.report_file&.attached? %> + +Regards, +<%= BetterTogether::ApplicationMailer.default[:from] %> diff --git a/app/views/better_together/metrics/reports/index.html.erb b/app/views/better_together/metrics/reports/index.html.erb index 1bb4fcdea..4b5fd5e73 100644 --- a/app/views/better_together/metrics/reports/index.html.erb +++ b/app/views/better_together/metrics/reports/index.html.erb @@ -1,36 +1,41 @@ <% content_for :page_title, 'Metrics Reports' %> -
+

Metrics Reports

-
+