feat(metrics): implement UTF-8 URL handling and validations for LinkClick and PageView models

Author: rsmithlal

app/models/better_together/metrics/link_click.rb

@@ -4,18 +4,42 @@
 module BetterTogether
   module Metrics
     class LinkClick < ApplicationRecord # rubocop:todo Style/Documentation
+      include Utf8UrlHandler
+
       # Validations
       VALID_URL_SCHEMES = %w[http https tel mailto].freeze
 
-      # Regular expression to match http, https, tel, and mailto URLs
-      VALID_URL_REGEX = /\A(http|https|tel|mailto):.+\z/
-
-      validates :url, presence: true,
-                      format: { with: VALID_URL_REGEX, message: 'must be a valid URL or tel/mailto link' }
-      validates :page_url, presence: true, format: URI::DEFAULT_PARSER.make_regexp(%w[http https])
+      validates :url, presence: true
+      validates :page_url, presence: true
       validates :locale, presence: true, inclusion: { in: I18n.available_locales.map(&:to_s) }
       validates :clicked_at, presence: true
       validates :internal, inclusion: { in: [true, false] }
+
+      # Custom validation for UTF-8 URL support
+      validate :url_must_be_valid
+      validate :page_url_must_be_valid
+
+      private
+
+      def url_must_be_valid
+        return if url.blank?
+
+        return if valid_utf8_url?(url)
+
+        errors.add(:url, 'must be a valid URL or tel/mailto link')
+      end
+
+      def page_url_must_be_valid
+        return if page_url.blank?
+
+        # For page_url, we're more lenient - it can be a relative path or full URL
+        uri = safe_parse_uri(page_url)
+
+        # If it parses as a URI and either has no scheme (relative) or has http/https scheme
+        return if uri && (uri.scheme.nil? || %w[http https].include?(uri.scheme&.downcase))
+
+        errors.add(:page_url, 'must be a valid URL')
+      end
     end
   end
 end

app/models/better_together/metrics/page_view.rb

@@ -4,6 +4,8 @@
 module BetterTogether
   module Metrics
     class PageView < ApplicationRecord # rubocop:todo Style/Documentation
+      include Utf8UrlHandler
+
       SENSITIVE_QUERY_PARAMS = %w[token password secret].freeze
 
       belongs_to :pageable, polymorphic: true
@@ -34,11 +36,15 @@ def set_page_url # rubocop:todo Metrics/AbcSize, Metrics/MethodLength
 
         return if url.blank?
 
-        uri = URI.parse(url)
-        @page_url_query = uri.query
-        self.page_url = uri.path
-      rescue URI::InvalidURIError
-        errors.add(:page_url, 'is invalid')
+        # Use our UTF-8 safe URI parser
+        uri = safe_parse_uri(url)
+        if uri
+          @page_url_query = uri.query
+          self.page_url = uri.path
+        else
+          # If we can't parse it at all, add an error
+          errors.add(:page_url, 'is invalid')
+        end
       end
 
       def page_url_without_sensitive_parameters

app/models/concerns/better_together/metrics/utf8_url_handler.rb

@@ -0,0 +1,132 @@
+# frozen_string_literal: true
+
+module BetterTogether
+  module Metrics
+    # Helper module for handling UTF-8 URLs in metrics models
+    module Utf8UrlHandler
+      extend ActiveSupport::Concern
+
+      private
+
+      # Parse a URL that may contain UTF-8 characters
+      # @param url [String] The URL to parse
+      # @return [URI::Generic, nil] Parsed URI or nil if invalid
+      def safe_parse_uri(url) # rubocop:todo Metrics/MethodLength
+        return if url.blank?
+
+        # First try with the URL as-is
+        begin
+          URI.parse(url)
+        rescue URI::InvalidURIError
+          # If that fails, try encoding it
+          encoded_url = encode_utf8_url(url)
+          begin
+            URI.parse(encoded_url)
+          rescue URI::InvalidURIError
+            nil
+          end
+        end
+      end
+
+      # Encode UTF-8 characters in a URL while preserving the structure
+      # @param url [String] The URL to encode
+      # @return [String] URL-encoded string
+      def encode_utf8_url(url) # rubocop:todo Metrics/AbcSize, Metrics/MethodLength, Metrics/PerceivedComplexity
+        return url if url.blank?
+
+        # Split URL into components to avoid encoding the protocol/scheme
+        if url.match(%r{\A([a-z]+://)}i)
+          scheme_and_authority, path_and_query = url.split('/', 3)
+          if path_and_query.present?
+            encoded_path = path_and_query.split('?').map do |part|
+              encode_utf8_component(part)
+            end.join('?')
+            "#{scheme_and_authority}/#{encoded_path}"
+          else
+            # Encode just the host part if no path
+            parts = url.split('//')
+            if parts.length > 1
+              protocol = parts[0]
+              host_and_rest = parts[1]
+              "#{protocol}//#{encode_host_component(host_and_rest)}"
+            else
+              url
+            end
+          end
+        else
+          # No protocol, encode the whole thing
+          encode_utf8_component(url)
+        end
+      end
+
+      # Encode UTF-8 characters in a URL component
+      # @param component [String] The URL component to encode
+      # @return [String] Encoded component
+      def encode_utf8_component(component)
+        return component if component.blank?
+
+        # Only encode non-ASCII characters
+        component.gsub(/[^\x00-\x7F]/) { |char| CGI.escape(char) }
+      end
+
+      # Encode UTF-8 characters in a host component (for IDN support)
+      # @param host_component [String] The host component to encode
+      # @return [String] Encoded host component
+      def encode_host_component(host_component) # rubocop:todo Metrics/MethodLength, Metrics/PerceivedComplexity
+        return host_component if host_component.blank?
+
+        # For international domain names, we need special handling
+        # Split by '/' to separate host from path
+        parts = host_component.split('/', 2)
+        host = parts[0]
+        path = parts[1]
+
+        # Convert international domain to punycode if needed
+        encoded_host = begin
+          # Try to convert to ASCII using Punycode for IDN support
+          if host.match?(/[^\x00-\x7F]/)
+            # For Ruby's built-in IDN support, we'll encode each part
+            host_parts = host.split('.')
+            encoded_parts = host_parts.map do |part|
+              if part.match?(/[^\x00-\x7F]/)
+                # Simple percent encoding for now
+                encode_utf8_component(part)
+              else
+                part
+              end
+            end
+            encoded_parts.join('.')
+          else
+            host
+          end
+        rescue StandardError
+          # Fallback to percent encoding
+          encode_utf8_component(host)
+        end
+
+        if path
+          "#{encoded_host}/#{encode_utf8_component(path)}"
+        else
+          encoded_host
+        end
+      end
+
+      # Validate if a URL is structurally valid for our purposes
+      # @param url [String] The URL to validate
+      # @return [Boolean] Whether the URL is valid
+      def valid_utf8_url?(url)
+        return false if url.blank?
+
+        uri = safe_parse_uri(url)
+        return false unless uri
+
+        # Check if it has a valid scheme
+        return false unless uri.scheme.present?
+
+        # For our metrics, we accept http, https, tel, mailto
+        allowed_schemes = %w[http https tel mailto]
+        allowed_schemes.include?(uri.scheme.downcase)
+      end
+    end
+  end
+end

spec/jobs/better_together/metrics/track_link_click_job_utf8_spec.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+
+RSpec.describe BetterTogether::Metrics::TrackLinkClickJob do
+  describe 'UTF-8 URL handling' do
+    let(:utf8_urls) do
+      [
+        'https://例え.テスト', # Japanese IDN
+        'https://тест.рф', # Cyrillic IDN
+        'https://example.com/café', # UTF-8 path
+        'https://example.com/页面', # Chinese characters in path
+        'https://bücher.example.com/straße', # German umlauts
+        'https://пример.испытание/тест' # Full Cyrillic URL
+      ]
+    end
+
+    let(:valid_params) do
+      {
+        page_url: 'https://example.com/test',
+        locale: 'en',
+        internal: false
+      }
+    end
+
+    it 'creates LinkClick records for UTF-8 URLs without errors' do
+      utf8_urls.each do |url|
+        expect do
+          described_class.perform_now(
+            url,
+            valid_params[:page_url],
+            valid_params[:locale],
+            valid_params[:internal]
+          )
+        end.to change(BetterTogether::Metrics::LinkClick, :count).by(1)
+
+        link_click = BetterTogether::Metrics::LinkClick.last
+        expect(link_click.url).to eq(url)
+        expect(link_click).to be_valid
+      end
+    end
+
+    it 'handles UTF-8 page URLs' do
+      utf8_urls.each do |page_url|
+        expect do
+          described_class.perform_now(
+            'https://example.com/link',
+            page_url,
+            'en',
+            false
+          )
+        end.to change(BetterTogether::Metrics::LinkClick, :count).by(1)
+
+        link_click = BetterTogether::Metrics::LinkClick.last
+        expect(link_click.page_url).to eq(page_url)
+        expect(link_click).to be_valid
+      end
+    end
+  end
+end

spec/jobs/better_together/metrics/track_page_view_job_utf8_spec.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+
+RSpec.describe BetterTogether::Metrics::TrackPageViewJob do
+  describe 'UTF-8 URL handling' do
+    let(:person) { create(:better_together_person) }
+    let(:pageable) { create(:better_together_page, content: 'Test page content') }
+
+    let(:utf8_urls) do
+      [
+        'https://example.com/café', # UTF-8 path
+        'https://example.com/页面', # Chinese characters in path
+        'https://bücher.example.com/straße' # German umlauts
+      ]
+    end
+
+    it 'creates PageView records without errors' do
+      # Test that the job can create page views successfully
+      # The UTF-8 handling is tested at the model level
+      expect do
+        described_class.perform_now(pageable, 'en')
+      end.to change(BetterTogether::Metrics::PageView, :count).by(1)
+
+      page_view = BetterTogether::Metrics::PageView.last
+      expect(page_view).to be_valid
+      expect(page_view.errors[:page_url]).to be_empty
+    end
+
+    it 'handles pageables with UTF-8 URLs' do
+      # Create a mock pageable that returns UTF-8 URL
+      utf8_pageable = instance_double(Page)
+      allow(utf8_pageable).to receive_messages(url: 'https://example.com/café', becomes: utf8_pageable,
+                                               class: double(base_class: double)) # rubocop:todo RSpec/VerifiedDoubles
+
+      expect do
+        described_class.perform_now(utf8_pageable, 'en')
+      end.to change(BetterTogether::Metrics::PageView, :count).by(1)
+
+      page_view = BetterTogether::Metrics::PageView.last
+      expect(page_view.errors[:page_url]).to be_empty
+    end
+  end
+end

spec/models/better_together/metrics/link_click_utf8_spec.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+
+RSpec.describe BetterTogether::Metrics::LinkClick do
+  describe 'UTF-8 URL validation' do
+    let(:utf8_urls) do
+      [
+        'https://例え.テスト', # Japanese IDN
+        'https://тест.рф', # Cyrillic IDN
+        'https://example.com/café', # UTF-8 path
+        'https://example.com/页面', # Chinese characters in path
+        'https://bücher.example.com/straße', # German umlauts
+        'https://пример.испытание/тест' # Full Cyrillic URL
+      ]
+    end
+
+    let(:valid_attributes) do
+      {
+        page_url: 'https://example.com/test',
+        locale: 'en',
+        clicked_at: Time.current,
+        internal: false
+      }
+    end
+
+    it 'accepts UTF-8 encoded URLs' do
+      utf8_urls.each do |url|
+        link_click = described_class.new(valid_attributes.merge(url: url))
+        expect(link_click).to be_valid, "URL #{url} should be valid but got errors: #{link_click.errors.full_messages}"
+      end
+    end
+
+    it 'accepts UTF-8 encoded page URLs' do
+      utf8_urls.each do |page_url|
+        link_click = described_class.new(valid_attributes.merge(page_url: page_url, url: 'https://example.com'))
+        expect(link_click).to be_valid,
+                              "Page URL #{page_url} should be valid but got errors: #{link_click.errors.full_messages}"
+      end
+    end
+
+    it 'handles URL encoding properly' do
+      # Test both encoded and unencoded versions
+      raw_url = 'https://example.com/café'
+      encoded_url = 'https://example.com/caf%C3%A9'
+
+      link_click1 = described_class.new(valid_attributes.merge(url: raw_url))
+      link_click2 = described_class.new(valid_attributes.merge(url: encoded_url))
+
+      expect(link_click1).to be_valid
+      expect(link_click2).to be_valid
+    end
+  end
+end