Use Addressable to heuristically parse invalid URLs and normalize them (#603)

Author: chvp

Gemfile

@@ -7,6 +7,7 @@ ruby '~> 3.3.1'
 
 gem 'rails', '~> 8.0.1'
 
+gem 'addressable' # More standards-compliant URI parser
 gem 'bcrypt' # Use Active Model has_secure_password
 gem 'bootsnap', require: false # Reduces boot times through caching; required in config/boot.rb
 gem 'feedjira' # Parse RSS feeds

Gemfile.lock

@@ -372,6 +372,7 @@ PLATFORMS
   ruby
 
 DEPENDENCIES
+  addressable
   annotaterb
   bcrypt
   bootsnap

app/models/entry.rb

@@ -27,6 +27,8 @@
 #
 #  fk_rails_...  (subscription_id => subscriptions.id)
 #
+require 'addressable/uri'
+
 class Entry < ApplicationRecord
   FEEDJIRA_KEYS_MAP = {
     author: :author,
@@ -72,8 +74,7 @@ def read?
   end
 
   def normalize_url(input)
-    # Some urls might contain spaces, so we replace these
-    uri = URI(input.gsub(' ', '%20'))
+    uri = Addressable::URI.heuristic_parse(input).normalize
     # Some entries might contain absolute/relative path to the page they were on
     uri = URI(url).merge(uri) if url.present?
     uri.to_s

test/models/entry_test.rb

@@ -71,10 +71,22 @@ class EntryTest < ActiveSupport::TestCase
   end
 
   # Methods
-  test 'should be able to normalize urls found in post' do
+  test 'should be able to normalize urls found in post when containing spaces' do
     entry = build(:entry, url: 'https://example.com/posts/first.html')
 
     assert_equal 'https://example.com/image%201.jpg', entry.normalize_url('https://example.com/image 1.jpg')
+  end
+
+  test 'should be able to normalize urls found in post when containing unicode' do
+    entry = build(:entry, url: 'https://example.com/posts/first.html')
+
+    assert_equal 'https://example.com/image%F0%9F%96%A41.jpg', entry.normalize_url('https://example.com/image🖤1.jpg')
+    assert_equal 'https://example.com/image%E2%80%941.jpg', entry.normalize_url('https://example.com/image—1.jpg')
+  end
+
+  test 'should be able to normalize urls found in post when missing host' do
+    entry = build(:entry, url: 'https://example.com/posts/first.html')
+
     assert_equal 'https://example.com/image.jpg', entry.normalize_url('/image.jpg')
   end
 end