feat(deps): use html2rss in latest development status

gildesmarais · gildesmarais · commit 11359812b477 · 2025-03-16T14:28:41.000+01:00
Signed-off-by: Gil Desmarais &lt;git@desmarais.de&gt;
diff --git a/Gemfile b/Gemfile
@@ -4,7 +4,9 @@ source 'https://rubygems.org'
 
 git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }
 
-gem 'html2rss', '~> 0.14'
+# gem 'html2rss', '~> 0.14'
+gem 'html2rss', github: 'html2rss/html2rss'
+
 gem 'html2rss-configs', github: 'html2rss/html2rss-configs'
 
 # Use these instead of the two above (uncomment them) when developing locally:
diff --git a/Gemfile.lock b/Gemfile.lock
@@ -1,3 +1,25 @@
+GIT
+  remote: https://github.com/html2rss/html2rss
+  revision: 19957e05b8859c7c7680a9319a90c560661cf439
+  specs:
+    html2rss (0.17.0)
+      addressable (~> 2.7)
+      dry-validation
+      faraday (> 2.0.1, < 3.0)
+      faraday-follow_redirects
+      kramdown
+      mime-types (> 3.0)
+      nokogiri (>= 1.10, < 2.0)
+      parallel
+      puppeteer-ruby
+      regexp_parser
+      reverse_markdown (~> 3.0)
+      rss
+      sanitize
+      thor
+      tzinfo
+      zeitwerk
+
 GIT
   remote: https://github.com/html2rss/html2rss-configs
   revision: 6aadc0cf35c10642a4f15022e3ee9902360d3171
@@ -22,6 +44,41 @@ GEM
     crass (1.0.6)
     diff-lcs (1.6.0)
     docile (1.4.1)
+    dry-configurable (1.3.0)
+      dry-core (~> 1.1)
+      zeitwerk (~> 2.6)
+    dry-core (1.1.0)
+      concurrent-ruby (~> 1.0)
+      logger
+      zeitwerk (~> 2.6)
+    dry-inflector (1.2.0)
+    dry-initializer (3.2.0)
+    dry-logic (1.6.0)
+      bigdecimal
+      concurrent-ruby (~> 1.0)
+      dry-core (~> 1.1)
+      zeitwerk (~> 2.6)
+    dry-schema (1.14.1)
+      concurrent-ruby (~> 1.0)
+      dry-configurable (~> 1.0, >= 1.0.1)
+      dry-core (~> 1.1)
+      dry-initializer (~> 3.2)
+      dry-logic (~> 1.5)
+      dry-types (~> 1.8)
+      zeitwerk (~> 2.6)
+    dry-types (1.8.2)
+      bigdecimal (~> 3.0)
+      concurrent-ruby (~> 1.0)
+      dry-core (~> 1.0)
+      dry-inflector (~> 1.0)
+      dry-logic (~> 1.4)
+      zeitwerk (~> 2.6)
+    dry-validation (1.11.1)
+      concurrent-ruby (~> 1.0)
+      dry-core (~> 1.1)
+      dry-initializer (~> 3.2)
+      dry-schema (~> 1.14)
+      zeitwerk (~> 2.6)
     erubi (1.13.1)
     faraday (2.12.2)
       faraday-net_http (>= 2.0, < 3.5)
@@ -32,22 +89,6 @@ GEM
     faraday-net_http (3.4.0)
       net-http (>= 0.5.0)
     hashdiff (1.1.2)
-    html2rss (0.17.0)
-      addressable (~> 2.7)
-      faraday (> 2.0.1, < 3.0)
-      faraday-follow_redirects
-      kramdown
-      mime-types (> 3.0)
-      nokogiri (>= 1.10, < 2.0)
-      parallel
-      puppeteer-ruby
-      regexp_parser
-      reverse_markdown (~> 3.0)
-      rss
-      sanitize (~> 6.0)
-      thor
-      tzinfo
-      zeitwerk
     json (2.10.2)
     kramdown (2.5.1)
       rexml (>= 3.3.9)
@@ -58,23 +99,25 @@ GEM
       logger
       mime-types-data (~> 3.2015)
     mime-types-data (3.2025.0304)
-    mini_portile2 (2.8.8)
     net-http (0.6.0)
       uri
     nio4r (2.7.4)
-    nokogiri (1.18.4)
-      mini_portile2 (~> 2.8.2)
-      racc (~> 1.4)
     nokogiri (1.18.4-aarch64-linux-gnu)
       racc (~> 1.4)
+    nokogiri (1.18.4-aarch64-linux-musl)
+      racc (~> 1.4)
     nokogiri (1.18.4-arm-linux-gnu)
       racc (~> 1.4)
+    nokogiri (1.18.4-arm-linux-musl)
+      racc (~> 1.4)
     nokogiri (1.18.4-arm64-darwin)
       racc (~> 1.4)
     nokogiri (1.18.4-x86_64-darwin)
       racc (~> 1.4)
     nokogiri (1.18.4-x86_64-linux-gnu)
       racc (~> 1.4)
+    nokogiri (1.18.4-x86_64-linux-musl)
+      racc (~> 1.4)
     parallel (1.26.3)
     parser (3.3.7.1)
       ast (~> 2.4.1)
@@ -144,9 +187,9 @@ GEM
       lint_roller (~> 1.1)
       rubocop (~> 1.72, >= 1.72.1)
     ruby-progressbar (1.13.0)
-    sanitize (6.1.3)
+    sanitize (7.0.0)
       crass (~> 1.0.2)
-      nokogiri (>= 1.12.0)
+      nokogiri (>= 1.16.8)
     sentry-ruby (5.23.0)
       bigdecimal
       concurrent-ruby (~> 1.0, >= 1.0.2)
@@ -180,19 +223,21 @@ GEM
     zeitwerk (2.7.2)
 
 PLATFORMS
-  aarch64-linux
-  arm-linux
+  aarch64-linux-gnu
+  aarch64-linux-musl
+  arm-linux-gnu
+  arm-linux-musl
   arm64-darwin
-  x86-linux
   x86_64-darwin
-  x86_64-linux
+  x86_64-linux-gnu
+  x86_64-linux-musl
 
 DEPENDENCIES
   base64
   byebug
   climate_control
   erubi
-  html2rss (~> 0.14)
+  html2rss!
   html2rss-configs!
   parallel
   puma
@@ -229,12 +274,20 @@ CHECKSUMS
   crass (1.0.6) sha256=dc516022a56e7b3b156099abc81b6d2b08ea1ed12676ac7a5657617f012bd45d
   diff-lcs (1.6.0) sha256=a1e7f7b272962f8fc769358ad00001b87cdcf32ba349d6c70c6b544613d2da2e
   docile (1.4.1) sha256=96159be799bfa73cdb721b840e9802126e4e03dfc26863db73647204c727f21e
+  dry-configurable (1.3.0) sha256=882d862858567fc1210d2549d4c090f34370fc1bb7c5c1933de3fe792e18afa8
+  dry-core (1.1.0) sha256=0903821a9707649a7da545a2cd88e20f3a663ab1c5288abd7f914fa7751ab195
+  dry-inflector (1.2.0) sha256=22f5d0b50fd57074ae57e2ca17e3b300e57564c218269dcf82ff3e42d3f38f2e
+  dry-initializer (3.2.0) sha256=37d59798f912dc0a1efe14a4db4a9306989007b302dcd5f25d0a2a20c166c4e3
+  dry-logic (1.6.0) sha256=da6fedbc0f90fc41f9b0cc7e6f05f5d529d1efaef6c8dcc8e0733f685745cea2
+  dry-schema (1.14.1) sha256=2fcd7539a7099cacae6a22f6a3a2c1846fe5afeb1c841cde432c89c6cb9b9ff1
+  dry-types (1.8.2) sha256=c84e9ada69419c727c3b12e191e0ed7d2c6d58d040d55e79ea16e0ebf8b3ec0f
+  dry-validation (1.11.1) sha256=70900bb5a2d911c8aab566d3e360c6bff389b8bf92ea8e04885ce51c41ff8085
   erubi (1.13.1) sha256=a082103b0885dbc5ecf1172fede897f9ebdb745a4b97a5e8dc63953db1ee4ad9
   faraday (2.12.2) sha256=157339c25c7b8bcb739f5cf1207cb0cefe8fa1c65027266bcbc34c90c84b9ad6
   faraday-follow_redirects (0.3.0) sha256=d92d975635e2c7fe525dd494fcd4b9bb7f0a4a0ec0d5f4c15c729530fdb807f9
   faraday-net_http (3.4.0) sha256=a1f1e4cd6a2cf21599c8221595e27582d9936819977bbd4089a601f24c64e54a
   hashdiff (1.1.2) sha256=2c30eeded6ed3dce8401d2b5b99e6963fe5f14ed85e60dd9e33c545a44b71a77
-  html2rss (0.17.0) sha256=f97c978272212fb9b67dfe8f9712a212ff22c9a812f574e9d64df703a8278370
+  html2rss (0.17.0)
   html2rss-configs (0.2.0)
   json (2.10.2) sha256=34e0eada93022b2a0a3345bb0b5efddb6e9ff5be7c48e409cfb54ff8a36a8b06
   kramdown (2.5.1) sha256=87bbb6abd9d3cebe4fc1f33e367c392b4500e6f8fa19dd61c0972cf4afe7368c
@@ -243,15 +296,16 @@ CHECKSUMS
   logger (1.6.6) sha256=dd618d24e637715472732e7eed02e33cfbdf56deaad225edd0f1f89d38024017
   mime-types (3.6.0) sha256=6f71db957840ceae44211531eff3e2f7e0dd4645fefb5f535dbaeb6307ab6464
   mime-types-data (3.2025.0304) sha256=7ed9385881432d708b7f4e9cf2a4a9e1d4e8bf8abcd08283c23a0d5cd7205097
-  mini_portile2 (2.8.8) sha256=8e47136cdac04ce81750bb6c09733b37895bf06962554e4b4056d78168d70a75
   net-http (0.6.0) sha256=9621b20c137898af9d890556848c93603716cab516dc2c89b01a38b894e259fb
   nio4r (2.7.4) sha256=d95dee68e0bb251b8ff90ac3423a511e3b784124e5db7ff5f4813a220ae73ca9
-  nokogiri (1.18.4) sha256=bb7820521c1bbae1d3e0092ff03b27a8e700912b37d80f962b7e4567947a64ac
   nokogiri (1.18.4-aarch64-linux-gnu) sha256=8f2263cef9953ce09bd5293d76c9bbd3013d2f94d1cca67783dfe6635c529deb
+  nokogiri (1.18.4-aarch64-linux-musl) sha256=4e231f8ba3128cfc2ef0cc0bdc807d7ce71fc62cb6a78216e817be8631fe6a96
   nokogiri (1.18.4-arm-linux-gnu) sha256=cc2945e2c19560a61a97737e6bd3b329edb1f82ca204d46a18e5e98ad0a550a6
+  nokogiri (1.18.4-arm-linux-musl) sha256=4fb7f44de0cd85abfa869e4cfb619410da174ebf9fbe26ae0caa65462b818bcb
   nokogiri (1.18.4-arm64-darwin) sha256=73902663b23b1123282b9c0b6d9654b1fb286dfee8d65cb1f6029087b7f0d037
   nokogiri (1.18.4-x86_64-darwin) sha256=e4776f58eea9b94d05caf8bf351e3c6aa1cce01edcc2ed530f3c302c13178965
   nokogiri (1.18.4-x86_64-linux-gnu) sha256=b1c6407b346b88704e97a342a80acd4755175324e624da34d0c5cfdc8d34191e
+  nokogiri (1.18.4-x86_64-linux-musl) sha256=ea7c0356a70f3d2d0d76315b533877013d20368d5c9f437c38e0bd462c4844dc
   parallel (1.26.3) sha256=d86babb7a2b814be9f4b81587bf0b6ce2da7d45969fab24d8ae4bf2bb4d4c7ef
   parser (3.3.7.1) sha256=7dbe61618025519024ac72402a6677ead02099587a5538e84371b76659e6aca1
   public_suffix (6.0.1) sha256=61d44e1cab5cbbbe5b31068481cf16976dd0dc1b6b07bd95617ef8c5e3e00c6f
@@ -282,7 +336,7 @@ CHECKSUMS
   rubocop-rspec (3.5.0) sha256=710c942fe1af884ba8eea75cbb8bdbb051929a2208880a6fc2e2dce1eed5304c
   rubocop-thread_safety (0.7.2) sha256=bd51449c420b1ddda5672b71a39706367402beb55aaf19fc020c1868717f31f6
   ruby-progressbar (1.13.0) sha256=80fc9c47a9b640d6834e0dc7b3c94c9df37f08cb072b7761e4a71e22cff29b33
-  sanitize (6.1.3) sha256=a42ce5f933d82765a8243599d5ab2e8867237e76cf5a699caaddfe60bb944152
+  sanitize (7.0.0) sha256=269d1b9d7326e69307723af5643ec032ff86ad616e72a3b36d301ac75a273984
   sentry-ruby (5.23.0) sha256=8e8bb2f9a56a267a50fcba947f2ae131b6542f45fc3bb5764c2c25ba68f385cc
   simplecov (0.22.0) sha256=fe2622c7834ff23b98066bb0a854284b2729a569ac659f82621fc22ef36213a5
   simplecov-html (0.13.1) sha256=5dab0b7ee612e60e9887ad57693832fdf4695b4c0c859eaea5f95c18791ef10b
diff --git a/app.rb b/app.rb
@@ -84,11 +84,38 @@ def self.development? = ENV['RACK_ENV'] == 'development'
         end
 
         r.on String, String do |folder_name, config_name_with_ext|
-          handle_html2rss_configs(request, folder_name, config_name_with_ext)
+          response['Content-Type'] = CONTENT_TYPE_RSS
+
+          name = "#{folder_name}/#{File.basename(config_name_with_ext, '.*')}"
+          config = Html2rss::Configs.find_by_name(name).dup
+
+          if (params = request.params).any?
+            config[:params] ||= {}
+            config[:params].merge!(params)
+          end
+
+          feed = Html2rss.feed(config)
+
+          HttpCache.expires(response, feed.channel.ttl.to_i * 60, cache_control: 'public')
+
+          feed.to_xml
         end
 
         r.on String do |config_name_with_ext|
-          handle_local_config_feeds(request, config_name_with_ext)
+          response['Content-Type'] = CONTENT_TYPE_RSS
+
+          config = LocalConfig.find(File.basename(config_name_with_ext, '.*'))
+
+          if (params = request.params).any?
+            config[:params] ||= {}
+            config[:params].merge!(params)
+          end
+
+          feed = Html2rss.feed(config)
+
+          HttpCache.expires(response, feed.channel.ttl.to_i * 60, cache_control: 'public')
+
+          feed.to_xml
         end
       end
 
diff --git a/app/html2rss_facade.rb b/app/html2rss_facade.rb
@@ -14,34 +14,47 @@ class Html2rssFacade
 
       attr_reader :feed_config, :typecast_params
 
-      ##
-      # @param name [String] the name of a html2rss-configs provided config.
-      # @param typecast_params [Object]
-      # @return [String] the serialized RSS feed
-      def self.from_config(name, typecast_params, &)
-        feed_config = Html2rss::Configs.find_by_name(name)
-        new(feed_config, typecast_params).feed(&)
-      end
+      class << self
+        ##
+        # @param name [String] the name of a html2rss-configs provided config.
+        # @param typecast_params [Object]
+        # @return [String] the serialized RSS feed
+        def from_config(name, typecast_params, &)
+          feed_config = Html2rss::Configs.find_by_name(name)
+          new(feed_config, typecast_params).feed(&)
+        end
 
-      ##
-      # @param name [String] the name of a feed in the file `config/feeds.yml`
-      # @param typecast_params [Object]
-      # @return [String] the serialized RSS feed
-      def self.from_local_config(name, typecast_params, &)
-        feed_config = LocalConfig.find(name)
-        new(feed_config, typecast_params).feed(&)
-      end
+        ##
+        # @param name [String] the name of a feed in the file `config/feeds.yml`
+        # @param typecast_params [Object]
+        # @return [String] the serialized RSS feed
+        def from_local_config(name, typecast_params, &)
+          feed_config = LocalConfig.find(name)
+          new(feed_config, typecast_params).feed(&)
+        end
 
-      ##
-      # @param feed_config [Hash<Symbol, Object>]
-      # @param typecast_params [Object]
-      # @param global_config [Hash<Symbol, Object>]
-      # @return [Html2rss::Config]
-      # @raise [Roda::RodaPlugins::TypecastParams::Error]
-      def self.feed_config_to_config(feed_config, typecast_params, global_config: LocalConfig.global)
-        dynamic_params = Html2rss::Config::Channel.required_params_for_config(feed_config[:channel])
-                                                  .to_h { |name| [name, typecast_params.str!(name)] }
-        Html2rss::Config.new(feed_config, global_config, dynamic_params)
+        ##
+        # @param feed_config [Hash<Symbol, Object>]
+        # @param typecast_params [Object]
+        # @param global_config [Hash<Symbol, Object>]
+        # @return [Html2rss::Config]
+        # @raise [Roda::RodaPlugins::TypecastParams::Error]
+        def feed_config_to_config(feed_config, typecast_params, global_config: LocalConfig.global)
+          params = required_params_for_config(feed_config[:channel])
+                   .to_h { |name| [name, typecast_params.str!(name)] }
+
+          config = global_config.merge(feed_config)
+          config[:params] = params if params.any?
+          config
+        end
+
+        private
+
+        def required_params_for_config(config)
+          config.each_with_object(Set.new) do |(_, value), required_params|
+            required_params.merge(value.scan(/%<(\w+)>[s|d]/).flatten) if value.is_a?(String)
+          end
+        end
       end
 
       ##
diff --git a/app/http_cache.rb b/app/http_cache.rb
@@ -15,6 +15,8 @@ module HttpCache
       # @param seconds [Integer]
       # @param cache_control [String, nil]
       def expires(response, seconds, cache_control: nil)
+        expires_now(response) and return if seconds <= 0
+
         response['Expires'] = (Time.now + seconds).httpdate
 
         cache_value = "max-age=#{seconds}"
diff --git a/app/ssrf_filter_strategy.rb b/app/ssrf_filter_strategy.rb
@@ -16,6 +16,7 @@ def execute
         response = SsrfFilter.get(ctx.url, headers:)
 
         Html2rss::RequestService::Response.new(body: response.body,
+                                               url: ctx.url,
                                                headers: response.to_hash.transform_values(&:first))
       end
     end
diff --git a/helpers/handle_error.rb b/helpers/handle_error.rb
@@ -8,12 +8,11 @@ module Web
     class App
       def handle_error(error) # rubocop:disable Metrics/MethodLength
         case error
-        when Html2rss::Config::ParamsMissing,
+        when Html2rss::Config::DynamicParams::ParamsMissing,
              Roda::RodaPlugins::TypecastParams::Error
           set_error_response('Parameters missing or invalid', 422)
-        when Html2rss::AttributePostProcessors::UnknownPostProcessorName,
-             Html2rss::ItemExtractors::UnknownExtractorName,
-             Html2rss::Config::ChannelMissing
+        when Html2rss::Selectors::PostProcessors::UnknownPostProcessorName,
+             Html2rss::Selectors::Extractors::UnknownExtractorName
           set_error_response('Invalid feed config', 422)
         when LocalConfig::NotFound,
              Html2rss::Configs::ConfigNotFound
diff --git a/helpers/handle_html2rss_configs.rb b/helpers/handle_html2rss_configs.rb
@@ -8,7 +8,8 @@ def handle_html2rss_configs(request, _folder_name, _config_name_with_ext)
 
         Html2rssFacade.from_config(path.full_config_name, typecast_params) do |config|
           response['Content-Type'] = CONTENT_TYPE_RSS
-          HttpCache.expires(response, config.ttl * 60, cache_control: 'public')
+
+          HttpCache.expires(response, config.dig(:channel, :ttl).to_i * 60, cache_control: 'public')
         end
       end
     end
diff --git a/helpers/handle_local_config_feeds.rb b/helpers/handle_local_config_feeds.rb
@@ -8,7 +8,8 @@ def handle_local_config_feeds(request, _config_name_with_ext)
 
         Html2rssFacade.from_local_config(path.full_config_name, typecast_params) do |config|
           response['Content-Type'] = CONTENT_TYPE_RSS
-          HttpCache.expires(response, config.ttl * 60, cache_control: 'public')
+
+          HttpCache.expires(response, config.dig(:channel, :ttl).to_i * 60, cache_control: 'public')
         end
       end
     end
diff --git a/routes/auto_source.rb b/routes/auto_source.rb
@@ -27,7 +27,7 @@ class App
 
               response['Content-Type'] = CONTENT_TYPE_RSS
 
-              url = Addressable::URI.parse Base64.urlsafe_decode64(encoded_url)
+              url = Addressable::URI.parse(Base64.urlsafe_decode64(encoded_url)).to_s
               rss = Html2rss.auto_source(url, strategy:)
 
               # Unfortunately, Ruby's rss gem does not provide a direct method to
