Fix: crawl the new docusaurus site.

Author: dblock

.rubocop_todo.yml

@@ -1,17 +1,23 @@
 # This configuration was generated by
 # `rubocop --auto-gen-config`
-# on 2023-03-19 01:49:28 UTC using RuboCop version 1.36.0.
+# on 2025-09-06 21:08:28 UTC using RuboCop version 1.36.0.
 # The point is for the user to remove these configuration records
 # one by one as the offenses are removed from the code base.
 # Note that changes in the inspected code, or installation of new
 # versions of RuboCop, may require this file to be generated again.
 
-# Offense count: 7
+# Offense count: 1
+# Configuration parameters: AllowKeywordBlockArguments.
+Lint/UnderscorePrefixedVariableName:
+  Exclude:
+    - 'tasks/lib/slack_api/methods_spider.rb'
+
+# Offense count: 4
 # Configuration parameters: AllowedMethods, AllowedPatterns, IgnoredMethods, CountRepeatedAttributes.
 Metrics/AbcSize:
-  Max: 41
+  Max: 45
 
-# Offense count: 2
+# Offense count: 3
 # Configuration parameters: CountComments, CountAsOne, ExcludedMethods, AllowedMethods, AllowedPatterns, IgnoredMethods.
 # AllowedMethods: refine
 Metrics/BlockLength:
@@ -20,19 +26,40 @@ Metrics/BlockLength:
 # Offense count: 1
 # Configuration parameters: CountComments, CountAsOne.
 Metrics/ClassLength:
-  Max: 153
+  Max: 125
 
 # Offense count: 2
 # Configuration parameters: AllowedMethods, AllowedPatterns, IgnoredMethods.
 Metrics/CyclomaticComplexity:
-  Max: 11
+  Max: 17
 
-# Offense count: 9
+# Offense count: 5
 # Configuration parameters: CountComments, CountAsOne, ExcludedMethods, AllowedMethods, AllowedPatterns, IgnoredMethods.
 Metrics/MethodLength:
   Max: 33
 
 # Offense count: 1
 # Configuration parameters: AllowedMethods, AllowedPatterns, IgnoredMethods.
 Metrics/PerceivedComplexity:
-  Max: 10
+  Max: 16
+
+# Offense count: 3
+Security/Open:
+  Exclude:
+    - 'tasks/lib/docs/downloader.rb'
+
+# Offense count: 1
+# Configuration parameters: AllowedConstants.
+Style/Documentation:
+  Exclude:
+    - 'spec/**/*'
+    - 'test/**/*'
+    - 'tasks/lib/docs/downloader.rb'
+
+# Offense count: 1
+# This cop supports unsafe autocorrection (--autocorrect-all).
+# Configuration parameters: EnforcedStyle.
+# SupportedStyles: always, always_true, never
+Style/FrozenStringLiteralComment:
+  Exclude:
+    - 'tasks/lib/docs/downloader.rb'

Rakefile

@@ -9,4 +9,5 @@ require 'fileutils'
 
 Bundler.setup :default, :development
 
+load 'tasks/download.rake'
 load 'tasks/update.rake'

tasks/download.rake

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+require_relative 'lib/docs/downloader'
+
+namespace :api do
+  namespace :ref do
+    desc 'Download JSON reference.'
+    task :download do
+      Rake::Task['api:ref:clean_files'].invoke('docs.slack.dev')
+      downloader = SlackApi::Docs::Downloader.new
+      downloader.download!
+      puts "\nFinished downloading reference."
+    end
+
+    desc 'Delete all generated files except undocumented ones.'
+    task :clean_files, :dirs do |_t, args|
+      files = Dir["./{#{Array(args[:dirs]).join(',')}}/*"].grep_v(%r{/undocumented\b})
+      FileUtils.rm_rf files
+    end
+  end
+end

tasks/lib/docs/downloader.rb

@@ -0,0 +1,74 @@
+module SlackApi
+  module Docs
+    class Downloader
+      def target_path
+        @target_path ||= File.expand_path('../../../docs.slack.dev', __dir__)
+      end
+
+      def events_dir
+        File.join(target_path, 'events')
+      end
+
+      def events_path
+        File.join(events_dir, 'events.json')
+      end
+
+      def methods_dir
+        File.join(target_path, 'methods')
+      end
+
+      def methods_path
+        File.join(methods_dir, 'methods.json')
+      end
+
+      def events_url
+        'https://docs.slack.dev/reference/events.json'
+      end
+
+      def methods_url
+        'https://docs.slack.dev/reference/methods.json'
+      end
+
+      def method_url(method)
+        "https://docs.slack.dev/reference/methods/#{method}.json"
+      end
+
+      def method_target_path(method_name)
+        File.join(methods_dir, "#{method_name}.json")
+      end
+
+      def download!
+        download_methods!
+        download_events!
+      end
+
+      def download_methods!
+        puts "#{methods_url} => #{methods_path}"
+        FileUtils.mkdir_p(methods_dir)
+        URI.open(methods_url) do |file|
+          json = JSON.parse(file.read)
+          File.write(methods_path, JSON.pretty_generate(json))
+          json.each do |method|
+            method_name = method['name']
+            method_url = method_url(method_name)
+            method_target_path = method_target_path(method_name)
+            puts "#{method_url} => #{method_target_path}"
+            URI.open(method_url) do |method_file|
+              method_json = JSON.parse(method_file.read)
+              File.write(method_target_path, JSON.pretty_generate(method_json))
+            end
+          end
+        end
+      end
+
+      def download_events!
+        puts "#{events_url} => #{events_path}"
+        URI.open(events_url) do |file|
+          json = JSON.parse(file.read)
+          FileUtils.mkdir_p(events_dir)
+          File.write(events_path, JSON.pretty_generate(json))
+        end
+      end
+    end
+  end
+end

tasks/lib/slack_api/events_spider.rb

@@ -3,57 +3,55 @@
 module SlackApi
   # Scrapes Slack events
   class EventsSpider < BaseSpider
-    handle 'https://api.slack.com/events', :process_list
+    handle 'https://docs.slack.dev/reference/events/', :process_list
+
+    def downloader
+      @downloader ||= SlackApi::Docs::Downloader.new
+    end
 
     def process_list(page, _default_data = {})
-      events_page = ensure!(page, '.apiEventsPage')
-      list = events_page.search('.apiEventPage__eventList')
-      ref = list.search('[data-automount-component=ApiDocsFilterableReferenceList]')
-      data = JSON.parse(ref.attribute('data-automount-props'))
-      raise(ElementNotFound, 'Could not parse events reference') unless data['items'].any?
-
-      data['items'].each do |event|
-        next unless event['isPublic']
-        next if event['isDeprecated']
-        next unless event['groups'].include?('RTM')
-
-        handle resolve_url(event['link'], page),
+      events = JSON.load_file(downloader.events_path)
+
+      events.each do |event|
+        next unless event['APIs']&.include?('RTM')
+
+        handle resolve_url(event['name'], page),
                :process_event,
                name: event['name'],
                desc: event['description'],
                required_scope: 'RTM'
       end
     end
 
-    def process_event(page, data = {})
-      event_page = ensure!(page, '.apiEventPage', data[:name])
-      descriptions = event_page.search('.apiDocsPage__markdownOutput p')
-      long_desc = descriptions.map(&:text).join(' ').gsub("\n", ' ').strip
+    def process_event(_page, data = {})
+      # event_page = ensure!(page, '#__docusaurus')
+      # descriptions = event_page.search('.apiDocsPage__markdownOutput p')
+      # long_desc = descriptions.map(&:text).join(' ').gsub("\n", ' ').strip
       # required_scopes = event_page.search('.apiReference__scope code').map(&:text).map(&:strip).join(', ')
 
       json_hash = {
         'name' => data[:name],
-        'desc' => "#{data[:desc]}.",
-        'long_desc' => long_desc,
-        'required_scope' => data[:required_scope]
+        'desc' => "#{data[:desc]}."
+        # 'long_desc' => long_desc,
+        # 'required_scope' => data[:required_scope]
       }
 
-      example = begin
-        JSON.parse(
-          event_page.search('.apiDocsPage__markdownOutput pre:first code')
-            .text
-            .gsub('…', '')
-            .gsub('...', '')
-            .gsub("\n", ' ')
-            .gsub(/\s+/, ' ')
-            .gsub(', }', '}')
-            .gsub(', ]', ']')
-        )
-      rescue StandardError
-        nil
-      end
-
-      json_hash['example'] = example if example
+      # example = begin
+      #   JSON.parse(
+      #     event_page.search('.apiDocsPage__markdownOutput pre:first code')
+      #       .text
+      #       .gsub('…', '')
+      #       .gsub('...', '')
+      #       .gsub("\n", ' ')
+      #       .gsub(/\s+/, ' ')
+      #       .gsub(', }', '}')
+      #       .gsub(', ]', ']')
+      #   )
+      # rescue StandardError
+      #   nil
+      # end
+
+      # json_hash['example'] = example if example
 
       record(file_name: "events/#{data[:name]}.json", json: JSON.pretty_generate(json_hash))
     end