Fix: crawn the new docusaurus site.

dblock · dblock · commit 31573924aa33 · 2025-09-06T16:45:06.000-04:00
diff --git a/Rakefile b/Rakefile
@@ -9,4 +9,5 @@ require 'fileutils'
 
 Bundler.setup :default, :development
 
+load 'tasks/download.rake'
 load 'tasks/update.rake'
diff --git a/tasks/download.rake b/tasks/download.rake
@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+require_relative 'lib/docs/downloader'
+
+namespace :api do
+  namespace :ref do
+    desc 'Download JSON reference.'
+    task :download do
+      Rake::Task['api:ref:clean_files'].invoke('docs.slack.dev')
+      downloader = SlackApi::Docs::Downloader.new
+      downloader.download!
+      puts "\nFinished downloading reference."
+    end
+
+    desc 'Delete all generated files except undocumented ones.'
+    task :clean_files, :dirs do |_t, args|
+      files = Dir["./{#{Array(args[:dirs]).join(',')}}/*"].grep_v(%r{/undocumented\b})
+      FileUtils.rm_rf files
+    end
+  end
+end
diff --git a/tasks/lib/docs/downloader.rb b/tasks/lib/docs/downloader.rb
@@ -0,0 +1,74 @@
+module SlackApi
+  module Docs
+    class Downloader
+      def target_path
+        @target_path ||= File.expand_path('../../../docs.slack.dev', __dir__)
+      end
+
+      def events_dir
+        File.join(target_path, 'events')
+      end
+
+      def events_path
+        File.join(events_dir, 'events.json')
+      end
+
+      def methods_dir
+        File.join(target_path, 'methods')
+      end
+
+      def methods_path
+        File.join(methods_dir, 'methods.json')
+      end
+
+      def events_url
+        'https://docs.slack.dev/reference/events.json'
+      end
+
+      def methods_url
+        'https://docs.slack.dev/reference/methods.json'
+      end
+
+      def method_url(method)
+        "https://docs.slack.dev/reference/methods/#{method}.json"
+      end
+
+      def method_target_path(method_name)
+        File.join(methods_dir, "#{method_name}.json")
+      end
+
+      def download!
+        download_methods!
+        download_events!
+      end
+
+      def download_methods!
+        puts "#{methods_url} => #{methods_path}"
+        FileUtils.mkdir_p(methods_dir)
+        URI.open(methods_url) do |file|
+          json = JSON.parse(file.read)
+          File.write(methods_path, JSON.pretty_generate(json))
+          json.each do |method|
+            method_name = method['name']
+            method_url = method_url(method_name)
+            method_target_path = method_target_path(method_name)
+            puts "#{method_url} => #{method_target_path}"
+            URI.open(method_url) do |method_file|
+              method_json = JSON.parse(method_file.read)
+              File.write(method_target_path, JSON.pretty_generate(method_json))
+            end
+          end
+        end
+      end
+
+      def download_events!
+        puts "#{events_url} => #{events_path}"
+        URI.open(events_url) do |file|
+          json = JSON.parse(file.read)
+          FileUtils.mkdir_p(events_dir)
+          File.write(events_path, JSON.pretty_generate(json))
+        end
+      end
+    end
+  end
+end
diff --git a/tasks/lib/slack_api/events_spider.rb b/tasks/lib/slack_api/events_spider.rb
@@ -15,7 +15,7 @@ def process_list(page, _default_data = {})
       data['items'].each do |event|
         next unless event['isPublic']
         next if event['isDeprecated']
-        next unless event['groups'].include?('RTM')
+        next unless event['groups']&.include?('RTM')
 
         handle resolve_url(event['link'], page),
                :process_event,
diff --git a/tasks/lib/slack_api/methods_spider.rb b/tasks/lib/slack_api/methods_spider.rb
@@ -5,26 +5,26 @@ module SlackApi
   class MethodsSpider < BaseSpider
     handle 'https://api.slack.com/methods', :process_list
 
+    def downloader
+      @downloader ||= SlackApi::Docs::Downloader.new
+    end
+
     def process_list(page, _default_data = {})
-      methods_page = ensure!(page, '.apiMethodPage')
-      list = methods_page.search('.apiMethodPage__methodList')
-      ref = list.search('[data-automount-component=ApiDocsFilterableReferenceList]')
-      data = JSON.parse(ref.attribute('data-automount-props'))
-      raise(ElementNotFound, 'Could not parse methods reference') unless data['items'].any?
+      methods = JSON.load_file(downloader.methods_path)
 
       groups = Set.new
-      data['items'].each do |method|
-        next unless method['isPublic']
-        next if method['isDeprecated']
 
-        groups += method['groups']
+      methods.each do |method|
+        groups += method['family']
         method_name = method['name']
-        method_group = method['groups'].first.split('.').first
+        method_group = method['family'].first.split('.').first
+        downloaded_filename = downloader.method_target_path(method_name)
         file_name = "methods/#{method_group}/#{method_name}.json"
-        method_url = resolve_url(method['link'], page)
+        method_url = resolve_url(downloader.method_url(method_name), page)
         handle method_url,
                :process_method,
                filename: file_name,
+               downloaded_filename: downloaded_filename,
                method_name: method_name,
                method_group: method_group,
                method_url: method_url
@@ -37,14 +37,15 @@ def process_list(page, _default_data = {})
       end
     end
 
-    def process_method(page, default_data = {})
-      method_page = ensure!(page, '.apiMethodPage', default_data[:method_name])
-      desc = method_page.search('.apiReference__mainDescription').text.gsub('’', "'")
-      return if desc.downcase.start_with? 'deprecated:'
+    def process_method(method_page, default_data = {})
+      method_data = JSON.load_file(default_data[:downloaded_filename])
 
-      args, arg_groups, fields = parse_args(method_page, default_data)
-      errors = parse_errors(method_page, default_data)
-      response = parse_response(method_page, default_data)
+      desc = method_data['desc'].gsub('’', "'")
+      return if desc.downcase.start_with? 'Deprecated:'
+
+      args, arg_groups, fields = parse_args(method_page, method_data)
+      errors = parse_errors(method_page, method_data)
+      response = parse_response(method_page, method_data)
 
       json_hash = {
         'group' => default_data[:method_group],
@@ -62,24 +63,25 @@ def process_method(page, default_data = {})
 
     private
 
-    def parse_args(api_page, default_data = {})
-      args_wrapper = ensure!(api_page, '.apiReference__arguments', default_data[:method_name])
-      rows = args_wrapper.search('.apiMethodPage__argumentRow')
+    def parse_args(_api_page, method_data = {})
       args = {}
       fields = {}
-      rows.each do |row|
-        name = row.search('.apiMethodPage__argument code').text
-        type = massage_type(name,
-                            row.search('.apiMethodPage__argumentType').text,
-                            default_data)
-        required = row.search('.apiMethodPage__argumentOptionality--required').any?
-
-        desc = row.search('.apiMethodPage__argumentDesc p')
-          .text
-          .tap { |t| t.slice!("\n") }
-          .tap { |t| t << '.' unless t.end_with?('.') }
-          .gsub('’', "'")
-        example = row.search('.apiReference__exampleCode code').first&.text
+      method_data['args']['properties'].each_pair do |name, arg|
+        arg['anyOf']&.each do |coll|
+          all = []
+          all << coll['type']
+          arg['type'] = all
+        end
+        arg.delete('anyOf')
+
+        type = massage_type(name, arg, method_data)
+        required = method_data['args']['required']&.include?(name)
+
+        desc = arg['desc']
+          &.tap { |t| t.slice!("\n") }
+          &.tap { |t| t << '.' unless t.end_with?('.') }
+          &.gsub('’', "'")
+        example = arg['example'] || arg['default']
 
         case name
         when 'token'
@@ -98,89 +100,80 @@ def parse_args(api_page, default_data = {})
         end
       end
 
-      arg_groups = parse_arg_groups(args_wrapper)
+      arg_groups = parse_arg_groups(_api_page, method_data)
 
-      [args, arg_groups, fields]
+      [args.sort.to_h, arg_groups, fields]
     end
 
-    def parse_arg_groups(args_wrapper)
-      # Look for groups of args that are interdependent
-      groups = args_wrapper.search('.apiMethodPage__argumentGroup')
-      groups = groups.map do |group|
-        # "At least one of" or "One of"
-        requirement = group.search('.apiMethodPage__argument em').text
-        mutually_exclusive = requirement.downcase == 'one of'
-
-        desc = group.search('.apiMethodPage__argumentGroupDesc p')
-          .text
-          .tap { |t| t.slice!("\n") }
-          .tap { |t| t << '.' unless t.end_with?('.') }
-          .gsub('’', "'")
-
-        rows = group.search('.apiMethodPage__argumentRow')
-        names = rows.map do |row|
-          row.search('.apiMethodPage__argument code').text
-        end
-
-        {
-          'args' => names,
-          'desc' => desc,
-          'mutually_exclusive' => mutually_exclusive
-        }
-      end
+    def parse_arg_groups(_api_page, _method_data = {})
+      groups = {}
+
+      # # Look for groups of args that are interdependent
+      # groups = args_wrapper.search('.apiMethodPage__argumentGroup')
+      # groups = groups.map do |group|
+      #   # "At least one of" or "One of"
+      #   requirement = group.search('.apiMethodPage__argument em').text
+      #   mutually_exclusive = requirement.downcase == 'one of'
+
+      #   desc = group.search('.apiMethodPage__argumentGroupDesc p')
+      #     .text
+      #     .tap { |t| t&.slice!("\n") }
+      #     .tap { |t| t << '.' unless t.end_with?('.') }
+      #     .gsub('’', "'")
+
+      #   rows = group.search('.apiMethodPage__argumentRow')
+      #   names = rows.map do |row|
+      #     row.search('.apiMethodPage__argument code').text
+      #   end
+
+      #   {
+      #     'args' => names,
+      #     'desc' => desc,
+      #     'mutually_exclusive' => mutually_exclusive
+      #   }
+      # end
 
       groups unless groups.empty?
     end
 
-    def massage_type(name, detected, default_data = {})
+    def massage_type(name, arg, data = {})
+      return 'enum' if arg.key?('enum')
+
       case name
       when 'date' then 'date'
       when 'latest', 'oldest', 'ts' then 'timestamp'
       when 'file' then 'file'
       when 'bot', 'user' then 'user'
       when 'channel'
-        case default_data[:method_group]
+        case data[:method_group]
         when 'im' then 'im'
         when 'mpim' then 'mpim'
         when 'groups' then 'group'
         else 'channel'
         end
       else
-        case detected
+        case detected = arg['type']
         when '', 'null' then nil
         else detected
         end
       end
     end
 
-    def parse_response(api_page, default_data = {})
-      response_wrapper = ensure!(api_page, '.apiReference__response', default_data[:method_name])
-      responses = response_wrapper.search('.apiReference__example')
+    def parse_response(_api_page, data = {})
       examples = []
-      responses.each do |response|
-        response.search('pre').each do |pre|
-          text = pre.text.strip
-          next unless text =~ /^\{.*}$/m
-
-          examples.push(text)
-        end
+      data['examples']&.each_pair do |_example_type, response|
+        example = JSON.pretty_generate(response['example'], indent: '    ')
+        example.gsub!(/\{\n\s*\}/, '{}')
+        example.gsub!(/\[\n\s*\]/, '[]')
+        examples.push example
       end
       { 'examples' => examples }
     end
 
-    def parse_errors(api_page, default_data = {})
-      errors_wrapper = ensure!(api_page, '.apiReference__errors', default_data[:method_name])
-      rows = errors_wrapper.search('.apiDocsTable tr')
+    def parse_errors(_api_page, data = {})
       errors = {}
-      rows.each do |row|
-        next if row.search('th').any?
-
-        name = row.search('[data-label=Error]').text
-        desc = row.search('[data-label=Description]').text
-          .tap { |t| t.slice!("\n") }
-          .tap { |t| t << '.' unless t.end_with?('.') }
-
-        errors[name] = desc
+      data['errors']&.each_pair do |name, desc|
+        errors[name] = desc['desc']
       end
       errors
     end
diff --git a/tasks/update.rake b/tasks/update.rake
@@ -74,6 +74,7 @@ namespace :api do
 
   desc 'Update scraped Slack events and Web API methods.'
   task :update do
+    # Rake::Task['api:ref:download'].invoke
     Rake::Task['api:methods:update'].invoke
     Rake::Task['api:events:update'].invoke
     Rake::Task['api:delete_undocumented'].invoke

Original file line number	Diff line number	Diff line change
`@@ -9,4 +9,5 @@ require 'fileutils'`
`9`	`9`
`10`	`10`	`Bundler.setup :default, :development`
`11`	`11`
	`12`	`+load 'tasks/download.rake'`
`12`	`13`	`load 'tasks/update.rake'`