Check for updates based on modified date for scrapers without release

Author: jmerle

docs/Scraper-Reference.md

@@ -187,7 +187,7 @@ More information about how filters work is available on the [Filter Reference](.
 
 ## Keeping scrapers up-to-date
 
-In order to keep scrapers up-to-date the `get_latest_version(options, &block)` method should be overridden by all scrapers that define the `self.release` attribute. This method should return the latest version of the documentation that is being scraped. The result of this method is periodically reported in a "Documentation versions report" issue which helps maintainers keep track of outdated documentations.
+In order to keep scrapers up-to-date the `get_latest_version(options, &block)` method should be overridden. If `self.release` is defined, this should return the latest version of the documentation. If `self.release` is not defined, it should return the Epoch time when the documentation was last modified. If the documentation will never change, simply return `1.0.0`. The result of this method is periodically reported in a "Documentation versions report" issue which helps maintainers keep track of outdated documentations.
 
 To make life easier, there are a few utility methods that you can use in `get_latest_version`:
 * `fetch(url, options, &block)`

lib/docs/core/doc.rb

@@ -152,7 +152,6 @@ def store_meta(store)
       end
     end
 
-
     def initialize
       raise NotImplementedError, "#{self.class} is an abstract class and cannot be instantiated." if self.class.abstract
     end
@@ -164,5 +163,108 @@ def build_page(id, &block)
     def build_pages(&block)
       raise NotImplementedError
     end
+
+    def get_scraper_version(opts, &block)
+      if self.class.method_defined?(:options) and !options[:release].nil?
+        block.call options[:release]
+      else
+        # If options[:release] does not exist, we return the Epoch timestamp of when the doc was last modified in DevDocs production
+        fetch_json('https://devdocs.io/docs.json', opts) do |json|
+          items = json.select {|item| item['name'] == self.class.name}
+          items = items.map {|item| item['mtime']}
+          block.call items.max
+        end
+      end
+    end
+
+    # Should return the latest version of this documentation
+    # If options[:release] is defined, it should be in the same format
+    # If options[:release] is not defined, it should return the Epoch timestamp of when the documentation was last updated
+    # If the docs will never change, simply return '1.0.0'
+    def get_latest_version(options, &block)
+      raise NotImplementedError
+    end
+
+    # Returns whether or not this scraper is outdated.
+    #
+    # The default implementation assumes the documentation uses a semver(-like) approach when it comes to versions.
+    # Patch updates are ignored because there are usually little to no documentation changes in bug-fix-only releases.
+    #
+    # Scrapers of documentations that do not use this versioning approach should override this method.
+    #
+    # Examples of the default implementation:
+    # 1 -> 2 = outdated
+    # 1.1 -> 1.2 = outdated
+    # 1.1.1 -> 1.1.2 = not outdated
+    def is_outdated(scraper_version, latest_version)
+      scraper_parts = scraper_version.to_s.split(/\./).map(&:to_i)
+      latest_parts = latest_version.to_s.split(/\./).map(&:to_i)
+
+      # Only check the first two parts, the third part is for patch updates
+      [0, 1].each do |i|
+        break if i >= scraper_parts.length or i >= latest_parts.length
+        return true if latest_parts[i] > scraper_parts[i]
+        return false if latest_parts[i] < scraper_parts[i]
+      end
+
+      false
+    end
+
+    private
+
+    #
+    # Utility methods for get_latest_version
+    #
+
+    def fetch(url, options, &block)
+      headers = {}
+
+      if options.key?(:github_token) and url.start_with?('https://api.github.com/')
+        headers['Authorization'] = "token #{options[:github_token]}"
+      end
+
+      options[:logger].debug("Fetching #{url}")
+
+      Request.run(url, { headers: headers }) do |response|
+        if response.success?
+          block.call response.body
+        else
+          options[:logger].error("Couldn't fetch #{url} (response code #{response.code})")
+          block.call nil
+        end
+      end
+    end
+
+    def fetch_doc(url, options, &block)
+      fetch(url, options) do |body|
+        block.call Nokogiri::HTML.parse(body, nil, 'UTF-8')
+      end
+    end
+
+    def fetch_json(url, options, &block)
+      fetch(url, options) do |body|
+        block.call JSON.parse(body)
+      end
+    end
+
+    def get_npm_version(package, options, &block)
+      fetch_json("https://registry.npmjs.com/#{package}", options) do |json|
+        block.call json['dist-tags']['latest']
+      end
+    end
+
+    def get_latest_github_release(owner, repo, options, &block)
+      fetch_json("https://api.github.com/repos/#{owner}/#{repo}/releases/latest", options, &block)
+    end
+
+    def get_github_tags(owner, repo, options, &block)
+      fetch_json("https://api.github.com/repos/#{owner}/#{repo}/tags", options, &block)
+    end
+
+    def get_github_file_contents(owner, repo, path, options, &block)
+      fetch_json("https://api.github.com/repos/#{owner}/#{repo}/contents/#{path}", options) do |json|
+        block.call(Base64.decode64(json['content']))
+      end
+    end
   end
 end

lib/docs/core/scraper.rb

@@ -132,35 +132,6 @@ def options
       end
     end
 
-    def get_latest_version(options, &block)
-      raise NotImplementedError
-    end
-
-    # Returns whether or not this scraper is outdated.
-    #
-    # The default implementation assumes the documentation uses a semver(-like) approach when it comes to versions.
-    # Patch updates are ignored because there are usually little to no documentation changes in bug-fix-only releases.
-    #
-    # Scrapers of documentations that do not use this versioning approach should override this method.
-    #
-    # Examples of the default implementation:
-    # 1 -> 2 = outdated
-    # 1.1 -> 1.2 = outdated
-    # 1.1.1 -> 1.1.2 = not outdated
-    def is_outdated(scraper_version, latest_version)
-      scraper_parts = scraper_version.split(/\./).map(&:to_i)
-      latest_parts = latest_version.split(/\./).map(&:to_i)
-
-      # Only check the first two parts, the third part is for patch updates
-      [0, 1].each do |i|
-        break if i >= scraper_parts.length or i >= latest_parts.length
-        return true if latest_parts[i] > scraper_parts[i]
-        return false if latest_parts[i] < scraper_parts[i]
-      end
-
-      false
-    end
-
     private
 
     def request_one(url)
@@ -231,62 +202,6 @@ def additional_options
       {}
     end
 
-    #
-    # Utility methods for get_latest_version
-    #
-
-    def fetch(url, options, &block)
-      headers = {}
-
-      if options.key?(:github_token) and url.start_with?('https://api.github.com/')
-        headers['Authorization'] = "token #{options[:github_token]}"
-      end
-
-      options[:logger].debug("Fetching #{url}")
-
-      Request.run(url, { headers: headers }) do |response|
-        if response.success?
-          block.call response.body
-        else
-          options[:logger].error("Couldn't fetch #{url} (response code #{response.code})")
-          block.call nil
-        end
-      end
-    end
-
-    def fetch_doc(url, options, &block)
-      fetch(url, options) do |body|
-        block.call Nokogiri::HTML.parse body, nil, 'UTF-8'
-      end
-    end
-
-    def fetch_json(url, options, &block)
-      fetch(url, options) do |body|
-        json = JSON.parse(body)
-        block.call json
-      end
-    end
-
-    def get_npm_version(package, options, &block)
-      fetch_json("https://registry.npmjs.com/#{package}", options) do |json|
-        block.call json['dist-tags']['latest']
-      end
-    end
-
-    def get_latest_github_release(owner, repo, options, &block)
-      fetch_json("https://api.github.com/repos/#{owner}/#{repo}/releases/latest", options, &block)
-    end
-
-    def get_github_tags(owner, repo, options, &block)
-      fetch_json("https://api.github.com/repos/#{owner}/#{repo}/tags", options, &block)
-    end
-
-    def get_github_file_contents(owner, repo, path, options, &block)
-      fetch_json("https://api.github.com/repos/#{owner}/#{repo}/contents/#{path}", options) do |json|
-        block.call(Base64.decode64(json['content']))
-      end
-    end
-
     module FixInternalUrlsBehavior
       def self.included(base)
         base.extend ClassMethods

lib/docs/scrapers/c.rb

@@ -26,6 +26,14 @@ class C < FileScraper
       Licensed under the Creative Commons Attribution-ShareAlike Unported License v3.0.
     HTML
 
+    def get_latest_version(options, &block)
+      fetch_doc('https://en.cppreference.com/w/Cppreference:Archives', options) do |doc|
+        link = doc.at_css('a[title^="File:"]')
+        date = link.content.scan(/(\d+)\./)[0][0]
+        block.call DateTime.strptime(date, '%Y%m%d').to_time.to_i
+      end
+    end
+
     private
 
     def file_path_for(*)

lib/docs/scrapers/chef.rb

@@ -49,9 +49,8 @@ class Chef < UrlScraper
     end
 
     def get_latest_version(options, &block)
-      fetch_doc('https://docs-archive.chef.io/', options) do |doc|
-        cell = doc.at_css('.main-archives > tr:nth-child(2) > td:nth-child(2)')
-        block.call cell.content.sub(/Chef Client /, '')
+      fetch_doc('https://downloads.chef.io/chef', options) do |doc|
+        block.call doc.at_css('h1.product-heading > span').content.strip
       end
     end
   end

lib/docs/scrapers/cpp.rb

@@ -34,6 +34,15 @@ class Cpp < FileScraper
       Licensed under the Creative Commons Attribution-ShareAlike Unported License v3.0.
     HTML
 
+    # Same as get_latest_version in lib/docs/scrapers/c.rb
+    def get_latest_version(options, &block)
+      fetch_doc('https://en.cppreference.com/w/Cppreference:Archives', options) do |doc|
+        link = doc.at_css('a[title^="File:"]')
+        date = link.content.scan(/(\d+)\./)[0][0]
+        block.call DateTime.strptime(date, '%Y%m%d').to_time.to_i
+      end
+    end
+
     private
 
     def file_path_for(*)

lib/docs/scrapers/haskell.rb

@@ -10,7 +10,7 @@ class Haskell < UrlScraper
 
     html_filters.push 'haskell/entries', 'haskell/clean_html'
 
-    options[:container] = ->(filter) { filter.subpath.start_with?('users_guide') ? '.body' : '#content' }
+    options[:container] = ->(filter) {filter.subpath.start_with?('users_guide') ? '.body' : '#content'}
 
     options[:only_patterns] = [/\Alibraries\//, /\Ausers_guide\//]
     options[:skip_patterns] = [
@@ -70,9 +70,10 @@ class Haskell < UrlScraper
     end
 
     def get_latest_version(options, &block)
-      fetch_doc('https://downloads.haskell.org/~ghc/latest/docs/html/users_guide/', options) do |doc|
-        label = doc.at_css('.related > ul > li:last-child').content
-        block.call label.scan(/([0-9.]+)/)[0][0]
+      fetch_doc('https://downloads.haskell.org/~ghc/latest/docs/html/', options) do |doc|
+        links = doc.css('a').to_a
+        versions = links.map {|link| link['href'].scan(/ghc-([0-9.]+)/)}
+        block.call versions.find {|version| !version.empty?}[0][0]
       end
     end
   end

lib/docs/scrapers/http.rb

@@ -7,6 +7,8 @@ class Http < Mdn
 
     html_filters.push 'http/clean_html', 'http/entries', 'title'
 
+    options[:mdn_tag] = 'HTTP'
+
     options[:root_title] = 'HTTP'
     options[:title] = ->(filter) { filter.current_url.host == 'tools.ietf.org' ? false : filter.default_title }
     options[:container] = ->(filter) { filter.current_url.host == 'tools.ietf.org' ? '.content' : nil }

lib/docs/scrapers/markdown.rb

@@ -13,5 +13,9 @@ class Markdown < UrlScraper
       &copy; 2004 John Gruber<br>
       Licensed under the BSD License.
     HTML
+
+    def get_latest_version(options, &block)
+      block.call '1.0.0'
+    end
   end
 end

lib/docs/scrapers/mdn/css.rb

@@ -6,6 +6,8 @@ class Css < Mdn
 
     html_filters.push 'css/clean_html', 'css/entries', 'title'
 
+    options[:mdn_tag] = 'CSS'
+
     options[:root_title] = 'CSS'
 
     options[:skip] = %w(/CSS3 /Media/Visual /paged_media /Media/TV /Media/Tactile)