penguins: use POST

It seems that EDI disallows GET.

Author: kou

lib/datasets/dataset.rb

@@ -33,8 +33,8 @@ def cache_path
       @cache_path ||= CachePath.new(@metadata.id)
     end
 
-    def download(output_path, url, *fallback_urls, &block)
-      downloader = Downloader.new(url, *fallback_urls)
+    def download(output_path, url, *fallback_urls, **options, &block)
+      downloader = Downloader.new(url, *fallback_urls, **options)
       downloader.download(output_path, &block)
     end
 

lib/datasets/downloader.rb

@@ -12,9 +12,11 @@ module Datasets
   class Downloader
     class TooManyRedirects < Error; end
 
-    def initialize(url, *fallback_urls)
+    def initialize(url, *fallback_urls, http_method: nil, http_parameters: nil)
       @url = normalize_url(url)
       @fallback_urls = fallback_urls.collect { |fallback_url| normalize_url(fallback_url) }
+      @http_method = http_method
+      @http_parameters = http_parameters
     end
 
     def download(output_path, &block)
@@ -151,7 +153,21 @@ def download(output_path, &block)
       http.start do
         path = url.path
         path += "?#{url.query}" if url.query
-        request = Net::HTTP::Get.new(path, headers)
+        if @http_method == :post
+          # TODO: We may want to add @http_content_type, @http_body
+          # and so on.
+          if @http_parameters
+            body = URI.encode_www_form(@http_parameters)
+            content_type = "application/x-www-form-urlencoded"
+            headers = {"Content-Type" => content_type}.merge(headers)
+          else
+            body = ""
+          end
+          request = Net::HTTP::Post.new(path, headers)
+          request.body = body
+        else
+          request = Net::HTTP::Get.new(path, headers)
+        end
         http.request(request) do |response|
           case response
           when Net::HTTPSuccess, Net::HTTPPartialContent

lib/datasets/penguins.rb

@@ -26,7 +26,9 @@ def initialize
         super
         species = self.class.name.split("::").last.downcase
         @metadata.id = "palmerpenguins-#{species}"
-        @metadata.url = self.class::URL
+        package_id = http_parameters["packageid"]
+        @metadata.url = "https://portal.edirepository.org/nis/mapbrowse" +
+                        "?packageid=#{package_id}"
         @metadata.licenses = ["CC0-1.0"]
         @data_path = cache_dir_path + "#{species}.csv"
       end
@@ -46,7 +48,10 @@ def each
       end
 
       private def open_data
-        download(data_path, metadata.url)
+        download(data_path,
+                 "https://portal.edirepository.org/nis/dataviewer",
+                 http_method: :post,
+                 http_parameters: http_parameters)
         CSV.open(data_path, headers: :first_row, converters: :all) do |csv|
           yield csv
         end
@@ -56,19 +61,37 @@ def each
     # Adelie penguin data from: https://doi.org/10.6073/pasta/abc50eed9138b75f54eaada0841b9b86
     class Adelie < SpeciesBase
       DOI = "doi.org/10.6073/pasta/abc50eed9138b75f54eaada0841b9b86".freeze
-      URL = "https://portal.edirepository.org/nis/dataviewer?packageid=knb-lter-pal.219.3&entityid=002f3893385f710df69eeebe893144ff".freeze
+
+      private def http_parameters
+        {
+          "packageid" => "knb-lter-pal.219.3",
+          "entityid" => "002f3893385f710df69eeebe893144ff",
+        }
+      end
     end
 
     # Chinstrap penguin data from: https://doi.org/10.6073/pasta/409c808f8fc9899d02401bdb04580af7
     class Chinstrap < SpeciesBase
       DOI = "doi.org/10.6073/pasta/409c808f8fc9899d02401bdb04580af7".freeze
-      URL = "https://portal.edirepository.org/nis/dataviewer?packageid=knb-lter-pal.221.2&entityid=fe853aa8f7a59aa84cdd3197619ef462".freeze
+
+      private def http_parameters
+        {
+          "packageid" => "knb-lter-pal.221.2",
+          "entityid" => "fe853aa8f7a59aa84cdd3197619ef462",
+        }
+      end
     end
 
     # Gentoo penguin data from: https://doi.org/10.6073/pasta/2b1cff60f81640f182433d23e68541ce
     class Gentoo < SpeciesBase
       DOI = "doi.org/10.6073/pasta/2b1cff60f81640f182433d23e68541ce".freeze
-      URL = "https://portal.edirepository.org/nis/dataviewer?packageid=knb-lter-pal.220.3&entityid=e03b43c924f226486f2f0ab6709d2381".freeze
+
+      private def http_parameters
+        {
+          "packageid" => "knb-lter-pal.220.3",
+          "entityid" => "e03b43c924f226486f2f0ab6709d2381",
+        }
+      end
     end
   end