Switch to the JSON output format for easier parsing

Author: pabs3

lib/wayback_machine_downloader.rb

@@ -84,7 +84,7 @@ def get_all_snapshots_to_consider
     # Note: Passing a page index parameter allow us to get more snapshots,
     # but from a less fresh index
     print "Getting snapshot pages"
-    snapshot_list_to_consider = ""
+    snapshot_list_to_consider = []
     snapshot_list_to_consider += get_raw_list_from_api(@base_url, nil)
     print "."
     unless @exact_url
@@ -95,17 +95,15 @@ def get_all_snapshots_to_consider
         print "."
       end
     end
-    puts " found #{snapshot_list_to_consider.lines.count} snaphots to consider."
+    puts " found #{snapshot_list_to_consider.length} snaphots to consider."
     puts
     snapshot_list_to_consider
   end
 
   def get_file_list_curated
     file_list_curated = Hash.new
-    get_all_snapshots_to_consider.each_line do |line|
-      next unless line.include?('/')
-      file_timestamp = line[0..13].to_i
-      file_url = line[15..-2]
+    get_all_snapshots_to_consider.each do |file_timestamp, file_url|
+      next unless file_url.include?('/')
       file_id = file_url.split('/')[3..-1].join('/')
       file_id = CGI::unescape file_id 
       file_id = file_id.tidy_bytes unless file_id == ""
@@ -130,10 +128,8 @@ def get_file_list_curated
 
   def get_file_list_all_timestamps
     file_list_curated = Hash.new
-    get_all_snapshots_to_consider.each_line do |line|
-      next unless line.include?('/')
-      file_timestamp = line[0..13].to_i
-      file_url = line[15..-2]
+    get_all_snapshots_to_consider.each do |file_timestamp, file_url|
+      next unless file_url.include?('/')
       file_id = file_url.split('/')[3..-1].join('/')
       file_id_and_timestamp = [file_timestamp, file_id].join('/')
       file_id_and_timestamp = CGI::unescape file_id_and_timestamp 

lib/wayback_machine_downloader/archive_api.rb

@@ -1,14 +1,23 @@
+require 'json'
 require 'uri'
 
 module ArchiveAPI
 
   def get_raw_list_from_api url, page_index
     request_url = URI("https://web.archive.org/cdx/search/xd")
-    params = [["url", url]]
+    params = [["output", "json"], ["url", url]]
     params += parameters_for_api page_index
     request_url.query = URI.encode_www_form(params)
 
-    URI.open(request_url).read
+    begin
+      json = JSON.parse(URI.open(request_url).read)
+      if (json[0] <=> ["timestamp","original"]) == 0
+        json.shift
+      end
+      json
+    rescue JSON::ParserError
+      []
+    end
   end
 
   def parameters_for_api page_index