Download and parse HTML

Author: s2t2

Rakefile

@@ -6,19 +6,3 @@ require_relative "./lib/us_news_rankings"
 RSpec::Core::RakeTask.new(:spec)
 
 task :default => :spec
-
-task :extract_tables do
-  html_files = Dir.glob("web/rankings/education/**/*page*.html") # Dir.glob("web/rankings/**/*.html")
-  puts "EXTRACTING RANKINGS TABLES FROM #{html_files.count} HTML PAGES"
-
-  html_files.each do |html_filepath|
-    puts html_filepath
-
-    begin
-      page = UsNewsRankings::Page.new(html_filepath)
-      page.parse
-    rescue UsNewsRankings::Page::InvalidTableError
-      puts "...skipping"
-    end
-  end
-end

lib/us_news_rankings/category.rb

@@ -6,8 +6,36 @@ def initialize(year)
       @year = year
     end
 
+    def source_urls
+      raise "Oh, please implement #source_urls on the child class."
+    end
+
+    def html_dir
+      raise "Oh, please implement #html_dir on the child class."
+    end
+
+    def pages
+      source_urls.each_with_index.map{|url, number| UsNewsRankings::Page.new({
+        category: self,
+        url: url,
+        number: number + 1
+      })}
+    end
+
+    # @example [{school_name: "abc", rank: 1}, {school_name: "def", rank: 2}, {school_name: "xyz", rank: 3}]
     def rankings
-      raise "please implement #rankings on the child class"
+      @rankings || extract_rankings.sort_by{|ranking| ranking[:rank].to_i }
+    end
+
+    def extract_rankings
+      extracted_rankings = []
+      pages.each do |page|
+        page.table_rows.each do |row|
+          ranking = UsNewsRankings::Education::GraduateSchools::LawClinical::Ranking.new(row)
+          extracted_rankings << ranking.to_h if ranking.ranked?
+        end
+      end
+      return extracted_rankings
     end
   end
 end

lib/us_news_rankings/education/graduate_schools/law_clinical/category.rb

@@ -16,24 +16,8 @@ def source_urls
             SOURCE_URLS[year]
           end
 
-          def pages
-            source_urls.map{|url| UsNewsRankings::Page.new(url) }
-          end
-
-          # @example [{school_name: "abc", rank: 1}, {school_name: "def", rank: 2}, {school_name: "xyz", rank: 3}]
-          def rankings
-            @rankings || extract_rankings.sort_by{|ranking| ranking[:rank].to_i }
-          end
-
-          def extract_rankings
-            extracted_rankings = []
-            pages.each do |page|
-              page.table_rows.each do |row|
-                ranking = UsNewsRankings::Education::GraduateSchools::LawClinical::Ranking.new(row)
-                extracted_rankings << ranking.to_h if ranking.ranked?
-              end
-            end
-            return extracted_rankings
+          def html_dir
+            "./web/education/graduate_schools/law_clinical"
           end
         end
       end

lib/us_news_rankings/page.rb

@@ -4,20 +4,42 @@
 
 module UsNewsRankings
   class Page
-    attr_reader :url
+    attr_reader :category, :url, :number
 
     # @param url [String] the page's source url
-    def initialize(url)
+    def initialize(category:, url:, number:)
+      @category = category
       @url = url
+      @number = number
+    end
+
+    def html_dir
+      File.join(category.html_dir, category.year.to_s)
+    end
+
+    def html_filepath
+      File.join(html_dir, "page#{number}.html")
+    end
+
+    def download_document
+      FileUtils.mkdir_p(html_dir)
+      File.open(html_filepath, 'w') do |file|
+        file.write(url_source.to_xhtml(indent: 2)) # file.write(table.to_xhtml(indent: 2))
+      end
+    end
+
+    def url_source
+      puts "REQUESTING URL SOURCE FROM #{url}"
+      @url_source ||= Nokogiri::HTML(open(url))
     end
 
     # @return [Nokogiri::HTML::Document]
     def document
-      @document ||= Nokogiri::HTML(open(url))
+      download_document unless File.exist?(html_filepath)
+      @document ||= Nokogiri::HTML(open(html_filepath))
     end
 
     def table
-      #raise "Oh, please implement #table in the child class"
       @table ||= (document.at_css("table.ranking-data") || document.at_css("table.searchresult") || document.at_css("table.flex-table"))
     end