Adding cache for fetching remote URLs to increase speed significantly

Author: Suszyński Krzysztof

.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

lib/puppet_forge_server/http/http_client.rb

@@ -17,13 +17,19 @@
 require 'open-uri'
 require 'open_uri_redirections'
 require 'timeout'
+require 'lrucache'
 require 'net/http'
 require 'net/http/post/multipart'
 
 
 module PuppetForgeServer::Http
   class HttpClient
 
+    def initialize(cache = nil)
+      cache = LRUCache.new(:ttl => 60 * 15, :max_size => 250) if cache.nil?
+      @cache = cache
+    end
+
     def post_file(url, file_hash, options = {})
       options = { :http => {}, :headers => {}}.merge(options)
 
@@ -46,10 +52,19 @@ def download(url)
     end
 
     private
+
     def open_uri(url)
-      ::Timeout.timeout(10) do
-        open(url, 'User-Agent' => "Puppet-Forge-Server/#{PuppetForgeServer::VERSION}", :allow_redirections => :safe)
+      contents = @cache.fetch(url) do
+        tmpfile = ::Timeout.timeout(10) do
+          PuppetForgeServer::Logger.get.debug "Fetching data for url: #{url} from remote server"
+          open(url, 'User-Agent' => "Puppet-Forge-Server/#{PuppetForgeServer::VERSION}", :allow_redirections => :safe)
+        end
+        contents = tmpfile.read
+        tmpfile.close
+        contents
       end
+      PuppetForgeServer::Logger.get.debug "Data for url: #{url} fetched, #{contents.size} bytes"
+      StringIO.new(contents)
     end
   end
 end

puppet-forge-server.gemspec

@@ -45,10 +45,12 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'deep_merge', '~> 1.0'
   spec.add_dependency 'multipart-post', '~> 2.0.0'
   spec.add_dependency 'redcarpet', '~> 3.3.0'
+  spec.add_dependency 'lrucache', '~> 0.1.4'
 
   spec.add_development_dependency 'rake', '~> 10.3'
   spec.add_development_dependency 'rspec', '~> 3.1'
   spec.add_development_dependency 'rspec-core', '~> 3.1'
+  spec.add_development_dependency 'simplecov', '~> 0.11.0'
 
   spec.required_ruby_version = '>= 1.9.3'
 end

spec/spec_helper.rb

@@ -23,8 +23,47 @@
 module RSpecMixin
   include Rack::Test::Methods
 end
+begin
+  gem 'simplecov'
+  require 'simplecov'
+  formatters = []
+  formatters << SimpleCov::Formatter::HTMLFormatter
+
+  begin
+    gem 'coveralls'
+    require 'coveralls'
+    formatters << Coveralls::SimpleCov::Formatter if ENV['TRAVIS']
+  rescue Gem::LoadError
+    # do nothing
+  end
+
+  begin
+    gem 'codeclimate-test-reporter'
+    require 'codeclimate-test-reporter'
+    formatters << CodeClimate::TestReporter::Formatter if (ENV['TRAVIS'] and ENV['CODECLIMATE_REPO_TOKEN'])
+  rescue Gem::LoadError
+    # do nothing
+  end
+  SimpleCov.formatter = SimpleCov::Formatter::MultiFormatter[*formatters]
+  SimpleCov.start do
+    add_filter "/spec/"
+    add_filter "/.vendor/"
+    add_filter "/vendor/"
+    add_filter "/gems/"
+    refuse_coverage_drop
+  end
+rescue Gem::LoadError
+  # do nothing
+end
+
+begin
+  gem 'pry'
+  require 'pry'
+rescue Gem::LoadError
+  # do nothing
+end
 
 RSpec.configure do |c|
   c.include RSpecMixin
   c.add_formatter(:documentation)
-end
+end

spec/unit/http/http_client_spec.rb

@@ -0,0 +1,71 @@
+require 'spec_helper'
+require 'puppet_forge_server'
+
+describe PuppetForgeServer::Http::HttpClient do
+  let(:port) do
+    require 'socket'
+    server = TCPServer.new('127.0.0.1', 0)
+    port = server.addr[1]
+    server.close
+    port
+  end
+  let(:uri) { "http://localhost:#{port}/" }
+  before(:each) do
+    @server = TCPServer.new('localhost', port)
+    @thr = Thread.new do
+      loop do
+        socket = @server.accept
+        response = "Hello!"
+        socket.print "HTTP/1.1 200 OK\r\n" +
+                     "Content-Type: text/plain\r\n" +
+                     "Content-Length: #{response.bytesize}\r\n" +
+                     "Connection: close\r\n"
+        socket.print "\r\n"
+        sleep 0.01
+        socket.print response
+        socket.close
+      end
+    end
+  end
+  after(:each) do
+    @server.close
+    @thr.kill
+  end
+  let(:instance) { described_class.new(cache) }
+  describe '#download' do
+    subject do
+      99.times { instance.download(uri) }
+      instance.download(uri)
+    end
+    context 'with 1sec LRU cache' do
+      let(:cache) do
+        require 'lrucache'
+        LRUCache.new(:ttl => 1)
+      end
+      it { expect(subject).not_to be_nil }
+      it { expect(subject).not_to be_closed }
+    end
+    context 'with default cache' do
+      let(:cache) { nil }
+      it { expect(subject).not_to be_nil }
+      it { expect(subject).not_to be_closed }
+    end
+  end
+  describe '#get' do
+    subject do
+      99.times { instance.get(uri) }
+      instance.get(uri)
+    end
+    context 'with 1sec LRU cache' do
+      let(:cache) do
+        require 'lrucache'
+        LRUCache.new(:ttl => 1)
+      end
+      it { expect(subject).to eq('Hello!') }
+    end
+    context 'with default cache' do
+      let(:cache) { nil }
+      it { expect(subject).to eq('Hello!') }
+    end
+  end
+end