Merge pull request #35 from coi-gov-pl/feature/add-cache-for-remote

Feature: Add cache to significantly speedup fetching from remote proxies

Author: Ilja Bobkevic

.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

lib/puppet_forge_server.rb

@@ -56,11 +56,14 @@ module Models
   end
 
   module Utils
+    autoload :CacheProvider, 'puppet_forge_server/utils/cache_provider'
+    autoload :FilteringInspecter, 'puppet_forge_server/utils/filtering_inspecter'
     autoload :Archiver, 'puppet_forge_server/utils/archiver'
     autoload :OptionParser, 'puppet_forge_server/utils/option_parser'
     autoload :Buffer, 'puppet_forge_server/utils/buffer'
     autoload :Http, 'puppet_forge_server/utils/http'
     autoload :MarkdownRenderer, 'puppet_forge_server/utils/md_renderer'
+    autoload :Encoding, 'puppet_forge_server/utils/encoding'
   end
 
   module Http

lib/puppet_forge_server/api/v3/releases.rb

@@ -16,6 +16,7 @@
 
 module PuppetForgeServer::Api::V3
   module Releases
+    include PuppetForgeServer::Utils::Encoding
     def get_releases(metadata)
       metadata.map do |element|
         author, name = element.metadata.name.split('-')
@@ -36,7 +37,7 @@ def get_releases(metadata)
             :file_uri => "/v3/files#{element.path}",
             :file_md5 => element.checksum,
             :deleted_at => element.deleted_at,
-            :readme => element.readme
+            :readme => to_utf8(element.readme)
         }
       end.uniq{|r| r[:version]}.sort_by { |r| Gem::Version.new(r[:version]) }
     end

lib/puppet_forge_server/app/public/css/puppetlabs.css

@@ -16,6 +16,7 @@
  * limitations under the License.
  */
 
+@import url(https://fonts.googleapis.com/css?family=PT+Sans&subset=latin,latin-ext);
 body {
   background: #fff;
   color: #5d5e59;

lib/puppet_forge_server/backends/proxy.rb

@@ -35,14 +35,18 @@ def initialize(url, cache_dir, http_client, file_path)
 
     def get_file_buffer(relative_path)
       file_name = relative_path.split('/').last
-      File.join(@cache_dir, file_name[0].downcase, file_name)
+      target_file = File.join(@cache_dir, file_name[0].downcase, file_name)
       path = Dir["#{@cache_dir}/**/#{file_name}"].first
       unless File.exist?("#{path}")
         buffer = download("#{@file_path.chomp('/')}/#{relative_path}")
-        File.open(File.join(@cache_dir, file_name[0].downcase, file_name), 'wb') do |file|
-          file.write(buffer.read)
+        File.open(target_file, 'wb') do |file|
+          bytes = buffer.read
+          file.write(bytes)
+          @log.debug("Saved #{bytes.size} bytes in filesystem cache for path: #{relative_path}, target file: #{target_file}")
         end
-        path = File.join(@cache_dir, file_name[0].downcase, file_name)
+        path = target_file
+      else
+        @log.info("Filesystem cache HIT for path: #{relative_path}")
       end
       File.open(path, 'rb')
     rescue => e
@@ -59,6 +63,27 @@ def upload(file_data)
     protected
     attr_reader :log
 
+    def get_non_mutable(relative_url)
+      file_name = relative_url.split('/').last
+      target_file = File.join(@cache_dir, file_name[0].downcase, file_name)
+      path = Dir["#{@cache_dir}/**/#{file_name}"].first
+      unless File.exist?("#{path}")
+        buffer = get(relative_url)
+        File.open(target_file, 'wb') do |file|
+          file.write(buffer)
+          @log.debug("Saved #{buffer} bytes in filesystem cache for url: #{relative_url}, target file: #{target_file}")
+        end
+        path = target_file
+      else
+        @log.info("Filesystem cache HIT for url: #{relative_url}")
+      end
+      File.binread(path)
+    rescue => e
+      @log.error("#{self.class.name} failed getting non-mutable url '#{relative_url}'")
+      @log.error("Error: #{e}")
+      return nil
+    end
+
     def get(relative_url)
       @http_client.get(url(relative_url))
     end

lib/puppet_forge_server/backends/proxy_v3.rb

@@ -30,11 +30,11 @@ def initialize(url, cache_dir, http_client = PuppetForgeServer::Http::HttpClient
     def get_metadata(author, name, options = {})
       query ="#{author}-#{name}"
       begin
-        releases = options[:version] ? [JSON.parse(get("/v3/releases/#{query}-#{options[:version]}"))] : get_all_result_pages("/v3/releases?module=#{query}")
+        releases = get_releases(query, options)
         get_modules(releases)
       rescue => e
-        @log.debug("#{self.class.name} failed querying metadata for '#{query}' with options #{options}")
-        @log.debug("Error: #{e}")
+        @log.error("#{self.class.name} failed querying metadata for '#{query}' with options #{options}")
+        @log.error("Error: #{e}")
         return nil
       end
     end
@@ -44,13 +44,26 @@ def query_metadata(query, options = {})
         releases = get_all_result_pages("/v3/modules?query=#{query}").map {|element| element['current_release']}
         get_modules(releases)
       rescue => e
-        @log.debug("#{self.class.name} failed querying metadata for '#{query}' with options #{options}")
-        @log.debug("Error: #{e}")
+        @log.error("#{self.class.name} failed querying metadata for '#{query}' with options #{options}")
+        @log.error("Error: #{e}")
         return nil
       end
     end
 
     private
+
+    def get_releases(query, options = {})
+      version = options[:version]
+      unless version.nil?
+        url = "/v3/releases/#{query}-#{version}"
+        buffer = get_non_mutable(url)
+        release = JSON.parse(buffer)
+        [ release ]
+      else
+        get_all_result_pages("/v3/releases?module=#{query}")
+      end
+    end
+
     def get_all_result_pages(next_page)
       results = []
       begin

lib/puppet_forge_server/http/http_client.rb

@@ -23,6 +23,15 @@
 
 module PuppetForgeServer::Http
   class HttpClient
+    include PuppetForgeServer::Utils::CacheProvider
+    include PuppetForgeServer::Utils::FilteringInspecter
+
+    def initialize(cache = nil)
+      cache = cache_instance if cache.nil?
+      cache.extend(PuppetForgeServer::Utils::FilteringInspecter)
+      @log = PuppetForgeServer::Logger.get
+      @cache = cache
+    end
 
     def post_file(url, file_hash, options = {})
       options = { :http => {}, :headers => {}}.merge(options)
@@ -45,11 +54,29 @@ def download(url)
       open_uri(url)
     end
 
+    def inspect
+      cache_inspected = @cache.inspect_without [ :@data ]
+      cache_inspected.gsub!(/>$/, ", @size=#{@cache.size}>")
+      inspected = inspect_without [ :@cache ]
+      inspected.gsub(/>$/, ", @cache=#{cache_inspected}>")
+    end
+
     private
+
     def open_uri(url)
-      ::Timeout.timeout(10) do
-        open(url, 'User-Agent' => "Puppet-Forge-Server/#{PuppetForgeServer::VERSION}", :allow_redirections => :safe)
+      hit_or_miss = @cache.include?(url) ? 'HIT' : 'MISS'
+      @log.info "Cache in RAM memory size: #{@cache.size}, #{hit_or_miss} for url: #{url}"
+      contents = @cache.fetch(url) do
+        tmpfile = ::Timeout.timeout(10) do
+          PuppetForgeServer::Logger.get.debug "Fetching data for url: #{url} from remote server"
+          open(url, 'User-Agent' => "Puppet-Forge-Server/#{PuppetForgeServer::VERSION}", :allow_redirections => :safe)
+        end
+        contents = tmpfile.read
+        tmpfile.close
+        contents
       end
+      @log.debug "Data for url: #{url} fetched, #{contents.size} bytes"
+      StringIO.new(contents)
     end
   end
 end

lib/puppet_forge_server/logger.rb

@@ -16,6 +16,7 @@
 
 
 require 'logger'
+require 'logger/colors'
 
 module PuppetForgeServer
   class Logger
@@ -45,7 +46,13 @@ def method_missing (method_name, *args, &block)
                       else
                         method_name
                     end
-      @loggers.each { |logger| logger.send(method_name, args.first) }
+      if args.size > 0
+        # setters
+        @loggers.each { |logger| logger.send(method_name, args.first) }
+      else
+        # getters
+        @loggers.collect { |logger| logger.send(method_name) }
+      end
     end
 
     def respond_to?(method_name, include_private = false)
@@ -70,4 +77,4 @@ def set(loggers= {})
       end
     end
   end
-end
+end

lib/puppet_forge_server/server.rb

@@ -19,6 +19,7 @@
 module PuppetForgeServer
   class Server
     include PuppetForgeServer::Utils::OptionParser
+    include PuppetForgeServer::Utils::CacheProvider
     include PuppetForgeServer::Utils::Http
 
     def go(args)
@@ -27,6 +28,7 @@ def go(args)
       begin
         options = parse_options(args)
         @log = logging(options)
+        configure_cache(options[:ram_cache_ttl], options[:ram_cache_size])
         backends = backends(options)
         server = build(backends, options[:webui_root])
         announce(options, backends)

lib/puppet_forge_server/utils/cache_provider.rb

@@ -0,0 +1,42 @@
+# -*- encoding: utf-8 -*-
+#
+# Copyright 2015 Centralny Osroder Informatyki (gov.pl)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'lrucache'
+
+module PuppetForgeServer::Utils
+  module CacheProvider
+
+    opts = PuppetForgeServer::Utils::OptionParser.DEFAULT_OPTIONS
+    @@CACHE = LRUCache.new(:ttl => opts[:ram_cache_ttl], :max_size => opts[:ram_cache_size])
+
+    # Method for fetching application wide cache for fetching HTTP requests
+    #
+    # @return [LRUCache] a instance of cache for application
+    def cache_instance
+      @@CACHE
+    end
+
+    # Configure a application wide cache using LSUCache implementation
+    #
+    # @param [int] ttl a time to live for elements
+    # @param [int] size a maximum size for cache
+    def configure_cache(ttl, size)
+      @@CACHE = LRUCache.new(:ttl => ttl, :max_size => size)
+      PuppetForgeServer::Logger.get.info("Using RAM memory LRUCache with time to live of #{ttl}sec and max size of #{size} elements")
+      nil
+    end
+  end
+end