Added caching, both in memory via LRUCache and in FileSystem persistent storage

Author: Suszyński Krzysztof

lib/puppet_forge_server.rb

@@ -56,6 +56,8 @@ module Models
   end
 
   module Utils
+    autoload :CacheProvider, 'puppet_forge_server/utils/cache_provider'
+    autoload :FilteringInspecter, 'puppet_forge_server/utils/filtering_inspecter'
     autoload :Archiver, 'puppet_forge_server/utils/archiver'
     autoload :OptionParser, 'puppet_forge_server/utils/option_parser'
     autoload :Buffer, 'puppet_forge_server/utils/buffer'

lib/puppet_forge_server/backends/proxy.rb

@@ -35,14 +35,16 @@ def initialize(url, cache_dir, http_client, file_path)
 
     def get_file_buffer(relative_path)
       file_name = relative_path.split('/').last
-      File.join(@cache_dir, file_name[0].downcase, file_name)
+      target_file = File.join(@cache_dir, file_name[0].downcase, file_name)
       path = Dir["#{@cache_dir}/**/#{file_name}"].first
       unless File.exist?("#{path}")
         buffer = download("#{@file_path.chomp('/')}/#{relative_path}")
-        File.open(File.join(@cache_dir, file_name[0].downcase, file_name), 'wb') do |file|
+        File.open(target_file, 'wb') do |file|
           file.write(buffer.read)
         end
-        path = File.join(@cache_dir, file_name[0].downcase, file_name)
+        path = target_file
+      else
+        @log.info("Filesystem cache HIT for path: #{relative_path}")
       end
       File.open(path, 'rb')
     rescue => e
@@ -59,6 +61,26 @@ def upload(file_data)
     protected
     attr_reader :log
 
+    def get_non_mutable(relative_url)
+      file_name = relative_url.split('/').last
+      target_file = File.join(@cache_dir, file_name[0].downcase, file_name)
+      path = Dir["#{@cache_dir}/**/#{file_name}"].first
+      unless File.exist?("#{path}")
+        buffer = get(relative_url)
+        File.open(target_file, 'wb') do |file|
+          file.write(buffer)
+        end
+        path = target_file
+      else
+        @log.info("Filesystem cache HIT for url: #{relative_url}")
+      end
+      File.binread(path)
+    rescue => e
+      @log.error("#{self.class.name} failed getting non-mutable url '#{relative_url}'")
+      @log.error("Error: #{e}")
+      return nil
+    end
+
     def get(relative_url)
       @http_client.get(url(relative_url))
     end

lib/puppet_forge_server/backends/proxy_v3.rb

@@ -30,11 +30,11 @@ def initialize(url, cache_dir, http_client = PuppetForgeServer::Http::HttpClient
     def get_metadata(author, name, options = {})
       query ="#{author}-#{name}"
       begin
-        releases = options[:version] ? [JSON.parse(get("/v3/releases/#{query}-#{options[:version]}"))] : get_all_result_pages("/v3/releases?module=#{query}")
+        releases = get_releases(query, options)
         get_modules(releases)
       rescue => e
-        @log.debug("#{self.class.name} failed querying metadata for '#{query}' with options #{options}")
-        @log.debug("Error: #{e}")
+        @log.error("#{self.class.name} failed querying metadata for '#{query}' with options #{options}")
+        @log.error("Error: #{e}")
         return nil
       end
     end
@@ -44,13 +44,26 @@ def query_metadata(query, options = {})
         releases = get_all_result_pages("/v3/modules?query=#{query}").map {|element| element['current_release']}
         get_modules(releases)
       rescue => e
-        @log.debug("#{self.class.name} failed querying metadata for '#{query}' with options #{options}")
-        @log.debug("Error: #{e}")
+        @log.error("#{self.class.name} failed querying metadata for '#{query}' with options #{options}")
+        @log.error("Error: #{e}")
         return nil
       end
     end
 
     private
+
+    def get_releases(query, options = {})
+      version = options[:version]
+      unless version.nil?
+        url = "/v3/releases/#{query}-#{version}"
+        buffer = get_non_mutable(url)
+        release = JSON.parse(buffer)
+        [ release ]
+      else
+        get_all_result_pages("/v3/releases?module=#{query}")
+      end
+    end
+
     def get_all_result_pages(next_page)
       results = []
       begin

lib/puppet_forge_server/http/http_client.rb

@@ -17,16 +17,19 @@
 require 'open-uri'
 require 'open_uri_redirections'
 require 'timeout'
-require 'lrucache'
 require 'net/http'
 require 'net/http/post/multipart'
 
 
 module PuppetForgeServer::Http
   class HttpClient
+    include PuppetForgeServer::Utils::CacheProvider
+    include PuppetForgeServer::Utils::FilteringInspecter
 
     def initialize(cache = nil)
-      cache = LRUCache.new(:ttl => 60 * 15, :max_size => 250) if cache.nil?
+      cache = cache_instance if cache.nil?
+      cache.extend(PuppetForgeServer::Utils::FilteringInspecter)
+      @log = PuppetForgeServer::Logger.get
       @cache = cache
     end
 
@@ -51,9 +54,18 @@ def download(url)
       open_uri(url)
     end
 
+    def inspect
+      cache_inspected = @cache.inspect_without [ :@data ]
+      cache_inspected.gsub!(/>$/, ", @size=#{@cache.size}>")
+      inspected = inspect_without [ :@cache ]
+      inspected.gsub(/>$/, ", @cache=#{cache_inspected}>")
+    end
+
     private
 
     def open_uri(url)
+      hit_or_miss = @cache.include?(url) ? 'HIT' : 'MISS'
+      @log.info "Cache in RAM memory size: #{@cache.size}, #{hit_or_miss} for url: #{url}"
       contents = @cache.fetch(url) do
         tmpfile = ::Timeout.timeout(10) do
           PuppetForgeServer::Logger.get.debug "Fetching data for url: #{url} from remote server"
@@ -63,7 +75,7 @@ def open_uri(url)
         tmpfile.close
         contents
       end
-      PuppetForgeServer::Logger.get.debug "Data for url: #{url} fetched, #{contents.size} bytes"
+      @log.debug "Data for url: #{url} fetched, #{contents.size} bytes"
       StringIO.new(contents)
     end
   end

lib/puppet_forge_server/server.rb

@@ -19,6 +19,7 @@
 module PuppetForgeServer
   class Server
     include PuppetForgeServer::Utils::OptionParser
+    include PuppetForgeServer::Utils::CacheProvider
     include PuppetForgeServer::Utils::Http
 
     def go(args)
@@ -27,6 +28,7 @@ def go(args)
       begin
         options = parse_options(args)
         @log = logging(options)
+        configure_cache(options[:ram_cache_ttl], options[:ram_cache_size])
         backends = backends(options)
         server = build(backends, options[:webui_root])
         announce(options, backends)

lib/puppet_forge_server/utils/cache_provider.rb

@@ -0,0 +1,42 @@
+# -*- encoding: utf-8 -*-
+#
+# Copyright 2015 Centralny Osroder Informatyki (gov.pl)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'lrucache'
+
+module PuppetForgeServer::Utils
+  module CacheProvider
+
+    opts = PuppetForgeServer::Utils::OptionParser.DEFAULT_OPTIONS
+    @@CACHE = LRUCache.new(:ttl => opts[:ram_cache_ttl], :max_size => opts[:ram_cache_size])
+
+    # Method for fetching application wide cache for fetching HTTP requests
+    #
+    # @return [LRUCache] a instance of cache for application
+    def cache_instance
+      @@CACHE
+    end
+
+    # Configure a application wide cache using LSUCache implementation
+    #
+    # @param [int] ttl a time to live for elements
+    # @param [int] size a maximum size for cache
+    def configure_cache(ttl, size)
+      @@CACHE = LRUCache.new(:ttl => ttl, :max_size => size)
+      PuppetForgeServer::Logger.get.info("Using RAM memory LRUCache with time to live of #{ttl}sec and max size of #{size} elements")
+      nil
+    end
+  end
+end

lib/puppet_forge_server/utils/filtering_inspecter.rb

@@ -0,0 +1,30 @@
+# -*- encoding: utf-8 -*-
+#
+# Copyright 2015 Centralny Osroder Informatyki (gov.pl)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module PuppetForgeServer::Utils
+  module FilteringInspecter
+    def self.inspect_without(object, variables)
+      filtered = object.instance_variables.reject { |n| variables.include? n }
+      vars = filtered.map { |n| "#{n}=#{object.instance_variable_get(n).inspect}" }
+      oid = object.object_id << 1
+      "#<%s:0x%x %s>" % [ object.class, oid, vars.join(', ') ]
+    end
+
+    def inspect_without(variables)
+      PuppetForgeServer::Utils::FilteringInspecter.inspect_without(self, variables)
+    end
+  end
+end

lib/puppet_forge_server/utils/option_parser.rb

@@ -28,9 +28,23 @@ module OptionParser
     @@DEFAULT_LOG_DIR = File.join(Dir.tmpdir.to_s, 'puppet-forge-server', 'log')
     @@DEFAULT_WEBUI_ROOT = File.expand_path('../app', File.dirname(__FILE__))
     @@DEFAULT_HOST = '0.0.0.0'
+    @@DEFAULT_RAM_CACHE_TTL = 60 * 30 # 30min
+    @@DEFAULT_RAM_CACHE_SIZE = 250
+
+    def self.DEFAULT_OPTIONS
+      {
+        :daemonize      => @@DEFAULT_DAEMONIZE,
+        :cache_basedir  => @@DEFAULT_CACHE_DIR,
+        :port           => @@DEFAULT_PORT,
+        :webui_root     => @@DEFAULT_WEBUI_ROOT,
+        :host           => @@DEFAULT_HOST,
+        :ram_cache_ttl  => @@DEFAULT_RAM_CACHE_TTL,
+        :ram_cache_size => @@DEFAULT_RAM_CACHE_SIZE
+      }
+    end
 
     def parse_options(args)
-      options = {:daemonize => @@DEFAULT_DAEMONIZE, :cache_basedir => @@DEFAULT_CACHE_DIR, :port => @@DEFAULT_PORT, :webui_root => @@DEFAULT_WEBUI_ROOT, :host => @@DEFAULT_HOST}
+      options = PuppetForgeServer::Utils::OptionParser.DEFAULT_OPTIONS
       option_parser = ::OptionParser.new do |opts|
         opts.banner = "Usage: #{File.basename $0} [options]"
         opts.version = PuppetForgeServer::VERSION
@@ -63,6 +77,14 @@ def parse_options(args)
           options[:cache_basedir] = cache_basedir
         end
 
+        opts.on('--ram-cache-ttl SECONDS', "The time to live in seconds for remote requests RAM cache (default: #{@@DEFAULT_RAM_CACHE_TTL})") do |ram_cache_ttl|
+          options[:ram_cache_ttl] = ram_cache_ttl
+        end
+
+        opts.on('--ram-cache-size ENTRIES', "The maximum number of enties in RAM cache for remote requests (default: #{@@DEFAULT_RAM_CACHE_SIZE})") do |ram_cache_size|
+          options[:ram_cache_size] = ram_cache_size
+        end
+
         opts.on('--log-dir DIR', "Log directory (default: #{@@DEFAULT_LOG_DIR})") do |log_dir|
           options[:log_dir] = log_dir
         end