Add S3 backend.

Dockerfile

@@ -0,0 +1,32 @@
+FROM ubuntu:14.04
+
+RUN apt-get update && \
+    apt-get -y upgrade
+
+RUN useradd forge
+RUN mkdir -p /opt/forge/log && \
+    mkdir -p /opt/forge/cache && \
+    mkdir -p /opt/forge/modules && \
+    chown -R forge /opt/forge
+
+ENV FORGE_S3BUCKET='##AWS_BUCKET##'
+ENV FORGE_AWS_SECRET='##AWS_SECRET_ACCESS_KEY##'
+ENV FORGE_AWS_KEY='##AWS_ACCESS_KEY_ID##'
+ENV FORGE_PORT=8080
+ENV FORGE_LOGDIR=/opt/forge/log
+ENV FORGE_CACHEDIR=/opt/forge/cache
+ENV FORGE_PROXY='https://forgeapi.puppetlabs.com'
+ENV FORGE_PIDFILE='/opt/forge/server.pid'
+ENV FORGE_AWS_REGION='##AWS_REGION##'
+
+RUN apt-get -y install ruby ruby-dev build-essential
+RUN gem install aws-sdk
+
+COPY puppet-forge-server-*.gem /
+COPY Gemfile-dockerfile /Gemfile
+RUN gem install bundle
+RUN bundle install --gemfile=/Gemfile
+RUN gem install --local /puppet-forge-server-*.gem
+
+USER forge
+CMD puppet-forge-server

Gemfile-dockerfile

@@ -0,0 +1,12 @@
+source 'https://rubygems.org'
+
+gem 'sinatra', '~> 1.4'
+gem 'sinatra-contrib', '~> 1.4'
+gem 'json', '~> 1.8'
+gem 'rack-mount', '~> 0.8'
+gem 'rack', '= 1.5.5'
+gem 'open4', '~> 1.3'
+gem 'open_uri_redirections', '~> 0.1'
+gem 'haml', '~> 4.0'
+gem 'deep_merge', '~> 1.0'
+gem 'multipart-post', '~> 2.0.0'

lib/puppet_forge_server.rb

@@ -46,6 +46,7 @@ module Backends
     autoload :Proxy, 'puppet_forge_server/backends/proxy'
     autoload :ProxyV1, 'puppet_forge_server/backends/proxy_v1'
     autoload :ProxyV3, 'puppet_forge_server/backends/proxy_v3'
+    autoload :S3, 'puppet_forge_server/backends/S3'
   end
 
   module Models

lib/puppet_forge_server/api/v3/releases.rb

@@ -30,7 +30,7 @@ def get_releases(metadata)
             :metadata => element.metadata.to_hash,
             :version => element.metadata.version,
             :tags => element.tags ? element.tags : [element.metadata.author, name],
-            :file_uri => "/v3/files#{element.path}",
+            :file_uri => "/v3/files/#{element.path}",
             :file_md5 => element.checksum,
             :deleted_at => element.deleted_at
         }

lib/puppet_forge_server/backends/S3.rb

@@ -0,0 +1,123 @@
+# -*- encoding: utf-8 -*-
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'json'
+require 'digest/md5'
+require 'aws-sdk'
+
+module PuppetForgeServer::Backends
+  class S3
+    include PuppetForgeServer::Utils::Archiver
+    # Priority should be lower than v3 API proxies as v3 requires less API calls
+    @@PRIORITY = 0
+    attr_reader :PRIORITY
+
+    def initialize(bucket, cache_dir, s3_client)
+        @bucket = bucket
+        @cache_dir = File.join(cache_dir, Digest::SHA1.hexdigest(@bucket))
+        @s3 = s3_client
+        # Create directory structure for all alphabetic letters
+        ('a'..'z').each do |i|
+          FileUtils.mkdir_p(File.join(@cache_dir, i))
+        end
+    end
+
+    def get_modules(filename, options)
+      options = ({:with_checksum => true}).merge(options)
+      modules = []
+      s3_filenames = []
+
+      bucket_list = @s3.list_objects({bucket: @bucket})
+      bucket_list.contents.each do |s3_object|
+        if /modules\/#{filename}/ =~ s3_object.key
+          s3_filenames << s3_object.key.gsub(/^modules\//, '')
+        end
+      end
+
+      s3_filenames.each do |exact_filename|
+        path = File.join(@cache_dir, exact_filename[0].downcase, exact_filename)
+        target_file = File.open(path, "w")
+        @s3.get_object({bucket: @bucket, key: "modules/#{exact_filename}", response_target: target_file})
+        modules << new_module(path, options, exact_filename)
+      end
+
+      modules.reject{|i| i.nil?}
+    end
+
+    def new_module(path, options, exact_filename)
+      metadata = read_metadata(path)
+      if metadata
+        thismodule = PuppetForgeServer::Models::Module.new({
+          :metadata => parse_dependencies(PuppetForgeServer::Models::Metadata.new(normalize_metadata(metadata))),
+          :checksum => options[:with_checksum] ? Digest::MD5.file(path).hexdigest : nil,
+          :path => exact_filename,
+          :private => ! @readonly
+          })
+      else
+        @log.error "Failed reading metadata from #{path}"
+        thismodule = nil
+      end
+      thismodule
+    end
+
+    def get_metadata(author, name, options = {})
+      version = options[:version] ? options[:version] : '.*'
+      get_modules("#{author}-#{name}-#{version}.tar.gz", options)
+    end
+
+    def query_metadata(query, options = {})
+      get_modules("*#{query}*.tar.gz", options)
+    end
+
+    def parse_dependencies(metadata)
+      metadata.dependencies = metadata.dependencies.dup.map do |dependency|
+        PuppetForgeServer::Models::Dependency.new({:name => dependency['name'], :version_requirement => dependency.key?('version_requirement') ? dependency['version_requirement'] : nil})
+      end.flatten
+      metadata
+    end
+
+    def normalize_metadata(metadata)
+      metadata['name'] = metadata['name'].gsub('/', '-')
+      metadata
+    end
+
+    def get_file_buffer(relative_path)
+      file_name = relative_path.split('/').last
+      File.join(@cache_dir, file_name[0].downcase, file_name)
+      path = Dir["#{@cache_dir}/**/#{file_name}"].first
+      unless File.exist?("#{path}")
+        buffer = download("#{@file_path}/#{relative_path}")
+        File.open(File.join(@cache_dir, file_name[0].downcase, file_name), 'wb') do |file|
+          file.write(buffer.read)
+        end
+        path = File.join(@cache_dir, file_name[0].downcase, file_name)
+      end
+      File.open(path, 'rb')
+      rescue => e
+      @log.error("#{self.class.name} failed downloading file '#{relative_path}'")
+      @log.error("Error: #{e}")
+      return nil
+    end
+
+    private
+    def read_metadata(archive_path)
+        metadata_file = read_from_archive(archive_path, %r[[^/]+/metadata\.json$])
+        JSON.parse(metadata_file)
+    rescue => error
+        warn "Error reading from module archive #{archive_path}: #{error}"
+        return nil
+    end
+
+  end
+end

lib/puppet_forge_server/server.rb

@@ -100,6 +100,13 @@ def backends(options)
             when 'Proxy'
               @log.info "Detecting API version for #{url}..."
               PuppetForgeServer::Backends.const_get("#{type}V#{get_api_version(url)}").new(url.chomp('/'), options[:cache_basedir])
+            when 'S3'
+              @log.info "S3 backend selected..."
+              PuppetForgeServer::Backends.const_get('S3').new(options[:backend]['S3'][0], options[:cache_basedir], Aws::S3::Client.new(
+                  region: options[:aws_region],
+                  access_key_id: options[:aws_key_id],
+                  secret_access_key: options[:aws_secret_key]
+                  ))
             else
               PuppetForgeServer::Backends.const_get(type).new(url)
           end

lib/puppet_forge_server/utils/option_parser.rb

@@ -35,44 +35,118 @@ def parse_options(args)
         opts.banner = "Usage: #{File.basename $0} [options]"
         opts.version = PuppetForgeServer::VERSION
 
-        opts.on('-p', '--port PORT', "Port number to bind to (default: #{@@DEFAULT_PORT})") do |port|
-          options[:port] = port
+        if ENV.has_key?('FORGE_PORT')
+          options[:port] = ENV['FORGE_PORT']
+        else
+          opts.on('-p', '--port PORT', "Port number to bind to (default: #{@@DEFAULT_PORT})") do |port|
+            options[:port] = port
+          end
         end
 
-        opts.on('-b', '--bind HOST', "Host name or IP address to bind to (default: #{@@DEFAULT_HOST})") do |host|
-          options[:host] = host
+        if ENV.has_key?('FORGE_BIND')
+          options[:host] = ENV['FORGE_BIND']
+        else
+          opts.on('-b', '--bind HOST', "Host name or IP address to bind to (default: #{@@DEFAULT_HOST})") do |host|
+            options[:host] = host
+          end
         end
 
-        opts.on('-D', '--daemonize', "Run server in the background (default: #{@@DEFAULT_DAEMONIZE})") do
+        if ENV.has_key?('FORGE_DAEMONIZE')
           options[:daemonize] = true
+        else
+          opts.on('-D', '--daemonize', "Run server in the background (default: #{@@DEFAULT_DAEMONIZE})") do
+            options[:daemonize] = true
+          end
         end
 
-        opts.on('--pidfile FILE', "Pid file location (default: #{@@DEFAULT_PID_FILE})") do |pidfile|
-          options[:pidfile] = pidfile
+        if ENV.has_key?('FORGE_PIDFILE')
+          options[pidfile] = ENV['FORGE_PIDFILE']
+        else
+          opts.on('--pidfile FILE', "Pid file location (default: #{@@DEFAULT_PID_FILE})") do |pidfile|
+            options[:pidfile] = pidfile
+          end
         end
 
-        options[:backend] = {'Directory' => [], 'Proxy' => [], 'Source' => []}
-        opts.on('-m', '--module-dir DIR', 'Directory containing packaged modules (recursively searched)') do |module_dir|
-          options[:backend]['Directory'] << module_dir
+        options[:backend] = {'Directory' => [], 'Proxy' => [], 'Source' => [], 'S3' => []}
+
+        if ENV.has_key?('FORGE_MODULEDIR')
+          options[:backend]['Directory'] << ENV['FORGE_MODULEDIR']
+        else
+          opts.on('-m', '--module-dir DIR', 'Directory containing packaged modules (recursively searched)') do |module_dir|
+            options[:backend]['Directory'] << module_dir
+          end
+        end
+
+        if ENV.has_key?('FORGE_PROXY')
+          options[:backend]['Proxy'] << ENV['FORGE_PROXY']
+        else
+          opts.on('-x', '--proxy URL', 'Remote forge URL') do |url|
+            options[:backend]['Proxy'] << url
+          end
+        end
+
+        if ENV.has_key?('FORGE_AWS_REGION')
+          options[:aws_region] = ENV['FORGE_AWS_REGION']
+        else
+          opts.on('--aws-region AWSREGION', 'AWS Region where the bucket is located') do |region|
+            options[:aws_region] = region
+          end
         end
-        opts.on('-x', '--proxy URL', 'Remote forge URL') do |url|
-          options[:backend]['Proxy'] << url
+
+        if ENV.has_key?('FORGE_AWS_KEY')
+          options[:aws_key_id] = ENV['FORGE_AWS_KEY']
+        else
+          opts.on('--aws-key KEY', 'AWS Key id') do |key|
+            options[:aws_key_id] = key
+          end
+        end
+
+        if ENV.has_key?('FORGE_AWS_SECRET')
+          options[:aws_secret_key] = ENV['FORGE_AWS_SECRET']
+        else
+          opts.on('--aws-secret SECRET', 'AWS Secret key') do |secret|
+            options[:aws_secret_key] = secret
+          end
+        end
+
+        if ENV.has_key?('FORGE_S3BUCKET')
+          options[:backend]['S3'] << ENV['FORGE_S3BUCKET']
+        else
+          opts.on('-s', '--s3 BucketName', 'use S3') do |bucket|
+            options[:backend]['S3'] << bucket
+          end
         end
 
-        opts.on('--cache-basedir DIR', "Proxy module cache base directory (default: #{@@DEFAULT_CACHE_DIR})") do |cache_basedir|
-          options[:cache_basedir] = cache_basedir
+        if ENV.has_key?('FORGE_CACHEDIR')
+          options[:cache_basedir] = ENV['FORGE_CACHEDIR']
+        else
+          opts.on('--cache-basedir DIR', "Proxy module cache base directory (default: #{@@DEFAULT_CACHE_DIR})") do |cache_basedir|
+            options[:cache_basedir] = cache_basedir
+          end
         end
 
-        opts.on('--log-dir DIR', "Log directory (default: #{@@DEFAULT_LOG_DIR})") do |log_dir|
-          options[:log_dir] = log_dir
+        if ENV.has_key?('FORGE_LOGDIR')
+          options[:log_dir] = ENV['FORGE_LOGDIR']
+        else
+          opts.on('--log-dir DIR', "Log directory (default: #{@@DEFAULT_LOG_DIR})") do |log_dir|
+            options[:log_dir] = log_dir
+          end
         end
 
-        opts.on('--webui-root DIR', "Directory containing views and other public files used for web UI: #{@@DEFAULT_WEBUI_ROOT})") do |webui_root|
-          options[:webui_root] = webui_root
+        if ENV.has_key?('FORGE_WEBUI_ROOT')
+          options[:webui_root] = ENV['FORGE_WEBUI_ROOT']
+        else
+          opts.on('--webui-root DIR', "Directory containing views and other public files used for web UI: #{@@DEFAULT_WEBUI_ROOT})") do |webui_root|
+            options[:webui_root] = webui_root
+          end
         end
 
-        opts.on('--debug', 'Log everything into STDERR') do
+        if ENV.has_key?('FORGE_DEBUG')
           options[:debug] = true
+        else
+          opts.on('--debug', 'Log everything into STDERR') do
+            options[:debug] = true
+          end
         end
       end
       begin

lib/puppet_forge_server/version.rb

@@ -15,5 +15,5 @@
 # limitations under the License.
 
 module PuppetForgeServer
-    VERSION = '1.8.0'
+    VERSION = '1.9.0'
 end