diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..1d45381 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,32 @@ +FROM ubuntu:14.04 + +RUN apt-get update && \ + apt-get -y upgrade + +RUN useradd forge +RUN mkdir -p /opt/forge/log && \ + mkdir -p /opt/forge/cache && \ + mkdir -p /opt/forge/modules && \ + chown -R forge /opt/forge + +ENV FORGE_S3BUCKET='##AWS_BUCKET##' +ENV FORGE_AWS_SECRET='##AWS_SECRET_ACCESS_KEY##' +ENV FORGE_AWS_KEY='##AWS_ACCESS_KEY_ID##' +ENV FORGE_PORT=8080 +ENV FORGE_LOGDIR=/opt/forge/log +ENV FORGE_CACHEDIR=/opt/forge/cache +ENV FORGE_PROXY='https://forgeapi.puppetlabs.com' +ENV FORGE_PIDFILE='/opt/forge/server.pid' +ENV FORGE_AWS_REGION='##AWS_REGION##' + +RUN apt-get -y install ruby ruby-dev build-essential +RUN gem install aws-sdk + +COPY puppet-forge-server-*.gem / +COPY Gemfile-dockerfile /Gemfile +RUN gem install bundle +RUN bundle install --gemfile=/Gemfile +RUN gem install --local /puppet-forge-server-*.gem + +USER forge +CMD puppet-forge-server diff --git a/Gemfile-dockerfile b/Gemfile-dockerfile new file mode 100644 index 0000000..2045e64 --- /dev/null +++ b/Gemfile-dockerfile @@ -0,0 +1,12 @@ +source 'https://rubygems.org' + +gem 'sinatra', '~> 1.4' +gem 'sinatra-contrib', '~> 1.4' +gem 'json', '~> 1.8' +gem 'rack-mount', '~> 0.8' +gem 'rack', '= 1.5.5' +gem 'open4', '~> 1.3' +gem 'open_uri_redirections', '~> 0.1' +gem 'haml', '~> 4.0' +gem 'deep_merge', '~> 1.0' +gem 'multipart-post', '~> 2.0.0' diff --git a/lib/puppet_forge_server.rb b/lib/puppet_forge_server.rb index a78e320..2d377f2 100644 --- a/lib/puppet_forge_server.rb +++ b/lib/puppet_forge_server.rb @@ -46,6 +46,7 @@ module Backends autoload :Proxy, 'puppet_forge_server/backends/proxy' autoload :ProxyV1, 'puppet_forge_server/backends/proxy_v1' autoload :ProxyV3, 'puppet_forge_server/backends/proxy_v3' + autoload :S3, 'puppet_forge_server/backends/S3' end module Models diff --git a/lib/puppet_forge_server/api/v3/releases.rb b/lib/puppet_forge_server/api/v3/releases.rb index d65ddf2..3666c52 100644 --- a/lib/puppet_forge_server/api/v3/releases.rb +++ b/lib/puppet_forge_server/api/v3/releases.rb @@ -30,7 +30,7 @@ def get_releases(metadata) :metadata => element.metadata.to_hash, :version => element.metadata.version, :tags => element.tags ? element.tags : [element.metadata.author, name], - :file_uri => "/v3/files#{element.path}", + :file_uri => "/v3/files/#{element.path}", :file_md5 => element.checksum, :deleted_at => element.deleted_at } diff --git a/lib/puppet_forge_server/backends/S3.rb b/lib/puppet_forge_server/backends/S3.rb new file mode 100644 index 0000000..7f0e438 --- /dev/null +++ b/lib/puppet_forge_server/backends/S3.rb @@ -0,0 +1,123 @@ +# -*- encoding: utf-8 -*- +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require 'json' +require 'digest/md5' +require 'aws-sdk' + +module PuppetForgeServer::Backends + class S3 + include PuppetForgeServer::Utils::Archiver + # Priority should be lower than v3 API proxies as v3 requires less API calls + @@PRIORITY = 0 + attr_reader :PRIORITY + + def initialize(bucket, cache_dir, s3_client) + @bucket = bucket + @cache_dir = File.join(cache_dir, Digest::SHA1.hexdigest(@bucket)) + @s3 = s3_client + # Create directory structure for all alphabetic letters + ('a'..'z').each do |i| + FileUtils.mkdir_p(File.join(@cache_dir, i)) + end + end + + def get_modules(filename, options) + options = ({:with_checksum => true}).merge(options) + modules = [] + s3_filenames = [] + + bucket_list = @s3.list_objects({bucket: @bucket}) + bucket_list.contents.each do |s3_object| + if /modules\/#{filename}/ =~ s3_object.key + s3_filenames << s3_object.key.gsub(/^modules\//, '') + end + end + + s3_filenames.each do |exact_filename| + path = File.join(@cache_dir, exact_filename[0].downcase, exact_filename) + target_file = File.open(path, "w") + @s3.get_object({bucket: @bucket, key: "modules/#{exact_filename}", response_target: target_file}) + modules << new_module(path, options, exact_filename) + end + + modules.reject{|i| i.nil?} + end + + def new_module(path, options, exact_filename) + metadata = read_metadata(path) + if metadata + thismodule = PuppetForgeServer::Models::Module.new({ + :metadata => parse_dependencies(PuppetForgeServer::Models::Metadata.new(normalize_metadata(metadata))), + :checksum => options[:with_checksum] ? Digest::MD5.file(path).hexdigest : nil, + :path => exact_filename, + :private => ! @readonly + }) + else + @log.error "Failed reading metadata from #{path}" + thismodule = nil + end + thismodule + end + + def get_metadata(author, name, options = {}) + version = options[:version] ? options[:version] : '.*' + get_modules("#{author}-#{name}-#{version}.tar.gz", options) + end + + def query_metadata(query, options = {}) + get_modules("*#{query}*.tar.gz", options) + end + + def parse_dependencies(metadata) + metadata.dependencies = metadata.dependencies.dup.map do |dependency| + PuppetForgeServer::Models::Dependency.new({:name => dependency['name'], :version_requirement => dependency.key?('version_requirement') ? dependency['version_requirement'] : nil}) + end.flatten + metadata + end + + def normalize_metadata(metadata) + metadata['name'] = metadata['name'].gsub('/', '-') + metadata + end + + def get_file_buffer(relative_path) + file_name = relative_path.split('/').last + File.join(@cache_dir, file_name[0].downcase, file_name) + path = Dir["#{@cache_dir}/**/#{file_name}"].first + unless File.exist?("#{path}") + buffer = download("#{@file_path}/#{relative_path}") + File.open(File.join(@cache_dir, file_name[0].downcase, file_name), 'wb') do |file| + file.write(buffer.read) + end + path = File.join(@cache_dir, file_name[0].downcase, file_name) + end + File.open(path, 'rb') + rescue => e + @log.error("#{self.class.name} failed downloading file '#{relative_path}'") + @log.error("Error: #{e}") + return nil + end + + private + def read_metadata(archive_path) + metadata_file = read_from_archive(archive_path, %r[[^/]+/metadata\.json$]) + JSON.parse(metadata_file) + rescue => error + warn "Error reading from module archive #{archive_path}: #{error}" + return nil + end + + end +end diff --git a/lib/puppet_forge_server/server.rb b/lib/puppet_forge_server/server.rb index 1baa383..591e12e 100644 --- a/lib/puppet_forge_server/server.rb +++ b/lib/puppet_forge_server/server.rb @@ -100,6 +100,13 @@ def backends(options) when 'Proxy' @log.info "Detecting API version for #{url}..." PuppetForgeServer::Backends.const_get("#{type}V#{get_api_version(url)}").new(url.chomp('/'), options[:cache_basedir]) + when 'S3' + @log.info "S3 backend selected..." + PuppetForgeServer::Backends.const_get('S3').new(options[:backend]['S3'][0], options[:cache_basedir], Aws::S3::Client.new( + region: options[:aws_region], + access_key_id: options[:aws_key_id], + secret_access_key: options[:aws_secret_key] + )) else PuppetForgeServer::Backends.const_get(type).new(url) end diff --git a/lib/puppet_forge_server/utils/option_parser.rb b/lib/puppet_forge_server/utils/option_parser.rb index 5aeba07..f06c95c 100644 --- a/lib/puppet_forge_server/utils/option_parser.rb +++ b/lib/puppet_forge_server/utils/option_parser.rb @@ -35,44 +35,118 @@ def parse_options(args) opts.banner = "Usage: #{File.basename $0} [options]" opts.version = PuppetForgeServer::VERSION - opts.on('-p', '--port PORT', "Port number to bind to (default: #{@@DEFAULT_PORT})") do |port| - options[:port] = port + if ENV.has_key?('FORGE_PORT') + options[:port] = ENV['FORGE_PORT'] + else + opts.on('-p', '--port PORT', "Port number to bind to (default: #{@@DEFAULT_PORT})") do |port| + options[:port] = port + end end - opts.on('-b', '--bind HOST', "Host name or IP address to bind to (default: #{@@DEFAULT_HOST})") do |host| - options[:host] = host + if ENV.has_key?('FORGE_BIND') + options[:host] = ENV['FORGE_BIND'] + else + opts.on('-b', '--bind HOST', "Host name or IP address to bind to (default: #{@@DEFAULT_HOST})") do |host| + options[:host] = host + end end - opts.on('-D', '--daemonize', "Run server in the background (default: #{@@DEFAULT_DAEMONIZE})") do + if ENV.has_key?('FORGE_DAEMONIZE') options[:daemonize] = true + else + opts.on('-D', '--daemonize', "Run server in the background (default: #{@@DEFAULT_DAEMONIZE})") do + options[:daemonize] = true + end end - opts.on('--pidfile FILE', "Pid file location (default: #{@@DEFAULT_PID_FILE})") do |pidfile| - options[:pidfile] = pidfile + if ENV.has_key?('FORGE_PIDFILE') + options[pidfile] = ENV['FORGE_PIDFILE'] + else + opts.on('--pidfile FILE', "Pid file location (default: #{@@DEFAULT_PID_FILE})") do |pidfile| + options[:pidfile] = pidfile + end end - options[:backend] = {'Directory' => [], 'Proxy' => [], 'Source' => []} - opts.on('-m', '--module-dir DIR', 'Directory containing packaged modules (recursively searched)') do |module_dir| - options[:backend]['Directory'] << module_dir + options[:backend] = {'Directory' => [], 'Proxy' => [], 'Source' => [], 'S3' => []} + + if ENV.has_key?('FORGE_MODULEDIR') + options[:backend]['Directory'] << ENV['FORGE_MODULEDIR'] + else + opts.on('-m', '--module-dir DIR', 'Directory containing packaged modules (recursively searched)') do |module_dir| + options[:backend]['Directory'] << module_dir + end + end + + if ENV.has_key?('FORGE_PROXY') + options[:backend]['Proxy'] << ENV['FORGE_PROXY'] + else + opts.on('-x', '--proxy URL', 'Remote forge URL') do |url| + options[:backend]['Proxy'] << url + end + end + + if ENV.has_key?('FORGE_AWS_REGION') + options[:aws_region] = ENV['FORGE_AWS_REGION'] + else + opts.on('--aws-region AWSREGION', 'AWS Region where the bucket is located') do |region| + options[:aws_region] = region + end end - opts.on('-x', '--proxy URL', 'Remote forge URL') do |url| - options[:backend]['Proxy'] << url + + if ENV.has_key?('FORGE_AWS_KEY') + options[:aws_key_id] = ENV['FORGE_AWS_KEY'] + else + opts.on('--aws-key KEY', 'AWS Key id') do |key| + options[:aws_key_id] = key + end + end + + if ENV.has_key?('FORGE_AWS_SECRET') + options[:aws_secret_key] = ENV['FORGE_AWS_SECRET'] + else + opts.on('--aws-secret SECRET', 'AWS Secret key') do |secret| + options[:aws_secret_key] = secret + end + end + + if ENV.has_key?('FORGE_S3BUCKET') + options[:backend]['S3'] << ENV['FORGE_S3BUCKET'] + else + opts.on('-s', '--s3 BucketName', 'use S3') do |bucket| + options[:backend]['S3'] << bucket + end end - opts.on('--cache-basedir DIR', "Proxy module cache base directory (default: #{@@DEFAULT_CACHE_DIR})") do |cache_basedir| - options[:cache_basedir] = cache_basedir + if ENV.has_key?('FORGE_CACHEDIR') + options[:cache_basedir] = ENV['FORGE_CACHEDIR'] + else + opts.on('--cache-basedir DIR', "Proxy module cache base directory (default: #{@@DEFAULT_CACHE_DIR})") do |cache_basedir| + options[:cache_basedir] = cache_basedir + end end - opts.on('--log-dir DIR', "Log directory (default: #{@@DEFAULT_LOG_DIR})") do |log_dir| - options[:log_dir] = log_dir + if ENV.has_key?('FORGE_LOGDIR') + options[:log_dir] = ENV['FORGE_LOGDIR'] + else + opts.on('--log-dir DIR', "Log directory (default: #{@@DEFAULT_LOG_DIR})") do |log_dir| + options[:log_dir] = log_dir + end end - opts.on('--webui-root DIR', "Directory containing views and other public files used for web UI: #{@@DEFAULT_WEBUI_ROOT})") do |webui_root| - options[:webui_root] = webui_root + if ENV.has_key?('FORGE_WEBUI_ROOT') + options[:webui_root] = ENV['FORGE_WEBUI_ROOT'] + else + opts.on('--webui-root DIR', "Directory containing views and other public files used for web UI: #{@@DEFAULT_WEBUI_ROOT})") do |webui_root| + options[:webui_root] = webui_root + end end - opts.on('--debug', 'Log everything into STDERR') do + if ENV.has_key?('FORGE_DEBUG') options[:debug] = true + else + opts.on('--debug', 'Log everything into STDERR') do + options[:debug] = true + end end end begin diff --git a/lib/puppet_forge_server/version.rb b/lib/puppet_forge_server/version.rb index cc808a3..7eca49b 100644 --- a/lib/puppet_forge_server/version.rb +++ b/lib/puppet_forge_server/version.rb @@ -15,5 +15,5 @@ # limitations under the License. module PuppetForgeServer - VERSION = '1.8.0' + VERSION = '1.9.0' end