diff --git a/lib/puppet_forge_server.rb b/lib/puppet_forge_server.rb index a78e320..2d377f2 100644 --- a/lib/puppet_forge_server.rb +++ b/lib/puppet_forge_server.rb @@ -46,6 +46,7 @@ module Backends autoload :Proxy, 'puppet_forge_server/backends/proxy' autoload :ProxyV1, 'puppet_forge_server/backends/proxy_v1' autoload :ProxyV3, 'puppet_forge_server/backends/proxy_v3' + autoload :S3, 'puppet_forge_server/backends/S3' end module Models diff --git a/lib/puppet_forge_server/backends/S3.rb b/lib/puppet_forge_server/backends/S3.rb new file mode 100644 index 0000000..7f0e438 --- /dev/null +++ b/lib/puppet_forge_server/backends/S3.rb @@ -0,0 +1,123 @@ +# -*- encoding: utf-8 -*- +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require 'json' +require 'digest/md5' +require 'aws-sdk' + +module PuppetForgeServer::Backends + class S3 + include PuppetForgeServer::Utils::Archiver + # Priority should be lower than v3 API proxies as v3 requires less API calls + @@PRIORITY = 0 + attr_reader :PRIORITY + + def initialize(bucket, cache_dir, s3_client) + @bucket = bucket + @cache_dir = File.join(cache_dir, Digest::SHA1.hexdigest(@bucket)) + @s3 = s3_client + # Create directory structure for all alphabetic letters + ('a'..'z').each do |i| + FileUtils.mkdir_p(File.join(@cache_dir, i)) + end + end + + def get_modules(filename, options) + options = ({:with_checksum => true}).merge(options) + modules = [] + s3_filenames = [] + + bucket_list = @s3.list_objects({bucket: @bucket}) + bucket_list.contents.each do |s3_object| + if /modules\/#{filename}/ =~ s3_object.key + s3_filenames << s3_object.key.gsub(/^modules\//, '') + end + end + + s3_filenames.each do |exact_filename| + path = File.join(@cache_dir, exact_filename[0].downcase, exact_filename) + target_file = File.open(path, "w") + @s3.get_object({bucket: @bucket, key: "modules/#{exact_filename}", response_target: target_file}) + modules << new_module(path, options, exact_filename) + end + + modules.reject{|i| i.nil?} + end + + def new_module(path, options, exact_filename) + metadata = read_metadata(path) + if metadata + thismodule = PuppetForgeServer::Models::Module.new({ + :metadata => parse_dependencies(PuppetForgeServer::Models::Metadata.new(normalize_metadata(metadata))), + :checksum => options[:with_checksum] ? Digest::MD5.file(path).hexdigest : nil, + :path => exact_filename, + :private => ! @readonly + }) + else + @log.error "Failed reading metadata from #{path}" + thismodule = nil + end + thismodule + end + + def get_metadata(author, name, options = {}) + version = options[:version] ? options[:version] : '.*' + get_modules("#{author}-#{name}-#{version}.tar.gz", options) + end + + def query_metadata(query, options = {}) + get_modules("*#{query}*.tar.gz", options) + end + + def parse_dependencies(metadata) + metadata.dependencies = metadata.dependencies.dup.map do |dependency| + PuppetForgeServer::Models::Dependency.new({:name => dependency['name'], :version_requirement => dependency.key?('version_requirement') ? dependency['version_requirement'] : nil}) + end.flatten + metadata + end + + def normalize_metadata(metadata) + metadata['name'] = metadata['name'].gsub('/', '-') + metadata + end + + def get_file_buffer(relative_path) + file_name = relative_path.split('/').last + File.join(@cache_dir, file_name[0].downcase, file_name) + path = Dir["#{@cache_dir}/**/#{file_name}"].first + unless File.exist?("#{path}") + buffer = download("#{@file_path}/#{relative_path}") + File.open(File.join(@cache_dir, file_name[0].downcase, file_name), 'wb') do |file| + file.write(buffer.read) + end + path = File.join(@cache_dir, file_name[0].downcase, file_name) + end + File.open(path, 'rb') + rescue => e + @log.error("#{self.class.name} failed downloading file '#{relative_path}'") + @log.error("Error: #{e}") + return nil + end + + private + def read_metadata(archive_path) + metadata_file = read_from_archive(archive_path, %r[[^/]+/metadata\.json$]) + JSON.parse(metadata_file) + rescue => error + warn "Error reading from module archive #{archive_path}: #{error}" + return nil + end + + end +end diff --git a/lib/puppet_forge_server/server.rb b/lib/puppet_forge_server/server.rb index 1baa383..591e12e 100644 --- a/lib/puppet_forge_server/server.rb +++ b/lib/puppet_forge_server/server.rb @@ -100,6 +100,13 @@ def backends(options) when 'Proxy' @log.info "Detecting API version for #{url}..." PuppetForgeServer::Backends.const_get("#{type}V#{get_api_version(url)}").new(url.chomp('/'), options[:cache_basedir]) + when 'S3' + @log.info "S3 backend selected..." + PuppetForgeServer::Backends.const_get('S3').new(options[:backend]['S3'][0], options[:cache_basedir], Aws::S3::Client.new( + region: options[:aws_region], + access_key_id: options[:aws_key_id], + secret_access_key: options[:aws_secret_key] + )) else PuppetForgeServer::Backends.const_get(type).new(url) end