Support for encoding of README to valid UTF-8

Author: Suszyński Krzysztof

lib/puppet_forge_server.rb

@@ -61,6 +61,7 @@ module Utils
     autoload :Buffer, 'puppet_forge_server/utils/buffer'
     autoload :Http, 'puppet_forge_server/utils/http'
     autoload :MarkdownRenderer, 'puppet_forge_server/utils/md_renderer'
+    autoload :Encoding, 'puppet_forge_server/utils/encoding'
   end
 
   module Http

lib/puppet_forge_server/api/v3/releases.rb

@@ -16,6 +16,7 @@
 
 module PuppetForgeServer::Api::V3
   module Releases
+    include PuppetForgeServer::Utils::Encoding
     def get_releases(metadata)
       metadata.map do |element|
         author, name = element.metadata.name.split('-')
@@ -36,7 +37,7 @@ def get_releases(metadata)
             :file_uri => "/v3/files#{element.path}",
             :file_md5 => element.checksum,
             :deleted_at => element.deleted_at,
-            :readme => element.readme
+            :readme => to_utf8(element.readme)
         }
       end.uniq{|r| r[:version]}.sort_by { |r| Gem::Version.new(r[:version]) }
     end

lib/puppet_forge_server/utils/encoding.rb

@@ -0,0 +1,46 @@
+# -*- encoding: utf-8 -*-
+#
+# Copyright 2015 Centralny Osroder Informatyki (gov.pl)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'iconv'
+
+module PuppetForgeServer::Utils
+  module Encoding
+
+    @@ic = Iconv.new('UTF-8//IGNORE', 'UTF-8')
+
+    # Converts give text to valid UTF-8
+    # @param [string] text given string, can be null
+    # @return [string] output string in utf-8
+    def to_utf8(text)
+      replaced = text
+      unless replaced.nil?
+        replaced = replaced.force_encoding("UTF-8") if is_ascii_8bit?(replaced)
+        replaced = cleanup_utf8(replaced)
+      end
+      replaced
+    end
+
+    private
+
+    def is_ascii_8bit?(text)
+      text.encoding.name == 'ASCII-8BIT'
+    end
+
+    def cleanup_utf8(text)
+      @@ic.iconv(text)
+    end
+  end
+end

puppet-forge-server.gemspec

@@ -46,12 +46,15 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'multipart-post', '~> 2.0.0'
   spec.add_dependency 'redcarpet', '~> 3.3.0'
   spec.add_dependency 'lrucache', '~> 0.1.4'
+  spec.add_dependency 'iconv', '~> 1.0.4'
 
   spec.add_development_dependency 'rake', '~> 10.3'
   spec.add_development_dependency 'rspec', '~> 3.1'
   spec.add_development_dependency 'rspec-core', '~> 3.1'
   spec.add_development_dependency 'simplecov', '~> 0.11.0'
   spec.add_development_dependency 'rspec-stopwatch', '~> 0.1.3'
+  spec.add_development_dependency 'pry'
+  spec.add_development_dependency 'pry-byebug'
 
   spec.required_ruby_version = '>= 1.9.3'
 end

spec/unit/utils/encoding_spec.rb

@@ -0,0 +1,34 @@
+require 'spec_helper'
+
+describe 'PuppetForgeServer::Utils::Encoding' do
+  let(:dummy_class) { Class.new { include PuppetForgeServer::Utils::Encoding } }
+  let(:dummy) { dummy_class.new }
+  describe '#to_utf8' do
+    subject { dummy.to_utf8(input_text) }
+    context 'for nil' do
+      let(:input_text) { nil }
+      it { expect(subject).to be_nil }
+    end
+
+    context 'for valid ascii text' do
+      let(:input_text) { 'a simple ascii text!' }
+      it { expect(subject).to eq('a simple ascii text!') }
+    end
+
+    context 'for valid utf-8 text' do
+      let(:input_text) { 'Zażółć gęślą jażń' }
+      it { expect(subject).to eq('Zażółć gęślą jażń') }
+    end
+
+    context 'for text with ISO-8859-1 encoded as UTF-8' do
+      let(:input_text) { "Co\xE2te d'Azur" }
+      it { expect(subject).to eq('Cote d\'Azur') }
+    end
+
+    context 'for text with UTF-8 not replaced characters' do
+      let(:multidot) { [0xE2, 0x80, 0xA6].pack("c*") }
+      let(:input_text) { "Read more#{multidot}" }
+      it { expect(subject).to eq('Read more…') }
+    end
+  end
+end