autoencoding with rchardet

Author: Fivell

README.md

@@ -1,6 +1,6 @@
 # ActiveAdminImport 
-The most fastest and efficient CSV import for Active Admin (based on activerecord-import gem) 
-with support of validations and bulk inserts 
+The most fastest and efficient CSV import for Active Admin
+with support of validations, bulk inserts and encodings handling
 
 
 
@@ -35,32 +35,16 @@ And then execute:
     $ bundle
 
 
-
-#Why yet another import for ActiveAdmin ? Now with activerecord-import ....
-
- <p>Because plain-vanilla, out-of-the-box ActiveRecord doesn’t provide support for inserting large amounts of data efficiently</p>
-
-Features of activerecord-import
-
-<ol>
-  <li>activerecord-import can perform validations (fast)</li>
-  <li>activerecord-import can perform on duplicate key updates (requires mysql)</li>
-</ol>
-
-    
-    
-
-
 # active_admin_import features
 <ol>
-  <li>Encoding handling</li>
-  <li>Preview before importing (Example 2)</li>
+  <li> Replacements (Ex 2)</li>
+  <li> Encoding handling (Ex 4, 5)</li>
   <li> CSV options</li>
   <li> Ability to prepend CSV headers automatically</li>
-  <li>Bulk import (activerecord-import)</li>
-  <li>Callbacks</li>
-  <li>Zip files</li>
-  <li>more...</li>
+  <li> Bulk import (activerecord-import)</li>
+  <li> Callbacks</li>
+  <li> Zip files</li>
+  <li> more...</li>
 </ol>
 
 
@@ -151,22 +135,33 @@ Features of activerecord-import
     end
 ```
 
-#### Example4 Importing without forcing to UTF-8 and disallow archives
+#### Example4 Importing  ISO-8859-1 encoded file and disallow archives
 
 
 ```ruby
     ActiveAdmin.register Post  do
         active_admin_import validate: true,
             template_object: ActiveAdminImport::Model.new(
-                hint: "file will be encoded to ISO-8859-1",
+                hint: "file encoded in ISO-8859-1",
                 force_encoding: "ISO-8859-1",
                 allow_archive: false
             )
     end
 ```
 
+#### Example5 Importing file with unknown encoding and autodetect it
+
 
-#### Example5 Callbacks for each bulk insert iteration
+```ruby
+    ActiveAdmin.register Post  do
+        active_admin_import validate: true,
+            template_object: ActiveAdminImport::Model.new(
+                force_encoding: :auto
+            )
+    end
+```
+
+#### Example6 Callbacks for each bulk insert iteration
 
 
 ```ruby
@@ -187,7 +182,7 @@ Features of activerecord-import
     end
 ```    
 
-#### Example6 dynamic CSV options, template overriding
+#### Example7 dynamic CSV options, template overriding
 
  -  put overrided template to ```app/views/import.html.erb```
 
@@ -228,10 +223,18 @@ Features of activerecord-import
     end                      
 ```
 
-#Links
-https://github.com/gregbell/active_admin
+## Dependencies
+
+Tool                  | Description
+--------------------- | -----------
+[rchardet]            | Character encoding auto-detection in Ruby. As smart as your browser. Open source.
+[activerecord-import] | Powerful library for bulk inserting data using ActiveRecord.
+
+[rchardet]: https://github.com/jmhodges/rchardet
+[activerecord-import]: https://github.com/jmhodges/rchardet
+
+
 
-https://github.com/zdennis/activerecord-import
 
 
 

active_admin_import.gemspec

@@ -17,6 +17,7 @@ Gem::Specification.new do |gem|
 
 
   gem.add_runtime_dependency 'activerecord-import', '~> 0.7.0'
+  gem.add_runtime_dependency 'rchardet', '~> 1.5.0'
 
   gem.add_runtime_dependency 'rubyzip', '~> 1.0', '>= 1.0.0'
   gem.add_dependency "rails", ">= 4.0"

lib/active_admin_import/model.rb

@@ -1,5 +1,7 @@
 # encoding: utf-8
 
+require 'rchardet'
+
 module ActiveAdminImport
   class Model
 
@@ -132,7 +134,7 @@ def define_methods_for(attr_name)
     end
 
     def encode(data)
-      data = data.force_encoding(force_encoding) if force_encoding?
+      data = content_encode(data) if force_encoding?
       data = data.encode('UTF-8',
                          invalid: :replace, undef: :replace)
       begin
@@ -142,6 +144,23 @@ def encode(data)
       end
     end
 
+    def detect_encoding?
+      force_encoding == :auto
+    end
+
+    def dynamic_encoding(data)
+      CharDet.detect(data)['encoding']
+    end
+
+    def content_encode(data)
+      encoding_name = if detect_encoding?
+                        dynamic_encoding(data)
+                      else
+                        force_encoding.to_s
+                      end
+      data.force_encoding(encoding_name)
+    end
+
     class <<self
       def define_set_method(attr_name)
         define_method(attr_name) { self.attributes[attr_name] } unless method_defined? attr_name

spec/import_spec.rb

@@ -2,6 +2,28 @@
 
 describe 'import', type: :feature do
 
+  shared_examples 'successful inserts' do |encoding, csv_file_name|
+    let(:options) do
+      attributes = { force_encoding: encoding }
+      { template_object: ActiveAdminImport::Model.new(attributes) }
+    end
+
+    before do
+      upload_file!(csv_file_name)
+    end
+
+    it "should import file with many records" do
+      expect(page).to have_content "Successfully imported 2 authors"
+      expect(Author.count).to eq(2)
+      Author.all.each do |author|
+        expect(author).to be_valid
+        expect(author.name).to be_present
+        expect(author.last_name).to be_present
+      end
+    end
+
+  end
+
   def with_zipped_csv(name, &block)
 
     zip_file = File.expand_path("./spec/fixtures/files/#{name}.zip")
@@ -25,7 +47,6 @@ def upload_file!(name, ext='csv')
   context "authors index" do
     before do
       add_author_resource
-
     end
 
     it "should navigate to import page" do
@@ -81,16 +102,14 @@ def upload_file!(name, ext='csv')
     end
 
     context "with hint defined" do
-      let(:options) {
-        {template_object: ActiveAdminImport::Model.new(hint: "hint")}
-      }
+      let(:options) do
+        { template_object: ActiveAdminImport::Model.new(hint: "hint") }
+      end
       it "renders hint at upload page" do
         expect(page).to have_content options[:template_object].hint
       end
-
     end
 
-
     context "when importing file" do
 
       [:empty, :only_headers].each do |file|
@@ -111,26 +130,16 @@ def upload_file!(name, ext='csv')
         end
       end
 
-      context "Win1251" do
-        let(:options) do
-          attributes = { force_encoding: "Windows-1251" }
-          { template_object: ActiveAdminImport::Model.new(attributes) }
-        end
-
-        before do
-          upload_file!(:authors_win1251_win_endline)
-        end
-
-        it "should import file with many records" do
-          expect(page).to have_content "Successfully imported 2 authors"
-          expect(Author.count).to eq(2)
-          Author.all.each do |author|
-            expect(author).to be_valid
-            expect(author.name).to be_present
-            expect(author.last_name).to be_present
-          end
-        end
+      context "auto detect encoding" do
+        include_examples 'successful inserts',
+                         :auto,
+                         :authors_win1251_win_endline
+      end
 
+      context "Win1251" do
+        include_examples 'successful inserts',
+                         'windows-1251',
+                         :authors_win1251_win_endline
       end
 
       context "BOM" do
@@ -205,7 +214,7 @@ def upload_file!(name, ext='csv')
 
 
         context "without validation" do
-          let(:options) { {validate: false} }
+          let(:options) { { validate: false } }
           it "should render error" do
             upload_file!(:author_invalid)
             expect(page).to have_content "Successfully imported 1 author"
@@ -248,7 +257,7 @@ def upload_file!(name, ext='csv')
       context "with different header attribute names" do
 
         let(:options) do
-          { headers_rewrites: { :'Second name' => :last_name } }
+          { headers_rewrites: {:'Second name' => :last_name} }
         end
 
         it "should import file" do
@@ -299,7 +308,7 @@ def upload_file!(name, ext='csv')
   end
 
   context "with invalid options" do
-    let(:options) { {invalid_option: :invalid_value} }
+    let(:options) { { invalid_option: :invalid_value } }
 
     it "should raise TypeError" do
       expect { add_author_resource(options) }.to raise_error(ArgumentError)