Partial data import

.gitignore

@@ -4,4 +4,5 @@ Gemfile.lock
 pkg/*
 reports/*
 bin/*
-*.log
+*.log
+.import_settings

lib/data-import.rb

@@ -5,9 +5,14 @@
 
 require "data-import/version"
 require "data-import/errors"
+require 'data-import/dictionary'
 require 'data-import/dependency_resolver'
 require 'data-import/execution_context'
 require 'data-import/runner'
+require 'data-import/migration_strategy'
+require 'data-import/full_migration'
+require 'data-import/partial_migration'
+require 'data-import/settings_store'
 require 'data-import/execution_plan'
 require 'data-import/dsl'
 require 'data-import/sequel/dataset'

lib/data-import/definition.rb

@@ -1,4 +1,4 @@
-require 'data-import/definition/lookup'
+require 'data-import/definition/id_mapping_container'
 require 'data-import/definition/mappings'
 require 'data-import/definition/simple'
 require 'data-import/definition/simple/importer'
@@ -10,13 +10,53 @@ class Definition
     attr_reader :source_database, :target_database
     attr_reader :dependencies
 
-    def initialize(name, source_database, target_database)
+    def initialize(name, source_database, target_database, id_mapping_container)
       @name = name
       @source_database = source_database
       @target_database = target_database
       @dependencies = []
+      @id_mapping_container = id_mapping_container
     end
 
+    def lookup_for(mapping_name, options = {})
+      warn "[DEPRECATION] `definition(...).lookup_for(...)` is deprecated and will be removed in later versions! Use `lookup_for(...)` in import and script blocks instead instead.\n#{caller[0]}"
+
+      attribute = options.fetch(:column) { mapping_name }
+
+      if @id_mapping_container.has_dictionary_for?(name, mapping_name)
+        raise ArgumentError, "lookup-table for column '#{attribute}' was already defined"
+      else
+        d = if options.fetch(:ignore_case) { false }
+              CaseIgnoringDictionary.new
+            else
+              Dictionary.new
+            end
+        @id_mapping_container.add_dictionary(name, mapping_name, attribute, d)
+      end
+    end
+
+    def row_imported(id, row)
+      warn "[DEPRECATION] `definition(...).row_imported(...)` is deprecated and will be removed in later versions! Use `id_mapping_for(...).add(...)` instead.\n#{caller[0]}"
+
+      @id_mapping_container.update_dictionaries(name, id, row)
+    end
+
+    def identify_by(mapping_name, value)
+      warn "[DEPRECATION] `definition(...).identify_by(...)` is deprecated and will be removed in later versions! Use `id_mapping_for(...).lookup(...)` instead.\n#{caller[0]}"
+
+      return if value.blank?
+      if @id_mapping_container.has_dictionary_for?(name, mapping_name)
+        @id_mapping_container.fetch(name, mapping_name).lookup(value)
+      else
+        raise ArgumentError, "no lookup-table defined named '#{name}'"
+      end
+    end
+
+    def has_lookup_table_on?(name)
+      @lookup_tables.has_key? name
+    end
+    private :has_lookup_table_on?
+
     def add_dependency(dependency)
       @dependencies << dependency
     end

lib/data-import/definition/id_mapping_container.rb

@@ -0,0 +1,88 @@
+module DataImport
+  class Definition
+    class IdMappingContainer
+      def initialize
+        @mapping_configs = Hash.new {|hash, key| hash[key] = []}
+      end
+
+      def add_dictionary(definition_name, mapping_name, attribute, dictionary)
+        @mapping_configs[definition_name] << IdMappingConfig.new(mapping_name, attribute, dictionary)
+      end
+
+      def fetch(definition_name, mapping_name)
+        if has_dictionary_for?(definition_name, mapping_name)
+          fetch_config(definition_name, mapping_name).dictionary
+        else
+          raise MissingIdMappingError.new(mapping_name)
+        end
+      end
+
+      def fetch_config(definition_name, mapping_name)
+        @mapping_configs[definition_name].detect {|config| config.name == mapping_name}
+      end
+      private :fetch_config
+
+      def has_dictionary_for?(definition_name, mapping_name)
+        @mapping_configs[definition_name].any? {|config| config.name == mapping_name}
+      end
+
+      def update_dictionaries(definition_name, new_id, row)
+        @mapping_configs[definition_name].each do |config|
+          next if row[config.attribute].blank?
+          config.dictionary.add(row[config.attribute], new_id)
+        end
+      end
+
+      def clear
+        @mapping_configs.each do |_name, configs|
+          configs.each do |config|
+            config.clear
+          end
+        end
+      end
+
+      def to_hash
+        @mapping_configs.each_with_object({}) do |(definition_name, configs), result|
+          result[definition_name] = configs.map(&:to_hash)
+        end
+      end
+
+      def load(mapping_data)
+        clear
+
+        mapping_data.each do |definition_name, configs|
+          configs.each do |config_data|
+            config = fetch_config(definition_name, config_data[:name])
+            if config.present? && config.attribute == config_data[:attribute]
+              config.load(config_data[:mappings])
+            end
+          end
+        end
+      end
+
+      class IdMappingConfig
+        attr_reader :name, :attribute, :dictionary
+
+        def initialize(name, attribute, dictionary)
+          @name = name
+          @attribute = attribute.to_sym
+          @dictionary = dictionary
+        end
+
+        def clear
+          dictionary.clear
+        end
+
+        def to_hash
+          {:name => name, :attribute => attribute, :mappings => dictionary.to_hash}
+        end
+
+        def load(mappings)
+          mappings.each do |key, value|
+            dictionary.add(key, value)
+          end
+        end
+      end
+    end
+  end
+end

lib/data-import/definition/mappings.rb

@@ -49,7 +49,7 @@ def initialize(referenced_definition, old_foreign_key, new_foreign_key, lookup_n
       end
 
       def apply!(definition, context, row, output_row)
-        output_row.merge!(@new_foreign_key => context.definition(@referenced_definition).identify_by(@lookup_name, row[@old_foreign_key]))
+        output_row.merge!(@new_foreign_key => context.id_mapping_for(@referenced_definition, @lookup_name).lookup(row[@old_foreign_key]))
       end
     end
 

lib/data-import/definition/script.rb

@@ -3,8 +3,6 @@ class Definition
     class Script < Definition
       attr_accessor :body
 
-      include Lookup
-
       def run(context)
         target_database.transaction do
           context.instance_exec &body

lib/data-import/definition/simple.rb

@@ -2,14 +2,12 @@ module DataImport
   class Definition
     class Simple < Definition
 
-      include Lookup
-
       attr_accessor :target_table_name
       attr_accessor :reader, :writer
       attr_accessor :after_blocks, :after_row_blocks
       attr_accessor :row_validation_blocks
 
-      def initialize(name, source_database, target_database)
+      def initialize(name, source_database, target_database, id_mapping_container)
         super
         @mappings = []
         @mode = :insert

lib/data-import/definition/simple/importer.rb

@@ -36,7 +36,7 @@ def import_row(row)
 
           if row_valid?(row_context)
             new_id = @definition.writer.write_row(mapped_row)
-            @definition.row_imported(new_id, row)
+            @context.id_mapping_container.update_dictionaries(@definition.name, new_id, row)
 
             @definition.after_row_blocks.each do |block|
               row_context.instance_exec(row_context, row, mapped_row, &block)

lib/data-import/dependency_resolver.rb

@@ -10,7 +10,7 @@ def resolve(options = {})
       resolved_dependencies = @strategy.new(dependency_graph).call(options)
       resolved_dependencies = resolved_dependencies.map {|definition| @plan.definition(definition) }
 
-      ExecutionPlan.new(resolved_dependencies)
+      ExecutionPlan.new(resolved_dependencies, @plan.id_mapping_container)
     end
 
     def dependency_graph

lib/data-import/dictionary.rb

@@ -0,0 +1,37 @@
+module DataImport
+  class Dictionary
+    def initialize
+      @mappings = {}
+    end
+
+    def add(key, value)
+      @mappings[key] = value
+    end
+
+    def lookup(key)
+      @mappings[key]
+    end
+
+    def to_hash
+      @mappings
+    end
+
+    def empty?
+      @mappings.empty?
+    end
+
+    def clear
+      @mappings.clear
+    end
+  end
+
+  class CaseIgnoringDictionary < Dictionary
+    def add(key, value)
+      super(key.nil? ? nil : key.downcase, value)
+    end
+
+    def lookup(key)
+      super(key.nil? ? nil : key.downcase)
+    end
+  end
+end

lib/data-import/dsl.rb

@@ -49,17 +49,17 @@ def target(*args)
     end
 
     def import(name, &block)
-      definition = DataImport::Definition::Simple.new(name, source_database, target_database)
+      definition = DataImport::Definition::Simple.new(name, source_database, target_database, @plan.id_mapping_container)
       @plan.add_definition(definition)
 
-      Import.new(definition).instance_eval &block
+      Import.new(definition, @plan.id_mapping_container).instance_eval &block
     end
 
     def script(name, &block)
-      definition = DataImport::Definition::Script.new(name, source_database, target_database)
+      definition = DataImport::Definition::Script.new(name, source_database, target_database, @plan.id_mapping_container)
       @plan.add_definition(definition)
 
-      Script.new(definition).instance_eval &block
+      Script.new(definition, @plan.id_mapping_container).instance_eval &block
     end
 
     def before_filter(&block)