Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
68 changes: 66 additions & 2 deletions Rakefile
Original file line number Diff line number Diff line change
Expand Up @@ -23,14 +23,16 @@ namespace 'project' do
Rake.application.invoke_task("project:upload_to_web[#{project_name}]")
Rake.application.invoke_task("project:remove_database[#{project_name}]")
Rake.application.invoke_task("project:remove_dump_from_tmp[#{project_name}]")
Rake.application.invoke_task("project:anonymize_database_with_scenerio[#{project_name},default,'']")
Rake.application.invoke_task("project:dump_database_with_scenerio[#{project_name},default,'']")
end

task :restore_database, [:project_name] do |_t, args|
project_name = args[:project_name]
database = {
host: CONFIG['database']['host'],
user: CONFIG['database']['user'],
pass: CONFIG['database']['pass']
password: CONFIG['database']['pass']
}

system(
Expand All @@ -43,7 +45,7 @@ namespace 'project' do
ShellHelper.restore_database(
project_name,
database,
CONFIG['tmp_path']
CONFIG['dump_server']['path']
)
)
end
Expand Down Expand Up @@ -75,6 +77,68 @@ namespace 'project' do
)
end

task :anonymize_database_with_scenerio, [:project_name, :scenerio, :params] do |_t, args|
project_name = args[:project_name]
scenerio = args[:scenerio]
params = args[:params]

anonymizer = Anonymizer.new project_name, nil, scenerio, params

db = Database.new anonymizer.config
db.anonymize
end

task :dump_database_with_scenerio, [:project_name, :scenerio, :params] do |_t, args|
project_name = args[:project_name]
scenerio = args[:scenerio]
params = args[:params]

anonymizer = Anonymizer.new project_name, nil, scenerio, params

db = Database.new anonymizer.config

unless anonymizer.config['dump_actions'].nil?

database = {
host: CONFIG['database']['host'],
user: CONFIG['database']['user'],
password: CONFIG['database']['pass']
}

file_name = anonymizer.config['dump_actions']['scenerios'][anonymizer.config['scenerio']]['file']
dump_file = anonymizer.config['dump_actions']['path'] + file_name
File.delete(dump_file) if File.exist?(dump_file)

if anonymizer.config['dump_actions']['scenerios'][scenerio]['tables'] == '*'
system(
ShellHelper.output_query_result(
project_name,
nil,
nil,
database,
dump_file
)
)
else
!anonymizer.config['dump_actions']['scenerios'][scenerio]['tables'].each do |table, condition|
where = if !condition['where'].nil? && !condition['where'].empty?
db.merge_query_with_params(condition['where'])
end

system(
ShellHelper.output_query_result(
project_name,
table,
where,
database,
dump_file
)
)
end
end
end
end

task :remove_database, [:project_name] do |_t, args|
project_name = args[:project_name]
database = {
Expand Down
13 changes: 13 additions & 0 deletions lib/anonymizer/helper/shell_helper.rb
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,19 @@ def self.dump_database(project_name, database, tmp_dir)
remove_white_space(command)
end

def self.output_query_result(project_name, table, where, database, file)
params = ' --skip-opt --no-create-info --compact --single-transaction'
cmd = if table.nil?
"mysqldump #{mysql_options(database)} #{project_name} #{params} >> #{file}"
elsif !where.nil?
"mysqldump #{mysql_options(database)} #{project_name} #{table} --where='#{where}' #{params} >> #{file}"
else
"mysqldump #{mysql_options(database)} #{project_name} #{table} --skip-opt #{params} >> #{file}"
end

remove_white_space(cmd)
end

def self.upload_to_web(project_name, database, web_server, tmp_dir, options = '')
random_string = "_#{database[:random_string]}" if database[:random_string]
command = "rsync -a #{ssh_option(web_server[:port])} #{options} " \
Expand Down
54 changes: 20 additions & 34 deletions lib/anonymizer/model/anonymizer.rb
Original file line number Diff line number Diff line change
Expand Up @@ -4,17 +4,17 @@
class Anonymizer
attr_accessor :config

def initialize(project_name, config = nil)
raise 'Invalid project name' unless project_name && project_name.is_a?(String)
def initialize(project_name, config = nil, scenerio = 'default', params = [])
@project_name = project_name

@config = config
@validator = Validator.new project_name, config, scenerio, params
@validator.validate_project_name
if config.nil?
project_file_path = project_file_path project_file_name project_name
raise 'Project not exists' unless project_file_path
@validator.validate_project_file project_file_path
config = read_config project_file_path
end

@config = prepare_config config
@config = prepare_config(config, scenerio, params)
end

def work(database)
Expand All @@ -23,43 +23,29 @@ def work(database)

private

def prepare_config(config)
# rubocop:disable Style/IfUnlessModifier
def prepare_config(config, scenerio, params)
if config['type'] == 'extended'
project_file_path = project_file_path project_file_name config['basic_type']
raise 'Basic type not exists' unless project_file_path
basic_config = read_config project_file_path
config = basic_config.deep_merge config
config = prepare_config_extended(config, scenerio, params)
end

validate_config config

@validator.validate_config config
config['database'] = { 'name' => @project_name }
@validator.config_overwrite config

config
end
# rubocop:enable Style/IfUnlessModifier

def validate_config(config)
if !config.key?('dump_server') || config['dump_server'].empty?
raise 'In project config file dump_server is not valid'
end

validate_dump_server config['dump_server']

config['tables'].each do |_table_name, columns|
columns.each do |column_name, info|
validate_column(column_name, info)
end
end
end

def validate_dump_server(dump_server)
%w[host user port path].each do |variable|
raise "In project config file dump_server #{variable} is not valid" unless dump_server.key?(variable)
end
end
def prepare_config_extended(config, scenerio, params)
project_file_path = project_file_path project_file_name config['basic_type']
@validator.validate_type(project_file_path)
basic_config = read_config project_file_path
config = basic_config.deep_merge config if config['merge_protect'].nil?
config['scenerio'] = scenerio
config['params'] = params

def validate_column(_column_name, info)
raise 'In project config file founded column without defined action' unless info['action']
config
end

def read_config(project_file_path)
Expand Down
92 changes: 70 additions & 22 deletions lib/anonymizer/model/database.rb
Original file line number Diff line number Diff line change
@@ -1,9 +1,13 @@
# frozen_string_literal: true

# rubocop:disable Metrics/ClassLength

# Basic class to communication with databese
class Database
attr_accessor :config, :name

# rubocop:enable Metrics/ClassLength

def initialize(config)
@config = config
@db = Sequel.mysql2(
Expand All @@ -14,6 +18,10 @@ def initialize(config)
)
end

# rubocop:disable Metrics/PerceivedComplexity
# rubocop:disable Metrics/MethodLength
# rubocop:disable Metrics/CyclomaticComplexity
# rubocop:disable Metrics/AbcSize
def anonymize
insert_fake_data

Expand All @@ -23,6 +31,7 @@ def anonymize
queries = column_query(table_name, columns)

queries.each do |query|
query = merge_query_with_params(query) if !@config['params'].nil? && !@config['params'].empty?
@db.run query
end
end
Expand All @@ -32,22 +41,75 @@ def anonymize
remove_fake_data
end

def scenerio_determine(info)
scenerio = if @config['scenerio'].nil? || info[@config['scenerio']].nil?
'default'
elsif @config['scenerio'] &&
!info[@config['scenerio']]['extended'].nil? &&
info[info[@config['scenerio']]['extended']].nil?
'default'
elsif @config['scenerio'] &&
!info[@config['scenerio']]['extended'].nil? &&
!info[info[@config['scenerio']]['extended']].nil?
info[@config['scenerio']]['extended']
else
@config['scenerio']
end

scenerio
end
# rubocop:enable Metrics/PerceivedComplexity
# rubocop:enable Metrics/MethodLength
# rubocop:enable Metrics/CyclomaticComplexity
# rubocop:enable Metrics/AbcSize

def column_query(table_name, columns)
queries = []

columns.each do |column_name, info|
Object.const_get(
"Database::#{translate_acton_to_class_name(info['action'])}"
).query(table_name, column_name, info).each do |query|
queries.push query
end
column, action = get_column_and_action(info)
arr = column_queries(table_name, column_name, column, action)
queries.concat arr

break if action == 'truncate'
end

queries
end

def column_queries(table_name, column_name, column, action)
queries = []

break if info['action'] == 'truncate'
Object.const_get(
"Database::#{translate_acton_to_class_name(action)}"
).query(table_name, column_name, column).each do |query|
queries.push query
end

queries
end

def get_column_and_action(info)
if config['version'].nil? || config['version'] < 2
column = info
action = info['action']
else
scenerio = scenerio_determine(info)
column = info[scenerio]
action = info[scenerio]['action']
end
[column, action]
end

def merge_query_with_params(query)
params = @config['params'].split(';')
params.each do |param, _value|
param = param.split(':')
query = query.gsub('%' + param[0] + '%', param[1].tr('|', ','))
end
query
end

def before_queries
if @config['custom_queries'] &&
@config['custom_queries']['before'] &&
Expand Down Expand Up @@ -84,26 +146,12 @@ def remove_fake_data
@db.drop_table :fake_user
end

# rubocop:disable Metrics/MethodLength
def translate_acton_to_class_name(action)
case action
when 'truncate'
class_name = 'Truncate'
when 'empty', 'set_static'
class_name = 'Static'
when 'eav_update'
class_name = 'Eav'
when 'json_update'
class_name = 'Json'
when 'multiple_update'
class_name = 'Multiple'
else
class_name = 'Column'
end
model = Model.new action
class_name = model.action_class

class_name
end
# rubocop:enable Metrics/MethodLength

def self.prepare_select_for_query(type)
query = if type == 'fullname'
Expand Down
15 changes: 15 additions & 0 deletions lib/anonymizer/model/database/delete.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
# frozen_string_literal: true

# Basic class to communication with databese
class Database
# Class to remove records
class Delete
def self.query(table_name, _column_name, info)
where = info['attributes']['where']
queries = []
query = "DELETE FROM #{table_name} WHERE #{where};"
queries.push query
queries
end
end
end
Loading