[zero] create core classes and initial measure

Author: Nikita

.rspec

@@ -0,0 +1 @@
+--require spec_helper

Gemfile

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+source "https://rubygems.org"
+ 
+
+gem 'stackprof'
+gem 'ruby-prof'
+gem 'memory_profiler'
+gem 'rspec'
+gem 'byebug'

Gemfile.lock

@@ -0,0 +1,35 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    byebug (11.1.3)
+    diff-lcs (1.6.0)
+    memory_profiler (1.1.0)
+    rspec (3.13.0)
+      rspec-core (~> 3.13.0)
+      rspec-expectations (~> 3.13.0)
+      rspec-mocks (~> 3.13.0)
+    rspec-core (3.13.3)
+      rspec-support (~> 3.13.0)
+    rspec-expectations (3.13.3)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.13.0)
+    rspec-mocks (3.13.2)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.13.0)
+    rspec-support (3.13.2)
+    ruby-prof (1.7.1)
+    stackprof (0.2.27)
+
+PLATFORMS
+  ruby
+  x86_64-linux
+
+DEPENDENCIES
+  byebug
+  memory_profiler
+  rspec
+  ruby-prof
+  stackprof
+
+BUNDLED WITH
+   2.5.18

main.rb

@@ -0,0 +1,147 @@
+class Main
+  PARTIAL_VOLUME_FILE_NAME = "dataN.txt"
+
+  def initialize(options: {})
+    @source_file_name = options[:source_file_name] || 'data_large.txt'
+    @count_lines = options[:count_lines]
+    @is_print_memory_usage = options[:is_print_memory_usage] || false
+  end
+
+  def call = work
+
+  private
+
+  attr_reader :source_file_name, :count_lines, :is_print_memory_usage
+
+  def work
+    `head -n #{count_lines} #{source_file_name} > #{PARTIAL_VOLUME_FILE_NAME}` if count_lines
+    file_lines = File.read(count_lines ? PARTIAL_VOLUME_FILE_NAME : source_file_name).split("\n")
+
+    users = []
+    sessions = []
+  
+    file_lines.each do |line|
+      cols = line.split(',')
+      users = users + [parse_user(line)] if cols[0] == 'user'
+      sessions = sessions + [parse_session(line)] if cols[0] == 'session'
+    end
+  
+    # Отчёт в json
+    #   - Сколько всего юзеров +
+    #   - Сколько всего уникальных браузеров +
+    #   - Сколько всего сессий +
+    #   - Перечислить уникальные браузеры в алфавитном порядке через запятую и капсом +
+    #
+    #   - По каждому пользователю
+    #     - сколько всего сессий +
+    #     - сколько всего времени +
+    #     - самая длинная сессия +
+    #     - браузеры через запятую +
+    #     - Хоть раз использовал IE? +
+    #     - Всегда использовал только Хром? +
+    #     - даты сессий в порядке убывания через запятую +
+  
+    report = {}
+  
+    report[:totalUsers] = users.count
+  
+    # Подсчёт количества уникальных браузеров
+    uniqueBrowsers = []
+    sessions.each do |session|
+      browser = session['browser']
+      uniqueBrowsers += [browser] if uniqueBrowsers.all? { |b| b != browser }
+    end
+  
+    report['uniqueBrowsersCount'] = uniqueBrowsers.count
+  
+    report['totalSessions'] = sessions.count
+  
+    report['allBrowsers'] =
+      sessions
+        .map { |s| s['browser'] }
+        .map { |b| b.upcase }
+        .sort
+        .uniq
+        .join(',')
+  
+    # Статистика по пользователям
+    users_objects = []
+  
+    users.each do |user|
+      attributes = user
+      user_sessions = sessions.select { |session| session['user_id'] == user['id'] }
+      user_object = User.new(attributes: attributes, sessions: user_sessions)
+      users_objects = users_objects + [user_object]
+    end
+  
+    report['usersStats'] = {}
+  
+    # Собираем количество сессий по пользователям
+    collect_stats_from_users(report, users_objects) do |user|
+      { 'sessionsCount' => user.sessions.count }
+    end
+  
+    # Собираем количество времени по пользователям
+    collect_stats_from_users(report, users_objects) do |user|
+      { 'totalTime' => user.sessions.map {|s| s['time']}.map {|t| t.to_i}.sum.to_s + ' min.' }
+    end
+  
+    # Выбираем самую длинную сессию пользователя
+    collect_stats_from_users(report, users_objects) do |user|
+      { 'longestSession' => user.sessions.map {|s| s['time']}.map {|t| t.to_i}.max.to_s + ' min.' }
+    end
+  
+    # Браузеры пользователя через запятую
+    collect_stats_from_users(report, users_objects) do |user|
+      { 'browsers' => user.sessions.map {|s| s['browser']}.map {|b| b.upcase}.sort.join(', ') }
+    end
+  
+    # Хоть раз использовал IE?
+    collect_stats_from_users(report, users_objects) do |user|
+      { 'usedIE' => user.sessions.map{|s| s['browser']}.any? { |b| b.upcase =~ /INTERNET EXPLORER/ } }
+    end
+  
+    # Всегда использовал только Chrome?
+    collect_stats_from_users(report, users_objects) do |user|
+      { 'alwaysUsedChrome' => user.sessions.map{|s| s['browser']}.all? { |b| b.upcase =~ /CHROME/ } }
+    end
+  
+    # Даты сессий через запятую в обратном порядке в формате iso8601
+    collect_stats_from_users(report, users_objects) do |user|
+      { 'dates' => user.sessions.map{|s| s['date']}.map {|d| Date.parse(d)}.sort.reverse.map { |d| d.iso8601 } }
+    end
+  
+    File.write('result.json', "#{report.to_json}\n")
+
+    puts "MEMORY USAGE: #{(`ps -o rss= -p #{Process.pid}`.to_i / 1024)} MB" if is_print_memory_usage
+  end
+
+  def parse_user(user)
+    fields = user.split(',')
+    parsed_result = {
+      'id' => fields[1],
+      'first_name' => fields[2],
+      'last_name' => fields[3],
+      'age' => fields[4],
+    }
+  end
+  
+  def parse_session(session)
+    fields = session.split(',')
+    parsed_result = {
+      'user_id' => fields[1],
+      'session_id' => fields[2],
+      'browser' => fields[3],
+      'time' => fields[4],
+      'date' => fields[5],
+    }
+  end
+  
+  def collect_stats_from_users(report, users_objects, &block)
+    users_objects.each do |user|
+      user_key = "#{user.attributes['first_name']}" + ' ' + "#{user.attributes['last_name']}"
+      report['usersStats'][user_key] ||= {}
+      report['usersStats'][user_key] = report['usersStats'][user_key].merge(block.call(user))
+    end
+  end
+end

measurer.rb

@@ -0,0 +1,10 @@
+require_relative 'main'
+
+class Measurer
+  def call
+    Main.new(options: { count_lines: 15_000, is_print_memory_usage: true }).call
+  end
+end
+
+# Zero iteration:
+# 15_000 lines: 103 MB

profiler.rb

@@ -0,0 +1,49 @@
+require 'memory_profiler'
+require 'stackprof'
+require 'ruby-prof'
+
+require_relative 'main'
+
+class Profiler
+  def initialize(count_lines:)
+    @count_lines = count_lines
+  end
+
+  def call
+    profile_by_memory_profiler
+    profile_memory_by_ruby_prof
+    profile_allocations_by_ruby_prof
+    profile_allocations_by_stack_prof
+  end
+
+  private
+
+  attr_reader :count_lines
+
+  def profile_by_memory_profiler
+    report = MemoryProfiler.report { action }
+    report.pretty_print(to_file: './profiles/memory_profiler')
+  end
+
+  def profile_memory_by_ruby_prof
+    RubyProf.measure_mode = RubyProf::MEMORY
+    result = RubyProf.profile { action }
+    printer = RubyProf::CallTreePrinter.new(result)
+    printer.print(:path => ".", :profile => "profiles/ruby_prof_memory_profile")
+  end
+
+  def profile_allocations_by_ruby_prof
+    RubyProf.measure_mode = RubyProf::ALLOCATIONS
+    result = RubyProf.profile { action }
+    printer = RubyProf::CallTreePrinter.new(result)
+    printer.print(:path => ".", :profile => "profiles/ruby_prof_allocations_profile")
+  end
+
+  def profile_allocations_by_stack_prof
+    StackProf.run(mode: :object, out: 'profiles/stackprof.dump') do
+      action
+    end
+  end
+
+  def action = Main.new(options: { count_lines: }).call
+end