From cd1ba3f5032a338a9477bbee85b1776946ce7c9d Mon Sep 17 00:00:00 2001 From: Bill Dueber Date: Wed, 25 Sep 2024 21:27:58 -0400 Subject: [PATCH 1/3] Implement and use semantic logger to create JSON logs --- Gemfile | 3 ++- Gemfile.lock | 15 +++++------ config/application.rb | 27 +++++++------------ config/load_local_config.rb | 2 -- lib/dromedary/abbreviated_json_format.rb | 33 ++++++++++++++++++++++++ lib/med_installer/logger.rb | 7 +++-- 6 files changed, 53 insertions(+), 34 deletions(-) create mode 100644 lib/dromedary/abbreviated_json_format.rb diff --git a/Gemfile b/Gemfile index cdd2be7..0c0c102 100644 --- a/Gemfile +++ b/Gemfile @@ -112,8 +112,9 @@ gem "traject", require: false # only for indexing # Semantic logging? gem "awesome_print" gem "semantic_logger" +gem "rails_semantic_logger" -gem "lograge", ">=0.11.1" +# gem "lograge", ">=0.11.1" # Contacts Email gem "mail_form", "~>1.7" diff --git a/Gemfile.lock b/Gemfile.lock index c16b816..010698b 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -191,11 +191,6 @@ GEM llhttp-ffi (0.4.0) ffi-compiler (~> 1.0) rake (~> 13.0) - lograge (0.12.0) - actionpack (>= 4) - activesupport (>= 4) - railties (>= 4) - request_store (~> 1.0) loofah (2.19.1) crass (~> 1.0.2) nokogiri (>= 1.5.9) @@ -274,6 +269,10 @@ GEM nokogiri (>= 1.6) rails-html-sanitizer (1.5.0) loofah (~> 2.19, >= 2.19.1) + rails_semantic_logger (4.17.0) + rack + railties (>= 5.1) + semantic_logger (~> 4.16) railties (5.2.8.1) actionpack (= 5.2.8.1) activesupport (= 5.2.8.1) @@ -291,8 +290,6 @@ GEM declarative (< 0.1.0) trailblazer-option (>= 0.1.1, < 0.2.0) uber (< 0.2.0) - request_store (1.5.1) - rack (>= 1.4) rexml (3.2.5) roda (3.83.0) rack @@ -355,7 +352,7 @@ GEM rexml (~> 3.2, >= 3.2.5) rubyzip (>= 1.2.2, < 3.0) websocket (~> 1.0) - semantic_logger (4.12.0) + semantic_logger (4.16.1) concurrent-ruby (~> 1.0) shrine (3.6.0) content_disposition (~> 1.0) @@ -464,7 +461,6 @@ DEPENDENCIES jquery-rails kaminari (>= 1.2.1) listen - lograge (>= 0.11.1) loofah (>= 2.3.1) mail_form (~> 1.7) middle_english_dictionary! @@ -480,6 +476,7 @@ DEPENDENCIES puma (>= 4.3.5) rack (>= 2.1.4) rails (~> 5.0) + rails_semantic_logger rake (~> 13.0) rsolr (>= 1.0) rspec-rails (~> 3.6) diff --git a/config/application.rb b/config/application.rb index 5c5a46b..157f777 100644 --- a/config/application.rb +++ b/config/application.rb @@ -2,6 +2,7 @@ require "rails/all" require "json" require_relative "../lib/dromedary/services" +require_relative "../lib/dromedary/abbreviated_json_format" # Require the gems listed in Gemfile, including any gems # you've limited to :test, :development, or :production. @@ -44,24 +45,14 @@ class Application < Rails::Application config.blacklight_url = Dromedary::Services[:solr_embedded_auth_url] config.log_level = :info - - config.lograge.enabled = false - - # add time to lograge - config.lograge.custom_options = lambda do |event| - {time: event.time} - end - - config.lograge.custom_payload do |controller| - { - host: controller.request.host, - ip: controller.request.ip, - query: controller.request.query_parameters - } - end - - config.lograge.formatter = Lograge::Formatters::Json.new - config.active_job.queue_adapter = :sidekiq + config.rails_semantic_logger.add_file_appender = false + config.semantic_logger.backtrace_level = :error + config.log_tags = { + ip: :remote_ip + } + + config.rails_semantic_logger.format = Dromedary::AbbreviatedJsonFormat.new + config.semantic_logger.add_appender(io: $stdout) config.active_record.yaml_column_permitted_classes = [ActiveSupport::HashWithIndifferentAccess] diff --git a/config/load_local_config.rb b/config/load_local_config.rb index 9ea86cd..983a798 100644 --- a/config/load_local_config.rb +++ b/config/load_local_config.rb @@ -31,9 +31,7 @@ def config end def hyp_to_bibid(collection: Dromedary::Services[:solr_current_collection]) - logger.info "Trying to get hyp_to_bibid for collection #{collection}" current_real_collection_name = underlying_real_collection_name(coll: collection) - logger.info "Real collection name identified as #{current_real_collection_name}" if @recorded_real_collection_name != current_real_collection_name @hyp_to_bibid = MedInstaller::HypToBibId.get_from_solr(collection: collection) @recorded_real_collection_name = current_real_collection_name diff --git a/lib/dromedary/abbreviated_json_format.rb b/lib/dromedary/abbreviated_json_format.rb new file mode 100644 index 0000000..16e247a --- /dev/null +++ b/lib/dromedary/abbreviated_json_format.rb @@ -0,0 +1,33 @@ +require "rails_semantic_logger" + +module Dromedary + class AbbreviatedJsonFormat < SemanticLogger::Formatters::Json + + + def application + "MED" + end + + def name; end + + def host; end + + def duration_ms; end + + def pid; end + + def level_index; end + + def thread_name; end + + def db_runtime; end + + def process_info; end + + def status_message; end + + def params + super.reject { |k, v| %x(utf8 authenticity_token redirect db_runtime).include? k } + end + end +end diff --git a/lib/med_installer/logger.rb b/lib/med_installer/logger.rb index 2be2e28..ee2d2ad 100644 --- a/lib/med_installer/logger.rb +++ b/lib/med_installer/logger.rb @@ -8,16 +8,15 @@ def process_info end end - Formatter = MEDFormatter.new(time_format: "%Y-%m-%d:%H:%M:%S") - SemanticLogger.add_appender(io: $stderr, level: :info, formatter: Formatter) + SemanticLogger.add_appender(io: $stdout, formatter: :color) + # Formatter = MEDFormatter.new(time_format: "%Y-%m-%d:%H:%M:%S") LOGGER = SemanticLogger["Dromedary"] def logger if defined? Rails Rails.logger - else - LOGGER end + LOGGER end end end From 8834027c34b2395612afa9933aa804bb2af73d58 Mon Sep 17 00:00:00 2001 From: Bill Dueber Date: Wed, 25 Sep 2024 21:40:28 -0400 Subject: [PATCH 2/3] Back out use of custom formatter for logging --- config/application.rb | 5 +++-- lib/dromedary/abbreviated_json_format.rb | 3 --- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/config/application.rb b/config/application.rb index 157f777..bfaa432 100644 --- a/config/application.rb +++ b/config/application.rb @@ -50,8 +50,9 @@ class Application < Rails::Application config.log_tags = { ip: :remote_ip } - - config.rails_semantic_logger.format = Dromedary::AbbreviatedJsonFormat.new + # Just use the standard :json logger and worry about filtering on the receiving end + # config.rails_semantic_logger.format = Dromedary::AbbreviatedJsonFormat.new + config.rails_semantic_logger.format = :json config.semantic_logger.add_appender(io: $stdout) config.active_record.yaml_column_permitted_classes = [ActiveSupport::HashWithIndifferentAccess] diff --git a/lib/dromedary/abbreviated_json_format.rb b/lib/dromedary/abbreviated_json_format.rb index 16e247a..e79e271 100644 --- a/lib/dromedary/abbreviated_json_format.rb +++ b/lib/dromedary/abbreviated_json_format.rb @@ -26,8 +26,5 @@ def process_info; end def status_message; end - def params - super.reject { |k, v| %x(utf8 authenticity_token redirect db_runtime).include? k } - end end end From f957d7b9784c11b0c4ed694d29d2171e396a7def Mon Sep 17 00:00:00 2001 From: Bill Dueber Date: Wed, 25 Sep 2024 21:46:42 -0400 Subject: [PATCH 3/3] Brought back the ip logging --- config/application.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/config/application.rb b/config/application.rb index bfaa432..e80d045 100644 --- a/config/application.rb +++ b/config/application.rb @@ -57,9 +57,9 @@ class Application < Rails::Application config.active_record.yaml_column_permitted_classes = [ActiveSupport::HashWithIndifferentAccess] - # config.log_tags = { - # ip: :remote_ip, - # } + config.log_tags = { + ip: :remote_ip, + } # config.rails_semantic_logger.quiet_assets = true # config.rails_semantic_logger.format = :json