From 91439bbcb65fe3624b8431bdec61af741a29d847 Mon Sep 17 00:00:00 2001 From: AASHISH SINGH Date: Fri, 10 Nov 2023 19:10:49 +0530 Subject: [PATCH 01/13] added trim functionality --- lib/chitragupta/constants.rb | 2 ++ lib/chitragupta/json_log_formatter.rb | 7 +++++- lib/chitragupta/logger.rb | 6 +++--- lib/chitragupta/util.rb | 31 ++++++++++++++++----------- 4 files changed, 30 insertions(+), 16 deletions(-) diff --git a/lib/chitragupta/constants.rb b/lib/chitragupta/constants.rb index 532ad38..21b0a50 100644 --- a/lib/chitragupta/constants.rb +++ b/lib/chitragupta/constants.rb @@ -10,5 +10,7 @@ module Constants CATEGORY_SERVER = "server" CATEGORY_PROCESS = "process" CATEGORY_WORKER = "worker" + + MAX_STR_LENGTH = 10 # 40000 end end diff --git a/lib/chitragupta/json_log_formatter.rb b/lib/chitragupta/json_log_formatter.rb index c93bc5e..78fe17f 100644 --- a/lib/chitragupta/json_log_formatter.rb +++ b/lib/chitragupta/json_log_formatter.rb @@ -1,8 +1,13 @@ module Chitragupta class JsonLogFormatter < Logger::Formatter + def initialize(should_trim_long_string=false) + @should_trim_long_string = should_trim_long_string + end def call(log_level, timestamp, _progname, message) - return Chitragupta::Util::sanitize_keys(log_level, timestamp, message) + puts("inside caa") + # return "inside_call - #{@should_trim_long_string}" + return Chitragupta::Util::sanitize_keys(log_level, timestamp, message, @should_trim_long_string) end end end diff --git a/lib/chitragupta/logger.rb b/lib/chitragupta/logger.rb index e2f65bf..ef37f43 100644 --- a/lib/chitragupta/logger.rb +++ b/lib/chitragupta/logger.rb @@ -1,10 +1,10 @@ require "syslog/logger" module Chitragupta - class Logger < Syslog::Logger - def initialize(*args) + class Loggerq < Syslog::Logger + def initialize(*args, should_trim_long_string: false) super(*args) - @formatter = Chitragupta::JsonLogFormatter.new + @formatter = Chitragupta::JsonLogFormatter.new(should_trim_long_string) end end end diff --git a/lib/chitragupta/util.rb b/lib/chitragupta/util.rb index 09e7370..262c7ed 100644 --- a/lib/chitragupta/util.rb +++ b/lib/chitragupta/util.rb @@ -5,8 +5,8 @@ module Chitragupta module Util extend self - def sanitize_keys(log_level, timestamp, message) - data = initialize_data(message) + def sanitize_keys(log_level, timestamp, message, should_trim_long_string) + data = initialize_data(message, should_trim_long_string) data[:log][:level] = log_level data[:meta][:timestamp] = timestamp @@ -37,7 +37,7 @@ def called_as_console? end private - def populate_server_data(data, message) + def populate_server_data(data, message, should_trim_long_string) data[:data][:request] = {} data[:data][:response] = {} data[:data][:request][:method] = Chitragupta.payload[:method] @@ -45,7 +45,7 @@ def populate_server_data(data, message) data[:data][:request][:ip] = Chitragupta.payload[:ip] data[:data][:request][:id] = Chitragupta.payload[:request_id] data[:data][:request][:user_id] = Chitragupta.payload[:user_id] - data[:data][:request][:params] = Chitragupta.payload[:params].to_json.to_s + data[:data][:request][:params] = trim_long_string(Chitragupta.payload[:params].to_json.to_s, should_trim_long_string) data[:data][:response][:status] = message[:status] rescue nil data[:data][:response][:duration] = message[:duration] rescue nil @@ -57,8 +57,8 @@ def populate_server_data(data, message) data[:log][:id] ||= Chitragupta.payload[:log_id] end - def populate_rails_server_data(data, message) - populate_server_data(data, message) + def populate_rails_server_data(data, message, should_trim_long_string) + populate_server_data(data, message, should_trim_long_string) data[:data][:request][:controller] = Chitragupta.payload[:controller] data[:data][:request][:action] = Chitragupta.payload[:action] @@ -66,8 +66,8 @@ def populate_rails_server_data(data, message) data[:data][:response][:db_query_duration] = message[:db] rescue nil end - def populate_ruby_server_data(data, message) - populate_server_data(data, message) + def populate_ruby_server_data(data, message, should_trim_long_string) + populate_server_data(data, message, should_trim_long_string) end def populate_task_data(data, message) @@ -93,7 +93,7 @@ def populate_worker_data(data, message) data[:data][:worker_name] = worker_name end - def initialize_data(message) + def initialize_data(message, should_trim_long_string) data = {} data[:data] = {} @@ -103,15 +103,15 @@ def initialize_data(message) else data[:log] = {} data[:meta] = {} - data[:log][:dynamic_data] = message.is_a?(String) ? message : message.inspect if message + data[:log][:dynamic_data] = message.is_a?(String) ? trim_long_string(message, should_trim_long_string) : trim_long_string(message.inspect, should_trim_long_string) if message end data[:meta][:format] ||= {} begin if called_as_rails_server? - populate_rails_server_data(data, message) + populate_rails_server_data(data, message, should_trim_long_string) elsif called_as_rack_server? - populate_ruby_server_data(data, message) + populate_ruby_server_data(data, message, should_trim_long_string) elsif called_as_rake? populate_task_data(data, message) elsif called_as_sidekiq? @@ -121,5 +121,12 @@ def initialize_data(message) return data end + def trim_long_string(input_str, should_trim_long_string) + if(should_trim_long_string) + return input_str.slice(0, Chitragupta::Constants::MAX_STR_LENGTH) + end + return input_str + end + end end From 143e1e11d77710d63cd5a4abdcf003f62e22955c Mon Sep 17 00:00:00 2001 From: AASHISH SINGH Date: Fri, 10 Nov 2023 19:12:20 +0530 Subject: [PATCH 02/13] fuxed typo --- lib/chitragupta/logger.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/chitragupta/logger.rb b/lib/chitragupta/logger.rb index ef37f43..50df064 100644 --- a/lib/chitragupta/logger.rb +++ b/lib/chitragupta/logger.rb @@ -1,7 +1,7 @@ require "syslog/logger" module Chitragupta - class Loggerq < Syslog::Logger + class Logger < Syslog::Logger def initialize(*args, should_trim_long_string: false) super(*args) @formatter = Chitragupta::JsonLogFormatter.new(should_trim_long_string) From 253f3ecdbf154ce02590cdf17821cc5efb848463 Mon Sep 17 00:00:00 2001 From: AASHISH SINGH Date: Fri, 10 Nov 2023 21:29:35 +0530 Subject: [PATCH 03/13] added --- lib/chitragupta.rb | 10 +++++----- lib/chitragupta/json_log_formatter.rb | 4 +++- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/lib/chitragupta.rb b/lib/chitragupta.rb index 456b96b..4fbf9c1 100644 --- a/lib/chitragupta.rb +++ b/lib/chitragupta.rb @@ -13,7 +13,7 @@ module Chitragupta # The gem can be used by adding the following in any of the rails initializations: application.rb / environment.rb # Chitragupta::setup_application_logger(RailsApplicationModule, current_user_function) - def setup_application_logger(app, current_user_caller=nil) + def setup_application_logger(app, current_user_caller=nil, should_trim_long_string=false) # Should be required only when the rails application is configuring the gem to be used. require "chitragupta/active_support/tagged_logging/formatter" @@ -38,10 +38,10 @@ def setup_application_logger(app, current_user_caller=nil) end if Chitragupta::Util.called_as_sidekiq? - Sidekiq.logger.formatter = JsonLogFormatter.new + Sidekiq.logger.formatter = JsonLogFormatter.new(should_trim_long_string) end - configure_app(app) + configure_app(app, should_trim_long_string) end @@ -50,9 +50,9 @@ def get_unique_log_id end private - def configure_app(app) + def configure_app(app, should_trim_long_string) app::Application.configure do - config.log_formatter = JsonLogFormatter.new if Chitragupta::Util.called_as_rails_server? || Chitragupta::Util.called_as_rake? || Chitragupta::Util.called_as_sidekiq? + config.log_formatter = JsonLogFormatter.new(should_trim_long_string) if Chitragupta::Util.called_as_rails_server? || Chitragupta::Util.called_as_rake? || Chitragupta::Util.called_as_sidekiq? if Chitragupta::Util.called_as_rails_server? require "chitragupta/request_log_formatter" config.lograge.enabled = true diff --git a/lib/chitragupta/json_log_formatter.rb b/lib/chitragupta/json_log_formatter.rb index 78fe17f..b17a06b 100644 --- a/lib/chitragupta/json_log_formatter.rb +++ b/lib/chitragupta/json_log_formatter.rb @@ -7,7 +7,9 @@ def initialize(should_trim_long_string=false) def call(log_level, timestamp, _progname, message) puts("inside caa") # return "inside_call - #{@should_trim_long_string}" - return Chitragupta::Util::sanitize_keys(log_level, timestamp, message, @should_trim_long_string) + final_str = Chitragupta::Util::sanitize_keys(log_level, timestamp, message, @should_trim_long_string) + puts("final_str - #{final_str}") + return final_str end end end From 7980dd3f719be7b95acf5c2770d4f1ea00cfbddd Mon Sep 17 00:00:00 2001 From: AASHISH SINGH Date: Sat, 11 Nov 2023 12:09:03 +0530 Subject: [PATCH 04/13] added debug logs --- lib/chitragupta/util.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/chitragupta/util.rb b/lib/chitragupta/util.rb index 262c7ed..a41e0e5 100644 --- a/lib/chitragupta/util.rb +++ b/lib/chitragupta/util.rb @@ -45,6 +45,7 @@ def populate_server_data(data, message, should_trim_long_string) data[:data][:request][:ip] = Chitragupta.payload[:ip] data[:data][:request][:id] = Chitragupta.payload[:request_id] data[:data][:request][:user_id] = Chitragupta.payload[:user_id] + data[:data][:request][:test] = "starting trimming of params" data[:data][:request][:params] = trim_long_string(Chitragupta.payload[:params].to_json.to_s, should_trim_long_string) data[:data][:response][:status] = message[:status] rescue nil From 2cf60a496ba320ab26414bd3539440e1ea01557d Mon Sep 17 00:00:00 2001 From: AASHISH SINGH Date: Sat, 11 Nov 2023 12:26:42 +0530 Subject: [PATCH 05/13] removed extra log --- lib/chitragupta/util.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/chitragupta/util.rb b/lib/chitragupta/util.rb index a41e0e5..262c7ed 100644 --- a/lib/chitragupta/util.rb +++ b/lib/chitragupta/util.rb @@ -45,7 +45,6 @@ def populate_server_data(data, message, should_trim_long_string) data[:data][:request][:ip] = Chitragupta.payload[:ip] data[:data][:request][:id] = Chitragupta.payload[:request_id] data[:data][:request][:user_id] = Chitragupta.payload[:user_id] - data[:data][:request][:test] = "starting trimming of params" data[:data][:request][:params] = trim_long_string(Chitragupta.payload[:params].to_json.to_s, should_trim_long_string) data[:data][:response][:status] = message[:status] rescue nil From 1cc7ebf771d8e1d79e2e3a41dbd44799cc167d4d Mon Sep 17 00:00:00 2001 From: AASHISH SINGH Date: Sat, 11 Nov 2023 12:31:01 +0530 Subject: [PATCH 06/13] removed unwantd logs --- lib/chitragupta/json_log_formatter.rb | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/lib/chitragupta/json_log_formatter.rb b/lib/chitragupta/json_log_formatter.rb index b17a06b..59431bb 100644 --- a/lib/chitragupta/json_log_formatter.rb +++ b/lib/chitragupta/json_log_formatter.rb @@ -5,11 +5,7 @@ def initialize(should_trim_long_string=false) end def call(log_level, timestamp, _progname, message) - puts("inside caa") - # return "inside_call - #{@should_trim_long_string}" - final_str = Chitragupta::Util::sanitize_keys(log_level, timestamp, message, @should_trim_long_string) - puts("final_str - #{final_str}") - return final_str + return Chitragupta::Util::sanitize_keys(log_level, timestamp, message, @should_trim_long_string) end end end From 16a3763554086b10013508d1fc4b7f93ba5ea68a Mon Sep 17 00:00:00 2001 From: AASHISH SINGH Date: Sat, 11 Nov 2023 12:35:03 +0530 Subject: [PATCH 07/13] updated length --- lib/chitragupta/constants.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/chitragupta/constants.rb b/lib/chitragupta/constants.rb index 21b0a50..e5bf966 100644 --- a/lib/chitragupta/constants.rb +++ b/lib/chitragupta/constants.rb @@ -11,6 +11,6 @@ module Constants CATEGORY_PROCESS = "process" CATEGORY_WORKER = "worker" - MAX_STR_LENGTH = 10 # 40000 + MAX_STR_LENGTH = 40000 end end From 9c656d3cbf848010a4c093a474315bec920a0725 Mon Sep 17 00:00:00 2001 From: AASHISH SINGH Date: Sat, 11 Nov 2023 12:46:40 +0530 Subject: [PATCH 08/13] updated max str length --- lib/chitragupta/constants.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/chitragupta/constants.rb b/lib/chitragupta/constants.rb index e5bf966..4670376 100644 --- a/lib/chitragupta/constants.rb +++ b/lib/chitragupta/constants.rb @@ -11,6 +11,6 @@ module Constants CATEGORY_PROCESS = "process" CATEGORY_WORKER = "worker" - MAX_STR_LENGTH = 40000 + MAX_STR_LENGTH = 400000 end end From 9b9894b8ca73b0d4a3318208e65ce9252943b383 Mon Sep 17 00:00:00 2001 From: AASHISH SINGH Date: Tue, 14 Nov 2023 13:16:21 +0530 Subject: [PATCH 09/13] changes contants to 10 --- lib/chitragupta/constants.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/chitragupta/constants.rb b/lib/chitragupta/constants.rb index 4670376..81c49d0 100644 --- a/lib/chitragupta/constants.rb +++ b/lib/chitragupta/constants.rb @@ -11,6 +11,6 @@ module Constants CATEGORY_PROCESS = "process" CATEGORY_WORKER = "worker" - MAX_STR_LENGTH = 400000 + MAX_STR_LENGTH = 10 # 400000 end end From 402166a0971808833e0e9cadf8c4e34cd8668ac8 Mon Sep 17 00:00:00 2001 From: AASHISH SINGH Date: Tue, 14 Nov 2023 19:28:33 +0530 Subject: [PATCH 10/13] updated constant --- lib/chitragupta/constants.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/chitragupta/constants.rb b/lib/chitragupta/constants.rb index 81c49d0..4670376 100644 --- a/lib/chitragupta/constants.rb +++ b/lib/chitragupta/constants.rb @@ -11,6 +11,6 @@ module Constants CATEGORY_PROCESS = "process" CATEGORY_WORKER = "worker" - MAX_STR_LENGTH = 10 # 400000 + MAX_STR_LENGTH = 400000 end end From d0c95e90521c62ec0bbd8811d7c63c2e37c5cb74 Mon Sep 17 00:00:00 2001 From: AASHISH SINGH Date: Tue, 14 Nov 2023 19:55:58 +0530 Subject: [PATCH 11/13] removed truncation of dynamic data --- lib/chitragupta/util.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/chitragupta/util.rb b/lib/chitragupta/util.rb index 262c7ed..e2ae186 100644 --- a/lib/chitragupta/util.rb +++ b/lib/chitragupta/util.rb @@ -103,7 +103,7 @@ def initialize_data(message, should_trim_long_string) else data[:log] = {} data[:meta] = {} - data[:log][:dynamic_data] = message.is_a?(String) ? trim_long_string(message, should_trim_long_string) : trim_long_string(message.inspect, should_trim_long_string) if message + data[:log][:dynamic_data] = message.is_a?(String) ? message : message.inspect if message end data[:meta][:format] ||= {} From ba915ffe0b34b85c34a96cfc583be998a93e164a Mon Sep 17 00:00:00 2001 From: AASHISH SINGH Date: Tue, 14 Nov 2023 20:19:38 +0530 Subject: [PATCH 12/13] add value to 10 --- lib/chitragupta/constants.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/chitragupta/constants.rb b/lib/chitragupta/constants.rb index 4670376..81c49d0 100644 --- a/lib/chitragupta/constants.rb +++ b/lib/chitragupta/constants.rb @@ -11,6 +11,6 @@ module Constants CATEGORY_PROCESS = "process" CATEGORY_WORKER = "worker" - MAX_STR_LENGTH = 400000 + MAX_STR_LENGTH = 10 # 400000 end end From 316388d2edee33ee0efaa519be990ee3bc3f5ec3 Mon Sep 17 00:00:00 2001 From: AASHISH SINGH Date: Tue, 14 Nov 2023 20:43:02 +0530 Subject: [PATCH 13/13] updated max value --- lib/chitragupta/constants.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/chitragupta/constants.rb b/lib/chitragupta/constants.rb index 81c49d0..4670376 100644 --- a/lib/chitragupta/constants.rb +++ b/lib/chitragupta/constants.rb @@ -11,6 +11,6 @@ module Constants CATEGORY_PROCESS = "process" CATEGORY_WORKER = "worker" - MAX_STR_LENGTH = 10 # 400000 + MAX_STR_LENGTH = 400000 end end