TD-788: Adds woody pool/connections metrics (#107)

* Adds overrides for linter deps
* Bumps woody w/ prometheus collectors
* Bumps woody & scoper
* Raises processing info log messages
* Adds woody event severity mapping example to `sys.config`

Author: nanodirijabl

apps/hellgate/src/hellgate.app.src

@@ -21,6 +21,8 @@
         payproc_errors,
         erl_health,
         limiter_proto,
+        prometheus,
+        prometheus_cowboy,
         opentelemetry_api,
         opentelemetry_exporter,
         opentelemetry

apps/hellgate/src/hellgate.erl

@@ -104,8 +104,15 @@ get_prometheus_route() ->
 
 -spec start(normal, any()) -> {ok, pid()} | {error, any()}.
 start(_StartType, _StartArgs) ->
+    ok = setup_metrics(),
     supervisor:start_link(?MODULE, []).
 
 -spec stop(any()) -> ok.
 stop(_State) ->
     ok.
+
+%%
+
+setup_metrics() ->
+    ok = woody_ranch_prometheus_collector:setup(),
+    ok = woody_hackney_prometheus_collector:setup().

apps/hellgate/src/hg_invoice_payment.erl

@@ -686,8 +686,8 @@ validate_recurrent_terms(RecurrentTerms, PaymentTool) ->
     _ =
         case hg_payment_tool:has_any_payment_method(PaymentTool, PMs) of
             false ->
-                logger:info("PaymentTool: ~p", [PaymentTool]),
-                logger:info("RecurrentPaymentMethods: ~p", [PMs]),
+                logger:notice("PaymentTool: ~p", [PaymentTool]),
+                logger:notice("RecurrentPaymentMethods: ~p", [PMs]),
                 throw_invalid_request(<<"Invalid payment method">>);
             true ->
                 ok
@@ -794,7 +794,7 @@ log_route_choice_meta(#{choice_meta := undefined}, _Revision) ->
     ok;
 log_route_choice_meta(#{choice_meta := ChoiceMeta}, Revision) ->
     Metadata = hg_routing:get_logger_metadata(ChoiceMeta, Revision),
-    logger:log(info, "Routing decision made", #{routing => Metadata}).
+    logger:log(notice, "Routing decision made", #{routing => Metadata}).
 
 maybe_log_misconfigurations({misconfiguration, _} = Error) ->
     {Format, Details} = hg_routing:prepare_log_message(Error),
@@ -810,14 +810,14 @@ log_rejected_routes(all, Routes, VS) ->
 log_rejected_routes(limit_misconfiguration, Routes, _VS) ->
     ?LOG_MD(warning, "Limiter hold error caused route candidates to be rejected: ~p", [Routes]);
 log_rejected_routes(limit_overflow, Routes, _VS) ->
-    ?LOG_MD(info, "Limit overflow caused route candidates to be rejected: ~p", [Routes]);
+    ?LOG_MD(notice, "Limit overflow caused route candidates to be rejected: ~p", [Routes]);
 log_rejected_routes(adapter_unavailable, Routes, _VS) ->
-    ?LOG_MD(info, "Adapter unavailability caused route candidates to be rejected: ~p", [Routes]);
+    ?LOG_MD(notice, "Adapter unavailability caused route candidates to be rejected: ~p", [Routes]);
 log_rejected_routes(provider_conversion_is_too_low, Routes, _VS) ->
-    ?LOG_MD(info, "Lacking conversion of provider caused route candidates to be rejected: ~p", [Routes]);
+    ?LOG_MD(notice, "Lacking conversion of provider caused route candidates to be rejected: ~p", [Routes]);
 log_rejected_routes(forbidden, Routes, VS) ->
-    ?LOG_MD(info, "Rejected routes found for varset: ~p", [VS]),
-    ?LOG_MD(info, "Rejected routes found, rejected routes: ~p", [Routes]);
+    ?LOG_MD(notice, "Rejected routes found for varset: ~p", [VS]),
+    ?LOG_MD(notice, "Rejected routes found, rejected routes: ~p", [Routes]);
 log_rejected_routes(_, _Routes, _VS) ->
     ok.
 
@@ -1932,7 +1932,7 @@ process_risk_score(Action, St) ->
         ok ->
             {next, {Events, hg_machine_action:set_timeout(0, Action)}};
         {error, risk_score_is_too_high = Reason} ->
-            logger:info("No route found, reason = ~p, varset: ~p", [Reason, VS1]),
+            logger:notice("No route found, reason = ~p, varset: ~p", [Reason, VS1]),
             handle_choose_route_error(Reason, Events, St, Action)
     end.
 
@@ -2030,15 +2030,15 @@ handle_choose_route_error(Error, Events, St, Action) ->
 
 %% NOTE See damsel payproc errors (proto/payment_processing_errors.thrift) for no route found
 
-construct_routing_failure({rejected_routes, {forbidden, RejectedRoutes}}) ->
-    construct_routing_failure([forbidden], genlib:format(RejectedRoutes));
 construct_routing_failure({rejected_routes, {SubCode, RejectedRoutes}}) when
     SubCode =:= limit_misconfiguration orelse
         SubCode =:= limit_overflow orelse
         SubCode =:= adapter_unavailable orelse
         SubCode =:= provider_conversion_is_too_low
 ->
     construct_routing_failure([rejected, SubCode], genlib:format(RejectedRoutes));
+construct_routing_failure({rejected_routes, {_SubCode, RejectedRoutes}}) ->
+    construct_routing_failure([forbidden], genlib:format(RejectedRoutes));
 construct_routing_failure({misconfiguration = Code, Details}) ->
     construct_routing_failure([unknown, {unknown_error, atom_to_binary(Code)}], genlib:format(Details));
 construct_routing_failure(Code = risk_score_is_too_high) ->
@@ -2336,7 +2336,7 @@ process_failure({payment, Step} = Activity, Events, Action, Failure, St, _Refund
     Target = get_target(St),
     case check_retry_possibility(Target, Failure, St) of
         {retry, Timeout} ->
-            _ = logger:info("Retry session after transient failure, wait ~p", [Timeout]),
+            _ = logger:notice("Retry session after transient failure, wait ~p", [Timeout]),
             {SessionEvents, SessionAction} = retry_session(Action, Target, Timeout),
             {next, {Events ++ SessionEvents, SessionAction}};
         fatal ->
@@ -3273,7 +3273,7 @@ log_cascade_attempt_context(
     #domain_PaymentsServiceTerms{attempt_limit = AttemptLimit},
     #st{routes = AttemptedRoutes}
 ) ->
-    ?LOG_MD(info, "Cascade context: merchant payment terms' attempt limit '~p', attempted routes: ~p", [
+    ?LOG_MD(notice, "Cascade context: merchant payment terms' attempt limit '~p', attempted routes: ~p", [
         AttemptLimit, AttemptedRoutes
     ]).
 

apps/hellgate/src/hg_invoice_payment_refund.erl

@@ -313,7 +313,7 @@ finish_session_processing({Events0, Action}, Session, Refund) ->
         {finished, ?session_failed(Failure)} ->
             case check_retry_possibility(Failure, Refund) of
                 {retry, Timeout} ->
-                    _ = logger:info("Retry session after transient failure, wait ~p", [Timeout]),
+                    _ = logger:notice("Retry session after transient failure, wait ~p", [Timeout]),
                     {SessionEvents, SessionAction} = retry_session(Action, Timeout),
                     {next, {Events1 ++ SessionEvents, SessionAction}};
                 fatal ->

apps/hellgate/src/hg_limiter.erl

@@ -86,7 +86,7 @@ check_limits_([T | TurnoverLimits], Context, Acc) ->
         true ->
             check_limits_(TurnoverLimits, Context, [Limit | Acc]);
         false ->
-            logger:info("Limit with id ~p overflowed, amount ~p upper boundary ~p", [
+            logger:notice("Limit with id ~p overflowed, amount ~p upper boundary ~p", [
                 LimitID,
                 LimiterAmount,
                 UpperBoundary

apps/hellgate/src/hg_machine.erl

@@ -346,7 +346,7 @@ dispatch_repair(Ns, Payload, Machine) ->
         marshal_repair_result(ok, Result, Machine)
     catch
         throw:{exception, Reason} = Error ->
-            logger:info("Process repair failed, ~p", [Reason]),
+            logger:notice("Process repair failed, ~p", [Reason]),
             woody_error:raise(business, marshal_repair_failed(Error))
     end.
 

apps/hellgate/src/hg_recurrent_paytool.erl

@@ -253,7 +253,7 @@ init(EncodedParams, #{id := RecPaymentToolID}) ->
         {ChosenRoute, ChoiceContext} = hg_routing:choose_route(NonFailRatedRoutes),
         ChosenPaymentRoute = hg_route:to_payment_route(ChosenRoute),
         LoggerMetadata = hg_routing:get_logger_metadata(ChoiceContext, Revision),
-        _ = logger:log(info, "Routing decision made", #{routing => LoggerMetadata}),
+        _ = logger:log(notice, "Routing decision made", #{routing => LoggerMetadata}),
         RecPaymentTool2 = set_minimal_payment_cost(RecPaymentTool, ChosenPaymentRoute, VS, Revision),
         {ok, {Changes, Action}} = start_session(),
         StartChanges = [
@@ -376,7 +376,7 @@ validate_risk_score(RiskScore) when RiskScore == low; RiskScore == high ->
     RiskScore.
 
 handle_route_error(risk_score_is_too_high = Reason, RecPaymentTool) ->
-    _ = logger:log(info, "No route found, reason = ~p", [Reason], logger:get_process_metadata()),
+    _ = logger:log(notice, "No route found, reason = ~p", [Reason], logger:get_process_metadata()),
     {misconfiguration, {'No route found for a recurrent payment tool', RecPaymentTool}}.
 handle_route_error({no_route_found, {Reason, RejectedRoutes}}, RecPaymentTool, Varset) ->
     LogFun = fun(Msg, Param) ->

apps/hellgate/src/hg_routing.erl

@@ -483,6 +483,7 @@ get_provider_status(Route, FDStats) ->
     AvailabilityStatus = get_adapter_availability_status(FdOverrides, AvailabilityServiceID, FDStats),
     ConversionStatus = get_provider_conversion_status(FdOverrides, ConversionServiceID, FDStats),
     {AvailabilityStatus, ConversionStatus}.
+
 get_adapter_availability_status(#domain_RouteFaultDetectorOverrides{enabled = true}, _FDID, _Stats) ->
     %% ignore fd statistic if set override
     {alive, 0.0};

config/sys.config

@@ -46,6 +46,15 @@
             event_handler_opts => #{
                 formatter_opts => #{
                     max_length => 1000
+                },
+                events_severity => #{
+                    %% Maybe disregard those events by lowering their severity level
+                    ['call service'] => info,
+                    ['service result'] => info,
+                    ['invoke service handler'] => info,
+                    ['service handler result'] => info,
+                    ['service handler result', error, business] => warning,
+                    ['client cache hit'] => info
                 }
             }
         }},
@@ -126,6 +135,10 @@
                 event_handler_opts => #{
                     formatter_opts => #{
                         max_length => 1000
+                    },
+                    events_severity => #{
+                        %% Was 'info'
+                        ['service handler result', error, business] => warning
                     }
                 }
             }}
@@ -150,6 +163,10 @@
                             event_handler_opts => #{
                                 formatter_opts => #{
                                     max_length => 1000
+                                },
+                                events_severity => #{
+                                    %% Was 'info'
+                                    ['service handler result', error, business] => warning
                                 }
                             }
                         }}

rebar.config

@@ -40,6 +40,8 @@
     {erl_health, {git, "https://github.com/valitydev/erlang-health.git", {branch, "master"}}},
     {fault_detector_proto, {git, "https://github.com/valitydev/fault-detector-proto.git", {branch, "master"}}},
     {limiter_proto, {git, "https://github.com/valitydev/limiter-proto.git", {branch, "master"}}},
+    {prometheus, "4.8.1"},
+    {prometheus_cowboy, "0.1.8"},
 
     %% OpenTelemetry deps
     {opentelemetry_api, "1.2.1"},
@@ -72,12 +74,6 @@
 {profiles, [
     {prod, [
         {deps, [
-            %% NOTE
-            %% Because of a dependency conflict, prometheus libs are only included in production build for now
-            %% https://github.com/project-fifo/rebar3_lint/issues/42
-            %% https://github.com/valitydev/hellgate/pull/2/commits/884724c1799703cee4d1033850fe32c17f986d9e
-            {prometheus, "4.8.1"},
-            {prometheus_cowboy, "0.1.8"},
             % for introspection on production
             {recon, "2.5.2"},
             {logger_logstash_formatter,
@@ -92,8 +88,6 @@
                 {tools, load},
                 {opentelemetry, temporary},
                 logger_logstash_formatter,
-                prometheus,
-                prometheus_cowboy,
                 sasl,
                 hellgate
             ]},
@@ -111,7 +105,7 @@
     ]}
 ]}.
 
-{plugins, [
+{project_plugins, [
     {covertool, "2.0.4"},
     {erlfmt, "1.0.0"},
     {rebar3_lint, "1.0.1"},
@@ -136,3 +130,11 @@
         "ct.coverdata"
     ]}
 ]}.
+
+%% NOTE
+%% It is needed to use rebar3 lint plugin
+{overrides, [
+    {del, accept, [{plugins, [{rebar3_archive_plugin, "0.0.2"}]}]},
+    {del, prometheus_cowboy, [{plugins, [{rebar3_archive_plugin, "0.0.1"}]}]},
+    {del, prometheus_httpd, [{plugins, [{rebar3_archive_plugin, "0.0.1"}]}]}
+]}.