MB-50306: Add cert expiration alerts for xdcr certs

timofey-barmin · timofey-barmin · commit acb26a5c1ff5 · 2022-03-23T23:27:52.000Z
Change-Id: Ia5757c34d6681383d992baff93e625867547b472 Reviewed-on: https://review.couchbase.org/c/ns_server/+/169188 Well-Formed: Build Bot <build@couchbase.com> Reviewed-by: Steve Watanabe <steve.watanabe@couchbase.com> Tested-by: Build Bot <build@couchbase.com> Tested-by: Timofey Barmin <timofey.barmin@couchbase.com>
diff --git a/src/goxdcr_rest.erl b/src/goxdcr_rest.erl
@@ -18,7 +18,8 @@
          find_all_replication_docs/0,
          all_local_replication_infos/0,
          stats/1,
-         get_replications_with_remote_info/0]).
+         get_replications_with_remote_info/0,
+         get_certificates/0]).
 
 convert_header_name(Header) when is_atom(Header) ->
     atom_to_list(Header);
@@ -234,3 +235,20 @@ get_replications_with_remote_info() ->
               ClusterName = proplists:get_value(RemoteClusterUUID, RemoteClusters, <<"unknown">>),
               [{Id, BucketName, binary_to_list(ClusterName), RemoteBucket} | Acc]
       end, [], find_all_replication_docs()).
+
+get_certificates() ->
+    get_from_goxdcr(
+      fun (Json) ->
+              Extract = fun (What) ->
+                            lists:flatmap(
+                              fun ({Cluster}) ->
+                                  case proplists:get_value(What, Cluster) of
+                                      undefined -> [];
+                                      B -> ns_server_cert:split_certs(B)
+                                  end
+                              end, Json)
+                        end,
+              TrustedCerts = Extract(<<"certificate">>),
+              ClientCerts = Extract(<<"clientCertificate">>),
+              #{trusted_certs => TrustedCerts, client_certs => ClientCerts}
+      end, "/pools/default/remoteClusters", 30000).
diff --git a/src/menelaus_web_alerts_srv.erl b/src/menelaus_web_alerts_srv.erl
@@ -121,7 +121,15 @@ errors(node_cert_expired) ->
 errors(ca_expires_soon) ->
     "Trusted CA certificate with ID=~b (subject: '~s') will expire at ~s.";
 errors(ca_expired) ->
-    "Trusted CA certificate with ID=~b (subject: '~s') has expired.".
+    "Trusted CA certificate with ID=~b (subject: '~s') has expired.";
+errors(client_xdcr_cert_expires_soon) ->
+    "Client XDCR certificate (subject: '~s') will expire at ~s.";
+errors(client_xdcr_cert_expired) ->
+    "Client XDCR certificate (subject: '~s') has expired.";
+errors(xdcr_ca_expires_soon) ->
+    "XDCR CA certificate (subject: '~s') will expire at ~s.";
+errors(xdcr_ca_expired) ->
+    "XDCR CA certificate (subject: '~s') has expired.".
 
 %% ------------------------------------------------------------------
 %% API Function Definitions
@@ -311,7 +319,7 @@ start_timer() ->
 global_checks() ->
     [oom, ip, write_fail, overhead, disk, audit_write_fail,
      indexer_ram_max_usage, cas_drift_threshold, communication_issue,
-     time_out_of_sync, disk_usage_analyzer_stuck, certs].
+     time_out_of_sync, disk_usage_analyzer_stuck, certs, xdcr_certs].
 
 %% @doc fires off various checks
 check_alerts(Opaque, Hist, Stats) ->
@@ -522,7 +530,8 @@ check(certs, Opaque, _History, _Stats) ->
             true ->
                 lists:flatmap(
                   fun (CAProps) ->
-                      ExpWarnings = ns_server_cert:expiration_warnings(CAProps),
+                      {_, ExpWarnings} =
+                          ns_server_cert:expiration_warnings(CAProps),
                       Subject = proplists:get_value(subject, CAProps),
                       Id = proplists:get_value(id, CAProps),
                       [{{ca, Id, Subject}, W} || W <- ExpWarnings]
@@ -535,11 +544,12 @@ check(certs, Opaque, _History, _Stats) ->
         case ns_config:read_key_fast({node, node(), node_cert}, undefined) of
             undefined -> [];
             Props ->
+                {_, ExpWarnings} = ns_server_cert:expiration_warnings(Props),
                 lists:map(
                   fun (W) ->
                       Subject = proplists:get_value(subject, Props),
                       {{node, Subject}, W}
-                  end, ns_server_cert:expiration_warnings(Props))
+                  end, ExpWarnings)
         end,
 
     lists:foreach(
@@ -562,7 +572,94 @@ check(certs, Opaque, _History, _Stats) ->
               global_alert({cert_expires_soon, {node, Host}}, Error)
       end, CAAlerts ++ LocalAlerts),
 
-    Opaque.
+    Opaque;
+
+check(xdcr_certs, Opaque, _History, _Stats) ->
+    case mb_master:master_node() == node() of
+        true -> check_xdcr_certs(Opaque);
+        false -> Opaque
+    end.
+
+check_xdcr_certs(Opaque) ->
+    try goxdcr_rest:get_certificates() of
+        Info ->
+            WarningDays = ns_config:read_key_fast(
+                            {cert, expiration_warning_days}, undefined),
+            Hash = erlang:phash2({Info, WarningDays}),
+            Now = calendar:datetime_to_gregorian_seconds(
+                    calendar:universal_time()),
+
+            {NewOpaque, AlertsList} =
+                case dict:find(xdcr_certs_check, Opaque) of
+                    {ok, #{hash := Hash,
+                           retry_time := RetryTime,
+                           result := Res}} when Now < RetryTime ->
+                        {Opaque, Res};
+                    _ ->
+                        {RetryTime, Alerts} = calculate_xdcr_cert_alerts(Info),
+                        {dict:store(xdcr_certs_check,
+                                    #{hash => Hash,
+                                      retry_time => RetryTime,
+                                      result => Alerts},
+                                    Opaque), Alerts}
+                end,
+
+            lists:foreach(
+              fun ({{ca, Subj}, expired}) ->
+                      Error = fmt_to_bin(errors(xdcr_ca_expired), [Subj]),
+                      global_alert({cert_expired, {xdcr, Subj}}, Error);
+                  ({{client_cert, Subj}, expired}) ->
+                      Error = fmt_to_bin(errors(client_xdcr_cert_expired),
+                                         [Subj]),
+                      global_alert({cert_expired, {client_xdcr, Subj}}, Error);
+                  ({{ca, Subj}, {expires_soon, UTCSeconds}}) ->
+                      Date = menelaus_web_cert:format_time(UTCSeconds),
+                      Error = fmt_to_bin(errors(xdcr_ca_expires_soon),
+                                         [Subj, Date]),
+                      global_alert({cert_expires_soon, {xdcr, Subj}}, Error);
+                  ({{client_cert, Subj}, {expires_soon, UTCSeconds}}) ->
+                      Date = menelaus_web_cert:format_time(UTCSeconds),
+                      Error = fmt_to_bin(errors(client_xdcr_cert_expires_soon),
+                                         [Subj, Date]),
+                      global_alert({cert_expires_soon, {client_xdcr, Subj}},
+                                   Error)
+              end, AlertsList),
+
+            NewOpaque
+
+    catch
+        C:E:ST ->
+            ?log_error("Failed to extract certs info from xdcr: ~p:~p~n~p",
+                       [C, E, ST]),
+            Opaque
+    end.
+
+calculate_xdcr_cert_alerts(#{trusted_certs := TrustedCerts,
+                             client_certs := ClientCerts}) ->
+    AlertsFun =
+        fun (Certs, Type) ->
+            lists:foldl(
+              fun (C, {WarningsAcc, RecheckAcc}) ->
+                  case ns_server_cert:decode_single_certificate(C) of
+                      {ok, D} ->
+                          Props = ns_server_cert:cert_props(D),
+                          {RecheckTime, Warnings} =
+                              ns_server_cert:expiration_warnings(Props),
+                          Subject = proplists:get_value(subject, Props),
+                          {[{{Type, Subject}, W} || W <- Warnings] ++
+                           WarningsAcc,
+                           min(RecheckTime, RecheckAcc)};
+                      {error, E} ->
+                          ?log_debug("Failed to decode xdcr cert: ~p",
+                                     [E]),
+                          {WarningsAcc, RecheckAcc}
+                  end
+              end, {[], infinity}, Certs)
+        end,
+
+    {CAAlerts, CARecheckTime} = AlertsFun(TrustedCerts, ca),
+    {ClientAlerts, ClientRecheckTime} = AlertsFun(ClientCerts, client_cert),
+    {min(CARecheckTime, ClientRecheckTime), CAAlerts ++ ClientAlerts}.
 
 alert_if_time_out_of_sync({time_offset_status, true}) ->
     Err = fmt_to_bin(errors(time_out_of_sync), [node()]),
diff --git a/src/ns_server_cert.erl b/src/ns_server_cert.erl
@@ -41,7 +41,9 @@
          generate_node_certs/1,
          filter_nodes_by_ca/2,
          inbox_chain_path/0,
-         expiration_warnings/1]).
+         expiration_warnings/1,
+         split_certs/1,
+         cert_props/1]).
 
 inbox_ca_path() ->
     filename:join(path_config:component_path(data, "inbox"), "CA").
@@ -1129,6 +1131,9 @@ read_ca_file(Path) ->
             {error, {read, Reason}}
     end.
 
+cert_props(DerCert) ->
+    cert_props(undefined, DerCert, []).
+
 cert_props(Type, DerCert, Extras) when is_binary(DerCert) ->
     {Sub, NotBefore, NotAfter} = get_der_info(DerCert),
     [{subject, iolist_to_binary(Sub)},
@@ -1174,7 +1179,7 @@ get_warnings() ->
                           generated -> [self_signed];
                           _ -> []
                       end,
-                  ExpWarnings = expiration_warnings(CAProps),
+                  {_, ExpWarnings} = expiration_warnings(CAProps),
                   Id = proplists:get_value(id, CAProps),
                   UnusedWarnings =
                       case proplists:get_value(type, CAProps) of
@@ -1194,17 +1199,18 @@ get_warnings() ->
 expiration_warnings(CertProps) ->
     Now = calendar:datetime_to_gregorian_seconds(calendar:universal_time()),
     WarningDays = ns_config:read_key_fast({cert, expiration_warning_days}, 7),
-    WarningThreshold = Now + WarningDays * 24 * 60 * 60,
+    WarningSeconds = WarningDays * 24 * 60 * 60,
+    WarningThreshold = Now + WarningSeconds,
 
     Expire = proplists:get_value(expires, CertProps), %% For pre-7.1 only
     NotAfter = proplists:get_value(not_after, CertProps, Expire),
     case NotAfter of
         A when is_integer(A) andalso A =< Now ->
-            [expired];
+            {infinity, [expired]};
         A when is_integer(A) andalso A =< WarningThreshold ->
-            [{expires_soon, A}];
+            {NotAfter, [{expires_soon, A}]};
         _ ->
-            []
+            {NotAfter - WarningSeconds, []}
     end.
 
 is_trusted(CAPem, TrustedCAs) ->
@@ -1237,7 +1243,7 @@ node_cert_warnings(TrustedCAs, NodeCertProps) ->
                 end
         end,
 
-    ExpirationWarnings = expiration_warnings(NodeCertProps),
+    {_, ExpirationWarnings} = expiration_warnings(NodeCertProps),
 
     SelfSignedWarnings =
         case proplists:get_value(type, NodeCertProps) of
