From 48701044297544edec2cd18ccf3b10a82a554b82 Mon Sep 17 00:00:00 2001 From: Steven Woods Date: Fri, 18 Jul 2025 16:58:56 +0100 Subject: [PATCH 1/3] CP-308811: Add an option to limit the span depth in tracing Adds a new span.depth key to the trace context baggage, and a configurable max_span_depth. This defaults to 100 and so will not limit traces, but is useful when wanting to analyse large traces which can often become slow if all the traces are recorded. Signed-off-by: Steven Woods --- ocaml/libs/tracing/tracing.ml | 98 +++++++++++++++++++++++++++++----- ocaml/libs/tracing/tracing.mli | 2 + ocaml/tests/test_observer.ml | 1 + ocaml/xapi/xapi_globs.ml | 7 +++ ocaml/xapi/xapi_observer.ml | 1 + 5 files changed, 95 insertions(+), 14 deletions(-) diff --git a/ocaml/libs/tracing/tracing.ml b/ocaml/libs/tracing/tracing.ml index d320fd6061b..389c5bafaa3 100644 --- a/ocaml/libs/tracing/tracing.ml +++ b/ocaml/libs/tracing/tracing.ml @@ -222,6 +222,8 @@ module TraceContext = struct let empty = {traceparent= None; baggage= None} + let depth_key = "span.depth" + let with_traceparent traceparent ctx = {ctx with traceparent} let with_baggage baggage ctx = {ctx with baggage} @@ -230,6 +232,20 @@ module TraceContext = struct let baggage_of ctx = ctx.baggage + let baggage_depth_of ctx = + Option.bind (baggage_of ctx) (List.assoc_opt depth_key) + |> Option.value ~default:"1" + |> int_of_string + + let update_with_baggage k v ctx = + let new_baggage = + baggage_of ctx + |> Option.value ~default:[] + |> List.remove_assoc k + |> List.cons (k, v) + in + with_baggage (Some new_baggage) ctx + let parse input = let open Astring.String in let trim_pair (key, value) = (trim key, trim value) in @@ -322,22 +338,36 @@ module Span = struct let start ?(attributes = Attributes.empty) ?(trace_context : TraceContext.t option) ~name ~parent ~span_kind () = - let trace_id, extra_context = + let trace_id, extra_context, depth = match parent with | None -> - (Trace_id.make (), TraceContext.empty) + (Trace_id.make (), TraceContext.empty, 1) | Some span_parent -> - (span_parent.context.trace_id, span_parent.context.trace_context) + ( span_parent.context.trace_id + , span_parent.context.trace_context + , TraceContext.baggage_depth_of span_parent.context.trace_context + 1 + ) in let span_id = Span_id.make () in + let extra_context_with_depth = + TraceContext.( + with_added_baggage depth_key (string_of_int depth) extra_context + ) + in let context : SpanContext.t = - {trace_id; span_id; trace_context= extra_context} + {trace_id; span_id; trace_context= extra_context_with_depth} in let context = - (* If trace_context is provided to the call, override any inherited trace context. *) - trace_context - |> Option.fold ~none:context - ~some:(Fun.flip SpanContext.with_trace_context context) + (* If trace_context is provided to the call, override any inherited trace + context except span.depth which should still be maintained. *) + match trace_context with + | Some tc -> + let tc_with_depth = + TraceContext.(with_added_baggage depth_key (string_of_int depth) tc) + in + SpanContext.with_trace_context tc_with_depth context + | None -> + context in (* Using gettimeofday over Mtime as it is better for sharing timestamps between the systems *) let begin_time = Unix.gettimeofday () in @@ -473,6 +503,11 @@ module Spans = struct let set_max_traces x = Atomic.set max_traces x + (* Default is much larger than the largest current traces, so effectively off *) + let max_depth = Atomic.make 100 + + let set_max_depth x = Atomic.set max_depth x + let finished_spans = Atomic.make ([], 0) let span_hashtbl_is_empty () = TraceMap.is_empty (Atomic.get spans) @@ -713,12 +748,18 @@ module Tracer = struct let get_tracer ~name:_ = TracerProvider.get_current () let span_of_span_context context name : Span.t = + let tc = SpanContext.context_of_span_context context in + let new_depth = TraceContext.baggage_depth_of tc in + let new_tc = + TraceContext.(with_added_baggage depth_key (string_of_int new_depth) tc) + in + let context = SpanContext.with_trace_context new_tc context in { context ; status= {status_code= Status.Unset; _description= None} ; name ; parent= None - ; span_kind= SpanKind.Client (* This will be the span of the client call*) + ; span_kind= SpanKind.Client (* This will be the span of the client call *) ; begin_time= Unix.gettimeofday () ; end_time= None ; links= [] @@ -730,10 +771,23 @@ module Tracer = struct ?(span_kind = SpanKind.Internal) ~name ~parent () : (Span.t option, exn) result = let open TracerProvider in - (* Do not start span if the TracerProvider is disabled*) + let parent_depth = + Option.fold ~none:1 + ~some:(fun parent -> + parent.Span.context + |> SpanContext.context_of_span_context + |> TraceContext.baggage_depth_of + ) + parent + in + (* Do not start span if the TracerProvider is disabled *) if not t.enabled then + ok_none (* Do not start span if the max depth has been reached *) + else if parent_depth >= Atomic.get Spans.max_depth then ( + let parent_trace_id = Option.fold ~none:"None" ~some:(fun p -> p.Span.context |> SpanContext.span_id_of_span_context |> Span_id.to_string) parent in + debug "Max_span_depth limit reached, not creating span %s (parent %s)" name parent_trace_id ; ok_none - else + ) else let attributes = Attributes.merge_into t.attributes attributes in let span = Span.start ~attributes ?trace_context ~name ~parent ~span_kind () @@ -750,8 +804,17 @@ module Tracer = struct |> Spans.remove_from_spans |> Option.map (fun existing_span -> let old_context = Span.get_context existing_span in + let parent_trace_context = Span.get_trace_context parent in + let new_depth = + TraceContext.baggage_depth_of parent_trace_context + 1 + in let new_context : SpanContext.t = - let trace_context = span.Span.context.trace_context in + let trace_context = + TraceContext.( + with_added_baggage depth_key (string_of_int new_depth) + span.Span.context.trace_context + ) + in SpanContext.context (SpanContext.trace_id_of_span_context parent.context) old_context.span_id @@ -759,7 +822,6 @@ module Tracer = struct in let updated_span = {existing_span with parent= Some parent} in let updated_span = {updated_span with context= new_context} in - let () = Spans.add_to_spans ~span:updated_span in updated_span ) @@ -926,7 +988,15 @@ module Propagator = struct let trace_context' = TraceContext.with_traceparent (Some traceparent) trace_context in - let carrier' = P.inject_into trace_context' carrier in + let new_depth = + TraceContext.baggage_depth_of trace_context' + 1 |> string_of_int + in + let trace_context'' = + TraceContext.( + with_added_baggage depth_key new_depth trace_context' + ) + in + let carrier' = P.inject_into trace_context'' carrier in f carrier' | _ -> f carrier diff --git a/ocaml/libs/tracing/tracing.mli b/ocaml/libs/tracing/tracing.mli index 8323346a443..ec33f4ac5ff 100644 --- a/ocaml/libs/tracing/tracing.mli +++ b/ocaml/libs/tracing/tracing.mli @@ -165,6 +165,8 @@ module Spans : sig val set_max_traces : int -> unit + val set_max_depth : int -> unit + val span_count : unit -> int val since : unit -> Span.t list * int diff --git a/ocaml/tests/test_observer.ml b/ocaml/tests/test_observer.ml index 2e2f8e6aa29..07d746e81c0 100644 --- a/ocaml/tests/test_observer.ml +++ b/ocaml/tests/test_observer.ml @@ -305,6 +305,7 @@ let verify_json_fields_and_values ~json = ; ("xs.host.uuid", `String _) ; ("xs.host.name", `String _) ; ("service.name", `String _) + ; ("span.depth", `String _) ] ) ; ("annotations", `List _) diff --git a/ocaml/xapi/xapi_globs.ml b/ocaml/xapi/xapi_globs.ml index 3688478dce8..f32d61443bc 100644 --- a/ocaml/xapi/xapi_globs.ml +++ b/ocaml/xapi/xapi_globs.ml @@ -1059,6 +1059,8 @@ let max_spans = ref 10000 let max_traces = ref 10000 +let max_span_depth = ref 100 + let use_xmlrpc = ref true let compress_tracing_files = ref true @@ -1783,6 +1785,11 @@ let other_options = , (fun () -> string_of_float !vm_sysprep_wait) , "Time in seconds to wait for VM to recognise inserted CD" ) + ; ( "max-span-depth" + , Arg.Set_int max_span_depth + , (fun () -> string_of_int !max_span_depth) + , "The maximum depth to which spans are recorded in a trace in Tracing" + ) ] (* The options can be set with the variable xapiflags in /etc/sysconfig/xapi. diff --git a/ocaml/xapi/xapi_observer.ml b/ocaml/xapi/xapi_observer.ml index 62d3ea4359c..a0f1f453b7d 100644 --- a/ocaml/xapi/xapi_observer.ml +++ b/ocaml/xapi/xapi_observer.ml @@ -599,6 +599,7 @@ let initialise_observer ~__context component = initialise_observer_component ~__context component let initialise ~__context = + Tracing.Spans.set_max_depth !Xapi_globs.max_span_depth ; List.iter (initialise_observer_meta ~__context) (startup_components ()) ; Db.Observer.get_all ~__context |> List.iter (fun self -> From 38767739e4fdc6a15daf0f3bd5207efbcbe1cc2d Mon Sep 17 00:00:00 2001 From: Steven Woods Date: Fri, 1 Aug 2025 17:02:59 +0100 Subject: [PATCH 2/3] CP-309305: Split Spans.since into chunks for exporting Http exporting appears to get overwhelmed when too many spans are exported at the same time. This adds the option to export a smaller chunk of spans at a time. This also reduces the size of the file exports as we only check for max file size after exporting all of the finished spans. Signed-off-by: Steven Woods --- ocaml/libs/tracing/tracing.ml | 23 ++++++++++++------ ocaml/libs/tracing/tracing_export.ml | 35 ++++++++++++++++++++++++--- ocaml/libs/tracing/tracing_export.mli | 7 ++++++ ocaml/xapi/xapi_globs.ml | 7 ++++++ ocaml/xapi/xapi_observer.ml | 1 + 5 files changed, 62 insertions(+), 11 deletions(-) diff --git a/ocaml/libs/tracing/tracing.ml b/ocaml/libs/tracing/tracing.ml index 389c5bafaa3..78ba3bc3ab6 100644 --- a/ocaml/libs/tracing/tracing.ml +++ b/ocaml/libs/tracing/tracing.ml @@ -351,7 +351,7 @@ module Span = struct let span_id = Span_id.make () in let extra_context_with_depth = TraceContext.( - with_added_baggage depth_key (string_of_int depth) extra_context + update_with_baggage depth_key (string_of_int depth) extra_context ) in let context : SpanContext.t = @@ -363,7 +363,7 @@ module Span = struct match trace_context with | Some tc -> let tc_with_depth = - TraceContext.(with_added_baggage depth_key (string_of_int depth) tc) + TraceContext.(update_with_baggage depth_key (string_of_int depth) tc) in SpanContext.with_trace_context tc_with_depth context | None -> @@ -751,7 +751,7 @@ module Tracer = struct let tc = SpanContext.context_of_span_context context in let new_depth = TraceContext.baggage_depth_of tc in let new_tc = - TraceContext.(with_added_baggage depth_key (string_of_int new_depth) tc) + TraceContext.(update_with_baggage depth_key (string_of_int new_depth) tc) in let context = SpanContext.with_trace_context new_tc context in { @@ -784,8 +784,17 @@ module Tracer = struct if not t.enabled then ok_none (* Do not start span if the max depth has been reached *) else if parent_depth >= Atomic.get Spans.max_depth then ( - let parent_trace_id = Option.fold ~none:"None" ~some:(fun p -> p.Span.context |> SpanContext.span_id_of_span_context |> Span_id.to_string) parent in - debug "Max_span_depth limit reached, not creating span %s (parent %s)" name parent_trace_id ; + let parent_trace_id = + Option.fold ~none:"None" + ~some:(fun p -> + p.Span.context + |> SpanContext.span_id_of_span_context + |> Span_id.to_string + ) + parent + in + debug "Max_span_depth limit reached, not creating span %s (parent %s)" + name parent_trace_id ; ok_none ) else let attributes = Attributes.merge_into t.attributes attributes in @@ -811,7 +820,7 @@ module Tracer = struct let new_context : SpanContext.t = let trace_context = TraceContext.( - with_added_baggage depth_key (string_of_int new_depth) + update_with_baggage depth_key (string_of_int new_depth) span.Span.context.trace_context ) in @@ -993,7 +1002,7 @@ module Propagator = struct in let trace_context'' = TraceContext.( - with_added_baggage depth_key new_depth trace_context' + update_with_baggage depth_key new_depth trace_context' ) in let carrier' = P.inject_into trace_context'' carrier in diff --git a/ocaml/libs/tracing/tracing_export.ml b/ocaml/libs/tracing/tracing_export.ml index 1162202b611..352d5d488e4 100644 --- a/ocaml/libs/tracing/tracing_export.ml +++ b/ocaml/libs/tracing/tracing_export.ml @@ -24,6 +24,10 @@ let export_interval = ref 30. let set_export_interval t = export_interval := t +let export_chunk_size = Atomic.make 10000 + +let set_export_chunk_size x = Atomic.set export_chunk_size x + let host_id = ref "localhost" let set_host_id id = host_id := id @@ -289,6 +293,22 @@ module Destination = struct with exn -> debug "Tracing: unable to export span : %s" (Printexc.to_string exn) + let rec span_info_chunks span_info batch_size = + let rec list_to_chunks_inner l n curr chunks = + if n = 0 then + if l <> [] then + list_to_chunks_inner l batch_size [] ((curr, batch_size) :: chunks) + else + (curr, batch_size) :: chunks + else + match l with + | [] -> + (curr, List.length curr) :: chunks + | h :: t -> + list_to_chunks_inner t (n - 1) (h :: curr) chunks + in + list_to_chunks_inner (fst span_info) batch_size [] [] + let flush_spans () = let ((_span_list, span_count) as span_info) = Spans.since () in let attributes = [("export.traces.count", string_of_int span_count)] in @@ -296,10 +316,17 @@ module Destination = struct with_tracing ~span_kind:Server ~trace_context:TraceContext.empty ~parent:None ~attributes ~name:"Tracing.flush_spans" in - TracerProvider.get_tracer_providers () - |> List.filter TracerProvider.get_enabled - |> List.concat_map TracerProvider.get_endpoints - |> List.iter (export_to_endpoint parent span_info) + let endpoints = + TracerProvider.get_tracer_providers () + |> List.filter TracerProvider.get_enabled + |> List.concat_map TracerProvider.get_endpoints + in + let span_info_chunks = + span_info_chunks span_info (Atomic.get export_chunk_size) + in + List.iter + (fun s_i -> List.iter (export_to_endpoint parent s_i) endpoints) + span_info_chunks let delay = Delay.make () diff --git a/ocaml/libs/tracing/tracing_export.mli b/ocaml/libs/tracing/tracing_export.mli index f322bd2404c..a857a4f523d 100644 --- a/ocaml/libs/tracing/tracing_export.mli +++ b/ocaml/libs/tracing/tracing_export.mli @@ -23,6 +23,13 @@ val set_export_interval : float -> unit Default is every [30.] seconds. *) +val set_export_chunk_size : int -> unit +(** [set_export_chunk_size size] sets the maximum number of finished spans that + can be exported in one chunk to [size]. + + Default is 10000 spans. + *) + val set_host_id : string -> unit (** [set_host_id id] sets the id of the host to [id]. diff --git a/ocaml/xapi/xapi_globs.ml b/ocaml/xapi/xapi_globs.ml index f32d61443bc..de2d68829fb 100644 --- a/ocaml/xapi/xapi_globs.ml +++ b/ocaml/xapi/xapi_globs.ml @@ -1055,6 +1055,8 @@ let trace_log_dir = ref "/var/log/dt/zipkinv2/json" let export_interval = ref 30. +let export_chunk_size = ref 10000 + let max_spans = ref 10000 let max_traces = ref 10000 @@ -1678,6 +1680,11 @@ let other_options = , (fun () -> string_of_float !export_interval) , "The interval for exports in Tracing" ) + ; ( "export-chunk-size" + , Arg.Set_int export_chunk_size + , (fun () -> string_of_int !export_chunk_size) + , "The span chunk size for exports in Tracing" + ) ; ( "max-spans" , Arg.Set_int max_spans , (fun () -> string_of_int !max_spans) diff --git a/ocaml/xapi/xapi_observer.ml b/ocaml/xapi/xapi_observer.ml index a0f1f453b7d..7a7163ff42f 100644 --- a/ocaml/xapi/xapi_observer.ml +++ b/ocaml/xapi/xapi_observer.ml @@ -600,6 +600,7 @@ let initialise_observer ~__context component = let initialise ~__context = Tracing.Spans.set_max_depth !Xapi_globs.max_span_depth ; + Tracing_export.set_export_chunk_size !Xapi_globs.export_chunk_size ; List.iter (initialise_observer_meta ~__context) (startup_components ()) ; Db.Observer.get_all ~__context |> List.iter (fun self -> From 78df7443fbb2c4ad5ac32b575586594ab8100620 Mon Sep 17 00:00:00 2001 From: Steven Woods Date: Mon, 1 Sep 2025 14:12:10 +0100 Subject: [PATCH 3/3] Use a forwarder so each component updates their depth and chunk size The other Observer components e.g. xenops need a forwarder to notify them of any changes to the export_chunk_size and max_depth as they do not have access to xapi_globs/xapi.conf Signed-off-by: Steven Woods --- ocaml/tests/test_cluster.ml | 2 ++ ocaml/xapi-idl/cluster/cli-help.t | 4 +++ ocaml/xapi-idl/lib/observer_helpers.ml | 16 ++++++++++ ocaml/xapi-idl/lib/observer_helpers.mli | 21 +++++++++++++ ocaml/xapi-idl/lib/observer_skeleton.ml | 4 +++ ocaml/xapi-idl/lib/observer_skeleton.mli | 4 +++ ocaml/xapi/xapi_observer.ml | 40 ++++++++++++++++++++++-- ocaml/xapi/xapi_xenops.ml | 10 ++++++ ocaml/xenopsd/lib/xenops_server.ml | 14 +++++++++ 9 files changed, 113 insertions(+), 2 deletions(-) diff --git a/ocaml/tests/test_cluster.ml b/ocaml/tests/test_cluster.ml index d24e36fe72c..9c945776cd9 100644 --- a/ocaml/tests/test_cluster.ml +++ b/ocaml/tests/test_cluster.ml @@ -34,9 +34,11 @@ let test_clusterd_rpc ~__context call = | "Observer.init" | "Observer.set_trace_log_dir" | "Observer.set_export_interval" + | "Observer.set_export_chunk_size" | "Observer.set_host_id" | "Observer.set_max_traces" | "Observer.set_max_spans" + | "Observer.set_max_depth" | "Observer.set_max_file_size" | "Observer.set_compress_tracing_files" ) , _ ) -> diff --git a/ocaml/xapi-idl/cluster/cli-help.t b/ocaml/xapi-idl/cluster/cli-help.t index 5b9362aa648..abe729544da 100644 --- a/ocaml/xapi-idl/cluster/cli-help.t +++ b/ocaml/xapi-idl/cluster/cli-help.t @@ -21,10 +21,14 @@ Observer.set_endpoints [OPTION]… dbg uuid endpoints + Observer.set_export_chunk_size [OPTION]… dbg int + Observer.set_export_interval [OPTION]… dbg float Observer.set_host_id [OPTION]… dbg string + Observer.set_max_depth [OPTION]… dbg int + Observer.set_max_file_size [OPTION]… dbg int Observer.set_max_spans [OPTION]… dbg int diff --git a/ocaml/xapi-idl/lib/observer_helpers.ml b/ocaml/xapi-idl/lib/observer_helpers.ml index 24f7ee3db46..c2ea58bb8d3 100644 --- a/ocaml/xapi-idl/lib/observer_helpers.ml +++ b/ocaml/xapi-idl/lib/observer_helpers.ml @@ -138,6 +138,10 @@ module ObserverAPI (R : RPC) = struct declare "Observer.set_export_interval" [] (dbg_p @-> float_p @-> returning unit_p err) + let set_export_chunk_size = + declare "Observer.set_export_chunk_size" [] + (dbg_p @-> int_p @-> returning unit_p err) + let set_max_spans = declare "Observer.set_max_spans" [] (dbg_p @-> int_p @-> returning unit_p err) @@ -146,6 +150,10 @@ module ObserverAPI (R : RPC) = struct declare "Observer.set_max_traces" [] (dbg_p @-> int_p @-> returning unit_p err) + let set_max_depth = + declare "Observer.set_max_depth" [] + (dbg_p @-> int_p @-> returning unit_p err) + let set_max_file_size = declare "Observer.set_max_file_size" [] (dbg_p @-> int_p @-> returning unit_p err) @@ -193,10 +201,14 @@ module type Server_impl = sig val set_export_interval : context -> dbg:debug_info -> interval:float -> unit + val set_export_chunk_size : context -> dbg:debug_info -> size:int -> unit + val set_max_spans : context -> dbg:debug_info -> spans:int -> unit val set_max_traces : context -> dbg:debug_info -> traces:int -> unit + val set_max_depth : context -> dbg:debug_info -> depth:int -> unit + val set_max_file_size : context -> dbg:debug_info -> file_size:int -> unit val set_host_id : context -> dbg:debug_info -> host_id:string -> unit @@ -227,8 +239,12 @@ module Server (Impl : Server_impl) () = struct S.set_export_interval (fun dbg interval -> Impl.set_export_interval () ~dbg ~interval ) ; + S.set_export_chunk_size (fun dbg size -> + Impl.set_export_chunk_size () ~dbg ~size + ) ; S.set_max_spans (fun dbg spans -> Impl.set_max_spans () ~dbg ~spans) ; S.set_max_traces (fun dbg traces -> Impl.set_max_traces () ~dbg ~traces) ; + S.set_max_depth (fun dbg depth -> Impl.set_max_depth () ~dbg ~depth) ; S.set_max_file_size (fun dbg file_size -> Impl.set_max_file_size () ~dbg ~file_size ) ; diff --git a/ocaml/xapi-idl/lib/observer_helpers.mli b/ocaml/xapi-idl/lib/observer_helpers.mli index cd23d2d1e80..489310a0847 100644 --- a/ocaml/xapi-idl/lib/observer_helpers.mli +++ b/ocaml/xapi-idl/lib/observer_helpers.mli @@ -77,6 +77,11 @@ module ObserverAPI : functor (R : Idl.RPC) -> sig (** [set_export_interval dbg interval] notifies the fowarder that the interval between trace exports has been set to [interval]. *) + val set_export_chunk_size : + (debug_info -> int -> (unit, Errors.error) R.comp) R.res + (** [set_export_chunk_size dbg size] notifies the fowarder that the max size + of each chunk of finished spans exported has been set to [size]. *) + val set_max_spans : (debug_info -> int -> (unit, Errors.error) R.comp) R.res (** [set_max_spans dbg spans] notifies the fowarder that the max number of spans has been set to [spans]. *) @@ -85,6 +90,10 @@ module ObserverAPI : functor (R : Idl.RPC) -> sig (** [set_max_traces dbg traces] notifies the fowarder that the max number of traces has been set to [traces]. *) + val set_max_depth : (debug_info -> int -> (unit, Errors.error) R.comp) R.res + (** [set_max_depth dbg depth] notifies the fowarder that the max depth of + a span in a trace has been set to [depth]. *) + val set_max_file_size : (debug_info -> int -> (unit, Errors.error) R.comp) R.res (** [set_max_file_size dbg file_size] notifies the fowarder that the max file @@ -135,10 +144,14 @@ module type Server_impl = sig val set_export_interval : context -> dbg:debug_info -> interval:float -> unit + val set_export_chunk_size : context -> dbg:debug_info -> size:int -> unit + val set_max_spans : context -> dbg:debug_info -> spans:int -> unit val set_max_traces : context -> dbg:debug_info -> traces:int -> unit + val set_max_depth : context -> dbg:debug_info -> depth:int -> unit + val set_max_file_size : context -> dbg:debug_info -> file_size:int -> unit val set_host_id : context -> dbg:debug_info -> host_id:string -> unit @@ -176,10 +189,14 @@ module Server : functor (_ : Server_impl) () -> sig val set_export_interval : (debug_info -> float -> unit) -> unit + val set_export_chunk_size : (debug_info -> int -> unit) -> unit + val set_max_spans : (debug_info -> int -> unit) -> unit val set_max_traces : (debug_info -> int -> unit) -> unit + val set_max_depth : (debug_info -> int -> unit) -> unit + val set_max_file_size : (debug_info -> int -> unit) -> unit val set_host_id : (debug_info -> string -> unit) -> unit @@ -215,10 +232,14 @@ module Client : sig val set_export_interval : debug_info -> float -> unit + val set_export_chunk_size : debug_info -> int -> unit + val set_max_spans : debug_info -> int -> unit val set_max_traces : debug_info -> int -> unit + val set_max_depth : debug_info -> int -> unit + val set_max_file_size : debug_info -> int -> unit val set_host_id : debug_info -> string -> unit diff --git a/ocaml/xapi-idl/lib/observer_skeleton.ml b/ocaml/xapi-idl/lib/observer_skeleton.ml index e53a45f958c..59df66d246e 100644 --- a/ocaml/xapi-idl/lib/observer_skeleton.ml +++ b/ocaml/xapi-idl/lib/observer_skeleton.ml @@ -36,10 +36,14 @@ module Observer = struct let set_export_interval ctx ~dbg ~interval = unimplemented __FUNCTION__ + let set_export_chunk_size ctx ~dbg ~size = unimplemented __FUNCTION__ + let set_max_spans ctx ~dbg ~spans = unimplemented __FUNCTION__ let set_max_traces ctx ~dbg ~traces = unimplemented __FUNCTION__ + let set_max_depth ctx ~dbg ~depth = unimplemented __FUNCTION__ + let set_max_file_size ctx ~dbg ~file_size = unimplemented __FUNCTION__ let set_host_id ctx ~dbg ~host_id = unimplemented __FUNCTION__ diff --git a/ocaml/xapi-idl/lib/observer_skeleton.mli b/ocaml/xapi-idl/lib/observer_skeleton.mli index c99b77f9a34..2b914ada71d 100644 --- a/ocaml/xapi-idl/lib/observer_skeleton.mli +++ b/ocaml/xapi-idl/lib/observer_skeleton.mli @@ -34,10 +34,14 @@ module Observer : sig val set_export_interval : context -> dbg:string -> interval:float -> unit + val set_export_chunk_size : context -> dbg:string -> size:int -> unit + val set_max_spans : context -> dbg:string -> spans:int -> unit val set_max_traces : context -> dbg:string -> traces:int -> unit + val set_max_depth : context -> dbg:string -> depth:int -> unit + val set_max_file_size : context -> dbg:string -> file_size:int -> unit val set_host_id : context -> dbg:string -> host_id:string -> unit diff --git a/ocaml/xapi/xapi_observer.ml b/ocaml/xapi/xapi_observer.ml index 7a7163ff42f..073e920cba2 100644 --- a/ocaml/xapi/xapi_observer.ml +++ b/ocaml/xapi/xapi_observer.ml @@ -48,10 +48,14 @@ module type ObserverInterface = sig val set_export_interval : __context:Context.t -> interval:float -> unit + val set_export_chunk_size : __context:Context.t -> size:int -> unit + val set_max_spans : __context:Context.t -> spans:int -> unit val set_max_traces : __context:Context.t -> traces:int -> unit + val set_max_depth : __context:Context.t -> depth:int -> unit + val set_max_file_size : __context:Context.t -> file_size:int -> unit val set_host_id : __context:Context.t -> host_id:string -> unit @@ -93,6 +97,10 @@ module Observer : ObserverInterface = struct debug "xapi Observer.set_export_interval" ; Tracing_export.set_export_interval interval + let set_export_chunk_size ~__context ~size = + debug "xapi Observer.set_export_chunk_size" ; + Tracing_export.set_export_chunk_size size + let set_max_spans ~__context ~spans = debug "xapi Observer.set_max_spans" ; Tracing.Spans.set_max_spans spans @@ -101,6 +109,10 @@ module Observer : ObserverInterface = struct debug "xapi Observer.set_max_traces" ; Tracing.Spans.set_max_traces traces + let set_max_depth ~__context ~depth = + debug "xapi Observer.set_max_depth" ; + Tracing.Spans.set_max_depth depth + let set_max_file_size ~__context ~file_size = debug "xapi Observer.set_max_file_size" ; Tracing_export.Destination.File.set_max_file_size file_size @@ -189,6 +201,12 @@ module Xapi_cluster = struct let dbg = Context.string_of_task __context in S.Observer.set_export_interval dbg interval + let set_export_chunk_size ~__context ~size = + debug "xapi_cluster Observer.set_export_chunk_size" ; + let module S = (val local_client ~__context : XAPI_CLUSTER) in + let dbg = Context.string_of_task __context in + S.Observer.set_export_chunk_size dbg size + let set_max_spans ~__context ~spans = debug "xapi_cluster Observer.set_max_spans" ; let module S = (val local_client ~__context : XAPI_CLUSTER) in @@ -201,6 +219,12 @@ module Xapi_cluster = struct let dbg = Context.string_of_task __context in S.Observer.set_max_traces dbg traces + let set_max_depth ~__context ~depth = + debug "xapi_cluster Observer.set_max_depth" ; + let module S = (val local_client ~__context : XAPI_CLUSTER) in + let dbg = Context.string_of_task __context in + S.Observer.set_max_depth dbg depth + let set_max_file_size ~__context ~file_size = debug "xapi_cluster Observer.set_max_file_size" ; let module S = (val local_client ~__context : XAPI_CLUSTER) in @@ -370,10 +394,14 @@ module Dom0ObserverConfig (ObserverComponent : OBSERVER_COMPONENT) : let set_export_interval ~__context:_ ~interval:_ = () + let set_export_chunk_size ~__context:_ ~size:_ = () + let set_max_spans ~__context:_ ~spans:_ = () let set_max_traces ~__context:_ ~traces:_ = () + let set_max_depth ~__context:_ ~depth:_ = () + let set_max_file_size ~__context:_ ~file_size:_ = () let set_host_id ~__context:_ ~host_id:_ = () @@ -542,6 +570,10 @@ let set_export_interval ~__context interval component = let module Forwarder = (val get_forwarder component : ObserverInterface) in Forwarder.set_export_interval ~__context ~interval +let set_export_chunk_size ~__context size component = + let module Forwarder = (val get_forwarder component : ObserverInterface) in + Forwarder.set_export_chunk_size ~__context ~size + let set_max_spans ~__context spans component = let module Forwarder = (val get_forwarder component : ObserverInterface) in Forwarder.set_max_spans ~__context ~spans @@ -550,6 +582,10 @@ let set_max_traces ~__context traces component = let module Forwarder = (val get_forwarder component : ObserverInterface) in Forwarder.set_max_traces ~__context ~traces +let set_max_depth ~__context depth component = + let module Forwarder = (val get_forwarder component : ObserverInterface) in + Forwarder.set_max_depth ~__context ~depth + let set_max_file_size ~__context file_size component = let module Forwarder = (val get_forwarder component : ObserverInterface) in Forwarder.set_max_file_size ~__context ~file_size @@ -585,8 +621,10 @@ let initialise_observer_component ~__context component = let initialise_observer_meta ~__context component = set_trace_log_dir ~__context !Xapi_globs.trace_log_dir component ; set_export_interval ~__context !Xapi_globs.export_interval component ; + set_export_chunk_size ~__context !Xapi_globs.export_chunk_size component ; set_max_spans ~__context !Xapi_globs.max_spans component ; set_max_traces ~__context !Xapi_globs.max_traces component ; + set_max_depth ~__context !Xapi_globs.max_span_depth component ; set_max_file_size ~__context !Xapi_globs.max_observer_file_size component ; set_host_id ~__context (Helpers.get_localhost_uuid ()) component ; set_compress_tracing_files ~__context @@ -599,8 +637,6 @@ let initialise_observer ~__context component = initialise_observer_component ~__context component let initialise ~__context = - Tracing.Spans.set_max_depth !Xapi_globs.max_span_depth ; - Tracing_export.set_export_chunk_size !Xapi_globs.export_chunk_size ; List.iter (initialise_observer_meta ~__context) (startup_components ()) ; Db.Observer.get_all ~__context |> List.iter (fun self -> diff --git a/ocaml/xapi/xapi_xenops.ml b/ocaml/xapi/xapi_xenops.ml index 7487d723ab3..9b12bcec5a6 100644 --- a/ocaml/xapi/xapi_xenops.ml +++ b/ocaml/xapi/xapi_xenops.ml @@ -4493,6 +4493,11 @@ module Observer = struct let dbg = Context.string_of_task __context in Client.Observer.set_export_interval dbg interval + let set_export_chunk_size ~__context ~size = + let module Client = (val make_client (default_xenopsd ()) : XENOPS) in + let dbg = Context.string_of_task __context in + Client.Observer.set_export_chunk_size dbg size + let set_max_spans ~__context ~spans = let module Client = (val make_client (default_xenopsd ()) : XENOPS) in let dbg = Context.string_of_task __context in @@ -4503,6 +4508,11 @@ module Observer = struct let dbg = Context.string_of_task __context in Client.Observer.set_max_traces dbg traces + let set_max_depth ~__context ~depth = + let module Client = (val make_client (default_xenopsd ()) : XENOPS) in + let dbg = Context.string_of_task __context in + Client.Observer.set_max_depth dbg depth + let set_max_file_size ~__context ~file_size = let module Client = (val make_client (default_xenopsd ()) : XENOPS) in let dbg = Context.string_of_task __context in diff --git a/ocaml/xenopsd/lib/xenops_server.ml b/ocaml/xenopsd/lib/xenops_server.ml index b47344a30e6..6a06b36ba14 100644 --- a/ocaml/xenopsd/lib/xenops_server.ml +++ b/ocaml/xenopsd/lib/xenops_server.ml @@ -4311,6 +4311,12 @@ module Observer = struct (fun () -> Tracing_export.set_export_interval interval) () + let set_export_chunk_size _ dbg size = + debug "Observer.set_export_chunk_size : dbg=%s" dbg ; + Debug.with_thread_associated dbg + (fun () -> Tracing_export.set_export_chunk_size size) + () + let set_max_spans _ dbg spans = debug "Observer.set_max_spans : dbg=%s" dbg ; Debug.with_thread_associated dbg @@ -4323,6 +4329,12 @@ module Observer = struct (fun () -> Tracing.Spans.set_max_traces traces) () + let set_max_depth _ dbg depth = + debug "Observer.set_max_depth : dbg=%s" dbg ; + Debug.with_thread_associated dbg + (fun () -> Tracing.Spans.set_max_depth depth) + () + let set_max_file_size _ dbg file_size = debug "Observer.set_max_file_size : dbg=%s" dbg ; Debug.with_thread_associated dbg @@ -4446,8 +4458,10 @@ let _ = Server.Observer.init (Observer.init ()) ; Server.Observer.set_trace_log_dir (Observer.set_trace_log_dir ()) ; Server.Observer.set_export_interval (Observer.set_export_interval ()) ; + Server.Observer.set_export_chunk_size (Observer.set_export_chunk_size ()) ; Server.Observer.set_max_spans (Observer.set_max_spans ()) ; Server.Observer.set_max_traces (Observer.set_max_traces ()) ; + Server.Observer.set_max_depth (Observer.set_max_depth ()) ; Server.Observer.set_max_file_size (Observer.set_max_file_size ()) ; Server.Observer.set_host_id (Observer.set_host_id ()) ; Server.Observer.set_compress_tracing_files