Skip to content

Commit f1617f4

Browse files
jpsamaroojpsamaroo
committed
MetricsTracker: Add metrics tracking package
1 parent 62f8307 commit f1617f4

File tree

11 files changed

+739
-0
lines changed

11 files changed

+739
-0
lines changed

‎lib/MetricsTracker/LICENSE.md‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
MetricsTracker.jl is licensed under the MIT "Expat" License:
2+
3+
> Copyright (c) 2024: Julian P Samaroo and contributors
4+
>
5+
> Permission is hereby granted, free of charge, to any person obtaining
6+
> a copy of this software and associated documentation files (the
7+
> "Software"), to deal in the Software without restriction, including
8+
> without limitation the rights to use, copy, modify, merge, publish,
9+
> distribute, sublicense, and/or sell copies of the Software, and to
10+
> permit persons to whom the Software is furnished to do so, subject to
11+
> the following conditions:
12+
>
13+
> The above copyright notice and this permission notice shall be
14+
> included in all copies or substantial portions of the Software.
15+
>
16+
> THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
17+
> EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
18+
> MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
19+
> IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
20+
> CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
21+
> TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
22+
> SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

‎lib/MetricsTracker/Project.toml‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
name = "MetricsTracker"
2+
uuid = "9a9c6fec-044d-4a27-aa18-2b01ca4026eb"
3+
authors = ["Julian P Samaroo <[email protected]>"]
4+
version = "0.1.0"
5+
6+
[deps]
7+
MacroTools = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09"
8+
ScopedValues = "7e506255-f358-4e82-b7e4-beb19740aa63"
9+
Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b"
10+
TaskLocalValues = "ed4db957-447d-4319-bfb6-7fa9ae7ecf34"
11+
12+
[compat]
13+
MacroTools = "0.5.13"
14+
ScopedValues = "1.2.1"
15+
Serialization = "1.11.0"
16+
TaskLocalValues = "0.1.1"

‎lib/MetricsTracker/src/MetricsTracker.jl‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
module MetricsTracker
2+
3+
import MacroTools: @capture
4+
import ScopedValues: ScopedValue, @with
5+
import TaskLocalValues: TaskLocalValue
6+
7+
include("types.jl")
8+
include("metrics.jl")
9+
include("lookup.jl")
10+
include("io.jl")
11+
include("builtins.jl")
12+
# FIXME
13+
#include("analysis.jl")
14+
#include("aggregate.jl")
15+
#include("decision.jl")
16+
17+
end # module MetricsTracker

‎lib/MetricsTracker/src/aggregate.jl‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
abstract type AbstractAggregator <: AbstractAnalysis end
2+
3+
#### Built-in Aggregators ####
4+
5+
struct SimpleAverageAggregator{T} <: AbstractAggregator
6+
inner::T
7+
end
8+
required_metrics(agg::SimpleAverageAggregator, ::Val{context}, ::Val{op}) where {context,op} =
9+
RequiredMetrics((context, op) => [agg.inner])
10+
function run_analysis(agg::SimpleAverageAggregator, ::Val{context}, ::Val{op}, @nospecialize(args...)) where {context,op}
11+
prev = fetch_metric_cached(agg, context, op, args...)
12+
next = fetch_metric(agg.inner, context, op, args...)
13+
if prev === nothing || next === nothing
14+
return next
15+
end
16+
return (prev + next) ÷ 2
17+
end

‎lib/MetricsTracker/src/analysis.jl‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
const RequiredMetrics = Dict{Tuple{Module,Symbol},Vector{AnalysisOrMetric}}
2+
const RequiredMetricsAny = Vector{AnalysisOrMetric}
3+
const NO_REQUIRED_METRICS = RequiredMetrics()
4+
required_metrics(::AnalysisOrMetric, _, _) = NO_REQUIRED_METRICS
5+
6+
function fetch_metric(m::AnalysisOrMetric, mod::Module, context::Symbol, key, extra; cached=false)
7+
@assert !COLLECTING_METRICS[] "Nesting analysis and metrics collection not yet supported"
8+
# Check if this is already cached
9+
cache = local_metrics_cache(mod, context, key)
10+
if cached
11+
return cache[m]
12+
end
13+
# FIXME: Proper invalidation support
14+
if m isa AbstractMetric
15+
if haskey(cache, m)
16+
value = cache[m]
17+
@debug "-- HIT for ($mod, $context) $m [$key] = $value"
18+
return value
19+
else
20+
# The metric isn't available yet
21+
@debug "-- MISS for ($mod, $context) $m [$key]"
22+
return nothing
23+
end
24+
elseif m isa AbstractAnalysis
25+
# Run the analysis
26+
@debug "Running ($mod, $context) $m [$key]"
27+
value = run_analysis(m, Val{nameof(mod)}(), Val{context}(), key, extra)
28+
# TODO: Allocate the correct Dict type
29+
get!(Dict, cache, m)[key] = value
30+
@debug "Finished ($mod, $context) $m [$key] = $value"
31+
return value
32+
end
33+
end
34+
35+
#### Built-in Analyses ####
36+
37+
struct RuntimeWithoutCompilation <: AbstractAnalysis end
38+
required_metrics(::RuntimeWithoutCompilation) =
39+
RequiredMetricsAny([TimeMetric(),
40+
CompileTimeMetric()])
41+
metric_type(::RuntimeWithoutCompilation) = UInt64
42+
function run_analysis(::RuntimeWithoutCompilation, mod, context, key, extra)
43+
time = fetch_metric(TimeMetric(), mod, context, key, extra)
44+
ctime = fetch_metric(CompileTimeMetric(), mod, context, key, extra)
45+
return time - ctime[1]
46+
end

‎lib/MetricsTracker/src/builtins.jl‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
#### Built-in Metrics ####
2+
3+
struct TimeMetric <: AbstractMetric end
4+
metric_applies(::TimeMetric, _) = true
5+
metric_type(::TimeMetric) = UInt64
6+
start_metric(::TimeMetric) = time_ns()
7+
stop_metric(::TimeMetric, last::UInt64) = time_ns() - last
8+
9+
struct ThreadTimeMetric <: AbstractMetric end
10+
metric_applies(::ThreadTimeMetric, _) = true
11+
metric_type(::ThreadTimeMetric) = UInt64
12+
start_metric(::ThreadTimeMetric) = cputhreadtime()
13+
stop_metric(::ThreadTimeMetric, last::UInt64) = cputhreadtime() - last
14+
15+
struct CompileTimeMetric <: AbstractMetric end
16+
metric_applies(::CompileTimeMetric, _) = true
17+
metric_type(::CompileTimeMetric) = Tuple{UInt64, UInt64}
18+
function start_metric(::CompileTimeMetric)
19+
Base.cumulative_compile_timing(true)
20+
return Base.cumulative_compile_time_ns()
21+
end
22+
function stop_metric(::CompileTimeMetric, last::Tuple{UInt64, UInt64})
23+
Base.cumulative_compile_timing(false)
24+
return Base.cumulative_compile_time_ns() .- last
25+
end
26+
27+
struct AllocMetric <: AbstractMetric end
28+
metric_applies(::AllocMetric, _) = true
29+
metric_type(::AllocMetric) = Base.GC_Diff
30+
start_metric(::AllocMetric) = Base.gc_num()
31+
stop_metric(::AllocMetric, last::Base.GC_Num) = Base.GC_Diff(Base.gc_num(), last)
32+
33+
struct ResultShapeMetric <: AbstractMetric end
34+
metric_applies(::ResultShapeMetric, _) = true
35+
metric_type(::ResultShapeMetric) = Union{Dims, Nothing}
36+
is_result_metric(::ResultShapeMetric) = true
37+
result_metric(m::ResultShapeMetric, result) =
38+
result isa AbstractArray ? size(result) : nothing
39+
40+
struct LoadAverageMetric <: AbstractMetric end
41+
metric_applies(::LoadAverageMetric, _) = true
42+
metric_type(::LoadAverageMetric) = Tuple{Float64, Float64, Float64}
43+
start_metric(::LoadAverageMetric) = nothing
44+
stop_metric(::LoadAverageMetric, _) = (Sys.loadavg()...,) ./ Sys.CPU_THREADS
45+
46+
# TODO: Useful metrics to add
47+
# perf performance counters
48+
# BPF probe-collected metrics

0 commit comments

Comments
 (0)