Merge pull request #29 from JuliaPerf/dev

Document updates, bug fixes and some other pending things

Author: Dvegrod

.perftests/t2-effmemtp.jl_PERFORMANCE.JLD2

@@ -1,16 +1,18 @@
 name = "PerfTest"
 uuid = "1dca261b-fc56-4a8c-a7e2-9798d8a75978"
 authors = ["Dvegrod <dvegrodu@gmail.com>, Samuel Omlin <samuel.omlin@cscs.ch>, and contributors"]
-version = "0.1.0"
+version = "0.1.5"
 
 [deps]
 BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
 Configurations = "5218b696-f38b-4ac9-8b61-a12ec717816d"
 CountFlops = "1db9610d-79e1-487a-8d40-77f3295c7593"
 CpuId = "adafc99b-e345-5852-983c-f28acb93d879"
 Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
+HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3"
 JLD2 = "033835bb-8acc-5ee8-8aae-3f567f8a3819"
 JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
+LibGit2 = "76f85450-5226-5b5a-8eaa-529ad045b433"
 LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
 MLStyle = "d8e11817-5142-5d16-987a-aa16d5891078"
 MacroTools = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09"

.perftests/t3-roofline.jl_PERFORMANCE.JLD2

@@ -22,7 +22,7 @@ MarkdownExtensions.expand_reflinks(reflinks; rootdir=DOCSRC)
 
 @info "Building documentation website using Documenter.jl..."
 makedocs(;
-    modules  = [PerfTest, PerfTest.Configuration],
+    modules  = [PerfTest, PerfTest.Configuration, PerfTest.BencherREST],
     authors  = "Daniel Sergio Vega Rodriguez, Samuel Omlin, and contributors",
     repo     = "https://github.com/JuliaPerf/PerfTest.jl/blob/{commit}{path}#{line}",
     sitename = "PerfTest.jl",
@@ -35,6 +35,7 @@ makedocs(;
         #assets           = [asset("https://img.shields.io/github/stars/JuliaPerf/PerfTest.jl.svg", class = :ico)],
         #warn_outdated    = true,
     ),
+    warnonly = [:missing_docs],
     pages   = [
         "Introduction"  => "index.md",
         "Usage"         => "usage.md",

.perftests/t5-recursive.jl_PERFORMANCE.JLD2

@@ -33,6 +33,43 @@ Order = [:function, :macro]
 ```
 #### Documentation
 ```@autodocs
-Modules = [PerfTest, PerfTest.Configuration, PerfTest.BencherInterface]
+Modules = [PerfTest, PerfTest.Configuration, PerfTest.BencherREST]
 Order   = [:function, :macro]
 ```
+
+
+## Translation ruleset and priority:
+
+PerfTest.jl translates a declarative recipe file for performance tests into a Julia expression that can actually execute them. The traslation prodedure is done by walking through the recipe code whilst trying to do pattern matching with a set of translation rules. The translation rules are totally ordered, which means that there is a clear hierarchy on which pattern is translated in case an expression fits two patterns at the same time. Below there is a full list of the rules, in decreasing order by priority. Given the way the macros in the package are declared there should not be ambiguous pattern matching but for further safety the priority is enforced.
+
+  1. @testset macro
+  2. @test macro
+  3. @test_throws macro
+  4. @test_logs macro
+  5. @inferred macro
+  6. @test_deprecated macro
+  7. @test_warn macro
+  8. @test_nowarn macro
+  9. @test_broken macro
+  10. @test_skip macro
+  11. @perftest macro
+  12. Back context token, internal use only
+  13. Prefix placement token, internal use only
+  14. Suffix placement token, internal use only
+  15. @config macro
+  16. @on_perftest_exec macro
+  17. @on_perftest_ignore macro
+  18. @define_memory_throughput macro
+  19. @define_metric macro
+  20. @define_benchmark macro
+  21  @roofline macro
+  22. @auxiliary_metric macro 
+  23. @perfcompare macro
+  24. Recursivity rule, used for includes inside testsets
+
+Inside a @perftest macro the expression has to be manipulated to be able tu use local variables in a global context (a requirement of [BenchmarkTools.jl]). The rules have the following priority:
+
+  1. Assignment pattern rule:  a = b
+  2. Function argument pattern rule:  f(arg)
+  3. Vectorized function call pattern rule: f.(arg)
+  4. Dot operator pattern rule: a.b

Project.toml

@@ -35,8 +35,6 @@ end
 
     a = rand(Int, N)
 
-    # To use a variable in macro formulas, it has to be exported (i.e. N)
-    @export_vars N
     # The ratio sets the threshold, being 1.0 the maximum empirical bandwidth and 0.6 = 60% of such maximum
     @define_eff_memory_throughput ratio=0.6 begin
         # The main block of the macro holds the formula for the bandwidth (therefore BYTES divided by SECONDS) on

docs/make.jl

@@ -37,8 +37,6 @@ end
     coeff = rand(N)
     x = 1.0
 
-    # To use a variable in macro formulas, it has to be exported (i.e. N)
-    @export_vars N
     @roofline actual_flops=:autoflop target_ratio=0.1 begin
         :autoflop / ((1 + N)*4)
     end

docs/src/api.MD

@@ -3,19 +3,30 @@ The package `PerfTest` provides the user with a performance regression unit test
 
 ## Dependencies
 `PerfTest` relies on:
- - [MacroTools]
- - [JLD2]
- - [MPI]
- - [STREAMBenchmark]
- - [GFlops]
- - [UnicodePlots]
- - [Test]
- - [Suppressor]
-
+  - BenchmarkTools
+  - Configurations
+  - CountFlops
+  - CpuId
+  - Dates
+  - HTTP
+  - JLD2
+  - JSON
+  - LibGit2
+  - LinearAlgebra
+  - MLStyle
+  - MacroTools
+  - Pkg
+  - Printf
+  - Revise
+  - STREAMBenchmark
+  - Suppressor
+  - TOML
+  - Test
+  - UnicodePlots
 ## Contributors
 This project has been developed as a Master's thesis by Daniel Sergio Vega Rodriguez.
 Thus, the contributors to this project have been so far:
-- Daniel Sergio Vega Rodriguez ([@Dvegrod](https://github.com/Dvegrod)), Università della Svizzera italiana (USI): developer
-- Dr. Samuel Omlin ([@omlins](https://github.com/omlins)), Swiss National Supercomputing Centre (CSCS), ETH Zurich: original idea and Master's thesis supervisor
-- Prof. Olaf Schenk, Università della Svizzera italiana (USI): Master's thesis supervisor
+- Daniel Sergio Vega Rodriguez ([@Dvegrod](https://github.com/Dvegrod)), Università della Svizzera italiana (USI): package developer
+- Dr. Samuel Omlin ([@omlins](https://github.com/omlins)), Swiss National Supercomputing Centre (CSCS), ETH Zurich: original idea and supervisor
+- Prof. Olaf Schenk, Università della Svizzera italiana (USI): PhD supervisor
 - Dr. Pasadakis Dimosthenis, Università della Svizzera italiana (USI): additional advisor

docs/src/examples/mock2-memorythroughput.md

@@ -9,6 +9,6 @@ There are a couple of things to keep into consideration when using the package:
 ## Features to be expected in the next versions:
 
 1. Easier access to performance suite results after execution
-2. Simplification of the package structure, there will be an emphasis on making the package easy to extend for unfamiliarised developers (hopefully by June 2025)
-3. Access to performance counter values through LIKWID (hopefully by June 2025)
+2. Simplification of the package structure, there will be an emphasis on making the package easy to extend for unfamiliarised developers
+3. Access to performance counter values through LIKWID
 4. Alternative regression testing against git commits instead of last execution for easier testing