From 1ce3da701259e07b4550acb06960e51c60486df7 Mon Sep 17 00:00:00 2001 From: Christoph Knittel Date: Fri, 11 Oct 2024 17:06:37 +0200 Subject: [PATCH 1/7] Syntax benchmark: cleanup --- tests/syntax_benchmarks/Benchmark.ml | 48 ++++++++++++---------------- 1 file changed, 20 insertions(+), 28 deletions(-) diff --git a/tests/syntax_benchmarks/Benchmark.ml b/tests/syntax_benchmarks/Benchmark.ml index 4f8e2db38a..80810c5b94 100644 --- a/tests/syntax_benchmarks/Benchmark.ml +++ b/tests/syntax_benchmarks/Benchmark.ml @@ -182,14 +182,8 @@ end = struct type action = Parse | Print let string_of_action action = match action with - | Parse -> "parser" - | Print -> "printer" - - (* TODO: we could at Reason here *) - type lang = Rescript - let string_of_lang lang = - match lang with - | Rescript -> "rescript" + | Parse -> "Parse" + | Print -> "Print" let parse_rescript src filename = let p = Parser.make src filename in @@ -197,19 +191,20 @@ end = struct assert (p.diagnostics == []); structure - let benchmark filename lang action = - let src = IO.read_file filename in - let name = - filename ^ " " ^ string_of_lang lang ^ " " ^ string_of_action action - in + let data_dir = "tests/syntax_benchmarks/data" + + let benchmark filename action = + let path = Filename.concat data_dir filename in + let src = IO.read_file path in + let name = string_of_action action ^ " " ^ filename in let benchmark_fn = - match (lang, action) with - | Rescript, Parse -> + match action with + | Parse -> fun _ -> - let _ = Sys.opaque_identity (parse_rescript src filename) in + let _ = Sys.opaque_identity (parse_rescript src path) in () - | Rescript, Print -> - let p = Parser.make src filename in + | Print -> + let p = Parser.make src path in let ast = ResParser.parse_implementation p in fun _ -> let _ = @@ -226,16 +221,13 @@ end = struct Benchmark.report b let run () = - let data_dir = "tests/syntax_benchmarks/data" in - benchmark (Filename.concat data_dir "RedBlackTree.res") Rescript Parse; - benchmark (Filename.concat data_dir "RedBlackTree.res") Rescript Print; - benchmark - (Filename.concat data_dir "RedBlackTreeNoComments.res") - Rescript Print; - benchmark (Filename.concat data_dir "Napkinscript.res") Rescript Parse; - benchmark (Filename.concat data_dir "Napkinscript.res") Rescript Print; - benchmark (Filename.concat data_dir "HeroGraphic.res") Rescript Parse; - benchmark (Filename.concat data_dir "HeroGraphic.res") Rescript Print + benchmark "RedBlackTree.res" Parse; + benchmark "RedBlackTree.res" Print; + benchmark "RedBlackTreeNoComments.res" Print; + benchmark "Napkinscript.res" Parse; + benchmark "Napkinscript.res" Print; + benchmark "HeroGraphic.res" Parse; + benchmark "HeroGraphic.res" Print end let () = Benchmarks.run () From 60768c966f5b51078a9590102ed6f6775e7d6b66 Mon Sep 17 00:00:00 2001 From: Christoph Knittel Date: Fri, 11 Oct 2024 17:05:18 +0200 Subject: [PATCH 2/7] Output benchmark results as JSON --- .github/workflows/ci.yml | 11 +++++- dune-project | 4 ++ rescript.opam | 1 + tests/syntax_benchmarks/Benchmark.ml | 59 ++++++++++++++++------------ tests/syntax_benchmarks/dune | 2 +- 5 files changed, 50 insertions(+), 27 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a19f176af3..2109dbc44e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -90,8 +90,9 @@ jobs: ocaml_compiler: ocaml-variants.5.2.0+options,ocaml-option-static upload_binaries: true upload_libs: true - # Build the playground compiler on the fastest runner + # Build the playground compiler and run the benchmarks on the fastest runner build_playground: true + benchmarks: true - os: buildjet-2vcpu-ubuntu-2204-arm # ARM ocaml_compiler: ocaml-variants.5.2.0+options,ocaml-option-static upload_binaries: true @@ -320,6 +321,14 @@ jobs: if: runner.os != 'Windows' run: make -C tests/gentype_tests/typescript-react-example clean test + - name: Build playground compiler + if: matrix.benchmarks + run: | + opam exec -- node packages/playground-bundling/scripts/generate_cmijs.js + opam exec -- dune build --profile browser + cp ./_build/default/compiler/jsoo/jsoo_playground_main.bc.js playground/compiler.js + + - name: Build playground compiler if: matrix.build_playground run: | diff --git a/dune-project b/dune-project index d0d3bdca87..2fa8675931 100644 --- a/dune-project +++ b/dune-project @@ -24,6 +24,10 @@ (and :with-test (= 0.26.2))) + (yojson + (and + :with-test + (= 2.2.2))) (ocaml-lsp-server (and :with-dev-setup diff --git a/rescript.opam b/rescript.opam index ff2913835d..bc35c92939 100644 --- a/rescript.opam +++ b/rescript.opam @@ -9,6 +9,7 @@ bug-reports: "https://github.com/rescript-lang/rescript-compiler/issues" depends: [ "ocaml" {>= "4.10"} "ocamlformat" {with-test & = "0.26.2"} + "yojson" {with-test & = "2.2.2"} "ocaml-lsp-server" {with-dev-setup & = "1.19.0"} "cppo" {= "1.6.9"} "js_of_ocaml" {= "5.8.1"} diff --git a/tests/syntax_benchmarks/Benchmark.ml b/tests/syntax_benchmarks/Benchmark.ml index 80810c5b94..075860cbad 100644 --- a/tests/syntax_benchmarks/Benchmark.ml +++ b/tests/syntax_benchmarks/Benchmark.ml @@ -79,7 +79,7 @@ module Benchmark : sig val make : name:string -> f:(t -> unit) -> unit -> t val launch : t -> unit - val report : t -> unit + val report : t -> Yojson.t list end = struct type t = { name: string; @@ -98,23 +98,26 @@ end = struct } let report b = - print_endline (Format.sprintf "Benchmark: %s" b.name); - print_endline (Format.sprintf "Nbr of iterations: %d" b.n); - print_endline - (Format.sprintf "Benchmark ran during: %fms" (Time.print b.duration)); - print_endline - (Format.sprintf "Avg time/op: %fms" - (Time.print b.duration /. float_of_int b.n)); - print_endline - (Format.sprintf "Allocs/op: %d" - (int_of_float (b.net_allocs /. float_of_int b.n))); - print_endline - (Format.sprintf "B/op: %d" - (int_of_float (b.net_bytes /. float_of_int b.n))); - - (* return (float64(r.Bytes) * float64(r.N) / 1e6) / r.T.Seconds() *) - print_newline (); - () + [ + `Assoc + [ + ("name", `String (Format.sprintf "%s - avg. time" b.name)); + ("unit", `String "ms"); + ("value", `Float (Time.print b.duration /. float_of_int b.n)); + ]; + `Assoc + [ + ("name", `String (Format.sprintf "%s - allocations" b.name)); + ("unit", `String ""); + ("value", `Int (int_of_float (b.net_allocs /. float_of_int b.n))); + ]; + `Assoc + [ + ("name", `String (Format.sprintf "%s - bytes allocated" b.name)); + ("unit", `String ""); + ("value", `Int (int_of_float (b.net_bytes /. float_of_int b.n))); + ]; + ] let make ~name ~f () = { @@ -221,13 +224,19 @@ end = struct Benchmark.report b let run () = - benchmark "RedBlackTree.res" Parse; - benchmark "RedBlackTree.res" Print; - benchmark "RedBlackTreeNoComments.res" Print; - benchmark "Napkinscript.res" Parse; - benchmark "Napkinscript.res" Print; - benchmark "HeroGraphic.res" Parse; - benchmark "HeroGraphic.res" Print + let results = + List.flatten + [ + benchmark "RedBlackTree.res" Parse; + benchmark "RedBlackTree.res" Print; + benchmark "RedBlackTreeNoComments.res" Print; + benchmark "Napkinscript.res" Parse; + benchmark "Napkinscript.res" Print; + benchmark "HeroGraphic.res" Parse; + benchmark "HeroGraphic.res" Print; + ] + in + print_endline (Yojson.pretty_to_string (`List results)) end let () = Benchmarks.run () diff --git a/tests/syntax_benchmarks/dune b/tests/syntax_benchmarks/dune index 0d234e3126..717ec85326 100644 --- a/tests/syntax_benchmarks/dune +++ b/tests/syntax_benchmarks/dune @@ -22,6 +22,6 @@ (foreign_stubs (language c) (names time)) - (libraries syntax)) + (libraries syntax yojson)) (data_only_dirs data) From 04cabc760ab6f8a8ac0c60d6966cc2177ba54a47 Mon Sep 17 00:00:00 2001 From: Christoph Knittel Date: Sat, 12 Oct 2024 09:44:46 +0200 Subject: [PATCH 3/7] Output JSON lazily --- tests/syntax_benchmarks/Benchmark.ml | 34 ++++++++++++++++------------ 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/tests/syntax_benchmarks/Benchmark.ml b/tests/syntax_benchmarks/Benchmark.ml index 075860cbad..6afcc83eb4 100644 --- a/tests/syntax_benchmarks/Benchmark.ml +++ b/tests/syntax_benchmarks/Benchmark.ml @@ -183,6 +183,7 @@ module Benchmarks : sig val run : unit -> unit end = struct type action = Parse | Print + let string_of_action action = match action with | Parse -> "Parse" @@ -196,7 +197,7 @@ end = struct let data_dir = "tests/syntax_benchmarks/data" - let benchmark filename action = + let benchmark (filename, action) = let path = Filename.concat data_dir filename in let src = IO.read_file path in let name = string_of_action action ^ " " ^ filename in @@ -223,20 +224,25 @@ end = struct Benchmark.launch b; Benchmark.report b + let specs = + [ + ("RedBlackTree.res", Parse); + ("RedBlackTree.res", Print); + ("RedBlackTreeNoComments.res", Print); + ("Napkinscript.res", Parse); + ("Napkinscript.res", Print); + ("HeroGraphic.res", Parse); + ("HeroGraphic.res", Print); + ] + let run () = - let results = - List.flatten - [ - benchmark "RedBlackTree.res" Parse; - benchmark "RedBlackTree.res" Print; - benchmark "RedBlackTreeNoComments.res" Print; - benchmark "Napkinscript.res" Parse; - benchmark "Napkinscript.res" Print; - benchmark "HeroGraphic.res" Parse; - benchmark "HeroGraphic.res" Print; - ] - in - print_endline (Yojson.pretty_to_string (`List results)) + List.to_seq specs + |> Seq.flat_map (fun spec -> benchmark spec |> List.to_seq) + |> Seq.iteri (fun i json -> + print_endline (if i == 0 then "[" else ","); + print_string (Yojson.to_string json)); + print_newline (); + print_endline "]" end let () = Benchmarks.run () From b6850bf0c24be6524b0f59e9879ceb26bb66be82 Mon Sep 17 00:00:00 2001 From: Christoph Knittel Date: Sat, 12 Oct 2024 09:13:41 +0200 Subject: [PATCH 4/7] Increment opam cache key --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2109dbc44e..659ef79163 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -151,7 +151,7 @@ jobs: # matrix.ocaml_compiler may contain commas - name: Get OPAM cache key shell: bash - run: echo "opam_cache_key=opam-env-v3-${{ matrix.os }}-${{ matrix.ocaml_compiler }}-${{ hashFiles('dune-project') }}" | sed 's/,/-/g' >> $GITHUB_ENV + run: echo "opam_cache_key=opam-env-v4-${{ matrix.os }}-${{ matrix.ocaml_compiler }}-${{ hashFiles('dune-project') }}" | sed 's/,/-/g' >> $GITHUB_ENV - name: Restore OPAM environment id: cache-opam-env From 5439c76643d35502ac6e5453ffaef8652a659123 Mon Sep 17 00:00:00 2001 From: Christoph Knittel Date: Sat, 12 Oct 2024 10:22:35 +0200 Subject: [PATCH 5/7] Build benchmarks only for OCaml >= 4.14.0 --- tests/syntax_benchmarks/dune | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/syntax_benchmarks/dune b/tests/syntax_benchmarks/dune index 717ec85326..d45805d291 100644 --- a/tests/syntax_benchmarks/dune +++ b/tests/syntax_benchmarks/dune @@ -9,6 +9,7 @@ (enabled_if (and (<> %{profile} browser) + (>= %{ocaml_version} "4.14.0") (or (= %{system} macosx) ; or one of Linuxes (see https://github.com/ocaml/ocaml/issues/10613) From 8e0c826a8e601a0e908b0f73da60a4345d6fe323 Mon Sep 17 00:00:00 2001 From: Christoph Knittel Date: Sat, 12 Oct 2024 09:10:39 +0200 Subject: [PATCH 6/7] Continuous benchmarking in CI --- .github/workflows/ci.yml | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 659ef79163..41ad7d1ef4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -321,13 +321,28 @@ jobs: if: runner.os != 'Windows' run: make -C tests/gentype_tests/typescript-react-example clean test - - name: Build playground compiler + - name: Run syntax benchmarks if: matrix.benchmarks - run: | - opam exec -- node packages/playground-bundling/scripts/generate_cmijs.js - opam exec -- dune build --profile browser - cp ./_build/default/compiler/jsoo/jsoo_playground_main.bc.js playground/compiler.js + run: ./_build/install/default/bin/syntax_benchmarks | tee tests/benchmark-output.json + - name: Download previous benchmark data + if: matrix.benchmarks + uses: actions/cache@v4 + with: + path: ./tests/benchmark-cache + key: syntax-benchmark-v1 + + - name: Store benchmark result + if: matrix.benchmarks + uses: benchmark-action/github-action-benchmark@v1 + with: + tool: "customSmallerIsBetter" + output-file-path: tests/benchmark-output.json + external-data-json-path: ./tests/benchmark-cache/benchmark-data.json + github-token: ${{ secrets.GITHUB_TOKEN }} + alert-threshold: "150%" + comment-always: true + comment-on-alert: true - name: Build playground compiler if: matrix.build_playground From 833b97fec0e20d1d559c348ecfa401930aaf3c77 Mon Sep 17 00:00:00 2001 From: Christoph Knittel Date: Sat, 12 Oct 2024 10:25:32 +0200 Subject: [PATCH 7/7] Permissions --- .github/workflows/ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 41ad7d1ef4..d921547ccb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -82,6 +82,10 @@ jobs: path: ${{ env.artifact_dir_name }} build-compiler: + permissions: + # benchmark-action/github-action-benchmark + pull-requests: write + strategy: fail-fast: false matrix: