diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a19f176af3..d921547ccb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -82,6 +82,10 @@ jobs: path: ${{ env.artifact_dir_name }} build-compiler: + permissions: + # benchmark-action/github-action-benchmark + pull-requests: write + strategy: fail-fast: false matrix: @@ -90,8 +94,9 @@ jobs: ocaml_compiler: ocaml-variants.5.2.0+options,ocaml-option-static upload_binaries: true upload_libs: true - # Build the playground compiler on the fastest runner + # Build the playground compiler and run the benchmarks on the fastest runner build_playground: true + benchmarks: true - os: buildjet-2vcpu-ubuntu-2204-arm # ARM ocaml_compiler: ocaml-variants.5.2.0+options,ocaml-option-static upload_binaries: true @@ -150,7 +155,7 @@ jobs: # matrix.ocaml_compiler may contain commas - name: Get OPAM cache key shell: bash - run: echo "opam_cache_key=opam-env-v3-${{ matrix.os }}-${{ matrix.ocaml_compiler }}-${{ hashFiles('dune-project') }}" | sed 's/,/-/g' >> $GITHUB_ENV + run: echo "opam_cache_key=opam-env-v4-${{ matrix.os }}-${{ matrix.ocaml_compiler }}-${{ hashFiles('dune-project') }}" | sed 's/,/-/g' >> $GITHUB_ENV - name: Restore OPAM environment id: cache-opam-env @@ -320,6 +325,29 @@ jobs: if: runner.os != 'Windows' run: make -C tests/gentype_tests/typescript-react-example clean test + - name: Run syntax benchmarks + if: matrix.benchmarks + run: ./_build/install/default/bin/syntax_benchmarks | tee tests/benchmark-output.json + + - name: Download previous benchmark data + if: matrix.benchmarks + uses: actions/cache@v4 + with: + path: ./tests/benchmark-cache + key: syntax-benchmark-v1 + + - name: Store benchmark result + if: matrix.benchmarks + uses: benchmark-action/github-action-benchmark@v1 + with: + tool: "customSmallerIsBetter" + output-file-path: tests/benchmark-output.json + external-data-json-path: ./tests/benchmark-cache/benchmark-data.json + github-token: ${{ secrets.GITHUB_TOKEN }} + alert-threshold: "150%" + comment-always: true + comment-on-alert: true + - name: Build playground compiler if: matrix.build_playground run: | diff --git a/dune-project b/dune-project index d0d3bdca87..2fa8675931 100644 --- a/dune-project +++ b/dune-project @@ -24,6 +24,10 @@ (and :with-test (= 0.26.2))) + (yojson + (and + :with-test + (= 2.2.2))) (ocaml-lsp-server (and :with-dev-setup diff --git a/rescript.opam b/rescript.opam index ff2913835d..bc35c92939 100644 --- a/rescript.opam +++ b/rescript.opam @@ -9,6 +9,7 @@ bug-reports: "https://github.com/rescript-lang/rescript-compiler/issues" depends: [ "ocaml" {>= "4.10"} "ocamlformat" {with-test & = "0.26.2"} + "yojson" {with-test & = "2.2.2"} "ocaml-lsp-server" {with-dev-setup & = "1.19.0"} "cppo" {= "1.6.9"} "js_of_ocaml" {= "5.8.1"} diff --git a/tests/syntax_benchmarks/Benchmark.ml b/tests/syntax_benchmarks/Benchmark.ml index 4f8e2db38a..6afcc83eb4 100644 --- a/tests/syntax_benchmarks/Benchmark.ml +++ b/tests/syntax_benchmarks/Benchmark.ml @@ -79,7 +79,7 @@ module Benchmark : sig val make : name:string -> f:(t -> unit) -> unit -> t val launch : t -> unit - val report : t -> unit + val report : t -> Yojson.t list end = struct type t = { name: string; @@ -98,23 +98,26 @@ end = struct } let report b = - print_endline (Format.sprintf "Benchmark: %s" b.name); - print_endline (Format.sprintf "Nbr of iterations: %d" b.n); - print_endline - (Format.sprintf "Benchmark ran during: %fms" (Time.print b.duration)); - print_endline - (Format.sprintf "Avg time/op: %fms" - (Time.print b.duration /. float_of_int b.n)); - print_endline - (Format.sprintf "Allocs/op: %d" - (int_of_float (b.net_allocs /. float_of_int b.n))); - print_endline - (Format.sprintf "B/op: %d" - (int_of_float (b.net_bytes /. float_of_int b.n))); - - (* return (float64(r.Bytes) * float64(r.N) / 1e6) / r.T.Seconds() *) - print_newline (); - () + [ + `Assoc + [ + ("name", `String (Format.sprintf "%s - avg. time" b.name)); + ("unit", `String "ms"); + ("value", `Float (Time.print b.duration /. float_of_int b.n)); + ]; + `Assoc + [ + ("name", `String (Format.sprintf "%s - allocations" b.name)); + ("unit", `String ""); + ("value", `Int (int_of_float (b.net_allocs /. float_of_int b.n))); + ]; + `Assoc + [ + ("name", `String (Format.sprintf "%s - bytes allocated" b.name)); + ("unit", `String ""); + ("value", `Int (int_of_float (b.net_bytes /. float_of_int b.n))); + ]; + ] let make ~name ~f () = { @@ -180,16 +183,11 @@ module Benchmarks : sig val run : unit -> unit end = struct type action = Parse | Print + let string_of_action action = match action with - | Parse -> "parser" - | Print -> "printer" - - (* TODO: we could at Reason here *) - type lang = Rescript - let string_of_lang lang = - match lang with - | Rescript -> "rescript" + | Parse -> "Parse" + | Print -> "Print" let parse_rescript src filename = let p = Parser.make src filename in @@ -197,19 +195,20 @@ end = struct assert (p.diagnostics == []); structure - let benchmark filename lang action = - let src = IO.read_file filename in - let name = - filename ^ " " ^ string_of_lang lang ^ " " ^ string_of_action action - in + let data_dir = "tests/syntax_benchmarks/data" + + let benchmark (filename, action) = + let path = Filename.concat data_dir filename in + let src = IO.read_file path in + let name = string_of_action action ^ " " ^ filename in let benchmark_fn = - match (lang, action) with - | Rescript, Parse -> + match action with + | Parse -> fun _ -> - let _ = Sys.opaque_identity (parse_rescript src filename) in + let _ = Sys.opaque_identity (parse_rescript src path) in () - | Rescript, Print -> - let p = Parser.make src filename in + | Print -> + let p = Parser.make src path in let ast = ResParser.parse_implementation p in fun _ -> let _ = @@ -225,17 +224,25 @@ end = struct Benchmark.launch b; Benchmark.report b + let specs = + [ + ("RedBlackTree.res", Parse); + ("RedBlackTree.res", Print); + ("RedBlackTreeNoComments.res", Print); + ("Napkinscript.res", Parse); + ("Napkinscript.res", Print); + ("HeroGraphic.res", Parse); + ("HeroGraphic.res", Print); + ] + let run () = - let data_dir = "tests/syntax_benchmarks/data" in - benchmark (Filename.concat data_dir "RedBlackTree.res") Rescript Parse; - benchmark (Filename.concat data_dir "RedBlackTree.res") Rescript Print; - benchmark - (Filename.concat data_dir "RedBlackTreeNoComments.res") - Rescript Print; - benchmark (Filename.concat data_dir "Napkinscript.res") Rescript Parse; - benchmark (Filename.concat data_dir "Napkinscript.res") Rescript Print; - benchmark (Filename.concat data_dir "HeroGraphic.res") Rescript Parse; - benchmark (Filename.concat data_dir "HeroGraphic.res") Rescript Print + List.to_seq specs + |> Seq.flat_map (fun spec -> benchmark spec |> List.to_seq) + |> Seq.iteri (fun i json -> + print_endline (if i == 0 then "[" else ","); + print_string (Yojson.to_string json)); + print_newline (); + print_endline "]" end let () = Benchmarks.run () diff --git a/tests/syntax_benchmarks/dune b/tests/syntax_benchmarks/dune index 0d234e3126..d45805d291 100644 --- a/tests/syntax_benchmarks/dune +++ b/tests/syntax_benchmarks/dune @@ -9,6 +9,7 @@ (enabled_if (and (<> %{profile} browser) + (>= %{ocaml_version} "4.14.0") (or (= %{system} macosx) ; or one of Linuxes (see https://github.com/ocaml/ocaml/issues/10613) @@ -22,6 +23,6 @@ (foreign_stubs (language c) (names time)) - (libraries syntax)) + (libraries syntax yojson)) (data_only_dirs data)