Add/Change: bench-dynamic-vs-lexical-binding and associated changes

Author: alphapapa

README.org

@@ -1346,32 +1346,38 @@ When called from an Org source block, it gives output like this:
 |---------------+----------+--------------------|
 |   1.657838266 |        3 | 1.4723854609999876 |
 
-**** ~bench-multi~
+**** ~bench-multi~ macros
 
-This macro makes comparing multiple forms easy:
+These macros make comparing multiple forms easy:
 
 #+BEGIN_SRC elisp :exports code :results silent
-  (cl-defmacro bench-multi (&key (times 1) forms ensure-equal)
+  (cl-defmacro bench-multi (&key (times 1) forms ensure-equal raw)
     "Return Org table as a list with benchmark results for FORMS.
-  Runs FORMS with `benchmark-run-compiled' for TIMES iterations.
+   Runs FORMS with `benchmark-run-compiled' for TIMES iterations.
 
-  When ENSURE-EQUAL is non-nil, the results of FORMS are compared,
-  and an error is raised if they aren't `equal'. If the results
-  are sequences, the difference between them is shown with
-  `seq-difference'.
+   When ENSURE-EQUAL is non-nil, the results of FORMS are compared,
+   and an error is raised if they aren't `equal'. If the results
+   are sequences, the difference between them is shown with
+   `seq-difference'.
 
-  If the first element of a form is a string, it's used as the
-  form's description in the bench-multi-results; otherwise, forms
-  are numbered from 0.
+   When RAW is non-nil, the raw results from
+   `benchmark-run-compiled' are returned instead of an Org table
+   list.
 
-  Before each form is run, `garbage-collect' is called."
+   If the first element of a form is a string, it's used as the
+   form's description in the bench-multi-results; otherwise, forms
+   are numbered from 0.
+
+   Before each form is run, `garbage-collect' is called."
     ;; MAYBE: Since `bench-multi-lexical' byte-compiles the file, I'm not sure if
     ;; `benchmark-run-compiled' is necessary over `benchmark-run', or if it matters.
     (declare (indent defun))
-    (let ((keys (gensym "keys"))
+    (let*((keys (gensym "keys"))
           (result-times (gensym "result-times"))
           (header '(("Form" "x faster than next" "Total runtime" "# of GCs" "Total GC runtime")
                     hline))
+          ;; Copy forms so that a subsequent call of the macro will get the original forms.
+          (forms (copy-list forms))
           (descriptions (cl-loop for form in forms
                                  for i from 0
                                  collect (if (stringp (car form))
@@ -1418,21 +1424,28 @@ This macro makes comparing multiple forms easy:
                                              (gethash (nth i ,keys) bench-multi-results)
                                              (gethash (nth (1+ i) ,keys) bench-multi-results)))))
                ;; Add factors to times and return table
-               (append ',header
-                       (cl-loop with length = (length ,result-times)
-                                for i from 0 to (1- length)
-                                for description = (car (nth i ,result-times))
-                                for factor = (if (< i (1- length))
-                                                 (format "%.2f" (/ (second (nth (1+ i) ,result-times))
-                                                                   (second (nth i ,result-times))))
-                                               "slowest")
-                                collect (append (list description factor)
-                                                (list (format "%.6f" (second (nth i ,result-times)))
-                                                      (third (nth i ,result-times))
-                                                      (if (> (fourth (nth i ,result-times)) 0)
-                                                          (format "%.6f" (fourth (nth i ,result-times)))
-                                                        0)))))))
+               (if ,raw
+                   ,result-times
+                 (append ',header
+                         (bench-multi-process-results ,result-times)))))
          (unintern 'bench-multi-results nil))))
+
+  (defun bench-multi-process-results (results)
+    "Return sorted RESULTS with factors added."
+    (setq results (sort results (-on #'< #'second)))
+    (cl-loop with length = (length results)
+             for i from 0 to (1- length)
+             for description = (car (nth i results))
+             for factor = (if (< i (1- length))
+                              (format "%.2f" (/ (second (nth (1+ i) results))
+                                                (second (nth i results))))
+                            "slowest")
+             collect (append (list description factor)
+                             (list (format "%.6f" (second (nth i results)))
+                                   (third (nth i results))
+                                   (if (> (fourth (nth i results)) 0)
+                                       (format "%.6f" (fourth (nth i results)))
+                                     0)))))
 #+END_SRC
 
 Used like:
@@ -1522,32 +1535,21 @@ So this macro showed which code is faster and helped catch a subtle bug.
 To evaluate forms with lexical binding enabled, use this macro:
 
 #+BEGIN_SRC elisp :exports code :results silent
-  (cl-defmacro bench-multi-lexical (&key (times 1) forms ensure-equal)
+  (cl-defmacro bench-multi-lexical (&key (times 1) forms ensure-equal raw)
     "Return Org table as a list with benchmark results for FORMS.
   Runs FORMS from a byte-compiled temp file with `lexical-binding'
   enabled, using `bench-multi', which see.
 
-  When ENSURE-EQUAL is non-nil, the results of FORMS are compared,
-  and an error is raised if they aren't `equal'.  If the results
-  are sequences, the difference between them is shown with
-  `seq-difference'.
-
-  If the first element of a form is a string, it's used as the
-  form's description in the bench-multi-results; otherwise, forms
-  are numbered from 0.
-
-  Before each form is run, `garbage-collect' is called.
-
   Afterward, the temp file is deleted and the function used to run
-  the benchmark is uninterned.."
+  the benchmark is uninterned."
     (declare (indent defun))
     `(let* ((temp-file (concat (make-temp-file "bench-multi-lexical-") ".el"))
             (fn (gensym "bench-multi-lexical-run-")))
        (with-temp-file temp-file
          (insert ";; -*- lexical-binding: t; -*-" "\n\n"
                  "(defvar bench-multi-results)" "\n\n"
-                 (format "(defun %s () (bench-multi :times %d :ensure-equal %s :forms %S))"
-                         fn ,times ,ensure-equal ',forms)))
+                 (format "(defun %s () (bench-multi :times %d :ensure-equal %s :raw %s :forms %S))"
+                         fn ,times ,ensure-equal ,raw ',forms)))
        (unwind-protect
            (if (byte-compile-file temp-file 'load)
                (funcall (intern (symbol-name fn)))
@@ -1618,6 +1620,48 @@ This shows that lexical-binding doesn't make much difference in this example.  B
 
 The ~buffer-local-value~ form improved by about 24% when using lexical binding.
 
+***** ~bench-dynamic-vs-lexical-binding~
+
+This macro compares dynamic and lexical binding.
+
+#+BEGIN_SRC elisp :exports code :results silent
+  (cl-defmacro bench-dynamic-vs-lexical-binding (&key (times 1) forms ensure-equal)
+    "Benchmark FORMS with both dynamic and lexical binding.
+  Calls `bench-multi' and `bench-multi-lexical', which see."
+    (declare (indent defun))
+    `(let ((dynamic (bench-multi :times ,times :ensure-equal ,ensure-equal :raw t
+                      :forms ,forms))
+           (lexical (bench-multi-lexical :times ,times :ensure-equal ,ensure-equal :raw t
+                      :forms ,forms))
+           (header '("Form" "x faster than next" "Total runtime" "# of GCs" "Total GC runtime")))
+       (cl-loop for result in-ref dynamic
+                do (setf (car result) (format "Dynamic: %s" (car result))))
+       (cl-loop for result in-ref lexical
+                do (setf (car result) (format "Lexical: %s" (car result))))
+       (append (list header)
+               (list 'hline)
+               (bench-multi-process-results (append dynamic lexical)))))
+#+END_SRC
+
+Example:
+
+#+BEGIN_SRC elisp :exports both :cache yes
+  (bench-dynamic-vs-lexical-binding :times 1000 :ensure-equal t
+    :forms (("buffer-local-value" (--filter (equal 'magit-status-mode (buffer-local-value 'major-mode it))
+                                            (buffer-list)))
+            ("with-current-buffer" (--filter (equal 'magit-status-mode (with-current-buffer it
+                                                                         major-mode))
+                                             (buffer-list)))))
+#+END_SRC
+
+#+RESULTS[73cc92a5949dd2d48f029cab9557eb6132bdf1cf]:
+| Form                         | x faster than next | Total runtime | # of GCs | Total GC runtime |
+|------------------------------+--------------------+---------------+----------+------------------|
+| Lexical: buffer-local-value  |               1.29 |      0.011170 |        0 |                0 |
+| Dynamic: buffer-local-value  |              67.10 |      0.014407 |        0 |                0 |
+| Lexical: with-current-buffer |               1.01 |      0.966651 |        0 |                0 |
+| Dynamic: with-current-buffer |            slowest |      0.974830 |        0 |                0 |
+
 **** =elp-profile=
 :PROPERTIES:
 :ID:       fd3fdece-0342-441c-8540-5a5c463890a5