Skip to content

Commit 39e82ce

Browse files
alphapapaalphapapa
committed
Add/Change: bench-multi-lexical and changes to bench-multi
bench-multi now produces code compatible with lexical-binding, and bench-multi-lexical benchmarks code in a byte-compiled file with lexical-binding enabled.
1 parent c189af2 commit 39e82ce

File tree

2 files changed

+1641
-1296
lines changed

2 files changed

+1641
-1296
lines changed

‎README.org‎

Lines changed: 180 additions & 70 deletions
Original file line numberDiff line numberDiff line change
@@ -182,6 +182,8 @@ Inserting strings into buffers with ~insert~ is generally fast, but it can slow
182182
| (loop do (insert ... | 1 | 0.001490397 | 0 | 0.0 |
183183

184184
The fastest method here is to call ~insert~ once with the result of calling ~concat~ once, using ~apply~ to pass all of the strings. With 100 iterations, it's about 6x faster than the next-fastest method, and even with 1 iteration, it's over 2x faster.
185+
186+
185187
*** Libraries :libraries:
186188
:PROPERTIES:
187189
:ID: 523aa766-36a3-4827-a114-6babf72edc6b
@@ -1348,24 +1350,26 @@ When called from an Org source block, it gives output like this:
13481350

13491351
This macro makes comparing multiple forms easy:
13501352

1351-
#+BEGIN_SRC elisp
1353+
#+BEGIN_SRC elisp :exports code :results silent
13521354
(cl-defmacro bench-multi (&key (times 1) forms ensure-equal)
1353-
"Run FORMS with `benchmark-run-compiled' for TIMES iterations, returning list suitable for Org source block evaluation.
1355+
"Return Org table as a list with benchmark results for FORMS.
1356+
Runs FORMS with `benchmark-run-compiled' for TIMES iterations.
13541357

1355-
When ENSURE-EQUAL is non-nil, compare the results of FORMS and
1356-
ensure they are `equal'. If they aren't, raise an error, and if
1357-
the results are sequences, show the difference between them using
1358+
When ENSURE-EQUAL is non-nil, the results of FORMS are compared,
1359+
and an error is raised if they aren't `equal'. If the results
1360+
are sequences, the difference between them is shown with
13581361
`seq-difference'.
13591362

1360-
If the first element of a form is a string, the string is used as
1361-
the form's description in the results; otherwise, forms are
1362-
numbered from 0.
1363+
If the first element of a form is a string, it's used as the
1364+
form's description in the bench-multi-results; otherwise, forms
1365+
are numbered from 0.
13631366

1364-
Before `benchmark-run-compiled' is called for each form,
1365-
`garbage-collect' is called."
1367+
Before each form is run, `garbage-collect' is called."
1368+
;; MAYBE: Since `bench-multi-lexical' byte-compiles the file, I'm not sure if
1369+
;; `benchmark-run-compiled' is necessary over `benchmark-run', or if it matters.
13661370
(declare (indent defun))
1367-
(let ((results (gensym))
1368-
(result-times (gensym))
1371+
(let ((keys (gensym "keys"))
1372+
(result-times (gensym "result-times"))
13691373
(header '(("Form" "x faster than next" "Total runtime" "# of GCs" "Total GC runtime")
13701374
hline))
13711375
(descriptions (cl-loop for form in forms
@@ -1374,54 +1378,61 @@ This macro makes comparing multiple forms easy:
13741378
(prog1 (car form)
13751379
(setf (nth i forms) (cadr (nth i forms))))
13761380
i))))
1377-
`(let* ((,results (make-hash-table))
1378-
(,result-times (sort (list ,@(cl-loop for form in forms
1379-
for i from 0
1380-
for description = (nth i descriptions)
1381-
collect `(progn
1382-
(garbage-collect)
1383-
(cons ,description
1384-
(benchmark-run-compiled ,times
1385-
,(if ensure-equal
1386-
`(puthash ,description ,form ,results)
1387-
form))))))
1388-
(lambda (a b)
1389-
(< (second a) (second b))))))
1390-
,(when ensure-equal
1391-
`(cl-loop with keys = (hash-table-keys ,results)
1392-
for i from 0 to (- (length keys) 2)
1393-
unless (equal (gethash (nth i keys) ,results)
1394-
(gethash (nth (1+ i) keys) ,results))
1395-
do (if (sequencep (gethash (car (hash-table-keys ,results)) ,results))
1396-
(let* ((k1) (k2)
1397-
;; If the difference in one order is nil, try in other order.
1398-
(difference (or (setq k1 (nth i keys)
1399-
k2 (nth (1+ i) keys)
1400-
difference (seq-difference (gethash k1 ,results)
1401-
(gethash k2 ,results)))
1402-
(setq k1 (nth (1+ i) keys)
1403-
k2 (nth i keys)
1404-
difference (seq-difference (gethash k1 ,results)
1405-
(gethash k2 ,results))))))
1406-
(user-error "Forms' results not equal: difference (%s - %s): %S"
1407-
k1 k2 difference))
1408-
;; Not a sequence
1409-
(user-error "Forms' results not equal: %s:%S %s:%S"
1410-
(nth i keys) (nth (1+ i) keys)
1411-
(gethash (nth i keys) ,results)
1412-
(gethash (nth (1+ i) keys) ,results)))))
1413-
;; Add factors to times and return table
1414-
(append ',header
1415-
(cl-loop with length = (length ,result-times)
1416-
for i from 0 to (1- length)
1417-
for time = (second (nth i ,result-times))
1418-
for description = (car (nth i ,result-times))
1419-
for factor = (if (< i (1- length))
1420-
(format "%.2f" (/ (second (nth (1+ i) ,result-times))
1421-
(second (nth i ,result-times))))
1422-
"slowest")
1423-
collect (append (list description factor)
1424-
(cdr (nth i ,result-times))))))))
1381+
`(unwind-protect
1382+
(progn
1383+
(defvar bench-multi-results nil)
1384+
(let* ((bench-multi-results (make-hash-table))
1385+
(,result-times (sort (list ,@(cl-loop for form in forms
1386+
for i from 0
1387+
for description = (nth i descriptions)
1388+
collect `(progn
1389+
(garbage-collect)
1390+
(cons ,description
1391+
(benchmark-run-compiled ,times
1392+
,(if ensure-equal
1393+
`(puthash ,description ,form bench-multi-results)
1394+
form))))))
1395+
(lambda (a b)
1396+
(< (second a) (second b))))))
1397+
,(when ensure-equal
1398+
`(cl-loop with ,keys = (hash-table-keys bench-multi-results)
1399+
for i from 0 to (- (length ,keys) 2)
1400+
unless (equal (gethash (nth i ,keys) bench-multi-results)
1401+
(gethash (nth (1+ i) ,keys) bench-multi-results))
1402+
do (if (sequencep (gethash (car (hash-table-keys bench-multi-results)) bench-multi-results))
1403+
(let* ((k1) (k2)
1404+
;; If the difference in one order is nil, try in other order.
1405+
(difference (or (setq k1 (nth i ,keys)
1406+
k2 (nth (1+ i) ,keys)
1407+
difference (seq-difference (gethash k1 bench-multi-results)
1408+
(gethash k2 bench-multi-results)))
1409+
(setq k1 (nth (1+ i) ,keys)
1410+
k2 (nth i ,keys)
1411+
difference (seq-difference (gethash k1 bench-multi-results)
1412+
(gethash k2 bench-multi-results))))))
1413+
(user-error "Forms' bench-multi-results not equal: difference (%s - %s): %S"
1414+
k1 k2 difference))
1415+
;; Not a sequence
1416+
(user-error "Forms' bench-multi-results not equal: %s:%S %s:%S"
1417+
(nth i ,keys) (nth (1+ i) ,keys)
1418+
(gethash (nth i ,keys) bench-multi-results)
1419+
(gethash (nth (1+ i) ,keys) bench-multi-results)))))
1420+
;; Add factors to times and return table
1421+
(append ',header
1422+
(cl-loop with length = (length ,result-times)
1423+
for i from 0 to (1- length)
1424+
for description = (car (nth i ,result-times))
1425+
for factor = (if (< i (1- length))
1426+
(format "%.2f" (/ (second (nth (1+ i) ,result-times))
1427+
(second (nth i ,result-times))))
1428+
"slowest")
1429+
collect (append (list description factor)
1430+
(list (format "%.6f" (second (nth i ,result-times)))
1431+
(third (nth i ,result-times))
1432+
(if (> (fourth (nth i ,result-times)) 0)
1433+
(format "%.6f" (fourth (nth i ,result-times)))
1434+
0)))))))
1435+
(unintern 'bench-multi-results nil))))
14251436
#+END_SRC
14261437

14271438
Used like:
@@ -1449,14 +1460,13 @@ Used like:
14491460
#+RESULTS[3316dc4375a3b162e32790bb7e72d715d7f756fb]:
14501461
| Form | x faster than next | Total runtime | # of GCs | Total GC runtime |
14511462
|-----------------+--------------------+---------------+----------+------------------|
1452-
| regexp | 62.50 | 0.043138672 | 0 | 0.0 |
1453-
| org-map-entries | slowest | 2.69609941 | 0 | 0.0 |
1463+
| regexp | 61.97 | 0.042440 | 0 | 0 |
1464+
| org-map-entries | slowest | 2.630019 | 0 | 0 |
14541465

14551466
It can also help catch bugs by ensuring that each form returns the same results. For example, the benchmark above contains a subtle bug: because ~case-fold-search~ in the =regexp= form is non-nil, the regexp is compared case-insensitively, so it matches Org headings which start with =Maybe= rather than only ones which start with =MAYBE=. Using the ~:ensure-equal t~ argument to ~bench-multi~ compares the results and raises an error showing the difference between the two sequences the forms evaluate to:
14561467

1457-
#+BEGIN_SRC elisp :exports code
1458-
(bench-multi
1459-
:ensure-equal t
1468+
#+BEGIN_SRC elisp :exports code :results silent
1469+
(bench-multi :ensure-equal t
14601470
:forms (("org-map-entries" (sort (org-map-entries (lambda ()
14611471
(nth 4 (org-heading-components)))
14621472
"/+MAYBE" 'agenda)
@@ -1480,8 +1490,7 @@ It can also help catch bugs by ensuring that each form returns the same results.
14801490
Fixing the error, by setting ~case-fold-search~ to ~nil~, not only makes the forms give the same result but, in this case, doubles the performance of the faster form:
14811491

14821492
#+BEGIN_SRC elisp :exports both :cache yes
1483-
(bench-multi
1484-
:ensure-equal t
1493+
(bench-multi :ensure-equal t
14851494
:forms (("org-map-entries" (sort (org-map-entries (lambda ()
14861495
(nth 4 (org-heading-components)))
14871496
"/+MAYBE" 'agenda)
@@ -1500,14 +1509,115 @@ Fixing the error, by setting ~case-fold-search~ to ~nil~, not only makes the for
15001509
#'string<))))
15011510
#+END_SRC
15021511

1503-
#+RESULTS[13bda5a63b7b851ff3aac91c6487a9eb22ad0fb7]:
1512+
#+RESULTS[773b94ff27f73dcfcb694429054710a581b7bec5]:
15041513
| Form | x faster than next | Total runtime | # of GCs | Total GC runtime |
15051514
|-----------------+--------------------+---------------+----------+------------------|
1506-
| regexp | 125.70 | 0.021080791 | 0 | 0.0 |
1507-
| org-map-entries | slowest | 2.649818971 | 0 | 0.0 |
1515+
| regexp | 134.09 | 0.019550 | 0 | 0 |
1516+
| org-map-entries | slowest | 2.621377 | 0 | 0 |
15081517

15091518
So this macro showed which code is faster and helped catch a subtle bug.
15101519

1520+
***** ~bench-multi-lexical~
1521+
1522+
To evaluate forms with lexical binding enabled, use this macro:
1523+
1524+
#+BEGIN_SRC elisp :exports code :results silent
1525+
(cl-defmacro bench-multi-lexical (&key (times 1) forms ensure-equal)
1526+
"Return Org table as a list with benchmark results for FORMS.
1527+
Runs FORMS from a byte-compiled temp file with `lexical-binding'
1528+
enabled, using `bench-multi', which see.
1529+
1530+
When ENSURE-EQUAL is non-nil, the results of FORMS are compared,
1531+
and an error is raised if they aren't `equal'. If the results
1532+
are sequences, the difference between them is shown with
1533+
`seq-difference'.
1534+
1535+
If the first element of a form is a string, it's used as the
1536+
form's description in the bench-multi-results; otherwise, forms
1537+
are numbered from 0.
1538+
1539+
Before each form is run, `garbage-collect' is called.
1540+
1541+
Afterward, the temp file is deleted and the function used to run
1542+
the benchmark is uninterned.."
1543+
(declare (indent defun))
1544+
`(let* ((temp-file (concat (make-temp-file "bench-multi-lexical-") ".el"))
1545+
(fn (gensym "bench-multi-lexical-run-")))
1546+
(with-temp-file temp-file
1547+
(insert ";; -*- lexical-binding: t; -*-" "\n\n"
1548+
"(defvar bench-multi-results)" "\n\n"
1549+
(format "(defun %s () (bench-multi :times %d :ensure-equal %s :forms %S))"
1550+
fn ,times ,ensure-equal ',forms)))
1551+
(unwind-protect
1552+
(if (byte-compile-file temp-file 'load)
1553+
(funcall (intern (symbol-name fn)))
1554+
(user-error "Error byte-compiling and loading temp file"))
1555+
(delete-file temp-file)
1556+
(unintern (symbol-name fn) nil))))
1557+
#+END_SRC
1558+
1559+
Used just like ~bench-multi~:
1560+
1561+
#+BEGIN_SRC elisp :exports both :cache yes
1562+
(bench-multi-lexical :ensure-equal t
1563+
:forms (("org-map-entries" (sort (org-map-entries (lambda ()
1564+
(nth 4 (org-heading-components)))
1565+
"/+MAYBE" 'agenda)
1566+
#'string<))
1567+
("regexp" (sort (-flatten
1568+
(-non-nil
1569+
(mapcar (lambda (file)
1570+
(let ((case-fold-search nil))
1571+
(with-current-buffer (find-buffer-visiting file)
1572+
(org-with-wide-buffer
1573+
(goto-char (point-min))
1574+
(cl-loop with regexp = (format org-heading-keyword-regexp-format "MAYBE")
1575+
while (re-search-forward regexp nil t)
1576+
collect (nth 4 (org-heading-components)))))))
1577+
(org-agenda-files))))
1578+
#'string<))))
1579+
#+END_SRC
1580+
1581+
#+RESULTS[a8ffc10fa4e21eb632122657312040f139b33204]:
1582+
| Form | x faster than next | Total runtime | # of GCs | Total GC runtime |
1583+
|-----------------+--------------------+---------------+----------+------------------|
1584+
| regexp | 134.26 | 0.019640 | 0 | 0 |
1585+
| org-map-entries | slowest | 2.636943 | 0 | 0 |
1586+
1587+
This shows that lexical-binding doesn't make much difference in this example. But in another example, it does:
1588+
1589+
#+BEGIN_SRC elisp :exports both :cache yes
1590+
(bench-multi :times 1000 :ensure-equal t
1591+
:forms (("buffer-local-value" (--filter (equal 'magit-status-mode (buffer-local-value 'major-mode it))
1592+
(buffer-list)))
1593+
("with-current-buffer" (--filter (equal 'magit-status-mode (with-current-buffer it
1594+
major-mode))
1595+
(buffer-list)))))
1596+
#+END_SRC
1597+
1598+
#+RESULTS[5ef620a1a1ab52900312e3adb52dd2e6a39dd9b2]:
1599+
| Form | x faster than next | Total runtime | # of GCs | Total GC runtime |
1600+
|---------------------+--------------------+---------------+----------+------------------|
1601+
| buffer-local-value | 69.64 | 0.013255 | 0 | 0 |
1602+
| with-current-buffer | slowest | 0.923159 | 0 | 0 |
1603+
1604+
#+BEGIN_SRC elisp :exports both :cache yes
1605+
(bench-multi-lexical :times 1000 :ensure-equal t
1606+
:forms (("buffer-local-value" (--filter (equal 'magit-status-mode (buffer-local-value 'major-mode it))
1607+
(buffer-list)))
1608+
("with-current-buffer" (--filter (equal 'magit-status-mode (with-current-buffer it
1609+
major-mode))
1610+
(buffer-list)))))
1611+
#+END_SRC
1612+
1613+
#+RESULTS[aa64952f5203400ddb1e700814a460fe81f9d2e6]:
1614+
| Form | x faster than next | Total runtime | # of GCs | Total GC runtime |
1615+
|---------------------+--------------------+---------------+----------+------------------|
1616+
| buffer-local-value | 86.98 | 0.010512 | 0 | 0 |
1617+
| with-current-buffer | slowest | 0.914274 | 0 | 0 |
1618+
1619+
The ~buffer-local-value~ form improved by about 24% when using lexical binding.
1620+
15111621
**** =elp-profile=
15121622
:PROPERTIES:
15131623
:ID: fd3fdece-0342-441c-8540-5a5c463890a5

0 commit comments

Comments
 (0)