Skip to content

gh-138946: list.sort enhancement proposal: Adaptivity for binarysort #138947

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 11 commits into from
Closed

gh-138946: list.sort enhancement proposal: Adaptivity for binarysort #138947

Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
0341e01
init
dg-pb Sep 15, 2025
e820eb7
minor edit
dg-pb Sep 15, 2025
93b69cd
v2
dg-pb Sep 16, 2025
71c82af
v3
dg-pb Sep 16, 2025
6e0269c
minor changes
dg-pb Sep 17, 2025
827d48e
minor edits
dg-pb Sep 17, 2025
589572f
micro change
dg-pb Sep 17, 2025
992b26f
minimum std condition added
dg-pb Sep 19, 2025
984bcd0
bit shift instead of div
dg-pb Sep 20, 2025
7f30b55
simplified to mu=j
dg-pb Sep 20, 2025
2ff75f8
noop
dg-pb Sep 21, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
288 changes: 259 additions & 29 deletions Objects/listobject.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1886,6 +1886,175 @@
return -1;
}

static int
abinarysort(MergeState *ms, const sortslice *ss, Py_ssize_t n, Py_ssize_t ok, int adapt)
{
Py_ssize_t k; /* for IFLT macro expansion */
PyObject ** const a = ss->keys;
PyObject ** const v = ss->values;
const bool has_values = v != NULL;
PyObject *pivot;

assert(0 <= ok && ok <= n && 1 <= n && n <= MAX_MINRUN);
/* assert a[:ok] is sorted */
if (! ok)
++ok;

Py_ssize_t M, L, R;
Py_ssize_t nsorted = ok;
Py_ssize_t mu = ok >> 1;
Py_ssize_t std = mu;
Py_ssize_t nbad = 0; // badness of fit

if (adapt) {
for (; ok < n; ++ok) {
pivot = a[ok];

// NOTE: If abinarysort is actually working, there will be gains
// And this is a relatively small insurance against adversity
// However, subject to be removed if not helpful in practice
if (std < (ok >> 2)) {
IFLT(pivot, a[mu]) {
L = 0;
R = mu;
if (L < R) {
std += !std;
M = R - std;
if (M < L)
M = L;
IFLT(pivot, a[M]) {
R = M;
if (L < R) {
M = R - std;
if (M < L)
M = L;
IFLT(pivot, a[M])
R = M;
else
L = M + 1;
}
}
else {
L = M + 1;
}
}
}
else {
L = mu + 1;
R = ok;
if (L < R) {
M = L + std;
if (M >= R)
M = R - 1;
IFLT(pivot, a[M]) {
R = M;
}
else {
L = M + 1;
if (L < R) {
M = L + std;
if (M >= R)
M = R - 1;
IFLT(pivot, a[M])
R = M;
else
L = M + 1;
}
}
}
}
// Binary Insertion
while (L < R) {
M = (L + R) >> 1;
IFLT(pivot, a[M])
R = M;
else
L = M + 1;
}
}
else {
// Binary Insertion
M = ok >> 1;
IFLT(pivot, a[M]) {
L = 0;
R = M;
}
else {
L = M + 1;
R = ok;
}
while (L < R) {
M = (L + R) >> 1;
IFLT(pivot, a[M])
R = M;
else
L = M + 1;
}
}

for (M = ok; M > L; --M)
a[M] = a[M - 1];
a[L] = pivot;
if (has_values) {
pivot = v[ok];
for (M = ok; M > L; --M)
v[M] = v[M - 1];
v[L] = pivot;
}

// Update Adaptive runvars
std = labs(L - mu);

Check warning on line 2006 in Objects/listobject.c

View workflow job for this annotation

GitHub Actions / Windows (free-threading) / Build and test (x64) 

'function': conversion from 'Py_ssize_t' to 'long', possible loss of data [D:\a\cpython\cpython\PCbuild\pythoncore.vcxproj]

Check warning on line 2006 in Objects/listobject.c

View workflow job for this annotation

GitHub Actions / Windows / Build and test (arm64) 

'function': conversion from 'Py_ssize_t' to 'long', possible loss of data [C:\a\cpython\cpython\PCbuild\pythoncore.vcxproj]

Check warning on line 2006 in Objects/listobject.c

View workflow job for this annotation

GitHub Actions / Windows (free-threading) / Build and test (arm64) 

'function': conversion from 'Py_ssize_t' to 'long', possible loss of data [C:\a\cpython\cpython\PCbuild\pythoncore.vcxproj]

Check warning on line 2006 in Objects/listobject.c

View workflow job for this annotation

GitHub Actions / Windows / Build and test (x64) 

'function': conversion from 'Py_ssize_t' to 'long', possible loss of data [D:\a\cpython\cpython\PCbuild\pythoncore.vcxproj]
nbad += std;
mu = L;
}
}
else {
for (; ok < n; ++ok) {
pivot = a[ok];

// Binary Insertion
M = ok >> 1;
IFLT(pivot, a[M]) {
L = 0;
R = M;
}
else {
L = M + 1;
R = ok;
}
while (L < R) {
M = (L + R) >> 1;
IFLT(pivot, a[M])
R = M;
else
L = M + 1;
}

for (M = ok; M > L; --M)
a[M] = a[M - 1];
a[L] = pivot;
if (has_values) {
pivot = v[ok];
for (M = ok; M > L; --M)
v[M] = v[M - 1];
v[L] = pivot;
}

// Update Adaptive runvars
std = labs(L - mu);

Check warning on line 2044 in Objects/listobject.c

View workflow job for this annotation

GitHub Actions / Windows (free-threading) / Build and test (x64) 

'function': conversion from 'Py_ssize_t' to 'long', possible loss of data [D:\a\cpython\cpython\PCbuild\pythoncore.vcxproj]

Check warning on line 2044 in Objects/listobject.c

View workflow job for this annotation

GitHub Actions / Windows / Build and test (arm64) 

'function': conversion from 'Py_ssize_t' to 'long', possible loss of data [C:\a\cpython\cpython\PCbuild\pythoncore.vcxproj]

Check warning on line 2044 in Objects/listobject.c

View workflow job for this annotation

GitHub Actions / Windows (free-threading) / Build and test (arm64) 

'function': conversion from 'Py_ssize_t' to 'long', possible loss of data [C:\a\cpython\cpython\PCbuild\pythoncore.vcxproj]

Check warning on line 2044 in Objects/listobject.c

View workflow job for this annotation

GitHub Actions / Windows / Build and test (x64) 

'function': conversion from 'Py_ssize_t' to 'long', possible loss of data [D:\a\cpython\cpython\PCbuild\pythoncore.vcxproj]
nbad += std;
mu = L;
}
}

// Return Adaptivity measure (max 1000)
// This is: 1000 * nbad / sum(range(nsorted:n))
return nbad * 2000 / ((n + 2 * nsorted - 1) * n);

Check warning on line 2052 in Objects/listobject.c

View workflow job for this annotation

GitHub Actions / Windows (free-threading) / Build and test (x64) 

'return': conversion from 'Py_ssize_t' to 'int', possible loss of data [D:\a\cpython\cpython\PCbuild\pythoncore.vcxproj]

Check warning on line 2052 in Objects/listobject.c

View workflow job for this annotation

GitHub Actions / Windows / Build and test (arm64) 

'return': conversion from 'Py_ssize_t' to 'int', possible loss of data [C:\a\cpython\cpython\PCbuild\pythoncore.vcxproj]

Check warning on line 2052 in Objects/listobject.c

View workflow job for this annotation

GitHub Actions / Windows (free-threading) / Build and test (arm64) 

'return': conversion from 'Py_ssize_t' to 'int', possible loss of data [C:\a\cpython\cpython\PCbuild\pythoncore.vcxproj]

Check warning on line 2052 in Objects/listobject.c

View workflow job for this annotation

GitHub Actions / Windows / Build and test (x64) 

'return': conversion from 'Py_ssize_t' to 'int', possible loss of data [D:\a\cpython\cpython\PCbuild\pythoncore.vcxproj]

fail:
return -1;
}

static void
sortslice_reverse(sortslice *s, Py_ssize_t n)
{
Expand Down Expand Up @@ -3074,36 +3243,97 @@
/* March over the array once, left to right, finding natural runs,
* and extending short natural runs to minrun elements.
*/
do {
Py_ssize_t n;

/* Identify next run. */
n = count_run(&ms, &lo, nremaining);
if (n < 0)
goto fail;
/* If short, extend to min(minrun, nremaining). */
minrun = minrun_next(&ms);
if (n < minrun) {
const Py_ssize_t force = nremaining <= minrun ?
nremaining : minrun;
if (binarysort(&ms, &lo, force, n) < 0)
// NOTE: Could turn on based on minlen or comparison type
int binary_adapt = ms.listlen >= 100;
if (!binary_adapt) {
do {
Py_ssize_t n;

/* Identify next run. */
n = count_run(&ms, &lo, nremaining);
if (n < 0)
goto fail;
n = force;
}
/* Maybe merge pending runs. */
assert(ms.n == 0 || ms.pending[ms.n -1].base.keys +
ms.pending[ms.n-1].len == lo.keys);
if (found_new_run(&ms, n) < 0)
goto fail;
/* Push new run on stack. */
assert(ms.n < MAX_MERGE_PENDING);
ms.pending[ms.n].base = lo;
ms.pending[ms.n].len = n;
++ms.n;
/* Advance to find next run. */
sortslice_advance(&lo, n);
nremaining -= n;
} while (nremaining);
/* If short, extend to min(minrun, nremaining). */
minrun = minrun_next(&ms);
if (n < minrun) {
const Py_ssize_t force = nremaining <= minrun ?
nremaining : minrun;
if (binarysort(&ms, &lo, force, n) < 0)
goto fail;
n = force;
}
/* Maybe merge pending runs. */
assert(ms.n == 0 || ms.pending[ms.n -1].base.keys +
ms.pending[ms.n-1].len == lo.keys);
if (found_new_run(&ms, n) < 0)
goto fail;
/* Push new run on stack. */
assert(ms.n < MAX_MERGE_PENDING);
ms.pending[ms.n].base = lo;
ms.pending[ms.n].len = n;
++ms.n;
/* Advance to find next run. */
sortslice_advance(&lo, n);
nremaining -= n;
} while (nremaining);
}
else {
int adapt = 0; // do not run binarysort adaptivity on 1st run
int cs = 0; // but do check goodness of adaptive fit
int cd = 1;
int abinret;
do {
Py_ssize_t n;

/* Identify next run. */
n = count_run(&ms, &lo, nremaining);
if (n < 0)
goto fail;
/* If short, extend to min(minrun, nremaining). */
minrun = minrun_next(&ms);
if (n < minrun) {
const Py_ssize_t force = nremaining <= minrun ?
nremaining : minrun;
if (cs) {
if (binarysort(&ms, &lo, force, n) < 0)
goto fail;
cs -= 1;
}
else {
abinret = abinarysort(&ms, &lo, force, n, adapt);
if (abinret < 0)
goto fail;
adapt = abinret < 250;
if (adapt)
cd = 1;
else if (cd >= 9)
cs = cd = 11;
else
cs = cd = cd + 2;
}
n = force;
}
else {
// After long monotonic run start adapting immediately
adapt = 1;
cs = 0;
cd = 1;
}
/* Maybe merge pending runs. */
assert(ms.n == 0 || ms.pending[ms.n -1].base.keys +
ms.pending[ms.n-1].len == lo.keys);
if (found_new_run(&ms, n) < 0)
goto fail;
/* Push new run on stack. */
assert(ms.n < MAX_MERGE_PENDING);
ms.pending[ms.n].base = lo;
ms.pending[ms.n].len = n;
++ms.n;
/* Advance to find next run. */
sortslice_advance(&lo, n);
nremaining -= n;
} while (nremaining);
}

if (merge_force_collapse(&ms) < 0)
goto fail;
Expand Down
Loading