Skip to content

Commit 5bec161

Browse files
notriddlenotriddle
committed
rustdoc-search: stringdex update with more packing
Before: 18M build/x86_64-unknown-linux-gnu/doc/search.index/ 57M build/x86_64-unknown-linux-gnu/compiler-doc/search.index/ After: 16M build/x86_64-unknown-linux-gnu/doc/search.index/ 49M build/x86_64-unknown-linux-gnu/compiler-doc/search.index/
1 parent ce4beeb commit 5bec161

File tree

3 files changed

+54
-30
lines changed

3 files changed

+54
-30
lines changed

‎Cargo.lock‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5233,9 +5233,9 @@ dependencies = [
52335233

52345234
[[package]]
52355235
name = "stringdex"
5236-
version = "0.0.1-alpha9"
5236+
version = "0.0.1-alpha10"
52375237
source = "registry+https://github.com/rust-lang/crates.io-index"
5238-
checksum = "7081029913fd7d591c0112182aba8c98ae886b4f12edb208130496cd17dc3c15"
5238+
checksum = "0fa846a7d509d1828a4f90962dc09810e161abcada7fc6a921e92c168d0811d7"
52395239
dependencies = [
52405240
"stacker",
52415241
]

‎src/librustdoc/Cargo.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ rustdoc-json-types = { path = "../rustdoc-json-types" }
2121
serde = { version = "1.0", features = ["derive"] }
2222
serde_json = "1.0"
2323
smallvec = "1.8.1"
24-
stringdex = { version = "0.0.1-alpha9" }
24+
stringdex = { version = "0.0.1-alpha10" }
2525
tempfile = "3"
2626
threadpool = "1.8.1"
2727
tracing = "0.1"

‎src/librustdoc/html/static/js/stringdex.js‎

Lines changed: 51 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -1108,22 +1108,39 @@ function loadDatabase(hooks) {
11081108
const id2 = id1 + ((nodeid[4] << 8) | nodeid[5]);
11091109
leaves = RoaringBitmap.makeSingleton(id1)
11101110
.union(RoaringBitmap.makeSingleton(id2));
1111+
} else if (!isWhole && (nodeid[0] & 0xf0) === 0x80) {
1112+
const id1 = ((nodeid[0] & 0x0f) << 16) | (nodeid[1] << 8) | nodeid[2];
1113+
const id2 = id1 + ((nodeid[3] << 4) | ((nodeid[4] >> 4) & 0x0f));
1114+
const id3 = id2 + (((nodeid[4] & 0x0f) << 8) | nodeid[5]);
1115+
leaves = RoaringBitmap.makeSingleton(id1)
1116+
.union(RoaringBitmap.makeSingleton(id2))
1117+
.union(RoaringBitmap.makeSingleton(id3));
11111118
} else {
11121119
leaves = RoaringBitmap.makeSingleton(
11131120
(nodeid[2] << 24) | (nodeid[3] << 16) |
11141121
(nodeid[4] << 8) | nodeid[5],
11151122
);
11161123
}
1117-
const data = (nodeid[0] & 0x20) !== 0 ?
1118-
Uint8Array.of(((nodeid[0] & 0x0f) << 4) | (nodeid[1] >> 4)) :
1119-
EMPTY_UINT8;
1120-
newPromise = Promise.resolve(new PrefixSearchTree(
1121-
EMPTY_SEARCH_TREE_BRANCHES,
1122-
EMPTY_SEARCH_TREE_BRANCHES,
1123-
data,
1124-
isWhole ? leaves : EMPTY_BITMAP,
1125-
isWhole ? EMPTY_BITMAP : leaves,
1126-
));
1124+
if (isWhole) {
1125+
const data = (nodeid[0] & 0x20) !== 0 ?
1126+
Uint8Array.of(((nodeid[0] & 0x0f) << 4) | (nodeid[1] >> 4)) :
1127+
EMPTY_UINT8;
1128+
newPromise = Promise.resolve(new PrefixSearchTree(
1129+
EMPTY_SEARCH_TREE_BRANCHES,
1130+
EMPTY_SEARCH_TREE_BRANCHES,
1131+
data,
1132+
leaves,
1133+
EMPTY_BITMAP,
1134+
));
1135+
} else {
1136+
const data = (nodeid[0] & 0xf0) === 0x80 ? 0 : (
1137+
((nodeid[0] & 0x0f) << 4) | (nodeid[1] >> 4));
1138+
newPromise = Promise.resolve(new SuffixSearchTree(
1139+
EMPTY_SEARCH_TREE_BRANCHES,
1140+
data,
1141+
leaves,
1142+
));
1143+
}
11271144
} else {
11281145
const hashHex = makeHexFromUint8Array(nodeid);
11291146
newPromise = new Promise((resolve, reject) => {
@@ -2748,6 +2765,7 @@ function loadDatabase(hooks) {
27482765
// because that's the canonical, hashed version of the data
27492766
let compression_tag = input[i];
27502767
const is_pure_suffixes_only_node = (compression_tag & 0x01) !== 0;
2768+
let no_leaves_flag;
27512769
if (compression_tag > 1) {
27522770
// compressed node
27532771
const is_long_compressed = (compression_tag & 0x04) !== 0;
@@ -2759,7 +2777,8 @@ function loadDatabase(hooks) {
27592777
compression_tag |= input[i] << 16;
27602778
i += 1;
27612779
}
2762-
let dlen = input[i];
2780+
let dlen = input[i] & 0x7F;
2781+
no_leaves_flag = input[i] & 0x80;
27632782
i += 1;
27642783
if (is_data_compressed) {
27652784
data = data_history[data_history.length - dlen - 1];
@@ -2786,10 +2805,15 @@ function loadDatabase(hooks) {
27862805
let whole;
27872806
let suffix;
27882807
if (is_pure_suffixes_only_node) {
2789-
suffix = input[i] === 0 ?
2790-
EMPTY_BITMAP1 :
2791-
new RoaringBitmap(input, i);
2792-
i += suffix.consumed_len_bytes;
2808+
if (no_leaves_flag) {
2809+
whole = EMPTY_BITMAP;
2810+
suffix = EMPTY_BITMAP;
2811+
} else {
2812+
suffix = input[i] === 0 ?
2813+
EMPTY_BITMAP1 :
2814+
new RoaringBitmap(input, i);
2815+
i += suffix.consumed_len_bytes;
2816+
}
27932817
tree = new SuffixSearchTree(
27942818
branches,
27952819
dlen,
@@ -2807,7 +2831,7 @@ function loadDatabase(hooks) {
28072831
let ci = 0;
28082832
canonical[ci] = 1;
28092833
ci += 1;
2810-
canonical[ci] = dlen;
2834+
canonical[ci] = dlen | no_leaves_flag;
28112835
ci += 1;
28122836
canonical[ci] = input[coffset]; // suffix child count
28132837
ci += 1;
@@ -2821,10 +2845,9 @@ function loadDatabase(hooks) {
28212845
}
28222846
siphashOfBytes(canonical.subarray(0, clen), 0, 0, 0, 0, hash);
28232847
} else {
2824-
if (input[i] === 0xff) {
2848+
if (no_leaves_flag) {
28252849
whole = EMPTY_BITMAP;
2826-
suffix = EMPTY_BITMAP1;
2827-
i += 1;
2850+
suffix = EMPTY_BITMAP;
28282851
} else {
28292852
whole = input[i] === 0 ?
28302853
EMPTY_BITMAP1 :
@@ -2856,7 +2879,7 @@ function loadDatabase(hooks) {
28562879
let ci = 0;
28572880
canonical[ci] = 0;
28582881
ci += 1;
2859-
canonical[ci] = dlen;
2882+
canonical[ci] = dlen | no_leaves_flag;
28602883
ci += 1;
28612884
canonical.set(data, ci);
28622885
ci += data.length;
@@ -2880,9 +2903,11 @@ function loadDatabase(hooks) {
28802903
}
28812904
hash[2] &= 0x7f;
28822905
} else {
2906+
i += 1;
28832907
// uncompressed node
2884-
const dlen = input [i + 1];
2885-
i += 2;
2908+
const dlen = input[i] & 0x7F;
2909+
no_leaves_flag = input[i] & 0x80;
2910+
i += 1;
28862911
if (dlen === 0 || is_pure_suffixes_only_node) {
28872912
data = EMPTY_UINT8;
28882913
} else {
@@ -2897,16 +2922,15 @@ function loadDatabase(hooks) {
28972922
i += branches_consumed_len_bytes;
28982923
let whole;
28992924
let suffix;
2900-
if (is_pure_suffixes_only_node) {
2925+
if (no_leaves_flag) {
2926+
whole = EMPTY_BITMAP;
2927+
suffix = EMPTY_BITMAP;
2928+
} else if (is_pure_suffixes_only_node) {
29012929
whole = EMPTY_BITMAP;
29022930
suffix = input[i] === 0 ?
29032931
EMPTY_BITMAP1 :
29042932
new RoaringBitmap(input, i);
29052933
i += suffix.consumed_len_bytes;
2906-
} else if (input[i] === 0xff) {
2907-
whole = EMPTY_BITMAP;
2908-
suffix = EMPTY_BITMAP;
2909-
i += 1;
29102934
} else {
29112935
whole = input[i] === 0 ?
29122936
EMPTY_BITMAP1 :

0 commit comments

Comments
 (0)