Skip to content

Commit e3d1112

Browse files
authored
Rewrite wrapper generators file to python and mark noGC functions using artifacts from hazard analysis reports (#649)
* rewrite generate_wrappers to python it's easier to do complex processing in python Signed-off-by: sagudev <[email protected]> * Use allFuctions and gcFunctions to create noGC and use this to make functions no GC in wrappers Signed-off-by: sagudev <[email protected]> --------- Signed-off-by: sagudev <[email protected]>
1 parent 77645ed commit e3d1112

File tree

6 files changed

+424
-319
lines changed

6 files changed

+424
-319
lines changed

.github/workflows/build.yml

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -101,7 +101,8 @@ jobs:
101101
# we generate wrappers only without debugmozjs
102102
if: ${{ matrix.features != 'debugmozjs' }}
103103
run: |
104-
bash ./mozjs/src/generate_wrappers.sh
104+
python3 ./mozjs/src/dl_and_gen_noGC.py
105+
python3 ./mozjs/src/generate_wrappers.py
105106
git add --all .
106107
git diff --staged --no-ext-diff --exit-code
107108

README.md

Lines changed: 11 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -141,22 +141,24 @@ In order to upgrade to a new version of SpiderMonkey:
141141
commit, including an artefact uploaded link, with a name of the form
142142
mozjs-*version*.tar.xz. Download it and save it locally.
143143

144-
3. Look at the patches in `mozjs-sys/etc/patches/*.patch`, and remove any that no longer apply
145-
(with a bit of luck this will be all of them).
144+
3. Go to <https://treeherder.mozilla.org/jobs?repo=mozilla-esr140&revision=${COMMIT}> and download artifacts `allFUnctions.txt.gz` and `gcFunctions.txt.gz` from job Linux debug > H
145+
146+
4. Create a new release on github with all files you downloaded. Name the new tag `mozjs-source-${COMMIT}`.
146147

147-
4. Run `python3 ./mozjs-sys/etc/update.py path/to/tarball`.
148+
5. Look at the patches in `mozjs-sys/etc/patches/*.patch`, and remove any that no longer apply
149+
(with a bit of luck this will be all of them).
148150

149-
5. Update `mozjs-sys/etc/COMMIT` with the commit number and mozjs-sys version with SpiderMonkey version.
151+
6. Run `python3 ./mozjs-sys/etc/update.py path/to/tarball`.
150152

151-
6. Run `./mozjs/src/generate_wrappers.sh` to regenerate wrappers.
153+
7. Update `mozjs-sys/etc/COMMIT` with the commit number and mozjs-sys version with SpiderMonkey version.
152154

153-
7. Build and test the bindings as above.
155+
8. Run `./mozjs/src/dl_and_gen_noGC.py` and `./mozjs/src/generate_wrappers.py` to regenerate wrappers.
154156

155-
8. Create a new release on github with the .tar.xz that you saved earlier. Name the new tag `mozjs-source-${COMMIT}` where `${COMMIT}` is the value stored in `mozjs/etc/COMMIT`.
157+
9. Build and test the bindings as above.
156158

157-
9. Submit a PR!
159+
10. Submit a PR!
158160

159-
10. Send companion PR to servo, as SpiderMonkey bump PR will not be merged
161+
11. Send companion PR to servo, as SpiderMonkey bump PR will not be merged
160162
until it's tested against servo.
161163

162164
## NixOS users

mozjs/src/dl_and_gen_noGC.py

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
#!/usr/bin/env python3
2+
3+
import gzip
4+
from urllib.request import urlretrieve
5+
6+
with open("mozjs-sys/etc/COMMIT", "r") as f:
7+
commit = f.read().strip()
8+
9+
print(f"Commit: {commit}")
10+
11+
url = f"https://github.com/servo/mozjs/releases/download/mozjs-source-{commit}"
12+
13+
urlretrieve(f"{url}/gcFunctions.txt.gz", "target/gcFunctions.txt.gz")
14+
urlretrieve(f"{url}/allFunctions.txt.gz", "target/allFunctions.txt.gz")
15+
16+
gc_functions = set()
17+
18+
with gzip.open("target/gcFunctions.txt.gz", "rt") as f:
19+
for line in f:
20+
if line.startswith("GC Function: "):
21+
stripped_line = line.removeprefix("GC Function: ").strip()
22+
gc_functions.add(stripped_line)
23+
24+
with gzip.open(
25+
"target/noGC.txt.gz", "wt"
26+
) as out_file: # 'wt' for text mode writing to a gzipped file
27+
with gzip.open("target/allFunctions.txt.gz", "rt") as f:
28+
for line in f:
29+
line_s = line.strip()
30+
if (
31+
"mozilla::dom" not in line_s
32+
and "mozilla::net" not in line_s
33+
and line_s not in gc_functions
34+
):
35+
out_file.write(line) # Write the line to the new file

mozjs/src/generate_wrappers.py

Lines changed: 171 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,171 @@
1+
#!/usr/bin/env python3
2+
import re
3+
import shutil
4+
from pathlib import Path
5+
import subprocess
6+
import gzip
7+
8+
9+
no_gc = set()
10+
11+
with gzip.open("target/noGC.txt.gz", "rt") as f:
12+
for line in f:
13+
no_gc.add(line.split(maxsplit=1)[0].split("$", maxsplit=1)[0])
14+
15+
16+
def read_file(file_path: Path):
17+
return file_path.read_text(encoding="utf-8")
18+
19+
20+
def write_file(file_path: Path, lines):
21+
file_path.parent.mkdir(parents=True, exist_ok=True)
22+
file_path.write_text("\n".join(lines) + "\n", encoding="utf-8")
23+
24+
25+
def grep_functions(file_path: Path) -> list[tuple[str, str | None]]:
26+
content = read_file(file_path)
27+
28+
# Match:
29+
# - optional #[link_name = "..."]
30+
# - followed by pub fn ...;
31+
pattern = re.compile(
32+
r'(?:#\s*\[\s*link_name\s*=\s*"(?P<link>[^"]+)"\s*\]\s*)?'
33+
r"(?P<sig>pub\s+fn[^;{]+)\s*;",
34+
re.MULTILINE,
35+
)
36+
37+
return [
38+
(
39+
re.sub(r"\s+", " ", m.group("sig").strip()),
40+
(m.group("link") or "").removeprefix("\\u{1}") or None,
41+
)
42+
for m in pattern.finditer(content)
43+
]
44+
45+
46+
def grep_heur(file_path: Path) -> list[str]:
47+
def no_link_name(fn: tuple[str, str | None]) -> str:
48+
sig, _ = fn
49+
return sig
50+
51+
def filter_pre(line: str) -> bool:
52+
return (
53+
"Handle" in line
54+
and "roxyHandler" not in line
55+
and "JS::IdVector" not in line
56+
and "pub fn Unbox" not in line
57+
and "CopyAsyncStack" not in line
58+
and "MutableHandleObjectVector" not in line
59+
)
60+
61+
def replace_in_line(line: str) -> str:
62+
return (
63+
line.replace("root::", "")
64+
.replace("JS::", "")
65+
.replace("js::", "")
66+
.replace("mozilla::", "")
67+
.replace("Handle<*mut JSObject>", "HandleObject")
68+
)
69+
70+
def filter_post(line: str) -> bool:
71+
return (
72+
# We are only wrapping handles in args not in results
73+
"-> Handle" not in line and "-> MutableHandle" not in line
74+
)
75+
76+
return list(
77+
filter(
78+
filter_post,
79+
map(
80+
replace_in_line,
81+
filter(filter_pre, map(no_link_name, grep_functions(file_path))),
82+
),
83+
)
84+
)
85+
86+
87+
# print(grep_functions(Path("./target/wrap_jsapi.rs")))
88+
# exit(0)
89+
90+
91+
def grep_heur2(file_path: Path) -> list[str]:
92+
def filter_pre(fn: tuple[str, str | None]) -> bool:
93+
sig, _ = fn
94+
return (
95+
("Handle" in sig or "JSContext" in sig)
96+
and "roxyHandler" not in sig
97+
and "JS::IdVector" not in sig
98+
and "pub fn Unbox" not in sig
99+
and "CopyAsyncStack" not in sig
100+
and "MutableHandleObjectVector" not in sig
101+
and "Opaque" not in sig
102+
and "pub fn JS_WrapPropertyDescriptor1" not in sig
103+
and "pub fn EncodeWideToUtf8" not in sig
104+
and "pub fn JS_NewContext" not in sig # returns jscontext
105+
# gc module causes problems in macro
106+
and "pub fn NewMemoryInfo" not in sig
107+
and "pub fn GetGCContext" not in sig
108+
and "pub fn SetDebuggerMalloc" not in sig
109+
and "pub fn GetDebuggerMallocSizeOf" not in sig
110+
and "pub fn FireOnGarbageCollectionHookRequired" not in sig
111+
and "pub fn ShouldAvoidSideEffects" not in sig
112+
# vargs
113+
and "..." not in sig
114+
and "VA(" not in sig
115+
)
116+
117+
def replace_in_line(fn: tuple[str, str | None]) -> str:
118+
sig, link_name = fn
119+
sig = (
120+
sig.replace("root::", "")
121+
.replace("JS::", "")
122+
.replace("js::", "")
123+
.replace("mozilla::", "")
124+
.replace("*mut JSContext", "&mut JSContext")
125+
.replace("*const JSContext", "&JSContext")
126+
)
127+
if link_name in no_gc:
128+
sig = sig.replace("&mut JSContext", "&JSContext")
129+
return sig
130+
131+
def filter_post(line: str) -> bool:
132+
return (
133+
# We are only wrapping handles in args not in results
134+
"-> Handle" not in line and "-> MutableHandle" not in line
135+
)
136+
137+
return list(
138+
filter(
139+
filter_post,
140+
map(replace_in_line, filter(filter_pre, grep_functions(file_path))),
141+
)
142+
)
143+
144+
145+
def find_latest_version_of_file_and_parse(
146+
input_file: str, out_module: str, heur_fn, extra: str = ""
147+
):
148+
target_dir = Path("target")
149+
files = list(target_dir.rglob(input_file))
150+
if not files:
151+
raise FileNotFoundError(f"No file found matching {input_file} in target/")
152+
153+
newest_file = max(files, key=lambda f: f.stat().st_mtime)
154+
155+
wrap_file = target_dir / f"wrap_{input_file}"
156+
shutil.copy(newest_file, wrap_file)
157+
158+
subprocess.run(
159+
["rustfmt", str(wrap_file), "--config", "max_width=1000"], check=True
160+
)
161+
162+
lines = heur_fn(wrap_file)
163+
out_file = Path("mozjs/src") / f"{out_module}{extra}_wrappers.in.rs"
164+
wrapped_lines = [f"wrap!({out_module}: {line});" for line in lines]
165+
write_file(out_file, wrapped_lines)
166+
167+
168+
find_latest_version_of_file_and_parse("jsapi.rs", "jsapi", grep_heur)
169+
find_latest_version_of_file_and_parse("gluebindings.rs", "glue", grep_heur)
170+
find_latest_version_of_file_and_parse("jsapi.rs", "jsapi", grep_heur2, "2")
171+
find_latest_version_of_file_and_parse("gluebindings.rs", "glue", grep_heur2, "2")

mozjs/src/generate_wrappers.sh

Lines changed: 0 additions & 104 deletions
This file was deleted.

0 commit comments

Comments
 (0)