Skip to content

Commit 3564b71

Browse files
kripkenkripken
authored
NaN fuzzing improvements (#1913)
* make DE_NAN avoid creating nan literals in the first place * add a reducer option `--denan` to not introduce nans in destructive reduction * add a `Literal::isNaN()` method * also remove the default exception logging from the fuzzer js glue, which is a source of non-useful VM differences (like nan nondeterminism) * added an option `--no-fuzz-nans` to make it easy to avoid nans when fuzzing (without hacking the source and recompiling). Background: trying to get fuzzing on jsc working despite this open issue: https://bugs.webkit.org/show_bug.cgi?id=175691
1 parent 8b820ed commit 3564b71

File tree

11 files changed

+764
-51
lines changed

11 files changed

+764
-51
lines changed

‎scripts/fuzz_opt.py‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -27,9 +27,12 @@
2727
# parameters
2828

2929

30-
LOG_LIMIT = 125
30+
FUZZ_OPTS = ['--mvp-features'] # may want to add '--no-fuzz-nans' for cross-VM testing
31+
3132
INPUT_SIZE_LIMIT = 250 * 1024
3233

34+
LOG_LIMIT = 125
35+
3336

3437
# utilities
3538

@@ -122,9 +125,8 @@ def run_vm(cmd):
122125
results = []
123126
# append to this list to add results from VMs
124127
results += [fix_output(run_vm([in_bin('wasm-opt'), prefix + 'wasm', '--fuzz-exec-before']))]
125-
results += [fix_output(run_vm([os.path.expanduser('d8'), '--experimental-wasm-sat_f2i_conversions', prefix + 'js', '--', prefix + 'wasm']))]
126-
results += [fix_output(run_vm([os.path.expanduser('d8-debug'), '--experimental-wasm-sat_f2i_conversions', '--wasm-tier-up', prefix + 'js', '--', prefix + 'wasm']))]
127-
results += [fix_output(run_vm([os.path.expanduser('d8-debug'), '--experimental-wasm-sat_f2i_conversions', '--no-wasm-tier-up', prefix + 'js', '--', prefix + 'wasm']))]
128+
results += [fix_output(run_vm([os.path.expanduser('d8'), prefix + 'js', '--', prefix + 'wasm']))]
129+
# results += [fix_output(run_vm([os.path.expanduser('~/.jsvu/jsc'), prefix + 'js', '--', prefix + 'wasm']))]
128130
# spec has no mechanism to not halt on a trap. so we just check until the first trap, basically
129131
# run(['../spec/interpreter/wasm', prefix + 'wasm'])
130132
# results += [fix_spec_output(run_unchecked(['../spec/interpreter/wasm', prefix + 'wasm', '-e', open(prefix + 'wat').read()]))]
@@ -146,7 +148,7 @@ def test_one(infile, opts):
146148

147149
# fuzz vms
148150
# gather VM outputs on input file
149-
run([in_bin('wasm-opt'), infile, '-ttf', '--emit-js-wrapper=a.js', '--emit-spec-wrapper=a.wat', '-o', 'a.wasm', '--mvp-features'])
151+
run([in_bin('wasm-opt'), infile, '-ttf', '--emit-js-wrapper=a.js', '--emit-spec-wrapper=a.wat', '-o', 'a.wasm'] + FUZZ_OPTS)
150152
wasm_size = os.stat('a.wasm').st_size
151153
bytes += wasm_size
152154
print('pre js size :', os.stat('a.js').st_size, ' wasm size:', wasm_size)

‎src/literal.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -110,6 +110,8 @@ class Literal {
110110
bool operator==(const Literal& other) const;
111111
bool operator!=(const Literal& other) const;
112112

113+
bool isNaN();
114+
113115
static uint32_t NaNPayload(float f);
114116
static uint64_t NaNPayload(double f);
115117
static float setQuietNaN(float f);

‎src/tools/fuzzing.h‎

Lines changed: 20 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -123,8 +123,15 @@ class TranslateToFuzzReader {
123123
std::cout << "shrink level: " << options.passOptions.shrinkLevel << '\n';
124124
}
125125

126-
void build(FeatureSet features_) {
126+
void setFeatures(FeatureSet features_) {
127127
features = features_;
128+
}
129+
130+
void setAllowNaNs(bool allowNaNs_) {
131+
allowNaNs = allowNaNs_;
132+
}
133+
134+
void build() {
128135
setupMemory();
129136
setupTable();
130137
setupGlobals();
@@ -137,7 +144,7 @@ class TranslateToFuzzReader {
137144
if (HANG_LIMIT > 0) {
138145
addHangLimitSupport();
139146
}
140-
if (DE_NAN) {
147+
if (!allowNaNs) {
141148
addDeNanSupport();
142149
}
143150
finalizeTable();
@@ -178,7 +185,7 @@ class TranslateToFuzzReader {
178185
// Optionally remove NaNs, which are a source of nondeterminism (which makes
179186
// cross-VM comparisons harder)
180187
// TODO: de-NaN SIMD values
181-
static const bool DE_NAN = true;
188+
bool allowNaNs = true;
182189

183190
// Features allowed to be emitted
184191
FeatureSet features = FeatureSet::All;
@@ -361,7 +368,7 @@ class TranslateToFuzzReader {
361368
}
362369

363370
Expression* makeDeNanOp(Expression* expr) {
364-
if (!DE_NAN) return expr;
371+
if (allowNaNs) return expr;
365372
if (expr->type == f32) {
366373
return builder.makeCall("deNan32", { expr }, f32);
367374
} else if (expr->type == f64) {
@@ -1213,7 +1220,7 @@ class TranslateToFuzzReader {
12131220
return store;
12141221
}
12151222

1216-
Literal makeLiteral(Type type) {
1223+
Literal makeArbitraryLiteral(Type type) {
12171224
if (type == v128) {
12181225
// generate each lane individually for random lane interpretation
12191226
switch (upTo(6)) {
@@ -1344,6 +1351,14 @@ class TranslateToFuzzReader {
13441351
WASM_UNREACHABLE();
13451352
}
13461353

1354+
Literal makeLiteral(Type type) {
1355+
auto ret = makeArbitraryLiteral(type);
1356+
if (!allowNaNs && ret.isNaN()) {
1357+
ret = Literal::makeFromInt32(0, type);
1358+
}
1359+
return ret;
1360+
}
1361+
13471362
Expression* makeConst(Type type) {
13481363
auto* ret = wasm.allocator.alloc<Const>();
13491364
ret->value = makeLiteral(type);

‎src/tools/js-wrapper.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -103,7 +103,7 @@ static std::string generateJSWrapper(Module& wasm) {
103103
}
104104
ret += ";\n";
105105
ret += "} catch (e) {\n";
106-
ret += " console.log('exception: ' + e);\n";
106+
ret += " console.log('exception!' /* + e */);\n";
107107
ret += "}\n";
108108
}
109109
return ret;

‎src/tools/wasm-opt.cpp‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,7 @@ int main(int argc, const char* argv[]) {
7171
std::string extraFuzzCommand;
7272
bool translateToFuzz = false;
7373
bool fuzzPasses = false;
74+
bool fuzzNaNs = true;
7475
std::string emitJSWrapper;
7576
std::string emitSpecWrapper;
7677
std::string inputSourceMapFilename;
@@ -112,6 +113,9 @@ int main(int argc, const char* argv[]) {
112113
.add("--fuzz-passes", "-fp", "Pick a random set of passes to run, useful for fuzzing. this depends on translate-to-fuzz (it picks the passes from the input)",
113114
Options::Arguments::Zero,
114115
[&](Options *o, const std::string& arguments) { fuzzPasses = true; })
116+
.add("--no-fuzz-nans", "", "don't emit NaNs when fuzzing, and remove them at runtime as well (helps avoid nondeterminism between VMs)",
117+
Options::Arguments::Zero,
118+
[&](Options *o, const std::string& arguments) { fuzzNaNs = false; })
115119
.add("--emit-js-wrapper", "-ejw", "Emit a JavaScript wrapper file that can run the wasm with some test values, useful for fuzzing",
116120
Options::Arguments::One,
117121
[&](Options *o, const std::string& arguments) { emitJSWrapper = arguments; })
@@ -166,7 +170,9 @@ int main(int argc, const char* argv[]) {
166170
if (fuzzPasses) {
167171
reader.pickPasses(options);
168172
}
169-
reader.build(options.getFeatures());
173+
reader.setFeatures(options.getFeatures());
174+
reader.setAllowNaNs(fuzzNaNs);
175+
reader.build();
170176
if (options.passOptions.validate) {
171177
if (!WasmValidator().validate(wasm, options.getFeatures())) {
172178
WasmPrinter::printModule(&wasm);

‎src/tools/wasm-reduce.cpp‎

Lines changed: 27 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -221,11 +221,12 @@ static std::unordered_set<Name> functionsWeTriedToRemove;
221221

222222
struct Reducer : public WalkerPass<PostWalker<Reducer, UnifiedExpressionVisitor<Reducer>>> {
223223
std::string command, test, working;
224-
bool binary, verbose, debugInfo;
224+
bool binary, deNan, verbose, debugInfo;
225225

226226
// test is the file we write to that the command will operate on
227227
// working is the current temporary state, the reduction so far
228-
Reducer(std::string command, std::string test, std::string working, bool binary, bool verbose, bool debugInfo) : command(command), test(test), working(working), binary(binary), verbose(verbose), debugInfo(debugInfo) {}
228+
Reducer(std::string command, std::string test, std::string working, bool binary, bool deNan, bool verbose, bool debugInfo) :
229+
command(command), test(test), working(working), binary(binary), deNan(deNan), verbose(verbose), debugInfo(debugInfo) {}
229230

230231
// runs passes in order to reduce, until we can't reduce any more
231232
// the criterion here is wasm binary size
@@ -360,8 +361,22 @@ struct Reducer : public WalkerPass<PostWalker<Reducer, UnifiedExpressionVisitor<
360361
return (counter % factor) <= bonus;
361362
}
362363

364+
bool isOkReplacement(Expression* with) {
365+
if (deNan) {
366+
if (auto* c = with->dynCast<Const>()) {
367+
if (c->value.isNaN()) {
368+
return false;
369+
}
370+
}
371+
}
372+
return true;
373+
}
374+
363375
// tests a reduction on the current traversal node, and undos if it failed
364376
bool tryToReplaceCurrent(Expression* with) {
377+
if (!isOkReplacement(with)) {
378+
return false;
379+
}
365380
auto* curr = getCurrent();
366381
//std::cerr << "try " << curr << " => " << with << '\n';
367382
if (curr->type != with->type) return false;
@@ -383,6 +398,9 @@ struct Reducer : public WalkerPass<PostWalker<Reducer, UnifiedExpressionVisitor<
383398

384399
// tests a reduction on an arbitrary child
385400
bool tryToReplaceChild(Expression*& child, Expression* with) {
401+
if (!isOkReplacement(with)) {
402+
return false;
403+
}
386404
if (child->type != with->type) return false;
387405
if (!shouldTryToReduce()) return false;
388406
auto* before = child;
@@ -865,6 +883,7 @@ struct Reducer : public WalkerPass<PostWalker<Reducer, UnifiedExpressionVisitor<
865883
int main(int argc, const char* argv[]) {
866884
std::string input, test, working, command;
867885
bool binary = true,
886+
deNan = false,
868887
verbose = false,
869888
debugInfo = false,
870889
force = false;
@@ -899,6 +918,11 @@ int main(int argc, const char* argv[]) {
899918
[&](Options* o, const std::string& argument) {
900919
binary = false;
901920
})
921+
.add("--denan", "", "Avoid nans when reducing",
922+
Options::Arguments::Zero,
923+
[&](Options* o, const std::string& argument) {
924+
deNan = true;
925+
})
902926
.add("--verbose", "-v", "Verbose output mode",
903927
Options::Arguments::Zero,
904928
[&](Options* o, const std::string& argument) {
@@ -997,7 +1021,7 @@ int main(int argc, const char* argv[]) {
9971021
bool stopping = false;
9981022

9991023
while (1) {
1000-
Reducer reducer(command, test, working, binary, verbose, debugInfo);
1024+
Reducer reducer(command, test, working, binary, deNan, verbose, debugInfo);
10011025

10021026
// run binaryen optimization passes to reduce. passes are fast to run
10031027
// and can often reduce large amounts of code efficiently, as opposed

‎src/wasm/literal.cpp‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -144,6 +144,17 @@ bool Literal::operator!=(const Literal& other) const {
144144
return !(*this == other);
145145
}
146146

147+
bool Literal::isNaN() {
148+
if (type == Type::f32 && std::isnan(getf32())) {
149+
return true;
150+
}
151+
if (type == Type::f64 && std::isnan(getf64())) {
152+
return true;
153+
}
154+
// TODO: SIMD?
155+
return false;
156+
}
157+
147158
uint32_t Literal::NaNPayload(float f) {
148159
assert(std::isnan(f) && "expected a NaN");
149160
// SEEEEEEE EFFFFFFF FFFFFFFF FFFFFFFF

‎test/passes/emit-js-wrapper=a.js.wast.js‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -51,33 +51,33 @@ try {
5151
console.log('[fuzz-exec] calling $add');
5252
console.log('[fuzz-exec] note result: $add => ' + literal(instance.exports.add(0, 0), 'i32'));
5353
} catch (e) {
54-
console.log('exception: ' + e);
54+
console.log('exception!' /* + e */);
5555
}
5656
if (instance.exports.hangLimitInitializer) instance.exports.hangLimitInitializer();
5757
try {
5858
console.log('[fuzz-exec] calling $no_return');
5959
instance.exports.no_return(0);
6060
} catch (e) {
61-
console.log('exception: ' + e);
61+
console.log('exception!' /* + e */);
6262
}
6363
if (instance.exports.hangLimitInitializer) instance.exports.hangLimitInitializer();
6464
try {
6565
console.log('[fuzz-exec] calling $types');
6666
instance.exports.types(0, 0, 0, 0, 0);
6767
} catch (e) {
68-
console.log('exception: ' + e);
68+
console.log('exception!' /* + e */);
6969
}
7070
if (instance.exports.hangLimitInitializer) instance.exports.hangLimitInitializer();
7171
try {
7272
console.log('[fuzz-exec] calling $types2');
7373
instance.exports.types2(0, 0, 0);
7474
} catch (e) {
75-
console.log('exception: ' + e);
75+
console.log('exception!' /* + e */);
7676
}
7777
if (instance.exports.hangLimitInitializer) instance.exports.hangLimitInitializer();
7878
try {
7979
console.log('[fuzz-exec] calling $types3');
8080
console.log('[fuzz-exec] note result: $types3 => ' + literal(instance.exports.types3(0, 0, 0), 'i32'));
8181
} catch (e) {
82-
console.log('exception: ' + e);
82+
console.log('exception!' /* + e */);
8383
}

‎test/passes/translate-to-fuzz.txt‎

Lines changed: 5 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,8 @@
1717
(import "fuzzing-support" "log-f64" (func $log-f64 (param f64)))
1818
(memory $0 (shared 1 1))
1919
(data (i32.const 0) "n\00\05E\00\00\00\00")
20-
(table $0 6 6 funcref)
21-
(elem (i32.const 0) $func_6 $func_12 $func_12 $func_12 $func_15 $func_16)
20+
(table $0 10 10 funcref)
21+
(elem (i32.const 0) $func_6 $func_12 $func_12 $func_12 $func_15 $func_16 $func_17 $func_17 $func_17 $func_17)
2222
(global $global$0 (mut f32) (f32.const 536870912))
2323
(global $global$1 (mut f32) (f32.const 2147483648))
2424
(global $global$2 (mut f64) (f64.const -1048576))
@@ -526,7 +526,7 @@
526526
(global.get $hangLimit)
527527
)
528528
(return
529-
(f32.const 865309568)
529+
(f32.const 185009408)
530530
)
531531
)
532532
(global.set $hangLimit
@@ -537,15 +537,9 @@
537537
)
538538
)
539539
(if (result f32)
540-
(i32.eqz
541-
(if (result i32)
542-
(i32.const 709182789)
543-
(i32.const -4)
544-
(i32.const 873467920)
545-
)
546-
)
540+
(i32.const 1230459474)
547541
(block $label$5 (result f32)
548-
(f32.const 59953536565248)
542+
(f32.const 121)
549543
)
550544
(block $label$6 (result f32)
551545
(f32.const 1)
@@ -557,24 +551,4 @@
557551
(i32.const 10)
558552
)
559553
)
560-
(func $deNan32 (; 19 ;) (param $0 f32) (result f32)
561-
(if (result f32)
562-
(f32.eq
563-
(local.get $0)
564-
(local.get $0)
565-
)
566-
(local.get $0)
567-
(f32.const 0)
568-
)
569-
)
570-
(func $deNan64 (; 20 ;) (param $0 f64) (result f64)
571-
(if (result f64)
572-
(f64.eq
573-
(local.get $0)
574-
(local.get $0)
575-
)
576-
(local.get $0)
577-
(f64.const 0)
578-
)
579-
)
580554
)

0 commit comments

Comments
 (0)