diff --git a/lib/std/Build.zig b/lib/std/Build.zig index e3006fa25513..70cf752a008b 100644 --- a/lib/std/Build.zig +++ b/lib/std/Build.zig @@ -856,6 +856,7 @@ pub const TestOptions = struct { name: []const u8 = "test", root_module: *Module, max_rss: usize = 0, + exact_filters: bool = false, filters: []const []const u8 = &.{}, test_runner: ?Step.Compile.TestRunner = null, use_llvm: ?bool = null, @@ -881,6 +882,7 @@ pub fn addTest(b: *Build, options: TestOptions) *Step.Compile { .kind = if (options.emit_object) .test_obj else .@"test", .root_module = options.root_module, .max_rss = options.max_rss, + .exact_filters = options.exact_filters, .filters = b.dupeStrings(options.filters), .test_runner = options.test_runner, .use_llvm = options.use_llvm, diff --git a/lib/std/Build/Step/Compile.zig b/lib/std/Build/Step/Compile.zig index 0f47a0b64781..c7dc61689d0f 100644 --- a/lib/std/Build/Step/Compile.zig +++ b/lib/std/Build/Step/Compile.zig @@ -55,6 +55,7 @@ global_base: ?u64 = null, /// Set via options; intended to be read-only after that. zig_lib_dir: ?LazyPath, exec_cmd_args: ?[]const ?[]const u8, +exact_filters: bool, filters: []const []const u8, test_runner: ?TestRunner, wasi_exec_model: ?std.builtin.WasiExecModel = null, @@ -273,6 +274,7 @@ pub const Options = struct { linkage: ?std.builtin.LinkMode = null, version: ?std.SemanticVersion = null, max_rss: usize = 0, + exact_filters: bool = false, filters: []const []const u8 = &.{}, test_runner: ?TestRunner = null, use_llvm: ?bool = null, @@ -424,6 +426,7 @@ pub fn create(owner: *std.Build, options: Options) *Compile { .installed_headers = std.array_list.Managed(HeaderInstallation).init(owner.allocator), .zig_lib_dir = null, .exec_cmd_args = null, + .exact_filters = options.exact_filters, .filters = options.filters, .test_runner = null, // set below .rdynamic = false, @@ -1464,7 +1467,8 @@ fn getZigArgs(compile: *Compile, fuzz: bool) ![][]const u8 { } for (compile.filters) |filter| { - try zig_args.append("--test-filter"); + const flag = if (compile.exact_filters) "--test-filter-exact" else "--test-filter"; + try zig_args.append(flag); try zig_args.append(filter); } diff --git a/src/Compilation.zig b/src/Compilation.zig index 86b1356a3f11..25f8f507bc17 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -252,6 +252,9 @@ mutex: if (builtin.single_threaded) struct { pub inline fn unlock(_: @This()) void {} } else std.Thread.Mutex = .{}, +/// Filter tests by exact fully qualified name +test_filter_exact: bool, + test_filters: []const []const u8, link_task_wait_group: WaitGroup = .{}, @@ -1415,6 +1418,7 @@ pub const MiscTask = enum { analyze_mod, docs_copy, docs_wasm, + test_filter_match, @"musl crt1.o", @"musl rcrt1.o", @@ -1763,6 +1767,7 @@ pub const CreateOptions = struct { native_system_include_paths: []const []const u8 = &.{}, clang_preprocessor_mode: ClangPreprocessorMode = .no, reference_trace: ?u32 = null, + test_filter_exact: bool = false, test_filters: []const []const u8 = &.{}, test_runner_path: ?[]const u8 = null, subsystem: ?std.Target.SubSystem = null, @@ -2263,6 +2268,7 @@ pub fn create(gpa: Allocator, arena: Allocator, diag: *CreateDiagnostic, options .reference_trace = options.reference_trace, .time_report = if (options.time_report) .init else null, .stack_report = options.stack_report, + .test_filter_exact = options.test_filter_exact, .test_filters = options.test_filters, .debug_compiler_runtime_libs = options.debug_compiler_runtime_libs, .debug_compile_errors = options.debug_compile_errors, @@ -2413,6 +2419,7 @@ pub fn create(gpa: Allocator, arena: Allocator, diag: *CreateDiagnostic, options hash.add(options.config.use_new_linker); hash.add(options.config.dll_export_fns); hash.add(options.config.is_test); + hash.add(options.test_filter_exact); hash.addListOfBytes(options.test_filters); hash.add(options.skip_linker_dependencies); hash.add(options.emit_h != .no); @@ -3114,6 +3121,46 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) UpdateE // The `test_functions` decl has been intentionally postponed until now, // at which point we must populate it with the list of test functions that // have been discovered and not filtered out. + + if (comp.test_filter_exact) { + if (comp.test_filters.len > zcu.test_functions.count()) { + const ip = &zcu.intern_pool; + + var eb: ErrorBundle.Wip = undefined; + eb.init(gpa) catch return comp.setAllocFailure(); + + seen_test: for (comp.test_filters) |filter| { + for (zcu.test_functions.keys()) |test_nav_index| { + const test_nav = ip.getNav(test_nav_index); + const test_nav_name = test_nav.fqn; + const test_name = test_nav_name.toSlice(&zcu.intern_pool); + if (std.mem.eql(u8, filter, test_name)) { + continue :seen_test; + } + } + eb.addRootErrorMessage(.{ + .msg = try eb.printString( + "no test '{s}' found", + .{filter}, + ), + }) catch return comp.setAllocFailure(); + } + + const children = eb.toOwnedBundle("") catch + return comp.setAllocFailure(); + comp.misc_failures.ensureUnusedCapacity(gpa, 1) catch + return comp.setAllocFailure(); + const msg = gpa.dupe(u8, "could not find all requested tests") catch + return comp.setAllocFailure(); + const gop = comp.misc_failures.getOrPutAssumeCapacity(.test_filter_match); + if (gop.found_existing) { + gop.value_ptr.deinit(gpa); + } + gop.value_ptr.* = .{ .msg = msg, .children = children }; + return; + } + } + try pt.populateTestFunctions(); } @@ -3463,6 +3510,7 @@ fn addNonIncrementalStuffToCacheManifest( try addModuleTableToCacheHash(zcu, arena, &man.hash, .{ .files = man }); // Synchronize with other matching comments: ZigOnlyHashStuff + man.hash.add(comp.test_filter_exact); man.hash.addListOfBytes(comp.test_filters); man.hash.add(comp.skip_linker_dependencies); //man.hash.add(zcu.emit_h != .no); diff --git a/src/Zcu.zig b/src/Zcu.zig index 642d743145ff..bfc84fde522d 100644 --- a/src/Zcu.zig +++ b/src/Zcu.zig @@ -4124,10 +4124,14 @@ fn resolveReferencesInner(zcu: *Zcu) !std.AutoHashMapUnmanaged(AnalUnit, ?Resolv const want_analysis = switch (decl.kind) { .@"const", .@"var" => unreachable, .@"comptime" => unreachable, - .unnamed_test => true, + .unnamed_test => !comp.test_filter_exact, .@"test", .decltest => a: { const fqn_slice = nav.fqn.toSlice(ip); - if (comp.test_filters.len > 0) { + if (comp.test_filter_exact) { + for (comp.test_filters) |test_filter| { + if (std.mem.eql(u8, fqn_slice, test_filter)) break; + } else break :a false; + } else if (comp.test_filters.len > 0) { for (comp.test_filters) |test_filter| { if (std.mem.indexOf(u8, fqn_slice, test_filter) != null) break; } else break :a false; diff --git a/src/Zcu/PerThread.zig b/src/Zcu/PerThread.zig index 62b8756c4948..f84497ecdf03 100644 --- a/src/Zcu/PerThread.zig +++ b/src/Zcu/PerThread.zig @@ -2752,12 +2752,18 @@ const ScanDeclIter = struct { // Perhaps we should add all test indiscriminately and filter at the end of the update. if (!comp.config.is_test) break :a false; if (file.mod != zcu.main_mod) break :a false; - if (is_named and comp.test_filters.len > 0) { + if (is_named) { const fqn_slice = fqn.toSlice(ip); - for (comp.test_filters) |test_filter| { - if (std.mem.indexOf(u8, fqn_slice, test_filter) != null) break; - } else break :a false; - } + if (comp.test_filter_exact) { + for (comp.test_filters) |test_filter| { + if (std.mem.eql(u8, fqn_slice, test_filter)) break; + } else break :a false; + } else if (comp.test_filters.len > 0) { + for (comp.test_filters) |test_filter| { + if (std.mem.indexOf(u8, fqn_slice, test_filter) != null) break; + } else break :a false; + } + } else if (comp.test_filter_exact) break :a false; try zcu.test_functions.put(gpa, nav, {}); break :a true; }, diff --git a/src/main.zig b/src/main.zig index 89a552de0b1b..15309f935674 100644 --- a/src/main.zig +++ b/src/main.zig @@ -652,6 +652,9 @@ const usage_build_generic = \\ \\Test Options: \\ --test-filter [text] Skip tests that do not match any filter + \\ --test-filter-exact [text] Include the test with specified fully qualified name; this flag + \\ can be passed multiple times to include more than one test. + \\ Cannot be used with --test-filter. \\ --test-cmd [arg] Specify test execution command one arg at a time \\ --test-cmd-bin Appends test binary path to test cmd args \\ --test-no-exec Compiles test binary without running it @@ -878,6 +881,7 @@ fn buildOutputType( var build_id: ?std.zig.BuildId = null; var runtime_args_start: ?usize = null; var test_filters: std.ArrayListUnmanaged([]const u8) = .empty; + var test_filter_mode_exact: ?bool = null; var test_runner_path: ?[]const u8 = null; var override_local_cache_dir: ?[]const u8 = try EnvVar.ZIG_LOCAL_CACHE_DIR.get(arena); var override_global_cache_dir: ?[]const u8 = try EnvVar.ZIG_GLOBAL_CACHE_DIR.get(arena); @@ -1299,6 +1303,18 @@ fn buildOutputType( } else if (mem.eql(u8, arg, "--libc")) { create_module.libc_paths_file = args_iter.nextOrFatal(); } else if (mem.eql(u8, arg, "--test-filter")) { + if (test_filter_mode_exact) |b| { + if (b) { + fatal("cannot use both --test-filter and --test-filter-exact", .{}); + } + } else test_filter_mode_exact = false; + try test_filters.append(arena, args_iter.nextOrFatal()); + } else if (mem.eql(u8, arg, "--test-filter-exact")) { + if (test_filter_mode_exact) |b| { + if (!b) { + fatal("cannot use both --test-filter and --test-filter-exact", .{}); + } + } else test_filter_mode_exact = true; try test_filters.append(arena, args_iter.nextOrFatal()); } else if (mem.eql(u8, arg, "--test-runner")) { test_runner_path = args_iter.nextOrFatal(); @@ -3465,6 +3481,7 @@ fn buildOutputType( .time_report = time_report, .stack_report = stack_report, .build_id = build_id, + .test_filter_exact = test_filter_mode_exact orelse false, .test_filters = test_filters.items, .test_runner_path = test_runner_path, .cache_mode = cache_mode, diff --git a/test/standalone/test_filter_exact/build.zig b/test/standalone/test_filter_exact/build.zig new file mode 100644 index 000000000000..92059361ce88 --- /dev/null +++ b/test/standalone/test_filter_exact/build.zig @@ -0,0 +1,107 @@ +pub fn build(b: *std.Build) !void { + const root_module = b.createModule(.{ + .root_source_file = b.path("main.zig"), + .target = b.graph.host, + }); + + const test_step = b.step("test", "Run the tests"); + + const test_runner: std.Build.Step.Compile.TestRunner = .{ + .path = b.path("test_runner.zig"), + .mode = .simple, + }; + + for (passing_filters) |filters| { + const test_exe = b.addTest(.{ + .root_module = root_module, + .filters = filters, + .exact_filters = true, + .test_runner = test_runner, + }); + + const run = b.addRunArtifact(test_exe); + + for (filters) |filter| { + run.addCheck(.{ .expect_stdout_match = filter }); + } + test_step.dependOn(&run.step); + } + + var errors: std.ArrayListUnmanaged([]const u8) = .empty; + + for (misspelt_filters) |f| { + const filters: [2][]const u8 = .{ f[0][0], if (f.len == 2) f[1][0] else undefined }; + + const test_exe = b.addTest(.{ + .root_module = root_module, + .filters = filters[0..f.len], + .exact_filters = true, + }); + + try errors.append(b.allocator, "error: could not find all requested tests"); + for (f) |x| { + if (x[1]) { + try errors.append(b.allocator, b.fmt("note: no test '{s}' found", .{x[0]})); + } + } + test_exe.expect_errors = .{ .exact = try errors.toOwnedSlice(b.allocator) }; + test_step.dependOn(&test_exe.step); + } + + const fqn_clash_exe = b.addTest(.{ + .root_module = b.createModule(.{ + .root_source_file = b.path("fqn_clash.zig"), + .target = b.graph.host, + }), + .filters = &.{"fqn_clash.test.test.name"}, + .exact_filters = true, + .test_runner = test_runner, + }); + + const fqn_run = b.addRunArtifact(fqn_clash_exe); + fqn_run.expectStdOutEqual( + \\fqn_clash.test.test.name + \\fqn_clash.test.test.name + \\ + ); + + test_step.dependOn(&fqn_run.step); +} + +const passing_filters = [_][]const []const u8{ + &.{"main.struct_name.test.testname"}, + &.{ "main.struct_name.test.testname", "main.test.struct_name.testname" }, + &.{ "main.test.struct_name.testname", "main.struct_name.test.testname" }, + &.{"main.test.struct_name.testname"}, +}; + +const misspelt_filters = [_][]const struct { []const u8, bool }{ + &.{.{ "main.struct_name.test.testnam", true }}, + &.{.{ "ain.struct_name.test.testname", true }}, + &.{.{ "main.test.struct_name.testnam", true }}, + &.{.{ "ain.test.struct_name.testname", true }}, + + &.{ .{ "main.struct_name.test.testnam", true }, .{ "main.test.struct_name.testname", false } }, + &.{ .{ "main.struct_name.test.testnam", true }, .{ "main.test.struct_name.testnam", true } }, + &.{ .{ "main.struct_name.test.testnam", true }, .{ "ain.test.struct_name.testname", true } }, + + &.{ .{ "ain.struct_name.test.testname", true }, .{ "main.test.struct_name.testname", false } }, + &.{ .{ "ain.struct_name.test.testname", true }, .{ "main.test.struct_name.testnam", true } }, + &.{ .{ "ain.struct_name.test.testname", true }, .{ "ain.test.struct_name.testname", true } }, + + &.{ .{ "main.test.struct_name.testnam", true }, .{ "main.struct_name.test.testname", false } }, + &.{ .{ "main.test.struct_name.testnam", true }, .{ "main.struct_name.test.testnam", true } }, + &.{ .{ "main.test.struct_name.testnam", true }, .{ "ain.struct_name.test.testname", true } }, + + &.{ .{ "ain.test.struct_name.testname", true }, .{ "main.struct_name.test.testname", false } }, + &.{ .{ "ain.test.struct_name.testname", true }, .{ "main.struct_name.test.testnam", true } }, + &.{ .{ "ain.test.struct_name.testname", true }, .{ "ain.struct_name.test.testname", true } }, + + &.{ .{ "main.struct_name.test.testname", false }, .{ "main.test.struct_name.testnam", true } }, + &.{ .{ "main.struct_name.test.testname", false }, .{ "ain.test.struct_name.testname", true } }, + + &.{ .{ "main.test.struct_name.testname", false }, .{ "main.struct_name.test.testnam", true } }, + &.{ .{ "main.test.struct_name.testname", false }, .{ "ain.struct_name.test.testname", true } }, +}; + +const std = @import("std"); diff --git a/test/standalone/test_filter_exact/fqn_clash.zig b/test/standalone/test_filter_exact/fqn_clash.zig new file mode 100644 index 000000000000..ecb31fd24e92 --- /dev/null +++ b/test/standalone/test_filter_exact/fqn_clash.zig @@ -0,0 +1,13 @@ +test "test.name" {} + +const @"test" = struct { + test "name" {} +}; + +test "failing" { + return error.bad; +} + +comptime { + _ = @"test"; +} diff --git a/test/standalone/test_filter_exact/main.zig b/test/standalone/test_filter_exact/main.zig new file mode 100644 index 000000000000..ea796ac92e15 --- /dev/null +++ b/test/standalone/test_filter_exact/main.zig @@ -0,0 +1,27 @@ +const struct_name = struct { + test "testname" {} + + fn func1() void {} + + test func1 { + return error.bad; + } +}; + +test struct_name { + return error.bad; +} + +comptime { + _ = struct_name; +} + +test "struct_name.testname" {} + +test "unfiltered" { + return error.bad; +} + +test { + return error.bad; +} diff --git a/test/standalone/test_filter_exact/test_runner.zig b/test/standalone/test_filter_exact/test_runner.zig new file mode 100644 index 000000000000..25f48f9e4bcc --- /dev/null +++ b/test/standalone/test_filter_exact/test_runner.zig @@ -0,0 +1,27 @@ +const builtin = @import("builtin"); +const std = @import("std"); + +pub fn main() u8 { + const stdout = std.fs.File.stdout(); + var buffer: [512]u8 = undefined; + var stdout_writer = stdout.writer(&buffer); + const writer = &stdout_writer.interface; + + var fail_count: usize = 0; + + for (builtin.test_functions) |test_fn| { + writer.print("{s}\n", .{test_fn.name}) catch @panic("failed writing test name"); + test_fn.func() catch |err| { + writer.print("err: {t}\n", .{err}) catch @panic("failed writing error name"); + fail_count += 1; + }; + } + + writer.flush() catch @panic("failed flushing"); + + if (fail_count > 0) { + return 1; + } + + return 0; +}