Merge pull request #22982 from mlugg/cache-mode

compiler: default to `.whole` cache mode for self-hosted backends

Author: jacobly0

src/Compilation.zig

@@ -2397,7 +2397,7 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) !void {
 
             // Work around windows `AccessDenied` if any files within this
             // directory are open by closing and reopening the file handles.
-            const need_writable_dance = w: {
+            const need_writable_dance: enum { no, lf_only, lf_and_debug } = w: {
                 if (builtin.os.tag == .windows) {
                     if (comp.bin_file) |lf| {
                         // We cannot just call `makeExecutable` as it makes a false
@@ -2410,11 +2410,13 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) !void {
                         if (lf.file) |f| {
                             f.close();
                             lf.file = null;
-                            break :w true;
+
+                            if (lf.closeDebugInfo()) break :w .lf_and_debug;
+                            break :w .lf_only;
                         }
                     }
                 }
-                break :w false;
+                break :w .no;
             };
 
             renameTmpIntoCache(comp.local_cache_directory, tmp_dir_sub_path, o_sub_path) catch |err| {
@@ -2441,8 +2443,13 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) !void {
                 };
 
                 // Has to be after the `wholeCacheModeSetBinFilePath` above.
-                if (need_writable_dance) {
-                    try lf.makeWritable();
+                switch (need_writable_dance) {
+                    .no => {},
+                    .lf_only => try lf.makeWritable(),
+                    .lf_and_debug => {
+                        try lf.makeWritable();
+                        try lf.reopenDebugInfo();
+                    },
                 }
             }
 

src/link.zig

@@ -600,6 +600,20 @@ pub const File = struct {
         }
     }
 
+    /// Some linkers create a separate file for debug info, which we might need to temporarily close
+    /// when moving the compilation result directory due to the host OS not allowing moving a
+    /// file/directory while a handle remains open.
+    /// Returns `true` if a debug info file was closed. In that case, `reopenDebugInfo` may be called.
+    pub fn closeDebugInfo(base: *File) bool {
+        const macho = base.cast(.macho) orelse return false;
+        return macho.closeDebugInfo();
+    }
+
+    pub fn reopenDebugInfo(base: *File) !void {
+        const macho = base.cast(.macho).?;
+        return macho.reopenDebugInfo();
+    }
+
     pub fn makeExecutable(base: *File) !void {
         dev.check(.make_executable);
         const comp = base.comp;

src/link/MachO.zig

@@ -3277,6 +3277,37 @@ const InitMetadataOptions = struct {
     program_code_size_hint: u64,
 };
 
+pub fn closeDebugInfo(self: *MachO) bool {
+    const d_sym = &(self.d_sym orelse return false);
+    d_sym.file.?.close();
+    d_sym.file = null;
+    return true;
+}
+
+pub fn reopenDebugInfo(self: *MachO) !void {
+    assert(self.d_sym.?.file == null);
+
+    assert(!self.base.comp.config.use_llvm);
+    assert(self.base.comp.config.debug_format == .dwarf);
+
+    const gpa = self.base.comp.gpa;
+    const sep = fs.path.sep_str;
+    const d_sym_path = try std.fmt.allocPrint(
+        gpa,
+        "{s}.dSYM" ++ sep ++ "Contents" ++ sep ++ "Resources" ++ sep ++ "DWARF",
+        .{self.base.emit.sub_path},
+    );
+    defer gpa.free(d_sym_path);
+
+    var d_sym_bundle = try self.base.emit.root_dir.handle.makeOpenPath(d_sym_path, .{});
+    defer d_sym_bundle.close();
+
+    self.d_sym.?.file = try d_sym_bundle.createFile(fs.path.basename(self.base.emit.sub_path), .{
+        .truncate = false,
+        .read = true,
+    });
+}
+
 // TODO: move to ZigObject
 fn initMetadata(self: *MachO, options: InitMetadataOptions) !void {
     if (!self.base.isRelocatable()) {
@@ -3333,25 +3364,8 @@ fn initMetadata(self: *MachO, options: InitMetadataOptions) !void {
         if (options.zo.dwarf) |*dwarf| {
             // Create dSYM bundle.
             log.debug("creating {s}.dSYM bundle", .{options.emit.sub_path});
-
-            const gpa = self.base.comp.gpa;
-            const sep = fs.path.sep_str;
-            const d_sym_path = try std.fmt.allocPrint(
-                gpa,
-                "{s}.dSYM" ++ sep ++ "Contents" ++ sep ++ "Resources" ++ sep ++ "DWARF",
-                .{options.emit.sub_path},
-            );
-            defer gpa.free(d_sym_path);
-
-            var d_sym_bundle = try options.emit.root_dir.handle.makeOpenPath(d_sym_path, .{});
-            defer d_sym_bundle.close();
-
-            const d_sym_file = try d_sym_bundle.createFile(options.emit.sub_path, .{
-                .truncate = false,
-                .read = true,
-            });
-
-            self.d_sym = .{ .allocator = gpa, .file = d_sym_file };
+            self.d_sym = .{ .allocator = self.base.comp.gpa, .file = null };
+            try self.reopenDebugInfo();
             try self.d_sym.?.initMetadata(self);
             try dwarf.initMetadata();
         }

src/link/MachO/DebugSymbols.zig

@@ -1,5 +1,5 @@
 allocator: Allocator,
-file: fs.File,
+file: ?fs.File,
 
 symtab_cmd: macho.symtab_command = .{},
 uuid_cmd: macho.uuid_command = .{ .uuid = [_]u8{0} ** 16 },
@@ -118,9 +118,9 @@ pub fn growSection(
         });
 
         if (requires_file_copy) {
-            const amt = try self.file.copyRangeAll(
+            const amt = try self.file.?.copyRangeAll(
                 sect.offset,
-                self.file,
+                self.file.?,
                 new_offset,
                 existing_size,
             );
@@ -129,7 +129,7 @@ pub fn growSection(
 
         sect.offset = @intCast(new_offset);
     } else if (sect.offset + allocated_size == std.math.maxInt(u64)) {
-        try self.file.setEndPos(sect.offset + needed_size);
+        try self.file.?.setEndPos(sect.offset + needed_size);
     }
 
     sect.size = needed_size;
@@ -165,7 +165,7 @@ fn detectAllocCollision(self: *DebugSymbols, start: u64, size: u64) !?u64 {
         }
     }
 
-    if (at_end) try self.file.setEndPos(end);
+    if (at_end) try self.file.?.setEndPos(end);
     return null;
 }
 
@@ -195,7 +195,7 @@ pub fn flushModule(self: *DebugSymbols, macho_file: *MachO) !void {
             sym_name,
             file_offset,
         });
-        try self.file.pwriteAll(mem.asBytes(&addr), file_offset);
+        try self.file.?.pwriteAll(mem.asBytes(&addr), file_offset);
     }
 
     self.finalizeDwarfSegment(macho_file);
@@ -208,7 +208,7 @@ pub fn flushModule(self: *DebugSymbols, macho_file: *MachO) !void {
 
 pub fn deinit(self: *DebugSymbols) void {
     const gpa = self.allocator;
-    self.file.close();
+    if (self.file) |file| file.close();
     self.segments.deinit(gpa);
     self.sections.deinit(gpa);
     self.relocs.deinit(gpa);
@@ -320,7 +320,7 @@ fn writeLoadCommands(self: *DebugSymbols, macho_file: *MachO) !struct { usize, u
 
     assert(stream.pos == needed_size);
 
-    try self.file.pwriteAll(buffer, @sizeOf(macho.mach_header_64));
+    try self.file.?.pwriteAll(buffer, @sizeOf(macho.mach_header_64));
 
     return .{ ncmds, buffer.len };
 }
@@ -346,7 +346,7 @@ fn writeHeader(self: *DebugSymbols, macho_file: *MachO, ncmds: usize, sizeofcmds
 
     log.debug("writing Mach-O header {}", .{header});
 
-    try self.file.pwriteAll(mem.asBytes(&header), 0);
+    try self.file.?.pwriteAll(mem.asBytes(&header), 0);
 }
 
 fn allocatedSize(self: *DebugSymbols, start: u64) u64 {
@@ -404,15 +404,15 @@ pub fn writeSymtab(self: *DebugSymbols, off: u32, macho_file: *MachO) !u32 {
         internal.writeSymtab(macho_file, self);
     }
 
-    try self.file.pwriteAll(mem.sliceAsBytes(self.symtab.items), cmd.symoff);
+    try self.file.?.pwriteAll(mem.sliceAsBytes(self.symtab.items), cmd.symoff);
 
     return off + cmd.nsyms * @sizeOf(macho.nlist_64);
 }
 
 pub fn writeStrtab(self: *DebugSymbols, off: u32) !u32 {
     const cmd = &self.symtab_cmd;
     cmd.stroff = off;
-    try self.file.pwriteAll(self.strtab.items, cmd.stroff);
+    try self.file.?.pwriteAll(self.strtab.items, cmd.stroff);
     return off + cmd.strsize;
 }
 

src/link/SpirV.zig

@@ -61,6 +61,18 @@ pub fn createEmpty(
     const gpa = comp.gpa;
     const target = comp.root_mod.resolved_target.result;
 
+    assert(!comp.config.use_lld); // Caught by Compilation.Config.resolve
+    assert(!comp.config.use_llvm); // Caught by Compilation.Config.resolve
+    assert(target.ofmt == .spirv); // Caught by Compilation.Config.resolve
+    switch (target.cpu.arch) {
+        .spirv, .spirv32, .spirv64 => {},
+        else => unreachable, // Caught by Compilation.Config.resolve.
+    }
+    switch (target.os.tag) {
+        .opencl, .opengl, .vulkan => {},
+        else => unreachable, // Caught by Compilation.Config.resolve.
+    }
+
     const self = try arena.create(SpirV);
     self.* = .{
         .base = .{
@@ -79,15 +91,11 @@ pub fn createEmpty(
     };
     errdefer self.deinit();
 
-    switch (target.cpu.arch) {
-        .spirv, .spirv32, .spirv64 => {},
-        else => unreachable, // Caught by Compilation.Config.resolve.
-    }
-
-    switch (target.os.tag) {
-        .opencl, .opengl, .vulkan => {},
-        else => unreachable, // Caught by Compilation.Config.resolve.
-    }
+    // TODO: read the file and keep valid parts instead of truncating
+    self.base.file = try emit.root_dir.handle.createFile(emit.sub_path, .{
+        .truncate = true,
+        .read = true,
+    });
 
     return self;
 }
@@ -98,24 +106,7 @@ pub fn open(
     emit: Path,
     options: link.File.OpenOptions,
 ) !*SpirV {
-    const target = comp.root_mod.resolved_target.result;
-    const use_lld = build_options.have_llvm and comp.config.use_lld;
-    const use_llvm = comp.config.use_llvm;
-
-    assert(!use_llvm); // Caught by Compilation.Config.resolve.
-    assert(!use_lld); // Caught by Compilation.Config.resolve.
-    assert(target.ofmt == .spirv); // Caught by Compilation.Config.resolve.
-
-    const spirv = try createEmpty(arena, comp, emit, options);
-    errdefer spirv.base.destroy();
-
-    // TODO: read the file and keep valid parts instead of truncating
-    const file = try emit.root_dir.handle.createFile(emit.sub_path, .{
-        .truncate = true,
-        .read = true,
-    });
-    spirv.base.file = file;
-    return spirv;
+    return createEmpty(arena, comp, emit, options);
 }
 
 pub fn deinit(self: *SpirV) void {

src/main.zig

@@ -3470,7 +3470,11 @@ fn buildOutputType(
         // incremental cache mode is used for LLVM backend too.
         if (create_module.resolved_options.use_llvm) break :b .whole;
 
-        break :b .incremental;
+        // Eventually, this default should be `.incremental`. However, since incremental
+        // compilation is currently an opt-in feature, it makes a strictly worse default cache mode
+        // than `.whole`.
+        // https://github.com/ziglang/zig/issues/21165
+        break :b .whole;
     };
 
     process.raiseFileDescriptorLimit();