x86_64: support rip-relative addressing to labels in inline asm

Author: jacobly0

src/arch/x86_64/CodeGen.zig

@@ -103178,6 +103178,19 @@ fn performReloc(self: *CodeGen, reloc: Mir.Inst.Index) void {
             .pseudo_j_z_and_np_inst, .pseudo_j_nz_or_p_inst => {},
             else => unreachable,
         },
+        .lea => switch (self.mir_instructions.items(.ops)[reloc]) {
+            .rm => {
+                const rx = self.mir_instructions.items(.data)[reloc].rx;
+                assert(rx.fixes == ._);
+                const mem_info: Mir.Memory.Info = @bitCast(
+                    self.mir_extra.items[rx.payload + std.meta.fieldIndex(Mir.Memory, "info").?],
+                );
+                assert(mem_info.base == .rip_inst);
+                self.mir_extra.items[rx.payload + std.meta.fieldIndex(Mir.Memory, "base").?] = next_inst;
+                return;
+            },
+            else => unreachable,
+        },
         else => unreachable,
     }
     self.mir_instructions.items(.data)[reloc].inst.inst = next_inst;
@@ -103461,7 +103474,10 @@ fn airAsm(self: *CodeGen, inst: Air.Inst.Index) !void {
             // for the string, we still use the next u32 for the null terminator.
             extra_i += clobber.len / 4 + 1;
 
-            if (std.mem.eql(u8, clobber, "") or std.mem.eql(u8, clobber, "memory")) {
+            if (std.mem.eql(u8, clobber, "") or std.mem.eql(u8, clobber, "memory") or
+                std.mem.eql(u8, clobber, "fpsr") or std.mem.eql(u8, clobber, "fpcr") or
+                std.mem.eql(u8, clobber, "mxcsr") or std.mem.eql(u8, clobber, "dirflag"))
+            {
                 // ok, sure
             } else if (std.mem.eql(u8, clobber, "cc") or
                 std.mem.eql(u8, clobber, "flags") or
@@ -103726,7 +103742,24 @@ fn airAsm(self: *CodeGen, inst: Air.Inst.Index) !void {
                 else
                     .@"1";
                 if (sib_it.next()) |_| return self.fail("invalid memory operand: '{s}'", .{op_str});
-                op.* = if (std.mem.eql(u8, base_str, "%%dx") and index_str.len == 0) .{ .reg = .dx } else .{ .mem = .{
+                op.* = if (std.mem.eql(u8, base_str, "%%dx") and index_str.len == 0)
+                    .{ .reg = .dx }
+                else if (std.mem.eql(u8, base_str, "%%rip") and index_str.len == 0 and
+                    Label.isValid(.reference, op_str[0..open]))
+                op: {
+                    const anon = std.ascii.isDigit(op_str[0]);
+                    const label_gop = try labels.getOrPut(self.gpa, op_str[0..if (anon) 1 else open]);
+                    if (anon and (op_str[1] == 'b' or op_str[1] == 'B') and !label_gop.found_existing)
+                        return self.fail("undefined label: '{s}'", .{op_str});
+                    if (!label_gop.found_existing) label_gop.value_ptr.* = .{};
+                    const pending_relocs = &label_gop.value_ptr.pending_relocs;
+                    if (if (anon)
+                        op_str[1] == 'f' or op_str[1] == 'F'
+                    else
+                        !label_gop.found_existing or pending_relocs.items.len > 0)
+                        try pending_relocs.append(self.gpa, @intCast(self.mir_instructions.len));
+                    break :op .{ .mem = .{ .base = .{ .rip_inst = label_gop.value_ptr.target } } };
+                } else .{ .mem = .{
                     .base = if (base_str.len > 0)
                         .{ .reg = parseRegName(base_str["%%".len..]) orelse
                             return self.fail("invalid base register: '{s}'", .{base_str}) }
@@ -103770,9 +103803,9 @@ fn airAsm(self: *CodeGen, inst: Air.Inst.Index) !void {
             } else if (Label.isValid(.reference, op_str)) {
                 const anon = std.ascii.isDigit(op_str[0]);
                 const label_gop = try labels.getOrPut(self.gpa, op_str[0..if (anon) 1 else op_str.len]);
-                if (!label_gop.found_existing) label_gop.value_ptr.* = .{};
                 if (anon and (op_str[1] == 'b' or op_str[1] == 'B') and !label_gop.found_existing)
                     return self.fail("undefined label: '{s}'", .{op_str});
+                if (!label_gop.found_existing) label_gop.value_ptr.* = .{};
                 const pending_relocs = &label_gop.value_ptr.pending_relocs;
                 if (if (anon)
                     op_str[1] == 'f' or op_str[1] == 'F'
@@ -105008,7 +105041,7 @@ fn genSetMem(
         .none => .{ .immediate = @bitCast(@as(i64, disp)) },
         .reg => |base_reg| .{ .register_offset = .{ .reg = base_reg, .off = disp } },
         .frame => |base_frame_index| .{ .lea_frame = .{ .index = base_frame_index, .off = disp } },
-        .table => unreachable,
+        .table, .rip_inst => unreachable,
         .reloc => |sym_index| .{ .lea_symbol = .{ .sym_index = sym_index, .off = disp } },
     };
     switch (src_mcv) {
@@ -105118,7 +105151,7 @@ fn genSetMem(
                     .index = frame_index,
                     .off = disp,
                 }).compare(.gte, src_align),
-                .table => unreachable,
+                .table, .rip_inst => unreachable,
                 .reloc => false,
             })).write(
                 self,
@@ -105767,7 +105800,7 @@ fn airCmpxchg(self: *CodeGen, inst: Air.Inst.Index) !void {
     const ptr_lock = switch (ptr_mem.base) {
         .none, .frame, .reloc => null,
         .reg => |reg| self.register_manager.lockReg(reg),
-        .table => unreachable,
+        .table, .rip_inst => unreachable,
     };
     defer if (ptr_lock) |lock| self.register_manager.unlockReg(lock);
 
@@ -105850,7 +105883,7 @@ fn atomicOp(
     const mem_lock = switch (ptr_mem.base) {
         .none, .frame, .reloc => null,
         .reg => |reg| self.register_manager.lockReg(reg),
-        .table => unreachable,
+        .table, .rip_inst => unreachable,
     };
     defer if (mem_lock) |lock| self.register_manager.unlockReg(lock);
 

src/arch/x86_64/Emit.zig

@@ -97,12 +97,14 @@ pub fn emitMir(emit: *Emit) Error!void {
                             op_index -= 1;
                             const op = lowered_inst.encoding.data.ops[op_index];
                             if (op == .none) continue;
-                            const enc_length: u4 = @intCast(
-                                std.math.divCeil(u7, @intCast(op.immBitSize()), 8) catch unreachable,
-                            );
+                            const is_mem = op.isMemory();
+                            const enc_length: u4 = if (is_mem) switch (lowered_inst.ops[op_index].mem.sib.base) {
+                                .rip_inst => 4,
+                                else => unreachable,
+                            } else @intCast(std.math.divCeil(u7, @intCast(op.immBitSize()), 8) catch unreachable);
                             reloc_offset -= enc_length;
-                            if (op_index == lowered_relocs[0].op_index)
-                                break :reloc_offset_length .{ reloc_offset, enc_length };
+                            if (op_index == lowered_relocs[0].op_index) break :reloc_offset_length .{ reloc_offset, enc_length };
+                            std.debug.assert(!is_mem);
                         }
                     };
                     try relocs.append(emit.lower.allocator, .{
@@ -434,7 +436,7 @@ pub fn emitMir(emit: *Emit) Error!void {
                                             loc_buf[0] = switch (mem.base()) {
                                                 .none => .{ .constu = 0 },
                                                 .reg => |reg| .{ .breg = reg.dwarfNum() },
-                                                .frame, .table => unreachable,
+                                                .frame, .table, .rip_inst => unreachable,
                                                 .reloc => |sym_index| .{ .addr = .{ .sym = sym_index } },
                                             };
                                             break :base &loc_buf[0];

src/arch/x86_64/Lower.zig

@@ -395,6 +395,7 @@ pub fn mem(lower: *Lower, op_index: InstOpIndex, payload: u32) Memory {
         .sib => |*sib| switch (sib.base) {
             else => {},
             .table => sib.disp = lower.reloc(op_index, .table, sib.disp).signed,
+            .rip_inst => |inst_index| sib.disp = lower.reloc(op_index, .{ .inst = inst_index }, sib.disp).signed,
         },
         else => {},
     }

src/arch/x86_64/Mir.zig

@@ -1742,6 +1742,7 @@ pub const Memory = struct {
                 .reg => |reg| @intFromEnum(reg),
                 .frame => |frame_index| @intFromEnum(frame_index),
                 .reloc => |sym_index| sym_index,
+                .rip_inst => |inst_index| inst_index,
             },
             .off = switch (mem.mod) {
                 .rm => |rm| @bitCast(rm.disp),
@@ -1769,6 +1770,7 @@ pub const Memory = struct {
                         .frame => .{ .frame = @enumFromInt(mem.base) },
                         .table => .table,
                         .reloc => .{ .reloc = mem.base },
+                        .rip_inst => .{ .rip_inst = mem.base },
                     },
                     .scale_index = switch (mem.info.index) {
                         .none => null,
@@ -1832,7 +1834,7 @@ pub fn resolveFrameAddr(mir: Mir, frame_addr: bits.FrameAddr) bits.RegisterOffse
 
 pub fn resolveFrameLoc(mir: Mir, mem: Memory) Memory {
     return switch (mem.info.base) {
-        .none, .reg, .table, .reloc => mem,
+        .none, .reg, .table, .reloc, .rip_inst => mem,
         .frame => if (mir.frame_locs.len > 0) .{
             .info = .{
                 .base = .reg,

src/arch/x86_64/bits.zig

@@ -4,6 +4,7 @@ const expect = std.testing.expect;
 
 const Allocator = std.mem.Allocator;
 const ArrayList = std.ArrayList;
+const Mir = @import("Mir.zig");
 
 /// EFLAGS condition codes
 pub const Condition = enum(u5) {
@@ -678,12 +679,13 @@ pub const Memory = struct {
         frame: FrameIndex,
         table,
         reloc: u32,
+        rip_inst: Mir.Inst.Index,
 
         pub const Tag = @typeInfo(Base).@"union".tag_type.?;
 
         pub fn isExtended(self: Base) bool {
             return switch (self) {
-                .none, .frame, .table, .reloc => false, // rsp, rbp, and rip are not extended
+                .none, .frame, .table, .reloc, .rip_inst => false, // rsp, rbp, and rip are not extended
                 .reg => |reg| reg.isExtended(),
             };
         }

src/arch/x86_64/encoder.zig

@@ -138,7 +138,7 @@ pub const Instruction = struct {
                 .moffs => true,
                 .rip => false,
                 .sib => |s| switch (s.base) {
-                    .none, .frame, .table, .reloc => false,
+                    .none, .frame, .table, .reloc, .rip_inst => false,
                     .reg => |reg| reg.class() == .segment,
                 },
             };
@@ -279,6 +279,7 @@ pub const Instruction = struct {
                             .frame => |frame_index| try writer.print("{}", .{frame_index}),
                             .table => try writer.print("Table", .{}),
                             .reloc => |sym_index| try writer.print("Symbol({d})", .{sym_index}),
+                            .rip_inst => |inst_index| try writer.print("RipInst({d})", .{inst_index}),
                         }
                         if (mem.scaleIndex()) |si| {
                             if (any) try writer.writeAll(" + ");
@@ -705,6 +706,10 @@ pub const Instruction = struct {
                     try encoder.modRm_indirectDisp32(operand_enc, 0);
                     try encoder.disp32(undefined);
                 } else return error.CannotEncode,
+                .rip_inst => {
+                    try encoder.modRm_RIPDisp32(operand_enc);
+                    try encoder.disp32(sib.disp);
+                },
             },
             .rip => |rip| {
                 try encoder.modRm_RIPDisp32(operand_enc);