Merge pull request #23355 from jacobly0/x86_64-rewrite

x86_64: start rewriting overflow operations

Author: jacobly0

lib/std/zig/AstGen.zig

@@ -8854,10 +8854,10 @@ fn asmExpr(
             return astgen.failNode(node, "assembly expression with no output must be marked volatile", .{});
         }
     }
-    if (full.outputs.len > 32) {
-        return astgen.failNode(full.outputs[32], "too many asm outputs", .{});
+    if (full.outputs.len >= 16) {
+        return astgen.failNode(full.outputs[16], "too many asm outputs", .{});
     }
-    var outputs_buffer: [32]Zir.Inst.Asm.Output = undefined;
+    var outputs_buffer: [15]Zir.Inst.Asm.Output = undefined;
     const outputs = outputs_buffer[0..full.outputs.len];
 
     var output_type_bits: u32 = 0;
@@ -8893,10 +8893,10 @@ fn asmExpr(
         }
     }
 
-    if (full.inputs.len > 32) {
+    if (full.inputs.len >= 32) {
         return astgen.failNode(full.inputs[32], "too many asm inputs", .{});
     }
-    var inputs_buffer: [32]Zir.Inst.Asm.Input = undefined;
+    var inputs_buffer: [31]Zir.Inst.Asm.Input = undefined;
     const inputs = inputs_buffer[0..full.inputs.len];
 
     for (full.inputs, 0..) |input_node, i| {
@@ -8912,7 +8912,7 @@ fn asmExpr(
         };
     }
 
-    var clobbers_buffer: [32]u32 = undefined;
+    var clobbers_buffer: [63]u32 = undefined;
     var clobber_i: usize = 0;
     if (full.first_clobber) |first_clobber| clobbers: {
         // asm ("foo" ::: "a", "b")
@@ -12932,14 +12932,14 @@ const GenZir = struct {
         }
         gz.astgen.extra.appendSliceAssumeCapacity(args.clobbers);
 
-        //  * 0b00000000_000XXXXX - `outputs_len`.
-        //  * 0b000000XX_XXX00000 - `inputs_len`.
-        //  * 0b0XXXXX00_00000000 - `clobbers_len`.
+        //  * 0b00000000_0000XXXX - `outputs_len`.
+        //  * 0b0000000X_XXXX0000 - `inputs_len`.
+        //  * 0b0XXXXXX0_00000000 - `clobbers_len`.
         //  * 0bX0000000_00000000 - is volatile
-        const small: u16 = @as(u16, @intCast(args.outputs.len)) |
-            @as(u16, @intCast(args.inputs.len << 5)) |
-            @as(u16, @intCast(args.clobbers.len << 10)) |
-            (@as(u16, @intFromBool(args.is_volatile)) << 15);
+        const small: u16 = @as(u16, @as(u4, @intCast(args.outputs.len))) << 0 |
+            @as(u16, @as(u5, @intCast(args.inputs.len))) << 4 |
+            @as(u16, @as(u6, @intCast(args.clobbers.len))) << 9 |
+            @as(u16, @intFromBool(args.is_volatile)) << 15;
 
         const new_index: Zir.Inst.Index = @enumFromInt(astgen.instructions.len);
         astgen.instructions.appendAssumeCapacity(.{

src/Sema.zig

@@ -16661,9 +16661,9 @@ fn zirAsm(
     const extra = sema.code.extraData(Zir.Inst.Asm, extended.operand);
     const src = block.nodeOffset(extra.data.src_node);
     const ret_ty_src = block.src(.{ .node_offset_asm_ret_ty = extra.data.src_node });
-    const outputs_len: u5 = @truncate(extended.small);
-    const inputs_len: u5 = @truncate(extended.small >> 5);
-    const clobbers_len: u5 = @truncate(extended.small >> 10);
+    const outputs_len: u4 = @truncate(extended.small);
+    const inputs_len: u5 = @truncate(extended.small >> 4);
+    const clobbers_len: u6 = @truncate(extended.small >> 9);
     const is_volatile = @as(u1, @truncate(extended.small >> 15)) != 0;
     const is_global_assembly = sema.func_index == .none;
     const zir_tags = sema.code.instructions.items(.tag);

src/arch/x86_64/CodeGen.zig

@@ -97,12 +97,14 @@ pub fn emitMir(emit: *Emit) Error!void {
                             op_index -= 1;
                             const op = lowered_inst.encoding.data.ops[op_index];
                             if (op == .none) continue;
-                            const enc_length: u4 = @intCast(
-                                std.math.divCeil(u7, @intCast(op.immBitSize()), 8) catch unreachable,
-                            );
+                            const is_mem = op.isMemory();
+                            const enc_length: u4 = if (is_mem) switch (lowered_inst.ops[op_index].mem.sib.base) {
+                                .rip_inst => 4,
+                                else => unreachable,
+                            } else @intCast(std.math.divCeil(u7, @intCast(op.immBitSize()), 8) catch unreachable);
                             reloc_offset -= enc_length;
-                            if (op_index == lowered_relocs[0].op_index)
-                                break :reloc_offset_length .{ reloc_offset, enc_length };
+                            if (op_index == lowered_relocs[0].op_index) break :reloc_offset_length .{ reloc_offset, enc_length };
+                            std.debug.assert(!is_mem);
                         }
                     };
                     try relocs.append(emit.lower.allocator, .{
@@ -434,7 +436,7 @@ pub fn emitMir(emit: *Emit) Error!void {
                                             loc_buf[0] = switch (mem.base()) {
                                                 .none => .{ .constu = 0 },
                                                 .reg => |reg| .{ .breg = reg.dwarfNum() },
-                                                .frame, .table => unreachable,
+                                                .frame, .table, .rip_inst => unreachable,
                                                 .reloc => |sym_index| .{ .addr = .{ .sym = sym_index } },
                                             };
                                             break :base &loc_buf[0];

src/arch/x86_64/Emit.zig

@@ -50,7 +50,7 @@ pub fn findByMnemonic(
         else => {},
     } else false;
     const rex_extended = for (ops) |op| {
-        if (op.isBaseExtended() or op.isIndexExtended()) break true;
+        if (op.baseExtEnc() != 0b00 or op.indexExtEnc() != 0b00) break true;
     } else false;
 
     if ((rex_required or rex_extended) and rex_invalid) return error.CannotEncode;

src/arch/x86_64/Encoding.zig

@@ -395,6 +395,7 @@ pub fn mem(lower: *Lower, op_index: InstOpIndex, payload: u32) Memory {
         .sib => |*sib| switch (sib.base) {
             else => {},
             .table => sib.disp = lower.reloc(op_index, .table, sib.disp).signed,
+            .rip_inst => |inst_index| sib.disp = lower.reloc(op_index, .{ .inst = inst_index }, sib.disp).signed,
         },
         else => {},
     }

src/arch/x86_64/Lower.zig

@@ -1742,6 +1742,7 @@ pub const Memory = struct {
                 .reg => |reg| @intFromEnum(reg),
                 .frame => |frame_index| @intFromEnum(frame_index),
                 .reloc => |sym_index| sym_index,
+                .rip_inst => |inst_index| inst_index,
             },
             .off = switch (mem.mod) {
                 .rm => |rm| @bitCast(rm.disp),
@@ -1769,6 +1770,7 @@ pub const Memory = struct {
                         .frame => .{ .frame = @enumFromInt(mem.base) },
                         .table => .table,
                         .reloc => .{ .reloc = mem.base },
+                        .rip_inst => .{ .rip_inst = mem.base },
                     },
                     .scale_index = switch (mem.info.index) {
                         .none => null,
@@ -1832,7 +1834,7 @@ pub fn resolveFrameAddr(mir: Mir, frame_addr: bits.FrameAddr) bits.RegisterOffse
 
 pub fn resolveFrameLoc(mir: Mir, mem: Memory) Memory {
     return switch (mem.info.base) {
-        .none, .reg, .table, .reloc => mem,
+        .none, .reg, .table, .reloc, .rip_inst => mem,
         .frame => if (mir.frame_locs.len > 0) .{
             .info = .{
                 .base = .reg,

src/arch/x86_64/Mir.zig

@@ -266,7 +266,8 @@ pub fn classifySystemV(ty: Type, zcu: *Zcu, target: *const std.Target, ctx: Cont
             // separately.".
             const ty_size = ty.abiSize(zcu);
             switch (ty.containerLayout(zcu)) {
-                .auto, .@"extern" => {},
+                .auto => unreachable,
+                .@"extern" => {},
                 .@"packed" => {
                     assert(ty_size <= 16);
                     result[0] = .integer;
@@ -345,15 +346,17 @@ fn classifySystemVStruct(
         );
         if (zcu.typeToStruct(field_ty)) |field_loaded_struct| {
             switch (field_loaded_struct.layout) {
-                .auto, .@"extern" => {
+                .auto => unreachable,
+                .@"extern" => {
                     byte_offset = classifySystemVStruct(result, byte_offset, field_loaded_struct, zcu, target);
                     continue;
                 },
                 .@"packed" => {},
             }
         } else if (zcu.typeToUnion(field_ty)) |field_loaded_union| {
             switch (field_loaded_union.flagsUnordered(ip).layout) {
-                .auto, .@"extern" => {
+                .auto => unreachable,
+                .@"extern" => {
                     byte_offset = classifySystemVUnion(result, byte_offset, field_loaded_union, zcu, target);
                     continue;
                 },
@@ -386,15 +389,17 @@ fn classifySystemVUnion(
         const field_ty = Type.fromInterned(loaded_union.field_types.get(ip)[field_index]);
         if (zcu.typeToStruct(field_ty)) |field_loaded_struct| {
             switch (field_loaded_struct.layout) {
-                .auto, .@"extern" => {
+                .auto => unreachable,
+                .@"extern" => {
                     _ = classifySystemVStruct(result, starting_byte_offset, field_loaded_struct, zcu, target);
                     continue;
                 },
                 .@"packed" => {},
             }
         } else if (zcu.typeToUnion(field_ty)) |field_loaded_union| {
             switch (field_loaded_union.flagsUnordered(ip).layout) {
-                .auto, .@"extern" => {
+                .auto => unreachable,
+                .@"extern" => {
                     _ = classifySystemVUnion(result, starting_byte_offset, field_loaded_union, zcu, target);
                     continue;
                 },

src/arch/x86_64/abi.zig

@@ -4,6 +4,7 @@ const expect = std.testing.expect;
 
 const Allocator = std.mem.Allocator;
 const ArrayList = std.ArrayList;
+const Mir = @import("Mir.zig");
 
 /// EFLAGS condition codes
 pub const Condition = enum(u5) {
@@ -359,11 +360,20 @@ pub const Register = enum(u8) {
 
     ah, ch, dh, bh,
 
-    ymm0, ymm1, ymm2,  ymm3,  ymm4,  ymm5,  ymm6,  ymm7,
-    ymm8, ymm9, ymm10, ymm11, ymm12, ymm13, ymm14, ymm15,
+    zmm0,  zmm1, zmm2,  zmm3,  zmm4,  zmm5,  zmm6,  zmm7,
+    zmm8,  zmm9, zmm10, zmm11, zmm12, zmm13, zmm14, zmm15,
+    zmm16, zmm17,zmm18, zmm19, zmm20, zmm21, zmm22, zmm23,
+    zmm24, zmm25,zmm26, zmm27, zmm28, zmm29, zmm30, zmm31,
 
-    xmm0, xmm1, xmm2,  xmm3,  xmm4,  xmm5,  xmm6,  xmm7,
-    xmm8, xmm9, xmm10, xmm11, xmm12, xmm13, xmm14, xmm15,
+    ymm0,  ymm1, ymm2,  ymm3,  ymm4,  ymm5,  ymm6,  ymm7,
+    ymm8,  ymm9, ymm10, ymm11, ymm12, ymm13, ymm14, ymm15,
+    ymm16, ymm17,ymm18, ymm19, ymm20, ymm21, ymm22, ymm23,
+    ymm24, ymm25,ymm26, ymm27, ymm28, ymm29, ymm30, ymm31,
+
+    xmm0,  xmm1, xmm2,  xmm3,  xmm4,  xmm5,  xmm6,  xmm7,
+    xmm8,  xmm9, xmm10, xmm11, xmm12, xmm13, xmm14, xmm15,
+    xmm16, xmm17,xmm18, xmm19, xmm20, xmm21, xmm22, xmm23,
+    xmm24, xmm25,xmm26, xmm27, xmm28, xmm29, xmm30, xmm31,
 
     mm0, mm1, mm2, mm3, mm4, mm5, mm6, mm7,
 
@@ -403,8 +413,9 @@ pub const Register = enum(u8) {
             @intFromEnum(Register.al)   ... @intFromEnum(Register.r15b)  => .general_purpose,
             @intFromEnum(Register.ah)   ... @intFromEnum(Register.bh)    => .gphi,
 
-            @intFromEnum(Register.ymm0) ... @intFromEnum(Register.ymm15) => .sse,
-            @intFromEnum(Register.xmm0) ... @intFromEnum(Register.xmm15) => .sse,
+            @intFromEnum(Register.zmm0) ... @intFromEnum(Register.zmm31) => .sse,
+            @intFromEnum(Register.ymm0) ... @intFromEnum(Register.ymm31) => .sse,
+            @intFromEnum(Register.xmm0) ... @intFromEnum(Register.xmm31) => .sse,
             @intFromEnum(Register.mm0)  ... @intFromEnum(Register.mm7)   => .mmx,
             @intFromEnum(Register.st0)  ... @intFromEnum(Register.st7)   => .x87,
 
@@ -427,13 +438,14 @@ pub const Register = enum(u8) {
             @intFromEnum(Register.al)   ... @intFromEnum(Register.r15b)  => @intFromEnum(Register.al),
             @intFromEnum(Register.ah)   ... @intFromEnum(Register.bh)    => @intFromEnum(Register.ah),
 
-            @intFromEnum(Register.ymm0) ... @intFromEnum(Register.ymm15) => @intFromEnum(Register.ymm0) - 16,
-            @intFromEnum(Register.xmm0) ... @intFromEnum(Register.xmm15) => @intFromEnum(Register.xmm0) - 16,
-            @intFromEnum(Register.mm0)  ... @intFromEnum(Register.mm7)   => @intFromEnum(Register.mm0) - 32,
-            @intFromEnum(Register.st0)  ... @intFromEnum(Register.st7)   => @intFromEnum(Register.st0) - 40,
-            @intFromEnum(Register.es)   ... @intFromEnum(Register.gs)    => @intFromEnum(Register.es) - 48,
-            @intFromEnum(Register.cr0)  ... @intFromEnum(Register.cr15)  => @intFromEnum(Register.cr0) - 54,
-            @intFromEnum(Register.dr0)  ... @intFromEnum(Register.dr15)  => @intFromEnum(Register.dr0) - 70,
+            @intFromEnum(Register.zmm0) ... @intFromEnum(Register.zmm31) => @intFromEnum(Register.zmm0) - 16,
+            @intFromEnum(Register.ymm0) ... @intFromEnum(Register.ymm31) => @intFromEnum(Register.ymm0) - 16,
+            @intFromEnum(Register.xmm0) ... @intFromEnum(Register.xmm31) => @intFromEnum(Register.xmm0) - 16,
+            @intFromEnum(Register.mm0)  ... @intFromEnum(Register.mm7)   => @intFromEnum(Register.mm0)  - 48,
+            @intFromEnum(Register.st0)  ... @intFromEnum(Register.st7)   => @intFromEnum(Register.st0)  - 56,
+            @intFromEnum(Register.es)   ... @intFromEnum(Register.gs)    => @intFromEnum(Register.es)   - 64,
+            @intFromEnum(Register.cr0)  ... @intFromEnum(Register.cr15)  => @intFromEnum(Register.cr0)  - 70,
+            @intFromEnum(Register.dr0)  ... @intFromEnum(Register.dr15)  => @intFromEnum(Register.dr0)  - 86,
 
             else => unreachable,
             // zig fmt: on
@@ -450,6 +462,7 @@ pub const Register = enum(u8) {
             @intFromEnum(Register.al)   ... @intFromEnum(Register.r15b)  => 8,
             @intFromEnum(Register.ah)   ... @intFromEnum(Register.bh)    => 8,
 
+            @intFromEnum(Register.zmm0) ... @intFromEnum(Register.zmm15) => 512,
             @intFromEnum(Register.ymm0) ... @intFromEnum(Register.ymm15) => 256,
             @intFromEnum(Register.xmm0) ... @intFromEnum(Register.xmm15) => 128,
             @intFromEnum(Register.mm0)  ... @intFromEnum(Register.mm7)   => 64,
@@ -473,8 +486,9 @@ pub const Register = enum(u8) {
             @intFromEnum(Register.r8w) ... @intFromEnum(Register.r15w)   => true,
             @intFromEnum(Register.r8b) ... @intFromEnum(Register.r15b)   => true,
 
-            @intFromEnum(Register.ymm8) ... @intFromEnum(Register.ymm15) => true,
-            @intFromEnum(Register.xmm8) ... @intFromEnum(Register.xmm15) => true,
+            @intFromEnum(Register.zmm8) ... @intFromEnum(Register.zmm31) => true,
+            @intFromEnum(Register.ymm8) ... @intFromEnum(Register.ymm31) => true,
+            @intFromEnum(Register.xmm8) ... @intFromEnum(Register.xmm31) => true,
 
             @intFromEnum(Register.cr8)  ... @intFromEnum(Register.cr15)  => true,
             @intFromEnum(Register.dr8)  ... @intFromEnum(Register.dr15)  => true,
@@ -484,7 +498,7 @@ pub const Register = enum(u8) {
         };
     }
 
-    pub fn enc(reg: Register) u4 {
+    pub fn enc(reg: Register) u5 {
         const base = switch (@intFromEnum(reg)) {
             // zig fmt: off
             @intFromEnum(Register.rax)  ... @intFromEnum(Register.r15)   => @intFromEnum(Register.rax),
@@ -509,10 +523,6 @@ pub const Register = enum(u8) {
         return @truncate(@intFromEnum(reg) - base);
     }
 
-    pub fn lowEnc(reg: Register) u3 {
-        return @truncate(reg.enc());
-    }
-
     pub fn toBitSize(reg: Register, bit_size: u64) Register {
         return switch (bit_size) {
             8 => reg.to8(),
@@ -557,11 +567,12 @@ pub const Register = enum(u8) {
         };
     }
 
-    fn sseBase(reg: Register) u7 {
+    fn sseBase(reg: Register) u8 {
         assert(reg.class() == .sse);
         return switch (@intFromEnum(reg)) {
-            @intFromEnum(Register.ymm0)...@intFromEnum(Register.ymm15) => @intFromEnum(Register.ymm0),
-            @intFromEnum(Register.xmm0)...@intFromEnum(Register.xmm15) => @intFromEnum(Register.xmm0),
+            @intFromEnum(Register.zmm0)...@intFromEnum(Register.zmm31) => @intFromEnum(Register.zmm0),
+            @intFromEnum(Register.ymm0)...@intFromEnum(Register.ymm31) => @intFromEnum(Register.ymm0),
+            @intFromEnum(Register.xmm0)...@intFromEnum(Register.xmm31) => @intFromEnum(Register.xmm0),
             else => unreachable,
         };
     }
@@ -678,15 +689,9 @@ pub const Memory = struct {
         frame: FrameIndex,
         table,
         reloc: u32,
+        rip_inst: Mir.Inst.Index,
 
         pub const Tag = @typeInfo(Base).@"union".tag_type.?;
-
-        pub fn isExtended(self: Base) bool {
-            return switch (self) {
-                .none, .frame, .table, .reloc => false, // rsp, rbp, and rip are not extended
-                .reg => |reg| reg.isExtended(),
-            };
-        }
     };
 
     pub const Mod = union(enum(u1)) {