[hardware] 🐛 Fix EMUL handling when reshffling more regs

Author: mp-17

hardware/src/ara_dispatcher.sv

@@ -165,12 +165,14 @@ module ara_dispatcher import ara_pkg::*; import rvv_pkg::*; #(
   // If the reg was not written, the content is unknown. No need to reshuffle
   // when writing with != EEW
   logic [31:0] eew_valid_d, eew_valid_q;
+  // Help for reshuffling
+  rvv_pkg::vlmul_e emul_vs1_d, emul_vs1_q, emul_vs2_d, emul_vs2_q;
   // Save eew information before reshuffling
   rvv_pkg::vew_e eew_old_buffer_d, eew_old_buffer_q, eew_new_buffer_d, eew_new_buffer_q;
   // Helpers to handle reshuffling with LMUL > 1
-  logic [2:0] rs_lmul_cnt_d, rs_lmul_cnt_q;
-  logic [2:0] rs_lmul_cnt_limit_d, rs_lmul_cnt_limit_q;
-  logic rs_mask_request_d, rs_mask_request_q;
+  logic [2:0] reg_lmul_cnt_d, reg_lmul_cnt_q;
+  logic [2:0] reg_lmul_cnt_limit_d, reg_lmul_cnt_limit_q;
+  logic reg_mask_request_d, reg_mask_request_q;
   // Save vreg to be reshuffled before reshuffling
   logic [4:0] vs_buffer_d, vs_buffer_q;
   // Keep track of the registers to be reshuffled |vs1|vs2|vd|
@@ -191,13 +193,15 @@ module ara_dispatcher import ara_pkg::*; import rvv_pkg::*; #(
       eew_new_buffer_q     <= rvv_pkg::EW8;
       vs_buffer_q          <= '0;
       reshuffle_req_q      <= '0;
-      rs_lmul_cnt_q        <= '0;
-      rs_lmul_cnt_limit_q  <= '0;
-      rs_mask_request_q    <= 1'b0;
+      reg_lmul_cnt_q       <= '0;
+      reg_lmul_cnt_limit_q <= '0;
+      reg_mask_request_q   <= 1'b0;
       reshuffle_eew_vs1_q  <= rvv_pkg::EW8;
       reshuffle_eew_vs2_q  <= rvv_pkg::EW8;
       reshuffle_eew_vd_q   <= rvv_pkg::EW8;
       pending_seg_mem_op_q <= 1'b0;
+      emul_vs1_q           <= LMUL_1;
+      emul_vs2_q           <= LMUL_1;
     end else begin
       state_q              <= state_d;
       state_qq             <= state_q;
@@ -207,13 +211,15 @@ module ara_dispatcher import ara_pkg::*; import rvv_pkg::*; #(
       eew_new_buffer_q     <= eew_new_buffer_d;
       vs_buffer_q          <= vs_buffer_d;
       reshuffle_req_q      <= reshuffle_req_d;
-      rs_lmul_cnt_q        <= rs_lmul_cnt_d;
-      rs_lmul_cnt_limit_q  <= rs_lmul_cnt_limit_d;
-      rs_mask_request_q    <= rs_mask_request_d;
+      reg_lmul_cnt_q       <= reg_lmul_cnt_d;
+      reg_lmul_cnt_limit_q <= reg_lmul_cnt_limit_d;
+      reg_mask_request_q   <= reg_mask_request_d;
       reshuffle_eew_vs1_q  <= reshuffle_eew_vs1_d;
       reshuffle_eew_vs2_q  <= reshuffle_eew_vs2_d;
       reshuffle_eew_vd_q   <= reshuffle_eew_vd_d;
       pending_seg_mem_op_q <= pending_seg_mem_op_d;
+      emul_vs1_q           <= emul_vs1_d;
+      emul_vs2_q           <= emul_vs2_d;
     end
   end
 
@@ -385,9 +391,9 @@ module ara_dispatcher import ara_pkg::*; import rvv_pkg::*; #(
 
     pending_seg_mem_op_d = pending_seg_mem_op_q;
 
-    rs_lmul_cnt_d       = '0;
-    rs_lmul_cnt_limit_d = '0;
-    rs_mask_request_d   = 1'b0;
+    reg_lmul_cnt_d       = '0;
+    reg_lmul_cnt_limit_d = '0;
+    reg_mask_request_d   = 1'b0;
 
     illegal_insn = 1'b0;
     illegal_insn_load  = 1'b0;
@@ -452,6 +458,8 @@ module ara_dispatcher import ara_pkg::*; import rvv_pkg::*; #(
 
     emul_vs1 = ara_req.emul;
     emul_vs2 = ara_req.emul;
+    emul_vs1_d = emul_vs1_q;
+    emul_vs2_d = emul_vs2_q;
 
     // Saturation in any lane will raise vxsat flag
     csr_vxsat_d |= |vxsat_flag_i;
@@ -489,9 +497,9 @@ module ara_dispatcher import ara_pkg::*; import rvv_pkg::*; #(
         acc_resp_o.resp_valid = 1'b0;
 
         // Handle LMUL > 1
-        rs_lmul_cnt_d       = rs_lmul_cnt_q;
-        rs_lmul_cnt_limit_d = rs_lmul_cnt_limit_q;
-        rs_mask_request_d   = 1'b0;
+        reg_lmul_cnt_d       = reg_lmul_cnt_q;
+        reg_lmul_cnt_limit_d = reg_lmul_cnt_limit_q;
+        reg_mask_request_d   = 1'b0;
 
         // Every single reshuffle request refers to LMUL == 1
         ara_req.emul = LMUL_1;
@@ -502,7 +510,7 @@ module ara_dispatcher import ara_pkg::*; import rvv_pkg::*; #(
         // These generate a reshuffle request to Ara's backend
         // When LMUL > 1, not all the regs that compose a large
         // register should always be reshuffled
-        ara_req_valid         = ~rs_mask_request_q;
+        ara_req_valid         = ~reg_mask_request_q;
         ara_req.use_scalar_op = 1'b1;
         ara_req.vs2           = vs_buffer_q;
         ara_req.eew_vs2       = eew_old_buffer_q;
@@ -525,8 +533,8 @@ module ara_dispatcher import ara_pkg::*; import rvv_pkg::*; #(
         // Backend ready - Decide what to do next
         if (ara_req_ready_i) begin
           // Register completely reshuffled
-          if (rs_lmul_cnt_q == rs_lmul_cnt_limit_q) begin
-            rs_lmul_cnt_d = 0;
+          if (reg_lmul_cnt_q == reg_lmul_cnt_limit_q) begin
+            reg_lmul_cnt_d = 0;
 
             // Delete the already processed vector register from the notebook -> |vs1|vs2|vd|
             unique casez (reshuffle_req_q)
@@ -544,11 +552,13 @@ module ara_dispatcher import ara_pkg::*; import rvv_pkg::*; #(
                 eew_old_buffer_d = eew_q[insn.vmem_type.rs2];
                 eew_new_buffer_d = reshuffle_eew_vs2_q;
                 vs_buffer_d      = insn.varith_type.rs2;
+                reg_lmul_cnt_limit_d = emul_vs2_q;
               end
               3'b100: begin
                 eew_old_buffer_d = eew_q[insn.vmem_type.rs1];
                 eew_new_buffer_d = reshuffle_eew_vs1_q;
                 vs_buffer_d      = insn.varith_type.rs1;
+                reg_lmul_cnt_limit_d = emul_vs1_q;
               end
               default:;
             endcase
@@ -570,7 +580,7 @@ module ara_dispatcher import ara_pkg::*; import rvv_pkg::*; #(
           // The register is not completely reshuffled (LMUL > 1)
           end else begin
             // Count up
-            rs_lmul_cnt_d = rs_lmul_cnt_q + 1;
+            reg_lmul_cnt_d = reg_lmul_cnt_q + 1;
 
             // Prepare the information to reshuffle the vector registers during the next cycles
             // Since LMUL > 1, we should go on and check if the next register needs a reshuffle
@@ -595,7 +605,7 @@ module ara_dispatcher import ara_pkg::*; import rvv_pkg::*; #(
             endcase
 
             // Mask the next request if we don't need to reshuffle the next reg
-            if (eew_new_buffer_d == eew_old_buffer_d) rs_mask_request_d = 1'b1;
+            if (eew_new_buffer_d == eew_old_buffer_d) reg_mask_request_d = 1'b1;
           end
         end
       end
@@ -1534,16 +1544,20 @@ module ara_dispatcher import ara_pkg::*; import rvv_pkg::*; #(
                     case (insn.varith_type.rs1)
                       5'b00000: begin
                         ara_req.op      = ara_pkg::VMVXS;
+                        ara_req.use_vs1 = 1'b0;
                         ara_req.vl      = 1;
                         // Scalar source
                         emul_vs2        = LMUL_1;
+                        ara_req.emul    = LMUL_1;
                       end
                       5'b10000: begin
                         ara_req.op      = ara_pkg::VCPOP;
+                        ara_req.use_vs1 = 1'b0;
                         ara_req.eew_vs2 = eew_q[ara_req.vs2];
                       end
                       5'b10001: begin
                         ara_req.op      = ara_pkg::VFIRST;
+                        ara_req.use_vs1 = 1'b0;
                         ara_req.eew_vs2 = eew_q[ara_req.vs2];
                       end
                       default :;
@@ -2030,6 +2044,7 @@ module ara_dispatcher import ara_pkg::*; import rvv_pkg::*; #(
                     ara_req.vl      = |csr_vl_q ? 1 : '0;
                     // This instruction ignores LMUL checks
                     skip_lmul_checks  = 1'b1;
+                    ara_req.emul      = LMUL_1;
                   end
                   // Divide instructions
                   6'b100000: ara_req.op = ara_pkg::VDIVU;
@@ -2288,6 +2303,8 @@ module ara_dispatcher import ara_pkg::*; import rvv_pkg::*; #(
                       ara_req.vstart     = '0;
                       skip_lmul_checks     = 1'b1;
                       ignore_zero_vl_check = 1'b1;
+                      emul_vs2           = LMUL_1;
+                      ara_req.emul       = LMUL_1;
 
                       // Zero-extend operands
                       unique case (csr_vtype_q.vsew)
@@ -2730,6 +2747,7 @@ module ara_dispatcher import ara_pkg::*; import rvv_pkg::*; #(
                       ara_req.vl      = |csr_vl_q ? 1 : '0;
                       // This instruction ignores LMUL checks
                       skip_lmul_checks  = 1'b1;
+                      ara_req.emul       = LMUL_1;
                     end
                     6'b010111: ara_req.op = ara_pkg::VMERGE;
                     6'b011000: begin
@@ -3753,27 +3771,38 @@ module ara_dispatcher import ara_pkg::*; import rvv_pkg::*; #(
         endcase
 
         // Mask the next request if we don't need to reshuffle the next reg
-        if (eew_new_buffer_d == eew_old_buffer_d) rs_mask_request_d = 1'b1;
+        if (eew_new_buffer_d == eew_old_buffer_d) reg_mask_request_d = 1'b1;
       end
 
       // Reshuffle if at least one of the three registers needs a reshuffle
       if (|reshuffle_req_d) begin
         // Instruction is of one of the RVV types
         automatic rvv_instruction_t insn = rvv_instruction_t'(instr.instr);
+        automatic rvv_pkg::vlmul_e reshuffle_emul;
 
         // Stall the interface, and inject a reshuffling instruction
         acc_resp_o.req_ready  = 1'b0;
         acc_resp_o.resp_valid = 1'b0;
         ara_req_valid  = 1'b0;
 
         // Initialize the reshuffle counter limit to handle LMUL > 1
-        unique case (ara_req.emul)
-          LMUL_2:  rs_lmul_cnt_limit_d = 1;
-          LMUL_4:  rs_lmul_cnt_limit_d = 3;
-          LMUL_8:  rs_lmul_cnt_limit_d = 7;
-          default: rs_lmul_cnt_limit_d = 0;
+        unique casez (reshuffle_req_d)
+          3'b??1: reshuffle_emul = ara_req.emul;
+          3'b?10: reshuffle_emul = emul_vs2;
+          3'b100: reshuffle_emul = emul_vs1;
         endcase
 
+        unique case (reshuffle_emul)
+          LMUL_2:  reg_lmul_cnt_limit_d = 1;
+          LMUL_4:  reg_lmul_cnt_limit_d = 3;
+          LMUL_8:  reg_lmul_cnt_limit_d = 7;
+          default: reg_lmul_cnt_limit_d = 0;
+        endcase
+
+        // Save vs1 and vs2 emul for reshuffling. vd, if needed, has been saved already.
+        emul_vs1_d = emul_vs1;
+        emul_vs2_d = emul_vs2;
+
         // Save info for next reshuffles
         reshuffle_eew_vs1_d = ara_req.eew_vs1;
         reshuffle_eew_vs2_d = ara_req.eew_vs2;