Implement Ziccid correctly and cleanly

The algorithm is as follows:
- The ITLB maintains inclusion of the I$
- The same address cannot be in any hart's ITLB if it is in
  any hart's STLB, and vice versa

The effect is that any word that can be stored-to cannot be in any I$.

The old scheme (periodically flush I$) was hacky and didn't correctly
respect the in-order fetch rule (i.e. fetches are in-order wrt.
each other, and so they see ordered stores in order).

Author: aswaterman

riscv/mmu.cc

@@ -384,6 +384,32 @@ void mmu_t::store_slow_path(reg_t original_addr, reg_t len, const uint8_t* bytes
   }
 }
 
+void mmu_t::flush_icache_vpn(reg_t vpn)
+{
+  for (size_t i = 0; i < ICACHE_ENTRIES; i++)
+    if ((icache[i].tag >> PGSHIFT) == vpn)
+      icache[i].tag = -1;
+}
+
+void mmu_t::flush_stlb_paddr(reg_t paddr)
+{
+  for (size_t i = 0; i < TLB_ENTRIES; i++)
+    if (tlb_store[i].data.target_addr == paddr)
+      tlb_store[i].tag = -1;
+}
+
+void mmu_t::flush_itlb_paddr(reg_t paddr)
+{
+  for (size_t i = 0; i < TLB_ENTRIES; i++) {
+    if (tlb_insn[i].data.target_addr == paddr) {
+      // Maintain inclusion of I$
+      flush_icache_vpn(tlb_insn[i].tag & ~TLB_FLAGS);
+
+      tlb_insn[i].tag = -1;
+    }
+  }
+}
+
 tlb_entry_t mmu_t::refill_tlb(reg_t vaddr, reg_t paddr, char* host_addr, access_type type)
 {
   reg_t idx = (vaddr >> PGSHIFT) % TLB_ENTRIES;
@@ -402,14 +428,31 @@ tlb_entry_t mmu_t::refill_tlb(reg_t vaddr, reg_t paddr, char* host_addr, access_
 
   switch (type) {
     case FETCH:
+      if (proc && proc->extension_enabled(EXT_ZICCID)) {
+        // Maintain inclusion of I$
+        if (tlb_insn[idx].tag != (reg_t)-1)
+          flush_icache_vpn(tlb_insn[idx].tag & ~TLB_FLAGS);
+
+        // Maintain exclusion with all store TLBs
+        for (auto [_, p2] : sim->get_harts())
+          p2->mmu->flush_stlb_paddr(entry.target_addr);
+      }
+
       tlb_insn[idx].data = entry;
       tlb_insn[idx].tag = expected_tag | (check_triggers_fetch ? TLB_CHECK_TRIGGERS : 0) | trace_flag | mmio_flag;
       break;
     case LOAD:
       tlb_load[idx].data = entry;
       tlb_load[idx].tag = expected_tag | (check_triggers_load ? TLB_CHECK_TRIGGERS : 0) | trace_flag | mmio_flag;
       break;
+
     case STORE:
+      if (proc && proc->extension_enabled(EXT_ZICCID)) {
+        // Maintain exclusion with all instruction TLBs
+        for (auto [_, p2] : sim->get_harts())
+          p2->mmu->flush_itlb_paddr(entry.target_addr);
+      }
+
       tlb_store[idx].data = entry;
       tlb_store[idx].tag = expected_tag | (check_triggers_store ? TLB_CHECK_TRIGGERS : 0) | trace_flag | mmio_flag;
       break;

riscv/mmu.h

@@ -372,7 +372,10 @@ class mmu_t
   }
 
   void flush_tlb();
+  void flush_itlb_paddr(reg_t paddr);
+  void flush_stlb_paddr(reg_t paddr);
   void flush_icache();
+  void flush_icache_vpn(reg_t vpn);
 
   void register_memtracer(memtracer_t*);