diff --git a/cpu/ppc/ppcmmu.cpp b/cpu/ppc/ppcmmu.cpp
index 1f3f3540d7..1f16ccdbc8 100644
--- a/cpu/ppc/ppcmmu.cpp
+++ b/cpu/ppc/ppcmmu.cpp
@@ -31,6 +31,7 @@ along with this program. If not, see .
#include
#include
#include
+#include
//#define MMU_PROFILING // uncomment this to enable MMU profiling
//#define TLB_PROFILING // uncomment this to enable SoftTLB profiling
@@ -471,15 +472,14 @@ enum TLBFlags : uint16_t {
TLBE_FROM_PAT = 1 << 4, // TLB entry has been translated with PAT
PAGE_WRITABLE = 1 << 5, // page is writable
PTE_SET_C = 1 << 6, // tells if C bit of the PTE needs to be updated
+ TLBE_CTX_TRACKED = 1 << 7, // entry pointer is in gTrackedIEntries or gTrackedDEntries
};
-// PAT context changes are frequent, so implicitly invalidate PAT-derived
-// entries via a generation counter (instead of walking the TLB array and
-// clearing every time).
-static uint32_t gTLBPatGeneration = 0;
+constexpr uint16_t TLBE_FROM_TRANSLATION =
+ TLBFlags::TLBE_FROM_BAT | TLBFlags::TLBE_FROM_PAT;
typedef struct TLBEntry {
- uint32_t tag; // use is_invalid() or matches_tag() for validity checks
+ uint32_t tag;
uint16_t flags;
uint16_t lru_bits;
union {
@@ -493,22 +493,93 @@ typedef struct TLBEntry {
};
};
uint32_t phys_tag;
- uint32_t pat_generation;
+ uint32_t reserved;
+} TLBEntry;
- bool is_invalid() const
- {
- return tag == TLB_INVALID_TAG ||
- ((flags & TLBFlags::TLBE_FROM_PAT) &&
- pat_generation != gTLBPatGeneration);
+// Track slots populated by either page or block address translation. Context
+// changes can then invalidate only populated slots without adding a generation
+// check to every TLB lookup.
+static std::vector gTrackedIEntries;
+static std::vector gTrackedDEntries;
+
+// TLBE_FROM_* sources awaiting invalidation at the next context sync.
+static uint16_t gPendingIInvalidationSources = 0;
+static uint16_t gPendingDInvalidationSources = 0;
+
+template
+static void track_translated_entry(TLBEntry *tlb_entry)
+{
+ if (!(tlb_entry->flags & TLBE_FROM_TRANSLATION) ||
+ (tlb_entry->flags & TLBFlags::TLBE_CTX_TRACKED))
+ return;
+
+ tlb_entry->flags |= TLBFlags::TLBE_CTX_TRACKED;
+ if (tlb_type == TLBType::ITLB)
+ gTrackedIEntries.push_back(tlb_entry);
+ else
+ gTrackedDEntries.push_back(tlb_entry);
+}
+
+template
+static void tlb_invalidate_tracked_entries()
+{
+ uint16_t *pending_sources;
+ std::vector *tracked_entries;
+
+ if (tlb_type == TLBType::ITLB) {
+ pending_sources = &gPendingIInvalidationSources;
+ tracked_entries = &gTrackedIEntries;
+ } else {
+ pending_sources = &gPendingDInvalidationSources;
+ tracked_entries = &gTrackedDEntries;
}
- bool matches_tag(uint32_t tag_in) const
- {
- return tag == tag_in &&
- (!(flags & TLBFlags::TLBE_FROM_PAT) ||
- pat_generation == gTLBPatGeneration);
+ if (!*pending_sources)
+ return;
+
+ size_t retained_count = 0;
+ for (TLBEntry *tlb_entry : *tracked_entries) {
+ uint16_t source = tlb_entry->flags & TLBE_FROM_TRANSLATION;
+ if (source & *pending_sources) {
+ tlb_entry->tag = TLB_INVALID_TAG;
+ tlb_entry->flags &= ~TLBFlags::TLBE_CTX_TRACKED;
+ } else if (source) {
+ (*tracked_entries)[retained_count++] = tlb_entry;
+ } else {
+ tlb_entry->flags &= ~TLBFlags::TLBE_CTX_TRACKED;
+ }
}
-} TLBEntry;
+ tracked_entries->resize(retained_count);
+ *pending_sources = 0;
+}
+
+static void schedule_tlb_invalidation(TLBType tlb_type, uint16_t sources_to_invalidate)
+{
+ uint16_t *pending_sources;
+
+ if (tlb_type == TLBType::ITLB)
+ pending_sources = &gPendingIInvalidationSources;
+ else
+ pending_sources = &gPendingDInvalidationSources;
+
+ if (!*pending_sources) {
+ if (tlb_type == TLBType::ITLB)
+ add_ctx_sync_action(&tlb_invalidate_tracked_entries);
+ else
+ add_ctx_sync_action(&tlb_invalidate_tracked_entries);
+ }
+ *pending_sources |= sources_to_invalidate;
+}
+
+template
+static void promote_tlb_entry(TLBEntry *tlb1_entry, const TLBEntry *tlb2_entry)
+{
+ uint16_t tracked = tlb1_entry->flags & TLBFlags::TLBE_CTX_TRACKED;
+ *tlb1_entry = *tlb2_entry;
+ tlb1_entry->flags &= ~TLBFlags::TLBE_CTX_TRACKED;
+ tlb1_entry->flags |= tracked;
+ track_translated_entry(tlb1_entry);
+}
// primary ITLB for all MMU modes
static std::array itlb1_mode1;
@@ -628,16 +699,16 @@ static TLBEntry* tlb2_target_entry(uint32_t gp_va)
}
// select the target from invalid blocks first
- if (tlb_entry[0].is_invalid()) {
+ if (tlb_entry[0].tag == TLB_INVALID_TAG) {
tlb2_touch_way<0>(tlb_entry);
return tlb_entry;
- } else if (tlb_entry[1].is_invalid()) {
+ } else if (tlb_entry[1].tag == TLB_INVALID_TAG) {
tlb2_touch_way<1>(tlb_entry);
return &tlb_entry[1];
- } else if (tlb_entry[2].is_invalid()) {
+ } else if (tlb_entry[2].tag == TLB_INVALID_TAG) {
tlb2_touch_way<2>(tlb_entry);
return &tlb_entry[2];
- } else if (tlb_entry[3].is_invalid()) {
+ } else if (tlb_entry[3].tag == TLB_INVALID_TAG) {
tlb2_touch_way<3>(tlb_entry);
return &tlb_entry[3];
} else { // no free entries, replace an existing one according with the hLRU policy
@@ -703,11 +774,12 @@ static TLBEntry* itlb2_refill(uint32_t guest_va)
const uint32_t tag = guest_va & ~0xFFFUL;
tlb_entry = tlb2_target_entry(tag);
tlb_entry->tag = tag;
- tlb_entry->flags = flags | TLBFlags::PAGE_MEM;
+ tlb_entry->flags = flags | TLBFlags::PAGE_MEM |
+ (tlb_entry->flags & TLBFlags::TLBE_CTX_TRACKED);
tlb_entry->host_va_offs_r = (int64_t)rgn_desc->mem_ptr - guest_va +
(phys_addr - rgn_desc->start);
tlb_entry->phys_tag = phys_addr & ~0xFFFUL;
- tlb_entry->pat_generation = gTLBPatGeneration;
+ track_translated_entry(tlb_entry);
} else {
ABORT_F("Instruction fetch from unmapped memory at 0x%08X!\n", phys_addr);
}
@@ -776,11 +848,13 @@ static TLBEntry* dtlb2_refill(uint32_t guest_va, int is_write, bool is_dbg = fal
tlb_entry = tlb2_target_entry(tag);
tlb_entry->tag = tag;
if (rgn_desc->type & RT_MMIO) { // MMIO region
- tlb_entry->flags = flags | TLBFlags::PAGE_IO;
+ tlb_entry->flags = flags | TLBFlags::PAGE_IO |
+ (tlb_entry->flags & TLBFlags::TLBE_CTX_TRACKED);
tlb_entry->rgn_desc = rgn_desc;
tlb_entry->dev_base_va = guest_va - (phys_addr - rgn_desc->start);
} else { // memory region backed by host memory
- tlb_entry->flags = flags | TLBFlags::PAGE_MEM;
+ tlb_entry->flags = flags | TLBFlags::PAGE_MEM |
+ (tlb_entry->flags & TLBFlags::TLBE_CTX_TRACKED);
tlb_entry->host_va_offs_r = (int64_t)rgn_desc->mem_ptr - guest_va +
(phys_addr - rgn_desc->start);
if (rgn_desc->type == RT_ROM) {
@@ -791,7 +865,7 @@ static TLBEntry* dtlb2_refill(uint32_t guest_va, int is_write, bool is_dbg = fal
}
}
tlb_entry->phys_tag = phys_addr & ~0xFFFUL;
- tlb_entry->pat_generation = gTLBPatGeneration;
+ track_translated_entry(tlb_entry);
return tlb_entry;
} else {
if (!is_dbg) {
@@ -820,15 +894,15 @@ static inline TLBEntry* lookup_secondary_tlb(uint32_t guest_va, uint32_t tag) {
tlb_entry = &pCurDTLB2[((guest_va >> PPC_PAGE_SIZE_BITS) & tlb_size_mask) * TLB2_WAYS];
}
- if (tlb_entry[0].matches_tag(tag)) {
+ if (tlb_entry[0].tag == tag) {
tlb2_touch_way<0>(tlb_entry);
- } else if (tlb_entry[1].matches_tag(tag)) {
+ } else if (tlb_entry[1].tag == tag) {
tlb2_touch_way<1>(tlb_entry);
tlb_entry = &tlb_entry[1];
- } else if (tlb_entry[2].matches_tag(tag)) {
+ } else if (tlb_entry[2].tag == tag) {
tlb2_touch_way<2>(tlb_entry);
tlb_entry = &tlb_entry[2];
- } else if (tlb_entry[3].matches_tag(tag)) {
+ } else if (tlb_entry[3].tag == tag) {
tlb2_touch_way<3>(tlb_entry);
tlb_entry = &tlb_entry[3];
} else {
@@ -853,7 +927,7 @@ static inline TLBLookupResult lookup_tlb(uint32_t guest_va, uint32_t tag)
primary_entry = &pCurDTLB1[(guest_va >> PPC_PAGE_SIZE_BITS) & tlb_size_mask];
}
- if (primary_entry->matches_tag(tag)) {
+ if (primary_entry->tag == tag) {
return TLBLookupResult{primary_entry, primary_entry, true};
}
return TLBLookupResult{
@@ -951,11 +1025,7 @@ uint8_t *mmu_translate_imem(uint32_t vaddr, uint32_t *paddr)
}
#endif
// refill the primary ITLB
- tlb1_entry->tag = tag;
- tlb1_entry->flags = tlb2_entry->flags;
- tlb1_entry->host_va_offs_r = tlb2_entry->host_va_offs_r;
- tlb1_entry->phys_tag = tlb2_entry->phys_tag;
- tlb1_entry->pat_generation = tlb2_entry->pat_generation;
+ promote_tlb_entry(tlb1_entry, tlb2_entry);
host_va = (uint8_t *)(tlb1_entry->host_va_offs_r + vaddr);
}
@@ -1002,79 +1072,6 @@ void tlb_flush_entry(uint32_t ea)
tlb_flush_secondary_entry(dtlb2_mode3, tag);
}
-template
-static void tlb_flush_entries(std::array &tlb, TLBFlags type) {
- for (auto &tlb_el : tlb) {
- if (tlb_el.tag != TLB_INVALID_TAG && tlb_el.flags & type) {
- tlb_el.tag = TLB_INVALID_TAG;
- }
- }
-}
-
-template
-void tlb_flush_entries(TLBFlags type)
-{
- // Mode 1 is real addressing and thus can't contain any PAT entries by definition.
- bool flush_mode1 = type != TLBE_FROM_PAT;
- if (tlb_type == TLBType::ITLB) {
- if (flush_mode1) {
- tlb_flush_entries(itlb1_mode1, type);
- }
- tlb_flush_entries(itlb1_mode2, type);
- tlb_flush_entries(itlb1_mode3, type);
- if (flush_mode1) {
- tlb_flush_entries(itlb2_mode1, type);
- }
- tlb_flush_entries(itlb2_mode2, type);
- tlb_flush_entries(itlb2_mode3, type);
- } else {
- if (flush_mode1) {
- tlb_flush_entries(dtlb1_mode1, type);
- }
- tlb_flush_entries(dtlb1_mode2, type);
- tlb_flush_entries(dtlb1_mode3, type);
- if (flush_mode1) {
- tlb_flush_entries(dtlb2_mode1, type);
- }
- tlb_flush_entries(dtlb2_mode2, type);
- tlb_flush_entries(dtlb2_mode3, type);
- }
-}
-
-static bool gTLBFlushIBatEntries = false;
-static bool gTLBFlushDBatEntries = false;
-static bool gTLBInvalidatePatEntries = false;
-
-template
-void tlb_flush_bat_entries()
-{
- if (tlb_type == TLBType::ITLB) {
- if (!gTLBFlushIBatEntries)
- return;
- tlb_flush_entries(TLBE_FROM_BAT);
- gTLBFlushIBatEntries = false;
- } else {
- if (!gTLBFlushDBatEntries)
- return;
- tlb_flush_entries(TLBE_FROM_BAT);
- gTLBFlushDBatEntries = false;
- }
-}
-
-static void tlb_invalidate_pat_entries()
-{
- if (!gTLBInvalidatePatEntries)
- return;
-
- gTLBPatGeneration++;
- if (gTLBPatGeneration == 0) { // Do a full flush when the counter wraps around
- tlb_flush_entries(TLBE_FROM_PAT);
- tlb_flush_entries(TLBE_FROM_PAT);
- }
-
- gTLBInvalidatePatEntries = false;
-}
-
static void mpc601_bat_update(uint32_t bat_reg)
{
PPC_BAT_entry *ibat_entry, *dbat_entry;
@@ -1105,16 +1102,9 @@ static void mpc601_bat_update(uint32_t bat_reg)
dbat_entry->valid = false;
}
- // MPC601 has unified BATs so we're going to flush both ITLB and DTLB
- if (!gTLBFlushIBatEntries) {
- gTLBFlushIBatEntries = true;
- add_ctx_sync_action(&tlb_flush_bat_entries);
- }
- if (!gTLBFlushDBatEntries) {
- gTLBFlushDBatEntries = true;
- add_ctx_sync_action(&tlb_flush_bat_entries);
- }
- mmu_pat_ctx_changed();
+ // MPC601 has unified BATs, so they affect both translation contexts.
+ schedule_tlb_invalidation(TLBType::ITLB, TLBE_FROM_TRANSLATION);
+ schedule_tlb_invalidation(TLBType::DTLB, TLBE_FROM_TRANSLATION);
}
static void mpc601_dbat_update(uint32_t /*bat_reg*/)
@@ -1140,11 +1130,8 @@ static void ppc_ibat_update(uint32_t bat_reg)
bat_entry->phys_hi = ppc_state.spr[upper_reg_num + 1] & hi_mask;
bat_entry->bepi = ppc_state.spr[upper_reg_num] & hi_mask;
- if (!gTLBFlushIBatEntries) {
- gTLBFlushIBatEntries = true;
- add_ctx_sync_action(&tlb_flush_bat_entries);
- }
- mmu_pat_ctx_changed();
+ // A new IBAT can shadow an existing page translation.
+ schedule_tlb_invalidation(TLBType::ITLB, TLBE_FROM_TRANSLATION);
}
static void ppc_dbat_update(uint32_t bat_reg)
@@ -1165,21 +1152,16 @@ static void ppc_dbat_update(uint32_t bat_reg)
bat_entry->phys_hi = ppc_state.spr[upper_reg_num + 1] & hi_mask;
bat_entry->bepi = ppc_state.spr[upper_reg_num] & hi_mask;
- if (!gTLBFlushDBatEntries) {
- gTLBFlushDBatEntries = true;
- add_ctx_sync_action(&tlb_flush_bat_entries);
- }
- mmu_pat_ctx_changed();
+ // A new DBAT can shadow an existing page translation.
+ schedule_tlb_invalidation(TLBType::DTLB, TLBE_FROM_TRANSLATION);
}
void mmu_pat_ctx_changed()
{
// Page address translation context changed so invalidate all PAT entries
// from both ITLB and DTLB.
- if (!gTLBInvalidatePatEntries) {
- gTLBInvalidatePatEntries = true;
- add_ctx_sync_action(&tlb_invalidate_pat_entries);
- }
+ schedule_tlb_invalidation(TLBType::ITLB, TLBFlags::TLBE_FROM_PAT);
+ schedule_tlb_invalidation(TLBType::DTLB, TLBFlags::TLBE_FROM_PAT);
}
#if SUPPORTS_PPC_LITTLE_ENDIAN_MODE
@@ -1265,7 +1247,7 @@ inline T mmu_read_vmem(uint32_t opcode, uint32_t guest_va)
if (tlb2_entry->flags & TLBFlags::PAGE_MEM) { // is it a real memory region?
// refill the primary TLB
- *tlb1_entry = *tlb2_entry;
+ promote_tlb_entry(tlb1_entry, tlb2_entry);
#if SUPPORTS_MEMORY_CTRL_ENDIAN_MODE
needs_swap = mem_ctrl_instance->needs_swap_endian(false);
@@ -1442,7 +1424,7 @@ inline void mmu_write_vmem(uint32_t opcode, uint32_t guest_va, T value)
if (tlb2_entry->flags & TLBFlags::PAGE_MEM) { // is it a real memory region?
// refill the primary TLB
- *tlb1_entry = *tlb2_entry;
+ promote_tlb_entry(tlb1_entry, tlb2_entry);
#if SUPPORTS_MEMORY_CTRL_ENDIAN_MODE
needs_swap = mem_ctrl_instance->needs_swap_endian(false);
@@ -1968,7 +1950,7 @@ bool mmu_translate_dbg(uint32_t guest_va, uint32_t &guest_pa) {
if (tlb2_entry->flags & TLBFlags::PAGE_MEM) { // is it a real memory region?
// refill the primary TLB
- *tlb1_entry = *tlb2_entry;
+ promote_tlb_entry(tlb1_entry, tlb2_entry);
}
else {
tlb1_entry = tlb2_entry;
@@ -2004,14 +1986,16 @@ static void invalidate_tlb_entries(std::array &tlb) {
tlb_el.host_va_offs_r = 0;
tlb_el.host_va_offs_w = 0;
tlb_el.phys_tag = 0;
- tlb_el.pat_generation = 0;
+ tlb_el.reserved = 0;
}
}
void ppc_mmu_init()
{
- gTLBPatGeneration = 0;
- gTLBInvalidatePatEntries = false;
+ gPendingIInvalidationSources = 0;
+ gPendingDInvalidationSources = 0;
+ gTrackedIEntries.clear();
+ gTrackedDEntries.clear();
last_ptab_area = {0xFFFFFFFF, 0xFFFFFFFF, 0, 0, nullptr, nullptr};
diff --git a/cpu/ppc/ppcopcodes.cpp b/cpu/ppc/ppcopcodes.cpp
index 255775d559..8a62153f92 100644
--- a/cpu/ppc/ppcopcodes.cpp
+++ b/cpu/ppc/ppcopcodes.cpp
@@ -1133,8 +1133,10 @@ void dppc_interpreter::ppc_mtspr(uint32_t opcode) {
case 533:
case 534:
case 535:
- ppc_state.spr[ref_spr] = val;
- ibat_update(ref_spr);
+ if (ppc_state.spr[ref_spr] != val) {
+ ppc_state.spr[ref_spr] = val;
+ ibat_update(ref_spr);
+ }
break;
case 536:
case 537:
@@ -1144,8 +1146,10 @@ void dppc_interpreter::ppc_mtspr(uint32_t opcode) {
case 541:
case 542:
case 543:
- ppc_state.spr[ref_spr] = val;
- dbat_update(ref_spr);
+ if (ppc_state.spr[ref_spr] != val) {
+ ppc_state.spr[ref_spr] = val;
+ dbat_update(ref_spr);
+ }
break;
case SPR::HID0:
ppc_state.spr[ref_spr] = val;
diff --git a/zdocs/developers/cpu/powerpc/mmuemu.md b/zdocs/developers/cpu/powerpc/mmuemu.md
index e7a8c37ba4..46fedbcaf8 100644
--- a/zdocs/developers/cpu/powerpc/mmuemu.md
+++ b/zdocs/developers/cpu/powerpc/mmuemu.md
@@ -105,3 +105,26 @@ else:
refill_primary_tlb(VA, PA)
perform_memory_access(PA, ART)
```
+
+## Translation context invalidation
+
+Guest operating systems can change BAT registers and page address translation
+state very frequently. XNU, for example, temporarily installs a DBAT mapping in
+some physical-memory access routines and restores the previous mapping shortly
+afterwards. DingusPPC applies the resulting TLB invalidation at the next context
+synchronization (`isync`, `rfi`, or `sc`), as the processor does. A BAT update
+must invalidate cached PAT translations as well as BAT translations because a
+new BAT can shadow a page mapping.
+
+Scanning every primary and secondary TLB array at each synchronization is
+expensive. Instead, DingusPPC records pointers to slots populated by BAT or PAT
+translation in separate instruction and data vectors. Pending invalidations are
+combined by translation source, and synchronization visits only those tracked
+slots. Writes that leave a BAT register unchanged are ignored before scheduling
+any invalidation work.
+
+Generation counters offer constant-time invalidation, but require an additional
+generation check on every TLB lookup. Tracking populated slots keeps the common
+hit path as a direct tag comparison. Its cost is paid at context synchronization
+and is proportional to the number of translated slots populated since the last
+invalidation.