diff --git a/cpu/ppc/ppcmmu.cpp b/cpu/ppc/ppcmmu.cpp index 1f3f3540d7..1f16ccdbc8 100644 --- a/cpu/ppc/ppcmmu.cpp +++ b/cpu/ppc/ppcmmu.cpp @@ -31,6 +31,7 @@ along with this program. If not, see . #include #include #include +#include //#define MMU_PROFILING // uncomment this to enable MMU profiling //#define TLB_PROFILING // uncomment this to enable SoftTLB profiling @@ -471,15 +472,14 @@ enum TLBFlags : uint16_t { TLBE_FROM_PAT = 1 << 4, // TLB entry has been translated with PAT PAGE_WRITABLE = 1 << 5, // page is writable PTE_SET_C = 1 << 6, // tells if C bit of the PTE needs to be updated + TLBE_CTX_TRACKED = 1 << 7, // entry pointer is in gTrackedIEntries or gTrackedDEntries }; -// PAT context changes are frequent, so implicitly invalidate PAT-derived -// entries via a generation counter (instead of walking the TLB array and -// clearing every time). -static uint32_t gTLBPatGeneration = 0; +constexpr uint16_t TLBE_FROM_TRANSLATION = + TLBFlags::TLBE_FROM_BAT | TLBFlags::TLBE_FROM_PAT; typedef struct TLBEntry { - uint32_t tag; // use is_invalid() or matches_tag() for validity checks + uint32_t tag; uint16_t flags; uint16_t lru_bits; union { @@ -493,22 +493,93 @@ typedef struct TLBEntry { }; }; uint32_t phys_tag; - uint32_t pat_generation; + uint32_t reserved; +} TLBEntry; - bool is_invalid() const - { - return tag == TLB_INVALID_TAG || - ((flags & TLBFlags::TLBE_FROM_PAT) && - pat_generation != gTLBPatGeneration); +// Track slots populated by either page or block address translation. Context +// changes can then invalidate only populated slots without adding a generation +// check to every TLB lookup. +static std::vector gTrackedIEntries; +static std::vector gTrackedDEntries; + +// TLBE_FROM_* sources awaiting invalidation at the next context sync. +static uint16_t gPendingIInvalidationSources = 0; +static uint16_t gPendingDInvalidationSources = 0; + +template +static void track_translated_entry(TLBEntry *tlb_entry) +{ + if (!(tlb_entry->flags & TLBE_FROM_TRANSLATION) || + (tlb_entry->flags & TLBFlags::TLBE_CTX_TRACKED)) + return; + + tlb_entry->flags |= TLBFlags::TLBE_CTX_TRACKED; + if (tlb_type == TLBType::ITLB) + gTrackedIEntries.push_back(tlb_entry); + else + gTrackedDEntries.push_back(tlb_entry); +} + +template +static void tlb_invalidate_tracked_entries() +{ + uint16_t *pending_sources; + std::vector *tracked_entries; + + if (tlb_type == TLBType::ITLB) { + pending_sources = &gPendingIInvalidationSources; + tracked_entries = &gTrackedIEntries; + } else { + pending_sources = &gPendingDInvalidationSources; + tracked_entries = &gTrackedDEntries; } - bool matches_tag(uint32_t tag_in) const - { - return tag == tag_in && - (!(flags & TLBFlags::TLBE_FROM_PAT) || - pat_generation == gTLBPatGeneration); + if (!*pending_sources) + return; + + size_t retained_count = 0; + for (TLBEntry *tlb_entry : *tracked_entries) { + uint16_t source = tlb_entry->flags & TLBE_FROM_TRANSLATION; + if (source & *pending_sources) { + tlb_entry->tag = TLB_INVALID_TAG; + tlb_entry->flags &= ~TLBFlags::TLBE_CTX_TRACKED; + } else if (source) { + (*tracked_entries)[retained_count++] = tlb_entry; + } else { + tlb_entry->flags &= ~TLBFlags::TLBE_CTX_TRACKED; + } } -} TLBEntry; + tracked_entries->resize(retained_count); + *pending_sources = 0; +} + +static void schedule_tlb_invalidation(TLBType tlb_type, uint16_t sources_to_invalidate) +{ + uint16_t *pending_sources; + + if (tlb_type == TLBType::ITLB) + pending_sources = &gPendingIInvalidationSources; + else + pending_sources = &gPendingDInvalidationSources; + + if (!*pending_sources) { + if (tlb_type == TLBType::ITLB) + add_ctx_sync_action(&tlb_invalidate_tracked_entries); + else + add_ctx_sync_action(&tlb_invalidate_tracked_entries); + } + *pending_sources |= sources_to_invalidate; +} + +template +static void promote_tlb_entry(TLBEntry *tlb1_entry, const TLBEntry *tlb2_entry) +{ + uint16_t tracked = tlb1_entry->flags & TLBFlags::TLBE_CTX_TRACKED; + *tlb1_entry = *tlb2_entry; + tlb1_entry->flags &= ~TLBFlags::TLBE_CTX_TRACKED; + tlb1_entry->flags |= tracked; + track_translated_entry(tlb1_entry); +} // primary ITLB for all MMU modes static std::array itlb1_mode1; @@ -628,16 +699,16 @@ static TLBEntry* tlb2_target_entry(uint32_t gp_va) } // select the target from invalid blocks first - if (tlb_entry[0].is_invalid()) { + if (tlb_entry[0].tag == TLB_INVALID_TAG) { tlb2_touch_way<0>(tlb_entry); return tlb_entry; - } else if (tlb_entry[1].is_invalid()) { + } else if (tlb_entry[1].tag == TLB_INVALID_TAG) { tlb2_touch_way<1>(tlb_entry); return &tlb_entry[1]; - } else if (tlb_entry[2].is_invalid()) { + } else if (tlb_entry[2].tag == TLB_INVALID_TAG) { tlb2_touch_way<2>(tlb_entry); return &tlb_entry[2]; - } else if (tlb_entry[3].is_invalid()) { + } else if (tlb_entry[3].tag == TLB_INVALID_TAG) { tlb2_touch_way<3>(tlb_entry); return &tlb_entry[3]; } else { // no free entries, replace an existing one according with the hLRU policy @@ -703,11 +774,12 @@ static TLBEntry* itlb2_refill(uint32_t guest_va) const uint32_t tag = guest_va & ~0xFFFUL; tlb_entry = tlb2_target_entry(tag); tlb_entry->tag = tag; - tlb_entry->flags = flags | TLBFlags::PAGE_MEM; + tlb_entry->flags = flags | TLBFlags::PAGE_MEM | + (tlb_entry->flags & TLBFlags::TLBE_CTX_TRACKED); tlb_entry->host_va_offs_r = (int64_t)rgn_desc->mem_ptr - guest_va + (phys_addr - rgn_desc->start); tlb_entry->phys_tag = phys_addr & ~0xFFFUL; - tlb_entry->pat_generation = gTLBPatGeneration; + track_translated_entry(tlb_entry); } else { ABORT_F("Instruction fetch from unmapped memory at 0x%08X!\n", phys_addr); } @@ -776,11 +848,13 @@ static TLBEntry* dtlb2_refill(uint32_t guest_va, int is_write, bool is_dbg = fal tlb_entry = tlb2_target_entry(tag); tlb_entry->tag = tag; if (rgn_desc->type & RT_MMIO) { // MMIO region - tlb_entry->flags = flags | TLBFlags::PAGE_IO; + tlb_entry->flags = flags | TLBFlags::PAGE_IO | + (tlb_entry->flags & TLBFlags::TLBE_CTX_TRACKED); tlb_entry->rgn_desc = rgn_desc; tlb_entry->dev_base_va = guest_va - (phys_addr - rgn_desc->start); } else { // memory region backed by host memory - tlb_entry->flags = flags | TLBFlags::PAGE_MEM; + tlb_entry->flags = flags | TLBFlags::PAGE_MEM | + (tlb_entry->flags & TLBFlags::TLBE_CTX_TRACKED); tlb_entry->host_va_offs_r = (int64_t)rgn_desc->mem_ptr - guest_va + (phys_addr - rgn_desc->start); if (rgn_desc->type == RT_ROM) { @@ -791,7 +865,7 @@ static TLBEntry* dtlb2_refill(uint32_t guest_va, int is_write, bool is_dbg = fal } } tlb_entry->phys_tag = phys_addr & ~0xFFFUL; - tlb_entry->pat_generation = gTLBPatGeneration; + track_translated_entry(tlb_entry); return tlb_entry; } else { if (!is_dbg) { @@ -820,15 +894,15 @@ static inline TLBEntry* lookup_secondary_tlb(uint32_t guest_va, uint32_t tag) { tlb_entry = &pCurDTLB2[((guest_va >> PPC_PAGE_SIZE_BITS) & tlb_size_mask) * TLB2_WAYS]; } - if (tlb_entry[0].matches_tag(tag)) { + if (tlb_entry[0].tag == tag) { tlb2_touch_way<0>(tlb_entry); - } else if (tlb_entry[1].matches_tag(tag)) { + } else if (tlb_entry[1].tag == tag) { tlb2_touch_way<1>(tlb_entry); tlb_entry = &tlb_entry[1]; - } else if (tlb_entry[2].matches_tag(tag)) { + } else if (tlb_entry[2].tag == tag) { tlb2_touch_way<2>(tlb_entry); tlb_entry = &tlb_entry[2]; - } else if (tlb_entry[3].matches_tag(tag)) { + } else if (tlb_entry[3].tag == tag) { tlb2_touch_way<3>(tlb_entry); tlb_entry = &tlb_entry[3]; } else { @@ -853,7 +927,7 @@ static inline TLBLookupResult lookup_tlb(uint32_t guest_va, uint32_t tag) primary_entry = &pCurDTLB1[(guest_va >> PPC_PAGE_SIZE_BITS) & tlb_size_mask]; } - if (primary_entry->matches_tag(tag)) { + if (primary_entry->tag == tag) { return TLBLookupResult{primary_entry, primary_entry, true}; } return TLBLookupResult{ @@ -951,11 +1025,7 @@ uint8_t *mmu_translate_imem(uint32_t vaddr, uint32_t *paddr) } #endif // refill the primary ITLB - tlb1_entry->tag = tag; - tlb1_entry->flags = tlb2_entry->flags; - tlb1_entry->host_va_offs_r = tlb2_entry->host_va_offs_r; - tlb1_entry->phys_tag = tlb2_entry->phys_tag; - tlb1_entry->pat_generation = tlb2_entry->pat_generation; + promote_tlb_entry(tlb1_entry, tlb2_entry); host_va = (uint8_t *)(tlb1_entry->host_va_offs_r + vaddr); } @@ -1002,79 +1072,6 @@ void tlb_flush_entry(uint32_t ea) tlb_flush_secondary_entry(dtlb2_mode3, tag); } -template -static void tlb_flush_entries(std::array &tlb, TLBFlags type) { - for (auto &tlb_el : tlb) { - if (tlb_el.tag != TLB_INVALID_TAG && tlb_el.flags & type) { - tlb_el.tag = TLB_INVALID_TAG; - } - } -} - -template -void tlb_flush_entries(TLBFlags type) -{ - // Mode 1 is real addressing and thus can't contain any PAT entries by definition. - bool flush_mode1 = type != TLBE_FROM_PAT; - if (tlb_type == TLBType::ITLB) { - if (flush_mode1) { - tlb_flush_entries(itlb1_mode1, type); - } - tlb_flush_entries(itlb1_mode2, type); - tlb_flush_entries(itlb1_mode3, type); - if (flush_mode1) { - tlb_flush_entries(itlb2_mode1, type); - } - tlb_flush_entries(itlb2_mode2, type); - tlb_flush_entries(itlb2_mode3, type); - } else { - if (flush_mode1) { - tlb_flush_entries(dtlb1_mode1, type); - } - tlb_flush_entries(dtlb1_mode2, type); - tlb_flush_entries(dtlb1_mode3, type); - if (flush_mode1) { - tlb_flush_entries(dtlb2_mode1, type); - } - tlb_flush_entries(dtlb2_mode2, type); - tlb_flush_entries(dtlb2_mode3, type); - } -} - -static bool gTLBFlushIBatEntries = false; -static bool gTLBFlushDBatEntries = false; -static bool gTLBInvalidatePatEntries = false; - -template -void tlb_flush_bat_entries() -{ - if (tlb_type == TLBType::ITLB) { - if (!gTLBFlushIBatEntries) - return; - tlb_flush_entries(TLBE_FROM_BAT); - gTLBFlushIBatEntries = false; - } else { - if (!gTLBFlushDBatEntries) - return; - tlb_flush_entries(TLBE_FROM_BAT); - gTLBFlushDBatEntries = false; - } -} - -static void tlb_invalidate_pat_entries() -{ - if (!gTLBInvalidatePatEntries) - return; - - gTLBPatGeneration++; - if (gTLBPatGeneration == 0) { // Do a full flush when the counter wraps around - tlb_flush_entries(TLBE_FROM_PAT); - tlb_flush_entries(TLBE_FROM_PAT); - } - - gTLBInvalidatePatEntries = false; -} - static void mpc601_bat_update(uint32_t bat_reg) { PPC_BAT_entry *ibat_entry, *dbat_entry; @@ -1105,16 +1102,9 @@ static void mpc601_bat_update(uint32_t bat_reg) dbat_entry->valid = false; } - // MPC601 has unified BATs so we're going to flush both ITLB and DTLB - if (!gTLBFlushIBatEntries) { - gTLBFlushIBatEntries = true; - add_ctx_sync_action(&tlb_flush_bat_entries); - } - if (!gTLBFlushDBatEntries) { - gTLBFlushDBatEntries = true; - add_ctx_sync_action(&tlb_flush_bat_entries); - } - mmu_pat_ctx_changed(); + // MPC601 has unified BATs, so they affect both translation contexts. + schedule_tlb_invalidation(TLBType::ITLB, TLBE_FROM_TRANSLATION); + schedule_tlb_invalidation(TLBType::DTLB, TLBE_FROM_TRANSLATION); } static void mpc601_dbat_update(uint32_t /*bat_reg*/) @@ -1140,11 +1130,8 @@ static void ppc_ibat_update(uint32_t bat_reg) bat_entry->phys_hi = ppc_state.spr[upper_reg_num + 1] & hi_mask; bat_entry->bepi = ppc_state.spr[upper_reg_num] & hi_mask; - if (!gTLBFlushIBatEntries) { - gTLBFlushIBatEntries = true; - add_ctx_sync_action(&tlb_flush_bat_entries); - } - mmu_pat_ctx_changed(); + // A new IBAT can shadow an existing page translation. + schedule_tlb_invalidation(TLBType::ITLB, TLBE_FROM_TRANSLATION); } static void ppc_dbat_update(uint32_t bat_reg) @@ -1165,21 +1152,16 @@ static void ppc_dbat_update(uint32_t bat_reg) bat_entry->phys_hi = ppc_state.spr[upper_reg_num + 1] & hi_mask; bat_entry->bepi = ppc_state.spr[upper_reg_num] & hi_mask; - if (!gTLBFlushDBatEntries) { - gTLBFlushDBatEntries = true; - add_ctx_sync_action(&tlb_flush_bat_entries); - } - mmu_pat_ctx_changed(); + // A new DBAT can shadow an existing page translation. + schedule_tlb_invalidation(TLBType::DTLB, TLBE_FROM_TRANSLATION); } void mmu_pat_ctx_changed() { // Page address translation context changed so invalidate all PAT entries // from both ITLB and DTLB. - if (!gTLBInvalidatePatEntries) { - gTLBInvalidatePatEntries = true; - add_ctx_sync_action(&tlb_invalidate_pat_entries); - } + schedule_tlb_invalidation(TLBType::ITLB, TLBFlags::TLBE_FROM_PAT); + schedule_tlb_invalidation(TLBType::DTLB, TLBFlags::TLBE_FROM_PAT); } #if SUPPORTS_PPC_LITTLE_ENDIAN_MODE @@ -1265,7 +1247,7 @@ inline T mmu_read_vmem(uint32_t opcode, uint32_t guest_va) if (tlb2_entry->flags & TLBFlags::PAGE_MEM) { // is it a real memory region? // refill the primary TLB - *tlb1_entry = *tlb2_entry; + promote_tlb_entry(tlb1_entry, tlb2_entry); #if SUPPORTS_MEMORY_CTRL_ENDIAN_MODE needs_swap = mem_ctrl_instance->needs_swap_endian(false); @@ -1442,7 +1424,7 @@ inline void mmu_write_vmem(uint32_t opcode, uint32_t guest_va, T value) if (tlb2_entry->flags & TLBFlags::PAGE_MEM) { // is it a real memory region? // refill the primary TLB - *tlb1_entry = *tlb2_entry; + promote_tlb_entry(tlb1_entry, tlb2_entry); #if SUPPORTS_MEMORY_CTRL_ENDIAN_MODE needs_swap = mem_ctrl_instance->needs_swap_endian(false); @@ -1968,7 +1950,7 @@ bool mmu_translate_dbg(uint32_t guest_va, uint32_t &guest_pa) { if (tlb2_entry->flags & TLBFlags::PAGE_MEM) { // is it a real memory region? // refill the primary TLB - *tlb1_entry = *tlb2_entry; + promote_tlb_entry(tlb1_entry, tlb2_entry); } else { tlb1_entry = tlb2_entry; @@ -2004,14 +1986,16 @@ static void invalidate_tlb_entries(std::array &tlb) { tlb_el.host_va_offs_r = 0; tlb_el.host_va_offs_w = 0; tlb_el.phys_tag = 0; - tlb_el.pat_generation = 0; + tlb_el.reserved = 0; } } void ppc_mmu_init() { - gTLBPatGeneration = 0; - gTLBInvalidatePatEntries = false; + gPendingIInvalidationSources = 0; + gPendingDInvalidationSources = 0; + gTrackedIEntries.clear(); + gTrackedDEntries.clear(); last_ptab_area = {0xFFFFFFFF, 0xFFFFFFFF, 0, 0, nullptr, nullptr}; diff --git a/cpu/ppc/ppcopcodes.cpp b/cpu/ppc/ppcopcodes.cpp index 255775d559..8a62153f92 100644 --- a/cpu/ppc/ppcopcodes.cpp +++ b/cpu/ppc/ppcopcodes.cpp @@ -1133,8 +1133,10 @@ void dppc_interpreter::ppc_mtspr(uint32_t opcode) { case 533: case 534: case 535: - ppc_state.spr[ref_spr] = val; - ibat_update(ref_spr); + if (ppc_state.spr[ref_spr] != val) { + ppc_state.spr[ref_spr] = val; + ibat_update(ref_spr); + } break; case 536: case 537: @@ -1144,8 +1146,10 @@ void dppc_interpreter::ppc_mtspr(uint32_t opcode) { case 541: case 542: case 543: - ppc_state.spr[ref_spr] = val; - dbat_update(ref_spr); + if (ppc_state.spr[ref_spr] != val) { + ppc_state.spr[ref_spr] = val; + dbat_update(ref_spr); + } break; case SPR::HID0: ppc_state.spr[ref_spr] = val; diff --git a/zdocs/developers/cpu/powerpc/mmuemu.md b/zdocs/developers/cpu/powerpc/mmuemu.md index e7a8c37ba4..46fedbcaf8 100644 --- a/zdocs/developers/cpu/powerpc/mmuemu.md +++ b/zdocs/developers/cpu/powerpc/mmuemu.md @@ -105,3 +105,26 @@ else: refill_primary_tlb(VA, PA) perform_memory_access(PA, ART) ``` + +## Translation context invalidation + +Guest operating systems can change BAT registers and page address translation +state very frequently. XNU, for example, temporarily installs a DBAT mapping in +some physical-memory access routines and restores the previous mapping shortly +afterwards. DingusPPC applies the resulting TLB invalidation at the next context +synchronization (`isync`, `rfi`, or `sc`), as the processor does. A BAT update +must invalidate cached PAT translations as well as BAT translations because a +new BAT can shadow a page mapping. + +Scanning every primary and secondary TLB array at each synchronization is +expensive. Instead, DingusPPC records pointers to slots populated by BAT or PAT +translation in separate instruction and data vectors. Pending invalidations are +combined by translation source, and synchronization visits only those tracked +slots. Writes that leave a BAT register unchanged are ignored before scheduling +any invalidation work. + +Generation counters offer constant-time invalidation, but require an additional +generation check on every TLB lookup. Tracking populated slots keeps the common +hit path as a direct tag comparison. Its cost is paid at context synchronization +and is proportional to the number of translated slots populated since the last +invalidation.