diff --git a/cpu/ppc/ppcmmu.cpp b/cpu/ppc/ppcmmu.cpp index 7155aa1612..8745936a9f 100644 --- a/cpu/ppc/ppcmmu.cpp +++ b/cpu/ppc/ppcmmu.cpp @@ -458,6 +458,58 @@ MapDmaResult mmu_map_dma_mem(uint32_t addr, uint32_t size, bool allow_mmio, bool return MapDmaResult{RT_NONE, false, nullptr, nullptr, 0}; } +constexpr uint32_t TLB_SIZE = 4096; +constexpr uint32_t TLB2_WAYS = 4; +constexpr uint32_t TLB_INVALID_TAG = 0xFFFFFFFF; +constexpr uint32_t TLB_VPS_MASK = 0x0FFFF000; // mask for TLB invalidation + +enum TLBFlags : uint16_t { + PAGE_MEM = 1 << 0, // memory page backed by host memory + PAGE_IO = 1 << 1, // memory mapped I/O page + PAGE_NOPHYS = 1 << 2, // no physical storage for this page (unmapped) + TLBE_FROM_BAT = 1 << 3, // TLB entry has been translated with BAT + TLBE_FROM_PAT = 1 << 4, // TLB entry has been translated with PAT + PAGE_WRITABLE = 1 << 5, // page is writable + PTE_SET_C = 1 << 6, // tells if C bit of the PTE needs to be updated +}; + +// PAT context changes are frequent, so implicitly invalidate PAT-derived +// entries via a generation counter (instead of walking the TLB array and +// clearing every time). +static uint32_t gTLBPatGeneration = 0; + +typedef struct TLBEntry { + uint32_t tag; // use is_invalid() or matches_tag() for validity checks + uint16_t flags; + uint16_t lru_bits; + union { + struct { // for memory pages + int64_t host_va_offs_r; + int64_t host_va_offs_w; + }; + struct { // for MMIO pages + AddressMapEntry* rgn_desc; + int64_t dev_base_va; + }; + }; + uint32_t phys_tag; + uint32_t pat_generation; + + bool is_invalid() const + { + return tag == TLB_INVALID_TAG || + ((flags & TLBFlags::TLBE_FROM_PAT) && + pat_generation != gTLBPatGeneration); + } + + bool matches_tag(uint32_t tag_in) const + { + return tag == tag_in && + (!(flags & TLBFlags::TLBE_FROM_PAT) || + pat_generation == gTLBPatGeneration); + } +} TLBEntry; + // primary ITLB for all MMU modes static std::array itlb1_mode1; static std::array itlb1_mode2; @@ -555,28 +607,28 @@ static TLBEntry* tlb2_target_entry(uint32_t gp_va) } // select the target from invalid blocks first - if (tlb_entry[0].tag == TLB_INVALID_TAG) { + if (tlb_entry[0].is_invalid()) { // update LRU bits tlb_entry[0].lru_bits = 0x3; tlb_entry[1].lru_bits = 0x2; tlb_entry[2].lru_bits &= 0x1; tlb_entry[3].lru_bits &= 0x1; return tlb_entry; - } else if (tlb_entry[1].tag == TLB_INVALID_TAG) { + } else if (tlb_entry[1].is_invalid()) { // update LRU bits tlb_entry[0].lru_bits = 0x2; tlb_entry[1].lru_bits = 0x3; tlb_entry[2].lru_bits &= 0x1; tlb_entry[3].lru_bits &= 0x1; return &tlb_entry[1]; - } else if (tlb_entry[2].tag == TLB_INVALID_TAG) { + } else if (tlb_entry[2].is_invalid()) { // update LRU bits tlb_entry[0].lru_bits &= 0x1; tlb_entry[1].lru_bits &= 0x1; tlb_entry[2].lru_bits = 0x3; tlb_entry[3].lru_bits = 0x2; return &tlb_entry[2]; - } else if (tlb_entry[3].tag == TLB_INVALID_TAG) { + } else if (tlb_entry[3].is_invalid()) { // update LRU bits tlb_entry[0].lru_bits &= 0x1; tlb_entry[1].lru_bits &= 0x1; @@ -666,6 +718,7 @@ static TLBEntry* itlb2_refill(uint32_t guest_va) tlb_entry->host_va_offs_r = (int64_t)rgn_desc->mem_ptr - guest_va + (phys_addr - rgn_desc->start); tlb_entry->phys_tag = phys_addr & ~0xFFFUL; + tlb_entry->pat_generation = gTLBPatGeneration; } else { ABORT_F("Instruction fetch from unmapped memory at 0x%08X!\n", phys_addr); } @@ -749,6 +802,7 @@ static TLBEntry* dtlb2_refill(uint32_t guest_va, int is_write, bool is_dbg = fal } } tlb_entry->phys_tag = phys_addr & ~0xFFFUL; + tlb_entry->pat_generation = gTLBPatGeneration; return tlb_entry; } else { if (!is_dbg) { @@ -777,27 +831,27 @@ static inline TLBEntry* lookup_secondary_tlb(uint32_t guest_va, uint32_t tag) { tlb_entry = &pCurDTLB2[((guest_va >> PPC_PAGE_SIZE_BITS) & tlb_size_mask) * TLB2_WAYS]; } - if (tlb_entry->tag == tag) { + if (tlb_entry[0].matches_tag(tag)) { // update LRU bits tlb_entry[0].lru_bits = 0x3; tlb_entry[1].lru_bits = 0x2; tlb_entry[2].lru_bits &= 0x1; tlb_entry[3].lru_bits &= 0x1; - } else if (tlb_entry[1].tag == tag) { + } else if (tlb_entry[1].matches_tag(tag)) { // update LRU bits tlb_entry[0].lru_bits = 0x2; tlb_entry[1].lru_bits = 0x3; tlb_entry[2].lru_bits &= 0x1; tlb_entry[3].lru_bits &= 0x1; tlb_entry = &tlb_entry[1]; - } else if (tlb_entry[2].tag == tag) { + } else if (tlb_entry[2].matches_tag(tag)) { // update LRU bits tlb_entry[0].lru_bits &= 0x1; tlb_entry[1].lru_bits &= 0x1; tlb_entry[2].lru_bits = 0x3; tlb_entry[3].lru_bits = 0x2; tlb_entry = &tlb_entry[2]; - } else if (tlb_entry[3].tag == tag) { + } else if (tlb_entry[3].matches_tag(tag)) { // update LRU bits tlb_entry[0].lru_bits &= 0x1; tlb_entry[1].lru_bits &= 0x1; @@ -829,7 +883,7 @@ uint8_t *mmu_translate_imem(uint32_t vaddr, uint32_t *paddr) // look up guest virtual address in the primary ITLB tlb1_entry = &pCurITLB1[(vaddr >> PPC_PAGE_SIZE_BITS) & tlb_size_mask]; - if (tlb1_entry->tag == tag) { // primary ITLB hit -> fast path + if (tlb1_entry->matches_tag(tag)) { // primary ITLB hit -> fast path #ifdef TLB_PROFILING num_primary_itlb_hits++; #endif @@ -855,6 +909,7 @@ uint8_t *mmu_translate_imem(uint32_t vaddr, uint32_t *paddr) tlb1_entry->flags = tlb2_entry->flags; tlb1_entry->host_va_offs_r = tlb2_entry->host_va_offs_r; tlb1_entry->phys_tag = tlb2_entry->phys_tag; + tlb1_entry->pat_generation = tlb2_entry->pat_generation; host_va = (uint8_t *)(tlb1_entry->host_va_offs_r + vaddr); } @@ -940,10 +995,9 @@ void tlb_flush_entries(TLBFlags type) } } -bool gTLBFlushIBatEntries = false; -bool gTLBFlushDBatEntries = false; -bool gTLBFlushIPatEntries = false; -bool gTLBFlushDPatEntries = false; +static bool gTLBFlushIBatEntries = false; +static bool gTLBFlushDBatEntries = false; +static bool gTLBInvalidatePatEntries = false; template void tlb_flush_bat_entries() @@ -961,38 +1015,18 @@ void tlb_flush_bat_entries() } } -template -void tlb_flush_pat_entries() +static void tlb_invalidate_pat_entries() { - if (tlb_type == TLBType::ITLB) { - if (!gTLBFlushIPatEntries) - return; + if (!gTLBInvalidatePatEntries) + return; + + gTLBPatGeneration++; + if (gTLBPatGeneration == 0) { // Do a full flush when the counter wraps around tlb_flush_entries(TLBE_FROM_PAT); - gTLBFlushIPatEntries = false; - } else { - if (!gTLBFlushDPatEntries) - return; tlb_flush_entries(TLBE_FROM_PAT); - gTLBFlushDPatEntries = false; } -} -template -void tlb_flush_all_entries() -{ - if (tlb_type == TLBType::ITLB) { - if (!gTLBFlushIBatEntries && !gTLBFlushIPatEntries) - return; - tlb_flush_entries((TLBFlags)(TLBE_FROM_BAT | TLBE_FROM_PAT)); - gTLBFlushIBatEntries = false; - gTLBFlushIPatEntries = false; - } else { - if (!gTLBFlushDBatEntries && !gTLBFlushDPatEntries) - return; - tlb_flush_entries((TLBFlags)(TLBE_FROM_BAT | TLBE_FROM_PAT)); - gTLBFlushDBatEntries = false; - gTLBFlushDPatEntries = false; - } + gTLBInvalidatePatEntries = false; } static void mpc601_bat_update(uint32_t bat_reg) @@ -1026,14 +1060,15 @@ static void mpc601_bat_update(uint32_t bat_reg) } // MPC601 has unified BATs so we're going to flush both ITLB and DTLB - if (!gTLBFlushIBatEntries || !gTLBFlushIPatEntries || !gTLBFlushDBatEntries || !gTLBFlushDPatEntries) { + if (!gTLBFlushIBatEntries) { gTLBFlushIBatEntries = true; - gTLBFlushIPatEntries = true; + add_ctx_sync_action(&tlb_flush_bat_entries); + } + if (!gTLBFlushDBatEntries) { gTLBFlushDBatEntries = true; - gTLBFlushDPatEntries = true; - add_ctx_sync_action(&tlb_flush_all_entries); - add_ctx_sync_action(&tlb_flush_all_entries); + add_ctx_sync_action(&tlb_flush_bat_entries); } + mmu_pat_ctx_changed(); } static void mpc601_dbat_update(uint32_t /*bat_reg*/) @@ -1059,11 +1094,11 @@ static void ppc_ibat_update(uint32_t bat_reg) bat_entry->phys_hi = ppc_state.spr[upper_reg_num + 1] & hi_mask; bat_entry->bepi = ppc_state.spr[upper_reg_num] & hi_mask; - if (!gTLBFlushIBatEntries || !gTLBFlushIPatEntries) { + if (!gTLBFlushIBatEntries) { gTLBFlushIBatEntries = true; - gTLBFlushIPatEntries = true; - add_ctx_sync_action(&tlb_flush_all_entries); + add_ctx_sync_action(&tlb_flush_bat_entries); } + mmu_pat_ctx_changed(); } static void ppc_dbat_update(uint32_t bat_reg) @@ -1084,22 +1119,20 @@ static void ppc_dbat_update(uint32_t bat_reg) bat_entry->phys_hi = ppc_state.spr[upper_reg_num + 1] & hi_mask; bat_entry->bepi = ppc_state.spr[upper_reg_num] & hi_mask; - if (!gTLBFlushDBatEntries || !gTLBFlushDPatEntries) { + if (!gTLBFlushDBatEntries) { gTLBFlushDBatEntries = true; - gTLBFlushDPatEntries = true; - add_ctx_sync_action(&tlb_flush_all_entries); + add_ctx_sync_action(&tlb_flush_bat_entries); } + mmu_pat_ctx_changed(); } void mmu_pat_ctx_changed() { - // Page address translation context changed so we need to flush - // all PAT entries from both ITLB and DTLB - if (!gTLBFlushIPatEntries || !gTLBFlushDPatEntries) { - gTLBFlushIPatEntries = true; - gTLBFlushDPatEntries = true; - add_ctx_sync_action(&tlb_flush_pat_entries); - add_ctx_sync_action(&tlb_flush_pat_entries); + // Page address translation context changed so invalidate all PAT entries + // from both ITLB and DTLB. + if (!gTLBInvalidatePatEntries) { + gTLBInvalidatePatEntries = true; + add_ctx_sync_action(&tlb_invalidate_pat_entries); } } @@ -1150,7 +1183,7 @@ inline T mmu_read_vmem(uint32_t opcode, uint32_t guest_va) bool needs_swap = false; #endif tlb1_entry = &pCurDTLB1[(guest_va >> PPC_PAGE_SIZE_BITS) & tlb_size_mask]; - if (tlb1_entry->tag == tag) { // primary TLB hit -> fast path + if (tlb1_entry->matches_tag(tag)) { // primary TLB hit -> fast path #ifdef TLB_PROFILING num_primary_dtlb_hits++; #endif @@ -1321,7 +1354,7 @@ inline void mmu_write_vmem(uint32_t opcode, uint32_t guest_va, T value) bool needs_swap = false; #endif tlb1_entry = &pCurDTLB1[(guest_va >> PPC_PAGE_SIZE_BITS) & tlb_size_mask]; - if (tlb1_entry->tag == tag) { // primary TLB hit -> fast path + if (tlb1_entry->matches_tag(tag)) { // primary TLB hit -> fast path #ifdef TLB_PROFILING num_primary_dtlb_hits++; #endif @@ -1895,7 +1928,7 @@ bool mmu_translate_dbg(uint32_t guest_va, uint32_t &guest_pa) { tlb1_entry = &pCurDTLB1[(guest_va >> PPC_PAGE_SIZE_BITS) & tlb_size_mask]; do { - if (tlb1_entry->tag != tag) { + if (!tlb1_entry->matches_tag(tag)) { // primary TLB miss -> look up address in the secondary TLB tlb2_entry = lookup_secondary_tlb(guest_va, tag); if (tlb2_entry == nullptr) { @@ -1946,12 +1979,15 @@ static void invalidate_tlb_entries(std::array &tlb) { tlb_el.host_va_offs_r = 0; tlb_el.host_va_offs_w = 0; tlb_el.phys_tag = 0; - tlb_el.reserved = 0; + tlb_el.pat_generation = 0; } } void ppc_mmu_init() { + gTLBPatGeneration = 0; + gTLBInvalidatePatEntries = false; + last_ptab_area = {0xFFFFFFFF, 0xFFFFFFFF, 0, 0, nullptr, nullptr}; mmu_exception_handler = ppc_exception_handler; diff --git a/cpu/ppc/ppcmmu.h b/cpu/ppc/ppcmmu.h index 0691eab327..0a0c19008b 100644 --- a/cpu/ppc/ppcmmu.h +++ b/cpu/ppc/ppcmmu.h @@ -24,8 +24,6 @@ along with this program. If not, see . #ifndef PPCMMU_H #define PPCMMU_H -#include - #include #include @@ -84,38 +82,6 @@ typedef struct MapDmaResult { constexpr uint32_t PPC_PAGE_SIZE_BITS = 12; constexpr uint32_t PPC_PAGE_SIZE = (1 << PPC_PAGE_SIZE_BITS); constexpr uint32_t PPC_PAGE_MASK = ~(PPC_PAGE_SIZE - 1); -constexpr uint32_t TLB_SIZE = 4096; -constexpr uint32_t TLB2_WAYS = 4; -constexpr uint32_t TLB_INVALID_TAG = 0xFFFFFFFF; -constexpr uint32_t TLB_VPS_MASK = 0x0FFFF000; // mask for TLB invalidation - -typedef struct TLBEntry { - uint32_t tag; - uint16_t flags; - uint16_t lru_bits; - union { - struct { // for memory pages - int64_t host_va_offs_r; - int64_t host_va_offs_w; - }; - struct { // for MMIO pages - AddressMapEntry* rgn_desc; - int64_t dev_base_va; - }; - }; - uint32_t phys_tag; - uint32_t reserved; -} TLBEntry; - -enum TLBFlags : uint16_t { - PAGE_MEM = 1 << 0, // memory page backed by host memory - PAGE_IO = 1 << 1, // memory mapped I/O page - PAGE_NOPHYS = 1 << 2, // no physical storage for this page (unmapped) - TLBE_FROM_BAT = 1 << 3, // TLB entry has been translated with BAT - TLBE_FROM_PAT = 1 << 4, // TLB entry has been translated with PAT - PAGE_WRITABLE = 1 << 5, // page is writable - PTE_SET_C = 1 << 6, // tells if C bit of the PTE needs to be updated -}; extern std::function ibat_update; extern std::function dbat_update; diff --git a/devices/common/dbdma.cpp b/devices/common/dbdma.cpp index fc7042ce1d..f4a37bc552 100644 --- a/devices/common/dbdma.cpp +++ b/devices/common/dbdma.cpp @@ -29,6 +29,7 @@ along with this program. If not, see . #include #include #include +#include #include #include