diff --git a/Source/Core/Core/IOS/Starlet/ARMCore.cpp b/Source/Core/Core/IOS/Starlet/ARMCore.cpp index 5e74bc8566..706a7c6b7e 100644 --- a/Source/Core/Core/IOS/Starlet/ARMCore.cpp +++ b/Source/Core/Core/IOS/Starlet/ARMCore.cpp @@ -190,7 +190,7 @@ u32 ARMCore::TranslateVirtualAddress(u32 address) const const u32 modified_address = address < 0x02000000 ? address | (m_cp15.process_id & 0xfe000000) : address; const u32 virtual_page = modified_address >> 10; - TLBEntry& entry = m_tlb[virtual_page & (TLB_ENTRY_COUNT - 1)]; + TLBEntry& entry = m_tlb[GetTLBIndex(virtual_page)]; if (entry.generation == m_tlb_generation && entry.virtual_page == virtual_page) return entry.physical_page | (modified_address & 0x3ff); @@ -600,8 +600,8 @@ bool ARMCore::TryEnterARMSliceStablePoll(u32 branch_instruction) // timer cannot advance again until the bus receives AdvanceCycles at the slice boundary, making // every remaining iteration in this slice observationally redundant. const u32 condition = branch_instruction >> 28; - if ((branch_instruction & 0x0e000000) != 0x0a000000 || condition >= 0xe || - !m_pc_written || HasUnmaskedInterrupt()) + if ((branch_instruction & 0x0e000000) != 0x0a000000 || condition >= 0xe || !m_pc_written || + HasUnmaskedInterrupt()) { return false; } @@ -864,15 +864,6 @@ size_t ARMCore::GetJitCompiledBlockCount() const #endif } -u64 ARMCore::GetJitBlockExecutionCount() const -{ -#if defined(_M_X86_64) - return m_jit ? m_jit->GetBlockExecutionCount() : 0; -#else - return 0; -#endif -} - u64 ARMCore::GetJitAddressTranslationCount() const { #if defined(_M_X86_64) @@ -1054,8 +1045,7 @@ u64 ARMCore::TakeJitFallbackIntervalCount() return std::exchange(m_jit_fallback_interval_count, 0); } -std::vector -ARMCore::TakeHotJitFallbackIntervalSamples(size_t maximum_count) +std::vector ARMCore::TakeHotJitFallbackIntervalSamples(size_t maximum_count) { std::vector> sorted(m_jit_fallback_interval_samples.begin(), m_jit_fallback_interval_samples.end()); @@ -2322,11 +2312,9 @@ void ARMCore::WriteCP15(u32 opcode1, u32 crn, u32 crm, u32 opcode2, u32 value) break; case 2: m_cp15.translation_table_base = value; - InvalidateTLB(); break; case 3: m_cp15.domain_access_control = value; - InvalidateTLB(); break; case 5: m_cp15.fault_status = value; @@ -2346,8 +2334,12 @@ void ARMCore::WriteCP15(u32 opcode1, u32 crn, u32 crm, u32 opcode2, u32 value) InvalidateTLB(); break; case 13: + // ARM926 FCSE is specifically designed to switch low-address process spaces without flushing + // either cache or TLB: their tags contain the modified virtual address (MVA), which already + // includes PID[31:25]. IOS writes c13 at virtually every process switch. Retaining those MVA + // translations is both the architectural behavior and essential to avoiding a page-table walk + // storm. Explicit c8 maintenance above remains the sole guest-visible TLB invalidation path. m_cp15.process_id = value; - InvalidateTLB(); break; default: break; diff --git a/Source/Core/Core/IOS/Starlet/ARMCore.h b/Source/Core/Core/IOS/Starlet/ARMCore.h index d6d16daf86..da402fa516 100644 --- a/Source/Core/Core/IOS/Starlet/ARMCore.h +++ b/Source/Core/Core/IOS/Starlet/ARMCore.h @@ -164,7 +164,6 @@ public: u64 GetJitExecutedInstructions() const; u64 GetJitNativeExecutedInstructions() const; size_t GetJitCompiledBlockCount() const; - u64 GetJitBlockExecutionCount() const; u64 GetJitAddressTranslationCount() const; u64 GetJitSlowReadCount() const; u64 GetJitSlowWriteCount() const; @@ -222,6 +221,14 @@ private: static constexpr size_t TLB_ENTRY_COUNT = 4096; static constexpr size_t INSTRUCTION_CACHE_ENTRY_COUNT = 16384; + static constexpr size_t GetTLBIndex(u32 virtual_page) + { + // FCSE places its seven-bit process identifier above bit 24 of the modified virtual address. + // Fold those bits into the direct-mapped software cache index so translations belonging to + // different IOS processes do not evict one another on every fast context switch. + return (virtual_page ^ (virtual_page >> 12)) & (TLB_ENTRY_COUNT - 1); + } + static bool IsValidMode(u32 mode); static s32 SignExtend(u32 value, unsigned bits); static AddResult AddWithCarry(u32 lhs, u32 rhs, bool carry); diff --git a/Source/Core/Core/IOS/Starlet/ARMJitX64.cpp b/Source/Core/Core/IOS/Starlet/ARMJitX64.cpp index 4f2bdc20a8..e068d12885 100644 --- a/Source/Core/Core/IOS/Starlet/ARMJitX64.cpp +++ b/Source/Core/Core/IOS/Starlet/ARMJitX64.cpp @@ -75,6 +75,7 @@ ARMJitX64::ARMJitX64(ARMCore& core) : m_core(core) static_cast(reinterpret_cast(&m_core.m_yield_requested) - base); m_executed_instructions_offset = static_cast(reinterpret_cast(&m_core.m_executed_instructions) - base); + m_control_offset = static_cast(reinterpret_cast(&m_core.m_cp15.control) - base); m_process_id_offset = static_cast(reinterpret_cast(&m_core.m_cp15.process_id) - base); m_tlb_generation_offset = @@ -115,9 +116,14 @@ void ARMJitX64::InvalidateTranslationContext() { // A TLB invalidation changes which physical page a virtual PC resolves to, but it does not // invalidate the ARM instruction cache. Keep already translated physical code and force the - // generated dispatcher to resolve the next virtual PC through the current page tables. This is - // particularly important for IOS, which flushes its TLB on every process switch. - std::ranges::fill(m_fast_entries, FastEntry{}); + // generated dispatcher to resolve the next virtual PC through the current page tables. Every + // fast entry carries the ARMCore TLB generation, so the common invalidation is O(1). This is + // critical for IOS, which flushes its TLB on virtually every process switch; clearing the whole + // 65,536-entry array here previously consumed most of the host CPU. Generation zero is skipped by + // ARMCore. If the 32-bit counter eventually wraps back to one, clear ancient generation-one + // entries once to prevent an alias after the wrap. + if (m_core.m_tlb_generation == 1) + std::ranges::fill(m_fast_entries, FastEntry{}); } u32 ARMJitX64::Run(u64 cycle_budget) @@ -162,18 +168,39 @@ void ARMJitX64::GenerateDispatcher() CMP(8, MatR(RAX), Imm8(0)); FixupBranch invalidated = J_CC(CC_NE, Jump::Near); - // Direct-mapped native block cache. The key includes CPSR.T in bit zero, matching the C++ map. + // Direct-mapped native block cache. The key includes CPSR.T in bit zero and uses the ARM926 + // modified virtual address (MVA) for low FCSE addresses. Different IOS process identifiers can + // therefore retain independent hot entries without invalidating the cache on every c13 write. MOV(32, R(EAX), MRegister(15)); MOV(32, R(ECX), MCPSR()); SHR(32, R(ECX), Imm8(5)); AND(32, R(ECX), Imm8(1)); OR(32, R(EAX), R(ECX)); + MOV(32, R(ECX), MDisp(JIT_CORE, m_control_offset)); + TEST(32, R(ECX), Imm32(1)); + FixupBranch mmu_disabled = J_CC(CC_Z, Jump::Near); + CMP(32, R(EAX), Imm32(0x02000000)); + FixupBranch outside_fcse = J_CC(CC_AE, Jump::Near); + MOV(32, R(ECX), MDisp(JIT_CORE, m_process_id_offset)); + AND(32, R(ECX), Imm32(0xfe000000)); + OR(32, R(EAX), R(ECX)); + SetJumpTarget(mmu_disabled); + SetJumpTarget(outside_fcse); + + // Fold the FCSE PID bits down into the cache index. A plain low-bit mask makes every process + // collide because PID occupies MVA[31:25]. The full MVA remains in FastEntry::key for safety. MOV(32, R(EDX), R(EAX)); SHR(32, R(EDX), Imm8(1)); + MOV(32, R(ECX), R(EAX)); + SHR(32, R(ECX), Imm8(17)); + XOR(32, R(EDX), R(ECX)); AND(32, R(EDX), Imm32(static_cast(FAST_ENTRY_COUNT - 1))); SHL(64, R(RDX), Imm8(4)); MOV(64, R(R11), ImmPtr(m_fast_entries.data())); ADD(64, R(R11), R(RDX)); + MOV(32, R(R8), MDisp(JIT_CORE, m_tlb_generation_offset)); + CMP(32, MDisp(R11, static_cast(offsetof(FastEntry, tlb_generation))), R(R8)); + FixupBranch stale_generation = J_CC(CC_NE); CMP(32, MDisp(R11, static_cast(offsetof(FastEntry, key))), R(EAX)); FixupBranch cache_miss = J_CC(CC_NE); MOV(64, R(R11), MDisp(R11, static_cast(offsetof(FastEntry, entry)))); @@ -181,6 +208,7 @@ void ARMJitX64::GenerateDispatcher() FixupBranch empty_entry = J_CC(CC_Z); JMPptr(R(R11)); + SetJumpTarget(stale_generation); SetJumpTarget(cache_miss); SetJumpTarget(empty_entry); MOV(64, R(ABI_PARAM1), ImmPtr(this)); @@ -208,6 +236,20 @@ void ARMJitX64::GenerateDispatcher() m_block_code_begin = AlignCode16(); } +u32 ARMJitX64::MakeFastEntryKey(u32 address, u32 control, u32 process_id) +{ + constexpr u32 CP15_CONTROL_MMU = 1U << 0; + const u32 virtual_address = address & ~1U; + if ((control & CP15_CONTROL_MMU) != 0 && virtual_address < 0x02000000) + return address | (process_id & 0xfe000000); + return address; +} + +size_t ARMJitX64::GetFastEntryIndex(u32 key) +{ + return ((key >> 1) ^ (key >> 17)) & (FAST_ENTRY_COUNT - 1); +} + const u8* ARMJitX64::Dispatch(ARMJitX64* jit) { const bool thumb = (jit->m_core.m_cpsr & ARMCore::CPSR_T) != 0; @@ -216,8 +258,11 @@ const u8* ARMJitX64::Dispatch(ARMJitX64* jit) if (!block || !block->runnable) return nullptr; - FastEntry& fast_entry = jit->m_fast_entries[(key >> 1) & (FAST_ENTRY_COUNT - 1)]; - fast_entry.key = key; + const u32 fast_key = + MakeFastEntryKey(key, jit->m_core.m_cp15.control, jit->m_core.m_cp15.process_id); + FastEntry& fast_entry = jit->m_fast_entries[GetFastEntryIndex(fast_key)]; + fast_entry.key = fast_key; + fast_entry.tlb_generation = jit->m_core.m_tlb_generation; fast_entry.entry = block->entry; return block->entry; } @@ -250,24 +295,113 @@ ARMJitX64::Block ARMJitX64::CompileBlock(u32 address, bool thumb) const u8* const body = AlignCode16(); LoadRegisterCache(); MOV(8, MPCWritten(), Imm8(0)); + const u8* const loop_body = GetCodePtr(); u32 current_address = address; + u32 last_instruction_address = address; u32 instruction_count = 0; u32 native_instruction_count = 0; bool terminated = false; bool dispatcher_exit = false; + bool block_exit_emitted = false; // The physical address in the cache key covers the first ARM TLB granule. End the block before // a 1 KiB boundary so a remapping of the following page can never reuse stale translated code. const u32 translation_granule = address & ~0x3ffU; + const auto emit_loop_back = [&](u32 target, u32 fallthrough, u32 branch_address, u32 condition) { + // Account for one complete guest loop before deciding whether the remaining slice can execute + // another. Conditional back edges use the same accounting on both the taken and fallthrough + // paths. This keeps small scheduler budgets exact while avoiding a dispatcher/register-cache + // round trip on every taken iteration of a native-only loop. + if (condition != 0xe) + EmitConditionResult(condition); + SUB(32, R(JIT_DOWNCOUNT), Imm32(instruction_count)); + ADD(32, R(JIT_EXECUTED_COUNT), Imm32(instruction_count)); + ADD(32, R(JIT_NATIVE_COUNT), Imm32(native_instruction_count)); + + FixupBranch not_taken; + if (condition != 0xe) + { + TEST(32, R(EAX), R(EAX)); + not_taken = J_CC(CC_Z, Jump::Near); + } + + CMP(32, R(JIT_DOWNCOUNT), Imm32(instruction_count)); + FixupBranch leave_budget = J_CC(CC_B, Jump::Near); + CMP(8, MYieldRequested(), Imm8(0)); + FixupBranch leave_yield = J_CC(CC_NE, Jump::Near); + CMP(8, MWaitingForInterrupt(), Imm8(0)); + FixupBranch leave_interrupt = J_CC(CC_NE, Jump::Near); + CMP(8, MWaitingForMemoryPoll(), Imm8(0)); + FixupBranch leave_poll = J_CC(CC_NE, Jump::Near); + MOV(64, R(RAX), ImmPtr(&m_clear_pending)); + CMP(8, MatR(RAX), Imm8(0)); + FixupBranch leave_invalidation = J_CC(CC_NE, Jump::Near); + JMP(loop_body); + + SetJumpTarget(leave_budget); + SetJumpTarget(leave_yield); + SetJumpTarget(leave_interrupt); + SetJumpTarget(leave_poll); + SetJumpTarget(leave_invalidation); + MOV(32, MRegister(15), Imm32(target)); + FixupBranch state_ready; + if (condition != 0xe) + { + state_ready = J(Jump::Near); + SetJumpTarget(not_taken); + MOV(32, MRegister(15), Imm32(fallthrough)); + SetJumpTarget(state_ready); + } + MOV(32, MInstructionAddress(), Imm32(branch_address)); + MOV(8, MPCWritten(), Imm8(1)); + FlushRegisterCache(); + JMP(m_dispatcher); + terminated = true; + block_exit_emitted = true; + }; while (instruction_count < MAX_BLOCK_INSTRUCTIONS && (current_address & ~0x3ffU) == translation_granule) { + last_instruction_address = current_address; ++instruction_count; m_compile_instruction_count = instruction_count; m_compile_native_instruction_count = native_instruction_count; if (thumb) { const u16 instruction = m_core.FetchThumbInstruction(current_address); + // A branch back to the block start uses a real host loop with an exact per-iteration budget + // check. Other branches keep their ordinary terminal block semantics. + if ((instruction & 0xf800) == 0xe000) + { + const s32 offset = ARMCore::SignExtend(instruction & 0x07ff, 11) * 2; + const u32 target = current_address + 4 + static_cast(offset); + if (target == address) + { + ++native_instruction_count; + emit_loop_back(target, current_address + 2, current_address, 0xe); + break; + } + } + else if ((instruction & 0xf000) == 0xd000) + { + const u32 condition = (instruction >> 8) & 0xf; + const s32 offset = ARMCore::SignExtend(instruction & 0xff, 8) * 2; + const u32 target = current_address + 4 + static_cast(offset); + bool canonical_memory_poll = false; + if (condition < 0xe && target == address && target + 4 == current_address) + { + const u16 load = m_core.FetchThumbInstruction(target); + const u16 compare = m_core.FetchThumbInstruction(target + 2); + canonical_memory_poll = (load & 0xf800) == 0x6800 && (compare & 0xf800) == 0x2800 && + (compare & 0xff) == 0 && ((compare >> 8) & 7) == (load & 7); + } + if (condition < 0xe && target == address && !canonical_memory_poll) + { + ++native_instruction_count; + emit_loop_back(target, current_address + 2, current_address, condition); + break; + } + } if (!EmitDirectThumb(instruction, current_address, &terminated)) { EmitFallbackThumb(instruction, current_address); @@ -285,6 +419,22 @@ ARMJitX64::Block ARMJitX64::CompileBlock(u32 address, bool thumb) else { const u32 instruction = m_core.FetchARMInstruction(current_address); + // Turn a branch to the block start into a bounded host loop. BL and every other branch keep + // the normal terminal path, including forward jumps whose target may be data or empty fill. + if ((instruction & 0x0e000000) == 0x0a000000 && (instruction & (1U << 24)) == 0 && + (instruction >> 28) != 0xf) + { + const u32 condition = instruction >> 28; + const s32 offset = ARMCore::SignExtend((instruction & 0x00ffffff) << 2, 26); + const u32 target = current_address + 8 + static_cast(offset); + const bool slice_stable_timer_poll = condition != 0xe && target + 16 == current_address; + if (target == address && !slice_stable_timer_poll) + { + ++native_instruction_count; + emit_loop_back(target, current_address + 4, current_address, condition); + break; + } + } if (!EmitDirectARM(instruction, current_address, &terminated, &dispatcher_exit)) { EmitFallbackARM(instruction, current_address); @@ -304,9 +454,10 @@ ARMJitX64::Block ARMJitX64::CompileBlock(u32 address, bool thumb) if (!terminated) { MOV(32, MRegister(15), Imm32(current_address)); - MOV(32, MInstructionAddress(), Imm32(current_address - (thumb ? 2 : 4))); + MOV(32, MInstructionAddress(), Imm32(last_instruction_address)); } - EmitBlockExit(instruction_count, native_instruction_count, dispatcher_exit); + if (!block_exit_emitted) + EmitBlockExit(instruction_count, native_instruction_count, dispatcher_exit); // The entry stub prevents a translated block from overrunning the scheduler's remaining ARM // budget. Keeping it separate lets us know the final block size without patching machine code. @@ -321,8 +472,7 @@ ARMJitX64::Block ARMJitX64::CompileBlock(u32 address, bool thumb) .runnable = native_instruction_count != 0}; } -bool ARMJitX64::EmitDirectARM(u32 instruction, u32 address, bool* terminal, - bool* dispatcher_exit) +bool ARMJitX64::EmitDirectARM(u32 instruction, u32 address, bool* terminal, bool* dispatcher_exit) { // MCR p15, 0, Rd, c7, c10, 4 is ARM926 Drain Write Buffer. It is an ordering barrier, not an // instruction-cache invalidation, and has no additional observable work in this single-host- @@ -461,14 +611,29 @@ bool ARMJitX64::EmitDirectARM(u32 instruction, u32 address, bool* terminal, return true; } - if ((instruction & 0x0ffffff0) == 0x012fff10 || - (instruction & 0x0ffffff0) == 0x012fff30) + if ((instruction & 0x0ffffff0) == 0x012fff10 || (instruction & 0x0ffffff0) == 0x012fff30) { - if ((instruction >> 28) != 0xe) - return false; const bool link = (instruction & 0x20) != 0; const u32 rm = instruction & 0xf; - EmitExchangeBranch(rm == 15 ? Imm32(address + 8) : MRegister(rm), link, address + 4); + if (condition == 0xf) + return false; + if (condition == 0xe) + { + EmitExchangeBranch(rm == 15 ? Imm32(address + 8) : MRegister(rm), link, address + 4); + } + else + { + // Conditional BX LR is a hot IOS scheduler return. Keep both outcomes terminal so IRQ, + // yield and translation state are sampled at the same architectural boundary as before. + EmitConditionResult(condition); + TEST(32, R(EAX), R(EAX)); + const FixupBranch not_taken = J_CC(CC_Z, Jump::Near); + EmitExchangeBranch(rm == 15 ? Imm32(address + 8) : MRegister(rm), link, address + 4); + const FixupBranch done = J(Jump::Near); + SetJumpTarget(not_taken); + MOV(32, MRegister(15), Imm32(address + 4)); + SetJumpTarget(done); + } MOV(32, MInstructionAddress(), Imm32(address)); MOV(8, MPCWritten(), Imm8(1)); *terminal = true; @@ -511,6 +676,22 @@ bool ARMJitX64::EmitDirectARM(u32 instruction, u32 address, bool* terminal, if (condition != 0xe) { + // Predication applies to the complete single-data-transfer instruction, including address + // writeback and any exact MMIO/SRAM helper. A failed predicate skips all of it and continues in + // the current block. This covers the hot LDREQ/LDRNE/STRNE forms in IOS without weakening the + // existing address and device guards. + if (condition != 0xf && (instruction & 0x0c000000) == 0x04000000 && + CanEmitARMMemory(instruction)) + { + EmitConditionResult(condition); + TEST(32, R(EAX), R(EAX)); + const FixupBranch predicate_failed = J_CC(CC_Z, Jump::Near); + const bool emitted = EmitARMMemory(instruction, address); + ASSERT(emitted); + SetJumpTarget(predicate_failed); + return true; + } + // ARM predication is pervasive in IOS's scheduler, allocator and error paths. Any data- // processing form already proven safe below can share the native predicate guard; a failed // condition simply skips the operation and continues at the next guest instruction. @@ -597,7 +778,7 @@ bool ARMJitX64::CanEmitARMDataProcessing(u32 instruction) const bool ARMJitX64::EmitARMMemory(u32 instruction, u32 address) { - if (!m_fastmem_base) + if (!CanEmitARMMemory(instruction)) return false; const bool preindex = (instruction & (1U << 24)) != 0; @@ -609,13 +790,7 @@ bool ARMJitX64::EmitARMMemory(u32 instruction, u32 address) const u32 rd = (instruction >> 12) & 0xf; const bool register_offset = (instruction & (1U << 25)) != 0; const u32 offset = instruction & 0xfff; - if (rd == 15 || (rn == 15 && (!preindex || writeback)) || (load && writeback && rn == rd)) - return false; - if (register_offset && (instruction & (1U << 4)) != 0) - return false; const u32 rm = instruction & 0xf; - if (register_offset && rm == 15) - return false; // RAM fast paths and their interpreter side exits share an explicit architectural boundary. // Materialize the two cached guest registers before either path can observe them. @@ -720,6 +895,22 @@ bool ARMJitX64::EmitARMMemory(u32 instruction, u32 address) return true; } +bool ARMJitX64::CanEmitARMMemory(u32 instruction) const +{ + if (!m_fastmem_base) + return false; + + const bool preindex = (instruction & (1U << 24)) != 0; + const bool writeback = !preindex || (instruction & (1U << 21)) != 0; + const bool load = (instruction & (1U << 20)) != 0; + const u32 rn = (instruction >> 16) & 0xf; + const u32 rd = (instruction >> 12) & 0xf; + const bool register_offset = (instruction & (1U << 25)) != 0; + const u32 rm = instruction & 0xf; + return rd != 15 && !(rn == 15 && (!preindex || writeback)) && !(load && writeback && rn == rd) && + !(register_offset && (instruction & (1U << 4)) != 0) && !(register_offset && rm == 15); +} + bool ARMJitX64::EmitARMHalfwordMemory(u32 instruction, u32 address) { if (!m_fastmem_base) @@ -735,9 +926,8 @@ bool ARMJitX64::EmitARMHalfwordMemory(u32 instruction, u32 address) const u32 type = (instruction >> 5) & 3; const u32 rm = instruction & 0xf; const u32 offset = ((instruction >> 4) & 0xf0) | (instruction & 0xf); - if (type == 0 || (!load && type != 1) || rd == 15 || - (rn == 15 && (!preindex || writeback)) || (load && writeback && rn == rd) || - (!immediate && rm == 15)) + if (type == 0 || (!load && type != 1) || rd == 15 || (rn == 15 && (!preindex || writeback)) || + (load && writeback && rn == rd) || (!immediate && rm == 15)) { return false; } @@ -903,7 +1093,24 @@ bool ARMJitX64::EmitARMBlockTransfer(u32 instruction, u32 address, bool* termina } const FixupBranch direct_done = J(Jump::Near); - EmitARMMemorySlowPath(slow_paths, instruction, address, direct_done); + for (const FixupBranch& slow_path : slow_paths) + SetJumpTarget(slow_path); + + // IOS keeps its kernel and IRQ stacks in Hollywood SRAM. Register-list transfers therefore miss + // the deliberately conservative SRAM fastmem aperture even though the instruction itself is + // already decoded. Calling the architectural transfer helper here preserves every ARMBus + // read/write (including split SRAM, boot0 protection and invalid apertures), but avoids returning + // to the generic ARM decoder and native dispatcher for each kernel push/pop. Transfers that load + // PC remain terminal through the encoding-derived flag above; ordinary stack transfers can keep + // executing the translated block. + FlushRegisterCache(); + MOV(64, R(ABI_PARAM1), ImmPtr(this)); + MOV(32, R(ABI_PARAM2), Imm32(instruction)); + MOV(32, R(ABI_PARAM3), Imm32(address)); + ABI_CallFunction(ExecuteUserBankBlockTransfer); + + SetJumpTarget(direct_done); + LoadRegisterCache(); return true; } @@ -1212,8 +1419,8 @@ bool ARMJitX64::EmitDirectThumb(u16 instruction, u32 address, bool* terminal) { const u16 load = m_core.FetchThumbInstruction(target); const u16 compare = m_core.FetchThumbInstruction(target + 2); - if ((load & 0xf800) == 0x6800 && (compare & 0xf800) == 0x2800 && - (compare & 0xff) == 0 && ((compare >> 8) & 7) == (load & 7)) + if ((load & 0xf800) == 0x6800 && (compare & 0xf800) == 0x2800 && (compare & 0xff) == 0 && + ((compare >> 8) & 7) == (load & 7)) { return false; } @@ -1515,8 +1722,8 @@ bool ARMJitX64::EmitThumbMemory(u16 instruction, u32 address) } void ARMJitX64::EmitFastmemAddress(std::vector* slow_paths, u32 access_size, - u32 range_size, SRAMFastmemAccess sram_access, - bool arm_unaligned_word) + u32 range_size, SRAMFastmemAccess sram_access, + bool arm_unaligned_word) { if (range_size == 0) range_size = access_size; @@ -1533,6 +1740,8 @@ void ARMJitX64::EmitFastmemAddress(std::vector* slow_paths, u32 acc MOV(32, R(EDX), R(EAX)); SHR(32, R(EDX), Imm8(10)); MOV(32, R(ECX), R(EDX)); + SHR(32, R(ECX), Imm8(12)); + XOR(32, R(ECX), R(EDX)); AND(32, R(ECX), Imm32(static_cast(ARMCore::TLB_ENTRY_COUNT - 1))); IMUL(32, ECX, R(ECX), Imm32(sizeof(ARMCore::TLBEntry))); MOV(64, R(R11), ImmPtr(m_core.m_tlb.data())); @@ -2086,11 +2295,6 @@ void ARMJitX64::EmitBlockExit(u32 instruction_count, u32 native_instruction_coun SUB(32, R(JIT_DOWNCOUNT), Imm32(instruction_count)); ADD(32, R(JIT_EXECUTED_COUNT), Imm32(instruction_count)); ADD(32, R(JIT_NATIVE_COUNT), Imm32(native_instruction_count)); - // Diagnostic only: one generated increment per completed native block lets the live profiler - // distinguish short-block dispatch overhead from slow bus helpers. The dispatcher overwrites - // the host flags before observing them, so this does not affect guest execution. - MOV(64, R(RAX), ImmPtr(&m_block_execution_count)); - ADD(64, MatR(RAX), Imm8(1)); JMP(dispatcher_exit ? m_dispatcher_exit : m_dispatcher); } @@ -2248,7 +2452,7 @@ u32 ARMJitX64::ReadMemorySlow(ARMJitX64* jit, u32 physical_address, u32 access_s case SlowMemoryRegion::SRAM: ++jit->m_slow_sram_read_count; ++(physical_address < 0xfff00000 ? jit->m_slow_sram_low_access_count : - jit->m_slow_sram_high_access_count); + jit->m_slow_sram_high_access_count); ++jit->m_slow_sram_page_read_access_count[(physical_address & 0x1ffff) >> 12]; break; case SlowMemoryRegion::MMIO: @@ -2279,7 +2483,7 @@ void ARMJitX64::WriteMemorySlow(ARMJitX64* jit, u32 physical_address, u32 access case SlowMemoryRegion::SRAM: ++jit->m_slow_sram_write_count; ++(physical_address < 0xfff00000 ? jit->m_slow_sram_low_access_count : - jit->m_slow_sram_high_access_count); + jit->m_slow_sram_high_access_count); ++jit->m_slow_sram_page_write_access_count[(physical_address & 0x1ffff) >> 12]; break; case SlowMemoryRegion::MMIO: @@ -2320,8 +2524,7 @@ void ARMJitX64::EnterSVCARM(ARMJitX64* jit, u32 address) ARMCore& core = jit->m_core; core.m_instruction_address = address; core.m_pc_written = false; - core.EnterException(ARMCore::Mode::Supervisor, STARLET_EXCEPTION_VECTOR_BASE + 0x08, - address + 4); + core.EnterException(ARMCore::Mode::Supervisor, STARLET_EXCEPTION_VECTOR_BASE + 0x08, address + 4); } void ARMJitX64::EnterSVCThumb(ARMJitX64* jit, u32 address) @@ -2329,8 +2532,7 @@ void ARMJitX64::EnterSVCThumb(ARMJitX64* jit, u32 address) ARMCore& core = jit->m_core; core.m_instruction_address = address; core.m_pc_written = false; - core.EnterException(ARMCore::Mode::Supervisor, STARLET_EXCEPTION_VECTOR_BASE + 0x08, - address + 2); + core.EnterException(ARMCore::Mode::Supervisor, STARLET_EXCEPTION_VECTOR_BASE + 0x08, address + 2); } } // namespace IOS::LLE diff --git a/Source/Core/Core/IOS/Starlet/ARMJitX64.h b/Source/Core/Core/IOS/Starlet/ARMJitX64.h index edd1dfb7a7..5d4de4064d 100644 --- a/Source/Core/Core/IOS/Starlet/ARMJitX64.h +++ b/Source/Core/Core/IOS/Starlet/ARMJitX64.h @@ -35,23 +35,13 @@ public: u64 GetExecutedInstructions() const { return m_executed_instructions; } u64 GetNativeExecutedInstructions() const { return m_native_executed_instructions; } size_t GetCompiledBlockCount() const { return m_blocks.size(); } - u64 GetBlockExecutionCount() const { return m_block_execution_count; } u64 GetAddressTranslationCount() const { return m_address_translation_count; } u64 GetSlowReadCount() const { return m_slow_read_count; } u64 GetSlowWriteCount() const { return m_slow_write_count; } u64 GetSlowRAMAccessCount() const { return m_slow_ram_read_count + m_slow_ram_write_count; } - u64 GetSlowSRAMAccessCount() const - { - return m_slow_sram_read_count + m_slow_sram_write_count; - } - u64 GetSlowMMIOAccessCount() const - { - return m_slow_mmio_read_count + m_slow_mmio_write_count; - } - u64 GetSlowOtherAccessCount() const - { - return m_slow_other_read_count + m_slow_other_write_count; - } + u64 GetSlowSRAMAccessCount() const { return m_slow_sram_read_count + m_slow_sram_write_count; } + u64 GetSlowMMIOAccessCount() const { return m_slow_mmio_read_count + m_slow_mmio_write_count; } + u64 GetSlowOtherAccessCount() const { return m_slow_other_read_count + m_slow_other_write_count; } u64 GetSlowSRAMLowAccessCount() const { return m_slow_sram_low_access_count; } u64 GetSlowSRAMHighAccessCount() const { return m_slow_sram_high_access_count; } u64 GetSlowSRAMReadAccessCount() const { return m_slow_sram_read_count; } @@ -94,17 +84,20 @@ private: struct FastEntry { u32 key = 0xffffffff; - u32 padding = 0; + u32 tlb_generation = 0; const u8* entry = nullptr; }; void PoisonMemory() override; void GenerateDispatcher(); + static u32 MakeFastEntryKey(u32 address, u32 control, u32 process_id); + static size_t GetFastEntryIndex(u32 key); static const u8* Dispatch(ARMJitX64* jit); Block* GetOrCompileBlock(u32 address); Block CompileBlock(u32 address, bool thumb); bool EmitDirectARM(u32 instruction, u32 address, bool* terminal, bool* dispatcher_exit); bool CanEmitARMDataProcessing(u32 instruction) const; + bool CanEmitARMMemory(u32 instruction) const; bool EmitARMMemory(u32 instruction, u32 address); bool EmitARMHalfwordMemory(u32 instruction, u32 address); bool EmitARMBlockTransfer(u32 instruction, u32 address, bool* terminal); @@ -152,8 +145,7 @@ private: static void ExecuteUserBankBlockTransfer(ARMJitX64* jit, u32 instruction, u32 address); static void ExecuteThumbPushPop(ARMJitX64* jit, u16 instruction, u32 address); static void ExceptionReturn(ARMJitX64* jit, u32 target); - static u32 ReadMemorySlow(ARMJitX64* jit, u32 physical_address, u32 access_size, - u32 byte_offset); + static u32 ReadMemorySlow(ARMJitX64* jit, u32 physical_address, u32 access_size, u32 byte_offset); static void WriteMemorySlow(ARMJitX64* jit, u32 physical_address, u32 access_size, u32 value); static u32 TranslateAddress(ARMJitX64* jit, u32 address); static void EnterUndefinedARM(ARMJitX64* jit, u32 instruction, u32 address); @@ -177,7 +169,6 @@ private: u8* m_block_code_begin = nullptr; u64 m_executed_instructions = 0; u64 m_native_executed_instructions = 0; - u64 m_block_execution_count = 0; u64 m_address_translation_count = 0; u64 m_slow_read_count = 0; u64 m_slow_write_count = 0; @@ -203,6 +194,7 @@ private: s32 m_waiting_for_memory_poll_offset = 0; s32 m_yield_requested_offset = 0; s32 m_executed_instructions_offset = 0; + s32 m_control_offset = 0; s32 m_process_id_offset = 0; s32 m_tlb_generation_offset = 0; u32 m_compile_instruction_count = 0; diff --git a/Source/Core/Core/IOS/Starlet/Starlet.cpp b/Source/Core/Core/IOS/Starlet/Starlet.cpp index 579bb95eb7..5ba431268b 100644 --- a/Source/Core/Core/IOS/Starlet/Starlet.cpp +++ b/Source/Core/Core/IOS/Starlet/Starlet.cpp @@ -52,6 +52,7 @@ bool Starlet::Init(const std::string& dump_directory, std::string* error) m_core->Reset(StarletMemory::BOOT_ROM_BASE); m_next_jit_diagnostic_instruction = 25'000'000; m_next_realtime_diagnostic_ms = Common::Timer::NowMs() + 1000; + m_realtime_diagnostic_slice_count = 0; m_last_realtime_ppc_pc = 0; m_realtime_ppc_pc_streak = 0; m_ppc_context_logged = false; @@ -196,8 +197,13 @@ void Starlet::RunSlice(s64 cycles_late) (ipc_handshake_active ? ARM_IPC_SLICE_CYCLES : ARM_ACTIVE_SLICE_CYCLES); m_core->RunCycles(arm_cycles); - const u64 now_ms = Common::Timer::NowMs(); - if (now_ms >= m_next_realtime_diagnostic_ms) + // Keep host-clock reads and diagnostic formatting off the normal scheduler path. + const bool check_realtime_diagnostic = + ++m_realtime_diagnostic_slice_count == REALTIME_DIAGNOSTIC_SLICE_INTERVAL; + if (check_realtime_diagnostic) + m_realtime_diagnostic_slice_count = 0; + const u64 now_ms = check_realtime_diagnostic ? Common::Timer::NowMs() : 0; + if (check_realtime_diagnostic && now_ms >= m_next_realtime_diagnostic_ms) { m_next_realtime_diagnostic_ms = now_ms + 1000; const PowerPC::PowerPCState& ppc = m_system.GetPPCState(); @@ -211,8 +217,7 @@ void Starlet::RunSlice(s64 cycles_late) "ppc-ctrl={:#04x} ppc-irq={:#010x}/{:#010x} arm-ctrl={:#04x} " "arm-irq={:#010x}/{:#010x} Starlet-PC={:#010x} wait-int={} wait-mem={}", ppc.pc, ppc.npc, instruction.valid, instruction.hex, ppc.msr.Hex, ppc.Exceptions, - ppc.downcount, ppc.gpr[1], LR(ppc), CTR(ppc), - pi.GetCause(), pi.GetMask(), + ppc.downcount, ppc.gpr[1], LR(ppc), CTR(ppc), pi.GetCause(), pi.GetMask(), m_system.GetWiiIPC().ReadStarletRegister(0x04), m_system.GetWiiIPC().ReadStarletRegister(0x30), m_system.GetWiiIPC().ReadStarletRegister(0x34), ipc_arm_ctrl, @@ -238,9 +243,9 @@ void Starlet::RunSlice(s64 cycles_late) pi.GetMask()); for (u32 base = 0; base < 32; base += 4) { - INFO_LOG_FMT(IOS, "Broadway GPR r{}={:#010x} r{}={:#010x} r{}={:#010x} r{}={:#010x}", - base, ppc.gpr[base], base + 1, ppc.gpr[base + 1], base + 2, - ppc.gpr[base + 2], base + 3, ppc.gpr[base + 3]); + INFO_LOG_FMT(IOS, "Broadway GPR r{}={:#010x} r{}={:#010x} r{}={:#010x} r{}={:#010x}", base, + ppc.gpr[base], base + 1, ppc.gpr[base + 1], base + 2, ppc.gpr[base + 2], + base + 3, ppc.gpr[base + 3]); } const u32 context_start = (ppc.pc - 0x40) & ~u32{3}; const u32 context_end = (ppc.pc + 0x80) & ~u32{3}; @@ -273,26 +278,22 @@ void Starlet::RunSlice(s64 cycles_late) { const u64 jit_executed = m_core->GetJitExecutedInstructions(); const u64 jit_native = m_core->GetJitNativeExecutedInstructions(); - const u64 block_executions = m_core->GetJitBlockExecutionCount(); const u64 interval_fallbacks = m_core->TakeJitFallbackIntervalCount(); INFO_LOG_FMT(IOS, - "Starlet live JIT: instructions={} native={:.1f}% fallbacks={} blocks={} " - "block-runs={} insns-per-block={:.1f} translations={} slow-reads={} " - "slow-writes={} slow-regions=ram:{}/sram:{}/mmio:{}/other:{} " - "interval-fallbacks={} idle-polls={} ipc-armctrl={:#04x} " - "PC={:#010x} " - "CPSR={:#010x} irqflag={:#010x} irqmask={:#010x} fiqmask={:#010x}", - jit_executed, jit_executed == 0 ? 0.0 : 100.0 * jit_native / jit_executed, - m_core->GetJitFallbackInstructionCount(), m_core->GetJitCompiledBlockCount(), - block_executions, - block_executions == 0 ? 0.0 : static_cast(jit_executed) / block_executions, - m_core->GetJitAddressTranslationCount(), m_core->GetJitSlowReadCount(), - m_core->GetJitSlowWriteCount(), - m_core->GetJitSlowRAMAccessCount(), m_core->GetJitSlowSRAMAccessCount(), - m_core->GetJitSlowMMIOAccessCount(), m_core->GetJitSlowOtherAccessCount(), - interval_fallbacks, m_core->GetMemoryPollEntryCount(), ipc_arm_ctrl, - m_core->GetRegister(15), m_core->GetCPSR(), - m_system.GetWiiIPC().ReadStarletRegister(0x38), + "Starlet live JIT: instructions={} native={:.1f}% fallbacks={} blocks={} " + "translations={} slow-reads={} " + "slow-writes={} slow-regions=ram:{}/sram:{}/mmio:{}/other:{} " + "interval-fallbacks={} idle-polls={} ipc-armctrl={:#04x} " + "PC={:#010x} " + "CPSR={:#010x} irqflag={:#010x} irqmask={:#010x} fiqmask={:#010x}", + jit_executed, jit_executed == 0 ? 0.0 : 100.0 * jit_native / jit_executed, + m_core->GetJitFallbackInstructionCount(), m_core->GetJitCompiledBlockCount(), + m_core->GetJitAddressTranslationCount(), m_core->GetJitSlowReadCount(), + m_core->GetJitSlowWriteCount(), m_core->GetJitSlowRAMAccessCount(), + m_core->GetJitSlowSRAMAccessCount(), m_core->GetJitSlowMMIOAccessCount(), + m_core->GetJitSlowOtherAccessCount(), interval_fallbacks, + m_core->GetMemoryPollEntryCount(), ipc_arm_ctrl, m_core->GetRegister(15), + m_core->GetCPSR(), m_system.GetWiiIPC().ReadStarletRegister(0x38), m_system.GetWiiIPC().ReadStarletRegister(0x3c), m_system.GetWiiIPC().ReadStarletRegister(0x40)); std::array hot_sram_read_pages{}; @@ -318,21 +319,18 @@ void Starlet::RunSlice(s64 cycles_late) { rank_sram_page(page, m_core->GetJitSlowSRAMPageReadAccessCount(page), hot_sram_read_pages, hot_sram_read_page_counts); - rank_sram_page(page, m_core->GetJitSlowSRAMPageWriteAccessCount(page), - hot_sram_write_pages, hot_sram_write_page_counts); + rank_sram_page(page, m_core->GetJitSlowSRAMPageWriteAccessCount(page), hot_sram_write_pages, + hot_sram_write_page_counts); } INFO_LOG_FMT(IOS, "Starlet slow SRAM aliases=low:{}/high:{} reads:{}/writes:{} " "hot-reads={:#04x}:{}, {:#04x}:{}, {:#04x}:{}, {:#04x}:{} " "hot-writes={:#04x}:{}, {:#04x}:{}, {:#04x}:{}, {:#04x}:{}", - m_core->GetJitSlowSRAMLowAccessCount(), - m_core->GetJitSlowSRAMHighAccessCount(), - m_core->GetJitSlowSRAMReadAccessCount(), - m_core->GetJitSlowSRAMWriteAccessCount(), hot_sram_read_pages[0], - hot_sram_read_page_counts[0], hot_sram_read_pages[1], - hot_sram_read_page_counts[1], hot_sram_read_pages[2], - hot_sram_read_page_counts[2], hot_sram_read_pages[3], - hot_sram_read_page_counts[3], hot_sram_write_pages[0], + m_core->GetJitSlowSRAMLowAccessCount(), m_core->GetJitSlowSRAMHighAccessCount(), + m_core->GetJitSlowSRAMReadAccessCount(), m_core->GetJitSlowSRAMWriteAccessCount(), + hot_sram_read_pages[0], hot_sram_read_page_counts[0], hot_sram_read_pages[1], + hot_sram_read_page_counts[1], hot_sram_read_pages[2], hot_sram_read_page_counts[2], + hot_sram_read_pages[3], hot_sram_read_page_counts[3], hot_sram_write_pages[0], hot_sram_write_page_counts[0], hot_sram_write_pages[1], hot_sram_write_page_counts[1], hot_sram_write_pages[2], hot_sram_write_page_counts[2], hot_sram_write_pages[3], @@ -356,11 +354,9 @@ void Starlet::RunSlice(s64 cycles_late) } m_jit_context_logged = true; } - for (const ARMCore::HotPCSample& sample : - m_core->TakeHotJitFallbackIntervalSamples(6)) + for (const ARMCore::HotPCSample& sample : m_core->TakeHotJitFallbackIntervalSamples(6)) { - INFO_LOG_FMT(IOS, - "Starlet interval fallback PC {:#010x} {} instruction={:#010x} samples={}", + INFO_LOG_FMT(IOS, "Starlet interval fallback PC {:#010x} {} instruction={:#010x} samples={}", sample.address, sample.thumb ? "Thumb" : "ARM", sample.instruction, sample.samples); } diff --git a/Source/Core/Core/IOS/Starlet/Starlet.h b/Source/Core/Core/IOS/Starlet/Starlet.h index d24defd6f0..289606ca3c 100644 --- a/Source/Core/Core/IOS/Starlet/Starlet.h +++ b/Source/Core/Core/IOS/Starlet/Starlet.h @@ -71,6 +71,9 @@ private: // idle quantum avoids hundreds of thousands of scheduler callbacks per second // while bounding interrupt wake latency. static constexpr u64 ARM_IDLE_SLICE_CYCLES = ARM_CLOCK / 10000; + // Do not read the host clock on every short scheduler slice: that would perturb the exact + // workload being measured. One check per 1024 callbacks keeps profiling overhead negligible. + static constexpr u64 REALTIME_DIAGNOSTIC_SLICE_INTERVAL = 1024; static void RunCallback(Core::System& system, u64 userdata, s64 cycles_late); void RunSlice(s64 cycles_late); @@ -81,6 +84,7 @@ private: CoreTiming::EventType* m_run_event = nullptr; u64 m_next_jit_diagnostic_instruction = 25'000'000; u64 m_next_realtime_diagnostic_ms = 0; + u64 m_realtime_diagnostic_slice_count = 0; u32 m_last_realtime_ppc_pc = 0; u32 m_realtime_ppc_pc_streak = 0; bool m_ppc_context_logged = false; diff --git a/Source/Core/Core/IOS/Starlet/StarletMemory.cpp b/Source/Core/Core/IOS/Starlet/StarletMemory.cpp index a82f4e889d..c45ec865b9 100644 --- a/Source/Core/Core/IOS/Starlet/StarletMemory.cpp +++ b/Source/Core/Core/IOS/Starlet/StarletMemory.cpp @@ -572,6 +572,7 @@ void StarletMemory::InitSDCard() void StarletMemory::Reset() { + const std::lock_guard wifi_sdio_lock(m_wifi_sdio_register_mutex); for (auto& wiimote : m_wiimotes) { if (wiimote) @@ -580,6 +581,7 @@ void StarletMemory::Reset() } m_sram.fill(0); m_registers.clear(); + ClearRegisterCache(); m_nand_overlay.clear(); m_nand_control_before_write = 0; ResetNANDOperationState(); @@ -659,8 +661,11 @@ void StarletMemory::Reset() void StarletMemory::DoState(PointerWrap& p) { + const std::lock_guard wifi_sdio_lock(m_wifi_sdio_register_mutex); p.DoArray(m_sram); p.Do(m_registers); + if (p.IsReadMode()) + ClearRegisterCache(); p.Do(m_nand_overlay); p.DoArray(m_nand_program_data); p.Do(m_nand_control_before_write); @@ -908,8 +913,18 @@ u8 StarletMemory::ReadMapped8(u32 address) const const u32 offset = GetSRAMOffset(address); return offset == INVALID_SRAM_OFFSET ? 0 : m_sram[offset]; } + auto& cache_entry = m_register_cache[GetRegisterCacheIndex(address)]; + const u64 cached = cache_entry.load(std::memory_order_relaxed); + if ((cached & REGISTER_CACHE_VALID) != 0 && + static_cast((cached >> 8) & 0xffffffff) == address) + { + return static_cast(cached); + } + const auto it = m_registers.find(address); - return it == m_registers.end() ? 0 : it->second; + const u8 value = it == m_registers.end() ? 0 : it->second; + cache_entry.store(EncodeRegisterCacheEntry(address, value), std::memory_order_relaxed); + return value; } void StarletMemory::WriteMapped8(u32 address, u8 value) @@ -927,6 +942,14 @@ void StarletMemory::WriteMapped8(u32 address, u8 value) return; } m_registers[address] = value; + m_register_cache[GetRegisterCacheIndex(address)].store(EncodeRegisterCacheEntry(address, value), + std::memory_order_relaxed); +} + +void StarletMemory::ClearRegisterCache() +{ + for (auto& entry : m_register_cache) + entry.store(0, std::memory_order_relaxed); } u32 StarletMemory::ReadRegister(u32 address) const @@ -2387,6 +2410,7 @@ void StarletMemory::HandleWiFiSDIOWrite(u32 address) u8 StarletMemory::ReadWiFiSDIOByte(u32 function, u32 address) const { + const std::lock_guard lock(m_wifi_sdio_register_mutex); if (function == 0) { // The three CIS pointers are little-endian 24-bit addresses in the common @@ -2531,6 +2555,7 @@ u8 StarletMemory::ReadWiFiSDIOByte(u32 function, u32 address) const void StarletMemory::WriteWiFiSDIOByte(u32 function, u32 address, u8 value) { + const std::lock_guard lock(m_wifi_sdio_register_mutex); address &= 0x1ffff; if (function == 1 && address < 0x10000) { @@ -2567,6 +2592,7 @@ void StarletMemory::WriteWiFiSDIOByte(u32 function, u32 address, u8 value) u32 StarletMemory::GetWiFiSDIOBackplaneAddress(u32 address) const { + const std::lock_guard lock(m_wifi_sdio_register_mutex); const auto get_window_byte = [this](u32 register_address, u8 default_value) { const auto it = m_wifi_sdio_registers.find((1U << 17) | register_address); return it == m_wifi_sdio_registers.end() ? default_value : it->second; @@ -3121,6 +3147,15 @@ u8 StarletMemory::Read8(u32 address) if (IsBootROMAddress(address)) return m_boot_rom[GetBootROMOffset(address)]; + // SRAM has no byte-access side effects. Resolve it before the large MMIO decoder; this retains + // the live boot0 overlay and A/B split mapping while avoiding dozens of unrelated device tests + // in the IOS scheduler's hottest memory path. + if (IsSRAMWindowAddress(address)) + { + const u32 offset = GetSRAMOffset(address); + return offset == INVALID_SRAM_OFFSET ? 0 : m_sram[offset]; + } + if (IsMemoryControllerIndirectRegister(address)) { const u16 value = ReadMemoryControllerHalfword(address); @@ -3294,6 +3329,16 @@ void StarletMemory::Write8(u32 address, u8 value) if (IsBootROMAddress(address)) return; + // Unlike Hollywood MMIO, SRAM writes have no completion or interrupt side effects. Handle the + // exact mapped byte here so normal IOS data stores do not traverse the complete device decoder. + if (IsSRAMWindowAddress(address)) + { + const u32 offset = GetSRAMOffset(address); + if (offset != INVALID_SRAM_OFFSET) + m_sram[offset] = value; + return; + } + // NAND_CTRL is normally written a word at a time. Keep its completed value // while the incoming acknowledgement word is assembled byte by byte, since // that write is a command rather than a replacement for the readable @@ -3500,6 +3545,19 @@ void StarletMemory::Write16(u32 address, u16 value) return; } + if (!IsBootROMAddress(address) && !IsBootROMAddress(address + 1) && + IsSRAMWindowAddress(address) && IsSRAMWindowAddress(address + 1)) + { + const u32 offset = GetSRAMOffset(address); + const u32 end_offset = GetSRAMOffset(address + 1); + if (offset != INVALID_SRAM_OFFSET && end_offset == offset + 1) + { + m_sram[offset] = static_cast(value >> 8); + m_sram[end_offset] = static_cast(value); + return; + } + } + ARMBus::Write16(address, value); } @@ -3526,6 +3584,21 @@ void StarletMemory::Write32(u32 address, u32 value) return; } + if (!IsBootROMAddress(address) && !IsBootROMAddress(address + 3) && + IsSRAMWindowAddress(address) && IsSRAMWindowAddress(address + 3)) + { + const u32 offset = GetSRAMOffset(address); + const u32 end_offset = GetSRAMOffset(address + 3); + if (offset != INVALID_SRAM_OFFSET && end_offset == offset + 3) + { + m_sram[offset] = static_cast(value >> 24); + m_sram[offset + 1] = static_cast(value >> 16); + m_sram[offset + 2] = static_cast(value >> 8); + m_sram[end_offset] = static_cast(value); + return; + } + } + ARMBus::Write32(address, value); } diff --git a/Source/Core/Core/IOS/Starlet/StarletMemory.h b/Source/Core/Core/IOS/Starlet/StarletMemory.h index 3ff47dc16c..6b50a3459c 100644 --- a/Source/Core/Core/IOS/Starlet/StarletMemory.h +++ b/Source/Core/Core/IOS/Starlet/StarletMemory.h @@ -4,10 +4,12 @@ #pragma once #include +#include #include #include #include #include +#include #include #include #include @@ -185,6 +187,20 @@ private: u32 GetGPIOInput() const; void UpdateGPIOInterrupt(); u32 GetTimer() const; + void ClearRegisterCache(); + + static constexpr size_t REGISTER_CACHE_SIZE = 4096; + static constexpr u64 REGISTER_CACHE_VALID = 1ULL << 40; + + static constexpr size_t GetRegisterCacheIndex(u32 address) + { + return (address ^ (address >> 12)) & (REGISTER_CACHE_SIZE - 1); + } + + static constexpr u64 EncodeRegisterCacheEntry(u32 address, u8 value) + { + return REGISTER_CACHE_VALID | (static_cast(address) << 8) | value; + } Core::System& m_system; std::array m_boot_rom{}; @@ -193,6 +209,11 @@ private: File::IOFile m_nand; File::IOFile m_sd_card; std::map m_registers; + // Hollywood exposes a sparse byte-addressed register file. IOS repeatedly polls a small hot + // subset, for which a tree lookup per byte is disproportionately expensive. Keep the map as the + // source of truth and use tagged entries only as a transparent read-through cache. Atomic slots + // allow read hits from Broadway and Starlet without adding a lock to this hot path. + mutable std::array, REGISTER_CACHE_SIZE> m_register_cache{}; std::map m_nand_overlay; NANDPage m_nand_program_data{}; u32 m_nand_control_before_write = 0; @@ -214,6 +235,11 @@ private: u32 m_sd_block_length = 512; u32 m_sdhc_status_before_write = 0; u32 m_wifi_sdio_status_before_write = 0; + // Broadway MMIO and the scheduled Starlet slice can both reach the shared Hollywood SDIO + // controller in dual-core mode. Protect the sparse register file while IOS resets a function. + // This is recursive because byte accesses consult the backplane-window registers through a + // helper that takes the same lock. + mutable std::recursive_mutex m_wifi_sdio_register_mutex; std::map m_wifi_sdio_registers; std::deque m_wifi_sdio_pio_read_data; u32 m_wifi_sdio_pio_write_function = 0; diff --git a/Source/Core/Core/PowerPC/MMU.cpp b/Source/Core/Core/PowerPC/MMU.cpp index d8a95e7ef1..5b5faada0c 100644 --- a/Source/Core/Core/PowerPC/MMU.cpp +++ b/Source/Core/Core/PowerPC/MMU.cpp @@ -285,8 +285,8 @@ T MMU::ReadFromHardware(u32 em_address) } else { - m_ppc_state.dCache.Read(m_memory, em_address, &value, sizeof(T), - HID0(m_ppc_state).DLOCK || flag != XCheckTLBFlag::Read); + value = m_ppc_state.dCache.ReadMainMemoryValue( + m_memory, em_address, HID0(m_ppc_state).DLOCK || flag != XCheckTLBFlag::Read); } return bswap(value); @@ -304,8 +304,9 @@ T MMU::ReadFromHardware(u32 em_address) } else { - m_ppc_state.dCache.Read(m_memory, em_address + 0x10000000, &value, sizeof(T), - HID0(m_ppc_state).DLOCK || flag != XCheckTLBFlag::Read); + value = m_ppc_state.dCache.ReadMainMemoryValue(m_memory, em_address + 0x10000000, + HID0(m_ppc_state).DLOCK || + flag != XCheckTLBFlag::Read); } return bswap(value); @@ -514,7 +515,29 @@ void MMU::WriteToHardware(u32 em_address, const u32 data, const u32 size) em_address &= m_memory.GetRamMask(); if (m_ppc_state.m_enable_dcache && !wi) - m_ppc_state.dCache.Write(m_memory, em_address, &swapped_data, size, HID0(m_ppc_state).DLOCK); + { + switch (size) + { + case 1: + m_ppc_state.dCache.WriteMainMemoryValue( + m_memory, em_address, static_cast(swapped_data), HID0(m_ppc_state).DLOCK); + break; + case 2: + m_ppc_state.dCache.WriteMainMemoryValue( + m_memory, em_address, static_cast(swapped_data), HID0(m_ppc_state).DLOCK); + break; + case 4: + m_ppc_state.dCache.WriteMainMemoryValue(m_memory, em_address, swapped_data, + HID0(m_ppc_state).DLOCK); + break; + default: + // Page-boundary splitting can legally produce a three-byte fragment. Keep the typed fast + // paths for normal accesses and preserve the generic byte-count behavior for that rarity. + m_ppc_state.dCache.Write(m_memory, em_address, &swapped_data, size, + HID0(m_ppc_state).DLOCK); + break; + } + } if (!m_ppc_state.m_enable_dcache || wi || flag != XCheckTLBFlag::Write) std::memcpy(&m_memory.GetRAM()[em_address], &swapped_data, size); @@ -529,8 +552,27 @@ void MMU::WriteToHardware(u32 em_address, const u32 data, const u32 size) if (m_ppc_state.m_enable_dcache && !wi) { - m_ppc_state.dCache.Write(m_memory, em_address + 0x10000000, &swapped_data, size, - HID0(m_ppc_state).DLOCK); + switch (size) + { + case 1: + m_ppc_state.dCache.WriteMainMemoryValue(m_memory, em_address + 0x10000000, + static_cast(swapped_data), + HID0(m_ppc_state).DLOCK); + break; + case 2: + m_ppc_state.dCache.WriteMainMemoryValue(m_memory, em_address + 0x10000000, + static_cast(swapped_data), + HID0(m_ppc_state).DLOCK); + break; + case 4: + m_ppc_state.dCache.WriteMainMemoryValue(m_memory, em_address + 0x10000000, + swapped_data, HID0(m_ppc_state).DLOCK); + break; + default: + m_ppc_state.dCache.Write(m_memory, em_address + 0x10000000, &swapped_data, size, + HID0(m_ppc_state).DLOCK); + break; + } } if (!m_ppc_state.m_enable_dcache || wi || flag != XCheckTLBFlag::Write) @@ -706,7 +748,8 @@ template T MMU::Read(const u32 address) { T var = ReadFromHardware(address); - Memcheck(address, var, false, sizeof(T)); + if (m_power_pc.GetMemChecks().HasAny()) + Memcheck(address, var, false, sizeof(T)); return var; } template u8 MMU::Read(const u32 address); @@ -762,7 +805,8 @@ template std::optional> MMU::HostTryRead(const Core::CPUThr template void MMU::Write(const Common::MakeAtLeastU32 var, const u32 address) { - Memcheck(address, var, true, sizeof(T)); + if (m_power_pc.GetMemChecks().HasAny()) + Memcheck(address, var, true, sizeof(T)); WriteToHardware(address, var, sizeof(T)); } template void MMU::Write(const u32 var, const u32 address); @@ -771,7 +815,8 @@ template void MMU::Write(const u32 var, const u32 address); template <> void MMU::Write(const u64 var, const u32 address) { - Memcheck(address, var, true, 8); + if (m_power_pc.GetMemChecks().HasAny()) + Memcheck(address, var, true, 8); WriteToHardware(address, static_cast(var >> 32), 4); WriteToHardware(address + sizeof(u32), static_cast(var), 4); } diff --git a/Source/Core/Core/PowerPC/PPCCache.cpp b/Source/Core/Core/PowerPC/PPCCache.cpp index 8935c5c19d..4cbd7146fe 100644 --- a/Source/Core/Core/PowerPC/PPCCache.cpp +++ b/Source/Core/Core/PowerPC/PPCCache.cpp @@ -18,13 +18,6 @@ namespace PowerPC { namespace { -constexpr std::array s_plru_mask{ - 11, 11, 19, 19, 37, 37, 69, 69, -}; -constexpr std::array s_plru_value{ - 11, 3, 17, 1, 36, 4, 64, 0, -}; - constexpr std::array s_way_from_valid = [] { std::array data{}; for (size_t m = 0; m < data.size(); m++) @@ -120,6 +113,68 @@ void Cache::Init(Memory::MemoryManager& memory) Reset(); } +u32 Cache::LoadCacheLine(Memory::MemoryManager& memory, u32 addr, u32 set) +{ + u32 way; + if (valid[set] != 0xff) + way = s_way_from_valid[valid[set]]; + else + way = s_way_from_plru[plru[set]]; + + if (valid[set] & (1 << way)) + { + // Store the evicted line back to main memory before replacing its lookup-table entry. + if (modified[set] & (1 << way)) + memory.CopyToEmu(addrs[set][way], data[set][way].data(), 32); + + if (addrs[set][way] & CACHE_VMEM_BIT) + lookup_table_vmem[(addrs[set][way] & memory.GetFakeVMemMask()) >> 5] = 0xff; + else if (addrs[set][way] & CACHE_EXRAM_BIT) + lookup_table_ex[(addrs[set][way] & memory.GetExRamMask()) >> 5] = 0xff; + else + lookup_table[(addrs[set][way] & memory.GetRamMask()) >> 5] = 0xff; + } + + memory.CopyFromEmu(data[set][way].data(), addr, 32); + + if (addr & CACHE_VMEM_BIT) + lookup_table_vmem[(addr & memory.GetFakeVMemMask()) >> 5] = way; + else if (addr & CACHE_EXRAM_BIT) + lookup_table_ex[(addr & memory.GetExRamMask()) >> 5] = way; + else + lookup_table[(addr & memory.GetRamMask()) >> 5] = way; + + addrs[set][way] = addr; + valid[set] |= 1 << way; + modified[set] &= ~(1 << way); + return way; +} + +DOLPHIN_FORCE_INLINE std::pair Cache::GetCache(Memory::MemoryManager& memory, u32 addr, + bool locked) +{ + addr &= ~31U; + const u32 set = (addr >> 5) & 0x7f; + u32 way; + + if (addr & CACHE_VMEM_BIT) + way = lookup_table_vmem[(addr & memory.GetFakeVMemMask()) >> 5]; + else if (addr & CACHE_EXRAM_BIT) + way = lookup_table_ex[(addr & memory.GetExRamMask()) >> 5]; + else + way = lookup_table[(addr & memory.GetRamMask()) >> 5]; + + // A hit is the overwhelmingly common path. Keep replacement, write-back and line filling out of + // this function so the compiler can inline the lookup into typed JIT reads and writes. + if (way == 0xff && !locked) + way = LoadCacheLine(memory, addr, set); + + if (way != 0xff) + plru[set] = (plru[set] & ~PLRU_MASK[way]) | PLRU_VALUE[way]; + + return {set, way}; +} + void InstructionCache::Init(Memory::MemoryManager& memory) { if (!m_config_callback_id) @@ -205,70 +260,6 @@ void Cache::Touch(Memory::MemoryManager& memory, u32 addr, bool store) GetCache(memory, addr, false); } -std::pair Cache::GetCache(Memory::MemoryManager& memory, u32 addr, bool locked) -{ - addr &= ~31; - u32 set = (addr >> 5) & 0x7f; - u32 way; - - if (addr & CACHE_VMEM_BIT) - { - way = lookup_table_vmem[(addr & memory.GetFakeVMemMask()) >> 5]; - } - else if (addr & CACHE_EXRAM_BIT) - { - way = lookup_table_ex[(addr & memory.GetExRamMask()) >> 5]; - } - else - { - way = lookup_table[(addr & memory.GetRamMask()) >> 5]; - } - - // load to the cache - if (!locked && way == 0xff) - { - // select a way - if (valid[set] != 0xff) - way = s_way_from_valid[valid[set]]; - else - way = s_way_from_plru[plru[set]]; - - if (valid[set] & (1 << way)) - { - // store the cache back to main memory - if (modified[set] & (1 << way)) - memory.CopyToEmu(addrs[set][way], data[set][way].data(), 32); - - if (addrs[set][way] & CACHE_VMEM_BIT) - lookup_table_vmem[(addrs[set][way] & memory.GetFakeVMemMask()) >> 5] = 0xff; - else if (addrs[set][way] & CACHE_EXRAM_BIT) - lookup_table_ex[(addrs[set][way] & memory.GetExRamMask()) >> 5] = 0xff; - else - lookup_table[(addrs[set][way] & memory.GetRamMask()) >> 5] = 0xff; - } - - // load - memory.CopyFromEmu(data[set][way].data(), (addr & ~0x1f), 32); - - if (addr & CACHE_VMEM_BIT) - lookup_table_vmem[(addr & memory.GetFakeVMemMask()) >> 5] = way; - else if (addr & CACHE_EXRAM_BIT) - lookup_table_ex[(addr & memory.GetExRamMask()) >> 5] = way; - else - lookup_table[(addr & memory.GetRamMask()) >> 5] = way; - - addrs[set][way] = addr; - valid[set] |= (1 << way); - modified[set] &= ~(1 << way); - } - - // update plru - if (way != 0xff) - plru[set] = (plru[set] & ~s_plru_mask[way]) | s_plru_value[way]; - - return {set, way}; -} - void Cache::Read(Memory::MemoryManager& memory, u32 addr, void* buffer, u32 len, bool locked) { auto* value = static_cast(buffer); @@ -296,6 +287,31 @@ void Cache::Read(Memory::MemoryManager& memory, u32 addr, void* buffer, u32 len, } } +template +T Cache::ReadValue(Memory::MemoryManager& memory, u32 addr, bool locked) +{ + T value; + const u32 offset_in_block = addr & 31; + if (offset_in_block <= 32 - sizeof(T)) + { + const auto [set, way] = GetCache(memory, addr, locked); + if (way != 0xff) + { + std::memcpy(&value, reinterpret_cast(data[set][way].data()) + offset_in_block, + sizeof(T)); + } + else + { + memory.CopyFromEmu(&value, addr, sizeof(T)); + } + return value; + } + + // Only an unaligned access at the end of a cache line needs the generic split path. + Read(memory, addr, &value, sizeof(T), locked); + return value; +} + void Cache::Write(Memory::MemoryManager& memory, u32 addr, const void* buffer, u32 len, bool locked) { auto* value = static_cast(buffer); @@ -324,6 +340,39 @@ void Cache::Write(Memory::MemoryManager& memory, u32 addr, const void* buffer, u } } +template +void Cache::WriteValue(Memory::MemoryManager& memory, u32 addr, T value, bool locked) +{ + const u32 offset_in_block = addr & 31; + if (offset_in_block <= 32 - sizeof(T)) + { + const auto [set, way] = GetCache(memory, addr, locked); + if (way != 0xff) + { + std::memcpy(reinterpret_cast(data[set][way].data()) + offset_in_block, &value, + sizeof(T)); + modified[set] |= 1 << way; + } + else + { + memory.CopyToEmu(addr, &value, sizeof(T)); + } + return; + } + + // Only an unaligned access at the end of a cache line needs the generic split path. + Write(memory, addr, &value, sizeof(T), locked); +} + +template u8 Cache::ReadValue(Memory::MemoryManager&, u32, bool); +template u16 Cache::ReadValue(Memory::MemoryManager&, u32, bool); +template u32 Cache::ReadValue(Memory::MemoryManager&, u32, bool); +template u64 Cache::ReadValue(Memory::MemoryManager&, u32, bool); +template void Cache::WriteValue(Memory::MemoryManager&, u32, u8, bool); +template void Cache::WriteValue(Memory::MemoryManager&, u32, u16, bool); +template void Cache::WriteValue(Memory::MemoryManager&, u32, u32, bool); +template void Cache::WriteValue(Memory::MemoryManager&, u32, u64, bool); + void Cache::DoState(Memory::MemoryManager& memory, PointerWrap& p) { if (p.IsReadMode()) diff --git a/Source/Core/Core/PowerPC/PPCCache.h b/Source/Core/Core/PowerPC/PPCCache.h index 890f308252..ff510b80f1 100644 --- a/Source/Core/Core/PowerPC/PPCCache.h +++ b/Source/Core/Core/PowerPC/PPCCache.h @@ -4,11 +4,13 @@ #pragma once #include +#include #include #include #include "Common/CommonTypes.h" #include "Common/Config/Config.h" +#include "Common/Inline.h" class JitInterface; namespace Memory @@ -33,6 +35,13 @@ constexpr u32 CACHE_VMEM_BIT = 0x20000000; struct Cache { + static constexpr std::array PLRU_MASK{ + 11, 11, 19, 19, 37, 37, 69, 69, + }; + static constexpr std::array PLRU_VALUE{ + 11, 3, 17, 1, 36, 4, 64, 0, + }; + std::array, CACHE_WAYS>, CACHE_SETS> data{}; // Stores the 32-byte aligned address of the start of each cache block. This consists of the cache @@ -58,7 +67,80 @@ struct Cache void FlushAll(Memory::MemoryManager& memory); - std::pair GetCache(Memory::MemoryManager& memory, u32 addr, bool locked); + u32 LoadCacheLine(Memory::MemoryManager& memory, u32 addr, u32 set); + DOLPHIN_FORCE_INLINE std::pair GetCache(Memory::MemoryManager& memory, u32 addr, + bool locked); + + template + T ReadValue(Memory::MemoryManager& memory, u32 addr, bool locked); + template + void WriteValue(Memory::MemoryManager& memory, u32 addr, T value, bool locked); + + // MMU has already classified and masked these addresses. Keeping this overwhelmingly common + // path inline avoids repeating the generic cache-region tests and the typed helper call for + // every Broadway JIT load/store while preserving the exact cache contents and PLRU behavior. + template + DOLPHIN_FORCE_INLINE T ReadMainMemoryValue(Memory::MemoryManager& memory, u32 addr, bool locked) + { + T value; + const u32 offset_in_block = addr & 31; + if (offset_in_block > 32 - sizeof(T)) + { + Read(memory, addr, &value, sizeof(T), locked); + return value; + } + + const u32 line_addr = addr & ~31U; + const u32 set = (line_addr >> 5) & 0x7f; + const u32 lookup_index = exram ? ((line_addr & 0x0fffffff) >> 5) : (line_addr >> 5); + u32 way = exram ? lookup_table_ex[lookup_index] : lookup_table[lookup_index]; + + if (way == 0xff && !locked) + way = LoadCacheLine(memory, line_addr, set); + + if (way == 0xff) + { + // A locked-cache miss bypasses the cache. Reuse the generic path for this rare case. + Read(memory, addr, &value, sizeof(T), locked); + return value; + } + + plru[set] = (plru[set] & ~PLRU_MASK[way]) | PLRU_VALUE[way]; + std::memcpy(&value, reinterpret_cast(data[set][way].data()) + offset_in_block, + sizeof(T)); + return value; + } + + template + DOLPHIN_FORCE_INLINE void WriteMainMemoryValue(Memory::MemoryManager& memory, u32 addr, T value, + bool locked) + { + const u32 offset_in_block = addr & 31; + if (offset_in_block > 32 - sizeof(T)) + { + Write(memory, addr, &value, sizeof(T), locked); + return; + } + + const u32 line_addr = addr & ~31U; + const u32 set = (line_addr >> 5) & 0x7f; + const u32 lookup_index = exram ? ((line_addr & 0x0fffffff) >> 5) : (line_addr >> 5); + u32 way = exram ? lookup_table_ex[lookup_index] : lookup_table[lookup_index]; + + if (way == 0xff && !locked) + way = LoadCacheLine(memory, line_addr, set); + + if (way == 0xff) + { + // A locked-cache miss bypasses the cache. Reuse the generic path for this rare case. + Write(memory, addr, &value, sizeof(T), locked); + return; + } + + plru[set] = (plru[set] & ~PLRU_MASK[way]) | PLRU_VALUE[way]; + std::memcpy(reinterpret_cast(data[set][way].data()) + offset_in_block, &value, sizeof(T)); + modified[set] |= 1 << way; + } void Read(Memory::MemoryManager& memory, u32 addr, void* buffer, u32 len, bool locked); void Write(Memory::MemoryManager& memory, u32 addr, const void* buffer, u32 len, bool locked); diff --git a/Source/UnitTests/Core/IOS/Starlet/ARMCoreTest.cpp b/Source/UnitTests/Core/IOS/Starlet/ARMCoreTest.cpp index e788c6552e..f42124a728 100644 --- a/Source/UnitTests/Core/IOS/Starlet/ARMCoreTest.cpp +++ b/Source/UnitTests/Core/IOS/Starlet/ARMCoreTest.cpp @@ -83,8 +83,7 @@ public: if (offset + 3 >= SRAM_SIZE) return 0; const u8* const sram = SRAMData(); - return (static_cast(sram[offset]) << 24) | - (static_cast(sram[offset + 1]) << 16) | + return (static_cast(sram[offset]) << 24) | (static_cast(sram[offset + 1]) << 16) | (static_cast(sram[offset + 2]) << 8) | sram[offset + 3]; } const size_t offset = ToOffset(address); @@ -197,8 +196,7 @@ public: { EXPECT_LT(offset + 3, SRAM_SIZE); const u8* const sram = SRAMData(); - return (static_cast(sram[offset]) << 24) | - (static_cast(sram[offset + 1]) << 16) | + return (static_cast(sram[offset]) << 24) | (static_cast(sram[offset + 1]) << 16) | (static_cast(sram[offset + 2]) << 8) | sram[offset + 3]; } @@ -288,6 +286,73 @@ TEST(WiiIPCCtrlRegister, ProducerBitsRemainLatchedUntilPeerAcknowledges) EXPECT_EQ(control.ppc() & 0x06, 0x00); } +TEST(StarletSRAM, WideAccessesPreserveAliasesSplitMappingAndBoot0Protection) +{ + constexpr u32 sram_low = StarletMemory::SRAM_BASE; + constexpr u32 sram_high = StarletMemory::SRAM_MIRROR_BASE; + constexpr u32 hardware_srnprot = 0x0d800060; + constexpr u32 sram_split_mode = 1U << 5; + Core::DeclareAsCPUThread(); + auto& system = Core::System::GetInstance(); + system.GetWiiIPC().Reset(); + StarletMemory memory(system); + memory.Reset(); + + memory.Write32(sram_low + 0x20, 0x11223344); + EXPECT_EQ(memory.Read32(sram_high + 0x20), 0x11223344u); + memory.Write16(sram_high + 0x24, 0xa1b2); + EXPECT_EQ(memory.Read16(sram_low + 0x24), 0xa1b2u); + memory.Write8(sram_low + 0x26, 0x5a); + EXPECT_EQ(memory.Read8(sram_high + 0x26), 0x5au); + + // With the reset (non-split) mapping, boot0 overlays the upper half of the top aperture. + const u32 boot0_word = memory.Read32(0xffff0020); + memory.Write32(0xffff0020, 0xdeadbeef); + EXPECT_EQ(memory.Read32(0xffff0020), boot0_word); + + memory.Write32(hardware_srnprot, sram_split_mode); + memory.Write32(sram_low + 0x20, 0x55667788); // Logical page zero now selects SRAM B. + EXPECT_EQ(memory.Read32(sram_high + 0x20), 0x55667788u); + EXPECT_EQ(memory.Read32(0xffff0020), 0x11223344u); // Upper aperture selects SRAM A. + memory.Write32(0xffff0020, 0xaabbccdd); + + // The split gap remains unmapped and ignores writes. + memory.Write32(sram_high + 0x8000, 0xcafebabe); + EXPECT_EQ(memory.Read32(sram_high + 0x8000), 0u); + + memory.Write32(hardware_srnprot, 0); + EXPECT_EQ(memory.Read32(sram_low + 0x20), 0xaabbccddu); +} + +TEST(StarletRegisters, CachePreservesSparseValuesAndCollisions) +{ + constexpr u32 address_a = 0x0d900100; + constexpr u32 address_b = address_a ^ 0x1001; + constexpr auto cache_index = [](u32 address) { return (address ^ (address >> 12)) & 0xfff; }; + static_assert(cache_index(address_a) == cache_index(address_b)); + + Core::DeclareAsCPUThread(); + auto& system = Core::System::GetInstance(); + system.GetWiiIPC().Reset(); + StarletMemory memory(system); + memory.Reset(); + + EXPECT_EQ(memory.Read8(address_a), 0u); + EXPECT_EQ(memory.Read8(address_b), 0u); + memory.Write8(address_a, 0x12); + EXPECT_EQ(memory.Read8(address_a), 0x12u); + memory.Write8(address_b, 0x34); + EXPECT_EQ(memory.Read8(address_b), 0x34u); + EXPECT_EQ(memory.Read8(address_a), 0x12u); + memory.Write8(address_a, 0x56); + EXPECT_EQ(memory.Read8(address_a), 0x56u); + EXPECT_EQ(memory.Read8(address_b), 0x34u); + + memory.Reset(); + EXPECT_EQ(memory.Read8(address_a), 0u); + EXPECT_EQ(memory.Read8(address_b), 0u); +} + TEST(StarletTimer, ZeroDelayAlarmMatchesImmediatelyAndUsesIRQW1C) { constexpr u32 hardware_base = 0x0d800000; @@ -956,6 +1021,133 @@ TEST(StarletARMCore, JitPreservedBlocksUseCurrentTLBGenerationForFastmem) #endif } +TEST(StarletARMCore, FCSESwitchPreservesTaggedTLBTranslations) +{ + TestBus bus(0x10000); + ARMCore core(bus); + bus.WriteARM(0x0000, 0xe5921000); // ldr r1, [r2] + bus.WriteARM(0x0004, 0xe3a00402); // mov r0, #0x02000000 + bus.WriteARM(0x0008, 0xee0d0f10); // mcr p15, 0, r0, c13, c0, 0 + bus.WriteARM(0x000c, 0xe5923000); // ldr r3, [r2] + bus.WriteARM(0x0010, 0xe3a00000); // mov r0, #0 + bus.WriteARM(0x0014, 0xee0d0f10); // mcr p15, 0, r0, c13, c0, 0 + bus.WriteARM(0x0018, 0xe5924000); // ldr r4, [r2] + bus.WriteARM(0x0100, 0x12345678); + bus.WriteARM(0x4000, 0x00000c02); // FCSE PID 0 low section -> PA 0 + bus.WriteARM(0x4080, 0x00000c02); // FCSE PID 1 low section -> PA 0 + bus.WriteARM(0x6000, 0x00000c02); // VA 0x80000000 section -> PA 0 + core.GetCP15State().translation_table_base = 0x4000; + core.GetCP15State().domain_access_control = 3; + core.GetCP15State().control |= 1; + core.SetRegister(2, 0x100); + core.SetRegister(15, 0x80000000); + bus.ResetReadCounts(); + + EXPECT_EQ(core.RunCycles(7), 7u); + EXPECT_EQ(core.GetRegister(1), 0x12345678u); + EXPECT_EQ(core.GetRegister(3), 0x12345678u); + EXPECT_EQ(core.GetRegister(4), 0x12345678u); + // Seven instruction fetches, three data reads, and exactly three first-level walks: kernel, + // PID 0 and PID 1. Switching back to PID 0 must reuse its MVA-tagged TLB entry. + EXPECT_EQ(bus.GetRead32Count(), 13u); +} + +TEST(StarletARMCore, JitFastCacheSeparatesFCSEProcesses) +{ +#if defined(_M_X86_64) + TestBus bus(0x204000); + ARMCore core(bus); + bus.WriteARM(0x000000, 0xe3a01001); // PID 0: mov r1, #1 + bus.WriteARM(0x000004, 0xea00003d); // b 0x100 + bus.WriteARM(0x100000, 0xe3a01002); // PID 1: mov r1, #2 + bus.WriteARM(0x100004, 0xea00003d); // b 0x100 + bus.WriteARM(0x200000, 0x00000c02); // FCSE PID 0 section -> PA 0 + bus.WriteARM(0x200080, 0x00100c02); // FCSE PID 1 section -> PA 0x00100000 + core.GetCP15State().translation_table_base = 0x200000; + core.GetCP15State().domain_access_control = 3; + core.GetCP15State().control |= 1; + core.SetJitEnabled(true); + + core.SetRegister(15, 0); + EXPECT_EQ(core.RunCycles(2), 2u); + EXPECT_EQ(core.GetRegister(1), 1u); + + core.GetCP15State().process_id = 0x02000000; + core.SetRegister(15, 0); + EXPECT_EQ(core.RunCycles(2), 2u); + EXPECT_EQ(core.GetRegister(1), 2u); + EXPECT_EQ(core.GetJitCompiledBlockCount(), 2u); + + // Both the software TLB and native fast-entry cache retain distinct MVA-tagged entries. Returning + // to PID 0 therefore needs neither a page-table read nor another block compilation. + bus.ResetReadCounts(); + core.GetCP15State().process_id = 0; + core.SetRegister(15, 0); + EXPECT_EQ(core.RunCycles(2), 2u); + EXPECT_EQ(core.GetRegister(1), 1u); + EXPECT_EQ(bus.GetRead32Count(), 0u); + EXPECT_EQ(core.GetJitCompiledBlockCount(), 2u); +#endif +} + +TEST(StarletARMCore, ARMJitCompilesConditionalBranchExchange) +{ +#if defined(_M_X86_64) + TestBus bus(0x100); + ARMCore core(bus); + bus.WriteARM(0x00, 0x012fff1e); // bxeq lr + core.SetJitEnabled(true); + core.SetRegister(14, 0x20); + core.SetCPSR(core.GetCPSR() | ARMCore::CPSR_Z); + + EXPECT_EQ(core.RunCycles(1), 1u); + EXPECT_EQ(core.GetRegister(15), 0x20u); + + core.SetRegister(15, 0); + core.SetCPSR(core.GetCPSR() & ~ARMCore::CPSR_Z); + EXPECT_EQ(core.RunCycles(1), 1u); + EXPECT_EQ(core.GetRegister(15), 4u); + EXPECT_EQ(core.GetJitFallbackInstructionCount(), 0u); + EXPECT_EQ(core.GetJitNativeExecutedInstructions(), 2u); +#endif +} + +TEST(StarletARMCore, ARMJitCompilesConditionalSingleDataTransfers) +{ +#if defined(_M_X86_64) + TestBus bus(0x200); + ARMCore core(bus); + bus.SetFastmemEnabled(true); + bus.WriteARM(0x00, 0x05921000); // ldreq r1, [r2] + bus.WriteARM(0x04, 0xea00003d); // b 0x100 + bus.WriteARM(0x20, 0x15823004); // strne r3, [r2, #4] + bus.WriteARM(0x24, 0xea000035); // b 0x100 + bus.WriteARM(0x100, 0x11223344); + bus.WriteARM(0x104, 0xaabbccdd); + core.SetJitEnabled(true); + core.SetRegister(2, 0x100); + core.SetRegister(3, 0x55667788); + core.SetCPSR(core.GetCPSR() | ARMCore::CPSR_Z); + + EXPECT_EQ(core.RunCycles(2), 2u); + EXPECT_EQ(core.GetRegister(1), 0x11223344u); + + core.SetRegister(15, 0x20); + EXPECT_EQ(core.RunCycles(2), 2u); + EXPECT_EQ(bus[0x104], 0xaau); + + core.SetRegister(15, 0x20); + core.SetCPSR(core.GetCPSR() & ~ARMCore::CPSR_Z); + EXPECT_EQ(core.RunCycles(2), 2u); + EXPECT_EQ(bus[0x104], 0x55u); + EXPECT_EQ(bus[0x105], 0x66u); + EXPECT_EQ(bus[0x106], 0x77u); + EXPECT_EQ(bus[0x107], 0x88u); + EXPECT_EQ(core.GetJitFallbackInstructionCount(), 0u); + EXPECT_EQ(core.GetJitNativeExecutedInstructions(), 6u); +#endif +} + TEST(StarletARMCore, WaitForInterruptFastForwardsAndWakesOnMaskedIRQ) { TestBus bus; @@ -1223,7 +1415,7 @@ TEST(StarletARMCore, ARMJitConditionalBranchesMatchInterpreter) { const u32 address = condition * 0x20; const u32 branch = (condition << 28) | 0x0a000000; - interpreter_bus.WriteARM(address, branch); // b address + 8 + interpreter_bus.WriteARM(address, branch); // b address + 8 jit_bus.WriteARM(address, branch); interpreter_bus.WriteARM(address + 4, 0xe3a00001); // mov r0, #1 jit_bus.WriteARM(address + 4, 0xe3a00001); @@ -1233,10 +1425,8 @@ TEST(StarletARMCore, ARMJitConditionalBranchesMatchInterpreter) for (u32 flags = 0; flags < 16; ++flags) { const u32 cpsr = static_cast(ARMCore::Mode::Supervisor) | - ((flags & 1) ? ARMCore::CPSR_N : 0) | - ((flags & 2) ? ARMCore::CPSR_Z : 0) | - ((flags & 4) ? ARMCore::CPSR_C : 0) | - ((flags & 8) ? ARMCore::CPSR_V : 0); + ((flags & 1) ? ARMCore::CPSR_N : 0) | ((flags & 2) ? ARMCore::CPSR_Z : 0) | + ((flags & 4) ? ARMCore::CPSR_C : 0) | ((flags & 8) ? ARMCore::CPSR_V : 0); interpreter.SetCPSR(cpsr); jit.SetCPSR(cpsr); interpreter.SetRegister(15, address); @@ -1272,7 +1462,7 @@ TEST(StarletARMCore, ARMJitCompilesPredicatedDataProcessing) bus.WriteARM(address + 0x04, (condition << 28) | 0x03530002); // cmp r3, #2 bus.WriteARM(address + 0x08, (condition << 28) | 0x03c44018); // bic r4, r4, #0x18 bus.WriteARM(address + 0x0c, (condition << 28) | 0x03a05007); // mov r5, #7 - bus.WriteARM(address + 0x10, 0xea000000); // b address + 0x18 + bus.WriteARM(address + 0x10, 0xea000000); // b address + 0x18 }; install_program(interpreter_bus); install_program(jit_bus); @@ -1280,10 +1470,8 @@ TEST(StarletARMCore, ARMJitCompilesPredicatedDataProcessing) for (u32 flags = 0; flags < 16; ++flags) { const u32 cpsr = static_cast(ARMCore::Mode::Supervisor) | - ((flags & 1) ? ARMCore::CPSR_N : 0) | - ((flags & 2) ? ARMCore::CPSR_Z : 0) | - ((flags & 4) ? ARMCore::CPSR_C : 0) | - ((flags & 8) ? ARMCore::CPSR_V : 0); + ((flags & 1) ? ARMCore::CPSR_N : 0) | ((flags & 2) ? ARMCore::CPSR_Z : 0) | + ((flags & 4) ? ARMCore::CPSR_C : 0) | ((flags & 8) ? ARMCore::CPSR_V : 0); interpreter.SetCPSR(cpsr); jit.SetCPSR(cpsr); for (ARMCore* core : {&interpreter, &jit}) @@ -1443,8 +1631,8 @@ TEST(StarletARMCore, ThumbJitCompilesRegisterLSREdgeCases) interpreter_bus.WriteThumb(0x02, 0xe7fe); // b . jit_bus.WriteThumb(0x02, 0xe7fe); - constexpr std::array values = {0, 1, 0x80000000, 0xffffffff, 0x12345678, - 0x7fffffff, 0xa5a5a5a5}; + constexpr std::array values = {0, 1, 0x80000000, 0xffffffff, + 0x12345678, 0x7fffffff, 0xa5a5a5a5}; constexpr std::array shifts = {0, 1, 2, 31, 32, 33, 63, 255}; for (const u32 value : values) { @@ -1775,8 +1963,50 @@ TEST(StarletARMCore, ARMJitDoesNotCrossFromSRAMIntoBoot0) core.SetJitEnabled(true); EXPECT_EQ(core.RunCycles(2), 2u); - EXPECT_EQ(core.GetJitFallbackInstructionCount(), 1u); - EXPECT_EQ(core.GetJitNativeExecutedInstructions(), 1u); + EXPECT_EQ(core.GetJitFallbackInstructionCount(), 0u); + EXPECT_EQ(core.GetJitNativeExecutedInstructions(), 2u); +#endif +} + +TEST(StarletARMCore, ARMJitKeepsSRAMStackBlockTransfersInsideNativeBlock) +{ +#if defined(_M_X86_64) + TestBus interpreter_bus(0x1000); + TestBus jit_bus(0x1000); + interpreter_bus.SetFastmemEnabled(true); + interpreter_bus.SetSRAMFastmemEnabled(true); + jit_bus.SetFastmemEnabled(true); + jit_bus.SetSRAMFastmemEnabled(true); + ARMCore interpreter(interpreter_bus); + ARMCore jit(jit_bus); + jit.SetJitEnabled(true); + const auto install_program = [](TestBus& bus) { + bus.WriteARM(0x00, 0xe92d4070); // push {r4-r6, lr} + bus.WriteARM(0x04, 0xe3a04000); // mov r4, #0 + bus.WriteARM(0x08, 0xe3a05000); // mov r5, #0 + bus.WriteARM(0x0c, 0xe3a06000); // mov r6, #0 + bus.WriteARM(0x10, 0xe3a0e000); // mov lr, #0 + bus.WriteARM(0x14, 0xe8bd4070); // pop {r4-r6, lr} + bus.WriteARM(0x18, 0xeafffffe); // b . + }; + install_program(interpreter_bus); + install_program(jit_bus); + for (ARMCore* core : {&interpreter, &jit}) + { + core->SetRegister(4, 0x11223344); + core->SetRegister(5, 0x55667788); + core->SetRegister(6, 0x99aabbcc); + core->SetRegister(13, 0xfff12000); + core->SetRegister(14, 0xddeeff00); + } + + ASSERT_EQ(interpreter.RunCycles(7), 7u); + ASSERT_EQ(jit.RunCycles(7), 7u); + for (u32 reg = 0; reg < 16; ++reg) + EXPECT_EQ(jit.GetRegister(reg), interpreter.GetRegister(reg)) << "r" << reg; + EXPECT_EQ(jit.GetCPSR(), interpreter.GetCPSR()); + EXPECT_EQ(jit.GetJitFallbackInstructionCount(), 0u); + EXPECT_EQ(jit.GetJitNativeExecutedInstructions(), 7u); #endif } @@ -2042,6 +2272,53 @@ TEST(StarletARMCore, RunCyclesStopsAtBudget) 2u); // Both words stay resident across loop iterations. } +TEST(StarletARMCore, ARMJitConditionalBackEdgeKeepsExactBudgetAndFallthrough) +{ +#if defined(_M_X86_64) + TestBus bus; + bus.SetFastmemEnabled(true); + ARMCore core(bus); + core.SetJitEnabled(true); + bus.WriteARM(0x00, 0xe2800001); // add r0, r0, #1 + bus.WriteARM(0x04, 0xe3500003); // cmp r0, #3 + bus.WriteARM(0x08, 0x1afffffc); // bne 0x00 + bus.WriteARM(0x0c, 0xe3a0102a); // mov r1, #42 + + EXPECT_EQ(core.RunCycles(9), 9u); + EXPECT_EQ(core.GetExecutedInstructions(), 9u); + EXPECT_EQ(core.GetRegister(0), 3u); + EXPECT_EQ(core.GetRegister(15), 0x0cu); + EXPECT_EQ(core.GetJitNativeExecutedInstructions(), 9u); + + EXPECT_EQ(core.RunCycles(1), 1u); + EXPECT_EQ(core.GetRegister(1), 42u); +#endif +} + +TEST(StarletARMCore, ThumbJitConditionalBackEdgeKeepsExactBudgetAndFallthrough) +{ +#if defined(_M_X86_64) + TestBus bus; + bus.SetFastmemEnabled(true); + ARMCore core(bus); + core.SetJitEnabled(true); + core.SetCPSR(static_cast(ARMCore::Mode::Supervisor) | ARMCore::CPSR_T); + bus.WriteThumb(0x00, 0x3001); // add r0, #1 + bus.WriteThumb(0x02, 0x2803); // cmp r0, #3 + bus.WriteThumb(0x04, 0xd1fc); // bne 0x00 + bus.WriteThumb(0x06, 0x212a); // mov r1, #42 + + EXPECT_EQ(core.RunCycles(9), 9u); + EXPECT_EQ(core.GetExecutedInstructions(), 9u); + EXPECT_EQ(core.GetRegister(0), 3u); + EXPECT_EQ(core.GetRegister(15), 0x06u); + EXPECT_EQ(core.GetJitNativeExecutedInstructions(), 9u); + + EXPECT_EQ(core.RunCycles(1), 1u); + EXPECT_EQ(core.GetRegister(1), 42u); +#endif +} + TEST(StarletARMCore, ARMJitFastForwardsSliceStableHollywoodTimerPoll) { TestBus bus; @@ -2073,14 +2350,14 @@ TEST(StarletARMCore, ARMJitFastForwardsSliceStableHollywoodTimerPoll) TEST(StarletARMCore, ARMJitKeepsNonblockingReceiveMessagePollGuestVisible) { const auto install_program = [](TestBus& bus, bool non_blocking) { - bus.WriteARM(0x00, 0xe5960000); // ldr r0, [r6] - bus.WriteARM(0x04, 0xe1a01008); // mov r1, r8 + bus.WriteARM(0x00, 0xe5960000); // ldr r0, [r6] + bus.WriteARM(0x04, 0xe1a01008); // mov r1, r8 bus.WriteARM(0x08, non_blocking ? 0xe3a02000 : 0xe3a02001); // mov r2, #block - bus.WriteARM(0x0c, 0xeb00000b); // bl 0x40 - bus.WriteARM(0x10, 0xe2505000); // subs r5, r0, #0 - bus.WriteARM(0x14, 0x1afffff9); // bne 0x00 - bus.WriteARM(0x40, 0xe60001d0); // IOS ReceiveMessage syscall 0x0e - bus.WriteARM(0x44, 0xe12fff1e); // bx lr + bus.WriteARM(0x0c, 0xeb00000b); // bl 0x40 + bus.WriteARM(0x10, 0xe2505000); // subs r5, r0, #0 + bus.WriteARM(0x14, 0x1afffff9); // bne 0x00 + bus.WriteARM(0x40, 0xe60001d0); // IOS ReceiveMessage syscall 0x0e + bus.WriteARM(0x44, 0xe12fff1e); // bx lr }; TestBus bus;