Yaya48/dolphin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

IOS: optimize Starlet LLE and accurate CPU cache

Browse Source 
Retain ARM926 FCSE translations, accelerate common Starlet JIT blocks and cache sparse Hollywood register reads. Inline the hot Broadway write-back cache path while preserving exact cache miss, locking and split-access behavior.
This commit is contained in:
Yaya48
2026-08-30 02:35:19 +02:00
parent 9389e38f75
commit 1bd3fac1f1
12 changed files with 968 additions and 223 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Source/Core/Core/IOS/Starlet/ARMCore.cpp
+9 -17
View File
@@ -190,7 +190,7 @@ u32 ARMCore::TranslateVirtualAddress(u32 address) const
const u32 modified_address =
address < 0x02000000 ? address | (m_cp15.process_id & 0xfe000000) : address;
const u32 virtual_page = modified_address >> 10;
TLBEntry& entry = m_tlb[virtual_page & (TLB_ENTRY_COUNT - 1)];
TLBEntry& entry = m_tlb[GetTLBIndex(virtual_page)];
if (entry.generation == m_tlb_generation && entry.virtual_page == virtual_page)
return entry.physical_page | (modified_address & 0x3ff);
@@ -600,8 +600,8 @@ bool ARMCore::TryEnterARMSliceStablePoll(u32 branch_instruction)
// timer cannot advance again until the bus receives AdvanceCycles at the slice boundary, making
// every remaining iteration in this slice observationally redundant.
const u32 condition = branch_instruction >> 28;
if ((branch_instruction & 0x0e000000) != 0x0a000000 || condition >= 0xe ||
!m_pc_written || HasUnmaskedInterrupt())
if ((branch_instruction & 0x0e000000) != 0x0a000000 || condition >= 0xe || !m_pc_written ||
HasUnmaskedInterrupt())
{
return false;
}
@@ -864,15 +864,6 @@ size_t ARMCore::GetJitCompiledBlockCount() const
#endif
}
u64 ARMCore::GetJitBlockExecutionCount() const
{
#if defined(_M_X86_64)
return m_jit ? m_jit->GetBlockExecutionCount() : 0;
#else
return 0;
#endif
}
u64 ARMCore::GetJitAddressTranslationCount() const
{
#if defined(_M_X86_64)
@@ -1054,8 +1045,7 @@ u64 ARMCore::TakeJitFallbackIntervalCount()
return std::exchange(m_jit_fallback_interval_count, 0);
}
std::vector<ARMCore::HotPCSample>
ARMCore::TakeHotJitFallbackIntervalSamples(size_t maximum_count)
std::vector<ARMCore::HotPCSample> ARMCore::TakeHotJitFallbackIntervalSamples(size_t maximum_count)
{
std::vector<std::pair<u32, u64>> sorted(m_jit_fallback_interval_samples.begin(),
m_jit_fallback_interval_samples.end());
@@ -2322,11 +2312,9 @@ void ARMCore::WriteCP15(u32 opcode1, u32 crn, u32 crm, u32 opcode2, u32 value)
break;
case 2:
m_cp15.translation_table_base = value;
InvalidateTLB();
break;
case 3:
m_cp15.domain_access_control = value;
InvalidateTLB();
break;
case 5:
m_cp15.fault_status = value;
@@ -2346,8 +2334,12 @@ void ARMCore::WriteCP15(u32 opcode1, u32 crn, u32 crm, u32 opcode2, u32 value)
InvalidateTLB();
break;
case 13:
// ARM926 FCSE is specifically designed to switch low-address process spaces without flushing
// either cache or TLB: their tags contain the modified virtual address (MVA), which already
// includes PID[31:25]. IOS writes c13 at virtually every process switch. Retaining those MVA
// translations is both the architectural behavior and essential to avoiding a page-table walk
// storm. Explicit c8 maintenance above remains the sole guest-visible TLB invalidation path.
m_cp15.process_id = value;
InvalidateTLB();
break;
default:
break;
Source/Core/Core/IOS/Starlet/ARMCore.h
+8 -1
View File
@@ -164,7 +164,6 @@ public:
u64 GetJitExecutedInstructions() const;
u64 GetJitNativeExecutedInstructions() const;
size_t GetJitCompiledBlockCount() const;
u64 GetJitBlockExecutionCount() const;
u64 GetJitAddressTranslationCount() const;
u64 GetJitSlowReadCount() const;
u64 GetJitSlowWriteCount() const;
@@ -222,6 +221,14 @@ private:
static constexpr size_t TLB_ENTRY_COUNT = 4096;
static constexpr size_t INSTRUCTION_CACHE_ENTRY_COUNT = 16384;
static constexpr size_t GetTLBIndex(u32 virtual_page)
{
// FCSE places its seven-bit process identifier above bit 24 of the modified virtual address.
// Fold those bits into the direct-mapped software cache index so translations belonging to
// different IOS processes do not evict one another on every fast context switch.
return (virtual_page ^ (virtual_page >> 12)) & (TLB_ENTRY_COUNT - 1);
}
static bool IsValidMode(u32 mode);
static s32 SignExtend(u32 value, unsigned bits);
static AddResult AddWithCarry(u32 lhs, u32 rhs, bool carry);
Source/Core/Core/IOS/Starlet/ARMJitX64.cpp
+243 -41
View File
@@ -75,6 +75,7 @@ ARMJitX64::ARMJitX64(ARMCore& core) : m_core(core)
static_cast<s32>(reinterpret_cast<const u8*>(&m_core.m_yield_requested) - base);
m_executed_instructions_offset =
static_cast<s32>(reinterpret_cast<const u8*>(&m_core.m_executed_instructions) - base);
m_control_offset = static_cast<s32>(reinterpret_cast<const u8*>(&m_core.m_cp15.control) - base);
m_process_id_offset =
static_cast<s32>(reinterpret_cast<const u8*>(&m_core.m_cp15.process_id) - base);
m_tlb_generation_offset =
@@ -115,9 +116,14 @@ void ARMJitX64::InvalidateTranslationContext()
{
// A TLB invalidation changes which physical page a virtual PC resolves to, but it does not
// invalidate the ARM instruction cache. Keep already translated physical code and force the
// generated dispatcher to resolve the next virtual PC through the current page tables. This is
// particularly important for IOS, which flushes its TLB on every process switch.
std::ranges::fill(m_fast_entries, FastEntry{});
// generated dispatcher to resolve the next virtual PC through the current page tables. Every
// fast entry carries the ARMCore TLB generation, so the common invalidation is O(1). This is
// critical for IOS, which flushes its TLB on virtually every process switch; clearing the whole
// 65,536-entry array here previously consumed most of the host CPU. Generation zero is skipped by
// ARMCore. If the 32-bit counter eventually wraps back to one, clear ancient generation-one
// entries once to prevent an alias after the wrap.
if (m_core.m_tlb_generation == 1)
std::ranges::fill(m_fast_entries, FastEntry{});
}
u32 ARMJitX64::Run(u64 cycle_budget)
@@ -162,18 +168,39 @@ void ARMJitX64::GenerateDispatcher()
CMP(8, MatR(RAX), Imm8(0));
FixupBranch invalidated = J_CC(CC_NE, Jump::Near);
// Direct-mapped native block cache. The key includes CPSR.T in bit zero, matching the C++ map.
// Direct-mapped native block cache. The key includes CPSR.T in bit zero and uses the ARM926
// modified virtual address (MVA) for low FCSE addresses. Different IOS process identifiers can
// therefore retain independent hot entries without invalidating the cache on every c13 write.
MOV(32, R(EAX), MRegister(15));
MOV(32, R(ECX), MCPSR());
SHR(32, R(ECX), Imm8(5));
AND(32, R(ECX), Imm8(1));
OR(32, R(EAX), R(ECX));
MOV(32, R(ECX), MDisp(JIT_CORE, m_control_offset));
TEST(32, R(ECX), Imm32(1));
FixupBranch mmu_disabled = J_CC(CC_Z, Jump::Near);
CMP(32, R(EAX), Imm32(0x02000000));
FixupBranch outside_fcse = J_CC(CC_AE, Jump::Near);
MOV(32, R(ECX), MDisp(JIT_CORE, m_process_id_offset));
AND(32, R(ECX), Imm32(0xfe000000));
OR(32, R(EAX), R(ECX));
SetJumpTarget(mmu_disabled);
SetJumpTarget(outside_fcse);
// Fold the FCSE PID bits down into the cache index. A plain low-bit mask makes every process
// collide because PID occupies MVA[31:25]. The full MVA remains in FastEntry::key for safety.
MOV(32, R(EDX), R(EAX));
SHR(32, R(EDX), Imm8(1));
MOV(32, R(ECX), R(EAX));
SHR(32, R(ECX), Imm8(17));
XOR(32, R(EDX), R(ECX));
AND(32, R(EDX), Imm32(static_cast<u32>(FAST_ENTRY_COUNT - 1)));
SHL(64, R(RDX), Imm8(4));
MOV(64, R(R11), ImmPtr(m_fast_entries.data()));
ADD(64, R(R11), R(RDX));
MOV(32, R(R8), MDisp(JIT_CORE, m_tlb_generation_offset));
CMP(32, MDisp(R11, static_cast<s32>(offsetof(FastEntry, tlb_generation))), R(R8));
FixupBranch stale_generation = J_CC(CC_NE);
CMP(32, MDisp(R11, static_cast<s32>(offsetof(FastEntry, key))), R(EAX));
FixupBranch cache_miss = J_CC(CC_NE);
MOV(64, R(R11), MDisp(R11, static_cast<s32>(offsetof(FastEntry, entry))));
@@ -181,6 +208,7 @@ void ARMJitX64::GenerateDispatcher()
FixupBranch empty_entry = J_CC(CC_Z);
JMPptr(R(R11));
SetJumpTarget(stale_generation);
SetJumpTarget(cache_miss);
SetJumpTarget(empty_entry);
MOV(64, R(ABI_PARAM1), ImmPtr(this));
@@ -208,6 +236,20 @@ void ARMJitX64::GenerateDispatcher()
m_block_code_begin = AlignCode16();
}
u32 ARMJitX64::MakeFastEntryKey(u32 address, u32 control, u32 process_id)
{
constexpr u32 CP15_CONTROL_MMU = 1U << 0;
const u32 virtual_address = address & ~1U;
if ((control & CP15_CONTROL_MMU) != 0 && virtual_address < 0x02000000)
return address | (process_id & 0xfe000000);
return address;
}
size_t ARMJitX64::GetFastEntryIndex(u32 key)
{
return ((key >> 1) ^ (key >> 17)) & (FAST_ENTRY_COUNT - 1);
}
const u8* ARMJitX64::Dispatch(ARMJitX64* jit)
{
const bool thumb = (jit->m_core.m_cpsr & ARMCore::CPSR_T) != 0;
@@ -216,8 +258,11 @@ const u8* ARMJitX64::Dispatch(ARMJitX64* jit)
if (!block || !block->runnable)
return nullptr;
FastEntry& fast_entry = jit->m_fast_entries[(key >> 1) & (FAST_ENTRY_COUNT - 1)];
fast_entry.key = key;
const u32 fast_key =
MakeFastEntryKey(key, jit->m_core.m_cp15.control, jit->m_core.m_cp15.process_id);
FastEntry& fast_entry = jit->m_fast_entries[GetFastEntryIndex(fast_key)];
fast_entry.key = fast_key;
fast_entry.tlb_generation = jit->m_core.m_tlb_generation;
fast_entry.entry = block->entry;
return block->entry;
}
@@ -250,24 +295,113 @@ ARMJitX64::Block ARMJitX64::CompileBlock(u32 address, bool thumb)
const u8* const body = AlignCode16();
LoadRegisterCache();
MOV(8, MPCWritten(), Imm8(0));
const u8* const loop_body = GetCodePtr();
u32 current_address = address;
u32 last_instruction_address = address;
u32 instruction_count = 0;
u32 native_instruction_count = 0;
bool terminated = false;
bool dispatcher_exit = false;
bool block_exit_emitted = false;
// The physical address in the cache key covers the first ARM TLB granule. End the block before
// a 1 KiB boundary so a remapping of the following page can never reuse stale translated code.
const u32 translation_granule = address & ~0x3ffU;
const auto emit_loop_back = [&](u32 target, u32 fallthrough, u32 branch_address, u32 condition) {
// Account for one complete guest loop before deciding whether the remaining slice can execute
// another. Conditional back edges use the same accounting on both the taken and fallthrough
// paths. This keeps small scheduler budgets exact while avoiding a dispatcher/register-cache
// round trip on every taken iteration of a native-only loop.
if (condition != 0xe)
EmitConditionResult(condition);
SUB(32, R(JIT_DOWNCOUNT), Imm32(instruction_count));
ADD(32, R(JIT_EXECUTED_COUNT), Imm32(instruction_count));
ADD(32, R(JIT_NATIVE_COUNT), Imm32(native_instruction_count));
FixupBranch not_taken;
if (condition != 0xe)
{
TEST(32, R(EAX), R(EAX));
not_taken = J_CC(CC_Z, Jump::Near);
}
CMP(32, R(JIT_DOWNCOUNT), Imm32(instruction_count));
FixupBranch leave_budget = J_CC(CC_B, Jump::Near);
CMP(8, MYieldRequested(), Imm8(0));
FixupBranch leave_yield = J_CC(CC_NE, Jump::Near);
CMP(8, MWaitingForInterrupt(), Imm8(0));
FixupBranch leave_interrupt = J_CC(CC_NE, Jump::Near);
CMP(8, MWaitingForMemoryPoll(), Imm8(0));
FixupBranch leave_poll = J_CC(CC_NE, Jump::Near);
MOV(64, R(RAX), ImmPtr(&m_clear_pending));
CMP(8, MatR(RAX), Imm8(0));
FixupBranch leave_invalidation = J_CC(CC_NE, Jump::Near);
JMP(loop_body);
SetJumpTarget(leave_budget);
SetJumpTarget(leave_yield);
SetJumpTarget(leave_interrupt);
SetJumpTarget(leave_poll);
SetJumpTarget(leave_invalidation);
MOV(32, MRegister(15), Imm32(target));
FixupBranch state_ready;
if (condition != 0xe)
{
state_ready = J(Jump::Near);
SetJumpTarget(not_taken);
MOV(32, MRegister(15), Imm32(fallthrough));
SetJumpTarget(state_ready);
}
MOV(32, MInstructionAddress(), Imm32(branch_address));
MOV(8, MPCWritten(), Imm8(1));
FlushRegisterCache();
JMP(m_dispatcher);
terminated = true;
block_exit_emitted = true;
};
while (instruction_count < MAX_BLOCK_INSTRUCTIONS &&
(current_address & ~0x3ffU) == translation_granule)
{
last_instruction_address = current_address;
++instruction_count;
m_compile_instruction_count = instruction_count;
m_compile_native_instruction_count = native_instruction_count;
if (thumb)
{
const u16 instruction = m_core.FetchThumbInstruction(current_address);
// A branch back to the block start uses a real host loop with an exact per-iteration budget
// check. Other branches keep their ordinary terminal block semantics.
if ((instruction & 0xf800) == 0xe000)
{
const s32 offset = ARMCore::SignExtend(instruction & 0x07ff, 11) * 2;
const u32 target = current_address + 4 + static_cast<u32>(offset);
if (target == address)
{
++native_instruction_count;
emit_loop_back(target, current_address + 2, current_address, 0xe);
break;
}
}
else if ((instruction & 0xf000) == 0xd000)
{
const u32 condition = (instruction >> 8) & 0xf;
const s32 offset = ARMCore::SignExtend(instruction & 0xff, 8) * 2;
const u32 target = current_address + 4 + static_cast<u32>(offset);
bool canonical_memory_poll = false;
if (condition < 0xe && target == address && target + 4 == current_address)
{
const u16 load = m_core.FetchThumbInstruction(target);
const u16 compare = m_core.FetchThumbInstruction(target + 2);
canonical_memory_poll = (load & 0xf800) == 0x6800 && (compare & 0xf800) == 0x2800 &&
(compare & 0xff) == 0 && ((compare >> 8) & 7) == (load & 7);
}
if (condition < 0xe && target == address && !canonical_memory_poll)
{
++native_instruction_count;
emit_loop_back(target, current_address + 2, current_address, condition);
break;
}
}
if (!EmitDirectThumb(instruction, current_address, &terminated))
{
EmitFallbackThumb(instruction, current_address);
@@ -285,6 +419,22 @@ ARMJitX64::Block ARMJitX64::CompileBlock(u32 address, bool thumb)
else
{
const u32 instruction = m_core.FetchARMInstruction(current_address);
// Turn a branch to the block start into a bounded host loop. BL and every other branch keep
// the normal terminal path, including forward jumps whose target may be data or empty fill.
if ((instruction & 0x0e000000) == 0x0a000000 && (instruction & (1U << 24)) == 0 &&
(instruction >> 28) != 0xf)
{
const u32 condition = instruction >> 28;
const s32 offset = ARMCore::SignExtend((instruction & 0x00ffffff) << 2, 26);
const u32 target = current_address + 8 + static_cast<u32>(offset);
const bool slice_stable_timer_poll = condition != 0xe && target + 16 == current_address;
if (target == address && !slice_stable_timer_poll)
{
++native_instruction_count;
emit_loop_back(target, current_address + 4, current_address, condition);
break;
}
}
if (!EmitDirectARM(instruction, current_address, &terminated, &dispatcher_exit))
{
EmitFallbackARM(instruction, current_address);
@@ -304,9 +454,10 @@ ARMJitX64::Block ARMJitX64::CompileBlock(u32 address, bool thumb)
if (!terminated)
{
MOV(32, MRegister(15), Imm32(current_address));
MOV(32, MInstructionAddress(), Imm32(current_address - (thumb ? 2 : 4)));
MOV(32, MInstructionAddress(), Imm32(last_instruction_address));
}
EmitBlockExit(instruction_count, native_instruction_count, dispatcher_exit);
if (!block_exit_emitted)
EmitBlockExit(instruction_count, native_instruction_count, dispatcher_exit);
// The entry stub prevents a translated block from overrunning the scheduler's remaining ARM
// budget. Keeping it separate lets us know the final block size without patching machine code.
@@ -321,8 +472,7 @@ ARMJitX64::Block ARMJitX64::CompileBlock(u32 address, bool thumb)
.runnable = native_instruction_count != 0};
}
bool ARMJitX64::EmitDirectARM(u32 instruction, u32 address, bool* terminal,
bool* dispatcher_exit)
bool ARMJitX64::EmitDirectARM(u32 instruction, u32 address, bool* terminal, bool* dispatcher_exit)
{
// MCR p15, 0, Rd, c7, c10, 4 is ARM926 Drain Write Buffer. It is an ordering barrier, not an
// instruction-cache invalidation, and has no additional observable work in this single-host-
@@ -461,14 +611,29 @@ bool ARMJitX64::EmitDirectARM(u32 instruction, u32 address, bool* terminal,
return true;
}
if ((instruction & 0x0ffffff0) == 0x012fff10 ||
(instruction & 0x0ffffff0) == 0x012fff30)
if ((instruction & 0x0ffffff0) == 0x012fff10 || (instruction & 0x0ffffff0) == 0x012fff30)
{
if ((instruction >> 28) != 0xe)
return false;
const bool link = (instruction & 0x20) != 0;
const u32 rm = instruction & 0xf;
EmitExchangeBranch(rm == 15 ? Imm32(address + 8) : MRegister(rm), link, address + 4);
if (condition == 0xf)
return false;
if (condition == 0xe)
{
EmitExchangeBranch(rm == 15 ? Imm32(address + 8) : MRegister(rm), link, address + 4);
}
else
{
// Conditional BX LR is a hot IOS scheduler return. Keep both outcomes terminal so IRQ,
// yield and translation state are sampled at the same architectural boundary as before.
EmitConditionResult(condition);
TEST(32, R(EAX), R(EAX));
const FixupBranch not_taken = J_CC(CC_Z, Jump::Near);
EmitExchangeBranch(rm == 15 ? Imm32(address + 8) : MRegister(rm), link, address + 4);
const FixupBranch done = J(Jump::Near);
SetJumpTarget(not_taken);
MOV(32, MRegister(15), Imm32(address + 4));
SetJumpTarget(done);
}
MOV(32, MInstructionAddress(), Imm32(address));
MOV(8, MPCWritten(), Imm8(1));
*terminal = true;
@@ -511,6 +676,22 @@ bool ARMJitX64::EmitDirectARM(u32 instruction, u32 address, bool* terminal,
if (condition != 0xe)
{
// Predication applies to the complete single-data-transfer instruction, including address
// writeback and any exact MMIO/SRAM helper. A failed predicate skips all of it and continues in
// the current block. This covers the hot LDREQ/LDRNE/STRNE forms in IOS without weakening the
// existing address and device guards.
if (condition != 0xf && (instruction & 0x0c000000) == 0x04000000 &&
CanEmitARMMemory(instruction))
{
EmitConditionResult(condition);
TEST(32, R(EAX), R(EAX));
const FixupBranch predicate_failed = J_CC(CC_Z, Jump::Near);
const bool emitted = EmitARMMemory(instruction, address);
ASSERT(emitted);
SetJumpTarget(predicate_failed);
return true;
}
// ARM predication is pervasive in IOS's scheduler, allocator and error paths. Any data-
// processing form already proven safe below can share the native predicate guard; a failed
// condition simply skips the operation and continues at the next guest instruction.
@@ -597,7 +778,7 @@ bool ARMJitX64::CanEmitARMDataProcessing(u32 instruction) const
bool ARMJitX64::EmitARMMemory(u32 instruction, u32 address)
{
if (!m_fastmem_base)
if (!CanEmitARMMemory(instruction))
return false;
const bool preindex = (instruction & (1U << 24)) != 0;
@@ -609,13 +790,7 @@ bool ARMJitX64::EmitARMMemory(u32 instruction, u32 address)
const u32 rd = (instruction >> 12) & 0xf;
const bool register_offset = (instruction & (1U << 25)) != 0;
const u32 offset = instruction & 0xfff;
if (rd == 15 || (rn == 15 && (!preindex || writeback)) || (load && writeback && rn == rd))
return false;
if (register_offset && (instruction & (1U << 4)) != 0)
return false;
const u32 rm = instruction & 0xf;
if (register_offset && rm == 15)
return false;
// RAM fast paths and their interpreter side exits share an explicit architectural boundary.
// Materialize the two cached guest registers before either path can observe them.
@@ -720,6 +895,22 @@ bool ARMJitX64::EmitARMMemory(u32 instruction, u32 address)
return true;
}
bool ARMJitX64::CanEmitARMMemory(u32 instruction) const
{
if (!m_fastmem_base)
return false;
const bool preindex = (instruction & (1U << 24)) != 0;
const bool writeback = !preindex || (instruction & (1U << 21)) != 0;
const bool load = (instruction & (1U << 20)) != 0;
const u32 rn = (instruction >> 16) & 0xf;
const u32 rd = (instruction >> 12) & 0xf;
const bool register_offset = (instruction & (1U << 25)) != 0;
const u32 rm = instruction & 0xf;
return rd != 15 && !(rn == 15 && (!preindex || writeback)) && !(load && writeback && rn == rd) &&
!(register_offset && (instruction & (1U << 4)) != 0) && !(register_offset && rm == 15);
}
bool ARMJitX64::EmitARMHalfwordMemory(u32 instruction, u32 address)
{
if (!m_fastmem_base)
@@ -735,9 +926,8 @@ bool ARMJitX64::EmitARMHalfwordMemory(u32 instruction, u32 address)
const u32 type = (instruction >> 5) & 3;
const u32 rm = instruction & 0xf;
const u32 offset = ((instruction >> 4) & 0xf0) | (instruction & 0xf);
if (type == 0 || (!load && type != 1) || rd == 15 ||
(rn == 15 && (!preindex || writeback)) || (load && writeback && rn == rd) ||
(!immediate && rm == 15))
if (type == 0 || (!load && type != 1) || rd == 15 || (rn == 15 && (!preindex || writeback)) ||
(load && writeback && rn == rd) || (!immediate && rm == 15))
{
return false;
}
@@ -903,7 +1093,24 @@ bool ARMJitX64::EmitARMBlockTransfer(u32 instruction, u32 address, bool* termina
}
const FixupBranch direct_done = J(Jump::Near);
EmitARMMemorySlowPath(slow_paths, instruction, address, direct_done);
for (const FixupBranch& slow_path : slow_paths)
SetJumpTarget(slow_path);
// IOS keeps its kernel and IRQ stacks in Hollywood SRAM. Register-list transfers therefore miss
// the deliberately conservative SRAM fastmem aperture even though the instruction itself is
// already decoded. Calling the architectural transfer helper here preserves every ARMBus
// read/write (including split SRAM, boot0 protection and invalid apertures), but avoids returning
// to the generic ARM decoder and native dispatcher for each kernel push/pop. Transfers that load
// PC remain terminal through the encoding-derived flag above; ordinary stack transfers can keep
// executing the translated block.
FlushRegisterCache();
MOV(64, R(ABI_PARAM1), ImmPtr(this));
MOV(32, R(ABI_PARAM2), Imm32(instruction));
MOV(32, R(ABI_PARAM3), Imm32(address));
ABI_CallFunction(ExecuteUserBankBlockTransfer);
SetJumpTarget(direct_done);
LoadRegisterCache();
return true;
}
@@ -1212,8 +1419,8 @@ bool ARMJitX64::EmitDirectThumb(u16 instruction, u32 address, bool* terminal)
{
const u16 load = m_core.FetchThumbInstruction(target);
const u16 compare = m_core.FetchThumbInstruction(target + 2);
if ((load & 0xf800) == 0x6800 && (compare & 0xf800) == 0x2800 &&
(compare & 0xff) == 0 && ((compare >> 8) & 7) == (load & 7))
if ((load & 0xf800) == 0x6800 && (compare & 0xf800) == 0x2800 && (compare & 0xff) == 0 &&
((compare >> 8) & 7) == (load & 7))
{
return false;
}
@@ -1515,8 +1722,8 @@ bool ARMJitX64::EmitThumbMemory(u16 instruction, u32 address)
}
void ARMJitX64::EmitFastmemAddress(std::vector<FixupBranch>* slow_paths, u32 access_size,
u32 range_size, SRAMFastmemAccess sram_access,
bool arm_unaligned_word)
u32 range_size, SRAMFastmemAccess sram_access,
bool arm_unaligned_word)
{
if (range_size == 0)
range_size = access_size;
@@ -1533,6 +1740,8 @@ void ARMJitX64::EmitFastmemAddress(std::vector<FixupBranch>* slow_paths, u32 acc
MOV(32, R(EDX), R(EAX));
SHR(32, R(EDX), Imm8(10));
MOV(32, R(ECX), R(EDX));
SHR(32, R(ECX), Imm8(12));
XOR(32, R(ECX), R(EDX));
AND(32, R(ECX), Imm32(static_cast<u32>(ARMCore::TLB_ENTRY_COUNT - 1)));
IMUL(32, ECX, R(ECX), Imm32(sizeof(ARMCore::TLBEntry)));
MOV(64, R(R11), ImmPtr(m_core.m_tlb.data()));
@@ -2086,11 +2295,6 @@ void ARMJitX64::EmitBlockExit(u32 instruction_count, u32 native_instruction_coun
SUB(32, R(JIT_DOWNCOUNT), Imm32(instruction_count));
ADD(32, R(JIT_EXECUTED_COUNT), Imm32(instruction_count));
ADD(32, R(JIT_NATIVE_COUNT), Imm32(native_instruction_count));
// Diagnostic only: one generated increment per completed native block lets the live profiler
// distinguish short-block dispatch overhead from slow bus helpers. The dispatcher overwrites
// the host flags before observing them, so this does not affect guest execution.
MOV(64, R(RAX), ImmPtr(&m_block_execution_count));
ADD(64, MatR(RAX), Imm8(1));
JMP(dispatcher_exit ? m_dispatcher_exit : m_dispatcher);
}
@@ -2248,7 +2452,7 @@ u32 ARMJitX64::ReadMemorySlow(ARMJitX64* jit, u32 physical_address, u32 access_s
case SlowMemoryRegion::SRAM:
++jit->m_slow_sram_read_count;
++(physical_address < 0xfff00000 ? jit->m_slow_sram_low_access_count :
jit->m_slow_sram_high_access_count);
jit->m_slow_sram_high_access_count);
++jit->m_slow_sram_page_read_access_count[(physical_address & 0x1ffff) >> 12];
break;
case SlowMemoryRegion::MMIO:
@@ -2279,7 +2483,7 @@ void ARMJitX64::WriteMemorySlow(ARMJitX64* jit, u32 physical_address, u32 access
case SlowMemoryRegion::SRAM:
++jit->m_slow_sram_write_count;
++(physical_address < 0xfff00000 ? jit->m_slow_sram_low_access_count :
jit->m_slow_sram_high_access_count);
jit->m_slow_sram_high_access_count);
++jit->m_slow_sram_page_write_access_count[(physical_address & 0x1ffff) >> 12];
break;
case SlowMemoryRegion::MMIO:
@@ -2320,8 +2524,7 @@ void ARMJitX64::EnterSVCARM(ARMJitX64* jit, u32 address)
ARMCore& core = jit->m_core;
core.m_instruction_address = address;
core.m_pc_written = false;
core.EnterException(ARMCore::Mode::Supervisor, STARLET_EXCEPTION_VECTOR_BASE + 0x08,
address + 4);
core.EnterException(ARMCore::Mode::Supervisor, STARLET_EXCEPTION_VECTOR_BASE + 0x08, address + 4);
}
void ARMJitX64::EnterSVCThumb(ARMJitX64* jit, u32 address)
@@ -2329,8 +2532,7 @@ void ARMJitX64::EnterSVCThumb(ARMJitX64* jit, u32 address)
ARMCore& core = jit->m_core;
core.m_instruction_address = address;
core.m_pc_written = false;
core.EnterException(ARMCore::Mode::Supervisor, STARLET_EXCEPTION_VECTOR_BASE + 0x08,
address + 2);
core.EnterException(ARMCore::Mode::Supervisor, STARLET_EXCEPTION_VECTOR_BASE + 0x08, address + 2);
}
} // namespace IOS::LLE
Source/Core/Core/IOS/Starlet/ARMJitX64.h
+9 -17
View File
@@ -35,23 +35,13 @@ public:
u64 GetExecutedInstructions() const { return m_executed_instructions; }
u64 GetNativeExecutedInstructions() const { return m_native_executed_instructions; }
size_t GetCompiledBlockCount() const { return m_blocks.size(); }
u64 GetBlockExecutionCount() const { return m_block_execution_count; }
u64 GetAddressTranslationCount() const { return m_address_translation_count; }
u64 GetSlowReadCount() const { return m_slow_read_count; }
u64 GetSlowWriteCount() const { return m_slow_write_count; }
u64 GetSlowRAMAccessCount() const { return m_slow_ram_read_count + m_slow_ram_write_count; }
u64 GetSlowSRAMAccessCount() const
{
return m_slow_sram_read_count + m_slow_sram_write_count;
}
u64 GetSlowMMIOAccessCount() const
{
return m_slow_mmio_read_count + m_slow_mmio_write_count;
}
u64 GetSlowOtherAccessCount() const
{
return m_slow_other_read_count + m_slow_other_write_count;
}
u64 GetSlowSRAMAccessCount() const { return m_slow_sram_read_count + m_slow_sram_write_count; }
u64 GetSlowMMIOAccessCount() const { return m_slow_mmio_read_count + m_slow_mmio_write_count; }
u64 GetSlowOtherAccessCount() const { return m_slow_other_read_count + m_slow_other_write_count; }
u64 GetSlowSRAMLowAccessCount() const { return m_slow_sram_low_access_count; }
u64 GetSlowSRAMHighAccessCount() const { return m_slow_sram_high_access_count; }
u64 GetSlowSRAMReadAccessCount() const { return m_slow_sram_read_count; }
@@ -94,17 +84,20 @@ private:
struct FastEntry
{
u32 key = 0xffffffff;
u32 padding = 0;
u32 tlb_generation = 0;
const u8* entry = nullptr;
};
void PoisonMemory() override;
void GenerateDispatcher();
static u32 MakeFastEntryKey(u32 address, u32 control, u32 process_id);
static size_t GetFastEntryIndex(u32 key);
static const u8* Dispatch(ARMJitX64* jit);
Block* GetOrCompileBlock(u32 address);
Block CompileBlock(u32 address, bool thumb);
bool EmitDirectARM(u32 instruction, u32 address, bool* terminal, bool* dispatcher_exit);
bool CanEmitARMDataProcessing(u32 instruction) const;
bool CanEmitARMMemory(u32 instruction) const;
bool EmitARMMemory(u32 instruction, u32 address);
bool EmitARMHalfwordMemory(u32 instruction, u32 address);
bool EmitARMBlockTransfer(u32 instruction, u32 address, bool* terminal);
@@ -152,8 +145,7 @@ private:
static void ExecuteUserBankBlockTransfer(ARMJitX64* jit, u32 instruction, u32 address);
static void ExecuteThumbPushPop(ARMJitX64* jit, u16 instruction, u32 address);
static void ExceptionReturn(ARMJitX64* jit, u32 target);
static u32 ReadMemorySlow(ARMJitX64* jit, u32 physical_address, u32 access_size,
u32 byte_offset);
static u32 ReadMemorySlow(ARMJitX64* jit, u32 physical_address, u32 access_size, u32 byte_offset);
static void WriteMemorySlow(ARMJitX64* jit, u32 physical_address, u32 access_size, u32 value);
static u32 TranslateAddress(ARMJitX64* jit, u32 address);
static void EnterUndefinedARM(ARMJitX64* jit, u32 instruction, u32 address);
@@ -177,7 +169,6 @@ private:
u8* m_block_code_begin = nullptr;
u64 m_executed_instructions = 0;
u64 m_native_executed_instructions = 0;
u64 m_block_execution_count = 0;
u64 m_address_translation_count = 0;
u64 m_slow_read_count = 0;
u64 m_slow_write_count = 0;
@@ -203,6 +194,7 @@ private:
s32 m_waiting_for_memory_poll_offset = 0;
s32 m_yield_requested_offset = 0;
s32 m_executed_instructions_offset = 0;
s32 m_control_offset = 0;
s32 m_process_id_offset = 0;
s32 m_tlb_generation_offset = 0;
u32 m_compile_instruction_count = 0;
Source/Core/Core/IOS/Starlet/Starlet.cpp
+35 -39
View File
@@ -52,6 +52,7 @@ bool Starlet::Init(const std::string& dump_directory, std::string* error)
m_core->Reset(StarletMemory::BOOT_ROM_BASE);
m_next_jit_diagnostic_instruction = 25'000'000;
m_next_realtime_diagnostic_ms = Common::Timer::NowMs() + 1000;
m_realtime_diagnostic_slice_count = 0;
m_last_realtime_ppc_pc = 0;
m_realtime_ppc_pc_streak = 0;
m_ppc_context_logged = false;
@@ -196,8 +197,13 @@ void Starlet::RunSlice(s64 cycles_late)
(ipc_handshake_active ? ARM_IPC_SLICE_CYCLES : ARM_ACTIVE_SLICE_CYCLES);
m_core->RunCycles(arm_cycles);
const u64 now_ms = Common::Timer::NowMs();
if (now_ms >= m_next_realtime_diagnostic_ms)
// Keep host-clock reads and diagnostic formatting off the normal scheduler path.
const bool check_realtime_diagnostic =
++m_realtime_diagnostic_slice_count == REALTIME_DIAGNOSTIC_SLICE_INTERVAL;
if (check_realtime_diagnostic)
m_realtime_diagnostic_slice_count = 0;
const u64 now_ms = check_realtime_diagnostic ? Common::Timer::NowMs() : 0;
if (check_realtime_diagnostic && now_ms >= m_next_realtime_diagnostic_ms)
{
m_next_realtime_diagnostic_ms = now_ms + 1000;
const PowerPC::PowerPCState& ppc = m_system.GetPPCState();
@@ -211,8 +217,7 @@ void Starlet::RunSlice(s64 cycles_late)
"ppc-ctrl={:#04x} ppc-irq={:#010x}/{:#010x} arm-ctrl={:#04x} "
"arm-irq={:#010x}/{:#010x} Starlet-PC={:#010x} wait-int={} wait-mem={}",
ppc.pc, ppc.npc, instruction.valid, instruction.hex, ppc.msr.Hex, ppc.Exceptions,
ppc.downcount, ppc.gpr[1], LR(ppc), CTR(ppc),
pi.GetCause(), pi.GetMask(),
ppc.downcount, ppc.gpr[1], LR(ppc), CTR(ppc), pi.GetCause(), pi.GetMask(),
m_system.GetWiiIPC().ReadStarletRegister(0x04),
m_system.GetWiiIPC().ReadStarletRegister(0x30),
m_system.GetWiiIPC().ReadStarletRegister(0x34), ipc_arm_ctrl,
@@ -238,9 +243,9 @@ void Starlet::RunSlice(s64 cycles_late)
pi.GetMask());
for (u32 base = 0; base < 32; base += 4)
{
INFO_LOG_FMT(IOS, "Broadway GPR r{}={:#010x} r{}={:#010x} r{}={:#010x} r{}={:#010x}",
base, ppc.gpr[base], base + 1, ppc.gpr[base + 1], base + 2,
ppc.gpr[base + 2], base + 3, ppc.gpr[base + 3]);
INFO_LOG_FMT(IOS, "Broadway GPR r{}={:#010x} r{}={:#010x} r{}={:#010x} r{}={:#010x}", base,
ppc.gpr[base], base + 1, ppc.gpr[base + 1], base + 2, ppc.gpr[base + 2],
base + 3, ppc.gpr[base + 3]);
}
const u32 context_start = (ppc.pc - 0x40) & ~u32{3};
const u32 context_end = (ppc.pc + 0x80) & ~u32{3};
@@ -273,26 +278,22 @@ void Starlet::RunSlice(s64 cycles_late)
{
const u64 jit_executed = m_core->GetJitExecutedInstructions();
const u64 jit_native = m_core->GetJitNativeExecutedInstructions();
const u64 block_executions = m_core->GetJitBlockExecutionCount();
const u64 interval_fallbacks = m_core->TakeJitFallbackIntervalCount();
INFO_LOG_FMT(IOS,
"Starlet live JIT: instructions={} native={:.1f}% fallbacks={} blocks={} "
"block-runs={} insns-per-block={:.1f} translations={} slow-reads={} "
"slow-writes={} slow-regions=ram:{}/sram:{}/mmio:{}/other:{} "
"interval-fallbacks={} idle-polls={} ipc-armctrl={:#04x} "
"PC={:#010x} "
"CPSR={:#010x} irqflag={:#010x} irqmask={:#010x} fiqmask={:#010x}",
jit_executed, jit_executed == 0 ? 0.0 : 100.0 * jit_native / jit_executed,
m_core->GetJitFallbackInstructionCount(), m_core->GetJitCompiledBlockCount(),
block_executions,
block_executions == 0 ? 0.0 : static_cast<double>(jit_executed) / block_executions,
m_core->GetJitAddressTranslationCount(), m_core->GetJitSlowReadCount(),
m_core->GetJitSlowWriteCount(),
m_core->GetJitSlowRAMAccessCount(), m_core->GetJitSlowSRAMAccessCount(),
m_core->GetJitSlowMMIOAccessCount(), m_core->GetJitSlowOtherAccessCount(),
interval_fallbacks, m_core->GetMemoryPollEntryCount(), ipc_arm_ctrl,
m_core->GetRegister(15), m_core->GetCPSR(),
m_system.GetWiiIPC().ReadStarletRegister(0x38),
"Starlet live JIT: instructions={} native={:.1f}% fallbacks={} blocks={} "
"translations={} slow-reads={} "
"slow-writes={} slow-regions=ram:{}/sram:{}/mmio:{}/other:{} "
"interval-fallbacks={} idle-polls={} ipc-armctrl={:#04x} "
"PC={:#010x} "
"CPSR={:#010x} irqflag={:#010x} irqmask={:#010x} fiqmask={:#010x}",
jit_executed, jit_executed == 0 ? 0.0 : 100.0 * jit_native / jit_executed,
m_core->GetJitFallbackInstructionCount(), m_core->GetJitCompiledBlockCount(),
m_core->GetJitAddressTranslationCount(), m_core->GetJitSlowReadCount(),
m_core->GetJitSlowWriteCount(), m_core->GetJitSlowRAMAccessCount(),
m_core->GetJitSlowSRAMAccessCount(), m_core->GetJitSlowMMIOAccessCount(),
m_core->GetJitSlowOtherAccessCount(), interval_fallbacks,
m_core->GetMemoryPollEntryCount(), ipc_arm_ctrl, m_core->GetRegister(15),
m_core->GetCPSR(), m_system.GetWiiIPC().ReadStarletRegister(0x38),
m_system.GetWiiIPC().ReadStarletRegister(0x3c),
m_system.GetWiiIPC().ReadStarletRegister(0x40));
std::array<u32, 4> hot_sram_read_pages{};
@@ -318,21 +319,18 @@ void Starlet::RunSlice(s64 cycles_late)
{
rank_sram_page(page, m_core->GetJitSlowSRAMPageReadAccessCount(page), hot_sram_read_pages,
hot_sram_read_page_counts);
rank_sram_page(page, m_core->GetJitSlowSRAMPageWriteAccessCount(page),
hot_sram_write_pages, hot_sram_write_page_counts);
rank_sram_page(page, m_core->GetJitSlowSRAMPageWriteAccessCount(page), hot_sram_write_pages,
hot_sram_write_page_counts);
}
INFO_LOG_FMT(IOS,
"Starlet slow SRAM aliases=low:{}/high:{} reads:{}/writes:{} "
"hot-reads={:#04x}:{}, {:#04x}:{}, {:#04x}:{}, {:#04x}:{} "
"hot-writes={:#04x}:{}, {:#04x}:{}, {:#04x}:{}, {:#04x}:{}",
m_core->GetJitSlowSRAMLowAccessCount(),
m_core->GetJitSlowSRAMHighAccessCount(),
m_core->GetJitSlowSRAMReadAccessCount(),
m_core->GetJitSlowSRAMWriteAccessCount(), hot_sram_read_pages[0],
hot_sram_read_page_counts[0], hot_sram_read_pages[1],
hot_sram_read_page_counts[1], hot_sram_read_pages[2],
hot_sram_read_page_counts[2], hot_sram_read_pages[3],
hot_sram_read_page_counts[3], hot_sram_write_pages[0],
m_core->GetJitSlowSRAMLowAccessCount(), m_core->GetJitSlowSRAMHighAccessCount(),
m_core->GetJitSlowSRAMReadAccessCount(), m_core->GetJitSlowSRAMWriteAccessCount(),
hot_sram_read_pages[0], hot_sram_read_page_counts[0], hot_sram_read_pages[1],
hot_sram_read_page_counts[1], hot_sram_read_pages[2], hot_sram_read_page_counts[2],
hot_sram_read_pages[3], hot_sram_read_page_counts[3], hot_sram_write_pages[0],
hot_sram_write_page_counts[0], hot_sram_write_pages[1],
hot_sram_write_page_counts[1], hot_sram_write_pages[2],
hot_sram_write_page_counts[2], hot_sram_write_pages[3],
@@ -356,11 +354,9 @@ void Starlet::RunSlice(s64 cycles_late)
}
m_jit_context_logged = true;
}
for (const ARMCore::HotPCSample& sample :
m_core->TakeHotJitFallbackIntervalSamples(6))
for (const ARMCore::HotPCSample& sample : m_core->TakeHotJitFallbackIntervalSamples(6))
{
INFO_LOG_FMT(IOS,
"Starlet interval fallback PC {:#010x} {} instruction={:#010x} samples={}",
INFO_LOG_FMT(IOS, "Starlet interval fallback PC {:#010x} {} instruction={:#010x} samples={}",
sample.address, sample.thumb ? "Thumb" : "ARM", sample.instruction,
sample.samples);
}
Source/Core/Core/IOS/Starlet/Starlet.h
+4
View File
@@ -71,6 +71,9 @@ private:
// idle quantum avoids hundreds of thousands of scheduler callbacks per second
// while bounding interrupt wake latency.
static constexpr u64 ARM_IDLE_SLICE_CYCLES = ARM_CLOCK / 10000;
// Do not read the host clock on every short scheduler slice: that would perturb the exact
// workload being measured. One check per 1024 callbacks keeps profiling overhead negligible.
static constexpr u64 REALTIME_DIAGNOSTIC_SLICE_INTERVAL = 1024;
static void RunCallback(Core::System& system, u64 userdata, s64 cycles_late);
void RunSlice(s64 cycles_late);
@@ -81,6 +84,7 @@ private:
CoreTiming::EventType* m_run_event = nullptr;
u64 m_next_jit_diagnostic_instruction = 25'000'000;
u64 m_next_realtime_diagnostic_ms = 0;
u64 m_realtime_diagnostic_slice_count = 0;
u32 m_last_realtime_ppc_pc = 0;
u32 m_realtime_ppc_pc_streak = 0;
bool m_ppc_context_logged = false;
Source/Core/Core/IOS/Starlet/StarletMemory.cpp
+74 -1
View File
@@ -572,6 +572,7 @@ void StarletMemory::InitSDCard()
void StarletMemory::Reset()
{
const std::lock_guard wifi_sdio_lock(m_wifi_sdio_register_mutex);
for (auto& wiimote : m_wiimotes)
{
if (wiimote)
@@ -580,6 +581,7 @@ void StarletMemory::Reset()
}
m_sram.fill(0);
m_registers.clear();
ClearRegisterCache();
m_nand_overlay.clear();
m_nand_control_before_write = 0;
ResetNANDOperationState();
@@ -659,8 +661,11 @@ void StarletMemory::Reset()
void StarletMemory::DoState(PointerWrap& p)
{
const std::lock_guard wifi_sdio_lock(m_wifi_sdio_register_mutex);
p.DoArray(m_sram);
p.Do(m_registers);
if (p.IsReadMode())
ClearRegisterCache();
p.Do(m_nand_overlay);
p.DoArray(m_nand_program_data);
p.Do(m_nand_control_before_write);
@@ -908,8 +913,18 @@ u8 StarletMemory::ReadMapped8(u32 address) const
const u32 offset = GetSRAMOffset(address);
return offset == INVALID_SRAM_OFFSET ? 0 : m_sram[offset];
}
auto& cache_entry = m_register_cache[GetRegisterCacheIndex(address)];
const u64 cached = cache_entry.load(std::memory_order_relaxed);
if ((cached & REGISTER_CACHE_VALID) != 0 &&
static_cast<u32>((cached >> 8) & 0xffffffff) == address)
{
return static_cast<u8>(cached);
}
const auto it = m_registers.find(address);
return it == m_registers.end() ? 0 : it->second;
const u8 value = it == m_registers.end() ? 0 : it->second;
cache_entry.store(EncodeRegisterCacheEntry(address, value), std::memory_order_relaxed);
return value;
}
void StarletMemory::WriteMapped8(u32 address, u8 value)
@@ -927,6 +942,14 @@ void StarletMemory::WriteMapped8(u32 address, u8 value)
return;
}
m_registers[address] = value;
m_register_cache[GetRegisterCacheIndex(address)].store(EncodeRegisterCacheEntry(address, value),
std::memory_order_relaxed);
}
void StarletMemory::ClearRegisterCache()
{
for (auto& entry : m_register_cache)
entry.store(0, std::memory_order_relaxed);
}
u32 StarletMemory::ReadRegister(u32 address) const
@@ -2387,6 +2410,7 @@ void StarletMemory::HandleWiFiSDIOWrite(u32 address)
u8 StarletMemory::ReadWiFiSDIOByte(u32 function, u32 address) const
{
const std::lock_guard lock(m_wifi_sdio_register_mutex);
if (function == 0)
{
// The three CIS pointers are little-endian 24-bit addresses in the common
@@ -2531,6 +2555,7 @@ u8 StarletMemory::ReadWiFiSDIOByte(u32 function, u32 address) const
void StarletMemory::WriteWiFiSDIOByte(u32 function, u32 address, u8 value)
{
const std::lock_guard lock(m_wifi_sdio_register_mutex);
address &= 0x1ffff;
if (function == 1 && address < 0x10000)
{
@@ -2567,6 +2592,7 @@ void StarletMemory::WriteWiFiSDIOByte(u32 function, u32 address, u8 value)
u32 StarletMemory::GetWiFiSDIOBackplaneAddress(u32 address) const
{
const std::lock_guard lock(m_wifi_sdio_register_mutex);
const auto get_window_byte = [this](u32 register_address, u8 default_value) {
const auto it = m_wifi_sdio_registers.find((1U << 17) | register_address);
return it == m_wifi_sdio_registers.end() ? default_value : it->second;
@@ -3121,6 +3147,15 @@ u8 StarletMemory::Read8(u32 address)
if (IsBootROMAddress(address))
return m_boot_rom[GetBootROMOffset(address)];
// SRAM has no byte-access side effects. Resolve it before the large MMIO decoder; this retains
// the live boot0 overlay and A/B split mapping while avoiding dozens of unrelated device tests
// in the IOS scheduler's hottest memory path.
if (IsSRAMWindowAddress(address))
{
const u32 offset = GetSRAMOffset(address);
return offset == INVALID_SRAM_OFFSET ? 0 : m_sram[offset];
}
if (IsMemoryControllerIndirectRegister(address))
{
const u16 value = ReadMemoryControllerHalfword(address);
@@ -3294,6 +3329,16 @@ void StarletMemory::Write8(u32 address, u8 value)
if (IsBootROMAddress(address))
return;
// Unlike Hollywood MMIO, SRAM writes have no completion or interrupt side effects. Handle the
// exact mapped byte here so normal IOS data stores do not traverse the complete device decoder.
if (IsSRAMWindowAddress(address))
{
const u32 offset = GetSRAMOffset(address);
if (offset != INVALID_SRAM_OFFSET)
m_sram[offset] = value;
return;
}
// NAND_CTRL is normally written a word at a time. Keep its completed value
// while the incoming acknowledgement word is assembled byte by byte, since
// that write is a command rather than a replacement for the readable
@@ -3500,6 +3545,19 @@ void StarletMemory::Write16(u32 address, u16 value)
return;
}
if (!IsBootROMAddress(address) && !IsBootROMAddress(address + 1) &&
IsSRAMWindowAddress(address) && IsSRAMWindowAddress(address + 1))
{
const u32 offset = GetSRAMOffset(address);
const u32 end_offset = GetSRAMOffset(address + 1);
if (offset != INVALID_SRAM_OFFSET && end_offset == offset + 1)
{
m_sram[offset] = static_cast<u8>(value >> 8);
m_sram[end_offset] = static_cast<u8>(value);
return;
}
}
ARMBus::Write16(address, value);
}
@@ -3526,6 +3584,21 @@ void StarletMemory::Write32(u32 address, u32 value)
return;
}
if (!IsBootROMAddress(address) && !IsBootROMAddress(address + 3) &&
IsSRAMWindowAddress(address) && IsSRAMWindowAddress(address + 3))
{
const u32 offset = GetSRAMOffset(address);
const u32 end_offset = GetSRAMOffset(address + 3);
if (offset != INVALID_SRAM_OFFSET && end_offset == offset + 3)
{
m_sram[offset] = static_cast<u8>(value >> 24);
m_sram[offset + 1] = static_cast<u8>(value >> 16);
m_sram[offset + 2] = static_cast<u8>(value >> 8);
m_sram[end_offset] = static_cast<u8>(value);
return;
}
}
ARMBus::Write32(address, value);
}
Source/Core/Core/IOS/Starlet/StarletMemory.h
+26
View File
@@ -4,10 +4,12 @@
#pragma once
#include <array>
#include <atomic>
#include <cstddef>
#include <deque>
#include <map>
#include <memory>
#include <mutex>
#include <optional>
#include <string>
#include <vector>
@@ -185,6 +187,20 @@ private:
u32 GetGPIOInput() const;
void UpdateGPIOInterrupt();
u32 GetTimer() const;
void ClearRegisterCache();
static constexpr size_t REGISTER_CACHE_SIZE = 4096;
static constexpr u64 REGISTER_CACHE_VALID = 1ULL << 40;
static constexpr size_t GetRegisterCacheIndex(u32 address)
{
return (address ^ (address >> 12)) & (REGISTER_CACHE_SIZE - 1);
}
static constexpr u64 EncodeRegisterCacheEntry(u32 address, u8 value)
{
return REGISTER_CACHE_VALID | (static_cast<u64>(address) << 8) | value;
}
Core::System& m_system;
std::array<u8, BOOT_ROM_SIZE> m_boot_rom{};
@@ -193,6 +209,11 @@ private:
File::IOFile m_nand;
File::IOFile m_sd_card;
std::map<u32, u8> m_registers;
// Hollywood exposes a sparse byte-addressed register file. IOS repeatedly polls a small hot
// subset, for which a tree lookup per byte is disproportionately expensive. Keep the map as the
// source of truth and use tagged entries only as a transparent read-through cache. Atomic slots
// allow read hits from Broadway and Starlet without adding a lock to this hot path.
mutable std::array<std::atomic<u64>, REGISTER_CACHE_SIZE> m_register_cache{};
std::map<u32, NANDPage> m_nand_overlay;
NANDPage m_nand_program_data{};
u32 m_nand_control_before_write = 0;
@@ -214,6 +235,11 @@ private:
u32 m_sd_block_length = 512;
u32 m_sdhc_status_before_write = 0;
u32 m_wifi_sdio_status_before_write = 0;
// Broadway MMIO and the scheduled Starlet slice can both reach the shared Hollywood SDIO
// controller in dual-core mode. Protect the sparse register file while IOS resets a function.
// This is recursive because byte accesses consult the backplane-window registers through a
// helper that takes the same lock.
mutable std::recursive_mutex m_wifi_sdio_register_mutex;
std::map<u32, u8> m_wifi_sdio_registers;
std::deque<u8> m_wifi_sdio_pio_read_data;
u32 m_wifi_sdio_pio_write_function = 0;
Source/Core/Core/PowerPC/MMU.cpp
+55 -10
View File
@@ -285,8 +285,8 @@ T MMU::ReadFromHardware(u32 em_address)
}
else
{
m_ppc_state.dCache.Read(m_memory, em_address, &value, sizeof(T),
HID0(m_ppc_state).DLOCK || flag != XCheckTLBFlag::Read);
value = m_ppc_state.dCache.ReadMainMemoryValue<T, false>(
m_memory, em_address, HID0(m_ppc_state).DLOCK || flag != XCheckTLBFlag::Read);
}
return bswap(value);
@@ -304,8 +304,9 @@ T MMU::ReadFromHardware(u32 em_address)
}
else
{
m_ppc_state.dCache.Read(m_memory, em_address + 0x10000000, &value, sizeof(T),
HID0(m_ppc_state).DLOCK || flag != XCheckTLBFlag::Read);
value = m_ppc_state.dCache.ReadMainMemoryValue<T, true>(m_memory, em_address + 0x10000000,
HID0(m_ppc_state).DLOCK ||
flag != XCheckTLBFlag::Read);
}
return bswap(value);
@@ -514,7 +515,29 @@ void MMU::WriteToHardware(u32 em_address, const u32 data, const u32 size)
em_address &= m_memory.GetRamMask();
if (m_ppc_state.m_enable_dcache && !wi)
m_ppc_state.dCache.Write(m_memory, em_address, &swapped_data, size, HID0(m_ppc_state).DLOCK);
{
switch (size)
{
case 1:
m_ppc_state.dCache.WriteMainMemoryValue<u8, false>(
m_memory, em_address, static_cast<u8>(swapped_data), HID0(m_ppc_state).DLOCK);
break;
case 2:
m_ppc_state.dCache.WriteMainMemoryValue<u16, false>(
m_memory, em_address, static_cast<u16>(swapped_data), HID0(m_ppc_state).DLOCK);
break;
case 4:
m_ppc_state.dCache.WriteMainMemoryValue<u32, false>(m_memory, em_address, swapped_data,
HID0(m_ppc_state).DLOCK);
break;
default:
// Page-boundary splitting can legally produce a three-byte fragment. Keep the typed fast
// paths for normal accesses and preserve the generic byte-count behavior for that rarity.
m_ppc_state.dCache.Write(m_memory, em_address, &swapped_data, size,
HID0(m_ppc_state).DLOCK);
break;
}
}
if (!m_ppc_state.m_enable_dcache || wi || flag != XCheckTLBFlag::Write)
std::memcpy(&m_memory.GetRAM()[em_address], &swapped_data, size);
@@ -529,8 +552,27 @@ void MMU::WriteToHardware(u32 em_address, const u32 data, const u32 size)
if (m_ppc_state.m_enable_dcache && !wi)
{
m_ppc_state.dCache.Write(m_memory, em_address + 0x10000000, &swapped_data, size,
HID0(m_ppc_state).DLOCK);
switch (size)
{
case 1:
m_ppc_state.dCache.WriteMainMemoryValue<u8, true>(m_memory, em_address + 0x10000000,
static_cast<u8>(swapped_data),
HID0(m_ppc_state).DLOCK);
break;
case 2:
m_ppc_state.dCache.WriteMainMemoryValue<u16, true>(m_memory, em_address + 0x10000000,
static_cast<u16>(swapped_data),
HID0(m_ppc_state).DLOCK);
break;
case 4:
m_ppc_state.dCache.WriteMainMemoryValue<u32, true>(m_memory, em_address + 0x10000000,
swapped_data, HID0(m_ppc_state).DLOCK);
break;
default:
m_ppc_state.dCache.Write(m_memory, em_address + 0x10000000, &swapped_data, size,
HID0(m_ppc_state).DLOCK);
break;
}
}
if (!m_ppc_state.m_enable_dcache || wi || flag != XCheckTLBFlag::Write)
@@ -706,7 +748,8 @@ template <std::unsigned_integral T>
T MMU::Read(const u32 address)
{
T var = ReadFromHardware<XCheckTLBFlag::Read, T>(address);
Memcheck(address, var, false, sizeof(T));
if (m_power_pc.GetMemChecks().HasAny())
Memcheck(address, var, false, sizeof(T));
return var;
}
template u8 MMU::Read<u8>(const u32 address);
@@ -762,7 +805,8 @@ template std::optional<ReadResult<u64>> MMU::HostTryRead<u64>(const Core::CPUThr
template <std::unsigned_integral T>
void MMU::Write(const Common::MakeAtLeastU32<T> var, const u32 address)
{
Memcheck(address, var, true, sizeof(T));
if (m_power_pc.GetMemChecks().HasAny())
Memcheck(address, var, true, sizeof(T));
WriteToHardware<XCheckTLBFlag::Write>(address, var, sizeof(T));
}
template void MMU::Write<u8>(const u32 var, const u32 address);
@@ -771,7 +815,8 @@ template void MMU::Write<u32>(const u32 var, const u32 address);
template <>
void MMU::Write<u64>(const u64 var, const u32 address)
{
Memcheck(address, var, true, 8);
if (m_power_pc.GetMemChecks().HasAny())
Memcheck(address, var, true, 8);
WriteToHardware<XCheckTLBFlag::Write>(address, static_cast<u32>(var >> 32), 4);
WriteToHardware<XCheckTLBFlag::Write>(address + sizeof(u32), static_cast<u32>(var), 4);
}
Source/Core/Core/PowerPC/PPCCache.cpp
+120 -71
View File
@@ -18,13 +18,6 @@ namespace PowerPC
{
namespace
{
constexpr std::array<u32, 8> s_plru_mask{
11, 11, 19, 19, 37, 37, 69, 69,
};
constexpr std::array<u32, 8> s_plru_value{
11, 3, 17, 1, 36, 4, 64, 0,
};
constexpr std::array<u32, 255> s_way_from_valid = [] {
std::array<u32, 255> data{};
for (size_t m = 0; m < data.size(); m++)
@@ -120,6 +113,68 @@ void Cache::Init(Memory::MemoryManager& memory)
Reset();
}
u32 Cache::LoadCacheLine(Memory::MemoryManager& memory, u32 addr, u32 set)
{
u32 way;
if (valid[set] != 0xff)
way = s_way_from_valid[valid[set]];
else
way = s_way_from_plru[plru[set]];
if (valid[set] & (1 << way))
{
// Store the evicted line back to main memory before replacing its lookup-table entry.
if (modified[set] & (1 << way))
memory.CopyToEmu(addrs[set][way], data[set][way].data(), 32);
if (addrs[set][way] & CACHE_VMEM_BIT)
lookup_table_vmem[(addrs[set][way] & memory.GetFakeVMemMask()) >> 5] = 0xff;
else if (addrs[set][way] & CACHE_EXRAM_BIT)
lookup_table_ex[(addrs[set][way] & memory.GetExRamMask()) >> 5] = 0xff;
else
lookup_table[(addrs[set][way] & memory.GetRamMask()) >> 5] = 0xff;
}
memory.CopyFromEmu(data[set][way].data(), addr, 32);
if (addr & CACHE_VMEM_BIT)
lookup_table_vmem[(addr & memory.GetFakeVMemMask()) >> 5] = way;
else if (addr & CACHE_EXRAM_BIT)
lookup_table_ex[(addr & memory.GetExRamMask()) >> 5] = way;
else
lookup_table[(addr & memory.GetRamMask()) >> 5] = way;
addrs[set][way] = addr;
valid[set] |= 1 << way;
modified[set] &= ~(1 << way);
return way;
}
DOLPHIN_FORCE_INLINE std::pair<u32, u32> Cache::GetCache(Memory::MemoryManager& memory, u32 addr,
bool locked)
{
addr &= ~31U;
const u32 set = (addr >> 5) & 0x7f;
u32 way;
if (addr & CACHE_VMEM_BIT)
way = lookup_table_vmem[(addr & memory.GetFakeVMemMask()) >> 5];
else if (addr & CACHE_EXRAM_BIT)
way = lookup_table_ex[(addr & memory.GetExRamMask()) >> 5];
else
way = lookup_table[(addr & memory.GetRamMask()) >> 5];
// A hit is the overwhelmingly common path. Keep replacement, write-back and line filling out of
// this function so the compiler can inline the lookup into typed JIT reads and writes.
if (way == 0xff && !locked)
way = LoadCacheLine(memory, addr, set);
if (way != 0xff)
plru[set] = (plru[set] & ~PLRU_MASK[way]) | PLRU_VALUE[way];
return {set, way};
}
void InstructionCache::Init(Memory::MemoryManager& memory)
{
if (!m_config_callback_id)
@@ -205,70 +260,6 @@ void Cache::Touch(Memory::MemoryManager& memory, u32 addr, bool store)
GetCache(memory, addr, false);
}
std::pair<u32, u32> Cache::GetCache(Memory::MemoryManager& memory, u32 addr, bool locked)
{
addr &= ~31;
u32 set = (addr >> 5) & 0x7f;
u32 way;
if (addr & CACHE_VMEM_BIT)
{
way = lookup_table_vmem[(addr & memory.GetFakeVMemMask()) >> 5];
}
else if (addr & CACHE_EXRAM_BIT)
{
way = lookup_table_ex[(addr & memory.GetExRamMask()) >> 5];
}
else
{
way = lookup_table[(addr & memory.GetRamMask()) >> 5];
}
// load to the cache
if (!locked && way == 0xff)
{
// select a way
if (valid[set] != 0xff)
way = s_way_from_valid[valid[set]];
else
way = s_way_from_plru[plru[set]];
if (valid[set] & (1 << way))
{
// store the cache back to main memory
if (modified[set] & (1 << way))
memory.CopyToEmu(addrs[set][way], data[set][way].data(), 32);
if (addrs[set][way] & CACHE_VMEM_BIT)
lookup_table_vmem[(addrs[set][way] & memory.GetFakeVMemMask()) >> 5] = 0xff;
else if (addrs[set][way] & CACHE_EXRAM_BIT)
lookup_table_ex[(addrs[set][way] & memory.GetExRamMask()) >> 5] = 0xff;
else
lookup_table[(addrs[set][way] & memory.GetRamMask()) >> 5] = 0xff;
}
// load
memory.CopyFromEmu(data[set][way].data(), (addr & ~0x1f), 32);
if (addr & CACHE_VMEM_BIT)
lookup_table_vmem[(addr & memory.GetFakeVMemMask()) >> 5] = way;
else if (addr & CACHE_EXRAM_BIT)
lookup_table_ex[(addr & memory.GetExRamMask()) >> 5] = way;
else
lookup_table[(addr & memory.GetRamMask()) >> 5] = way;
addrs[set][way] = addr;
valid[set] |= (1 << way);
modified[set] &= ~(1 << way);
}
// update plru
if (way != 0xff)
plru[set] = (plru[set] & ~s_plru_mask[way]) | s_plru_value[way];
return {set, way};
}
void Cache::Read(Memory::MemoryManager& memory, u32 addr, void* buffer, u32 len, bool locked)
{
auto* value = static_cast<u8*>(buffer);
@@ -296,6 +287,31 @@ void Cache::Read(Memory::MemoryManager& memory, u32 addr, void* buffer, u32 len,
}
}
template <typename T>
T Cache::ReadValue(Memory::MemoryManager& memory, u32 addr, bool locked)
{
T value;
const u32 offset_in_block = addr & 31;
if (offset_in_block <= 32 - sizeof(T))
{
const auto [set, way] = GetCache(memory, addr, locked);
if (way != 0xff)
{
std::memcpy(&value, reinterpret_cast<const u8*>(data[set][way].data()) + offset_in_block,
sizeof(T));
}
else
{
memory.CopyFromEmu(&value, addr, sizeof(T));
}
return value;
}
// Only an unaligned access at the end of a cache line needs the generic split path.
Read(memory, addr, &value, sizeof(T), locked);
return value;
}
void Cache::Write(Memory::MemoryManager& memory, u32 addr, const void* buffer, u32 len, bool locked)
{
auto* value = static_cast<const u8*>(buffer);
@@ -324,6 +340,39 @@ void Cache::Write(Memory::MemoryManager& memory, u32 addr, const void* buffer, u
}
}
template <typename T>
void Cache::WriteValue(Memory::MemoryManager& memory, u32 addr, T value, bool locked)
{
const u32 offset_in_block = addr & 31;
if (offset_in_block <= 32 - sizeof(T))
{
const auto [set, way] = GetCache(memory, addr, locked);
if (way != 0xff)
{
std::memcpy(reinterpret_cast<u8*>(data[set][way].data()) + offset_in_block, &value,
sizeof(T));
modified[set] |= 1 << way;
}
else
{
memory.CopyToEmu(addr, &value, sizeof(T));
}
return;
}
// Only an unaligned access at the end of a cache line needs the generic split path.
Write(memory, addr, &value, sizeof(T), locked);
}
template u8 Cache::ReadValue<u8>(Memory::MemoryManager&, u32, bool);
template u16 Cache::ReadValue<u16>(Memory::MemoryManager&, u32, bool);
template u32 Cache::ReadValue<u32>(Memory::MemoryManager&, u32, bool);
template u64 Cache::ReadValue<u64>(Memory::MemoryManager&, u32, bool);
template void Cache::WriteValue<u8>(Memory::MemoryManager&, u32, u8, bool);
template void Cache::WriteValue<u16>(Memory::MemoryManager&, u32, u16, bool);
template void Cache::WriteValue<u32>(Memory::MemoryManager&, u32, u32, bool);
template void Cache::WriteValue<u64>(Memory::MemoryManager&, u32, u64, bool);
void Cache::DoState(Memory::MemoryManager& memory, PointerWrap& p)
{
if (p.IsReadMode())
Source/Core/Core/PowerPC/PPCCache.h
+83 -1
View File
@@ -4,11 +4,13 @@
#pragma once
#include <array>
#include <cstring>
#include <optional>
#include <vector>
#include "Common/CommonTypes.h"
#include "Common/Config/Config.h"
#include "Common/Inline.h"
class JitInterface;
namespace Memory
@@ -33,6 +35,13 @@ constexpr u32 CACHE_VMEM_BIT = 0x20000000;
struct Cache
{
static constexpr std::array<u8, CACHE_WAYS> PLRU_MASK{
11, 11, 19, 19, 37, 37, 69, 69,
};
static constexpr std::array<u8, CACHE_WAYS> PLRU_VALUE{
11, 3, 17, 1, 36, 4, 64, 0,
};
std::array<std::array<std::array<u32, CACHE_BLOCK_SIZE>, CACHE_WAYS>, CACHE_SETS> data{};
// Stores the 32-byte aligned address of the start of each cache block. This consists of the cache
@@ -58,7 +67,80 @@ struct Cache
void FlushAll(Memory::MemoryManager& memory);
std::pair<u32, u32> GetCache(Memory::MemoryManager& memory, u32 addr, bool locked);
u32 LoadCacheLine(Memory::MemoryManager& memory, u32 addr, u32 set);
DOLPHIN_FORCE_INLINE std::pair<u32, u32> GetCache(Memory::MemoryManager& memory, u32 addr,
bool locked);
template <typename T>
T ReadValue(Memory::MemoryManager& memory, u32 addr, bool locked);
template <typename T>
void WriteValue(Memory::MemoryManager& memory, u32 addr, T value, bool locked);
// MMU has already classified and masked these addresses. Keeping this overwhelmingly common
// path inline avoids repeating the generic cache-region tests and the typed helper call for
// every Broadway JIT load/store while preserving the exact cache contents and PLRU behavior.
template <typename T, bool exram>
DOLPHIN_FORCE_INLINE T ReadMainMemoryValue(Memory::MemoryManager& memory, u32 addr, bool locked)
{
T value;
const u32 offset_in_block = addr & 31;
if (offset_in_block > 32 - sizeof(T))
{
Read(memory, addr, &value, sizeof(T), locked);
return value;
}
const u32 line_addr = addr & ~31U;
const u32 set = (line_addr >> 5) & 0x7f;
const u32 lookup_index = exram ? ((line_addr & 0x0fffffff) >> 5) : (line_addr >> 5);
u32 way = exram ? lookup_table_ex[lookup_index] : lookup_table[lookup_index];
if (way == 0xff && !locked)
way = LoadCacheLine(memory, line_addr, set);
if (way == 0xff)
{
// A locked-cache miss bypasses the cache. Reuse the generic path for this rare case.
Read(memory, addr, &value, sizeof(T), locked);
return value;
}
plru[set] = (plru[set] & ~PLRU_MASK[way]) | PLRU_VALUE[way];
std::memcpy(&value, reinterpret_cast<const u8*>(data[set][way].data()) + offset_in_block,
sizeof(T));
return value;
}
template <typename T, bool exram>
DOLPHIN_FORCE_INLINE void WriteMainMemoryValue(Memory::MemoryManager& memory, u32 addr, T value,
bool locked)
{
const u32 offset_in_block = addr & 31;
if (offset_in_block > 32 - sizeof(T))
{
Write(memory, addr, &value, sizeof(T), locked);
return;
}
const u32 line_addr = addr & ~31U;
const u32 set = (line_addr >> 5) & 0x7f;
const u32 lookup_index = exram ? ((line_addr & 0x0fffffff) >> 5) : (line_addr >> 5);
u32 way = exram ? lookup_table_ex[lookup_index] : lookup_table[lookup_index];
if (way == 0xff && !locked)
way = LoadCacheLine(memory, line_addr, set);
if (way == 0xff)
{
// A locked-cache miss bypasses the cache. Reuse the generic path for this rare case.
Write(memory, addr, &value, sizeof(T), locked);
return;
}
plru[set] = (plru[set] & ~PLRU_MASK[way]) | PLRU_VALUE[way];
std::memcpy(reinterpret_cast<u8*>(data[set][way].data()) + offset_in_block, &value, sizeof(T));
modified[set] |= 1 << way;
}
void Read(Memory::MemoryManager& memory, u32 addr, void* buffer, u32 len, bool locked);
void Write(Memory::MemoryManager& memory, u32 addr, const void* buffer, u32 len, bool locked);
Source/UnitTests/Core/IOS/Starlet/ARMCoreTest.cpp
+302 -25
View File
@@ -83,8 +83,7 @@ public:
if (offset + 3 >= SRAM_SIZE)
return 0;
const u8* const sram = SRAMData();
return (static_cast<u32>(sram[offset]) << 24) |
(static_cast<u32>(sram[offset + 1]) << 16) |
return (static_cast<u32>(sram[offset]) << 24) | (static_cast<u32>(sram[offset + 1]) << 16) |
(static_cast<u32>(sram[offset + 2]) << 8) | sram[offset + 3];
}
const size_t offset = ToOffset(address);
@@ -197,8 +196,7 @@ public:
{
EXPECT_LT(offset + 3, SRAM_SIZE);
const u8* const sram = SRAMData();
return (static_cast<u32>(sram[offset]) << 24) |
(static_cast<u32>(sram[offset + 1]) << 16) |
return (static_cast<u32>(sram[offset]) << 24) | (static_cast<u32>(sram[offset + 1]) << 16) |
(static_cast<u32>(sram[offset + 2]) << 8) | sram[offset + 3];
}
@@ -288,6 +286,73 @@ TEST(WiiIPCCtrlRegister, ProducerBitsRemainLatchedUntilPeerAcknowledges)
EXPECT_EQ(control.ppc() & 0x06, 0x00);
}
TEST(StarletSRAM, WideAccessesPreserveAliasesSplitMappingAndBoot0Protection)
{
constexpr u32 sram_low = StarletMemory::SRAM_BASE;
constexpr u32 sram_high = StarletMemory::SRAM_MIRROR_BASE;
constexpr u32 hardware_srnprot = 0x0d800060;
constexpr u32 sram_split_mode = 1U << 5;
Core::DeclareAsCPUThread();
auto& system = Core::System::GetInstance();
system.GetWiiIPC().Reset();
StarletMemory memory(system);
memory.Reset();
memory.Write32(sram_low + 0x20, 0x11223344);
EXPECT_EQ(memory.Read32(sram_high + 0x20), 0x11223344u);
memory.Write16(sram_high + 0x24, 0xa1b2);
EXPECT_EQ(memory.Read16(sram_low + 0x24), 0xa1b2u);
memory.Write8(sram_low + 0x26, 0x5a);
EXPECT_EQ(memory.Read8(sram_high + 0x26), 0x5au);
// With the reset (non-split) mapping, boot0 overlays the upper half of the top aperture.
const u32 boot0_word = memory.Read32(0xffff0020);
memory.Write32(0xffff0020, 0xdeadbeef);
EXPECT_EQ(memory.Read32(0xffff0020), boot0_word);
memory.Write32(hardware_srnprot, sram_split_mode);
memory.Write32(sram_low + 0x20, 0x55667788); // Logical page zero now selects SRAM B.
EXPECT_EQ(memory.Read32(sram_high + 0x20), 0x55667788u);
EXPECT_EQ(memory.Read32(0xffff0020), 0x11223344u); // Upper aperture selects SRAM A.
memory.Write32(0xffff0020, 0xaabbccdd);
// The split gap remains unmapped and ignores writes.
memory.Write32(sram_high + 0x8000, 0xcafebabe);
EXPECT_EQ(memory.Read32(sram_high + 0x8000), 0u);
memory.Write32(hardware_srnprot, 0);
EXPECT_EQ(memory.Read32(sram_low + 0x20), 0xaabbccddu);
}
TEST(StarletRegisters, CachePreservesSparseValuesAndCollisions)
{
constexpr u32 address_a = 0x0d900100;
constexpr u32 address_b = address_a ^ 0x1001;
constexpr auto cache_index = [](u32 address) { return (address ^ (address >> 12)) & 0xfff; };
static_assert(cache_index(address_a) == cache_index(address_b));
Core::DeclareAsCPUThread();
auto& system = Core::System::GetInstance();
system.GetWiiIPC().Reset();
StarletMemory memory(system);
memory.Reset();
EXPECT_EQ(memory.Read8(address_a), 0u);
EXPECT_EQ(memory.Read8(address_b), 0u);
memory.Write8(address_a, 0x12);
EXPECT_EQ(memory.Read8(address_a), 0x12u);
memory.Write8(address_b, 0x34);
EXPECT_EQ(memory.Read8(address_b), 0x34u);
EXPECT_EQ(memory.Read8(address_a), 0x12u);
memory.Write8(address_a, 0x56);
EXPECT_EQ(memory.Read8(address_a), 0x56u);
EXPECT_EQ(memory.Read8(address_b), 0x34u);
memory.Reset();
EXPECT_EQ(memory.Read8(address_a), 0u);
EXPECT_EQ(memory.Read8(address_b), 0u);
}
TEST(StarletTimer, ZeroDelayAlarmMatchesImmediatelyAndUsesIRQW1C)
{
constexpr u32 hardware_base = 0x0d800000;
@@ -956,6 +1021,133 @@ TEST(StarletARMCore, JitPreservedBlocksUseCurrentTLBGenerationForFastmem)
#endif
}
TEST(StarletARMCore, FCSESwitchPreservesTaggedTLBTranslations)
{
TestBus bus(0x10000);
ARMCore core(bus);
bus.WriteARM(0x0000, 0xe5921000); // ldr r1, [r2]
bus.WriteARM(0x0004, 0xe3a00402); // mov r0, #0x02000000
bus.WriteARM(0x0008, 0xee0d0f10); // mcr p15, 0, r0, c13, c0, 0
bus.WriteARM(0x000c, 0xe5923000); // ldr r3, [r2]
bus.WriteARM(0x0010, 0xe3a00000); // mov r0, #0
bus.WriteARM(0x0014, 0xee0d0f10); // mcr p15, 0, r0, c13, c0, 0
bus.WriteARM(0x0018, 0xe5924000); // ldr r4, [r2]
bus.WriteARM(0x0100, 0x12345678);
bus.WriteARM(0x4000, 0x00000c02); // FCSE PID 0 low section -> PA 0
bus.WriteARM(0x4080, 0x00000c02); // FCSE PID 1 low section -> PA 0
bus.WriteARM(0x6000, 0x00000c02); // VA 0x80000000 section -> PA 0
core.GetCP15State().translation_table_base = 0x4000;
core.GetCP15State().domain_access_control = 3;
core.GetCP15State().control |= 1;
core.SetRegister(2, 0x100);
core.SetRegister(15, 0x80000000);
bus.ResetReadCounts();
EXPECT_EQ(core.RunCycles(7), 7u);
EXPECT_EQ(core.GetRegister(1), 0x12345678u);
EXPECT_EQ(core.GetRegister(3), 0x12345678u);
EXPECT_EQ(core.GetRegister(4), 0x12345678u);
// Seven instruction fetches, three data reads, and exactly three first-level walks: kernel,
// PID 0 and PID 1. Switching back to PID 0 must reuse its MVA-tagged TLB entry.
EXPECT_EQ(bus.GetRead32Count(), 13u);
}
TEST(StarletARMCore, JitFastCacheSeparatesFCSEProcesses)
{
#if defined(_M_X86_64)
TestBus bus(0x204000);
ARMCore core(bus);
bus.WriteARM(0x000000, 0xe3a01001); // PID 0: mov r1, #1
bus.WriteARM(0x000004, 0xea00003d); // b 0x100
bus.WriteARM(0x100000, 0xe3a01002); // PID 1: mov r1, #2
bus.WriteARM(0x100004, 0xea00003d); // b 0x100
bus.WriteARM(0x200000, 0x00000c02); // FCSE PID 0 section -> PA 0
bus.WriteARM(0x200080, 0x00100c02); // FCSE PID 1 section -> PA 0x00100000
core.GetCP15State().translation_table_base = 0x200000;
core.GetCP15State().domain_access_control = 3;
core.GetCP15State().control |= 1;
core.SetJitEnabled(true);
core.SetRegister(15, 0);
EXPECT_EQ(core.RunCycles(2), 2u);
EXPECT_EQ(core.GetRegister(1), 1u);
core.GetCP15State().process_id = 0x02000000;
core.SetRegister(15, 0);
EXPECT_EQ(core.RunCycles(2), 2u);
EXPECT_EQ(core.GetRegister(1), 2u);
EXPECT_EQ(core.GetJitCompiledBlockCount(), 2u);
// Both the software TLB and native fast-entry cache retain distinct MVA-tagged entries. Returning
// to PID 0 therefore needs neither a page-table read nor another block compilation.
bus.ResetReadCounts();
core.GetCP15State().process_id = 0;
core.SetRegister(15, 0);
EXPECT_EQ(core.RunCycles(2), 2u);
EXPECT_EQ(core.GetRegister(1), 1u);
EXPECT_EQ(bus.GetRead32Count(), 0u);
EXPECT_EQ(core.GetJitCompiledBlockCount(), 2u);
#endif
}
TEST(StarletARMCore, ARMJitCompilesConditionalBranchExchange)
{
#if defined(_M_X86_64)
TestBus bus(0x100);
ARMCore core(bus);
bus.WriteARM(0x00, 0x012fff1e); // bxeq lr
core.SetJitEnabled(true);
core.SetRegister(14, 0x20);
core.SetCPSR(core.GetCPSR() | ARMCore::CPSR_Z);
EXPECT_EQ(core.RunCycles(1), 1u);
EXPECT_EQ(core.GetRegister(15), 0x20u);
core.SetRegister(15, 0);
core.SetCPSR(core.GetCPSR() & ~ARMCore::CPSR_Z);
EXPECT_EQ(core.RunCycles(1), 1u);
EXPECT_EQ(core.GetRegister(15), 4u);
EXPECT_EQ(core.GetJitFallbackInstructionCount(), 0u);
EXPECT_EQ(core.GetJitNativeExecutedInstructions(), 2u);
#endif
}
TEST(StarletARMCore, ARMJitCompilesConditionalSingleDataTransfers)
{
#if defined(_M_X86_64)
TestBus bus(0x200);
ARMCore core(bus);
bus.SetFastmemEnabled(true);
bus.WriteARM(0x00, 0x05921000); // ldreq r1, [r2]
bus.WriteARM(0x04, 0xea00003d); // b 0x100
bus.WriteARM(0x20, 0x15823004); // strne r3, [r2, #4]
bus.WriteARM(0x24, 0xea000035); // b 0x100
bus.WriteARM(0x100, 0x11223344);
bus.WriteARM(0x104, 0xaabbccdd);
core.SetJitEnabled(true);
core.SetRegister(2, 0x100);
core.SetRegister(3, 0x55667788);
core.SetCPSR(core.GetCPSR() | ARMCore::CPSR_Z);
EXPECT_EQ(core.RunCycles(2), 2u);
EXPECT_EQ(core.GetRegister(1), 0x11223344u);
core.SetRegister(15, 0x20);
EXPECT_EQ(core.RunCycles(2), 2u);
EXPECT_EQ(bus[0x104], 0xaau);
core.SetRegister(15, 0x20);
core.SetCPSR(core.GetCPSR() & ~ARMCore::CPSR_Z);
EXPECT_EQ(core.RunCycles(2), 2u);
EXPECT_EQ(bus[0x104], 0x55u);
EXPECT_EQ(bus[0x105], 0x66u);
EXPECT_EQ(bus[0x106], 0x77u);
EXPECT_EQ(bus[0x107], 0x88u);
EXPECT_EQ(core.GetJitFallbackInstructionCount(), 0u);
EXPECT_EQ(core.GetJitNativeExecutedInstructions(), 6u);
#endif
}
TEST(StarletARMCore, WaitForInterruptFastForwardsAndWakesOnMaskedIRQ)
{
TestBus bus;
@@ -1223,7 +1415,7 @@ TEST(StarletARMCore, ARMJitConditionalBranchesMatchInterpreter)
{
const u32 address = condition * 0x20;
const u32 branch = (condition << 28) | 0x0a000000;
interpreter_bus.WriteARM(address, branch); // b<cond> address + 8
interpreter_bus.WriteARM(address, branch); // b<cond> address + 8
jit_bus.WriteARM(address, branch);
interpreter_bus.WriteARM(address + 4, 0xe3a00001); // mov r0, #1
jit_bus.WriteARM(address + 4, 0xe3a00001);
@@ -1233,10 +1425,8 @@ TEST(StarletARMCore, ARMJitConditionalBranchesMatchInterpreter)
for (u32 flags = 0; flags < 16; ++flags)
{
const u32 cpsr = static_cast<u32>(ARMCore::Mode::Supervisor) |
((flags & 1) ? ARMCore::CPSR_N : 0) |
((flags & 2) ? ARMCore::CPSR_Z : 0) |
((flags & 4) ? ARMCore::CPSR_C : 0) |
((flags & 8) ? ARMCore::CPSR_V : 0);
((flags & 1) ? ARMCore::CPSR_N : 0) | ((flags & 2) ? ARMCore::CPSR_Z : 0) |
((flags & 4) ? ARMCore::CPSR_C : 0) | ((flags & 8) ? ARMCore::CPSR_V : 0);
interpreter.SetCPSR(cpsr);
jit.SetCPSR(cpsr);
interpreter.SetRegister(15, address);
@@ -1272,7 +1462,7 @@ TEST(StarletARMCore, ARMJitCompilesPredicatedDataProcessing)
bus.WriteARM(address + 0x04, (condition << 28) | 0x03530002); // cmp<cond> r3, #2
bus.WriteARM(address + 0x08, (condition << 28) | 0x03c44018); // bic<cond> r4, r4, #0x18
bus.WriteARM(address + 0x0c, (condition << 28) | 0x03a05007); // mov<cond> r5, #7
bus.WriteARM(address + 0x10, 0xea000000); // b address + 0x18
bus.WriteARM(address + 0x10, 0xea000000); // b address + 0x18
};
install_program(interpreter_bus);
install_program(jit_bus);
@@ -1280,10 +1470,8 @@ TEST(StarletARMCore, ARMJitCompilesPredicatedDataProcessing)
for (u32 flags = 0; flags < 16; ++flags)
{
const u32 cpsr = static_cast<u32>(ARMCore::Mode::Supervisor) |
((flags & 1) ? ARMCore::CPSR_N : 0) |
((flags & 2) ? ARMCore::CPSR_Z : 0) |
((flags & 4) ? ARMCore::CPSR_C : 0) |
((flags & 8) ? ARMCore::CPSR_V : 0);
((flags & 1) ? ARMCore::CPSR_N : 0) | ((flags & 2) ? ARMCore::CPSR_Z : 0) |
((flags & 4) ? ARMCore::CPSR_C : 0) | ((flags & 8) ? ARMCore::CPSR_V : 0);
interpreter.SetCPSR(cpsr);
jit.SetCPSR(cpsr);
for (ARMCore* core : {&interpreter, &jit})
@@ -1443,8 +1631,8 @@ TEST(StarletARMCore, ThumbJitCompilesRegisterLSREdgeCases)
interpreter_bus.WriteThumb(0x02, 0xe7fe); // b .
jit_bus.WriteThumb(0x02, 0xe7fe);
constexpr std::array<u32, 7> values = {0, 1, 0x80000000, 0xffffffff, 0x12345678,
0x7fffffff, 0xa5a5a5a5};
constexpr std::array<u32, 7> values = {0, 1, 0x80000000, 0xffffffff,
0x12345678, 0x7fffffff, 0xa5a5a5a5};
constexpr std::array<u32, 8> shifts = {0, 1, 2, 31, 32, 33, 63, 255};
for (const u32 value : values)
{
@@ -1775,8 +1963,50 @@ TEST(StarletARMCore, ARMJitDoesNotCrossFromSRAMIntoBoot0)
core.SetJitEnabled(true);
EXPECT_EQ(core.RunCycles(2), 2u);
EXPECT_EQ(core.GetJitFallbackInstructionCount(), 1u);
EXPECT_EQ(core.GetJitNativeExecutedInstructions(), 1u);
EXPECT_EQ(core.GetJitFallbackInstructionCount(), 0u);
EXPECT_EQ(core.GetJitNativeExecutedInstructions(), 2u);
#endif
}
TEST(StarletARMCore, ARMJitKeepsSRAMStackBlockTransfersInsideNativeBlock)
{
#if defined(_M_X86_64)
TestBus interpreter_bus(0x1000);
TestBus jit_bus(0x1000);
interpreter_bus.SetFastmemEnabled(true);
interpreter_bus.SetSRAMFastmemEnabled(true);
jit_bus.SetFastmemEnabled(true);
jit_bus.SetSRAMFastmemEnabled(true);
ARMCore interpreter(interpreter_bus);
ARMCore jit(jit_bus);
jit.SetJitEnabled(true);
const auto install_program = [](TestBus& bus) {
bus.WriteARM(0x00, 0xe92d4070); // push {r4-r6, lr}
bus.WriteARM(0x04, 0xe3a04000); // mov r4, #0
bus.WriteARM(0x08, 0xe3a05000); // mov r5, #0
bus.WriteARM(0x0c, 0xe3a06000); // mov r6, #0
bus.WriteARM(0x10, 0xe3a0e000); // mov lr, #0
bus.WriteARM(0x14, 0xe8bd4070); // pop {r4-r6, lr}
bus.WriteARM(0x18, 0xeafffffe); // b .
};
install_program(interpreter_bus);
install_program(jit_bus);
for (ARMCore* core : {&interpreter, &jit})
{
core->SetRegister(4, 0x11223344);
core->SetRegister(5, 0x55667788);
core->SetRegister(6, 0x99aabbcc);
core->SetRegister(13, 0xfff12000);
core->SetRegister(14, 0xddeeff00);
}
ASSERT_EQ(interpreter.RunCycles(7), 7u);
ASSERT_EQ(jit.RunCycles(7), 7u);
for (u32 reg = 0; reg < 16; ++reg)
EXPECT_EQ(jit.GetRegister(reg), interpreter.GetRegister(reg)) << "r" << reg;
EXPECT_EQ(jit.GetCPSR(), interpreter.GetCPSR());
EXPECT_EQ(jit.GetJitFallbackInstructionCount(), 0u);
EXPECT_EQ(jit.GetJitNativeExecutedInstructions(), 7u);
#endif
}
@@ -2042,6 +2272,53 @@ TEST(StarletARMCore, RunCyclesStopsAtBudget)
2u); // Both words stay resident across loop iterations.
}
TEST(StarletARMCore, ARMJitConditionalBackEdgeKeepsExactBudgetAndFallthrough)
{
#if defined(_M_X86_64)
TestBus bus;
bus.SetFastmemEnabled(true);
ARMCore core(bus);
core.SetJitEnabled(true);
bus.WriteARM(0x00, 0xe2800001); // add r0, r0, #1
bus.WriteARM(0x04, 0xe3500003); // cmp r0, #3
bus.WriteARM(0x08, 0x1afffffc); // bne 0x00
bus.WriteARM(0x0c, 0xe3a0102a); // mov r1, #42
EXPECT_EQ(core.RunCycles(9), 9u);
EXPECT_EQ(core.GetExecutedInstructions(), 9u);
EXPECT_EQ(core.GetRegister(0), 3u);
EXPECT_EQ(core.GetRegister(15), 0x0cu);
EXPECT_EQ(core.GetJitNativeExecutedInstructions(), 9u);
EXPECT_EQ(core.RunCycles(1), 1u);
EXPECT_EQ(core.GetRegister(1), 42u);
#endif
}
TEST(StarletARMCore, ThumbJitConditionalBackEdgeKeepsExactBudgetAndFallthrough)
{
#if defined(_M_X86_64)
TestBus bus;
bus.SetFastmemEnabled(true);
ARMCore core(bus);
core.SetJitEnabled(true);
core.SetCPSR(static_cast<u32>(ARMCore::Mode::Supervisor) | ARMCore::CPSR_T);
bus.WriteThumb(0x00, 0x3001); // add r0, #1
bus.WriteThumb(0x02, 0x2803); // cmp r0, #3
bus.WriteThumb(0x04, 0xd1fc); // bne 0x00
bus.WriteThumb(0x06, 0x212a); // mov r1, #42
EXPECT_EQ(core.RunCycles(9), 9u);
EXPECT_EQ(core.GetExecutedInstructions(), 9u);
EXPECT_EQ(core.GetRegister(0), 3u);
EXPECT_EQ(core.GetRegister(15), 0x06u);
EXPECT_EQ(core.GetJitNativeExecutedInstructions(), 9u);
EXPECT_EQ(core.RunCycles(1), 1u);
EXPECT_EQ(core.GetRegister(1), 42u);
#endif
}
TEST(StarletARMCore, ARMJitFastForwardsSliceStableHollywoodTimerPoll)
{
TestBus bus;
@@ -2073,14 +2350,14 @@ TEST(StarletARMCore, ARMJitFastForwardsSliceStableHollywoodTimerPoll)
TEST(StarletARMCore, ARMJitKeepsNonblockingReceiveMessagePollGuestVisible)
{
const auto install_program = [](TestBus& bus, bool non_blocking) {
bus.WriteARM(0x00, 0xe5960000); // ldr r0, [r6]
bus.WriteARM(0x04, 0xe1a01008); // mov r1, r8
bus.WriteARM(0x00, 0xe5960000); // ldr r0, [r6]
bus.WriteARM(0x04, 0xe1a01008); // mov r1, r8
bus.WriteARM(0x08, non_blocking ? 0xe3a02000 : 0xe3a02001); // mov r2, #block
bus.WriteARM(0x0c, 0xeb00000b); // bl 0x40
bus.WriteARM(0x10, 0xe2505000); // subs r5, r0, #0
bus.WriteARM(0x14, 0x1afffff9); // bne 0x00
bus.WriteARM(0x40, 0xe60001d0); // IOS ReceiveMessage syscall 0x0e
bus.WriteARM(0x44, 0xe12fff1e); // bx lr
bus.WriteARM(0x0c, 0xeb00000b); // bl 0x40
bus.WriteARM(0x10, 0xe2505000); // subs r5, r0, #0
bus.WriteARM(0x14, 0x1afffff9); // bne 0x00
bus.WriteARM(0x40, 0xe60001d0); // IOS ReceiveMessage syscall 0x0e
bus.WriteARM(0x44, 0xe12fff1e); // bx lr
};
TestBus bus;