diff --git a/Run-Wii-IOS-LLE.ps1 b/Run-Wii-IOS-LLE.ps1 index 9925801118..bbee81bcea 100644 --- a/Run-Wii-IOS-LLE.ps1 +++ b/Run-Wii-IOS-LLE.ps1 @@ -3,6 +3,8 @@ param( [string]$UserDirectory = '.starlet_user3', + [switch]$DisableJIT, + [switch]$Wait ) @@ -17,6 +19,12 @@ if (-not (Test-Path -LiteralPath $dolphinPath)) { $dolphin = Start-Process -FilePath $dolphinPath -ArgumentList @( '-u', $userPath, '-n', '0000000100000002', + '-C', "Dolphin.Core.WiiStarletJIT=$(-not $DisableJIT)", + # Keep the next BootMii run diagnostic rather than observational. BootMii reports its loader + # stages and panic codes through GPIO bits 16-23; the Starlet bus logs those writes under IOS. + '-C', 'Logger.Options.WriteToFile=True', + '-C', 'Logger.Options.Verbosity=4', + '-C', 'Logger.Logs.IOS=True', '-v', 'D3D', '-p', 'win32' ) -WorkingDirectory $repoRoot -PassThru diff --git a/Source/Core/Core/CMakeLists.txt b/Source/Core/Core/CMakeLists.txt index bcd2ba9645..a73b84bcf0 100644 --- a/Source/Core/Core/CMakeLists.txt +++ b/Source/Core/Core/CMakeLists.txt @@ -612,6 +612,8 @@ target_sources(core PRIVATE AchievementApprovedHash.h) if(_M_X86_64) target_sources(core PRIVATE + IOS/Starlet/ARMJitX64.cpp + IOS/Starlet/ARMJitX64.h DSP/Jit/x64/DSPEmitter.cpp DSP/Jit/x64/DSPEmitter.h DSP/Jit/x64/DSPJitArithmetic.cpp diff --git a/Source/Core/Core/Config/MainSettings.cpp b/Source/Core/Core/Config/MainSettings.cpp index 82eed03039..9ae5d6f1de 100644 --- a/Source/Core/Core/Config/MainSettings.cpp +++ b/Source/Core/Core/Config/MainSettings.cpp @@ -257,6 +257,7 @@ const Info MAIN_GPU_DETERMINISM_MODE{{System::Main, "Core", "GPUDet "auto"}; const Info MAIN_OVERRIDE_BOOT_IOS{{System::Main, "Core", "OverrideBootIOS"}, -1}; const Info MAIN_WII_IOS_LLE{{System::Main, "Core", "WiiIOSLLE"}, false}; +const Info MAIN_WII_STARLET_JIT{{System::Main, "Core", "WiiStarletJIT"}, true}; GPUDeterminismMode GetGPUDeterminismMode() { diff --git a/Source/Core/Core/Config/MainSettings.h b/Source/Core/Core/Config/MainSettings.h index 639abe8796..aa804b6010 100644 --- a/Source/Core/Core/Config/MainSettings.h +++ b/Source/Core/Core/Config/MainSettings.h @@ -167,6 +167,7 @@ extern const Info MAIN_FALLBACK_REGION; extern const Info MAIN_REAL_WII_REMOTE_REPEAT_REPORTS; extern const Info MAIN_OVERRIDE_BOOT_IOS; extern const Info MAIN_WII_IOS_LLE; +extern const Info MAIN_WII_STARLET_JIT; extern const Info MAIN_WII_NUS_SHOP_URL; extern const Info MAIN_WII_WIILINK_ENABLE; diff --git a/Source/Core/Core/IOS/Starlet/ARMCore.cpp b/Source/Core/Core/IOS/Starlet/ARMCore.cpp index 064346260a..5e74bc8566 100644 --- a/Source/Core/Core/IOS/Starlet/ARMCore.cpp +++ b/Source/Core/Core/IOS/Starlet/ARMCore.cpp @@ -8,8 +8,14 @@ #include #include #include +#include #include "Common/ChunkFile.h" +#if defined(_M_X86_64) +// ARMCore owns ARMJitX64 by value through unique_ptr construction in this translation unit, so a +// private JIT layout change must rebuild this file as well as the emitter. +#include "Core/IOS/Starlet/ARMJitX64.h" +#endif namespace IOS::LLE { @@ -30,6 +36,19 @@ ARMCore::ARMCore(ARMBus& bus) : m_bus(bus) Reset(); } +ARMCore::~ARMCore() = default; + +void ARMCore::SetJitEnabled(bool enabled) +{ +#if defined(_M_X86_64) + if (enabled && !m_jit) + m_jit = std::make_unique(*this); + m_jit_enabled = enabled; +#else + m_jit_enabled = false; +#endif +} + void ARMCore::Reset(u32 reset_vector) { m_registers.fill(0); @@ -57,11 +76,19 @@ void ARMCore::Reset(u32 reset_vector) m_waiting_for_interrupt = false; m_waiting_for_memory_poll = false; m_yield_requested = false; + m_slice_fast_forward_requested = false; m_memory_poll_address = 0; m_pc_written = false; m_instruction_address = m_registers[15]; m_last_undefined_instruction = 0; m_executed_instructions = 0; + m_memory_poll_entry_count = 0; + m_jit_fallback_instruction_count = 0; + m_jit_fallback_samples.clear(); + m_jit_fallback_interval_count = 0; + m_jit_fallback_interval_samples.clear(); + m_next_profile_instruction = PROFILE_SAMPLE_INTERVAL; + m_pc_samples.clear(); } u32 ARMCore::GetRegister(size_t index) const @@ -192,7 +219,7 @@ u32 ARMCore::TranslateVirtualAddress(u32 address) const case 3: // 4 KiB small page; extended small pages share this mapping shape. return cache_translation((second_level & 0xfffff000) | (modified_address & 0xfff)); default: - return modified_address; + return cache_translation(modified_address); } } case 2: // 1 MiB section. @@ -210,14 +237,16 @@ u32 ARMCore::TranslateVirtualAddress(u32 address) const case 3: // 1 KiB tiny page. return cache_translation((second_level & 0xfffffc00) | (modified_address & 0x3ff)); default: - return modified_address; + return cache_translation(modified_address); } } default: // Fault entry. Abort delivery and access-permission checks are // intentionally introduced after basic page-table translation; identity - // fallback keeps diagnostics observable meanwhile. - return modified_address; + // fallback keeps diagnostics observable meanwhile. Cache that provisional translation just + // like a mapped page: real software must invalidate the TLB after changing a page-table entry, + // and without this cache the JIT would side-exit forever on every identity access. + return cache_translation(modified_address); } } @@ -229,6 +258,10 @@ void ARMCore::InvalidateTLB() entry.generation = 0; m_tlb_generation = 1; } +#if defined(_M_X86_64) + if (m_jit) + m_jit->InvalidateTranslationContext(); +#endif } u32 ARMCore::FetchARMInstruction(u32 address) @@ -277,6 +310,10 @@ void ARMCore::InvalidateInstructionCache() entry.generation = 0; m_instruction_cache_generation = 1; } +#if defined(_M_X86_64) + if (m_jit) + m_jit->Clear(); +#endif } u8 ARMCore::ReadByte(u32 address) const @@ -552,6 +589,53 @@ bool ARMCore::TryEnterThumbMemoryPoll(u16 branch_instruction) m_memory_poll_address = address; m_waiting_for_memory_poll = true; + ++m_memory_poll_entry_count; + return true; +} + +bool ARMCore::TryEnterARMSliceStablePoll(u32 branch_instruction) +{ + // IOS's early timer delay has this ARM shape (the middle literal load initializes the base only + // on entry): LDR value,[base]; B compare; literal/base setup; CMP value,target; Bcc loop. The + // timer cannot advance again until the bus receives AdvanceCycles at the slice boundary, making + // every remaining iteration in this slice observationally redundant. + const u32 condition = branch_instruction >> 28; + if ((branch_instruction & 0x0e000000) != 0x0a000000 || condition >= 0xe || + !m_pc_written || HasUnmaskedInterrupt()) + { + return false; + } + + const u32 loop_address = m_registers[15]; + if (loop_address + 16 != m_instruction_address) + return false; + + const u32 load = FetchARMInstruction(loop_address); + const u32 skip = FetchARMInstruction(loop_address + 4); + const u32 compare = FetchARMInstruction(loop_address + 12); + if ((load & 0xfff00000) != 0xe5900000 || (skip & 0xff000000) != 0xea000000 || + (compare & 0xfff00ff0) != 0xe1500000) + { + return false; + } + + const s32 skip_offset = SignExtend((skip & 0x00ffffff) << 2, 26); + if (loop_address + 12 != loop_address + 12 + static_cast(skip_offset)) + return false; + + const u32 load_destination = (load >> 12) & 0xf; + if (((compare >> 16) & 0xf) != load_destination) + return false; + + const u32 base = ReadRegisterOperand((load >> 16) & 0xf); + const u32 poll_address = base + (load & 0xfff); + const u32 physical_address = TranslateVirtualAddress(poll_address); + if (!m_bus.IsSliceStablePollAddress(physical_address, sizeof(u32))) + return false; + + m_memory_poll_address = poll_address; + m_waiting_for_memory_poll = true; + ++m_memory_poll_entry_count; return true; } @@ -563,13 +647,23 @@ int ARMCore::StepInternal(bool advance_bus) // entry. if (m_waiting_for_memory_poll) { - if (!HasUnmaskedInterrupt() && Read32(m_memory_poll_address) == 0) + const u32 physical_poll_address = TranslateVirtualAddress(m_memory_poll_address); + if (m_bus.IsSliceStablePollAddress(physical_poll_address, sizeof(u32))) + { + // Step() advances the device clock itself, so only RunCycles can collapse the rest of a + // scheduler slice. Resume normally here. + m_waiting_for_memory_poll = false; + } + else if (!HasUnmaskedInterrupt() && Read32(m_memory_poll_address) == 0) { if (advance_bus) m_bus.AdvanceCycles(1); return 1; } - m_waiting_for_memory_poll = false; + else + { + m_waiting_for_memory_poll = false; + } } if (m_waiting_for_interrupt && !m_irq_line && !m_fiq_line) { @@ -623,6 +717,7 @@ int ARMCore::StepInternal(bool advance_bus) { ExecuteARM(instruction); } + TryEnterARMSliceStablePoll(instruction); } if (!m_pc_written) @@ -648,6 +743,16 @@ u64 ARMCore::RunCycles(u64 cycle_budget) { if (m_waiting_for_memory_poll) { + const u32 physical_poll_address = TranslateVirtualAddress(m_memory_poll_address); + if (!HasUnmaskedInterrupt() && + m_bus.IsSliceStablePollAddress(physical_poll_address, sizeof(u32))) + { + // This MMIO value cannot change before AdvanceCycles below. Consume the unused slice and + // retry the guest loop against the newly advanced device state next time. + m_waiting_for_memory_poll = false; + cycles = cycle_budget; + break; + } if (HasUnmaskedInterrupt() || Read32(m_memory_poll_address) != 0) { m_waiting_for_memory_poll = false; @@ -663,7 +768,32 @@ u64 ARMCore::RunCycles(u64 cycle_budget) cycles = cycle_budget; break; } +#if defined(_M_X86_64) + if (m_jit_enabled && m_jit && !HasUnmaskedInterrupt()) + { + const u32 executed = m_jit->Run(cycle_budget - cycles); + if (executed != 0) + { + cycles += executed; + SampleExecutionPC(); + if (m_slice_fast_forward_requested) + { + m_slice_fast_forward_requested = false; + cycles = cycle_budget; + break; + } + continue; + } + } +#endif cycles += static_cast(StepInternal(false)); + SampleExecutionPC(); + if (m_slice_fast_forward_requested) + { + m_slice_fast_forward_requested = false; + cycles = cycle_budget; + break; + } } // Starlet interrupt lines are sampled by the scheduler at slice boundaries. // Updating timers and peripherals once here is therefore architecturally @@ -674,6 +804,281 @@ u64 ARMCore::RunCycles(u64 cycle_budget) return cycles; } +void ARMCore::SampleExecutionPC() +{ + if (m_executed_instructions < m_next_profile_instruction) + return; + + const u32 key = m_registers[15] | ((m_cpsr & CPSR_T) != 0 ? 1U : 0U); + ++m_pc_samples[key]; + m_next_profile_instruction += PROFILE_SAMPLE_INTERVAL; +} + +std::vector ARMCore::GetHotPCSamples(size_t maximum_count) +{ + std::vector> sorted(m_pc_samples.begin(), m_pc_samples.end()); + std::ranges::sort(sorted, {}, [](const auto& entry) { return entry.second; }); + if (sorted.size() > maximum_count) + sorted.resize(maximum_count); + + std::vector result; + result.reserve(sorted.size()); + for (auto it = sorted.rbegin(); it != sorted.rend(); ++it) + { + const bool thumb = (it->first & 1) != 0; + const u32 address = it->first & ~1U; + result.push_back( + {.address = address, + .instruction = thumb ? FetchThumbInstruction(address) : FetchARMInstruction(address), + .samples = it->second, + .thumb = thumb}); + } + return result; +} + +u64 ARMCore::GetJitExecutedInstructions() const +{ +#if defined(_M_X86_64) + return m_jit ? m_jit->GetExecutedInstructions() : 0; +#else + return 0; +#endif +} + +u64 ARMCore::GetJitNativeExecutedInstructions() const +{ +#if defined(_M_X86_64) + // Kept out of line so changes to the x64 JIT's diagnostic layout rebuild this translation unit. + return m_jit ? m_jit->GetNativeExecutedInstructions() : 0; +#else + return 0; +#endif +} + +size_t ARMCore::GetJitCompiledBlockCount() const +{ +#if defined(_M_X86_64) + return m_jit ? m_jit->GetCompiledBlockCount() : 0; +#else + return 0; +#endif +} + +u64 ARMCore::GetJitBlockExecutionCount() const +{ +#if defined(_M_X86_64) + return m_jit ? m_jit->GetBlockExecutionCount() : 0; +#else + return 0; +#endif +} + +u64 ARMCore::GetJitAddressTranslationCount() const +{ +#if defined(_M_X86_64) + return m_jit ? m_jit->GetAddressTranslationCount() : 0; +#else + return 0; +#endif +} + +u64 ARMCore::GetJitSlowReadCount() const +{ +#if defined(_M_X86_64) + return m_jit ? m_jit->GetSlowReadCount() : 0; +#else + return 0; +#endif +} + +u64 ARMCore::GetJitSlowWriteCount() const +{ +#if defined(_M_X86_64) + return m_jit ? m_jit->GetSlowWriteCount() : 0; +#else + return 0; +#endif +} + +u64 ARMCore::GetJitSlowRAMAccessCount() const +{ +#if defined(_M_X86_64) + return m_jit ? m_jit->GetSlowRAMAccessCount() : 0; +#else + return 0; +#endif +} + +u64 ARMCore::GetJitSlowSRAMAccessCount() const +{ +#if defined(_M_X86_64) + return m_jit ? m_jit->GetSlowSRAMAccessCount() : 0; +#else + return 0; +#endif +} + +u64 ARMCore::GetJitSlowMMIOAccessCount() const +{ +#if defined(_M_X86_64) + return m_jit ? m_jit->GetSlowMMIOAccessCount() : 0; +#else + return 0; +#endif +} + +u64 ARMCore::GetJitSlowOtherAccessCount() const +{ +#if defined(_M_X86_64) + return m_jit ? m_jit->GetSlowOtherAccessCount() : 0; +#else + return 0; +#endif +} + +u64 ARMCore::GetJitSlowSRAMLowAccessCount() const +{ +#if defined(_M_X86_64) + return m_jit ? m_jit->GetSlowSRAMLowAccessCount() : 0; +#else + return 0; +#endif +} + +u64 ARMCore::GetJitSlowSRAMHighAccessCount() const +{ +#if defined(_M_X86_64) + return m_jit ? m_jit->GetSlowSRAMHighAccessCount() : 0; +#else + return 0; +#endif +} + +u64 ARMCore::GetJitSlowSRAMReadAccessCount() const +{ +#if defined(_M_X86_64) + return m_jit ? m_jit->GetSlowSRAMReadAccessCount() : 0; +#else + return 0; +#endif +} + +u64 ARMCore::GetJitSlowSRAMWriteAccessCount() const +{ +#if defined(_M_X86_64) + return m_jit ? m_jit->GetSlowSRAMWriteAccessCount() : 0; +#else + return 0; +#endif +} + +u64 ARMCore::GetJitSlowSRAMPageAccessCount(size_t page) const +{ +#if defined(_M_X86_64) + return m_jit ? m_jit->GetSlowSRAMPageAccessCount(page) : 0; +#else + return 0; +#endif +} + +u64 ARMCore::GetJitSlowSRAMPageReadAccessCount(size_t page) const +{ +#if defined(_M_X86_64) + return m_jit ? m_jit->GetSlowSRAMPageReadAccessCount(page) : 0; +#else + return 0; +#endif +} + +u64 ARMCore::GetJitSlowSRAMPageWriteAccessCount(size_t page) const +{ +#if defined(_M_X86_64) + return m_jit ? m_jit->GetSlowSRAMPageWriteAccessCount(page) : 0; +#else + return 0; +#endif +} + +void ARMCore::RecordJitFallback(u32 address, bool thumb) +{ + ++m_jit_fallback_instruction_count; + ++m_jit_fallback_interval_count; + // Sampling keeps profiling effectively free even when hundreds of millions of instructions + // use an interpreter side exit. + if ((m_jit_fallback_instruction_count & 0xfff) == 0) + { + ++m_jit_fallback_samples[address | (thumb ? 1U : 0U)]; + ++m_jit_fallback_interval_samples[address | (thumb ? 1U : 0U)]; + } +} + +std::vector ARMCore::GetHotJitFallbackSamples(size_t maximum_count) +{ + std::vector> sorted(m_jit_fallback_samples.begin(), + m_jit_fallback_samples.end()); + std::ranges::sort(sorted, {}, [](const auto& entry) { return entry.second; }); + if (sorted.size() > maximum_count) + sorted.erase(sorted.begin(), sorted.end() - maximum_count); + + std::vector result; + result.reserve(sorted.size()); + for (auto it = sorted.rbegin(); it != sorted.rend(); ++it) + { + const bool thumb = (it->first & 1) != 0; + const u32 address = it->first & ~1U; + result.push_back( + {.address = address, + .instruction = thumb ? FetchThumbInstruction(address) : FetchARMInstruction(address), + .samples = it->second, + .thumb = thumb}); + } + return result; +} + +ARMCore::HotPCSample ARMCore::GetCurrentPCSample() +{ + const bool thumb = (m_cpsr & CPSR_T) != 0; + return GetInstructionSample(m_registers[15], thumb); +} + +ARMCore::HotPCSample ARMCore::GetInstructionSample(u32 address, bool thumb) +{ + return {.address = address, + .instruction = thumb ? FetchThumbInstruction(address) : FetchARMInstruction(address), + .samples = 1, + .thumb = thumb}; +} + +u64 ARMCore::TakeJitFallbackIntervalCount() +{ + return std::exchange(m_jit_fallback_interval_count, 0); +} + +std::vector +ARMCore::TakeHotJitFallbackIntervalSamples(size_t maximum_count) +{ + std::vector> sorted(m_jit_fallback_interval_samples.begin(), + m_jit_fallback_interval_samples.end()); + m_jit_fallback_interval_samples.clear(); + std::ranges::sort(sorted, {}, [](const auto& entry) { return entry.second; }); + if (sorted.size() > maximum_count) + sorted.erase(sorted.begin(), sorted.end() - maximum_count); + + std::vector result; + result.reserve(sorted.size()); + for (auto it = sorted.rbegin(); it != sorted.rend(); ++it) + { + const bool thumb = (it->first & 1) != 0; + const u32 address = it->first & ~1U; + result.push_back( + {.address = address, + .instruction = thumb ? FetchThumbInstruction(address) : FetchARMInstruction(address), + .samples = it->second, + .thumb = thumb}); + } + return result; +} + void ARMCore::DoState(PointerWrap& p) { p.DoArray(m_registers); @@ -696,6 +1101,7 @@ void ARMCore::DoState(PointerWrap& p) p.Do(m_waiting_for_interrupt); p.Do(m_waiting_for_memory_poll); p.Do(m_yield_requested); + p.Do(m_slice_fast_forward_requested); p.Do(m_memory_poll_address); p.Do(m_pc_written); p.Do(m_instruction_address); @@ -1928,9 +2334,13 @@ void ARMCore::WriteCP15(u32 opcode1, u32 crn, u32 crm, u32 opcode2, u32 value) case 6: m_cp15.fault_address = value; break; - case 7: // Treat cache maintenance conservatively as a whole instruction-cache - // invalidation. - InvalidateInstructionCache(); + case 7: + // ARM926 separates I-cache and D-cache maintenance. In particular c7,c10,4 is Drain Write + // Buffer, an ordering operation used heavily by IOS; invalidating translated code for it is + // both architecturally wrong and catastrophically expensive. Only operations which include + // the instruction cache make previously modified instructions visible. + if (crm == 5 || crm == 7) + InvalidateInstructionCache(); break; case 8: // TLB maintenance. InvalidateTLB(); diff --git a/Source/Core/Core/IOS/Starlet/ARMCore.h b/Source/Core/Core/IOS/Starlet/ARMCore.h index 553282e7d7..d6d16daf86 100644 --- a/Source/Core/Core/IOS/Starlet/ARMCore.h +++ b/Source/Core/Core/IOS/Starlet/ARMCore.h @@ -5,6 +5,9 @@ #include #include +#include +#include +#include #include "Common/CommonTypes.h" @@ -12,6 +15,10 @@ class PointerWrap; namespace IOS::LLE { +#if defined(_M_X86_64) +class ARMJitX64; +#endif + // Byte-addressed bus used by the Starlet ARM core. Keeping endianness in the // CPU is intentional: ARM926 can change its data endianness through CP15 while // the underlying devices remain byte addressed. @@ -45,6 +52,18 @@ public: } virtual void AdvanceCycles(u64 cycles) {} virtual bool IsIdlePollAddress(u32 address, u32 size) const { return false; } + // True only for read-only MMIO counters whose value cannot change until AdvanceCycles runs. + // This lets a side-effect-free guest polling loop yield the remainder of the current scheduler + // slice without skipping any observable device transition. + virtual bool IsSliceStablePollAddress(u32 address, u32 size) const { return false; } + // Returns Dolphin's 4 GiB physical fastmem view when this bus is backed by Wii RAM. Device, + // boot-ROM and SRAM addresses must still use the bus callbacks. + virtual u8* GetFastmemBase() const { return nullptr; } + // Optional direct Starlet SRAM view. The two state pointers let generated code preserve the + // boot0 overlay and SRAM A/B split mapping exactly while avoiding a host call for ordinary SRAM. + virtual u8* GetFastmemSRAMBase() const { return nullptr; } + virtual const bool* GetFastmemBoot0Mapped() const { return nullptr; } + virtual const bool* GetFastmemSRAMSplitMode() const { return nullptr; } }; class ARMCore final @@ -71,6 +90,14 @@ public: u32 process_id = 0; }; + struct HotPCSample + { + u32 address = 0; + u32 instruction = 0; + u64 samples = 0; + bool thumb = false; + }; + static constexpr u32 CPSR_N = 1U << 31; static constexpr u32 CPSR_Z = 1U << 30; static constexpr u32 CPSR_C = 1U << 29; @@ -82,6 +109,7 @@ public: static constexpr u32 CPSR_MODE_MASK = 0x1f; explicit ARMCore(ARMBus& bus); + ~ARMCore(); void Reset(u32 reset_vector = 0); int Step(); @@ -131,8 +159,39 @@ public: u32 GetLastUndefinedInstruction() const { return m_last_undefined_instruction; } u64 GetExecutedInstructions() const { return m_executed_instructions; } + void SetJitEnabled(bool enabled); + bool IsJitEnabled() const { return m_jit_enabled; } + u64 GetJitExecutedInstructions() const; + u64 GetJitNativeExecutedInstructions() const; + size_t GetJitCompiledBlockCount() const; + u64 GetJitBlockExecutionCount() const; + u64 GetJitAddressTranslationCount() const; + u64 GetJitSlowReadCount() const; + u64 GetJitSlowWriteCount() const; + u64 GetJitSlowRAMAccessCount() const; + u64 GetJitSlowSRAMAccessCount() const; + u64 GetJitSlowMMIOAccessCount() const; + u64 GetJitSlowOtherAccessCount() const; + u64 GetJitSlowSRAMLowAccessCount() const; + u64 GetJitSlowSRAMHighAccessCount() const; + u64 GetJitSlowSRAMReadAccessCount() const; + u64 GetJitSlowSRAMWriteAccessCount() const; + u64 GetJitSlowSRAMPageAccessCount(size_t page) const; + u64 GetJitSlowSRAMPageReadAccessCount(size_t page) const; + u64 GetJitSlowSRAMPageWriteAccessCount(size_t page) const; + u64 GetJitFallbackInstructionCount() const { return m_jit_fallback_instruction_count; } + u64 GetMemoryPollEntryCount() const { return m_memory_poll_entry_count; } + std::vector GetHotPCSamples(size_t maximum_count); + HotPCSample GetCurrentPCSample(); + HotPCSample GetInstructionSample(u32 address, bool thumb); + std::vector GetHotJitFallbackSamples(size_t maximum_count); + u64 TakeJitFallbackIntervalCount(); + std::vector TakeHotJitFallbackIntervalSamples(size_t maximum_count); private: +#if defined(_M_X86_64) + friend class ARMJitX64; +#endif struct ShiftResult { u32 value; @@ -169,6 +228,9 @@ private: int StepInternal(bool advance_bus); bool HasUnmaskedInterrupt() const; bool TryEnterThumbMemoryPoll(u16 branch_instruction); + bool TryEnterARMSliceStablePoll(u32 branch_instruction); + void RecordJitFallback(u32 address, bool thumb); + void SampleExecutionPC(); u16 Read16(u32 address) const; u32 Read32(u32 address) const; @@ -251,10 +313,23 @@ private: bool m_waiting_for_interrupt = false; bool m_waiting_for_memory_poll = false; bool m_yield_requested = false; + bool m_slice_fast_forward_requested = false; u32 m_memory_poll_address = 0; bool m_pc_written = false; u32 m_instruction_address = 0; u32 m_last_undefined_instruction = 0; u64 m_executed_instructions = 0; + u64 m_memory_poll_entry_count = 0; + u64 m_jit_fallback_instruction_count = 0; + std::unordered_map m_jit_fallback_samples; + u64 m_jit_fallback_interval_count = 0; + std::unordered_map m_jit_fallback_interval_samples; + static constexpr u64 PROFILE_SAMPLE_INTERVAL = 8192; + u64 m_next_profile_instruction = PROFILE_SAMPLE_INTERVAL; + std::unordered_map m_pc_samples; + bool m_jit_enabled = false; +#if defined(_M_X86_64) + std::unique_ptr m_jit; +#endif }; } // namespace IOS::LLE diff --git a/Source/Core/Core/IOS/Starlet/ARMJitX64.cpp b/Source/Core/Core/IOS/Starlet/ARMJitX64.cpp new file mode 100644 index 0000000000..4f2bdc20a8 --- /dev/null +++ b/Source/Core/Core/IOS/Starlet/ARMJitX64.cpp @@ -0,0 +1,2336 @@ +// Copyright 2026 Dolphin Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +#include "Core/IOS/Starlet/ARMJitX64.h" + +#include +#include +#include +#include + +#include "Common/BitSet.h" +#include "Common/CommonTypes.h" +#include "Common/x64ABI.h" +#include "Core/HW/Memmap.h" +#include "Core/IOS/Starlet/ARMCore.h" + +using namespace Gen; + +namespace IOS::LLE +{ +namespace +{ +// Keep the same long-lived execution model as the Broadway JIT: these host registers survive +// dispatcher helper calls and remain live for the whole Starlet slice. +constexpr BitSet32 JIT_CALLEE_SAVED = BitSet32{RBX, RBP, R12, R13, R14, R15}; +constexpr X64Reg JIT_NATIVE_COUNT = R12; +constexpr X64Reg JIT_EXECUTED_COUNT = R13; +constexpr X64Reg JIT_DOWNCOUNT = R14; +constexpr X64Reg JIT_CORE = R15; +constexpr u32 STARLET_EXCEPTION_VECTOR_BASE = 0xffff0000; + +enum class SlowMemoryRegion +{ + RAM, + SRAM, + MMIO, + Other, +}; + +SlowMemoryRegion ClassifySlowMemoryAddress(u32 address) +{ + if (address < Memory::MEM1_SIZE_RETAIL || + (address >= 0x10000000 && address < 0x10000000 + Memory::MEM2_SIZE_RETAIL)) + { + return SlowMemoryRegion::RAM; + } + if ((address >= 0x0d400000 && address < 0x0d420000) || address >= 0xfff00000) + return SlowMemoryRegion::SRAM; + if (address >= 0x0d800000 && address < 0x0e000000) + return SlowMemoryRegion::MMIO; + return SlowMemoryRegion::Other; +} +} // namespace + +ARMJitX64::ARMJitX64(ARMCore& core) : m_core(core) +{ + m_fastmem_base = m_core.m_bus.GetFastmemBase(); + // SRAM is a plain Hollywood memory array just like MEM1/MEM2. The generated address checks + // below still side-exit for boot0 overlays and the unmapped split-window gap, so only actual + // SRAM reaches this pointer; MMIO and ROM writes retain the exact bus path. + m_sram_base = m_core.m_bus.GetFastmemSRAMBase(); + m_boot0_mapped = m_core.m_bus.GetFastmemBoot0Mapped(); + m_sram_split_mode = m_core.m_bus.GetFastmemSRAMSplitMode(); + const auto* const base = reinterpret_cast(&m_core); + m_registers_offset = static_cast(reinterpret_cast(&m_core.m_registers) - base); + m_cpsr_offset = static_cast(reinterpret_cast(&m_core.m_cpsr) - base); + m_instruction_address_offset = + static_cast(reinterpret_cast(&m_core.m_instruction_address) - base); + m_pc_written_offset = static_cast(reinterpret_cast(&m_core.m_pc_written) - base); + m_waiting_for_interrupt_offset = + static_cast(reinterpret_cast(&m_core.m_waiting_for_interrupt) - base); + m_waiting_for_memory_poll_offset = + static_cast(reinterpret_cast(&m_core.m_waiting_for_memory_poll) - base); + m_yield_requested_offset = + static_cast(reinterpret_cast(&m_core.m_yield_requested) - base); + m_executed_instructions_offset = + static_cast(reinterpret_cast(&m_core.m_executed_instructions) - base); + m_process_id_offset = + static_cast(reinterpret_cast(&m_core.m_cp15.process_id) - base); + m_tlb_generation_offset = + static_cast(reinterpret_cast(&m_core.m_tlb_generation) - base); + + AllocCodeSpace(CODE_SIZE); + // Poison once at allocation time. Runtime CP15 invalidations only reset the allocation cursor; + // rewriting the entire cache on every guest I-cache operation is prohibitively expensive. + ClearCodeSpace(); + GenerateDispatcher(); +} + +ARMJitX64::~ARMJitX64() +{ + FreeCodeSpace(); +} + +void ARMJitX64::PoisonMemory() +{ + std::memset(region, 0xcc, region_size); +} + +void ARMJitX64::Clear() +{ + // CP15 I-cache maintenance can be reached through a fallback at the end of the currently + // executing host block. Defer destruction until its RET has brought us back to C++. + if (m_is_running) + { + m_clear_pending = true; + return; + } + m_blocks.clear(); + std::ranges::fill(m_fast_entries, FastEntry{}); + SetCodePtr(m_block_code_begin, region + region_size); +} + +void ARMJitX64::InvalidateTranslationContext() +{ + // A TLB invalidation changes which physical page a virtual PC resolves to, but it does not + // invalidate the ARM instruction cache. Keep already translated physical code and force the + // generated dispatcher to resolve the next virtual PC through the current page tables. This is + // particularly important for IOS, which flushes its TLB on every process switch. + std::ranges::fill(m_fast_entries, FastEntry{}); +} + +u32 ARMJitX64::Run(u64 cycle_budget) +{ + if (cycle_budget == 0 || !m_run_entry) + return 0; + + m_is_running = true; + const u64 result = + m_run_entry(static_cast(std::min(cycle_budget, std::numeric_limits::max()))); + m_is_running = false; + if (m_clear_pending) + { + m_clear_pending = false; + Clear(); + } + const u32 executed = static_cast(result); + m_executed_instructions += executed; + m_native_executed_instructions += result >> 32; + return executed; +} + +void ARMJitX64::GenerateDispatcher() +{ + m_run_entry = reinterpret_cast(AlignCode16()); + ABI_PushRegistersAndAdjustStack(JIT_CALLEE_SAVED, 8); + MOV(64, R(JIT_CORE), ImmPtr(&m_core)); + MOV(32, R(JIT_DOWNCOUNT), R(ABI_PARAM1)); + XOR(32, R(JIT_EXECUTED_COUNT), R(JIT_EXECUTED_COUNT)); + XOR(32, R(JIT_NATIVE_COUNT), R(JIT_NATIVE_COUNT)); + + m_dispatcher = GetCodePtr(); + TEST(32, R(JIT_DOWNCOUNT), R(JIT_DOWNCOUNT)); + FixupBranch budget_exhausted = J_CC(CC_Z, Jump::Near); + CMP(8, MYieldRequested(), Imm8(0)); + FixupBranch yielded = J_CC(CC_NE, Jump::Near); + CMP(8, MWaitingForInterrupt(), Imm8(0)); + FixupBranch waiting_for_interrupt = J_CC(CC_NE, Jump::Near); + CMP(8, MWaitingForMemoryPoll(), Imm8(0)); + FixupBranch waiting_for_memory_poll = J_CC(CC_NE, Jump::Near); + MOV(64, R(RAX), ImmPtr(&m_clear_pending)); + CMP(8, MatR(RAX), Imm8(0)); + FixupBranch invalidated = J_CC(CC_NE, Jump::Near); + + // Direct-mapped native block cache. The key includes CPSR.T in bit zero, matching the C++ map. + MOV(32, R(EAX), MRegister(15)); + MOV(32, R(ECX), MCPSR()); + SHR(32, R(ECX), Imm8(5)); + AND(32, R(ECX), Imm8(1)); + OR(32, R(EAX), R(ECX)); + MOV(32, R(EDX), R(EAX)); + SHR(32, R(EDX), Imm8(1)); + AND(32, R(EDX), Imm32(static_cast(FAST_ENTRY_COUNT - 1))); + SHL(64, R(RDX), Imm8(4)); + MOV(64, R(R11), ImmPtr(m_fast_entries.data())); + ADD(64, R(R11), R(RDX)); + CMP(32, MDisp(R11, static_cast(offsetof(FastEntry, key))), R(EAX)); + FixupBranch cache_miss = J_CC(CC_NE); + MOV(64, R(R11), MDisp(R11, static_cast(offsetof(FastEntry, entry)))); + TEST(64, R(R11), R(R11)); + FixupBranch empty_entry = J_CC(CC_Z); + JMPptr(R(R11)); + + SetJumpTarget(cache_miss); + SetJumpTarget(empty_entry); + MOV(64, R(ABI_PARAM1), ImmPtr(this)); + ABI_CallFunction(Dispatch); + TEST(64, R(RAX), R(RAX)); + FixupBranch cannot_compile = J_CC(CC_Z, Jump::Near); + JMPptr(R(RAX)); + + m_dispatcher_exit = GetCodePtr(); + SetJumpTarget(budget_exhausted); + SetJumpTarget(yielded); + SetJumpTarget(waiting_for_interrupt); + SetJumpTarget(waiting_for_memory_poll); + SetJumpTarget(invalidated); + SetJumpTarget(cannot_compile); + MOV(32, R(EAX), R(JIT_EXECUTED_COUNT)); + MOV(64, R(RDX), R(JIT_NATIVE_COUNT)); + SHL(64, R(RDX), Imm8(32)); + OR(64, R(RAX), R(RDX)); + MOV(64, R(RCX), R(JIT_EXECUTED_COUNT)); + ADD(64, MExecutedInstructions(), R(RCX)); + ABI_PopRegistersAndAdjustStack(JIT_CALLEE_SAVED, 8); + RET(); + + m_block_code_begin = AlignCode16(); +} + +const u8* ARMJitX64::Dispatch(ARMJitX64* jit) +{ + const bool thumb = (jit->m_core.m_cpsr & ARMCore::CPSR_T) != 0; + const u32 key = jit->m_core.m_registers[15] | (thumb ? 1U : 0U); + Block* const block = jit->GetOrCompileBlock(key); + if (!block || !block->runnable) + return nullptr; + + FastEntry& fast_entry = jit->m_fast_entries[(key >> 1) & (FAST_ENTRY_COUNT - 1)]; + fast_entry.key = key; + fast_entry.entry = block->entry; + return block->entry; +} + +ARMJitX64::Block* ARMJitX64::GetOrCompileBlock(u32 address) +{ + const bool thumb = (address & 1) != 0; + const u32 virtual_address = address & ~1U; + const u32 physical_address = m_core.TranslateVirtualAddress(virtual_address); + const u64 block_key = (static_cast(physical_address) << 32) | address; + if (const auto it = m_blocks.find(block_key); it != m_blocks.end()) + return &it->second; + + if (IsAlmostFull()) + { + if (m_is_running) + { + m_clear_pending = true; + return nullptr; + } + Clear(); + } + + Block block = CompileBlock(virtual_address, thumb); + return &m_blocks.emplace(block_key, block).first->second; +} + +ARMJitX64::Block ARMJitX64::CompileBlock(u32 address, bool thumb) +{ + const u8* const body = AlignCode16(); + LoadRegisterCache(); + MOV(8, MPCWritten(), Imm8(0)); + + u32 current_address = address; + u32 instruction_count = 0; + u32 native_instruction_count = 0; + bool terminated = false; + bool dispatcher_exit = false; + // The physical address in the cache key covers the first ARM TLB granule. End the block before + // a 1 KiB boundary so a remapping of the following page can never reuse stale translated code. + const u32 translation_granule = address & ~0x3ffU; + while (instruction_count < MAX_BLOCK_INSTRUCTIONS && + (current_address & ~0x3ffU) == translation_granule) + { + ++instruction_count; + m_compile_instruction_count = instruction_count; + m_compile_native_instruction_count = native_instruction_count; + if (thumb) + { + const u16 instruction = m_core.FetchThumbInstruction(current_address); + if (!EmitDirectThumb(instruction, current_address, &terminated)) + { + EmitFallbackThumb(instruction, current_address); + terminated = true; + break; + } + else + { + ++native_instruction_count; + } + if (terminated) + break; + current_address += 2; + } + else + { + const u32 instruction = m_core.FetchARMInstruction(current_address); + if (!EmitDirectARM(instruction, current_address, &terminated, &dispatcher_exit)) + { + EmitFallbackARM(instruction, current_address); + terminated = true; + break; + } + else + { + ++native_instruction_count; + } + if (terminated) + break; + current_address += 4; + } + } + + if (!terminated) + { + MOV(32, MRegister(15), Imm32(current_address)); + MOV(32, MInstructionAddress(), Imm32(current_address - (thumb ? 2 : 4))); + } + EmitBlockExit(instruction_count, native_instruction_count, dispatcher_exit); + + // The entry stub prevents a translated block from overrunning the scheduler's remaining ARM + // budget. Keeping it separate lets us know the final block size without patching machine code. + const u8* const entry = AlignCode16(); + CMP(32, R(JIT_DOWNCOUNT), Imm32(instruction_count)); + J_CC(CC_B, m_dispatcher_exit); + JMP(body); + + return {.entry = entry, + .instruction_count = instruction_count, + .native_instruction_count = native_instruction_count, + .runnable = native_instruction_count != 0}; +} + +bool ARMJitX64::EmitDirectARM(u32 instruction, u32 address, bool* terminal, + bool* dispatcher_exit) +{ + // MCR p15, 0, Rd, c7, c10, 4 is ARM926 Drain Write Buffer. It is an ordering barrier, not an + // instruction-cache invalidation, and has no additional observable work in this single-host- + // thread memory model. IOS executes it in hot synchronization paths. + if ((instruction & 0xffff0fff) == 0xee070f9a) + return true; + + // IOS's normal kernel ABI does not use SVC. User modules issue an intentionally undefined + // instruction, E6000010 | (syscall_number << 5), and the undefined-instruction vector decodes + // it. Enter that architectural exception directly instead of paying for a generic interpreter + // fallback at every syscall boundary. + if ((instruction & 0xffffe01f) == 0xe6000010) + { + FlushRegisterCache(); + MOV(64, R(ABI_PARAM1), ImmPtr(this)); + MOV(32, R(ABI_PARAM2), Imm32(instruction)); + MOV(32, R(ABI_PARAM3), Imm32(address)); + ABI_CallFunction(EnterUndefinedARM); + *terminal = true; + return true; + } + + // Production IOS also retains SVC 0xAB for debugger semihosting. Preserve the real supervisor + // exception path; only the generic decode/dispatch overhead is removed here. + if ((instruction & 0xff000000) == 0xef000000) + { + FlushRegisterCache(); + MOV(64, R(ABI_PARAM1), ImmPtr(this)); + MOV(32, R(ABI_PARAM2), Imm32(address)); + ABI_CallFunction(EnterSVCARM); + *terminal = true; + return true; + } + + // IOS enters the kernel through an undefined-instruction syscall and switches register banks + // several times before returning. These are architectural ARM926 operations, not HLE of the + // syscall itself. Keeping them in the translated block avoids five interpreter/dispatcher + // round trips for virtually every IOS service call. + const u32 condition = instruction >> 28; + const bool is_mrs = (instruction & 0x0fbf0fff) == 0x010f0000; + const bool is_msr_register = (instruction & 0x0fb0fff0) == 0x0120f000; + const bool is_msr_immediate = (instruction & 0x0fb0f000) == 0x0320f000; + if (condition == 0xe && is_mrs) + { + const bool spsr = (instruction & (1U << 22)) != 0; + const u32 rd = (instruction >> 12) & 0xf; + if (!spsr) + { + MOV(32, R(EAX), MCPSR()); + MOV(32, MRegister(rd), R(EAX)); + return true; + } + + FlushRegisterCache(); + MOV(64, R(ABI_PARAM1), ImmPtr(this)); + ABI_CallFunction(ReadSPSR); + MOV(32, MStoredRegister(rd), R(EAX)); + LoadRegisterCache(); + return true; + } + if (condition == 0xe && (is_msr_register || is_msr_immediate)) + { + const bool spsr = (instruction & (1U << 22)) != 0; + const u32 field_mask = (instruction >> 16) & 0xf; + FlushRegisterCache(); + MOV(64, R(ABI_PARAM1), ImmPtr(this)); + MOV(32, R(ABI_PARAM2), Imm32(spsr ? 1 : 0)); + MOV(32, R(ABI_PARAM3), Imm32(field_mask)); + if (is_msr_immediate) + { + const u32 rotate = ((instruction >> 8) & 0xf) * 2; + MOV(32, R(ABI_PARAM4), Imm32(std::rotr(instruction & 0xff, rotate))); + } + else + { + MOV(32, R(ABI_PARAM4), MStoredRegister(instruction & 0xf)); + } + ABI_CallFunction(WritePSR); + if (spsr) + { + LoadRegisterCache(); + } + else + { + // A CPSR control-field write can switch banked registers or ARM/Thumb state. End the block + // after the native operation so the dispatcher observes that new architectural context. + MOV(32, MStoredRegister(15), Imm32(address + 4)); + MOV(32, MInstructionAddress(), Imm32(address)); + MOV(8, MPCWritten(), Imm8(1)); + *terminal = true; + // IRQ/FIQ lines are sampled by ARMCore outside the native dispatcher. A CPSR control write + // can unmask a line that was already asserted when this JIT slice began, so executing even + // one more guest instruction here could miss the only unmasked window of IOS's interrupt + // save/restore sequence. + *dispatcher_exit = true; + } + return true; + } + + // MOVS PC, Rm is the canonical exception return at the end of the IOS syscall and IRQ paths. + if (condition == 0xe && (instruction & 0x0ffffff0) == 0x01b0f000) + { + FlushRegisterCache(); + MOV(64, R(ABI_PARAM1), ImmPtr(this)); + MOV(32, R(ABI_PARAM2), MStoredRegister(instruction & 0xf)); + ABI_CallFunction(ExceptionReturn); + MOV(32, MInstructionAddress(), Imm32(address)); + MOV(8, MPCWritten(), Imm8(1)); + *terminal = true; + // The SPSR restored by an exception return can re-enable IRQ/FIQ just like MSR CPSR_c. + *dispatcher_exit = true; + return true; + } + + // Reading r15 as an ARM data-processing operand observes the architectural PC value (the + // current instruction address plus eight). IOS uses MOV LR, PC in its syscall return path. + // Keep the general PC-operand forms on the interpreter, but compile this exact hot form. + if (condition == 0xe && instruction == 0xe1a0e00f) + { + MOV(32, MRegister(14), Imm32(address + 8)); + return true; + } + + // The syscall prologue/epilogue saves and restores the user register bank with STM/LDM ^. + // Execute the exact existing banked-transfer implementation as an in-block helper. Neither + // canonical form loads PC, so translation can continue without a dispatcher exit. + if (condition == 0xe && (instruction & 0x0e000000) == 0x08000000 && + (instruction & (1U << 22)) != 0 && (instruction & (1U << 15)) == 0) + { + FlushRegisterCache(); + MOV(64, R(ABI_PARAM1), ImmPtr(this)); + MOV(32, R(ABI_PARAM2), Imm32(instruction)); + MOV(32, R(ABI_PARAM3), Imm32(address)); + ABI_CallFunction(ExecuteUserBankBlockTransfer); + LoadRegisterCache(); + return true; + } + + if ((instruction & 0x0ffffff0) == 0x012fff10 || + (instruction & 0x0ffffff0) == 0x012fff30) + { + if ((instruction >> 28) != 0xe) + return false; + const bool link = (instruction & 0x20) != 0; + const u32 rm = instruction & 0xf; + EmitExchangeBranch(rm == 15 ? Imm32(address + 8) : MRegister(rm), link, address + 4); + MOV(32, MInstructionAddress(), Imm32(address)); + MOV(8, MPCWritten(), Imm8(1)); + *terminal = true; + return true; + } + + if ((instruction & 0x0e000000) == 0x0a000000 && (instruction >> 28) != 0xf) + { + const bool link = (instruction & (1U << 24)) != 0; + const s32 offset = ARMCore::SignExtend((instruction & 0x00ffffff) << 2, 26); + const u32 target = address + 8 + static_cast(offset); + // Preserve the interpreter boundary for IOS's timer polling loop. It recognizes that the + // Hollywood timer is immutable until the end of this scheduler slice and can yield safely. + if (condition != 0xe && target + 16 == address) + return false; + if (condition == 0xe) + { + if (link) + MOV(32, MRegister(14), Imm32(address + 4)); + MOV(32, MRegister(15), Imm32(target)); + } + else + { + EmitConditionResult(condition); + TEST(32, R(EAX), R(EAX)); + FixupBranch not_taken = J_CC(CC_Z); + if (link) + MOV(32, MRegister(14), Imm32(address + 4)); + MOV(32, MRegister(15), Imm32(target)); + FixupBranch done = J(); + SetJumpTarget(not_taken); + MOV(32, MRegister(15), Imm32(address + 4)); + SetJumpTarget(done); + } + MOV(32, MInstructionAddress(), Imm32(address)); + MOV(8, MPCWritten(), Imm8(1)); + *terminal = true; + return true; + } + + if (condition != 0xe) + { + // ARM predication is pervasive in IOS's scheduler, allocator and error paths. Any data- + // processing form already proven safe below can share the native predicate guard; a failed + // condition simply skips the operation and continues at the next guest instruction. + if (condition == 0xf || !CanEmitARMDataProcessing(instruction)) + return false; + + EmitConditionResult(condition); + TEST(32, R(EAX), R(EAX)); + const FixupBranch predicate_failed = J_CC(CC_Z, Jump::Near); + EmitARMDataProcessing(instruction); + SetJumpTarget(predicate_failed); + return true; + } + + if ((instruction & 0x0c000000) == 0x04000000) + return EmitARMMemory(instruction, address); + + if ((instruction & 0x0e000090) == 0x00000090) + return EmitARMHalfwordMemory(instruction, address); + + if ((instruction & 0x0e000000) == 0x08000000) + return EmitARMBlockTransfer(instruction, address, terminal); + + if (!CanEmitARMDataProcessing(instruction)) + return false; + + EmitARMDataProcessing(instruction); + return true; +} + +bool ARMJitX64::CanEmitARMDataProcessing(u32 instruction) const +{ + // The special encodings in the data-processing space stay on the exact interpreter path. The + // common ALU subset below covers straight-line boot and IOS code. + // Zero is a valid ANDEQ encoding, but it is also the erased/uninitialized instruction fill used + // throughout IOS images and our bus. Keep it as a block boundary so translation cannot run into + // data or precompile code that a loader has not populated yet. + if (instruction == 0) + return false; + if ((instruction & 0x0c000000) != 0) + return false; + if ((instruction & 0x0fff0ff0) == 0x016f0f10 || (instruction & 0x0f8000f0) == 0x00800090 || + (instruction & 0x0fc000f0) == 0x00000090 || (instruction & 0x0fb00ff0) == 0x01000090 || + (instruction & 0x0fbf0fff) == 0x010f0000 || (instruction & 0x0fb0fff0) == 0x0120f000 || + (instruction & 0x0fb0f000) == 0x0320f000 || (instruction & 0x0e000090) == 0x00000090) + { + return false; + } + + const bool immediate = (instruction & (1U << 25)) != 0; + const u32 opcode = (instruction >> 21) & 0xf; + const bool set_flags = (instruction & (1U << 20)) != 0; + const bool writes_result = opcode < 8 || opcode > 0xb; + const u32 rn = (instruction >> 16) & 0xf; + const u32 rd = (instruction >> 12) & 0xf; + const u32 rm = instruction & 0xf; + if ((opcode != 0xd && opcode != 0xf && rn == 15) || (writes_result && rd == 15) || + (!immediate && rm == 15)) + { + return false; + } + if (opcode == 5 || opcode == 6 || opcode == 7) + return false; + + const bool shifted_register_operand = !immediate && (instruction & 0xff0) != 0; + const bool shift_by_register = shifted_register_operand && (instruction & (1U << 4)) != 0; + const u32 rs = (instruction >> 8) & 0xf; + // Carry-out from a shifted operand is only architecturally visible for logical flag-setting + // operations. Keep those on the interpreter for now; all non-flag-setting ALU forms can use the + // value-only native shifter exactly. + if (shifted_register_operand && (set_flags || !writes_result)) + return false; + if (shift_by_register && rs == 15) + return false; + + const bool logical = opcode == 0 || opcode == 1 || opcode == 8 || opcode == 9 || opcode == 0xc || + opcode == 0xd || opcode == 0xe || opcode == 0xf; + const u32 rotate = ((instruction >> 8) & 0xf) * 2; + if (logical && (set_flags || !writes_result) && immediate && rotate != 0) + return false; + + return true; +} + +bool ARMJitX64::EmitARMMemory(u32 instruction, u32 address) +{ + if (!m_fastmem_base) + return false; + + const bool preindex = (instruction & (1U << 24)) != 0; + const bool add_offset = (instruction & (1U << 23)) != 0; + const bool byte = (instruction & (1U << 22)) != 0; + const bool writeback = !preindex || (instruction & (1U << 21)) != 0; + const bool load = (instruction & (1U << 20)) != 0; + const u32 rn = (instruction >> 16) & 0xf; + const u32 rd = (instruction >> 12) & 0xf; + const bool register_offset = (instruction & (1U << 25)) != 0; + const u32 offset = instruction & 0xfff; + if (rd == 15 || (rn == 15 && (!preindex || writeback)) || (load && writeback && rn == rd)) + return false; + if (register_offset && (instruction & (1U << 4)) != 0) + return false; + const u32 rm = instruction & 0xf; + if (register_offset && rm == 15) + return false; + + // RAM fast paths and their interpreter side exits share an explicit architectural boundary. + // Materialize the two cached guest registers before either path can observe them. + FlushRegisterCache(); + + // R10D keeps the indexed/writeback value while EAX is translated to a physical address. + if (rn == 15) + MOV(32, R(R10), Imm32(address + 8)); + else + MOV(32, R(R10), MRegister(rn)); + if (register_offset) + { + MOV(32, R(EDX), MRegister(rm)); + const u32 shift_type = (instruction >> 5) & 3; + const u32 amount = (instruction >> 7) & 0x1f; + if (shift_type == 0) + { + if (amount != 0) + SHL(32, R(EDX), Imm8(amount)); + } + else if (shift_type == 1) + { + if (amount == 0) + XOR(32, R(EDX), R(EDX)); + else + SHR(32, R(EDX), Imm8(amount)); + } + else if (shift_type == 2) + { + SAR(32, R(EDX), Imm8(amount == 0 ? 31 : amount)); + } + else if (amount == 0) + { + // RRX uses the old CPSR carry as bit 31. + MOV(32, R(ECX), MCPSR()); + SHR(32, R(ECX), Imm8(29)); + SHL(32, R(ECX), Imm8(31)); + SHR(32, R(EDX), Imm8(1)); + OR(32, R(EDX), R(ECX)); + } + else + { + ROR(32, R(EDX), Imm8(amount)); + } + if (add_offset) + ADD(32, R(R10), R(EDX)); + else + SUB(32, R(R10), R(EDX)); + } + else if (offset != 0) + { + if (add_offset) + ADD(32, R(R10), Imm32(offset)); + else + SUB(32, R(R10), Imm32(offset)); + } + if (preindex) + MOV(32, R(EAX), R(R10)); + else + { + if (rn == 15) + MOV(32, R(EAX), Imm32(address + 8)); + else + MOV(32, R(EAX), MRegister(rn)); + } + + const u32 access_size = byte ? 1 : 4; + std::vector slow_paths; + EmitFastmemAddress(&slow_paths, access_size, 0, + load ? SRAMFastmemAccess::Read : SRAMFastmemAccess::Write, !byte); + + if (load) + { + if (byte) + { + MOVZX(32, 8, EAX, MatR(R11)); + } + else + { + MOV(32, R(EAX), MatR(R11)); + if (m_core.m_big_endian) + BSWAP(32, EAX); + // ARM926 word loads align the bus access down and rotate the fetched word according to + // the original address. EmitFastmemAddress preserved that byte offset in ECX. + SHL(32, R(ECX), Imm8(3)); + ROR(32, R(EAX), R(ECX)); + } + MOV(32, MRegister(rd), R(EAX)); + } + else + { + MOV(32, R(EDX), MRegister(rd)); + if (!byte && m_core.m_big_endian) + BSWAP(32, EDX); + MOV(access_size * 8, MatR(R11), R(EDX)); + } + if (writeback) + MOV(32, MRegister(rn), R(R10)); + + const FixupBranch direct_done = J(); + EmitARMMemorySlowPath(slow_paths, instruction, address, direct_done); + return true; +} + +bool ARMJitX64::EmitARMHalfwordMemory(u32 instruction, u32 address) +{ + if (!m_fastmem_base) + return false; + + const bool preindex = (instruction & (1U << 24)) != 0; + const bool add_offset = (instruction & (1U << 23)) != 0; + const bool immediate = (instruction & (1U << 22)) != 0; + const bool writeback = !preindex || (instruction & (1U << 21)) != 0; + const bool load = (instruction & (1U << 20)) != 0; + const u32 rn = (instruction >> 16) & 0xf; + const u32 rd = (instruction >> 12) & 0xf; + const u32 type = (instruction >> 5) & 3; + const u32 rm = instruction & 0xf; + const u32 offset = ((instruction >> 4) & 0xf0) | (instruction & 0xf); + if (type == 0 || (!load && type != 1) || rd == 15 || + (rn == 15 && (!preindex || writeback)) || (load && writeback && rn == rd) || + (!immediate && rm == 15)) + { + return false; + } + + FlushRegisterCache(); + if (rn == 15) + MOV(32, R(R10), Imm32(address + 8)); + else + MOV(32, R(R10), MRegister(rn)); + if (immediate) + { + if (offset != 0) + { + if (add_offset) + ADD(32, R(R10), Imm32(offset)); + else + SUB(32, R(R10), Imm32(offset)); + } + } + else if (add_offset) + { + ADD(32, R(R10), MRegister(rm)); + } + else + { + SUB(32, R(R10), MRegister(rm)); + } + + if (preindex) + MOV(32, R(EAX), R(R10)); + else if (rn == 15) + MOV(32, R(EAX), Imm32(address + 8)); + else + MOV(32, R(EAX), MRegister(rn)); + + const u32 access_size = type == 2 ? 1 : 2; + std::vector slow_paths; + EmitFastmemAddress(&slow_paths, access_size, 0, + load ? SRAMFastmemAccess::Read : SRAMFastmemAccess::Write); + if (load) + { + if (type == 1) + { + MOVZX(32, 16, EAX, MatR(R11)); + if (m_core.m_big_endian) + ROL(16, R(EAX), Imm8(8)); + } + else if (type == 2) + { + MOVSX(32, 8, EAX, MatR(R11)); + } + else + { + MOVZX(32, 16, EAX, MatR(R11)); + if (m_core.m_big_endian) + ROL(16, R(EAX), Imm8(8)); + MOVSX(32, 16, EAX, R(EAX)); + } + MOV(32, MRegister(rd), R(EAX)); + } + else + { + MOV(32, R(EDX), MRegister(rd)); + if (m_core.m_big_endian) + ROL(16, R(EDX), Imm8(8)); + MOV(16, MatR(R11), R(EDX)); + } + if (writeback) + MOV(32, MRegister(rn), R(R10)); + + const FixupBranch direct_done = J(Jump::Near); + EmitARMMemorySlowPath(slow_paths, instruction, address, direct_done); + return true; +} + +bool ARMJitX64::EmitARMBlockTransfer(u32 instruction, u32 address, bool* terminal) +{ + if (!m_fastmem_base) + return false; + + const bool preindex = (instruction & (1U << 24)) != 0; + const bool increment = (instruction & (1U << 23)) != 0; + const bool psr_or_user = (instruction & (1U << 22)) != 0; + const bool writeback = (instruction & (1U << 21)) != 0; + const bool load = (instruction & (1U << 20)) != 0; + const u32 rn = (instruction >> 16) & 0xf; + const u32 register_list = instruction & 0xffff; + const u32 count = std::popcount(register_list); + if (psr_or_user || rn == 15 || register_list == 0 || + (writeback && (register_list & (1U << rn)) != 0)) + { + return false; + } + + const u32 range_size = count * 4; + FlushRegisterCache(); + MOV(32, R(R10), MRegister(rn)); + MOV(32, R(EAX), R(R10)); + if (increment) + { + if (preindex) + ADD(32, R(EAX), Imm32(4)); + } + else + { + SUB(32, R(EAX), Imm32(preindex ? range_size : range_size - 4)); + } + + std::vector slow_paths; + if ((m_core.m_cp15.control & 1U) != 0) + { + MOV(32, R(EDX), R(EAX)); + AND(32, R(EDX), Imm32(0x3ff)); + CMP(32, R(EDX), Imm32(0x400 - range_size)); + slow_paths.push_back(J_CC(CC_A, Jump::Near)); + } + // Register-list transfers are deliberately excluded from SRAM fastmem. Their boundary and + // writeback behavior needs a separate full-chain regression before they can bypass the bus. + EmitFastmemAddress(&slow_paths, 4, range_size, SRAMFastmemAccess::None); + + u32 memory_offset = 0; + for (u32 reg = 0; reg < 16; ++reg) + { + if ((register_list & (1U << reg)) == 0) + continue; + const OpArg memory = memory_offset == 0 ? MatR(R11) : MDisp(R11, memory_offset); + if (load) + { + MOV(32, R(EAX), memory); + if (m_core.m_big_endian) + BSWAP(32, EAX); + MOV(32, MRegister(reg), R(EAX)); + } + else + { + if (reg == 15) + MOV(32, R(EDX), Imm32(address + 12)); + else + MOV(32, R(EDX), MRegister(reg)); + if (m_core.m_big_endian) + BSWAP(32, EDX); + MOV(32, memory, R(EDX)); + } + memory_offset += 4; + } + + if (writeback) + { + if (increment) + ADD(32, R(R10), Imm32(range_size)); + else + SUB(32, R(R10), Imm32(range_size)); + MOV(32, MRegister(rn), R(R10)); + } + + if (load && (register_list & (1U << 15)) != 0) + { + MOV(32, R(EAX), MRegister(15)); + EmitExchangeBranch(R(EAX), false, 0); + MOV(32, MInstructionAddress(), Imm32(address)); + MOV(8, MPCWritten(), Imm8(1)); + *terminal = true; + } + + const FixupBranch direct_done = J(Jump::Near); + EmitARMMemorySlowPath(slow_paths, instruction, address, direct_done); + return true; +} + +void ARMJitX64::EmitARMDataProcessing(u32 instruction) +{ + const u32 opcode = (instruction >> 21) & 0xf; + const bool set_flags = (instruction & (1U << 20)) != 0; + const bool immediate = (instruction & (1U << 25)) != 0; + const u32 rn = (instruction >> 16) & 0xf; + const u32 rd = (instruction >> 12) & 0xf; + const u32 rm = instruction & 0xf; + const bool writes_result = opcode < 8 || opcode > 0xb; + + if (immediate) + { + const u32 rotate = ((instruction >> 8) & 0xf) * 2; + MOV(32, R(EDX), Imm32(std::rotr(instruction & 0xff, rotate))); + } + else + { + MOV(32, R(EDX), MRegister(rm)); + const u32 shift_type = (instruction >> 5) & 3; + if ((instruction & 0xff0) != 0) + { + if ((instruction & (1U << 4)) != 0) + { + const u32 rs = (instruction >> 8) & 0xf; + MOV(32, R(ECX), MRegister(rs)); + AND(32, R(ECX), Imm32(0xff)); + if (shift_type == 3) + { + // x86 and ARM both reduce register ROR counts modulo 32 for the result value. + ROR(32, R(EDX), R(ECX)); + } + else + { + // x86 masks variable LSL/LSR/ASR counts, while ARM saturates counts >= 32. + CMP(32, R(ECX), Imm32(32)); + const FixupBranch below_32 = J_CC(CC_B, Jump::Near); + if (shift_type == 2) + SAR(32, R(EDX), Imm8(31)); + else + XOR(32, R(EDX), R(EDX)); + const FixupBranch shift_done = J(Jump::Near); + SetJumpTarget(below_32); + if (shift_type == 0) + SHL(32, R(EDX), R(ECX)); + else if (shift_type == 1) + SHR(32, R(EDX), R(ECX)); + else + SAR(32, R(EDX), R(ECX)); + SetJumpTarget(shift_done); + } + } + else + { + const u32 amount = (instruction >> 7) & 0x1f; + if (shift_type == 0) + { + if (amount != 0) + SHL(32, R(EDX), Imm8(amount)); + } + else if (shift_type == 1) + { + if (amount == 0) + XOR(32, R(EDX), R(EDX)); + else + SHR(32, R(EDX), Imm8(amount)); + } + else if (shift_type == 2) + { + SAR(32, R(EDX), Imm8(amount == 0 ? 31 : amount)); + } + else if (amount == 0) + { + // ROR #0 is RRX: C becomes bit 31 and the operand shifts right by one. + MOV(32, R(ECX), MCPSR()); + SHR(32, R(ECX), Imm8(29)); + SHL(32, R(ECX), Imm8(31)); + SHR(32, R(EDX), Imm8(1)); + OR(32, R(EDX), R(ECX)); + } + else + { + ROR(32, R(EDX), Imm8(amount)); + } + } + } + } + if (opcode != 0xd && opcode != 0xf) + MOV(32, R(EAX), MRegister(rn)); + + bool arithmetic = false; + switch (opcode) + { + case 0x0: // AND + case 0x8: // TST + AND(32, R(EAX), R(EDX)); + break; + case 0x1: // EOR + case 0x9: // TEQ + XOR(32, R(EAX), R(EDX)); + break; + case 0x2: // SUB + case 0xa: // CMP + SUB(32, R(EAX), R(EDX)); + arithmetic = true; + break; + case 0x3: // RSB + SUB(32, R(EDX), R(EAX)); + MOV(32, R(EAX), R(EDX)); + arithmetic = true; + break; + case 0x4: // ADD + case 0xb: // CMN + ADD(32, R(EAX), R(EDX)); + arithmetic = true; + break; + case 0xc: // ORR + OR(32, R(EAX), R(EDX)); + break; + case 0xd: // MOV + MOV(32, R(EAX), R(EDX)); + break; + case 0xe: // BIC + NOT(32, R(EDX)); + AND(32, R(EAX), R(EDX)); + break; + case 0xf: // MVN + MOV(32, R(EAX), R(EDX)); + NOT(32, R(EAX)); + break; + default: + ASSERT(false); + } + + if (writes_result) + MOV(32, MRegister(rd), R(EAX)); + if (set_flags || !writes_result) + { + if (arithmetic) + EmitArithmeticFlags(opcode == 2 || opcode == 3 || opcode == 0xa); + else + EmitLogicalFlags(EAX); + } +} + +void ARMJitX64::EmitConditionResult(u32 condition) +{ + MOV(32, R(EAX), MCPSR()); + switch (condition) + { + case 0x0: // EQ: Z + SHR(32, R(EAX), Imm8(30)); + AND(32, R(EAX), Imm8(1)); + return; + case 0x1: // NE: !Z + SHR(32, R(EAX), Imm8(30)); + XOR(32, R(EAX), Imm8(1)); + AND(32, R(EAX), Imm8(1)); + return; + case 0x2: // CS: C + SHR(32, R(EAX), Imm8(29)); + AND(32, R(EAX), Imm8(1)); + return; + case 0x3: // CC: !C + SHR(32, R(EAX), Imm8(29)); + XOR(32, R(EAX), Imm8(1)); + AND(32, R(EAX), Imm8(1)); + return; + case 0x4: // MI: N + SHR(32, R(EAX), Imm8(31)); + return; + case 0x5: // PL: !N + SHR(32, R(EAX), Imm8(31)); + XOR(32, R(EAX), Imm8(1)); + return; + case 0x6: // VS: V + SHR(32, R(EAX), Imm8(28)); + AND(32, R(EAX), Imm8(1)); + return; + case 0x7: // VC: !V + SHR(32, R(EAX), Imm8(28)); + XOR(32, R(EAX), Imm8(1)); + AND(32, R(EAX), Imm8(1)); + return; + case 0x8: // HI: C && !Z + MOV(32, R(EDX), R(EAX)); + SHR(32, R(EAX), Imm8(29)); + AND(32, R(EAX), Imm8(1)); + SHR(32, R(EDX), Imm8(30)); + XOR(32, R(EDX), Imm8(1)); + AND(32, R(EAX), R(EDX)); + return; + case 0x9: // LS: !C || Z + MOV(32, R(EDX), R(EAX)); + SHR(32, R(EAX), Imm8(29)); + XOR(32, R(EAX), Imm8(1)); + AND(32, R(EAX), Imm8(1)); + SHR(32, R(EDX), Imm8(30)); + AND(32, R(EDX), Imm8(1)); + OR(32, R(EAX), R(EDX)); + return; + case 0xa: // GE: N == V + MOV(32, R(EDX), R(EAX)); + SHR(32, R(EAX), Imm8(31)); + SHR(32, R(EDX), Imm8(28)); + XOR(32, R(EAX), R(EDX)); + XOR(32, R(EAX), Imm8(1)); + AND(32, R(EAX), Imm8(1)); + return; + case 0xb: // LT: N != V + MOV(32, R(EDX), R(EAX)); + SHR(32, R(EAX), Imm8(31)); + SHR(32, R(EDX), Imm8(28)); + XOR(32, R(EAX), R(EDX)); + AND(32, R(EAX), Imm8(1)); + return; + case 0xc: // GT: !Z && N == V + MOV(32, R(EDX), R(EAX)); + SHR(32, R(EDX), Imm8(30)); + XOR(32, R(EDX), Imm8(1)); + AND(32, R(EDX), Imm8(1)); + MOV(32, R(ECX), R(EAX)); + SHR(32, R(EAX), Imm8(31)); + SHR(32, R(ECX), Imm8(28)); + XOR(32, R(EAX), R(ECX)); + XOR(32, R(EAX), Imm8(1)); + AND(32, R(EAX), R(EDX)); + AND(32, R(EAX), Imm8(1)); + return; + case 0xd: // LE: Z || N != V + MOV(32, R(EDX), R(EAX)); + SHR(32, R(EDX), Imm8(30)); + AND(32, R(EDX), Imm8(1)); + MOV(32, R(ECX), R(EAX)); + SHR(32, R(EAX), Imm8(31)); + SHR(32, R(ECX), Imm8(28)); + XOR(32, R(EAX), R(ECX)); + OR(32, R(EAX), R(EDX)); + AND(32, R(EAX), Imm8(1)); + return; + default: // AL + MOV(32, R(EAX), Imm32(1)); + return; + } +} + +void ARMJitX64::EmitExchangeBranch(OpArg target, bool link, u32 return_address) +{ + MOV(32, R(EAX), target); + if (link) + MOV(32, MRegister(14), Imm32(return_address)); + + MOV(32, R(ECX), R(EAX)); + AND(32, R(ECX), Imm8(1)); + SHL(32, R(ECX), Imm8(5)); + MOV(32, R(EDX), MCPSR()); + AND(32, R(EDX), Imm32(~ARMCore::CPSR_T)); + OR(32, R(EDX), R(ECX)); + MOV(32, MCPSR(), R(EDX)); + + TEST(32, R(EAX), Imm32(1)); + FixupBranch thumb = J_CC(CC_NZ); + AND(32, R(EAX), Imm32(~3U)); + FixupBranch aligned = J(); + SetJumpTarget(thumb); + AND(32, R(EAX), Imm32(~1U)); + SetJumpTarget(aligned); + MOV(32, MRegister(15), R(EAX)); +} + +bool ARMJitX64::EmitDirectThumb(u16 instruction, u32 address, bool* terminal) +{ + if ((instruction & 0xff00) == 0xdf00) + { + FlushRegisterCache(); + MOV(64, R(ABI_PARAM1), ImmPtr(this)); + MOV(32, R(ABI_PARAM2), Imm32(address)); + ABI_CallFunction(EnterSVCThumb); + *terminal = true; + return true; + } + + if ((instruction & 0xf800) == 0xe000) + { + const s32 offset = ARMCore::SignExtend((instruction & 0x07ff) << 1, 12); + MOV(32, MRegister(15), Imm32(address + 4 + static_cast(offset))); + MOV(32, MInstructionAddress(), Imm32(address)); + MOV(8, MPCWritten(), Imm8(1)); + *terminal = true; + return true; + } + + if ((instruction & 0xf000) == 0xd000) + { + const u32 condition = (instruction >> 8) & 0xf; + if (condition >= 0xe) + return false; + const s32 offset = ARMCore::SignExtend(instruction & 0xff, 8) * 2; + const u32 target = address + 4 + static_cast(offset); + + // Preserve the interpreter side exit for IOS's canonical LDR/CMP/branch idle loop. It is the + // boundary that recognizes a safe RAM/SRAM poll and fast-forwards the sleeping ARM clock. + if (target + 4 == address) + { + const u16 load = m_core.FetchThumbInstruction(target); + const u16 compare = m_core.FetchThumbInstruction(target + 2); + if ((load & 0xf800) == 0x6800 && (compare & 0xf800) == 0x2800 && + (compare & 0xff) == 0 && ((compare >> 8) & 7) == (load & 7)) + { + return false; + } + } + + EmitConditionResult(condition); + TEST(32, R(EAX), R(EAX)); + FixupBranch not_taken = J_CC(CC_Z); + MOV(32, MRegister(15), Imm32(target)); + FixupBranch done = J(); + SetJumpTarget(not_taken); + MOV(32, MRegister(15), Imm32(address + 2)); + SetJumpTarget(done); + MOV(32, MInstructionAddress(), Imm32(address)); + MOV(8, MPCWritten(), Imm8(1)); + *terminal = true; + return true; + } + + if ((instruction & 0xf800) == 0x1800) + { + EmitThumbAddSub(instruction); + return true; + } + + if ((instruction & 0xe000) == 0x0000) + { + EmitThumbShiftImmediate(instruction); + return true; + } + + if ((instruction & 0xe000) == 0x2000) + { + EmitThumbImmediate(instruction); + return true; + } + + if ((instruction & 0xfc00) == 0x4000) + { + const u32 opcode = (instruction >> 6) & 0xf; + if (opcode != 0x2 && opcode != 0x4 && opcode != 0x5 && opcode != 0x6 && opcode != 0x7) + { + EmitThumbALU(instruction); + return true; + } + } + + if ((instruction & 0xfc00) == 0x4400) + { + const u32 opcode = (instruction >> 8) & 3; + const u32 rd = (instruction & 7) | ((instruction >> 4) & 8); + const u32 rs = ((instruction >> 3) & 7) | ((instruction >> 3) & 8); + const OpArg rhs = rs == 15 ? Imm32(address + 4) : MRegister(rs); + if (opcode == 3) + { + EmitExchangeBranch(rhs, (instruction & 0x0080) != 0, (address + 2) | 1); + MOV(32, MInstructionAddress(), Imm32(address)); + MOV(8, MPCWritten(), Imm8(1)); + *terminal = true; + return true; + } + + if (rd == 15) + { + if (opcode == 1) + return false; + if (opcode == 0) + { + MOV(32, R(EAX), Imm32(address + 4)); + ADD(32, R(EAX), rhs); + } + else + { + MOV(32, R(EAX), rhs); + } + AND(32, R(EAX), Imm32(~1U)); + MOV(32, MRegister(15), R(EAX)); + MOV(32, MInstructionAddress(), Imm32(address)); + MOV(8, MPCWritten(), Imm8(1)); + *terminal = true; + return true; + } + + if (opcode == 0) + { + MOV(32, R(EAX), MRegister(rd)); + ADD(32, R(EAX), rhs); + MOV(32, MRegister(rd), R(EAX)); + } + else if (opcode == 1) + { + MOV(32, R(EAX), MRegister(rd)); + SUB(32, R(EAX), rhs); + EmitArithmeticFlags(true); + } + else + { + MOV(32, R(EAX), rhs); + MOV(32, MRegister(rd), R(EAX)); + } + return true; + } + + if ((instruction & 0xf800) == 0xf000) + { + const s32 high_offset = ARMCore::SignExtend(instruction & 0x7ff, 11) * 4096; + MOV(32, MRegister(14), Imm32(address + 4 + static_cast(high_offset))); + return true; + } + + if ((instruction & 0xf800) == 0xf800 || (instruction & 0xf800) == 0xe800) + { + const bool exchange = (instruction & 0xf800) == 0xe800; + MOV(32, R(EAX), MRegister(14)); + ADD(32, R(EAX), Imm32((instruction & 0x7ff) << 1)); + MOV(32, MRegister(14), Imm32((address + 2) | 1)); + if (exchange) + { + AND(32, MCPSR(), Imm32(~ARMCore::CPSR_T)); + AND(32, R(EAX), Imm32(~3U)); + } + else + { + AND(32, R(EAX), Imm32(~1U)); + } + MOV(32, MRegister(15), R(EAX)); + MOV(32, MInstructionAddress(), Imm32(address)); + MOV(8, MPCWritten(), Imm8(1)); + *terminal = true; + return true; + } + + if ((instruction & 0xf000) == 0xa000) + { + const bool use_sp = (instruction & 0x0800) != 0; + const u32 rd = (instruction >> 8) & 7; + const u32 offset = (instruction & 0xff) << 2; + if (use_sp) + { + MOV(32, R(EAX), MRegister(13)); + ADD(32, R(EAX), Imm32(offset)); + } + else + { + MOV(32, R(EAX), Imm32(((address + 4) & ~3U) + offset)); + } + MOV(32, MRegister(rd), R(EAX)); + return true; + } + + if ((instruction & 0xff00) == 0xb000) + { + const u32 offset = (instruction & 0x7f) << 2; + if (instruction & 0x0080) + SUB(32, MRegister(13), Imm32(offset)); + else + ADD(32, MRegister(13), Imm32(offset)); + return true; + } + + // PUSH/POP without PC is pervasive in Thumb IOS modules. Keep the exact MMU and bank behavior + // in the existing implementation, but do not terminate the translated block around it. + if ((instruction & 0xf600) == 0xb400 && + !((instruction & 0x0800) != 0 && (instruction & 0x0100) != 0)) + { + FlushRegisterCache(); + MOV(64, R(ABI_PARAM1), ImmPtr(this)); + MOV(32, R(ABI_PARAM2), Imm32(instruction)); + MOV(32, R(ABI_PARAM3), Imm32(address)); + ABI_CallFunction(ExecuteThumbPushPop); + LoadRegisterCache(); + return true; + } + + if (EmitThumbMemory(instruction, address)) + return true; + + return false; +} + +bool ARMJitX64::EmitThumbMemory(u16 instruction, u32 address) +{ + if (!m_fastmem_base) + return false; + + FlushRegisterCache(); + + bool load = false; + bool sign_extend = false; + u32 access_size = 0; + u32 rd = 0; + + if ((instruction & 0xf800) == 0x4800) // LDR Rd, [PC, #imm] + { + load = true; + access_size = 4; + rd = (instruction >> 8) & 7; + MOV(32, R(EAX), Imm32(((address + 4) & ~3U) + ((instruction & 0xff) << 2))); + } + else if ((instruction & 0xf000) == 0x5000) // Register-offset loads and stores. + { + const u32 operation = (instruction >> 9) & 7; + const u32 ro = (instruction >> 6) & 7; + const u32 rb = (instruction >> 3) & 7; + rd = instruction & 7; + load = operation >= 3; + sign_extend = operation == 3 || operation == 7; + if (operation == 0 || operation == 4) + access_size = 4; + else if (operation == 1 || operation == 5 || operation == 7) + access_size = 2; + else + access_size = 1; + MOV(32, R(EAX), MRegister(rb)); + ADD(32, R(EAX), MRegister(ro)); + } + else if ((instruction & 0xe000) == 0x6000) // STR/LDR[B] Rd, [Rb, #imm] + { + const bool byte = (instruction & 0x1000) != 0; + load = (instruction & 0x0800) != 0; + access_size = byte ? 1 : 4; + rd = instruction & 7; + const u32 rb = (instruction >> 3) & 7; + u32 offset = (instruction >> 6) & 0x1f; + if (!byte) + offset <<= 2; + MOV(32, R(EAX), MRegister(rb)); + if (offset != 0) + ADD(32, R(EAX), Imm32(offset)); + } + else if ((instruction & 0xf000) == 0x8000) // STRH/LDRH Rd, [Rb, #imm] + { + load = (instruction & 0x0800) != 0; + access_size = 2; + rd = instruction & 7; + const u32 rb = (instruction >> 3) & 7; + const u32 offset = ((instruction >> 6) & 0x1f) << 1; + MOV(32, R(EAX), MRegister(rb)); + if (offset != 0) + ADD(32, R(EAX), Imm32(offset)); + } + else if ((instruction & 0xf000) == 0x9000) // STR/LDR Rd, [SP, #imm] + { + load = (instruction & 0x0800) != 0; + access_size = 4; + rd = (instruction >> 8) & 7; + MOV(32, R(EAX), MRegister(13)); + const u32 offset = (instruction & 0xff) << 2; + if (offset != 0) + ADD(32, R(EAX), Imm32(offset)); + } + else + { + return false; + } + + std::vector slow_paths; + EmitFastmemAddress(&slow_paths, access_size, 0, + load ? SRAMFastmemAccess::Read : SRAMFastmemAccess::Write); + + if (load) + { + if (access_size == 1) + { + if (sign_extend) + MOVSX(32, 8, EAX, MatR(R11)); + else + MOVZX(32, 8, EAX, MatR(R11)); + } + else if (access_size == 2) + { + MOVZX(32, 16, EAX, MatR(R11)); + if (m_core.m_big_endian) + ROL(16, R(EAX), Imm8(8)); + if (sign_extend) + MOVSX(32, 16, EAX, R(EAX)); + } + else + { + MOV(32, R(EAX), MatR(R11)); + if (m_core.m_big_endian) + BSWAP(32, EAX); + } + MOV(32, MRegister(rd), R(EAX)); + } + else + { + MOV(32, R(EDX), MRegister(rd)); + if (access_size == 2 && m_core.m_big_endian) + ROL(16, R(EDX), Imm8(8)); + else if (access_size == 4 && m_core.m_big_endian) + BSWAP(32, EDX); + MOV(access_size * 8, MatR(R11), R(EDX)); + } + + const FixupBranch direct_done = J(); + EmitThumbMemorySlowPath(slow_paths, instruction, address, direct_done); + return true; +} + +void ARMJitX64::EmitFastmemAddress(std::vector* slow_paths, u32 access_size, + u32 range_size, SRAMFastmemAccess sram_access, + bool arm_unaligned_word) +{ + if (range_size == 0) + range_size = access_size; + constexpr u32 CP15_CONTROL_MMU = 1U << 0; + if ((m_core.m_cp15.control & CP15_CONTROL_MMU) != 0) + { + CMP(32, R(EAX), Imm32(0x02000000)); + FixupBranch no_fcse = J_CC(CC_AE); + MOV(32, R(EDX), MDisp(JIT_CORE, m_process_id_offset)); + AND(32, R(EDX), Imm32(0xfe000000)); + OR(32, R(EAX), R(EDX)); + SetJumpTarget(no_fcse); + + MOV(32, R(EDX), R(EAX)); + SHR(32, R(EDX), Imm8(10)); + MOV(32, R(ECX), R(EDX)); + AND(32, R(ECX), Imm32(static_cast(ARMCore::TLB_ENTRY_COUNT - 1))); + IMUL(32, ECX, R(ECX), Imm32(sizeof(ARMCore::TLBEntry))); + MOV(64, R(R11), ImmPtr(m_core.m_tlb.data())); + ADD(64, R(R11), R(RCX)); + MOV(32, R(R8), MDisp(JIT_CORE, m_tlb_generation_offset)); + CMP(32, MDisp(R11, static_cast(offsetof(ARMCore::TLBEntry, generation))), R(R8)); + const FixupBranch stale_tlb_entry = J_CC(CC_NE, Jump::Near); + CMP(32, MDisp(R11, static_cast(offsetof(ARMCore::TLBEntry, virtual_page))), R(EDX)); + const FixupBranch different_tlb_page = J_CC(CC_NE, Jump::Near); + AND(32, R(EAX), Imm32(0x3ff)); + OR(32, R(EAX), MDisp(R11, static_cast(offsetof(ARMCore::TLBEntry, physical_page)))); + const FixupBranch translation_ready = J(Jump::Near); + + // IOS invalidates its unified TLB on every process switch. Previously the first load/store on + // every page after such a switch executed the whole guest instruction in the interpreter and + // terminated its native block. Refill only the software translation cache here and resume the + // same generated fastmem access. RBX is available as a callee-saved scratch register because + // every memory emitter flushes the r0/r1 register cache before reaching this routine. + SetJumpTarget(stale_tlb_entry); + SetJumpTarget(different_tlb_page); + MOV(32, R(RBX), R(R10)); + MOV(32, R(ABI_PARAM2), R(EAX)); + MOV(64, R(ABI_PARAM1), ImmPtr(this)); + ABI_CallFunction(TranslateAddress); + MOV(32, R(R10), R(RBX)); + SetJumpTarget(translation_ready); + } + + // Preserve the exact translated bus address before alignment checks and SRAM aperture mapping + // reuse EAX. Slow MMIO/protected-memory helpers must see the original physical address, not a + // masked SRAM offset left behind by a rejected fast path. + MOV(32, R(R9), R(EAX)); + + if (arm_unaligned_word) + { + // ARMv5 single-data-transfer words always perform an aligned bus transaction. LDR rotates + // the fetched word by this saved byte offset; STR simply ignores the bottom two bits. + MOV(32, R(ECX), R(EAX)); + AND(32, R(ECX), Imm32(3)); + AND(32, R(EAX), Imm32(~3U)); + } + else if (access_size > 1) + { + TEST(32, R(EAX), Imm32(access_size - 1)); + slow_paths->push_back(J_CC(CC_NZ, Jump::Near)); + } + + // Return the resolved host pointer in R11. Wii RAM uses Dolphin's 4 GiB fastmem view. + CMP(32, R(EAX), Imm32(Memory::MEM1_SIZE_RETAIL - range_size)); + FixupBranch in_mem1 = J_CC(CC_BE, Jump::Near); + CMP(32, R(EAX), Imm32(0x10000000)); + FixupBranch below_mem2 = J_CC(CC_B, Jump::Near); + CMP(32, R(EAX), Imm32(0x10000000 + Memory::MEM2_SIZE_RETAIL - range_size)); + FixupBranch in_mem2 = J_CC(CC_BE, Jump::Near); + + // Addresses above MEM2 are only directly accessible when they hit the high Starlet SRAM + // mirror. Everything else remains on the exact bus/MMIO path. + if (sram_access == SRAMFastmemAccess::None || !m_sram_base || !m_boot0_mapped || + !m_sram_split_mode) + { + slow_paths->push_back(J(Jump::Near)); + SetJumpTarget(below_mem2); + slow_paths->push_back(J(Jump::Near)); + } + else + { + MOV(32, R(R9), R(EAX)); + CMP(32, R(EAX), Imm32(0xfff00000)); + slow_paths->push_back(J_CC(CC_B, Jump::Near)); + + // boot0 overrides one half of the upper 128 KiB aperture. Consult the live Hollywood state; + // reads from that overlay must keep using the bus so ROM write protection is preserved. + MOV(64, R(R8), ImmPtr(m_boot0_mapped)); + CMP(8, MatR(R8), Imm8(0)); + FixupBranch boot0_disabled = J_CC(CC_Z, Jump::Near); + CMP(32, R(EAX), Imm32(0xfffe0000)); + FixupBranch below_boot0 = J_CC(CC_B, Jump::Near); + MOV(32, R(EDX), R(EAX)); + AND(32, R(EDX), Imm32(0x1ffff)); + MOV(64, R(R8), ImmPtr(m_sram_split_mode)); + CMP(8, MatR(R8), Imm8(0)); + FixupBranch nonsplit_boot0 = J_CC(CC_Z, Jump::Near); + CMP(32, R(EDX), Imm32(0x10000)); + slow_paths->push_back(J_CC(CC_B, Jump::Near)); + FixupBranch boot0_checked = J(Jump::Near); + SetJumpTarget(nonsplit_boot0); + CMP(32, R(EDX), Imm32(0x10000)); + slow_paths->push_back(J_CC(CC_AE, Jump::Near)); + if (range_size > 1) + { + // A multi-register store beginning at the end of SRAM A must not write through the + // following boot0 half of the aperture. + CMP(32, R(EDX), Imm32(0x10000 - range_size)); + slow_paths->push_back(J_CC(CC_A, Jump::Near)); + } + SetJumpTarget(boot0_checked); + SetJumpTarget(boot0_disabled); + SetJumpTarget(below_boot0); + FixupBranch map_sram = J(Jump::Near); + + // The low 0x0d400000 aperture aliases the same SRAM and can never select boot0. + SetJumpTarget(below_mem2); + CMP(32, R(EAX), Imm32(0x0d400000)); + slow_paths->push_back(J_CC(CC_B, Jump::Near)); + CMP(32, R(EAX), Imm32(0x0d420000 - range_size)); + slow_paths->push_back(J_CC(CC_A, Jump::Near)); + MOV(32, R(R9), R(EAX)); + SetJumpTarget(map_sram); + + MOV(32, R(EAX), R(R9)); + AND(32, R(EAX), Imm32(0x1ffff)); + + // Full-chain profiling gives a narrow safe aperture for reads. Direct writes to the measured + // IOS pages 0x00/0x19 passed isolated bounds tests, but reproducibly terminated BootMii during + // its hardware setup. Keep every SRAM write on the exact bus path until write coherency with + // the live boot chain is understood. Boot0, split-window holes and register-list transfers also + // remain on the bus. + MOV(32, R(EDX), R(EAX)); + SHR(32, R(EDX), Imm8(12)); + if (sram_access == SRAMFastmemAccess::Write) + { + slow_paths->push_back(J(Jump::Near)); + } + else + { + CMP(32, R(EDX), Imm32(0x00)); + const FixupBranch profiled_read_page_00 = J_CC(CC_E, Jump::Near); + CMP(32, R(EDX), Imm32(0x12)); + const FixupBranch profiled_read_page_12 = J_CC(CC_E, Jump::Near); + CMP(32, R(EDX), Imm32(0x14)); + const FixupBranch profiled_read_page_14 = J_CC(CC_E, Jump::Near); + CMP(32, R(EDX), Imm32(0x19)); + const FixupBranch profiled_read_page_19 = J_CC(CC_E, Jump::Near); + CMP(32, R(EDX), Imm32(0x1e)); + slow_paths->push_back(J_CC(CC_NE, Jump::Near)); + SetJumpTarget(profiled_read_page_00); + SetJumpTarget(profiled_read_page_12); + SetJumpTarget(profiled_read_page_14); + SetJumpTarget(profiled_read_page_19); + } + + MOV(64, R(R8), ImmPtr(m_sram_split_mode)); + CMP(8, MatR(R8), Imm8(0)); + FixupBranch split = J_CC(CC_NE, Jump::Near); + + // Unsplit: A then B occupy offsets [0, 0x18000). + CMP(32, R(EAX), Imm32(0x18000 - range_size)); + slow_paths->push_back(J_CC(CC_A, Jump::Near)); + FixupBranch sram_mapped = J(Jump::Near); + + // Split: B occupies [0, 0x8000), A occupies [0x10000, 0x20000). + SetJumpTarget(split); + CMP(32, R(EAX), Imm32(0x8000 - range_size)); + FixupBranch split_b = J_CC(CC_BE, Jump::Near); + CMP(32, R(EAX), Imm32(0x10000)); + slow_paths->push_back(J_CC(CC_B, Jump::Near)); + CMP(32, R(EAX), Imm32(0x20000 - range_size)); + slow_paths->push_back(J_CC(CC_A, Jump::Near)); + SUB(32, R(EAX), Imm32(0x10000)); + FixupBranch split_done = J(Jump::Near); + SetJumpTarget(split_b); + ADD(32, R(EAX), Imm32(0x10000)); + SetJumpTarget(split_done); + SetJumpTarget(sram_mapped); + MOV(64, R(R11), ImmPtr(m_sram_base)); + ADD(64, R(R11), R(RAX)); + FixupBranch pointer_done = J(Jump::Near); + + SetJumpTarget(in_mem1); + SetJumpTarget(in_mem2); + MOV(64, R(R11), ImmPtr(m_fastmem_base)); + ADD(64, R(R11), R(RAX)); + SetJumpTarget(pointer_done); + return; + } + + SetJumpTarget(in_mem1); + SetJumpTarget(in_mem2); + MOV(64, R(R11), ImmPtr(m_fastmem_base)); + ADD(64, R(R11), R(RAX)); +} + +void ARMJitX64::EmitThumbMemorySlowPath(const std::vector& slow_paths, u16 instruction, + u32 address, FixupBranch direct_done) +{ + for (const FixupBranch& slow_path : slow_paths) + SetJumpTarget(slow_path); + EmitFallbackThumb(instruction, address); + EmitBlockExit(m_compile_instruction_count, m_compile_native_instruction_count); + SetJumpTarget(direct_done); + LoadRegisterCache(); +} + +void ARMJitX64::EmitThumbShiftImmediate(u16 instruction) +{ + const u32 type = (instruction >> 11) & 3; + const u32 amount = (instruction >> 6) & 0x1f; + const u32 rs = (instruction >> 3) & 7; + const u32 rd = instruction & 7; + MOV(32, R(EAX), MRegister(rs)); + + if (amount == 0 && type == 0) // LSL #0 preserves carry. + { + MOV(32, MRegister(rd), R(EAX)); + EmitLogicalFlags(EAX); + return; + } + + if (amount == 0) + { + MOV(32, R(R10), R(EAX)); + SHR(32, R(R10), Imm8(31)); + if (type == 1) // LSR #0 means LSR #32. + XOR(32, R(EAX), R(EAX)); + else // ASR #0 means ASR #32. + SAR(32, R(EAX), Imm8(31)); + } + else + { + if (type == 0) + SHL(32, R(EAX), Imm8(amount)); + else if (type == 1) + SHR(32, R(EAX), Imm8(amount)); + else + SAR(32, R(EAX), Imm8(amount)); + SETcc(CC_C, R(R10)); + MOVZX(32, 8, R10, R(R10)); + } + MOV(32, MRegister(rd), R(EAX)); + EmitLogicalFlagsWithCarry(EAX, R10); +} + +void ARMJitX64::EmitARMMemorySlowPath(const std::vector& slow_paths, u32 instruction, + u32 address, FixupBranch direct_done) +{ + for (const FixupBranch& slow_path : slow_paths) + SetJumpTarget(slow_path); + + // A normal ARM single-data transfer that misses RAM/SRAM fastmem is still a fully decoded + // instruction. Execute only its physical bus transaction here and stay inside the translated + // block. The previous path re-entered the generic interpreter and native dispatcher for every + // SDIO register word; IOS58's WLAN module performs millions of those transfers while bringing + // up D11, making exact LLE orders of magnitude slower than the emulated hardware. + // + // This is not an HLE shortcut: ReadMemorySlow/WriteMemorySlow call the same ARMBus methods as the + // interpreter, preserve ARM926 unaligned-word rotation and current CP15 endian state, and device + // clocks still advance at the same RunCycles slice boundary. + if ((instruction & 0x0c000000) == 0x04000000) + { + const bool preindex = (instruction & (1U << 24)) != 0; + const bool byte = (instruction & (1U << 22)) != 0; + const bool writeback = !preindex || (instruction & (1U << 21)) != 0; + const bool load = (instruction & (1U << 20)) != 0; + const u32 rn = (instruction >> 16) & 0xf; + const u32 rd = (instruction >> 12) & 0xf; + const u32 access_size = byte ? 1 : 4; + + // Helper calls may clobber R10 and the byte-offset scratch register. RBX/RBP are available + // here because EmitARMMemory flushed the r0/r1 cache before address generation. + if (writeback) + MOV(32, R(RBX), R(R10)); + if (!byte) + MOV(32, R(RBP), R(ECX)); + + if (load) + { + // Windows x64 uses R9 for ABI_PARAM4, which is also our preserved physical-address + // scratch. Copy the address argument before filling the fourth parameter. + MOV(32, R(ABI_PARAM2), R(R9)); + if (byte) + MOV(32, R(ABI_PARAM4), Imm32(0)); + else + MOV(32, R(ABI_PARAM4), R(RBP)); + MOV(32, R(ABI_PARAM3), Imm32(access_size)); + MOV(64, R(ABI_PARAM1), ImmPtr(this)); + ABI_CallFunction(ReadMemorySlow); + MOV(32, MStoredRegister(rd), R(EAX)); + } + else + { + MOV(32, R(ABI_PARAM2), R(R9)); + MOV(32, R(ABI_PARAM4), MStoredRegister(rd)); + MOV(32, R(ABI_PARAM3), Imm32(access_size)); + MOV(64, R(ABI_PARAM1), ImmPtr(this)); + ABI_CallFunction(WriteMemorySlow); + } + if (writeback) + MOV(32, MStoredRegister(rn), R(RBX)); + + SetJumpTarget(direct_done); + LoadRegisterCache(); + return; + } + + // MMIO, TLB misses and protected SRAM overlays are architectural boundaries. The exact + // interpreter path updates all CPU/device state, then the dispatcher re-samples IRQ/FIQ, yield + // and translation state before another native block can run. + EmitFallbackARM(instruction, address); + EmitBlockExit(m_compile_instruction_count, m_compile_native_instruction_count); + SetJumpTarget(direct_done); + LoadRegisterCache(); +} + +void ARMJitX64::EmitThumbAddSub(u16 instruction) +{ + const bool immediate = (instruction & 0x0400) != 0; + const bool subtract = (instruction & 0x0200) != 0; + const u32 operand = (instruction >> 6) & 7; + const u32 rs = (instruction >> 3) & 7; + const u32 rd = instruction & 7; + + MOV(32, R(EAX), MRegister(rs)); + if (subtract) + { + if (immediate) + SUB(32, R(EAX), Imm32(operand)); + else + SUB(32, R(EAX), MRegister(operand)); + } + else + { + if (immediate) + ADD(32, R(EAX), Imm32(operand)); + else + ADD(32, R(EAX), MRegister(operand)); + } + MOV(32, MRegister(rd), R(EAX)); + EmitArithmeticFlags(subtract); +} + +void ARMJitX64::EmitThumbImmediate(u16 instruction) +{ + const u32 opcode = (instruction >> 11) & 3; + const u32 rd = (instruction >> 8) & 7; + const u32 immediate = instruction & 0xff; + if (opcode == 0) // MOV + { + MOV(32, R(EAX), Imm32(immediate)); + MOV(32, MRegister(rd), R(EAX)); + EmitLogicalFlags(EAX); + return; + } + + MOV(32, R(EAX), MRegister(rd)); + if (opcode == 2) // ADD + ADD(32, R(EAX), Imm32(immediate)); + else + SUB(32, R(EAX), Imm32(immediate)); + + if (opcode != 1) // CMP does not write a result. + MOV(32, MRegister(rd), R(EAX)); + EmitArithmeticFlags(opcode != 2); +} + +void ARMJitX64::EmitThumbALU(u16 instruction) +{ + const u32 opcode = (instruction >> 6) & 0xf; + const u32 rs = (instruction >> 3) & 7; + const u32 rd = instruction & 7; + + MOV(32, R(EAX), MRegister(rd)); + switch (opcode) + { + case 0x0: // AND + AND(32, R(EAX), MRegister(rs)); + MOV(32, MRegister(rd), R(EAX)); + EmitLogicalFlags(EAX); + return; + case 0x1: // EOR + XOR(32, R(EAX), MRegister(rs)); + MOV(32, MRegister(rd), R(EAX)); + EmitLogicalFlags(EAX); + return; + case 0x3: // LSR (register) + { + // ARM uses the low byte of the register as the shift amount. Unlike x86, counts at and above + // 32 do not wrap, and a zero count preserves C. Spell out all four architectural cases. + MOV(32, R(ECX), MRegister(rs)); + AND(32, R(ECX), Imm32(0xff)); + TEST(32, R(ECX), R(ECX)); + const FixupBranch zero = J_CC(CC_Z, Jump::Near); + CMP(32, R(ECX), Imm32(32)); + const FixupBranch below_32 = J_CC(CC_B, Jump::Near); + const FixupBranch equal_32 = J_CC(CC_E, Jump::Near); + + XOR(32, R(EAX), R(EAX)); + XOR(32, R(EDX), R(EDX)); + const FixupBranch carry_ready_above_32 = J(Jump::Near); + + SetJumpTarget(equal_32); + MOV(32, R(EDX), R(EAX)); + SHR(32, R(EDX), Imm8(31)); + XOR(32, R(EAX), R(EAX)); + const FixupBranch carry_ready_equal_32 = J(Jump::Near); + + SetJumpTarget(below_32); + MOV(32, R(EDX), R(EAX)); + SUB(32, R(ECX), Imm8(1)); + SHR(32, R(EDX), R(ECX)); + AND(32, R(EDX), Imm32(1)); + ADD(32, R(ECX), Imm8(1)); + SHR(32, R(EAX), R(ECX)); + + SetJumpTarget(carry_ready_above_32); + SetJumpTarget(carry_ready_equal_32); + MOV(32, MRegister(rd), R(EAX)); + EmitLogicalFlagsWithCarry(EAX, EDX); + const FixupBranch done = J(Jump::Near); + + SetJumpTarget(zero); + MOV(32, MRegister(rd), R(EAX)); + EmitLogicalFlags(EAX); + SetJumpTarget(done); + return; + } + case 0x8: // TST + TEST(32, R(EAX), MRegister(rs)); + SETcc(CC_S, R(R8)); + SETcc(CC_Z, R(R9)); + MOVZX(32, 8, R8, R(R8)); + MOVZX(32, 8, R9, R(R9)); + SHL(32, R(R8), Imm8(31)); + SHL(32, R(R9), Imm8(30)); + MOV(32, R(ECX), MCPSR()); + AND(32, R(ECX), Imm32(~(ARMCore::CPSR_N | ARMCore::CPSR_Z))); + OR(32, R(ECX), R(R8)); + OR(32, R(ECX), R(R9)); + MOV(32, MCPSR(), R(ECX)); + return; + case 0x9: // NEG + MOV(32, R(EAX), MRegister(rs)); + NEG(32, R(EAX)); + MOV(32, MRegister(rd), R(EAX)); + EmitArithmeticFlags(true); + return; + case 0xa: // CMP + CMP(32, R(EAX), MRegister(rs)); + EmitArithmeticFlags(true); + return; + case 0xb: // CMN + ADD(32, R(EAX), MRegister(rs)); + EmitArithmeticFlags(false); + return; + case 0xc: // ORR + OR(32, R(EAX), MRegister(rs)); + MOV(32, MRegister(rd), R(EAX)); + EmitLogicalFlags(EAX); + return; + case 0xd: // MUL + IMUL(32, EAX, MRegister(rs)); + MOV(32, MRegister(rd), R(EAX)); + EmitLogicalFlags(EAX); + return; + case 0xe: // BIC + MOV(32, R(EDX), MRegister(rs)); + NOT(32, R(EDX)); + AND(32, R(EAX), R(EDX)); + MOV(32, MRegister(rd), R(EAX)); + EmitLogicalFlags(EAX); + return; + case 0xf: // MVN + MOV(32, R(EAX), MRegister(rs)); + NOT(32, R(EAX)); + MOV(32, MRegister(rd), R(EAX)); + EmitLogicalFlags(EAX); + return; + default: + ASSERT(false); + } +} + +void ARMJitX64::EmitLogicalFlags(X64Reg result) +{ + TEST(32, R(result), R(result)); + SETcc(CC_S, R(R8)); + SETcc(CC_Z, R(R9)); + MOVZX(32, 8, R8, R(R8)); + MOVZX(32, 8, R9, R(R9)); + SHL(32, R(R8), Imm8(31)); + SHL(32, R(R9), Imm8(30)); + MOV(32, R(ECX), MCPSR()); + AND(32, R(ECX), Imm32(~(ARMCore::CPSR_N | ARMCore::CPSR_Z))); + OR(32, R(ECX), R(R8)); + OR(32, R(ECX), R(R9)); + MOV(32, MCPSR(), R(ECX)); +} + +void ARMJitX64::EmitLogicalFlagsWithCarry(X64Reg result, X64Reg carry) +{ + TEST(32, R(result), R(result)); + SETcc(CC_S, R(R8)); + SETcc(CC_Z, R(R9)); + MOVZX(32, 8, R8, R(R8)); + MOVZX(32, 8, R9, R(R9)); + SHL(32, R(R8), Imm8(31)); + SHL(32, R(R9), Imm8(30)); + SHL(32, R(carry), Imm8(29)); + MOV(32, R(ECX), MCPSR()); + AND(32, R(ECX), Imm32(~(ARMCore::CPSR_N | ARMCore::CPSR_Z | ARMCore::CPSR_C))); + OR(32, R(ECX), R(R8)); + OR(32, R(ECX), R(R9)); + OR(32, R(ECX), R(carry)); + MOV(32, MCPSR(), R(ECX)); +} + +void ARMJitX64::EmitArithmeticFlags(bool subtraction) +{ + SETcc(CC_S, R(R8)); + SETcc(CC_Z, R(R9)); + SETcc(subtraction ? CC_NC : CC_C, R(R10)); + SETcc(CC_O, R(R11)); + MOVZX(32, 8, R8, R(R8)); + MOVZX(32, 8, R9, R(R9)); + MOVZX(32, 8, R10, R(R10)); + MOVZX(32, 8, R11, R(R11)); + SHL(32, R(R8), Imm8(31)); + SHL(32, R(R9), Imm8(30)); + SHL(32, R(R10), Imm8(29)); + SHL(32, R(R11), Imm8(28)); + MOV(32, R(ECX), MCPSR()); + AND(32, R(ECX), Imm32(~(ARMCore::CPSR_N | ARMCore::CPSR_Z | ARMCore::CPSR_C | ARMCore::CPSR_V))); + OR(32, R(ECX), R(R8)); + OR(32, R(ECX), R(R9)); + OR(32, R(ECX), R(R10)); + OR(32, R(ECX), R(R11)); + MOV(32, MCPSR(), R(ECX)); +} + +void ARMJitX64::EmitFallbackThumb(u16 instruction, u32 address) +{ + FlushRegisterCache(); + MOV(64, R(ABI_PARAM1), ImmPtr(this)); + MOV(32, R(ABI_PARAM2), Imm32(instruction)); + MOV(32, R(ABI_PARAM3), Imm32(address)); + ABI_CallFunction(FallbackThumb); +} + +void ARMJitX64::EmitFallbackARM(u32 instruction, u32 address) +{ + FlushRegisterCache(); + MOV(64, R(ABI_PARAM1), ImmPtr(this)); + MOV(32, R(ABI_PARAM2), Imm32(instruction)); + MOV(32, R(ABI_PARAM3), Imm32(address)); + ABI_CallFunction(FallbackARM); +} + +void ARMJitX64::EmitBlockExit(u32 instruction_count, u32 native_instruction_count, + bool dispatcher_exit) +{ + FlushRegisterCache(); + SUB(32, R(JIT_DOWNCOUNT), Imm32(instruction_count)); + ADD(32, R(JIT_EXECUTED_COUNT), Imm32(instruction_count)); + ADD(32, R(JIT_NATIVE_COUNT), Imm32(native_instruction_count)); + // Diagnostic only: one generated increment per completed native block lets the live profiler + // distinguish short-block dispatch overhead from slow bus helpers. The dispatcher overwrites + // the host flags before observing them, so this does not affect guest execution. + MOV(64, R(RAX), ImmPtr(&m_block_execution_count)); + ADD(64, MatR(RAX), Imm8(1)); + JMP(dispatcher_exit ? m_dispatcher_exit : m_dispatcher); +} + +OpArg ARMJitX64::MRegister(u32 index) const +{ + if (m_register_cache_active) + { + if (index == 0) + return R(RBX); + if (index == 1) + return R(RBP); + } + return MStoredRegister(index); +} + +OpArg ARMJitX64::MStoredRegister(u32 index) const +{ + return MDisp(JIT_CORE, m_registers_offset + static_cast(index * sizeof(u32))); +} + +void ARMJitX64::LoadRegisterCache() +{ + if (m_register_cache_active) + return; + MOV(32, R(RBX), MStoredRegister(0)); + MOV(32, R(RBP), MStoredRegister(1)); + m_register_cache_active = true; +} + +void ARMJitX64::FlushRegisterCache() +{ + if (!m_register_cache_active) + return; + MOV(32, MStoredRegister(0), R(RBX)); + MOV(32, MStoredRegister(1), R(RBP)); + m_register_cache_active = false; +} + +OpArg ARMJitX64::MCPSR() const +{ + return MDisp(R15, m_cpsr_offset); +} + +OpArg ARMJitX64::MInstructionAddress() const +{ + return MDisp(R15, m_instruction_address_offset); +} + +OpArg ARMJitX64::MPCWritten() const +{ + return MDisp(R15, m_pc_written_offset); +} + +OpArg ARMJitX64::MWaitingForInterrupt() const +{ + return MDisp(R15, m_waiting_for_interrupt_offset); +} + +OpArg ARMJitX64::MWaitingForMemoryPoll() const +{ + return MDisp(R15, m_waiting_for_memory_poll_offset); +} + +OpArg ARMJitX64::MYieldRequested() const +{ + return MDisp(R15, m_yield_requested_offset); +} + +OpArg ARMJitX64::MExecutedInstructions() const +{ + return MDisp(R15, m_executed_instructions_offset); +} + +void ARMJitX64::FallbackThumb(ARMJitX64* jit, u16 instruction, u32 address) +{ + ARMCore* const core = &jit->m_core; + core->RecordJitFallback(address, true); + core->m_instruction_address = address; + core->m_pc_written = false; + core->ExecuteThumb(instruction); + core->TryEnterThumbMemoryPoll(instruction); + if (!core->m_pc_written) + core->m_registers[15] = address + ((core->m_cpsr & ARMCore::CPSR_T) ? 2 : 4); +} + +void ARMJitX64::FallbackARM(ARMJitX64* jit, u32 instruction, u32 address) +{ + ARMCore* const core = &jit->m_core; + core->RecordJitFallback(address, false); + core->m_instruction_address = address; + core->m_pc_written = false; + if ((instruction & 0xfe000000) == 0xfa000000) + { + const u32 h = (instruction >> 24) & 1; + const s32 offset = + ARMCore::SignExtend((instruction & 0x00ffffff) << 2, 26) + static_cast(h << 1); + core->m_registers[14] = address + 4; + core->m_cpsr |= ARMCore::CPSR_T; + core->WritePC(address + 8 + static_cast(offset)); + } + else if (core->ConditionPassed(instruction >> 28)) + { + core->ExecuteARM(instruction); + } + core->TryEnterARMSliceStablePoll(instruction); + if (!core->m_pc_written) + core->m_registers[15] = address + ((core->m_cpsr & ARMCore::CPSR_T) ? 2 : 4); +} + +u32 ARMJitX64::ReadSPSR(ARMJitX64* jit) +{ + ARMCore& core = jit->m_core; + if (const u32* const spsr = core.GetSPSR(core.GetMode())) + return *spsr; + return core.m_cpsr; +} + +void ARMJitX64::WritePSR(ARMJitX64* jit, u32 spsr, u32 field_mask, u32 value) +{ + jit->m_core.WritePSR(spsr != 0, field_mask, value); +} + +void ARMJitX64::ExecuteUserBankBlockTransfer(ARMJitX64* jit, u32 instruction, u32 address) +{ + ARMCore& core = jit->m_core; + core.m_instruction_address = address; + core.m_pc_written = false; + core.ExecuteBlockDataTransfer(instruction); +} + +void ARMJitX64::ExecuteThumbPushPop(ARMJitX64* jit, u16 instruction, u32 address) +{ + ARMCore& core = jit->m_core; + core.m_instruction_address = address; + core.m_pc_written = false; + core.ExecuteThumb(instruction); +} + +void ARMJitX64::ExceptionReturn(ARMJitX64* jit, u32 target) +{ + ARMCore& core = jit->m_core; + core.RestoreCPSRFromSPSR(); + core.WritePC(target); +} + +u32 ARMJitX64::ReadMemorySlow(ARMJitX64* jit, u32 physical_address, u32 access_size, + u32 byte_offset) +{ + ++jit->m_slow_read_count; + switch (ClassifySlowMemoryAddress(physical_address)) + { + case SlowMemoryRegion::RAM: + ++jit->m_slow_ram_read_count; + break; + case SlowMemoryRegion::SRAM: + ++jit->m_slow_sram_read_count; + ++(physical_address < 0xfff00000 ? jit->m_slow_sram_low_access_count : + jit->m_slow_sram_high_access_count); + ++jit->m_slow_sram_page_read_access_count[(physical_address & 0x1ffff) >> 12]; + break; + case SlowMemoryRegion::MMIO: + ++jit->m_slow_mmio_read_count; + break; + case SlowMemoryRegion::Other: + ++jit->m_slow_other_read_count; + break; + } + ARMCore& core = jit->m_core; + if (access_size == 1) + return core.m_bus.Read8(physical_address); + + u32 value = core.m_bus.Read32(physical_address & ~3U); + if (!core.m_big_endian) + value = std::byteswap(value); + return std::rotr(value, static_cast((byte_offset & 3) * 8)); +} + +void ARMJitX64::WriteMemorySlow(ARMJitX64* jit, u32 physical_address, u32 access_size, u32 value) +{ + ++jit->m_slow_write_count; + switch (ClassifySlowMemoryAddress(physical_address)) + { + case SlowMemoryRegion::RAM: + ++jit->m_slow_ram_write_count; + break; + case SlowMemoryRegion::SRAM: + ++jit->m_slow_sram_write_count; + ++(physical_address < 0xfff00000 ? jit->m_slow_sram_low_access_count : + jit->m_slow_sram_high_access_count); + ++jit->m_slow_sram_page_write_access_count[(physical_address & 0x1ffff) >> 12]; + break; + case SlowMemoryRegion::MMIO: + ++jit->m_slow_mmio_write_count; + break; + case SlowMemoryRegion::Other: + ++jit->m_slow_other_write_count; + break; + } + ARMCore& core = jit->m_core; + if (access_size == 1) + { + core.m_bus.Write8(physical_address, static_cast(value)); + return; + } + + if (!core.m_big_endian) + value = std::byteswap(value); + core.m_bus.Write32(physical_address & ~3U, value); +} + +u32 ARMJitX64::TranslateAddress(ARMJitX64* jit, u32 address) +{ + ++jit->m_address_translation_count; + return jit->m_core.TranslateVirtualAddress(address); +} + +void ARMJitX64::EnterUndefinedARM(ARMJitX64* jit, u32 instruction, u32 address) +{ + ARMCore& core = jit->m_core; + core.m_instruction_address = address; + core.m_pc_written = false; + core.UndefinedInstruction(instruction); +} + +void ARMJitX64::EnterSVCARM(ARMJitX64* jit, u32 address) +{ + ARMCore& core = jit->m_core; + core.m_instruction_address = address; + core.m_pc_written = false; + core.EnterException(ARMCore::Mode::Supervisor, STARLET_EXCEPTION_VECTOR_BASE + 0x08, + address + 4); +} + +void ARMJitX64::EnterSVCThumb(ARMJitX64* jit, u32 address) +{ + ARMCore& core = jit->m_core; + core.m_instruction_address = address; + core.m_pc_written = false; + core.EnterException(ARMCore::Mode::Supervisor, STARLET_EXCEPTION_VECTOR_BASE + 0x08, + address + 2); +} + +} // namespace IOS::LLE diff --git a/Source/Core/Core/IOS/Starlet/ARMJitX64.h b/Source/Core/Core/IOS/Starlet/ARMJitX64.h new file mode 100644 index 0000000000..edd1dfb7a7 --- /dev/null +++ b/Source/Core/Core/IOS/Starlet/ARMJitX64.h @@ -0,0 +1,212 @@ +// Copyright 2026 Dolphin Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +#pragma once + +#include +#include +#include +#include + +#include "Common/CommonTypes.h" +#include "Common/x64Emitter.h" + +namespace IOS::LLE +{ +class ARMCore; + +// Starlet JIT for x86-64 hosts. Like Dolphin's Broadway JIT, translated blocks run below one +// persistent native dispatcher. Hot block lookup therefore stays in generated code instead of +// crossing C++ and the host ABI at every ARM basic block. Instructions which are not translated +// still use the exact interpreter implementation so MMIO and privileged ordering stay intact. +class ARMJitX64 final : public Gen::X64CodeBlock +{ +public: + explicit ARMJitX64(ARMCore& core); + ~ARMJitX64() override; + + ARMJitX64(const ARMJitX64&) = delete; + ARMJitX64& operator=(const ARMJitX64&) = delete; + + u32 Run(u64 cycle_budget); + void Clear(); + void InvalidateTranslationContext(); + + u64 GetExecutedInstructions() const { return m_executed_instructions; } + u64 GetNativeExecutedInstructions() const { return m_native_executed_instructions; } + size_t GetCompiledBlockCount() const { return m_blocks.size(); } + u64 GetBlockExecutionCount() const { return m_block_execution_count; } + u64 GetAddressTranslationCount() const { return m_address_translation_count; } + u64 GetSlowReadCount() const { return m_slow_read_count; } + u64 GetSlowWriteCount() const { return m_slow_write_count; } + u64 GetSlowRAMAccessCount() const { return m_slow_ram_read_count + m_slow_ram_write_count; } + u64 GetSlowSRAMAccessCount() const + { + return m_slow_sram_read_count + m_slow_sram_write_count; + } + u64 GetSlowMMIOAccessCount() const + { + return m_slow_mmio_read_count + m_slow_mmio_write_count; + } + u64 GetSlowOtherAccessCount() const + { + return m_slow_other_read_count + m_slow_other_write_count; + } + u64 GetSlowSRAMLowAccessCount() const { return m_slow_sram_low_access_count; } + u64 GetSlowSRAMHighAccessCount() const { return m_slow_sram_high_access_count; } + u64 GetSlowSRAMReadAccessCount() const { return m_slow_sram_read_count; } + u64 GetSlowSRAMWriteAccessCount() const { return m_slow_sram_write_count; } + u64 GetSlowSRAMPageAccessCount(size_t page) const + { + return GetSlowSRAMPageReadAccessCount(page) + GetSlowSRAMPageWriteAccessCount(page); + } + u64 GetSlowSRAMPageReadAccessCount(size_t page) const + { + return page < m_slow_sram_page_read_access_count.size() ? + m_slow_sram_page_read_access_count[page] : + 0; + } + u64 GetSlowSRAMPageWriteAccessCount(size_t page) const + { + return page < m_slow_sram_page_write_access_count.size() ? + m_slow_sram_page_write_access_count[page] : + 0; + } + +private: + using RunEntry = u64 (*)(u32); + + enum class SRAMFastmemAccess + { + None, + Read, + Write, + }; + + struct Block + { + const u8* entry = nullptr; + u32 instruction_count = 0; + u32 native_instruction_count = 0; + bool runnable = false; + }; + + struct FastEntry + { + u32 key = 0xffffffff; + u32 padding = 0; + const u8* entry = nullptr; + }; + + void PoisonMemory() override; + void GenerateDispatcher(); + static const u8* Dispatch(ARMJitX64* jit); + Block* GetOrCompileBlock(u32 address); + Block CompileBlock(u32 address, bool thumb); + bool EmitDirectARM(u32 instruction, u32 address, bool* terminal, bool* dispatcher_exit); + bool CanEmitARMDataProcessing(u32 instruction) const; + bool EmitARMMemory(u32 instruction, u32 address); + bool EmitARMHalfwordMemory(u32 instruction, u32 address); + bool EmitARMBlockTransfer(u32 instruction, u32 address, bool* terminal); + void EmitARMDataProcessing(u32 instruction); + bool EmitDirectThumb(u16 instruction, u32 address, bool* terminal); + bool EmitThumbMemory(u16 instruction, u32 address); + void EmitConditionResult(u32 condition); + void EmitExchangeBranch(Gen::OpArg target, bool link, u32 return_address); + void EmitThumbShiftImmediate(u16 instruction); + void EmitFastmemAddress(std::vector* slow_paths, u32 access_size, + u32 range_size = 0, + SRAMFastmemAccess sram_access = SRAMFastmemAccess::None, + bool arm_unaligned_word = false); + void EmitThumbMemorySlowPath(const std::vector& slow_paths, u16 instruction, + u32 address, Gen::FixupBranch direct_done); + void EmitARMMemorySlowPath(const std::vector& slow_paths, u32 instruction, + u32 address, Gen::FixupBranch direct_done); + void EmitThumbAddSub(u16 instruction); + void EmitThumbImmediate(u16 instruction); + void EmitThumbALU(u16 instruction); + void EmitLogicalFlags(Gen::X64Reg result); + void EmitLogicalFlagsWithCarry(Gen::X64Reg result, Gen::X64Reg carry); + void EmitArithmeticFlags(bool subtraction); + void EmitFallbackThumb(u16 instruction, u32 address); + void EmitFallbackARM(u32 instruction, u32 address); + void EmitBlockExit(u32 instruction_count, u32 native_instruction_count, + bool dispatcher_exit = false); + void LoadRegisterCache(); + void FlushRegisterCache(); + + Gen::OpArg MRegister(u32 index) const; + Gen::OpArg MStoredRegister(u32 index) const; + Gen::OpArg MCPSR() const; + Gen::OpArg MInstructionAddress() const; + Gen::OpArg MPCWritten() const; + Gen::OpArg MWaitingForInterrupt() const; + Gen::OpArg MWaitingForMemoryPoll() const; + Gen::OpArg MYieldRequested() const; + Gen::OpArg MExecutedInstructions() const; + + static void FallbackThumb(ARMJitX64* jit, u16 instruction, u32 address); + static void FallbackARM(ARMJitX64* jit, u32 instruction, u32 address); + static u32 ReadSPSR(ARMJitX64* jit); + static void WritePSR(ARMJitX64* jit, u32 spsr, u32 field_mask, u32 value); + static void ExecuteUserBankBlockTransfer(ARMJitX64* jit, u32 instruction, u32 address); + static void ExecuteThumbPushPop(ARMJitX64* jit, u16 instruction, u32 address); + static void ExceptionReturn(ARMJitX64* jit, u32 target); + static u32 ReadMemorySlow(ARMJitX64* jit, u32 physical_address, u32 access_size, + u32 byte_offset); + static void WriteMemorySlow(ARMJitX64* jit, u32 physical_address, u32 access_size, u32 value); + static u32 TranslateAddress(ARMJitX64* jit, u32 address); + static void EnterUndefinedARM(ARMJitX64* jit, u32 instruction, u32 address); + static void EnterSVCARM(ARMJitX64* jit, u32 address); + static void EnterSVCThumb(ARMJitX64* jit, u32 address); + + static constexpr size_t CODE_SIZE = 32 * 1024 * 1024; + static constexpr u32 MAX_BLOCK_INSTRUCTIONS = 32; + static constexpr size_t FAST_ENTRY_COUNT = 1 << 16; + + ARMCore& m_core; + std::unordered_map m_blocks; + std::array m_fast_entries{}; + u8* m_fastmem_base = nullptr; + u8* m_sram_base = nullptr; + const bool* m_boot0_mapped = nullptr; + const bool* m_sram_split_mode = nullptr; + RunEntry m_run_entry = nullptr; + const u8* m_dispatcher = nullptr; + const u8* m_dispatcher_exit = nullptr; + u8* m_block_code_begin = nullptr; + u64 m_executed_instructions = 0; + u64 m_native_executed_instructions = 0; + u64 m_block_execution_count = 0; + u64 m_address_translation_count = 0; + u64 m_slow_read_count = 0; + u64 m_slow_write_count = 0; + u64 m_slow_ram_read_count = 0; + u64 m_slow_ram_write_count = 0; + u64 m_slow_sram_read_count = 0; + u64 m_slow_sram_write_count = 0; + u64 m_slow_mmio_read_count = 0; + u64 m_slow_mmio_write_count = 0; + u64 m_slow_other_read_count = 0; + u64 m_slow_other_write_count = 0; + u64 m_slow_sram_low_access_count = 0; + u64 m_slow_sram_high_access_count = 0; + std::array m_slow_sram_page_read_access_count{}; + std::array m_slow_sram_page_write_access_count{}; + bool m_is_running = false; + bool m_clear_pending = false; + s32 m_registers_offset = 0; + s32 m_cpsr_offset = 0; + s32 m_instruction_address_offset = 0; + s32 m_pc_written_offset = 0; + s32 m_waiting_for_interrupt_offset = 0; + s32 m_waiting_for_memory_poll_offset = 0; + s32 m_yield_requested_offset = 0; + s32 m_executed_instructions_offset = 0; + s32 m_process_id_offset = 0; + s32 m_tlb_generation_offset = 0; + u32 m_compile_instruction_count = 0; + u32 m_compile_native_instruction_count = 0; + bool m_register_cache_active = false; +}; +} // namespace IOS::LLE diff --git a/Source/Core/Core/IOS/Starlet/Starlet.cpp b/Source/Core/Core/IOS/Starlet/Starlet.cpp index 315f679b19..579bb95eb7 100644 --- a/Source/Core/Core/IOS/Starlet/Starlet.cpp +++ b/Source/Core/Core/IOS/Starlet/Starlet.cpp @@ -7,11 +7,17 @@ #include #include "Common/ChunkFile.h" +#include "Common/GekkoDisassembler.h" #include "Common/Logging/Log.h" +#include "Common/Timer.h" +#include "Core/Config/MainSettings.h" #include "Core/CoreTiming.h" +#include "Core/HW/ProcessorInterface.h" #include "Core/HW/WII_IPC.h" #include "Core/IOS/Starlet/ARMCore.h" #include "Core/IOS/Starlet/StarletMemory.h" +#include "Core/PowerPC/MMU.h" +#include "Core/PowerPC/PowerPC.h" #include "Core/System.h" namespace IOS::LLE @@ -27,6 +33,8 @@ bool Starlet::Init(const std::string& dump_directory, std::string* error) // Construct the complete physical bus (including its persistent controller // reset state) before the CPU so the ARM core always observes native-width // memory accesses and the same Hollywood device state. + // Allocate the concrete bus here; its device-state members are part of the + // allocation size and must stay in lockstep with StarletMemory's definition. m_memory = std::make_unique(m_system); if (!m_memory->Init(dump_directory, error)) { @@ -36,11 +44,23 @@ bool Starlet::Init(const std::string& dump_directory, std::string* error) // ARMCore owns generation-tagged software TLB and instruction caches, so // construct it only after the complete physical bus exists. m_core = std::make_unique(*m_memory); +#if defined(_M_X86_64) + m_core->SetJitEnabled(Config::Get(Config::MAIN_WII_STARLET_JIT)); +#else + m_core->SetJitEnabled(false); +#endif m_core->Reset(StarletMemory::BOOT_ROM_BASE); + m_next_jit_diagnostic_instruction = 25'000'000; + m_next_realtime_diagnostic_ms = Common::Timer::NowMs() + 1000; + m_last_realtime_ppc_pc = 0; + m_realtime_ppc_pc_streak = 0; + m_ppc_context_logged = false; + m_jit_context_logged = false; m_run_event = m_system.GetCoreTiming().RegisterEvent("StarletLLE", RunCallback); m_initialized = true; m_system.GetCoreTiming().ScheduleEvent(0, m_run_event); - INFO_LOG_FMT(IOS, "Starlet LLE started at PC {:#010x}", m_core->GetRegister(15)); + INFO_LOG_FMT(IOS, "Starlet LLE started at PC {:#010x} ({})", m_core->GetRegister(15), + m_core->IsJitEnabled() ? "JIT" : "interpreter"); return true; } @@ -48,6 +68,30 @@ void Starlet::Shutdown() { if (m_run_event) m_system.GetCoreTiming().RemoveEvent(m_run_event); + if (m_core && m_core->GetJitExecutedInstructions() != 0) + { + INFO_LOG_FMT(IOS, "Starlet JIT executed {} instructions ({} native, {:.1f}%)", + m_core->GetJitExecutedInstructions(), m_core->GetJitNativeExecutedInstructions(), + 100.0 * m_core->GetJitNativeExecutedInstructions() / + m_core->GetJitExecutedInstructions()); + for (const ARMCore::HotPCSample& sample : m_core->GetHotPCSamples(16)) + { + INFO_LOG_FMT(IOS, "Starlet hot PC {:#010x} {} instruction={:#010x} samples={}", + sample.address, sample.thumb ? "Thumb" : "ARM", sample.instruction, + sample.samples); + } + INFO_LOG_FMT(IOS, + "Starlet JIT compiled {} blocks, used {} fallback instructions and entered " + "{} idle memory polls", + m_core->GetJitCompiledBlockCount(), m_core->GetJitFallbackInstructionCount(), + m_core->GetMemoryPollEntryCount()); + for (const ARMCore::HotPCSample& sample : m_core->GetHotJitFallbackSamples(16)) + { + INFO_LOG_FMT(IOS, "Starlet fallback PC {:#010x} {} instruction={:#010x} samples={}", + sample.address, sample.thumb ? "Thumb" : "ARM", sample.instruction, + sample.samples); + } + } m_initialized = false; m_run_event = nullptr; m_core.reset(); @@ -139,13 +183,190 @@ void Starlet::RunSlice(s64 cycles_late) // Active and sleeping states use different scheduling quanta so busy IOS code // amortizes host callbacks while a sleeping core retains prompt // external-interrupt wakeups. - const bool ipc_handshake_active = (m_system.GetWiiIPC().ReadStarletRegister(0x0c) & 0x0f) != 0; + const u32 ipc_arm_ctrl = m_system.GetWiiIPC().ReadStarletRegister(0x0c); + // In ARMCTRL, X1/X2 are the two inputs produced by Broadway (bits 2 and 1). Only those + // require the tighter interleave while Starlet is consuming a request. Y1/Y2 are Starlet's + // outputs; keeping the 256-cycle quantum until Broadway acknowledges them can leave the whole + // console permanently on the expensive IPC path even though Starlet has already yielded at the + // write which exposed the reply. + const bool ipc_handshake_active = (ipc_arm_ctrl & 0x06) != 0; const u64 arm_cycles = m_core->IsWaitingForExternalEvent() ? ARM_IDLE_SLICE_CYCLES : (ipc_handshake_active ? ARM_IPC_SLICE_CYCLES : ARM_ACTIVE_SLICE_CYCLES); m_core->RunCycles(arm_cycles); + const u64 now_ms = Common::Timer::NowMs(); + if (now_ms >= m_next_realtime_diagnostic_ms) + { + m_next_realtime_diagnostic_ms = now_ms + 1000; + const PowerPC::PowerPCState& ppc = m_system.GetPPCState(); + const PowerPC::TryReadInstResult instruction = m_system.GetMMU().TryReadInstruction(ppc.pc); + const auto& pi = m_system.GetProcessorInterface(); + INFO_LOG_FMT(IOS, + "Broadway live: PC={:#010x} NPC={:#010x} instruction-valid={} " + "instruction={:#010x} MSR={:#010x} " + "exceptions={:#010x} downcount={} r1={:#010x} LR={:#010x} CTR={:#010x} " + "pi-irq={:#010x}/{:#010x} " + "ppc-ctrl={:#04x} ppc-irq={:#010x}/{:#010x} arm-ctrl={:#04x} " + "arm-irq={:#010x}/{:#010x} Starlet-PC={:#010x} wait-int={} wait-mem={}", + ppc.pc, ppc.npc, instruction.valid, instruction.hex, ppc.msr.Hex, ppc.Exceptions, + ppc.downcount, ppc.gpr[1], LR(ppc), CTR(ppc), + pi.GetCause(), pi.GetMask(), + m_system.GetWiiIPC().ReadStarletRegister(0x04), + m_system.GetWiiIPC().ReadStarletRegister(0x30), + m_system.GetWiiIPC().ReadStarletRegister(0x34), ipc_arm_ctrl, + m_system.GetWiiIPC().ReadStarletRegister(0x38), + m_system.GetWiiIPC().ReadStarletRegister(0x3c), m_core->GetRegister(15), + m_core->IsWaitingForInterrupt(), m_core->IsWaitingForMemoryPoll()); + + if (ppc.pc == m_last_realtime_ppc_pc) + ++m_realtime_ppc_pc_streak; + else + { + m_last_realtime_ppc_pc = ppc.pc; + m_realtime_ppc_pc_streak = 1; + m_ppc_context_logged = false; + } + + if (!m_ppc_context_logged && m_realtime_ppc_pc_streak >= 3 && ppc.pc != 0xfffffffc) + { + INFO_LOG_FMT(IOS, + "Broadway repeated-PC context: PC={:#010x} LR={:#010x} CTR={:#010x} " + "CR={:#010x} XER={:#010x} PI={:#010x}/{:#010x}", + ppc.pc, LR(ppc), CTR(ppc), ppc.cr.Get(), ppc.spr[SPR_XER], pi.GetCause(), + pi.GetMask()); + for (u32 base = 0; base < 32; base += 4) + { + INFO_LOG_FMT(IOS, "Broadway GPR r{}={:#010x} r{}={:#010x} r{}={:#010x} r{}={:#010x}", + base, ppc.gpr[base], base + 1, ppc.gpr[base + 1], base + 2, + ppc.gpr[base + 2], base + 3, ppc.gpr[base + 3]); + } + const u32 context_start = (ppc.pc - 0x40) & ~u32{3}; + const u32 context_end = (ppc.pc + 0x80) & ~u32{3}; + for (u32 address = context_start; address <= context_end; address += 4) + { + const PowerPC::TryReadInstResult op = m_system.GetMMU().TryReadInstruction(address); + if (op.valid) + { + INFO_LOG_FMT(IOS, "Broadway code {:#010x}: {:#010x} {}", address, op.hex, + Common::GekkoDisassembler::Disassemble(op.hex, address)); + } + } + const u32 lr_start = (LR(ppc) - 0x20) & ~u32{3}; + const u32 lr_end = (LR(ppc) + 0x20) & ~u32{3}; + for (u32 address = lr_start; address <= lr_end; address += 4) + { + const PowerPC::TryReadInstResult op = m_system.GetMMU().TryReadInstruction(address); + if (op.valid) + { + INFO_LOG_FMT(IOS, "Broadway LR code {:#010x}: {:#010x} {}", address, op.hex, + Common::GekkoDisassembler::Disassemble(op.hex, address)); + } + } + m_ppc_context_logged = true; + } + } + + const u64 executed = m_core->GetExecutedInstructions(); + if (m_core->IsJitEnabled() && executed >= m_next_jit_diagnostic_instruction) + { + const u64 jit_executed = m_core->GetJitExecutedInstructions(); + const u64 jit_native = m_core->GetJitNativeExecutedInstructions(); + const u64 block_executions = m_core->GetJitBlockExecutionCount(); + const u64 interval_fallbacks = m_core->TakeJitFallbackIntervalCount(); + INFO_LOG_FMT(IOS, + "Starlet live JIT: instructions={} native={:.1f}% fallbacks={} blocks={} " + "block-runs={} insns-per-block={:.1f} translations={} slow-reads={} " + "slow-writes={} slow-regions=ram:{}/sram:{}/mmio:{}/other:{} " + "interval-fallbacks={} idle-polls={} ipc-armctrl={:#04x} " + "PC={:#010x} " + "CPSR={:#010x} irqflag={:#010x} irqmask={:#010x} fiqmask={:#010x}", + jit_executed, jit_executed == 0 ? 0.0 : 100.0 * jit_native / jit_executed, + m_core->GetJitFallbackInstructionCount(), m_core->GetJitCompiledBlockCount(), + block_executions, + block_executions == 0 ? 0.0 : static_cast(jit_executed) / block_executions, + m_core->GetJitAddressTranslationCount(), m_core->GetJitSlowReadCount(), + m_core->GetJitSlowWriteCount(), + m_core->GetJitSlowRAMAccessCount(), m_core->GetJitSlowSRAMAccessCount(), + m_core->GetJitSlowMMIOAccessCount(), m_core->GetJitSlowOtherAccessCount(), + interval_fallbacks, m_core->GetMemoryPollEntryCount(), ipc_arm_ctrl, + m_core->GetRegister(15), m_core->GetCPSR(), + m_system.GetWiiIPC().ReadStarletRegister(0x38), + m_system.GetWiiIPC().ReadStarletRegister(0x3c), + m_system.GetWiiIPC().ReadStarletRegister(0x40)); + std::array hot_sram_read_pages{}; + std::array hot_sram_read_page_counts{}; + std::array hot_sram_write_pages{}; + std::array hot_sram_write_page_counts{}; + const auto rank_sram_page = [](u32 page, u64 count, auto& pages, auto& counts) { + for (size_t rank = 0; rank < counts.size(); ++rank) + { + if (count <= counts[rank]) + continue; + for (size_t move = counts.size() - 1; move > rank; --move) + { + counts[move] = counts[move - 1]; + pages[move] = pages[move - 1]; + } + counts[rank] = count; + pages[rank] = page; + break; + } + }; + for (u32 page = 0; page < 32; ++page) + { + rank_sram_page(page, m_core->GetJitSlowSRAMPageReadAccessCount(page), hot_sram_read_pages, + hot_sram_read_page_counts); + rank_sram_page(page, m_core->GetJitSlowSRAMPageWriteAccessCount(page), + hot_sram_write_pages, hot_sram_write_page_counts); + } + INFO_LOG_FMT(IOS, + "Starlet slow SRAM aliases=low:{}/high:{} reads:{}/writes:{} " + "hot-reads={:#04x}:{}, {:#04x}:{}, {:#04x}:{}, {:#04x}:{} " + "hot-writes={:#04x}:{}, {:#04x}:{}, {:#04x}:{}, {:#04x}:{}", + m_core->GetJitSlowSRAMLowAccessCount(), + m_core->GetJitSlowSRAMHighAccessCount(), + m_core->GetJitSlowSRAMReadAccessCount(), + m_core->GetJitSlowSRAMWriteAccessCount(), hot_sram_read_pages[0], + hot_sram_read_page_counts[0], hot_sram_read_pages[1], + hot_sram_read_page_counts[1], hot_sram_read_pages[2], + hot_sram_read_page_counts[2], hot_sram_read_pages[3], + hot_sram_read_page_counts[3], hot_sram_write_pages[0], + hot_sram_write_page_counts[0], hot_sram_write_pages[1], + hot_sram_write_page_counts[1], hot_sram_write_pages[2], + hot_sram_write_page_counts[2], hot_sram_write_pages[3], + hot_sram_write_page_counts[3]); + const ARMCore::HotPCSample current = m_core->GetCurrentPCSample(); + INFO_LOG_FMT(IOS, "Starlet current PC {:#010x} {} instruction={:#010x}", current.address, + current.thumb ? "Thumb" : "ARM", current.instruction); + if (!m_jit_context_logged && current.address == 0xffff0758) + { + for (u32 address = 0xffff0738; address <= 0xffff0778; address += 4) + { + const ARMCore::HotPCSample sample = m_core->GetInstructionSample(address, false); + INFO_LOG_FMT(IOS, "Starlet stuck context {:#010x} instruction={:#010x}", sample.address, + sample.instruction); + } + for (u32 address = 0xffff49a0; address <= 0xffff49e0; address += 4) + { + const ARMCore::HotPCSample sample = m_core->GetInstructionSample(address, false); + INFO_LOG_FMT(IOS, "Starlet stuck target {:#010x} instruction={:#010x}", sample.address, + sample.instruction); + } + m_jit_context_logged = true; + } + for (const ARMCore::HotPCSample& sample : + m_core->TakeHotJitFallbackIntervalSamples(6)) + { + INFO_LOG_FMT(IOS, + "Starlet interval fallback PC {:#010x} {} instruction={:#010x} samples={}", + sample.address, sample.thumb ? "Thumb" : "ARM", sample.instruction, + sample.samples); + } + m_next_jit_diagnostic_instruction = executed + 25'000'000; + } + const u64 broadway_cycles = arm_cycles * BROADWAY_CLOCK / ARM_CLOCK; const s64 next = std::max(1, static_cast(broadway_cycles) - cycles_late); m_system.GetCoreTiming().ScheduleEvent(next, m_run_event); diff --git a/Source/Core/Core/IOS/Starlet/Starlet.h b/Source/Core/Core/IOS/Starlet/Starlet.h index 3bb70a6cbf..d24defd6f0 100644 --- a/Source/Core/Core/IOS/Starlet/Starlet.h +++ b/Source/Core/Core/IOS/Starlet/Starlet.h @@ -79,6 +79,12 @@ private: std::unique_ptr m_memory; std::unique_ptr m_core; CoreTiming::EventType* m_run_event = nullptr; + u64 m_next_jit_diagnostic_instruction = 25'000'000; + u64 m_next_realtime_diagnostic_ms = 0; + u32 m_last_realtime_ppc_pc = 0; + u32 m_realtime_ppc_pc_streak = 0; + bool m_ppc_context_logged = false; + bool m_jit_context_logged = false; bool m_initialized = false; }; } // namespace IOS::LLE diff --git a/Source/Core/Core/IOS/Starlet/StarletMemory.cpp b/Source/Core/Core/IOS/Starlet/StarletMemory.cpp index c19ba22803..a82f4e889d 100644 --- a/Source/Core/Core/IOS/Starlet/StarletMemory.cpp +++ b/Source/Core/Core/IOS/Starlet/StarletMemory.cpp @@ -400,6 +400,51 @@ constexpr u32 GPIO_EEP_CLK = 0x800; constexpr u32 GPIO_EEP_MOSI = 0x1000; constexpr u32 GPIO_EEP_MISO = 0x2000; constexpr u32 GPIO_VALID_MASK = 0x00ffffff; +constexpr u32 GPIO_DEBUG_MASK = 0x00ff0000; + +constexpr std::string_view GetBootMiiDebugCodeMeaning(u8 code) +{ + // BootMii's public loader and stubs deliberately publish these values on GPIO[23:16]. Some + // values are reused by more than one layer, so keep the descriptions broad and preserve the + // exact byte in the log. + switch (code) + { + case 0x03: + return "hardware setup begin"; + case 0x12: + return "SD mount failed (following byte is the FAT error)"; + case 0x42: + return "loader entry"; + case 0x43: + return "loader stack ready"; + case 0x44: + return "loader BSS cleared"; + case 0x73: + return "hardware setup complete"; + case 0xc0: + return "stub bypass/power-button path"; + case 0xc1: + return "stub fallback payload loaded"; + case 0xc8: + return "fall back to NAND boot2"; + case 0xc9: + return "handoff to NAND boot2"; + case 0xe3: + return "panic: invalid ELF header"; + case 0xe4: + return "panic: ELF has no program headers"; + case 0xf0: + return "BootMii loader main"; + case 0xf1: + return "loader/stub initialized"; + case 0xf2: + return "SD payload selected"; + case 0xf3: + return "handoff to ARM payload"; + default: + return "unclassified"; + } +} constexpr u32 SRNPROT_SRAM_SPLIT_MODE = 1U << 5; constexpr u32 BOOT0_DISABLE = 1U << 12; @@ -775,6 +820,37 @@ bool StarletMemory::IsIdlePollAddress(u32 address, u32 size) const (IsSRAMWindowAddress(address) && IsSRAMWindowAddress(last_address)); } +bool StarletMemory::IsSliceStablePollAddress(u32 address, u32 size) const +{ + // The Hollywood timer is derived solely from m_arm_cycles, which changes in AdvanceCycles at + // the end of an ARM scheduler slice. Re-reading it inside the same slice cannot observe a new + // value, so a side-effect-free timer loop may yield the unused instructions exactly. + return address == HW_TIMER && size == sizeof(u32); +} + +u8* StarletMemory::GetFastmemBase() const +{ + return m_system.GetMemory().GetPhysicalBase(); +} + +u8* StarletMemory::GetFastmemSRAMBase() const +{ + // The JIT admits only measured SRAM pages and directions, and checks the live boot0 overlay and + // A/B split state before dereferencing this view. Register-list transfers and all nonprofiled + // writes remain on the exact bus path until each additional class has a full-chain regression. + return const_cast(m_sram.data()); +} + +const bool* StarletMemory::GetFastmemBoot0Mapped() const +{ + return &m_boot0_mapped; +} + +const bool* StarletMemory::GetFastmemSRAMSplitMode() const +{ + return &m_sram_split_mode; +} + bool StarletMemory::IsMemoryAddress(u32 address) { return address < Memory::MEM1_SIZE_RETAIL || @@ -4278,6 +4354,15 @@ void StarletMemory::HandleOTPCommand(u32 command) void StarletMemory::HandleGPIOWrite(u32 value) { + const u8 old_debug_code = static_cast((m_gpio_out & GPIO_DEBUG_MASK) >> 16); + const u8 debug_code = static_cast((value & GPIO_DEBUG_MASK) >> 16); + if (debug_code != old_debug_code) + { + const Starlet* const starlet = m_system.GetStarlet(); + INFO_LOG_FMT(IOS, "Starlet GPIO debug port: code={:#04x} stage='{}' PC={:#010x}", debug_code, + GetBootMiiDebugCodeMeaning(debug_code), starlet ? starlet->GetPC() : 0); + } + const bool old_cs = (m_gpio_out & GPIO_EEP_CS) != 0; const bool old_clock = (m_gpio_out & GPIO_EEP_CLK) != 0; const bool chip_selected = (value & GPIO_EEP_CS) != 0; diff --git a/Source/Core/Core/IOS/Starlet/StarletMemory.h b/Source/Core/Core/IOS/Starlet/StarletMemory.h index 846d2e3dde..3ff47dc16c 100644 --- a/Source/Core/Core/IOS/Starlet/StarletMemory.h +++ b/Source/Core/Core/IOS/Starlet/StarletMemory.h @@ -61,6 +61,11 @@ public: void Write32(u32 address, u32 value) override; void AdvanceCycles(u64 cycles) override; bool IsIdlePollAddress(u32 address, u32 size) const override; + bool IsSliceStablePollAddress(u32 address, u32 size) const override; + u8* GetFastmemBase() const override; + u8* GetFastmemSRAMBase() const override; + const bool* GetFastmemBoot0Mapped() const override; + const bool* GetFastmemSRAMSplitMode() const override; u64 GetCycles() const { return m_arm_cycles; } std::optional TryReadBroadwayResetInstruction(u32 address) const; diff --git a/Source/UnitTests/Core/IOS/Starlet/ARMCoreTest.cpp b/Source/UnitTests/Core/IOS/Starlet/ARMCoreTest.cpp index 619b23104a..e788c6552e 100644 --- a/Source/UnitTests/Core/IOS/Starlet/ARMCoreTest.cpp +++ b/Source/UnitTests/Core/IOS/Starlet/ARMCoreTest.cpp @@ -1,7 +1,11 @@ // Copyright 2026 Dolphin Emulator Project +// ARMCore is instantiated by value below, so changes to its private layout must rebuild this TU. // SPDX-License-Identifier: GPL-2.0-or-later +#include #include +#include +#include #include #include @@ -17,16 +21,23 @@ namespace IOS::LLE { namespace { +// Several integration tests construct the concrete Starlet bus directly, so +// changes to its device-state layout must rebuild this translation unit. // Native-width bus counters also verify that the CPU's generation-tagged // caches bypass redundant traffic without changing architectural results. class TestBus final : public ARMBus { public: - explicit TestBus(size_t size = 0x1000) : m_memory(size) {} + explicit TestBus(size_t size = 0x1000) + : m_memory(size), m_sram(SRAM_SIZE + 2 * SRAM_GUARD_SIZE, SRAM_GUARD_VALUE) + { + } u8 Read8(u32 address) override { ++m_read8_count; + if (address >= MMIO_WORD_ADDRESS && address < MMIO_WORD_ADDRESS + 4) + return static_cast(m_mmio_word >> (24 - (address & 3) * 8)); const size_t offset = ToOffset(address); EXPECT_LT(offset, m_memory.size()); return offset < m_memory.size() ? m_memory[offset] : 0; @@ -45,6 +56,37 @@ public: u32 Read32(u32 address) override { ++m_read32_count; + if (address == MMIO_WORD_ADDRESS) + return m_mmio_word; + if (m_sram_fastmem_enabled && address >= 0xfff00000) + { + size_t offset = address & 0x1ffff; + if (m_boot0_mapped && address >= 0xfffe0000 && + (m_sram_split_mode ? offset < 0x10000 : offset >= 0x10000)) + { + return 0; + } + if (m_sram_split_mode) + { + if (offset < 0x8000) + offset += 0x10000; + else if (offset >= 0x10000) + offset -= 0x10000; + else + return 0; + } + else if (offset >= 0x18000) + { + return 0; + } + EXPECT_LT(offset + 3, SRAM_SIZE); + if (offset + 3 >= SRAM_SIZE) + return 0; + const u8* const sram = SRAMData(); + return (static_cast(sram[offset]) << 24) | + (static_cast(sram[offset + 1]) << 16) | + (static_cast(sram[offset + 2]) << 8) | sram[offset + 3]; + } const size_t offset = ToOffset(address); EXPECT_LT(offset + 3, m_memory.size()); if (offset + 3 >= m_memory.size()) @@ -56,6 +98,37 @@ public: void Write8(u32 address, u8 value) override { + if (address >= MMIO_WORD_ADDRESS && address < MMIO_WORD_ADDRESS + 4) + { + const u32 shift = 24 - (address & 3) * 8; + m_mmio_word = (m_mmio_word & ~(0xffU << shift)) | (static_cast(value) << shift); + return; + } + if (m_sram_fastmem_enabled && address >= 0xfff00000) + { + size_t offset = address & 0x1ffff; + if (m_boot0_mapped && address >= 0xfffe0000 && + (m_sram_split_mode ? offset < 0x10000 : offset >= 0x10000)) + { + return; + } + if (m_sram_split_mode) + { + if (offset < 0x8000) + offset += 0x10000; + else if (offset >= 0x10000) + offset -= 0x10000; + else + return; + } + else if (offset >= 0x18000) + { + return; + } + ASSERT_LT(offset, SRAM_SIZE); + SRAMData()[offset] = value; + return; + } const size_t offset = ToOffset(address); ASSERT_LT(offset, m_memory.size()); m_memory[offset] = value; @@ -74,7 +147,68 @@ public: size <= m_memory.size() - offset; } + bool IsSliceStablePollAddress(u32 address, u32 size) const override + { + return m_slice_stable_poll_safe && address == m_slice_stable_poll_address && size == 4; + } + + u8* GetFastmemBase() const override + { + return m_fastmem_enabled ? const_cast(m_memory.data()) : nullptr; + } + + u8* GetFastmemSRAMBase() const override + { + return m_sram_fastmem_enabled ? const_cast(SRAMData()) : nullptr; + } + + const bool* GetFastmemBoot0Mapped() const override + { + return m_sram_fastmem_enabled ? &m_boot0_mapped : nullptr; + } + + const bool* GetFastmemSRAMSplitMode() const override + { + return m_sram_fastmem_enabled ? &m_sram_split_mode : nullptr; + } + void SetIdlePollSafe(bool safe) { m_idle_poll_safe = safe; } + void SetSliceStablePollAddress(u32 address) + { + m_slice_stable_poll_safe = true; + m_slice_stable_poll_address = address; + } + void SetFastmemEnabled(bool enabled) { m_fastmem_enabled = enabled; } + void SetSRAMFastmemEnabled(bool enabled) { m_sram_fastmem_enabled = enabled; } + void SetBoot0Mapped(bool mapped) { m_boot0_mapped = mapped; } + void SetSRAMSplitMode(bool split) { m_sram_split_mode = split; } + + void WriteSRAM32(u32 offset, u32 value) + { + ASSERT_LT(offset + 3, SRAM_SIZE); + u8* const sram = SRAMData(); + sram[offset] = static_cast(value >> 24); + sram[offset + 1] = static_cast(value >> 16); + sram[offset + 2] = static_cast(value >> 8); + sram[offset + 3] = static_cast(value); + } + + u32 ReadSRAM32(u32 offset) const + { + EXPECT_LT(offset + 3, SRAM_SIZE); + const u8* const sram = SRAMData(); + return (static_cast(sram[offset]) << 24) | + (static_cast(sram[offset + 1]) << 16) | + (static_cast(sram[offset + 2]) << 8) | sram[offset + 3]; + } + + bool SRAMCanariesIntact() const + { + return std::all_of(m_sram.begin(), m_sram.begin() + SRAM_GUARD_SIZE, + [](u8 value) { return value == SRAM_GUARD_VALUE; }) && + std::all_of(m_sram.end() - SRAM_GUARD_SIZE, m_sram.end(), + [](u8 value) { return value == SRAM_GUARD_VALUE; }); + } void WriteARM(u32 address, u32 instruction) { @@ -98,6 +232,7 @@ public: u64 GetRead8Count() const { return m_read8_count; } u64 GetRead16Count() const { return m_read16_count; } u64 GetRead32Count() const { return m_read32_count; } + u32 GetMMIOWord() const { return m_mmio_word; } void ResetReadCounts() { m_read8_count = 0; @@ -106,18 +241,34 @@ public: } private: + static constexpr u32 MMIO_WORD_ADDRESS = 0x0d800000; + static constexpr size_t SRAM_SIZE = 0x18000; + static constexpr size_t SRAM_GUARD_SIZE = 64; + static constexpr u8 SRAM_GUARD_VALUE = 0xa5; + + u8* SRAMData() { return m_sram.data() + SRAM_GUARD_SIZE; } + const u8* SRAMData() const { return m_sram.data() + SRAM_GUARD_SIZE; } + static size_t ToOffset(u32 address) { return address >= 0xffff0000 ? address - 0xffff0000 : address; } std::vector m_memory; + std::vector m_sram; u64 m_cycles = 0; u64 m_advance_calls = 0; u64 m_read8_count = 0; u64 m_read16_count = 0; u64 m_read32_count = 0; + u32 m_mmio_word = 0; bool m_idle_poll_safe = true; + bool m_slice_stable_poll_safe = false; + u32 m_slice_stable_poll_address = 0; + bool m_fastmem_enabled = false; + bool m_sram_fastmem_enabled = false; + bool m_boot0_mapped = true; + bool m_sram_split_mode = false; }; TEST(WiiIPCCtrlRegister, ProducerBitsRemainLatchedUntilPeerAcknowledges) @@ -703,6 +854,108 @@ TEST(StarletARMCore, CP15InstructionCacheMaintenanceInvalidatesCachedCode) EXPECT_EQ(core.GetRegister(1), 2u); } +TEST(StarletARMCore, CP15DrainWriteBufferPreservesInstructionCache) +{ + TestBus bus; + ARMCore core(bus); + bus.WriteARM(0x00, 0xe3a01001); // mov r1, #1 + bus.WriteARM(0x04, 0xee070f9a); // mcr p15, 0, r0, c7, c10, 4 (drain write buffer) + + core.Step(); + ASSERT_EQ(core.GetRegister(1), 1u); + bus.WriteARM(0x00, 0xe3a01002); + + core.SetRegister(15, 4); + core.Step(); + core.SetRegister(15, 0); + core.Step(); + EXPECT_EQ(core.GetRegister(1), 1u); +} + +TEST(StarletARMCore, JitCompilesDrainWriteBufferNatively) +{ + TestBus bus; + ARMCore core(bus); + core.SetJitEnabled(true); + bus.WriteARM(0x00, 0xe3a01001); // mov r1, #1 + bus.WriteARM(0x04, 0xee070f9a); // drain write buffer + bus.WriteARM(0x08, 0xe2811001); // add r1, r1, #1 + + EXPECT_EQ(core.RunCycles(4), 4u); + EXPECT_EQ(core.GetRegister(1), 2u); + EXPECT_EQ(core.GetJitNativeExecutedInstructions(), 3u); + EXPECT_EQ(core.GetJitFallbackInstructionCount(), 1u); +} + +TEST(StarletARMCore, JitDefersCP15CacheInvalidationUntilTheHostBlockReturns) +{ + TestBus bus; + ARMCore core(bus); + core.SetJitEnabled(true); + bus.WriteARM(0x00, 0xe3a01001); // mov r1, #1 (native JIT instruction) + bus.WriteARM(0x04, + 0xee070f15); // mcr p15, 0, r0, c7, c5, 0 (fallback and JIT clear) + bus.WriteARM(0x08, 0xe2811001); // add r1, r1, #1 (compiled after the clear) + + EXPECT_EQ(core.RunCycles(3), 3u); + EXPECT_EQ(core.GetRegister(1), 2u); + EXPECT_EQ(core.GetRegister(15), 0x0cu); + EXPECT_GT(core.GetJitExecutedInstructions(), 0u); +} + +TEST(StarletARMCore, JitPreservesPhysicalBlocksAcrossTLBMaintenance) +{ +#if defined(_M_X86_64) + TestBus bus; + ARMCore core(bus); + core.SetJitEnabled(true); + bus.WriteARM(0x00, 0xe3a01001); // mov r1, #1 (native JIT instruction) + bus.WriteARM(0x04, + 0xee080f17); // mcr p15, 0, r0, c8, c7, 0 (invalidate unified TLB) + + EXPECT_EQ(core.RunCycles(2), 2u); + ASSERT_EQ(core.GetJitCompiledBlockCount(), 1u); + + core.SetRegister(15, 0); + EXPECT_EQ(core.RunCycles(2), 2u); + EXPECT_EQ(core.GetRegister(1), 1u); + EXPECT_EQ(core.GetJitCompiledBlockCount(), 1u); +#endif +} + +TEST(StarletARMCore, JitPreservedBlocksUseCurrentTLBGenerationForFastmem) +{ +#if defined(_M_X86_64) + TestBus bus(0x10000); + ARMCore core(bus); + bus.SetFastmemEnabled(true); + bus.WriteARM(0x0000, 0xe5921000); // ldr r1, [r2] + bus.WriteARM(0x0004, + 0xee080f17); // mcr p15, 0, r0, c8, c7, 0 (invalidate unified TLB) + bus.WriteARM(0x0040, 0x11223344); + bus.WriteARM(0x6000, + 0x00000c02); // VA 0x80000000 section -> PA 0, full access + core.GetCP15State().translation_table_base = 0x4000; + core.GetCP15State().domain_access_control = 3; + core.GetCP15State().control |= 1; + core.SetRegister(2, 0x80000040); + core.SetRegister(15, 0x80000000); + core.SetJitEnabled(true); + + EXPECT_EQ(core.RunCycles(2), 2u); + ASSERT_EQ(core.GetRegister(1), 0x11223344u); + ASSERT_EQ(core.GetJitFallbackInstructionCount(), 1u); + ASSERT_EQ(core.GetJitCompiledBlockCount(), 1u); + + core.SetRegister(1, 0); + core.SetRegister(15, 0x80000000); + EXPECT_EQ(core.RunCycles(1), 1u); + EXPECT_EQ(core.GetRegister(1), 0x11223344u); + EXPECT_EQ(core.GetJitFallbackInstructionCount(), 1u); + EXPECT_EQ(core.GetJitCompiledBlockCount(), 1u); +#endif +} + TEST(StarletARMCore, WaitForInterruptFastForwardsAndWakesOnMaskedIRQ) { TestBus bus; @@ -766,6 +1019,997 @@ TEST(StarletARMCore, ThumbExecutionAndConditions) EXPECT_EQ(core.GetRegister(15), 0x10au); } +TEST(StarletARMCore, ThumbJitMatchesInterpreterAcrossALUBranchAndMemory) +{ + TestBus interpreter_bus; + TestBus jit_bus; + ARMCore interpreter(interpreter_bus); + ARMCore jit(jit_bus); + interpreter.SetJitEnabled(false); + jit.SetJitEnabled(true); + + const auto install_program = [](TestBus& bus) { + bus.WriteThumb(0x00, 0x2000); // mov r0, #0 + bus.WriteThumb(0x02, 0x210a); // mov r1, #10 + bus.WriteThumb(0x04, 0x3001); // add r0, #1 + bus.WriteThumb(0x06, 0x280a); // cmp r0, #10 + bus.WriteThumb(0x08, 0xd1fc); // bne 0x04 + bus.WriteThumb(0x0a, 0x6010); // str r0, [r2] + }; + install_program(interpreter_bus); + install_program(jit_bus); + const u32 thumb_cpsr = static_cast(ARMCore::Mode::Supervisor) | ARMCore::CPSR_T; + interpreter.SetCPSR(thumb_cpsr); + jit.SetCPSR(thumb_cpsr); + interpreter.SetRegister(2, 0x100); + jit.SetRegister(2, 0x100); + + EXPECT_EQ(interpreter.RunCycles(33), 33u); + EXPECT_EQ(jit.RunCycles(33), 33u); + for (u32 reg = 0; reg < 16; ++reg) + EXPECT_EQ(jit.GetRegister(reg), interpreter.GetRegister(reg)) << "r" << reg; + EXPECT_EQ(jit.GetCPSR(), interpreter.GetCPSR()); + EXPECT_EQ(jit_bus[0x100], interpreter_bus[0x100]); + EXPECT_EQ(jit_bus[0x103], interpreter_bus[0x103]); + EXPECT_GT(jit.GetJitExecutedInstructions(), 0u); +} + +TEST(StarletARMCore, ThumbJitUsesDirectFastmemForAlignedRAM) +{ + TestBus bus; + bus.SetFastmemEnabled(true); + ARMCore core(bus); + core.SetJitEnabled(true); + bus.WriteThumb(0x00, 0x202a); // mov r0, #42 + bus.WriteThumb(0x02, 0x6010); // str r0, [r2] + bus.WriteThumb(0x04, 0x6811); // ldr r1, [r2] + bus.WriteThumb(0x06, 0xbe00); // BKPT boundary/fallback + core.SetCPSR(static_cast(ARMCore::Mode::Supervisor) | ARMCore::CPSR_T); + core.SetRegister(2, 0x100); + + EXPECT_EQ(core.RunCycles(4), 4u); + EXPECT_EQ(core.GetRegister(1), 42u); + EXPECT_EQ(bus[0x100], 0u); + EXPECT_EQ(bus[0x103], 42u); + EXPECT_GE(core.GetJitNativeExecutedInstructions(), 3u); +} + +TEST(StarletARMCore, ARMJitCompilesRegisterOffsetLoadAndExchangeReturn) +{ + TestBus bus; + bus.SetFastmemEnabled(true); + ARMCore core(bus); + core.SetJitEnabled(true); + bus.WriteARM(0x00, 0xe7923101); // ldr r3, [r2, r1, lsl #2] + bus.WriteARM(0x04, 0xeafffffe); // b . + bus.WriteARM(0x104, 0x12345678); // data + core.SetRegister(1, 1); + core.SetRegister(2, 0x100); + + EXPECT_EQ(core.RunCycles(2), 2u); + EXPECT_EQ(core.GetRegister(3), 0x12345678u); + EXPECT_EQ(core.GetRegister(15), 0x04u); + EXPECT_EQ(core.GetJitFallbackInstructionCount(), 0u); + EXPECT_EQ(core.GetJitNativeExecutedInstructions(), 2u); +} + +TEST(StarletARMCore, ARMJitCompilesIOSSyscallBankSwitchAndExceptionReturn) +{ +#if defined(_M_X86_64) + TestBus interpreter_bus(0x2000); + TestBus jit_bus(0x2000); + ARMCore interpreter(interpreter_bus); + ARMCore jit(jit_bus); + jit.SetJitEnabled(true); + + const auto install_program = [](TestBus& bus) { + bus.WriteARM(0x00000000, 0xe6000010); // IOS syscall 0 -> undefined exception + bus.WriteARM(0x00000004, 0xeafffffe); // return target: b . + bus.WriteARM(0xffff0004, 0xea00003d); // b 0xffff0100 + bus.WriteARM(0xffff0100, 0xe14f8000); // mrs r8, spsr + bus.WriteARM(0xffff0104, 0xe3a0b01f); // mov r11, #System + bus.WriteARM(0xffff0108, 0xe121f00b); // msr cpsr_c, r11 + bus.WriteARM(0xffff010c, 0xe3a0002a); // mov r0, #42 in the user/system bank + bus.WriteARM(0xffff0110, 0xe3a0b0db); // mov r11, #Undefined + IRQ/FIQ masked + bus.WriteARM(0xffff0114, 0xe121f00b); // msr cpsr_c, r11 + bus.WriteARM(0xffff0118, 0xe1a0b008); // mov r11, r8 + bus.WriteARM(0xffff011c, 0xe16ff00b); // msr spsr_fsxc, r11 + bus.WriteARM(0xffff0120, 0xe1b0f00e); // movs pc, lr + }; + install_program(interpreter_bus); + install_program(jit_bus); + const u32 original_cpsr = static_cast(ARMCore::Mode::System) | ARMCore::CPSR_C; + interpreter.SetCPSR(original_cpsr); + jit.SetCPSR(original_cpsr); + + ASSERT_EQ(interpreter.RunCycles(11), 11u); + ASSERT_EQ(jit.RunCycles(11), 11u); + for (u32 reg = 0; reg < 16; ++reg) + EXPECT_EQ(jit.GetRegister(reg), interpreter.GetRegister(reg)) << "r" << reg; + EXPECT_EQ(jit.GetCPSR(), interpreter.GetCPSR()); + EXPECT_EQ(jit.GetCPSR(), original_cpsr); + EXPECT_EQ(jit.GetRegister(0), 42u); + EXPECT_EQ(jit.GetRegister(15), 4u); + EXPECT_EQ(jit.GetJitFallbackInstructionCount(), 0u); + EXPECT_EQ(jit.GetJitNativeExecutedInstructions(), 11u); +#endif +} + +TEST(StarletARMCore, ARMJitSamplesPendingIRQImmediatelyAfterCPSRUnmask) +{ +#if defined(_M_X86_64) + TestBus bus(0x1000); + ARMCore core(bus); + core.SetJitEnabled(true); + bus.WriteARM(0x00, 0xe10f1000); // mrs r1, cpsr + bus.WriteARM(0x04, 0xe3c11080); // bic r1, r1, #CPSR_I + bus.WriteARM(0x08, 0xe121f001); // msr cpsr_c, r1 + bus.WriteARM(0x0c, 0xe3a02055); // must not execute before the pending IRQ + + core.SetCPSR(static_cast(ARMCore::Mode::System) | ARMCore::CPSR_I); + core.SetIRQLine(true); + + // The exception entry costs three cycles and can take RunCycles beyond its requested boundary. + // What matters is that IRQ is sampled between MSR and the following guest instruction. + EXPECT_GE(core.RunCycles(4), 4u); + EXPECT_EQ(core.GetMode(), ARMCore::Mode::IRQ); + EXPECT_EQ(core.GetRegister(15), 0xffff0018u); + EXPECT_EQ(core.GetRegister(2), 0u); + EXPECT_EQ(core.GetJitFallbackInstructionCount(), 0u); +#endif +} + +TEST(StarletARMCore, ThumbJitKeepsPushPopInsideNativeBlock) +{ +#if defined(_M_X86_64) + TestBus interpreter_bus(0x1000); + TestBus jit_bus(0x1000); + ARMCore interpreter(interpreter_bus); + ARMCore jit(jit_bus); + jit.SetJitEnabled(true); + const auto install_program = [](TestBus& bus) { + bus.WriteThumb(0x00, 0xb403); // push {r0, r1} + bus.WriteThumb(0x02, 0xbc0c); // pop {r2, r3} + bus.WriteThumb(0x04, 0xe7fe); // b . + }; + install_program(interpreter_bus); + install_program(jit_bus); + const u32 cpsr = static_cast(ARMCore::Mode::System) | ARMCore::CPSR_T; + for (ARMCore* core : {&interpreter, &jit}) + { + core->SetCPSR(cpsr); + core->SetRegister(0, 0x11223344); + core->SetRegister(1, 0x55667788); + core->SetRegister(13, 0x200); + } + + ASSERT_EQ(interpreter.RunCycles(3), 3u); + ASSERT_EQ(jit.RunCycles(3), 3u); + for (u32 reg = 0; reg < 16; ++reg) + EXPECT_EQ(jit.GetRegister(reg), interpreter.GetRegister(reg)) << "r" << reg; + EXPECT_EQ(jit.GetCPSR(), interpreter.GetCPSR()); + EXPECT_EQ(jit.GetRegister(2), 0x11223344u); + EXPECT_EQ(jit.GetRegister(3), 0x55667788u); + EXPECT_EQ(jit.GetJitFallbackInstructionCount(), 0u); + EXPECT_EQ(jit.GetJitNativeExecutedInstructions(), 3u); +#endif +} + +TEST(StarletARMCore, ARMJitCompilesExchangeReturn) +{ + TestBus bus; + ARMCore core(bus); + core.SetJitEnabled(true); + bus.WriteARM(0x00, 0xe12fff1e); // bx lr + bus.WriteARM(0x20, 0xeafffffe); // b . + core.SetRegister(14, 0x20); + + EXPECT_EQ(core.RunCycles(1), 1u); + EXPECT_EQ(core.GetRegister(15), 0x20u); + EXPECT_EQ(core.GetCPSR() & ARMCore::CPSR_T, 0u); + EXPECT_EQ(core.GetJitFallbackInstructionCount(), 0u); + EXPECT_EQ(core.GetJitNativeExecutedInstructions(), 1u); +} + +TEST(StarletARMCore, ARMJitConditionalBranchesMatchInterpreter) +{ + TestBus interpreter_bus(0x1000); + TestBus jit_bus(0x1000); + ARMCore interpreter(interpreter_bus); + ARMCore jit(jit_bus); + jit.SetJitEnabled(true); + + for (u32 condition = 0; condition < 15; ++condition) + { + const u32 address = condition * 0x20; + const u32 branch = (condition << 28) | 0x0a000000; + interpreter_bus.WriteARM(address, branch); // b address + 8 + jit_bus.WriteARM(address, branch); + interpreter_bus.WriteARM(address + 4, 0xe3a00001); // mov r0, #1 + jit_bus.WriteARM(address + 4, 0xe3a00001); + interpreter_bus.WriteARM(address + 8, 0xe3a00002); // mov r0, #2 + jit_bus.WriteARM(address + 8, 0xe3a00002); + + for (u32 flags = 0; flags < 16; ++flags) + { + const u32 cpsr = static_cast(ARMCore::Mode::Supervisor) | + ((flags & 1) ? ARMCore::CPSR_N : 0) | + ((flags & 2) ? ARMCore::CPSR_Z : 0) | + ((flags & 4) ? ARMCore::CPSR_C : 0) | + ((flags & 8) ? ARMCore::CPSR_V : 0); + interpreter.SetCPSR(cpsr); + jit.SetCPSR(cpsr); + interpreter.SetRegister(15, address); + jit.SetRegister(15, address); + interpreter.SetRegister(0, 0); + jit.SetRegister(0, 0); + + ASSERT_EQ(interpreter.RunCycles(2), 2u); + ASSERT_EQ(jit.RunCycles(2), 2u); + ASSERT_EQ(jit.GetRegister(0), interpreter.GetRegister(0)) + << "condition=" << condition << " flags=" << flags; + ASSERT_EQ(jit.GetRegister(15), interpreter.GetRegister(15)) + << "condition=" << condition << " flags=" << flags; + ASSERT_EQ(jit.GetCPSR(), interpreter.GetCPSR()) + << "condition=" << condition << " flags=" << flags; + } + } +} + +TEST(StarletARMCore, ARMJitCompilesPredicatedDataProcessing) +{ + TestBus interpreter_bus(0x1000); + TestBus jit_bus(0x1000); + ARMCore interpreter(interpreter_bus); + ARMCore jit(jit_bus); + jit.SetJitEnabled(true); + + for (u32 condition = 0; condition < 15; ++condition) + { + const u32 address = condition * 0x20; + const auto install_program = [&](TestBus& bus) { + bus.WriteARM(address + 0x00, (condition << 28) | 0x02822003); // add r2, r2, #3 + bus.WriteARM(address + 0x04, (condition << 28) | 0x03530002); // cmp r3, #2 + bus.WriteARM(address + 0x08, (condition << 28) | 0x03c44018); // bic r4, r4, #0x18 + bus.WriteARM(address + 0x0c, (condition << 28) | 0x03a05007); // mov r5, #7 + bus.WriteARM(address + 0x10, 0xea000000); // b address + 0x18 + }; + install_program(interpreter_bus); + install_program(jit_bus); + + for (u32 flags = 0; flags < 16; ++flags) + { + const u32 cpsr = static_cast(ARMCore::Mode::Supervisor) | + ((flags & 1) ? ARMCore::CPSR_N : 0) | + ((flags & 2) ? ARMCore::CPSR_Z : 0) | + ((flags & 4) ? ARMCore::CPSR_C : 0) | + ((flags & 8) ? ARMCore::CPSR_V : 0); + interpreter.SetCPSR(cpsr); + jit.SetCPSR(cpsr); + for (ARMCore* core : {&interpreter, &jit}) + { + core->SetRegister(2, 10); + core->SetRegister(3, 2); + core->SetRegister(4, 0xff); + core->SetRegister(5, 0); + core->SetRegister(15, address); + } + + ASSERT_EQ(interpreter.RunCycles(5), 5u); + ASSERT_EQ(jit.RunCycles(5), 5u); + for (u32 reg = 0; reg < 16; ++reg) + { + ASSERT_EQ(jit.GetRegister(reg), interpreter.GetRegister(reg)) + << "condition=" << condition << " flags=" << flags << " r" << reg; + } + ASSERT_EQ(jit.GetCPSR(), interpreter.GetCPSR()) + << "condition=" << condition << " flags=" << flags; + } + } + + EXPECT_EQ(jit.GetJitFallbackInstructionCount(), 0u); + EXPECT_EQ(jit.GetJitNativeExecutedInstructions(), 15u * 16u * 5u); +} + +TEST(StarletARMCore, ARMJitCompilesMovLinkFromArchitecturalPC) +{ + TestBus interpreter_bus; + TestBus jit_bus; + ARMCore interpreter(interpreter_bus); + ARMCore jit(jit_bus); + jit.SetJitEnabled(true); + interpreter_bus.WriteARM(0x00, 0xe1a0e00f); // mov lr, pc + jit_bus.WriteARM(0x00, 0xe1a0e00f); + interpreter_bus.WriteARM(0x04, 0xeafffffe); // b . + jit_bus.WriteARM(0x04, 0xeafffffe); + + ASSERT_EQ(interpreter.RunCycles(2), 2u); + ASSERT_EQ(jit.RunCycles(2), 2u); + EXPECT_EQ(jit.GetRegister(14), interpreter.GetRegister(14)); + EXPECT_EQ(jit.GetRegister(14), 8u); + EXPECT_EQ(jit.GetRegister(15), interpreter.GetRegister(15)); + EXPECT_EQ(jit.GetCPSR(), interpreter.GetCPSR()); + EXPECT_EQ(jit.GetJitFallbackInstructionCount(), 0u); + EXPECT_EQ(jit.GetJitNativeExecutedInstructions(), 2u); +} + +TEST(StarletARMCore, ARMJitEntersIOSUndefinedSyscallAndSVCNatively) +{ + { + TestBus bus; + ARMCore core(bus); + core.SetJitEnabled(true); + constexpr u32 syscall = 0xe6000010 | (0x36 << 5); + bus.WriteARM(0x00, syscall); + core.SetCPSR(static_cast(ARMCore::Mode::System)); + + ASSERT_EQ(core.RunCycles(1), 1u); + EXPECT_EQ(core.GetMode(), ARMCore::Mode::Undefined); + EXPECT_EQ(core.GetRegister(14), 4u); + EXPECT_EQ(core.GetRegister(15), 0xffff0004u); + EXPECT_EQ(core.GetLastUndefinedInstruction(), syscall); + EXPECT_EQ(core.GetJitFallbackInstructionCount(), 0u); + EXPECT_EQ(core.GetJitNativeExecutedInstructions(), 1u); + } + + { + TestBus bus; + ARMCore core(bus); + core.SetJitEnabled(true); + bus.WriteARM(0x00, 0xef0000ab); // svc 0xab (IOS debugger semihosting) + core.SetCPSR(static_cast(ARMCore::Mode::System)); + + ASSERT_EQ(core.RunCycles(1), 1u); + EXPECT_EQ(core.GetMode(), ARMCore::Mode::Supervisor); + EXPECT_EQ(core.GetRegister(14), 4u); + EXPECT_EQ(core.GetRegister(15), 0xffff0008u); + EXPECT_EQ(core.GetJitFallbackInstructionCount(), 0u); + EXPECT_EQ(core.GetJitNativeExecutedInstructions(), 1u); + } + + { + TestBus bus; + ARMCore core(bus); + core.SetJitEnabled(true); + bus.WriteThumb(0x00, 0xdfab); // svc 0xab + core.SetCPSR(static_cast(ARMCore::Mode::System) | ARMCore::CPSR_T); + + ASSERT_EQ(core.RunCycles(1), 1u); + EXPECT_EQ(core.GetMode(), ARMCore::Mode::Supervisor); + EXPECT_EQ(core.GetRegister(14), 2u); + EXPECT_EQ(core.GetRegister(15), 0xffff0008u); + EXPECT_EQ(core.GetJitFallbackInstructionCount(), 0u); + EXPECT_EQ(core.GetJitNativeExecutedInstructions(), 1u); + } +} + +TEST(StarletARMCore, ThumbJitCompilesShiftBranchExchangeAndRegisterMemory) +{ + TestBus interpreter_bus(0x1000); + TestBus jit_bus(0x1000); + interpreter_bus.SetFastmemEnabled(true); + jit_bus.SetFastmemEnabled(true); + ARMCore interpreter(interpreter_bus); + ARMCore jit(jit_bus); + jit.SetJitEnabled(true); + const auto install_program = [](TestBus& bus) { + bus.WriteThumb(0x00, 0x00c8); // lsl r0, r1, #3 + bus.WriteThumb(0x02, 0x0882); // lsr r2, r0, #2 + bus.WriteThumb(0x04, 0x1053); // asr r3, r2, #1 + bus.WriteThumb(0x06, 0x5163); // str r3, [r4, r5] + bus.WriteThumb(0x08, 0x5966); // ldr r6, [r4, r5] + bus.WriteThumb(0x0a, 0x429e); // cmp r6, r3 + bus.WriteThumb(0x0c, 0xd100); // bne 0x10 (not taken) + bus.WriteThumb(0x0e, 0x2755); // mov r7, #0x55 + bus.WriteThumb(0x10, 0x4740); // bx r8 + bus.WriteThumb(0x20, 0x27aa); // mov r7, #0xaa + bus.WriteThumb(0x22, 0xbe00); // BKPT boundary + }; + install_program(interpreter_bus); + install_program(jit_bus); + const u32 cpsr = static_cast(ARMCore::Mode::Supervisor) | ARMCore::CPSR_T; + interpreter.SetCPSR(cpsr); + jit.SetCPSR(cpsr); + for (ARMCore* core : {&interpreter, &jit}) + { + core->SetRegister(1, 0xf0000003); + core->SetRegister(4, 0x100); + core->SetRegister(5, 4); + core->SetRegister(8, 0x21); + } + + EXPECT_EQ(interpreter.RunCycles(10), 10u); + EXPECT_EQ(jit.RunCycles(10), 10u); + for (u32 reg = 0; reg < 16; ++reg) + EXPECT_EQ(jit.GetRegister(reg), interpreter.GetRegister(reg)) << "r" << reg; + EXPECT_EQ(jit.GetCPSR(), interpreter.GetCPSR()); + for (u32 byte = 0; byte < 4; ++byte) + EXPECT_EQ(jit_bus[0x104 + byte], interpreter_bus[0x104 + byte]); + EXPECT_EQ(jit.GetJitFallbackInstructionCount(), 0u); + // The final target MOV is interpreted because its two-instruction block would exceed the one + // cycle left in this deliberately exact budget. Everything before the indirect branch is native. + EXPECT_EQ(jit.GetJitNativeExecutedInstructions(), 9u); +} + +TEST(StarletARMCore, ThumbJitCompilesRegisterLSREdgeCases) +{ + TestBus interpreter_bus; + TestBus jit_bus; + ARMCore interpreter(interpreter_bus); + ARMCore jit(jit_bus); + jit.SetJitEnabled(true); + interpreter_bus.WriteThumb(0x00, 0x40c3); // lsr r3, r0 + jit_bus.WriteThumb(0x00, 0x40c3); + interpreter_bus.WriteThumb(0x02, 0xe7fe); // b . + jit_bus.WriteThumb(0x02, 0xe7fe); + + constexpr std::array values = {0, 1, 0x80000000, 0xffffffff, 0x12345678, + 0x7fffffff, 0xa5a5a5a5}; + constexpr std::array shifts = {0, 1, 2, 31, 32, 33, 63, 255}; + for (const u32 value : values) + { + for (const u32 shift : shifts) + { + for (const bool carry : {false, true}) + { + const u32 cpsr = static_cast(ARMCore::Mode::Supervisor) | ARMCore::CPSR_T | + ARMCore::CPSR_V | (carry ? ARMCore::CPSR_C : 0); + for (ARMCore* core : {&interpreter, &jit}) + { + core->SetCPSR(cpsr); + core->SetRegister(0, shift); + core->SetRegister(3, value); + core->SetRegister(15, 0); + } + + ASSERT_EQ(interpreter.RunCycles(2), 2u); + ASSERT_EQ(jit.RunCycles(2), 2u); + EXPECT_EQ(jit.GetRegister(3), interpreter.GetRegister(3)) + << "value=0x" << std::hex << value << " shift=" << std::dec << shift; + EXPECT_EQ(jit.GetRegister(15), interpreter.GetRegister(15)); + EXPECT_EQ(jit.GetCPSR(), interpreter.GetCPSR()) + << "value=0x" << std::hex << value << " shift=" << std::dec << shift; + } + } + } + EXPECT_EQ(jit.GetJitFallbackInstructionCount(), 0u); + EXPECT_EQ(jit.GetJitNativeExecutedInstructions(), values.size() * shifts.size() * 2 * 2); +} + +TEST(StarletARMCore, ARMJitMatchesInterpreterAcrossALUBranchAndMemory) +{ + TestBus interpreter_bus; + TestBus jit_bus; + ARMCore interpreter(interpreter_bus); + ARMCore jit(jit_bus); + interpreter.SetJitEnabled(false); + jit.SetJitEnabled(true); + + const auto install_program = [](TestBus& bus) { + bus.WriteARM(0x00, 0xe3a00000); // mov r0, #0 + bus.WriteARM(0x04, 0xe3a0100a); // mov r1, #10 + bus.WriteARM(0x08, 0xe2800001); // add r0, r0, #1 + bus.WriteARM(0x0c, 0xe350000a); // cmp r0, #10 + bus.WriteARM(0x10, 0x1afffffc); // bne 0x08 + bus.WriteARM(0x14, 0xe5820000); // str r0, [r2] + }; + install_program(interpreter_bus); + install_program(jit_bus); + interpreter.SetRegister(2, 0x100); + jit.SetRegister(2, 0x100); + + EXPECT_EQ(interpreter.RunCycles(33), 33u); + EXPECT_EQ(jit.RunCycles(33), 33u); + for (u32 reg = 0; reg < 16; ++reg) + EXPECT_EQ(jit.GetRegister(reg), interpreter.GetRegister(reg)) << "r" << reg; + EXPECT_EQ(jit.GetCPSR(), interpreter.GetCPSR()); + EXPECT_EQ(jit_bus[0x100], interpreter_bus[0x100]); + EXPECT_EQ(jit_bus[0x103], interpreter_bus[0x103]); + EXPECT_GT(jit.GetJitExecutedInstructions(), 0u); + EXPECT_GT(jit.GetJitNativeExecutedInstructions(), 0u); +} + +TEST(StarletARMCore, ARMJitUsesDirectFastmemForAlignedRAM) +{ + TestBus bus; + bus.SetFastmemEnabled(true); + ARMCore core(bus); + core.SetJitEnabled(true); + bus.WriteARM(0x00, 0xe3a0002a); // mov r0, #42 + bus.WriteARM(0x04, 0xe5820000); // str r0, [r2] + bus.WriteARM(0x08, 0xe5921000); // ldr r1, [r2] + bus.WriteARM(0x0c, 0x00000000); // unsupported boundary/fallback + core.SetRegister(2, 0x100); + + EXPECT_EQ(core.RunCycles(4), 4u); + EXPECT_EQ(core.GetRegister(1), 42u); + EXPECT_EQ(bus[0x100], 0u); + EXPECT_EQ(bus[0x103], 42u); + EXPECT_GE(core.GetJitNativeExecutedInstructions(), 3u); +} + +TEST(StarletARMCore, ARMJitCallsExactBusWithoutInterpreterExitForMMIOWordTransfers) +{ +#if defined(_M_X86_64) + TestBus interpreter_bus; + TestBus jit_bus; + jit_bus.SetFastmemEnabled(true); + ARMCore interpreter(interpreter_bus); + ARMCore jit(jit_bus); + jit.SetJitEnabled(true); + + const auto install_program = [](TestBus& bus) { + bus.WriteARM(0x00, 0xe5821000); // str r1, [r2] + bus.WriteARM(0x04, 0xe5923000); // ldr r3, [r2] + bus.WriteARM(0x08, 0xe2834001); // add r4, r3, #1 + bus.WriteARM(0x0c, 0xeafffffe); // b 0x0c + }; + install_program(interpreter_bus); + install_program(jit_bus); + for (ARMCore* core : {&interpreter, &jit}) + { + core->SetRegister(1, 0x12345678); + core->SetRegister(2, 0x0d800000); + } + + ASSERT_EQ(interpreter.RunCycles(4), 4u); + ASSERT_EQ(jit.RunCycles(4), 4u); + EXPECT_EQ(jit_bus.GetMMIOWord(), interpreter_bus.GetMMIOWord()); + EXPECT_EQ(jit.GetRegister(3), interpreter.GetRegister(3)); + EXPECT_EQ(jit.GetRegister(4), interpreter.GetRegister(4)); + EXPECT_EQ(jit.GetRegister(4), 0x12345679u); + EXPECT_EQ(jit.GetJitFallbackInstructionCount(), 0u); + EXPECT_EQ(jit.GetJitNativeExecutedInstructions(), 4u); +#endif +} + +TEST(StarletARMCore, ARMJitMatchesARM926UnalignedWordTransfersWithoutFallback) +{ +#if defined(_M_X86_64) + TestBus interpreter_bus; + TestBus jit_bus; + jit_bus.SetFastmemEnabled(true); + ARMCore interpreter(interpreter_bus); + ARMCore jit(jit_bus); + jit.SetJitEnabled(true); + + const auto install_program = [](TestBus& bus) { + bus.WriteARM(0x00, 0xe5921000); // ldr r1, [r2] -- rotate by 8 + bus.WriteARM(0x04, 0xe5943000); // ldr r3, [r4] -- rotate by 16 + bus.WriteARM(0x08, 0xe5965000); // ldr r5, [r6] -- rotate by 24 + bus.WriteARM(0x0c, 0xe5887000); // str r7, [r8] -- align down + bus.WriteARM(0x10, 0xeafffffe); // b 0x10 + bus.WriteARM(0x100, 0x11223344); + }; + install_program(interpreter_bus); + install_program(jit_bus); + for (ARMCore* core : {&interpreter, &jit}) + { + core->SetRegister(2, 0x101); + core->SetRegister(4, 0x102); + core->SetRegister(6, 0x103); + core->SetRegister(7, 0xaabbccdd); + core->SetRegister(8, 0x10b); + } + + ASSERT_EQ(interpreter.RunCycles(5), 5u); + ASSERT_EQ(jit.RunCycles(5), 5u); + for (u32 reg : {1u, 3u, 5u}) + EXPECT_EQ(jit.GetRegister(reg), interpreter.GetRegister(reg)) << "r" << reg; + EXPECT_EQ(jit.GetRegister(1), 0x44112233u); + EXPECT_EQ(jit.GetRegister(3), 0x33441122u); + EXPECT_EQ(jit.GetRegister(5), 0x22334411u); + for (u32 byte = 0x108; byte < 0x10c; ++byte) + EXPECT_EQ(jit_bus[byte], interpreter_bus[byte]); + EXPECT_EQ(jit.GetJitFallbackInstructionCount(), 0u); + EXPECT_EQ(jit.GetJitNativeExecutedInstructions(), 5u); +#endif +} + +TEST(StarletARMCore, ARMJitUsesReadFastmemForProfiledSplitSRAMMirrorPages) +{ +#if defined(_M_X86_64) + TestBus bus; + ARMCore core(bus); + bus.SetFastmemEnabled(true); + bus.SetSRAMFastmemEnabled(true); + bus.SetBoot0Mapped(false); + bus.SetSRAMSplitMode(true); + bus.WriteARM(0x00, 0xe5921000); // ldr r1, [r2] + bus.WriteARM(0x04, 0xe5943000); // ldr r3, [r4] + bus.WriteARM(0x08, 0xe5965000); // ldr r5, [r6] + bus.WriteARM(0x0c, 0xe5987000); // ldr r7, [r8] + bus.WriteARM(0x10, 0xe59a9000); // ldr r9, [r10] + bus.WriteARM(0x14, 0xeafffffe); // b 0x14 + // In split mode, aperture page 0x00 selects SRAM B while the other profiled pages select SRAM + // A. These are the read hot pages observed across BootMii, IOS reload and System Menu phases. + bus.WriteSRAM32(0x10000, 0x01234567); + bus.WriteSRAM32(0x2000, 0x89abcdef); + bus.WriteSRAM32(0x4000, 0x12345678); + bus.WriteSRAM32(0x9000, 0x55aa33cc); + bus.WriteSRAM32(0xe000, 0xc001d00d); + core.SetRegister(2, 0xfff00000); + core.SetRegister(4, 0xfff12000); + core.SetRegister(6, 0xfff14000); + core.SetRegister(8, 0xfff19000); + core.SetRegister(10, 0xfff1e000); + core.SetJitEnabled(true); + bus.ResetReadCounts(); + + EXPECT_EQ(core.RunCycles(6), 6u); + EXPECT_EQ(core.GetRegister(1), 0x01234567u); + EXPECT_EQ(core.GetRegister(3), 0x89abcdefu); + EXPECT_EQ(core.GetRegister(5), 0x12345678u); + EXPECT_EQ(core.GetRegister(7), 0x55aa33ccu); + EXPECT_EQ(core.GetRegister(9), 0xc001d00du); + // Six instruction fetches compile the block. None of the SRAM data reads may call the bus. + EXPECT_EQ(bus.GetRead32Count(), 6u); + EXPECT_EQ(core.GetJitFallbackInstructionCount(), 0u); + EXPECT_EQ(core.GetJitNativeExecutedInstructions(), 6u); + EXPECT_TRUE(bus.SRAMCanariesIntact()); +#endif +} + +TEST(StarletARMCore, ARMJitProfilesSlowSRAMPagesByDirection) +{ +#if defined(_M_X86_64) + TestBus bus; + ARMCore core(bus); + bus.SetFastmemEnabled(true); + bus.SetSRAMFastmemEnabled(true); + bus.SetBoot0Mapped(false); + bus.SetSRAMSplitMode(true); + bus.WriteARM(0x00, 0xe5921000); // ldr r1, [r2] + bus.WriteARM(0x04, 0xe5843000); // str r3, [r4] + bus.WriteARM(0x08, 0xeafffffe); // b 0x08 + // Page 0x1f is deliberately outside both direction-specific fastmem whitelists. + bus.WriteSRAM32(0xf000, 0x12345678); + core.SetRegister(2, 0xfff1f000); + core.SetRegister(3, 0x89abcdef); + core.SetRegister(4, 0xfff1f004); + core.SetJitEnabled(true); + + EXPECT_EQ(core.RunCycles(3), 3u); + EXPECT_EQ(core.GetRegister(1), 0x12345678u); + EXPECT_EQ(bus.ReadSRAM32(0xf004), 0x89abcdefu); + EXPECT_EQ(core.GetJitSlowSRAMPageReadAccessCount(0x1f), 1u); + EXPECT_EQ(core.GetJitSlowSRAMPageWriteAccessCount(0x1f), 1u); + EXPECT_EQ(core.GetJitSlowSRAMPageAccessCount(0x1f), 2u); + EXPECT_TRUE(bus.SRAMCanariesIntact()); +#endif +} + +TEST(StarletARMCore, ARMJitKeepsSplitSRAMMirrorWritesOnExactBusPath) +{ +#if defined(_M_X86_64) + TestBus bus; + ARMCore core(bus); + bus.SetFastmemEnabled(true); + bus.SetSRAMFastmemEnabled(true); + bus.SetBoot0Mapped(false); + bus.SetSRAMSplitMode(true); + bus.WriteARM(0x00, 0xe5821000); // str r1, [r2] + bus.WriteARM(0x04, 0xe5843000); // str r3, [r4] + bus.WriteARM(0x08, 0xeafffffe); // b 0x08 + core.SetRegister(1, 0x89abcdef); + core.SetRegister(2, 0xfff00000); + core.SetRegister(3, 0x12345678); + core.SetRegister(4, 0xfff19000); + core.SetJitEnabled(true); + + EXPECT_EQ(core.RunCycles(3), 3u); + EXPECT_EQ(bus.ReadSRAM32(0x10000), 0x89abcdefu); + EXPECT_EQ(bus.ReadSRAM32(0x9000), 0x12345678u); + EXPECT_EQ(core.GetJitSlowSRAMPageWriteAccessCount(0x00), 1u); + EXPECT_EQ(core.GetJitSlowSRAMPageWriteAccessCount(0x19), 1u); + EXPECT_EQ(core.GetJitFallbackInstructionCount(), 0u); + EXPECT_EQ(core.GetJitNativeExecutedInstructions(), 3u); + EXPECT_TRUE(bus.SRAMCanariesIntact()); +#endif +} + +TEST(StarletARMCore, ARMJitMapsSplitSRAMAAndBWithoutCrossingTheGap) +{ +#if defined(_M_X86_64) + TestBus bus; + ARMCore core(bus); + bus.SetFastmemEnabled(true); + bus.SetSRAMFastmemEnabled(true); + bus.SetBoot0Mapped(false); + bus.SetSRAMSplitMode(true); + bus.WriteARM(0x00, 0xe5821000); // str r1, [r2] -- high SRAM A + bus.WriteARM(0x04, 0xe5843000); // str r3, [r4] -- high SRAM B + bus.WriteARM(0x08, 0xeafffffe); // b 0x08 + core.SetRegister(1, 0x11223344); + core.SetRegister(2, 0xffff0000); + core.SetRegister(3, 0xaabbccdd); + core.SetRegister(4, 0xfff00000); + core.SetJitEnabled(true); + + EXPECT_EQ(core.RunCycles(3), 3u); + EXPECT_EQ(bus.ReadSRAM32(0x00000), 0x11223344u); + EXPECT_EQ(bus.ReadSRAM32(0x10000), 0xaabbccddu); + EXPECT_EQ(core.GetJitFallbackInstructionCount(), 0u); + EXPECT_EQ(core.GetJitNativeExecutedInstructions(), 3u); +#endif +} + +TEST(StarletARMCore, ARMJitKeepsBoot0OverlayAndInvalidSRAMGapOnTheBus) +{ +#if defined(_M_X86_64) + TestBus bus; + ARMCore core(bus); + bus.SetFastmemEnabled(true); + bus.SetSRAMFastmemEnabled(true); + bus.SetBoot0Mapped(true); + bus.SetSRAMSplitMode(false); + bus.WriteARM(0x00, 0xe5821000); // str r1, [r2] -- boot0 in non-split mode + bus.WriteARM(0x04, 0xe5843000); // str r3, [r4] -- invalid non-split tail + bus.WriteARM(0x08, 0xeafffffe); // b 0x08 + core.SetRegister(1, 0x11223344); + core.SetRegister(2, 0xffff0000); + core.SetRegister(3, 0xaabbccdd); + core.SetRegister(4, 0xfff18000); + core.SetJitEnabled(true); + + EXPECT_EQ(core.RunCycles(3), 3u); + EXPECT_EQ(core.GetJitFallbackInstructionCount(), 0u); + EXPECT_EQ(core.GetJitNativeExecutedInstructions(), 3u); +#endif +} + +TEST(StarletARMCore, ARMJitDoesNotCrossFromSRAMIntoBoot0) +{ +#if defined(_M_X86_64) + TestBus bus; + ARMCore core(bus); + bus.SetFastmemEnabled(true); + bus.SetSRAMFastmemEnabled(true); + bus.SetBoot0Mapped(true); + bus.SetSRAMSplitMode(false); + bus.WriteARM(0x00, 0xe8a2000a); // stmia r2!, {r1, r3} + bus.WriteARM(0x04, 0xeafffffe); // b 0x04 + core.SetRegister(1, 0x11223344); + core.SetRegister(2, 0xfffefffc); + core.SetRegister(3, 0xaabbccdd); + core.SetJitEnabled(true); + + EXPECT_EQ(core.RunCycles(2), 2u); + EXPECT_EQ(core.GetJitFallbackInstructionCount(), 1u); + EXPECT_EQ(core.GetJitNativeExecutedInstructions(), 1u); +#endif +} + +TEST(StarletARMCore, ARMJitCompilesPredicationShiftsHalfwordsAndRegisterLists) +{ +#if defined(_M_X86_64) + TestBus interpreter_bus(0x1000); + TestBus jit_bus(0x1000); + interpreter_bus.SetFastmemEnabled(true); + jit_bus.SetFastmemEnabled(true); + ARMCore interpreter(interpreter_bus); + ARMCore jit(jit_bus); + jit.SetJitEnabled(true); + const auto install_program = [](TestBus& bus) { + bus.WriteARM(0x00, 0xe1a04802); // mov r4, r2, lsl #16 + bus.WriteARM(0x04, 0x03a03055); // moveq r3, #0x55 + bus.WriteARM(0x08, 0x13a030aa); // movne r3, #0xaa (not executed) + bus.WriteARM(0x0c, 0xe1c052b2); // strh r5, [r0, #0x22] + bus.WriteARM(0x10, 0xe1d062b2); // ldrh r6, [r0, #0x22] + bus.WriteARM(0x14, 0xe92d4014); // push {r2, r4, lr} + bus.WriteARM(0x18, 0xe3a02000); // mov r2, #0 + bus.WriteARM(0x1c, 0xe3a04000); // mov r4, #0 + bus.WriteARM(0x20, 0xe3a0e000); // mov lr, #0 + bus.WriteARM(0x24, 0xe8bd4014); // pop {r2, r4, lr} + bus.WriteARM(0x28, 0xe1c77813); // bic r7, r7, r3, lsl r8 + bus.WriteARM(0x2c, 0xeafffffe); // b . + }; + install_program(interpreter_bus); + install_program(jit_bus); + const u32 cpsr = static_cast(ARMCore::Mode::Supervisor) | ARMCore::CPSR_Z; + interpreter.SetCPSR(cpsr); + jit.SetCPSR(cpsr); + for (ARMCore* core : {&interpreter, &jit}) + { + core->SetRegister(0, 0x200); + core->SetRegister(2, 0x1234); + core->SetRegister(5, 0xa1b2); + core->SetRegister(7, 0xffffffff); + core->SetRegister(8, 4); + core->SetRegister(13, 0x300); + core->SetRegister(14, 0x87654320); + } + + ASSERT_EQ(interpreter.RunCycles(12), 12u); + ASSERT_EQ(jit.RunCycles(12), 12u); + for (u32 reg = 0; reg < 16; ++reg) + EXPECT_EQ(jit.GetRegister(reg), interpreter.GetRegister(reg)) << "r" << reg; + EXPECT_EQ(jit.GetCPSR(), interpreter.GetCPSR()); + for (u32 byte = 0x220; byte < 0x224; ++byte) + EXPECT_EQ(jit_bus[byte], interpreter_bus[byte]); + for (u32 byte = 0x2f4; byte < 0x300; ++byte) + EXPECT_EQ(jit_bus[byte], interpreter_bus[byte]); + EXPECT_EQ(jit.GetJitFallbackInstructionCount(), 0u); + EXPECT_EQ(jit.GetJitNativeExecutedInstructions(), 12u); +#endif +} + +TEST(StarletARMCore, ARMJitRandomizedDifferential) +{ + constexpr u32 case_count = 10000; + TestBus interpreter_bus(0x40000); + TestBus jit_bus(0x40000); + ARMCore interpreter(interpreter_bus); + ARMCore jit(jit_bus); + jit.SetJitEnabled(true); + + u32 random = 0x9e3779b9; + const auto next_random = [&random] { + random ^= random << 13; + random ^= random >> 17; + random ^= random << 5; + return random; + }; + constexpr std::array opcodes = {0, 1, 2, 3, 4, 8, 9, 10, 11, 12, 13, 14, 15}; + + for (u32 test_case = 0; test_case < case_count; ++test_case) + { + const u32 address = test_case * 8; + const u32 opcode = opcodes[next_random() % opcodes.size()]; + const bool writes_result = opcode < 8 || opcode > 0xb; + const bool immediate = (next_random() & 1) != 0; + const bool set_flags = !writes_result || (next_random() & 1) != 0; + const u32 rn = next_random() & 7; + const u32 rd = next_random() & 7; + const u32 operand = immediate ? (next_random() & 0xff) : (next_random() & 7); + const u32 instruction = 0xe0000000 | (static_cast(immediate) << 25) | (opcode << 21) | + (static_cast(set_flags) << 20) | (rn << 16) | (rd << 12) | operand; + interpreter_bus.WriteARM(address, instruction); + jit_bus.WriteARM(address, instruction); + interpreter_bus.WriteARM(address + 4, 0xeafffffe); // b . + jit_bus.WriteARM(address + 4, 0xeafffffe); + + const u32 cpsr = + static_cast(ARMCore::Mode::Supervisor) | + (next_random() & (ARMCore::CPSR_N | ARMCore::CPSR_Z | ARMCore::CPSR_C | ARMCore::CPSR_V)); + interpreter.SetCPSR(cpsr); + jit.SetCPSR(cpsr); + for (u32 reg = 0; reg < 15; ++reg) + { + const u32 value = next_random(); + interpreter.SetRegister(reg, value); + jit.SetRegister(reg, value); + } + interpreter.SetRegister(15, address); + jit.SetRegister(15, address); + + SCOPED_TRACE(testing::Message() + << "case=" << test_case << " instruction=0x" << std::hex << instruction); + ASSERT_EQ(interpreter.RunCycles(2), 2u); + ASSERT_EQ(jit.RunCycles(2), 2u); + for (u32 reg = 0; reg < 16; ++reg) + ASSERT_EQ(jit.GetRegister(reg), interpreter.GetRegister(reg)) << "r" << reg; + ASSERT_EQ(jit.GetCPSR(), interpreter.GetCPSR()); + } + EXPECT_GE(jit.GetJitNativeExecutedInstructions(), case_count * 2); +} + +TEST(StarletARMCore, ThumbJitRandomizedDifferential) +{ + constexpr u32 case_count = 10000; + TestBus interpreter_bus(0x20000); + TestBus jit_bus(0x20000); + ARMCore interpreter(interpreter_bus); + ARMCore jit(jit_bus); + jit.SetJitEnabled(true); + + u32 random = 0x243f6a88; + const auto next_random = [&random] { + random ^= random << 13; + random ^= random >> 17; + random ^= random << 5; + return random; + }; + constexpr std::array alu_opcodes = {0, 1, 8, 9, 10, 11, 12, 13, 14, 15}; + + for (u32 test_case = 0; test_case < case_count; ++test_case) + { + const u32 address = test_case * 4; + u16 instruction = 0; + switch (next_random() % 5) + { + case 0: + instruction = + static_cast(0x1800 | ((next_random() & 3) << 9) | ((next_random() & 7) << 6) | + ((next_random() & 7) << 3) | (next_random() & 7)); + break; + case 1: + instruction = static_cast(0x2000 | ((next_random() & 3) << 11) | + ((next_random() & 7) << 8) | (next_random() & 0xff)); + break; + case 2: + instruction = + static_cast(0x4000 | (alu_opcodes[next_random() % alu_opcodes.size()] << 6) | + ((next_random() & 7) << 3) | (next_random() & 7)); + break; + case 3: + instruction = static_cast(0xa000 | ((next_random() & 1) << 11) | + ((next_random() & 7) << 8) | (next_random() & 0xff)); + break; + default: + instruction = static_cast(0xb000 | (next_random() & 0xff)); + break; + } + interpreter_bus.WriteThumb(address, instruction); + jit_bus.WriteThumb(address, instruction); + interpreter_bus.WriteThumb(address + 2, 0xe7fe); // b . + jit_bus.WriteThumb(address + 2, 0xe7fe); + + const u32 cpsr = + static_cast(ARMCore::Mode::Supervisor) | ARMCore::CPSR_T | + (next_random() & (ARMCore::CPSR_N | ARMCore::CPSR_Z | ARMCore::CPSR_C | ARMCore::CPSR_V)); + interpreter.SetCPSR(cpsr); + jit.SetCPSR(cpsr); + for (u32 reg = 0; reg < 15; ++reg) + { + const u32 value = next_random(); + interpreter.SetRegister(reg, value); + jit.SetRegister(reg, value); + } + interpreter.SetRegister(15, address); + jit.SetRegister(15, address); + + SCOPED_TRACE(testing::Message() + << "case=" << test_case << " instruction=0x" << std::hex << instruction); + ASSERT_EQ(interpreter.RunCycles(2), 2u); + ASSERT_EQ(jit.RunCycles(2), 2u); + for (u32 reg = 0; reg < 16; ++reg) + ASSERT_EQ(jit.GetRegister(reg), interpreter.GetRegister(reg)) << "r" << reg; + ASSERT_EQ(jit.GetCPSR(), interpreter.GetCPSR()); + } + EXPECT_GE(jit.GetJitNativeExecutedInstructions(), case_count * 2); +} + +// This benchmark is intentionally excluded from the normal test run. Invoke it explicitly when +// changing the JIT so performance measurements never turn into timing-dependent correctness tests. +TEST(StarletARMCore, DISABLED_JitThroughputBenchmark) +{ + constexpr u64 cycle_count = 8'000'000; + TestBus interpreter_bus; + TestBus jit_bus; + ARMCore interpreter(interpreter_bus); + ARMCore jit(jit_bus); + interpreter.SetJitEnabled(false); + jit.SetJitEnabled(true); + const auto install_program = [](TestBus& bus) { + bus.WriteARM(0x00, 0xe2800001); // add r0, r0, #1 + bus.WriteARM(0x04, 0xe2811003); // add r1, r1, #3 + bus.WriteARM(0x08, 0xe0222000); // eor r2, r2, r0 + bus.WriteARM(0x0c, 0xeafffffb); // b 0x00 + }; + install_program(interpreter_bus); + install_program(jit_bus); + + const auto interpreter_start = std::chrono::steady_clock::now(); + EXPECT_EQ(interpreter.RunCycles(cycle_count), cycle_count); + const auto interpreter_end = std::chrono::steady_clock::now(); + const auto jit_start = std::chrono::steady_clock::now(); + EXPECT_EQ(jit.RunCycles(cycle_count), cycle_count); + const auto jit_end = std::chrono::steady_clock::now(); + const auto interpreter_us = + std::chrono::duration_cast(interpreter_end - interpreter_start) + .count(); + const auto jit_us = + std::chrono::duration_cast(jit_end - jit_start).count(); + + EXPECT_EQ(jit.GetRegister(0), interpreter.GetRegister(0)); + EXPECT_EQ(jit.GetRegister(1), interpreter.GetRegister(1)); + EXPECT_EQ(jit.GetRegister(2), interpreter.GetRegister(2)); + EXPECT_GT(jit.GetJitNativeExecutedInstructions(), cycle_count / 2); + std::cout << "Starlet ARM interpreter: " << interpreter_us << " us, JIT: " << jit_us + << " us, speedup: " << static_cast(interpreter_us) / jit_us << "x\n"; +} + TEST(StarletARMCore, MultiplyAndLongMultiply) { TestBus bus; @@ -798,6 +2042,72 @@ TEST(StarletARMCore, RunCyclesStopsAtBudget) 2u); // Both words stay resident across loop iterations. } +TEST(StarletARMCore, ARMJitFastForwardsSliceStableHollywoodTimerPoll) +{ + TestBus bus; + bus.SetFastmemEnabled(true); + bus.SetSliceStablePollAddress(0x100); + ARMCore core(bus); + core.SetJitEnabled(true); + bus.WriteARM(0x00, 0xe5932000); // ldr r2, [r3] + bus.WriteARM(0x04, 0xea000000); // b 0x0c + bus.WriteARM(0x08, 0xe3a03000); // entry-only base setup, skipped by the polling loop + bus.WriteARM(0x0c, 0xe1520000); // cmp r2, r0 + bus.WriteARM(0x10, 0x3afffffa); // bcc 0x00 + bus.WriteARM(0x14, 0xeafffffe); // b . + core.SetRegister(0, 10); + core.SetRegister(3, 0x100); + bus.Write32(0x100, 5); + + EXPECT_EQ(core.RunCycles(4096), 4096u); + EXPECT_EQ(core.GetExecutedInstructions(), 4u); + EXPECT_EQ(core.GetMemoryPollEntryCount(), 1u); + EXPECT_EQ(bus.GetCycles(), 4096u); + + bus.Write32(0x100, 12); + EXPECT_EQ(core.RunCycles(4), 4u); + EXPECT_EQ(core.GetRegister(15), 0x14u); + EXPECT_EQ(core.GetMemoryPollEntryCount(), 1u); +} + +TEST(StarletARMCore, ARMJitKeepsNonblockingReceiveMessagePollGuestVisible) +{ + const auto install_program = [](TestBus& bus, bool non_blocking) { + bus.WriteARM(0x00, 0xe5960000); // ldr r0, [r6] + bus.WriteARM(0x04, 0xe1a01008); // mov r1, r8 + bus.WriteARM(0x08, non_blocking ? 0xe3a02000 : 0xe3a02001); // mov r2, #block + bus.WriteARM(0x0c, 0xeb00000b); // bl 0x40 + bus.WriteARM(0x10, 0xe2505000); // subs r5, r0, #0 + bus.WriteARM(0x14, 0x1afffff9); // bne 0x00 + bus.WriteARM(0x40, 0xe60001d0); // IOS ReceiveMessage syscall 0x0e + bus.WriteARM(0x44, 0xe12fff1e); // bx lr + }; + + TestBus bus; + install_program(bus, true); + ARMCore core(bus); + core.SetJitEnabled(true); + core.SetRegister(0, 0xfffffffa); // Empty queue error returned by the completed syscall. + core.SetRegister(15, 0x10); + + // The kernel syscall must remain guest-visible on every iteration. IOS can schedule another + // Starlet thread while servicing it even when no hardware IRQ/FIQ is asserted, so treating this + // as a hardware wait can deadlock the Broadway side of IOS IPC. + EXPECT_EQ(core.RunCycles(14), 14u); + EXPECT_EQ(core.GetExecutedInstructions(), 14u); + EXPECT_FALSE(core.IsWaitingForExternalEvent()); + EXPECT_EQ(bus.GetCycles(), 14u); + + TestBus blocking_bus; + install_program(blocking_bus, false); + ARMCore blocking_core(blocking_bus); + blocking_core.SetJitEnabled(true); + blocking_core.SetRegister(0, 0xfffffffa); + blocking_core.SetRegister(15, 0x10); + EXPECT_EQ(blocking_core.RunCycles(2), 2u); + EXPECT_FALSE(blocking_core.IsWaitingForExternalEvent()); +} + TEST(StarletARMCore, ThumbMemoryPollingFastForwardsSafeRAMUntilItChanges) { TestBus bus; @@ -820,6 +2130,32 @@ TEST(StarletARMCore, ThumbMemoryPollingFastForwardsSafeRAMUntilItChanges) EXPECT_FALSE(core.IsWaitingForExternalEvent()); } +TEST(StarletARMCore, ThumbJitFastmemPreservesMemoryPollingFastForward) +{ + TestBus bus; + bus.SetFastmemEnabled(true); + ARMCore core(bus); + core.SetJitEnabled(true); + bus.WriteThumb(0x00, 0x6823); // ldr r3, [r4] + bus.WriteThumb(0x02, 0x2b00); // cmp r3, #0 + bus.WriteThumb(0x04, 0xd0fc); // beq 0x00 + core.SetCPSR(static_cast(ARMCore::Mode::Supervisor) | ARMCore::CPSR_T); + core.SetRegister(4, 0x100); + + EXPECT_EQ(core.RunCycles(10000), 10000u); + EXPECT_EQ(core.GetExecutedInstructions(), 3u); + EXPECT_EQ(core.GetMemoryPollEntryCount(), 1u); + EXPECT_EQ(core.GetRegister(3), 0u); + EXPECT_EQ(core.GetRegister(15), 0u); + EXPECT_NE(core.GetCPSR() & ARMCore::CPSR_Z, 0u); + EXPECT_TRUE(core.IsWaitingForExternalEvent()); + + bus.Write32(0x100, 1); + EXPECT_EQ(core.RunCycles(2), 2u); + EXPECT_EQ(core.GetRegister(3), 1u); + EXPECT_FALSE(core.IsWaitingForExternalEvent()); +} + TEST(StarletARMCore, ThumbMemoryPollingDoesNotSkipMMIO) { TestBus bus; @@ -872,5 +2208,31 @@ TEST(StarletARMCore, MMUSectionTranslation) // translation cache. EXPECT_EQ(bus.GetRead32Count(), 3u); } + +TEST(StarletARMCore, JitCachesProvisionalIdentityTranslationForFaultPages) +{ +#if defined(_M_X86_64) + TestBus bus(0x10000); + bus.SetFastmemEnabled(true); + ARMCore core(bus); + bus.WriteARM(0x0000, 0xe5921000); // ldr r1, [r2] + bus.WriteARM(0x0004, 0xe5923000); // ldr r3, [r2] + bus.WriteARM(0x0008, 0xeafffffe); // b 0x08 + bus.WriteARM(0x1400, 0x12345678); + // The zero first-level descriptors intentionally select the current provisional identity + // behavior. The first data access refills the software TLB without terminating the native + // block; the second must remain in fastmem as well. + core.GetCP15State().translation_table_base = 0x4000; + core.GetCP15State().control |= 1; + core.SetRegister(2, 0x1400); + core.SetJitEnabled(true); + + EXPECT_EQ(core.RunCycles(3), 3u); + EXPECT_EQ(core.GetRegister(1), 0x12345678u); + EXPECT_EQ(core.GetRegister(3), 0x12345678u); + EXPECT_EQ(core.GetJitFallbackInstructionCount(), 0u); + EXPECT_EQ(core.GetJitNativeExecutedInstructions(), 3u); +#endif +} } // namespace } // namespace IOS::LLE diff --git a/docs/Wii_IOS_LLE.md b/docs/Wii_IOS_LLE.md index b28aedb617..c0b94f4848 100644 --- a/docs/Wii_IOS_LLE.md +++ b/docs/Wii_IOS_LLE.md @@ -63,8 +63,8 @@ Dolphin CoreTiming (Broadway clock domain, 729 MHz) +-- WFI idle: 72,900 Broadway cycles --> 24,300 Starlet cycles | v - ARMv5TE interpreter - + software TLB/I-cache + ARMv5TE interpreter or x64 JIT + + software TLB/I-cache | +-------------------------------+------------------------------+ | | | @@ -118,6 +118,13 @@ X2 and immediately submit the next request with X1 before Starlet is scheduled a timer, NAND, OHCI, and Wiimote clocks once per 4,096-cycle scheduler slice instead of once per interpreted instruction; external IRQ delivery remains bounded to about 16.9 microseconds and WFI polling to 100 microseconds. +- An x86-64 Starlet JIT translates the observed ARM and Thumb integer, branch, interworking and + memory-transfer subset. Its inline generation-tagged TLB and direct fastmem paths cover ordinary + MEM1/MEM2 accesses plus a measured, direction-specific subset of single SRAM reads. Every SRAM + write, register-list transfer, TLB miss, MMIO access, protected boot0 overlay, invalid SRAM + aperture or unsupported instruction remains an architectural side exit: registers are flushed, + the exact interpreter/device operation runs, and the dispatcher re-samples IRQ/FIQ, IPC yield, + CP15 and translation state before another native block executes. - Big-endian Starlet address space, 96 KiB of physical SRAM (64 KiB bank A plus 32 KiB bank B) exposed through the hardware's unusual 128 KiB windows, plus shared MEM1/MEM2 access. - Raw NAND reads, chip identification/status, Wii ECC generation, ECC-enabled page programming @@ -135,7 +142,9 @@ X2 and immediately submit the next request with X1 before Starlet is scheduled a read, write-enable/disable, word write/erase, and whole-array write/erase commands in COW memory. - Hollywood's Starlet GPIO bank, including enable/output/direction/input, ownership, straps, interrupt level/mask, and write-one-to-clear interrupt flags. POWER and EJECT have their idle - levels and the EEPROM MISO pin remains connected to the existing serial model. + levels and the EEPROM MISO pin remains connected to the existing serial model. Writes to the + public BootMii diagnostic byte on GPIO bits 23:16 are logged as stage codes with the Starlet PC; + this observes the firmware's own progress/panic channel without modifying its control flow. - Immediate AHM memory-flush acknowledgement, the indirect DDR/SEQ/BIST register banks used by boot1 training, and the hardware-controlled boot0 ROM overlay/SRAM-bank swap. - PPC/ARM IPC mailboxes, Starlet-side access to both control registers, and Broadway @@ -305,6 +314,18 @@ An isolated boot probe using the local, mutually matching dumps has executed thi `IPC_BOOT2_RUN(1, 2)` immediately. Cold-boot BootMii and HBC -> IOS254 -> MINI -> BootMii both remain in the original interactive menu, and keyboard-backed GameCube navigation was validated without modifying `armboot.bin`, `ppcboot.elf`, or `bootmii.ini`. +22. Directional JIT profiling separated slow SRAM reads from writes and counted them per 4 KiB + aperture page. During the IOS-to-Menu phase, about 71% of slow SRAM accesses were writes; + pages `0x00` and `0x19` dominated those writes, while page `0x1e` dominated the remaining + reads. Single reads from the previously validated `0x00`/`0x12`/`0x14`/`0x19` pages and the + newly measured `0x1e` page now use guarded split-bank fastmem. Direct writes to `0x00`/`0x19` + passed synthetic bounds and canary tests but reproducibly stopped BootMii at its public + `hardware setup begin` GPIO marker, so every SRAM write was restored to the exact bus path. + The resulting build passed all 61 targeted Starlet/JIT tests and a fresh cold-boot regression: + BootMii completed hardware setup, the Wii selection re-entered NAND boot2, original IOS released + Broadway 54.1 seconds later, and the populated System Menu rendered without a crash or firmware + patch. The still-dominant exact writes are a measured coherency problem, not permission to skip + their bus semantics. The probe never prints ROM, NAND, key, or firmware instruction bytes. The committed unit suite covers ARM-to-Thumb loads into PC, high Starlet exception vectors, privileged `LDM ... ^` user-bank @@ -354,10 +375,12 @@ The next firmware stages need substantially more hardware fidelity: enumeration, Bluetooth HCI/ACL, and one paired remote's L2CAP/HID input path are implemented. - Remaining IOS reload/reset edge cases, device timing, and scheduler accuracy needed by timing-sensitive original exploits. -- Performance: the Menu's original IOS idle path now runs at full speed through safe RAM/SRAM poll - fast-forwarding. ARM-heavy boot and transient driver workloads still use the interpreter and can - take longer than real hardware; a block cache or ARM JIT remains the next performance frontier - for sustained workloads that do not enter the scheduler's idle loop. +- Performance: the Menu's original IOS idle path runs at full speed through safe RAM/SRAM poll + fast-forwarding, and the x64 Starlet JIT now executes most ordinary ARM/Thumb instructions + natively. Boot and transient driver workloads still cross many architectural side exits for + MMIO, CP15 operations and the not-yet-native instruction subset. Those exits must be reduced by + proven instruction translations or register-specific MMIO stubs, never by treating every device + access as a resumable C helper. Until those items are implemented, this is an end-to-end experimental LLE implementation rather than a drop-in replacement for Dolphin's mature IOS HLE mode. @@ -389,11 +412,80 @@ than a drop-in replacement for Dolphin's mature IOS HLE mode. - [ARM926EJ-S Technical Reference Manual](https://developer.arm.com/documentation/ddi0198/latest/) for ARMv5TE, CP15, banked-register, exception, and interworking semantics. +## JIT correctness and BootMii performance model + +BootMii is two distinct Starlet programs before its Broadway UI appears. The boot2-style loader +performs Hollywood setup, mounts the external SD controller and loads `/bootmii/armboot.bin`. +MINI then configures the ARM926 MMU/caches and IRQs, initializes NAND, IPC and SDHC, loads +`/bootmii/ppcboot.elf`, writes the Broadway EXI reset stub, releases the reset lines and finally +sleeps in its IPC loop. MINI has no Wi-Fi stack. Consequently, Wi-Fi SDIO traffic observed later +belongs to Nintendo IOS/System Menu startup and is not a valid optimization target for a black +screen inside BootMii. + +The public loader also gives deterministic stage telemetry. `debug_output()` writes one byte to +GPIO bits 23:16: `42/43/44` cover entry, stack and BSS setup; `F0/F1` enter and finish loader setup; +`F2/F3` select and hand off the SD payload; `C8/C9` fall back and hand off to NAND boot2; `E3/E4` +are ELF-loader panics. A panic then alternates its error byte with zero around repeated 500 ms +timer delays. Therefore a trace fixed at the loader's `udelay` loop for millions of scheduler +slices is a panic, not evidence that SDHC or the GUI merely needs more time. + +The JIT follows the same invariant used by Dolphin's Broadway JIT and QEMU TCG: a slow access can +resume inside a translated block only when all guest architectural state at that instruction is +recoverable and the helper cannot change scheduling, interrupt, translation or code-cache state. +Dolphin flushes registers before its safe slow loads/stores and checks memory exceptions; QEMU +records a host-PC to guest-PC/state map so faults restore the precise instruction boundary. The +Starlet JIT does not yet have that per-instruction recovery metadata. Generic MMIO therefore exits +to the dispatcher. Future performance work can specialize proven side-effect-free registers +(for example read-only status or timer reads), while complex writes, IRQ acknowledgements, IPC and +reset transitions must remain exits. + +The ARM926 manual is also explicit about the boundaries relevant here: wait-for-interrupt drains +the write buffer and sleeps until IRQ/FIQ/debug; disabling and re-enabling the MMU preserves TLB +contents; D-cache clean, write-buffer drain, I-cache invalidation and TLB invalidation are separate +CP15 operations. Collapsing all `c7` maintenance into a code-cache flush is both inaccurate and +slow, while ignoring the instruction-cache operations breaks self-modifying boot code. + +### IOS syscall execution and evaluated acceleration strategies + +IOS has two superficially similar software-exception ABIs. Normal kernel calls are deliberately +undefined ARM words of the form `0xE6000010 | (syscall_number << 5)`. The Undefined vector saves +the complete thread context, extracts bits 12:5, switches to System mode and dispatches through the +IOS-version-specific syscall table. ARM/Thumb `SVC 0xAB` is a separate RealView semihosting ABI; +production IOS retains essentially only debug-string output (`r0 = 4`, string in `r1`). Replacing +either path with host-side IOS HLE would skip the original scheduler, message queues, permission +checks and exploit-relevant kernel behavior, so LLE keeps the guest handlers. The JIT only enters +the architecturally correct Undefined/Supervisor exception directly, avoiding a redundant generic +instruction-decoder fallback. + +The acceleration options were evaluated as follows: + +| Method | Expected value here | Decision | +|---|---|---| +| Per-instruction interpreter | Reference accuracy, very low throughput | Keep as the exact fallback/oracle only | +| IOS syscall HLE | Very fast for ordinary titles | Reject for original IOS and exploit compatibility | +| QEMU TCG/Unicorn ARM core | Mature ARM system translation | Valuable reference, but integrating Dolphin memory, Hollywood MMIO, CoreTiming, precise cache invalidation and dual-CPU IPC would duplicate most of the current machine model | +| LLVM/whole-function recompilation | Strong global optimization | Excessive compile latency and difficult precise MMIO/exception recovery during boot and self-modifying code | +| Custom basic-block JIT | Shares Dolphin memory and timing directly | Selected; continue with measured native coverage and exact side exits | +| Native ARM predication | Removes common IOS conditional-ALU fallbacks | Implemented for every already-validated data-processing form | +| Native Undefined/SVC entry | Removes decoder fallback at each software exception | Implemented without bypassing the guest kernel handler | +| Direct block chaining | Removes dispatcher lookup on hot edges | Next high-value CPU optimization, provided TLB/I-cache invalidation unlinks every affected edge | +| Wider register allocation / traces | Reduces repeated guest-register loads and stores | Medium-term; add only after branch/memory profiles identify stable hot traces | +| Register-specific MMIO fast paths | Can remove very hot safe status reads | Only after sampled effective-address traces; generic MMIO continuation is forbidden | + +This ordering follows the same split documented by +[QEMU TCG](https://github.com/qemu/qemu/blob/master/docs/devel/tcg.rst): RAM/ROM accesses use cached +host offsets, MMIO calls device code, and direct block chains must be removable when translated +pages change. The current microbenchmark reaches roughly two billion simple translated ARM +instructions per host second, while complete IOS traces are orders of magnitude slower. The raw +x86 emitter is therefore not the limiting component; remaining work is native coverage, side-exit +frequency, guest scheduling/idle detection and device-access cost. + ## Code map | Area | Files | |---|---| | ARM CPU | `Core/IOS/Starlet/ARMCore.{h,cpp}` | +| x86-64 ARM/Thumb JIT | `Core/IOS/Starlet/ARMJitX64.{h,cpp}` | | Starlet scheduler/lifetime | `Core/IOS/Starlet/Starlet.{h,cpp}` | | Address space and devices | `Core/IOS/Starlet/StarletMemory.{h,cpp}` | | Bluetooth pairing and Wii Remote HID | `Core/IOS/USB/Bluetooth/{BTBase,WiimoteDevice}.{h,cpp}` |