Yaya48/QEMU-S5L8950X
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
main
QEMU-S5L8950X/hw/i386/amd_iommu.c
T
Yaya48 cf256aa081 Import QEMU upstream snapshot d2e570c 
Upstream: https://gitlab.com/qemu-project/qemu.git

Upstream-Commit: d2e570cc0f97b936902a5b1b86b73c0f5998b475
2026-08-31 02:15:30 +02:00

2838 lines
91 KiB
C
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
/*
* QEMU emulation of AMD IOMMU (AMD-Vi)
*
* Copyright (C) 2011 Eduard - Gabriel Munteanu
* Copyright (C) 2015, 2016 David Kiarie Kahurani
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
* You should have received a copy of the GNU General Public License along
* with this program; if not, see <http://www.gnu.org/licenses/>.
*
* Cache implementation inspired by hw/i386/intel_iommu.c
*/
#include "qemu/osdep.h"
#include "hw/i386/pc.h"
#include "hw/pci/msi.h"
#include "hw/pci/pci_bus.h"
#include "migration/vmstate.h"
#include "amd_iommu.h"
#include "qapi/error.h"
#include "qemu/error-report.h"
#include "hw/i386/apic_internal.h"
#include "trace.h"
#include "hw/i386/apic-msidef.h"
#include "hw/core/qdev-properties.h"
#include "kvm/kvm_i386.h"
#include "qemu/iova-tree.h"
#include "hw/core/registerfields.h"
struct AMDVIAddressSpace {
PCIBus *bus; /* PCIBus (for bus number) */
uint8_t devfn; /* device function */
AMDVIState *iommu_state; /* AMDVI - one per machine */
MemoryRegion root; /* AMDVI Root memory map region */
IOMMUMemoryRegion iommu; /* Device's address translation region */
MemoryRegion iommu_nodma; /* Alias of shared nodma memory region */
MemoryRegion iommu_ir; /* Device's interrupt remapping region */
AddressSpace as; /* device's corresponding address space */
/* DMA address translation support */
IOMMUNotifierFlag notifier_flags;
/* entry in list of Address spaces with registered notifiers */
QLIST_ENTRY(AMDVIAddressSpace) next;
/* Record DMA translation ranges */
IOVATree *iova_tree;
/* DMA address translation active */
bool addr_translation;
};
/* AMDVI cache entry */
typedef struct AMDVIIOTLBEntry {
uint16_t domid; /* assigned domain id */
uint16_t devid; /* device owning entry */
uint64_t perms; /* access permissions */
uint64_t translated_addr; /* translated address */
uint64_t page_mask; /* physical page size */
} AMDVIIOTLBEntry;
/*
* These 'fault' reasons have an overloaded meaning since they are not only
* intended for describing reasons that generate an IO_PAGE_FAULT as per the AMD
* IOMMU specification, but are also used to signal internal errors in the
* emulation code.
*/
typedef enum AMDVIFaultReason {
AMDVI_FR_DTE_RTR_ERR = 1, /* Failure to retrieve DTE */
AMDVI_FR_DTE_V, /* DTE[V] = 0 */
AMDVI_FR_DTE_TV, /* DTE[TV] = 0 */
AMDVI_FR_PT_ROOT_INV, /* Page Table Root ptr invalid */
AMDVI_FR_PT_ENTRY_INV, /* Failure to read PTE from guest memory */
} AMDVIFaultReason;
typedef struct AMDVIAsKey {
PCIBus *bus;
uint8_t devfn;
} AMDVIAsKey;
typedef struct AMDVIIOTLBKey {
uint64_t gfn;
uint16_t devid;
} AMDVIIOTLBKey;
typedef struct AMDVIIrteGA {
uint64_t ga_lo;
uint64_t ga_hi;
} AMDVIIrteGA;
/* XT IOMMU General Interrupt Control Register layout */
FIELD(AMDVI_XT_GEN_INTR, DEST_MODE, 2, 1)
FIELD(AMDVI_XT_GEN_INTR, DEST_LO, 8, 24)
FIELD(AMDVI_XT_GEN_INTR, VECTOR, 32, 8)
FIELD(AMDVI_XT_GEN_INTR, DELIVERY_MODE, 40, 1)
FIELD(AMDVI_XT_GEN_INTR, DEST_HI, 56, 8)
/* Interrupt Remapping Table Fields Formats */
/* Basic 32-bit IRTE layout (GAEn=0) */
FIELD(AMDVI_IRTE, VALID, 0, 1)
FIELD(AMDVI_IRTE, SUP_IOPF, 1, 1)
FIELD(AMDVI_IRTE, INT_TYPE, 2, 3)
FIELD(AMDVI_IRTE, RQ_EOI, 5, 1)
FIELD(AMDVI_IRTE, DM, 6, 1)
FIELD(AMDVI_IRTE, GUEST_MODE, 7, 1)
FIELD(AMDVI_IRTE, DESTINATION, 8, 8)
FIELD(AMDVI_IRTE, VECTOR, 16, 8)
/* 128-bit IRTE layout (GAEn=1) */
FIELD(AMDVI_IRTE_GA_LO, VALID, 0, 1)
FIELD(AMDVI_IRTE_GA_LO, SUP_IOPF, 1, 1)
FIELD(AMDVI_IRTE_GA_LO, INT_TYPE, 2, 3)
FIELD(AMDVI_IRTE_GA_LO, RQ_EOI, 5, 1)
FIELD(AMDVI_IRTE_GA_LO, DM, 6, 1)
FIELD(AMDVI_IRTE_GA_LO, GUEST_MODE, 7, 1)
/*
* In the 128-bit IRTE format, XT mode uses IRTE_GA_LOW.Destination[23:0]
* together with IRTE_GA_HI.DestinationHi[7:0] to construct a 32-bit x2APIC
* destination.
* Without XTEn (i.e. when x2APIC support is not enabled), only
* IRTE_GA_LOW.Destination[7:0] is used.
*/
FIELD(AMDVI_IRTE_GA_LO, DESTINATION, 8, 24)
FIELD(AMDVI_IRTE_GA_HI, VECTOR, 0, 8)
FIELD(AMDVI_IRTE_GA_HI, DESTINATION_HI, 56, 8)
uint64_t amdvi_extended_feature_register(AMDVIState *s)
{
uint64_t feature = AMDVI_DEFAULT_EXT_FEATURES;
if (s->xtsup) {
feature |= AMDVI_FEATURE_XT;
}
if (!s->iommu.dma_translation) {
feature |= AMDVI_HATS_MODE_RESERVED;
}
return feature;
}
/* configure MMIO registers at startup/reset */
static void amdvi_set_quad(AMDVIState *s, hwaddr addr, uint64_t val,
uint64_t romask, uint64_t w1cmask)
{
stq_le_p(&s->mmior[addr], val);
stq_le_p(&s->romask[addr], romask);
stq_le_p(&s->w1cmask[addr], w1cmask);
}
static uint16_t amdvi_readw(AMDVIState *s, hwaddr addr)
{
return lduw_le_p(&s->mmior[addr]);
}
static uint32_t amdvi_readl(AMDVIState *s, hwaddr addr)
{
return ldl_le_p(&s->mmior[addr]);
}
static uint64_t amdvi_readq(AMDVIState *s, hwaddr addr)
{
return ldq_le_p(&s->mmior[addr]);
}
/* internal write */
static void amdvi_writeq_raw(AMDVIState *s, hwaddr addr, uint64_t val)
{
stq_le_p(&s->mmior[addr], val);
}
/* external write */
static void amdvi_writew(AMDVIState *s, hwaddr addr, uint16_t val)
{
uint16_t romask = lduw_le_p(&s->romask[addr]);
uint16_t w1cmask = lduw_le_p(&s->w1cmask[addr]);
uint16_t oldval = lduw_le_p(&s->mmior[addr]);
uint16_t oldval_preserved = oldval & (romask | w1cmask);
uint16_t newval_write = val & ~romask;
uint16_t newval_w1c_set = val & w1cmask;
stw_le_p(&s->mmior[addr],
(oldval_preserved | newval_write) & ~newval_w1c_set);
}
static void amdvi_writel(AMDVIState *s, hwaddr addr, uint32_t val)
{
uint32_t romask = ldl_le_p(&s->romask[addr]);
uint32_t w1cmask = ldl_le_p(&s->w1cmask[addr]);
uint32_t oldval = ldl_le_p(&s->mmior[addr]);
uint32_t oldval_preserved = oldval & (romask | w1cmask);
uint32_t newval_write = val & ~romask;
uint32_t newval_w1c_set = val & w1cmask;
stl_le_p(&s->mmior[addr],
(oldval_preserved | newval_write) & ~newval_w1c_set);
}
static void amdvi_writeq(AMDVIState *s, hwaddr addr, uint64_t val)
{
uint64_t romask = ldq_le_p(&s->romask[addr]);
uint64_t w1cmask = ldq_le_p(&s->w1cmask[addr]);
uint64_t oldval = ldq_le_p(&s->mmior[addr]);
uint64_t oldval_preserved = oldval & (romask | w1cmask);
uint64_t newval_write = val & ~romask;
uint64_t newval_w1c_set = val & w1cmask;
stq_le_p(&s->mmior[addr],
(oldval_preserved | newval_write) & ~newval_w1c_set);
}
/* AND a 64-bit register with a 64-bit value */
static bool amdvi_test_mask(AMDVIState *s, hwaddr addr, uint64_t val)
{
return amdvi_readq(s, addr) & val;
}
/* OR a 64-bit register with a 64-bit value storing result in the register */
static void amdvi_assign_orq(AMDVIState *s, hwaddr addr, uint64_t val)
{
amdvi_writeq_raw(s, addr, amdvi_readq(s, addr) | val);
}
/* AND a 64-bit register with a 64-bit value storing result in the register */
static void amdvi_assign_andq(AMDVIState *s, hwaddr addr, uint64_t val)
{
amdvi_writeq_raw(s, addr, amdvi_readq(s, addr) & val);
}
static void amdvi_build_xt_msi_msg(AMDVIState *s, MSIMessage *msg)
{
uint64_t xt_reg = amdvi_readq(s, AMDVI_MMIO_XT_GEN_INTR);
X86IOMMUIrq irq = {
.vector = FIELD_EX64(xt_reg, AMDVI_XT_GEN_INTR, VECTOR),
.delivery_mode = FIELD_EX64(xt_reg, AMDVI_XT_GEN_INTR, DELIVERY_MODE),
.dest_mode = FIELD_EX64(xt_reg, AMDVI_XT_GEN_INTR, DEST_MODE),
.dest = (FIELD_EX64(xt_reg, AMDVI_XT_GEN_INTR, DEST_HI) << 24) |
FIELD_EX64(xt_reg, AMDVI_XT_GEN_INTR, DEST_LO),
.trigger_mode = 0,
.redir_hint = 0,
};
x86_iommu_irq_to_msi_message(&irq, msg);
}
static void amdvi_generate_msi_interrupt(AMDVIState *s)
{
MSIMessage msg = {};
if (s->intcapxten) {
trace_amdvi_generate_msi_interrupt("XT GEN");
amdvi_build_xt_msi_msg(s, &msg);
} else if (msi_enabled(&s->pci->dev)) {
trace_amdvi_generate_msi_interrupt("MSI");
msg = msi_get_message(&s->pci->dev, 0);
} else {
trace_amdvi_generate_msi_interrupt("NO MSI");
return;
}
apic_get_class(NULL)->send_msi(&msg);
}
static uint32_t get_next_eventlog_entry(AMDVIState *s)
{
uint32_t evtlog_size = s->evtlog_len * AMDVI_EVENT_LEN;
return (s->evtlog_tail + AMDVI_EVENT_LEN) % evtlog_size;
}
static void amdvi_log_event(AMDVIState *s, uint64_t *evt)
{
uint64_t le_evt[2];
uint32_t evtlog_tail_next;
/* event logging not enabled */
if (!s->evtlog_enabled || amdvi_test_mask(s, AMDVI_MMIO_STATUS,
AMDVI_MMIO_STATUS_EVT_OVF)) {
return;
}
evtlog_tail_next = get_next_eventlog_entry(s);
/* event log buffer full */
if (evtlog_tail_next == s->evtlog_head) {
/* generate overflow interrupt */
if (s->evtlog_intr) {
amdvi_assign_orq(s, AMDVI_MMIO_STATUS, AMDVI_MMIO_STATUS_EVT_OVF);
amdvi_generate_msi_interrupt(s);
}
return;
}
/*
* Convert event buffer to little-endian before writing it to guest memory.
*/
le_evt[0] = cpu_to_le64(evt[0]);
le_evt[1] = cpu_to_le64(evt[1]);
if (dma_memory_write(&address_space_memory, s->evtlog + s->evtlog_tail,
le_evt, AMDVI_EVENT_LEN, MEMTXATTRS_UNSPECIFIED)) {
trace_amdvi_evntlog_fail(s->evtlog, s->evtlog_tail);
}
s->evtlog_tail = evtlog_tail_next;
amdvi_writeq_raw(s, AMDVI_MMIO_EVENT_TAIL, s->evtlog_tail);
if (s->evtlog_intr) {
amdvi_assign_orq(s, AMDVI_MMIO_STATUS, AMDVI_MMIO_STATUS_EVENT_INT);
amdvi_generate_msi_interrupt(s);
}
}
static void amdvi_setevent_bits(uint64_t *buffer, uint64_t value, int start,
int length)
{
int index = start / 64, bitpos = start % 64;
uint64_t mask = MAKE_64BIT_MASK(bitpos, length);
buffer[index] &= ~mask;
buffer[index] |= (value << bitpos) & mask;
}
/*
* AMDVi event structure
* 0:15 -> DeviceID
* 48:63 -> event type + miscellaneous info
* 64:127 -> related address
*/
static void amdvi_encode_event(uint64_t *evt, uint16_t devid, uint64_t addr,
uint16_t info)
{
evt[0] = 0;
evt[1] = 0;
amdvi_setevent_bits(evt, devid, 0, 16);
amdvi_setevent_bits(evt, info, 48, 16);
amdvi_setevent_bits(evt, addr, 64, 64);
}
/* log an error encountered during a page walk
*
* @addr: virtual address in translation request
*/
static void amdvi_page_fault(AMDVIState *s, uint16_t devid,
hwaddr addr, uint16_t info)
{
uint64_t evt[2];
info |= AMDVI_EVENT_IOPF_I | AMDVI_EVENT_IOPF;
amdvi_encode_event(evt, devid, addr, info);
amdvi_log_event(s, evt);
pci_word_test_and_set_mask(s->pci->dev.config + PCI_STATUS,
PCI_STATUS_SIG_TARGET_ABORT);
}
/*
* log a master abort accessing device table
* @devtab : address of device table entry
* @info : error flags
*/
static void amdvi_log_devtab_error(AMDVIState *s, uint16_t devid,
hwaddr devtab, uint16_t info)
{
uint64_t evt[2];
info |= AMDVI_EVENT_DEV_TAB_HW_ERROR;
amdvi_encode_event(evt, devid, devtab, info);
amdvi_log_event(s, evt);
pci_word_test_and_set_mask(s->pci->dev.config + PCI_STATUS,
PCI_STATUS_SIG_TARGET_ABORT);
}
/* log an event trying to access command buffer
* @addr : address that couldn't be accessed
*/
static void amdvi_log_command_error(AMDVIState *s, hwaddr addr)
{
uint64_t evt[2];
uint16_t info = AMDVI_EVENT_COMMAND_HW_ERROR;
amdvi_encode_event(evt, 0, addr, info);
amdvi_log_event(s, evt);
pci_word_test_and_set_mask(s->pci->dev.config + PCI_STATUS,
PCI_STATUS_SIG_TARGET_ABORT);
}
/* log an illegal command event
* @addr : address of illegal command
*/
static void amdvi_log_illegalcom_error(AMDVIState *s, uint16_t info,
hwaddr addr)
{
uint64_t evt[2];
info |= AMDVI_EVENT_ILLEGAL_COMMAND_ERROR;
amdvi_encode_event(evt, 0, addr, info);
amdvi_log_event(s, evt);
}
/* log an error accessing device table
*
* @devid : device owning the table entry
* @devtab : address of device table entry
* @info : error flags
*/
static void amdvi_log_illegaldevtab_error(AMDVIState *s, uint16_t devid,
hwaddr addr, uint16_t info)
{
uint64_t evt[2];
info |= AMDVI_EVENT_ILLEGAL_DEVTAB_ENTRY;
amdvi_encode_event(evt, devid, addr, info);
amdvi_log_event(s, evt);
}
/* log an error accessing a PTE entry
* @addr : address that couldn't be accessed
*/
static void amdvi_log_pagetab_error(AMDVIState *s, uint16_t devid,
hwaddr addr, uint16_t info)
{
uint64_t evt[2];
info |= AMDVI_EVENT_PAGE_TAB_HW_ERROR;
amdvi_encode_event(evt, devid, addr, info);
amdvi_log_event(s, evt);
pci_word_test_and_set_mask(s->pci->dev.config + PCI_STATUS,
PCI_STATUS_SIG_TARGET_ABORT);
}
static gboolean amdvi_as_equal(gconstpointer v1, gconstpointer v2)
{
const AMDVIAsKey *key1 = v1;
const AMDVIAsKey *key2 = v2;
return key1->bus == key2->bus && key1->devfn == key2->devfn;
}
static guint amdvi_as_hash(gconstpointer v)
{
const AMDVIAsKey *key = v;
guint bus = (guint)(uintptr_t)key->bus;
return (guint)(bus << 8 | (guint)key->devfn);
}
static AMDVIAddressSpace *amdvi_as_lookup(AMDVIState *s, PCIBus *bus,
uint8_t devfn)
{
const AMDVIAsKey key = { .bus = bus, .devfn = devfn };
return g_hash_table_lookup(s->address_spaces, &key);
}
static gboolean amdvi_find_as_by_devid(gpointer key, gpointer value,
gpointer user_data)
{
const AMDVIAsKey *as = key;
const uint16_t *devidp = user_data;
return *devidp == PCI_BUILD_BDF(pci_bus_num(as->bus), as->devfn);
}
static AMDVIAddressSpace *amdvi_get_as_by_devid(AMDVIState *s, uint16_t devid)
{
return g_hash_table_find(s->address_spaces,
amdvi_find_as_by_devid, &devid);
}
static gboolean amdvi_iotlb_equal(gconstpointer v1, gconstpointer v2)
{
const AMDVIIOTLBKey *key1 = v1;
const AMDVIIOTLBKey *key2 = v2;
return key1->devid == key2->devid && key1->gfn == key2->gfn;
}
static guint amdvi_iotlb_hash(gconstpointer v)
{
const AMDVIIOTLBKey *key = v;
/* Use GPA and DEVID to find the bucket */
return (guint)(key->gfn << AMDVI_PAGE_SHIFT_4K |
(key->devid & ~AMDVI_PAGE_MASK_4K));
}
static AMDVIIOTLBEntry *amdvi_iotlb_lookup(AMDVIState *s, hwaddr addr,
uint64_t devid)
{
AMDVIIOTLBKey key = {
.gfn = AMDVI_GET_IOTLB_GFN(addr),
.devid = devid,
};
return g_hash_table_lookup(s->iotlb, &key);
}
static void amdvi_iotlb_reset(AMDVIState *s)
{
assert(s->iotlb);
trace_amdvi_iotlb_reset();
g_hash_table_remove_all(s->iotlb);
}
static gboolean amdvi_iotlb_remove_by_devid(gpointer key, gpointer value,
gpointer user_data)
{
AMDVIIOTLBEntry *entry = (AMDVIIOTLBEntry *)value;
uint16_t devid = *(uint16_t *)user_data;
return entry->devid == devid;
}
static void amdvi_iotlb_remove_page(AMDVIState *s, hwaddr addr,
uint64_t devid)
{
AMDVIIOTLBKey key = {
.gfn = AMDVI_GET_IOTLB_GFN(addr),
.devid = devid,
};
g_hash_table_remove(s->iotlb, &key);
}
static void amdvi_update_iotlb(AMDVIState *s, uint16_t devid,
uint64_t gpa, IOMMUTLBEntry to_cache,
uint16_t domid)
{
/* don't cache erroneous translations */
if (to_cache.perm != IOMMU_NONE) {
AMDVIIOTLBEntry *entry = g_new(AMDVIIOTLBEntry, 1);
AMDVIIOTLBKey *key = g_new(AMDVIIOTLBKey, 1);
key->gfn = AMDVI_GET_IOTLB_GFN(gpa);
key->devid = devid;
trace_amdvi_cache_update(domid, PCI_BUS_NUM(devid), PCI_SLOT(devid),
PCI_FUNC(devid), gpa, to_cache.translated_addr);
if (g_hash_table_size(s->iotlb) >= AMDVI_IOTLB_MAX_SIZE) {
amdvi_iotlb_reset(s);
}
entry->domid = domid;
entry->perms = to_cache.perm;
entry->translated_addr = to_cache.translated_addr;
entry->page_mask = to_cache.addr_mask;
entry->devid = devid;
g_hash_table_replace(s->iotlb, key, entry);
}
}
static void amdvi_completion_wait(AMDVIState *s, uint64_t *cmd)
{
/* pad the last 3 bits */
hwaddr addr = extract64(cmd[0], 3, 49) << 3;
uint64_t data = cmd[1];
/* Format the data to be written to guest memory as little-endian */
uint64_t le_data = cpu_to_le64(data);
if (extract64(cmd[0], 52, 8)) {
amdvi_log_illegalcom_error(s, extract64(cmd[0], 60, 4),
s->cmdbuf + s->cmdbuf_head);
}
if (extract64(cmd[0], 0, 1)) {
if (dma_memory_write(&address_space_memory, addr, &le_data,
AMDVI_COMPLETION_DATA_SIZE,
MEMTXATTRS_UNSPECIFIED)) {
trace_amdvi_completion_wait_fail(addr);
}
}
/* set completion interrupt */
if (extract64(cmd[0], 1, 1)) {
amdvi_assign_orq(s, AMDVI_MMIO_STATUS, AMDVI_MMIO_STATUS_COMP_INT);
/* generate interrupt */
amdvi_generate_msi_interrupt(s);
}
trace_amdvi_completion_wait(addr, data);
}
static inline uint64_t amdvi_get_perms(uint64_t entry)
{
return (entry & (AMDVI_DEV_PERM_READ | AMDVI_DEV_PERM_WRITE)) >>
AMDVI_DEV_PERM_SHIFT;
}
/* validate that reserved bits are honoured */
static bool amdvi_validate_dte(AMDVIState *s, uint16_t devid,
uint64_t *dte)
{
uint64_t root;
if ((dte[0] & AMDVI_DTE_QUAD0_RESERVED) ||
(dte[1] & AMDVI_DTE_QUAD1_RESERVED) ||
(dte[2] & AMDVI_DTE_QUAD2_RESERVED) ||
(dte[3] & AMDVI_DTE_QUAD3_RESERVED)) {
amdvi_log_illegaldevtab_error(s, devid,
s->devtab +
devid * AMDVI_DEVTAB_ENTRY_SIZE, 0);
return false;
}
/*
* 1 = Host Address Translation is not supported. Value in MMIO Offset
* 0030h[HATS] is not meaningful. A non-zero host page table root pointer
* in the DTE would result in an ILLEGAL_DEV_TABLE_ENTRY event.
*/
root = (dte[0] & AMDVI_DEV_PT_ROOT_MASK) >> 12;
if (root && !s->iommu.dma_translation) {
amdvi_log_illegaldevtab_error(s, devid,
s->devtab +
devid * AMDVI_DEVTAB_ENTRY_SIZE, 0);
return false;
}
return true;
}
/* get a device table entry given the devid */
static bool amdvi_get_dte(AMDVIState *s, int devid, uint64_t *entry)
{
uint32_t offset = devid * AMDVI_DEVTAB_ENTRY_SIZE;
if (dma_memory_read(&address_space_memory, s->devtab + offset, entry,
AMDVI_DEVTAB_ENTRY_SIZE, MEMTXATTRS_UNSPECIFIED)) {
trace_amdvi_dte_get_fail(s->devtab, offset);
/* log error accessing dte */
amdvi_log_devtab_error(s, devid, s->devtab + offset, 0);
return false;
}
*entry = le64_to_cpu(*entry);
if (!amdvi_validate_dte(s, devid, entry)) {
trace_amdvi_invalid_dte(entry[0]);
return false;
}
return true;
}
/* get pte translation mode */
static inline uint8_t get_pte_translation_mode(uint64_t pte)
{
return (pte >> AMDVI_DEV_MODE_RSHIFT) & AMDVI_DEV_MODE_MASK;
}
static inline uint64_t amdvi_get_pte_entry(AMDVIState *s, uint64_t pte_addr,
uint16_t devid)
{
uint64_t pte;
if (dma_memory_read(&address_space_memory, pte_addr,
&pte, sizeof(pte), MEMTXATTRS_UNSPECIFIED)) {
trace_amdvi_get_pte_hwerror(pte_addr);
amdvi_log_pagetab_error(s, devid, pte_addr, 0);
pte = (uint64_t)-1;
return pte;
}
pte = le64_to_cpu(pte);
return pte;
}
static int amdvi_as_to_dte(AMDVIAddressSpace *as, uint64_t *dte)
{
uint16_t devid = PCI_BUILD_BDF(pci_bus_num(as->bus), as->devfn);
AMDVIState *s = as->iommu_state;
if (!amdvi_get_dte(s, devid, dte)) {
/* Unable to retrieve DTE for devid */
return -AMDVI_FR_DTE_RTR_ERR;
}
if (!(dte[0] & AMDVI_DEV_VALID)) {
/* DTE[V] not set, address is passed untranslated for devid */
return -AMDVI_FR_DTE_V;
}
if (!(dte[0] & AMDVI_DEV_TRANSLATION_VALID)) {
/* DTE[TV] not set, host page table not valid for devid */
return -AMDVI_FR_DTE_TV;
}
return 0;
}
/*
* For a PTE encoding a large page, return the page size it encodes as described
* by the AMD IOMMU Specification Table 14: Example Page Size Encodings.
* No need to adjust the value of the PTE to point to the first PTE in the large
* page since the encoding guarantees all "base" PTEs in the large page are the
* same.
*/
static uint64_t large_pte_page_size(uint64_t pte)
{
assert(PTE_NEXT_LEVEL(pte) == 7);
/* Determine size of the large/contiguous page encoded in the PTE */
return PTE_LARGE_PAGE_SIZE(pte);
}
/*
* Validate DTE fields and extract permissions and top level data required to
* initiate the page table walk.
*
* On success, returns 0 and stores:
* - top_level: highest page-table level encoded in DTE[Mode]
* - dte_perms: effective permissions from the DTE
*
* On failure, returns -AMDVI_FR_PT_ROOT_INV. This includes cases where:
* - DTE permissions disallow read AND write
* - DTE[Mode] is invalid for translation
* - IOVA exceeds the address width supported by DTE[Mode]
* In all such cases a page walk must be aborted.
*/
static int amdvi_get_top_pt_level_and_perms(hwaddr address, uint64_t dte,
uint8_t *top_level,
IOMMUAccessFlags *dte_perms)
{
*dte_perms = amdvi_get_perms(dte);
if (*dte_perms == IOMMU_NONE) {
return -AMDVI_FR_PT_ROOT_INV;
}
/* Verifying a valid mode is encoded in DTE */
*top_level = get_pte_translation_mode(dte);
/*
* Page Table Root pointer is only valid for GPA->SPA translation on
* supported modes.
*/
if (*top_level == 0 || *top_level > 6) {
return -AMDVI_FR_PT_ROOT_INV;
}
/*
* If IOVA is larger than the max supported by the highest pgtable level,
* there is nothing to do.
*/
if (address > PT_LEVEL_MAX_ADDR(*top_level)) {
/* IOVA too large for the current DTE */
return -AMDVI_FR_PT_ROOT_INV;
}
return 0;
}
/*
* Helper function to fetch a PTE using AMD v1 pgtable format.
* On successful page walk, returns 0 and pte parameter points to a valid PTE.
* On failure, returns:
* -AMDVI_FR_PT_ROOT_INV: A page walk is not possible due to conditions like DTE
* with invalid permissions, Page Table Root can not be read from DTE, or a
* larger IOVA than supported by page table level encoded in DTE[Mode].
* -AMDVI_FR_PT_ENTRY_INV: A PTE could not be read from guest memory during a
* page table walk. This means that the DTE has valid data, but one of the
* lower level entries in the Page Table could not be read.
*/
static int fetch_pte(AMDVIAddressSpace *as, hwaddr address, uint64_t dte,
uint64_t *pte, hwaddr *page_size)
{
uint64_t pte_addr;
uint8_t pt_level, next_pt_level;
IOMMUAccessFlags perms;
int ret;
*page_size = 0;
/*
* Verify the DTE is properly configured before page walk, and extract
* top pagetable level and permissions.
*/
ret = amdvi_get_top_pt_level_and_perms(address, dte, &pt_level, &perms);
if (ret < 0) {
return ret;
}
/*
* Retrieve the top pagetable entry by following the DTE Page Table Root
* Pointer and indexing the top level table using the IOVA from the request.
*/
pte_addr = NEXT_PTE_ADDR(dte, pt_level, address);
*pte = amdvi_get_pte_entry(as->iommu_state, pte_addr, as->devfn);
if (*pte == (uint64_t)-1) {
/*
* A returned PTE of -1 here indicates a failure to read the top level
* page table from guest memory. A page walk is not possible and page
* size must be returned as 0.
*/
return -AMDVI_FR_PT_ROOT_INV;
}
/*
* Calculate page size for the top level page table entry.
* This ensures correct results for a single level Page Table setup.
*/
*page_size = PTE_LEVEL_PAGE_SIZE(pt_level);
/*
* The root page table entry and its level have been determined. Begin the
* page walk.
*/
while (pt_level > 0) {
/* Permission bits are ANDed at every level, including the DTE */
perms &= amdvi_get_perms(*pte);
if (perms == IOMMU_NONE) {
return 0;
}
/* Not Present */
if (!IOMMU_PTE_PRESENT(*pte)) {
return 0;
}
next_pt_level = PTE_NEXT_LEVEL(*pte);
/* Large or Leaf PTE found */
if (next_pt_level == 0 || next_pt_level == 7) {
/* Leaf PTE found */
break;
}
/* Next level must always be less than current level */
if (pt_level <= next_pt_level) {
return -AMDVI_FR_PT_ENTRY_INV;
}
pt_level = next_pt_level;
/*
* The current entry is a Page Directory Entry. Descend to the lower
* page table level encoded in current pte, and index the new table
* using the appropriate IOVA bits to retrieve the new entry.
*/
*page_size = PTE_LEVEL_PAGE_SIZE(pt_level);
pte_addr = NEXT_PTE_ADDR(*pte, pt_level, address);
*pte = amdvi_get_pte_entry(as->iommu_state, pte_addr, as->devfn);
if (*pte == (uint64_t)-1) {
/* Failure to read PTE. Page walk skips a page_size chunk */
return -AMDVI_FR_PT_ENTRY_INV;
}
}
assert(PTE_NEXT_LEVEL(*pte) == 0 || PTE_NEXT_LEVEL(*pte) == 7);
/*
* Page walk ends when Next Level field on PTE shows that either a leaf PTE
* or a series of large PTEs have been reached. In the latter case, even if
* the range starts in the middle of a contiguous page, the returned PTE
* must be the first PTE of the series.
*/
if (PTE_NEXT_LEVEL(*pte) == 7) {
/* Update page_size with the large PTE page size */
*page_size = large_pte_page_size(*pte);
}
return 0;
}
/*
* Invoke notifiers registered for the address space. Update record of mapped
* ranges in IOVA Tree.
*/
static void amdvi_notify_iommu(AMDVIAddressSpace *as, IOMMUTLBEvent *event)
{
IOMMUTLBEntry *entry = &event->entry;
DMAMap target = {
.iova = entry->iova,
.size = entry->addr_mask,
.translated_addr = entry->translated_addr,
.perm = entry->perm,
};
/*
* Search the IOVA Tree for an existing translation for the target, and skip
* the notification if the mapping is already recorded.
* When the guest uses large pages, comparing against the record makes it
* possible to determine the size of the original MAP and adjust the UNMAP
* request to match it. This avoids failed checks against the mappings kept
* by the VFIO kernel driver.
*/
const DMAMap *mapped = iova_tree_find(as->iova_tree, &target);
if (event->type == IOMMU_NOTIFIER_UNMAP) {
if (!mapped) {
/* No record exists of this mapping, nothing to do */
return;
}
/*
* Adjust the size based on the original record. This is essential to
* determine when large/contiguous pages are used, since the guest has
* already cleared the PTE (erasing the pagesize encoded on it) before
* issuing the invalidation command.
*/
if (mapped->size != target.size) {
assert(mapped->size > target.size);
target.size = mapped->size;
/* Adjust event to invoke notifier with correct range */
entry->addr_mask = mapped->size;
}
iova_tree_remove(as->iova_tree, target);
} else { /* IOMMU_NOTIFIER_MAP */
if (mapped) {
/*
* If a mapping is present and matches the request, skip the
* notification.
*/
if (!memcmp(mapped, &target, sizeof(DMAMap))) {
return;
} else {
/*
* This should never happen unless a buggy guest OS omits or
* sends incorrect invalidation(s). Report an error in the event
* it does happen.
*/
error_report("Found conflicting translation. This could be due "
"to an incorrect or missing invalidation command");
}
}
/* Record the new mapping */
iova_tree_insert(as->iova_tree, &target);
}
/* Invoke the notifiers registered for this address space */
memory_region_notify_iommu(&as->iommu, 0, *event);
}
/*
* Walk the guest page table for an IOVA and range and signal the registered
* notifiers to sync the shadow page tables in the host.
* Must be called with a valid DTE for DMA remapping i.e. V=1,TV=1
*/
static void amdvi_sync_shadow_page_table_range(AMDVIAddressSpace *as,
uint64_t *dte, hwaddr addr,
uint64_t size, bool send_unmap)
{
IOMMUTLBEvent event;
hwaddr page_mask, pagesize;
hwaddr iova = addr;
hwaddr end = iova + size - 1;
uint64_t pte;
int ret;
while (iova < end) {
ret = fetch_pte(as, iova, dte[0], &pte, &pagesize);
if (ret == -AMDVI_FR_PT_ROOT_INV) {
/*
* Invalid conditions such as the IOVA being larger than supported
* by current page table mode as configured in the DTE, or a failure
* to fetch the Page Table from the Page Table Root Pointer in DTE.
*/
assert(pagesize == 0);
return;
}
/* PTE has been validated for major errors and pagesize is set */
assert(pagesize);
page_mask = ~(pagesize - 1);
if (ret == -AMDVI_FR_PT_ENTRY_INV) {
/*
* Failure to read PTE from memory, the pagesize matches the current
* level. Unable to determine the region type, so a safe strategy is
* to skip the range and continue the page walk.
*/
goto next;
}
event.entry.target_as = &address_space_memory;
event.entry.iova = iova & page_mask;
/* translated_addr is irrelevant for the unmap case */
event.entry.translated_addr = (pte & AMDVI_DEV_PT_ROOT_MASK) &
page_mask;
event.entry.addr_mask = ~page_mask;
event.entry.perm = amdvi_get_perms(pte);
/*
* In cases where the leaf PTE is not found, or it has invalid
* permissions, an UNMAP type notification is sent, but only if the
* caller requested it.
*/
if (!IOMMU_PTE_PRESENT(pte) || (event.entry.perm == IOMMU_NONE)) {
if (!send_unmap) {
goto next;
}
event.type = IOMMU_NOTIFIER_UNMAP;
} else {
event.type = IOMMU_NOTIFIER_MAP;
}
/*
* The following call might need to adjust event.entry.size in cases
* where the guest unmapped a series of large pages.
*/
amdvi_notify_iommu(as, &event);
/*
* In the special scenario where the guest is unmapping a large page,
* addr_mask has been adjusted before sending the notification. Update
* pagesize accordingly in order to correctly compute the next IOVA.
*/
pagesize = event.entry.addr_mask + 1;
next:
iova &= ~(pagesize - 1);
/* Check for 64-bit overflow and terminate walk in such cases */
if ((iova + pagesize) < iova) {
break;
} else {
iova += pagesize;
}
}
}
/*
* Unmap entire range that the notifier registered for i.e. the full AS.
*
* This is seemingly technically equivalent to directly calling
* memory_region_unmap_iommu_notifier_range(), but it allows to check for
* notifier boundaries and issue notifications with ranges within those bounds.
*/
static void amdvi_address_space_unmap(AMDVIAddressSpace *as, IOMMUNotifier *n)
{
hwaddr start = n->start;
hwaddr end = n->end;
hwaddr remain;
DMAMap map;
assert(start <= end);
remain = end - start + 1;
/*
* Divide the notifier range into chunks that are aligned and do not exceed
* the notifier boundaries.
*/
while (remain >= AMDVI_PAGE_SIZE) {
IOMMUTLBEvent event;
uint64_t mask = dma_aligned_pow2_mask(start, end, 64);
event.type = IOMMU_NOTIFIER_UNMAP;
IOMMUTLBEntry entry = {
.target_as = &address_space_memory,
.iova = start,
.translated_addr = 0, /* irrelevant for unmap case */
.addr_mask = mask,
.perm = IOMMU_NONE,
};
event.entry = entry;
/* Call notifier registered for updates on this address space */
memory_region_notify_iommu_one(n, &event);
start += mask + 1;
remain -= mask + 1;
}
assert(!remain);
map.iova = n->start;
map.size = n->end - n->start;
iova_tree_remove(as->iova_tree, map);
}
/*
* For all the address spaces with notifiers registered, unmap the entire range
* the notifier registered for i.e. clear all the address spaces managed by the
* IOMMU.
*/
static void amdvi_address_space_unmap_all(AMDVIState *s)
{
AMDVIAddressSpace *as;
IOMMUNotifier *n;
QLIST_FOREACH(as, &s->amdvi_as_with_notifiers, next) {
IOMMU_NOTIFIER_FOREACH(n, &as->iommu) {
amdvi_address_space_unmap(as, n);
}
}
}
/*
* For every translation present in the IOMMU, construct IOMMUTLBEntry data
* and pass it as parameter to notifier callback.
*/
static void amdvi_iommu_replay(IOMMUMemoryRegion *iommu_mr, IOMMUNotifier *n)
{
AMDVIAddressSpace *as = container_of(iommu_mr, AMDVIAddressSpace, iommu);
uint64_t dte[4] = { 0 };
if (!(n->notifier_flags & IOMMU_NOTIFIER_MAP)) {
return;
}
if (amdvi_as_to_dte(as, dte)) {
return;
}
/* Dropping all mappings for the address space. Also clears the IOVA tree */
amdvi_address_space_unmap(as, n);
amdvi_sync_shadow_page_table_range(as, &dte[0], 0, UINT64_MAX, false);
}
static void amdvi_address_space_sync(AMDVIAddressSpace *as)
{
IOMMUNotifier *n;
uint64_t dte[4] = { 0 };
/* If only UNMAP notifiers are registered, drop all existing mappings */
if (!(as->notifier_flags & IOMMU_NOTIFIER_MAP)) {
IOMMU_NOTIFIER_FOREACH(n, &as->iommu) {
/*
* Directly calling memory_region_unmap_iommu_notifier_range() does
* not guarantee that the addr_mask eventually passed as parameter
* to the notifier is valid. Use amdvi_address_space_unmap() which
* ensures the notifier range is divided into properly aligned
* regions, and issues notifications for each one.
*/
amdvi_address_space_unmap(as, n);
}
return;
}
if (amdvi_as_to_dte(as, dte)) {
return;
}
amdvi_sync_shadow_page_table_range(as, &dte[0], 0, UINT64_MAX, true);
}
/*
* This differs from the replay() method in that it issues both MAP and UNMAP
* notifications since it is called after global invalidation events in order to
* re-sync all address spaces.
*/
static void amdvi_iommu_address_space_sync_all(AMDVIState *s)
{
AMDVIAddressSpace *as;
QLIST_FOREACH(as, &s->amdvi_as_with_notifiers, next) {
amdvi_address_space_sync(as);
}
}
/*
* Toggle between address translation and passthrough modes by enabling the
* corresponding memory regions.
*/
static void amdvi_switch_address_space(AMDVIAddressSpace *amdvi_as)
{
AMDVIState *s = amdvi_as->iommu_state;
if (s->dma_remap && amdvi_as->addr_translation) {
/* Enabling DMA region */
memory_region_set_enabled(&amdvi_as->iommu_nodma, false);
memory_region_set_enabled(MEMORY_REGION(&amdvi_as->iommu), true);
} else {
/* Disabling DMA region, using passthrough */
memory_region_set_enabled(MEMORY_REGION(&amdvi_as->iommu), false);
memory_region_set_enabled(&amdvi_as->iommu_nodma, true);
}
}
/*
* For all existing address spaces managed by the IOMMU, enable/disable the
* corresponding memory regions to reset the address translation mode and
* use passthrough by default.
*/
static void amdvi_reset_address_translation_all(AMDVIState *s)
{
AMDVIAddressSpace *iommu_as;
GHashTableIter as_it;
g_hash_table_iter_init(&as_it, s->address_spaces);
while (g_hash_table_iter_next(&as_it, NULL, (void **)&iommu_as)) {
/* Use passthrough as default mode after reset */
iommu_as->addr_translation = false;
amdvi_switch_address_space(iommu_as);
}
}
static void enable_dma_mode(AMDVIAddressSpace *as, bool inval_current)
{
/*
* When enabling DMA mode for the purpose of isolating guest devices on
* a failure to retrieve or invalid DTE, all existing mappings must be
* dropped.
*/
if (inval_current) {
IOMMUNotifier *n;
IOMMU_NOTIFIER_FOREACH(n, &as->iommu) {
amdvi_address_space_unmap(as, n);
}
}
if (as->addr_translation) {
return;
}
/* Installing DTE enabling translation, activate region */
as->addr_translation = true;
amdvi_switch_address_space(as);
/* Sync shadow page tables */
amdvi_address_space_sync(as);
}
/*
* If paging was previously in use in the address space
* - invalidate all existing mappings
* - switch to no_dma memory region
*/
static void enable_nodma_mode(AMDVIAddressSpace *as)
{
IOMMUNotifier *n;
if (!as->addr_translation) {
/* passthrough is already active, nothing to do */
return;
}
as->addr_translation = false;
IOMMU_NOTIFIER_FOREACH(n, &as->iommu) {
/* Drop all mappings for the address space */
amdvi_address_space_unmap(as, n);
}
amdvi_switch_address_space(as);
}
/*
* A guest driver must issue the INVALIDATE_DEVTAB_ENTRY command to the IOMMU
* after changing a Device Table entry. We can use this fact to detect when a
* Device Table entry is created for a device attached to a paging domain and
* enable the corresponding IOMMU memory region to allow for DMA translation if
* appropriate.
*/
static void amdvi_update_addr_translation_mode(AMDVIState *s, uint16_t devid)
{
uint8_t dte_mode;
AMDVIAddressSpace *as;
uint64_t dte[4] = { 0 };
int ret;
as = amdvi_get_as_by_devid(s, devid);
if (!as) {
return;
}
ret = amdvi_as_to_dte(as, dte);
if (!ret) {
dte_mode = (dte[0] >> AMDVI_DEV_MODE_RSHIFT) & AMDVI_DEV_MODE_MASK;
}
switch (ret) {
case 0:
/* DTE was successfully retrieved */
if (!dte_mode) {
enable_nodma_mode(as); /* DTE[V]=1 && DTE[Mode]=0 => passthrough */
} else {
enable_dma_mode(as, false); /* Enable DMA translation */
}
break;
case -AMDVI_FR_DTE_V:
/* DTE[V]=0, address is passed untranslated */
enable_nodma_mode(as);
break;
case -AMDVI_FR_DTE_RTR_ERR:
case -AMDVI_FR_DTE_TV:
/*
* Enforce isolation by using DMA in rare scenarios where the DTE cannot
* be retrieved or DTE[TV]=0. Existing mappings are dropped.
*/
enable_dma_mode(as, true);
break;
}
}
/* log error without aborting since linux seems to be using reserved bits */
static void amdvi_inval_devtab_entry(AMDVIState *s, uint64_t *cmd)
{
uint16_t devid = extract64(cmd[0], 0, 16);
trace_amdvi_devtab_inval(PCI_BUS_NUM(devid), PCI_SLOT(devid),
PCI_FUNC(devid));
/* This command should invalidate internal caches of which there isn't */
if (extract64(cmd[0], 16, 44) || cmd[1]) {
amdvi_log_illegalcom_error(s, extract64(cmd[0], 60, 4),
s->cmdbuf + s->cmdbuf_head);
return;
}
/*
* When DMA remapping capability is enabled, check if updated DTE is setup
* for paging or not, and configure the corresponding memory regions.
*/
if (s->dma_remap) {
amdvi_update_addr_translation_mode(s, devid);
}
}
static void amdvi_complete_ppr(AMDVIState *s, uint64_t *cmd)
{
if (extract64(cmd[0], 16, 16) || extract64(cmd[0], 52, 8) ||
extract64(cmd[1], 0, 2) || extract64(cmd[1], 3, 29)
|| extract64(cmd[1], 48, 16)) {
amdvi_log_illegalcom_error(s, extract64(cmd[0], 60, 4),
s->cmdbuf + s->cmdbuf_head);
}
trace_amdvi_ppr_exec();
}
static void amdvi_intremap_inval_notify_all(AMDVIState *s, bool global,
uint32_t index, uint32_t mask)
{
x86_iommu_iec_notify_all(X86_IOMMU_DEVICE(s), global, index, mask);
}
static void amdvi_inval_all(AMDVIState *s, uint64_t *cmd)
{
if (extract64(cmd[0], 0, 60) || cmd[1]) {
amdvi_log_illegalcom_error(s, extract64(cmd[0], 60, 4),
s->cmdbuf + s->cmdbuf_head);
}
/* Notify global invalidation */
amdvi_intremap_inval_notify_all(s, true, 0, 0);
amdvi_iotlb_reset(s);
/*
* Fully replay the address space i.e. send both UNMAP and MAP events in
* order to synchronize guest and host IO page tables tables.
*/
amdvi_iommu_address_space_sync_all(s);
trace_amdvi_all_inval();
}
static gboolean amdvi_iotlb_remove_by_domid(gpointer key, gpointer value,
gpointer user_data)
{
AMDVIIOTLBEntry *entry = (AMDVIIOTLBEntry *)value;
uint16_t domid = *(uint16_t *)user_data;
return entry->domid == domid;
}
/*
* Helper to decode the size of the range to invalidate encoded in the
* INVALIDATE_IOMMU_PAGES Command format.
* The size of the region to invalidate depends on the S bit and address.
* S bit value:
* 0 : Invalidation size is 4 Kbytes.
* 1 : Invalidation size is determined by first zero bit in the address
* starting from Address[12].
*
* In the AMD IOMMU Linux driver, an invalidation command with address
* ((1 << 63) - 1) is sent when intending to clear the entire cache.
* However, Table 14: Example Page Size Encodings shows that an address of
* ((1ULL << 51) - 1) encodes the entire cache, so effectively any address with
* first zero at bit 51 or larger is a request to invalidate the entire address
* space.
*/
static uint64_t amdvi_decode_invalidation_size(hwaddr addr, uint16_t flags)
{
uint64_t size = AMDVI_PAGE_SIZE;
uint8_t fzbit = 0;
if (flags & AMDVI_CMD_INVAL_IOMMU_PAGES_S) {
fzbit = cto64(addr | 0xFFF);
if (fzbit >= 51) {
size = AMDVI_INV_ALL_PAGES;
} else {
size = 1ULL << (fzbit + 1);
}
}
return size;
}
/*
* Synchronize the guest page tables with the shadow page tables kept in the
* host for the specified range.
* The invalidation command issued by the guest and intercepted by the VMM
* does not specify a device, but a domain, since all devices in the same domain
* share the same page tables. However, vIOMMU emulation creates separate
* address spaces per device, so it is necessary to traverse the list of all of
* address spaces (i.e. devices) that have notifiers registered in order to
* propagate the changes to the host page tables.
* We cannot return early from this function once a matching domain has been
* identified and its page tables synced (based on the fact that all devices in
* the same domain share the page tables). The reason is that different devices
* (i.e. address spaces) could have different notifiers registered, and by
* skipping address spaces that appear later on the amdvi_as_with_notifiers list
* their notifiers (which could differ from the ones registered for the first
* device/address space) would not be invoked.
*/
static void amdvi_sync_domain(AMDVIState *s, uint16_t domid, uint64_t addr,
uint16_t flags)
{
AMDVIAddressSpace *as;
uint64_t size = amdvi_decode_invalidation_size(addr, flags);
if (size == AMDVI_INV_ALL_PAGES) {
addr = 0; /* Set start address to 0 and invalidate entire AS */
} else {
addr &= ~(size - 1);
}
/*
* Call notifiers that have registered for each address space matching the
* domain ID, in order to sync the guest pagetable state with the host.
*/
QLIST_FOREACH(as, &s->amdvi_as_with_notifiers, next) {
uint64_t dte[4] = { 0 };
/*
* Retrieve the Device Table entry for the devid corresponding to the
* current address space, and verify the DomainID matches i.e. the page
* tables to be synced belong to devices in the domain.
*/
if (amdvi_as_to_dte(as, dte)) {
continue;
}
/* Only need to sync the Page Tables for a matching domain */
if (domid != (dte[1] & AMDVI_DEV_DOMID_ID_MASK)) {
continue;
}
/*
* We have determined that there is a valid Device Table Entry for a
* device matching the DomainID in the INV_IOMMU_PAGES command issued by
* the guest. Walk the guest page table to sync shadow page table.
*/
if (as->notifier_flags & IOMMU_NOTIFIER_MAP) {
/* Sync guest IOMMU mappings with host */
amdvi_sync_shadow_page_table_range(as, &dte[0], addr, size, true);
}
}
}
/* we don't have devid - we can't remove pages by address */
static void amdvi_inval_pages(AMDVIState *s, uint64_t *cmd)
{
uint16_t domid = extract64(cmd[0], 32, 16);
uint64_t addr = extract64(cmd[1], 12, 52) << 12;
uint16_t flags = extract64(cmd[1], 0, 3);
if (extract64(cmd[0], 20, 12) || extract64(cmd[0], 48, 12) ||
extract64(cmd[1], 3, 9)) {
amdvi_log_illegalcom_error(s, extract64(cmd[0], 60, 4),
s->cmdbuf + s->cmdbuf_head);
}
g_hash_table_foreach_remove(s->iotlb, amdvi_iotlb_remove_by_domid,
&domid);
amdvi_sync_domain(s, domid, addr, flags);
trace_amdvi_pages_inval(domid);
}
static void amdvi_prefetch_pages(AMDVIState *s, uint64_t *cmd)
{
if (extract64(cmd[0], 16, 8) || extract64(cmd[0], 52, 8) ||
extract64(cmd[1], 1, 1) || extract64(cmd[1], 3, 1) ||
extract64(cmd[1], 5, 7)) {
amdvi_log_illegalcom_error(s, extract64(cmd[0], 60, 4),
s->cmdbuf + s->cmdbuf_head);
}
trace_amdvi_prefetch_pages();
}
static void amdvi_inval_inttable(AMDVIState *s, uint64_t *cmd)
{
if (extract64(cmd[0], 16, 44) || cmd[1]) {
amdvi_log_illegalcom_error(s, extract64(cmd[0], 60, 4),
s->cmdbuf + s->cmdbuf_head);
return;
}
/* Notify global invalidation */
amdvi_intremap_inval_notify_all(s, true, 0, 0);
trace_amdvi_intr_inval();
}
/* FIXME: Try to work with the specified size instead of all the pages
* when the S bit is on
*/
static void iommu_inval_iotlb(AMDVIState *s, uint64_t *cmd)
{
uint16_t devid = extract64(cmd[0], 0, 16);
if (extract64(cmd[1], 1, 1) || extract64(cmd[1], 3, 1) ||
extract64(cmd[1], 6, 6)) {
amdvi_log_illegalcom_error(s, extract64(cmd[0], 60, 4),
s->cmdbuf + s->cmdbuf_head);
return;
}
if (extract64(cmd[1], 0, 1)) {
g_hash_table_foreach_remove(s->iotlb, amdvi_iotlb_remove_by_devid,
&devid);
} else {
amdvi_iotlb_remove_page(s, extract64(cmd[1], 12, 52) << 12,
devid);
}
trace_amdvi_iotlb_inval();
}
/* not honouring reserved bits is regarded as an illegal command */
static void amdvi_cmdbuf_exec(AMDVIState *s)
{
uint64_t cmd[2];
if (dma_memory_read(&address_space_memory, s->cmdbuf + s->cmdbuf_head,
cmd, AMDVI_COMMAND_SIZE, MEMTXATTRS_UNSPECIFIED)) {
trace_amdvi_command_read_fail(s->cmdbuf, s->cmdbuf_head);
amdvi_log_command_error(s, s->cmdbuf + s->cmdbuf_head);
return;
}
/*
* Commands in guest memory are little-endian. Convert once after reading
* so that command handlers can decode values in host native endianness.
* Convert back to little-endian only when writing data to guest memory via
* dma_memory_write().
*/
cmd[0] = le64_to_cpu(cmd[0]);
cmd[1] = le64_to_cpu(cmd[1]);
switch (extract64(cmd[0], 60, 4)) {
case AMDVI_CMD_COMPLETION_WAIT:
amdvi_completion_wait(s, cmd);
break;
case AMDVI_CMD_INVAL_DEVTAB_ENTRY:
amdvi_inval_devtab_entry(s, cmd);
break;
case AMDVI_CMD_INVAL_AMDVI_PAGES:
amdvi_inval_pages(s, cmd);
break;
case AMDVI_CMD_INVAL_IOTLB_PAGES:
iommu_inval_iotlb(s, cmd);
break;
case AMDVI_CMD_INVAL_INTR_TABLE:
amdvi_inval_inttable(s, cmd);
break;
case AMDVI_CMD_PREFETCH_AMDVI_PAGES:
amdvi_prefetch_pages(s, cmd);
break;
case AMDVI_CMD_COMPLETE_PPR_REQUEST:
amdvi_complete_ppr(s, cmd);
break;
case AMDVI_CMD_INVAL_AMDVI_ALL:
amdvi_inval_all(s, cmd);
break;
default:
trace_amdvi_unhandled_command(extract64(cmd[0], 60, 4));
/* log illegal command */
amdvi_log_illegalcom_error(s, extract64(cmd[0], 60, 4),
s->cmdbuf + s->cmdbuf_head);
}
}
static void amdvi_cmdbuf_run(AMDVIState *s)
{
if (!s->cmdbuf_enabled) {
trace_amdvi_command_error(amdvi_readq(s, AMDVI_MMIO_CONTROL));
return;
}
/* check if there is work to do. */
while (s->cmdbuf_head != s->cmdbuf_tail) {
trace_amdvi_command_exec(s->cmdbuf_head, s->cmdbuf_tail, s->cmdbuf);
amdvi_cmdbuf_exec(s);
s->cmdbuf_head += AMDVI_COMMAND_SIZE;
/* wrap head pointer */
if (s->cmdbuf_head >= s->cmdbuf_len * AMDVI_COMMAND_SIZE) {
s->cmdbuf_head = 0;
}
amdvi_writeq_raw(s, AMDVI_MMIO_COMMAND_HEAD, s->cmdbuf_head);
}
}
static inline
const char *amdvi_mmio_get_name(hwaddr addr)
{
/* Return MMIO names as string literals */
switch (addr) {
#define MMIO_REG_TO_STRING(mmio_reg) case mmio_reg: return #mmio_reg
MMIO_REG_TO_STRING(AMDVI_MMIO_DEVICE_TABLE);
MMIO_REG_TO_STRING(AMDVI_MMIO_COMMAND_BASE);
MMIO_REG_TO_STRING(AMDVI_MMIO_EVENT_BASE);
MMIO_REG_TO_STRING(AMDVI_MMIO_CONTROL);
MMIO_REG_TO_STRING(AMDVI_MMIO_EXCL_BASE);
MMIO_REG_TO_STRING(AMDVI_MMIO_EXCL_LIMIT);
MMIO_REG_TO_STRING(AMDVI_MMIO_EXT_FEATURES);
MMIO_REG_TO_STRING(AMDVI_MMIO_COMMAND_HEAD);
MMIO_REG_TO_STRING(AMDVI_MMIO_COMMAND_TAIL);
MMIO_REG_TO_STRING(AMDVI_MMIO_EVENT_HEAD);
MMIO_REG_TO_STRING(AMDVI_MMIO_EVENT_TAIL);
MMIO_REG_TO_STRING(AMDVI_MMIO_STATUS);
MMIO_REG_TO_STRING(AMDVI_MMIO_PPR_BASE);
MMIO_REG_TO_STRING(AMDVI_MMIO_PPR_HEAD);
MMIO_REG_TO_STRING(AMDVI_MMIO_PPR_TAIL);
MMIO_REG_TO_STRING(AMDVI_MMIO_XT_GEN_INTR);
#undef MMIO_REG_TO_STRING
default:
return "UNHANDLED";
}
}
static uint64_t amdvi_mmio_read(void *opaque, hwaddr addr, unsigned size)
{
AMDVIState *s = opaque;
uint64_t val = -1;
if (addr + size > AMDVI_MMIO_SIZE) {
trace_amdvi_mmio_read_invalid(AMDVI_MMIO_SIZE, addr, size);
return (uint64_t)-1;
}
if (size == 2) {
val = amdvi_readw(s, addr);
} else if (size == 4) {
val = amdvi_readl(s, addr);
} else if (size == 8) {
val = amdvi_readq(s, addr);
}
trace_amdvi_mmio_read(amdvi_mmio_get_name(addr), addr, size, addr & ~0x07);
return val;
}
static void amdvi_handle_control_write(AMDVIState *s)
{
unsigned long control = amdvi_readq(s, AMDVI_MMIO_CONTROL);
s->enabled = !!(control & AMDVI_MMIO_CONTROL_AMDVIEN);
s->evtlog_enabled = s->enabled && !!(control &
AMDVI_MMIO_CONTROL_EVENTLOGEN);
s->evtlog_intr = !!(control & AMDVI_MMIO_CONTROL_EVENTINTEN);
s->completion_wait_intr = !!(control & AMDVI_MMIO_CONTROL_COMWAITINTEN);
s->cmdbuf_enabled = s->enabled && !!(control &
AMDVI_MMIO_CONTROL_CMDBUFLEN);
s->ga_enabled = !!(control & AMDVI_MMIO_CONTROL_GAEN);
s->xten = !!(control & AMDVI_MMIO_CONTROL_XTEN) && s->xtsup &&
s->ga_enabled;
/*
* IntCapXTEn controls whether IOMMU-originated interrupts are sent based
* on the information in XT IOMMU Interrupt Control Registers rather than
* the IOMMU’s MSI capability registers. Therefore it requires IOMMU
* x2APIC support capabilities (i.e. XTSup=1), but it is independent of
* whether a driver chooses to enable x2APIC mode for interrupt remapping
* (i.e. XTEn=1).
*/
s->intcapxten = !!(control & AMDVI_MMIO_CONTROL_INTCAPXTEN) && s->xtsup;
/* update the flags depending on the control register */
if (s->cmdbuf_enabled) {
amdvi_assign_orq(s, AMDVI_MMIO_STATUS, AMDVI_MMIO_STATUS_CMDBUF_RUN);
} else {
amdvi_assign_andq(s, AMDVI_MMIO_STATUS, ~AMDVI_MMIO_STATUS_CMDBUF_RUN);
}
if (s->evtlog_enabled) {
amdvi_assign_orq(s, AMDVI_MMIO_STATUS, AMDVI_MMIO_STATUS_EVT_RUN);
} else {
amdvi_assign_andq(s, AMDVI_MMIO_STATUS, ~AMDVI_MMIO_STATUS_EVT_RUN);
}
trace_amdvi_control_status(control);
amdvi_cmdbuf_run(s);
}
static inline void amdvi_handle_devtab_write(AMDVIState *s)
{
uint64_t val = amdvi_readq(s, AMDVI_MMIO_DEVICE_TABLE);
s->devtab = (val & AMDVI_MMIO_DEVTAB_BASE_MASK);
/* set device table length (i.e. number of entries table can hold) */
s->devtab_len = (((val & AMDVI_MMIO_DEVTAB_SIZE_MASK) + 1) *
(AMDVI_MMIO_DEVTAB_SIZE_UNIT /
AMDVI_MMIO_DEVTAB_ENTRY_SIZE));
}
static inline void amdvi_handle_cmdhead_write(AMDVIState *s)
{
s->cmdbuf_head = amdvi_readq(s, AMDVI_MMIO_COMMAND_HEAD)
& AMDVI_MMIO_CMDBUF_HEAD_MASK
& (s->cmdbuf_len * AMDVI_COMMAND_SIZE - 1);
amdvi_cmdbuf_run(s);
}
static inline void amdvi_handle_cmdbase_write(AMDVIState *s)
{
s->cmdbuf = amdvi_readq(s, AMDVI_MMIO_COMMAND_BASE)
& AMDVI_MMIO_CMDBUF_BASE_MASK;
s->cmdbuf_len = 1UL << (amdvi_readq(s, AMDVI_MMIO_CMDBUF_SIZE_BYTE)
& AMDVI_MMIO_CMDBUF_SIZE_MASK);
s->cmdbuf_head = s->cmdbuf_tail = 0;
}
static inline void amdvi_handle_cmdtail_write(AMDVIState *s)
{
s->cmdbuf_tail = amdvi_readq(s, AMDVI_MMIO_COMMAND_TAIL)
& AMDVI_MMIO_CMDBUF_TAIL_MASK
& (s->cmdbuf_len * AMDVI_COMMAND_SIZE - 1);
amdvi_cmdbuf_run(s);
}
static inline void amdvi_handle_excllim_write(AMDVIState *s)
{
uint64_t val = amdvi_readq(s, AMDVI_MMIO_EXCL_LIMIT);
s->excl_limit = (val & AMDVI_MMIO_EXCL_LIMIT_MASK) |
AMDVI_MMIO_EXCL_LIMIT_LOW;
}
static inline void amdvi_handle_evtbase_write(AMDVIState *s)
{
uint64_t val = amdvi_readq(s, AMDVI_MMIO_EVENT_BASE);
if (amdvi_readq(s, AMDVI_MMIO_STATUS) & AMDVI_MMIO_STATUS_EVENT_INT)
/* Do not reset if eventlog interrupt bit is set*/
return;
s->evtlog = val & AMDVI_MMIO_EVTLOG_BASE_MASK;
s->evtlog_len = 1UL << (amdvi_readq(s, AMDVI_MMIO_EVTLOG_SIZE_BYTE)
& AMDVI_MMIO_EVTLOG_SIZE_MASK);
/* clear tail and head pointer to 0 when event base is updated */
s->evtlog_tail = s->evtlog_head = 0;
amdvi_writeq_raw(s, AMDVI_MMIO_EVENT_HEAD, s->evtlog_head);
amdvi_writeq_raw(s, AMDVI_MMIO_EVENT_TAIL, s->evtlog_tail);
}
static inline void amdvi_handle_evttail_write(AMDVIState *s)
{
uint64_t val = amdvi_readq(s, AMDVI_MMIO_EVENT_TAIL);
s->evtlog_tail = val & AMDVI_MMIO_EVTLOG_TAIL_MASK;
}
static inline void amdvi_handle_evthead_write(AMDVIState *s)
{
uint64_t val = amdvi_readq(s, AMDVI_MMIO_EVENT_HEAD);
s->evtlog_head = val & AMDVI_MMIO_EVTLOG_HEAD_MASK;
}
static inline void amdvi_handle_pprbase_write(AMDVIState *s)
{
uint64_t val = amdvi_readq(s, AMDVI_MMIO_PPR_BASE);
s->ppr_log = val & AMDVI_MMIO_PPRLOG_BASE_MASK;
s->pprlog_len = 1UL << (amdvi_readq(s, AMDVI_MMIO_PPRLOG_SIZE_BYTE)
& AMDVI_MMIO_PPRLOG_SIZE_MASK);
}
static inline void amdvi_handle_pprhead_write(AMDVIState *s)
{
uint64_t val = amdvi_readq(s, AMDVI_MMIO_PPR_HEAD);
s->pprlog_head = val & AMDVI_MMIO_PPRLOG_HEAD_MASK;
}
static inline void amdvi_handle_pprtail_write(AMDVIState *s)
{
uint64_t val = amdvi_readq(s, AMDVI_MMIO_PPR_TAIL);
s->pprlog_tail = val & AMDVI_MMIO_PPRLOG_TAIL_MASK;
}
/* FIXME: something might go wrong if System Software writes in chunks
* of one byte but linux writes in chunks of 4 bytes so currently it
* works correctly with linux but will definitely be busted if software
* reads/writes 8 bytes
*/
static void amdvi_mmio_reg_write(AMDVIState *s, unsigned size, uint64_t val,
hwaddr addr)
{
if (size == 2) {
amdvi_writew(s, addr, val);
} else if (size == 4) {
amdvi_writel(s, addr, val);
} else if (size == 8) {
amdvi_writeq(s, addr, val);
}
}
static void amdvi_mmio_write(void *opaque, hwaddr addr, uint64_t val,
unsigned size)
{
AMDVIState *s = opaque;
unsigned long offset = addr & 0x07;
if (addr + size > AMDVI_MMIO_SIZE) {
trace_amdvi_mmio_write("error: addr outside region: max ",
(uint64_t)AMDVI_MMIO_SIZE, size, val, offset);
return;
}
trace_amdvi_mmio_write(amdvi_mmio_get_name(addr), addr, size, val, offset);
switch (addr & ~0x07) {
case AMDVI_MMIO_CONTROL:
amdvi_mmio_reg_write(s, size, val, addr);
amdvi_handle_control_write(s);
break;
case AMDVI_MMIO_DEVICE_TABLE:
amdvi_mmio_reg_write(s, size, val, addr);
/* set device table address
* This also suffers from inability to tell whether software
* is done writing
*/
if (offset || (size == 8)) {
amdvi_handle_devtab_write(s);
}
break;
case AMDVI_MMIO_COMMAND_HEAD:
amdvi_mmio_reg_write(s, size, val, addr);
amdvi_handle_cmdhead_write(s);
break;
case AMDVI_MMIO_COMMAND_BASE:
amdvi_mmio_reg_write(s, size, val, addr);
/* FIXME - make sure System Software has finished writing in case
* it writes in chucks less than 8 bytes in a robust way.As for
* now, this hacks works for the linux driver
*/
if (offset || (size == 8)) {
amdvi_handle_cmdbase_write(s);
}
break;
case AMDVI_MMIO_COMMAND_TAIL:
amdvi_mmio_reg_write(s, size, val, addr);
amdvi_handle_cmdtail_write(s);
break;
case AMDVI_MMIO_EVENT_BASE:
amdvi_mmio_reg_write(s, size, val, addr);
amdvi_handle_evtbase_write(s);
break;
case AMDVI_MMIO_EVENT_HEAD:
amdvi_mmio_reg_write(s, size, val, addr);
amdvi_handle_evthead_write(s);
break;
case AMDVI_MMIO_EVENT_TAIL:
amdvi_mmio_reg_write(s, size, val, addr);
amdvi_handle_evttail_write(s);
break;
case AMDVI_MMIO_EXCL_LIMIT:
amdvi_mmio_reg_write(s, size, val, addr);
amdvi_handle_excllim_write(s);
break;
/* PPR log base - unused for now */
case AMDVI_MMIO_PPR_BASE:
amdvi_mmio_reg_write(s, size, val, addr);
amdvi_handle_pprbase_write(s);
break;
/* PPR log head - also unused for now */
case AMDVI_MMIO_PPR_HEAD:
amdvi_mmio_reg_write(s, size, val, addr);
amdvi_handle_pprhead_write(s);
break;
/* PPR log tail - unused for now */
case AMDVI_MMIO_PPR_TAIL:
amdvi_mmio_reg_write(s, size, val, addr);
amdvi_handle_pprtail_write(s);
break;
case AMDVI_MMIO_STATUS:
amdvi_mmio_reg_write(s, size, val, addr);
break;
case AMDVI_MMIO_XT_GEN_INTR:
amdvi_mmio_reg_write(s, size, val, addr);
break;
}
}
static void amdvi_page_walk(AMDVIAddressSpace *as, uint64_t *dte,
IOMMUTLBEntry *ret, unsigned perms,
hwaddr addr)
{
hwaddr page_mask, pagesize = 0;
uint8_t mode;
uint64_t pte;
int fetch_ret;
/* make sure the DTE has TV = 1 */
if (!(dte[0] & AMDVI_DEV_TRANSLATION_VALID)) {
/*
* A DTE with V=1, TV=0 does not have a valid Page Table Root Pointer.
* An IOMMU processing a request that requires a table walk terminates
* the walk when it encounters this condition. Do the same and return
* instead of assuming that the address is forwarded without translation
* i.e. the passthrough case, as it is done for the case where DTE[V]=0.
*/
return;
}
mode = get_pte_translation_mode(dte[0]);
if (mode >= 7) {
trace_amdvi_mode_invalid(mode, addr);
return;
}
if (mode == 0) {
goto no_remap;
}
/* Attempt to fetch the PTE to determine if a valid mapping exists */
fetch_ret = fetch_pte(as, addr, dte[0], &pte, &pagesize);
/*
* If walking the page table results in an error of any type, returns an
* empty PTE i.e. no mapping, or the permissions do not match, return since
* there is no translation available.
*/
if (fetch_ret < 0 || !IOMMU_PTE_PRESENT(pte) ||
perms != (perms & amdvi_get_perms(pte))) {
amdvi_page_fault(as->iommu_state, as->devfn, addr, perms);
trace_amdvi_page_fault(addr);
return;
}
/* A valid PTE and page size has been retrieved */
assert(pagesize);
page_mask = ~(pagesize - 1);
/* get access permissions from pte */
ret->iova = addr & page_mask;
ret->translated_addr = (pte & AMDVI_DEV_PT_ROOT_MASK) & page_mask;
ret->addr_mask = ~page_mask;
ret->perm = amdvi_get_perms(pte);
return;
no_remap:
ret->iova = addr & AMDVI_PAGE_MASK_4K;
ret->translated_addr = addr & AMDVI_PAGE_MASK_4K;
ret->addr_mask = ~AMDVI_PAGE_MASK_4K;
ret->perm = amdvi_get_perms(dte[0]);
}
static void amdvi_do_translate(AMDVIAddressSpace *as, hwaddr addr,
bool is_write, IOMMUTLBEntry *ret)
{
AMDVIState *s = as->iommu_state;
uint16_t devid = PCI_BUILD_BDF(pci_bus_num(as->bus), as->devfn);
AMDVIIOTLBEntry *iotlb_entry = amdvi_iotlb_lookup(s, addr, devid);
uint64_t entry[4];
int dte_ret;
if (iotlb_entry) {
trace_amdvi_iotlb_hit(PCI_BUS_NUM(devid), PCI_SLOT(devid),
PCI_FUNC(devid), addr, iotlb_entry->translated_addr);
ret->iova = addr & ~iotlb_entry->page_mask;
ret->translated_addr = iotlb_entry->translated_addr;
ret->addr_mask = iotlb_entry->page_mask;
ret->perm = iotlb_entry->perms;
return;
}
dte_ret = amdvi_as_to_dte(as, entry);
if (dte_ret < 0) {
if (dte_ret == -AMDVI_FR_DTE_V) {
/* DTE[V]=0, address is passed untranslated */
goto out;
}
return;
}
amdvi_page_walk(as, entry, ret,
is_write ? AMDVI_PERM_WRITE : AMDVI_PERM_READ, addr);
amdvi_update_iotlb(s, devid, addr, *ret,
entry[1] & AMDVI_DEV_DOMID_ID_MASK);
return;
out:
ret->iova = addr & AMDVI_PAGE_MASK_4K;
ret->translated_addr = addr & AMDVI_PAGE_MASK_4K;
ret->addr_mask = ~AMDVI_PAGE_MASK_4K;
ret->perm = IOMMU_RW;
}
static inline bool amdvi_is_interrupt_addr(hwaddr addr)
{
return addr >= AMDVI_INT_ADDR_FIRST && addr <= AMDVI_INT_ADDR_LAST;
}
static IOMMUTLBEntry amdvi_translate(IOMMUMemoryRegion *iommu, hwaddr addr,
IOMMUAccessFlags flag, int iommu_idx)
{
AMDVIAddressSpace *as = container_of(iommu, AMDVIAddressSpace, iommu);
AMDVIState *s = as->iommu_state;
IOMMUTLBEntry ret = {
.target_as = &address_space_memory,
.iova = addr,
.translated_addr = 0,
.addr_mask = ~(hwaddr)0,
.perm = IOMMU_NONE
};
if (!s->enabled) {
/* AMDVI disabled - corresponds to iommu=off not
* failure to provide any parameter
*/
ret.iova = addr & AMDVI_PAGE_MASK_4K;
ret.translated_addr = addr & AMDVI_PAGE_MASK_4K;
ret.addr_mask = ~AMDVI_PAGE_MASK_4K;
ret.perm = IOMMU_RW;
return ret;
} else if (amdvi_is_interrupt_addr(addr)) {
ret.iova = addr & AMDVI_PAGE_MASK_4K;
ret.translated_addr = addr & AMDVI_PAGE_MASK_4K;
ret.addr_mask = ~AMDVI_PAGE_MASK_4K;
ret.perm = IOMMU_WO;
return ret;
}
amdvi_do_translate(as, addr, flag & IOMMU_WO, &ret);
trace_amdvi_translation_result(pci_bus_num(as->bus), PCI_SLOT(as->devfn),
PCI_FUNC(as->devfn), addr, ret.translated_addr);
return ret;
}
static int amdvi_get_irte(AMDVIState *s, MSIMessage *origin, uint64_t *dte,
uint32_t *irte, uint16_t devid)
{
uint64_t irte_root, offset;
irte_root = dte[2] & AMDVI_IR_PHYS_ADDR_MASK;
offset = (origin->data & AMDVI_IRTE_OFFSET) << 2;
trace_amdvi_ir_irte(irte_root, offset);
if (dma_memory_read(&address_space_memory, irte_root + offset,
irte, sizeof(*irte), MEMTXATTRS_UNSPECIFIED)) {
trace_amdvi_ir_err("failed to get irte");
return -AMDVI_IR_GET_IRTE;
}
*irte = le32_to_cpu(*irte);
trace_amdvi_ir_irte_val(*irte);
return 0;
}
static int amdvi_int_remap_legacy(AMDVIState *iommu,
MSIMessage *origin,
MSIMessage *translated,
uint64_t *dte,
X86IOMMUIrq *irq,
uint16_t sid)
{
uint8_t int_type;
uint32_t irte;
int ret;
/* get interrupt remapping table */
ret = amdvi_get_irte(iommu, origin, dte, &irte, sid);
if (ret < 0) {
return ret;
}
if (!FIELD_EX32(irte, AMDVI_IRTE, VALID)) {
trace_amdvi_ir_target_abort("RemapEn is disabled");
return -AMDVI_IR_TARGET_ABORT;
}
if (FIELD_EX32(irte, AMDVI_IRTE, GUEST_MODE)) {
error_report_once("guest mode is not zero");
return -AMDVI_IR_ERR;
}
int_type = FIELD_EX32(irte, AMDVI_IRTE, INT_TYPE);
if (int_type > AMDVI_IOAPIC_INT_TYPE_ARBITRATED) {
error_report_once("reserved int_type");
return -AMDVI_IR_ERR;
}
irq->delivery_mode = int_type;
irq->vector = FIELD_EX32(irte, AMDVI_IRTE, VECTOR);
irq->dest_mode = FIELD_EX32(irte, AMDVI_IRTE, DM);
irq->redir_hint = FIELD_EX32(irte, AMDVI_IRTE, RQ_EOI);
irq->dest = FIELD_EX32(irte, AMDVI_IRTE, DESTINATION);
return 0;
}
static int amdvi_get_irte_ga(AMDVIState *s, MSIMessage *origin, uint64_t *dte,
AMDVIIrteGA *irte, uint16_t devid)
{
uint64_t irte_root, offset;
irte_root = dte[2] & AMDVI_IR_PHYS_ADDR_MASK;
offset = (origin->data & AMDVI_IRTE_OFFSET) << 4;
trace_amdvi_ir_irte(irte_root, offset);
if (dma_memory_read(&address_space_memory, irte_root + offset,
irte, sizeof(*irte), MEMTXATTRS_UNSPECIFIED)) {
trace_amdvi_ir_err("failed to get irte_ga");
return -AMDVI_IR_GET_IRTE;
}
irte->ga_lo = le64_to_cpu(irte->ga_lo);
irte->ga_hi = le64_to_cpu(irte->ga_hi);
trace_amdvi_ir_irte_ga_val(irte->ga_hi, irte->ga_lo);
return 0;
}
static int amdvi_int_remap_ga(AMDVIState *iommu,
MSIMessage *origin,
MSIMessage *translated,
uint64_t *dte,
X86IOMMUIrq *irq,
uint16_t sid)
{
AMDVIIrteGA irte;
uint8_t int_type;
int ret;
/* get interrupt remapping table */
ret = amdvi_get_irte_ga(iommu, origin, dte, &irte, sid);
if (ret < 0) {
return ret;
}
if (!FIELD_EX64(irte.ga_lo, AMDVI_IRTE_GA_LO, VALID)) {
trace_amdvi_ir_target_abort("RemapEn is disabled");
return -AMDVI_IR_TARGET_ABORT;
}
if (FIELD_EX64(irte.ga_lo, AMDVI_IRTE_GA_LO, GUEST_MODE)) {
error_report_once("guest mode is not zero");
return -AMDVI_IR_ERR;
}
int_type = FIELD_EX64(irte.ga_lo, AMDVI_IRTE_GA_LO, INT_TYPE);
if (int_type > AMDVI_IOAPIC_INT_TYPE_ARBITRATED) {
error_report_once("reserved int_type is set");
return -AMDVI_IR_ERR;
}
irq->delivery_mode = int_type;
irq->vector = FIELD_EX64(irte.ga_hi, AMDVI_IRTE_GA_HI, VECTOR);
irq->dest_mode = FIELD_EX64(irte.ga_lo, AMDVI_IRTE_GA_LO, DM);
irq->redir_hint = FIELD_EX64(irte.ga_lo, AMDVI_IRTE_GA_LO, RQ_EOI);
if (iommu->xten) {
irq->dest = FIELD_EX64(irte.ga_lo, AMDVI_IRTE_GA_LO, DESTINATION) |
(FIELD_EX64(irte.ga_hi, AMDVI_IRTE_GA_HI, DESTINATION_HI)
<< 24);
} else {
irq->dest = FIELD_EX64(irte.ga_lo, AMDVI_IRTE_GA_LO, DESTINATION) &
0xff;
}
return 0;
}
static int __amdvi_int_remap_msi(AMDVIState *iommu,
MSIMessage *origin,
MSIMessage *translated,
uint64_t *dte,
X86IOMMUIrq *irq,
uint16_t sid)
{
int ret;
uint8_t int_ctl;
int_ctl = (dte[2] >> AMDVI_IR_INTCTL_SHIFT) & 3;
trace_amdvi_ir_intctl(int_ctl);
switch (int_ctl) {
case AMDVI_IR_INTCTL_PASS:
memcpy(translated, origin, sizeof(*origin));
return 0;
case AMDVI_IR_INTCTL_REMAP:
break;
case AMDVI_IR_INTCTL_ABORT:
trace_amdvi_ir_target_abort("int_ctl abort");
return -AMDVI_IR_TARGET_ABORT;
default:
trace_amdvi_ir_err("int_ctl reserved");
return -AMDVI_IR_ERR;
}
if (iommu->ga_enabled) {
ret = amdvi_int_remap_ga(iommu, origin, translated, dte, irq, sid);
} else {
ret = amdvi_int_remap_legacy(iommu, origin, translated, dte, irq, sid);
}
return ret;
}
/* Interrupt remapping for MSI/MSI-X entry */
static int amdvi_int_remap_msi(AMDVIState *iommu,
MSIMessage *origin,
MSIMessage *translated,
uint16_t sid)
{
int ret = 0;
uint64_t pass = 0;
uint64_t dte[4] = { 0 };
X86IOMMUIrq irq = { 0 };
uint8_t dest_mode, delivery_mode;
assert(origin && translated);
/*
* When IOMMU is enabled, interrupt remap request will come either from
* IO-APIC or PCI device. If interrupt is from PCI device then it will
* have a valid requester id but if the interrupt is from IO-APIC
* then requester id will be invalid.
*/
if (sid == X86_IOMMU_SID_INVALID) {
sid = AMDVI_IOAPIC_SB_DEVID;
}
trace_amdvi_ir_remap_msi_req(origin->address, origin->data, sid);
/* check if device table entry is set before we go further. */
if (!iommu || !iommu->devtab_len) {
memcpy(translated, origin, sizeof(*origin));
goto out;
}
if (!amdvi_get_dte(iommu, sid, dte)) {
return -AMDVI_IR_ERR;
}
/* Check if IR is enabled in DTE */
if (!(dte[2] & AMDVI_IR_REMAP_ENABLE)) {
memcpy(translated, origin, sizeof(*origin));
goto out;
}
/* validate that we are configure with intremap=on */
if (!x86_iommu_ir_supported(X86_IOMMU_DEVICE(iommu))) {
trace_amdvi_err("Interrupt remapping is enabled in the guest but "
"not in the host. Use intremap=on to enable interrupt "
"remapping in amd-iommu.");
return -AMDVI_IR_ERR;
}
if (origin->address < AMDVI_INT_ADDR_FIRST ||
origin->address + sizeof(origin->data) > AMDVI_INT_ADDR_LAST + 1) {
trace_amdvi_err("MSI is not from IOAPIC.");
return -AMDVI_IR_ERR;
}
/*
* The MSI data register [10:8] are used to get the upstream interrupt type.
*
* See MSI/MSI-X format:
* https://pdfs.semanticscholar.org/presentation/9420/c279e942eca568157711ef5c92b800c40a79.pdf
* (page 5)
*/
delivery_mode = (origin->data >> MSI_DATA_DELIVERY_MODE_SHIFT) & 7;
switch (delivery_mode) {
case AMDVI_IOAPIC_INT_TYPE_FIXED:
case AMDVI_IOAPIC_INT_TYPE_ARBITRATED:
trace_amdvi_ir_delivery_mode("fixed/arbitrated");
ret = __amdvi_int_remap_msi(iommu, origin, translated, dte, &irq, sid);
if (ret < 0) {
goto remap_fail;
} else {
/* Translate IRQ to MSI messages */
x86_iommu_irq_to_msi_message(&irq, translated);
goto out;
}
break;
case AMDVI_IOAPIC_INT_TYPE_SMI:
error_report("SMI is not supported!");
ret = -AMDVI_IR_ERR;
break;
case AMDVI_IOAPIC_INT_TYPE_NMI:
pass = dte[2] & AMDVI_DEV_NMI_PASS_MASK;
trace_amdvi_ir_delivery_mode("nmi");
break;
case AMDVI_IOAPIC_INT_TYPE_INIT:
pass = dte[2] & AMDVI_DEV_INT_PASS_MASK;
trace_amdvi_ir_delivery_mode("init");
break;
case AMDVI_IOAPIC_INT_TYPE_EINT:
pass = dte[2] & AMDVI_DEV_EINT_PASS_MASK;
trace_amdvi_ir_delivery_mode("eint");
break;
default:
trace_amdvi_ir_delivery_mode("unsupported delivery_mode");
ret = -AMDVI_IR_ERR;
break;
}
if (ret < 0) {
goto remap_fail;
}
/*
* The MSI address register bit[2] is used to get the destination
* mode. The dest_mode 1 is valid for fixed and arbitrated interrupts
* only.
*/
dest_mode = (origin->address >> MSI_ADDR_DEST_MODE_SHIFT) & 1;
if (dest_mode) {
trace_amdvi_ir_err("invalid dest_mode");
ret = -AMDVI_IR_ERR;
goto remap_fail;
}
if (pass) {
memcpy(translated, origin, sizeof(*origin));
} else {
trace_amdvi_ir_err("passthrough is not enabled");
ret = -AMDVI_IR_ERR;
goto remap_fail;
}
out:
trace_amdvi_ir_remap_msi(origin->address, origin->data,
translated->address, translated->data);
return 0;
remap_fail:
return ret;
}
static int amdvi_int_remap(X86IOMMUState *iommu,
MSIMessage *origin,
MSIMessage *translated,
uint16_t sid)
{
return amdvi_int_remap_msi(AMD_IOMMU_DEVICE(iommu), origin,
translated, sid);
}
static MemTxResult amdvi_mem_ir_write(void *opaque, hwaddr addr,
uint64_t value, unsigned size,
MemTxAttrs attrs)
{
int ret;
MSIMessage from = { 0, 0 }, to = { 0, 0 };
uint16_t sid = AMDVI_IOAPIC_SB_DEVID;
from.address = (uint64_t) addr + AMDVI_INT_ADDR_FIRST;
from.data = (uint32_t) value;
trace_amdvi_mem_ir_write_req(addr, value, size);
if (!attrs.unspecified) {
/* We have explicit Source ID */
sid = attrs.requester_id;
}
ret = amdvi_int_remap_msi(opaque, &from, &to, sid);
if (ret < 0) {
/* TODO: log the event using IOMMU log event interface */
error_report_once("failed to remap interrupt from devid 0x%x", sid);
return MEMTX_ERROR;
}
apic_get_class(NULL)->send_msi(&to);
trace_amdvi_mem_ir_write(to.address, to.data);
return MEMTX_OK;
}
static MemTxResult amdvi_mem_ir_read(void *opaque, hwaddr addr,
uint64_t *data, unsigned size,
MemTxAttrs attrs)
{
return MEMTX_OK;
}
static const MemoryRegionOps amdvi_ir_ops = {
.read_with_attrs = amdvi_mem_ir_read,
.write_with_attrs = amdvi_mem_ir_write,
.endianness = DEVICE_LITTLE_ENDIAN,
.impl = {
.min_access_size = 4,
.max_access_size = 4,
},
.valid = {
.min_access_size = 4,
.max_access_size = 4,
}
};
static AddressSpace *amdvi_host_dma_iommu(PCIBus *bus, void *opaque, int devfn)
{
char name[128];
AMDVIState *s = opaque;
AMDVIAddressSpace *amdvi_dev_as;
AMDVIAsKey *key;
amdvi_dev_as = amdvi_as_lookup(s, bus, devfn);
/* allocate memory during the first run */
if (!amdvi_dev_as) {
snprintf(name, sizeof(name), "amd_iommu_devfn_%d", devfn);
amdvi_dev_as = g_new0(AMDVIAddressSpace, 1);
key = g_new0(AMDVIAsKey, 1);
amdvi_dev_as->bus = bus;
amdvi_dev_as->devfn = (uint8_t)devfn;
amdvi_dev_as->iommu_state = s;
amdvi_dev_as->notifier_flags = IOMMU_NOTIFIER_NONE;
amdvi_dev_as->iova_tree = iova_tree_new();
amdvi_dev_as->addr_translation = false;
key->bus = bus;
key->devfn = devfn;
g_hash_table_insert(s->address_spaces, key, amdvi_dev_as);
/*
* Memory region relationships looks like (Address range shows
* only lower 32 bits to make it short in length...):
*
* |--------------------+-------------------+----------|
* | Name | Address range | Priority |
* |--------------------+-------------------+----------+
* | amdvi-root | 00000000-ffffffff | 0 |
* | amdvi-iommu_nodma | 00000000-ffffffff | 0 |
* | amdvi-iommu_ir | fee00000-feefffff | 1 |
* |--------------------+-------------------+----------|
*/
memory_region_init_iommu(&amdvi_dev_as->iommu,
sizeof(amdvi_dev_as->iommu),
TYPE_AMD_IOMMU_MEMORY_REGION,
OBJECT(s),
"amd_iommu", UINT64_MAX);
memory_region_init(&amdvi_dev_as->root, OBJECT(s),
"amdvi_root", UINT64_MAX);
address_space_init(&amdvi_dev_as->as, &amdvi_dev_as->root, name);
memory_region_add_subregion_overlap(&amdvi_dev_as->root, 0,
MEMORY_REGION(&amdvi_dev_as->iommu),
0);
/* Build the DMA Disabled alias to shared memory */
memory_region_init_alias(&amdvi_dev_as->iommu_nodma, OBJECT(s),
"amdvi-sys", &s->mr_sys, 0,
memory_region_size(&s->mr_sys));
memory_region_add_subregion_overlap(&amdvi_dev_as->root, 0,
&amdvi_dev_as->iommu_nodma,
0);
/* Build the Interrupt Remapping alias to shared memory */
memory_region_init_alias(&amdvi_dev_as->iommu_ir, OBJECT(s),
"amdvi-ir", &s->mr_ir, 0,
memory_region_size(&s->mr_ir));
memory_region_add_subregion_overlap(MEMORY_REGION(&amdvi_dev_as->iommu),
AMDVI_INT_ADDR_FIRST,
&amdvi_dev_as->iommu_ir, 1);
amdvi_switch_address_space(amdvi_dev_as);
}
return &amdvi_dev_as->as;
}
static const PCIIOMMUOps amdvi_iommu_ops = {
.get_address_space = amdvi_host_dma_iommu,
};
static const MemoryRegionOps mmio_mem_ops = {
.read = amdvi_mmio_read,
.write = amdvi_mmio_write,
.endianness = DEVICE_LITTLE_ENDIAN,
.impl = {
.min_access_size = 1,
.max_access_size = 8,
.unaligned = false,
},
.valid = {
.min_access_size = 1,
.max_access_size = 8,
}
};
static int amdvi_iommu_notify_flag_changed(IOMMUMemoryRegion *iommu,
IOMMUNotifierFlag old,
IOMMUNotifierFlag new,
Error **errp)
{
AMDVIAddressSpace *as = container_of(iommu, AMDVIAddressSpace, iommu);
AMDVIState *s = as->iommu_state;
/*
* Accurate synchronization of the vIOMMU page tables required to support
* MAP notifiers is provided by the dma-remap feature. In addition, this
* also requires that the vIOMMU presents the NpCache capability, so a guest
* driver issues invalidations for both map() and unmap() operations. The
* capability is already set by default as part of AMDVI_CAPAB_FEATURES and
* written to the configuration in amdvi_pci_realize().
*/
if (!s->dma_remap && (new & IOMMU_NOTIFIER_MAP)) {
error_setg_errno(errp, ENOTSUP,
"device %02x.%02x.%x requires dma-remap=1",
pci_bus_num(as->bus), PCI_SLOT(as->devfn), PCI_FUNC(as->devfn));
return -ENOTSUP;
}
/*
* Update notifier flags for address space and the list of address spaces
* with registered notifiers.
*/
as->notifier_flags = new;
if (old == IOMMU_NOTIFIER_NONE) {
QLIST_INSERT_HEAD(&s->amdvi_as_with_notifiers, as, next);
} else if (new == IOMMU_NOTIFIER_NONE) {
QLIST_REMOVE(as, next);
}
return 0;
}
static void amdvi_init(AMDVIState *s)
{
amdvi_iotlb_reset(s);
s->devtab_len = 0;
s->cmdbuf_len = 0;
s->cmdbuf_head = 0;
s->cmdbuf_tail = 0;
s->evtlog_head = 0;
s->evtlog_tail = 0;
s->excl_enabled = false;
s->excl_allow = false;
s->mmio_enabled = false;
s->enabled = false;
s->cmdbuf_enabled = false;
s->xten = false;
s->intcapxten = false;
/* reset MMIO */
memset(s->mmior, 0, AMDVI_MMIO_SIZE);
amdvi_set_quad(s, AMDVI_MMIO_EXT_FEATURES,
amdvi_extended_feature_register(s),
0xffffffffffffffef, 0);
amdvi_set_quad(s, AMDVI_MMIO_STATUS, 0, 0x98, 0x67);
}
static void amdvi_pci_realize(PCIDevice *pdev, Error **errp)
{
AMDVIPCIState *s = AMD_IOMMU_PCI(pdev);
int ret;
ret = pci_add_capability(pdev, AMDVI_CAPAB_ID_SEC, 0,
AMDVI_CAPAB_SIZE, errp);
if (ret < 0) {
return;
}
s->capab_offset = ret;
ret = pci_add_capability(pdev, PCI_CAP_ID_MSI, 0,
AMDVI_CAPAB_REG_SIZE, errp);
if (ret < 0) {
return;
}
ret = pci_add_capability(pdev, PCI_CAP_ID_HT, 0,
AMDVI_CAPAB_REG_SIZE, errp);
if (ret < 0) {
return;
}
if (msi_init(pdev, 0, 1, true, false, errp) < 0) {
return;
}
/* reset device ident */
pci_config_set_prog_interface(pdev->config, 0);
/* reset AMDVI specific capabilities, all r/o */
pci_set_long(pdev->config + s->capab_offset, AMDVI_CAPAB_FEATURES);
pci_set_long(pdev->config + s->capab_offset + AMDVI_CAPAB_BAR_LOW,
AMDVI_BASE_ADDR & MAKE_64BIT_MASK(14, 18));
pci_set_long(pdev->config + s->capab_offset + AMDVI_CAPAB_BAR_HIGH,
AMDVI_BASE_ADDR >> 32);
pci_set_long(pdev->config + s->capab_offset + AMDVI_CAPAB_RANGE,
0xff000000);
pci_set_long(pdev->config + s->capab_offset + AMDVI_CAPAB_MISC, 0);
pci_set_long(pdev->config + s->capab_offset + AMDVI_CAPAB_MISC,
AMDVI_MAX_PH_ADDR | AMDVI_MAX_GVA_ADDR | AMDVI_MAX_VA_ADDR);
}
static void amdvi_sysbus_reset(DeviceState *dev)
{
AMDVIState *s = AMD_IOMMU_DEVICE(dev);
msi_reset(&s->pci->dev);
amdvi_init(s);
/* Discard all mappings on device reset */
amdvi_address_space_unmap_all(s);
amdvi_reset_address_translation_all(s);
}
static const VMStateDescription vmstate_xt = {
.name = "amd-iommu-xt",
.version_id = 1,
.minimum_version_id = 1,
.fields = (VMStateField[]) {
VMSTATE_BOOL(xten, AMDVIState),
VMSTATE_BOOL(intcapxten, AMDVIState),
VMSTATE_END_OF_LIST()
}
};
static const VMStateDescription vmstate_amdvi_sysbus_migratable = {
.name = "amd-iommu",
.version_id = 1,
.minimum_version_id = 1,
.priority = MIG_PRI_IOMMU,
.fields = (VMStateField[]) {
/* Updated in amdvi_handle_control_write() */
VMSTATE_BOOL(enabled, AMDVIState),
VMSTATE_BOOL(ga_enabled, AMDVIState),
/* bool ats_enabled is obsolete */
VMSTATE_UNUSED(1), /* was ats_enabled */
VMSTATE_BOOL(cmdbuf_enabled, AMDVIState),
VMSTATE_BOOL(completion_wait_intr, AMDVIState),
VMSTATE_BOOL(evtlog_enabled, AMDVIState),
VMSTATE_BOOL(evtlog_intr, AMDVIState),
/* Updated in amdvi_handle_devtab_write() */
VMSTATE_UINT64(devtab, AMDVIState),
VMSTATE_UINT64(devtab_len, AMDVIState),
/* Updated in amdvi_handle_cmdbase_write() */
VMSTATE_UINT64(cmdbuf, AMDVIState),
VMSTATE_UINT64(cmdbuf_len, AMDVIState),
/* Updated in amdvi_handle_cmdhead_write() */
VMSTATE_UINT32(cmdbuf_head, AMDVIState),
/* Updated in amdvi_handle_cmdtail_write() */
VMSTATE_UINT32(cmdbuf_tail, AMDVIState),
/* Updated in amdvi_handle_evtbase_write() */
VMSTATE_UINT64(evtlog, AMDVIState),
VMSTATE_UINT32(evtlog_len, AMDVIState),
/* Updated in amdvi_handle_evthead_write() */
VMSTATE_UINT32(evtlog_head, AMDVIState),
/* Updated in amdvi_handle_evttail_write() */
VMSTATE_UINT32(evtlog_tail, AMDVIState),
/* Updated in amdvi_handle_pprbase_write() */
VMSTATE_UINT64(ppr_log, AMDVIState),
VMSTATE_UINT32(pprlog_len, AMDVIState),
/* Updated in amdvi_handle_pprhead_write() */
VMSTATE_UINT32(pprlog_head, AMDVIState),
/* Updated in amdvi_handle_tailhead_write() */
VMSTATE_UINT32(pprlog_tail, AMDVIState),
/* MMIO registers */
VMSTATE_UINT8_ARRAY(mmior, AMDVIState, AMDVI_MMIO_SIZE),
VMSTATE_UINT8_ARRAY(romask, AMDVIState, AMDVI_MMIO_SIZE),
VMSTATE_UINT8_ARRAY(w1cmask, AMDVIState, AMDVI_MMIO_SIZE),
VMSTATE_END_OF_LIST()
},
.subsections = (const VMStateDescription *const []) {
&vmstate_xt,
NULL
}
};
static void amdvi_sysbus_realize(DeviceState *dev, Error **errp)
{
DeviceClass *dc = (DeviceClass *) object_get_class(OBJECT(dev));
AMDVIState *s = AMD_IOMMU_DEVICE(dev);
MachineState *ms = MACHINE(qdev_get_machine());
PCMachineState *pcms = PC_MACHINE(ms);
X86MachineState *x86ms = X86_MACHINE(ms);
PCIBus *bus = pcms->pcibus;
if (s->pci_id) {
PCIDevice *pdev = NULL;
int ret = pci_qdev_find_device(s->pci_id, &pdev);
if (ret) {
error_report("Cannot find PCI device '%s'", s->pci_id);
return;
}
if (!object_dynamic_cast(OBJECT(pdev), TYPE_AMD_IOMMU_PCI)) {
error_report("Device '%s' must be an AMDVI-PCI device type", s->pci_id);
return;
}
s->pci = AMD_IOMMU_PCI(pdev);
dc->vmsd = &vmstate_amdvi_sysbus_migratable;
} else {
s->pci = AMD_IOMMU_PCI(object_new(TYPE_AMD_IOMMU_PCI));
/* This device should take care of IOMMU PCI properties */
if (!qdev_realize(DEVICE(s->pci), &bus->qbus, errp)) {
return;
}
}
s->iotlb = g_hash_table_new_full(amdvi_iotlb_hash,
amdvi_iotlb_equal, g_free, g_free);
s->address_spaces = g_hash_table_new_full(amdvi_as_hash,
amdvi_as_equal, g_free, g_free);
/* set up MMIO */
memory_region_init_io(&s->mr_mmio, OBJECT(s), &mmio_mem_ops, s,
"amdvi-mmio", AMDVI_MMIO_SIZE);
memory_region_add_subregion(get_system_memory(), AMDVI_BASE_ADDR,
&s->mr_mmio);
/* Create the share memory regions by all devices */
memory_region_init(&s->mr_sys, OBJECT(s), "amdvi-sys", UINT64_MAX);
/* set up the DMA disabled memory region */
memory_region_init_alias(&s->mr_nodma, OBJECT(s),
"amdvi-nodma", get_system_memory(), 0,
memory_region_size(get_system_memory()));
memory_region_add_subregion_overlap(&s->mr_sys, 0,
&s->mr_nodma, 0);
/* set up the Interrupt Remapping memory region */
memory_region_init_io(&s->mr_ir, OBJECT(s), &amdvi_ir_ops,
s, "amdvi-ir", AMDVI_INT_ADDR_SIZE);
memory_region_add_subregion_overlap(&s->mr_sys, AMDVI_INT_ADDR_FIRST,
&s->mr_ir, 1);
/* Pseudo address space under root PCI bus. */
x86ms->ioapic_as = amdvi_host_dma_iommu(bus, s, AMDVI_IOAPIC_SB_DEVID);
if (kvm_enabled() && x86ms->apic_id_limit > 255 && !s->xtsup) {
error_report("AMD IOMMU with x2APIC configuration requires xtsup=on");
exit(EXIT_FAILURE);
}
if (s->xtsup) {
if (kvm_irqchip_is_split() && !kvm_enable_x2apic()) {
error_report("AMD IOMMU xtsup=on requires x2APIC support on "
"the KVM side");
exit(EXIT_FAILURE);
}
}
pci_setup_iommu(bus, &amdvi_iommu_ops, s);
amdvi_init(s);
}
static const Property amdvi_properties[] = {
DEFINE_PROP_BOOL("xtsup", AMDVIState, xtsup, false),
DEFINE_PROP_STRING("pci-id", AMDVIState, pci_id),
DEFINE_PROP_BOOL("dma-remap", AMDVIState, dma_remap, false),
};
static const VMStateDescription vmstate_amdvi_sysbus = {
.name = "amd-iommu",
.unmigratable = 1
};
static void amdvi_sysbus_class_init(ObjectClass *klass, const void *data)
{
DeviceClass *dc = DEVICE_CLASS(klass);
X86IOMMUClass *dc_class = X86_IOMMU_DEVICE_CLASS(klass);
device_class_set_legacy_reset(dc, amdvi_sysbus_reset);
dc->vmsd = &vmstate_amdvi_sysbus;
dc->hotpluggable = false;
dc_class->realize = amdvi_sysbus_realize;
dc_class->int_remap = amdvi_int_remap;
set_bit(DEVICE_CATEGORY_MISC, dc->categories);
dc->desc = "AMD IOMMU (AMD-Vi) DMA Remapping device";
device_class_set_props(dc, amdvi_properties);
}
static const TypeInfo amdvi_sysbus = {
.name = TYPE_AMD_IOMMU_DEVICE,
.parent = TYPE_X86_IOMMU_DEVICE,
.instance_size = sizeof(AMDVIState),
.class_init = amdvi_sysbus_class_init
};
static void amdvi_pci_class_init(ObjectClass *klass, const void *data)
{
DeviceClass *dc = DEVICE_CLASS(klass);
PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
k->vendor_id = PCI_VENDOR_ID_AMD;
k->device_id = 0x1419;
k->class_id = 0x0806;
k->realize = amdvi_pci_realize;
set_bit(DEVICE_CATEGORY_MISC, dc->categories);
dc->desc = "AMD IOMMU (AMD-Vi) DMA Remapping device";
}
static const TypeInfo amdvi_pci = {
.name = TYPE_AMD_IOMMU_PCI,
.parent = TYPE_PCI_DEVICE,
.instance_size = sizeof(AMDVIPCIState),
.class_init = amdvi_pci_class_init,
.interfaces = (const InterfaceInfo[]) {
{ INTERFACE_CONVENTIONAL_PCI_DEVICE },
{ },
},
};
static void amdvi_iommu_memory_region_class_init(ObjectClass *klass,
const void *data)
{
IOMMUMemoryRegionClass *imrc = IOMMU_MEMORY_REGION_CLASS(klass);
imrc->translate = amdvi_translate;
imrc->notify_flag_changed = amdvi_iommu_notify_flag_changed;
imrc->replay = amdvi_iommu_replay;
}
static const TypeInfo amdvi_iommu_memory_region_info = {
.parent = TYPE_IOMMU_MEMORY_REGION,
.name = TYPE_AMD_IOMMU_MEMORY_REGION,
.class_init = amdvi_iommu_memory_region_class_init,
};
static void amdvi_register_types(void)
{
type_register_static(&amdvi_pci);
type_register_static(&amdvi_sysbus);
type_register_static(&amdvi_iommu_memory_region_info);
}
type_init(amdvi_register_types);
Reference in New Issue View Git Blame Copy Permalink