Import QEMU upstream snapshot d2e570c
Upstream: https://gitlab.com/qemu-project/qemu.git Upstream-Commit: d2e570cc0f97b936902a5b1b86b73c0f5998b475
This commit is contained in:
@@ -0,0 +1,5 @@
|
||||
config NVME_PCI
|
||||
bool
|
||||
default y if PCI_DEVICES || PCIE_DEVICES
|
||||
depends on PCI
|
||||
select SPDM_SOCKET
|
||||
+10696
File diff suppressed because it is too large
Load Diff
+711
@@ -0,0 +1,711 @@
|
||||
/*
|
||||
* QEMU NVM Express End-to-End Data Protection support
|
||||
*
|
||||
* Copyright (c) 2021 Samsung Electronics Co., Ltd.
|
||||
*
|
||||
* Authors:
|
||||
* Klaus Jensen <[email protected]>
|
||||
* Gollu Appalanaidu <[email protected]>
|
||||
*/
|
||||
|
||||
#include "qemu/osdep.h"
|
||||
#include "qapi/error.h"
|
||||
#include "system/block-backend.h"
|
||||
|
||||
#include "nvme.h"
|
||||
#include "dif.h"
|
||||
#include "trace.h"
|
||||
|
||||
uint16_t nvme_check_prinfo(NvmeNamespace *ns, uint8_t prinfo, uint64_t slba,
|
||||
uint64_t reftag)
|
||||
{
|
||||
uint64_t mask = ns->pif ? 0xffffffffffff : 0xffffffff;
|
||||
|
||||
if ((NVME_ID_NS_DPS_TYPE(ns->id_ns.dps) == NVME_ID_NS_DPS_TYPE_1) &&
|
||||
(prinfo & NVME_PRINFO_PRCHK_REF) && (slba & mask) != reftag) {
|
||||
return NVME_INVALID_PROT_INFO | NVME_DNR;
|
||||
}
|
||||
|
||||
if ((NVME_ID_NS_DPS_TYPE(ns->id_ns.dps) == NVME_ID_NS_DPS_TYPE_3) &&
|
||||
(prinfo & NVME_PRINFO_PRCHK_REF)) {
|
||||
return NVME_INVALID_PROT_INFO;
|
||||
}
|
||||
|
||||
return NVME_SUCCESS;
|
||||
}
|
||||
|
||||
/* from Linux kernel (crypto/crct10dif_common.c) */
|
||||
static uint16_t crc16_t10dif(uint16_t crc, const unsigned char *buffer,
|
||||
size_t len)
|
||||
{
|
||||
unsigned int i;
|
||||
|
||||
for (i = 0; i < len; i++) {
|
||||
crc = (crc << 8) ^ crc16_t10dif_table[((crc >> 8) ^ buffer[i]) & 0xff];
|
||||
}
|
||||
|
||||
return crc;
|
||||
}
|
||||
|
||||
/* from Linux kernel (lib/crc64.c) */
|
||||
static uint64_t crc64_nvme(uint64_t crc, const unsigned char *buffer,
|
||||
size_t len)
|
||||
{
|
||||
size_t i;
|
||||
|
||||
for (i = 0; i < len; i++) {
|
||||
crc = (crc >> 8) ^ crc64_nvme_table[(crc & 0xff) ^ buffer[i]];
|
||||
}
|
||||
|
||||
return crc ^ (uint64_t)~0;
|
||||
}
|
||||
|
||||
static void nvme_dif_pract_generate_dif_crc16(NvmeNamespace *ns, uint8_t *buf,
|
||||
size_t len, uint8_t *mbuf,
|
||||
size_t mlen, uint16_t apptag,
|
||||
uint64_t *reftag)
|
||||
{
|
||||
uint8_t *end = buf + len;
|
||||
int16_t pil = 0;
|
||||
|
||||
if (!(ns->id_ns.dps & NVME_ID_NS_DPS_FIRST_EIGHT)) {
|
||||
pil = ns->lbaf.ms - nvme_pi_tuple_size(ns);
|
||||
}
|
||||
|
||||
trace_pci_nvme_dif_pract_generate_dif_crc16(len, ns->lbasz,
|
||||
ns->lbasz + pil, apptag,
|
||||
*reftag);
|
||||
|
||||
for (; buf < end; buf += ns->lbasz, mbuf += ns->lbaf.ms) {
|
||||
NvmeDifTuple *dif = (NvmeDifTuple *)(mbuf + pil);
|
||||
uint16_t crc = crc16_t10dif(0x0, buf, ns->lbasz);
|
||||
|
||||
if (pil) {
|
||||
crc = crc16_t10dif(crc, mbuf, pil);
|
||||
}
|
||||
|
||||
dif->g16.guard = cpu_to_be16(crc);
|
||||
dif->g16.apptag = cpu_to_be16(apptag);
|
||||
dif->g16.reftag = cpu_to_be32(*reftag);
|
||||
|
||||
if (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps) != NVME_ID_NS_DPS_TYPE_3) {
|
||||
(*reftag)++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static void nvme_dif_pract_generate_dif_crc64(NvmeNamespace *ns, uint8_t *buf,
|
||||
size_t len, uint8_t *mbuf,
|
||||
size_t mlen, uint16_t apptag,
|
||||
uint64_t *reftag)
|
||||
{
|
||||
uint8_t *end = buf + len;
|
||||
int16_t pil = 0;
|
||||
|
||||
if (!(ns->id_ns.dps & NVME_ID_NS_DPS_FIRST_EIGHT)) {
|
||||
pil = ns->lbaf.ms - 16;
|
||||
}
|
||||
|
||||
trace_pci_nvme_dif_pract_generate_dif_crc64(len, ns->lbasz,
|
||||
ns->lbasz + pil, apptag,
|
||||
*reftag);
|
||||
|
||||
for (; buf < end; buf += ns->lbasz, mbuf += ns->lbaf.ms) {
|
||||
NvmeDifTuple *dif = (NvmeDifTuple *)(mbuf + pil);
|
||||
uint64_t crc = crc64_nvme(~0ULL, buf, ns->lbasz);
|
||||
|
||||
if (pil) {
|
||||
crc = crc64_nvme(~crc, mbuf, pil);
|
||||
}
|
||||
|
||||
dif->g64.guard = cpu_to_be64(crc);
|
||||
dif->g64.apptag = cpu_to_be16(apptag);
|
||||
|
||||
dif->g64.sr[0] = *reftag >> 40;
|
||||
dif->g64.sr[1] = *reftag >> 32;
|
||||
dif->g64.sr[2] = *reftag >> 24;
|
||||
dif->g64.sr[3] = *reftag >> 16;
|
||||
dif->g64.sr[4] = *reftag >> 8;
|
||||
dif->g64.sr[5] = *reftag;
|
||||
|
||||
if (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps) != NVME_ID_NS_DPS_TYPE_3) {
|
||||
(*reftag)++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void nvme_dif_pract_generate_dif(NvmeNamespace *ns, uint8_t *buf, size_t len,
|
||||
uint8_t *mbuf, size_t mlen, uint16_t apptag,
|
||||
uint64_t *reftag)
|
||||
{
|
||||
switch (ns->pif) {
|
||||
case NVME_PI_GUARD_16:
|
||||
return nvme_dif_pract_generate_dif_crc16(ns, buf, len, mbuf, mlen,
|
||||
apptag, reftag);
|
||||
case NVME_PI_GUARD_64:
|
||||
return nvme_dif_pract_generate_dif_crc64(ns, buf, len, mbuf, mlen,
|
||||
apptag, reftag);
|
||||
}
|
||||
|
||||
abort();
|
||||
}
|
||||
|
||||
static uint16_t nvme_dif_prchk_crc16(NvmeNamespace *ns, NvmeDifTuple *dif,
|
||||
uint8_t *buf, uint8_t *mbuf, size_t pil,
|
||||
uint8_t prinfo, uint16_t apptag,
|
||||
uint16_t appmask, uint64_t reftag)
|
||||
{
|
||||
switch (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps)) {
|
||||
case NVME_ID_NS_DPS_TYPE_3:
|
||||
if (be32_to_cpu(dif->g16.reftag) != 0xffffffff) {
|
||||
break;
|
||||
}
|
||||
|
||||
/* fallthrough */
|
||||
case NVME_ID_NS_DPS_TYPE_1:
|
||||
case NVME_ID_NS_DPS_TYPE_2:
|
||||
if (be16_to_cpu(dif->g16.apptag) != 0xffff) {
|
||||
break;
|
||||
}
|
||||
|
||||
trace_pci_nvme_dif_prchk_disabled_crc16(be16_to_cpu(dif->g16.apptag),
|
||||
be32_to_cpu(dif->g16.reftag));
|
||||
|
||||
return NVME_SUCCESS;
|
||||
}
|
||||
|
||||
if (prinfo & NVME_PRINFO_PRCHK_GUARD) {
|
||||
uint16_t crc = crc16_t10dif(0x0, buf, ns->lbasz);
|
||||
|
||||
if (pil) {
|
||||
crc = crc16_t10dif(crc, mbuf, pil);
|
||||
}
|
||||
|
||||
trace_pci_nvme_dif_prchk_guard_crc16(be16_to_cpu(dif->g16.guard), crc);
|
||||
|
||||
if (be16_to_cpu(dif->g16.guard) != crc) {
|
||||
return NVME_E2E_GUARD_ERROR;
|
||||
}
|
||||
}
|
||||
|
||||
if (prinfo & NVME_PRINFO_PRCHK_APP) {
|
||||
trace_pci_nvme_dif_prchk_apptag(be16_to_cpu(dif->g16.apptag), apptag,
|
||||
appmask);
|
||||
|
||||
if ((be16_to_cpu(dif->g16.apptag) & appmask) != (apptag & appmask)) {
|
||||
return NVME_E2E_APP_ERROR;
|
||||
}
|
||||
}
|
||||
|
||||
if (prinfo & NVME_PRINFO_PRCHK_REF) {
|
||||
trace_pci_nvme_dif_prchk_reftag_crc16(be32_to_cpu(dif->g16.reftag),
|
||||
reftag);
|
||||
|
||||
if (be32_to_cpu(dif->g16.reftag) != reftag) {
|
||||
return NVME_E2E_REF_ERROR;
|
||||
}
|
||||
}
|
||||
|
||||
return NVME_SUCCESS;
|
||||
}
|
||||
|
||||
static uint16_t nvme_dif_prchk_crc64(NvmeNamespace *ns, NvmeDifTuple *dif,
|
||||
uint8_t *buf, uint8_t *mbuf, size_t pil,
|
||||
uint8_t prinfo, uint16_t apptag,
|
||||
uint16_t appmask, uint64_t reftag)
|
||||
{
|
||||
uint64_t r = 0;
|
||||
|
||||
r |= (uint64_t)dif->g64.sr[0] << 40;
|
||||
r |= (uint64_t)dif->g64.sr[1] << 32;
|
||||
r |= (uint64_t)dif->g64.sr[2] << 24;
|
||||
r |= (uint64_t)dif->g64.sr[3] << 16;
|
||||
r |= (uint64_t)dif->g64.sr[4] << 8;
|
||||
r |= (uint64_t)dif->g64.sr[5];
|
||||
|
||||
switch (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps)) {
|
||||
case NVME_ID_NS_DPS_TYPE_3:
|
||||
if (r != 0xffffffffffff) {
|
||||
break;
|
||||
}
|
||||
|
||||
/* fallthrough */
|
||||
case NVME_ID_NS_DPS_TYPE_1:
|
||||
case NVME_ID_NS_DPS_TYPE_2:
|
||||
if (be16_to_cpu(dif->g64.apptag) != 0xffff) {
|
||||
break;
|
||||
}
|
||||
|
||||
trace_pci_nvme_dif_prchk_disabled_crc64(be16_to_cpu(dif->g16.apptag),
|
||||
r);
|
||||
|
||||
return NVME_SUCCESS;
|
||||
}
|
||||
|
||||
if (prinfo & NVME_PRINFO_PRCHK_GUARD) {
|
||||
uint64_t crc = crc64_nvme(~0ULL, buf, ns->lbasz);
|
||||
|
||||
if (pil) {
|
||||
crc = crc64_nvme(~crc, mbuf, pil);
|
||||
}
|
||||
|
||||
trace_pci_nvme_dif_prchk_guard_crc64(be64_to_cpu(dif->g64.guard), crc);
|
||||
|
||||
if (be64_to_cpu(dif->g64.guard) != crc) {
|
||||
return NVME_E2E_GUARD_ERROR;
|
||||
}
|
||||
}
|
||||
|
||||
if (prinfo & NVME_PRINFO_PRCHK_APP) {
|
||||
trace_pci_nvme_dif_prchk_apptag(be16_to_cpu(dif->g64.apptag), apptag,
|
||||
appmask);
|
||||
|
||||
if ((be16_to_cpu(dif->g64.apptag) & appmask) != (apptag & appmask)) {
|
||||
return NVME_E2E_APP_ERROR;
|
||||
}
|
||||
}
|
||||
|
||||
if (prinfo & NVME_PRINFO_PRCHK_REF) {
|
||||
trace_pci_nvme_dif_prchk_reftag_crc64(r, reftag);
|
||||
|
||||
if (r != reftag) {
|
||||
return NVME_E2E_REF_ERROR;
|
||||
}
|
||||
}
|
||||
|
||||
return NVME_SUCCESS;
|
||||
}
|
||||
|
||||
static uint16_t nvme_dif_prchk(NvmeNamespace *ns, NvmeDifTuple *dif,
|
||||
uint8_t *buf, uint8_t *mbuf, size_t pil,
|
||||
uint8_t prinfo, uint16_t apptag,
|
||||
uint16_t appmask, uint64_t reftag)
|
||||
{
|
||||
switch (ns->pif) {
|
||||
case NVME_PI_GUARD_16:
|
||||
return nvme_dif_prchk_crc16(ns, dif, buf, mbuf, pil, prinfo, apptag,
|
||||
appmask, reftag);
|
||||
case NVME_PI_GUARD_64:
|
||||
return nvme_dif_prchk_crc64(ns, dif, buf, mbuf, pil, prinfo, apptag,
|
||||
appmask, reftag);
|
||||
}
|
||||
|
||||
abort();
|
||||
}
|
||||
|
||||
uint16_t nvme_dif_check(NvmeNamespace *ns, uint8_t *buf, size_t len,
|
||||
uint8_t *mbuf, size_t mlen, uint8_t prinfo,
|
||||
uint64_t slba, uint16_t apptag,
|
||||
uint16_t appmask, uint64_t *reftag)
|
||||
{
|
||||
uint8_t *bufp, *end = buf + len;
|
||||
int16_t pil = 0;
|
||||
uint16_t status;
|
||||
|
||||
status = nvme_check_prinfo(ns, prinfo, slba, *reftag);
|
||||
if (status) {
|
||||
return status;
|
||||
}
|
||||
|
||||
if (!(ns->id_ns.dps & NVME_ID_NS_DPS_FIRST_EIGHT)) {
|
||||
pil = ns->lbaf.ms - nvme_pi_tuple_size(ns);
|
||||
}
|
||||
|
||||
trace_pci_nvme_dif_check(prinfo, ns->lbasz + pil);
|
||||
|
||||
for (bufp = buf; bufp < end; bufp += ns->lbasz, mbuf += ns->lbaf.ms) {
|
||||
NvmeDifTuple *dif = (NvmeDifTuple *)(mbuf + pil);
|
||||
status = nvme_dif_prchk(ns, dif, bufp, mbuf, pil, prinfo, apptag,
|
||||
appmask, *reftag);
|
||||
if (status) {
|
||||
/*
|
||||
* The first block of a 'raw' image is always allocated, so we
|
||||
* cannot reliably know if the block is all zeroes or not. For
|
||||
* CRC16 this works fine because the T10 CRC16 is 0x0 for all
|
||||
* zeroes, but the Rocksoft CRC64 is not. Thus, if a guard error is
|
||||
* detected for the first block, check if it is zeroed and manually
|
||||
* set the protection information to all ones to disable protection
|
||||
* information checking.
|
||||
*/
|
||||
if (status == NVME_E2E_GUARD_ERROR && slba == 0x0 && bufp == buf) {
|
||||
g_autofree uint8_t *zeroes = g_malloc0(ns->lbasz);
|
||||
|
||||
if (memcmp(bufp, zeroes, ns->lbasz) == 0) {
|
||||
memset(mbuf + pil, 0xff, nvme_pi_tuple_size(ns));
|
||||
}
|
||||
} else {
|
||||
return status;
|
||||
}
|
||||
}
|
||||
|
||||
if (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps) != NVME_ID_NS_DPS_TYPE_3) {
|
||||
(*reftag)++;
|
||||
}
|
||||
}
|
||||
|
||||
return NVME_SUCCESS;
|
||||
}
|
||||
|
||||
uint16_t nvme_dif_mangle_mdata(NvmeNamespace *ns, uint8_t *mbuf, size_t mlen,
|
||||
uint64_t slba)
|
||||
{
|
||||
BlockBackend *blk = ns->blkconf.blk;
|
||||
BlockDriverState *bs = blk_bs(blk);
|
||||
|
||||
int64_t moffset = 0, offset = nvme_l2b(ns, slba);
|
||||
uint8_t *mbufp, *end;
|
||||
bool zeroed;
|
||||
int16_t pil = 0;
|
||||
int64_t bytes = (mlen / ns->lbaf.ms) << ns->lbaf.ds;
|
||||
int64_t pnum = 0;
|
||||
|
||||
Error *err = NULL;
|
||||
|
||||
|
||||
if (!(ns->id_ns.dps & NVME_ID_NS_DPS_FIRST_EIGHT)) {
|
||||
pil = ns->lbaf.ms - nvme_pi_tuple_size(ns);
|
||||
}
|
||||
|
||||
do {
|
||||
int ret;
|
||||
|
||||
bytes -= pnum;
|
||||
|
||||
ret = bdrv_block_status(bs, offset, bytes, &pnum, NULL, NULL);
|
||||
if (ret < 0) {
|
||||
error_setg_errno(&err, -ret, "unable to get block status");
|
||||
error_report_err(err);
|
||||
|
||||
return NVME_INTERNAL_DEV_ERROR;
|
||||
}
|
||||
|
||||
zeroed = !!(ret & BDRV_BLOCK_ZERO);
|
||||
|
||||
trace_pci_nvme_block_status(offset, bytes, pnum, ret, zeroed);
|
||||
|
||||
if (zeroed) {
|
||||
mbufp = mbuf + moffset;
|
||||
mlen = (pnum >> ns->lbaf.ds) * ns->lbaf.ms;
|
||||
end = mbufp + mlen;
|
||||
|
||||
for (; mbufp < end; mbufp += ns->lbaf.ms) {
|
||||
memset(mbufp + pil, 0xff, nvme_pi_tuple_size(ns));
|
||||
}
|
||||
}
|
||||
|
||||
moffset += (pnum >> ns->lbaf.ds) * ns->lbaf.ms;
|
||||
offset += pnum;
|
||||
} while (pnum != bytes);
|
||||
|
||||
return NVME_SUCCESS;
|
||||
}
|
||||
|
||||
static void nvme_dif_rw_cb(void *opaque, int ret)
|
||||
{
|
||||
NvmeBounceContext *ctx = opaque;
|
||||
NvmeRequest *req = ctx->req;
|
||||
NvmeNamespace *ns = req->ns;
|
||||
BlockBackend *blk = ns->blkconf.blk;
|
||||
|
||||
trace_pci_nvme_dif_rw_cb(nvme_cid(req), blk_name(blk));
|
||||
|
||||
qemu_iovec_destroy(&ctx->data.iov);
|
||||
g_free(ctx->data.bounce);
|
||||
|
||||
qemu_iovec_destroy(&ctx->mdata.iov);
|
||||
g_free(ctx->mdata.bounce);
|
||||
|
||||
g_free(ctx);
|
||||
|
||||
nvme_rw_complete_cb(req, ret);
|
||||
}
|
||||
|
||||
static void nvme_dif_rw_check_cb(void *opaque, int ret)
|
||||
{
|
||||
NvmeBounceContext *ctx = opaque;
|
||||
NvmeRequest *req = ctx->req;
|
||||
NvmeNamespace *ns = req->ns;
|
||||
NvmeCtrl *n = nvme_ctrl(req);
|
||||
NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
|
||||
uint64_t slba = le64_to_cpu(rw->slba);
|
||||
uint8_t prinfo = NVME_RW_PRINFO(le16_to_cpu(rw->control));
|
||||
uint16_t apptag = le16_to_cpu(rw->apptag);
|
||||
uint16_t appmask = le16_to_cpu(rw->appmask);
|
||||
uint64_t reftag = le32_to_cpu(rw->reftag);
|
||||
uint64_t cdw3 = le32_to_cpu(rw->cdw3);
|
||||
uint16_t status;
|
||||
|
||||
reftag |= cdw3 << 32;
|
||||
|
||||
trace_pci_nvme_dif_rw_check_cb(nvme_cid(req), prinfo, apptag, appmask,
|
||||
reftag);
|
||||
|
||||
if (ret) {
|
||||
goto out;
|
||||
}
|
||||
|
||||
status = nvme_dif_mangle_mdata(ns, ctx->mdata.bounce, ctx->mdata.iov.size,
|
||||
slba);
|
||||
if (status) {
|
||||
req->status = status;
|
||||
goto out;
|
||||
}
|
||||
|
||||
status = nvme_dif_check(ns, ctx->data.bounce, ctx->data.iov.size,
|
||||
ctx->mdata.bounce, ctx->mdata.iov.size, prinfo,
|
||||
slba, apptag, appmask, &reftag);
|
||||
if (status) {
|
||||
req->status = status;
|
||||
goto out;
|
||||
}
|
||||
|
||||
status = nvme_bounce_data(n, ctx->data.bounce, ctx->data.iov.size,
|
||||
NVME_TX_DIRECTION_FROM_DEVICE, req);
|
||||
if (status) {
|
||||
req->status = status;
|
||||
goto out;
|
||||
}
|
||||
|
||||
if (prinfo & NVME_PRINFO_PRACT && ns->lbaf.ms == nvme_pi_tuple_size(ns)) {
|
||||
goto out;
|
||||
}
|
||||
|
||||
status = nvme_bounce_mdata(n, ctx->mdata.bounce, ctx->mdata.iov.size,
|
||||
NVME_TX_DIRECTION_FROM_DEVICE, req);
|
||||
if (status) {
|
||||
req->status = status;
|
||||
}
|
||||
|
||||
out:
|
||||
nvme_dif_rw_cb(ctx, ret);
|
||||
}
|
||||
|
||||
static void nvme_dif_rw_mdata_in_cb(void *opaque, int ret)
|
||||
{
|
||||
NvmeBounceContext *ctx = opaque;
|
||||
NvmeRequest *req = ctx->req;
|
||||
NvmeNamespace *ns = req->ns;
|
||||
NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
|
||||
uint64_t slba = le64_to_cpu(rw->slba);
|
||||
uint32_t nlb = le16_to_cpu(rw->nlb) + 1;
|
||||
size_t mlen = nvme_m2b(ns, nlb);
|
||||
uint64_t offset = nvme_moff(ns, slba);
|
||||
BlockBackend *blk = ns->blkconf.blk;
|
||||
|
||||
trace_pci_nvme_dif_rw_mdata_in_cb(nvme_cid(req), blk_name(blk));
|
||||
|
||||
if (ret) {
|
||||
goto out;
|
||||
}
|
||||
|
||||
ctx->mdata.bounce = g_malloc(mlen);
|
||||
|
||||
qemu_iovec_reset(&ctx->mdata.iov);
|
||||
qemu_iovec_add(&ctx->mdata.iov, ctx->mdata.bounce, mlen);
|
||||
|
||||
req->aiocb = blk_aio_preadv(blk, offset, &ctx->mdata.iov, 0,
|
||||
nvme_dif_rw_check_cb, ctx);
|
||||
return;
|
||||
|
||||
out:
|
||||
nvme_dif_rw_cb(ctx, ret);
|
||||
}
|
||||
|
||||
static void nvme_dif_rw_mdata_out_cb(void *opaque, int ret)
|
||||
{
|
||||
NvmeBounceContext *ctx = opaque;
|
||||
NvmeRequest *req = ctx->req;
|
||||
NvmeNamespace *ns = req->ns;
|
||||
NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
|
||||
uint64_t slba = le64_to_cpu(rw->slba);
|
||||
uint64_t offset = nvme_moff(ns, slba);
|
||||
BlockBackend *blk = ns->blkconf.blk;
|
||||
|
||||
trace_pci_nvme_dif_rw_mdata_out_cb(nvme_cid(req), blk_name(blk));
|
||||
|
||||
if (ret) {
|
||||
goto out;
|
||||
}
|
||||
|
||||
req->aiocb = blk_aio_pwritev(blk, offset, &ctx->mdata.iov, 0,
|
||||
nvme_dif_rw_cb, ctx);
|
||||
return;
|
||||
|
||||
out:
|
||||
nvme_dif_rw_cb(ctx, ret);
|
||||
}
|
||||
|
||||
uint16_t nvme_dif_rw(NvmeCtrl *n, NvmeRequest *req)
|
||||
{
|
||||
NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
|
||||
NvmeNamespace *ns = req->ns;
|
||||
BlockBackend *blk = ns->blkconf.blk;
|
||||
bool wrz = rw->opcode == NVME_CMD_WRITE_ZEROES;
|
||||
uint32_t nlb = le16_to_cpu(rw->nlb) + 1;
|
||||
uint64_t slba = le64_to_cpu(rw->slba);
|
||||
size_t len = nvme_l2b(ns, nlb);
|
||||
size_t mlen = nvme_m2b(ns, nlb);
|
||||
size_t mapped_len = len;
|
||||
int64_t offset = nvme_l2b(ns, slba);
|
||||
uint8_t prinfo = NVME_RW_PRINFO(le16_to_cpu(rw->control));
|
||||
uint16_t apptag = le16_to_cpu(rw->apptag);
|
||||
uint16_t appmask = le16_to_cpu(rw->appmask);
|
||||
uint64_t reftag = le32_to_cpu(rw->reftag);
|
||||
uint64_t cdw3 = le32_to_cpu(rw->cdw3);
|
||||
bool pract = !!(prinfo & NVME_PRINFO_PRACT);
|
||||
NvmeBounceContext *ctx;
|
||||
uint16_t status;
|
||||
|
||||
reftag |= cdw3 << 32;
|
||||
|
||||
trace_pci_nvme_dif_rw(pract, prinfo);
|
||||
|
||||
ctx = g_new0(NvmeBounceContext, 1);
|
||||
ctx->req = req;
|
||||
|
||||
if (wrz) {
|
||||
BdrvRequestFlags flags = BDRV_REQ_MAY_UNMAP;
|
||||
|
||||
if (prinfo & NVME_PRINFO_PRCHK_MASK) {
|
||||
status = NVME_INVALID_PROT_INFO | NVME_DNR;
|
||||
goto err;
|
||||
}
|
||||
|
||||
if (pract) {
|
||||
uint8_t *mbuf, *end;
|
||||
int16_t pil = ns->lbaf.ms - nvme_pi_tuple_size(ns);
|
||||
|
||||
flags = 0;
|
||||
|
||||
ctx->mdata.bounce = g_malloc0(mlen);
|
||||
|
||||
qemu_iovec_init(&ctx->mdata.iov, 1);
|
||||
qemu_iovec_add(&ctx->mdata.iov, ctx->mdata.bounce, mlen);
|
||||
|
||||
mbuf = ctx->mdata.bounce;
|
||||
end = mbuf + mlen;
|
||||
|
||||
if (ns->id_ns.dps & NVME_ID_NS_DPS_FIRST_EIGHT) {
|
||||
pil = 0;
|
||||
}
|
||||
|
||||
for (; mbuf < end; mbuf += ns->lbaf.ms) {
|
||||
NvmeDifTuple *dif = (NvmeDifTuple *)(mbuf + pil);
|
||||
|
||||
switch (ns->pif) {
|
||||
case NVME_PI_GUARD_16:
|
||||
dif->g16.apptag = cpu_to_be16(apptag);
|
||||
dif->g16.reftag = cpu_to_be32(reftag);
|
||||
|
||||
break;
|
||||
|
||||
case NVME_PI_GUARD_64:
|
||||
dif->g64.guard = cpu_to_be64(0x6482d367eb22b64e);
|
||||
dif->g64.apptag = cpu_to_be16(apptag);
|
||||
|
||||
dif->g64.sr[0] = reftag >> 40;
|
||||
dif->g64.sr[1] = reftag >> 32;
|
||||
dif->g64.sr[2] = reftag >> 24;
|
||||
dif->g64.sr[3] = reftag >> 16;
|
||||
dif->g64.sr[4] = reftag >> 8;
|
||||
dif->g64.sr[5] = reftag;
|
||||
|
||||
break;
|
||||
|
||||
default:
|
||||
abort();
|
||||
}
|
||||
|
||||
switch (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps)) {
|
||||
case NVME_ID_NS_DPS_TYPE_1:
|
||||
case NVME_ID_NS_DPS_TYPE_2:
|
||||
reftag++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
req->aiocb = blk_aio_pwrite_zeroes(blk, offset, len, flags,
|
||||
nvme_dif_rw_mdata_out_cb, ctx);
|
||||
return NVME_NO_COMPLETE;
|
||||
}
|
||||
|
||||
if (nvme_ns_ext(ns) && !(pract && ns->lbaf.ms == nvme_pi_tuple_size(ns))) {
|
||||
mapped_len += mlen;
|
||||
}
|
||||
|
||||
status = nvme_map_dptr(n, &req->sg, mapped_len, &req->cmd);
|
||||
if (status) {
|
||||
goto err;
|
||||
}
|
||||
|
||||
ctx->data.bounce = g_malloc(len);
|
||||
|
||||
qemu_iovec_init(&ctx->data.iov, 1);
|
||||
qemu_iovec_add(&ctx->data.iov, ctx->data.bounce, len);
|
||||
|
||||
if (req->cmd.opcode == NVME_CMD_READ) {
|
||||
block_acct_start(blk_get_stats(blk), &req->acct, ctx->data.iov.size,
|
||||
BLOCK_ACCT_READ);
|
||||
|
||||
req->aiocb = blk_aio_preadv(ns->blkconf.blk, offset, &ctx->data.iov, 0,
|
||||
nvme_dif_rw_mdata_in_cb, ctx);
|
||||
return NVME_NO_COMPLETE;
|
||||
}
|
||||
|
||||
status = nvme_bounce_data(n, ctx->data.bounce, ctx->data.iov.size,
|
||||
NVME_TX_DIRECTION_TO_DEVICE, req);
|
||||
if (status) {
|
||||
goto err;
|
||||
}
|
||||
|
||||
ctx->mdata.bounce = g_malloc(mlen);
|
||||
|
||||
qemu_iovec_init(&ctx->mdata.iov, 1);
|
||||
qemu_iovec_add(&ctx->mdata.iov, ctx->mdata.bounce, mlen);
|
||||
|
||||
if (!(pract && ns->lbaf.ms == nvme_pi_tuple_size(ns))) {
|
||||
status = nvme_bounce_mdata(n, ctx->mdata.bounce, ctx->mdata.iov.size,
|
||||
NVME_TX_DIRECTION_TO_DEVICE, req);
|
||||
if (status) {
|
||||
goto err;
|
||||
}
|
||||
}
|
||||
|
||||
status = nvme_check_prinfo(ns, prinfo, slba, reftag);
|
||||
if (status) {
|
||||
goto err;
|
||||
}
|
||||
|
||||
if (pract) {
|
||||
/* splice generated protection information into the buffer */
|
||||
nvme_dif_pract_generate_dif(ns, ctx->data.bounce, ctx->data.iov.size,
|
||||
ctx->mdata.bounce, ctx->mdata.iov.size,
|
||||
apptag, &reftag);
|
||||
} else {
|
||||
status = nvme_dif_check(ns, ctx->data.bounce, ctx->data.iov.size,
|
||||
ctx->mdata.bounce, ctx->mdata.iov.size, prinfo,
|
||||
slba, apptag, appmask, &reftag);
|
||||
if (status) {
|
||||
goto err;
|
||||
}
|
||||
}
|
||||
|
||||
block_acct_start(blk_get_stats(blk), &req->acct, ctx->data.iov.size,
|
||||
BLOCK_ACCT_WRITE);
|
||||
|
||||
req->aiocb = blk_aio_pwritev(ns->blkconf.blk, offset, &ctx->data.iov, 0,
|
||||
nvme_dif_rw_mdata_out_cb, ctx);
|
||||
|
||||
return NVME_NO_COMPLETE;
|
||||
|
||||
err:
|
||||
qemu_iovec_destroy(&ctx->data.iov);
|
||||
g_free(ctx->data.bounce);
|
||||
|
||||
qemu_iovec_destroy(&ctx->mdata.iov);
|
||||
g_free(ctx->mdata.bounce);
|
||||
|
||||
g_free(ctx);
|
||||
|
||||
return status;
|
||||
}
|
||||
+191
@@ -0,0 +1,191 @@
|
||||
#ifndef HW_NVME_DIF_H
|
||||
#define HW_NVME_DIF_H
|
||||
|
||||
/* from Linux kernel (crypto/crct10dif_common.c) */
|
||||
static const uint16_t crc16_t10dif_table[256] = {
|
||||
0x0000, 0x8BB7, 0x9CD9, 0x176E, 0xB205, 0x39B2, 0x2EDC, 0xA56B,
|
||||
0xEFBD, 0x640A, 0x7364, 0xF8D3, 0x5DB8, 0xD60F, 0xC161, 0x4AD6,
|
||||
0x54CD, 0xDF7A, 0xC814, 0x43A3, 0xE6C8, 0x6D7F, 0x7A11, 0xF1A6,
|
||||
0xBB70, 0x30C7, 0x27A9, 0xAC1E, 0x0975, 0x82C2, 0x95AC, 0x1E1B,
|
||||
0xA99A, 0x222D, 0x3543, 0xBEF4, 0x1B9F, 0x9028, 0x8746, 0x0CF1,
|
||||
0x4627, 0xCD90, 0xDAFE, 0x5149, 0xF422, 0x7F95, 0x68FB, 0xE34C,
|
||||
0xFD57, 0x76E0, 0x618E, 0xEA39, 0x4F52, 0xC4E5, 0xD38B, 0x583C,
|
||||
0x12EA, 0x995D, 0x8E33, 0x0584, 0xA0EF, 0x2B58, 0x3C36, 0xB781,
|
||||
0xD883, 0x5334, 0x445A, 0xCFED, 0x6A86, 0xE131, 0xF65F, 0x7DE8,
|
||||
0x373E, 0xBC89, 0xABE7, 0x2050, 0x853B, 0x0E8C, 0x19E2, 0x9255,
|
||||
0x8C4E, 0x07F9, 0x1097, 0x9B20, 0x3E4B, 0xB5FC, 0xA292, 0x2925,
|
||||
0x63F3, 0xE844, 0xFF2A, 0x749D, 0xD1F6, 0x5A41, 0x4D2F, 0xC698,
|
||||
0x7119, 0xFAAE, 0xEDC0, 0x6677, 0xC31C, 0x48AB, 0x5FC5, 0xD472,
|
||||
0x9EA4, 0x1513, 0x027D, 0x89CA, 0x2CA1, 0xA716, 0xB078, 0x3BCF,
|
||||
0x25D4, 0xAE63, 0xB90D, 0x32BA, 0x97D1, 0x1C66, 0x0B08, 0x80BF,
|
||||
0xCA69, 0x41DE, 0x56B0, 0xDD07, 0x786C, 0xF3DB, 0xE4B5, 0x6F02,
|
||||
0x3AB1, 0xB106, 0xA668, 0x2DDF, 0x88B4, 0x0303, 0x146D, 0x9FDA,
|
||||
0xD50C, 0x5EBB, 0x49D5, 0xC262, 0x6709, 0xECBE, 0xFBD0, 0x7067,
|
||||
0x6E7C, 0xE5CB, 0xF2A5, 0x7912, 0xDC79, 0x57CE, 0x40A0, 0xCB17,
|
||||
0x81C1, 0x0A76, 0x1D18, 0x96AF, 0x33C4, 0xB873, 0xAF1D, 0x24AA,
|
||||
0x932B, 0x189C, 0x0FF2, 0x8445, 0x212E, 0xAA99, 0xBDF7, 0x3640,
|
||||
0x7C96, 0xF721, 0xE04F, 0x6BF8, 0xCE93, 0x4524, 0x524A, 0xD9FD,
|
||||
0xC7E6, 0x4C51, 0x5B3F, 0xD088, 0x75E3, 0xFE54, 0xE93A, 0x628D,
|
||||
0x285B, 0xA3EC, 0xB482, 0x3F35, 0x9A5E, 0x11E9, 0x0687, 0x8D30,
|
||||
0xE232, 0x6985, 0x7EEB, 0xF55C, 0x5037, 0xDB80, 0xCCEE, 0x4759,
|
||||
0x0D8F, 0x8638, 0x9156, 0x1AE1, 0xBF8A, 0x343D, 0x2353, 0xA8E4,
|
||||
0xB6FF, 0x3D48, 0x2A26, 0xA191, 0x04FA, 0x8F4D, 0x9823, 0x1394,
|
||||
0x5942, 0xD2F5, 0xC59B, 0x4E2C, 0xEB47, 0x60F0, 0x779E, 0xFC29,
|
||||
0x4BA8, 0xC01F, 0xD771, 0x5CC6, 0xF9AD, 0x721A, 0x6574, 0xEEC3,
|
||||
0xA415, 0x2FA2, 0x38CC, 0xB37B, 0x1610, 0x9DA7, 0x8AC9, 0x017E,
|
||||
0x1F65, 0x94D2, 0x83BC, 0x080B, 0xAD60, 0x26D7, 0x31B9, 0xBA0E,
|
||||
0xF0D8, 0x7B6F, 0x6C01, 0xE7B6, 0x42DD, 0xC96A, 0xDE04, 0x55B3
|
||||
};
|
||||
|
||||
#define CRC64_NVME_POLY 0x9A6C9329AC4BC9B5ULL
|
||||
|
||||
static const uint64_t crc64_nvme_table[] = {
|
||||
0x0000000000000000ULL, 0x7F6EF0C830358979ULL,
|
||||
0xFEDDE190606B12F2ULL, 0x81B31158505E9B8BULL,
|
||||
0xC962E5739841B68FULL, 0xB60C15BBA8743FF6ULL,
|
||||
0x37BF04E3F82AA47DULL, 0x48D1F42BC81F2D04ULL,
|
||||
0xA61CECB46814FE75ULL, 0xD9721C7C5821770CULL,
|
||||
0x58C10D24087FEC87ULL, 0x27AFFDEC384A65FEULL,
|
||||
0x6F7E09C7F05548FAULL, 0x1010F90FC060C183ULL,
|
||||
0x91A3E857903E5A08ULL, 0xEECD189FA00BD371ULL,
|
||||
0x78E0FF3B88BE6F81ULL, 0x078E0FF3B88BE6F8ULL,
|
||||
0x863D1EABE8D57D73ULL, 0xF953EE63D8E0F40AULL,
|
||||
0xB1821A4810FFD90EULL, 0xCEECEA8020CA5077ULL,
|
||||
0x4F5FFBD87094CBFCULL, 0x30310B1040A14285ULL,
|
||||
0xDEFC138FE0AA91F4ULL, 0xA192E347D09F188DULL,
|
||||
0x2021F21F80C18306ULL, 0x5F4F02D7B0F40A7FULL,
|
||||
0x179EF6FC78EB277BULL, 0x68F0063448DEAE02ULL,
|
||||
0xE943176C18803589ULL, 0x962DE7A428B5BCF0ULL,
|
||||
0xF1C1FE77117CDF02ULL, 0x8EAF0EBF2149567BULL,
|
||||
0x0F1C1FE77117CDF0ULL, 0x7072EF2F41224489ULL,
|
||||
0x38A31B04893D698DULL, 0x47CDEBCCB908E0F4ULL,
|
||||
0xC67EFA94E9567B7FULL, 0xB9100A5CD963F206ULL,
|
||||
0x57DD12C379682177ULL, 0x28B3E20B495DA80EULL,
|
||||
0xA900F35319033385ULL, 0xD66E039B2936BAFCULL,
|
||||
0x9EBFF7B0E12997F8ULL, 0xE1D10778D11C1E81ULL,
|
||||
0x606216208142850AULL, 0x1F0CE6E8B1770C73ULL,
|
||||
0x8921014C99C2B083ULL, 0xF64FF184A9F739FAULL,
|
||||
0x77FCE0DCF9A9A271ULL, 0x08921014C99C2B08ULL,
|
||||
0x4043E43F0183060CULL, 0x3F2D14F731B68F75ULL,
|
||||
0xBE9E05AF61E814FEULL, 0xC1F0F56751DD9D87ULL,
|
||||
0x2F3DEDF8F1D64EF6ULL, 0x50531D30C1E3C78FULL,
|
||||
0xD1E00C6891BD5C04ULL, 0xAE8EFCA0A188D57DULL,
|
||||
0xE65F088B6997F879ULL, 0x9931F84359A27100ULL,
|
||||
0x1882E91B09FCEA8BULL, 0x67EC19D339C963F2ULL,
|
||||
0xD75ADABD7A6E2D6FULL, 0xA8342A754A5BA416ULL,
|
||||
0x29873B2D1A053F9DULL, 0x56E9CBE52A30B6E4ULL,
|
||||
0x1E383FCEE22F9BE0ULL, 0x6156CF06D21A1299ULL,
|
||||
0xE0E5DE5E82448912ULL, 0x9F8B2E96B271006BULL,
|
||||
0x71463609127AD31AULL, 0x0E28C6C1224F5A63ULL,
|
||||
0x8F9BD7997211C1E8ULL, 0xF0F5275142244891ULL,
|
||||
0xB824D37A8A3B6595ULL, 0xC74A23B2BA0EECECULL,
|
||||
0x46F932EAEA507767ULL, 0x3997C222DA65FE1EULL,
|
||||
0xAFBA2586F2D042EEULL, 0xD0D4D54EC2E5CB97ULL,
|
||||
0x5167C41692BB501CULL, 0x2E0934DEA28ED965ULL,
|
||||
0x66D8C0F56A91F461ULL, 0x19B6303D5AA47D18ULL,
|
||||
0x980521650AFAE693ULL, 0xE76BD1AD3ACF6FEAULL,
|
||||
0x09A6C9329AC4BC9BULL, 0x76C839FAAAF135E2ULL,
|
||||
0xF77B28A2FAAFAE69ULL, 0x8815D86ACA9A2710ULL,
|
||||
0xC0C42C4102850A14ULL, 0xBFAADC8932B0836DULL,
|
||||
0x3E19CDD162EE18E6ULL, 0x41773D1952DB919FULL,
|
||||
0x269B24CA6B12F26DULL, 0x59F5D4025B277B14ULL,
|
||||
0xD846C55A0B79E09FULL, 0xA72835923B4C69E6ULL,
|
||||
0xEFF9C1B9F35344E2ULL, 0x90973171C366CD9BULL,
|
||||
0x1124202993385610ULL, 0x6E4AD0E1A30DDF69ULL,
|
||||
0x8087C87E03060C18ULL, 0xFFE938B633338561ULL,
|
||||
0x7E5A29EE636D1EEAULL, 0x0134D92653589793ULL,
|
||||
0x49E52D0D9B47BA97ULL, 0x368BDDC5AB7233EEULL,
|
||||
0xB738CC9DFB2CA865ULL, 0xC8563C55CB19211CULL,
|
||||
0x5E7BDBF1E3AC9DECULL, 0x21152B39D3991495ULL,
|
||||
0xA0A63A6183C78F1EULL, 0xDFC8CAA9B3F20667ULL,
|
||||
0x97193E827BED2B63ULL, 0xE877CE4A4BD8A21AULL,
|
||||
0x69C4DF121B863991ULL, 0x16AA2FDA2BB3B0E8ULL,
|
||||
0xF86737458BB86399ULL, 0x8709C78DBB8DEAE0ULL,
|
||||
0x06BAD6D5EBD3716BULL, 0x79D4261DDBE6F812ULL,
|
||||
0x3105D23613F9D516ULL, 0x4E6B22FE23CC5C6FULL,
|
||||
0xCFD833A67392C7E4ULL, 0xB0B6C36E43A74E9DULL,
|
||||
0x9A6C9329AC4BC9B5ULL, 0xE50263E19C7E40CCULL,
|
||||
0x64B172B9CC20DB47ULL, 0x1BDF8271FC15523EULL,
|
||||
0x530E765A340A7F3AULL, 0x2C608692043FF643ULL,
|
||||
0xADD397CA54616DC8ULL, 0xD2BD67026454E4B1ULL,
|
||||
0x3C707F9DC45F37C0ULL, 0x431E8F55F46ABEB9ULL,
|
||||
0xC2AD9E0DA4342532ULL, 0xBDC36EC59401AC4BULL,
|
||||
0xF5129AEE5C1E814FULL, 0x8A7C6A266C2B0836ULL,
|
||||
0x0BCF7B7E3C7593BDULL, 0x74A18BB60C401AC4ULL,
|
||||
0xE28C6C1224F5A634ULL, 0x9DE29CDA14C02F4DULL,
|
||||
0x1C518D82449EB4C6ULL, 0x633F7D4A74AB3DBFULL,
|
||||
0x2BEE8961BCB410BBULL, 0x548079A98C8199C2ULL,
|
||||
0xD53368F1DCDF0249ULL, 0xAA5D9839ECEA8B30ULL,
|
||||
0x449080A64CE15841ULL, 0x3BFE706E7CD4D138ULL,
|
||||
0xBA4D61362C8A4AB3ULL, 0xC52391FE1CBFC3CAULL,
|
||||
0x8DF265D5D4A0EECEULL, 0xF29C951DE49567B7ULL,
|
||||
0x732F8445B4CBFC3CULL, 0x0C41748D84FE7545ULL,
|
||||
0x6BAD6D5EBD3716B7ULL, 0x14C39D968D029FCEULL,
|
||||
0x95708CCEDD5C0445ULL, 0xEA1E7C06ED698D3CULL,
|
||||
0xA2CF882D2576A038ULL, 0xDDA178E515432941ULL,
|
||||
0x5C1269BD451DB2CAULL, 0x237C997575283BB3ULL,
|
||||
0xCDB181EAD523E8C2ULL, 0xB2DF7122E51661BBULL,
|
||||
0x336C607AB548FA30ULL, 0x4C0290B2857D7349ULL,
|
||||
0x04D364994D625E4DULL, 0x7BBD94517D57D734ULL,
|
||||
0xFA0E85092D094CBFULL, 0x856075C11D3CC5C6ULL,
|
||||
0x134D926535897936ULL, 0x6C2362AD05BCF04FULL,
|
||||
0xED9073F555E26BC4ULL, 0x92FE833D65D7E2BDULL,
|
||||
0xDA2F7716ADC8CFB9ULL, 0xA54187DE9DFD46C0ULL,
|
||||
0x24F29686CDA3DD4BULL, 0x5B9C664EFD965432ULL,
|
||||
0xB5517ED15D9D8743ULL, 0xCA3F8E196DA80E3AULL,
|
||||
0x4B8C9F413DF695B1ULL, 0x34E26F890DC31CC8ULL,
|
||||
0x7C339BA2C5DC31CCULL, 0x035D6B6AF5E9B8B5ULL,
|
||||
0x82EE7A32A5B7233EULL, 0xFD808AFA9582AA47ULL,
|
||||
0x4D364994D625E4DAULL, 0x3258B95CE6106DA3ULL,
|
||||
0xB3EBA804B64EF628ULL, 0xCC8558CC867B7F51ULL,
|
||||
0x8454ACE74E645255ULL, 0xFB3A5C2F7E51DB2CULL,
|
||||
0x7A894D772E0F40A7ULL, 0x05E7BDBF1E3AC9DEULL,
|
||||
0xEB2AA520BE311AAFULL, 0x944455E88E0493D6ULL,
|
||||
0x15F744B0DE5A085DULL, 0x6A99B478EE6F8124ULL,
|
||||
0x224840532670AC20ULL, 0x5D26B09B16452559ULL,
|
||||
0xDC95A1C3461BBED2ULL, 0xA3FB510B762E37ABULL,
|
||||
0x35D6B6AF5E9B8B5BULL, 0x4AB846676EAE0222ULL,
|
||||
0xCB0B573F3EF099A9ULL, 0xB465A7F70EC510D0ULL,
|
||||
0xFCB453DCC6DA3DD4ULL, 0x83DAA314F6EFB4ADULL,
|
||||
0x0269B24CA6B12F26ULL, 0x7D0742849684A65FULL,
|
||||
0x93CA5A1B368F752EULL, 0xECA4AAD306BAFC57ULL,
|
||||
0x6D17BB8B56E467DCULL, 0x12794B4366D1EEA5ULL,
|
||||
0x5AA8BF68AECEC3A1ULL, 0x25C64FA09EFB4AD8ULL,
|
||||
0xA4755EF8CEA5D153ULL, 0xDB1BAE30FE90582AULL,
|
||||
0xBCF7B7E3C7593BD8ULL, 0xC399472BF76CB2A1ULL,
|
||||
0x422A5673A732292AULL, 0x3D44A6BB9707A053ULL,
|
||||
0x759552905F188D57ULL, 0x0AFBA2586F2D042EULL,
|
||||
0x8B48B3003F739FA5ULL, 0xF42643C80F4616DCULL,
|
||||
0x1AEB5B57AF4DC5ADULL, 0x6585AB9F9F784CD4ULL,
|
||||
0xE436BAC7CF26D75FULL, 0x9B584A0FFF135E26ULL,
|
||||
0xD389BE24370C7322ULL, 0xACE74EEC0739FA5BULL,
|
||||
0x2D545FB4576761D0ULL, 0x523AAF7C6752E8A9ULL,
|
||||
0xC41748D84FE75459ULL, 0xBB79B8107FD2DD20ULL,
|
||||
0x3ACAA9482F8C46ABULL, 0x45A459801FB9CFD2ULL,
|
||||
0x0D75ADABD7A6E2D6ULL, 0x721B5D63E7936BAFULL,
|
||||
0xF3A84C3BB7CDF024ULL, 0x8CC6BCF387F8795DULL,
|
||||
0x620BA46C27F3AA2CULL, 0x1D6554A417C62355ULL,
|
||||
0x9CD645FC4798B8DEULL, 0xE3B8B53477AD31A7ULL,
|
||||
0xAB69411FBFB21CA3ULL, 0xD407B1D78F8795DAULL,
|
||||
0x55B4A08FDFD90E51ULL, 0x2ADA5047EFEC8728ULL,
|
||||
};
|
||||
|
||||
static inline size_t nvme_pi_tuple_size(NvmeNamespace *ns)
|
||||
{
|
||||
return ns->pif ? 16 : 8;
|
||||
}
|
||||
|
||||
uint16_t nvme_check_prinfo(NvmeNamespace *ns, uint8_t prinfo, uint64_t slba,
|
||||
uint64_t reftag);
|
||||
uint16_t nvme_dif_mangle_mdata(NvmeNamespace *ns, uint8_t *mbuf, size_t mlen,
|
||||
uint64_t slba);
|
||||
void nvme_dif_pract_generate_dif(NvmeNamespace *ns, uint8_t *buf, size_t len,
|
||||
uint8_t *mbuf, size_t mlen, uint16_t apptag,
|
||||
uint64_t *reftag);
|
||||
uint16_t nvme_dif_check(NvmeNamespace *ns, uint8_t *buf, size_t len,
|
||||
uint8_t *mbuf, size_t mlen, uint8_t prinfo,
|
||||
uint64_t slba, uint16_t apptag,
|
||||
uint16_t appmask, uint64_t *reftag);
|
||||
uint16_t nvme_dif_rw(NvmeCtrl *n, NvmeRequest *req);
|
||||
|
||||
#endif /* HW_NVME_DIF_H */
|
||||
@@ -0,0 +1 @@
|
||||
system_ss.add(when: 'CONFIG_NVME_PCI', if_true: files('ctrl.c', 'dif.c', 'ns.c', 'subsys.c', 'nguid.c'))
|
||||
+187
@@ -0,0 +1,187 @@
|
||||
/*
|
||||
* QEMU NVMe NGUID functions
|
||||
*
|
||||
* Copyright 2024 Google LLC
|
||||
*
|
||||
* This program is free software; you can redistribute it and/or modify it
|
||||
* under the terms of the GNU General Public License as published by the
|
||||
* Free Software Foundation; either version 2 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
|
||||
* for more details.
|
||||
*/
|
||||
|
||||
#include "qemu/osdep.h"
|
||||
#include "qapi/visitor.h"
|
||||
#include "qemu/ctype.h"
|
||||
#include "nvme.h"
|
||||
|
||||
#define NGUID_SEPARATOR '-'
|
||||
|
||||
#define NGUID_VALUE_AUTO "auto"
|
||||
|
||||
#define NGUID_FMT \
|
||||
"%02hhx%02hhx%02hhx%02hhx" \
|
||||
"%02hhx%02hhx%02hhx%02hhx" \
|
||||
"%02hhx%02hhx%02hhx%02hhx" \
|
||||
"%02hhx%02hhx%02hhx%02hhx"
|
||||
|
||||
#define NGUID_STR_LEN (2 * NGUID_LEN + 1)
|
||||
|
||||
bool nvme_nguid_is_null(const NvmeNGUID *nguid)
|
||||
{
|
||||
static NvmeNGUID null_nguid;
|
||||
return memcmp(nguid, &null_nguid, sizeof(NvmeNGUID)) == 0;
|
||||
}
|
||||
|
||||
static void nvme_nguid_generate(NvmeNGUID *out)
|
||||
{
|
||||
int i;
|
||||
uint32_t x;
|
||||
|
||||
QEMU_BUILD_BUG_ON((NGUID_LEN % sizeof(x)) != 0);
|
||||
|
||||
for (i = 0; i < NGUID_LEN; i += sizeof(x)) {
|
||||
x = g_random_int();
|
||||
memcpy(&out->data[i], &x, sizeof(x));
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* The Linux Kernel typically prints the NGUID of an NVMe namespace using the
|
||||
* same format as the UUID. For instance:
|
||||
*
|
||||
* $ cat /sys/class/block/nvme0n1/nguid
|
||||
* e9accd3b-8390-4e13-167c-f0593437f57d
|
||||
*
|
||||
* When there is no UUID but there is NGUID the Kernel will print the NGUID as
|
||||
* wwid and it won't use the UUID format:
|
||||
*
|
||||
* $ cat /sys/class/block/nvme0n1/wwid
|
||||
* eui.e9accd3b83904e13167cf0593437f57d
|
||||
*
|
||||
* The NGUID has different fields compared to the UUID, so the grouping used in
|
||||
* the UUID format has no relation with the 3 fields of the NGUID.
|
||||
*
|
||||
* This implementation won't expect a strict format as the UUID one and instead
|
||||
* it will admit any string of hexadecimal digits. Byte groups could be created
|
||||
* using the '-' separator. The number of bytes needs to be exactly 16 and the
|
||||
* separator '-' has to be exactly in a byte boundary. The following are
|
||||
* examples of accepted formats for the NGUID string:
|
||||
*
|
||||
* nguid="e9accd3b-8390-4e13-167c-f0593437f57d"
|
||||
* nguid="e9accd3b83904e13167cf0593437f57d"
|
||||
* nguid="FEDCBA9876543210-ABCDEF-0123456789"
|
||||
*/
|
||||
static bool nvme_nguid_is_valid(const char *str)
|
||||
{
|
||||
int i;
|
||||
int digit_count = 0;
|
||||
|
||||
for (i = 0; i < strlen(str); i++) {
|
||||
const char c = str[i];
|
||||
if (qemu_isxdigit(c)) {
|
||||
digit_count++;
|
||||
continue;
|
||||
}
|
||||
if (c == NGUID_SEPARATOR) {
|
||||
/*
|
||||
* We need to make sure the separator is in a byte boundary, the
|
||||
* string does not start with the separator and they are not back to
|
||||
* back "--".
|
||||
*/
|
||||
if ((i > 0) && (str[i - 1] != NGUID_SEPARATOR) &&
|
||||
(digit_count % 2) == 0) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
/*
|
||||
* The string should have the correct byte length and not finish with the
|
||||
* separator
|
||||
*/
|
||||
return (digit_count == (2 * NGUID_LEN)) && (str[i - 1] != NGUID_SEPARATOR);
|
||||
}
|
||||
|
||||
static int nvme_nguid_parse(const char *str, NvmeNGUID *nguid)
|
||||
{
|
||||
uint8_t *id = &nguid->data[0];
|
||||
int ret = 0;
|
||||
int i;
|
||||
const char *ptr = str;
|
||||
|
||||
if (!nvme_nguid_is_valid(str)) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
for (i = 0; i < NGUID_LEN; i++) {
|
||||
ret = sscanf(ptr, "%02hhx", &id[i]);
|
||||
if (ret != 1) {
|
||||
return -1;
|
||||
}
|
||||
ptr += 2;
|
||||
if (*ptr == NGUID_SEPARATOR) {
|
||||
ptr++;
|
||||
}
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/*
|
||||
* When converted back to string this implementation will use a raw hex number
|
||||
* with no separators, for instance:
|
||||
*
|
||||
* "e9accd3b83904e13167cf0593437f57d"
|
||||
*/
|
||||
static void nvme_nguid_stringify(const NvmeNGUID *nguid, char *out)
|
||||
{
|
||||
const uint8_t *id = &nguid->data[0];
|
||||
snprintf(out, NGUID_STR_LEN, NGUID_FMT,
|
||||
id[0], id[1], id[2], id[3], id[4], id[5], id[6], id[7],
|
||||
id[8], id[9], id[10], id[11], id[12], id[13], id[14], id[15]);
|
||||
}
|
||||
|
||||
static void get_nguid(Object *obj, Visitor *v, const char *name, void *opaque,
|
||||
Error **errp)
|
||||
{
|
||||
const Property *prop = opaque;
|
||||
NvmeNGUID *nguid = object_field_prop_ptr(obj, prop);
|
||||
char buffer[NGUID_STR_LEN];
|
||||
char *p = buffer;
|
||||
|
||||
nvme_nguid_stringify(nguid, buffer);
|
||||
|
||||
visit_type_str(v, name, &p, errp);
|
||||
}
|
||||
|
||||
static void set_nguid(Object *obj, Visitor *v, const char *name, void *opaque,
|
||||
Error **errp)
|
||||
{
|
||||
const Property *prop = opaque;
|
||||
NvmeNGUID *nguid = object_field_prop_ptr(obj, prop);
|
||||
char *str;
|
||||
|
||||
if (!visit_type_str(v, name, &str, errp)) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (!strcmp(str, NGUID_VALUE_AUTO)) {
|
||||
nvme_nguid_generate(nguid);
|
||||
} else if (nvme_nguid_parse(str, nguid) < 0) {
|
||||
error_set_from_qdev_prop_error(errp, EINVAL, obj, name, str);
|
||||
}
|
||||
g_free(str);
|
||||
}
|
||||
|
||||
const PropertyInfo qdev_prop_nguid = {
|
||||
.type = "str",
|
||||
.description =
|
||||
"NGUID or \"" NGUID_VALUE_AUTO "\" for random value",
|
||||
.get = get_nguid,
|
||||
.set = set_nguid,
|
||||
};
|
||||
+1139
File diff suppressed because it is too large
Load Diff
+770
@@ -0,0 +1,770 @@
|
||||
/*
|
||||
* QEMU NVM Express
|
||||
*
|
||||
* Copyright (c) 2012 Intel Corporation
|
||||
* Copyright (c) 2021 Minwoo Im
|
||||
* Copyright (c) 2021 Samsung Electronics Co., Ltd.
|
||||
*
|
||||
* Authors:
|
||||
* Keith Busch <[email protected]>
|
||||
* Klaus Jensen <[email protected]>
|
||||
* Gollu Appalanaidu <[email protected]>
|
||||
* Dmitry Fomichev <[email protected]>
|
||||
* Minwoo Im <[email protected]>
|
||||
*
|
||||
* This code is licensed under the GNU GPL v2 or later.
|
||||
*/
|
||||
|
||||
#ifndef HW_NVME_NVME_H
|
||||
#define HW_NVME_NVME_H
|
||||
|
||||
#include "qemu/uuid.h"
|
||||
#include "hw/pci/pci_device.h"
|
||||
#include "hw/block/block.h"
|
||||
|
||||
#include "block/nvme.h"
|
||||
|
||||
#define NVME_MAX_CONTROLLERS 256
|
||||
#define NVME_MAX_NAMESPACES 256
|
||||
#define NVME_EUI64_DEFAULT ((uint64_t)0x5254000000000000)
|
||||
#define NVME_FDP_MAX_EVENTS 63
|
||||
#define NVME_FDP_MAXPIDS 128
|
||||
|
||||
/*
|
||||
* The controller only supports Submission and Completion Queue Entry Sizes of
|
||||
* 64 and 16 bytes respectively.
|
||||
*/
|
||||
#define NVME_SQES 6
|
||||
#define NVME_CQES 4
|
||||
|
||||
QEMU_BUILD_BUG_ON(NVME_MAX_NAMESPACES > NVME_NSID_BROADCAST - 1);
|
||||
|
||||
typedef struct NvmeCtrl NvmeCtrl;
|
||||
typedef struct NvmeNamespace NvmeNamespace;
|
||||
|
||||
#define TYPE_NVME_BUS "nvme-bus"
|
||||
OBJECT_DECLARE_SIMPLE_TYPE(NvmeBus, NVME_BUS)
|
||||
|
||||
typedef struct NvmeBus {
|
||||
BusState parent_bus;
|
||||
} NvmeBus;
|
||||
|
||||
#define TYPE_NVME_SUBSYS "nvme-subsys"
|
||||
#define NVME_SUBSYS(obj) \
|
||||
OBJECT_CHECK(NvmeSubsystem, (obj), TYPE_NVME_SUBSYS)
|
||||
#define SUBSYS_SLOT_RSVD (void *)0xFFFF
|
||||
|
||||
typedef struct NvmeReclaimUnit {
|
||||
uint64_t ruamw;
|
||||
} NvmeReclaimUnit;
|
||||
|
||||
typedef struct NvmeRuHandle {
|
||||
uint8_t ruht;
|
||||
uint8_t ruha;
|
||||
uint64_t event_filter;
|
||||
uint8_t lbafi;
|
||||
uint64_t ruamw;
|
||||
|
||||
/* reclaim units indexed by reclaim group */
|
||||
NvmeReclaimUnit *rus;
|
||||
} NvmeRuHandle;
|
||||
|
||||
typedef struct NvmeFdpEventBuffer {
|
||||
NvmeFdpEvent events[NVME_FDP_MAX_EVENTS];
|
||||
unsigned int nelems;
|
||||
unsigned int start;
|
||||
unsigned int next;
|
||||
} NvmeFdpEventBuffer;
|
||||
|
||||
typedef struct NvmeEnduranceGroup {
|
||||
uint8_t event_conf;
|
||||
|
||||
struct {
|
||||
NvmeFdpEventBuffer host_events, ctrl_events;
|
||||
|
||||
uint16_t nruh;
|
||||
uint16_t nrg;
|
||||
uint8_t rgif;
|
||||
uint64_t runs;
|
||||
|
||||
uint64_t hbmw;
|
||||
uint64_t mbmw;
|
||||
uint64_t mbe;
|
||||
|
||||
bool enabled;
|
||||
|
||||
NvmeRuHandle *ruhs;
|
||||
} fdp;
|
||||
} NvmeEnduranceGroup;
|
||||
|
||||
typedef struct NvmeSubsystem {
|
||||
DeviceState parent_obj;
|
||||
NvmeBus bus;
|
||||
uint8_t subnqn[256];
|
||||
char *serial;
|
||||
|
||||
NvmeCtrl *ctrls[NVME_MAX_CONTROLLERS];
|
||||
NvmeNamespace *namespaces[NVME_MAX_NAMESPACES + 1];
|
||||
NvmeEnduranceGroup endgrp;
|
||||
|
||||
struct {
|
||||
char *nqn;
|
||||
|
||||
struct {
|
||||
bool enabled;
|
||||
uint64_t runs;
|
||||
uint16_t nruh;
|
||||
uint32_t nrg;
|
||||
} fdp;
|
||||
} params;
|
||||
} NvmeSubsystem;
|
||||
|
||||
int nvme_subsys_register_ctrl(NvmeCtrl *n, Error **errp);
|
||||
void nvme_subsys_unregister_ctrl(NvmeSubsystem *subsys, NvmeCtrl *n);
|
||||
|
||||
static inline NvmeCtrl *nvme_subsys_ctrl(NvmeSubsystem *subsys,
|
||||
uint32_t cntlid)
|
||||
{
|
||||
if (!subsys || cntlid >= NVME_MAX_CONTROLLERS) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
if (subsys->ctrls[cntlid] == SUBSYS_SLOT_RSVD) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
return subsys->ctrls[cntlid];
|
||||
}
|
||||
|
||||
static inline NvmeNamespace *nvme_subsys_ns(NvmeSubsystem *subsys,
|
||||
uint32_t nsid)
|
||||
{
|
||||
if (!subsys || !nsid || nsid > NVME_MAX_NAMESPACES) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
return subsys->namespaces[nsid];
|
||||
}
|
||||
|
||||
#define TYPE_NVME_NS "nvme-ns"
|
||||
#define NVME_NS(obj) \
|
||||
OBJECT_CHECK(NvmeNamespace, (obj), TYPE_NVME_NS)
|
||||
|
||||
typedef struct NvmeZone {
|
||||
NvmeZoneDescr d;
|
||||
uint64_t w_ptr;
|
||||
QTAILQ_ENTRY(NvmeZone) entry;
|
||||
} NvmeZone;
|
||||
|
||||
#define FDP_EVT_MAX 0xff
|
||||
#define NVME_FDP_MAX_NS_RUHS 32u
|
||||
#define FDPVSS 0
|
||||
|
||||
/*
|
||||
* NOTE: Apart from event type 0, any event type with a shift value of 0 is
|
||||
* considered unsupported and thus skipped in get/set features calls.
|
||||
*
|
||||
* NOTE: NvmeRuHandle uses a 64bit event mask - refactor to support event types
|
||||
* of 63 or greater.
|
||||
*/
|
||||
static const uint8_t nvme_fdp_evf_shifts[FDP_EVT_MAX + 1] = {
|
||||
/* Host events */
|
||||
[FDP_EVT_RU_NOT_FULLY_WRITTEN] = 0,
|
||||
[FDP_EVT_RU_ATL_EXCEEDED] = 1,
|
||||
[FDP_EVT_CTRL_RESET_RUH] = 2,
|
||||
[FDP_EVT_INVALID_PID] = 3,
|
||||
/* CTRL events */
|
||||
[FDP_EVT_MEDIA_REALLOC] = 32,
|
||||
[FDP_EVT_RUH_IMPLICIT_RU_CHANGE] = 33,
|
||||
};
|
||||
|
||||
#define NGUID_LEN 16
|
||||
|
||||
typedef struct {
|
||||
uint8_t data[NGUID_LEN];
|
||||
} NvmeNGUID;
|
||||
|
||||
bool nvme_nguid_is_null(const NvmeNGUID *nguid);
|
||||
|
||||
extern const PropertyInfo qdev_prop_nguid;
|
||||
|
||||
#define DEFINE_PROP_NGUID_NODEFAULT(_name, _state, _field) \
|
||||
DEFINE_PROP(_name, _state, _field, qdev_prop_nguid, NvmeNGUID)
|
||||
|
||||
typedef struct NvmeNamespaceParams {
|
||||
bool detached;
|
||||
bool shared;
|
||||
uint32_t nsid;
|
||||
QemuUUID uuid;
|
||||
NvmeNGUID nguid;
|
||||
uint64_t eui64;
|
||||
bool eui64_default;
|
||||
|
||||
uint16_t ms;
|
||||
uint8_t mset;
|
||||
uint8_t pi;
|
||||
uint8_t pil;
|
||||
uint8_t pif;
|
||||
|
||||
uint16_t mssrl;
|
||||
uint32_t mcl;
|
||||
uint8_t msrc;
|
||||
|
||||
bool zoned;
|
||||
bool cross_zone_read;
|
||||
uint64_t zone_size_bs;
|
||||
uint64_t zone_cap_bs;
|
||||
uint32_t max_active_zones;
|
||||
uint32_t max_open_zones;
|
||||
uint32_t zd_extension_size;
|
||||
|
||||
uint32_t numzrwa;
|
||||
uint64_t zrwas;
|
||||
uint64_t zrwafg;
|
||||
|
||||
struct {
|
||||
char *ruhs;
|
||||
} fdp;
|
||||
|
||||
struct {
|
||||
uint16_t nawun;
|
||||
uint16_t nawupf;
|
||||
uint16_t nabsn;
|
||||
uint16_t nabspf;
|
||||
uint16_t nabo;
|
||||
} atomic;
|
||||
} NvmeNamespaceParams;
|
||||
|
||||
typedef struct NvmeAtomic {
|
||||
uint32_t atomic_max_write_size;
|
||||
uint64_t atomic_boundary;
|
||||
uint64_t atomic_nabo;
|
||||
bool atomic_writes;
|
||||
} NvmeAtomic;
|
||||
|
||||
typedef struct NvmeNamespace {
|
||||
DeviceState parent_obj;
|
||||
BlockConf blkconf;
|
||||
int32_t bootindex;
|
||||
char bootindex_suffix[24];
|
||||
int64_t size;
|
||||
int64_t moff;
|
||||
NvmeIdNs id_ns;
|
||||
NvmeIdNsNvm id_ns_nvm;
|
||||
NvmeIdNsInd id_ns_ind;
|
||||
NvmeLBAF lbaf;
|
||||
unsigned int nlbaf;
|
||||
size_t lbasz;
|
||||
uint8_t csi;
|
||||
uint16_t status;
|
||||
int attached;
|
||||
uint8_t pif;
|
||||
|
||||
struct {
|
||||
uint16_t zrwas;
|
||||
uint16_t zrwafg;
|
||||
uint32_t numzrwa;
|
||||
} zns;
|
||||
|
||||
QTAILQ_ENTRY(NvmeNamespace) entry;
|
||||
|
||||
NvmeIdNsZoned *id_ns_zoned;
|
||||
NvmeZone *zone_array;
|
||||
QTAILQ_HEAD(, NvmeZone) exp_open_zones;
|
||||
QTAILQ_HEAD(, NvmeZone) imp_open_zones;
|
||||
QTAILQ_HEAD(, NvmeZone) closed_zones;
|
||||
QTAILQ_HEAD(, NvmeZone) full_zones;
|
||||
uint32_t num_zones;
|
||||
uint64_t zone_size;
|
||||
uint64_t zone_capacity;
|
||||
uint32_t zone_size_log2;
|
||||
uint8_t *zd_extensions;
|
||||
int32_t nr_open_zones;
|
||||
int32_t nr_active_zones;
|
||||
|
||||
NvmeNamespaceParams params;
|
||||
NvmeSubsystem *subsys;
|
||||
NvmeEnduranceGroup *endgrp;
|
||||
|
||||
/* NULL for shared namespaces; set to specific controller if private */
|
||||
NvmeCtrl *ctrl;
|
||||
|
||||
struct {
|
||||
uint32_t err_rec;
|
||||
} features;
|
||||
|
||||
struct {
|
||||
uint16_t nphs;
|
||||
/* reclaim unit handle identifiers indexed by placement handle */
|
||||
uint16_t *phs;
|
||||
} fdp;
|
||||
|
||||
NvmeAtomic atomic;
|
||||
} NvmeNamespace;
|
||||
|
||||
static inline uint32_t nvme_nsid(NvmeNamespace *ns)
|
||||
{
|
||||
if (ns) {
|
||||
return ns->params.nsid;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static inline size_t nvme_l2b(NvmeNamespace *ns, uint64_t lba)
|
||||
{
|
||||
return lba << ns->lbaf.ds;
|
||||
}
|
||||
|
||||
static inline size_t nvme_m2b(NvmeNamespace *ns, uint64_t lba)
|
||||
{
|
||||
return ns->lbaf.ms * lba;
|
||||
}
|
||||
|
||||
static inline int64_t nvme_moff(NvmeNamespace *ns, uint64_t lba)
|
||||
{
|
||||
return ns->moff + nvme_m2b(ns, lba);
|
||||
}
|
||||
|
||||
static inline bool nvme_ns_ext(NvmeNamespace *ns)
|
||||
{
|
||||
return !!NVME_ID_NS_FLBAS_EXTENDED(ns->id_ns.flbas);
|
||||
}
|
||||
|
||||
static inline NvmeZoneState nvme_get_zone_state(NvmeZone *zone)
|
||||
{
|
||||
return zone->d.zs >> 4;
|
||||
}
|
||||
|
||||
static inline void nvme_set_zone_state(NvmeZone *zone, NvmeZoneState state)
|
||||
{
|
||||
zone->d.zs = state << 4;
|
||||
}
|
||||
|
||||
static inline uint64_t nvme_zone_rd_boundary(NvmeNamespace *ns, NvmeZone *zone)
|
||||
{
|
||||
return zone->d.zslba + ns->zone_size;
|
||||
}
|
||||
|
||||
static inline uint64_t nvme_zone_wr_boundary(NvmeZone *zone)
|
||||
{
|
||||
return zone->d.zslba + zone->d.zcap;
|
||||
}
|
||||
|
||||
static inline bool nvme_wp_is_valid(NvmeZone *zone)
|
||||
{
|
||||
uint8_t st = nvme_get_zone_state(zone);
|
||||
|
||||
return st != NVME_ZONE_STATE_FULL &&
|
||||
st != NVME_ZONE_STATE_READ_ONLY &&
|
||||
st != NVME_ZONE_STATE_OFFLINE;
|
||||
}
|
||||
|
||||
static inline uint8_t *nvme_get_zd_extension(NvmeNamespace *ns,
|
||||
uint32_t zone_idx)
|
||||
{
|
||||
return &ns->zd_extensions[zone_idx * ns->params.zd_extension_size];
|
||||
}
|
||||
|
||||
static inline void nvme_aor_inc_open(NvmeNamespace *ns)
|
||||
{
|
||||
assert(ns->nr_open_zones >= 0);
|
||||
if (ns->params.max_open_zones) {
|
||||
ns->nr_open_zones++;
|
||||
assert(ns->nr_open_zones <= ns->params.max_open_zones);
|
||||
}
|
||||
}
|
||||
|
||||
static inline void nvme_aor_dec_open(NvmeNamespace *ns)
|
||||
{
|
||||
if (ns->params.max_open_zones) {
|
||||
assert(ns->nr_open_zones > 0);
|
||||
ns->nr_open_zones--;
|
||||
}
|
||||
assert(ns->nr_open_zones >= 0);
|
||||
}
|
||||
|
||||
static inline void nvme_aor_inc_active(NvmeNamespace *ns)
|
||||
{
|
||||
assert(ns->nr_active_zones >= 0);
|
||||
if (ns->params.max_active_zones) {
|
||||
ns->nr_active_zones++;
|
||||
assert(ns->nr_active_zones <= ns->params.max_active_zones);
|
||||
}
|
||||
}
|
||||
|
||||
static inline void nvme_aor_dec_active(NvmeNamespace *ns)
|
||||
{
|
||||
if (ns->params.max_active_zones) {
|
||||
assert(ns->nr_active_zones > 0);
|
||||
ns->nr_active_zones--;
|
||||
assert(ns->nr_active_zones >= ns->nr_open_zones);
|
||||
}
|
||||
assert(ns->nr_active_zones >= 0);
|
||||
}
|
||||
|
||||
static inline void nvme_fdp_stat_inc(uint64_t *a, uint64_t b)
|
||||
{
|
||||
uint64_t ret = *a + b;
|
||||
*a = ret < *a ? UINT64_MAX : ret;
|
||||
}
|
||||
|
||||
void nvme_ns_init_format(NvmeNamespace *ns);
|
||||
int nvme_ns_setup(NvmeNamespace *ns, Error **errp);
|
||||
void nvme_ns_drain(NvmeNamespace *ns);
|
||||
void nvme_ns_shutdown(NvmeNamespace *ns);
|
||||
void nvme_ns_cleanup(NvmeNamespace *ns);
|
||||
|
||||
typedef struct NvmeAsyncEvent {
|
||||
QTAILQ_ENTRY(NvmeAsyncEvent) entry;
|
||||
NvmeAerResult result;
|
||||
} NvmeAsyncEvent;
|
||||
|
||||
enum {
|
||||
NVME_SG_ALLOC = 1 << 0,
|
||||
NVME_SG_DMA = 1 << 1,
|
||||
};
|
||||
|
||||
typedef struct NvmeSg {
|
||||
int flags;
|
||||
|
||||
union {
|
||||
QEMUSGList qsg;
|
||||
QEMUIOVector iov;
|
||||
};
|
||||
} NvmeSg;
|
||||
|
||||
typedef enum NvmeTxDirection {
|
||||
NVME_TX_DIRECTION_TO_DEVICE = 0,
|
||||
NVME_TX_DIRECTION_FROM_DEVICE = 1,
|
||||
} NvmeTxDirection;
|
||||
|
||||
typedef struct NvmeRequest {
|
||||
struct NvmeSQueue *sq;
|
||||
struct NvmeNamespace *ns;
|
||||
BlockAIOCB *aiocb;
|
||||
uint16_t status;
|
||||
void *opaque;
|
||||
NvmeCqe cqe;
|
||||
NvmeCmd cmd;
|
||||
BlockAcctCookie acct;
|
||||
NvmeSg sg;
|
||||
bool atomic_write;
|
||||
QTAILQ_ENTRY(NvmeRequest)entry;
|
||||
/*
|
||||
* If you add a new field here, please make sure to update
|
||||
* nvme_vmstate_request, pre_save_validate_aer_req() and
|
||||
* pre_save_validate_cq_req().
|
||||
*/
|
||||
} NvmeRequest;
|
||||
|
||||
typedef struct NvmeBounceContext {
|
||||
NvmeRequest *req;
|
||||
|
||||
struct {
|
||||
QEMUIOVector iov;
|
||||
uint8_t *bounce;
|
||||
} data, mdata;
|
||||
} NvmeBounceContext;
|
||||
|
||||
static inline const char *nvme_adm_opc_str(uint8_t opc)
|
||||
{
|
||||
switch (opc) {
|
||||
case NVME_ADM_CMD_DELETE_SQ: return "NVME_ADM_CMD_DELETE_SQ";
|
||||
case NVME_ADM_CMD_CREATE_SQ: return "NVME_ADM_CMD_CREATE_SQ";
|
||||
case NVME_ADM_CMD_GET_LOG_PAGE: return "NVME_ADM_CMD_GET_LOG_PAGE";
|
||||
case NVME_ADM_CMD_DELETE_CQ: return "NVME_ADM_CMD_DELETE_CQ";
|
||||
case NVME_ADM_CMD_CREATE_CQ: return "NVME_ADM_CMD_CREATE_CQ";
|
||||
case NVME_ADM_CMD_IDENTIFY: return "NVME_ADM_CMD_IDENTIFY";
|
||||
case NVME_ADM_CMD_ABORT: return "NVME_ADM_CMD_ABORT";
|
||||
case NVME_ADM_CMD_SET_FEATURES: return "NVME_ADM_CMD_SET_FEATURES";
|
||||
case NVME_ADM_CMD_GET_FEATURES: return "NVME_ADM_CMD_GET_FEATURES";
|
||||
case NVME_ADM_CMD_ASYNC_EV_REQ: return "NVME_ADM_CMD_ASYNC_EV_REQ";
|
||||
case NVME_ADM_CMD_NS_ATTACHMENT: return "NVME_ADM_CMD_NS_ATTACHMENT";
|
||||
case NVME_ADM_CMD_DIRECTIVE_SEND: return "NVME_ADM_CMD_DIRECTIVE_SEND";
|
||||
case NVME_ADM_CMD_VIRT_MNGMT: return "NVME_ADM_CMD_VIRT_MNGMT";
|
||||
case NVME_ADM_CMD_DIRECTIVE_RECV: return "NVME_ADM_CMD_DIRECTIVE_RECV";
|
||||
case NVME_ADM_CMD_DBBUF_CONFIG: return "NVME_ADM_CMD_DBBUF_CONFIG";
|
||||
case NVME_ADM_CMD_FORMAT_NVM: return "NVME_ADM_CMD_FORMAT_NVM";
|
||||
case NVME_ADM_CMD_SECURITY_SEND: return "NVME_ADM_CMD_SECURITY_SEND";
|
||||
case NVME_ADM_CMD_SECURITY_RECV: return "NVME_ADM_CMD_SECURITY_RECV";
|
||||
default: return "NVME_ADM_CMD_UNKNOWN";
|
||||
}
|
||||
}
|
||||
|
||||
static inline const char *nvme_io_opc_str(uint8_t opc)
|
||||
{
|
||||
switch (opc) {
|
||||
case NVME_CMD_FLUSH: return "NVME_NVM_CMD_FLUSH";
|
||||
case NVME_CMD_WRITE: return "NVME_NVM_CMD_WRITE";
|
||||
case NVME_CMD_READ: return "NVME_NVM_CMD_READ";
|
||||
case NVME_CMD_COMPARE: return "NVME_NVM_CMD_COMPARE";
|
||||
case NVME_CMD_WRITE_ZEROES: return "NVME_NVM_CMD_WRITE_ZEROES";
|
||||
case NVME_CMD_DSM: return "NVME_NVM_CMD_DSM";
|
||||
case NVME_CMD_VERIFY: return "NVME_NVM_CMD_VERIFY";
|
||||
case NVME_CMD_COPY: return "NVME_NVM_CMD_COPY";
|
||||
case NVME_CMD_ZONE_MGMT_SEND: return "NVME_ZONED_CMD_MGMT_SEND";
|
||||
case NVME_CMD_ZONE_MGMT_RECV: return "NVME_ZONED_CMD_MGMT_RECV";
|
||||
case NVME_CMD_ZONE_APPEND: return "NVME_ZONED_CMD_ZONE_APPEND";
|
||||
default: return "NVME_NVM_CMD_UNKNOWN";
|
||||
}
|
||||
}
|
||||
|
||||
typedef struct NvmeSQueue {
|
||||
struct NvmeCtrl *ctrl;
|
||||
uint16_t sqid;
|
||||
uint16_t cqid;
|
||||
uint32_t head;
|
||||
uint32_t tail;
|
||||
uint32_t size;
|
||||
uint64_t dma_addr;
|
||||
uint64_t db_addr;
|
||||
uint64_t ei_addr;
|
||||
QEMUBH *bh;
|
||||
EventNotifier notifier;
|
||||
bool ioeventfd_enabled;
|
||||
NvmeRequest *io_req;
|
||||
QTAILQ_HEAD(, NvmeRequest) req_list;
|
||||
QTAILQ_HEAD(, NvmeRequest) out_req_list;
|
||||
QTAILQ_ENTRY(NvmeSQueue) entry;
|
||||
} NvmeSQueue;
|
||||
|
||||
typedef struct NvmeCQueue {
|
||||
struct NvmeCtrl *ctrl;
|
||||
uint8_t phase;
|
||||
uint16_t cqid;
|
||||
uint16_t irq_enabled;
|
||||
uint32_t head;
|
||||
uint32_t tail;
|
||||
uint32_t vector;
|
||||
uint32_t size;
|
||||
uint64_t dma_addr;
|
||||
uint64_t db_addr;
|
||||
uint64_t ei_addr;
|
||||
QEMUBH *bh;
|
||||
EventNotifier notifier;
|
||||
bool ioeventfd_enabled;
|
||||
QTAILQ_HEAD(, NvmeSQueue) sq_list;
|
||||
QTAILQ_HEAD(, NvmeRequest) req_list;
|
||||
} NvmeCQueue;
|
||||
|
||||
#define TYPE_NVME "nvme"
|
||||
#define NVME(obj) \
|
||||
OBJECT_CHECK(NvmeCtrl, (obj), TYPE_NVME)
|
||||
|
||||
typedef struct NvmeParams {
|
||||
char *serial;
|
||||
char *model;
|
||||
char *firmware_version;
|
||||
uint32_t num_queues; /* deprecated since 5.1 */
|
||||
uint32_t max_ioqpairs;
|
||||
uint16_t msix_qsize;
|
||||
uint16_t mqes;
|
||||
uint32_t cmb_size_mb;
|
||||
uint8_t aerl;
|
||||
uint32_t aer_max_queued;
|
||||
uint8_t mdts;
|
||||
uint8_t vsl;
|
||||
bool use_intel_id;
|
||||
uint8_t zasl;
|
||||
bool auto_transition_zones;
|
||||
bool legacy_cmb;
|
||||
bool ioeventfd;
|
||||
bool dbcs;
|
||||
uint16_t sriov_max_vfs;
|
||||
uint16_t sriov_vq_flexible;
|
||||
uint16_t sriov_vi_flexible;
|
||||
uint32_t sriov_max_vq_per_vf;
|
||||
uint32_t sriov_max_vi_per_vf;
|
||||
bool msix_exclusive_bar;
|
||||
bool ocp;
|
||||
|
||||
struct {
|
||||
bool mem;
|
||||
} ctratt;
|
||||
|
||||
uint16_t atomic_awun;
|
||||
uint16_t atomic_awupf;
|
||||
bool atomic_dn;
|
||||
} NvmeParams;
|
||||
|
||||
typedef struct NvmeCtrl {
|
||||
PCIDevice parent_obj;
|
||||
MemoryRegion bar0;
|
||||
MemoryRegion iomem;
|
||||
NvmeBar bar;
|
||||
NvmeParams params;
|
||||
NvmeBus bus;
|
||||
|
||||
uint16_t cntlid;
|
||||
bool qs_created;
|
||||
uint32_t page_size;
|
||||
uint16_t page_bits;
|
||||
uint16_t max_prp_ents;
|
||||
uint32_t max_q_ents;
|
||||
uint8_t outstanding_aers;
|
||||
uint32_t irq_status;
|
||||
int cq_pending;
|
||||
uint64_t host_timestamp; /* Timestamp sent by the host */
|
||||
uint64_t timestamp_set_qemu_clock_ms; /* QEMU clock time */
|
||||
uint64_t starttime_ms;
|
||||
uint16_t temperature;
|
||||
uint8_t smart_critical_warning;
|
||||
uint32_t conf_msix_qsize;
|
||||
uint32_t conf_ioqpairs;
|
||||
uint64_t dbbuf_dbs;
|
||||
uint64_t dbbuf_eis;
|
||||
bool dbbuf_enabled;
|
||||
|
||||
struct {
|
||||
uint32_t acs[256];
|
||||
struct {
|
||||
uint32_t nvm[256];
|
||||
uint32_t zoned[256];
|
||||
} iocs;
|
||||
} cse;
|
||||
|
||||
struct {
|
||||
MemoryRegion mem;
|
||||
uint8_t *buf;
|
||||
bool cmse;
|
||||
hwaddr cba;
|
||||
} cmb;
|
||||
|
||||
struct {
|
||||
HostMemoryBackend *dev;
|
||||
bool cmse;
|
||||
hwaddr cba;
|
||||
} pmr;
|
||||
|
||||
uint8_t aer_mask;
|
||||
NvmeRequest **aer_reqs;
|
||||
QTAILQ_HEAD(, NvmeAsyncEvent) aer_queue;
|
||||
int aer_queued;
|
||||
|
||||
uint32_t dmrsl;
|
||||
|
||||
/* Namespace ID is started with 1 so bitmap should be 1-based */
|
||||
#define NVME_CHANGED_NSID_SIZE (NVME_MAX_NAMESPACES + 1)
|
||||
DECLARE_BITMAP(changed_nsids, NVME_CHANGED_NSID_SIZE);
|
||||
|
||||
NvmeSubsystem *subsys;
|
||||
|
||||
NvmeNamespace namespace;
|
||||
NvmeNamespace *namespaces[NVME_MAX_NAMESPACES + 1];
|
||||
uint32_t num_queues;
|
||||
NvmeSQueue **sq;
|
||||
NvmeCQueue **cq;
|
||||
NvmeSQueue admin_sq;
|
||||
NvmeCQueue admin_cq;
|
||||
NvmeIdCtrl id_ctrl;
|
||||
|
||||
struct {
|
||||
struct {
|
||||
uint16_t temp_thresh_hi;
|
||||
uint16_t temp_thresh_low;
|
||||
};
|
||||
|
||||
uint32_t async_config;
|
||||
NvmeHostBehaviorSupport hbs;
|
||||
} features;
|
||||
|
||||
NvmePriCtrlCap pri_ctrl_cap;
|
||||
uint32_t nr_sec_ctrls;
|
||||
NvmeSecCtrlEntry *sec_ctrl_list;
|
||||
struct {
|
||||
uint16_t vqrfap;
|
||||
uint16_t virfap;
|
||||
} next_pri_ctrl_cap; /* These override pri_ctrl_cap after reset */
|
||||
uint32_t dn; /* Disable Normal */
|
||||
NvmeAtomic atomic;
|
||||
|
||||
/* Socket mapping to SPDM over NVMe Security In/Out commands */
|
||||
int spdm_socket;
|
||||
|
||||
/* Migration-related stuff */
|
||||
Error *migration_blocker;
|
||||
} NvmeCtrl;
|
||||
|
||||
typedef enum NvmeResetType {
|
||||
NVME_RESET_FUNCTION = 0,
|
||||
NVME_RESET_CONTROLLER = 1,
|
||||
} NvmeResetType;
|
||||
|
||||
static inline NvmeNamespace *nvme_ns(NvmeCtrl *n, uint32_t nsid)
|
||||
{
|
||||
if (!nsid || nsid > NVME_MAX_NAMESPACES) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
return n->namespaces[nsid];
|
||||
}
|
||||
|
||||
static inline NvmeCQueue *nvme_cq(NvmeRequest *req)
|
||||
{
|
||||
NvmeSQueue *sq = req->sq;
|
||||
NvmeCtrl *n = sq->ctrl;
|
||||
|
||||
return n->cq[sq->cqid];
|
||||
}
|
||||
|
||||
static inline NvmeCtrl *nvme_ctrl(NvmeRequest *req)
|
||||
{
|
||||
NvmeSQueue *sq = req->sq;
|
||||
return sq->ctrl;
|
||||
}
|
||||
|
||||
static inline uint16_t nvme_cid(NvmeRequest *req)
|
||||
{
|
||||
if (!req) {
|
||||
return 0xffff;
|
||||
}
|
||||
|
||||
return le16_to_cpu(req->cqe.cid);
|
||||
}
|
||||
|
||||
static inline NvmeSecCtrlEntry *nvme_sctrl(NvmeCtrl *n)
|
||||
{
|
||||
PCIDevice *pci_dev = &n->parent_obj;
|
||||
NvmeCtrl *pf = NVME(pcie_sriov_get_pf(pci_dev));
|
||||
|
||||
if (pci_is_vf(pci_dev)) {
|
||||
return &pf->sec_ctrl_list[pcie_sriov_vf_number(pci_dev)];
|
||||
}
|
||||
|
||||
return NULL;
|
||||
}
|
||||
|
||||
static inline NvmeSecCtrlEntry *nvme_sctrl_for_cntlid(NvmeCtrl *n,
|
||||
uint16_t cntlid)
|
||||
{
|
||||
NvmeSecCtrlEntry *list = n->sec_ctrl_list;
|
||||
uint8_t i;
|
||||
|
||||
for (i = 0; i < n->nr_sec_ctrls; i++) {
|
||||
if (le16_to_cpu(list[i].scid) == cntlid) {
|
||||
return &list[i];
|
||||
}
|
||||
}
|
||||
|
||||
return NULL;
|
||||
}
|
||||
|
||||
void nvme_attach_ns(NvmeCtrl *n, NvmeNamespace *ns);
|
||||
uint16_t nvme_bounce_data(NvmeCtrl *n, void *ptr, uint32_t len,
|
||||
NvmeTxDirection dir, NvmeRequest *req);
|
||||
uint16_t nvme_bounce_mdata(NvmeCtrl *n, void *ptr, uint32_t len,
|
||||
NvmeTxDirection dir, NvmeRequest *req);
|
||||
void nvme_rw_complete_cb(void *opaque, int ret);
|
||||
uint16_t nvme_map_dptr(NvmeCtrl *n, NvmeSg *sg, size_t len,
|
||||
NvmeCmd *cmd);
|
||||
|
||||
void nvme_atomic_configure_max_write_size(bool dn, uint16_t awun,
|
||||
uint16_t awupf, NvmeAtomic *atomic);
|
||||
void nvme_ns_atomic_configure_boundary(bool dn, uint16_t nabsn,
|
||||
uint16_t nabspf, NvmeAtomic *atomic);
|
||||
|
||||
extern const VMStateDescription nvme_vmstate_atomic;
|
||||
extern const VMStateDescription nvme_vmstate_ns;
|
||||
|
||||
#endif /* HW_NVME_NVME_H */
|
||||
@@ -0,0 +1,247 @@
|
||||
/*
|
||||
* QEMU NVM Express Subsystem: nvme-subsys
|
||||
*
|
||||
* Copyright (c) 2021 Minwoo Im <[email protected]>
|
||||
*
|
||||
* This code is licensed under the GNU GPL v2. Refer COPYING.
|
||||
*/
|
||||
|
||||
#include "qemu/osdep.h"
|
||||
#include "qemu/units.h"
|
||||
#include "qapi/error.h"
|
||||
#include "hw/core/qdev.h"
|
||||
|
||||
#include "nvme.h"
|
||||
|
||||
#define NVME_DEFAULT_RU_SIZE (96 * MiB)
|
||||
|
||||
static int nvme_subsys_reserve_cntlids(NvmeCtrl *n, int start, int num)
|
||||
{
|
||||
NvmeSubsystem *subsys = n->subsys;
|
||||
NvmeSecCtrlEntry *list = n->sec_ctrl_list;
|
||||
NvmeSecCtrlEntry *sctrl;
|
||||
int i, cnt = 0;
|
||||
|
||||
for (i = start; i < ARRAY_SIZE(subsys->ctrls) && cnt < num; i++) {
|
||||
if (!subsys->ctrls[i]) {
|
||||
sctrl = &list[cnt];
|
||||
sctrl->scid = cpu_to_le16(i);
|
||||
subsys->ctrls[i] = SUBSYS_SLOT_RSVD;
|
||||
cnt++;
|
||||
}
|
||||
}
|
||||
|
||||
return cnt;
|
||||
}
|
||||
|
||||
static void nvme_subsys_unreserve_cntlids(NvmeCtrl *n)
|
||||
{
|
||||
NvmeSubsystem *subsys = n->subsys;
|
||||
NvmeSecCtrlEntry *list = n->sec_ctrl_list;
|
||||
NvmeSecCtrlEntry *sctrl;
|
||||
int i, cntlid;
|
||||
|
||||
for (i = 0; i < n->params.sriov_max_vfs; i++) {
|
||||
sctrl = &list[i];
|
||||
cntlid = le16_to_cpu(sctrl->scid);
|
||||
|
||||
if (cntlid) {
|
||||
assert(subsys->ctrls[cntlid] == SUBSYS_SLOT_RSVD);
|
||||
subsys->ctrls[cntlid] = NULL;
|
||||
sctrl->scid = 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
int nvme_subsys_register_ctrl(NvmeCtrl *n, Error **errp)
|
||||
{
|
||||
NvmeSubsystem *subsys = n->subsys;
|
||||
NvmeSecCtrlEntry *sctrl = nvme_sctrl(n);
|
||||
int cntlid, num_rsvd, num_vfs = n->params.sriov_max_vfs;
|
||||
|
||||
if (pci_is_vf(&n->parent_obj)) {
|
||||
cntlid = le16_to_cpu(sctrl->scid);
|
||||
} else {
|
||||
n->sec_ctrl_list = g_new0(NvmeSecCtrlEntry, num_vfs);
|
||||
|
||||
for (cntlid = 0; cntlid < ARRAY_SIZE(subsys->ctrls); cntlid++) {
|
||||
if (!subsys->ctrls[cntlid]) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (cntlid == ARRAY_SIZE(subsys->ctrls)) {
|
||||
error_setg(errp, "no more free controller id");
|
||||
return -1;
|
||||
}
|
||||
|
||||
num_rsvd = nvme_subsys_reserve_cntlids(n, cntlid + 1, num_vfs);
|
||||
if (num_rsvd != num_vfs) {
|
||||
nvme_subsys_unreserve_cntlids(n);
|
||||
error_setg(errp,
|
||||
"no more free controller ids for secondary controllers");
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
if (!subsys->serial) {
|
||||
subsys->serial = g_strdup(n->params.serial);
|
||||
} else if (strcmp(subsys->serial, n->params.serial)) {
|
||||
error_setg(errp, "invalid controller serial");
|
||||
return -1;
|
||||
}
|
||||
|
||||
subsys->ctrls[cntlid] = n;
|
||||
|
||||
return cntlid;
|
||||
}
|
||||
|
||||
void nvme_subsys_unregister_ctrl(NvmeSubsystem *subsys, NvmeCtrl *n)
|
||||
{
|
||||
if (pci_is_vf(&n->parent_obj)) {
|
||||
subsys->ctrls[n->cntlid] = SUBSYS_SLOT_RSVD;
|
||||
} else {
|
||||
subsys->ctrls[n->cntlid] = NULL;
|
||||
nvme_subsys_unreserve_cntlids(n);
|
||||
}
|
||||
|
||||
n->cntlid = -1;
|
||||
}
|
||||
|
||||
static bool nvme_calc_rgif(uint16_t nruh, uint16_t nrg, uint8_t *rgif)
|
||||
{
|
||||
uint16_t val;
|
||||
unsigned int i;
|
||||
|
||||
if (unlikely(nrg == 1)) {
|
||||
/* PIDRG_NORGI scenario, all of pid is used for PHID */
|
||||
*rgif = 0;
|
||||
return true;
|
||||
}
|
||||
|
||||
val = nrg;
|
||||
i = 0;
|
||||
while (val) {
|
||||
val >>= 1;
|
||||
i++;
|
||||
}
|
||||
*rgif = i;
|
||||
|
||||
/* ensure remaining bits suffice to represent number of phids in a RG */
|
||||
if (unlikely((UINT16_MAX >> i) < nruh)) {
|
||||
*rgif = 0;
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool nvme_subsys_setup_fdp(NvmeSubsystem *subsys, Error **errp)
|
||||
{
|
||||
NvmeEnduranceGroup *endgrp = &subsys->endgrp;
|
||||
|
||||
if (!subsys->params.fdp.runs) {
|
||||
error_setg(errp, "fdp.runs must be non-zero");
|
||||
return false;
|
||||
}
|
||||
|
||||
endgrp->fdp.runs = subsys->params.fdp.runs;
|
||||
|
||||
if (!subsys->params.fdp.nrg) {
|
||||
error_setg(errp, "fdp.nrg must be non-zero");
|
||||
return false;
|
||||
}
|
||||
|
||||
endgrp->fdp.nrg = subsys->params.fdp.nrg;
|
||||
|
||||
if (!subsys->params.fdp.nruh ||
|
||||
subsys->params.fdp.nruh > NVME_FDP_MAXPIDS) {
|
||||
error_setg(errp, "fdp.nruh must be non-zero and less than %u",
|
||||
NVME_FDP_MAXPIDS);
|
||||
return false;
|
||||
}
|
||||
|
||||
endgrp->fdp.nruh = subsys->params.fdp.nruh;
|
||||
|
||||
if (!nvme_calc_rgif(endgrp->fdp.nruh, endgrp->fdp.nrg, &endgrp->fdp.rgif)) {
|
||||
error_setg(errp,
|
||||
"cannot derive a valid rgif (nruh %"PRIu16" nrg %"PRIu32")",
|
||||
endgrp->fdp.nruh, endgrp->fdp.nrg);
|
||||
return false;
|
||||
}
|
||||
|
||||
endgrp->fdp.ruhs = g_new(NvmeRuHandle, endgrp->fdp.nruh);
|
||||
|
||||
for (uint16_t ruhid = 0; ruhid < endgrp->fdp.nruh; ruhid++) {
|
||||
endgrp->fdp.ruhs[ruhid] = (NvmeRuHandle) {
|
||||
.ruht = NVME_RUHT_INITIALLY_ISOLATED,
|
||||
.ruha = NVME_RUHA_UNUSED,
|
||||
};
|
||||
|
||||
endgrp->fdp.ruhs[ruhid].rus = g_new(NvmeReclaimUnit, endgrp->fdp.nrg);
|
||||
}
|
||||
|
||||
endgrp->fdp.enabled = true;
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool nvme_subsys_setup(NvmeSubsystem *subsys, Error **errp)
|
||||
{
|
||||
const char *nqn = subsys->params.nqn ?
|
||||
subsys->params.nqn : subsys->parent_obj.id;
|
||||
|
||||
snprintf((char *)subsys->subnqn, sizeof(subsys->subnqn),
|
||||
"nqn.2019-08.org.qemu:%s", nqn);
|
||||
|
||||
if (subsys->params.fdp.enabled && !nvme_subsys_setup_fdp(subsys, errp)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
static void nvme_subsys_realize(DeviceState *dev, Error **errp)
|
||||
{
|
||||
NvmeSubsystem *subsys = NVME_SUBSYS(dev);
|
||||
|
||||
qbus_init(&subsys->bus, sizeof(NvmeBus), TYPE_NVME_BUS, dev, dev->id);
|
||||
qbus_set_bus_hotplug_handler(BUS(&subsys->bus));
|
||||
|
||||
nvme_subsys_setup(subsys, errp);
|
||||
}
|
||||
|
||||
static const Property nvme_subsystem_props[] = {
|
||||
DEFINE_PROP_STRING("nqn", NvmeSubsystem, params.nqn),
|
||||
DEFINE_PROP_BOOL("fdp", NvmeSubsystem, params.fdp.enabled, false),
|
||||
DEFINE_PROP_SIZE("fdp.runs", NvmeSubsystem, params.fdp.runs,
|
||||
NVME_DEFAULT_RU_SIZE),
|
||||
DEFINE_PROP_UINT32("fdp.nrg", NvmeSubsystem, params.fdp.nrg, 1),
|
||||
DEFINE_PROP_UINT16("fdp.nruh", NvmeSubsystem, params.fdp.nruh, 0),
|
||||
};
|
||||
|
||||
static void nvme_subsys_class_init(ObjectClass *oc, const void *data)
|
||||
{
|
||||
DeviceClass *dc = DEVICE_CLASS(oc);
|
||||
|
||||
set_bit(DEVICE_CATEGORY_STORAGE, dc->categories);
|
||||
|
||||
dc->realize = nvme_subsys_realize;
|
||||
dc->desc = "Virtual NVMe subsystem";
|
||||
|
||||
device_class_set_props(dc, nvme_subsystem_props);
|
||||
}
|
||||
|
||||
static const TypeInfo nvme_subsys_info = {
|
||||
.name = TYPE_NVME_SUBSYS,
|
||||
.parent = TYPE_DEVICE,
|
||||
.class_init = nvme_subsys_class_init,
|
||||
.instance_size = sizeof(NvmeSubsystem),
|
||||
};
|
||||
|
||||
static void nvme_subsys_register_types(void)
|
||||
{
|
||||
type_register_static(&nvme_subsys_info);
|
||||
}
|
||||
|
||||
type_init(nvme_subsys_register_types)
|
||||
@@ -0,0 +1,229 @@
|
||||
# successful events
|
||||
pci_nvme_irq_msix(uint32_t vector) "raising MSI-X IRQ vector %u"
|
||||
pci_nvme_irq_pin(void) "pulsing IRQ pin"
|
||||
pci_nvme_irq_masked(void) "IRQ is masked"
|
||||
pci_nvme_dma_read(uint64_t prp1, uint64_t prp2) "DMA read, prp1=0x%"PRIx64" prp2=0x%"PRIx64""
|
||||
pci_nvme_dbbuf_config(uint64_t dbs_addr, uint64_t eis_addr) "dbs_addr=0x%"PRIx64" eis_addr=0x%"PRIx64""
|
||||
pci_nvme_map_addr(uint64_t addr, uint64_t len) "addr 0x%"PRIx64" len %"PRIu64""
|
||||
pci_nvme_map_addr_cmb(uint64_t addr, uint64_t len) "addr 0x%"PRIx64" len %"PRIu64""
|
||||
pci_nvme_map_prp(uint64_t trans_len, uint32_t len, uint64_t prp1, uint64_t prp2, int num_prps) "trans_len %"PRIu64" len %"PRIu32" prp1 0x%"PRIx64" prp2 0x%"PRIx64" num_prps %d"
|
||||
pci_nvme_pre_save_enter(void *n) "n=%p"
|
||||
pci_nvme_pre_save_ns_drain(void *n, int i) "n=%p i=%d"
|
||||
pci_nvme_pre_save_sq_out_req_check(void *n, int i, uint32_t head, uint32_t tail, uint32_t size) "n=%p i=%d head=0x%"PRIx32" tail=0x%"PRIx32" size=0x%"PRIx32""
|
||||
pci_nvme_pre_save_cq_req_check(void *n, int i, uint32_t head, uint32_t tail, uint32_t size) "n=%p i=%d head=0x%"PRIx32" tail=0x%"PRIx32" size=0x%"PRIx32""
|
||||
pci_nvme_pre_save_cq_unposted_cqe(void *n, int i, uint16_t cid, uint32_t nsid, uint32_t dw0, uint32_t dw1, uint16_t status, uint8_t opc) "n=%p i=%d cid %"PRIu16" nsid %"PRIu32" dw0 0x%"PRIx32" dw1 0x%"PRIx32" status 0x%"PRIx16" opc 0x%"PRIx8""
|
||||
pci_nvme_pre_save_aer(uint8_t typ, uint8_t info, uint8_t log_page) "type 0x%"PRIx8" info 0x%"PRIx8" lid 0x%"PRIx8""
|
||||
pci_nvme_post_load_enter(void *n) "n=%p"
|
||||
pci_nvme_post_load_restore_cq(void *n, int i, uint32_t head, uint32_t tail, uint32_t size) "n=%p i=%d head=0x%"PRIx32" tail=0x%"PRIx32" size=0x%"PRIx32""
|
||||
pci_nvme_post_load_restore_sq(void *n, int i, uint32_t head, uint32_t tail, uint32_t size) "n=%p i=%d head=0x%"PRIx32" tail=0x%"PRIx32" size=0x%"PRIx32""
|
||||
pci_nvme_post_load_aer(uint8_t typ, uint8_t info, uint8_t log_page) "type 0x%"PRIx8" info 0x%"PRIx8" lid 0x%"PRIx8""
|
||||
pci_nvme_map_sgl(uint8_t typ, uint64_t len) "type 0x%"PRIx8" len %"PRIu64""
|
||||
pci_nvme_io_cmd(uint16_t cid, uint32_t nsid, uint16_t sqid, uint8_t opcode, const char *opname) "cid %"PRIu16" nsid 0x%"PRIx32" sqid %"PRIu16" opc 0x%"PRIx8" opname '%s'"
|
||||
pci_nvme_admin_cmd(uint16_t cid, uint16_t sqid, uint8_t opcode, const char *opname) "cid %"PRIu16" sqid %"PRIu16" opc 0x%"PRIx8" opname '%s'"
|
||||
pci_nvme_flush_ns(uint32_t nsid) "nsid 0x%"PRIx32""
|
||||
pci_nvme_format_set(uint32_t nsid, uint8_t lbaf, uint8_t mset, uint8_t pi, uint8_t pil) "nsid %"PRIu32" lbaf %"PRIu8" mset %"PRIu8" pi %"PRIu8" pil %"PRIu8""
|
||||
pci_nvme_read(uint16_t cid, uint32_t nsid, uint32_t nlb, uint64_t count, uint64_t lba) "cid %"PRIu16" nsid %"PRIu32" nlb %"PRIu32" count %"PRIu64" lba 0x%"PRIx64""
|
||||
pci_nvme_write(uint16_t cid, const char *verb, uint32_t nsid, uint32_t nlb, uint64_t count, uint64_t lba) "cid %"PRIu16" opname '%s' nsid %"PRIu32" nlb %"PRIu32" count %"PRIu64" lba 0x%"PRIx64""
|
||||
pci_nvme_rw_cb(uint16_t cid, const char *blkname) "cid %"PRIu16" blk '%s'"
|
||||
pci_nvme_misc_cb(uint16_t cid) "cid %"PRIu16""
|
||||
pci_nvme_dif_rw(uint8_t pract, uint8_t prinfo) "pract 0x%"PRIx8" prinfo 0x%"PRIx8""
|
||||
pci_nvme_dif_rw_cb(uint16_t cid, const char *blkname) "cid %"PRIu16" blk '%s'"
|
||||
pci_nvme_dif_rw_mdata_in_cb(uint16_t cid, const char *blkname) "cid %"PRIu16" blk '%s'"
|
||||
pci_nvme_dif_rw_mdata_out_cb(uint16_t cid, const char *blkname) "cid %"PRIu16" blk '%s'"
|
||||
pci_nvme_dif_rw_check_cb(uint16_t cid, uint8_t prinfo, uint16_t apptag, uint16_t appmask, uint32_t reftag) "cid %"PRIu16" prinfo 0x%"PRIx8" apptag 0x%"PRIx16" appmask 0x%"PRIx16" reftag 0x%"PRIx32""
|
||||
pci_nvme_dif_pract_generate_dif_crc16(size_t len, size_t lba_size, size_t chksum_len, uint16_t apptag, uint32_t reftag) "len %zu lba_size %zu chksum_len %zu apptag 0x%"PRIx16" reftag 0x%"PRIx32""
|
||||
pci_nvme_dif_pract_generate_dif_crc64(size_t len, size_t lba_size, size_t chksum_len, uint16_t apptag, uint64_t reftag) "len %zu lba_size %zu chksum_len %zu apptag 0x%"PRIx16" reftag 0x%"PRIx64""
|
||||
pci_nvme_dif_check(uint8_t prinfo, uint16_t chksum_len) "prinfo 0x%"PRIx8" chksum_len %"PRIu16""
|
||||
pci_nvme_dif_prchk_disabled_crc16(uint16_t apptag, uint32_t reftag) "apptag 0x%"PRIx16" reftag 0x%"PRIx32""
|
||||
pci_nvme_dif_prchk_disabled_crc64(uint16_t apptag, uint64_t reftag) "apptag 0x%"PRIx16" reftag 0x%"PRIx64""
|
||||
pci_nvme_dif_prchk_guard_crc16(uint16_t guard, uint16_t crc) "guard 0x%"PRIx16" crc 0x%"PRIx16""
|
||||
pci_nvme_dif_prchk_guard_crc64(uint64_t guard, uint64_t crc) "guard 0x%"PRIx64" crc 0x%"PRIx64""
|
||||
pci_nvme_dif_prchk_apptag(uint16_t apptag, uint16_t elbat, uint16_t elbatm) "apptag 0x%"PRIx16" elbat 0x%"PRIx16" elbatm 0x%"PRIx16""
|
||||
pci_nvme_dif_prchk_reftag_crc16(uint32_t reftag, uint32_t elbrt) "reftag 0x%"PRIx32" elbrt 0x%"PRIx32""
|
||||
pci_nvme_dif_prchk_reftag_crc64(uint64_t reftag, uint64_t elbrt) "reftag 0x%"PRIx64" elbrt 0x%"PRIx64""
|
||||
pci_nvme_copy(uint16_t cid, uint32_t nsid, uint16_t nr, uint8_t format) "cid %"PRIu16" nsid %"PRIu32" nr %"PRIu16" format 0x%"PRIx8""
|
||||
pci_nvme_copy_source_range(uint64_t slba, uint32_t nlb) "slba 0x%"PRIx64" nlb %"PRIu32""
|
||||
pci_nvme_copy_out(uint64_t slba, uint32_t nlb) "slba 0x%"PRIx64" nlb %"PRIu32""
|
||||
pci_nvme_verify(uint16_t cid, uint32_t nsid, uint64_t slba, uint32_t nlb) "cid %"PRIu16" nsid %"PRIu32" slba 0x%"PRIx64" nlb %"PRIu32""
|
||||
pci_nvme_verify_mdata_in_cb(uint16_t cid, const char *blkname) "cid %"PRIu16" blk '%s'"
|
||||
pci_nvme_verify_cb(uint16_t cid, uint8_t prinfo, uint16_t apptag, uint16_t appmask, uint32_t reftag) "cid %"PRIu16" prinfo 0x%"PRIx8" apptag 0x%"PRIx16" appmask 0x%"PRIx16" reftag 0x%"PRIx32""
|
||||
pci_nvme_rw_complete_cb(uint16_t cid, const char *blkname) "cid %"PRIu16" blk '%s'"
|
||||
pci_nvme_block_status(int64_t offset, int64_t bytes, int64_t pnum, int ret, bool zeroed) "offset %"PRId64" bytes %"PRId64" pnum %"PRId64" ret 0x%x zeroed %d"
|
||||
pci_nvme_dsm(uint32_t nr, uint32_t attr) "nr %"PRIu32" attr 0x%"PRIx32""
|
||||
pci_nvme_dsm_deallocate(uint64_t slba, uint32_t nlb) "slba %"PRIu64" nlb %"PRIu32""
|
||||
pci_nvme_dsm_single_range_limit_exceeded(uint32_t nlb, uint32_t dmrsl) "nlb %"PRIu32" dmrsl %"PRIu32""
|
||||
pci_nvme_compare(uint16_t cid, uint32_t nsid, uint64_t slba, uint32_t nlb) "cid %"PRIu16" nsid %"PRIu32" slba 0x%"PRIx64" nlb %"PRIu32""
|
||||
pci_nvme_compare_data_cb(uint16_t cid) "cid %"PRIu16""
|
||||
pci_nvme_compare_mdata_cb(uint16_t cid) "cid %"PRIu16""
|
||||
pci_nvme_aio_discard_cb(uint16_t cid) "cid %"PRIu16""
|
||||
pci_nvme_aio_copy_in_cb(uint16_t cid) "cid %"PRIu16""
|
||||
pci_nvme_aio_flush_cb(uint16_t cid, const char *blkname) "cid %"PRIu16" blk '%s'"
|
||||
pci_nvme_create_sq(uint64_t addr, uint16_t sqid, uint16_t cqid, uint16_t qsize, uint16_t qflags) "create submission queue, addr=0x%"PRIx64", sqid=%"PRIu16", cqid=%"PRIu16", qsize=%"PRIu16", qflags=%"PRIu16""
|
||||
pci_nvme_create_cq(uint64_t addr, uint16_t cqid, uint16_t vector, uint16_t size, uint16_t qflags, int ien) "create completion queue, addr=0x%"PRIx64", cqid=%"PRIu16", vector=%"PRIu16", qsize=%"PRIu16", qflags=%"PRIu16", ien=%d"
|
||||
pci_nvme_del_sq(uint16_t qid) "deleting submission queue sqid=%"PRIu16""
|
||||
pci_nvme_del_cq(uint16_t cqid) "deleted completion queue, cqid=%"PRIu16""
|
||||
pci_nvme_identify(uint16_t cid, uint8_t cns, uint16_t ctrlid, uint8_t csi) "cid %"PRIu16" cns 0x%"PRIx8" ctrlid %"PRIu16" csi 0x%"PRIx8""
|
||||
pci_nvme_identify_ctrl(void) "identify controller"
|
||||
pci_nvme_identify_ctrl_csi(uint8_t csi) "identify controller, csi=0x%"PRIx8""
|
||||
pci_nvme_identify_ns(uint32_t ns) "nsid %"PRIu32""
|
||||
pci_nvme_identify_ns_ind(uint32_t nsid) "nsid %"PRIu32""
|
||||
pci_nvme_identify_ctrl_list(uint8_t cns, uint16_t cntid) "cns 0x%"PRIx8" cntid %"PRIu16""
|
||||
pci_nvme_identify_pri_ctrl_cap(uint16_t cntlid) "identify primary controller capabilities cntlid=%"PRIu16""
|
||||
pci_nvme_identify_sec_ctrl_list(uint16_t cntlid, uint8_t numcntl) "identify secondary controller list cntlid=%"PRIu16" numcntl=%"PRIu8""
|
||||
pci_nvme_identify_ns_csi(uint32_t ns, uint8_t csi) "nsid=%"PRIu32", csi=0x%"PRIx8""
|
||||
pci_nvme_identify_nslist(uint32_t ns) "nsid %"PRIu32""
|
||||
pci_nvme_identify_nslist_csi(uint16_t ns, uint8_t csi) "nsid=%"PRIu16", csi=0x%"PRIx8""
|
||||
pci_nvme_identify_cmd_set(void) "identify i/o command set"
|
||||
pci_nvme_identify_ns_descr_list(uint32_t ns) "nsid %"PRIu32""
|
||||
pci_nvme_get_log(uint16_t cid, uint8_t lid, uint8_t lsp, uint8_t rae, uint32_t len, uint64_t off) "cid %"PRIu16" lid 0x%"PRIx8" lsp 0x%"PRIx8" rae 0x%"PRIx8" len %"PRIu32" off %"PRIu64""
|
||||
pci_nvme_getfeat(uint16_t cid, uint32_t nsid, uint8_t fid, uint8_t sel, uint32_t cdw11) "cid %"PRIu16" nsid 0x%"PRIx32" fid 0x%"PRIx8" sel 0x%"PRIx8" cdw11 0x%"PRIx32""
|
||||
pci_nvme_setfeat(uint16_t cid, uint32_t nsid, uint8_t fid, uint8_t save, uint32_t cdw11) "cid %"PRIu16" nsid 0x%"PRIx32" fid 0x%"PRIx8" save 0x%"PRIx8" cdw11 0x%"PRIx32""
|
||||
pci_nvme_getfeat_vwcache(const char* result) "get feature volatile write cache, result=%s"
|
||||
pci_nvme_getfeat_numq(int result) "get feature number of queues, result=%d"
|
||||
pci_nvme_setfeat_numq(int reqcq, int reqsq, int gotcq, int gotsq) "requested cq_count=%d sq_count=%d, responding with cq_count=%d sq_count=%d"
|
||||
pci_nvme_setfeat_timestamp(uint64_t ts) "set feature timestamp = 0x%"PRIx64""
|
||||
pci_nvme_getfeat_timestamp(uint64_t ts) "get feature timestamp = 0x%"PRIx64""
|
||||
pci_nvme_process_aers(int queued) "queued %d"
|
||||
pci_nvme_aer(uint16_t cid) "cid %"PRIu16""
|
||||
pci_nvme_aer_aerl_exceeded(void) "aerl exceeded"
|
||||
pci_nvme_aer_masked(uint8_t type, uint8_t mask) "type 0x%"PRIx8" mask 0x%"PRIx8""
|
||||
pci_nvme_aer_post_cqe(uint8_t typ, uint8_t info, uint8_t log_page) "type 0x%"PRIx8" info 0x%"PRIx8" lid 0x%"PRIx8""
|
||||
pci_nvme_ns_attachment(uint16_t cid, uint8_t sel) "cid %"PRIu16", sel=0x%"PRIx8""
|
||||
pci_nvme_ns_attachment_attach(uint16_t cntlid, uint32_t nsid) "cntlid=0x%"PRIx16", nsid=0x%"PRIx32""
|
||||
pci_nvme_enqueue_event(uint8_t typ, uint8_t info, uint8_t log_page) "type 0x%"PRIx8" info 0x%"PRIx8" lid 0x%"PRIx8""
|
||||
pci_nvme_enqueue_event_noqueue(int queued) "queued %d"
|
||||
pci_nvme_enqueue_event_masked(uint8_t typ) "type 0x%"PRIx8""
|
||||
pci_nvme_no_outstanding_aers(void) "ignoring event; no outstanding AERs"
|
||||
pci_nvme_enqueue_req_completion(uint16_t cid, uint16_t cqid, uint32_t dw0, uint32_t dw1, uint16_t status) "cid %"PRIu16" cqid %"PRIu16" dw0 0x%"PRIx32" dw1 0x%"PRIx32" status 0x%"PRIx16""
|
||||
pci_nvme_update_cq_eventidx(uint16_t cqid, uint16_t new_eventidx) "cqid %"PRIu16" new_eventidx %"PRIu16""
|
||||
pci_nvme_update_sq_eventidx(uint16_t sqid, uint16_t new_eventidx) "sqid %"PRIu16" new_eventidx %"PRIu16""
|
||||
pci_nvme_mmio_read(uint64_t addr, unsigned size) "addr 0x%"PRIx64" size %d"
|
||||
pci_nvme_mmio_write(uint64_t addr, uint64_t data, unsigned size) "addr 0x%"PRIx64" data 0x%"PRIx64" size %d"
|
||||
pci_nvme_mmio_doorbell_cq(uint16_t cqid, uint16_t new_head) "cqid %"PRIu16" new_head %"PRIu16""
|
||||
pci_nvme_mmio_doorbell_sq(uint16_t sqid, uint16_t new_tail) "sqid %"PRIu16" new_tail %"PRIu16""
|
||||
pci_nvme_mmio_intm_set(uint64_t data, uint64_t new_mask) "wrote MMIO, interrupt mask set, data=0x%"PRIx64", new_mask=0x%"PRIx64""
|
||||
pci_nvme_mmio_intm_clr(uint64_t data, uint64_t new_mask) "wrote MMIO, interrupt mask clr, data=0x%"PRIx64", new_mask=0x%"PRIx64""
|
||||
pci_nvme_mmio_cfg(uint64_t data) "wrote MMIO, config controller config=0x%"PRIx64""
|
||||
pci_nvme_mmio_aqattr(uint64_t data) "wrote MMIO, admin queue attributes=0x%"PRIx64""
|
||||
pci_nvme_mmio_asqaddr(uint64_t data) "wrote MMIO, admin submission queue address=0x%"PRIx64""
|
||||
pci_nvme_mmio_acqaddr(uint64_t data) "wrote MMIO, admin completion queue address=0x%"PRIx64""
|
||||
pci_nvme_mmio_asqaddr_hi(uint64_t data, uint64_t new_addr) "wrote MMIO, admin submission queue high half=0x%"PRIx64", new_address=0x%"PRIx64""
|
||||
pci_nvme_mmio_acqaddr_hi(uint64_t data, uint64_t new_addr) "wrote MMIO, admin completion queue high half=0x%"PRIx64", new_address=0x%"PRIx64""
|
||||
pci_nvme_mmio_start_success(void) "setting controller enable bit succeeded"
|
||||
pci_nvme_mmio_stopped(void) "cleared controller enable bit"
|
||||
pci_nvme_mmio_shutdown_set(void) "shutdown bit set"
|
||||
pci_nvme_mmio_shutdown_cleared(void) "shutdown bit cleared"
|
||||
pci_nvme_update_cq_head(uint16_t cqid, uint16_t new_head) "cqid %"PRIu16" new_head %"PRIu16""
|
||||
pci_nvme_update_sq_tail(uint16_t sqid, uint16_t new_tail) "sqid %"PRIu16" new_tail %"PRIu16""
|
||||
pci_nvme_open_zone(uint64_t slba, uint32_t zone_idx, int all) "open zone, slba=%"PRIu64", idx=%"PRIu32", all=%"PRIi32""
|
||||
pci_nvme_close_zone(uint64_t slba, uint32_t zone_idx, int all) "close zone, slba=%"PRIu64", idx=%"PRIu32", all=%"PRIi32""
|
||||
pci_nvme_finish_zone(uint64_t slba, uint32_t zone_idx, int all) "finish zone, slba=%"PRIu64", idx=%"PRIu32", all=%"PRIi32""
|
||||
pci_nvme_reset_zone(uint64_t slba, uint32_t zone_idx, int all) "reset zone, slba=%"PRIu64", idx=%"PRIu32", all=%"PRIi32""
|
||||
pci_nvme_zns_zone_reset(uint64_t zslba) "zslba 0x%"PRIx64""
|
||||
pci_nvme_offline_zone(uint64_t slba, uint32_t zone_idx, int all) "offline zone, slba=%"PRIu64", idx=%"PRIu32", all=%"PRIi32""
|
||||
pci_nvme_set_descriptor_extension(uint64_t slba, uint32_t zone_idx) "set zone descriptor extension, slba=%"PRIu64", idx=%"PRIu32""
|
||||
pci_nvme_zd_extension_set(uint32_t zone_idx) "set descriptor extension for zone_idx=%"PRIu32""
|
||||
pci_nvme_clear_ns_close(uint32_t state, uint64_t slba) "zone state=%"PRIu32", slba=%"PRIu64" transitioned to Closed state"
|
||||
pci_nvme_clear_ns_reset(uint32_t state, uint64_t slba) "zone state=%"PRIu32", slba=%"PRIu64" transitioned to Empty state"
|
||||
pci_nvme_zoned_zrwa_implicit_flush(uint64_t zslba, uint32_t nlb) "zslba 0x%"PRIx64" nlb %"PRIu32""
|
||||
pci_nvme_pci_reset(void) "PCI Function Level Reset"
|
||||
pci_nvme_virt_mngmt(uint16_t cid, uint16_t act, uint16_t cntlid, const char* rt, uint16_t nr) "cid %"PRIu16", act=0x%"PRIx16", ctrlid=%"PRIu16" %s nr=%"PRIu16""
|
||||
pci_nvme_fdp_ruh_change(uint16_t rgid, uint16_t ruhid) "change RU on RUH rgid=%"PRIu16", ruhid=%"PRIu16""
|
||||
|
||||
# error conditions
|
||||
pci_nvme_err_mdts(size_t len) "len %zu"
|
||||
pci_nvme_err_zasl(size_t len) "len %zu"
|
||||
pci_nvme_err_req_status(uint16_t cid, uint32_t nsid, uint16_t status, uint8_t opc) "cid %"PRIu16" nsid %"PRIu32" status 0x%"PRIx16" opc 0x%"PRIx8""
|
||||
pci_nvme_err_addr_read(uint64_t addr) "addr 0x%"PRIx64""
|
||||
pci_nvme_err_addr_write(uint64_t addr) "addr 0x%"PRIx64""
|
||||
pci_nvme_err_cfs(void) "controller fatal status"
|
||||
pci_nvme_err_aio(uint16_t cid, const char *errname, uint16_t status) "cid %"PRIu16" err '%s' status 0x%"PRIx16""
|
||||
pci_nvme_err_copy_invalid_format(uint8_t format) "format 0x%"PRIx8""
|
||||
pci_nvme_err_invalid_sgld(uint16_t cid, uint8_t typ) "cid %"PRIu16" type 0x%"PRIx8""
|
||||
pci_nvme_err_invalid_num_sgld(uint16_t cid, uint8_t typ) "cid %"PRIu16" type 0x%"PRIx8""
|
||||
pci_nvme_err_invalid_sgl_excess_length(uint32_t residual) "residual %"PRIu32""
|
||||
pci_nvme_err_invalid_dma(void) "PRP/SGL is too small for transfer size"
|
||||
pci_nvme_err_invalid_prplist_ent(uint64_t prplist) "PRP list entry is not page aligned: 0x%"PRIx64""
|
||||
pci_nvme_err_invalid_prp2_align(uint64_t prp2) "PRP2 is not page aligned: 0x%"PRIx64""
|
||||
pci_nvme_err_invalid_opc(uint8_t opc) "invalid opcode 0x%"PRIx8""
|
||||
pci_nvme_err_invalid_admin_opc(uint8_t opc) "invalid admin opcode 0x%"PRIx8""
|
||||
pci_nvme_err_invalid_lba_range(uint64_t start, uint64_t len, uint64_t limit) "Invalid LBA start=%"PRIu64" len=%"PRIu64" limit=%"PRIu64""
|
||||
pci_nvme_err_invalid_log_page_offset(uint64_t ofs, uint64_t size) "must be <= %"PRIu64", got %"PRIu64""
|
||||
pci_nvme_err_cmb_invalid_cba(uint64_t cmbmsc) "cmbmsc 0x%"PRIx64""
|
||||
pci_nvme_err_cmb_not_enabled(uint64_t cmbmsc) "cmbmsc 0x%"PRIx64""
|
||||
pci_nvme_err_unaligned_zone_cmd(uint8_t action, uint64_t slba, uint64_t zslba) "unaligned zone op 0x%"PRIx32", got slba=%"PRIu64", zslba=%"PRIu64""
|
||||
pci_nvme_err_invalid_zone_state_transition(uint8_t action, uint64_t slba, uint8_t attrs) "action=0x%"PRIx8", slba=%"PRIu64", attrs=0x%"PRIx32""
|
||||
pci_nvme_err_write_not_at_wp(uint64_t slba, uint64_t zone, uint64_t wp) "writing at slba=%"PRIu64", zone=%"PRIu64", but wp=%"PRIu64""
|
||||
pci_nvme_err_append_not_at_start(uint64_t slba, uint64_t zone) "appending at slba=%"PRIu64", but zone=%"PRIu64""
|
||||
pci_nvme_err_zone_is_full(uint64_t zslba) "zslba 0x%"PRIx64""
|
||||
pci_nvme_err_zone_is_read_only(uint64_t zslba) "zslba 0x%"PRIx64""
|
||||
pci_nvme_err_zone_is_offline(uint64_t zslba) "zslba 0x%"PRIx64""
|
||||
pci_nvme_err_zone_boundary(uint64_t slba, uint32_t nlb, uint64_t zcap) "lba 0x%"PRIx64" nlb %"PRIu32" zcap 0x%"PRIx64""
|
||||
pci_nvme_err_zone_invalid_write(uint64_t slba, uint64_t wp) "lba 0x%"PRIx64" wp 0x%"PRIx64""
|
||||
pci_nvme_err_zone_write_not_ok(uint64_t slba, uint32_t nlb, uint16_t status) "slba=%"PRIu64", nlb=%"PRIu32", status=0x%"PRIx16""
|
||||
pci_nvme_err_zone_read_not_ok(uint64_t slba, uint32_t nlb, uint16_t status) "slba=%"PRIu64", nlb=%"PRIu32", status=0x%"PRIx16""
|
||||
pci_nvme_err_insuff_active_res(uint32_t max_active) "max_active=%"PRIu32" zone limit exceeded"
|
||||
pci_nvme_err_insuff_open_res(uint32_t max_open) "max_open=%"PRIu32" zone limit exceeded"
|
||||
pci_nvme_err_zd_extension_map_error(uint32_t zone_idx) "can't map descriptor extension for zone_idx=%"PRIu32""
|
||||
pci_nvme_err_invalid_iocsci(uint32_t idx) "unsupported command set combination index %"PRIu32""
|
||||
pci_nvme_err_invalid_del_sq(uint16_t qid) "invalid submission queue deletion, sid=%"PRIu16""
|
||||
pci_nvme_err_invalid_create_sq_cqid(uint16_t cqid) "failed creating submission queue, invalid cqid=%"PRIu16""
|
||||
pci_nvme_err_invalid_create_sq_sqid(uint16_t sqid) "failed creating submission queue, invalid sqid=%"PRIu16""
|
||||
pci_nvme_err_invalid_create_sq_size(uint16_t qsize) "failed creating submission queue, invalid qsize=%"PRIu16""
|
||||
pci_nvme_err_invalid_create_sq_addr(uint64_t addr) "failed creating submission queue, addr=0x%"PRIx64""
|
||||
pci_nvme_err_invalid_create_sq_qflags(uint16_t qflags) "failed creating submission queue, qflags=%"PRIu16""
|
||||
pci_nvme_err_invalid_del_cq_cqid(uint16_t cqid) "failed deleting completion queue, cqid=%"PRIu16""
|
||||
pci_nvme_err_invalid_del_cq_notempty(uint16_t cqid) "failed deleting completion queue, it is not empty, cqid=%"PRIu16""
|
||||
pci_nvme_err_invalid_create_cq_cqid(uint16_t cqid) "failed creating completion queue, cqid=%"PRIu16""
|
||||
pci_nvme_err_invalid_create_cq_size(uint16_t size) "failed creating completion queue, size=%"PRIu16""
|
||||
pci_nvme_err_invalid_create_cq_addr(uint64_t addr) "failed creating completion queue, addr=0x%"PRIx64""
|
||||
pci_nvme_err_invalid_create_cq_vector(uint16_t vector) "failed creating completion queue, vector=%"PRIu16""
|
||||
pci_nvme_err_invalid_create_cq_qflags(uint16_t qflags) "failed creating completion queue, qflags=%"PRIu16""
|
||||
pci_nvme_err_invalid_create_cq_entry_size(uint8_t iosqes, uint8_t iocqes) "iosqes %"PRIu8" iocqes %"PRIu8""
|
||||
pci_nvme_err_invalid_identify_cns(uint16_t cns) "identify, invalid cns=0x%"PRIx16""
|
||||
pci_nvme_err_invalid_getfeat(int dw10) "invalid get features, dw10=0x%"PRIx32""
|
||||
pci_nvme_err_invalid_setfeat(uint32_t dw10) "invalid set features, dw10=0x%"PRIx32""
|
||||
pci_nvme_err_invalid_log_page(uint16_t cid, uint16_t lid) "cid %"PRIu16" lid 0x%"PRIx16""
|
||||
pci_nvme_err_startfail_cq(void) "nvme_start_ctrl failed because there are non-admin completion queues"
|
||||
pci_nvme_err_startfail_sq(void) "nvme_start_ctrl failed because there are non-admin submission queues"
|
||||
pci_nvme_err_startfail_asq_misaligned(uint64_t addr) "nvme_start_ctrl failed because the admin submission queue address is misaligned: 0x%"PRIx64""
|
||||
pci_nvme_err_startfail_acq_misaligned(uint64_t addr) "nvme_start_ctrl failed because the admin completion queue address is misaligned: 0x%"PRIx64""
|
||||
pci_nvme_err_startfail_page_too_small(uint8_t log2ps, uint8_t maxlog2ps) "nvme_start_ctrl failed because the page size is too small: log2size=%u, min=%u"
|
||||
pci_nvme_err_startfail_page_too_large(uint8_t log2ps, uint8_t maxlog2ps) "nvme_start_ctrl failed because the page size is too large: log2size=%u, max=%u"
|
||||
pci_nvme_err_startfail_cqent_too_small(uint8_t log2ps, uint8_t maxlog2ps) "nvme_start_ctrl failed because the completion queue entry size is too small: log2size=%u, min=%u"
|
||||
pci_nvme_err_startfail_cqent_too_large(uint8_t log2ps, uint8_t maxlog2ps) "nvme_start_ctrl failed because the completion queue entry size is too large: log2size=%u, max=%u"
|
||||
pci_nvme_err_startfail_sqent_too_small(uint8_t log2ps, uint8_t maxlog2ps) "nvme_start_ctrl failed because the submission queue entry size is too small: log2size=%u, min=%u"
|
||||
pci_nvme_err_startfail_sqent_too_large(uint8_t log2ps, uint8_t maxlog2ps) "nvme_start_ctrl failed because the submission queue entry size is too large: log2size=%u, max=%u"
|
||||
pci_nvme_err_startfail_css(uint8_t css) "nvme_start_ctrl failed because invalid command set selected:%u"
|
||||
pci_nvme_err_startfail_asqent_sz_zero(void) "nvme_start_ctrl failed because the admin submission queue size is zero"
|
||||
pci_nvme_err_startfail_acqent_sz_zero(void) "nvme_start_ctrl failed because the admin completion queue size is zero"
|
||||
pci_nvme_err_startfail_zasl_too_small(uint32_t zasl, uint32_t pagesz) "nvme_start_ctrl failed because zone append size limit %"PRIu32" is too small, needs to be >= %"PRIu32""
|
||||
pci_nvme_err_startfail(void) "setting controller enable bit failed"
|
||||
pci_nvme_err_startfail_virt_state(uint16_t vq, uint16_t vi) "nvme_start_ctrl failed due to ctrl state: vi=%u vq=%u"
|
||||
pci_nvme_err_invalid_mgmt_action(uint8_t action) "action=0x%"PRIx8""
|
||||
pci_nvme_err_ignored_mmio_vf_offline(uint64_t addr, unsigned size) "addr 0x%"PRIx64" size %d"
|
||||
|
||||
# undefined behavior
|
||||
pci_nvme_ub_mmiowr_misaligned32(uint64_t offset) "MMIO write not 32-bit aligned, offset=0x%"PRIx64""
|
||||
pci_nvme_ub_mmiowr_toosmall(uint64_t offset, unsigned size) "MMIO write smaller than 32 bits, offset=0x%"PRIx64", size=%u"
|
||||
pci_nvme_ub_mmiowr_intmask_with_msix(void) "undefined access to interrupt mask set when MSI-X is enabled"
|
||||
pci_nvme_ub_mmiowr_ro_csts(void) "attempted to set a read only bit of controller status"
|
||||
pci_nvme_ub_mmiowr_ssreset_w1c_unsupported(void) "attempted to W1C CSTS.NSSRO but CAP.NSSRS is zero (not supported)"
|
||||
pci_nvme_ub_mmiowr_ssreset_unsupported(void) "attempted NVM subsystem reset but CAP.NSSRS is zero (not supported)"
|
||||
pci_nvme_ub_mmiowr_cmbloc_reserved(void) "invalid write to reserved CMBLOC when CMBSZ is zero, ignored"
|
||||
pci_nvme_ub_mmiowr_cmbsz_readonly(void) "invalid write to read only CMBSZ, ignored"
|
||||
pci_nvme_ub_mmiowr_pmrcap_readonly(void) "invalid write to read only PMRCAP, ignored"
|
||||
pci_nvme_ub_mmiowr_pmrsts_readonly(void) "invalid write to read only PMRSTS, ignored"
|
||||
pci_nvme_ub_mmiowr_pmrebs_readonly(void) "invalid write to read only PMREBS, ignored"
|
||||
pci_nvme_ub_mmiowr_pmrswtp_readonly(void) "invalid write to read only PMRSWTP, ignored"
|
||||
pci_nvme_ub_mmiowr_invalid(uint64_t offset, uint64_t data) "invalid MMIO write, offset=0x%"PRIx64", data=0x%"PRIx64""
|
||||
pci_nvme_ub_mmiord_misaligned32(uint64_t offset) "MMIO read not 32-bit aligned, offset=0x%"PRIx64""
|
||||
pci_nvme_ub_mmiord_toosmall(uint64_t offset) "MMIO read smaller than 32-bits, offset=0x%"PRIx64""
|
||||
pci_nvme_ub_mmiord_invalid_ofs(uint64_t offset) "MMIO read beyond last register, offset=0x%"PRIx64", returning 0"
|
||||
pci_nvme_ub_db_wr_misaligned(uint64_t offset) "doorbell write not 32-bit aligned, offset=0x%"PRIx64", ignoring"
|
||||
pci_nvme_ub_db_wr_invalid_cq(uint32_t qid) "completion queue doorbell write for nonexistent queue, cqid=%"PRIu32", ignoring"
|
||||
pci_nvme_ub_db_wr_invalid_cqhead(uint32_t qid, uint16_t new_head) "completion queue doorbell write value beyond queue size, cqid=%"PRIu32", new_head=%"PRIu16", ignoring"
|
||||
pci_nvme_ub_db_wr_invalid_sq(uint32_t qid) "submission queue doorbell write for nonexistent queue, sqid=%"PRIu32", ignoring"
|
||||
pci_nvme_ub_db_wr_invalid_sqtail(uint32_t qid, uint16_t new_tail) "submission queue doorbell write value beyond queue size, sqid=%"PRIu32", new_head=%"PRIu16", ignoring"
|
||||
pci_nvme_ub_unknown_css_value(void) "unknown value in cc.css field"
|
||||
pci_nvme_ub_too_many_mappings(void) "too many prp/sgl mappings"
|
||||
@@ -0,0 +1 @@
|
||||
#include "trace/trace-hw_nvme.h"
|
||||
Reference in New Issue
Block a user