motorhead/qemu
1
0
Fork 0
mirror of https://github.com/Motorhead1991/qemu.git synced 2025-08-06 17:23:56 -06:00
Code Issues Projects Releases Packages Wiki Activity Actions

hw/cxl: Multi-Region CXL Type-3 Devices (Volatile and Persistent)

Browse source 
This commit enables each CXL Type-3 device to contain one volatile
memory region and one persistent region.

Two new properties have been added to cxl-type3 device initialization:
    [volatile-memdev] and [persistent-memdev]

The existing [memdev] property has been deprecated and will default the
memory region to a persistent memory region (although a user may assign
the region to a ram or file backed region). It cannot be used in
combination with the new [persistent-memdev] property.

Partitioning volatile memory from persistent memory is not yet supported.

Volatile memory is mapped at DPA(0x0), while Persistent memory is mapped
at DPA(vmem->size), per CXL Spec 8.2.9.8.2.0 - Get Partition Info.

Signed-off-by: Gregory Price <gregory.price@memverge.com>
Reviewed-by: Davidlohr Bueso <dave@stgolabs.net>
Reviewed-by: Fan Ni <fan.ni@samsung.com>
Tested-by: Fan Ni <fan.ni@samsung.com>
Signed-off-by: Jonathan Cameron <Jonathan.Cameron@huawei.com>

Message-Id: <20230421160827.2227-4-Jonathan.Cameron@huawei.com>
Reviewed-by: Michael S. Tsirkin <mst@redhat.com>
Signed-off-by: Michael S. Tsirkin <mst@redhat.com>
This commit is contained in:
Gregory Price 2023-04-21 17:08:27 +01:00 • committed by Michael S. Tsirkin
parent 3521176526
commit adacc814f5
7 changed files with 371 additions and 119 deletions
Download patch file Download diff file Expand all files Collapse all files

32
hw/cxl/cxl-mailbox-utils.c
View file

@ -141,7 +141,8 @@ static ret_code cmd_firmware_update_get_info(struct cxl_cmd *cmd,
} QEMU_PACKED *fw_info;
QEMU_BUILD_BUG_ON(sizeof(*fw_info) != 0x50);
if (cxl_dstate->pmem_size < CXL_CAPACITY_MULTIPLIER) {
if ((cxl_dstate->vmem_size < CXL_CAPACITY_MULTIPLIER) ||
(cxl_dstate->pmem_size < CXL_CAPACITY_MULTIPLIER)) {
return CXL_MBOX_INTERNAL_ERROR;
}
@ -288,21 +289,21 @@ static ret_code cmd_identify_memory_device(struct cxl_cmd *cmd,
CXLType3Dev *ct3d = container_of(cxl_dstate, CXLType3Dev, cxl_dstate);
CXLType3Class *cvc = CXL_TYPE3_GET_CLASS(ct3d);
uint64_t size = cxl_dstate->pmem_size;
if (!QEMU_IS_ALIGNED(size, CXL_CAPACITY_MULTIPLIER)) {
if ((!QEMU_IS_ALIGNED(cxl_dstate->vmem_size, CXL_CAPACITY_MULTIPLIER)) ||
(!QEMU_IS_ALIGNED(cxl_dstate->pmem_size, CXL_CAPACITY_MULTIPLIER))) {
return CXL_MBOX_INTERNAL_ERROR;
}
id = (void *)cmd->payload;
memset(id, 0, sizeof(*id));
/* PMEM only */
snprintf(id->fw_revision, 0x10, "BWFW VERSION %02d", 0);
id->total_capacity = size / CXL_CAPACITY_MULTIPLIER;
id->persistent_capacity = size / CXL_CAPACITY_MULTIPLIER;
id->lsa_size = cvc->get_lsa_size(ct3d);
stq_le_p(&id->total_capacity, cxl_dstate->mem_size / CXL_CAPACITY_MULTIPLIER);
stq_le_p(&id->persistent_capacity, cxl_dstate->pmem_size / CXL_CAPACITY_MULTIPLIER);
stq_le_p(&id->volatile_capacity, cxl_dstate->vmem_size / CXL_CAPACITY_MULTIPLIER);
stl_le_p(&id->lsa_size, cvc->get_lsa_size(ct3d));
*len = sizeof(*id);
return CXL_MBOX_SUCCESS;
@ -319,17 +320,20 @@ static ret_code cmd_ccls_get_partition_info(struct cxl_cmd *cmd,
uint64_t next_pmem;
} QEMU_PACKED *part_info = (void *)cmd->payload;
QEMU_BUILD_BUG_ON(sizeof(*part_info) != 0x20);
uint64_t size = cxl_dstate->pmem_size;
if (!QEMU_IS_ALIGNED(size, CXL_CAPACITY_MULTIPLIER)) {
if ((!QEMU_IS_ALIGNED(cxl_dstate->vmem_size, CXL_CAPACITY_MULTIPLIER)) ||
(!QEMU_IS_ALIGNED(cxl_dstate->pmem_size, CXL_CAPACITY_MULTIPLIER))) {
return CXL_MBOX_INTERNAL_ERROR;
}
/* PMEM only */
part_info->active_vmem = 0;
part_info->next_vmem = 0;
part_info->active_pmem = size / CXL_CAPACITY_MULTIPLIER;
part_info->next_pmem = 0;
stq_le_p(&part_info->active_vmem, cxl_dstate->vmem_size / CXL_CAPACITY_MULTIPLIER);
/*
* When both next_vmem and next_pmem are 0, there is no pending change to
* partitioning.
*/
stq_le_p(&part_info->next_vmem, 0);
stq_le_p(&part_info->active_pmem, cxl_dstate->pmem_size / CXL_CAPACITY_MULTIPLIER);
stq_le_p(&part_info->next_pmem, 0);
*len = sizeof(*part_info);
return CXL_MBOX_SUCCESS;

306
hw/mem/cxl_type3.c
View file

@ -31,7 +31,8 @@ enum {
};
static int ct3_build_cdat_entries_for_mr(CDATSubHeader **cdat_table,
int dsmad_handle, MemoryRegion *mr)
int dsmad_handle, MemoryRegion *mr,
bool is_pmem, uint64_t dpa_base)
{
g_autofree CDATDsmas *dsmas = NULL;
g_autofree CDATDslbis *dslbis0 = NULL;
@ -50,8 +51,8 @@ static int ct3_build_cdat_entries_for_mr(CDATSubHeader **cdat_table,
.length = sizeof(*dsmas),
},
.DSMADhandle = dsmad_handle,
.flags = CDAT_DSMAS_FLAG_NV,
.DPA_base = 0,
.flags = is_pmem ? CDAT_DSMAS_FLAG_NV : 0,
.DPA_base = dpa_base,
.DPA_length = memory_region_size(mr),
};
@ -130,8 +131,11 @@ static int ct3_build_cdat_entries_for_mr(CDATSubHeader **cdat_table,
.length = sizeof(*dsemts),
},
.DSMAS_handle = dsmad_handle,
/* Reserved - the non volatile from DSMAS matters */
.EFI_memory_type_attr = 2,
/*
* NV: Reserved - the non volatile from DSMAS matters
* V: EFI_MEMORY_SP
*/
.EFI_memory_type_attr = is_pmem ? 2 : 1,
.DPA_offset = 0,
.DPA_length = memory_region_size(mr),
};
@ -150,33 +154,68 @@ static int ct3_build_cdat_entries_for_mr(CDATSubHeader **cdat_table,
static int ct3_build_cdat_table(CDATSubHeader ***cdat_table, void *priv)
{
g_autofree CDATSubHeader **table = NULL;
MemoryRegion *nonvolatile_mr;
CXLType3Dev *ct3d = priv;
MemoryRegion *volatile_mr = NULL, *nonvolatile_mr = NULL;
int dsmad_handle = 0;
int rc;
int cur_ent = 0;
int len = 0;
int rc, i;
if (!ct3d->hostmem) {
if (!ct3d->hostpmem && !ct3d->hostvmem) {
return 0;
}
nonvolatile_mr = host_memory_backend_get_memory(ct3d->hostmem);
if (!nonvolatile_mr) {
return -EINVAL;
if (ct3d->hostvmem) {
volatile_mr = host_memory_backend_get_memory(ct3d->hostvmem);
if (!volatile_mr) {
return -EINVAL;
}
len += CT3_CDAT_NUM_ENTRIES;
}
table = g_malloc0(CT3_CDAT_NUM_ENTRIES * sizeof(*table));
if (ct3d->hostpmem) {
nonvolatile_mr = host_memory_backend_get_memory(ct3d->hostpmem);
if (!nonvolatile_mr) {
return -EINVAL;
}
len += CT3_CDAT_NUM_ENTRIES;
}
table = g_malloc0(len * sizeof(*table));
if (!table) {
return -ENOMEM;
}
rc = ct3_build_cdat_entries_for_mr(table, dsmad_handle++, nonvolatile_mr);
if (rc < 0) {
return rc;
/* Now fill them in */
if (volatile_mr) {
rc = ct3_build_cdat_entries_for_mr(table, dsmad_handle++, volatile_mr,
false, 0);
if (rc < 0) {
return rc;
}
cur_ent = CT3_CDAT_NUM_ENTRIES;
}
if (nonvolatile_mr) {
rc = ct3_build_cdat_entries_for_mr(&(table[cur_ent]), dsmad_handle++,
nonvolatile_mr, true,
(volatile_mr ?
memory_region_size(volatile_mr) : 0));
if (rc < 0) {
goto error_cleanup;
}
cur_ent += CT3_CDAT_NUM_ENTRIES;
}
assert(len == cur_ent);
*cdat_table = g_steal_pointer(&table);
return CT3_CDAT_NUM_ENTRIES;
return len;
error_cleanup:
for (i = 0; i < cur_ent; i++) {
g_free(table[i]);
}
return rc;
}
static void ct3_free_cdat_table(CDATSubHeader **cdat_table, int num, void *priv)
@ -264,16 +303,42 @@ static void build_dvsecs(CXLType3Dev *ct3d)
{
CXLComponentState *cxl_cstate = &ct3d->cxl_cstate;
uint8_t *dvsec;
uint32_t range1_size_hi, range1_size_lo,
range1_base_hi = 0, range1_base_lo = 0,
range2_size_hi = 0, range2_size_lo = 0,
range2_base_hi = 0, range2_base_lo = 0;
/*
* Volatile memory is mapped as (0x0)
* Persistent memory is mapped at (volatile->size)
*/
if (ct3d->hostvmem) {
range1_size_hi = ct3d->hostvmem->size >> 32;
range1_size_lo = (2 << 5) | (2 << 2) | 0x3 |
(ct3d->hostvmem->size & 0xF0000000);
if (ct3d->hostpmem) {
range2_size_hi = ct3d->hostpmem->size >> 32;
range2_size_lo = (2 << 5) | (2 << 2) | 0x3 |
(ct3d->hostpmem->size & 0xF0000000);
}
} else {
range1_size_hi = ct3d->hostpmem->size >> 32;
range1_size_lo = (2 << 5) | (2 << 2) | 0x3 |
(ct3d->hostpmem->size & 0xF0000000);
}
dvsec = (uint8_t *)&(CXLDVSECDevice){
.cap = 0x1e,
.ctrl = 0x2,
.status2 = 0x2,
.range1_size_hi = ct3d->hostmem->size >> 32,
.range1_size_lo = (2 << 5) | (2 << 2) | 0x3 |
(ct3d->hostmem->size & 0xF0000000),
.range1_base_hi = 0,
.range1_base_lo = 0,
.range1_size_hi = range1_size_hi,
.range1_size_lo = range1_size_lo,
.range1_base_hi = range1_base_hi,
.range1_base_lo = range1_base_lo,
.range2_size_hi = range2_size_hi,
.range2_size_lo = range2_size_lo,
.range2_base_hi = range2_base_hi,
.range2_base_lo = range2_base_lo,
};
cxl_component_create_dvsec(cxl_cstate, CXL2_TYPE3_DEVICE,
PCIE_CXL_DEVICE_DVSEC_LENGTH,
@ -514,36 +579,69 @@ static void ct3d_reg_write(void *opaque, hwaddr offset, uint64_t value,
static bool cxl_setup_memory(CXLType3Dev *ct3d, Error **errp)
{
DeviceState *ds = DEVICE(ct3d);
MemoryRegion *mr;
char *name;
if (!ct3d->hostmem) {
error_setg(errp, "memdev property must be set");
if (!ct3d->hostmem && !ct3d->hostvmem && !ct3d->hostpmem) {
error_setg(errp, "at least one memdev property must be set");
return false;
} else if (ct3d->hostmem && ct3d->hostpmem) {
error_setg(errp, "[memdev] cannot be used with new "
"[persistent-memdev] property");
return false;
} else if (ct3d->hostmem) {
/* Use of hostmem property implies pmem */
ct3d->hostpmem = ct3d->hostmem;
ct3d->hostmem = NULL;
}
if (ct3d->hostpmem && !ct3d->lsa) {
error_setg(errp, "lsa property must be set for persistent devices");
return false;
}
mr = host_memory_backend_get_memory(ct3d->hostmem);
if (!mr) {
error_setg(errp, "memdev property must be set");
return false;
if (ct3d->hostvmem) {
MemoryRegion *vmr;
char *v_name;
vmr = host_memory_backend_get_memory(ct3d->hostvmem);
if (!vmr) {
error_setg(errp, "volatile memdev must have backing device");
return false;
}
memory_region_set_nonvolatile(vmr, false);
memory_region_set_enabled(vmr, true);
host_memory_backend_set_mapped(ct3d->hostvmem, true);
if (ds->id) {
v_name = g_strdup_printf("cxl-type3-dpa-vmem-space:%s", ds->id);
} else {
v_name = g_strdup("cxl-type3-dpa-vmem-space");
}
address_space_init(&ct3d->hostvmem_as, vmr, v_name);
ct3d->cxl_dstate.vmem_size = memory_region_size(vmr);
ct3d->cxl_dstate.mem_size += memory_region_size(vmr);
g_free(v_name);
}
memory_region_set_nonvolatile(mr, true);
memory_region_set_enabled(mr, true);
host_memory_backend_set_mapped(ct3d->hostmem, true);
if (ds->id) {
name = g_strdup_printf("cxl-type3-dpa-space:%s", ds->id);
} else {
name = g_strdup("cxl-type3-dpa-space");
}
address_space_init(&ct3d->hostmem_as, mr, name);
g_free(name);
if (ct3d->hostpmem) {
MemoryRegion *pmr;
char *p_name;
ct3d->cxl_dstate.pmem_size = ct3d->hostmem->size;
if (!ct3d->lsa) {
error_setg(errp, "lsa property must be set");
return false;
pmr = host_memory_backend_get_memory(ct3d->hostpmem);
if (!pmr) {
error_setg(errp, "persistent memdev must have backing device");
return false;
}
memory_region_set_nonvolatile(pmr, true);
memory_region_set_enabled(pmr, true);
host_memory_backend_set_mapped(ct3d->hostpmem, true);
if (ds->id) {
p_name = g_strdup_printf("cxl-type3-dpa-pmem-space:%s", ds->id);
} else {
p_name = g_strdup("cxl-type3-dpa-pmem-space");
}
address_space_init(&ct3d->hostpmem_as, pmr, p_name);
ct3d->cxl_dstate.pmem_size = memory_region_size(pmr);
ct3d->cxl_dstate.mem_size += memory_region_size(pmr);
g_free(p_name);
}
return true;
@ -633,7 +731,12 @@ err_release_cdat:
err_free_special_ops:
g_free(regs->special_ops);
err_address_space_free:
address_space_destroy(&ct3d->hostmem_as);
if (ct3d->hostpmem) {
address_space_destroy(&ct3d->hostpmem_as);
}
if (ct3d->hostvmem) {
address_space_destroy(&ct3d->hostvmem_as);
}
return;
}
@ -646,7 +749,12 @@ static void ct3_exit(PCIDevice *pci_dev)
pcie_aer_exit(pci_dev);
cxl_doe_cdat_release(cxl_cstate);
g_free(regs->special_ops);
address_space_destroy(&ct3d->hostmem_as);
if (ct3d->hostpmem) {
address_space_destroy(&ct3d->hostpmem_as);
}
if (ct3d->hostvmem) {
address_space_destroy(&ct3d->hostvmem_as);
}
}
/* TODO: Support multiple HDM decoders and DPA skip */
@ -681,51 +789,77 @@ static bool cxl_type3_dpa(CXLType3Dev *ct3d, hwaddr host_addr, uint64_t *dpa)
return true;
}
static int cxl_type3_hpa_to_as_and_dpa(CXLType3Dev *ct3d,
hwaddr host_addr,
unsigned int size,
AddressSpace **as,
uint64_t *dpa_offset)
{
MemoryRegion *vmr = NULL, *pmr = NULL;
if (ct3d->hostvmem) {
vmr = host_memory_backend_get_memory(ct3d->hostvmem);
}
if (ct3d->hostpmem) {
pmr = host_memory_backend_get_memory(ct3d->hostpmem);
}
if (!vmr && !pmr) {
return -ENODEV;
}
if (!cxl_type3_dpa(ct3d, host_addr, dpa_offset)) {
return -EINVAL;
}
if (*dpa_offset > ct3d->cxl_dstate.mem_size) {
return -EINVAL;
}
if (vmr) {
if (*dpa_offset < memory_region_size(vmr)) {
*as = &ct3d->hostvmem_as;
} else {
*as = &ct3d->hostpmem_as;
*dpa_offset -= memory_region_size(vmr);
}
} else {
*as = &ct3d->hostpmem_as;
}
return 0;
}
MemTxResult cxl_type3_read(PCIDevice *d, hwaddr host_addr, uint64_t *data,
unsigned size, MemTxAttrs attrs)
{
CXLType3Dev *ct3d = CXL_TYPE3(d);
uint64_t dpa_offset;
MemoryRegion *mr;
uint64_t dpa_offset = 0;
AddressSpace *as = NULL;
int res;
/* TODO support volatile region */
mr = host_memory_backend_get_memory(ct3d->hostmem);
if (!mr) {
res = cxl_type3_hpa_to_as_and_dpa(CXL_TYPE3(d), host_addr, size,
&as, &dpa_offset);
if (res) {
return MEMTX_ERROR;
}
if (!cxl_type3_dpa(ct3d, host_addr, &dpa_offset)) {
return MEMTX_ERROR;
}
if (dpa_offset > memory_region_size(mr)) {
return MEMTX_ERROR;
}
return address_space_read(&ct3d->hostmem_as, dpa_offset, attrs, data, size);
return address_space_read(as, dpa_offset, attrs, data, size);
}
MemTxResult cxl_type3_write(PCIDevice *d, hwaddr host_addr, uint64_t data,
unsigned size, MemTxAttrs attrs)
{
CXLType3Dev *ct3d = CXL_TYPE3(d);
uint64_t dpa_offset;
MemoryRegion *mr;
uint64_t dpa_offset = 0;
AddressSpace *as = NULL;
int res;
mr = host_memory_backend_get_memory(ct3d->hostmem);
if (!mr) {
return MEMTX_OK;
res = cxl_type3_hpa_to_as_and_dpa(CXL_TYPE3(d), host_addr, size,
&as, &dpa_offset);
if (res) {
return MEMTX_ERROR;
}
if (!cxl_type3_dpa(ct3d, host_addr, &dpa_offset)) {
return MEMTX_OK;
}
if (dpa_offset > memory_region_size(mr)) {
return MEMTX_OK;
}
return address_space_write(&ct3d->hostmem_as, dpa_offset, attrs,
&data, size);
return address_space_write(as, dpa_offset, attrs, &data, size);
}
static void ct3d_reset(DeviceState *dev)
@ -740,7 +874,11 @@ static void ct3d_reset(DeviceState *dev)
static Property ct3_props[] = {
DEFINE_PROP_LINK("memdev", CXLType3Dev, hostmem, TYPE_MEMORY_BACKEND,
HostMemoryBackend *),
HostMemoryBackend *), /* for backward compatibility */
DEFINE_PROP_LINK("persistent-memdev", CXLType3Dev, hostpmem,
TYPE_MEMORY_BACKEND, HostMemoryBackend *),
DEFINE_PROP_LINK("volatile-memdev", CXLType3Dev, hostvmem,
TYPE_MEMORY_BACKEND, HostMemoryBackend *),
DEFINE_PROP_LINK("lsa", CXLType3Dev, lsa, TYPE_MEMORY_BACKEND,
HostMemoryBackend *),
DEFINE_PROP_UINT64("sn", CXLType3Dev, sn, UI64_NULL),
@ -752,6 +890,10 @@ static uint64_t get_lsa_size(CXLType3Dev *ct3d)
{
MemoryRegion *mr;
if (!ct3d->lsa) {
return 0;
}
mr = host_memory_backend_get_memory(ct3d->lsa);
return memory_region_size(mr);
}
@ -769,6 +911,10 @@ static uint64_t get_lsa(CXLType3Dev *ct3d, void *buf, uint64_t size,
MemoryRegion *mr;
void *lsa;
if (!ct3d->lsa) {
return 0;
}
mr = host_memory_backend_get_memory(ct3d->lsa);
validate_lsa_access(mr, size, offset);
@ -784,6 +930,10 @@ static void set_lsa(CXLType3Dev *ct3d, const void *buf, uint64_t size,
MemoryRegion *mr;
void *lsa;
if (!ct3d->lsa) {
return;
}
mr = host_memory_backend_get_memory(ct3d->lsa);
validate_lsa_access(mr, size, offset);
@ -955,7 +1105,7 @@ static void ct3_class_init(ObjectClass *oc, void *data)
pc->config_read = ct3d_config_read;
set_bit(DEVICE_CATEGORY_STORAGE, dc->categories);
dc->desc = "CXL PMEM Device (Type 3)";
dc->desc = "CXL Memory Device (Type 3)";
dc->reset = ct3d_reset;
device_class_set_props(dc, ct3_props);