From a191a209f0c46642e43b5a9ddfe2371ad1b06f8a Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Thu, 14 Sep 2023 20:29:52 -0700 Subject: [PATCH 001/138] cxl/port: Quiet warning messages from the cxl_test environment commit 7914992b37d5a4b165e65a32a7723afde9356720 upstream. The cxl_test platform device CXL port hierarchy is useful for testing, but throws warning messages of the form: cxl_mem mem2: at cxl_root_port.1 no parent for dport: platform cxl_mem mem3: at cxl_root_port.2 no parent for dport: platform cxl_mem mem4: at cxl_root_port.3 no parent for dport: platform cxl_mem mem5: at cxl_root_port.0 no parent for dport: platform cxl_mem mem6: at cxl_root_port.1 no parent for dport: platform cxl_mem mem7: at cxl_root_port.2 no parent for dport: platform cxl_mem mem8: at cxl_root_port.3 no parent for dport: platform cxl_mem mem9: at cxl_root_port.4 no parent for dport: platform cxl_mem mem10: at cxl_root_port.4 no parent for dport: platform ...and this message when running testing in QEMU: cxl_region region4: Bypassing cpu_cache_invalidate_memregion() for testing! Noisy cxl_test warnings have caused other regressions to be missed. In the interest of using cxl_test for early detection of dev_err() and dev_warn() messages, silence platform device topology and cache-invalidation messages. Signed-off-by: Dan Williams Signed-off-by: Yi Liu Signed-off-by: Bo Li --- drivers/cxl/core/port.c | 6 +++++- drivers/cxl/core/region.c | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/drivers/cxl/core/port.c b/drivers/cxl/core/port.c index 7f28d1021fa99..058c36dd476b8 100644 --- a/drivers/cxl/core/port.c +++ b/drivers/cxl/core/port.c @@ -1500,7 +1500,11 @@ int devm_cxl_enumerate_ports(struct cxl_memdev *cxlmd) struct cxl_dport *dport; struct cxl_port *port; - if (!dport_dev) + /* + * The terminal "grandparent" in PCI is NULL and @platform_bus + * for platform devices + */ + if (!dport_dev || dport_dev == &platform_bus) return 0; uport_dev = dport_dev->parent; diff --git a/drivers/cxl/core/region.c b/drivers/cxl/core/region.c index 1728cae1e8409..4f26e672ab3fe 100644 --- a/drivers/cxl/core/region.c +++ b/drivers/cxl/core/region.c @@ -123,7 +123,7 @@ static int cxl_region_invalidate_memregion(struct cxl_region *cxlr) { if (!cpu_cache_has_invalidate_memregion()) { if (IS_ENABLED(CONFIG_CXL_REGION_INVALIDATION_TEST)) { - dev_warn_once( + dev_info_once( &cxlr->dev, "Bypassing cpu_cache_invalidate_memregion() for testing!\n"); return 0; From 3c2cfb6f80059910863ea9fc19e39ff809792ddf Mon Sep 17 00:00:00 2001 From: Ira Weiny Date: Wed, 17 May 2023 14:28:11 -0700 Subject: [PATCH 002/138] cxl/pci: Update comment commit 1b27978d69670282a1586465de640cff363a34d8 upstream. The existence of struct cxl_dev_id containing a single member is odd. The comment made sense when I wrote it but could be clarified. Update the comment and place it next to the odd looking structure. Signed-off-by: Ira Weiny Reviewed-by: Jonathan Cameron Reviewed-by: Dave Jiang Link: https://lore.kernel.org/r/20230426-cxl-fixes-v1-2-870c4c8b463a@intel.com Signed-off-by: Dan Williams Signed-off-by: Yi Liu Signed-off-by: Bo Li --- drivers/cxl/pci.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/drivers/cxl/pci.c b/drivers/cxl/pci.c index aacd93f9067da..b84af5866aa23 100644 --- a/drivers/cxl/pci.c +++ b/drivers/cxl/pci.c @@ -85,6 +85,10 @@ static int cxl_pci_mbox_wait_for_doorbell(struct cxl_dev_state *cxlds) status & CXLMDEV_DEV_FATAL ? " fatal" : "", \ status & CXLMDEV_FW_HALT ? " firmware-halt" : "") +/* + * Threaded irq dev_id's must be globally unique. cxl_dev_id provides a unique + * wrapper object for each irq within the same cxlds. + */ struct cxl_dev_id { struct cxl_dev_state *cxlds; }; @@ -95,7 +99,6 @@ static int cxl_request_irq(struct cxl_dev_state *cxlds, int irq, struct device *dev = cxlds->dev; struct cxl_dev_id *dev_id; - /* dev_id must be globally unique and must contain the cxlds */ dev_id = devm_kzalloc(dev, sizeof(*dev_id), GFP_KERNEL); if (!dev_id) return -ENOMEM; From 3f3cba8f1475c588a6340a4707b522cd51808894 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Sun, 1 Oct 2023 07:35:29 -0700 Subject: [PATCH 003/138] ACPICA: Add defines for CDAT SSLBIS commit 178e1ea6a68f12967ee0e9afc4d79a2939acd43c upstream. Add upstream port and any port definition for SSLBIS. Link: https://github.com/acpica/acpica/pull/898 Reviewed-by: Jonathan Cameron Signed-off-by: Dave Jiang Signed-off-by: Rafael J. Wysocki Signed-off-by: Yi Liu Signed-off-by: Bo Li --- include/acpi/actbl1.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/include/acpi/actbl1.h b/include/acpi/actbl1.h index 8d5572ad48cbe..a33375e055ad7 100644 --- a/include/acpi/actbl1.h +++ b/include/acpi/actbl1.h @@ -465,6 +465,9 @@ struct acpi_cdat_sslbe { u16 reserved; }; +#define ACPI_CDAT_SSLBIS_US_PORT 0x0100 +#define ACPI_CDAT_SSLBIS_ANY_PORT 0xffff + /******************************************************************************* * * CEDT - CXL Early Discovery Table From f9addaab1cdeb96782a89ac7f0cc70463bfe4f10 Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Tue, 3 Oct 2023 20:12:12 -0700 Subject: [PATCH 004/138] cxl/pci: Remove hardirq handler for cxl_request_irq() commit 08b8a8c05423174e3ef4fb0bd514de20088cf5ac upstream. Now that all callers of cxl_request_irq() are using threaded irqs, drop the hardirq handler option. Reviewed-by: Jonathan Cameron Reviewed-by: Ira Weiny Reviewed-by: Davidlohr Bueso Reviewed-by: Dave Jiang Signed-off-by: Dan Williams Signed-off-by: Yi Liu Signed-off-by: Bo Li --- drivers/cxl/pci.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/drivers/cxl/pci.c b/drivers/cxl/pci.c index b84af5866aa23..310bc7d91a004 100644 --- a/drivers/cxl/pci.c +++ b/drivers/cxl/pci.c @@ -94,7 +94,7 @@ struct cxl_dev_id { }; static int cxl_request_irq(struct cxl_dev_state *cxlds, int irq, - irq_handler_t handler, irq_handler_t thread_fn) + irq_handler_t thread_fn) { struct device *dev = cxlds->dev; struct cxl_dev_id *dev_id; @@ -104,9 +104,9 @@ static int cxl_request_irq(struct cxl_dev_state *cxlds, int irq, return -ENOMEM; dev_id->cxlds = cxlds; - return devm_request_threaded_irq(dev, irq, handler, thread_fn, - IRQF_SHARED | IRQF_ONESHOT, - NULL, dev_id); + return devm_request_threaded_irq(dev, irq, NULL, thread_fn, + IRQF_SHARED | IRQF_ONESHOT, NULL, + dev_id); } static bool cxl_mbox_background_complete(struct cxl_dev_state *cxlds) @@ -448,7 +448,7 @@ static int cxl_pci_setup_mailbox(struct cxl_memdev_state *mds) if (irq < 0) return 0; - if (cxl_request_irq(cxlds, irq, NULL, cxl_pci_mbox_irq)) + if (cxl_request_irq(cxlds, irq, cxl_pci_mbox_irq)) return 0; dev_dbg(cxlds->dev, "Mailbox interrupts enabled\n"); @@ -646,7 +646,7 @@ static int cxl_event_req_irq(struct cxl_dev_state *cxlds, u8 setting) if (irq < 0) return irq; - return cxl_request_irq(cxlds, irq, NULL, cxl_event_thread); + return cxl_request_irq(cxlds, irq, cxl_event_thread); } static int cxl_event_get_int_policy(struct cxl_memdev_state *mds, From 1d2abdd7e923b558abfb0d01b7a60b6b84ae68ca Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Wed, 4 Oct 2023 16:56:34 -0700 Subject: [PATCH 005/138] tools/testing/cxl: Make cxl_memdev_state available to other command emulation commit 501b3d9fb036d58e88da434bc6473cec5f75644c upstream. Move @mds out of the event specific 'struct mock_event_store' and into the base 'struct cxl_mockmem_data' directly. This is in preparation for enabling cxl_test to exercise the notifier flow for 'sanitize' operation completion. Reviewed-by: Ira Weiny Reviewed-by: Dave Jiang Signed-off-by: Dan Williams Signed-off-by: Yi Liu Signed-off-by: Bo Li --- tools/testing/cxl/test/mem.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/testing/cxl/test/mem.c b/tools/testing/cxl/test/mem.c index 0ed100617d993..40db734d92a68 100644 --- a/tools/testing/cxl/test/mem.c +++ b/tools/testing/cxl/test/mem.c @@ -134,7 +134,6 @@ struct mock_event_log { }; struct mock_event_store { - struct cxl_memdev_state *mds; struct mock_event_log mock_logs[CXL_EVENT_TYPE_MAX]; u32 ev_status; }; @@ -151,6 +150,7 @@ struct cxl_mockmem_data { int user_limit; int master_limit; struct mock_event_store mes; + struct cxl_memdev_state *mds; u8 event_buf[SZ_4K]; u64 timestamp; }; @@ -327,7 +327,7 @@ static void cxl_mock_event_trigger(struct device *dev) event_reset_log(log); } - cxl_mem_get_event_records(mes->mds, mes->ev_status); + cxl_mem_get_event_records(mdata->mds, mes->ev_status); } struct cxl_event_record_raw maint_needed = { @@ -1416,6 +1416,7 @@ static int cxl_mock_mem_probe(struct platform_device *pdev) if (IS_ERR(mds)) return PTR_ERR(mds); + mdata->mds = mds; mds->mbox_send = cxl_mock_mbox_send; mds->payload_size = SZ_4K; mds->event.buf = (struct cxl_get_event_payload *) mdata->event_buf; @@ -1448,7 +1449,6 @@ static int cxl_mock_mem_probe(struct platform_device *pdev) if (rc) return rc; - mdata->mes.mds = mds; cxl_mock_add_event_logs(&mdata->mes); cxlmd = devm_cxl_add_memdev(&pdev->dev, cxlds); From 10c718c601efa7508acef7ccde02ae3bb6e4a17e Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Wed, 4 Oct 2023 16:50:09 -0700 Subject: [PATCH 006/138] tools/testing/cxl: Add 'sanitize notifier' support commit cf009d4ec38cb3acc09b4248674c67abe916ceb5 upstream. Allow for cxl_test regression of the sanitize notifier. Reuse the core setup infrastructure, and trigger notifications upon any sanitize submission with a programmable notification delay. Cc: Davidlohr Bueso Reviewed-by: Ira Weiny Reviewed-by: Dave Jiang Signed-off-by: Dan Williams Signed-off-by: Yi Liu Signed-off-by: Bo Li --- tools/testing/cxl/test/mem.c | 68 +++++++++++++++++++++++++++++++++++- 1 file changed, 67 insertions(+), 1 deletion(-) diff --git a/tools/testing/cxl/test/mem.c b/tools/testing/cxl/test/mem.c index 40db734d92a68..ccbc67091c4c9 100644 --- a/tools/testing/cxl/test/mem.c +++ b/tools/testing/cxl/test/mem.c @@ -90,6 +90,12 @@ static struct cxl_cel_entry mock_cel[] = { .effect = cpu_to_le16(EFFECT(CONF_CHANGE_COLD_RESET) | EFFECT(CONF_CHANGE_IMMEDIATE)), }, + { + .opcode = cpu_to_le16(CXL_MBOX_OP_SANITIZE), + .effect = cpu_to_le16(EFFECT(DATA_CHANGE_IMMEDIATE) | + EFFECT(SECURITY_CHANGE_IMMEDIATE) | + EFFECT(BACKGROUND_OP)), + }, }; /* See CXL 2.0 Table 181 Get Health Info Output Payload */ @@ -153,6 +159,7 @@ struct cxl_mockmem_data { struct cxl_memdev_state *mds; u8 event_buf[SZ_4K]; u64 timestamp; + unsigned long sanitize_timeout; }; static struct mock_event_log *event_find_log(struct device *dev, int log_type) @@ -568,9 +575,26 @@ static int mock_partition_info(struct cxl_mbox_cmd *cmd) return 0; } +void cxl_mockmem_sanitize_work(struct work_struct *work) +{ + struct cxl_memdev_state *mds = + container_of(work, typeof(*mds), security.poll_dwork.work); + + mutex_lock(&mds->mbox_mutex); + if (mds->security.sanitize_node) + sysfs_notify_dirent(mds->security.sanitize_node); + mds->security.sanitize_active = false; + mutex_unlock(&mds->mbox_mutex); + + dev_dbg(mds->cxlds.dev, "sanitize complete\n"); +} + static int mock_sanitize(struct cxl_mockmem_data *mdata, struct cxl_mbox_cmd *cmd) { + struct cxl_memdev_state *mds = mdata->mds; + int rc = 0; + if (cmd->size_in != 0) return -EINVAL; @@ -586,7 +610,16 @@ static int mock_sanitize(struct cxl_mockmem_data *mdata, return -ENXIO; } - return 0; /* assume less than 2 secs, no bg */ + mutex_lock(&mds->mbox_mutex); + if (schedule_delayed_work(&mds->security.poll_dwork, + msecs_to_jiffies(mdata->sanitize_timeout))) { + mds->security.sanitize_active = true; + dev_dbg(mds->cxlds.dev, "sanitize issued\n"); + } else + rc = -EBUSY; + mutex_unlock(&mds->mbox_mutex); + + return rc; } static int mock_secure_erase(struct cxl_mockmem_data *mdata, @@ -1420,6 +1453,7 @@ static int cxl_mock_mem_probe(struct platform_device *pdev) mds->mbox_send = cxl_mock_mbox_send; mds->payload_size = SZ_4K; mds->event.buf = (struct cxl_get_event_payload *) mdata->event_buf; + INIT_DELAYED_WORK(&mds->security.poll_dwork, cxl_mockmem_sanitize_work); cxlds = &mds->cxlds; cxlds->serial = pdev->id; @@ -1459,6 +1493,10 @@ static int cxl_mock_mem_probe(struct platform_device *pdev) if (rc) return rc; + rc = devm_cxl_sanitize_setup_notifier(&pdev->dev, cxlmd); + if (rc) + return rc; + cxl_mem_get_event_records(mds, CXLDEV_EVENT_STATUS_ALL); return 0; @@ -1527,10 +1565,38 @@ static ssize_t fw_buf_checksum_show(struct device *dev, static DEVICE_ATTR_RO(fw_buf_checksum); +static ssize_t sanitize_timeout_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct cxl_mockmem_data *mdata = dev_get_drvdata(dev); + + return sysfs_emit(buf, "%lu\n", mdata->sanitize_timeout); +} + +static ssize_t sanitize_timeout_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct cxl_mockmem_data *mdata = dev_get_drvdata(dev); + unsigned long val; + int rc; + + rc = kstrtoul(buf, 0, &val); + if (rc) + return rc; + + mdata->sanitize_timeout = val; + + return count; +} + +static DEVICE_ATTR_RW(sanitize_timeout); + static struct attribute *cxl_mock_mem_attrs[] = { &dev_attr_security_lock.attr, &dev_attr_event_trigger.attr, &dev_attr_fw_buf_checksum.attr, + &dev_attr_sanitize_timeout.attr, NULL }; ATTRIBUTE_GROUPS(cxl_mock_mem); From 99127f383f5a6d0c3b4e170611a036907de0026c Mon Sep 17 00:00:00 2001 From: Alison Schofield Date: Tue, 15 Aug 2023 10:20:52 -0700 Subject: [PATCH 007/138] cxl/mbox: Remove useless cast in cxl_mem_create_range_info() commit 9214c9d56c470b183f3d374c7aed7c8756f80fd9 upstream. DEFINE_RES_MEM() is a wrapper around the DEFINE_RES_NAMED() macro which already has the (struct resource) for the compound literal. The user of the macro should not repeat the cast. Cleans up these sparse warnings: drivers/cxl/core/mbox.c:1184:18: warning: cast to non-scalar drivers/cxl/core/mbox.c:1184:18: warning: cast from non-scalar Fixes: 52c4d11f1dce ("resource: Convert DEFINE_RES_NAMED() to be compound literal") Signed-off-by: Alison Schofield Reviewed-by: Jonathan Cameron Reviewed-by: Davidlohr Bueso Reviewed-by: Andy Shevchenko Link: https://lore.kernel.org/r/20230815172052.22514-1-alison.schofield@intel.com Signed-off-by: Dan Williams Signed-off-by: Yi Liu Signed-off-by: Bo Li --- drivers/cxl/core/mbox.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/cxl/core/mbox.c b/drivers/cxl/core/mbox.c index fecaa18f4dd20..41c21865ae5d9 100644 --- a/drivers/cxl/core/mbox.c +++ b/drivers/cxl/core/mbox.c @@ -1247,8 +1247,7 @@ int cxl_mem_create_range_info(struct cxl_memdev_state *mds) return 0; } - cxlds->dpa_res = - (struct resource)DEFINE_RES_MEM(0, mds->total_bytes); + cxlds->dpa_res = DEFINE_RES_MEM(0, mds->total_bytes); if (mds->partition_align_bytes == 0) { rc = add_dpa_res(dev, &cxlds->dpa_res, &cxlds->ram_res, 0, From 29c56514660453ef29db302cc028d7b6db9f435d Mon Sep 17 00:00:00 2001 From: Vishal Verma Date: Thu, 26 Oct 2023 11:43:20 -0600 Subject: [PATCH 008/138] MAINTAINERS: Add tools/testing/cxl files to CXL commit fae6389f912eb0b6aea1366814b5501077b5dbac upstream. tools/testing/cxl contains the unit test infrastructure for mocking CXL hierarchies. These are under the purview of the CXL subsystem maintainers. Add the 'F:' entry for this to MAINTAINERS so that get_maintainer.pl works as expected for patches to this area. Signed-off-by: Vishal Verma Link: https://lore.kernel.org/r/20231026-vv-mainteners-fix-v1-1-0a0f25634073@intel.com Signed-off-by: Dan Williams Signed-off-by: Yi Liu Signed-off-by: Bo Li --- MAINTAINERS | 1 + 1 file changed, 1 insertion(+) diff --git a/MAINTAINERS b/MAINTAINERS index b413d5c97d413..cf5fb6c0c34b3 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -5223,6 +5223,7 @@ L: linux-cxl@vger.kernel.org S: Maintained F: drivers/cxl/ F: include/uapi/linux/cxl_mem.h +F: tools/testing/cxl/ COMPUTE EXPRESS LINK PMU (CPMU) M: Jonathan Cameron From 34e21a31d2353399062ec68b28874f2ed771c237 Mon Sep 17 00:00:00 2001 From: Vishal Verma Date: Thu, 26 Oct 2023 11:32:41 -0600 Subject: [PATCH 009/138] tools/testing/cxl: Slow down the mock firmware transfer commit 8f61d48c83f6e3525a770e44692604595693f787 upstream. The cxl-cli unit test for firmware update does operations like starting an asynchronous firmware update, making sure it is in progress, and attempting to cancel it. In some cases, such as with no or minimal dynamic debugging turned on, the firmware update completes too quickly, not allowing the test to have a chance to verify it was in progress. This caused a failure of the signature: expected fw_update_in_progress:true test/cxl-update-firmware.sh: failed at line 88 Fix this by adding a delay (~1.5 - 2 ms) to each firmware transfer request handled by the mocked interface. Reported-by: Dan Williams Tested-by: Dan Williams Signed-off-by: Vishal Verma Link: https://lore.kernel.org/r/20231026-vv-fw_upd_test_fix-v2-1-5282fd193883@intel.com Signed-off-by: Dan Williams Signed-off-by: Yi Liu Signed-off-by: Bo Li --- tools/testing/cxl/test/mem.c | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/testing/cxl/test/mem.c b/tools/testing/cxl/test/mem.c index ccbc67091c4c9..e2196408faa2e 100644 --- a/tools/testing/cxl/test/mem.c +++ b/tools/testing/cxl/test/mem.c @@ -1271,6 +1271,7 @@ static int mock_transfer_fw(struct cxl_mockmem_data *mdata, } memcpy(fw + offset, transfer->data, length); + usleep_range(1500, 2000); return 0; } From 35a9769dbd62a674b58d8e9463c48be0567d23a6 Mon Sep 17 00:00:00 2001 From: Robert Richter Date: Wed, 18 Oct 2023 19:16:57 +0200 Subject: [PATCH 010/138] cxl/port: Rename @comp_map to @reg_map in struct cxl_register_map commit d8add49263a98d766e5758dc2ec9f83c3b685c12 upstream. Name the field @reg_map, because @reg_map->host will be used for mapping operations beyond component registers (i.e. AER registers). This is valid for all occurrences of @comp_map. Change them all. Signed-off-by: Robert Richter Reviewed-by: Jonathan Cameron Link: https://lore.kernel.org/r/20231018171713.1883517-5-rrichter@amd.com Signed-off-by: Dan Williams Signed-off-by: Yi Liu Signed-off-by: Bo Li --- drivers/cxl/core/port.c | 6 +++--- drivers/cxl/cxl.h | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/drivers/cxl/core/port.c b/drivers/cxl/core/port.c index 058c36dd476b8..82cd949babb9c 100644 --- a/drivers/cxl/core/port.c +++ b/drivers/cxl/core/port.c @@ -721,7 +721,7 @@ static int cxl_port_setup_regs(struct cxl_port *port, { if (dev_is_platform(port->uport_dev)) return 0; - return cxl_setup_comp_regs(&port->dev, &port->comp_map, + return cxl_setup_comp_regs(&port->dev, &port->reg_map, component_reg_phys); } @@ -738,9 +738,9 @@ static int cxl_dport_setup_regs(struct device *host, struct cxl_dport *dport, * register probing, and fixup @host after the fact, since @host may be * NULL. */ - rc = cxl_setup_comp_regs(dport->dport_dev, &dport->comp_map, + rc = cxl_setup_comp_regs(dport->dport_dev, &dport->reg_map, component_reg_phys); - dport->comp_map.host = host; + dport->reg_map.host = host; return rc; } diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h index bcfcbf65d18a0..e6e7d1d3a07d1 100644 --- a/drivers/cxl/cxl.h +++ b/drivers/cxl/cxl.h @@ -572,7 +572,7 @@ struct cxl_dax_region { * @regions: cxl_region_ref instances, regions mapped by this port * @parent_dport: dport that points to this port in the parent * @decoder_ida: allocator for decoder ids - * @comp_map: component register capability mappings + * @reg_map: component and ras register mapping parameters * @nr_dports: number of entries in @dports * @hdm_end: track last allocated HDM decoder instance for allocation ordering * @commit_end: cursor to track highest committed decoder for commit ordering @@ -592,7 +592,7 @@ struct cxl_port { struct xarray regions; struct cxl_dport *parent_dport; struct ida decoder_ida; - struct cxl_register_map comp_map; + struct cxl_register_map reg_map; int nr_dports; int hdm_end; int commit_end; @@ -620,7 +620,7 @@ struct cxl_rcrb_info { /** * struct cxl_dport - CXL downstream port * @dport_dev: PCI bridge or firmware device representing the downstream link - * @comp_map: component register capability mappings + * @reg_map: component and ras register mapping parameters * @port_id: unique hardware identifier for dport in decoder target list * @rcrb: Data about the Root Complex Register Block layout * @rch: Indicate whether this dport was enumerated in RCH or VH mode @@ -628,7 +628,7 @@ struct cxl_rcrb_info { */ struct cxl_dport { struct device *dport_dev; - struct cxl_register_map comp_map; + struct cxl_register_map reg_map; int port_id; struct cxl_rcrb_info rcrb; bool rch; From f5bf437ed71ecb7eb52354d97293307024e6475d Mon Sep 17 00:00:00 2001 From: Robert Richter Date: Wed, 18 Oct 2023 19:16:58 +0200 Subject: [PATCH 011/138] cxl/port: Pre-initialize component register mappings commit 4d758764e7f9db83806135f3bfcff1ab64f16e60 upstream. The component registers of a component may not exist and cxl_setup_comp_regs() will fail for that reason. In another case, Software may not use and set those registers up. cxl_setup_comp_regs() is then called with a base address of CXL_RESOURCE_NONE. Both are valid cases, but the function returns without initializing the register map. Now, a missing component register block is not necessarily a reason to fail (feature is optional or its existence checked later). Change cxl_setup_comp_regs() to also use components with the component register block missing. Thus, always initialize struct cxl_register_map with valid values, set @dev and make @resource CXL_RESOURCE_NONE. The change is in preparation of follow-on patches. Signed-off-by: Terry Bowman Signed-off-by: Robert Richter Reviewed-by: Jonathan Cameron Link: https://lore.kernel.org/r/20231018171713.1883517-6-rrichter@amd.com Signed-off-by: Dan Williams Signed-off-by: Yi Liu Signed-off-by: Bo Li --- drivers/cxl/core/port.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/drivers/cxl/core/port.c b/drivers/cxl/core/port.c index 82cd949babb9c..98473ba5c919c 100644 --- a/drivers/cxl/core/port.c +++ b/drivers/cxl/core/port.c @@ -703,16 +703,18 @@ static struct cxl_port *cxl_port_alloc(struct device *uport_dev, static int cxl_setup_comp_regs(struct device *host, struct cxl_register_map *map, resource_size_t component_reg_phys) { - if (component_reg_phys == CXL_RESOURCE_NONE) - return 0; - *map = (struct cxl_register_map) { .host = host, - .reg_type = CXL_REGLOC_RBI_COMPONENT, + .reg_type = CXL_REGLOC_RBI_EMPTY, .resource = component_reg_phys, - .max_size = CXL_COMPONENT_REG_BLOCK_SIZE, }; + if (component_reg_phys == CXL_RESOURCE_NONE) + return 0; + + map->reg_type = CXL_REGLOC_RBI_COMPONENT; + map->max_size = CXL_COMPONENT_REG_BLOCK_SIZE; + return cxl_setup_regs(map); } From 15295e78df0f0e3cd5b4847a81b7d2373939629a Mon Sep 17 00:00:00 2001 From: Robert Richter Date: Wed, 18 Oct 2023 19:16:59 +0200 Subject: [PATCH 012/138] cxl/pci: Store the endpoint's Component Register mappings in struct cxl_dev_state commit 2dd18279202f6247904e6e23738c1ec6a86b24b1 upstream. Same as for ports and dports, also store the endpoint's Component Register mappings, use struct cxl_dev_state for that. Keep the Component Register base address @component_reg_phys a bit to not break functionality. It will be removed after the transition in a later patch. Signed-off-by: Terry Bowman Signed-off-by: Robert Richter Reviewed-by: Jonathan Cameron Reviewed-by: Dave Jiang Link: https://lore.kernel.org/r/20231018171713.1883517-7-rrichter@amd.com Signed-off-by: Dan Williams Signed-off-by: Yi Liu Signed-off-by: Bo Li --- drivers/cxl/core/mbox.c | 2 ++ drivers/cxl/cxlmem.h | 2 ++ drivers/cxl/pci.c | 9 +++++---- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/drivers/cxl/core/mbox.c b/drivers/cxl/core/mbox.c index 41c21865ae5d9..f1bcd9a439198 100644 --- a/drivers/cxl/core/mbox.c +++ b/drivers/cxl/core/mbox.c @@ -1398,6 +1398,8 @@ struct cxl_memdev_state *cxl_memdev_state_create(struct device *dev) mutex_init(&mds->mbox_mutex); mutex_init(&mds->event.log_lock); mds->cxlds.dev = dev; + mds->cxlds.reg_map.host = dev; + mds->cxlds.reg_map.resource = CXL_RESOURCE_NONE; mds->cxlds.type = CXL_DEVTYPE_CLASSMEM; return mds; diff --git a/drivers/cxl/cxlmem.h b/drivers/cxl/cxlmem.h index edb46123e3eb0..36c90eafc5fb1 100644 --- a/drivers/cxl/cxlmem.h +++ b/drivers/cxl/cxlmem.h @@ -400,6 +400,7 @@ enum cxl_devtype { * * @dev: The device associated with this CXL state * @cxlmd: The device representing the CXL.mem capabilities of @dev + * @reg_map: component and ras register mapping parameters * @regs: Parsed register blocks * @cxl_dvsec: Offset to the PCIe device DVSEC * @rcd: operating in RCD mode (CXL 3.0 9.11.8 CXL Devices Attached to an RCH) @@ -414,6 +415,7 @@ enum cxl_devtype { struct cxl_dev_state { struct device *dev; struct cxl_memdev *cxlmd; + struct cxl_register_map reg_map; struct cxl_regs regs; int cxl_dvsec; bool rcd; diff --git a/drivers/cxl/pci.c b/drivers/cxl/pci.c index 310bc7d91a004..b1da00a044322 100644 --- a/drivers/cxl/pci.c +++ b/drivers/cxl/pci.c @@ -828,15 +828,16 @@ static int cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) * still be useful for management functions so don't return an error. */ cxlds->component_reg_phys = CXL_RESOURCE_NONE; - rc = cxl_pci_setup_regs(pdev, CXL_REGLOC_RBI_COMPONENT, &map); + rc = cxl_pci_setup_regs(pdev, CXL_REGLOC_RBI_COMPONENT, + &cxlds->reg_map); if (rc) dev_warn(&pdev->dev, "No component registers (%d)\n", rc); - else if (!map.component_map.ras.valid) + else if (!cxlds->reg_map.component_map.ras.valid) dev_dbg(&pdev->dev, "RAS registers not found\n"); - cxlds->component_reg_phys = map.resource; + cxlds->component_reg_phys = cxlds->reg_map.resource; - rc = cxl_map_component_regs(&map, &cxlds->regs.component, + rc = cxl_map_component_regs(&cxlds->reg_map, &cxlds->regs.component, BIT(CXL_CM_CAP_CAP_ID_RAS)); if (rc) dev_dbg(&pdev->dev, "Failed to map RAS capability.\n"); From 98bfc30f9563caae9629d1b7043ab1d58336d585 Mon Sep 17 00:00:00 2001 From: Robert Richter Date: Wed, 18 Oct 2023 19:17:00 +0200 Subject: [PATCH 013/138] cxl/hdm: Use stored Component Register mappings to map HDM decoder capability commit 8ce520fdea245c9e17ebc0659973984362bc1fde upstream. Now, that the Component Register mappings are stored, use them to enable and map the HDM decoder capabilities. The Component Registers do not need to be probed again for this, remove probing code. The HDM capability applies to Endpoints, USPs and VH Host Bridges. The Endpoint's component register mappings are located in the cxlds and else in the port's structure. Duplicate the cxlds->reg_map in port->reg_map for endpoint ports. Signed-off-by: Terry Bowman Signed-off-by: Robert Richter Reviewed-by: Dave Jiang [rework to drop cxl_port_get_comp_map()] Link: https://lore.kernel.org/r/20231018171713.1883517-8-rrichter@amd.com Signed-off-by: Dan Williams Signed-off-by: Yi Liu Signed-off-by: Bo Li --- drivers/cxl/core/hdm.c | 48 ++++++++++++++++------------------------- drivers/cxl/core/port.c | 29 +++++++++++++++++++------ drivers/cxl/mem.c | 5 ++--- 3 files changed, 43 insertions(+), 39 deletions(-) diff --git a/drivers/cxl/core/hdm.c b/drivers/cxl/core/hdm.c index 36b20b844b109..7737fa7631d45 100644 --- a/drivers/cxl/core/hdm.c +++ b/drivers/cxl/core/hdm.c @@ -94,26 +94,6 @@ static void parse_hdm_decoder_caps(struct cxl_hdm *cxlhdm) cxlhdm->iw_cap_mask |= BIT(16); } -static int map_hdm_decoder_regs(struct cxl_port *port, void __iomem *crb, - struct cxl_component_regs *regs) -{ - struct cxl_register_map map = { - .host = &port->dev, - .resource = port->component_reg_phys, - .base = crb, - .max_size = CXL_COMPONENT_REG_BLOCK_SIZE, - }; - - cxl_probe_component_regs(&port->dev, crb, &map.component_map); - if (!map.component_map.hdm_decoder.valid) { - dev_dbg(&port->dev, "HDM decoder registers not implemented\n"); - /* unique error code to indicate no HDM decoder capability */ - return -ENODEV; - } - - return cxl_map_component_regs(&map, regs, BIT(CXL_CM_CAP_CAP_ID_HDM)); -} - static bool should_emulate_decoders(struct cxl_endpoint_dvsec_info *info) { struct cxl_hdm *cxlhdm; @@ -166,9 +146,9 @@ static bool should_emulate_decoders(struct cxl_endpoint_dvsec_info *info) struct cxl_hdm *devm_cxl_setup_hdm(struct cxl_port *port, struct cxl_endpoint_dvsec_info *info) { + struct cxl_register_map *reg_map = &port->reg_map; struct device *dev = &port->dev; struct cxl_hdm *cxlhdm; - void __iomem *crb; int rc; cxlhdm = devm_kzalloc(dev, sizeof(*cxlhdm), GFP_KERNEL); @@ -177,19 +157,29 @@ struct cxl_hdm *devm_cxl_setup_hdm(struct cxl_port *port, cxlhdm->port = port; dev_set_drvdata(dev, cxlhdm); - crb = ioremap(port->component_reg_phys, CXL_COMPONENT_REG_BLOCK_SIZE); - if (!crb && info && info->mem_enabled) { + /* Memory devices can configure device HDM using DVSEC range regs. */ + if (reg_map->resource == CXL_RESOURCE_NONE) { + if (!info && !info->mem_enabled) { + dev_err(dev, "No component registers mapped\n"); + return ERR_PTR(-ENXIO); + } + cxlhdm->decoder_count = info->ranges; return cxlhdm; - } else if (!crb) { - dev_err(dev, "No component registers mapped\n"); - return ERR_PTR(-ENXIO); } - rc = map_hdm_decoder_regs(port, crb, &cxlhdm->regs); - iounmap(crb); - if (rc) + if (!reg_map->component_map.hdm_decoder.valid) { + dev_dbg(&port->dev, "HDM decoder registers not implemented\n"); + /* unique error code to indicate no HDM decoder capability */ + return ERR_PTR(-ENODEV); + } + + rc = cxl_map_component_regs(reg_map, &cxlhdm->regs, + BIT(CXL_CM_CAP_CAP_ID_HDM)); + if (rc) { + dev_err(dev, "Failed to map HDM capability.\n"); return ERR_PTR(rc); + } parse_hdm_decoder_caps(cxlhdm); if (cxlhdm->decoder_count == 0) { diff --git a/drivers/cxl/core/port.c b/drivers/cxl/core/port.c index 98473ba5c919c..450b7bc5be095 100644 --- a/drivers/cxl/core/port.c +++ b/drivers/cxl/core/port.c @@ -760,16 +760,31 @@ static struct cxl_port *__devm_cxl_add_port(struct device *host, return port; dev = &port->dev; - if (is_cxl_memdev(uport_dev)) + if (is_cxl_memdev(uport_dev)) { + struct cxl_memdev *cxlmd = to_cxl_memdev(uport_dev); + struct cxl_dev_state *cxlds = cxlmd->cxlds; + rc = dev_set_name(dev, "endpoint%d", port->id); - else if (parent_dport) + if (rc) + goto err; + + /* + * The endpoint driver already enumerated the component and RAS + * registers. Reuse that enumeration while prepping them to be + * mapped by the cxl_port driver. + */ + port->reg_map = cxlds->reg_map; + port->reg_map.host = &port->dev; + } else if (parent_dport) { rc = dev_set_name(dev, "port%d", port->id); - else - rc = dev_set_name(dev, "root%d", port->id); - if (rc) - goto err; + if (rc) + goto err; - rc = cxl_port_setup_regs(port, component_reg_phys); + rc = cxl_port_setup_regs(port, component_reg_phys); + if (rc) + goto err; + } else + rc = dev_set_name(dev, "root%d", port->id); if (rc) goto err; diff --git a/drivers/cxl/mem.c b/drivers/cxl/mem.c index 317c7548e4e9b..04107058739bc 100644 --- a/drivers/cxl/mem.c +++ b/drivers/cxl/mem.c @@ -49,7 +49,6 @@ static int devm_cxl_add_endpoint(struct device *host, struct cxl_memdev *cxlmd, struct cxl_dport *parent_dport) { struct cxl_port *parent_port = parent_dport->port; - struct cxl_dev_state *cxlds = cxlmd->cxlds; struct cxl_port *endpoint, *iter, *down; int rc; @@ -65,8 +64,8 @@ static int devm_cxl_add_endpoint(struct device *host, struct cxl_memdev *cxlmd, ep->next = down; } - endpoint = devm_cxl_add_port(host, &cxlmd->dev, - cxlds->component_reg_phys, + /* Note: endpoint port component registers are derived from @cxlds */ + endpoint = devm_cxl_add_port(host, &cxlmd->dev, CXL_RESOURCE_NONE, parent_dport); if (IS_ERR(endpoint)) return PTR_ERR(endpoint); From f415dc2274f4d857b117152804564c535146a91a Mon Sep 17 00:00:00 2001 From: Robert Richter Date: Wed, 18 Oct 2023 19:17:01 +0200 Subject: [PATCH 014/138] cxl/pci: Remove Component Register base address from struct cxl_dev_state commit f611d98a003644f76ad8fea7c3ca786a8ca69aff upstream. The Component Register base address @component_reg_phys is no longer used after the rework of the Component Register setup which now uses struct member @reg_map instead. Remove the base address. Signed-off-by: Terry Bowman Signed-off-by: Robert Richter Reviewed-by: Jonathan Cameron Reviewed-by: Dave Jiang Link: https://lore.kernel.org/r/20231018171713.1883517-9-rrichter@amd.com Signed-off-by: Dan Williams Signed-off-by: Yi Liu Signed-off-by: Bo Li --- drivers/cxl/cxlmem.h | 2 -- drivers/cxl/pci.c | 3 --- tools/testing/cxl/test/mem.c | 4 +--- 3 files changed, 1 insertion(+), 8 deletions(-) diff --git a/drivers/cxl/cxlmem.h b/drivers/cxl/cxlmem.h index 36c90eafc5fb1..a72468912fc86 100644 --- a/drivers/cxl/cxlmem.h +++ b/drivers/cxl/cxlmem.h @@ -408,7 +408,6 @@ enum cxl_devtype { * @dpa_res: Overall DPA resource tree for the device * @pmem_res: Active Persistent memory capacity configuration * @ram_res: Active Volatile memory capacity configuration - * @component_reg_phys: register base of component registers * @serial: PCIe Device Serial Number * @type: Generic Memory Class device or Vendor Specific Memory device */ @@ -423,7 +422,6 @@ struct cxl_dev_state { struct resource dpa_res; struct resource pmem_res; struct resource ram_res; - resource_size_t component_reg_phys; u64 serial; enum cxl_devtype type; }; diff --git a/drivers/cxl/pci.c b/drivers/cxl/pci.c index b1da00a044322..67194d70c2dda 100644 --- a/drivers/cxl/pci.c +++ b/drivers/cxl/pci.c @@ -827,7 +827,6 @@ static int cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) * If the component registers can't be found, the cxl_pci driver may * still be useful for management functions so don't return an error. */ - cxlds->component_reg_phys = CXL_RESOURCE_NONE; rc = cxl_pci_setup_regs(pdev, CXL_REGLOC_RBI_COMPONENT, &cxlds->reg_map); if (rc) @@ -835,8 +834,6 @@ static int cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) else if (!cxlds->reg_map.component_map.ras.valid) dev_dbg(&pdev->dev, "RAS registers not found\n"); - cxlds->component_reg_phys = cxlds->reg_map.resource; - rc = cxl_map_component_regs(&cxlds->reg_map, &cxlds->regs.component, BIT(CXL_CM_CAP_CAP_ID_RAS)); if (rc) diff --git a/tools/testing/cxl/test/mem.c b/tools/testing/cxl/test/mem.c index e2196408faa2e..a6257430d73a0 100644 --- a/tools/testing/cxl/test/mem.c +++ b/tools/testing/cxl/test/mem.c @@ -1458,10 +1458,8 @@ static int cxl_mock_mem_probe(struct platform_device *pdev) cxlds = &mds->cxlds; cxlds->serial = pdev->id; - if (is_rcd(pdev)) { + if (is_rcd(pdev)) cxlds->rcd = true; - cxlds->component_reg_phys = CXL_RESOURCE_NONE; - } rc = cxl_enumerate_cmds(mds); if (rc) From cfe12bf63210164015f13aa8896624f706f7030f Mon Sep 17 00:00:00 2001 From: Robert Richter Date: Wed, 18 Oct 2023 19:17:02 +0200 Subject: [PATCH 015/138] cxl/port: Remove Component Register base address from struct cxl_port commit a2fcb84a1978e4e855d632a07412030e99819cb8 upstream. The Component Register base address @component_reg_phys is no longer used after the rework of the Component Register setup which now uses struct member @reg_map instead. Remove the base address. Signed-off-by: Terry Bowman Signed-off-by: Robert Richter Reviewed-by: Jonathan Cameron Reviewed-by: Dave Jiang Link: https://lore.kernel.org/r/20231018171713.1883517-10-rrichter@amd.com Signed-off-by: Dan Williams Signed-off-by: Yi Liu Signed-off-by: Bo Li --- drivers/cxl/core/port.c | 4 +--- drivers/cxl/cxl.h | 2 -- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/drivers/cxl/core/port.c b/drivers/cxl/core/port.c index 450b7bc5be095..e2a1eada43da1 100644 --- a/drivers/cxl/core/port.c +++ b/drivers/cxl/core/port.c @@ -628,7 +628,6 @@ static int devm_cxl_link_parent_dport(struct device *host, static struct lock_class_key cxl_port_key; static struct cxl_port *cxl_port_alloc(struct device *uport_dev, - resource_size_t component_reg_phys, struct cxl_dport *parent_dport) { struct cxl_port *port; @@ -679,7 +678,6 @@ static struct cxl_port *cxl_port_alloc(struct device *uport_dev, } else dev->parent = uport_dev; - port->component_reg_phys = component_reg_phys; ida_init(&port->decoder_ida); port->hdm_end = -1; port->commit_end = -1; @@ -755,7 +753,7 @@ static struct cxl_port *__devm_cxl_add_port(struct device *host, struct device *dev; int rc; - port = cxl_port_alloc(uport_dev, component_reg_phys, parent_dport); + port = cxl_port_alloc(uport_dev, parent_dport); if (IS_ERR(port)) return port; diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h index e6e7d1d3a07d1..bec24d3a41969 100644 --- a/drivers/cxl/cxl.h +++ b/drivers/cxl/cxl.h @@ -576,7 +576,6 @@ struct cxl_dax_region { * @nr_dports: number of entries in @dports * @hdm_end: track last allocated HDM decoder instance for allocation ordering * @commit_end: cursor to track highest committed decoder for commit ordering - * @component_reg_phys: component register capability base address (optional) * @dead: last ep has been removed, force port re-creation * @depth: How deep this port is relative to the root. depth 0 is the root. * @cdat: Cached CDAT data @@ -596,7 +595,6 @@ struct cxl_port { int nr_dports; int hdm_end; int commit_end; - resource_size_t component_reg_phys; bool dead; unsigned int depth; struct cxl_cdat { From 1a64e39d14ced8a6d3b639788dca73c832e3037c Mon Sep 17 00:00:00 2001 From: Robert Richter Date: Fri, 27 Oct 2023 15:08:06 -0700 Subject: [PATCH 016/138] cxl/pci: Add RCH downstream port AER register discovery commit f05fd10d138d8ba795fcd72ace99e9c8eb7e777e upstream. Restricted CXL host (RCH) downstream port AER information is not currently logged while in the error state. One problem preventing the error logging is the AER and RAS registers are not accessible. The CXL driver requires changes to find RCH downstream port AER and RAS registers for purpose of error logging. RCH downstream ports are not enumerated during a PCI bus scan and are instead discovered using system firmware, ACPI in this case.[1] The downstream port is implemented as a Root Complex Register Block (RCRB). The RCRB is a 4k memory block containing PCIe registers based on the PCIe root port.[2] The RCRB includes AER extended capability registers used for reporting errors. Note, the RCH's AER Capability is located in the RCRB memory space instead of PCI configuration space, thus its register access is different. Existing kernel PCIe AER functions can not be used to manage the downstream port AER capabilities and RAS registers because the port was not enumerated during PCI scan and the registers are not PCI config accessible. Discover RCH downstream port AER extended capability registers. Use MMIO accesses to search for extended AER capability in RCRB register space. [1] CXL 3.0 Spec, 9.11.2 - System Firmware View of CXL 1.1 Hierarchy [2] CXL 3.0 Spec, 8.2.1.1 - RCH Downstream Port RCRB Co-developed-by: Robert Richter Signed-off-by: Terry Bowman Signed-off-by: Robert Richter Reviewed-by: Jonathan Cameron Reviewed-by: Dave Jiang Link: https://lore.kernel.org/r/20231018171713.1883517-12-rrichter@amd.com Signed-off-by: Dan Williams Signed-off-by: Yi Liu Signed-off-by: Bo Li --- drivers/cxl/core/core.h | 1 + drivers/cxl/core/pci.c | 15 +++++++++++++++ drivers/cxl/core/regs.c | 36 ++++++++++++++++++++++++++++++++++++ drivers/cxl/cxl.h | 7 +++++++ drivers/cxl/mem.c | 2 ++ 5 files changed, 61 insertions(+) diff --git a/drivers/cxl/core/core.h b/drivers/cxl/core/core.h index 6444cc827c9ce..713d8754cc7e9 100644 --- a/drivers/cxl/core/core.h +++ b/drivers/cxl/core/core.h @@ -80,6 +80,7 @@ struct cxl_rcrb_info; resource_size_t __rcrb_to_component(struct device *dev, struct cxl_rcrb_info *ri, enum cxl_rcrb which); +u16 cxl_rcrb_to_aer(struct device *dev, resource_size_t rcrb); extern struct rw_semaphore cxl_dpa_rwsem; extern struct rw_semaphore cxl_region_rwsem; diff --git a/drivers/cxl/core/pci.c b/drivers/cxl/core/pci.c index 6edfd05466737..fdf76b05ac2b7 100644 --- a/drivers/cxl/core/pci.c +++ b/drivers/cxl/core/pci.c @@ -712,6 +712,21 @@ static bool cxl_report_and_clear(struct cxl_dev_state *cxlds) return true; } +#ifdef CONFIG_PCIEAER_CXL + +void cxl_setup_parent_dport(struct device *host, struct cxl_dport *dport) +{ + struct device *dport_dev = dport->dport_dev; + struct pci_host_bridge *host_bridge; + + host_bridge = to_pci_host_bridge(dport_dev); + if (host_bridge->native_cxl_error) + dport->rcrb.aer_cap = cxl_rcrb_to_aer(dport_dev, dport->rcrb.base); +} +EXPORT_SYMBOL_NS_GPL(cxl_setup_parent_dport, CXL); + +#endif + pci_ers_result_t cxl_error_detected(struct pci_dev *pdev, pci_channel_state_t state) { diff --git a/drivers/cxl/core/regs.c b/drivers/cxl/core/regs.c index 92ef68849fbea..f4602b6709e99 100644 --- a/drivers/cxl/core/regs.c +++ b/drivers/cxl/core/regs.c @@ -471,6 +471,42 @@ int cxl_setup_regs(struct cxl_register_map *map) } EXPORT_SYMBOL_NS_GPL(cxl_setup_regs, CXL); +u16 cxl_rcrb_to_aer(struct device *dev, resource_size_t rcrb) +{ + void __iomem *addr; + u16 offset = 0; + u32 cap_hdr; + + if (WARN_ON_ONCE(rcrb == CXL_RESOURCE_NONE)) + return 0; + + if (!request_mem_region(rcrb, SZ_4K, dev_name(dev))) + return 0; + + addr = ioremap(rcrb, SZ_4K); + if (!addr) + goto out; + + cap_hdr = readl(addr + offset); + while (PCI_EXT_CAP_ID(cap_hdr) != PCI_EXT_CAP_ID_ERR) { + offset = PCI_EXT_CAP_NEXT(cap_hdr); + + /* Offset 0 terminates capability list. */ + if (!offset) + break; + cap_hdr = readl(addr + offset); + } + + if (offset) + dev_dbg(dev, "found AER extended capability (0x%x)\n", offset); + + iounmap(addr); +out: + release_mem_region(rcrb, SZ_4K); + + return offset; +} + resource_size_t __rcrb_to_component(struct device *dev, struct cxl_rcrb_info *ri, enum cxl_rcrb which) { diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h index bec24d3a41969..ec96c68768b02 100644 --- a/drivers/cxl/cxl.h +++ b/drivers/cxl/cxl.h @@ -706,6 +706,13 @@ struct cxl_dport *devm_cxl_add_rch_dport(struct cxl_port *port, struct device *dport_dev, int port_id, resource_size_t rcrb); +#ifdef CONFIG_PCIEAER_CXL +void cxl_setup_parent_dport(struct device *host, struct cxl_dport *dport); +#else +static inline void cxl_setup_parent_dport(struct device *host, + struct cxl_dport *dport) { } +#endif + struct cxl_decoder *to_cxl_decoder(struct device *dev); struct cxl_root_decoder *to_cxl_root_decoder(struct device *dev); struct cxl_switch_decoder *to_cxl_switch_decoder(struct device *dev); diff --git a/drivers/cxl/mem.c b/drivers/cxl/mem.c index 04107058739bc..e087febf9af04 100644 --- a/drivers/cxl/mem.c +++ b/drivers/cxl/mem.c @@ -157,6 +157,8 @@ static int cxl_mem_probe(struct device *dev) else endpoint_parent = &parent_port->dev; + cxl_setup_parent_dport(dev, dport); + device_lock(endpoint_parent); if (!endpoint_parent->driver) { dev_err(dev, "CXL port topology %s not enabled\n", From 20f4a413ea031577c266e6eeec4ae1de4e1d16ab Mon Sep 17 00:00:00 2001 From: Terry Bowman Date: Wed, 18 Oct 2023 19:17:05 +0200 Subject: [PATCH 017/138] PCI/AER: Refactor cper_print_aer() for use by CXL driver module commit 6777877eb7a3290cf0a8a6b621e46f72f9d94b6b upstream. The CXL driver plans to use cper_print_aer() for logging restricted CXL host (RCH) AER errors. cper_print_aer() is not currently exported and therefore not usable by the CXL drivers built as loadable modules. Export the cper_print_aer() function. Use the EXPORT_SYMBOL_NS_GPL() variant to restrict the export to CXL drivers. The CONFIG_ACPI_APEI_PCIEAER kernel config is currently used to enable cper_print_aer(). cper_print_aer() logs the AER registers and is useful in PCIE AER logging outside of APEI. Remove the CONFIG_ACPI_APEI_PCIEAER dependency to enable cper_print_aer(). The cper_print_aer() function name implies CPER specific use but is useful in non-CPER cases as well. Rename cper_print_aer() to pci_print_aer(). Also, update cxl_core to import CXL namespace imports. Co-developed-by: Robert Richter Signed-off-by: Terry Bowman Signed-off-by: Robert Richter Cc: Mahesh J Salgaonkar Cc: Oliver O'Halloran Cc: Bjorn Helgaas Cc: linux-pci@vger.kernel.org Reviewed-by: Jonathan Cameron Acked-by: Bjorn Helgaas Reviewed-by: Dave Jiang Link: https://lore.kernel.org/r/20231018171713.1883517-13-rrichter@amd.com Signed-off-by: Dan Williams Signed-off-by: Yi Liu Signed-off-by: Bo Li --- drivers/cxl/core/port.c | 1 + drivers/pci/pcie/aer.c | 9 +++++---- include/linux/aer.h | 2 +- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/drivers/cxl/core/port.c b/drivers/cxl/core/port.c index e2a1eada43da1..ca69f0f9144ae 100644 --- a/drivers/cxl/core/port.c +++ b/drivers/cxl/core/port.c @@ -2116,3 +2116,4 @@ static void cxl_core_exit(void) subsys_initcall(cxl_core_init); module_exit(cxl_core_exit); MODULE_LICENSE("GPL v2"); +MODULE_IMPORT_NS(CXL); diff --git a/drivers/pci/pcie/aer.c b/drivers/pci/pcie/aer.c index 8e700020ee0bc..81805e63cf1c2 100644 --- a/drivers/pci/pcie/aer.c +++ b/drivers/pci/pcie/aer.c @@ -760,9 +760,10 @@ int cper_severity_to_aer(int cper_severity) } } EXPORT_SYMBOL_GPL(cper_severity_to_aer); +#endif -void cper_print_aer(struct pci_dev *dev, int aer_severity, - struct aer_capability_regs *aer) +void pci_print_aer(struct pci_dev *dev, int aer_severity, + struct aer_capability_regs *aer) { int layer, agent, tlp_header_valid = 0; u32 status, mask; @@ -801,7 +802,7 @@ void cper_print_aer(struct pci_dev *dev, int aer_severity, trace_aer_event(dev_name(&dev->dev), (status & ~mask), aer_severity, tlp_header_valid, &aer->header_log); } -#endif +EXPORT_SYMBOL_NS_GPL(pci_print_aer, CXL); /** * add_error_device - list device to be handled @@ -1002,7 +1003,7 @@ static void aer_recover_work_func(struct work_struct *work) PCI_SLOT(entry.devfn), PCI_FUNC(entry.devfn)); continue; } - cper_print_aer(pdev, entry.severity, entry.regs); + pci_print_aer(pdev, entry.severity, entry.regs); /* * Memory for aer_capability_regs(entry.regs) is being allocated from the * ghes_estatus_pool to protect it from overwriting when multiple sections diff --git a/include/linux/aer.h b/include/linux/aer.h index 29cc10220952d..f6ea2f57d8089 100644 --- a/include/linux/aer.h +++ b/include/linux/aer.h @@ -51,7 +51,7 @@ static inline int pci_aer_clear_nonfatal_status(struct pci_dev *dev) static inline int pcie_aer_is_native(struct pci_dev *dev) { return 0; } #endif -void cper_print_aer(struct pci_dev *dev, int aer_severity, +void pci_print_aer(struct pci_dev *dev, int aer_severity, struct aer_capability_regs *aer); int cper_severity_to_aer(int cper_severity); void aer_recover_queue(int domain, unsigned int bus, unsigned int devfn, From 5aa96fd8e0245b5598419f26622ae347b0442b95 Mon Sep 17 00:00:00 2001 From: Terry Bowman Date: Wed, 18 Oct 2023 19:17:06 +0200 Subject: [PATCH 018/138] cxl/pci: Update CXL error logging to use RAS register address commit bf6c9fa846e2a0f7db2a2eabd52ad4f8d4335bcb upstream. The CXL error handler currently only logs endpoint RAS status. The CXL topology includes several components providing RAS details to be logged during error handling.[1] Update the current handler's RAS logging to use a RAS register address. Also, update the error handler function names to be consistent with correctable and uncorrectable RAS. This will allow for adding support to log other CXL component's RAS details in the future. [1] CXL3.0 Table 8-22 CXL_Capability_ID Assignment Co-developed-by: Robert Richter Signed-off-by: Terry Bowman Signed-off-by: Robert Richter Reviewed-by: Jonathan Cameron Reviewed-by: Dave Jiang Link: https://lore.kernel.org/r/20231018171713.1883517-14-rrichter@amd.com Signed-off-by: Dan Williams Signed-off-by: Yi Liu Signed-off-by: Bo Li --- drivers/cxl/core/pci.c | 44 +++++++++++++++++++++++++++++------------- 1 file changed, 31 insertions(+), 13 deletions(-) diff --git a/drivers/cxl/core/pci.c b/drivers/cxl/core/pci.c index fdf76b05ac2b7..bf8a7ed976ef1 100644 --- a/drivers/cxl/core/pci.c +++ b/drivers/cxl/core/pci.c @@ -640,32 +640,36 @@ void read_cdat_data(struct cxl_port *port) } EXPORT_SYMBOL_NS_GPL(read_cdat_data, CXL); -void cxl_cor_error_detected(struct pci_dev *pdev) +static void __cxl_handle_cor_ras(struct cxl_dev_state *cxlds, + void __iomem *ras_base) { - struct cxl_dev_state *cxlds = pci_get_drvdata(pdev); void __iomem *addr; u32 status; - if (!cxlds->regs.ras) + if (!ras_base) return; - addr = cxlds->regs.ras + CXL_RAS_CORRECTABLE_STATUS_OFFSET; + addr = ras_base + CXL_RAS_CORRECTABLE_STATUS_OFFSET; status = readl(addr); if (status & CXL_RAS_CORRECTABLE_STATUS_MASK) { writel(status & CXL_RAS_CORRECTABLE_STATUS_MASK, addr); trace_cxl_aer_correctable_error(cxlds->cxlmd, status); } } -EXPORT_SYMBOL_NS_GPL(cxl_cor_error_detected, CXL); + +static void cxl_handle_endpoint_cor_ras(struct cxl_dev_state *cxlds) +{ + return __cxl_handle_cor_ras(cxlds, cxlds->regs.ras); +} /* CXL spec rev3.0 8.2.4.16.1 */ -static void header_log_copy(struct cxl_dev_state *cxlds, u32 *log) +static void header_log_copy(void __iomem *ras_base, u32 *log) { void __iomem *addr; u32 *log_addr; int i, log_u32_size = CXL_HEADERLOG_SIZE / sizeof(u32); - addr = cxlds->regs.ras + CXL_RAS_HEADER_LOG_OFFSET; + addr = ras_base + CXL_RAS_HEADER_LOG_OFFSET; log_addr = log; for (i = 0; i < log_u32_size; i++) { @@ -679,17 +683,18 @@ static void header_log_copy(struct cxl_dev_state *cxlds, u32 *log) * Log the state of the RAS status registers and prepare them to log the * next error status. Return 1 if reset needed. */ -static bool cxl_report_and_clear(struct cxl_dev_state *cxlds) +static bool __cxl_handle_ras(struct cxl_dev_state *cxlds, + void __iomem *ras_base) { u32 hl[CXL_HEADERLOG_SIZE_U32]; void __iomem *addr; u32 status; u32 fe; - if (!cxlds->regs.ras) + if (!ras_base) return false; - addr = cxlds->regs.ras + CXL_RAS_UNCORRECTABLE_STATUS_OFFSET; + addr = ras_base + CXL_RAS_UNCORRECTABLE_STATUS_OFFSET; status = readl(addr); if (!(status & CXL_RAS_UNCORRECTABLE_STATUS_MASK)) return false; @@ -697,7 +702,7 @@ static bool cxl_report_and_clear(struct cxl_dev_state *cxlds) /* If multiple errors, log header points to first error from ctrl reg */ if (hweight32(status) > 1) { void __iomem *rcc_addr = - cxlds->regs.ras + CXL_RAS_CAP_CONTROL_OFFSET; + ras_base + CXL_RAS_CAP_CONTROL_OFFSET; fe = BIT(FIELD_GET(CXL_RAS_CAP_CONTROL_FE_MASK, readl(rcc_addr))); @@ -705,13 +710,18 @@ static bool cxl_report_and_clear(struct cxl_dev_state *cxlds) fe = status; } - header_log_copy(cxlds, hl); + header_log_copy(ras_base, hl); trace_cxl_aer_uncorrectable_error(cxlds->cxlmd, status, fe, hl); writel(status & CXL_RAS_UNCORRECTABLE_STATUS_MASK, addr); return true; } +static bool cxl_handle_endpoint_ras(struct cxl_dev_state *cxlds) +{ + return __cxl_handle_ras(cxlds, cxlds->regs.ras); +} + #ifdef CONFIG_PCIEAER_CXL void cxl_setup_parent_dport(struct device *host, struct cxl_dport *dport) @@ -727,6 +737,14 @@ EXPORT_SYMBOL_NS_GPL(cxl_setup_parent_dport, CXL); #endif +void cxl_cor_error_detected(struct pci_dev *pdev) +{ + struct cxl_dev_state *cxlds = pci_get_drvdata(pdev); + + cxl_handle_endpoint_cor_ras(cxlds); +} +EXPORT_SYMBOL_NS_GPL(cxl_cor_error_detected, CXL); + pci_ers_result_t cxl_error_detected(struct pci_dev *pdev, pci_channel_state_t state) { @@ -741,7 +759,7 @@ pci_ers_result_t cxl_error_detected(struct pci_dev *pdev, * chance the situation is recoverable dump the status of the RAS * capability registers and bounce the active state of the memdev. */ - ue = cxl_report_and_clear(cxlds); + ue = cxl_handle_endpoint_ras(cxlds); switch (state) { case pci_channel_io_normal: From f2348ca9971ab3cb96e84314332948d94a49195d Mon Sep 17 00:00:00 2001 From: Terry Bowman Date: Wed, 18 Oct 2023 19:17:07 +0200 Subject: [PATCH 019/138] cxl/pci: Map RCH downstream AER registers for logging protocol errors commit 6c5f3aacb2963d49a11d4f8accb1188db6a6404b upstream. The restricted CXL host (RCH) error handler will log protocol errors using AER and RAS status registers. The AER and RAS registers need to be virtually memory mapped before enabling interrupts. Create the initializer function devm_cxl_setup_parent_dport() for this when the endpoint is connected with the dport. The initialization sets up the RCH RAS and AER mappings. Add 'struct cxl_regs' to 'struct cxl_dport' for saving a pointer to the RCH downstream port's AER and RAS registers. Signed-off-by: Terry Bowman Co-developed-by: Robert Richter Signed-off-by: Robert Richter Reviewed-by: Jonathan Cameron Link: https://lore.kernel.org/r/20231018171713.1883517-15-rrichter@amd.com Signed-off-by: Dan Williams Signed-off-by: Yi Liu Signed-off-by: Bo Li --- drivers/cxl/core/pci.c | 36 ++++++++++++++++++++++++++++++++++++ drivers/cxl/cxl.h | 10 ++++++++++ 2 files changed, 46 insertions(+) diff --git a/drivers/cxl/core/pci.c b/drivers/cxl/core/pci.c index bf8a7ed976ef1..4361cae2832b8 100644 --- a/drivers/cxl/core/pci.c +++ b/drivers/cxl/core/pci.c @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -724,6 +725,38 @@ static bool cxl_handle_endpoint_ras(struct cxl_dev_state *cxlds) #ifdef CONFIG_PCIEAER_CXL +static void cxl_dport_map_rch_aer(struct cxl_dport *dport) +{ + struct cxl_rcrb_info *ri = &dport->rcrb; + void __iomem *dport_aer = NULL; + resource_size_t aer_phys; + struct device *host; + + if (dport->rch && ri->aer_cap) { + host = dport->reg_map.host; + aer_phys = ri->aer_cap + ri->base; + dport_aer = devm_cxl_iomap_block(host, aer_phys, + sizeof(struct aer_capability_regs)); + } + + dport->regs.dport_aer = dport_aer; +} + +static void cxl_dport_map_regs(struct cxl_dport *dport) +{ + struct cxl_register_map *map = &dport->reg_map; + struct device *dev = dport->dport_dev; + + if (!map->component_map.ras.valid) + dev_dbg(dev, "RAS registers not found\n"); + else if (cxl_map_component_regs(map, &dport->regs.component, + BIT(CXL_CM_CAP_CAP_ID_RAS))) + dev_dbg(dev, "Failed to map RAS capability.\n"); + + if (dport->rch) + cxl_dport_map_rch_aer(dport); +} + void cxl_setup_parent_dport(struct device *host, struct cxl_dport *dport) { struct device *dport_dev = dport->dport_dev; @@ -732,6 +765,9 @@ void cxl_setup_parent_dport(struct device *host, struct cxl_dport *dport) host_bridge = to_pci_host_bridge(dport_dev); if (host_bridge->native_cxl_error) dport->rcrb.aer_cap = cxl_rcrb_to_aer(dport_dev, dport->rcrb.base); + + dport->reg_map.host = host; + cxl_dport_map_regs(dport); } EXPORT_SYMBOL_NS_GPL(cxl_setup_parent_dport, CXL); diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h index ec96c68768b02..9f62259177d36 100644 --- a/drivers/cxl/cxl.h +++ b/drivers/cxl/cxl.h @@ -223,6 +223,14 @@ struct cxl_regs { struct_group_tagged(cxl_pmu_regs, pmu_regs, void __iomem *pmu; ); + + /* + * RCH downstream port specific RAS register + * @aer: CXL 3.0 8.2.1.1 RCH Downstream Port RCRB + */ + struct_group_tagged(cxl_rch_regs, rch_regs, + void __iomem *dport_aer; + ); }; struct cxl_reg_map { @@ -623,6 +631,7 @@ struct cxl_rcrb_info { * @rcrb: Data about the Root Complex Register Block layout * @rch: Indicate whether this dport was enumerated in RCH or VH mode * @port: reference to cxl_port that contains this downstream port + * @regs: Dport parsed register blocks */ struct cxl_dport { struct device *dport_dev; @@ -631,6 +640,7 @@ struct cxl_dport { struct cxl_rcrb_info rcrb; bool rch; struct cxl_port *port; + struct cxl_regs regs; }; /** From d048b7a4a25c698cf3bfa629ea0a0e96a76c9366 Mon Sep 17 00:00:00 2001 From: Terry Bowman Date: Wed, 18 Oct 2023 19:17:08 +0200 Subject: [PATCH 020/138] cxl/pci: Add RCH downstream port error logging commit 6ac07883dbb5f60f7bc56a13b7a84a382aa9c1ab upstream. RCH downstream port error logging is missing in the current CXL driver. The missing AER and RAS error logging is needed for communicating driver error details to userspace. Update the driver to include PCIe AER and CXL RAS error logging. Add RCH downstream port error handling into the existing RCiEP handler. The downstream port error handler is added to the RCiEP error handler because the downstream port is implemented in a RCRB, is not PCI enumerable, and as a result is not directly accessible to the PCI AER root port driver. The AER root port driver calls the RCiEP handler for handling RCD errors and RCH downstream port protocol errors. Update existing RCiEP correctable and uncorrectable handlers to also call the RCH handler. The RCH handler will read the RCH AER registers, check for error severity, and if an error exists will log using an existing kernel AER trace routine. The RCH handler will also log downstream port RAS errors if they exist. Co-developed-by: Robert Richter Signed-off-by: Terry Bowman Signed-off-by: Robert Richter Reviewed-by: Jonathan Cameron Reviewed-by: Dave Jiang Link: https://lore.kernel.org/r/20231018171713.1883517-16-rrichter@amd.com Signed-off-by: Dan Williams Signed-off-by: Yi Liu Signed-off-by: Bo Li --- drivers/cxl/core/pci.c | 96 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) diff --git a/drivers/cxl/core/pci.c b/drivers/cxl/core/pci.c index 4361cae2832b8..259d7b94296f8 100644 --- a/drivers/cxl/core/pci.c +++ b/drivers/cxl/core/pci.c @@ -771,12 +771,105 @@ void cxl_setup_parent_dport(struct device *host, struct cxl_dport *dport) } EXPORT_SYMBOL_NS_GPL(cxl_setup_parent_dport, CXL); +static void cxl_handle_rdport_cor_ras(struct cxl_dev_state *cxlds, + struct cxl_dport *dport) +{ + return __cxl_handle_cor_ras(cxlds, dport->regs.ras); +} + +static bool cxl_handle_rdport_ras(struct cxl_dev_state *cxlds, + struct cxl_dport *dport) +{ + return __cxl_handle_ras(cxlds, dport->regs.ras); +} + +/* + * Copy the AER capability registers using 32 bit read accesses. + * This is necessary because RCRB AER capability is MMIO mapped. Clear the + * status after copying. + * + * @aer_base: base address of AER capability block in RCRB + * @aer_regs: destination for copying AER capability + */ +static bool cxl_rch_get_aer_info(void __iomem *aer_base, + struct aer_capability_regs *aer_regs) +{ + int read_cnt = sizeof(struct aer_capability_regs) / sizeof(u32); + u32 *aer_regs_buf = (u32 *)aer_regs; + int n; + + if (!aer_base) + return false; + + /* Use readl() to guarantee 32-bit accesses */ + for (n = 0; n < read_cnt; n++) + aer_regs_buf[n] = readl(aer_base + n * sizeof(u32)); + + writel(aer_regs->uncor_status, aer_base + PCI_ERR_UNCOR_STATUS); + writel(aer_regs->cor_status, aer_base + PCI_ERR_COR_STATUS); + + return true; +} + +/* Get AER severity. Return false if there is no error. */ +static bool cxl_rch_get_aer_severity(struct aer_capability_regs *aer_regs, + int *severity) +{ + if (aer_regs->uncor_status & ~aer_regs->uncor_mask) { + if (aer_regs->uncor_status & PCI_ERR_ROOT_FATAL_RCV) + *severity = AER_FATAL; + else + *severity = AER_NONFATAL; + return true; + } + + if (aer_regs->cor_status & ~aer_regs->cor_mask) { + *severity = AER_CORRECTABLE; + return true; + } + + return false; +} + +static void cxl_handle_rdport_errors(struct cxl_dev_state *cxlds) +{ + struct pci_dev *pdev = to_pci_dev(cxlds->dev); + struct aer_capability_regs aer_regs; + struct cxl_dport *dport; + struct cxl_port *port; + int severity; + + port = cxl_pci_find_port(pdev, &dport); + if (!port) + return; + + put_device(&port->dev); + + if (!cxl_rch_get_aer_info(dport->regs.dport_aer, &aer_regs)) + return; + + if (!cxl_rch_get_aer_severity(&aer_regs, &severity)) + return; + + pci_print_aer(pdev, severity, &aer_regs); + + if (severity == AER_CORRECTABLE) + cxl_handle_rdport_cor_ras(cxlds, dport); + else + cxl_handle_rdport_ras(cxlds, dport); +} + +#else +static void cxl_handle_rdport_errors(struct cxl_dev_state *cxlds) { } #endif void cxl_cor_error_detected(struct pci_dev *pdev) { struct cxl_dev_state *cxlds = pci_get_drvdata(pdev); + if (cxlds->rcd) + cxl_handle_rdport_errors(cxlds); + cxl_handle_endpoint_cor_ras(cxlds); } EXPORT_SYMBOL_NS_GPL(cxl_cor_error_detected, CXL); @@ -789,6 +882,9 @@ pci_ers_result_t cxl_error_detected(struct pci_dev *pdev, struct device *dev = &cxlmd->dev; bool ue; + if (cxlds->rcd) + cxl_handle_rdport_errors(cxlds); + /* * A frozen channel indicates an impending reset which is fatal to * CXL.mem operation, and will likely crash the system. On the off From f7d58d9b5ee44568c47eb75af56917189ccd335a Mon Sep 17 00:00:00 2001 From: Terry Bowman Date: Wed, 18 Oct 2023 19:17:09 +0200 Subject: [PATCH 021/138] cxl/pci: Disable root port interrupts in RCH mode commit d1a9def33d7043df7445114cb89c0aa65818ae91 upstream. The RCH root port contains root command AER registers that should not be enabled.[1] Disable these to prevent root port interrupts. [1] CXL 3.0 - 12.2.1.1 RCH Downstream Port-detected Errors Signed-off-by: Terry Bowman Signed-off-by: Robert Richter Reviewed-by: Jonathan Cameron Reviewed-by: Dave Jiang Link: https://lore.kernel.org/r/20231018171713.1883517-17-rrichter@amd.com Signed-off-by: Dan Williams Signed-off-by: Yi Liu Signed-off-by: Bo Li --- drivers/cxl/core/pci.c | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/drivers/cxl/core/pci.c b/drivers/cxl/core/pci.c index 259d7b94296f8..764bc02e6c664 100644 --- a/drivers/cxl/core/pci.c +++ b/drivers/cxl/core/pci.c @@ -757,6 +757,35 @@ static void cxl_dport_map_regs(struct cxl_dport *dport) cxl_dport_map_rch_aer(dport); } +static void cxl_disable_rch_root_ints(struct cxl_dport *dport) +{ + void __iomem *aer_base = dport->regs.dport_aer; + struct pci_host_bridge *bridge; + u32 aer_cmd_mask, aer_cmd; + + if (!aer_base) + return; + + bridge = to_pci_host_bridge(dport->dport_dev); + + /* + * Disable RCH root port command interrupts. + * CXL 3.0 12.2.1.1 - RCH Downstream Port-detected Errors + * + * This sequence may not be necessary. CXL spec states disabling + * the root cmd register's interrupts is required. But, PCI spec + * shows these are disabled by default on reset. + */ + if (bridge->native_cxl_error) { + aer_cmd_mask = (PCI_ERR_ROOT_CMD_COR_EN | + PCI_ERR_ROOT_CMD_NONFATAL_EN | + PCI_ERR_ROOT_CMD_FATAL_EN); + aer_cmd = readl(aer_base + PCI_ERR_ROOT_COMMAND); + aer_cmd &= ~aer_cmd_mask; + writel(aer_cmd, aer_base + PCI_ERR_ROOT_COMMAND); + } +} + void cxl_setup_parent_dport(struct device *host, struct cxl_dport *dport) { struct device *dport_dev = dport->dport_dev; @@ -768,6 +797,9 @@ void cxl_setup_parent_dport(struct device *host, struct cxl_dport *dport) dport->reg_map.host = host; cxl_dport_map_regs(dport); + + if (dport->rch) + cxl_disable_rch_root_ints(dport); } EXPORT_SYMBOL_NS_GPL(cxl_setup_parent_dport, CXL); From b8ef52054b79dd51b130f5e19bc2ec371647e7e2 Mon Sep 17 00:00:00 2001 From: Robert Richter Date: Wed, 18 Oct 2023 19:17:10 +0200 Subject: [PATCH 022/138] PCI/AER: Forward RCH downstream port-detected errors to the CXL.mem dev handler commit 0a867568bb0d203ca3d28634a611a1367d7c892d upstream. In Restricted CXL Device (RCD) mode a CXL device is exposed as an RCiEP, but CXL downstream and upstream ports are not enumerated and not visible in the PCIe hierarchy. [1] Protocol and link errors from these non-enumerated ports are signaled as internal AER errors, either Uncorrectable Internal Error (UIE) or Corrected Internal Errors (CIE) via an RCEC. Restricted CXL host (RCH) downstream port-detected errors have the Requester ID of the RCEC set in the RCEC's AER Error Source ID register. A CXL handler must then inspect the error status in various CXL registers residing in the dport's component register space (CXL RAS capability) or the dport's RCRB (PCIe AER extended capability). [2] Errors showing up in the RCEC's error handler must be handled and connected to the CXL subsystem. Implement this by forwarding the error to all CXL devices below the RCEC. Since the entire CXL device is controlled only using PCIe Configuration Space of device 0, function 0, only pass it there [3]. The error handling is limited to currently supported devices with the Memory Device class code set (CXL Type 3 Device, PCI_CLASS_MEMORY_CXL, 502h), handle downstream port errors in the device's cxl_pci driver. Support for other CXL Device Types (e.g. a CXL.cache Device) can be added later. To handle downstream port errors in addition to errors directed to the CXL endpoint device, a handler must also inspect the CXL RAS and PCIe AER capabilities of the CXL downstream port the device is connected to. Since CXL downstream port errors are signaled using internal errors, the handler requires those errors to be unmasked. This is subject of a follow-on patch. The reason for choosing this implementation is that the AER service driver claims the RCEC device, but does not allow it to register a custom specific handler to support CXL. Connecting the RCEC hard-wired with a CXL handler does not work, as the CXL subsystem might not be present all the time. The alternative to add an implementation to the portdrv to allow the registration of a custom RCEC error handler isn't worth doing it as CXL would be its only user. Instead, just check for an CXL RCEC and pass it down to the connected CXL device's error handler. With this approach the code can entirely be implemented in the PCIe AER driver and is independent of the CXL subsystem. The CXL driver only provides the handler. [1] CXL 3.0 spec: 9.11.8 CXL Devices Attached to an RCH [2] CXL 3.0 spec, 12.2.1.1 RCH Downstream Port-detected Errors [3] CXL 3.0 spec, 8.1.3 PCIe DVSEC for CXL Devices Co-developed-by: Terry Bowman Signed-off-by: Terry Bowman Signed-off-by: Robert Richter Cc: Oliver O'Halloran Cc: Bjorn Helgaas Cc: linuxppc-dev@lists.ozlabs.org Cc: linux-pci@vger.kernel.org Acked-by: Bjorn Helgaas Reviewed-by: Jonathan Cameron Reviewed-by: Dave Jiang Link: https://lore.kernel.org/r/20231018171713.1883517-18-rrichter@amd.com Signed-off-by: Dan Williams Signed-off-by: Yi Liu Signed-off-by: Bo Li --- drivers/pci/pcie/Kconfig | 9 ++++ drivers/pci/pcie/aer.c | 93 +++++++++++++++++++++++++++++++++++++++- 2 files changed, 100 insertions(+), 2 deletions(-) diff --git a/drivers/pci/pcie/Kconfig b/drivers/pci/pcie/Kconfig index 228652a59f27e..8999fcebde6aa 100644 --- a/drivers/pci/pcie/Kconfig +++ b/drivers/pci/pcie/Kconfig @@ -49,6 +49,15 @@ config PCIEAER_INJECT gotten from: https://git.kernel.org/cgit/linux/kernel/git/gong.chen/aer-inject.git/ +config PCIEAER_CXL + bool "PCI Express CXL RAS support" + default y + depends on PCIEAER && CXL_PCI + help + Enables CXL error handling. + + If unsure, say Y. + # # PCI Express ECRC # diff --git a/drivers/pci/pcie/aer.c b/drivers/pci/pcie/aer.c index 81805e63cf1c2..9f7fdb6878133 100644 --- a/drivers/pci/pcie/aer.c +++ b/drivers/pci/pcie/aer.c @@ -940,14 +940,97 @@ static bool find_source_device(struct pci_dev *parent, return true; } +#ifdef CONFIG_PCIEAER_CXL + +static bool is_cxl_mem_dev(struct pci_dev *dev) +{ + /* + * The capability, status, and control fields in Device 0, + * Function 0 DVSEC control the CXL functionality of the + * entire device (CXL 3.0, 8.1.3). + */ + if (dev->devfn != PCI_DEVFN(0, 0)) + return false; + + /* + * CXL Memory Devices must have the 502h class code set (CXL + * 3.0, 8.1.12.1). + */ + if ((dev->class >> 8) != PCI_CLASS_MEMORY_CXL) + return false; + + return true; +} + +static bool cxl_error_is_native(struct pci_dev *dev) +{ + struct pci_host_bridge *host = pci_find_host_bridge(dev->bus); + + return (pcie_ports_native || host->native_aer); +} + +static bool is_internal_error(struct aer_err_info *info) +{ + if (info->severity == AER_CORRECTABLE) + return info->status & PCI_ERR_COR_INTERNAL; + + return info->status & PCI_ERR_UNC_INTN; +} + +static int cxl_rch_handle_error_iter(struct pci_dev *dev, void *data) +{ + struct aer_err_info *info = (struct aer_err_info *)data; + const struct pci_error_handlers *err_handler; + + if (!is_cxl_mem_dev(dev) || !cxl_error_is_native(dev)) + return 0; + + /* protect dev->driver */ + device_lock(&dev->dev); + + err_handler = dev->driver ? dev->driver->err_handler : NULL; + if (!err_handler) + goto out; + + if (info->severity == AER_CORRECTABLE) { + if (err_handler->cor_error_detected) + err_handler->cor_error_detected(dev); + } else if (err_handler->error_detected) { + if (info->severity == AER_NONFATAL) + err_handler->error_detected(dev, pci_channel_io_normal); + else if (info->severity == AER_FATAL) + err_handler->error_detected(dev, pci_channel_io_frozen); + } +out: + device_unlock(&dev->dev); + return 0; +} + +static void cxl_rch_handle_error(struct pci_dev *dev, struct aer_err_info *info) +{ + /* + * Internal errors of an RCEC indicate an AER error in an + * RCH's downstream port. Check and handle them in the CXL.mem + * device driver. + */ + if (pci_pcie_type(dev) == PCI_EXP_TYPE_RC_EC && + is_internal_error(info)) + pcie_walk_rcec(dev, cxl_rch_handle_error_iter, info); +} + +#else +static inline void cxl_rch_handle_error(struct pci_dev *dev, + struct aer_err_info *info) { } +#endif + /** - * handle_error_source - handle logging error into an event log + * pci_aer_handle_error - handle logging error into an event log * @dev: pointer to pci_dev data structure of error source device * @info: comprehensive error information * * Invoked when an error being detected by Root Port. */ -static void handle_error_source(struct pci_dev *dev, struct aer_err_info *info) +static void pci_aer_handle_error(struct pci_dev *dev, struct aer_err_info *info) { int aer = dev->aer_cap; @@ -971,6 +1054,12 @@ static void handle_error_source(struct pci_dev *dev, struct aer_err_info *info) pcie_do_recovery(dev, pci_channel_io_normal, aer_root_reset); else if (info->severity == AER_FATAL) pcie_do_recovery(dev, pci_channel_io_frozen, aer_root_reset); +} + +static void handle_error_source(struct pci_dev *dev, struct aer_err_info *info) +{ + cxl_rch_handle_error(dev, info); + pci_aer_handle_error(dev, info); pci_dev_put(dev); } From aa0cbf8d2249a769800ecd6b71cbfc3ec9f6cdd1 Mon Sep 17 00:00:00 2001 From: Robert Richter Date: Wed, 18 Oct 2023 19:17:11 +0200 Subject: [PATCH 023/138] PCI/AER: Unmask RCEC internal errors to enable RCH downstream port error handling commit b7e9392d5d46a67fb5b66dbb2c257dd0d48eec70 upstream. AER corrected and uncorrectable internal errors (CIE/UIE) are masked in their corresponding mask registers per default once in power-up state. [1][2] Enable internal errors for RCECs to receive CXL downstream port errors of Restricted CXL Hosts (RCHs). [1] CXL 3.0 Spec, 12.2.1.1 - RCH Downstream Port Detected Errors [2] PCIe Base Spec r6.0, 7.8.4.3 Uncorrectable Error Mask Register, 7.8.4.6 Correctable Error Mask Register Co-developed-by: Terry Bowman Signed-off-by: Terry Bowman Signed-off-by: Robert Richter Acked-by: Bjorn Helgaas Reviewed-by: Jonathan Cameron Reviewed-by: Dave Jiang Link: https://lore.kernel.org/r/20231018171713.1883517-19-rrichter@amd.com Signed-off-by: Dan Williams Signed-off-by: Yi Liu Signed-off-by: Bo Li --- drivers/pci/pcie/aer.c | 57 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/drivers/pci/pcie/aer.c b/drivers/pci/pcie/aer.c index 9f7fdb6878133..76a5f96434f41 100644 --- a/drivers/pci/pcie/aer.c +++ b/drivers/pci/pcie/aer.c @@ -942,6 +942,30 @@ static bool find_source_device(struct pci_dev *parent, #ifdef CONFIG_PCIEAER_CXL +/** + * pci_aer_unmask_internal_errors - unmask internal errors + * @dev: pointer to the pcie_dev data structure + * + * Unmasks internal errors in the Uncorrectable and Correctable Error + * Mask registers. + * + * Note: AER must be enabled and supported by the device which must be + * checked in advance, e.g. with pcie_aer_is_native(). + */ +static void pci_aer_unmask_internal_errors(struct pci_dev *dev) +{ + int aer = dev->aer_cap; + u32 mask; + + pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_MASK, &mask); + mask &= ~PCI_ERR_UNC_INTN; + pci_write_config_dword(dev, aer + PCI_ERR_UNCOR_MASK, mask); + + pci_read_config_dword(dev, aer + PCI_ERR_COR_MASK, &mask); + mask &= ~PCI_ERR_COR_INTERNAL; + pci_write_config_dword(dev, aer + PCI_ERR_COR_MASK, mask); +} + static bool is_cxl_mem_dev(struct pci_dev *dev) { /* @@ -1018,7 +1042,39 @@ static void cxl_rch_handle_error(struct pci_dev *dev, struct aer_err_info *info) pcie_walk_rcec(dev, cxl_rch_handle_error_iter, info); } +static int handles_cxl_error_iter(struct pci_dev *dev, void *data) +{ + bool *handles_cxl = data; + + if (!*handles_cxl) + *handles_cxl = is_cxl_mem_dev(dev) && cxl_error_is_native(dev); + + /* Non-zero terminates iteration */ + return *handles_cxl; +} + +static bool handles_cxl_errors(struct pci_dev *rcec) +{ + bool handles_cxl = false; + + if (pci_pcie_type(rcec) == PCI_EXP_TYPE_RC_EC && + pcie_aer_is_native(rcec)) + pcie_walk_rcec(rcec, handles_cxl_error_iter, &handles_cxl); + + return handles_cxl; +} + +static void cxl_rch_enable_rcec(struct pci_dev *rcec) +{ + if (!handles_cxl_errors(rcec)) + return; + + pci_aer_unmask_internal_errors(rcec); + pci_info(rcec, "CXL: Internal errors unmasked"); +} + #else +static inline void cxl_rch_enable_rcec(struct pci_dev *dev) { } static inline void cxl_rch_handle_error(struct pci_dev *dev, struct aer_err_info *info) { } #endif @@ -1427,6 +1483,7 @@ static int aer_probe(struct pcie_device *dev) return status; } + cxl_rch_enable_rcec(port); aer_enable_rootport(rpc); pci_info(port, "enabled with IRQ %d\n", dev->irq); return 0; From 36dcb1d9176acdb58f4f3110a3ebd8289b61e341 Mon Sep 17 00:00:00 2001 From: Robert Richter Date: Wed, 18 Oct 2023 19:17:12 +0200 Subject: [PATCH 024/138] cxl/core/regs: Rename phys_addr in cxl_map_component_regs() commit d3970f006f084e5aab5091a865203899259e4d70 upstream. Trivial change that renames variable phys_addr in cxl_map_component_regs() to shorten its length to keep the 80 char size limit for the line and also for consistency between the different paths. Signed-off-by: Terry Bowman Signed-off-by: Robert Richter Reviewed-by: Dave Jiang Reviewed-by: Jonathan Cameron Link: https://lore.kernel.org/r/20231018171713.1883517-20-rrichter@amd.com Signed-off-by: Dan Williams Signed-off-by: Yi Liu Signed-off-by: Bo Li --- drivers/cxl/core/regs.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/cxl/core/regs.c b/drivers/cxl/core/regs.c index f4602b6709e99..35542a0b9f262 100644 --- a/drivers/cxl/core/regs.c +++ b/drivers/cxl/core/regs.c @@ -216,16 +216,16 @@ int cxl_map_component_regs(const struct cxl_register_map *map, for (i = 0; i < ARRAY_SIZE(mapinfo); i++) { struct mapinfo *mi = &mapinfo[i]; - resource_size_t phys_addr; + resource_size_t addr; resource_size_t length; if (!mi->rmap->valid) continue; if (!test_bit(mi->rmap->id, &map_mask)) continue; - phys_addr = map->resource + mi->rmap->offset; + addr = map->resource + mi->rmap->offset; length = mi->rmap->size; - *(mi->addr) = devm_cxl_iomap_block(host, phys_addr, length); + *(mi->addr) = devm_cxl_iomap_block(host, addr, length); if (!*(mi->addr)) return -ENOMEM; } From be1d3898c404c71c8f53144f076e953fe1b68924 Mon Sep 17 00:00:00 2001 From: Robert Richter Date: Wed, 18 Oct 2023 19:17:13 +0200 Subject: [PATCH 025/138] cxl/core/regs: Rework cxl_map_pmu_regs() to use map->dev for devm commit e8db0701605bccbeb8d7907ecd2e50f346a725bd upstream. struct cxl_register_map carries a @dev parameter for devm operations. Simplify the function interface to use that instead of a separate @dev argument. Signed-off-by: Terry Bowman Signed-off-by: Robert Richter Reviewed-by: Jonathan Cameron Link: https://lore.kernel.org/r/20231018171713.1883517-21-rrichter@amd.com Signed-off-by: Dan Williams Signed-off-by: Yi Liu Signed-off-by: Bo Li --- drivers/cxl/core/regs.c | 5 ++--- drivers/cxl/cxl.h | 3 +-- drivers/cxl/pci.c | 2 +- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/drivers/cxl/core/regs.c b/drivers/cxl/core/regs.c index 35542a0b9f262..7eebbcbe2e098 100644 --- a/drivers/cxl/core/regs.c +++ b/drivers/cxl/core/regs.c @@ -387,10 +387,9 @@ int cxl_count_regblock(struct pci_dev *pdev, enum cxl_regloc_type type) } EXPORT_SYMBOL_NS_GPL(cxl_count_regblock, CXL); -int cxl_map_pmu_regs(struct pci_dev *pdev, struct cxl_pmu_regs *regs, - struct cxl_register_map *map) +int cxl_map_pmu_regs(struct cxl_register_map *map, struct cxl_pmu_regs *regs) { - struct device *dev = &pdev->dev; + struct device *dev = map->host; resource_size_t phys_addr; phys_addr = map->resource; diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h index 9f62259177d36..1e7230529f30c 100644 --- a/drivers/cxl/cxl.h +++ b/drivers/cxl/cxl.h @@ -288,8 +288,7 @@ int cxl_map_component_regs(const struct cxl_register_map *map, unsigned long map_mask); int cxl_map_device_regs(const struct cxl_register_map *map, struct cxl_device_regs *regs); -int cxl_map_pmu_regs(struct pci_dev *pdev, struct cxl_pmu_regs *regs, - struct cxl_register_map *map); +int cxl_map_pmu_regs(struct cxl_register_map *map, struct cxl_pmu_regs *regs); enum cxl_regloc_type; int cxl_count_regblock(struct pci_dev *pdev, enum cxl_regloc_type type); diff --git a/drivers/cxl/pci.c b/drivers/cxl/pci.c index 67194d70c2dda..2d8ff9de000f3 100644 --- a/drivers/cxl/pci.c +++ b/drivers/cxl/pci.c @@ -895,7 +895,7 @@ static int cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) break; } - rc = cxl_map_pmu_regs(pdev, &pmu_regs, &map); + rc = cxl_map_pmu_regs(&map, &pmu_regs); if (rc) { dev_dbg(&pdev->dev, "Could not map PMU regs\n"); break; From 48841d6dd5ec18efbab8d031d1223d25225e2d30 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Mon, 16 Oct 2023 10:57:54 -0700 Subject: [PATCH 026/138] cxl: Add decoders_committed sysfs attribute to cxl_port commit 05e37b2138a6deb1f23daf1282dc86b29968a1ab upstream. This attribute allows cxl-cli to determine whether there are decoders committed to a memdev. This is only a snapshot of the state, and doesn't offer any protection or serialization against a concurrent disable-region operation. Reviewed-by: Jim Harris Suggested-by: Dan Williams Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/169747907439.272156.10261062080830155662.stgit@djiang5-mobl3 Signed-off-by: Dan Williams Signed-off-by: Yi Liu Signed-off-by: Bo Li --- Documentation/ABI/testing/sysfs-bus-cxl | 15 +++++++++++++++ drivers/cxl/core/port.c | 25 +++++++++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/Documentation/ABI/testing/sysfs-bus-cxl b/Documentation/ABI/testing/sysfs-bus-cxl index 087f762ebfd53..432610f41aeec 100644 --- a/Documentation/ABI/testing/sysfs-bus-cxl +++ b/Documentation/ABI/testing/sysfs-bus-cxl @@ -178,6 +178,21 @@ Description: hardware decoder target list. +What: /sys/bus/cxl/devices/portX/decoders_committed +Date: October, 2023 +KernelVersion: v6.7 +Contact: linux-cxl@vger.kernel.org +Description: + (RO) A memory device is considered active when any of its + decoders are in the "committed" state (See CXL 3.0 8.2.4.19.7 + CXL HDM Decoder n Control Register). Hotplug and destructive + operations like "sanitize" are blocked while device is actively + decoding a Host Physical Address range. Note that this number + may be elevated without any regionX objects active or even + enumerated, as this may be due to decoders established by + platform firwmare or a previous kernel (kexec). + + What: /sys/bus/cxl/devices/decoderX.Y Date: June, 2021 KernelVersion: v5.14 diff --git a/drivers/cxl/core/port.c b/drivers/cxl/core/port.c index ca69f0f9144ae..cf97c3d36acfb 100644 --- a/drivers/cxl/core/port.c +++ b/drivers/cxl/core/port.c @@ -530,8 +530,33 @@ static void cxl_port_release(struct device *dev) kfree(port); } +static ssize_t decoders_committed_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct cxl_port *port = to_cxl_port(dev); + int rc; + + down_read(&cxl_region_rwsem); + rc = sysfs_emit(buf, "%d\n", cxl_num_decoders_committed(port)); + up_read(&cxl_region_rwsem); + + return rc; +} + +static DEVICE_ATTR_RO(decoders_committed); + +static struct attribute *cxl_port_attrs[] = { + &dev_attr_decoders_committed.attr, + NULL, +}; + +static struct attribute_group cxl_port_attribute_group = { + .attrs = cxl_port_attrs, +}; + static const struct attribute_group *cxl_port_attribute_groups[] = { &cxl_base_attribute_group, + &cxl_port_attribute_group, NULL, }; From 01cc081dc360e02785e39f5140d19916e41694ef Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Thu, 12 Oct 2023 11:53:37 -0700 Subject: [PATCH 027/138] cxl: Export QTG ids from CFMWS to sysfs as qos_class attribute commit 529c0a44045e59c3c067f1f2c5887759644c50ae upstream. Export the QoS Throttling Group ID from the CXL Fixed Memory Window Structure (CFMWS) under the root decoder sysfs attributes as qos_class. CXL rev3.0 9.17.1.3 CXL Fixed Memory Window Structure (CFMWS) cxl cli will use this id to match with the _DSM retrieved id for a hot-plugged CXL memory device DPA memory range to make sure that the DPA range is under the right CFMWS window. Reviewed-by: Davidlohr Bueso Reviewed-by: Ira Weiny Reviewed-by: Jonathan Cameron Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/169713681699.2205276.14475306324720093079.stgit@djiang5-mobl3 Signed-off-by: Dan Williams Signed-off-by: Yi Liu Signed-off-by: Bo Li --- Documentation/ABI/testing/sysfs-bus-cxl | 15 +++++++++++++++ drivers/cxl/acpi.c | 3 +++ drivers/cxl/core/port.c | 11 +++++++++++ drivers/cxl/cxl.h | 3 +++ 4 files changed, 32 insertions(+) diff --git a/Documentation/ABI/testing/sysfs-bus-cxl b/Documentation/ABI/testing/sysfs-bus-cxl index 432610f41aeec..e76c3600607f8 100644 --- a/Documentation/ABI/testing/sysfs-bus-cxl +++ b/Documentation/ABI/testing/sysfs-bus-cxl @@ -384,6 +384,21 @@ Description: provided it is currently idle / not bound to a driver. +What: /sys/bus/cxl/devices/decoderX.Y/qos_class +Date: May, 2023 +KernelVersion: v6.5 +Contact: linux-cxl@vger.kernel.org +Description: + (RO) For CXL host platforms that support "QoS Telemmetry" this + root-decoder-only attribute conveys a platform specific cookie + that identifies a QoS performance class for the CXL Window. + This class-id can be compared against a similar "qos_class" + published for each memory-type that an endpoint supports. While + it is not required that endpoints map their local memory-class + to a matching platform class, mismatches are not recommended and + there are platform specific side-effects that may result. + + What: /sys/bus/cxl/devices/regionZ/uuid Date: May, 2022 KernelVersion: v6.0 diff --git a/drivers/cxl/acpi.c b/drivers/cxl/acpi.c index 9a881a764cf35..fd4ca778ee367 100644 --- a/drivers/cxl/acpi.c +++ b/drivers/cxl/acpi.c @@ -285,6 +285,9 @@ static int __cxl_parse_cfmws(struct acpi_cedt_cfmws *cfmws, } } } + + cxlrd->qos_class = cfmws->qtg_id; + rc = cxl_decoder_add(cxld, target_map); err_xormap: if (rc) diff --git a/drivers/cxl/core/port.c b/drivers/cxl/core/port.c index cf97c3d36acfb..9bc6f900ac063 100644 --- a/drivers/cxl/core/port.c +++ b/drivers/cxl/core/port.c @@ -287,6 +287,15 @@ static ssize_t interleave_ways_show(struct device *dev, static DEVICE_ATTR_RO(interleave_ways); +static ssize_t qos_class_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev); + + return sysfs_emit(buf, "%d\n", cxlrd->qos_class); +} +static DEVICE_ATTR_RO(qos_class); + static struct attribute *cxl_decoder_base_attrs[] = { &dev_attr_start.attr, &dev_attr_size.attr, @@ -306,6 +315,7 @@ static struct attribute *cxl_decoder_root_attrs[] = { &dev_attr_cap_type2.attr, &dev_attr_cap_type3.attr, &dev_attr_target_list.attr, + &dev_attr_qos_class.attr, SET_CXL_REGION_ATTR(create_pmem_region) SET_CXL_REGION_ATTR(create_ram_region) SET_CXL_REGION_ATTR(delete_region) @@ -1763,6 +1773,7 @@ struct cxl_root_decoder *cxl_root_decoder_alloc(struct cxl_port *port, } atomic_set(&cxlrd->region_id, rc); + cxlrd->qos_class = CXL_QOS_CLASS_INVALID; return cxlrd; } EXPORT_SYMBOL_NS_GPL(cxl_root_decoder_alloc, CXL); diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h index 1e7230529f30c..aecfb186ed09f 100644 --- a/drivers/cxl/cxl.h +++ b/drivers/cxl/cxl.h @@ -330,6 +330,7 @@ enum cxl_decoder_type { */ #define CXL_DECODER_MAX_INTERLEAVE 16 +#define CXL_QOS_CLASS_INVALID -1 /** * struct cxl_decoder - Common CXL HDM Decoder Attributes @@ -439,6 +440,7 @@ typedef struct cxl_dport *(*cxl_calc_hb_fn)(struct cxl_root_decoder *cxlrd, * @calc_hb: which host bridge covers the n'th position by granularity * @platform_data: platform specific configuration data * @range_lock: sync region autodiscovery by address range + * @qos_class: QoS performance class cookie * @cxlsd: base cxl switch decoder */ struct cxl_root_decoder { @@ -447,6 +449,7 @@ struct cxl_root_decoder { cxl_calc_hb_fn calc_hb; void *platform_data; struct mutex range_lock; + int qos_class; struct cxl_switch_decoder cxlsd; }; From 8bb91624722f79f45b0bf62fed2a66deb734c8e1 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Thu, 12 Oct 2023 11:53:42 -0700 Subject: [PATCH 028/138] cxl: Add checksum verification to CDAT from CXL commit 670e4e88f3b1a88a5a089be329b95c51592973ee upstream. A CDAT table is available from a CXL device. The table is read by the driver and cached in software. With the CXL subsystem needing to parse the CDAT table, the checksum should be verified. Add checksum verification after the CDAT table is read from device. Reviewed-by: Ira Weiny Reviewed-by: Jonathan Cameron Reviewed-by: Davidlohr Bueso Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/169713682277.2205276.2687265961314933628.stgit@djiang5-mobl3 Signed-off-by: Dan Williams Signed-off-by: Yi Liu Signed-off-by: Bo Li --- drivers/cxl/core/pci.c | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/drivers/cxl/core/pci.c b/drivers/cxl/core/pci.c index 764bc02e6c664..8f8c3520e0907 100644 --- a/drivers/cxl/core/pci.c +++ b/drivers/cxl/core/pci.c @@ -590,6 +590,16 @@ static int cxl_cdat_read_table(struct device *dev, return 0; } +static unsigned char cdat_checksum(void *buf, size_t size) +{ + unsigned char sum, *data = buf; + size_t i; + + for (sum = 0, i = 0; i < size; i++) + sum += data[i]; + return sum; +} + /** * read_cdat_data - Read the CDAT data on this port * @port: Port to read data from @@ -629,15 +639,21 @@ void read_cdat_data(struct cxl_port *port) return; rc = cxl_cdat_read_table(dev, cdat_doe, cdat_table, &cdat_length); - if (rc) { - /* Don't leave table data allocated on error */ - devm_kfree(dev, cdat_table); - dev_err(dev, "CDAT data read error\n"); - return; - } + if (rc) + goto err; - port->cdat.table = cdat_table + sizeof(__le32); + cdat_table = cdat_table + sizeof(__le32); + if (cdat_checksum(cdat_table, cdat_length)) + goto err; + + port->cdat.table = cdat_table; port->cdat.length = cdat_length; + return; + +err: + /* Don't leave table data allocated on error */ + devm_kfree(dev, cdat_table); + dev_err(dev, "Failed to read/validate CDAT.\n"); } EXPORT_SYMBOL_NS_GPL(read_cdat_data, CXL); From 93e18c5f61f09839622a0f2465325b160d3f8d37 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Thu, 12 Oct 2023 11:53:48 -0700 Subject: [PATCH 029/138] cxl: Add support for reading CXL switch CDAT table commit 8358e8f1596b0b23d3bbc4cf5df5e5e55afc0122 upstream. Add read_cdat_data() call in cxl_switch_port_probe() to allow reading of CDAT data for CXL switches. read_cdat_data() needs to be adjusted for the retrieving of the PCIe device depending on if the passed in port is endpoint or switch. Reviewed-by: Davidlohr Bueso Reviewed-by: Ira Weiny Reviewed-by: Jonathan Cameron Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/169713682855.2205276.6418370379144967443.stgit@djiang5-mobl3 Signed-off-by: Dan Williams Signed-off-by: Yi Liu Signed-off-by: Bo Li --- drivers/cxl/core/pci.c | 22 +++++++++++++++++----- drivers/cxl/port.c | 3 +++ 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/drivers/cxl/core/pci.c b/drivers/cxl/core/pci.c index 8f8c3520e0907..f20acbe211ee2 100644 --- a/drivers/cxl/core/pci.c +++ b/drivers/cxl/core/pci.c @@ -608,18 +608,30 @@ static unsigned char cdat_checksum(void *buf, size_t size) */ void read_cdat_data(struct cxl_port *port) { - struct cxl_memdev *cxlmd = to_cxl_memdev(port->uport_dev); - struct device *host = cxlmd->dev.parent; + struct device *uport = port->uport_dev; struct device *dev = &port->dev; struct pci_doe_mb *cdat_doe; + struct pci_dev *pdev = NULL; + struct cxl_memdev *cxlmd; size_t cdat_length; void *cdat_table; int rc; - if (!dev_is_pci(host)) + if (is_cxl_memdev(uport)) { + struct device *host; + + cxlmd = to_cxl_memdev(uport); + host = cxlmd->dev.parent; + if (dev_is_pci(host)) + pdev = to_pci_dev(host); + } else if (dev_is_pci(uport)) { + pdev = to_pci_dev(uport); + } + + if (!pdev) return; - cdat_doe = pci_find_doe_mailbox(to_pci_dev(host), - PCI_DVSEC_VENDOR_ID_CXL, + + cdat_doe = pci_find_doe_mailbox(pdev, PCI_DVSEC_VENDOR_ID_CXL, CXL_DOE_PROTOCOL_TABLE_ACCESS); if (!cdat_doe) { dev_dbg(dev, "No CDAT mailbox\n"); diff --git a/drivers/cxl/port.c b/drivers/cxl/port.c index 6240e05b95424..47bc8e0b85907 100644 --- a/drivers/cxl/port.c +++ b/drivers/cxl/port.c @@ -62,6 +62,9 @@ static int cxl_switch_port_probe(struct cxl_port *port) struct cxl_hdm *cxlhdm; int rc; + /* Cache the data early to ensure is_visible() works */ + read_cdat_data(port); + rc = devm_cxl_port_enumerate_dports(port); if (rc < 0) return rc; From 954ab2268af8130de1cabc0ffc10fc7b88a10389 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Thu, 12 Oct 2023 11:53:54 -0700 Subject: [PATCH 030/138] acpi: Move common tables helper functions to common lib commit a103f46633fdcddc2aaca506420f177e8803a2bd upstream. Some of the routines in ACPI driver/acpi/tables.c can be shared with parsing CDAT. CDAT is a device-provided data structure that is formatted similar to a platform provided ACPI table. CDAT is used by CXL and can exist on platforms that do not use ACPI. Split out the common routine from ACPI to accommodate platforms that do not support ACPI and move that to /lib. The common routines can be built outside of ACPI if FIRMWARE_TABLES is selected. Link: https://lore.kernel.org/linux-cxl/CAJZ5v0jipbtTNnsA0-o5ozOk8ZgWnOg34m34a9pPenTyRLj=6A@mail.gmail.com/ Suggested-by: "Rafael J. Wysocki" Reviewed-by: Hanjun Guo Reviewed-by: Jonathan Cameron Signed-off-by: Dave Jiang Acked-by: "Rafael J. Wysocki" Link: https://lore.kernel.org/r/169713683430.2205276.17899451119920103445.stgit@djiang5-mobl3 Signed-off-by: Dan Williams Signed-off-by: Yi Liu Signed-off-by: Bo Li --- MAINTAINERS | 2 + drivers/acpi/Kconfig | 1 + drivers/acpi/tables.c | 173 ----------------------------------- include/linux/acpi.h | 42 +++------ include/linux/fw_table.h | 43 +++++++++ lib/Kconfig | 3 + lib/Makefile | 2 + lib/fw_table.c | 189 +++++++++++++++++++++++++++++++++++++++ 8 files changed, 251 insertions(+), 204 deletions(-) create mode 100644 include/linux/fw_table.h create mode 100644 lib/fw_table.c diff --git a/MAINTAINERS b/MAINTAINERS index cf5fb6c0c34b3..eadcca709f12b 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -294,6 +294,8 @@ F: drivers/pnp/pnpacpi/ F: include/acpi/ F: include/linux/acpi.h F: include/linux/fwnode.h +F: include/linux/fw_table.h +F: lib/fw_table.c F: tools/power/acpi/ ACPI APEI diff --git a/drivers/acpi/Kconfig b/drivers/acpi/Kconfig index 648228831f5e8..9d5e09a79fbd4 100644 --- a/drivers/acpi/Kconfig +++ b/drivers/acpi/Kconfig @@ -12,6 +12,7 @@ menuconfig ACPI select PNP select NLS select CRC32 + select FIRMWARE_TABLE default y if X86 help Advanced Configuration and Power Interface (ACPI) support for diff --git a/drivers/acpi/tables.c b/drivers/acpi/tables.c index ffa1888844fe6..87557d5fa3aa3 100644 --- a/drivers/acpi/tables.c +++ b/drivers/acpi/tables.c @@ -37,18 +37,6 @@ static struct acpi_table_desc initial_tables[ACPI_MAX_TABLES] __initdata; static int acpi_apic_instance __initdata_or_acpilib; -enum acpi_subtable_type { - ACPI_SUBTABLE_COMMON, - ACPI_SUBTABLE_HMAT, - ACPI_SUBTABLE_PRMT, - ACPI_SUBTABLE_CEDT, -}; - -struct acpi_subtable_entry { - union acpi_subtable_headers *hdr; - enum acpi_subtable_type type; -}; - /* * Disable table checksum verification for the early stage due to the size * limitation of the current x86 early mapping implementation. @@ -251,167 +239,6 @@ void acpi_table_print_madt_entry(struct acpi_subtable_header *header) } } -static unsigned long __init_or_acpilib -acpi_get_entry_type(struct acpi_subtable_entry *entry) -{ - switch (entry->type) { - case ACPI_SUBTABLE_COMMON: - return entry->hdr->common.type; - case ACPI_SUBTABLE_HMAT: - return entry->hdr->hmat.type; - case ACPI_SUBTABLE_PRMT: - return 0; - case ACPI_SUBTABLE_CEDT: - return entry->hdr->cedt.type; - } - return 0; -} - -static unsigned long __init_or_acpilib -acpi_get_entry_length(struct acpi_subtable_entry *entry) -{ - switch (entry->type) { - case ACPI_SUBTABLE_COMMON: - return entry->hdr->common.length; - case ACPI_SUBTABLE_HMAT: - return entry->hdr->hmat.length; - case ACPI_SUBTABLE_PRMT: - return entry->hdr->prmt.length; - case ACPI_SUBTABLE_CEDT: - return entry->hdr->cedt.length; - } - return 0; -} - -static unsigned long __init_or_acpilib -acpi_get_subtable_header_length(struct acpi_subtable_entry *entry) -{ - switch (entry->type) { - case ACPI_SUBTABLE_COMMON: - return sizeof(entry->hdr->common); - case ACPI_SUBTABLE_HMAT: - return sizeof(entry->hdr->hmat); - case ACPI_SUBTABLE_PRMT: - return sizeof(entry->hdr->prmt); - case ACPI_SUBTABLE_CEDT: - return sizeof(entry->hdr->cedt); - } - return 0; -} - -static enum acpi_subtable_type __init_or_acpilib -acpi_get_subtable_type(char *id) -{ - if (strncmp(id, ACPI_SIG_HMAT, 4) == 0) - return ACPI_SUBTABLE_HMAT; - if (strncmp(id, ACPI_SIG_PRMT, 4) == 0) - return ACPI_SUBTABLE_PRMT; - if (strncmp(id, ACPI_SIG_CEDT, 4) == 0) - return ACPI_SUBTABLE_CEDT; - return ACPI_SUBTABLE_COMMON; -} - -static __init_or_acpilib bool has_handler(struct acpi_subtable_proc *proc) -{ - return proc->handler || proc->handler_arg; -} - -static __init_or_acpilib int call_handler(struct acpi_subtable_proc *proc, - union acpi_subtable_headers *hdr, - unsigned long end) -{ - if (proc->handler) - return proc->handler(hdr, end); - if (proc->handler_arg) - return proc->handler_arg(hdr, proc->arg, end); - return -EINVAL; -} - -/** - * acpi_parse_entries_array - for each proc_num find a suitable subtable - * - * @id: table id (for debugging purposes) - * @table_size: size of the root table - * @table_header: where does the table start? - * @proc: array of acpi_subtable_proc struct containing entry id - * and associated handler with it - * @proc_num: how big proc is? - * @max_entries: how many entries can we process? - * - * For each proc_num find a subtable with proc->id and run proc->handler - * on it. Assumption is that there's only single handler for particular - * entry id. - * - * The table_size is not the size of the complete ACPI table (the length - * field in the header struct), but only the size of the root table; i.e., - * the offset from the very first byte of the complete ACPI table, to the - * first byte of the very first subtable. - * - * On success returns sum of all matching entries for all proc handlers. - * Otherwise, -ENODEV or -EINVAL is returned. - */ -static int __init_or_acpilib acpi_parse_entries_array( - char *id, unsigned long table_size, - struct acpi_table_header *table_header, struct acpi_subtable_proc *proc, - int proc_num, unsigned int max_entries) -{ - struct acpi_subtable_entry entry; - unsigned long table_end, subtable_len, entry_len; - int count = 0; - int errs = 0; - int i; - - table_end = (unsigned long)table_header + table_header->length; - - /* Parse all entries looking for a match. */ - - entry.type = acpi_get_subtable_type(id); - entry.hdr = (union acpi_subtable_headers *) - ((unsigned long)table_header + table_size); - subtable_len = acpi_get_subtable_header_length(&entry); - - while (((unsigned long)entry.hdr) + subtable_len < table_end) { - if (max_entries && count >= max_entries) - break; - - for (i = 0; i < proc_num; i++) { - if (acpi_get_entry_type(&entry) != proc[i].id) - continue; - if (!has_handler(&proc[i]) || - (!errs && - call_handler(&proc[i], entry.hdr, table_end))) { - errs++; - continue; - } - - proc[i].count++; - break; - } - if (i != proc_num) - count++; - - /* - * If entry->length is 0, break from this loop to avoid - * infinite loop. - */ - entry_len = acpi_get_entry_length(&entry); - if (entry_len == 0) { - pr_err("[%4.4s:0x%02x] Invalid zero length\n", id, proc->id); - return -EINVAL; - } - - entry.hdr = (union acpi_subtable_headers *) - ((unsigned long)entry.hdr + entry_len); - } - - if (max_entries && count > max_entries) { - pr_warn("[%4.4s:0x%02x] found the maximum %i entries\n", - id, proc->id, count); - } - - return errs ? -EINVAL : count; -} - int __init_or_acpilib acpi_table_parse_entries_array( char *id, unsigned long table_size, struct acpi_subtable_proc *proc, int proc_num, unsigned int max_entries) diff --git a/include/linux/acpi.h b/include/linux/acpi.h index 7c6f4006389da..c9743462d4827 100644 --- a/include/linux/acpi.h +++ b/include/linux/acpi.h @@ -15,6 +15,7 @@ #include #include #include +#include struct irq_domain; struct irq_domain_ops; @@ -24,6 +25,16 @@ struct irq_domain_ops; #endif #include +#ifdef CONFIG_ACPI_TABLE_LIB +#define EXPORT_SYMBOL_ACPI_LIB(x) EXPORT_SYMBOL_NS_GPL(x, ACPI) +#define __init_or_acpilib +#define __initdata_or_acpilib +#else +#define EXPORT_SYMBOL_ACPI_LIB(x) +#define __init_or_acpilib __init +#define __initdata_or_acpilib __initdata +#endif + #ifdef CONFIG_ACPI #include @@ -119,21 +130,8 @@ enum acpi_address_range_id { /* Table Handlers */ -union acpi_subtable_headers { - struct acpi_subtable_header common; - struct acpi_hmat_structure hmat; - struct acpi_prmt_module_header prmt; - struct acpi_cedt_header cedt; -}; - typedef int (*acpi_tbl_table_handler)(struct acpi_table_header *table); -typedef int (*acpi_tbl_entry_handler)(union acpi_subtable_headers *header, - const unsigned long end); - -typedef int (*acpi_tbl_entry_handler_arg)(union acpi_subtable_headers *header, - void *arg, const unsigned long end); - /* Debugger support */ struct acpi_debugger_ops { @@ -207,14 +205,6 @@ static inline int acpi_debugger_notify_command_complete(void) (!entry) || (unsigned long)entry + sizeof(*entry) > end || \ ((struct acpi_subtable_header *)entry)->length < sizeof(*entry)) -struct acpi_subtable_proc { - int id; - acpi_tbl_entry_handler handler; - acpi_tbl_entry_handler_arg handler_arg; - void *arg; - int count; -}; - void __iomem *__acpi_map_table(unsigned long phys, unsigned long size); void __acpi_unmap_table(void __iomem *map, unsigned long size); int early_acpi_boot_init(void); @@ -229,16 +219,6 @@ void acpi_reserve_initial_tables (void); void acpi_table_init_complete (void); int acpi_table_init (void); -#ifdef CONFIG_ACPI_TABLE_LIB -#define EXPORT_SYMBOL_ACPI_LIB(x) EXPORT_SYMBOL_NS_GPL(x, ACPI) -#define __init_or_acpilib -#define __initdata_or_acpilib -#else -#define EXPORT_SYMBOL_ACPI_LIB(x) -#define __init_or_acpilib __init -#define __initdata_or_acpilib __initdata -#endif - int acpi_table_parse(char *id, acpi_tbl_table_handler handler); int __init_or_acpilib acpi_table_parse_entries(char *id, unsigned long table_size, int entry_id, diff --git a/include/linux/fw_table.h b/include/linux/fw_table.h new file mode 100644 index 0000000000000..ff8fa58d5818b --- /dev/null +++ b/include/linux/fw_table.h @@ -0,0 +1,43 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * fw_tables.h - Parsing support for ACPI and ACPI-like tables provided by + * platform or device firmware + * + * Copyright (C) 2001 Paul Diefenbaugh + * Copyright (C) 2023 Intel Corp. + */ +#ifndef _FW_TABLE_H_ +#define _FW_TABLE_H_ + +union acpi_subtable_headers; + +typedef int (*acpi_tbl_entry_handler)(union acpi_subtable_headers *header, + const unsigned long end); + +typedef int (*acpi_tbl_entry_handler_arg)(union acpi_subtable_headers *header, + void *arg, const unsigned long end); + +struct acpi_subtable_proc { + int id; + acpi_tbl_entry_handler handler; + acpi_tbl_entry_handler_arg handler_arg; + void *arg; + int count; +}; + +#include +#include + +union acpi_subtable_headers { + struct acpi_subtable_header common; + struct acpi_hmat_structure hmat; + struct acpi_prmt_module_header prmt; + struct acpi_cedt_header cedt; +}; + +int acpi_parse_entries_array(char *id, unsigned long table_size, + struct acpi_table_header *table_header, + struct acpi_subtable_proc *proc, + int proc_num, unsigned int max_entries); + +#endif diff --git a/lib/Kconfig b/lib/Kconfig index 43d69669465af..b27a54257d1bc 100644 --- a/lib/Kconfig +++ b/lib/Kconfig @@ -759,3 +759,6 @@ config ASN1_ENCODER config POLYNOMIAL tristate + +config FIRMWARE_TABLE + bool diff --git a/lib/Makefile b/lib/Makefile index b9d2577fbbe19..9b2d631aaf6fb 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -407,6 +407,8 @@ CFLAGS_longest_symbol_kunit.o += $(call cc-disable-warning, missing-prototypes) obj-$(CONFIG_GENERIC_LIB_DEVMEM_IS_ALLOWED) += devmem_is_allowed.o +obj-$(CONFIG_FIRMWARE_TABLE) += fw_table.o + # FORTIFY_SOURCE compile-time behavior tests TEST_FORTIFY_SRCS = $(wildcard $(srctree)/$(src)/test_fortify/*-*.c) TEST_FORTIFY_LOGS = $(patsubst $(srctree)/$(src)/%.c, %.log, $(TEST_FORTIFY_SRCS)) diff --git a/lib/fw_table.c b/lib/fw_table.c new file mode 100644 index 0000000000000..e84bd0866e104 --- /dev/null +++ b/lib/fw_table.c @@ -0,0 +1,189 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * fw_tables.c - Parsing support for ACPI and ACPI-like tables provided by + * platform or device firmware + * + * Copyright (C) 2001 Paul Diefenbaugh + * Copyright (C) 2023 Intel Corp. + */ +#include +#include +#include +#include +#include +#include + +enum acpi_subtable_type { + ACPI_SUBTABLE_COMMON, + ACPI_SUBTABLE_HMAT, + ACPI_SUBTABLE_PRMT, + ACPI_SUBTABLE_CEDT, +}; + +struct acpi_subtable_entry { + union acpi_subtable_headers *hdr; + enum acpi_subtable_type type; +}; + +static unsigned long __init_or_acpilib +acpi_get_entry_type(struct acpi_subtable_entry *entry) +{ + switch (entry->type) { + case ACPI_SUBTABLE_COMMON: + return entry->hdr->common.type; + case ACPI_SUBTABLE_HMAT: + return entry->hdr->hmat.type; + case ACPI_SUBTABLE_PRMT: + return 0; + case ACPI_SUBTABLE_CEDT: + return entry->hdr->cedt.type; + } + return 0; +} + +static unsigned long __init_or_acpilib +acpi_get_entry_length(struct acpi_subtable_entry *entry) +{ + switch (entry->type) { + case ACPI_SUBTABLE_COMMON: + return entry->hdr->common.length; + case ACPI_SUBTABLE_HMAT: + return entry->hdr->hmat.length; + case ACPI_SUBTABLE_PRMT: + return entry->hdr->prmt.length; + case ACPI_SUBTABLE_CEDT: + return entry->hdr->cedt.length; + } + return 0; +} + +static unsigned long __init_or_acpilib +acpi_get_subtable_header_length(struct acpi_subtable_entry *entry) +{ + switch (entry->type) { + case ACPI_SUBTABLE_COMMON: + return sizeof(entry->hdr->common); + case ACPI_SUBTABLE_HMAT: + return sizeof(entry->hdr->hmat); + case ACPI_SUBTABLE_PRMT: + return sizeof(entry->hdr->prmt); + case ACPI_SUBTABLE_CEDT: + return sizeof(entry->hdr->cedt); + } + return 0; +} + +static enum acpi_subtable_type __init_or_acpilib +acpi_get_subtable_type(char *id) +{ + if (strncmp(id, ACPI_SIG_HMAT, 4) == 0) + return ACPI_SUBTABLE_HMAT; + if (strncmp(id, ACPI_SIG_PRMT, 4) == 0) + return ACPI_SUBTABLE_PRMT; + if (strncmp(id, ACPI_SIG_CEDT, 4) == 0) + return ACPI_SUBTABLE_CEDT; + return ACPI_SUBTABLE_COMMON; +} + +static __init_or_acpilib bool has_handler(struct acpi_subtable_proc *proc) +{ + return proc->handler || proc->handler_arg; +} + +static __init_or_acpilib int call_handler(struct acpi_subtable_proc *proc, + union acpi_subtable_headers *hdr, + unsigned long end) +{ + if (proc->handler) + return proc->handler(hdr, end); + if (proc->handler_arg) + return proc->handler_arg(hdr, proc->arg, end); + return -EINVAL; +} + +/** + * acpi_parse_entries_array - for each proc_num find a suitable subtable + * + * @id: table id (for debugging purposes) + * @table_size: size of the root table + * @table_header: where does the table start? + * @proc: array of acpi_subtable_proc struct containing entry id + * and associated handler with it + * @proc_num: how big proc is? + * @max_entries: how many entries can we process? + * + * For each proc_num find a subtable with proc->id and run proc->handler + * on it. Assumption is that there's only single handler for particular + * entry id. + * + * The table_size is not the size of the complete ACPI table (the length + * field in the header struct), but only the size of the root table; i.e., + * the offset from the very first byte of the complete ACPI table, to the + * first byte of the very first subtable. + * + * On success returns sum of all matching entries for all proc handlers. + * Otherwise, -ENODEV or -EINVAL is returned. + */ +int __init_or_acpilib +acpi_parse_entries_array(char *id, unsigned long table_size, + struct acpi_table_header *table_header, + struct acpi_subtable_proc *proc, + int proc_num, unsigned int max_entries) +{ + unsigned long table_end, subtable_len, entry_len; + struct acpi_subtable_entry entry; + int count = 0; + int errs = 0; + int i; + + table_end = (unsigned long)table_header + table_header->length; + + /* Parse all entries looking for a match. */ + + entry.type = acpi_get_subtable_type(id); + entry.hdr = (union acpi_subtable_headers *) + ((unsigned long)table_header + table_size); + subtable_len = acpi_get_subtable_header_length(&entry); + + while (((unsigned long)entry.hdr) + subtable_len < table_end) { + if (max_entries && count >= max_entries) + break; + + for (i = 0; i < proc_num; i++) { + if (acpi_get_entry_type(&entry) != proc[i].id) + continue; + if (!has_handler(&proc[i]) || + (!errs && + call_handler(&proc[i], entry.hdr, table_end))) { + errs++; + continue; + } + + proc[i].count++; + break; + } + if (i != proc_num) + count++; + + /* + * If entry->length is 0, break from this loop to avoid + * infinite loop. + */ + entry_len = acpi_get_entry_length(&entry); + if (entry_len == 0) { + pr_err("[%4.4s:0x%02x] Invalid zero length\n", id, proc->id); + return -EINVAL; + } + + entry.hdr = (union acpi_subtable_headers *) + ((unsigned long)entry.hdr + entry_len); + } + + if (max_entries && count > max_entries) { + pr_warn("[%4.4s:0x%02x] found the maximum %i entries\n", + id, proc->id, count); + } + + return errs ? -EINVAL : count; +} +EXPORT_SYMBOL_GPL(acpi_parse_entries_array); From 05a81aeb338c0212940426b64c826aa10440ab98 Mon Sep 17 00:00:00 2001 From: Heiner Kallweit Date: Sun, 15 Oct 2023 23:34:25 +0200 Subject: [PATCH 031/138] ACPI: Add helper acpi_use_parent_companion commit 29166faac5486cbc34228431cd049e1c1be451ac upstream. In several drivers devices use the ACPI companion of the parent. Add a helper for this use case to avoid code duplication. Signed-off-by: Heiner Kallweit Acked-by: Rafael J. Wysocki Reviewed-by: Andi Shyti Signed-off-by: Wolfram Sang Signed-off-by: Yi Liu Signed-off-by: Bo Li --- include/linux/acpi.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/include/linux/acpi.h b/include/linux/acpi.h index c9743462d4827..dd1537115b57f 100644 --- a/include/linux/acpi.h +++ b/include/linux/acpi.h @@ -1535,4 +1535,9 @@ static inline void acpi_device_notify(struct device *dev) { } static inline void acpi_device_notify_remove(struct device *dev) { } #endif +static inline void acpi_use_parent_companion(struct device *dev) +{ + ACPI_COMPANION_SET(dev, ACPI_COMPANION(dev->parent)); +} + #endif /*_LINUX_ACPI_H*/ From 2d094696fdefcbbf5c34f2dbf304cc7e7cc518c0 Mon Sep 17 00:00:00 2001 From: Dan Carpenter Date: Tue, 31 Oct 2023 12:53:52 +0300 Subject: [PATCH 032/138] cxl/hdm: Fix && vs || bug commit 69d56b15a7941680aba8c3175b165221ecdf54b6 upstream. If "info" is NULL then this code will crash. || was intended instead of &&. Fixes: 8ce520fdea24 ("cxl/hdm: Use stored Component Register mappings to map HDM decoder capability") Signed-off-by: Dan Carpenter Reviewed-by: Robert Richter Link: https://lore.kernel.org/r/60028378-d3d5-4d6d-90fd-f915f061e731@moroto.mountain Signed-off-by: Dan Williams Signed-off-by: Yi Liu Signed-off-by: Bo Li --- drivers/cxl/core/hdm.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/cxl/core/hdm.c b/drivers/cxl/core/hdm.c index 7737fa7631d45..b653f570d0cce 100644 --- a/drivers/cxl/core/hdm.c +++ b/drivers/cxl/core/hdm.c @@ -159,7 +159,7 @@ struct cxl_hdm *devm_cxl_setup_hdm(struct cxl_port *port, /* Memory devices can configure device HDM using DVSEC range regs. */ if (reg_map->resource == CXL_RESOURCE_NONE) { - if (!info && !info->mem_enabled) { + if (!info || !info->mem_enabled) { dev_err(dev, "No component registers mapped\n"); return ERR_PTR(-ENXIO); } From a7ab948ebe87a9226c98d3a1a8190db4773820fc Mon Sep 17 00:00:00 2001 From: Terry Bowman Date: Thu, 2 Nov 2023 10:52:32 -0500 Subject: [PATCH 033/138] cxl/pci: Change CXL AER support check to use native AER commit b3741ac86c8e648709506102f7ab51905d50df43 upstream. Native CXL protocol errors are delivered to the OS through AER reporting. The owner of AER owns CXL Protocol error management with respect to _OSC negotiation.[1] CXL device errors are handled by a separate interrupt with native control gated by _OSC control field 'CXL Memory Error Reporting Control'. The CXL driver incorrectly checks for 'CXL Memory Error Reporting Control' before accessing AER registers and caching RCH downport AER registers. Replace the current check in these 2 cases with native AER checks. [1] CXL 3.0 - 9.17.2 CXL _OSC, Table-9-26, Interpretation of CXL _OSC Support Fields, p.641 Fixes: f05fd10d138d ("cxl/pci: Add RCH downstream port AER register discovery") Signed-off-by: Terry Bowman Reviewed-by: Smita Koralahalli Link: https://lore.kernel.org/r/20231102155232.1421261-1-terry.bowman@amd.com Signed-off-by: Dan Williams Signed-off-by: Yi Liu Signed-off-by: Bo Li --- drivers/cxl/core/pci.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/cxl/core/pci.c b/drivers/cxl/core/pci.c index f20acbe211ee2..f6c890bfe8422 100644 --- a/drivers/cxl/core/pci.c +++ b/drivers/cxl/core/pci.c @@ -804,7 +804,7 @@ static void cxl_disable_rch_root_ints(struct cxl_dport *dport) * the root cmd register's interrupts is required. But, PCI spec * shows these are disabled by default on reset. */ - if (bridge->native_cxl_error) { + if (bridge->native_aer) { aer_cmd_mask = (PCI_ERR_ROOT_CMD_COR_EN | PCI_ERR_ROOT_CMD_NONFATAL_EN | PCI_ERR_ROOT_CMD_FATAL_EN); @@ -820,7 +820,7 @@ void cxl_setup_parent_dport(struct device *host, struct cxl_dport *dport) struct pci_host_bridge *host_bridge; host_bridge = to_pci_host_bridge(dport_dev); - if (host_bridge->native_cxl_error) + if (host_bridge->native_aer) dport->rcrb.aer_cap = cxl_rcrb_to_aer(dport_dev, dport->rcrb.base); dport->reg_map.host = host; From 2bdd537d69e3df161a0c00314ff7bccb8cd44d73 Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Thu, 2 Nov 2023 15:07:02 -0700 Subject: [PATCH 034/138] lib/fw_table: Remove acpi_parse_entries_array() export commit 4b92894064b3df472b2cf5741c7f080e16dcd1ec upstream. Stephen reports that the ACPI helper library rework, CONFIG_FIRMWARE_TABLE, introduces a new compiler warning: WARNING: modpost: vmlinux: acpi_parse_entries_array: EXPORT_SYMBOL used for init symbol. Remove __init or EXPORT_SYMBOL. Delete this export as it turns out it is unneeded, and future work wraps this in another exported helper. Note that in general EXPORT_SYMBOL_ACPI_LIB() is needed for exporting symbols that are marked __init_or_acpilib, but in this case no export is required. Fixes: a103f46633fd ("acpi: Move common tables helper functions to common lib") Cc: Dave Jiang Reported-by: Stephen Rothwell Closes: http://lore.kernel.org/r/20231030160523.670a7569@canb.auug.org.au Reviewed-by: Dave Jiang Link: https://lore.kernel.org/r/169896282222.70775.940454758280866379.stgit@dwillia2-xfh.jf.intel.com Signed-off-by: Dan Williams Signed-off-by: Yi Liu Signed-off-by: Bo Li --- lib/fw_table.c | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/fw_table.c b/lib/fw_table.c index e84bd0866e104..b51f30a28e47b 100644 --- a/lib/fw_table.c +++ b/lib/fw_table.c @@ -186,4 +186,3 @@ acpi_parse_entries_array(char *id, unsigned long table_size, return errs ? -EINVAL : count; } -EXPORT_SYMBOL_GPL(acpi_parse_entries_array); From 0b555d31cd722f7c97418a5102041a81f6284f7a Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Wed, 22 Nov 2023 08:33:53 -0700 Subject: [PATCH 035/138] ACPI: Fix ARM32 platforms compile issue introduced by fw_table changes commit 35732699f5d2922ff674e711e566cf44a4bd86d2 upstream. Linus reported that: After commit a103f46633fd the kernel stopped compiling for several ARM32 platforms that I am building with a bare metal compiler. Bare metal compilers (arm-none-eabi-) don't define __linux__. This is because the header is now in the include path for : CC arch/arm/kernel/irq.o CC kernel/sysctl.o CC crypto/api.o In file included from ../include/acpi/acpi.h:22, from ../include/linux/fw_table.h:29, from ../include/linux/acpi.h:18, from ../include/linux/irqchip.h:14, from ../arch/arm/kernel/irq.c:25: ../include/acpi/platform/acenv.h:218:2: error: #error Unknown target environment 218 | #error Unknown target environment | ^~~~~ The issue is caused by the introducing of splitting out the ACPI code to support the new generic fw_table code. Rafael suggested [1] moving the fw_table.h include in linux/acpi.h to below the linux/mutex.h. Remove the two includes in fw_table.h. Replace linux/fw_table.h include in fw_table.c with linux/acpi.h. Link: https://lore.kernel.org/linux-acpi/CAJZ5v0idWdJq3JSqQWLG5q+b+b=zkEdWR55rGYEoxh7R6N8kFQ@mail.gmail.com/ Fixes: a103f46633fd ("acpi: Move common tables helper functions to common lib") Closes: https://lore.kernel.org/linux-acpi/20231114-arm-build-bug-v1-1-458745fe32a4@linaro.org/ Reported-by: Linus Walleij Suggested-by: Rafael J. Wysocki Tested-by: Linus Walleij Signed-off-by: Dave Jiang Acked-by: Rafael J. Wysocki Signed-off-by: Rafael J. Wysocki Signed-off-by: Yi Liu Signed-off-by: Bo Li --- include/linux/acpi.h | 22 +++++++++++----------- include/linux/fw_table.h | 3 --- lib/fw_table.c | 2 +- 3 files changed, 12 insertions(+), 15 deletions(-) diff --git a/include/linux/acpi.h b/include/linux/acpi.h index dd1537115b57f..305695ff21aa1 100644 --- a/include/linux/acpi.h +++ b/include/linux/acpi.h @@ -15,7 +15,6 @@ #include #include #include -#include struct irq_domain; struct irq_domain_ops; @@ -25,22 +24,13 @@ struct irq_domain_ops; #endif #include -#ifdef CONFIG_ACPI_TABLE_LIB -#define EXPORT_SYMBOL_ACPI_LIB(x) EXPORT_SYMBOL_NS_GPL(x, ACPI) -#define __init_or_acpilib -#define __initdata_or_acpilib -#else -#define EXPORT_SYMBOL_ACPI_LIB(x) -#define __init_or_acpilib __init -#define __initdata_or_acpilib __initdata -#endif - #ifdef CONFIG_ACPI #include #include #include #include +#include #include #include @@ -48,6 +38,16 @@ struct irq_domain_ops; #include #include +#ifdef CONFIG_ACPI_TABLE_LIB +#define EXPORT_SYMBOL_ACPI_LIB(x) EXPORT_SYMBOL_NS_GPL(x, ACPI) +#define __init_or_acpilib +#define __initdata_or_acpilib +#else +#define EXPORT_SYMBOL_ACPI_LIB(x) +#define __init_or_acpilib __init +#define __initdata_or_acpilib __initdata +#endif + static inline acpi_handle acpi_device_handle(struct acpi_device *adev) { return adev ? adev->handle : NULL; diff --git a/include/linux/fw_table.h b/include/linux/fw_table.h index ff8fa58d5818b..ca49947f0a775 100644 --- a/include/linux/fw_table.h +++ b/include/linux/fw_table.h @@ -25,9 +25,6 @@ struct acpi_subtable_proc { int count; }; -#include -#include - union acpi_subtable_headers { struct acpi_subtable_header common; struct acpi_hmat_structure hmat; diff --git a/lib/fw_table.c b/lib/fw_table.c index b51f30a28e47b..294df54e33b6f 100644 --- a/lib/fw_table.c +++ b/lib/fw_table.c @@ -7,7 +7,7 @@ * Copyright (C) 2023 Intel Corp. */ #include -#include +#include #include #include #include From dd0f4ee09a77a2677069b402a054081cb3ddec25 Mon Sep 17 00:00:00 2001 From: Davidlohr Bueso Date: Tue, 29 Aug 2023 08:20:14 -0700 Subject: [PATCH 036/138] cxl: Add Support for Get Timestamp commit cb46fca88d14939da2785567253d0a297f31be27 upstream. Add the call to the UAPI such that userspace may corelate the timestamps from the device log with system wall time, if, for example there's any sort of inaccuracy or skew in the device. Signed-off-by: Davidlohr Bueso Reviewed-by: Dave Jiang Reviewed-by: Jonathan Cameron Link: https://lore.kernel.org/r/20230829152014.15452-1-dave@stgolabs.net Signed-off-by: Dan Williams Signed-off-by: Yi Liu Signed-off-by: Bo Li --- drivers/cxl/core/mbox.c | 1 + drivers/cxl/cxlmem.h | 1 + include/uapi/linux/cxl_mem.h | 1 + 3 files changed, 3 insertions(+) diff --git a/drivers/cxl/core/mbox.c b/drivers/cxl/core/mbox.c index f1bcd9a439198..e05013e6bc069 100644 --- a/drivers/cxl/core/mbox.c +++ b/drivers/cxl/core/mbox.c @@ -63,6 +63,7 @@ static struct cxl_mem_command cxl_mem_commands[CXL_MEM_COMMAND_ID_MAX] = { CXL_CMD(GET_SHUTDOWN_STATE, 0, 0x1, 0), CXL_CMD(SET_SHUTDOWN_STATE, 0x1, 0, 0), CXL_CMD(GET_SCAN_MEDIA_CAPS, 0x10, 0x4, 0), + CXL_CMD(GET_TIMESTAMP, 0, 0x8, 0), }; /* diff --git a/drivers/cxl/cxlmem.h b/drivers/cxl/cxlmem.h index a72468912fc86..bf4ddf18730bb 100644 --- a/drivers/cxl/cxlmem.h +++ b/drivers/cxl/cxlmem.h @@ -503,6 +503,7 @@ enum cxl_opcode { CXL_MBOX_OP_GET_FW_INFO = 0x0200, CXL_MBOX_OP_TRANSFER_FW = 0x0201, CXL_MBOX_OP_ACTIVATE_FW = 0x0202, + CXL_MBOX_OP_GET_TIMESTAMP = 0x0300, CXL_MBOX_OP_SET_TIMESTAMP = 0x0301, CXL_MBOX_OP_GET_SUPPORTED_LOGS = 0x0400, CXL_MBOX_OP_GET_LOG = 0x0401, diff --git a/include/uapi/linux/cxl_mem.h b/include/uapi/linux/cxl_mem.h index 14bc6e7421483..42066f4eb8903 100644 --- a/include/uapi/linux/cxl_mem.h +++ b/include/uapi/linux/cxl_mem.h @@ -46,6 +46,7 @@ ___C(GET_SCAN_MEDIA_CAPS, "Get Scan Media Capabilities"), \ ___DEPRECATED(SCAN_MEDIA, "Scan Media"), \ ___DEPRECATED(GET_SCAN_MEDIA, "Get Scan Media Results"), \ + ___C(GET_TIMESTAMP, "Get Timestamp"), \ ___C(MAX, "invalid / last command") #define ___C(a, b) CXL_MEM_COMMAND_ID_##a From fcd642e3219e1bec5d56f7e5194b1f3cd404cd6a Mon Sep 17 00:00:00 2001 From: Ira Weiny Date: Thu, 16 Nov 2023 16:03:29 -0800 Subject: [PATCH 037/138] cxl/cdat: Free correct buffer on checksum error commit c65efe3685f5d150eeca5599afeabdc85da899d1 upstream. The new 6.7-rc1 kernel now checks the checksum on CDAT data. While using a branch of Fan's DCD qemu work (and specifying DCD devices), the following splat was observed. WARNING: CPU: 1 PID: 1384 at drivers/base/devres.c:1064 devm_kfree+0x4f/0x60 ... RIP: 0010:devm_kfree+0x4f/0x60 ... ? devm_kfree+0x4f/0x60 read_cdat_data+0x1a0/0x2a0 [cxl_core] cxl_port_probe+0xdf/0x200 [cxl_port] ... The issue in qemu is still unknown but the spat is a straight forward bug in the CDAT checksum processing code. Use a CDAT buffer variable to ensure the devm_free() works correctly on error. Fixes: 670e4e88f3b1 ("cxl: Add checksum verification to CDAT from CXL") Signed-off-by: Ira Weiny Reviewed-by: Dave Jiang Reviewed-by: Fan Ni Reviewed-by: Robert Richter Link: http://lore.kernel.org/r/20231116-fix-cdat-devm-free-v1-1-b148b40707d7@intel.com Signed-off-by: Dan Williams Signed-off-by: Yi Liu Signed-off-by: Bo Li --- drivers/cxl/core/pci.c | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/drivers/cxl/core/pci.c b/drivers/cxl/core/pci.c index f6c890bfe8422..a0d0e237911cc 100644 --- a/drivers/cxl/core/pci.c +++ b/drivers/cxl/core/pci.c @@ -614,7 +614,7 @@ void read_cdat_data(struct cxl_port *port) struct pci_dev *pdev = NULL; struct cxl_memdev *cxlmd; size_t cdat_length; - void *cdat_table; + void *cdat_table, *cdat_buf; int rc; if (is_cxl_memdev(uport)) { @@ -645,16 +645,15 @@ void read_cdat_data(struct cxl_port *port) return; } - cdat_table = devm_kzalloc(dev, cdat_length + sizeof(__le32), - GFP_KERNEL); - if (!cdat_table) + cdat_buf = devm_kzalloc(dev, cdat_length + sizeof(__le32), GFP_KERNEL); + if (!cdat_buf) return; - rc = cxl_cdat_read_table(dev, cdat_doe, cdat_table, &cdat_length); + rc = cxl_cdat_read_table(dev, cdat_doe, cdat_buf, &cdat_length); if (rc) goto err; - cdat_table = cdat_table + sizeof(__le32); + cdat_table = cdat_buf + sizeof(__le32); if (cdat_checksum(cdat_table, cdat_length)) goto err; @@ -664,7 +663,7 @@ void read_cdat_data(struct cxl_port *port) err: /* Don't leave table data allocated on error */ - devm_kfree(dev, cdat_table); + devm_kfree(dev, cdat_buf); dev_err(dev, "Failed to read/validate CDAT.\n"); } EXPORT_SYMBOL_NS_GPL(read_cdat_data, CXL); From 6ebc2f2ab6e22a1f88f57c147fead02c806da4db Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Thu, 14 Dec 2023 12:13:58 -0700 Subject: [PATCH 038/138] cxl: Fix unregister_region() callback parameter assignment commit ace196de694ebea5e2b3161e21ad169eb45accc6 upstream. In devm_cxl_add_region(), devm_add_action_or_reset() is called by passing in unregister_region() with data ptr of 'cxlr'. However, in unregister_region(), the passed in parameter is incorrectly assumed to be a 'struct device' rather than the 'cxlr' pointer. The code has been working because 'struct device' is the first member of 'struct cxl_region'. Issue found by inspection. Fix the assignment so that cxlr is pointing directly to the passed in parameter. Not flagged for -stable since there is no functional impact of this fix. Signed-off-by: Dave Jiang Reviewed-by: Ira Weiny Link: https://lore.kernel.org/r/170258123810.952211.3907381447996426480.stgit@djiang5-mobl3 Signed-off-by: Dan Williams Signed-off-by: Yi Liu Signed-off-by: Bo Li --- drivers/cxl/core/region.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/cxl/core/region.c b/drivers/cxl/core/region.c index 4f26e672ab3fe..ca8c818313f67 100644 --- a/drivers/cxl/core/region.c +++ b/drivers/cxl/core/region.c @@ -2194,13 +2194,13 @@ static struct cxl_region *to_cxl_region(struct device *dev) return container_of(dev, struct cxl_region, dev); } -static void unregister_region(void *dev) +static void unregister_region(void *_cxlr) { - struct cxl_region *cxlr = to_cxl_region(dev); + struct cxl_region *cxlr = _cxlr; struct cxl_region_params *p = &cxlr->params; int i; - device_del(dev); + device_del(&cxlr->dev); /* * Now that region sysfs is shutdown, the parameter block is now @@ -2211,7 +2211,7 @@ static void unregister_region(void *dev) detach_target(cxlr, i); cxl_region_iomem_release(cxlr); - put_device(dev); + put_device(&cxlr->dev); } static struct lock_class_key cxl_region_key; From ec69327d868515ae7c69aa8e2a957dfae017bb10 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Thu, 21 Dec 2023 15:02:31 -0700 Subject: [PATCH 039/138] lib/firmware_table: tables: Add CDAT table parsing support commit 60e43fe5285e2077ce9904d78cd42a230d03b788 upstream. The CDAT table is very similar to ACPI tables when it comes to sub-table and entry structures. The helper functions can be also used to parse the CDAT table. Add support to the helper functions to deal with an external CDAT table, and also handle the endieness since CDAT can be processed by a BE host. Export a function cdat_table_parse() for CXL driver to parse a CDAT table. In order to minimize ACPICA code changes, __force is being utilized to deal with the case of a big endian (BE) host parsing a CDAT. All CDAT data structure variables are being force casted to __leX as appropriate. Cc: Rafael J. Wysocki Cc: Len Brown Reviewed-by: Jonathan Cameron Signed-off-by: Dave Jiang Acked-by: Rafael J. Wysocki Link: https://lore.kernel.org/r/170319615131.2212653.10932785667981494238.stgit@djiang5-mobl3 Signed-off-by: Dan Williams Signed-off-by: Yi Liu Signed-off-by: Bo Li --- drivers/acpi/tables.c | 5 +-- include/linux/fw_table.h | 21 ++++++++++- lib/fw_table.c | 75 +++++++++++++++++++++++++++++++++------- 3 files changed, 86 insertions(+), 15 deletions(-) diff --git a/drivers/acpi/tables.c b/drivers/acpi/tables.c index 87557d5fa3aa3..c59a3617bca7f 100644 --- a/drivers/acpi/tables.c +++ b/drivers/acpi/tables.c @@ -265,8 +265,9 @@ int __init_or_acpilib acpi_table_parse_entries_array( return -ENODEV; } - count = acpi_parse_entries_array(id, table_size, table_header, - proc, proc_num, max_entries); + count = acpi_parse_entries_array(id, table_size, + (union fw_table_header *)table_header, + proc, proc_num, max_entries); acpi_put_table(table_header); return count; diff --git a/include/linux/fw_table.h b/include/linux/fw_table.h index ca49947f0a775..95421860397a2 100644 --- a/include/linux/fw_table.h +++ b/include/linux/fw_table.h @@ -25,16 +25,35 @@ struct acpi_subtable_proc { int count; }; +union fw_table_header { + struct acpi_table_header acpi; + struct acpi_table_cdat cdat; +}; + union acpi_subtable_headers { struct acpi_subtable_header common; struct acpi_hmat_structure hmat; struct acpi_prmt_module_header prmt; struct acpi_cedt_header cedt; + struct acpi_cdat_header cdat; }; int acpi_parse_entries_array(char *id, unsigned long table_size, - struct acpi_table_header *table_header, + union fw_table_header *table_header, struct acpi_subtable_proc *proc, int proc_num, unsigned int max_entries); +int cdat_table_parse(enum acpi_cdat_type type, + acpi_tbl_entry_handler_arg handler_arg, void *arg, + struct acpi_table_cdat *table_header); + +/* CXL is the only non-ACPI consumer of the FIRMWARE_TABLE library */ +#if IS_ENABLED(CONFIG_ACPI) && !IS_ENABLED(CONFIG_CXL_BUS) +#define EXPORT_SYMBOL_FWTBL_LIB(x) EXPORT_SYMBOL_ACPI_LIB(x) +#define __init_or_fwtbl_lib __init_or_acpilib +#else +#define EXPORT_SYMBOL_FWTBL_LIB(x) EXPORT_SYMBOL_NS_GPL(x, CXL) +#define __init_or_fwtbl_lib +#endif + #endif diff --git a/lib/fw_table.c b/lib/fw_table.c index 294df54e33b6f..1e5e0b2f70120 100644 --- a/lib/fw_table.c +++ b/lib/fw_table.c @@ -12,12 +12,14 @@ #include #include #include +#include enum acpi_subtable_type { ACPI_SUBTABLE_COMMON, ACPI_SUBTABLE_HMAT, ACPI_SUBTABLE_PRMT, ACPI_SUBTABLE_CEDT, + CDAT_SUBTABLE, }; struct acpi_subtable_entry { @@ -25,7 +27,7 @@ struct acpi_subtable_entry { enum acpi_subtable_type type; }; -static unsigned long __init_or_acpilib +static unsigned long __init_or_fwtbl_lib acpi_get_entry_type(struct acpi_subtable_entry *entry) { switch (entry->type) { @@ -37,11 +39,13 @@ acpi_get_entry_type(struct acpi_subtable_entry *entry) return 0; case ACPI_SUBTABLE_CEDT: return entry->hdr->cedt.type; + case CDAT_SUBTABLE: + return entry->hdr->cdat.type; } return 0; } -static unsigned long __init_or_acpilib +static unsigned long __init_or_fwtbl_lib acpi_get_entry_length(struct acpi_subtable_entry *entry) { switch (entry->type) { @@ -53,11 +57,16 @@ acpi_get_entry_length(struct acpi_subtable_entry *entry) return entry->hdr->prmt.length; case ACPI_SUBTABLE_CEDT: return entry->hdr->cedt.length; + case CDAT_SUBTABLE: { + __le16 length = (__force __le16)entry->hdr->cdat.length; + + return le16_to_cpu(length); + } } return 0; } -static unsigned long __init_or_acpilib +static unsigned long __init_or_fwtbl_lib acpi_get_subtable_header_length(struct acpi_subtable_entry *entry) { switch (entry->type) { @@ -69,11 +78,13 @@ acpi_get_subtable_header_length(struct acpi_subtable_entry *entry) return sizeof(entry->hdr->prmt); case ACPI_SUBTABLE_CEDT: return sizeof(entry->hdr->cedt); + case CDAT_SUBTABLE: + return sizeof(entry->hdr->cdat); } return 0; } -static enum acpi_subtable_type __init_or_acpilib +static enum acpi_subtable_type __init_or_fwtbl_lib acpi_get_subtable_type(char *id) { if (strncmp(id, ACPI_SIG_HMAT, 4) == 0) @@ -82,17 +93,32 @@ acpi_get_subtable_type(char *id) return ACPI_SUBTABLE_PRMT; if (strncmp(id, ACPI_SIG_CEDT, 4) == 0) return ACPI_SUBTABLE_CEDT; + if (strncmp(id, ACPI_SIG_CDAT, 4) == 0) + return CDAT_SUBTABLE; return ACPI_SUBTABLE_COMMON; } -static __init_or_acpilib bool has_handler(struct acpi_subtable_proc *proc) +static unsigned long __init_or_fwtbl_lib +acpi_table_get_length(enum acpi_subtable_type type, + union fw_table_header *header) +{ + if (type == CDAT_SUBTABLE) { + __le32 length = (__force __le32)header->cdat.length; + + return le32_to_cpu(length); + } + + return header->acpi.length; +} + +static __init_or_fwtbl_lib bool has_handler(struct acpi_subtable_proc *proc) { return proc->handler || proc->handler_arg; } -static __init_or_acpilib int call_handler(struct acpi_subtable_proc *proc, - union acpi_subtable_headers *hdr, - unsigned long end) +static __init_or_fwtbl_lib int call_handler(struct acpi_subtable_proc *proc, + union acpi_subtable_headers *hdr, + unsigned long end) { if (proc->handler) return proc->handler(hdr, end); @@ -124,23 +150,26 @@ static __init_or_acpilib int call_handler(struct acpi_subtable_proc *proc, * On success returns sum of all matching entries for all proc handlers. * Otherwise, -ENODEV or -EINVAL is returned. */ -int __init_or_acpilib +int __init_or_fwtbl_lib acpi_parse_entries_array(char *id, unsigned long table_size, - struct acpi_table_header *table_header, + union fw_table_header *table_header, struct acpi_subtable_proc *proc, int proc_num, unsigned int max_entries) { unsigned long table_end, subtable_len, entry_len; struct acpi_subtable_entry entry; + enum acpi_subtable_type type; int count = 0; int errs = 0; int i; - table_end = (unsigned long)table_header + table_header->length; + type = acpi_get_subtable_type(id); + table_end = (unsigned long)table_header + + acpi_table_get_length(type, table_header); /* Parse all entries looking for a match. */ - entry.type = acpi_get_subtable_type(id); + entry.type = type; entry.hdr = (union acpi_subtable_headers *) ((unsigned long)table_header + table_size); subtable_len = acpi_get_subtable_header_length(&entry); @@ -186,3 +215,25 @@ acpi_parse_entries_array(char *id, unsigned long table_size, return errs ? -EINVAL : count; } + +int __init_or_fwtbl_lib +cdat_table_parse(enum acpi_cdat_type type, + acpi_tbl_entry_handler_arg handler_arg, + void *arg, + struct acpi_table_cdat *table_header) +{ + struct acpi_subtable_proc proc = { + .id = type, + .handler_arg = handler_arg, + .arg = arg, + }; + + if (!table_header) + return -EINVAL; + + return acpi_parse_entries_array(ACPI_SIG_CDAT, + sizeof(struct acpi_table_cdat), + (union fw_table_header *)table_header, + &proc, 1, 0); +} +EXPORT_SYMBOL_FWTBL_LIB(cdat_table_parse); From 99f553ae1178b0d9dc44382d8931a2aaeb3449fa Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Thu, 21 Dec 2023 15:03:07 -0700 Subject: [PATCH 040/138] acpi: numa: Add helper function to retrieve the performance attributes commit ca53543d8e340070fb37fde93f36ed9012c76b90 upstream. Add helper to retrieve the performance attributes based on the device handle. The helper function is exported so the CXL driver can use that to acquire the performance data between the CPU and the CXL host bridge. Reviewed-by: Jonathan Cameron Signed-off-by: Dave Jiang Acked-by: Rafael J. Wysocki Link: https://lore.kernel.org/r/170319618721.2212653.5552947472849081786.stgit@djiang5-mobl3 Signed-off-by: Dan Williams Signed-off-by: Yi Liu Signed-off-by: Bo Li --- drivers/acpi/numa/hmat.c | 41 ++++++++++++++++++++++++++++++++++++++++ include/linux/acpi.h | 11 +++++++++++ 2 files changed, 52 insertions(+) diff --git a/drivers/acpi/numa/hmat.c b/drivers/acpi/numa/hmat.c index 1d6450c139225..0e2b5fede76b6 100644 --- a/drivers/acpi/numa/hmat.c +++ b/drivers/acpi/numa/hmat.c @@ -106,6 +106,47 @@ static struct memory_target *find_mem_target(unsigned int mem_pxm) return NULL; } +static struct memory_target *acpi_find_genport_target(u32 uid) +{ + struct memory_target *target; + u32 target_uid; + u8 *uid_ptr; + + list_for_each_entry(target, &targets, node) { + uid_ptr = target->gen_port_device_handle + 8; + target_uid = *(u32 *)uid_ptr; + if (uid == target_uid) + return target; + } + + return NULL; +} + +/** + * acpi_get_genport_coordinates - Retrieve the access coordinates for a generic port + * @uid: ACPI unique id + * @coord: The access coordinates written back out for the generic port + * + * Return: 0 on success. Errno on failure. + * + * Only supports device handles that are ACPI. Assume ACPI0016 HID for CXL. + */ +int acpi_get_genport_coordinates(u32 uid, + struct access_coordinate *coord) +{ + struct memory_target *target; + + guard(mutex)(&target_lock); + target = acpi_find_genport_target(uid); + if (!target) + return -ENOENT; + + *coord = target->coord[NODE_ACCESS_CLASS_GENPORT_SINK]; + + return 0; +} +EXPORT_SYMBOL_NS_GPL(acpi_get_genport_coordinates, CXL); + static __init void alloc_memory_initiator(unsigned int cpu_pxm) { struct memory_initiator *initiator; diff --git a/include/linux/acpi.h b/include/linux/acpi.h index 305695ff21aa1..3973069877123 100644 --- a/include/linux/acpi.h +++ b/include/linux/acpi.h @@ -15,6 +15,7 @@ #include #include #include +#include struct irq_domain; struct irq_domain_ops; @@ -419,6 +420,16 @@ extern int acpi_blacklisted(void); extern void acpi_osi_setup(char *str); extern bool acpi_osi_is_win8(void); +#ifdef CONFIG_ACPI_HMAT +int acpi_get_genport_coordinates(u32 uid, struct access_coordinate *coord); +#else +static inline int acpi_get_genport_coordinates(u32 uid, + struct access_coordinate *coord) +{ + return -EOPNOTSUPP; +} +#endif + #ifdef CONFIG_ACPI_NUMA int acpi_map_pxm_to_node(int pxm); int acpi_get_node(acpi_handle handle); From c1fa91f0de528ba32127df48ff31a77fef5cc089 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Thu, 21 Dec 2023 15:03:13 -0700 Subject: [PATCH 041/138] cxl: Add callback to parse the DSMAS subtables from CDAT commit ad6f04c0269b0b7908f09621d3b3c90def39a297 upstream. Provide a callback function to the CDAT parser in order to parse the Device Scoped Memory Affinity Structure (DSMAS). Each DSMAS structure contains the DPA range and its associated attributes in each entry. See the CDAT specification for details. The device handle and the DPA range is saved and to be associated with the DSLBIS locality data when the DSLBIS entries are parsed. The xarray is a local variable. When the total path performance data is calculated and storred this xarray can be discarded. Coherent Device Attribute Table 1.03 2.1 Device Scoped memory Affinity Structure (DSMAS) Reviewed-by: Jonathan Cameron Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/170319619355.2212653.2675953129671561293.stgit@djiang5-mobl3 Signed-off-by: Dan Williams Signed-off-by: Yi Liu Signed-off-by: Bo Li --- drivers/cxl/Kconfig | 3 ++ drivers/cxl/core/Makefile | 1 + drivers/cxl/core/cdat.c | 92 +++++++++++++++++++++++++++++++++++++++ drivers/cxl/cxl.h | 2 + drivers/cxl/port.c | 1 + tools/testing/cxl/Kbuild | 1 + 6 files changed, 100 insertions(+) create mode 100644 drivers/cxl/core/cdat.c diff --git a/drivers/cxl/Kconfig b/drivers/cxl/Kconfig index 8ea1d340e4385..67998dbd1d46b 100644 --- a/drivers/cxl/Kconfig +++ b/drivers/cxl/Kconfig @@ -5,6 +5,7 @@ menuconfig CXL_BUS select FW_LOADER select FW_UPLOAD select PCI_DOE + select FIRMWARE_TABLE help CXL is a bus that is electrically compatible with PCI Express, but layers three protocols on that signalling (CXL.io, CXL.cache, and @@ -54,8 +55,10 @@ config CXL_MEM_RAW_COMMANDS config CXL_ACPI tristate "CXL ACPI: Platform Support" depends on ACPI + depends on ACPI_NUMA default CXL_BUS select ACPI_TABLE_LIB + select ACPI_HMAT help Enable support for host managed device memory (HDM) resources published by a platform's ACPI CXL memory layout description. See diff --git a/drivers/cxl/core/Makefile b/drivers/cxl/core/Makefile index 1f66b5d4d9355..9259bcc6773c8 100644 --- a/drivers/cxl/core/Makefile +++ b/drivers/cxl/core/Makefile @@ -13,5 +13,6 @@ cxl_core-y += mbox.o cxl_core-y += pci.o cxl_core-y += hdm.o cxl_core-y += pmu.o +cxl_core-y += cdat.o cxl_core-$(CONFIG_TRACING) += trace.o cxl_core-$(CONFIG_CXL_REGION) += region.o diff --git a/drivers/cxl/core/cdat.c b/drivers/cxl/core/cdat.c new file mode 100644 index 0000000000000..9bf4f53bf77fe --- /dev/null +++ b/drivers/cxl/core/cdat.c @@ -0,0 +1,92 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright(c) 2023 Intel Corporation. All rights reserved. */ +#include +#include +#include +#include "cxlpci.h" +#include "cxl.h" + +struct dsmas_entry { + struct range dpa_range; + u8 handle; +}; + +static int cdat_dsmas_handler(union acpi_subtable_headers *header, void *arg, + const unsigned long end) +{ + struct acpi_cdat_header *hdr = &header->cdat; + struct acpi_cdat_dsmas *dsmas; + int size = sizeof(*hdr) + sizeof(*dsmas); + struct xarray *dsmas_xa = arg; + struct dsmas_entry *dent; + u16 len; + int rc; + + len = le16_to_cpu((__force __le16)hdr->length); + if (len != size || (unsigned long)hdr + len > end) { + pr_warn("Malformed DSMAS table length: (%u:%u)\n", size, len); + return -EINVAL; + } + + /* Skip common header */ + dsmas = (struct acpi_cdat_dsmas *)(hdr + 1); + + dent = kzalloc(sizeof(*dent), GFP_KERNEL); + if (!dent) + return -ENOMEM; + + dent->handle = dsmas->dsmad_handle; + dent->dpa_range.start = le64_to_cpu((__force __le64)dsmas->dpa_base_address); + dent->dpa_range.end = le64_to_cpu((__force __le64)dsmas->dpa_base_address) + + le64_to_cpu((__force __le64)dsmas->dpa_length) - 1; + + rc = xa_insert(dsmas_xa, dent->handle, dent, GFP_KERNEL); + if (rc) { + kfree(dent); + return rc; + } + + return 0; +} + +static int cxl_cdat_endpoint_process(struct cxl_port *port, + struct xarray *dsmas_xa) +{ + return cdat_table_parse(ACPI_CDAT_TYPE_DSMAS, cdat_dsmas_handler, + dsmas_xa, port->cdat.table); +} + +static void discard_dsmas(struct xarray *xa) +{ + unsigned long index; + void *ent; + + xa_for_each(xa, index, ent) { + xa_erase(xa, index); + kfree(ent); + } + xa_destroy(xa); +} +DEFINE_FREE(dsmas, struct xarray *, if (_T) discard_dsmas(_T)) + +void cxl_endpoint_parse_cdat(struct cxl_port *port) +{ + struct xarray __dsmas_xa; + struct xarray *dsmas_xa __free(dsmas) = &__dsmas_xa; + int rc; + + xa_init(&__dsmas_xa); + if (!port->cdat.table) + return; + + rc = cxl_cdat_endpoint_process(port, dsmas_xa); + if (rc < 0) { + dev_dbg(&port->dev, "Failed to parse CDAT: %d\n", rc); + return; + } + + /* Performance data processing */ +} +EXPORT_SYMBOL_NS_GPL(cxl_endpoint_parse_cdat, CXL); + +MODULE_IMPORT_NS(CXL); diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h index aecfb186ed09f..2c914c5ddbfa3 100644 --- a/drivers/cxl/cxl.h +++ b/drivers/cxl/cxl.h @@ -840,6 +840,8 @@ static inline struct cxl_dax_region *to_cxl_dax_region(struct device *dev) } #endif +void cxl_endpoint_parse_cdat(struct cxl_port *port); + /* * Unit test builds overrides this to __weak, find the 'strong' version * of these symbols in tools/testing/cxl/. diff --git a/drivers/cxl/port.c b/drivers/cxl/port.c index 47bc8e0b85907..a889c4e6cb272 100644 --- a/drivers/cxl/port.c +++ b/drivers/cxl/port.c @@ -109,6 +109,7 @@ static int cxl_endpoint_port_probe(struct cxl_port *port) /* Cache the data early to ensure is_visible() works */ read_cdat_data(port); + cxl_endpoint_parse_cdat(port); get_device(&cxlmd->dev); rc = devm_add_action_or_reset(&port->dev, schedule_detach, cxlmd); diff --git a/tools/testing/cxl/Kbuild b/tools/testing/cxl/Kbuild index 95dc58b94178b..0b12c36902d82 100644 --- a/tools/testing/cxl/Kbuild +++ b/tools/testing/cxl/Kbuild @@ -58,6 +58,7 @@ cxl_core-y += $(CXL_CORE_SRC)/mbox.o cxl_core-y += $(CXL_CORE_SRC)/pci.o cxl_core-y += $(CXL_CORE_SRC)/hdm.o cxl_core-y += $(CXL_CORE_SRC)/pmu.o +cxl_core-y += $(CXL_CORE_SRC)/cdat.o cxl_core-$(CONFIG_TRACING) += $(CXL_CORE_SRC)/trace.o cxl_core-$(CONFIG_CXL_REGION) += $(CXL_CORE_SRC)/region.o cxl_core-y += config_check.o From afceb9c696dba6aafaba6f7e6d5db85c74811f59 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Thu, 21 Dec 2023 15:03:20 -0700 Subject: [PATCH 042/138] cxl: Add callback to parse the DSLBIS subtable from CDAT commit 63cef81b9dca6ddf1c34d697016f830ddcfadf28 upstream. Provide a callback to parse the Device Scoped Latency and Bandwidth Information Structure (DSLBIS) in the CDAT structures. The DSLBIS contains the bandwidth and latency information that's tied to a DSMAS handle. The driver will retrieve the read and write latency and bandwidth associated with the DSMAS which is tied to a DPA range. Coherent Device Attribute Table 1.03 2.1 Device Scoped Latency and Bandwidth Information Structure (DSLBIS) Reviewed-by: Jonathan Cameron Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/170319620005.2212653.7475488478229720542.stgit@djiang5-mobl3 Signed-off-by: Dan Williams Signed-off-by: Yi Liu Signed-off-by: Bo Li --- drivers/cxl/core/cdat.c | 102 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 100 insertions(+), 2 deletions(-) diff --git a/drivers/cxl/core/cdat.c b/drivers/cxl/core/cdat.c index 9bf4f53bf77fe..97d8ef8848c66 100644 --- a/drivers/cxl/core/cdat.c +++ b/drivers/cxl/core/cdat.c @@ -3,12 +3,15 @@ #include #include #include +#include +#include #include "cxlpci.h" #include "cxl.h" struct dsmas_entry { struct range dpa_range; u8 handle; + struct access_coordinate coord; }; static int cdat_dsmas_handler(union acpi_subtable_headers *header, void *arg, @@ -49,11 +52,106 @@ static int cdat_dsmas_handler(union acpi_subtable_headers *header, void *arg, return 0; } +static void cxl_access_coordinate_set(struct access_coordinate *coord, + int access, unsigned int val) +{ + switch (access) { + case ACPI_HMAT_ACCESS_LATENCY: + coord->read_latency = val; + coord->write_latency = val; + break; + case ACPI_HMAT_READ_LATENCY: + coord->read_latency = val; + break; + case ACPI_HMAT_WRITE_LATENCY: + coord->write_latency = val; + break; + case ACPI_HMAT_ACCESS_BANDWIDTH: + coord->read_bandwidth = val; + coord->write_bandwidth = val; + break; + case ACPI_HMAT_READ_BANDWIDTH: + coord->read_bandwidth = val; + break; + case ACPI_HMAT_WRITE_BANDWIDTH: + coord->write_bandwidth = val; + break; + } +} + +static int cdat_dslbis_handler(union acpi_subtable_headers *header, void *arg, + const unsigned long end) +{ + struct acpi_cdat_header *hdr = &header->cdat; + struct acpi_cdat_dslbis *dslbis; + int size = sizeof(*hdr) + sizeof(*dslbis); + struct xarray *dsmas_xa = arg; + struct dsmas_entry *dent; + __le64 le_base; + __le16 le_val; + u64 val; + u16 len; + int rc; + + len = le16_to_cpu((__force __le16)hdr->length); + if (len != size || (unsigned long)hdr + len > end) { + pr_warn("Malformed DSLBIS table length: (%u:%u)\n", size, len); + return -EINVAL; + } + + /* Skip common header */ + dslbis = (struct acpi_cdat_dslbis *)(hdr + 1); + + /* Skip unrecognized data type */ + if (dslbis->data_type > ACPI_HMAT_WRITE_BANDWIDTH) + return 0; + + /* Not a memory type, skip */ + if ((dslbis->flags & ACPI_HMAT_MEMORY_HIERARCHY) != ACPI_HMAT_MEMORY) + return 0; + + dent = xa_load(dsmas_xa, dslbis->handle); + if (!dent) { + pr_warn("No matching DSMAS entry for DSLBIS entry.\n"); + return 0; + } + + le_base = (__force __le64)dslbis->entry_base_unit; + le_val = (__force __le16)dslbis->entry[0]; + rc = check_mul_overflow(le64_to_cpu(le_base), + le16_to_cpu(le_val), &val); + if (rc) + pr_warn("DSLBIS value overflowed.\n"); + + cxl_access_coordinate_set(&dent->coord, dslbis->data_type, val); + + return 0; +} + +static int cdat_table_parse_output(int rc) +{ + if (rc < 0) + return rc; + if (rc == 0) + return -ENOENT; + + return 0; +} + static int cxl_cdat_endpoint_process(struct cxl_port *port, struct xarray *dsmas_xa) { - return cdat_table_parse(ACPI_CDAT_TYPE_DSMAS, cdat_dsmas_handler, - dsmas_xa, port->cdat.table); + int rc; + + rc = cdat_table_parse(ACPI_CDAT_TYPE_DSMAS, cdat_dsmas_handler, + dsmas_xa, port->cdat.table); + rc = cdat_table_parse_output(rc); + if (rc) + return rc; + + rc = cdat_table_parse(ACPI_CDAT_TYPE_DSLBIS, cdat_dslbis_handler, + dsmas_xa, port->cdat.table); + return cdat_table_parse_output(rc); } static void discard_dsmas(struct xarray *xa) From 971cfba75bafd2ec46b85187bbc43cff95862480 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Thu, 21 Dec 2023 15:03:26 -0700 Subject: [PATCH 043/138] cxl: Add callback to parse the SSLBIS subtable from CDAT commit 80aa780dda20618be76162bf991d49cf962fda38 upstream. Provide a callback to parse the Switched Scoped Latency and Bandwidth Information Structure (SSLBIS) in the CDAT structures. The SSLBIS contains the bandwidth and latency information that's tied to the CXL switch that the data table has been read from. The extracted values are stored to the cxl_dport correlated by the port_id depending on the SSLBIS entry. Coherent Device Attribute Table 1.03 2.1 Switched Scoped Latency and Bandwidth Information Structure (DSLBIS) Reviewed-by: Jonathan Cameron Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/170319620635.2212653.5194389158785365150.stgit@djiang5-mobl3 Signed-off-by: Dan Williams Signed-off-by: Yi Liu Signed-off-by: Bo Li --- drivers/cxl/core/cdat.c | 98 +++++++++++++++++++++++++++++++++++++++++ drivers/cxl/cxl.h | 4 ++ drivers/cxl/port.c | 2 + 3 files changed, 104 insertions(+) diff --git a/drivers/cxl/core/cdat.c b/drivers/cxl/core/cdat.c index 97d8ef8848c66..b3ab47d250e16 100644 --- a/drivers/cxl/core/cdat.c +++ b/drivers/cxl/core/cdat.c @@ -187,4 +187,102 @@ void cxl_endpoint_parse_cdat(struct cxl_port *port) } EXPORT_SYMBOL_NS_GPL(cxl_endpoint_parse_cdat, CXL); +static int cdat_sslbis_handler(union acpi_subtable_headers *header, void *arg, + const unsigned long end) +{ + struct acpi_cdat_sslbis *sslbis; + int size = sizeof(header->cdat) + sizeof(*sslbis); + struct cxl_port *port = arg; + struct device *dev = &port->dev; + struct acpi_cdat_sslbe *entry; + int remain, entries, i; + u16 len; + + len = le16_to_cpu((__force __le16)header->cdat.length); + remain = len - size; + if (!remain || remain % sizeof(*entry) || + (unsigned long)header + len > end) { + dev_warn(dev, "Malformed SSLBIS table length: (%u)\n", len); + return -EINVAL; + } + + /* Skip common header */ + sslbis = (struct acpi_cdat_sslbis *)((unsigned long)header + + sizeof(header->cdat)); + + /* Unrecognized data type, we can skip */ + if (sslbis->data_type > ACPI_HMAT_WRITE_BANDWIDTH) + return 0; + + entries = remain / sizeof(*entry); + entry = (struct acpi_cdat_sslbe *)((unsigned long)header + sizeof(*sslbis)); + + for (i = 0; i < entries; i++) { + u16 x = le16_to_cpu((__force __le16)entry->portx_id); + u16 y = le16_to_cpu((__force __le16)entry->porty_id); + __le64 le_base; + __le16 le_val; + struct cxl_dport *dport; + unsigned long index; + u16 dsp_id; + u64 val; + + switch (x) { + case ACPI_CDAT_SSLBIS_US_PORT: + dsp_id = y; + break; + case ACPI_CDAT_SSLBIS_ANY_PORT: + switch (y) { + case ACPI_CDAT_SSLBIS_US_PORT: + dsp_id = x; + break; + case ACPI_CDAT_SSLBIS_ANY_PORT: + dsp_id = ACPI_CDAT_SSLBIS_ANY_PORT; + break; + default: + dsp_id = y; + break; + } + break; + default: + dsp_id = x; + break; + } + + le_base = (__force __le64)sslbis->entry_base_unit; + le_val = (__force __le16)entry->latency_or_bandwidth; + + if (check_mul_overflow(le64_to_cpu(le_base), + le16_to_cpu(le_val), &val)) + dev_warn(dev, "SSLBIS value overflowed!\n"); + + xa_for_each(&port->dports, index, dport) { + if (dsp_id == ACPI_CDAT_SSLBIS_ANY_PORT || + dsp_id == dport->port_id) + cxl_access_coordinate_set(&dport->sw_coord, + sslbis->data_type, + val); + } + + entry++; + } + + return 0; +} + +void cxl_switch_parse_cdat(struct cxl_port *port) +{ + int rc; + + if (!port->cdat.table) + return; + + rc = cdat_table_parse(ACPI_CDAT_TYPE_SSLBIS, cdat_sslbis_handler, + port, port->cdat.table); + rc = cdat_table_parse_output(rc); + if (rc) + dev_dbg(&port->dev, "Failed to parse SSLBIS: %d\n", rc); +} +EXPORT_SYMBOL_NS_GPL(cxl_switch_parse_cdat, CXL); + MODULE_IMPORT_NS(CXL); diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h index 2c914c5ddbfa3..b50c8acf26a53 100644 --- a/drivers/cxl/cxl.h +++ b/drivers/cxl/cxl.h @@ -8,6 +8,7 @@ #include #include #include +#include #include /** @@ -634,6 +635,7 @@ struct cxl_rcrb_info { * @rch: Indicate whether this dport was enumerated in RCH or VH mode * @port: reference to cxl_port that contains this downstream port * @regs: Dport parsed register blocks + * @sw_coord: access coordinates (performance) for switch from CDAT */ struct cxl_dport { struct device *dport_dev; @@ -643,6 +645,7 @@ struct cxl_dport { bool rch; struct cxl_port *port; struct cxl_regs regs; + struct access_coordinate sw_coord; }; /** @@ -841,6 +844,7 @@ static inline struct cxl_dax_region *to_cxl_dax_region(struct device *dev) #endif void cxl_endpoint_parse_cdat(struct cxl_port *port); +void cxl_switch_parse_cdat(struct cxl_port *port); /* * Unit test builds overrides this to __weak, find the 'strong' version diff --git a/drivers/cxl/port.c b/drivers/cxl/port.c index a889c4e6cb272..da3c3a08bd626 100644 --- a/drivers/cxl/port.c +++ b/drivers/cxl/port.c @@ -69,6 +69,8 @@ static int cxl_switch_port_probe(struct cxl_port *port) if (rc < 0) return rc; + cxl_switch_parse_cdat(port); + cxlhdm = devm_cxl_setup_hdm(port, NULL); if (!IS_ERR(cxlhdm)) return devm_cxl_enumerate_decoders(cxlhdm, NULL); From fa13f7d10647b2b7e29a1980c12e4dd38ab0444b Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Thu, 21 Dec 2023 15:03:32 -0700 Subject: [PATCH 044/138] cxl: Add support for _DSM Function for retrieving QTG ID commit 790815902ec61ba1715fd67d3cb9036e13c942bc upstream. CXL spec v3.0 9.17.3 CXL Root Device Specific Methods (_DSM) Add support to retrieve QTG ID via ACPI _DSM call. The _DSM call requires an input of an ACPI package with 4 dwords (read latency, write latency, read bandwidth, write bandwidth). The call returns a package with 1 WORD that provides the max supported QTG ID and a package that may contain 0 or more WORDs as the recommended QTG IDs in the recommended order. Create a cxl_root container for the root cxl_port and provide a callback ->get_qos_class() in order to retrieve the QoS class. For the ACPI case, the _DSM helper is used to retrieve the QTG ID and returned. A devm_cxl_add_root() function is added for root port setup and registration of the cxl_root callback operation(s). Signed-off-by: Dave Jiang Reviewed-by: Jonathan Cameron Link: https://lore.kernel.org/r/170319621294.2212653.1649682083061569256.stgit@djiang5-mobl3 Signed-off-by: Dan Williams Signed-off-by: Yi Liu Signed-off-by: Bo Li --- drivers/cxl/acpi.c | 132 +++++++++++++++++++++++++++++++++++++++- drivers/cxl/core/port.c | 49 ++++++++++++--- drivers/cxl/cxl.h | 25 ++++++++ 3 files changed, 193 insertions(+), 13 deletions(-) diff --git a/drivers/cxl/acpi.c b/drivers/cxl/acpi.c index fd4ca778ee367..dfbf471f3b5eb 100644 --- a/drivers/cxl/acpi.c +++ b/drivers/cxl/acpi.c @@ -6,6 +6,7 @@ #include #include #include +#include #include #include "cxlpci.h" #include "cxl.h" @@ -17,6 +18,10 @@ struct cxl_cxims_data { u64 xormaps[] __counted_by(nr_maps); }; +static const guid_t acpi_cxl_qtg_id_guid = + GUID_INIT(0xF365F9A6, 0xA7DE, 0x4071, + 0xA6, 0x6A, 0xB4, 0x0C, 0x0B, 0x4F, 0x8E, 0x52); + /* * Find a targets entry (n) in the host bridge interleave list. * CXL Specification 3.0 Table 9-22 @@ -194,6 +199,125 @@ struct cxl_cfmws_context { int id; }; +/** + * cxl_acpi_evaluate_qtg_dsm - Retrieve QTG ids via ACPI _DSM + * @handle: ACPI handle + * @coord: performance access coordinates + * @entries: number of QTG IDs to return + * @qos_class: int array provided by caller to return QTG IDs + * + * Return: number of QTG IDs returned, or -errno for errors + * + * Issue QTG _DSM with accompanied bandwidth and latency data in order to get + * the QTG IDs that are suitable for the performance point in order of most + * suitable to least suitable. Write back array of QTG IDs and return the + * actual number of QTG IDs written back. + */ +static int +cxl_acpi_evaluate_qtg_dsm(acpi_handle handle, struct access_coordinate *coord, + int entries, int *qos_class) +{ + union acpi_object *out_obj, *out_buf, *obj; + union acpi_object in_array[4] = { + [0].integer = { ACPI_TYPE_INTEGER, coord->read_latency }, + [1].integer = { ACPI_TYPE_INTEGER, coord->write_latency }, + [2].integer = { ACPI_TYPE_INTEGER, coord->read_bandwidth }, + [3].integer = { ACPI_TYPE_INTEGER, coord->write_bandwidth }, + }; + union acpi_object in_obj = { + .package = { + .type = ACPI_TYPE_PACKAGE, + .count = 4, + .elements = in_array, + }, + }; + int count, pkg_entries, i; + u16 max_qtg; + int rc; + + if (!entries) + return -EINVAL; + + out_obj = acpi_evaluate_dsm(handle, &acpi_cxl_qtg_id_guid, 1, 1, &in_obj); + if (!out_obj) + return -ENXIO; + + if (out_obj->type != ACPI_TYPE_PACKAGE) { + rc = -ENXIO; + goto out; + } + + /* Check Max QTG ID */ + obj = &out_obj->package.elements[0]; + if (obj->type != ACPI_TYPE_INTEGER) { + rc = -ENXIO; + goto out; + } + + max_qtg = obj->integer.value; + + /* It's legal to have 0 QTG entries */ + pkg_entries = out_obj->package.count; + if (pkg_entries <= 1) { + rc = 0; + goto out; + } + + /* Retrieve QTG IDs package */ + obj = &out_obj->package.elements[1]; + if (obj->type != ACPI_TYPE_PACKAGE) { + rc = -ENXIO; + goto out; + } + + pkg_entries = obj->package.count; + count = min(entries, pkg_entries); + for (i = 0; i < count; i++) { + u16 qtg_id; + + out_buf = &obj->package.elements[i]; + if (out_buf->type != ACPI_TYPE_INTEGER) { + rc = -ENXIO; + goto out; + } + + qtg_id = out_buf->integer.value; + if (qtg_id > max_qtg) + pr_warn("QTG ID %u greater than MAX %u\n", + qtg_id, max_qtg); + + qos_class[i] = qtg_id; + } + rc = count; + +out: + ACPI_FREE(out_obj); + return rc; +} + +static int cxl_acpi_qos_class(struct cxl_port *root_port, + struct access_coordinate *coord, int entries, + int *qos_class) +{ + acpi_handle handle; + struct device *dev; + + dev = root_port->uport_dev; + + if (!dev_is_platform(dev)) + return -ENODEV; + + handle = ACPI_HANDLE(dev); + if (!handle) + return -ENODEV; + + return cxl_acpi_evaluate_qtg_dsm(handle, coord, entries, qos_class); +} + +static const struct cxl_root_ops acpi_root_ops = { + .qos_class = cxl_acpi_qos_class, +}; + static int __cxl_parse_cfmws(struct acpi_cedt_cfmws *cfmws, struct cxl_cfmws_context *ctx) { @@ -666,6 +790,7 @@ static int cxl_acpi_probe(struct platform_device *pdev) { int rc; struct resource *cxl_res; + struct cxl_root *cxl_root; struct cxl_port *root_port; struct device *host = &pdev->dev; struct acpi_device *adev = ACPI_COMPANION(host); @@ -685,9 +810,10 @@ static int cxl_acpi_probe(struct platform_device *pdev) cxl_res->end = -1; cxl_res->flags = IORESOURCE_MEM; - root_port = devm_cxl_add_port(host, host, CXL_RESOURCE_NONE, NULL); - if (IS_ERR(root_port)) - return PTR_ERR(root_port); + cxl_root = devm_cxl_add_root(host, &acpi_root_ops); + if (IS_ERR(cxl_root)) + return PTR_ERR(cxl_root); + root_port = &cxl_root->port; rc = bus_for_each_dev(adev->dev.bus, NULL, root_port, add_host_bridge_dport); diff --git a/drivers/cxl/core/port.c b/drivers/cxl/core/port.c index 9bc6f900ac063..3d89ffae55154 100644 --- a/drivers/cxl/core/port.c +++ b/drivers/cxl/core/port.c @@ -537,7 +537,10 @@ static void cxl_port_release(struct device *dev) xa_destroy(&port->dports); xa_destroy(&port->regions); ida_free(&cxl_port_ida, port->id); - kfree(port); + if (is_cxl_root(port)) + kfree(to_cxl_root(port)); + else + kfree(port); } static ssize_t decoders_committed_show(struct device *dev, @@ -665,17 +668,31 @@ static struct lock_class_key cxl_port_key; static struct cxl_port *cxl_port_alloc(struct device *uport_dev, struct cxl_dport *parent_dport) { - struct cxl_port *port; + struct cxl_root *cxl_root __free(kfree) = NULL; + struct cxl_port *port, *_port __free(kfree) = NULL; struct device *dev; int rc; - port = kzalloc(sizeof(*port), GFP_KERNEL); - if (!port) - return ERR_PTR(-ENOMEM); + /* No parent_dport, root cxl_port */ + if (!parent_dport) { + cxl_root = kzalloc(sizeof(*cxl_root), GFP_KERNEL); + if (!cxl_root) + return ERR_PTR(-ENOMEM); + } else { + _port = kzalloc(sizeof(*port), GFP_KERNEL); + if (!_port) + return ERR_PTR(-ENOMEM); + } rc = ida_alloc(&cxl_port_ida, GFP_KERNEL); if (rc < 0) - goto err; + return ERR_PTR(rc); + + if (cxl_root) + port = &no_free_ptr(cxl_root)->port; + else + port = no_free_ptr(_port); + port->id = rc; port->uport_dev = uport_dev; @@ -727,10 +744,6 @@ static struct cxl_port *cxl_port_alloc(struct device *uport_dev, dev->type = &cxl_port_type; return port; - -err: - kfree(port); - return ERR_PTR(rc); } static int cxl_setup_comp_regs(struct device *host, struct cxl_register_map *map, @@ -880,6 +893,22 @@ struct cxl_port *devm_cxl_add_port(struct device *host, } EXPORT_SYMBOL_NS_GPL(devm_cxl_add_port, CXL); +struct cxl_root *devm_cxl_add_root(struct device *host, + const struct cxl_root_ops *ops) +{ + struct cxl_root *cxl_root; + struct cxl_port *port; + + port = devm_cxl_add_port(host, host, CXL_RESOURCE_NONE, NULL); + if (IS_ERR(port)) + return (struct cxl_root *)port; + + cxl_root = to_cxl_root(port); + cxl_root->ops = ops; + return cxl_root; +} +EXPORT_SYMBOL_NS_GPL(devm_cxl_add_root, CXL); + struct pci_bus *cxl_port_to_pci_bus(struct cxl_port *port) { /* There is no pci_bus associated with a CXL platform-root port */ diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h index b50c8acf26a53..895828f27984f 100644 --- a/drivers/cxl/cxl.h +++ b/drivers/cxl/cxl.h @@ -615,6 +615,29 @@ struct cxl_port { bool cdat_available; }; +struct cxl_root_ops { + int (*qos_class)(struct cxl_port *root_port, + struct access_coordinate *coord, int entries, + int *qos_class); +}; + +/** + * struct cxl_root - logical collection of root cxl_port items + * + * @port: cxl_port member + * @ops: cxl root operations + */ +struct cxl_root { + struct cxl_port port; + const struct cxl_root_ops *ops; +}; + +static inline struct cxl_root * +to_cxl_root(const struct cxl_port *port) +{ + return container_of(port, struct cxl_root, port); +} + static inline struct cxl_dport * cxl_find_dport_by_dev(struct cxl_port *port, const struct device *dport_dev) { @@ -704,6 +727,8 @@ struct cxl_port *devm_cxl_add_port(struct device *host, struct device *uport_dev, resource_size_t component_reg_phys, struct cxl_dport *parent_dport); +struct cxl_root *devm_cxl_add_root(struct device *host, + const struct cxl_root_ops *ops); struct cxl_port *find_cxl_root(struct cxl_port *port); int devm_cxl_enumerate_ports(struct cxl_memdev *cxlmd); void cxl_bus_rescan(void); From e994308fc745542c94e22f237577477d053769cb Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Thu, 21 Dec 2023 15:03:39 -0700 Subject: [PATCH 045/138] cxl: Calculate and store PCI link latency for the downstream ports commit 4d07a05397c8c15c37c8c3abb7afaea1dcd2f0e7 upstream. The latency is calculated by dividing the flit size over the bandwidth. Add support to retrieve the flit size for the CXL switch device and calculate the latency of the PCIe link. Cache the latency number with cxl_dport. Reviewed-by: Jonathan Cameron Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/170319621931.2212653.6800240203604822886.stgit@djiang5-mobl3 Signed-off-by: Dan Williams Signed-off-by: Yi Liu Signed-off-by: Bo Li --- drivers/cxl/core/core.h | 2 ++ drivers/cxl/core/pci.c | 36 ++++++++++++++++++++++++++++++++++++ drivers/cxl/core/port.c | 6 ++++++ drivers/cxl/cxl.h | 4 ++++ drivers/cxl/cxlpci.h | 13 +++++++++++++ drivers/pci/pci.c | 37 ++++++++++++++++++++++++++++++++++++- include/linux/pci.h | 1 + 7 files changed, 98 insertions(+), 1 deletion(-) diff --git a/drivers/cxl/core/core.h b/drivers/cxl/core/core.h index 713d8754cc7e9..76c2861e20027 100644 --- a/drivers/cxl/core/core.h +++ b/drivers/cxl/core/core.h @@ -95,4 +95,6 @@ enum cxl_poison_trace_type { CXL_POISON_TRACE_CLEAR, }; +long cxl_pci_get_latency(struct pci_dev *pdev); + #endif /* __CXL_CORE_H__ */ diff --git a/drivers/cxl/core/pci.c b/drivers/cxl/core/pci.c index a0d0e237911cc..d2803e114c5c4 100644 --- a/drivers/cxl/core/pci.c +++ b/drivers/cxl/core/pci.c @@ -1,5 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-only /* Copyright(c) 2021 Intel Corporation. All rights reserved. */ +#include #include #include #include @@ -973,3 +974,38 @@ pci_ers_result_t cxl_error_detected(struct pci_dev *pdev, return PCI_ERS_RESULT_NEED_RESET; } EXPORT_SYMBOL_NS_GPL(cxl_error_detected, CXL); + +static int cxl_flit_size(struct pci_dev *pdev) +{ + if (cxl_pci_flit_256(pdev)) + return 256; + + return 68; +} + +/** + * cxl_pci_get_latency - calculate the link latency for the PCIe link + * @pdev: PCI device + * + * return: calculated latency or 0 for no latency + * + * CXL Memory Device SW Guide v1.0 2.11.4 Link latency calculation + * Link latency = LinkPropagationLatency + FlitLatency + RetimerLatency + * LinkProgationLatency is negligible, so 0 will be used + * RetimerLatency is assumed to be negligible and 0 will be used + * FlitLatency = FlitSize / LinkBandwidth + * FlitSize is defined by spec. CXL rev3.0 4.2.1. + * 68B flit is used up to 32GT/s. >32GT/s, 256B flit size is used. + * The FlitLatency is converted to picoseconds. + */ +long cxl_pci_get_latency(struct pci_dev *pdev) +{ + long bw; + + bw = pcie_link_speed_mbps(pdev); + if (bw < 0) + return 0; + bw /= BITS_PER_BYTE; + + return cxl_flit_size(pdev) * MEGA / bw; +} diff --git a/drivers/cxl/core/port.c b/drivers/cxl/core/port.c index 3d89ffae55154..5a5637988020e 100644 --- a/drivers/cxl/core/port.c +++ b/drivers/cxl/core/port.c @@ -850,6 +850,9 @@ static struct cxl_port *__devm_cxl_add_port(struct device *host, if (rc) return ERR_PTR(rc); + if (parent_dport && dev_is_pci(uport_dev)) + port->pci_latency = cxl_pci_get_latency(to_pci_dev(uport_dev)); + return port; err: @@ -1133,6 +1136,9 @@ __devm_cxl_add_dport(struct cxl_port *port, struct device *dport_dev, if (rc) return ERR_PTR(rc); + if (dev_is_pci(dport_dev)) + dport->link_latency = cxl_pci_get_latency(to_pci_dev(dport_dev)); + return dport; } diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h index 895828f27984f..ebd96e24560d0 100644 --- a/drivers/cxl/cxl.h +++ b/drivers/cxl/cxl.h @@ -591,6 +591,7 @@ struct cxl_dax_region { * @depth: How deep this port is relative to the root. depth 0 is the root. * @cdat: Cached CDAT data * @cdat_available: Should a CDAT attribute be available in sysfs + * @pci_latency: Upstream latency in picoseconds */ struct cxl_port { struct device dev; @@ -613,6 +614,7 @@ struct cxl_port { size_t length; } cdat; bool cdat_available; + long pci_latency; }; struct cxl_root_ops { @@ -659,6 +661,7 @@ struct cxl_rcrb_info { * @port: reference to cxl_port that contains this downstream port * @regs: Dport parsed register blocks * @sw_coord: access coordinates (performance) for switch from CDAT + * @link_latency: calculated PCIe downstream latency */ struct cxl_dport { struct device *dport_dev; @@ -669,6 +672,7 @@ struct cxl_dport { struct cxl_port *port; struct cxl_regs regs; struct access_coordinate sw_coord; + long link_latency; }; /** diff --git a/drivers/cxl/cxlpci.h b/drivers/cxl/cxlpci.h index 0fa4799ea316c..711b05d9a370e 100644 --- a/drivers/cxl/cxlpci.h +++ b/drivers/cxl/cxlpci.h @@ -85,6 +85,19 @@ struct cdat_entry_header { __le16 length; } __packed; +/* + * CXL v3.0 6.2.3 Table 6-4 + * The table indicates that if PCIe Flit Mode is set, then CXL is in 256B flits + * mode, otherwise it's 68B flits mode. + */ +static inline bool cxl_pci_flit_256(struct pci_dev *pdev) +{ + u16 lnksta2; + + pcie_capability_read_word(pdev, PCI_EXP_LNKSTA2, &lnksta2); + return lnksta2 & PCI_EXP_LNKSTA2_FLIT; +} + int devm_cxl_port_enumerate_dports(struct cxl_port *port); struct cxl_dev_state; int cxl_hdm_decode_init(struct cxl_dev_state *cxlds, struct cxl_hdm *cxlhdm, diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c index 26af9e1470751..c9fab7aa428cd 100644 --- a/drivers/pci/pci.c +++ b/drivers/pci/pci.c @@ -6290,6 +6290,41 @@ int pcie_set_mps(struct pci_dev *dev, int mps) } EXPORT_SYMBOL(pcie_set_mps); +static enum pci_bus_speed to_pcie_link_speed(u16 lnksta) +{ + return pcie_link_speed[FIELD_GET(PCI_EXP_LNKSTA_CLS, lnksta)]; +} + +int pcie_link_speed_mbps(struct pci_dev *pdev) +{ + u16 lnksta; + int err; + + err = pcie_capability_read_word(pdev, PCI_EXP_LNKSTA, &lnksta); + if (err) + return err; + + switch (to_pcie_link_speed(lnksta)) { + case PCIE_SPEED_2_5GT: + return 2500; + case PCIE_SPEED_5_0GT: + return 5000; + case PCIE_SPEED_8_0GT: + return 8000; + case PCIE_SPEED_16_0GT: + return 16000; + case PCIE_SPEED_32_0GT: + return 32000; + case PCIE_SPEED_64_0GT: + return 64000; + default: + break; + } + + return -EINVAL; +} +EXPORT_SYMBOL(pcie_link_speed_mbps); + /** * pcie_bandwidth_available - determine minimum link settings of a PCIe * device and its bandwidth limitation @@ -6323,7 +6358,7 @@ u32 pcie_bandwidth_available(struct pci_dev *dev, struct pci_dev **limiting_dev, while (dev) { pcie_capability_read_word(dev, PCI_EXP_LNKSTA, &lnksta); - next_speed = pcie_link_speed[lnksta & PCI_EXP_LNKSTA_CLS]; + next_speed = to_pcie_link_speed(lnksta); next_width = FIELD_GET(PCI_EXP_LNKSTA_NLW, lnksta); next_bw = next_width * PCIE_SPEED2MBS_ENC(next_speed); diff --git a/include/linux/pci.h b/include/linux/pci.h index 2f48988278077..d23851dedd138 100644 --- a/include/linux/pci.h +++ b/include/linux/pci.h @@ -1367,6 +1367,7 @@ int pcie_set_mps(struct pci_dev *dev, int mps); u32 pcie_bandwidth_available(struct pci_dev *dev, struct pci_dev **limiting_dev, enum pci_bus_speed *speed, enum pcie_link_width *width); +int pcie_link_speed_mbps(struct pci_dev *pdev); void pcie_print_link_status(struct pci_dev *dev); int pcie_reset_flr(struct pci_dev *dev, bool probe); int pcie_flr(struct pci_dev *dev); From 8523840282bd57bc0c1c5b2920ca3cf65ffd538e Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Thu, 21 Dec 2023 15:03:45 -0700 Subject: [PATCH 046/138] tools/testing/cxl: Add hostbridge UID string for cxl_test mock hb devices commit f2202f990456acc52b50f6b14c95b232ac14429b upstream. In order to support acpi_device_uid() call, add static string to acpi_device->pnp.unique_id. Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/170319622564.2212653.1534465446670631698.stgit@djiang5-mobl3 Signed-off-by: Dan Williams Signed-off-by: Yi Liu Signed-off-by: Bo Li --- tools/testing/cxl/test/cxl.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tools/testing/cxl/test/cxl.c b/tools/testing/cxl/test/cxl.c index ed1361c8b96c4..606af1fe6a348 100644 --- a/tools/testing/cxl/test/cxl.c +++ b/tools/testing/cxl/test/cxl.c @@ -68,15 +68,19 @@ static struct acpi_device acpi0017_mock; static struct acpi_device host_bridge[NR_BRIDGES] = { [0] = { .handle = &host_bridge[0], + .pnp.unique_id = "0", }, [1] = { .handle = &host_bridge[1], + .pnp.unique_id = "1", }, [2] = { .handle = &host_bridge[2], + .pnp.unique_id = "2", }, [3] = { .handle = &host_bridge[3], + .pnp.unique_id = "3", }, }; From bbf55675b1d2036195358f5cb31b1343034736e6 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Thu, 21 Dec 2023 15:03:51 -0700 Subject: [PATCH 047/138] cxl: Store the access coordinates for the generic ports commit 1037b82fccfe9c001ffa7a883651bb4cde7b705c upstream. Each CXL host bridge is represented by an ACPI0016 device. A generic port device handle that is an ACPI device is represented by a string of ACPI0016 device HID and UID. Create a device handle from the ACPI device and retrieve the access coordinates from the stored memory targets. The access coordinates are stored under the cxl_dport that is associated with the CXL host bridge. The access coordinates struct is dynamically allocated under cxl_dport in order for code later on to detect whether the data exists or not. Reviewed-by: Jonathan Cameron Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/170319623196.2212653.17916695743464172534.stgit@djiang5-mobl3 Signed-off-by: Dan Williams Signed-off-by: Yi Liu Signed-off-by: Bo Li --- drivers/cxl/acpi.c | 25 +++++++++++++++++++++++++ drivers/cxl/cxl.h | 2 ++ 2 files changed, 27 insertions(+) diff --git a/drivers/cxl/acpi.c b/drivers/cxl/acpi.c index dfbf471f3b5eb..476b1674c940a 100644 --- a/drivers/cxl/acpi.c +++ b/drivers/cxl/acpi.c @@ -523,8 +523,29 @@ static int cxl_get_chbs(struct device *dev, struct acpi_device *hb, return 0; } +static int get_genport_coordinates(struct device *dev, struct cxl_dport *dport) +{ + struct acpi_device *hb = to_cxl_host_bridge(NULL, dev); + u32 uid; + int rc; + + if (kstrtou32(acpi_device_uid(hb), 0, &uid)) + return -EINVAL; + + rc = acpi_get_genport_coordinates(uid, &dport->hb_coord); + if (rc < 0) + return rc; + + /* Adjust back to picoseconds from nanoseconds */ + dport->hb_coord.read_latency *= 1000; + dport->hb_coord.write_latency *= 1000; + + return 0; +} + static int add_host_bridge_dport(struct device *match, void *arg) { + int ret; acpi_status rc; struct device *bridge; struct cxl_dport *dport; @@ -574,6 +595,10 @@ static int add_host_bridge_dport(struct device *match, void *arg) if (IS_ERR(dport)) return PTR_ERR(dport); + ret = get_genport_coordinates(match, dport); + if (ret) + dev_dbg(match, "Failed to get generic port perf coordinates.\n"); + return 0; } diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h index ebd96e24560d0..f77993fb478ac 100644 --- a/drivers/cxl/cxl.h +++ b/drivers/cxl/cxl.h @@ -661,6 +661,7 @@ struct cxl_rcrb_info { * @port: reference to cxl_port that contains this downstream port * @regs: Dport parsed register blocks * @sw_coord: access coordinates (performance) for switch from CDAT + * @hb_coord: access coordinates (performance) from ACPI generic port (host bridge) * @link_latency: calculated PCIe downstream latency */ struct cxl_dport { @@ -672,6 +673,7 @@ struct cxl_dport { struct cxl_port *port; struct cxl_regs regs; struct access_coordinate sw_coord; + struct access_coordinate hb_coord; long link_latency; }; From 4192c905264f7121db426d89da0a0e76fae8cd86 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Thu, 21 Dec 2023 15:03:58 -0700 Subject: [PATCH 048/138] cxl: Add helper function that calculate performance data for downstream ports commit 14a6960b3e928ccea22d687fb0626237885a20bd upstream. The CDAT information from the switch, Switch Scoped Latency and Bandwidth Information Structure (SSLBIS), is parsed and stored under a cxl_dport based on the correlated downstream port id from the SSLBIS entry. Walk the entire CXL port paths and collect all the performance data. Also pick up the link latency number that's stored under the dports. The entire path PCIe bandwidth can be retrieved using the pcie_bandwidth_available() call. Reviewed-by: Jonathan Cameron Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/170319623824.2212653.10302079766473698427.stgit@djiang5-mobl3 Signed-off-by: Dan Williams Signed-off-by: Yi Liu Signed-off-by: Bo Li --- drivers/cxl/core/port.c | 75 +++++++++++++++++++++++++++++++++++++++++ drivers/cxl/cxl.h | 3 ++ 2 files changed, 78 insertions(+) diff --git a/drivers/cxl/core/port.c b/drivers/cxl/core/port.c index 5a5637988020e..217c9d780493d 100644 --- a/drivers/cxl/core/port.c +++ b/drivers/cxl/core/port.c @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -2093,6 +2094,80 @@ bool schedule_cxl_memdev_detach(struct cxl_memdev *cxlmd) } EXPORT_SYMBOL_NS_GPL(schedule_cxl_memdev_detach, CXL); +static void combine_coordinates(struct access_coordinate *c1, + struct access_coordinate *c2) +{ + if (c2->write_bandwidth) + c1->write_bandwidth = min(c1->write_bandwidth, + c2->write_bandwidth); + c1->write_latency += c2->write_latency; + + if (c2->read_bandwidth) + c1->read_bandwidth = min(c1->read_bandwidth, + c2->read_bandwidth); + c1->read_latency += c2->read_latency; +} + +/** + * cxl_endpoint_get_perf_coordinates - Retrieve performance numbers stored in dports + * of CXL path + * @port: endpoint cxl_port + * @coord: output performance data + * + * Return: errno on failure, 0 on success. + */ +int cxl_endpoint_get_perf_coordinates(struct cxl_port *port, + struct access_coordinate *coord) +{ + struct access_coordinate c = { + .read_bandwidth = UINT_MAX, + .write_bandwidth = UINT_MAX, + }; + struct cxl_port *iter = port; + struct cxl_dport *dport; + struct pci_dev *pdev; + unsigned int bw; + + if (!is_cxl_endpoint(port)) + return -EINVAL; + + dport = iter->parent_dport; + + /* + * Exit the loop when the parent port of the current port is cxl root. + * The iterative loop starts at the endpoint and gathers the + * latency of the CXL link from the current iter to the next downstream + * port each iteration. If the parent is cxl root then there is + * nothing to gather. + */ + while (iter && !is_cxl_root(to_cxl_port(iter->dev.parent))) { + combine_coordinates(&c, &dport->sw_coord); + c.write_latency += dport->link_latency; + c.read_latency += dport->link_latency; + + iter = to_cxl_port(iter->dev.parent); + dport = iter->parent_dport; + } + + /* Augment with the generic port (host bridge) perf data */ + combine_coordinates(&c, &dport->hb_coord); + + /* Get the calculated PCI paths bandwidth */ + pdev = to_pci_dev(port->uport_dev->parent); + bw = pcie_bandwidth_available(pdev, NULL, NULL, NULL); + if (bw == 0) + return -ENXIO; + bw /= BITS_PER_BYTE; + + c.write_bandwidth = min(c.write_bandwidth, bw); + c.read_bandwidth = min(c.read_bandwidth, bw); + + *coord = c; + + return 0; +} +EXPORT_SYMBOL_NS_GPL(cxl_endpoint_get_perf_coordinates, CXL); + /* for user tooling to ensure port disable work has completed */ static ssize_t flush_store(const struct bus_type *bus, const char *buf, size_t count) { diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h index f77993fb478ac..bc25457644d7b 100644 --- a/drivers/cxl/cxl.h +++ b/drivers/cxl/cxl.h @@ -877,6 +877,9 @@ static inline struct cxl_dax_region *to_cxl_dax_region(struct device *dev) void cxl_endpoint_parse_cdat(struct cxl_port *port); void cxl_switch_parse_cdat(struct cxl_port *port); +int cxl_endpoint_get_perf_coordinates(struct cxl_port *port, + struct access_coordinate *coord); + /* * Unit test builds overrides this to __weak, find the 'strong' version * of these symbols in tools/testing/cxl/. From 8a10396fcf4fd2828bf84e6efa5ce0bfa8c9fa30 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Thu, 21 Dec 2023 15:04:04 -0700 Subject: [PATCH 049/138] cxl: Compute the entire CXL path latency and bandwidth data commit 7a4f148dd8d518bc1e012aa738b0ed6244959293 upstream. CXL Memory Device SW Guide [1] rev1.0 2.11.2 provides instruction on how to calculate latency and bandwidth for CXL memory device. Calculate minimum bandwidth and total latency for the path from the CXL device to the root port. The QTG id is retrieved by providing the performance data as input and calling the root port callback ->get_qos_class(). The retrieved id is stored with the cxl_port of the CXL device. For example for a device that is directly attached to a host bus: Total Latency = Device Latency (from CDAT) + Dev to Host Bus (HB) Link Latency + Generic Port Latency Min Bandwidth = Min bandwidth for link bandwidth between HB and CXL device, device CDAT bandwidth, and Generic Port Bandwidth For a device that has a switch in between host bus and CXL device: Total Latency = Device (CDAT) Latency + Dev to Switch Link Latency + Switch (CDAT) Latency + Switch to HB Link Latency + Generic Port Latency Min Bandwidth = Min bandwidth for link bandwidth between CXL device to CXL switch, CXL device CDAT bandwidth, CXL switch CDAT bandwidth, CXL switch to HB bandwidth, and Generic Port Bandwidth. [1]: https://cdrdv2-public.intel.com/643805/643805_CXL%20Memory%20Device%20SW%20Guide_Rev1p0.pdf Reviewed-by: Jonathan Cameron Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/170319624458.2212653.13252496567443656371.stgit@djiang5-mobl3 Signed-off-by: Dan Williams Signed-off-by: Yi Liu Signed-off-by: Bo Li --- drivers/cxl/core/cdat.c | 59 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 58 insertions(+), 1 deletion(-) diff --git a/drivers/cxl/core/cdat.c b/drivers/cxl/core/cdat.c index b3ab47d250e16..43dfef80fb847 100644 --- a/drivers/cxl/core/cdat.c +++ b/drivers/cxl/core/cdat.c @@ -12,6 +12,9 @@ struct dsmas_entry { struct range dpa_range; u8 handle; struct access_coordinate coord; + + int entries; + int qos_class; }; static int cdat_dsmas_handler(union acpi_subtable_headers *header, void *arg, @@ -154,6 +157,55 @@ static int cxl_cdat_endpoint_process(struct cxl_port *port, return cdat_table_parse_output(rc); } +static int cxl_port_perf_data_calculate(struct cxl_port *port, + struct xarray *dsmas_xa) +{ + struct access_coordinate c; + struct cxl_port *root_port; + struct cxl_root *cxl_root; + struct dsmas_entry *dent; + int valid_entries = 0; + unsigned long index; + int rc; + + rc = cxl_endpoint_get_perf_coordinates(port, &c); + if (rc) { + dev_dbg(&port->dev, "Failed to retrieve perf coordinates.\n"); + return rc; + } + + root_port = find_cxl_root(port); + cxl_root = to_cxl_root(root_port); + if (!cxl_root->ops || !cxl_root->ops->qos_class) + return -EOPNOTSUPP; + + xa_for_each(dsmas_xa, index, dent) { + int qos_class; + + dent->coord.read_latency = dent->coord.read_latency + + c.read_latency; + dent->coord.write_latency = dent->coord.write_latency + + c.write_latency; + dent->coord.read_bandwidth = min_t(int, c.read_bandwidth, + dent->coord.read_bandwidth); + dent->coord.write_bandwidth = min_t(int, c.write_bandwidth, + dent->coord.write_bandwidth); + + dent->entries = 1; + rc = cxl_root->ops->qos_class(root_port, &dent->coord, 1, &qos_class); + if (rc != 1) + continue; + + valid_entries++; + dent->qos_class = qos_class; + } + + if (!valid_entries) + return -ENOENT; + + return 0; +} + static void discard_dsmas(struct xarray *xa) { unsigned long index; @@ -183,7 +235,12 @@ void cxl_endpoint_parse_cdat(struct cxl_port *port) return; } - /* Performance data processing */ + rc = cxl_port_perf_data_calculate(port, dsmas_xa); + if (rc) { + dev_dbg(&port->dev, "Failed to do perf coord calculations.\n"); + return; + } + } EXPORT_SYMBOL_NS_GPL(cxl_endpoint_parse_cdat, CXL); From 2db53dbbfb8bb51b5bd6761bf4253acab03460ea Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Thu, 21 Dec 2023 15:04:11 -0700 Subject: [PATCH 050/138] cxl: Store QTG IDs and related info to the CXL memory device context commit 86557b7edf77d2a3835136c325c8baa6fe803234 upstream. Once the QTG ID _DSM is executed successfully, the QTG ID is retrieved from the return package. Create a list of entries in the cxl_memdev context and store the QTG ID as qos_class token and the associated DPA range. This information can be exposed to user space via sysfs in order to help region setup for hot-plugged CXL memory devices. Reviewed-by: Jonathan Cameron Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/170319625109.2212653.11872111896220384056.stgit@djiang5-mobl3 Signed-off-by: Dan Williams Signed-off-by: Yi Liu Signed-off-by: Bo Li --- drivers/cxl/core/cdat.c | 69 +++++++++++++++++++++++++++++++++++++++++ drivers/cxl/core/mbox.c | 2 ++ drivers/cxl/cxlmem.h | 21 +++++++++++++ 3 files changed, 92 insertions(+) diff --git a/drivers/cxl/core/cdat.c b/drivers/cxl/core/cdat.c index 43dfef80fb847..6189d967f399c 100644 --- a/drivers/cxl/core/cdat.c +++ b/drivers/cxl/core/cdat.c @@ -6,6 +6,7 @@ #include #include #include "cxlpci.h" +#include "cxlmem.h" #include "cxl.h" struct dsmas_entry { @@ -206,6 +207,71 @@ static int cxl_port_perf_data_calculate(struct cxl_port *port, return 0; } +static void add_perf_entry(struct device *dev, struct dsmas_entry *dent, + struct list_head *list) +{ + struct cxl_dpa_perf *dpa_perf; + + dpa_perf = kzalloc(sizeof(*dpa_perf), GFP_KERNEL); + if (!dpa_perf) + return; + + dpa_perf->dpa_range = dent->dpa_range; + dpa_perf->coord = dent->coord; + dpa_perf->qos_class = dent->qos_class; + list_add_tail(&dpa_perf->list, list); + dev_dbg(dev, + "DSMAS: dpa: %#llx qos: %d read_bw: %d write_bw %d read_lat: %d write_lat: %d\n", + dent->dpa_range.start, dpa_perf->qos_class, + dent->coord.read_bandwidth, dent->coord.write_bandwidth, + dent->coord.read_latency, dent->coord.write_latency); +} + +static void free_perf_ents(void *data) +{ + struct cxl_memdev_state *mds = data; + struct cxl_dpa_perf *dpa_perf, *n; + LIST_HEAD(discard); + + list_splice_tail_init(&mds->ram_perf_list, &discard); + list_splice_tail_init(&mds->pmem_perf_list, &discard); + list_for_each_entry_safe(dpa_perf, n, &discard, list) { + list_del(&dpa_perf->list); + kfree(dpa_perf); + } +} + +static void cxl_memdev_set_qos_class(struct cxl_dev_state *cxlds, + struct xarray *dsmas_xa) +{ + struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds); + struct device *dev = cxlds->dev; + struct range pmem_range = { + .start = cxlds->pmem_res.start, + .end = cxlds->pmem_res.end, + }; + struct range ram_range = { + .start = cxlds->ram_res.start, + .end = cxlds->ram_res.end, + }; + struct dsmas_entry *dent; + unsigned long index; + + xa_for_each(dsmas_xa, index, dent) { + if (resource_size(&cxlds->ram_res) && + range_contains(&ram_range, &dent->dpa_range)) + add_perf_entry(dev, dent, &mds->ram_perf_list); + else if (resource_size(&cxlds->pmem_res) && + range_contains(&pmem_range, &dent->dpa_range)) + add_perf_entry(dev, dent, &mds->pmem_perf_list); + else + dev_dbg(dev, "no partition for dsmas dpa: %#llx\n", + dent->dpa_range.start); + } + + devm_add_action_or_reset(&cxlds->cxlmd->dev, free_perf_ents, mds); +} + static void discard_dsmas(struct xarray *xa) { unsigned long index; @@ -221,6 +287,8 @@ DEFINE_FREE(dsmas, struct xarray *, if (_T) discard_dsmas(_T)) void cxl_endpoint_parse_cdat(struct cxl_port *port) { + struct cxl_memdev *cxlmd = to_cxl_memdev(port->uport_dev); + struct cxl_dev_state *cxlds = cxlmd->cxlds; struct xarray __dsmas_xa; struct xarray *dsmas_xa __free(dsmas) = &__dsmas_xa; int rc; @@ -241,6 +309,7 @@ void cxl_endpoint_parse_cdat(struct cxl_port *port) return; } + cxl_memdev_set_qos_class(cxlds, dsmas_xa); } EXPORT_SYMBOL_NS_GPL(cxl_endpoint_parse_cdat, CXL); diff --git a/drivers/cxl/core/mbox.c b/drivers/cxl/core/mbox.c index e05013e6bc069..2fb3378de8674 100644 --- a/drivers/cxl/core/mbox.c +++ b/drivers/cxl/core/mbox.c @@ -1402,6 +1402,8 @@ struct cxl_memdev_state *cxl_memdev_state_create(struct device *dev) mds->cxlds.reg_map.host = dev; mds->cxlds.reg_map.resource = CXL_RESOURCE_NONE; mds->cxlds.type = CXL_DEVTYPE_CLASSMEM; + INIT_LIST_HEAD(&mds->ram_perf_list); + INIT_LIST_HEAD(&mds->pmem_perf_list); return mds; } diff --git a/drivers/cxl/cxlmem.h b/drivers/cxl/cxlmem.h index bf4ddf18730bb..f62cdc25099d1 100644 --- a/drivers/cxl/cxlmem.h +++ b/drivers/cxl/cxlmem.h @@ -6,6 +6,7 @@ #include #include #include +#include #include "cxl.h" /* CXL 2.0 8.2.8.5.1.1 Memory Device Status Register */ @@ -391,6 +392,20 @@ enum cxl_devtype { CXL_DEVTYPE_CLASSMEM, }; +/** + * struct cxl_dpa_perf - DPA performance property entry + * @list - list entry + * @dpa_range - range for DPA address + * @coord - QoS performance data (i.e. latency, bandwidth) + * @qos_class - QoS Class cookies + */ +struct cxl_dpa_perf { + struct list_head list; + struct range dpa_range; + struct access_coordinate coord; + int qos_class; +}; + /** * struct cxl_dev_state - The driver device state * @@ -455,6 +470,8 @@ struct cxl_dev_state { * @security: security driver state info * @fw: firmware upload / activation state * @mbox_send: @dev specific transport for transmitting mailbox commands + * @ram_perf_list: performance data entries matched to RAM + * @pmem_perf_list: performance data entries matched to PMEM * * See CXL 3.0 8.2.9.8.2 Capacity Configuration and Label Storage for * details on capacity parameters. @@ -475,6 +492,10 @@ struct cxl_memdev_state { u64 active_persistent_bytes; u64 next_volatile_bytes; u64 next_persistent_bytes; + + struct list_head ram_perf_list; + struct list_head pmem_perf_list; + struct cxl_event_state event; struct cxl_poison_state poison; struct cxl_security_state security; From b49d3d7551c40a0a0197491a4c3bf26fc74e33bf Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Thu, 21 Dec 2023 15:04:16 -0700 Subject: [PATCH 051/138] cxl: Export sysfs attributes for memory device QoS class commit 42834b17cf1f00fa79ff1f02134f9c576a125252 upstream. Export qos_class sysfs attributes for the CXL memory device. The QoS clas should show up as /sys/bus/cxl/devices/memX/ram/qos_class for the volatile partition and /sys/bus/cxl/devices/memX/pmem/qos_class for the persistent partition. The QTG ID is retrieved via _DSM after supplying the calculated bandwidth and latency for the entire CXL path from device to the CPU. This ID is used to match up to the root decoder QoS class to determine which CFMWS the memory range of a hotplugged CXL mem device should be assigned under. While there may be multiple DSMAS exported by the device CDAT, the driver will only expose the first QTG ID per partition in sysfs for now. In the future when multiple QTG IDs are necessary, they can be exposed. [1] [1]: https://lore.kernel.org/linux-cxl/167571650007.587790.10040913293130712882.stgit@djiang5-mobl3.local/T/#md2a47b1ead3e1ba08f50eab29a4af1aed1d215ab Suggested-by: Dan Williams Reviewed-by: Jonathan Cameron Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/170319625698.2212653.17544381274847420961.stgit@djiang5-mobl3 Signed-off-by: Dan Williams Signed-off-by: Yi Liu Signed-off-by: Bo Li --- Documentation/ABI/testing/sysfs-bus-cxl | 34 +++++++++++++ drivers/cxl/mem.c | 67 ++++++++++++++++++++++--- 2 files changed, 95 insertions(+), 6 deletions(-) diff --git a/Documentation/ABI/testing/sysfs-bus-cxl b/Documentation/ABI/testing/sysfs-bus-cxl index e76c3600607f8..fff2581b80335 100644 --- a/Documentation/ABI/testing/sysfs-bus-cxl +++ b/Documentation/ABI/testing/sysfs-bus-cxl @@ -28,6 +28,23 @@ Description: Payload in the CXL-2.0 specification. +What: /sys/bus/cxl/devices/memX/ram/qos_class +Date: May, 2023 +KernelVersion: v6.8 +Contact: linux-cxl@vger.kernel.org +Description: + (RO) For CXL host platforms that support "QoS Telemmetry" + this attribute conveys a comma delimited list of platform + specific cookies that identifies a QoS performance class + for the volatile partition of the CXL mem device. These + class-ids can be compared against a similar "qos_class" + published for a root decoder. While it is not required + that the endpoints map their local memory-class to a + matching platform class, mismatches are not recommended + and there are platform specific performance related + side-effects that may result. First class-id is displayed. + + What: /sys/bus/cxl/devices/memX/pmem/size Date: December, 2020 KernelVersion: v5.12 @@ -38,6 +55,23 @@ Description: Payload in the CXL-2.0 specification. +What: /sys/bus/cxl/devices/memX/pmem/qos_class +Date: May, 2023 +KernelVersion: v6.8 +Contact: linux-cxl@vger.kernel.org +Description: + (RO) For CXL host platforms that support "QoS Telemmetry" + this attribute conveys a comma delimited list of platform + specific cookies that identifies a QoS performance class + for the persistent partition of the CXL mem device. These + class-ids can be compared against a similar "qos_class" + published for a root decoder. While it is not required + that the endpoints map their local memory-class to a + matching platform class, mismatches are not recommended + and there are platform specific performance related + side-effects that may result. First class-id is displayed. + + What: /sys/bus/cxl/devices/memX/serial Date: January, 2022 KernelVersion: v5.18 diff --git a/drivers/cxl/mem.c b/drivers/cxl/mem.c index e087febf9af04..c5c9d8e0d88d6 100644 --- a/drivers/cxl/mem.c +++ b/drivers/cxl/mem.c @@ -215,23 +215,78 @@ static ssize_t trigger_poison_list_store(struct device *dev, } static DEVICE_ATTR_WO(trigger_poison_list); +static ssize_t ram_qos_class_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct cxl_memdev *cxlmd = to_cxl_memdev(dev); + struct cxl_dev_state *cxlds = cxlmd->cxlds; + struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds); + struct cxl_dpa_perf *dpa_perf; + + if (!dev->driver) + return -ENOENT; + + if (list_empty(&mds->ram_perf_list)) + return -ENOENT; + + dpa_perf = list_first_entry(&mds->ram_perf_list, struct cxl_dpa_perf, + list); + + return sysfs_emit(buf, "%d\n", dpa_perf->qos_class); +} + +static struct device_attribute dev_attr_ram_qos_class = + __ATTR(qos_class, 0444, ram_qos_class_show, NULL); + +static ssize_t pmem_qos_class_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct cxl_memdev *cxlmd = to_cxl_memdev(dev); + struct cxl_dev_state *cxlds = cxlmd->cxlds; + struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds); + struct cxl_dpa_perf *dpa_perf; + + if (!dev->driver) + return -ENOENT; + + if (list_empty(&mds->pmem_perf_list)) + return -ENOENT; + + dpa_perf = list_first_entry(&mds->pmem_perf_list, struct cxl_dpa_perf, + list); + + return sysfs_emit(buf, "%d\n", dpa_perf->qos_class); +} + +static struct device_attribute dev_attr_pmem_qos_class = + __ATTR(qos_class, 0444, pmem_qos_class_show, NULL); + static umode_t cxl_mem_visible(struct kobject *kobj, struct attribute *a, int n) { - if (a == &dev_attr_trigger_poison_list.attr) { - struct device *dev = kobj_to_dev(kobj); - struct cxl_memdev *cxlmd = to_cxl_memdev(dev); - struct cxl_memdev_state *mds = - to_cxl_memdev_state(cxlmd->cxlds); + struct device *dev = kobj_to_dev(kobj); + struct cxl_memdev *cxlmd = to_cxl_memdev(dev); + struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds); + if (a == &dev_attr_trigger_poison_list.attr) if (!test_bit(CXL_POISON_ENABLED_LIST, mds->poison.enabled_cmds)) return 0; - } + + if (a == &dev_attr_pmem_qos_class.attr) + if (list_empty(&mds->pmem_perf_list)) + return 0; + + if (a == &dev_attr_ram_qos_class.attr) + if (list_empty(&mds->ram_perf_list)) + return 0; + return a->mode; } static struct attribute *cxl_mem_attrs[] = { &dev_attr_trigger_poison_list.attr, + &dev_attr_ram_qos_class.attr, + &dev_attr_pmem_qos_class.attr, NULL }; From 661f190726c88b0c06a5216e9ad8f3bece43e66a Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Thu, 21 Dec 2023 15:04:23 -0700 Subject: [PATCH 052/138] cxl: Check qos_class validity on memdev probe commit 185c1a489f873cb71520fc089401e02dbf302dcd upstream. Add a check to make sure the qos_class for the device will match one of the root decoders qos_class. If no match is found, then the qos_class for the device is set to invalid. Also add a check to ensure that the device's host bridge matches to one of the root decoder's downstream targets. Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/170319626313.2212653.9021004640856081917.stgit@djiang5-mobl3 Signed-off-by: Dan Williams Signed-off-by: Yi Liu Signed-off-by: Bo Li --- drivers/cxl/core/cdat.c | 101 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) diff --git a/drivers/cxl/core/cdat.c b/drivers/cxl/core/cdat.c index 6189d967f399c..10ac76bed4772 100644 --- a/drivers/cxl/core/cdat.c +++ b/drivers/cxl/core/cdat.c @@ -8,6 +8,7 @@ #include "cxlpci.h" #include "cxlmem.h" #include "cxl.h" +#include "core.h" struct dsmas_entry { struct range dpa_range; @@ -272,6 +273,105 @@ static void cxl_memdev_set_qos_class(struct cxl_dev_state *cxlds, devm_add_action_or_reset(&cxlds->cxlmd->dev, free_perf_ents, mds); } +static int match_cxlrd_qos_class(struct device *dev, void *data) +{ + int dev_qos_class = *(int *)data; + struct cxl_root_decoder *cxlrd; + + if (!is_root_decoder(dev)) + return 0; + + cxlrd = to_cxl_root_decoder(dev); + if (cxlrd->qos_class == CXL_QOS_CLASS_INVALID) + return 0; + + if (cxlrd->qos_class == dev_qos_class) + return 1; + + return 0; +} + +static void cxl_qos_match(struct cxl_port *root_port, + struct list_head *work_list, + struct list_head *discard_list) +{ + struct cxl_dpa_perf *dpa_perf, *n; + + list_for_each_entry_safe(dpa_perf, n, work_list, list) { + int rc; + + if (dpa_perf->qos_class == CXL_QOS_CLASS_INVALID) + return; + + rc = device_for_each_child(&root_port->dev, + (void *)&dpa_perf->qos_class, + match_cxlrd_qos_class); + if (!rc) + list_move_tail(&dpa_perf->list, discard_list); + } +} + +static int match_cxlrd_hb(struct device *dev, void *data) +{ + struct device *host_bridge = data; + struct cxl_switch_decoder *cxlsd; + struct cxl_root_decoder *cxlrd; + + if (!is_root_decoder(dev)) + return 0; + + cxlrd = to_cxl_root_decoder(dev); + cxlsd = &cxlrd->cxlsd; + + guard(rwsem_read)(&cxl_region_rwsem); + for (int i = 0; i < cxlsd->nr_targets; i++) { + if (host_bridge == cxlsd->target[i]->dport_dev) + return 1; + } + + return 0; +} + +static void discard_dpa_perf(struct list_head *list) +{ + struct cxl_dpa_perf *dpa_perf, *n; + + list_for_each_entry_safe(dpa_perf, n, list, list) { + list_del(&dpa_perf->list); + kfree(dpa_perf); + } +} +DEFINE_FREE(dpa_perf, struct list_head *, if (!list_empty(_T)) discard_dpa_perf(_T)) + +static int cxl_qos_class_verify(struct cxl_memdev *cxlmd) +{ + struct cxl_dev_state *cxlds = cxlmd->cxlds; + struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds); + struct cxl_port *root_port __free(put_device) = NULL; + LIST_HEAD(__discard); + struct list_head *discard __free(dpa_perf) = &__discard; + int rc; + + root_port = find_cxl_root(cxlmd->endpoint); + if (!root_port) + return -ENODEV; + + /* Check that the QTG IDs are all sane between end device and root decoders */ + cxl_qos_match(root_port, &mds->ram_perf_list, discard); + cxl_qos_match(root_port, &mds->pmem_perf_list, discard); + + /* Check to make sure that the device's host bridge is under a root decoder */ + rc = device_for_each_child(&root_port->dev, + (void *)cxlmd->endpoint->host_bridge, + match_cxlrd_hb); + if (!rc) { + list_splice_tail_init(&mds->ram_perf_list, discard); + list_splice_tail_init(&mds->pmem_perf_list, discard); + } + + return rc; +} + static void discard_dsmas(struct xarray *xa) { unsigned long index; @@ -310,6 +410,7 @@ void cxl_endpoint_parse_cdat(struct cxl_port *port) } cxl_memdev_set_qos_class(cxlds, dsmas_xa); + cxl_qos_class_verify(cxlmd); } EXPORT_SYMBOL_NS_GPL(cxl_endpoint_parse_cdat, CXL); From 688060e263ef726e8d054e9135892f33f9c7142a Mon Sep 17 00:00:00 2001 From: Alison Schofield Date: Fri, 22 Dec 2023 16:47:40 -0800 Subject: [PATCH 053/138] cxl/region: Add dev_dbg() detail on failure to allocate HPA space commit 7984d22f1315bf30433e11e5010e4ce09ca22037 upstream. When the region driver fails while allocating HPA space for a new region it can be because the parent resource, the CXL Window, has no more available space. In that case, the debug user sees this message: cxl_core:alloc_hpa:555: cxl region2: failed to allocate HPA: -34 Expand the message like this: cxl_core:alloc_hpa:555: cxl region8: HPA allocation error (-34) for size:0x20000000 in CXL Window 0 [mem 0xf010000000-0xf04fffffff flags 0x200] Now the debug user can examine /proc/iomem and consider actions like removing other allocations in that space or reducing the size of their region request. Suggested-by: Dan Williams Signed-off-by: Alison Schofield Reviewed-by: Dave Jiang Reviewed-by: Vishal Verma Reviewed-by: Fan Ni Link: https://lore.kernel.org/r/20231223004740.1401858-1-alison.schofield@intel.com Signed-off-by: Dan Williams Signed-off-by: Yi Liu Signed-off-by: Bo Li --- drivers/cxl/core/region.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/drivers/cxl/core/region.c b/drivers/cxl/core/region.c index ca8c818313f67..774e198bca2b6 100644 --- a/drivers/cxl/core/region.c +++ b/drivers/cxl/core/region.c @@ -536,8 +536,9 @@ static int alloc_hpa(struct cxl_region *cxlr, resource_size_t size) res = alloc_free_mem_region(cxlrd->res, size, SZ_256M, dev_name(&cxlr->dev)); if (IS_ERR(res)) { - dev_dbg(&cxlr->dev, "failed to allocate HPA: %ld\n", - PTR_ERR(res)); + dev_dbg(&cxlr->dev, + "HPA allocation error (%ld) for size:%#llx in %s %pr\n", + PTR_ERR(res), size, cxlrd->res->name, cxlrd->res); return PTR_ERR(res); } From 83093810c60488f41dc6ffd32669b823052da3d5 Mon Sep 17 00:00:00 2001 From: Randy Dunlap Date: Tue, 2 Jan 2024 09:39:17 -0800 Subject: [PATCH 054/138] cxl/region: use %pap format to print resource_size_t commit 58f1e9d3a30438042fc9ed65b3dc56b2e5f7886a upstream. Use "%pap" to print a resource_size_t (phys_addr_t derived type) to prevent build warnings on 32-bit arches (seen on i386 and riscv-32). ../drivers/cxl/core/region.c: In function 'alloc_hpa': ../drivers/cxl/core/region.c:556:25: warning: format '%llx' expects argument of type 'long long unsigned int', but argument 5 has type 'resource_size_t' {aka 'unsigned int'} [-Wformat=] 556 | "HPA allocation error (%ld) for size:%#llx in %s %pr\n", Fixes: 7984d22f1315 ("cxl/region: Add dev_dbg() detail on failure to allocate HPA space") Signed-off-by: Randy Dunlap Cc: Fan Ni Cc: Davidlohr Bueso Cc: Jonathan Cameron Cc: Dave Jiang Cc: Alison Schofield Cc: Vishal Verma Cc: Ira Weiny Cc: Dan Williams Cc: Link: https://lore.kernel.org/r/20240102173917.19718-1-rdunlap@infradead.org Signed-off-by: Dan Williams Signed-off-by: Yi Liu Signed-off-by: Bo Li --- drivers/cxl/core/region.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/cxl/core/region.c b/drivers/cxl/core/region.c index 774e198bca2b6..4b89b5258bb5d 100644 --- a/drivers/cxl/core/region.c +++ b/drivers/cxl/core/region.c @@ -537,8 +537,8 @@ static int alloc_hpa(struct cxl_region *cxlr, resource_size_t size) dev_name(&cxlr->dev)); if (IS_ERR(res)) { dev_dbg(&cxlr->dev, - "HPA allocation error (%ld) for size:%#llx in %s %pr\n", - PTR_ERR(res), size, cxlrd->res->name, cxlrd->res); + "HPA allocation error (%ld) for size:%pap in %s %pr\n", + PTR_ERR(res), &size, cxlrd->res->name, cxlrd->res); return PTR_ERR(res); } From 6830c5fc2b38a46fdba48e7111ffc14ea620fd9f Mon Sep 17 00:00:00 2001 From: Ira Weiny Date: Wed, 20 Dec 2023 16:17:28 -0800 Subject: [PATCH 055/138] cxl/trace: Pass UUID explicitly to event traces commit 6d0fc416c42a98b39a74151376928d577873941c upstream. CXL CPER events are identified by the CPER Section Type GUID. The GUID correlates with the CXL UUID for the event record. It turns out that a CXL CPER record is a strict subset of the CXL event record, only the UUID header field is chopped. In order to unify handling between native and CPER flavors of CXL events, prepare the code for the UUID to be passed in rather than inferred from the record itself. Later patches update the passed in record to only refer to the common data between the formats. Pass the UUID explicitly to each trace event to be able to remove the UUID from the event structures. Originally it was desirable to remove the UUID from the well known event because the UUID value was redundant. However, the trace API was already in place.[1] Signed-off-by: Ira Weiny Link: https://lore.kernel.org/all/36f2d12934d64a278f2c0313cbd01abc@huawei.com [1] Link: https://lore.kernel.org/r/20231220-cxl-cper-v5-1-1bb8a4ca2c7a@intel.com Reviewed-by: Jonathan Cameron Acked-by: Ard Biesheuvel Signed-off-by: Dan Williams Signed-off-by: Yi Liu Signed-off-by: Bo Li --- drivers/cxl/core/mbox.c | 8 ++++---- drivers/cxl/core/trace.h | 28 ++++++++++++++-------------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/drivers/cxl/core/mbox.c b/drivers/cxl/core/mbox.c index 2fb3378de8674..fcc11fd1c9049 100644 --- a/drivers/cxl/core/mbox.c +++ b/drivers/cxl/core/mbox.c @@ -871,19 +871,19 @@ static void cxl_event_trace_record(const struct cxl_memdev *cxlmd, struct cxl_event_gen_media *rec = (struct cxl_event_gen_media *)record; - trace_cxl_general_media(cxlmd, type, rec); + trace_cxl_general_media(cxlmd, type, id, rec); } else if (uuid_equal(id, &dram_event_uuid)) { struct cxl_event_dram *rec = (struct cxl_event_dram *)record; - trace_cxl_dram(cxlmd, type, rec); + trace_cxl_dram(cxlmd, type, id, rec); } else if (uuid_equal(id, &mem_mod_event_uuid)) { struct cxl_event_mem_module *rec = (struct cxl_event_mem_module *)record; - trace_cxl_memory_module(cxlmd, type, rec); + trace_cxl_memory_module(cxlmd, type, id, rec); } else { /* For unknown record types print just the header */ - trace_cxl_generic_event(cxlmd, type, record); + trace_cxl_generic_event(cxlmd, type, id, record); } } diff --git a/drivers/cxl/core/trace.h b/drivers/cxl/core/trace.h index 1e1e50d39bf63..2c9cfb4654773 100644 --- a/drivers/cxl/core/trace.h +++ b/drivers/cxl/core/trace.h @@ -198,12 +198,12 @@ TRACE_EVENT(cxl_overflow, __field(u8, hdr_length) \ __field(u8, hdr_maint_op_class) -#define CXL_EVT_TP_fast_assign(cxlmd, l, hdr) \ +#define CXL_EVT_TP_fast_assign(cxlmd, l, uuid, hdr) \ __assign_str(memdev, dev_name(&(cxlmd)->dev)); \ __assign_str(host, dev_name((cxlmd)->dev.parent)); \ __entry->log = (l); \ __entry->serial = (cxlmd)->cxlds->serial; \ - memcpy(&__entry->hdr_uuid, &(hdr).id, sizeof(uuid_t)); \ + memcpy(&__entry->hdr_uuid, (uuid), sizeof(uuid_t)); \ __entry->hdr_length = (hdr).length; \ __entry->hdr_flags = get_unaligned_le24((hdr).flags); \ __entry->hdr_handle = le16_to_cpu((hdr).handle); \ @@ -225,9 +225,9 @@ TRACE_EVENT(cxl_overflow, TRACE_EVENT(cxl_generic_event, TP_PROTO(const struct cxl_memdev *cxlmd, enum cxl_event_log_type log, - struct cxl_event_record_raw *rec), + const uuid_t *uuid, struct cxl_event_record_raw *rec), - TP_ARGS(cxlmd, log, rec), + TP_ARGS(cxlmd, log, uuid, rec), TP_STRUCT__entry( CXL_EVT_TP_entry @@ -235,7 +235,7 @@ TRACE_EVENT(cxl_generic_event, ), TP_fast_assign( - CXL_EVT_TP_fast_assign(cxlmd, log, rec->hdr); + CXL_EVT_TP_fast_assign(cxlmd, log, uuid, rec->hdr); memcpy(__entry->data, &rec->data, CXL_EVENT_RECORD_DATA_LENGTH); ), @@ -315,9 +315,9 @@ TRACE_EVENT(cxl_generic_event, TRACE_EVENT(cxl_general_media, TP_PROTO(const struct cxl_memdev *cxlmd, enum cxl_event_log_type log, - struct cxl_event_gen_media *rec), + const uuid_t *uuid, struct cxl_event_gen_media *rec), - TP_ARGS(cxlmd, log, rec), + TP_ARGS(cxlmd, log, uuid, rec), TP_STRUCT__entry( CXL_EVT_TP_entry @@ -336,7 +336,7 @@ TRACE_EVENT(cxl_general_media, ), TP_fast_assign( - CXL_EVT_TP_fast_assign(cxlmd, log, rec->hdr); + CXL_EVT_TP_fast_assign(cxlmd, log, uuid, rec->hdr); /* General Media */ __entry->dpa = le64_to_cpu(rec->phys_addr); @@ -409,9 +409,9 @@ TRACE_EVENT(cxl_general_media, TRACE_EVENT(cxl_dram, TP_PROTO(const struct cxl_memdev *cxlmd, enum cxl_event_log_type log, - struct cxl_event_dram *rec), + const uuid_t *uuid, struct cxl_event_dram *rec), - TP_ARGS(cxlmd, log, rec), + TP_ARGS(cxlmd, log, uuid, rec), TP_STRUCT__entry( CXL_EVT_TP_entry @@ -433,7 +433,7 @@ TRACE_EVENT(cxl_dram, ), TP_fast_assign( - CXL_EVT_TP_fast_assign(cxlmd, log, rec->hdr); + CXL_EVT_TP_fast_assign(cxlmd, log, uuid, rec->hdr); /* DRAM */ __entry->dpa = le64_to_cpu(rec->phys_addr); @@ -558,9 +558,9 @@ TRACE_EVENT(cxl_dram, TRACE_EVENT(cxl_memory_module, TP_PROTO(const struct cxl_memdev *cxlmd, enum cxl_event_log_type log, - struct cxl_event_mem_module *rec), + const uuid_t *uuid, struct cxl_event_mem_module *rec), - TP_ARGS(cxlmd, log, rec), + TP_ARGS(cxlmd, log, uuid, rec), TP_STRUCT__entry( CXL_EVT_TP_entry @@ -580,7 +580,7 @@ TRACE_EVENT(cxl_memory_module, ), TP_fast_assign( - CXL_EVT_TP_fast_assign(cxlmd, log, rec->hdr); + CXL_EVT_TP_fast_assign(cxlmd, log, uuid, rec->hdr); /* Memory Module Event */ __entry->event_type = rec->event_type; From 78dd661b716ea994167313c2c214e318343a1efe Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Fri, 5 Jan 2024 15:07:34 -0700 Subject: [PATCH 056/138] cxl: Introduce put_cxl_root() helper commit 98856b2ea3065d8f60e90f423d7707f4a4706ec5 upstream. Add a helper function put_cxl_root() to maintain symmetry for find_cxl_root() function instead of relying on open coding of the put_device() in order to dereference the 'struct device' that happens via get_device() in find_cxl_root(). Suggested-by: Robert Richter Reviewed-by: Ira Weiny Signed-off-by: Dave Jiang Reviewed-by: Robert Richter Link: https://lore.kernel.org/r/170449245417.3779673.4566146351673989387.stgit@djiang5-mobl3 Signed-off-by: Dan Williams Signed-off-by: Yi Liu Signed-off-by: Bo Li --- drivers/cxl/core/port.c | 9 +++++++++ drivers/cxl/cxl.h | 3 +++ 2 files changed, 12 insertions(+) diff --git a/drivers/cxl/core/port.c b/drivers/cxl/core/port.c index 217c9d780493d..c2d7a9566b86a 100644 --- a/drivers/cxl/core/port.c +++ b/drivers/cxl/core/port.c @@ -982,6 +982,15 @@ struct cxl_port *find_cxl_root(struct cxl_port *port) } EXPORT_SYMBOL_NS_GPL(find_cxl_root, CXL); +void put_cxl_root(struct cxl_root *cxl_root) +{ + if (!cxl_root) + return; + + put_device(&cxl_root->port.dev); +} +EXPORT_SYMBOL_NS_GPL(put_cxl_root, CXL); + static struct cxl_dport *find_dport(struct cxl_port *port, int id) { struct cxl_dport *dport; diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h index bc25457644d7b..f0fd35696c210 100644 --- a/drivers/cxl/cxl.h +++ b/drivers/cxl/cxl.h @@ -736,6 +736,9 @@ struct cxl_port *devm_cxl_add_port(struct device *host, struct cxl_root *devm_cxl_add_root(struct device *host, const struct cxl_root_ops *ops); struct cxl_port *find_cxl_root(struct cxl_port *port); +void put_cxl_root(struct cxl_root *cxl_root); +DEFINE_FREE(put_cxl_root, struct cxl_root *, if (_T) put_cxl_root(_T)) + int devm_cxl_enumerate_ports(struct cxl_memdev *cxlmd); void cxl_bus_rescan(void); void cxl_bus_drain(void); From 75c12b3c47fdcc08b44d85c139aa004dce75dfbe Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Fri, 5 Jan 2024 15:07:40 -0700 Subject: [PATCH 057/138] cxl: Convert find_cxl_root() to return a 'struct cxl_root *' commit 44cd71ef7bacdfcf7c3e8a7b13f11a7eba69533d upstream. Commit 790815902ec6 ("cxl: Add support for _DSM Function for retrieving QTG ID") introduced 'struct cxl_root', however all usages have been worked indirectly through cxl_port. Refactor code such as find_cxl_root() function to use 'struct cxl_root' directly. Suggested-by: Dan Williams Reviewed-by: Ira Weiny Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/170449246044.3779673.13035770941393418591.stgit@djiang5-mobl3 Signed-off-by: Dan Williams Signed-off-by: Yi Liu Signed-off-by: Bo Li --- drivers/cxl/acpi.c | 6 ++---- drivers/cxl/core/cdat.c | 17 ++++++++++------- drivers/cxl/core/pmem.c | 6 ++++-- drivers/cxl/core/port.c | 4 ++-- drivers/cxl/cxl.h | 14 +++++++------- drivers/cxl/port.c | 4 +++- 6 files changed, 28 insertions(+), 23 deletions(-) diff --git a/drivers/cxl/acpi.c b/drivers/cxl/acpi.c index 476b1674c940a..8b48d0dceae0e 100644 --- a/drivers/cxl/acpi.c +++ b/drivers/cxl/acpi.c @@ -295,14 +295,12 @@ cxl_acpi_evaluate_qtg_dsm(acpi_handle handle, struct access_coordinate *coord, return rc; } -static int cxl_acpi_qos_class(struct cxl_port *root_port, +static int cxl_acpi_qos_class(struct cxl_root *cxl_root, struct access_coordinate *coord, int entries, int *qos_class) { + struct device *dev = cxl_root->port.uport_dev; acpi_handle handle; - struct device *dev; - - dev = root_port->uport_dev; if (!dev_is_platform(dev)) return -ENODEV; diff --git a/drivers/cxl/core/cdat.c b/drivers/cxl/core/cdat.c index 10ac76bed4772..3807100219fdb 100644 --- a/drivers/cxl/core/cdat.c +++ b/drivers/cxl/core/cdat.c @@ -163,7 +163,6 @@ static int cxl_port_perf_data_calculate(struct cxl_port *port, struct xarray *dsmas_xa) { struct access_coordinate c; - struct cxl_port *root_port; struct cxl_root *cxl_root; struct dsmas_entry *dent; int valid_entries = 0; @@ -176,8 +175,7 @@ static int cxl_port_perf_data_calculate(struct cxl_port *port, return rc; } - root_port = find_cxl_root(port); - cxl_root = to_cxl_root(root_port); + cxl_root = find_cxl_root(port); if (!cxl_root->ops || !cxl_root->ops->qos_class) return -EOPNOTSUPP; @@ -194,7 +192,8 @@ static int cxl_port_perf_data_calculate(struct cxl_port *port, dent->coord.write_bandwidth); dent->entries = 1; - rc = cxl_root->ops->qos_class(root_port, &dent->coord, 1, &qos_class); + rc = cxl_root->ops->qos_class(cxl_root, &dent->coord, 1, + &qos_class); if (rc != 1) continue; @@ -347,15 +346,19 @@ static int cxl_qos_class_verify(struct cxl_memdev *cxlmd) { struct cxl_dev_state *cxlds = cxlmd->cxlds; struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds); - struct cxl_port *root_port __free(put_device) = NULL; LIST_HEAD(__discard); struct list_head *discard __free(dpa_perf) = &__discard; + struct cxl_port *root_port; int rc; - root_port = find_cxl_root(cxlmd->endpoint); - if (!root_port) + struct cxl_root *cxl_root __free(put_cxl_root) = + find_cxl_root(cxlmd->endpoint); + + if (!cxl_root) return -ENODEV; + root_port = &cxl_root->port; + /* Check that the QTG IDs are all sane between end device and root decoders */ cxl_qos_match(root_port, &mds->ram_perf_list, discard); cxl_qos_match(root_port, &mds->pmem_perf_list, discard); diff --git a/drivers/cxl/core/pmem.c b/drivers/cxl/core/pmem.c index fc94f52403271..da92a901b9e8c 100644 --- a/drivers/cxl/core/pmem.c +++ b/drivers/cxl/core/pmem.c @@ -64,12 +64,14 @@ static int match_nvdimm_bridge(struct device *dev, void *data) struct cxl_nvdimm_bridge *cxl_find_nvdimm_bridge(struct cxl_memdev *cxlmd) { - struct cxl_port *port = find_cxl_root(cxlmd->endpoint); + struct cxl_root *cxl_root = find_cxl_root(cxlmd->endpoint); + struct cxl_port *port; struct device *dev; - if (!port) + if (!cxl_root) return NULL; + port = &cxl_root->port; dev = device_find_child(&port->dev, NULL, match_nvdimm_bridge); put_device(&port->dev); diff --git a/drivers/cxl/core/port.c b/drivers/cxl/core/port.c index c2d7a9566b86a..1ff0c730b1c7b 100644 --- a/drivers/cxl/core/port.c +++ b/drivers/cxl/core/port.c @@ -968,7 +968,7 @@ static bool dev_is_cxl_root_child(struct device *dev) return false; } -struct cxl_port *find_cxl_root(struct cxl_port *port) +struct cxl_root *find_cxl_root(struct cxl_port *port) { struct cxl_port *iter = port; @@ -978,7 +978,7 @@ struct cxl_port *find_cxl_root(struct cxl_port *port) if (!iter) return NULL; get_device(&iter->dev); - return iter; + return to_cxl_root(iter); } EXPORT_SYMBOL_NS_GPL(find_cxl_root, CXL); diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h index f0fd35696c210..d983eccc80ba3 100644 --- a/drivers/cxl/cxl.h +++ b/drivers/cxl/cxl.h @@ -617,12 +617,6 @@ struct cxl_port { long pci_latency; }; -struct cxl_root_ops { - int (*qos_class)(struct cxl_port *root_port, - struct access_coordinate *coord, int entries, - int *qos_class); -}; - /** * struct cxl_root - logical collection of root cxl_port items * @@ -640,6 +634,12 @@ to_cxl_root(const struct cxl_port *port) return container_of(port, struct cxl_root, port); } +struct cxl_root_ops { + int (*qos_class)(struct cxl_root *cxl_root, + struct access_coordinate *coord, int entries, + int *qos_class); +}; + static inline struct cxl_dport * cxl_find_dport_by_dev(struct cxl_port *port, const struct device *dport_dev) { @@ -735,7 +735,7 @@ struct cxl_port *devm_cxl_add_port(struct device *host, struct cxl_dport *parent_dport); struct cxl_root *devm_cxl_add_root(struct device *host, const struct cxl_root_ops *ops); -struct cxl_port *find_cxl_root(struct cxl_port *port); +struct cxl_root *find_cxl_root(struct cxl_port *port); void put_cxl_root(struct cxl_root *cxl_root); DEFINE_FREE(put_cxl_root, struct cxl_root *, if (_T) put_cxl_root(_T)) diff --git a/drivers/cxl/port.c b/drivers/cxl/port.c index da3c3a08bd626..4f3a08fdc9e94 100644 --- a/drivers/cxl/port.c +++ b/drivers/cxl/port.c @@ -94,6 +94,7 @@ static int cxl_endpoint_port_probe(struct cxl_port *port) struct cxl_endpoint_dvsec_info info = { .port = port }; struct cxl_memdev *cxlmd = to_cxl_memdev(port->uport_dev); struct cxl_dev_state *cxlds = cxlmd->cxlds; + struct cxl_root *cxl_root; struct cxl_hdm *cxlhdm; struct cxl_port *root; int rc; @@ -130,7 +131,8 @@ static int cxl_endpoint_port_probe(struct cxl_port *port) * This can't fail in practice as CXL root exit unregisters all * descendant ports and that in turn synchronizes with cxl_port_probe() */ - root = find_cxl_root(port); + cxl_root = find_cxl_root(port); + root = &cxl_root->port; /* * Now that all endpoint decoders are successfully enumerated, try to From 8dcfe2e6bbac6f66faedaa0469bac5fb2611ea70 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Fri, 5 Jan 2024 15:07:46 -0700 Subject: [PATCH 058/138] cxl: Fix device reference leak in cxl_port_perf_data_calculate() commit 98e7ab3345e105339b7d919b4918f3c1879052ed upstream. cxl_port_perf_data_calculate() calls find_cxl_root() and does not dereference the 'struct device' in the cxl_root->port. find_cxl_root() calls get_device() and takes a reference on the port 'struct device' member. Use the __free() macro to ensure the dereference happens. Fixes: 7a4f148dd8d5 ("cxl: Compute the entire CXL path latency and bandwidth data") Reviewed-by: Ira Weiny Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/170449246681.3779673.2288926019977963333.stgit@djiang5-mobl3 Signed-off-by: Dan Williams Signed-off-by: Yi Liu Signed-off-by: Bo Li --- drivers/cxl/core/cdat.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/drivers/cxl/core/cdat.c b/drivers/cxl/core/cdat.c index 3807100219fdb..473c588f86ad2 100644 --- a/drivers/cxl/core/cdat.c +++ b/drivers/cxl/core/cdat.c @@ -163,7 +163,6 @@ static int cxl_port_perf_data_calculate(struct cxl_port *port, struct xarray *dsmas_xa) { struct access_coordinate c; - struct cxl_root *cxl_root; struct dsmas_entry *dent; int valid_entries = 0; unsigned long index; @@ -175,7 +174,11 @@ static int cxl_port_perf_data_calculate(struct cxl_port *port, return rc; } - cxl_root = find_cxl_root(port); + struct cxl_root *cxl_root __free(put_cxl_root) = find_cxl_root(port); + + if (!cxl_root) + return -ENODEV; + if (!cxl_root->ops || !cxl_root->ops->qos_class) return -EOPNOTSUPP; From 29ddf59213ee4ed895ee543bee3bb72769b57e2b Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Fri, 5 Jan 2024 15:07:53 -0700 Subject: [PATCH 059/138] cxl: Refactor to use __free() for cxl_root allocation in cxl_find_nvdimm_bridge() commit 66f11890d35a609012037cccfd9e63ed98474f99 upstream. Use scope-based resource management __free() macro to drop the open coded put_device() in cxl_find_nvdimm_bridge(). Reviewed-by: Ira Weiny Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/170449247353.3779673.5963704495491343135.stgit@djiang5-mobl3 Signed-off-by: Dan Williams Signed-off-by: Yi Liu Signed-off-by: Bo Li --- drivers/cxl/core/pmem.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/drivers/cxl/core/pmem.c b/drivers/cxl/core/pmem.c index da92a901b9e8c..e69625a8d6a1d 100644 --- a/drivers/cxl/core/pmem.c +++ b/drivers/cxl/core/pmem.c @@ -64,16 +64,14 @@ static int match_nvdimm_bridge(struct device *dev, void *data) struct cxl_nvdimm_bridge *cxl_find_nvdimm_bridge(struct cxl_memdev *cxlmd) { - struct cxl_root *cxl_root = find_cxl_root(cxlmd->endpoint); - struct cxl_port *port; + struct cxl_root *cxl_root __free(put_cxl_root) = + find_cxl_root(cxlmd->endpoint); struct device *dev; if (!cxl_root) return NULL; - port = &cxl_root->port; - dev = device_find_child(&port->dev, NULL, match_nvdimm_bridge); - put_device(&port->dev); + dev = device_find_child(&cxl_root->port.dev, NULL, match_nvdimm_bridge); if (!dev) return NULL; From 743a424f856a8a7357e46837b421812823cc7dbb Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Fri, 5 Jan 2024 15:07:59 -0700 Subject: [PATCH 060/138] cxl: Refactor to use __free() for cxl_root allocation in cxl_endpoint_port_probe() commit 321dd36c286b3f982b341ce9a273b0f66f0e00ed upstream. Use scope-based resource management __free() macro to drop the open coded put_device() in cxl_endpoint_port_probe(). Reviewed-by: Ira Weiny Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/170449247973.3779673.15088722836135359275.stgit@djiang5-mobl3 Signed-off-by: Dan Williams Signed-off-by: Yi Liu Signed-off-by: Bo Li --- drivers/cxl/port.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/drivers/cxl/port.c b/drivers/cxl/port.c index 4f3a08fdc9e94..97c21566677aa 100644 --- a/drivers/cxl/port.c +++ b/drivers/cxl/port.c @@ -94,7 +94,6 @@ static int cxl_endpoint_port_probe(struct cxl_port *port) struct cxl_endpoint_dvsec_info info = { .port = port }; struct cxl_memdev *cxlmd = to_cxl_memdev(port->uport_dev); struct cxl_dev_state *cxlds = cxlmd->cxlds; - struct cxl_root *cxl_root; struct cxl_hdm *cxlhdm; struct cxl_port *root; int rc; @@ -131,7 +130,8 @@ static int cxl_endpoint_port_probe(struct cxl_port *port) * This can't fail in practice as CXL root exit unregisters all * descendant ports and that in turn synchronizes with cxl_port_probe() */ - cxl_root = find_cxl_root(port); + struct cxl_root *cxl_root __free(put_cxl_root) = find_cxl_root(port); + root = &cxl_root->port; /* @@ -139,7 +139,6 @@ static int cxl_endpoint_port_probe(struct cxl_port *port) * assemble regions from committed decoders */ device_for_each_child(&port->dev, root, discover_region); - put_device(&root->dev); return 0; } From 90707ab2e1887af849176705651e9d944b73885d Mon Sep 17 00:00:00 2001 From: Ira Weiny Date: Wed, 20 Dec 2023 16:17:29 -0800 Subject: [PATCH 061/138] cxl/events: Promote CXL event structures to a core header commit 26a1a86dd093a10d0653429bf013dae6e95dccbf upstream. UEFI code can process CXL events through CPER records. Those records use almost the same format as the CXL events. Lift the CXL event structures to a core header to be shared in later patches. [jic123: drop "CXL rev 3.0" mention] Signed-off-by: Ira Weiny Link: https://lore.kernel.org/r/20231220-cxl-cper-v5-2-1bb8a4ca2c7a@intel.com [djbw: add F: entry to maintainers for include/linux/cxl-event.h] Reviewed-by: Jonathan Cameron Acked-by: Ard Biesheuvel Signed-off-by: Dan Williams Signed-off-by: Yi Liu Signed-off-by: Bo Li --- MAINTAINERS | 1 + drivers/cxl/cxlmem.h | 90 +------------------------------------ include/linux/cxl-event.h | 95 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 97 insertions(+), 89 deletions(-) create mode 100644 include/linux/cxl-event.h diff --git a/MAINTAINERS b/MAINTAINERS index eadcca709f12b..5871b31669415 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -5224,6 +5224,7 @@ M: Dan Williams L: linux-cxl@vger.kernel.org S: Maintained F: drivers/cxl/ +F: include/linux/cxl-event.h F: include/uapi/linux/cxl_mem.h F: tools/testing/cxl/ diff --git a/drivers/cxl/cxlmem.h b/drivers/cxl/cxlmem.h index f62cdc25099d1..b2406a3710512 100644 --- a/drivers/cxl/cxlmem.h +++ b/drivers/cxl/cxlmem.h @@ -7,6 +7,7 @@ #include #include #include +#include #include "cxl.h" /* CXL 2.0 8.2.8.5.1.1 Memory Device Status Register */ @@ -601,27 +602,6 @@ struct cxl_mbox_identify { u8 qos_telemetry_caps; } __packed; -/* - * Common Event Record Format - * CXL rev 3.0 section 8.2.9.2.1; Table 8-42 - */ -struct cxl_event_record_hdr { - uuid_t id; - u8 length; - u8 flags[3]; - __le16 handle; - __le16 related_handle; - __le64 timestamp; - u8 maint_op_class; - u8 reserved[15]; -} __packed; - -#define CXL_EVENT_RECORD_DATA_LENGTH 0x50 -struct cxl_event_record_raw { - struct cxl_event_record_hdr hdr; - u8 data[CXL_EVENT_RECORD_DATA_LENGTH]; -} __packed; - /* * Get Event Records output payload * CXL rev 3.0 section 8.2.9.2.2; Table 8-50 @@ -663,74 +643,6 @@ struct cxl_mbox_clear_event_payload { } __packed; #define CXL_CLEAR_EVENT_MAX_HANDLES U8_MAX -/* - * General Media Event Record - * CXL rev 3.0 Section 8.2.9.2.1.1; Table 8-43 - */ -#define CXL_EVENT_GEN_MED_COMP_ID_SIZE 0x10 -struct cxl_event_gen_media { - struct cxl_event_record_hdr hdr; - __le64 phys_addr; - u8 descriptor; - u8 type; - u8 transaction_type; - u8 validity_flags[2]; - u8 channel; - u8 rank; - u8 device[3]; - u8 component_id[CXL_EVENT_GEN_MED_COMP_ID_SIZE]; - u8 reserved[46]; -} __packed; - -/* - * DRAM Event Record - DER - * CXL rev 3.0 section 8.2.9.2.1.2; Table 3-44 - */ -#define CXL_EVENT_DER_CORRECTION_MASK_SIZE 0x20 -struct cxl_event_dram { - struct cxl_event_record_hdr hdr; - __le64 phys_addr; - u8 descriptor; - u8 type; - u8 transaction_type; - u8 validity_flags[2]; - u8 channel; - u8 rank; - u8 nibble_mask[3]; - u8 bank_group; - u8 bank; - u8 row[3]; - u8 column[2]; - u8 correction_mask[CXL_EVENT_DER_CORRECTION_MASK_SIZE]; - u8 reserved[0x17]; -} __packed; - -/* - * Get Health Info Record - * CXL rev 3.0 section 8.2.9.8.3.1; Table 8-100 - */ -struct cxl_get_health_info { - u8 health_status; - u8 media_status; - u8 add_status; - u8 life_used; - u8 device_temp[2]; - u8 dirty_shutdown_cnt[4]; - u8 cor_vol_err_cnt[4]; - u8 cor_per_err_cnt[4]; -} __packed; - -/* - * Memory Module Event Record - * CXL rev 3.0 section 8.2.9.2.1.3; Table 8-45 - */ -struct cxl_event_mem_module { - struct cxl_event_record_hdr hdr; - u8 event_type; - struct cxl_get_health_info info; - u8 reserved[0x3d]; -} __packed; - struct cxl_mbox_get_partition_info { __le64 active_volatile_cap; __le64 active_persistent_cap; diff --git a/include/linux/cxl-event.h b/include/linux/cxl-event.h new file mode 100644 index 0000000000000..0fc068123f8ed --- /dev/null +++ b/include/linux/cxl-event.h @@ -0,0 +1,95 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2023 Intel Corporation. */ +#ifndef _LINUX_CXL_EVENT_H +#define _LINUX_CXL_EVENT_H + +/* + * Common Event Record Format + * CXL rev 3.0 section 8.2.9.2.1; Table 8-42 + */ +struct cxl_event_record_hdr { + uuid_t id; + u8 length; + u8 flags[3]; + __le16 handle; + __le16 related_handle; + __le64 timestamp; + u8 maint_op_class; + u8 reserved[15]; +} __packed; + +#define CXL_EVENT_RECORD_DATA_LENGTH 0x50 +struct cxl_event_record_raw { + struct cxl_event_record_hdr hdr; + u8 data[CXL_EVENT_RECORD_DATA_LENGTH]; +} __packed; + +/* + * General Media Event Record + * CXL rev 3.0 Section 8.2.9.2.1.1; Table 8-43 + */ +#define CXL_EVENT_GEN_MED_COMP_ID_SIZE 0x10 +struct cxl_event_gen_media { + struct cxl_event_record_hdr hdr; + __le64 phys_addr; + u8 descriptor; + u8 type; + u8 transaction_type; + u8 validity_flags[2]; + u8 channel; + u8 rank; + u8 device[3]; + u8 component_id[CXL_EVENT_GEN_MED_COMP_ID_SIZE]; + u8 reserved[46]; +} __packed; + +/* + * DRAM Event Record - DER + * CXL rev 3.0 section 8.2.9.2.1.2; Table 3-44 + */ +#define CXL_EVENT_DER_CORRECTION_MASK_SIZE 0x20 +struct cxl_event_dram { + struct cxl_event_record_hdr hdr; + __le64 phys_addr; + u8 descriptor; + u8 type; + u8 transaction_type; + u8 validity_flags[2]; + u8 channel; + u8 rank; + u8 nibble_mask[3]; + u8 bank_group; + u8 bank; + u8 row[3]; + u8 column[2]; + u8 correction_mask[CXL_EVENT_DER_CORRECTION_MASK_SIZE]; + u8 reserved[0x17]; +} __packed; + +/* + * Get Health Info Record + * CXL rev 3.0 section 8.2.9.8.3.1; Table 8-100 + */ +struct cxl_get_health_info { + u8 health_status; + u8 media_status; + u8 add_status; + u8 life_used; + u8 device_temp[2]; + u8 dirty_shutdown_cnt[4]; + u8 cor_vol_err_cnt[4]; + u8 cor_per_err_cnt[4]; +} __packed; + +/* + * Memory Module Event Record + * CXL rev 3.0 section 8.2.9.2.1.3; Table 8-45 + */ +struct cxl_event_mem_module { + struct cxl_event_record_hdr hdr; + u8 event_type; + struct cxl_get_health_info info; + u8 reserved[0x3d]; +} __packed; + +#endif /* _LINUX_CXL_EVENT_H */ From e00f93090e0e882d6d41cc8e0a694a03b27d8e2d Mon Sep 17 00:00:00 2001 From: Ira Weiny Date: Wed, 20 Dec 2023 16:17:30 -0800 Subject: [PATCH 062/138] cxl/events: Create common event UUID defines commit 4c115c9c1f81a6efe2bd68fcefec6836f7f3dc71 upstream. Dan points out in review that the cxl_test code could be made better through the use of UUID's defines rather than being open coded.[1] Create UUID defines and use them rather than open coding them. Suggested-by: Dan Williams Signed-off-by: Ira Weiny Link: http://lore.kernel.org/r/65738d09e30e2_45e0129451@dwillia2-xfh.jf.intel.com.notmuch [1] Link: https://lore.kernel.org/r/20231220-cxl-cper-v5-3-1bb8a4ca2c7a@intel.com [djbw: clang-format uuid definitions] Reviewed-by: Jonathan Cameron Acked-by: Ard Biesheuvel Signed-off-by: Dan Williams Signed-off-by: Yi Liu Signed-off-by: Bo Li --- drivers/cxl/core/mbox.c | 30 +++--------------------------- drivers/cxl/cxlmem.h | 24 ++++++++++++++++++++++++ tools/testing/cxl/test/mem.c | 9 +++------ 3 files changed, 30 insertions(+), 33 deletions(-) diff --git a/drivers/cxl/core/mbox.c b/drivers/cxl/core/mbox.c index fcc11fd1c9049..33fe056397304 100644 --- a/drivers/cxl/core/mbox.c +++ b/drivers/cxl/core/mbox.c @@ -837,46 +837,22 @@ int cxl_enumerate_cmds(struct cxl_memdev_state *mds) } EXPORT_SYMBOL_NS_GPL(cxl_enumerate_cmds, CXL); -/* - * General Media Event Record - * CXL rev 3.0 Section 8.2.9.2.1.1; Table 8-43 - */ -static const uuid_t gen_media_event_uuid = - UUID_INIT(0xfbcd0a77, 0xc260, 0x417f, - 0x85, 0xa9, 0x08, 0x8b, 0x16, 0x21, 0xeb, 0xa6); - -/* - * DRAM Event Record - * CXL rev 3.0 section 8.2.9.2.1.2; Table 8-44 - */ -static const uuid_t dram_event_uuid = - UUID_INIT(0x601dcbb3, 0x9c06, 0x4eab, - 0xb8, 0xaf, 0x4e, 0x9b, 0xfb, 0x5c, 0x96, 0x24); - -/* - * Memory Module Event Record - * CXL rev 3.0 section 8.2.9.2.1.3; Table 8-45 - */ -static const uuid_t mem_mod_event_uuid = - UUID_INIT(0xfe927475, 0xdd59, 0x4339, - 0xa5, 0x86, 0x79, 0xba, 0xb1, 0x13, 0xb7, 0x74); - static void cxl_event_trace_record(const struct cxl_memdev *cxlmd, enum cxl_event_log_type type, struct cxl_event_record_raw *record) { uuid_t *id = &record->hdr.id; - if (uuid_equal(id, &gen_media_event_uuid)) { + if (uuid_equal(id, &CXL_EVENT_GEN_MEDIA_UUID)) { struct cxl_event_gen_media *rec = (struct cxl_event_gen_media *)record; trace_cxl_general_media(cxlmd, type, id, rec); - } else if (uuid_equal(id, &dram_event_uuid)) { + } else if (uuid_equal(id, &CXL_EVENT_DRAM_UUID)) { struct cxl_event_dram *rec = (struct cxl_event_dram *)record; trace_cxl_dram(cxlmd, type, id, rec); - } else if (uuid_equal(id, &mem_mod_event_uuid)) { + } else if (uuid_equal(id, &CXL_EVENT_MEM_MODULE_UUID)) { struct cxl_event_mem_module *rec = (struct cxl_event_mem_module *)record; diff --git a/drivers/cxl/cxlmem.h b/drivers/cxl/cxlmem.h index b2406a3710512..6a88cdcab9501 100644 --- a/drivers/cxl/cxlmem.h +++ b/drivers/cxl/cxlmem.h @@ -602,6 +602,30 @@ struct cxl_mbox_identify { u8 qos_telemetry_caps; } __packed; +/* + * General Media Event Record UUID + * CXL rev 3.0 Section 8.2.9.2.1.1; Table 8-43 + */ +#define CXL_EVENT_GEN_MEDIA_UUID \ + UUID_INIT(0xfbcd0a77, 0xc260, 0x417f, 0x85, 0xa9, 0x08, 0x8b, 0x16, \ + 0x21, 0xeb, 0xa6) + +/* + * DRAM Event Record UUID + * CXL rev 3.0 section 8.2.9.2.1.2; Table 8-44 + */ +#define CXL_EVENT_DRAM_UUID \ + UUID_INIT(0x601dcbb3, 0x9c06, 0x4eab, 0xb8, 0xaf, 0x4e, 0x9b, 0xfb, \ + 0x5c, 0x96, 0x24) + +/* + * Memory Module Event Record UUID + * CXL rev 3.0 section 8.2.9.2.1.3; Table 8-45 + */ +#define CXL_EVENT_MEM_MODULE_UUID \ + UUID_INIT(0xfe927475, 0xdd59, 0x4339, 0xa5, 0x86, 0x79, 0xba, 0xb1, \ + 0x13, 0xb7, 0x74) + /* * Get Event Records output payload * CXL rev 3.0 section 8.2.9.2.2; Table 8-50 diff --git a/tools/testing/cxl/test/mem.c b/tools/testing/cxl/test/mem.c index a6257430d73a0..22f91ecade25f 100644 --- a/tools/testing/cxl/test/mem.c +++ b/tools/testing/cxl/test/mem.c @@ -363,8 +363,7 @@ struct cxl_event_record_raw hardware_replace = { struct cxl_event_gen_media gen_media = { .hdr = { - .id = UUID_INIT(0xfbcd0a77, 0xc260, 0x417f, - 0x85, 0xa9, 0x08, 0x8b, 0x16, 0x21, 0xeb, 0xa6), + .id = CXL_EVENT_GEN_MEDIA_UUID, .length = sizeof(struct cxl_event_gen_media), .flags[0] = CXL_EVENT_RECORD_FLAG_PERMANENT, /* .handle = Set dynamically */ @@ -381,8 +380,7 @@ struct cxl_event_gen_media gen_media = { struct cxl_event_dram dram = { .hdr = { - .id = UUID_INIT(0x601dcbb3, 0x9c06, 0x4eab, - 0xb8, 0xaf, 0x4e, 0x9b, 0xfb, 0x5c, 0x96, 0x24), + .id = CXL_EVENT_DRAM_UUID, .length = sizeof(struct cxl_event_dram), .flags[0] = CXL_EVENT_RECORD_FLAG_PERF_DEGRADED, /* .handle = Set dynamically */ @@ -401,8 +399,7 @@ struct cxl_event_dram dram = { struct cxl_event_mem_module mem_module = { .hdr = { - .id = UUID_INIT(0xfe927475, 0xdd59, 0x4339, - 0xa5, 0x86, 0x79, 0xba, 0xb1, 0x13, 0xb7, 0x74), + .id = CXL_EVENT_MEM_MODULE_UUID, .length = sizeof(struct cxl_event_mem_module), /* .handle = Set dynamically */ .related_handle = cpu_to_le16(0), From ee3b706d0f7457f0c815fd69911d6e0383a1f80f Mon Sep 17 00:00:00 2001 From: Ira Weiny Date: Wed, 20 Dec 2023 16:17:31 -0800 Subject: [PATCH 063/138] cxl/events: Remove passing a UUID to known event traces commit 207a1f82301de0b4123f00a8d26ea55bb2484757 upstream. The UUID data is redundant in the known event trace types. The addition of static defines allows the trace macros to create the UUID data inside the trace thus removing unnecessary code. Have well known trace events use static data to set the uuid field based on the event type. Suggested-by: Jonathan Cameron Signed-off-by: Ira Weiny Link: https://lore.kernel.org/r/20231220-cxl-cper-v5-4-1bb8a4ca2c7a@intel.com Acked-by: Ard Biesheuvel Signed-off-by: Dan Williams Signed-off-by: Yi Liu Signed-off-by: Bo Li --- drivers/cxl/core/mbox.c | 6 +++--- drivers/cxl/core/trace.h | 28 ++++++++++++++++------------ 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/drivers/cxl/core/mbox.c b/drivers/cxl/core/mbox.c index 33fe056397304..1a58dbf824cc7 100644 --- a/drivers/cxl/core/mbox.c +++ b/drivers/cxl/core/mbox.c @@ -847,16 +847,16 @@ static void cxl_event_trace_record(const struct cxl_memdev *cxlmd, struct cxl_event_gen_media *rec = (struct cxl_event_gen_media *)record; - trace_cxl_general_media(cxlmd, type, id, rec); + trace_cxl_general_media(cxlmd, type, rec); } else if (uuid_equal(id, &CXL_EVENT_DRAM_UUID)) { struct cxl_event_dram *rec = (struct cxl_event_dram *)record; - trace_cxl_dram(cxlmd, type, id, rec); + trace_cxl_dram(cxlmd, type, rec); } else if (uuid_equal(id, &CXL_EVENT_MEM_MODULE_UUID)) { struct cxl_event_mem_module *rec = (struct cxl_event_mem_module *)record; - trace_cxl_memory_module(cxlmd, type, id, rec); + trace_cxl_memory_module(cxlmd, type, rec); } else { /* For unknown record types print just the header */ trace_cxl_generic_event(cxlmd, type, id, record); diff --git a/drivers/cxl/core/trace.h b/drivers/cxl/core/trace.h index 2c9cfb4654773..13a821042c8ce 100644 --- a/drivers/cxl/core/trace.h +++ b/drivers/cxl/core/trace.h @@ -181,6 +181,7 @@ TRACE_EVENT(cxl_overflow, * 1) Add CXL_EVT_TP_entry to TP_STRUCT__entry * 2) Use CXL_EVT_TP_fast_assign within TP_fast_assign; * pass the dev, log, and CXL event header + * NOTE: The uuid must be assigned by the specific trace event * 3) Use CXL_EVT_TP_printk() instead of TP_printk() * * See the generic_event tracepoint as an example. @@ -198,12 +199,11 @@ TRACE_EVENT(cxl_overflow, __field(u8, hdr_length) \ __field(u8, hdr_maint_op_class) -#define CXL_EVT_TP_fast_assign(cxlmd, l, uuid, hdr) \ +#define CXL_EVT_TP_fast_assign(cxlmd, l, hdr) \ __assign_str(memdev, dev_name(&(cxlmd)->dev)); \ __assign_str(host, dev_name((cxlmd)->dev.parent)); \ __entry->log = (l); \ __entry->serial = (cxlmd)->cxlds->serial; \ - memcpy(&__entry->hdr_uuid, (uuid), sizeof(uuid_t)); \ __entry->hdr_length = (hdr).length; \ __entry->hdr_flags = get_unaligned_le24((hdr).flags); \ __entry->hdr_handle = le16_to_cpu((hdr).handle); \ @@ -235,7 +235,8 @@ TRACE_EVENT(cxl_generic_event, ), TP_fast_assign( - CXL_EVT_TP_fast_assign(cxlmd, log, uuid, rec->hdr); + CXL_EVT_TP_fast_assign(cxlmd, log, rec->hdr); + memcpy(&__entry->hdr_uuid, uuid, sizeof(uuid_t)); memcpy(__entry->data, &rec->data, CXL_EVENT_RECORD_DATA_LENGTH); ), @@ -315,9 +316,9 @@ TRACE_EVENT(cxl_generic_event, TRACE_EVENT(cxl_general_media, TP_PROTO(const struct cxl_memdev *cxlmd, enum cxl_event_log_type log, - const uuid_t *uuid, struct cxl_event_gen_media *rec), + struct cxl_event_gen_media *rec), - TP_ARGS(cxlmd, log, uuid, rec), + TP_ARGS(cxlmd, log, rec), TP_STRUCT__entry( CXL_EVT_TP_entry @@ -336,7 +337,8 @@ TRACE_EVENT(cxl_general_media, ), TP_fast_assign( - CXL_EVT_TP_fast_assign(cxlmd, log, uuid, rec->hdr); + CXL_EVT_TP_fast_assign(cxlmd, log, rec->hdr); + memcpy(&__entry->hdr_uuid, &CXL_EVENT_GEN_MEDIA_UUID, sizeof(uuid_t)); /* General Media */ __entry->dpa = le64_to_cpu(rec->phys_addr); @@ -409,9 +411,9 @@ TRACE_EVENT(cxl_general_media, TRACE_EVENT(cxl_dram, TP_PROTO(const struct cxl_memdev *cxlmd, enum cxl_event_log_type log, - const uuid_t *uuid, struct cxl_event_dram *rec), + struct cxl_event_dram *rec), - TP_ARGS(cxlmd, log, uuid, rec), + TP_ARGS(cxlmd, log, rec), TP_STRUCT__entry( CXL_EVT_TP_entry @@ -433,7 +435,8 @@ TRACE_EVENT(cxl_dram, ), TP_fast_assign( - CXL_EVT_TP_fast_assign(cxlmd, log, uuid, rec->hdr); + CXL_EVT_TP_fast_assign(cxlmd, log, rec->hdr); + memcpy(&__entry->hdr_uuid, &CXL_EVENT_DRAM_UUID, sizeof(uuid_t)); /* DRAM */ __entry->dpa = le64_to_cpu(rec->phys_addr); @@ -558,9 +561,9 @@ TRACE_EVENT(cxl_dram, TRACE_EVENT(cxl_memory_module, TP_PROTO(const struct cxl_memdev *cxlmd, enum cxl_event_log_type log, - const uuid_t *uuid, struct cxl_event_mem_module *rec), + struct cxl_event_mem_module *rec), - TP_ARGS(cxlmd, log, uuid, rec), + TP_ARGS(cxlmd, log, rec), TP_STRUCT__entry( CXL_EVT_TP_entry @@ -580,7 +583,8 @@ TRACE_EVENT(cxl_memory_module, ), TP_fast_assign( - CXL_EVT_TP_fast_assign(cxlmd, log, uuid, rec->hdr); + CXL_EVT_TP_fast_assign(cxlmd, log, rec->hdr); + memcpy(&__entry->hdr_uuid, &CXL_EVENT_MEM_MODULE_UUID, sizeof(uuid_t)); /* Memory Module Event */ __entry->event_type = rec->event_type; From 87925249207067f7fca321d28abd253c01e57e32 Mon Sep 17 00:00:00 2001 From: Ira Weiny Date: Wed, 20 Dec 2023 16:17:32 -0800 Subject: [PATCH 064/138] cxl/events: Separate UUID from event structures commit 6eade110754c085cee9e46f4d87d2c3ea4e59e8c upstream. The UEFI CXL CPER structure does not include the UUID. Now that the UUID is passed separately to the trace event there is no need to have the UUID in those structures. Move UUID from the event record header to the raw structures. Adjust cxl-test to Create dummy structures for creating test records. Signed-off-by: Ira Weiny Link: https://lore.kernel.org/r/20231220-cxl-cper-v5-5-1bb8a4ca2c7a@intel.com Reviewed-by: Jonathan Cameron Acked-by: Ard Biesheuvel Signed-off-by: Dan Williams Signed-off-by: Yi Liu Signed-off-by: Bo Li --- drivers/cxl/core/mbox.c | 2 +- include/linux/cxl-event.h | 6 +- tools/testing/cxl/test/mem.c | 129 ++++++++++++++++++++--------------- 3 files changed, 81 insertions(+), 56 deletions(-) diff --git a/drivers/cxl/core/mbox.c b/drivers/cxl/core/mbox.c index 1a58dbf824cc7..5a77d6f2c0c02 100644 --- a/drivers/cxl/core/mbox.c +++ b/drivers/cxl/core/mbox.c @@ -841,7 +841,7 @@ static void cxl_event_trace_record(const struct cxl_memdev *cxlmd, enum cxl_event_log_type type, struct cxl_event_record_raw *record) { - uuid_t *id = &record->hdr.id; + uuid_t *id = &record->id; if (uuid_equal(id, &CXL_EVENT_GEN_MEDIA_UUID)) { struct cxl_event_gen_media *rec = diff --git a/include/linux/cxl-event.h b/include/linux/cxl-event.h index 0fc068123f8ed..3d9b5954d0c11 100644 --- a/include/linux/cxl-event.h +++ b/include/linux/cxl-event.h @@ -8,7 +8,6 @@ * CXL rev 3.0 section 8.2.9.2.1; Table 8-42 */ struct cxl_event_record_hdr { - uuid_t id; u8 length; u8 flags[3]; __le16 handle; @@ -18,8 +17,13 @@ struct cxl_event_record_hdr { u8 reserved[15]; } __packed; +/* + * Common Event Record Format + * CXL rev 3.0 section 8.2.9.2.1; Table 8-42 + */ #define CXL_EVENT_RECORD_DATA_LENGTH 0x50 struct cxl_event_record_raw { + uuid_t id; struct cxl_event_record_hdr hdr; u8 data[CXL_EVENT_RECORD_DATA_LENGTH]; } __packed; diff --git a/tools/testing/cxl/test/mem.c b/tools/testing/cxl/test/mem.c index 22f91ecade25f..799ecd1d53c5a 100644 --- a/tools/testing/cxl/test/mem.c +++ b/tools/testing/cxl/test/mem.c @@ -338,9 +338,9 @@ static void cxl_mock_event_trigger(struct device *dev) } struct cxl_event_record_raw maint_needed = { + .id = UUID_INIT(0xBA5EBA11, 0xABCD, 0xEFEB, + 0xa5, 0x5a, 0xa5, 0x5a, 0xa5, 0xa5, 0x5a, 0xa5), .hdr = { - .id = UUID_INIT(0xBA5EBA11, 0xABCD, 0xEFEB, - 0xa5, 0x5a, 0xa5, 0x5a, 0xa5, 0xa5, 0x5a, 0xa5), .length = sizeof(struct cxl_event_record_raw), .flags[0] = CXL_EVENT_RECORD_FLAG_MAINT_NEEDED, /* .handle = Set dynamically */ @@ -350,9 +350,9 @@ struct cxl_event_record_raw maint_needed = { }; struct cxl_event_record_raw hardware_replace = { + .id = UUID_INIT(0xABCDEFEB, 0xBA11, 0xBA5E, + 0xa5, 0x5a, 0xa5, 0x5a, 0xa5, 0xa5, 0x5a, 0xa5), .hdr = { - .id = UUID_INIT(0xABCDEFEB, 0xBA11, 0xBA5E, - 0xa5, 0x5a, 0xa5, 0x5a, 0xa5, 0xa5, 0x5a, 0xa5), .length = sizeof(struct cxl_event_record_raw), .flags[0] = CXL_EVENT_RECORD_FLAG_HW_REPLACE, /* .handle = Set dynamically */ @@ -361,61 +361,82 @@ struct cxl_event_record_raw hardware_replace = { .data = { 0xDE, 0xAD, 0xBE, 0xEF }, }; -struct cxl_event_gen_media gen_media = { - .hdr = { - .id = CXL_EVENT_GEN_MEDIA_UUID, - .length = sizeof(struct cxl_event_gen_media), - .flags[0] = CXL_EVENT_RECORD_FLAG_PERMANENT, - /* .handle = Set dynamically */ - .related_handle = cpu_to_le16(0), +struct cxl_test_gen_media { + uuid_t id; + struct cxl_event_gen_media rec; +} __packed; + +struct cxl_test_gen_media gen_media = { + .id = CXL_EVENT_GEN_MEDIA_UUID, + .rec = { + .hdr = { + .length = sizeof(struct cxl_test_gen_media), + .flags[0] = CXL_EVENT_RECORD_FLAG_PERMANENT, + /* .handle = Set dynamically */ + .related_handle = cpu_to_le16(0), + }, + .phys_addr = cpu_to_le64(0x2000), + .descriptor = CXL_GMER_EVT_DESC_UNCORECTABLE_EVENT, + .type = CXL_GMER_MEM_EVT_TYPE_DATA_PATH_ERROR, + .transaction_type = CXL_GMER_TRANS_HOST_WRITE, + /* .validity_flags = */ + .channel = 1, + .rank = 30 }, - .phys_addr = cpu_to_le64(0x2000), - .descriptor = CXL_GMER_EVT_DESC_UNCORECTABLE_EVENT, - .type = CXL_GMER_MEM_EVT_TYPE_DATA_PATH_ERROR, - .transaction_type = CXL_GMER_TRANS_HOST_WRITE, - /* .validity_flags = */ - .channel = 1, - .rank = 30 }; -struct cxl_event_dram dram = { - .hdr = { - .id = CXL_EVENT_DRAM_UUID, - .length = sizeof(struct cxl_event_dram), - .flags[0] = CXL_EVENT_RECORD_FLAG_PERF_DEGRADED, - /* .handle = Set dynamically */ - .related_handle = cpu_to_le16(0), +struct cxl_test_dram { + uuid_t id; + struct cxl_event_dram rec; +} __packed; + +struct cxl_test_dram dram = { + .id = CXL_EVENT_DRAM_UUID, + .rec = { + .hdr = { + .length = sizeof(struct cxl_test_dram), + .flags[0] = CXL_EVENT_RECORD_FLAG_PERF_DEGRADED, + /* .handle = Set dynamically */ + .related_handle = cpu_to_le16(0), + }, + .phys_addr = cpu_to_le64(0x8000), + .descriptor = CXL_GMER_EVT_DESC_THRESHOLD_EVENT, + .type = CXL_GMER_MEM_EVT_TYPE_INV_ADDR, + .transaction_type = CXL_GMER_TRANS_INTERNAL_MEDIA_SCRUB, + /* .validity_flags = */ + .channel = 1, + .bank_group = 5, + .bank = 2, + .column = {0xDE, 0xAD}, }, - .phys_addr = cpu_to_le64(0x8000), - .descriptor = CXL_GMER_EVT_DESC_THRESHOLD_EVENT, - .type = CXL_GMER_MEM_EVT_TYPE_INV_ADDR, - .transaction_type = CXL_GMER_TRANS_INTERNAL_MEDIA_SCRUB, - /* .validity_flags = */ - .channel = 1, - .bank_group = 5, - .bank = 2, - .column = {0xDE, 0xAD}, }; -struct cxl_event_mem_module mem_module = { - .hdr = { - .id = CXL_EVENT_MEM_MODULE_UUID, - .length = sizeof(struct cxl_event_mem_module), - /* .handle = Set dynamically */ - .related_handle = cpu_to_le16(0), +struct cxl_test_mem_module { + uuid_t id; + struct cxl_event_mem_module rec; +} __packed; + +struct cxl_test_mem_module mem_module = { + .id = CXL_EVENT_MEM_MODULE_UUID, + .rec = { + .hdr = { + .length = sizeof(struct cxl_test_mem_module), + /* .handle = Set dynamically */ + .related_handle = cpu_to_le16(0), + }, + .event_type = CXL_MMER_TEMP_CHANGE, + .info = { + .health_status = CXL_DHI_HS_PERFORMANCE_DEGRADED, + .media_status = CXL_DHI_MS_ALL_DATA_LOST, + .add_status = (CXL_DHI_AS_CRITICAL << 2) | + (CXL_DHI_AS_WARNING << 4) | + (CXL_DHI_AS_WARNING << 5), + .device_temp = { 0xDE, 0xAD}, + .dirty_shutdown_cnt = { 0xde, 0xad, 0xbe, 0xef }, + .cor_vol_err_cnt = { 0xde, 0xad, 0xbe, 0xef }, + .cor_per_err_cnt = { 0xde, 0xad, 0xbe, 0xef }, + } }, - .event_type = CXL_MMER_TEMP_CHANGE, - .info = { - .health_status = CXL_DHI_HS_PERFORMANCE_DEGRADED, - .media_status = CXL_DHI_MS_ALL_DATA_LOST, - .add_status = (CXL_DHI_AS_CRITICAL << 2) | - (CXL_DHI_AS_WARNING << 4) | - (CXL_DHI_AS_WARNING << 5), - .device_temp = { 0xDE, 0xAD}, - .dirty_shutdown_cnt = { 0xde, 0xad, 0xbe, 0xef }, - .cor_vol_err_cnt = { 0xde, 0xad, 0xbe, 0xef }, - .cor_per_err_cnt = { 0xde, 0xad, 0xbe, 0xef }, - } }; static int mock_set_timestamp(struct cxl_dev_state *cxlds, @@ -437,11 +458,11 @@ static int mock_set_timestamp(struct cxl_dev_state *cxlds, static void cxl_mock_add_event_logs(struct mock_event_store *mes) { put_unaligned_le16(CXL_GMER_VALID_CHANNEL | CXL_GMER_VALID_RANK, - &gen_media.validity_flags); + &gen_media.rec.validity_flags); put_unaligned_le16(CXL_DER_VALID_CHANNEL | CXL_DER_VALID_BANK_GROUP | CXL_DER_VALID_BANK | CXL_DER_VALID_COLUMN, - &dram.validity_flags); + &dram.rec.validity_flags); mes_add_event(mes, CXL_EVENT_TYPE_INFO, &maint_needed); mes_add_event(mes, CXL_EVENT_TYPE_INFO, From 95113a6cb698089783426b85a7899504b10c3592 Mon Sep 17 00:00:00 2001 From: Ira Weiny Date: Wed, 20 Dec 2023 16:17:33 -0800 Subject: [PATCH 065/138] cxl/events: Create a CXL event union commit f9c683386f5bc0364615138ce2b14be50848dbcf upstream. The CXL CPER and event log records share everything but a UUID/GUID in their structures. Define a cxl_event union without the UUID/GUID to be shared between the CPER and event log record formats. Adjust the code to use this union. Signed-off-by: Ira Weiny Link: https://lore.kernel.org/r/20231220-cxl-cper-v5-6-1bb8a4ca2c7a@intel.com Reviewed-by: Jonathan Cameron Acked-by: Ard Biesheuvel Signed-off-by: Dan Williams Signed-off-by: Yi Liu Signed-off-by: Bo Li --- drivers/cxl/core/mbox.c | 32 +++++++++++++------------------- drivers/cxl/core/trace.h | 8 ++++---- include/linux/cxl-event.h | 23 +++++++++++++++++------ tools/testing/cxl/test/mem.c | 31 ++++++++++++++++++------------- 4 files changed, 52 insertions(+), 42 deletions(-) diff --git a/drivers/cxl/core/mbox.c b/drivers/cxl/core/mbox.c index 5a77d6f2c0c02..f3b983b9472d5 100644 --- a/drivers/cxl/core/mbox.c +++ b/drivers/cxl/core/mbox.c @@ -841,26 +841,17 @@ static void cxl_event_trace_record(const struct cxl_memdev *cxlmd, enum cxl_event_log_type type, struct cxl_event_record_raw *record) { + union cxl_event *evt = &record->event; uuid_t *id = &record->id; - if (uuid_equal(id, &CXL_EVENT_GEN_MEDIA_UUID)) { - struct cxl_event_gen_media *rec = - (struct cxl_event_gen_media *)record; - - trace_cxl_general_media(cxlmd, type, rec); - } else if (uuid_equal(id, &CXL_EVENT_DRAM_UUID)) { - struct cxl_event_dram *rec = (struct cxl_event_dram *)record; - - trace_cxl_dram(cxlmd, type, rec); - } else if (uuid_equal(id, &CXL_EVENT_MEM_MODULE_UUID)) { - struct cxl_event_mem_module *rec = - (struct cxl_event_mem_module *)record; - - trace_cxl_memory_module(cxlmd, type, rec); - } else { - /* For unknown record types print just the header */ - trace_cxl_generic_event(cxlmd, type, id, record); - } + if (uuid_equal(id, &CXL_EVENT_GEN_MEDIA_UUID)) + trace_cxl_general_media(cxlmd, type, &evt->gen_media); + else if (uuid_equal(id, &CXL_EVENT_DRAM_UUID)) + trace_cxl_dram(cxlmd, type, &evt->dram); + else if (uuid_equal(id, &CXL_EVENT_MEM_MODULE_UUID)) + trace_cxl_memory_module(cxlmd, type, &evt->mem_module); + else + trace_cxl_generic_event(cxlmd, type, id, &evt->generic); } static int cxl_clear_event_record(struct cxl_memdev_state *mds, @@ -903,7 +894,10 @@ static int cxl_clear_event_record(struct cxl_memdev_state *mds, */ i = 0; for (cnt = 0; cnt < total; cnt++) { - payload->handles[i++] = get_pl->records[cnt].hdr.handle; + struct cxl_event_record_raw *raw = &get_pl->records[cnt]; + struct cxl_event_generic *gen = &raw->event.generic; + + payload->handles[i++] = gen->hdr.handle; dev_dbg(mds->cxlds.dev, "Event log '%d': Clearing %u\n", log, le16_to_cpu(payload->handles[i - 1])); diff --git a/drivers/cxl/core/trace.h b/drivers/cxl/core/trace.h index 13a821042c8ce..e110125bc75c2 100644 --- a/drivers/cxl/core/trace.h +++ b/drivers/cxl/core/trace.h @@ -225,9 +225,9 @@ TRACE_EVENT(cxl_overflow, TRACE_EVENT(cxl_generic_event, TP_PROTO(const struct cxl_memdev *cxlmd, enum cxl_event_log_type log, - const uuid_t *uuid, struct cxl_event_record_raw *rec), + const uuid_t *uuid, struct cxl_event_generic *gen_rec), - TP_ARGS(cxlmd, log, uuid, rec), + TP_ARGS(cxlmd, log, uuid, gen_rec), TP_STRUCT__entry( CXL_EVT_TP_entry @@ -235,9 +235,9 @@ TRACE_EVENT(cxl_generic_event, ), TP_fast_assign( - CXL_EVT_TP_fast_assign(cxlmd, log, rec->hdr); + CXL_EVT_TP_fast_assign(cxlmd, log, gen_rec->hdr); memcpy(&__entry->hdr_uuid, uuid, sizeof(uuid_t)); - memcpy(__entry->data, &rec->data, CXL_EVENT_RECORD_DATA_LENGTH); + memcpy(__entry->data, gen_rec->data, CXL_EVENT_RECORD_DATA_LENGTH); ), CXL_EVT_TP_printk("%s", diff --git a/include/linux/cxl-event.h b/include/linux/cxl-event.h index 3d9b5954d0c11..4d6c05f535f80 100644 --- a/include/linux/cxl-event.h +++ b/include/linux/cxl-event.h @@ -17,13 +17,8 @@ struct cxl_event_record_hdr { u8 reserved[15]; } __packed; -/* - * Common Event Record Format - * CXL rev 3.0 section 8.2.9.2.1; Table 8-42 - */ #define CXL_EVENT_RECORD_DATA_LENGTH 0x50 -struct cxl_event_record_raw { - uuid_t id; +struct cxl_event_generic { struct cxl_event_record_hdr hdr; u8 data[CXL_EVENT_RECORD_DATA_LENGTH]; } __packed; @@ -96,4 +91,20 @@ struct cxl_event_mem_module { u8 reserved[0x3d]; } __packed; +union cxl_event { + struct cxl_event_generic generic; + struct cxl_event_gen_media gen_media; + struct cxl_event_dram dram; + struct cxl_event_mem_module mem_module; +} __packed; + +/* + * Common Event Record Format; in event logs + * CXL rev 3.0 section 8.2.9.2.1; Table 8-42 + */ +struct cxl_event_record_raw { + uuid_t id; + union cxl_event event; +} __packed; + #endif /* _LINUX_CXL_EVENT_H */ diff --git a/tools/testing/cxl/test/mem.c b/tools/testing/cxl/test/mem.c index 799ecd1d53c5a..0d0ca63de8629 100644 --- a/tools/testing/cxl/test/mem.c +++ b/tools/testing/cxl/test/mem.c @@ -252,7 +252,8 @@ static int mock_get_event(struct device *dev, struct cxl_mbox_cmd *cmd) for (i = 0; i < CXL_TEST_EVENT_CNT && !event_log_empty(log); i++) { memcpy(&pl->records[i], event_get_current(log), sizeof(pl->records[i])); - pl->records[i].hdr.handle = event_get_cur_event_handle(log); + pl->records[i].event.generic.hdr.handle = + event_get_cur_event_handle(log); log->cur_idx++; } @@ -340,25 +341,29 @@ static void cxl_mock_event_trigger(struct device *dev) struct cxl_event_record_raw maint_needed = { .id = UUID_INIT(0xBA5EBA11, 0xABCD, 0xEFEB, 0xa5, 0x5a, 0xa5, 0x5a, 0xa5, 0xa5, 0x5a, 0xa5), - .hdr = { - .length = sizeof(struct cxl_event_record_raw), - .flags[0] = CXL_EVENT_RECORD_FLAG_MAINT_NEEDED, - /* .handle = Set dynamically */ - .related_handle = cpu_to_le16(0xa5b6), + .event.generic = { + .hdr = { + .length = sizeof(struct cxl_event_record_raw), + .flags[0] = CXL_EVENT_RECORD_FLAG_MAINT_NEEDED, + /* .handle = Set dynamically */ + .related_handle = cpu_to_le16(0xa5b6), + }, + .data = { 0xDE, 0xAD, 0xBE, 0xEF }, }, - .data = { 0xDE, 0xAD, 0xBE, 0xEF }, }; struct cxl_event_record_raw hardware_replace = { .id = UUID_INIT(0xABCDEFEB, 0xBA11, 0xBA5E, 0xa5, 0x5a, 0xa5, 0x5a, 0xa5, 0xa5, 0x5a, 0xa5), - .hdr = { - .length = sizeof(struct cxl_event_record_raw), - .flags[0] = CXL_EVENT_RECORD_FLAG_HW_REPLACE, - /* .handle = Set dynamically */ - .related_handle = cpu_to_le16(0xb6a5), + .event.generic = { + .hdr = { + .length = sizeof(struct cxl_event_record_raw), + .flags[0] = CXL_EVENT_RECORD_FLAG_HW_REPLACE, + /* .handle = Set dynamically */ + .related_handle = cpu_to_le16(0xb6a5), + }, + .data = { 0xDE, 0xAD, 0xBE, 0xEF }, }, - .data = { 0xDE, 0xAD, 0xBE, 0xEF }, }; struct cxl_test_gen_media { From f7bec55906c8949d17b7acb51cc45421b1023826 Mon Sep 17 00:00:00 2001 From: Ira Weiny Date: Wed, 20 Dec 2023 16:17:34 -0800 Subject: [PATCH 066/138] acpi/ghes: Process CXL Component Events commit 671a794c33c6e048ca5cedd5ad6af44d52d5d7e5 upstream. BIOS can configure memory devices as firmware first. This will send CXL events to the firmware instead of the OS. The firmware can then send these events to the OS via UEFI. UEFI v2.10 section N.2.14 defines a Common Platform Error Record (CPER) format for CXL Component Events. The format is mostly the same as the CXL Common Event Record Format. The difference is the use of a GUID in the Section Type rather than a UUID as part of the event itself. Add GHES support to detect CXL CPER records and call a registered callback with the event. A notifier chain was considered for the callback but the complexity did not justify the use case as only the CXL subsystem requires this event. Enforce that only one callback can be registered at any time. Cc: Ard Biesheuvel Cc: Rafael J. Wysocki Signed-off-by: Ira Weiny Link: https://lore.kernel.org/r/20231220-cxl-cper-v5-7-1bb8a4ca2c7a@intel.com [djbw: fixup checkpatch errors] Reviewed-by: Jonathan Cameron Acked-by: Ard Biesheuvel Signed-off-by: Dan Williams Signed-off-by: Yi Liu Signed-off-by: Bo Li --- drivers/acpi/apei/ghes.c | 89 +++++++++++++++++++++++++++++++++++++++ include/linux/cxl-event.h | 50 ++++++++++++++++++++++ 2 files changed, 139 insertions(+) diff --git a/drivers/acpi/apei/ghes.c b/drivers/acpi/apei/ghes.c index e768dfd345fb2..9677ae6835151 100644 --- a/drivers/acpi/apei/ghes.c +++ b/drivers/acpi/apei/ghes.c @@ -27,6 +27,7 @@ #include #include #include +#include #include #include #include @@ -675,6 +676,78 @@ static void ghes_defer_non_standard_event(struct acpi_hest_generic_data *gdata, schedule_work(&entry->work); } +/* + * Only a single callback can be registered for CXL CPER events. + */ +static DECLARE_RWSEM(cxl_cper_rw_sem); +static cxl_cper_callback cper_callback; + +/* CXL Event record UUIDs are formatted as GUIDs and reported in section type */ + +/* + * General Media Event Record + * CXL rev 3.0 Section 8.2.9.2.1.1; Table 8-43 + */ +#define CPER_SEC_CXL_GEN_MEDIA_GUID \ + GUID_INIT(0xfbcd0a77, 0xc260, 0x417f, \ + 0x85, 0xa9, 0x08, 0x8b, 0x16, 0x21, 0xeb, 0xa6) + +/* + * DRAM Event Record + * CXL rev 3.0 section 8.2.9.2.1.2; Table 8-44 + */ +#define CPER_SEC_CXL_DRAM_GUID \ + GUID_INIT(0x601dcbb3, 0x9c06, 0x4eab, \ + 0xb8, 0xaf, 0x4e, 0x9b, 0xfb, 0x5c, 0x96, 0x24) + +/* + * Memory Module Event Record + * CXL rev 3.0 section 8.2.9.2.1.3; Table 8-45 + */ +#define CPER_SEC_CXL_MEM_MODULE_GUID \ + GUID_INIT(0xfe927475, 0xdd59, 0x4339, \ + 0xa5, 0x86, 0x79, 0xba, 0xb1, 0x13, 0xb7, 0x74) + +static void cxl_cper_post_event(enum cxl_event_type event_type, + struct cxl_cper_event_rec *rec) +{ + if (rec->hdr.length <= sizeof(rec->hdr) || + rec->hdr.length > sizeof(*rec)) { + pr_err(FW_WARN "CXL CPER Invalid section length (%u)\n", + rec->hdr.length); + return; + } + + if (!(rec->hdr.validation_bits & CPER_CXL_COMP_EVENT_LOG_VALID)) { + pr_err(FW_WARN "CXL CPER invalid event\n"); + return; + } + + guard(rwsem_read)(&cxl_cper_rw_sem); + if (cper_callback) + cper_callback(event_type, rec); +} + +int cxl_cper_register_callback(cxl_cper_callback callback) +{ + guard(rwsem_write)(&cxl_cper_rw_sem); + if (cper_callback) + return -EINVAL; + cper_callback = callback; + return 0; +} +EXPORT_SYMBOL_GPL(cxl_cper_register_callback); + +int cxl_cper_unregister_callback(cxl_cper_callback callback) +{ + guard(rwsem_write)(&cxl_cper_rw_sem); + if (callback != cper_callback) + return -EINVAL; + cper_callback = NULL; + return 0; +} +EXPORT_SYMBOL_GPL(cxl_cper_unregister_callback); + static bool ghes_do_proc(struct ghes *ghes, const struct acpi_hest_generic_status *estatus) { @@ -709,6 +782,22 @@ static bool ghes_do_proc(struct ghes *ghes, } else if (guid_equal(sec_type, &CPER_SEC_PROC_ARM)) { queued = ghes_handle_arm_hw_error(gdata, sev, sync); + } else if (guid_equal(sec_type, &CPER_SEC_CXL_GEN_MEDIA_GUID)) { + struct cxl_cper_event_rec *rec = + acpi_hest_get_payload(gdata); + + cxl_cper_post_event(CXL_CPER_EVENT_GEN_MEDIA, rec); + } else if (guid_equal(sec_type, &CPER_SEC_CXL_DRAM_GUID)) { + struct cxl_cper_event_rec *rec = + acpi_hest_get_payload(gdata); + + cxl_cper_post_event(CXL_CPER_EVENT_DRAM, rec); + } else if (guid_equal(sec_type, + &CPER_SEC_CXL_MEM_MODULE_GUID)) { + struct cxl_cper_event_rec *rec = + acpi_hest_get_payload(gdata); + + cxl_cper_post_event(CXL_CPER_EVENT_MEM_MODULE, rec); } else { void *err = acpi_hest_get_payload(gdata); diff --git a/include/linux/cxl-event.h b/include/linux/cxl-event.h index 4d6c05f535f80..95841750a383b 100644 --- a/include/linux/cxl-event.h +++ b/include/linux/cxl-event.h @@ -107,4 +107,54 @@ struct cxl_event_record_raw { union cxl_event event; } __packed; +enum cxl_event_type { + CXL_CPER_EVENT_GEN_MEDIA, + CXL_CPER_EVENT_DRAM, + CXL_CPER_EVENT_MEM_MODULE, +}; + +#define CPER_CXL_DEVICE_ID_VALID BIT(0) +#define CPER_CXL_DEVICE_SN_VALID BIT(1) +#define CPER_CXL_COMP_EVENT_LOG_VALID BIT(2) +struct cxl_cper_event_rec { + struct { + u32 length; + u64 validation_bits; + struct cper_cxl_event_devid { + u16 vendor_id; + u16 device_id; + u8 func_num; + u8 device_num; + u8 bus_num; + u16 segment_num; + u16 slot_num; /* bits 2:0 reserved */ + u8 reserved; + } __packed device_id; + struct cper_cxl_event_sn { + u32 lower_dw; + u32 upper_dw; + } __packed dev_serial_num; + } __packed hdr; + + union cxl_event event; +} __packed; + +typedef void (*cxl_cper_callback)(enum cxl_event_type type, + struct cxl_cper_event_rec *rec); + +#ifdef CONFIG_ACPI_APEI_GHES +int cxl_cper_register_callback(cxl_cper_callback callback); +int cxl_cper_unregister_callback(cxl_cper_callback callback); +#else +static inline int cxl_cper_register_callback(cxl_cper_callback callback) +{ + return 0; +} + +static inline int cxl_cper_unregister_callback(cxl_cper_callback callback) +{ + return 0; +} +#endif + #endif /* _LINUX_CXL_EVENT_H */ From c47253ea8bca099cafd756a354537a77b8e1d94a Mon Sep 17 00:00:00 2001 From: Ira Weiny Date: Wed, 20 Dec 2023 16:17:36 -0800 Subject: [PATCH 067/138] cxl/pci: Register for and process CPER events commit dc97f6344f205b0dfa144e1b3e16d6dc05383d57 upstream. If the firmware has configured CXL event support to be firmware first the OS can process those events through CPER records. The CXL layer has unique DPA to HPA knowledge and standard event trace parsing in place. CPER records contain Bus, Device, Function information which can be used to identify the PCI device which is sending the event. Change the PCI driver registration to include registration of a CXL CPER callback to process events through the trace subsystem. Use new scoped based management to simplify the handling of the PCI device object. Tested-by: Smita-Koralahalli Reviewed-by: Smita-Koralahalli Link: https://lore.kernel.org/r/20231220-cxl-cper-v5-9-1bb8a4ca2c7a@intel.com Signed-off-by: Ira Weiny [djbw: use new pci_dev guard, flip init order] Reviewed-by: Jonathan Cameron Acked-by: Ard Biesheuvel Signed-off-by: Dan Williams Signed-off-by: Yi Liu Signed-off-by: Bo Li --- drivers/cxl/core/mbox.c | 40 +++++++++++++++++++-------- drivers/cxl/cxlmem.h | 4 +++ drivers/cxl/pci.c | 58 ++++++++++++++++++++++++++++++++++++++- include/linux/cxl-event.h | 1 + 4 files changed, 90 insertions(+), 13 deletions(-) diff --git a/drivers/cxl/core/mbox.c b/drivers/cxl/core/mbox.c index f3b983b9472d5..7dc2d6a3266d1 100644 --- a/drivers/cxl/core/mbox.c +++ b/drivers/cxl/core/mbox.c @@ -837,21 +837,37 @@ int cxl_enumerate_cmds(struct cxl_memdev_state *mds) } EXPORT_SYMBOL_NS_GPL(cxl_enumerate_cmds, CXL); -static void cxl_event_trace_record(const struct cxl_memdev *cxlmd, - enum cxl_event_log_type type, - struct cxl_event_record_raw *record) +void cxl_event_trace_record(const struct cxl_memdev *cxlmd, + enum cxl_event_log_type type, + enum cxl_event_type event_type, + const uuid_t *uuid, union cxl_event *evt) { - union cxl_event *evt = &record->event; - uuid_t *id = &record->id; - - if (uuid_equal(id, &CXL_EVENT_GEN_MEDIA_UUID)) + if (event_type == CXL_CPER_EVENT_GEN_MEDIA) trace_cxl_general_media(cxlmd, type, &evt->gen_media); - else if (uuid_equal(id, &CXL_EVENT_DRAM_UUID)) + else if (event_type == CXL_CPER_EVENT_DRAM) trace_cxl_dram(cxlmd, type, &evt->dram); - else if (uuid_equal(id, &CXL_EVENT_MEM_MODULE_UUID)) + else if (event_type == CXL_CPER_EVENT_MEM_MODULE) trace_cxl_memory_module(cxlmd, type, &evt->mem_module); else - trace_cxl_generic_event(cxlmd, type, id, &evt->generic); + trace_cxl_generic_event(cxlmd, type, uuid, &evt->generic); +} +EXPORT_SYMBOL_NS_GPL(cxl_event_trace_record, CXL); + +static void __cxl_event_trace_record(const struct cxl_memdev *cxlmd, + enum cxl_event_log_type type, + struct cxl_event_record_raw *record) +{ + enum cxl_event_type ev_type = CXL_CPER_EVENT_GENERIC; + const uuid_t *uuid = &record->id; + + if (uuid_equal(uuid, &CXL_EVENT_GEN_MEDIA_UUID)) + ev_type = CXL_CPER_EVENT_GEN_MEDIA; + else if (uuid_equal(uuid, &CXL_EVENT_DRAM_UUID)) + ev_type = CXL_CPER_EVENT_DRAM; + else if (uuid_equal(uuid, &CXL_EVENT_MEM_MODULE_UUID)) + ev_type = CXL_CPER_EVENT_MEM_MODULE; + + cxl_event_trace_record(cxlmd, type, ev_type, uuid, &record->event); } static int cxl_clear_event_record(struct cxl_memdev_state *mds, @@ -960,8 +976,8 @@ static void cxl_mem_get_records_log(struct cxl_memdev_state *mds, break; for (i = 0; i < nr_rec; i++) - cxl_event_trace_record(cxlmd, type, - &payload->records[i]); + __cxl_event_trace_record(cxlmd, type, + &payload->records[i]); if (payload->flags & CXL_GET_EVENT_FLAG_OVERFLOW) trace_cxl_overflow(cxlmd, type, payload); diff --git a/drivers/cxl/cxlmem.h b/drivers/cxl/cxlmem.h index 6a88cdcab9501..0e9c2fb229c3d 100644 --- a/drivers/cxl/cxlmem.h +++ b/drivers/cxl/cxlmem.h @@ -824,6 +824,10 @@ void set_exclusive_cxl_commands(struct cxl_memdev_state *mds, void clear_exclusive_cxl_commands(struct cxl_memdev_state *mds, unsigned long *cmds); void cxl_mem_get_event_records(struct cxl_memdev_state *mds, u32 status); +void cxl_event_trace_record(const struct cxl_memdev *cxlmd, + enum cxl_event_log_type type, + enum cxl_event_type event_type, + const uuid_t *uuid, union cxl_event *evt); int cxl_set_timestamp(struct cxl_memdev_state *mds); int cxl_poison_state_init(struct cxl_memdev_state *mds); int cxl_mem_get_poison(struct cxl_memdev *cxlmd, u64 offset, u64 len, diff --git a/drivers/cxl/pci.c b/drivers/cxl/pci.c index 2d8ff9de000f3..1d9dd6b019df7 100644 --- a/drivers/cxl/pci.c +++ b/drivers/cxl/pci.c @@ -1,5 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-only /* Copyright(c) 2020 Intel Corporation. All rights reserved. */ +#include #include #include #include @@ -968,6 +969,61 @@ static struct pci_driver cxl_pci_driver = { }, }; +#define CXL_EVENT_HDR_FLAGS_REC_SEVERITY GENMASK(1, 0) +static void cxl_cper_event_call(enum cxl_event_type ev_type, + struct cxl_cper_event_rec *rec) +{ + struct cper_cxl_event_devid *device_id = &rec->hdr.device_id; + struct pci_dev *pdev __free(pci_dev_put) = NULL; + enum cxl_event_log_type log_type; + struct cxl_dev_state *cxlds; + unsigned int devfn; + u32 hdr_flags; + + devfn = PCI_DEVFN(device_id->device_num, device_id->func_num); + pdev = pci_get_domain_bus_and_slot(device_id->segment_num, + device_id->bus_num, devfn); + if (!pdev) + return; + + guard(pci_dev)(pdev); + if (pdev->driver != &cxl_pci_driver) + return; + + cxlds = pci_get_drvdata(pdev); + if (!cxlds) + return; + + /* Fabricate a log type */ + hdr_flags = get_unaligned_le24(rec->event.generic.hdr.flags); + log_type = FIELD_GET(CXL_EVENT_HDR_FLAGS_REC_SEVERITY, hdr_flags); + + cxl_event_trace_record(cxlds->cxlmd, log_type, ev_type, + &uuid_null, &rec->event); +} + +static int __init cxl_pci_driver_init(void) +{ + int rc; + + rc = cxl_cper_register_callback(cxl_cper_event_call); + if (rc) + return rc; + + rc = pci_register_driver(&cxl_pci_driver); + if (rc) + cxl_cper_unregister_callback(cxl_cper_event_call); + + return rc; +} + +static void __exit cxl_pci_driver_exit(void) +{ + pci_unregister_driver(&cxl_pci_driver); + cxl_cper_unregister_callback(cxl_cper_event_call); +} + +module_init(cxl_pci_driver_init); +module_exit(cxl_pci_driver_exit); MODULE_LICENSE("GPL v2"); -module_pci_driver(cxl_pci_driver); MODULE_IMPORT_NS(CXL); diff --git a/include/linux/cxl-event.h b/include/linux/cxl-event.h index 95841750a383b..91125eca4c8ab 100644 --- a/include/linux/cxl-event.h +++ b/include/linux/cxl-event.h @@ -108,6 +108,7 @@ struct cxl_event_record_raw { } __packed; enum cxl_event_type { + CXL_CPER_EVENT_GENERIC, CXL_CPER_EVENT_GEN_MEDIA, CXL_CPER_EVENT_DRAM, CXL_CPER_EVENT_MEM_MODULE, From b5041381270042062b06dc725922ddd7ee0255f3 Mon Sep 17 00:00:00 2001 From: Shiyang Ruan Date: Fri, 12 Jan 2024 14:27:09 +0800 Subject: [PATCH 068/138] cxl/core: use sysfs_emit() for attr's _show() commit 73bf93edeeea866b0b6efbc8d2595bdaaba7f1a5 upstream. sprintf() is deprecated for sysfs, use preferred sysfs_emit() instead. Signed-off-by: Shiyang Ruan Reviewed-by: Jonathan Cameron Reviewed-by: Dave Jiang Reviewed-by: Fan Ni Link: https://lore.kernel.org/r/20240112062709.2490947-1-ruansy.fnst@fujitsu.com Signed-off-by: Dan Williams Signed-off-by: Yi Liu Signed-off-by: Bo Li --- drivers/cxl/core/memdev.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/cxl/core/memdev.c b/drivers/cxl/core/memdev.c index eb895c70043fd..f607ab96ba9d4 100644 --- a/drivers/cxl/core/memdev.c +++ b/drivers/cxl/core/memdev.c @@ -114,7 +114,7 @@ static DEVICE_ATTR_RO(serial); static ssize_t numa_node_show(struct device *dev, struct device_attribute *attr, char *buf) { - return sprintf(buf, "%d\n", dev_to_node(dev)); + return sysfs_emit(buf, "%d\n", dev_to_node(dev)); } static DEVICE_ATTR_RO(numa_node); From 539ec1af2b5648e68db0805bd0b03cd82d98e839 Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Tue, 16 Jan 2024 13:17:17 -0800 Subject: [PATCH 069/138] tools/testing/cxl: Disable "missing prototypes / declarations" warnings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit commit 68deb9972079c9904fe714c049a7f08bd997a9ee upstream. Prevent warnings of the form: tools/testing/cxl/test/mock.c:44:6: error: no previous prototype for ‘__wrap_is_acpi_device_node’ [-Werror=missing-prototypes] tools/testing/cxl/test/mock.c:63:5: error: no previous prototype for ‘__wrap_acpi_table_parse_cedt’ [-Werror=missing-prototypes] tools/testing/cxl/test/mock.c:81:13: error: no previous prototype for ‘__wrap_acpi_evaluate_integer’ [-Werror=missing-prototypes] ...by locally disabling some warnings. It turns out that: Commit 0fcb70851fbf ("Makefile.extrawarn: turn on missing-prototypes globally") ...in addition to expanding in-tree coverage, also impacts out-of-tree module builds like those in tools/testing/cxl/. Filter out the warning options on unit test code that does not effect mainline builds. Reviewed-by: Alison Schofield Link: https://lore.kernel.org/r/170543983780.460832.10920261849128601697.stgit@dwillia2-xfh.jf.intel.com Signed-off-by: Dan Williams Signed-off-by: Yi Liu Signed-off-by: Bo Li --- tools/testing/cxl/Kbuild | 2 ++ tools/testing/cxl/test/Kbuild | 2 ++ 2 files changed, 4 insertions(+) diff --git a/tools/testing/cxl/Kbuild b/tools/testing/cxl/Kbuild index 0b12c36902d82..caff3834671f9 100644 --- a/tools/testing/cxl/Kbuild +++ b/tools/testing/cxl/Kbuild @@ -65,4 +65,6 @@ cxl_core-y += config_check.o cxl_core-y += cxl_core_test.o cxl_core-y += cxl_core_exports.o +KBUILD_CFLAGS := $(filter-out -Wmissing-prototypes -Wmissing-declarations, $(KBUILD_CFLAGS)) + obj-m += test/ diff --git a/tools/testing/cxl/test/Kbuild b/tools/testing/cxl/test/Kbuild index 61d5f7bcddf9a..6b19278978561 100644 --- a/tools/testing/cxl/test/Kbuild +++ b/tools/testing/cxl/test/Kbuild @@ -8,3 +8,5 @@ obj-m += cxl_mock_mem.o cxl_test-y := cxl.o cxl_mock-y := mock.o cxl_mock_mem-y := mem.o + +KBUILD_CFLAGS := $(filter-out -Wmissing-prototypes -Wmissing-declarations, $(KBUILD_CFLAGS)) From ad0c65d41daf8cf298d5a7564cb9d6ffc9131231 Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Tue, 16 Jan 2024 13:17:23 -0800 Subject: [PATCH 070/138] tools/testing/nvdimm: Disable "missing prototypes / declarations" warnings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit commit c97dac57c804b53eab492ca0230d86356729d633 upstream. Prevent warnings of the form: tools/testing/nvdimm/config_check.c:4:6: error: no previous prototype for ‘check’ [-Werror=missing-prototypes] ...by locally disabling some warnings. It turns out that: Commit 0fcb70851fbf ("Makefile.extrawarn: turn on missing-prototypes globally") ...in addition to expanding in-tree coverage, also impacts out-of-tree module builds like those in tools/testing/nvdimm/. Filter out the warning options on unit test code that does not effect mainline builds. Reviewed-by: Alison Schofield Link: https://lore.kernel.org/r/170543984331.460832.1780246477583036191.stgit@dwillia2-xfh.jf.intel.com Signed-off-by: Dan Williams Signed-off-by: Yi Liu Signed-off-by: Bo Li --- tools/testing/nvdimm/Kbuild | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/testing/nvdimm/Kbuild b/tools/testing/nvdimm/Kbuild index 8153251ea389a..91a3627f301a7 100644 --- a/tools/testing/nvdimm/Kbuild +++ b/tools/testing/nvdimm/Kbuild @@ -82,4 +82,6 @@ libnvdimm-$(CONFIG_NVDIMM_KEYS) += $(NVDIMM_SRC)/security.o libnvdimm-y += libnvdimm_test.o libnvdimm-y += config_check.o +KBUILD_CFLAGS := $(filter-out -Wmissing-prototypes -Wmissing-declarations, $(KBUILD_CFLAGS)) + obj-m += test/ From d4be63d1d68512c06552d085e42a174b8b2143fd Mon Sep 17 00:00:00 2001 From: Ira Weiny Date: Wed, 17 Jan 2024 17:24:01 -0800 Subject: [PATCH 071/138] cxl/pci: Skip irq features if MSI/MSI-X are not supported commit d72a4caf685989e353dff0a97d7376ee10edbf87 upstream. CXL 3.1 Section 3.1.1 states: "A Function on a CXL device must not generate INTx messages if that Function participates in CXL.cache protocol or CXL.mem protocols." The generic CXL memory driver only supports devices which use the CXL.mem protocol. The current driver attempts to allocate MSI/MSI-X vectors in anticipation of their need for mailbox interrupts or event processing. However, the above requirement does not require a device to support interrupts, only that they use MSI/MSI-X. For example, a device may disable mailbox interrupts and either be configured for firmware first or skip event processing and function. Dave Larsen reported that the following Intel / Agilex card does not support interrupts on function 0. CXL: Intel Corporation Device 0ddb (rev 01) (prog-if 10 [CXL Memory Device (CXL 2.x)]) Rather than fail device probe if interrupts are not supported; flag that irqs are not enabled and avoid features which require interrupts. Emit messages appropriate for the situation to aid in debugging should device behavior be unexpected due to a failure to allocate vectors. Note that it is possible for a device to have host based event processing through polling. However, the driver does not support polling and it is not anticipated to be generally required. Leave that functionality to a future patch if such a device comes along. Reported-by: Dave Larsen Reviewed-by: Dave Jiang Reviewed-by: Fan Ni Signed-off-by: Ira Weiny Reviewed-and-tested-by: Davidlohr Bueso Link: https://lore.kernel.org/r/20240117-dont-fail-irq-v2-1-f33f26b0e365@intel.com Signed-off-by: Dan Williams Signed-off-by: Yi Liu Signed-off-by: Bo Li --- drivers/cxl/pci.c | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/drivers/cxl/pci.c b/drivers/cxl/pci.c index 1d9dd6b019df7..5d2672d767e9f 100644 --- a/drivers/cxl/pci.c +++ b/drivers/cxl/pci.c @@ -382,7 +382,7 @@ static int cxl_pci_mbox_send(struct cxl_memdev_state *mds, return rc; } -static int cxl_pci_setup_mailbox(struct cxl_memdev_state *mds) +static int cxl_pci_setup_mailbox(struct cxl_memdev_state *mds, bool irq_avail) { struct cxl_dev_state *cxlds = &mds->cxlds; const int cap = readl(cxlds->regs.mbox + CXLDEV_MBOX_CAPS_OFFSET); @@ -441,7 +441,7 @@ static int cxl_pci_setup_mailbox(struct cxl_memdev_state *mds) INIT_DELAYED_WORK(&mds->security.poll_dwork, cxl_mbox_sanitize_work); /* background command interrupts are optional */ - if (!(cap & CXLDEV_MBOX_CAP_BG_CMD_IRQ)) + if (!(cap & CXLDEV_MBOX_CAP_BG_CMD_IRQ) || !irq_avail) return 0; msgnum = FIELD_GET(CXLDEV_MBOX_CAP_IRQ_MSGNUM_MASK, cap); @@ -588,7 +588,7 @@ static int cxl_mem_alloc_event_buf(struct cxl_memdev_state *mds) return devm_add_action_or_reset(mds->cxlds.dev, free_event_buf, buf); } -static int cxl_alloc_irq_vectors(struct pci_dev *pdev) +static bool cxl_alloc_irq_vectors(struct pci_dev *pdev) { int nvecs; @@ -605,9 +605,9 @@ static int cxl_alloc_irq_vectors(struct pci_dev *pdev) PCI_IRQ_MSIX | PCI_IRQ_MSI); if (nvecs < 1) { dev_dbg(&pdev->dev, "Failed to alloc irq vectors: %d\n", nvecs); - return -ENXIO; + return false; } - return 0; + return true; } static irqreturn_t cxl_event_thread(int irq, void *id) @@ -743,7 +743,7 @@ static bool cxl_event_int_is_fw(u8 setting) } static int cxl_event_config(struct pci_host_bridge *host_bridge, - struct cxl_memdev_state *mds) + struct cxl_memdev_state *mds, bool irq_avail) { struct cxl_event_interrupt_policy policy; int rc; @@ -755,6 +755,11 @@ static int cxl_event_config(struct pci_host_bridge *host_bridge, if (!host_bridge->native_cxl_error) return 0; + if (!irq_avail) { + dev_info(mds->cxlds.dev, "No interrupt support, disable event processing.\n"); + return 0; + } + rc = cxl_mem_alloc_event_buf(mds); if (rc) return rc; @@ -789,6 +794,7 @@ static int cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) struct cxl_register_map map; struct cxl_memdev *cxlmd; int i, rc, pmu_count; + bool irq_avail; /* * Double check the anonymous union trickery in struct cxl_regs @@ -846,11 +852,9 @@ static int cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) else dev_warn(&pdev->dev, "Media not active (%d)\n", rc); - rc = cxl_alloc_irq_vectors(pdev); - if (rc) - return rc; + irq_avail = cxl_alloc_irq_vectors(pdev); - rc = cxl_pci_setup_mailbox(mds); + rc = cxl_pci_setup_mailbox(mds, irq_avail); if (rc) return rc; @@ -909,7 +913,7 @@ static int cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) } } - rc = cxl_event_config(host_bridge, mds); + rc = cxl_event_config(host_bridge, mds, irq_avail); if (rc) return rc; From 35e06b59ddf845fd8b489b97a275eabf3672a8f7 Mon Sep 17 00:00:00 2001 From: Li Ming Date: Mon, 29 Jan 2024 13:18:56 +0000 Subject: [PATCH 072/138] cxl/pci: Skip to handle RAS errors if CXL.mem device is detached commit eef5c7b28dbecd6b141987a96db6c54e49828102 upstream. The PCI AER model is an awkward fit for CXL error handling. While the expectation is that a PCI device can escalate to link reset to recover from an AER event, the same reset on CXL amounts to a surprise memory hotplug of massive amounts of memory. At present, the CXL error handler attempts some optimistic error handling to unbind the device from the cxl_mem driver after reaping some RAS register values. This results in a "hopeful" attempt to unplug the memory, but there is no guarantee that will succeed. A subsequent AER notification after the memdev unbind event can no longer assume the registers are mapped. Check for memdev bind before reaping status register values to avoid crashes of the form: BUG: unable to handle page fault for address: ffa00000195e9100 #PF: supervisor read access in kernel mode #PF: error_code(0x0000) - not-present page [...] RIP: 0010:__cxl_handle_ras+0x30/0x110 [cxl_core] [...] Call Trace: ? __die+0x24/0x70 ? page_fault_oops+0x82/0x160 ? kernelmode_fixup_or_oops+0x84/0x110 ? exc_page_fault+0x113/0x170 ? asm_exc_page_fault+0x26/0x30 ? __pfx_dpc_reset_link+0x10/0x10 ? __cxl_handle_ras+0x30/0x110 [cxl_core] ? find_cxl_port+0x59/0x80 [cxl_core] cxl_handle_rp_ras+0xbc/0xd0 [cxl_core] cxl_error_detected+0x6c/0xf0 [cxl_core] report_error_detected+0xc7/0x1c0 pci_walk_bus+0x73/0x90 pcie_do_recovery+0x23f/0x330 Longer term, the unbind and PCI_ERS_RESULT_DISCONNECT behavior might need to be replaced with a new PCI_ERS_RESULT_PANIC. Fixes: 6ac07883dbb5 ("cxl/pci: Add RCH downstream port error logging") Cc: stable@vger.kernel.org Suggested-by: Dan Williams Signed-off-by: Li Ming Link: https://lore.kernel.org/r/20240129131856.2458980-1-ming4.li@intel.com Signed-off-by: Dan Williams Signed-off-by: Yi Liu Signed-off-by: Bo Li --- drivers/cxl/core/pci.c | 43 ++++++++++++++++++++++++++++++------------ 1 file changed, 31 insertions(+), 12 deletions(-) diff --git a/drivers/cxl/core/pci.c b/drivers/cxl/core/pci.c index d2803e114c5c4..47e980e611c50 100644 --- a/drivers/cxl/core/pci.c +++ b/drivers/cxl/core/pci.c @@ -926,11 +926,21 @@ static void cxl_handle_rdport_errors(struct cxl_dev_state *cxlds) { } void cxl_cor_error_detected(struct pci_dev *pdev) { struct cxl_dev_state *cxlds = pci_get_drvdata(pdev); + struct device *dev = &cxlds->cxlmd->dev; + + scoped_guard(device, dev) { + if (!dev->driver) { + dev_warn(&pdev->dev, + "%s: memdev disabled, abort error handling\n", + dev_name(dev)); + return; + } - if (cxlds->rcd) - cxl_handle_rdport_errors(cxlds); + if (cxlds->rcd) + cxl_handle_rdport_errors(cxlds); - cxl_handle_endpoint_cor_ras(cxlds); + cxl_handle_endpoint_cor_ras(cxlds); + } } EXPORT_SYMBOL_NS_GPL(cxl_cor_error_detected, CXL); @@ -942,16 +952,25 @@ pci_ers_result_t cxl_error_detected(struct pci_dev *pdev, struct device *dev = &cxlmd->dev; bool ue; - if (cxlds->rcd) - cxl_handle_rdport_errors(cxlds); + scoped_guard(device, dev) { + if (!dev->driver) { + dev_warn(&pdev->dev, + "%s: memdev disabled, abort error handling\n", + dev_name(dev)); + return PCI_ERS_RESULT_DISCONNECT; + } + + if (cxlds->rcd) + cxl_handle_rdport_errors(cxlds); + /* + * A frozen channel indicates an impending reset which is fatal to + * CXL.mem operation, and will likely crash the system. On the off + * chance the situation is recoverable dump the status of the RAS + * capability registers and bounce the active state of the memdev. + */ + ue = cxl_handle_endpoint_ras(cxlds); + } - /* - * A frozen channel indicates an impending reset which is fatal to - * CXL.mem operation, and will likely crash the system. On the off - * chance the situation is recoverable dump the status of the RAS - * capability registers and bounce the active state of the memdev. - */ - ue = cxl_handle_endpoint_ras(cxlds); switch (state) { case pci_channel_io_normal: From 3151567ed487f64705af0db920c3653725195901 Mon Sep 17 00:00:00 2001 From: Ira Weiny Date: Wed, 31 Jan 2024 15:55:38 -0800 Subject: [PATCH 073/138] cxl/cper: Fix errant CPER prints for CXL events commit 54ce1927eb787f7bbb7ee664841c8f5932703f39 upstream. Jonathan reports that CXL CPER events dump an extra generic error message. {1}[Hardware Error]: Hardware error from APEI Generic Hardware Error Source: 1 {1}[Hardware Error]: event severity: recoverable {1}[Hardware Error]: Error 0, type: recoverable {1}[Hardware Error]: section type: unknown, fbcd0a77-c260-417f-85a9-088b1621eba6 {1}[Hardware Error]: section length: 0x90 {1}[Hardware Error]: 00000000: 00000090 00000007 00000000 0d938086 ................ {1}[Hardware Error]: 00000010: 00100000 00000000 00040000 00000000 ................ ... CXL events were rerouted though the CXL subsystem for additional processing. However, when that work was done it was missed that cper_estatus_print_section() continued with a generic error message which is confusing. Teach CPER print code to ignore printing details of some section types. Assign the CXL event GUIDs to this set to prevent confusing unknown prints. Reported-by: Jonathan Cameron Suggested-by: Jonathan Cameron Signed-off-by: Ira Weiny Reviewed-by: Dave Jiang Reviewed-by: Jonathan Cameron Reviewed-by: Alison Schofield Signed-off-by: Ard Biesheuvel Signed-off-by: Yi Liu Signed-off-by: Bo Li --- drivers/acpi/apei/ghes.c | 26 -------------------------- drivers/firmware/efi/cper.c | 19 +++++++++++++++++++ include/linux/cper.h | 23 +++++++++++++++++++++++ 3 files changed, 42 insertions(+), 26 deletions(-) diff --git a/drivers/acpi/apei/ghes.c b/drivers/acpi/apei/ghes.c index 9677ae6835151..7bf0d9ee24e80 100644 --- a/drivers/acpi/apei/ghes.c +++ b/drivers/acpi/apei/ghes.c @@ -682,32 +682,6 @@ static void ghes_defer_non_standard_event(struct acpi_hest_generic_data *gdata, static DECLARE_RWSEM(cxl_cper_rw_sem); static cxl_cper_callback cper_callback; -/* CXL Event record UUIDs are formatted as GUIDs and reported in section type */ - -/* - * General Media Event Record - * CXL rev 3.0 Section 8.2.9.2.1.1; Table 8-43 - */ -#define CPER_SEC_CXL_GEN_MEDIA_GUID \ - GUID_INIT(0xfbcd0a77, 0xc260, 0x417f, \ - 0x85, 0xa9, 0x08, 0x8b, 0x16, 0x21, 0xeb, 0xa6) - -/* - * DRAM Event Record - * CXL rev 3.0 section 8.2.9.2.1.2; Table 8-44 - */ -#define CPER_SEC_CXL_DRAM_GUID \ - GUID_INIT(0x601dcbb3, 0x9c06, 0x4eab, \ - 0xb8, 0xaf, 0x4e, 0x9b, 0xfb, 0x5c, 0x96, 0x24) - -/* - * Memory Module Event Record - * CXL rev 3.0 section 8.2.9.2.1.3; Table 8-45 - */ -#define CPER_SEC_CXL_MEM_MODULE_GUID \ - GUID_INIT(0xfe927475, 0xdd59, 0x4339, \ - 0xa5, 0x86, 0x79, 0xba, 0xb1, 0x13, 0xb7, 0x74) - static void cxl_cper_post_event(enum cxl_event_type event_type, struct cxl_cper_event_rec *rec) { diff --git a/drivers/firmware/efi/cper.c b/drivers/firmware/efi/cper.c index 74ffaf44d74ce..481af0b6f22c4 100644 --- a/drivers/firmware/efi/cper.c +++ b/drivers/firmware/efi/cper.c @@ -583,6 +583,17 @@ static void cper_print_tstamp(const char *pfx, } } +struct ignore_section { + guid_t guid; + const char *name; +}; + +static const struct ignore_section ignore_sections[] = { + { .guid = CPER_SEC_CXL_GEN_MEDIA_GUID, .name = "CXL General Media Event" }, + { .guid = CPER_SEC_CXL_DRAM_GUID, .name = "CXL DRAM Event" }, + { .guid = CPER_SEC_CXL_MEM_MODULE_GUID, .name = "CXL Memory Module Event" }, +}; + static void cper_estatus_print_section(const char *pfx, struct acpi_hest_generic_data *gdata, int sec_no) @@ -603,6 +614,14 @@ cper_estatus_print_section(const char *pfx, struct acpi_hest_generic_data *gdata printk("%s""fru_text: %.20s\n", pfx, gdata->fru_text); snprintf(newpfx, sizeof(newpfx), "%s ", pfx); + + for (int i = 0; i < ARRAY_SIZE(ignore_sections); i++) { + if (guid_equal(sec_type, &ignore_sections[i].guid)) { + printk("%ssection_type: %s\n", newpfx, ignore_sections[i].name); + return; + } + } + if (guid_equal(sec_type, &CPER_SEC_PROC_GENERIC)) { struct cper_sec_proc_generic *proc_err = acpi_hest_get_payload(gdata); diff --git a/include/linux/cper.h b/include/linux/cper.h index ad1ed24730917..3670b866ac119 100644 --- a/include/linux/cper.h +++ b/include/linux/cper.h @@ -90,6 +90,29 @@ enum { GUID_INIT(0x667DD791, 0xC6B3, 0x4c27, 0x8A, 0x6B, 0x0F, 0x8E, \ 0x72, 0x2D, 0xEB, 0x41) +/* CXL Event record UUIDs are formatted as GUIDs and reported in section type */ +/* + * General Media Event Record + * CXL rev 3.0 Section 8.2.9.2.1.1; Table 8-43 + */ +#define CPER_SEC_CXL_GEN_MEDIA_GUID \ + GUID_INIT(0xfbcd0a77, 0xc260, 0x417f, \ + 0x85, 0xa9, 0x08, 0x8b, 0x16, 0x21, 0xeb, 0xa6) +/* + * DRAM Event Record + * CXL rev 3.0 section 8.2.9.2.1.2; Table 8-44 + */ +#define CPER_SEC_CXL_DRAM_GUID \ + GUID_INIT(0x601dcbb3, 0x9c06, 0x4eab, \ + 0xb8, 0xaf, 0x4e, 0x9b, 0xfb, 0x5c, 0x96, 0x24) +/* + * Memory Module Event Record + * CXL rev 3.0 section 8.2.9.2.1.3; Table 8-45 + */ +#define CPER_SEC_CXL_MEM_MODULE_GUID \ + GUID_INIT(0xfe927475, 0xdd59, 0x4339, \ + 0xa5, 0x86, 0x79, 0xba, 0xb1, 0x13, 0xb7, 0x74) + /* * Flags bits definitions for flags in struct cper_record_header * If set, the error has been recovered From f74adf0321854bf6f5d487eb61f9ed9019067a24 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Tue, 6 Feb 2024 12:03:37 -0700 Subject: [PATCH 074/138] cxl: Change 'struct cxl_memdev_state' *_perf_list to single 'struct cxl_dpa_perf' commit 00413c15068276f6e5a50215849a8d4bd812b443 upstream. In order to address the issue with being able to expose qos_class sysfs attributes under 'ram' and 'pmem' sub-directories, the attributes must be defined as static attributes rather than under driver->dev_groups. To avoid implementing locking for accessing the 'struct cxl_dpa_perf` lists, convert the list to a single 'struct cxl_dpa_perf' entry in preparation to move the attributes to statically defined. While theoretically a partition may have multiple qos_class via CDAT, this has not been encountered with testing on available hardware. The code is simplified for now to not support the complex case until a use case is needed to support that. Link: https://lore.kernel.org/linux-cxl/65b200ba228f_2d43c29468@dwillia2-mobl3.amr.corp.intel.com.notmuch/ Suggested-by: Dan Williams Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/20240206190431.1810289-2-dave.jiang@intel.com Reviewed-by: Jonathan Cameron Signed-off-by: Dan Williams Signed-off-by: Yi Liu Signed-off-by: Bo Li --- drivers/cxl/core/cdat.c | 82 ++++++++++++----------------------------- drivers/cxl/core/mbox.c | 4 +- drivers/cxl/cxlmem.h | 10 ++--- drivers/cxl/mem.c | 28 ++------------ 4 files changed, 34 insertions(+), 90 deletions(-) diff --git a/drivers/cxl/core/cdat.c b/drivers/cxl/core/cdat.c index 473c588f86ad2..d44285695c072 100644 --- a/drivers/cxl/core/cdat.c +++ b/drivers/cxl/core/cdat.c @@ -210,19 +210,12 @@ static int cxl_port_perf_data_calculate(struct cxl_port *port, return 0; } -static void add_perf_entry(struct device *dev, struct dsmas_entry *dent, - struct list_head *list) +static void update_perf_entry(struct device *dev, struct dsmas_entry *dent, + struct cxl_dpa_perf *dpa_perf) { - struct cxl_dpa_perf *dpa_perf; - - dpa_perf = kzalloc(sizeof(*dpa_perf), GFP_KERNEL); - if (!dpa_perf) - return; - dpa_perf->dpa_range = dent->dpa_range; dpa_perf->coord = dent->coord; dpa_perf->qos_class = dent->qos_class; - list_add_tail(&dpa_perf->list, list); dev_dbg(dev, "DSMAS: dpa: %#llx qos: %d read_bw: %d write_bw %d read_lat: %d write_lat: %d\n", dent->dpa_range.start, dpa_perf->qos_class, @@ -230,20 +223,6 @@ static void add_perf_entry(struct device *dev, struct dsmas_entry *dent, dent->coord.read_latency, dent->coord.write_latency); } -static void free_perf_ents(void *data) -{ - struct cxl_memdev_state *mds = data; - struct cxl_dpa_perf *dpa_perf, *n; - LIST_HEAD(discard); - - list_splice_tail_init(&mds->ram_perf_list, &discard); - list_splice_tail_init(&mds->pmem_perf_list, &discard); - list_for_each_entry_safe(dpa_perf, n, &discard, list) { - list_del(&dpa_perf->list); - kfree(dpa_perf); - } -} - static void cxl_memdev_set_qos_class(struct cxl_dev_state *cxlds, struct xarray *dsmas_xa) { @@ -263,16 +242,14 @@ static void cxl_memdev_set_qos_class(struct cxl_dev_state *cxlds, xa_for_each(dsmas_xa, index, dent) { if (resource_size(&cxlds->ram_res) && range_contains(&ram_range, &dent->dpa_range)) - add_perf_entry(dev, dent, &mds->ram_perf_list); + update_perf_entry(dev, dent, &mds->ram_perf); else if (resource_size(&cxlds->pmem_res) && range_contains(&pmem_range, &dent->dpa_range)) - add_perf_entry(dev, dent, &mds->pmem_perf_list); + update_perf_entry(dev, dent, &mds->pmem_perf); else dev_dbg(dev, "no partition for dsmas dpa: %#llx\n", dent->dpa_range.start); } - - devm_add_action_or_reset(&cxlds->cxlmd->dev, free_perf_ents, mds); } static int match_cxlrd_qos_class(struct device *dev, void *data) @@ -293,24 +270,24 @@ static int match_cxlrd_qos_class(struct device *dev, void *data) return 0; } -static void cxl_qos_match(struct cxl_port *root_port, - struct list_head *work_list, - struct list_head *discard_list) +static void reset_dpa_perf(struct cxl_dpa_perf *dpa_perf) { - struct cxl_dpa_perf *dpa_perf, *n; + *dpa_perf = (struct cxl_dpa_perf) { + .qos_class = CXL_QOS_CLASS_INVALID, + }; +} - list_for_each_entry_safe(dpa_perf, n, work_list, list) { - int rc; +static bool cxl_qos_match(struct cxl_port *root_port, + struct cxl_dpa_perf *dpa_perf) +{ + if (dpa_perf->qos_class == CXL_QOS_CLASS_INVALID) + return false; - if (dpa_perf->qos_class == CXL_QOS_CLASS_INVALID) - return; + if (!device_for_each_child(&root_port->dev, &dpa_perf->qos_class, + match_cxlrd_qos_class)) + return false; - rc = device_for_each_child(&root_port->dev, - (void *)&dpa_perf->qos_class, - match_cxlrd_qos_class); - if (!rc) - list_move_tail(&dpa_perf->list, discard_list); - } + return true; } static int match_cxlrd_hb(struct device *dev, void *data) @@ -334,23 +311,10 @@ static int match_cxlrd_hb(struct device *dev, void *data) return 0; } -static void discard_dpa_perf(struct list_head *list) -{ - struct cxl_dpa_perf *dpa_perf, *n; - - list_for_each_entry_safe(dpa_perf, n, list, list) { - list_del(&dpa_perf->list); - kfree(dpa_perf); - } -} -DEFINE_FREE(dpa_perf, struct list_head *, if (!list_empty(_T)) discard_dpa_perf(_T)) - static int cxl_qos_class_verify(struct cxl_memdev *cxlmd) { struct cxl_dev_state *cxlds = cxlmd->cxlds; struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds); - LIST_HEAD(__discard); - struct list_head *discard __free(dpa_perf) = &__discard; struct cxl_port *root_port; int rc; @@ -363,16 +327,18 @@ static int cxl_qos_class_verify(struct cxl_memdev *cxlmd) root_port = &cxl_root->port; /* Check that the QTG IDs are all sane between end device and root decoders */ - cxl_qos_match(root_port, &mds->ram_perf_list, discard); - cxl_qos_match(root_port, &mds->pmem_perf_list, discard); + if (!cxl_qos_match(root_port, &mds->ram_perf)) + reset_dpa_perf(&mds->ram_perf); + if (!cxl_qos_match(root_port, &mds->pmem_perf)) + reset_dpa_perf(&mds->pmem_perf); /* Check to make sure that the device's host bridge is under a root decoder */ rc = device_for_each_child(&root_port->dev, (void *)cxlmd->endpoint->host_bridge, match_cxlrd_hb); if (!rc) { - list_splice_tail_init(&mds->ram_perf_list, discard); - list_splice_tail_init(&mds->pmem_perf_list, discard); + reset_dpa_perf(&mds->ram_perf); + reset_dpa_perf(&mds->pmem_perf); } return rc; diff --git a/drivers/cxl/core/mbox.c b/drivers/cxl/core/mbox.c index 7dc2d6a3266d1..65185c9fa0013 100644 --- a/drivers/cxl/core/mbox.c +++ b/drivers/cxl/core/mbox.c @@ -1388,8 +1388,8 @@ struct cxl_memdev_state *cxl_memdev_state_create(struct device *dev) mds->cxlds.reg_map.host = dev; mds->cxlds.reg_map.resource = CXL_RESOURCE_NONE; mds->cxlds.type = CXL_DEVTYPE_CLASSMEM; - INIT_LIST_HEAD(&mds->ram_perf_list); - INIT_LIST_HEAD(&mds->pmem_perf_list); + mds->ram_perf.qos_class = CXL_QOS_CLASS_INVALID; + mds->pmem_perf.qos_class = CXL_QOS_CLASS_INVALID; return mds; } diff --git a/drivers/cxl/cxlmem.h b/drivers/cxl/cxlmem.h index 0e9c2fb229c3d..675fe674ff3e7 100644 --- a/drivers/cxl/cxlmem.h +++ b/drivers/cxl/cxlmem.h @@ -395,13 +395,11 @@ enum cxl_devtype { /** * struct cxl_dpa_perf - DPA performance property entry - * @list - list entry * @dpa_range - range for DPA address * @coord - QoS performance data (i.e. latency, bandwidth) * @qos_class - QoS Class cookies */ struct cxl_dpa_perf { - struct list_head list; struct range dpa_range; struct access_coordinate coord; int qos_class; @@ -471,8 +469,8 @@ struct cxl_dev_state { * @security: security driver state info * @fw: firmware upload / activation state * @mbox_send: @dev specific transport for transmitting mailbox commands - * @ram_perf_list: performance data entries matched to RAM - * @pmem_perf_list: performance data entries matched to PMEM + * @ram_perf: performance data entry matched to RAM partition + * @pmem_perf: performance data entry matched to PMEM partition * * See CXL 3.0 8.2.9.8.2 Capacity Configuration and Label Storage for * details on capacity parameters. @@ -494,8 +492,8 @@ struct cxl_memdev_state { u64 next_volatile_bytes; u64 next_persistent_bytes; - struct list_head ram_perf_list; - struct list_head pmem_perf_list; + struct cxl_dpa_perf ram_perf; + struct cxl_dpa_perf pmem_perf; struct cxl_event_state event; struct cxl_poison_state poison; diff --git a/drivers/cxl/mem.c b/drivers/cxl/mem.c index c5c9d8e0d88d6..547f5a145fc54 100644 --- a/drivers/cxl/mem.c +++ b/drivers/cxl/mem.c @@ -221,18 +221,8 @@ static ssize_t ram_qos_class_show(struct device *dev, struct cxl_memdev *cxlmd = to_cxl_memdev(dev); struct cxl_dev_state *cxlds = cxlmd->cxlds; struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds); - struct cxl_dpa_perf *dpa_perf; - if (!dev->driver) - return -ENOENT; - - if (list_empty(&mds->ram_perf_list)) - return -ENOENT; - - dpa_perf = list_first_entry(&mds->ram_perf_list, struct cxl_dpa_perf, - list); - - return sysfs_emit(buf, "%d\n", dpa_perf->qos_class); + return sysfs_emit(buf, "%d\n", mds->ram_perf.qos_class); } static struct device_attribute dev_attr_ram_qos_class = @@ -244,18 +234,8 @@ static ssize_t pmem_qos_class_show(struct device *dev, struct cxl_memdev *cxlmd = to_cxl_memdev(dev); struct cxl_dev_state *cxlds = cxlmd->cxlds; struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds); - struct cxl_dpa_perf *dpa_perf; - - if (!dev->driver) - return -ENOENT; - - if (list_empty(&mds->pmem_perf_list)) - return -ENOENT; - - dpa_perf = list_first_entry(&mds->pmem_perf_list, struct cxl_dpa_perf, - list); - return sysfs_emit(buf, "%d\n", dpa_perf->qos_class); + return sysfs_emit(buf, "%d\n", mds->pmem_perf.qos_class); } static struct device_attribute dev_attr_pmem_qos_class = @@ -273,11 +253,11 @@ static umode_t cxl_mem_visible(struct kobject *kobj, struct attribute *a, int n) return 0; if (a == &dev_attr_pmem_qos_class.attr) - if (list_empty(&mds->pmem_perf_list)) + if (mds->pmem_perf.qos_class == CXL_QOS_CLASS_INVALID) return 0; if (a == &dev_attr_ram_qos_class.attr) - if (list_empty(&mds->ram_perf_list)) + if (mds->ram_perf.qos_class == CXL_QOS_CLASS_INVALID) return 0; return a->mode; From c800d0b51c83fee3edf2625672cb540900d6eae7 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Tue, 6 Feb 2024 12:03:38 -0700 Subject: [PATCH 075/138] cxl: Remove unnecessary type cast in cxl_qos_class_verify() commit 10cb393d769bed9473e1fa8d34e4d6d389db9155 upstream. The passed in host bridge parameter for device_for_each_child() has unnecessary void * type cast. Remove the type cast. Suggested-by: Jonathan Cameron Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/20240206190431.1810289-3-dave.jiang@intel.com Reviewed-by: Jonathan Cameron Signed-off-by: Dan Williams Signed-off-by: Yi Liu Signed-off-by: Bo Li --- drivers/cxl/core/cdat.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/cxl/core/cdat.c b/drivers/cxl/core/cdat.c index d44285695c072..0a0c01c079f6b 100644 --- a/drivers/cxl/core/cdat.c +++ b/drivers/cxl/core/cdat.c @@ -334,8 +334,7 @@ static int cxl_qos_class_verify(struct cxl_memdev *cxlmd) /* Check to make sure that the device's host bridge is under a root decoder */ rc = device_for_each_child(&root_port->dev, - (void *)cxlmd->endpoint->host_bridge, - match_cxlrd_hb); + cxlmd->endpoint->host_bridge, match_cxlrd_hb); if (!rc) { reset_dpa_perf(&mds->ram_perf); reset_dpa_perf(&mds->pmem_perf); From e4c037c4e55bdfa27dfd78e910054584c90abdb3 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Tue, 6 Feb 2024 12:03:39 -0700 Subject: [PATCH 076/138] cxl: Fix sysfs export of qos_class for memdev commit cc214417f06f6b0a8f6da09220c8bcdb742210b9 upstream. Current implementation exports only to /sys/bus/cxl/devices/.../memN/qos_class. With both ram and pmem exposed, the second registered sysfs attribute is rejected as duplicate. It's not possible to create qos_class under the dev_groups via the driver due to the ram and pmem sysfs sub-directories already created by the device sysfs groups. Move the ram and pmem qos_class to the device sysfs groups and add a call to sysfs_update() after the perf data are validated so the qos_class can be visible. The end results should be /sys/bus/cxl/devices/.../memN/ram/qos_class and /sys/bus/cxl/devices/.../memN/pmem/qos_class. Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/20240206190431.1810289-4-dave.jiang@intel.com Reviewed-by: Jonathan Cameron Signed-off-by: Dan Williams Signed-off-by: Yi Liu Signed-off-by: Bo Li --- drivers/cxl/core/cdat.c | 1 + drivers/cxl/core/memdev.c | 63 +++++++++++++++++++++++++++++++++++++++ drivers/cxl/cxl.h | 2 ++ drivers/cxl/mem.c | 36 ---------------------- 4 files changed, 66 insertions(+), 36 deletions(-) diff --git a/drivers/cxl/core/cdat.c b/drivers/cxl/core/cdat.c index 0a0c01c079f6b..6bffd3551765b 100644 --- a/drivers/cxl/core/cdat.c +++ b/drivers/cxl/core/cdat.c @@ -382,6 +382,7 @@ void cxl_endpoint_parse_cdat(struct cxl_port *port) cxl_memdev_set_qos_class(cxlds, dsmas_xa); cxl_qos_class_verify(cxlmd); + cxl_memdev_update_perf(cxlmd); } EXPORT_SYMBOL_NS_GPL(cxl_endpoint_parse_cdat, CXL); diff --git a/drivers/cxl/core/memdev.c b/drivers/cxl/core/memdev.c index f607ab96ba9d4..0277726afd049 100644 --- a/drivers/cxl/core/memdev.c +++ b/drivers/cxl/core/memdev.c @@ -403,13 +403,41 @@ static struct attribute *cxl_memdev_attributes[] = { NULL, }; +static ssize_t pmem_qos_class_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct cxl_memdev *cxlmd = to_cxl_memdev(dev); + struct cxl_dev_state *cxlds = cxlmd->cxlds; + struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds); + + return sysfs_emit(buf, "%d\n", mds->pmem_perf.qos_class); +} + +static struct device_attribute dev_attr_pmem_qos_class = + __ATTR(qos_class, 0444, pmem_qos_class_show, NULL); + static struct attribute *cxl_memdev_pmem_attributes[] = { &dev_attr_pmem_size.attr, + &dev_attr_pmem_qos_class.attr, NULL, }; +static ssize_t ram_qos_class_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct cxl_memdev *cxlmd = to_cxl_memdev(dev); + struct cxl_dev_state *cxlds = cxlmd->cxlds; + struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds); + + return sysfs_emit(buf, "%d\n", mds->ram_perf.qos_class); +} + +static struct device_attribute dev_attr_ram_qos_class = + __ATTR(qos_class, 0444, ram_qos_class_show, NULL); + static struct attribute *cxl_memdev_ram_attributes[] = { &dev_attr_ram_size.attr, + &dev_attr_ram_qos_class.attr, NULL, }; @@ -433,14 +461,42 @@ static struct attribute_group cxl_memdev_attribute_group = { .is_visible = cxl_memdev_visible, }; +static umode_t cxl_ram_visible(struct kobject *kobj, struct attribute *a, int n) +{ + struct device *dev = kobj_to_dev(kobj); + struct cxl_memdev *cxlmd = to_cxl_memdev(dev); + struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds); + + if (a == &dev_attr_ram_qos_class.attr) + if (mds->ram_perf.qos_class == CXL_QOS_CLASS_INVALID) + return 0; + + return a->mode; +} + static struct attribute_group cxl_memdev_ram_attribute_group = { .name = "ram", .attrs = cxl_memdev_ram_attributes, + .is_visible = cxl_ram_visible, }; +static umode_t cxl_pmem_visible(struct kobject *kobj, struct attribute *a, int n) +{ + struct device *dev = kobj_to_dev(kobj); + struct cxl_memdev *cxlmd = to_cxl_memdev(dev); + struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds); + + if (a == &dev_attr_pmem_qos_class.attr) + if (mds->pmem_perf.qos_class == CXL_QOS_CLASS_INVALID) + return 0; + + return a->mode; +} + static struct attribute_group cxl_memdev_pmem_attribute_group = { .name = "pmem", .attrs = cxl_memdev_pmem_attributes, + .is_visible = cxl_pmem_visible, }; static umode_t cxl_memdev_security_visible(struct kobject *kobj, @@ -475,6 +531,13 @@ static const struct attribute_group *cxl_memdev_attribute_groups[] = { NULL, }; +void cxl_memdev_update_perf(struct cxl_memdev *cxlmd) +{ + sysfs_update_group(&cxlmd->dev.kobj, &cxl_memdev_ram_attribute_group); + sysfs_update_group(&cxlmd->dev.kobj, &cxl_memdev_pmem_attribute_group); +} +EXPORT_SYMBOL_NS_GPL(cxl_memdev_update_perf, CXL); + static const struct device_type cxl_memdev_type = { .name = "cxl_memdev", .release = cxl_memdev_release, diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h index d983eccc80ba3..d6bee32b9c7cc 100644 --- a/drivers/cxl/cxl.h +++ b/drivers/cxl/cxl.h @@ -883,6 +883,8 @@ void cxl_switch_parse_cdat(struct cxl_port *port); int cxl_endpoint_get_perf_coordinates(struct cxl_port *port, struct access_coordinate *coord); +void cxl_memdev_update_perf(struct cxl_memdev *cxlmd); + /* * Unit test builds overrides this to __weak, find the 'strong' version * of these symbols in tools/testing/cxl/. diff --git a/drivers/cxl/mem.c b/drivers/cxl/mem.c index 547f5a145fc54..0c79d9ce877cc 100644 --- a/drivers/cxl/mem.c +++ b/drivers/cxl/mem.c @@ -215,32 +215,6 @@ static ssize_t trigger_poison_list_store(struct device *dev, } static DEVICE_ATTR_WO(trigger_poison_list); -static ssize_t ram_qos_class_show(struct device *dev, - struct device_attribute *attr, char *buf) -{ - struct cxl_memdev *cxlmd = to_cxl_memdev(dev); - struct cxl_dev_state *cxlds = cxlmd->cxlds; - struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds); - - return sysfs_emit(buf, "%d\n", mds->ram_perf.qos_class); -} - -static struct device_attribute dev_attr_ram_qos_class = - __ATTR(qos_class, 0444, ram_qos_class_show, NULL); - -static ssize_t pmem_qos_class_show(struct device *dev, - struct device_attribute *attr, char *buf) -{ - struct cxl_memdev *cxlmd = to_cxl_memdev(dev); - struct cxl_dev_state *cxlds = cxlmd->cxlds; - struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds); - - return sysfs_emit(buf, "%d\n", mds->pmem_perf.qos_class); -} - -static struct device_attribute dev_attr_pmem_qos_class = - __ATTR(qos_class, 0444, pmem_qos_class_show, NULL); - static umode_t cxl_mem_visible(struct kobject *kobj, struct attribute *a, int n) { struct device *dev = kobj_to_dev(kobj); @@ -252,21 +226,11 @@ static umode_t cxl_mem_visible(struct kobject *kobj, struct attribute *a, int n) mds->poison.enabled_cmds)) return 0; - if (a == &dev_attr_pmem_qos_class.attr) - if (mds->pmem_perf.qos_class == CXL_QOS_CLASS_INVALID) - return 0; - - if (a == &dev_attr_ram_qos_class.attr) - if (mds->ram_perf.qos_class == CXL_QOS_CLASS_INVALID) - return 0; - return a->mode; } static struct attribute *cxl_mem_attrs[] = { &dev_attr_trigger_poison_list.attr, - &dev_attr_ram_qos_class.attr, - &dev_attr_pmem_qos_class.attr, NULL }; From 59bf11660e699291f0991eba6e3fcd2d8405be25 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Tue, 6 Feb 2024 12:03:40 -0700 Subject: [PATCH 077/138] cxl/test: Add support for qos_class checking commit 117132edc6900621337b77d29e953d0d01f32a5f upstream. Set a fake qos_class to a unique value in order to do simple testing of qos_class for root decoders and mem devs via user cxl_test. A mock function is added to set the fake qos_class values for memory device and overrides cxl_endpoint_parse_cdat() in cxl driver code. Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/20240206190431.1810289-5-dave.jiang@intel.com Reviewed-by: Jonathan Cameron Signed-off-by: Dan Williams Signed-off-by: Yi Liu Signed-off-by: Bo Li --- tools/testing/cxl/Kbuild | 1 + tools/testing/cxl/test/cxl.c | 63 ++++++++++++++++++++++++++++++----- tools/testing/cxl/test/mock.c | 14 ++++++++ tools/testing/cxl/test/mock.h | 1 + 4 files changed, 70 insertions(+), 9 deletions(-) diff --git a/tools/testing/cxl/Kbuild b/tools/testing/cxl/Kbuild index caff3834671f9..030b388800f05 100644 --- a/tools/testing/cxl/Kbuild +++ b/tools/testing/cxl/Kbuild @@ -13,6 +13,7 @@ ldflags-y += --wrap=cxl_hdm_decode_init ldflags-y += --wrap=cxl_dvsec_rr_decode ldflags-y += --wrap=devm_cxl_add_rch_dport ldflags-y += --wrap=cxl_rcd_component_reg_phys +ldflags-y += --wrap=cxl_endpoint_parse_cdat DRIVERS := ../../../drivers CXL_SRC := $(DRIVERS)/cxl diff --git a/tools/testing/cxl/test/cxl.c b/tools/testing/cxl/test/cxl.c index 606af1fe6a348..6da9df7c878ed 100644 --- a/tools/testing/cxl/test/cxl.c +++ b/tools/testing/cxl/test/cxl.c @@ -15,6 +15,8 @@ static int interleave_arithmetic; +#define FAKE_QTG_ID 42 + #define NR_CXL_HOST_BRIDGES 2 #define NR_CXL_SINGLE_HOST 1 #define NR_CXL_RCH 1 @@ -209,7 +211,7 @@ static struct { .granularity = 4, .restrictions = ACPI_CEDT_CFMWS_RESTRICT_TYPE3 | ACPI_CEDT_CFMWS_RESTRICT_VOLATILE, - .qtg_id = 0, + .qtg_id = FAKE_QTG_ID, .window_size = SZ_256M * 4UL, }, .target = { 0 }, @@ -224,7 +226,7 @@ static struct { .granularity = 4, .restrictions = ACPI_CEDT_CFMWS_RESTRICT_TYPE3 | ACPI_CEDT_CFMWS_RESTRICT_VOLATILE, - .qtg_id = 1, + .qtg_id = FAKE_QTG_ID, .window_size = SZ_256M * 8UL, }, .target = { 0, 1, }, @@ -239,7 +241,7 @@ static struct { .granularity = 4, .restrictions = ACPI_CEDT_CFMWS_RESTRICT_TYPE3 | ACPI_CEDT_CFMWS_RESTRICT_PMEM, - .qtg_id = 2, + .qtg_id = FAKE_QTG_ID, .window_size = SZ_256M * 4UL, }, .target = { 0 }, @@ -254,7 +256,7 @@ static struct { .granularity = 4, .restrictions = ACPI_CEDT_CFMWS_RESTRICT_TYPE3 | ACPI_CEDT_CFMWS_RESTRICT_PMEM, - .qtg_id = 3, + .qtg_id = FAKE_QTG_ID, .window_size = SZ_256M * 8UL, }, .target = { 0, 1, }, @@ -269,7 +271,7 @@ static struct { .granularity = 4, .restrictions = ACPI_CEDT_CFMWS_RESTRICT_TYPE3 | ACPI_CEDT_CFMWS_RESTRICT_PMEM, - .qtg_id = 4, + .qtg_id = FAKE_QTG_ID, .window_size = SZ_256M * 4UL, }, .target = { 2 }, @@ -284,7 +286,7 @@ static struct { .granularity = 4, .restrictions = ACPI_CEDT_CFMWS_RESTRICT_TYPE3 | ACPI_CEDT_CFMWS_RESTRICT_VOLATILE, - .qtg_id = 5, + .qtg_id = FAKE_QTG_ID, .window_size = SZ_256M, }, .target = { 3 }, @@ -301,7 +303,7 @@ static struct { .granularity = 4, .restrictions = ACPI_CEDT_CFMWS_RESTRICT_TYPE3 | ACPI_CEDT_CFMWS_RESTRICT_PMEM, - .qtg_id = 0, + .qtg_id = FAKE_QTG_ID, .window_size = SZ_256M * 8UL, }, .target = { 0, }, @@ -317,7 +319,7 @@ static struct { .granularity = 0, .restrictions = ACPI_CEDT_CFMWS_RESTRICT_TYPE3 | ACPI_CEDT_CFMWS_RESTRICT_PMEM, - .qtg_id = 1, + .qtg_id = FAKE_QTG_ID, .window_size = SZ_256M * 8UL, }, .target = { 0, 1, }, @@ -333,7 +335,7 @@ static struct { .granularity = 0, .restrictions = ACPI_CEDT_CFMWS_RESTRICT_TYPE3 | ACPI_CEDT_CFMWS_RESTRICT_PMEM, - .qtg_id = 0, + .qtg_id = FAKE_QTG_ID, .window_size = SZ_256M * 16UL, }, .target = { 0, 1, 0, 1, }, @@ -976,6 +978,48 @@ static int mock_cxl_port_enumerate_dports(struct cxl_port *port) return 0; } +/* + * Faking the cxl_dpa_perf for the memdev when appropriate. + */ +static void dpa_perf_setup(struct cxl_port *endpoint, struct range *range, + struct cxl_dpa_perf *dpa_perf) +{ + dpa_perf->qos_class = FAKE_QTG_ID; + dpa_perf->dpa_range = *range; + dpa_perf->coord.read_latency = 500; + dpa_perf->coord.write_latency = 500; + dpa_perf->coord.read_bandwidth = 1000; + dpa_perf->coord.write_bandwidth = 1000; +} + +static void mock_cxl_endpoint_parse_cdat(struct cxl_port *port) +{ + struct cxl_root *cxl_root __free(put_cxl_root) = + find_cxl_root(port); + struct cxl_memdev *cxlmd = to_cxl_memdev(port->uport_dev); + struct cxl_dev_state *cxlds = cxlmd->cxlds; + struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds); + struct range pmem_range = { + .start = cxlds->pmem_res.start, + .end = cxlds->pmem_res.end, + }; + struct range ram_range = { + .start = cxlds->ram_res.start, + .end = cxlds->ram_res.end, + }; + + if (!cxl_root) + return; + + if (range_len(&ram_range)) + dpa_perf_setup(port, &ram_range, &mds->ram_perf); + + if (range_len(&pmem_range)) + dpa_perf_setup(port, &pmem_range, &mds->pmem_perf); + + cxl_memdev_update_perf(cxlmd); +} + static struct cxl_mock_ops cxl_mock_ops = { .is_mock_adev = is_mock_adev, .is_mock_bridge = is_mock_bridge, @@ -989,6 +1033,7 @@ static struct cxl_mock_ops cxl_mock_ops = { .devm_cxl_setup_hdm = mock_cxl_setup_hdm, .devm_cxl_add_passthrough_decoder = mock_cxl_add_passthrough_decoder, .devm_cxl_enumerate_decoders = mock_cxl_enumerate_decoders, + .cxl_endpoint_parse_cdat = mock_cxl_endpoint_parse_cdat, .list = LIST_HEAD_INIT(cxl_mock_ops.list), }; diff --git a/tools/testing/cxl/test/mock.c b/tools/testing/cxl/test/mock.c index 1a61e68e30950..6f737941dc0e1 100644 --- a/tools/testing/cxl/test/mock.c +++ b/tools/testing/cxl/test/mock.c @@ -285,6 +285,20 @@ resource_size_t __wrap_cxl_rcd_component_reg_phys(struct device *dev, } EXPORT_SYMBOL_NS_GPL(__wrap_cxl_rcd_component_reg_phys, CXL); +void __wrap_cxl_endpoint_parse_cdat(struct cxl_port *port) +{ + int index; + struct cxl_mock_ops *ops = get_cxl_mock_ops(&index); + struct cxl_memdev *cxlmd = to_cxl_memdev(port->uport_dev); + + if (ops && ops->is_mock_dev(cxlmd->dev.parent)) + ops->cxl_endpoint_parse_cdat(port); + else + cxl_endpoint_parse_cdat(port); + put_cxl_mock_ops(index); +} +EXPORT_SYMBOL_NS_GPL(__wrap_cxl_endpoint_parse_cdat, CXL); + MODULE_LICENSE("GPL v2"); MODULE_IMPORT_NS(ACPI); MODULE_IMPORT_NS(CXL); diff --git a/tools/testing/cxl/test/mock.h b/tools/testing/cxl/test/mock.h index a94223750346c..d1b0271d28220 100644 --- a/tools/testing/cxl/test/mock.h +++ b/tools/testing/cxl/test/mock.h @@ -25,6 +25,7 @@ struct cxl_mock_ops { int (*devm_cxl_add_passthrough_decoder)(struct cxl_port *port); int (*devm_cxl_enumerate_decoders)( struct cxl_hdm *hdm, struct cxl_endpoint_dvsec_info *info); + void (*cxl_endpoint_parse_cdat)(struct cxl_port *port); }; void register_cxl_mock_ops(struct cxl_mock_ops *ops); From 2740cf48ea753923472e256674028ca2983e44be Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Sun, 18 Feb 2024 14:28:57 -0800 Subject: [PATCH 078/138] acpi/ghes: Remove CXL CPER notifications commit f3e6b3ae9cfc128af11b665c6ef4022ba2683778 upstream. Initial tests with the CXL CPER implementation identified that error reports were being duplicated in the log and the trace event [1]. Then it was discovered that the notification handler took sleeping locks while the GHES event handling runs in spin_lock_irqsave() context [2] While the duplicate reporting was fixed in v6.8-rc4, the fix for the sleeping-lock-vs-atomic collision would enjoy more time to settle and gain some test cycles. Given how late it is in the development cycle, remove the CXL hookup for now and try again during the next merge window. Note that end result is that v6.8 does not emit CXL CPER payloads to the kernel log, but this is in line with the CXL trend to move error reporting to trace events instead of the kernel log. Cc: Ard Biesheuvel Cc: Rafael J. Wysocki Cc: Jonathan Cameron Reviewed-by: Ira Weiny Link: http://lore.kernel.org/r/20240108165855.00002f5a@Huawei.com [1] Closes: http://lore.kernel.org/r/b963c490-2c13-4b79-bbe7-34c6568423c7@moroto.mountain [2] Signed-off-by: Dan Williams Signed-off-by: Yi Liu Signed-off-by: Bo Li --- drivers/acpi/apei/ghes.c | 63 --------------------------------------- drivers/cxl/pci.c | 57 +---------------------------------- include/linux/cxl-event.h | 18 ----------- 3 files changed, 1 insertion(+), 137 deletions(-) diff --git a/drivers/acpi/apei/ghes.c b/drivers/acpi/apei/ghes.c index 7bf0d9ee24e80..e768dfd345fb2 100644 --- a/drivers/acpi/apei/ghes.c +++ b/drivers/acpi/apei/ghes.c @@ -27,7 +27,6 @@ #include #include #include -#include #include #include #include @@ -676,52 +675,6 @@ static void ghes_defer_non_standard_event(struct acpi_hest_generic_data *gdata, schedule_work(&entry->work); } -/* - * Only a single callback can be registered for CXL CPER events. - */ -static DECLARE_RWSEM(cxl_cper_rw_sem); -static cxl_cper_callback cper_callback; - -static void cxl_cper_post_event(enum cxl_event_type event_type, - struct cxl_cper_event_rec *rec) -{ - if (rec->hdr.length <= sizeof(rec->hdr) || - rec->hdr.length > sizeof(*rec)) { - pr_err(FW_WARN "CXL CPER Invalid section length (%u)\n", - rec->hdr.length); - return; - } - - if (!(rec->hdr.validation_bits & CPER_CXL_COMP_EVENT_LOG_VALID)) { - pr_err(FW_WARN "CXL CPER invalid event\n"); - return; - } - - guard(rwsem_read)(&cxl_cper_rw_sem); - if (cper_callback) - cper_callback(event_type, rec); -} - -int cxl_cper_register_callback(cxl_cper_callback callback) -{ - guard(rwsem_write)(&cxl_cper_rw_sem); - if (cper_callback) - return -EINVAL; - cper_callback = callback; - return 0; -} -EXPORT_SYMBOL_GPL(cxl_cper_register_callback); - -int cxl_cper_unregister_callback(cxl_cper_callback callback) -{ - guard(rwsem_write)(&cxl_cper_rw_sem); - if (callback != cper_callback) - return -EINVAL; - cper_callback = NULL; - return 0; -} -EXPORT_SYMBOL_GPL(cxl_cper_unregister_callback); - static bool ghes_do_proc(struct ghes *ghes, const struct acpi_hest_generic_status *estatus) { @@ -756,22 +709,6 @@ static bool ghes_do_proc(struct ghes *ghes, } else if (guid_equal(sec_type, &CPER_SEC_PROC_ARM)) { queued = ghes_handle_arm_hw_error(gdata, sev, sync); - } else if (guid_equal(sec_type, &CPER_SEC_CXL_GEN_MEDIA_GUID)) { - struct cxl_cper_event_rec *rec = - acpi_hest_get_payload(gdata); - - cxl_cper_post_event(CXL_CPER_EVENT_GEN_MEDIA, rec); - } else if (guid_equal(sec_type, &CPER_SEC_CXL_DRAM_GUID)) { - struct cxl_cper_event_rec *rec = - acpi_hest_get_payload(gdata); - - cxl_cper_post_event(CXL_CPER_EVENT_DRAM, rec); - } else if (guid_equal(sec_type, - &CPER_SEC_CXL_MEM_MODULE_GUID)) { - struct cxl_cper_event_rec *rec = - acpi_hest_get_payload(gdata); - - cxl_cper_post_event(CXL_CPER_EVENT_MEM_MODULE, rec); } else { void *err = acpi_hest_get_payload(gdata); diff --git a/drivers/cxl/pci.c b/drivers/cxl/pci.c index 5d2672d767e9f..0b37f15f3a600 100644 --- a/drivers/cxl/pci.c +++ b/drivers/cxl/pci.c @@ -973,61 +973,6 @@ static struct pci_driver cxl_pci_driver = { }, }; -#define CXL_EVENT_HDR_FLAGS_REC_SEVERITY GENMASK(1, 0) -static void cxl_cper_event_call(enum cxl_event_type ev_type, - struct cxl_cper_event_rec *rec) -{ - struct cper_cxl_event_devid *device_id = &rec->hdr.device_id; - struct pci_dev *pdev __free(pci_dev_put) = NULL; - enum cxl_event_log_type log_type; - struct cxl_dev_state *cxlds; - unsigned int devfn; - u32 hdr_flags; - - devfn = PCI_DEVFN(device_id->device_num, device_id->func_num); - pdev = pci_get_domain_bus_and_slot(device_id->segment_num, - device_id->bus_num, devfn); - if (!pdev) - return; - - guard(pci_dev)(pdev); - if (pdev->driver != &cxl_pci_driver) - return; - - cxlds = pci_get_drvdata(pdev); - if (!cxlds) - return; - - /* Fabricate a log type */ - hdr_flags = get_unaligned_le24(rec->event.generic.hdr.flags); - log_type = FIELD_GET(CXL_EVENT_HDR_FLAGS_REC_SEVERITY, hdr_flags); - - cxl_event_trace_record(cxlds->cxlmd, log_type, ev_type, - &uuid_null, &rec->event); -} - -static int __init cxl_pci_driver_init(void) -{ - int rc; - - rc = cxl_cper_register_callback(cxl_cper_event_call); - if (rc) - return rc; - - rc = pci_register_driver(&cxl_pci_driver); - if (rc) - cxl_cper_unregister_callback(cxl_cper_event_call); - - return rc; -} - -static void __exit cxl_pci_driver_exit(void) -{ - pci_unregister_driver(&cxl_pci_driver); - cxl_cper_unregister_callback(cxl_cper_event_call); -} - -module_init(cxl_pci_driver_init); -module_exit(cxl_pci_driver_exit); +module_pci_driver(cxl_pci_driver); MODULE_LICENSE("GPL v2"); MODULE_IMPORT_NS(CXL); diff --git a/include/linux/cxl-event.h b/include/linux/cxl-event.h index 91125eca4c8ab..03fa6d50d46fe 100644 --- a/include/linux/cxl-event.h +++ b/include/linux/cxl-event.h @@ -140,22 +140,4 @@ struct cxl_cper_event_rec { union cxl_event event; } __packed; -typedef void (*cxl_cper_callback)(enum cxl_event_type type, - struct cxl_cper_event_rec *rec); - -#ifdef CONFIG_ACPI_APEI_GHES -int cxl_cper_register_callback(cxl_cper_callback callback); -int cxl_cper_unregister_callback(cxl_cper_callback callback); -#else -static inline int cxl_cper_register_callback(cxl_cper_callback callback) -{ - return 0; -} - -static inline int cxl_cper_unregister_callback(cxl_cper_callback callback) -{ - return 0; -} -#endif - #endif /* _LINUX_CXL_EVENT_H */ From 871b2c2950c344cb217f8fcd45d9f8a657bcd56e Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Fri, 8 Mar 2024 14:59:22 -0700 Subject: [PATCH 079/138] ACPI: HMAT: Introduce 2 levels of generic port access class commit 1745a7b364dfd339ab2696b7d51d7ed950ed2598 upstream. In order to compute access0 and access1 classes for CXL memory, 2 levels of generic port information must be stored. Access0 will indicate the generic port access coordinates to the closest initiator and access1 will indicate the generic port access coordinates to the cloest CPU. Cc: Rafael J. Wysocki Reviewed-by: Jonathan Cameron Tested-by: Jonathan Cameron Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/20240308220055.2172956-4-dave.jiang@intel.com Signed-off-by: Dan Williams Signed-off-by: Yi Liu Signed-off-by: Bo Li --- drivers/acpi/numa/hmat.c | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/drivers/acpi/numa/hmat.c b/drivers/acpi/numa/hmat.c index 0e2b5fede76b6..6ac2e68c430b0 100644 --- a/drivers/acpi/numa/hmat.c +++ b/drivers/acpi/numa/hmat.c @@ -59,7 +59,8 @@ struct target_cache { }; enum { - NODE_ACCESS_CLASS_GENPORT_SINK = ACCESS_COORDINATE_MAX, + NODE_ACCESS_CLASS_GENPORT_SINK_LOCAL = ACCESS_COORDINATE_MAX, + NODE_ACCESS_CLASS_GENPORT_SINK_CPU, NODE_ACCESS_CLASS_MAX, }; @@ -141,7 +142,7 @@ int acpi_get_genport_coordinates(u32 uid, if (!target) return -ENOENT; - *coord = target->coord[NODE_ACCESS_CLASS_GENPORT_SINK]; + *coord = target->coord[NODE_ACCESS_CLASS_GENPORT_SINK_LOCAL]; return 0; } @@ -695,7 +696,8 @@ static void hmat_update_target_attrs(struct memory_target *target, int i; /* Don't update for generic port if there's no device handle */ - if (access == NODE_ACCESS_CLASS_GENPORT_SINK && + if ((access == NODE_ACCESS_CLASS_GENPORT_SINK_LOCAL || + access == NODE_ACCESS_CLASS_GENPORT_SINK_CPU) && !(*(u16 *)target->gen_port_device_handle)) return; @@ -736,7 +738,8 @@ static void hmat_update_target_attrs(struct memory_target *target, list_for_each_entry(initiator, &initiators, node) { u32 value; - if (access == ACCESS_COORDINATE_CPU && + if ((access == ACCESS_COORDINATE_CPU || + access == NODE_ACCESS_CLASS_GENPORT_SINK_CPU) && !initiator->has_cpu) { clear_bit(initiator->processor_pxm, p_nodes); continue; @@ -775,7 +778,9 @@ static void hmat_update_generic_target(struct memory_target *target) static DECLARE_BITMAP(p_nodes, MAX_NUMNODES); hmat_update_target_attrs(target, p_nodes, - NODE_ACCESS_CLASS_GENPORT_SINK); + NODE_ACCESS_CLASS_GENPORT_SINK_LOCAL); + hmat_update_target_attrs(target, p_nodes, + NODE_ACCESS_CLASS_GENPORT_SINK_CPU); } static void hmat_register_target_initiators(struct memory_target *target) From 5fd9a0caedb56865770f7afeec9cf21f496494b9 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Fri, 8 Mar 2024 14:59:23 -0700 Subject: [PATCH 080/138] ACPI: HMAT / cxl: Add retrieval of generic port coordinates for both access classes commit bd98cbbbf82a3086423865816e1b5ab4bb4b6c60 upstream. Update acpi_get_genport_coordinates() to allow retrieval of both access classes of the 'struct access_coordinate' for a generic target. The update will allow CXL code to compute access coordinates for both access class. Cc: Rafael J. Wysocki Reviewed-by: Jonathan Cameron Tested-by: Jonathan Cameron Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/20240308220055.2172956-5-dave.jiang@intel.com Signed-off-by: Dan Williams Signed-off-by: Yi Liu Signed-off-by: Bo Li --- drivers/acpi/numa/hmat.c | 8 ++++++-- drivers/cxl/acpi.c | 8 +++++--- drivers/cxl/core/port.c | 2 +- drivers/cxl/cxl.h | 2 +- 4 files changed, 13 insertions(+), 7 deletions(-) diff --git a/drivers/acpi/numa/hmat.c b/drivers/acpi/numa/hmat.c index 6ac2e68c430b0..c170691b38f2a 100644 --- a/drivers/acpi/numa/hmat.c +++ b/drivers/acpi/numa/hmat.c @@ -126,7 +126,8 @@ static struct memory_target *acpi_find_genport_target(u32 uid) /** * acpi_get_genport_coordinates - Retrieve the access coordinates for a generic port * @uid: ACPI unique id - * @coord: The access coordinates written back out for the generic port + * @coord: The access coordinates written back out for the generic port. + * Expect 2 levels array. * * Return: 0 on success. Errno on failure. * @@ -142,7 +143,10 @@ int acpi_get_genport_coordinates(u32 uid, if (!target) return -ENOENT; - *coord = target->coord[NODE_ACCESS_CLASS_GENPORT_SINK_LOCAL]; + coord[ACCESS_COORDINATE_LOCAL] = + target->coord[NODE_ACCESS_CLASS_GENPORT_SINK_LOCAL]; + coord[ACCESS_COORDINATE_CPU] = + target->coord[NODE_ACCESS_CLASS_GENPORT_SINK_CPU]; return 0; } diff --git a/drivers/cxl/acpi.c b/drivers/cxl/acpi.c index 8b48d0dceae0e..122974a3fe029 100644 --- a/drivers/cxl/acpi.c +++ b/drivers/cxl/acpi.c @@ -530,13 +530,15 @@ static int get_genport_coordinates(struct device *dev, struct cxl_dport *dport) if (kstrtou32(acpi_device_uid(hb), 0, &uid)) return -EINVAL; - rc = acpi_get_genport_coordinates(uid, &dport->hb_coord); + rc = acpi_get_genport_coordinates(uid, dport->hb_coord); if (rc < 0) return rc; /* Adjust back to picoseconds from nanoseconds */ - dport->hb_coord.read_latency *= 1000; - dport->hb_coord.write_latency *= 1000; + for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) { + dport->hb_coord[i].read_latency *= 1000; + dport->hb_coord[i].write_latency *= 1000; + } return 0; } diff --git a/drivers/cxl/core/port.c b/drivers/cxl/core/port.c index 1ff0c730b1c7b..f6e41a42beb62 100644 --- a/drivers/cxl/core/port.c +++ b/drivers/cxl/core/port.c @@ -2159,7 +2159,7 @@ int cxl_endpoint_get_perf_coordinates(struct cxl_port *port, } /* Augment with the generic port (host bridge) perf data */ - combine_coordinates(&c, &dport->hb_coord); + combine_coordinates(&c, &dport->hb_coord[ACCESS_COORDINATE_LOCAL]); /* Get the calculated PCI paths bandwidth */ pdev = to_pci_dev(port->uport_dev->parent); diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h index d6bee32b9c7cc..ea0ae2e0a25f3 100644 --- a/drivers/cxl/cxl.h +++ b/drivers/cxl/cxl.h @@ -673,7 +673,7 @@ struct cxl_dport { struct cxl_port *port; struct cxl_regs regs; struct access_coordinate sw_coord; - struct access_coordinate hb_coord; + struct access_coordinate hb_coord[ACCESS_COORDINATE_MAX]; long link_latency; }; From a90a132db0179960a090eee8e7a5418b33e4ecf1 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Fri, 8 Mar 2024 14:59:24 -0700 Subject: [PATCH 081/138] cxl: Split out combine_coordinates() for common shared usage commit 032f7b37adff6985e22516053698b77131c2ce96 upstream. Refactor the common code of combining coordinates in order to reduce code. Create a new function cxl_cooordinates_combine() it combine two 'struct access_coordinate'. Reviewed-by: Jonathan Cameron Tested-by: Jonathan Cameron Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/20240308220055.2172956-6-dave.jiang@intel.com Signed-off-by: Dan Williams Signed-off-by: Yi Liu Signed-off-by: Bo Li --- drivers/cxl/core/cdat.c | 32 +++++++++++++++++++++++--------- drivers/cxl/core/port.c | 18 ++---------------- drivers/cxl/cxl.h | 4 ++++ 3 files changed, 29 insertions(+), 25 deletions(-) diff --git a/drivers/cxl/core/cdat.c b/drivers/cxl/core/cdat.c index 6bffd3551765b..ac1e0a29af2e3 100644 --- a/drivers/cxl/core/cdat.c +++ b/drivers/cxl/core/cdat.c @@ -185,15 +185,7 @@ static int cxl_port_perf_data_calculate(struct cxl_port *port, xa_for_each(dsmas_xa, index, dent) { int qos_class; - dent->coord.read_latency = dent->coord.read_latency + - c.read_latency; - dent->coord.write_latency = dent->coord.write_latency + - c.write_latency; - dent->coord.read_bandwidth = min_t(int, c.read_bandwidth, - dent->coord.read_bandwidth); - dent->coord.write_bandwidth = min_t(int, c.write_bandwidth, - dent->coord.write_bandwidth); - + cxl_coordinates_combine(&dent->coord, &dent->coord, &c); dent->entries = 1; rc = cxl_root->ops->qos_class(cxl_root, &dent->coord, 1, &qos_class); @@ -484,4 +476,26 @@ void cxl_switch_parse_cdat(struct cxl_port *port) } EXPORT_SYMBOL_NS_GPL(cxl_switch_parse_cdat, CXL); +/** + * cxl_coordinates_combine - Combine the two input coordinates + * + * @out: Output coordinate of c1 and c2 combined + * @c1: input coordinates + * @c2: input coordinates + */ +void cxl_coordinates_combine(struct access_coordinate *out, + struct access_coordinate *c1, + struct access_coordinate *c2) +{ + if (c1->write_bandwidth && c2->write_bandwidth) + out->write_bandwidth = min(c1->write_bandwidth, + c2->write_bandwidth); + out->write_latency = c1->write_latency + c2->write_latency; + + if (c1->read_bandwidth && c2->read_bandwidth) + out->read_bandwidth = min(c1->read_bandwidth, + c2->read_bandwidth); + out->read_latency = c1->read_latency + c2->read_latency; +} + MODULE_IMPORT_NS(CXL); diff --git a/drivers/cxl/core/port.c b/drivers/cxl/core/port.c index f6e41a42beb62..eac1cec735ec0 100644 --- a/drivers/cxl/core/port.c +++ b/drivers/cxl/core/port.c @@ -2103,20 +2103,6 @@ bool schedule_cxl_memdev_detach(struct cxl_memdev *cxlmd) } EXPORT_SYMBOL_NS_GPL(schedule_cxl_memdev_detach, CXL); -static void combine_coordinates(struct access_coordinate *c1, - struct access_coordinate *c2) -{ - if (c2->write_bandwidth) - c1->write_bandwidth = min(c1->write_bandwidth, - c2->write_bandwidth); - c1->write_latency += c2->write_latency; - - if (c2->read_bandwidth) - c1->read_bandwidth = min(c1->read_bandwidth, - c2->read_bandwidth); - c1->read_latency += c2->read_latency; -} - /** * cxl_endpoint_get_perf_coordinates - Retrieve performance numbers stored in dports * of CXL path @@ -2150,7 +2136,7 @@ int cxl_endpoint_get_perf_coordinates(struct cxl_port *port, * nothing to gather. */ while (iter && !is_cxl_root(to_cxl_port(iter->dev.parent))) { - combine_coordinates(&c, &dport->sw_coord); + cxl_coordinates_combine(&c, &c, &dport->sw_coord); c.write_latency += dport->link_latency; c.read_latency += dport->link_latency; @@ -2159,7 +2145,7 @@ int cxl_endpoint_get_perf_coordinates(struct cxl_port *port, } /* Augment with the generic port (host bridge) perf data */ - combine_coordinates(&c, &dport->hb_coord[ACCESS_COORDINATE_LOCAL]); + cxl_coordinates_combine(&c, &c, &dport->hb_coord[ACCESS_COORDINATE_LOCAL]); /* Get the calculated PCI paths bandwidth */ pdev = to_pci_dev(port->uport_dev->parent); diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h index ea0ae2e0a25f3..8a11655c3de6c 100644 --- a/drivers/cxl/cxl.h +++ b/drivers/cxl/cxl.h @@ -885,6 +885,10 @@ int cxl_endpoint_get_perf_coordinates(struct cxl_port *port, void cxl_memdev_update_perf(struct cxl_memdev *cxlmd); +void cxl_coordinates_combine(struct access_coordinate *out, + struct access_coordinate *c1, + struct access_coordinate *c2); + /* * Unit test builds overrides this to __weak, find the 'strong' version * of these symbols in tools/testing/cxl/. From c6c99b424225f488e1d9b4ef8458815995130110 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Fri, 8 Mar 2024 14:59:25 -0700 Subject: [PATCH 082/138] cxl: Split out host bridge access coordinates commit 863027d40993f13155451bd898bfe4c4e9b7002f upstream. The difference between access class 0 and access class 1 for 'struct access_coordinate', if any, is that class 0 is for the distance from the target to the closest initiator and that class 1 is for the distance from the target to the closest CPU. For CXL memory, the nearest initiator may not necessarily be a CPU node. The performance path from the CXL endpoint to the host bridge should remain the same. However, the numbers extracted and stored from HMAT is the difference for the two access classes. Split out the performance numbers for the host bridge (generic target) from the calculation of the entire path in order to allow calculation of both access classes for a CXL region. Reviewed-by: Jonathan Cameron Tested-by: Jonathan Cameron Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/20240308220055.2172956-7-dave.jiang@intel.com Signed-off-by: Dan Williams Signed-off-by: Yi Liu Signed-off-by: Bo Li --- drivers/cxl/core/cdat.c | 28 ++++++++++++++++++++++------ drivers/cxl/core/port.c | 35 ++++++++++++++++++++++++++++++++--- drivers/cxl/cxl.h | 2 ++ 3 files changed, 56 insertions(+), 9 deletions(-) diff --git a/drivers/cxl/core/cdat.c b/drivers/cxl/core/cdat.c index ac1e0a29af2e3..2d835fdd5c6f5 100644 --- a/drivers/cxl/core/cdat.c +++ b/drivers/cxl/core/cdat.c @@ -162,15 +162,22 @@ static int cxl_cdat_endpoint_process(struct cxl_port *port, static int cxl_port_perf_data_calculate(struct cxl_port *port, struct xarray *dsmas_xa) { - struct access_coordinate c; + struct access_coordinate ep_c; + struct access_coordinate coord[ACCESS_COORDINATE_MAX]; struct dsmas_entry *dent; int valid_entries = 0; unsigned long index; int rc; - rc = cxl_endpoint_get_perf_coordinates(port, &c); + rc = cxl_endpoint_get_perf_coordinates(port, &ep_c); if (rc) { - dev_dbg(&port->dev, "Failed to retrieve perf coordinates.\n"); + dev_dbg(&port->dev, "Failed to retrieve ep perf coordinates.\n"); + return rc; + } + + rc = cxl_hb_get_perf_coordinates(port, coord); + if (rc) { + dev_dbg(&port->dev, "Failed to retrieve hb perf coordinates.\n"); return rc; } @@ -185,10 +192,19 @@ static int cxl_port_perf_data_calculate(struct cxl_port *port, xa_for_each(dsmas_xa, index, dent) { int qos_class; - cxl_coordinates_combine(&dent->coord, &dent->coord, &c); + cxl_coordinates_combine(&dent->coord, &dent->coord, &ep_c); + /* + * Keeping the host bridge coordinates separate from the dsmas + * coordinates in order to allow calculation of access class + * 0 and 1 for region later. + */ + cxl_coordinates_combine(&coord[ACCESS_COORDINATE_LOCAL], + &coord[ACCESS_COORDINATE_LOCAL], + &dent->coord); dent->entries = 1; - rc = cxl_root->ops->qos_class(cxl_root, &dent->coord, 1, - &qos_class); + rc = cxl_root->ops->qos_class(cxl_root, + &coord[ACCESS_COORDINATE_LOCAL], + 1, &qos_class); if (rc != 1) continue; diff --git a/drivers/cxl/core/port.c b/drivers/cxl/core/port.c index eac1cec735ec0..3b33779065802 100644 --- a/drivers/cxl/core/port.c +++ b/drivers/cxl/core/port.c @@ -2103,6 +2103,38 @@ bool schedule_cxl_memdev_detach(struct cxl_memdev *cxlmd) } EXPORT_SYMBOL_NS_GPL(schedule_cxl_memdev_detach, CXL); +/** + * cxl_hb_get_perf_coordinates - Retrieve performance numbers between initiator + * and host bridge + * + * @port: endpoint cxl_port + * @coord: output access coordinates + * + * Return: errno on failure, 0 on success. + */ +int cxl_hb_get_perf_coordinates(struct cxl_port *port, + struct access_coordinate *coord) +{ + struct cxl_port *iter = port; + struct cxl_dport *dport; + + if (!is_cxl_endpoint(port)) + return -EINVAL; + + dport = iter->parent_dport; + while (iter && !is_cxl_root(to_cxl_port(iter->dev.parent))) { + iter = to_cxl_port(iter->dev.parent); + dport = iter->parent_dport; + } + + coord[ACCESS_COORDINATE_LOCAL] = + dport->hb_coord[ACCESS_COORDINATE_LOCAL]; + coord[ACCESS_COORDINATE_CPU] = + dport->hb_coord[ACCESS_COORDINATE_CPU]; + + return 0; +} + /** * cxl_endpoint_get_perf_coordinates - Retrieve performance numbers stored in dports * of CXL path @@ -2144,9 +2176,6 @@ int cxl_endpoint_get_perf_coordinates(struct cxl_port *port, dport = iter->parent_dport; } - /* Augment with the generic port (host bridge) perf data */ - cxl_coordinates_combine(&c, &c, &dport->hb_coord[ACCESS_COORDINATE_LOCAL]); - /* Get the calculated PCI paths bandwidth */ pdev = to_pci_dev(port->uport_dev->parent); bw = pcie_bandwidth_available(pdev, NULL, NULL, NULL); diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h index 8a11655c3de6c..a1c6dfbc054ce 100644 --- a/drivers/cxl/cxl.h +++ b/drivers/cxl/cxl.h @@ -882,6 +882,8 @@ void cxl_switch_parse_cdat(struct cxl_port *port); int cxl_endpoint_get_perf_coordinates(struct cxl_port *port, struct access_coordinate *coord); +int cxl_hb_get_perf_coordinates(struct cxl_port *port, + struct access_coordinate *coord); void cxl_memdev_update_perf(struct cxl_memdev *cxlmd); From 62ffe434c15dbf2115be229ea24ec1a03f386f32 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Fri, 8 Mar 2024 14:59:26 -0700 Subject: [PATCH 083/138] cxl: Move QoS class to be calculated from the nearest CPU commit 6ef83c4e19e9ce20a05127eec8a10911cf3516a7 upstream. Retrieve the qos_class (QTG ID) using the access coordinates from the nearest CPU rather than the nearst initiator that may not be a CPU. This may be the more appropriate number that applications care about. For most cases, access0 and access1 have the same values. Link: https://lore.kernel.org/linux-cxl/20240112113023.00006c50@Huawei.com/ Suggested-by: Jonathan Cameron Reviewed-by: Jonathan Cameron Tested-by: Jonathan Cameron Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/20240308220055.2172956-8-dave.jiang@intel.com Signed-off-by: Dan Williams Signed-off-by: Yi Liu Signed-off-by: Bo Li --- drivers/cxl/core/cdat.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/cxl/core/cdat.c b/drivers/cxl/core/cdat.c index 2d835fdd5c6f5..a71ec3746caf1 100644 --- a/drivers/cxl/core/cdat.c +++ b/drivers/cxl/core/cdat.c @@ -198,12 +198,12 @@ static int cxl_port_perf_data_calculate(struct cxl_port *port, * coordinates in order to allow calculation of access class * 0 and 1 for region later. */ - cxl_coordinates_combine(&coord[ACCESS_COORDINATE_LOCAL], - &coord[ACCESS_COORDINATE_LOCAL], + cxl_coordinates_combine(&coord[ACCESS_COORDINATE_CPU], + &coord[ACCESS_COORDINATE_CPU], &dent->coord); dent->entries = 1; rc = cxl_root->ops->qos_class(cxl_root, - &coord[ACCESS_COORDINATE_LOCAL], + &coord[ACCESS_COORDINATE_CPU], 1, &qos_class); if (rc != 1) continue; From 6146c8f33a6c16e0d7f9d9417f7ec704829e05c4 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Fri, 8 Mar 2024 14:59:27 -0700 Subject: [PATCH 084/138] cxl: Set cxlmd->endpoint before adding port device commit 3d8be8b398e3d315200d6c139f0166fe5f1bd576 upstream. Move setting of cxlmd->endpoint to before calling add_device() on the port device. Otherwise when referencing cxlmd->endpoint in region discovery code that is triggered by the port driver probe function, the endpoint port pointer is not valid. Current code does not hit this issue yet since cxlmd->endpoint is not being referenced during region discovery. However follow on code that does performance calculations will. Tested-by: Wonjae Lee Reviewed-by: Jonathan Cameron Tested-by: Jonathan Cameron Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/20240308220055.2172956-9-dave.jiang@intel.com Signed-off-by: Dan Williams Signed-off-by: Yi Liu Signed-off-by: Bo Li --- drivers/cxl/core/port.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/cxl/core/port.c b/drivers/cxl/core/port.c index 3b33779065802..de61b526954df 100644 --- a/drivers/cxl/core/port.c +++ b/drivers/cxl/core/port.c @@ -822,6 +822,7 @@ static struct cxl_port *__devm_cxl_add_port(struct device *host, */ port->reg_map = cxlds->reg_map; port->reg_map.host = &port->dev; + cxlmd->endpoint = port; } else if (parent_dport) { rc = dev_set_name(dev, "port%d", port->id); if (rc) @@ -1374,7 +1375,6 @@ int cxl_endpoint_autoremove(struct cxl_memdev *cxlmd, struct cxl_port *endpoint) get_device(host); get_device(&endpoint->dev); - cxlmd->endpoint = endpoint; cxlmd->depth = endpoint->depth; return devm_add_action_or_reset(dev, delete_endpoint, cxlmd); } From 0d5aa3c9d97fdcfdcb0372308f902351bfc77585 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Fri, 8 Mar 2024 14:59:28 -0700 Subject: [PATCH 085/138] cxl/region: Calculate performance data for a region commit 3d9f4a197230d6f4d5f816bcae0e0497b0eec410 upstream. Calculate and store the performance data for a CXL region. Find the worst read and write latency for all the included ranges from each of the devices that attributes to the region and designate that as the latency data. Sum all the read and write bandwidth data for each of the device region and that is the total bandwidth for the region. The perf list is expected to be constructed before the endpoint decoders are registered and thus there should be no early reading of the entries from the region assemble action. The calling of the region qos calculate function is under the protection of cxl_dpa_rwsem and will ensure that all DPA associated work has completed. Reviewed-by: Jonathan Cameron Tested-by: Jonathan Cameron Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/20240308220055.2172956-10-dave.jiang@intel.com Signed-off-by: Dan Williams Signed-off-by: Yi Liu Signed-off-by: Bo Li --- drivers/cxl/core/cdat.c | 64 +++++++++++++++++++++++++++++++++++++++ drivers/cxl/core/region.c | 2 ++ drivers/cxl/cxl.h | 4 +++ 3 files changed, 70 insertions(+) diff --git a/drivers/cxl/core/cdat.c b/drivers/cxl/core/cdat.c index a71ec3746caf1..ab4f21a69df48 100644 --- a/drivers/cxl/core/cdat.c +++ b/drivers/cxl/core/cdat.c @@ -515,3 +515,67 @@ void cxl_coordinates_combine(struct access_coordinate *out, } MODULE_IMPORT_NS(CXL); + +void cxl_region_perf_data_calculate(struct cxl_region *cxlr, + struct cxl_endpoint_decoder *cxled) +{ + struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); + struct cxl_port *port = cxlmd->endpoint; + struct cxl_dev_state *cxlds = cxlmd->cxlds; + struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds); + struct access_coordinate hb_coord[ACCESS_COORDINATE_MAX]; + struct access_coordinate coord; + struct range dpa = { + .start = cxled->dpa_res->start, + .end = cxled->dpa_res->end, + }; + struct cxl_dpa_perf *perf; + int rc; + + switch (cxlr->mode) { + case CXL_DECODER_RAM: + perf = &mds->ram_perf; + break; + case CXL_DECODER_PMEM: + perf = &mds->pmem_perf; + break; + default: + return; + } + + lockdep_assert_held(&cxl_dpa_rwsem); + + if (!range_contains(&perf->dpa_range, &dpa)) + return; + + rc = cxl_hb_get_perf_coordinates(port, hb_coord); + if (rc) { + dev_dbg(&port->dev, "Failed to retrieve hb perf coordinates.\n"); + return; + } + + for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) { + /* Pickup the host bridge coords */ + cxl_coordinates_combine(&coord, &hb_coord[i], &perf->coord); + + /* Get total bandwidth and the worst latency for the cxl region */ + cxlr->coord[i].read_latency = max_t(unsigned int, + cxlr->coord[i].read_latency, + coord.read_latency); + cxlr->coord[i].write_latency = max_t(unsigned int, + cxlr->coord[i].write_latency, + coord.write_latency); + cxlr->coord[i].read_bandwidth += coord.read_bandwidth; + cxlr->coord[i].write_bandwidth += coord.write_bandwidth; + + /* + * Convert latency to nanosec from picosec to be consistent + * with the resulting latency coordinates computed by the + * HMAT_REPORTING code. + */ + cxlr->coord[i].read_latency = + DIV_ROUND_UP(cxlr->coord[i].read_latency, 1000); + cxlr->coord[i].write_latency = + DIV_ROUND_UP(cxlr->coord[i].write_latency, 1000); + } +} diff --git a/drivers/cxl/core/region.c b/drivers/cxl/core/region.c index 4b89b5258bb5d..516c00f32fe55 100644 --- a/drivers/cxl/core/region.c +++ b/drivers/cxl/core/region.c @@ -1839,6 +1839,8 @@ static int cxl_region_attach(struct cxl_region *cxlr, return -EINVAL; } + cxl_region_perf_data_calculate(cxlr, cxled); + if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) { int i; diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h index a1c6dfbc054ce..6748740e512a7 100644 --- a/drivers/cxl/cxl.h +++ b/drivers/cxl/cxl.h @@ -519,6 +519,7 @@ struct cxl_region_params { * @cxlr_pmem: (for pmem regions) cached copy of the nvdimm bridge * @flags: Region state flags * @params: active + config params for the region + * @coord: QoS access coordinates for the region */ struct cxl_region { struct device dev; @@ -529,6 +530,7 @@ struct cxl_region { struct cxl_pmem_region *cxlr_pmem; unsigned long flags; struct cxl_region_params params; + struct access_coordinate coord[ACCESS_COORDINATE_MAX]; }; struct cxl_nvdimm_bridge { @@ -884,6 +886,8 @@ int cxl_endpoint_get_perf_coordinates(struct cxl_port *port, struct access_coordinate *coord); int cxl_hb_get_perf_coordinates(struct cxl_port *port, struct access_coordinate *coord); +void cxl_region_perf_data_calculate(struct cxl_region *cxlr, + struct cxl_endpoint_decoder *cxled); void cxl_memdev_update_perf(struct cxl_memdev *cxlmd); From 89503e2e69771ee14cd32208df5f75782af50490 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Fri, 8 Mar 2024 14:59:29 -0700 Subject: [PATCH 086/138] cxl/region: Add sysfs attribute for locality attributes of CXL regions commit c20eaf44113eac090b0d77fa2036143a836b9f56 upstream. Add read/write latencies and bandwidth sysfs attributes for the enabled CXL region. The bandwidth is the aggregated bandwidth of all devices that contribute to the CXL region. The latency is the worst latency of the device amongst all the devices that contribute to the CXL region. Reviewed-by: Jonathan Cameron Tested-by: Jonathan Cameron Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/20240308220055.2172956-11-dave.jiang@intel.com Signed-off-by: Dan Williams Signed-off-by: Yi Liu Signed-off-by: Bo Li --- Documentation/ABI/testing/sysfs-bus-cxl | 34 +++++++++ drivers/cxl/core/region.c | 94 +++++++++++++++++++++++++ 2 files changed, 128 insertions(+) diff --git a/Documentation/ABI/testing/sysfs-bus-cxl b/Documentation/ABI/testing/sysfs-bus-cxl index fff2581b80335..3f5627a1210a1 100644 --- a/Documentation/ABI/testing/sysfs-bus-cxl +++ b/Documentation/ABI/testing/sysfs-bus-cxl @@ -552,3 +552,37 @@ Description: attribute is only visible for devices supporting the capability. The retrieved errors are logged as kernel events when cxl_poison event tracing is enabled. + + +What: /sys/bus/cxl/devices/regionZ/accessY/read_bandwidth + /sys/bus/cxl/devices/regionZ/accessY/write_banwidth +Date: Jan, 2024 +KernelVersion: v6.9 +Contact: linux-cxl@vger.kernel.org +Description: + (RO) The aggregated read or write bandwidth of the region. The + number is the accumulated read or write bandwidth of all CXL memory + devices that contributes to the region in MB/s. It is + identical data that should appear in + /sys/devices/system/node/nodeX/accessY/initiators/read_bandwidth or + /sys/devices/system/node/nodeX/accessY/initiators/write_bandwidth. + See Documentation/ABI/stable/sysfs-devices-node. access0 provides + the number to the closest initiator and access1 provides the + number to the closest CPU. + + +What: /sys/bus/cxl/devices/regionZ/accessY/read_latency + /sys/bus/cxl/devices/regionZ/accessY/write_latency +Date: Jan, 2024 +KernelVersion: v6.9 +Contact: linux-cxl@vger.kernel.org +Description: + (RO) The read or write latency of the region. The number is + the worst read or write latency of all CXL memory devices that + contributes to the region in nanoseconds. It is identical data + that should appear in + /sys/devices/system/node/nodeX/accessY/initiators/read_latency or + /sys/devices/system/node/nodeX/accessY/initiators/write_latency. + See Documentation/ABI/stable/sysfs-devices-node. access0 provides + the number to the closest initiator and access1 provides the + number to the closest CPU. diff --git a/drivers/cxl/core/region.c b/drivers/cxl/core/region.c index 516c00f32fe55..20d5e0da5dc3e 100644 --- a/drivers/cxl/core/region.c +++ b/drivers/cxl/core/region.c @@ -30,6 +30,98 @@ static struct cxl_region *to_cxl_region(struct device *dev); +#define __ACCESS_ATTR_RO(_level, _name) { \ + .attr = { .name = __stringify(_name), .mode = 0444 }, \ + .show = _name##_access##_level##_show, \ +} + +#define ACCESS_DEVICE_ATTR_RO(level, name) \ + struct device_attribute dev_attr_access##level##_##name = __ACCESS_ATTR_RO(level, name) + +#define ACCESS_ATTR_RO(level, attrib) \ +static ssize_t attrib##_access##level##_show(struct device *dev, \ + struct device_attribute *attr, \ + char *buf) \ +{ \ + struct cxl_region *cxlr = to_cxl_region(dev); \ + \ + if (cxlr->coord[level].attrib == 0) \ + return -ENOENT; \ + \ + return sysfs_emit(buf, "%u\n", cxlr->coord[level].attrib); \ +} \ +static ACCESS_DEVICE_ATTR_RO(level, attrib) + +ACCESS_ATTR_RO(0, read_bandwidth); +ACCESS_ATTR_RO(0, read_latency); +ACCESS_ATTR_RO(0, write_bandwidth); +ACCESS_ATTR_RO(0, write_latency); + +#define ACCESS_ATTR_DECLARE(level, attrib) \ + (&dev_attr_access##level##_##attrib.attr) + +static struct attribute *access0_coordinate_attrs[] = { + ACCESS_ATTR_DECLARE(0, read_bandwidth), + ACCESS_ATTR_DECLARE(0, write_bandwidth), + ACCESS_ATTR_DECLARE(0, read_latency), + ACCESS_ATTR_DECLARE(0, write_latency), + NULL +}; + +ACCESS_ATTR_RO(1, read_bandwidth); +ACCESS_ATTR_RO(1, read_latency); +ACCESS_ATTR_RO(1, write_bandwidth); +ACCESS_ATTR_RO(1, write_latency); + +static struct attribute *access1_coordinate_attrs[] = { + ACCESS_ATTR_DECLARE(1, read_bandwidth), + ACCESS_ATTR_DECLARE(1, write_bandwidth), + ACCESS_ATTR_DECLARE(1, read_latency), + ACCESS_ATTR_DECLARE(1, write_latency), + NULL +}; + +#define ACCESS_VISIBLE(level) \ +static umode_t cxl_region_access##level##_coordinate_visible( \ + struct kobject *kobj, struct attribute *a, int n) \ +{ \ + struct device *dev = kobj_to_dev(kobj); \ + struct cxl_region *cxlr = to_cxl_region(dev); \ + \ + if (a == &dev_attr_access##level##_read_latency.attr && \ + cxlr->coord[level].read_latency == 0) \ + return 0; \ + \ + if (a == &dev_attr_access##level##_write_latency.attr && \ + cxlr->coord[level].write_latency == 0) \ + return 0; \ + \ + if (a == &dev_attr_access##level##_read_bandwidth.attr && \ + cxlr->coord[level].read_bandwidth == 0) \ + return 0; \ + \ + if (a == &dev_attr_access##level##_write_bandwidth.attr && \ + cxlr->coord[level].write_bandwidth == 0) \ + return 0; \ + \ + return a->mode; \ +} + +ACCESS_VISIBLE(0); +ACCESS_VISIBLE(1); + +static const struct attribute_group cxl_region_access0_coordinate_group = { + .name = "access0", + .attrs = access0_coordinate_attrs, + .is_visible = cxl_region_access0_coordinate_visible, +}; + +static const struct attribute_group cxl_region_access1_coordinate_group = { + .name = "access1", + .attrs = access1_coordinate_attrs, + .is_visible = cxl_region_access1_coordinate_visible, +}; + static ssize_t uuid_show(struct device *dev, struct device_attribute *attr, char *buf) { @@ -2150,6 +2242,8 @@ static const struct attribute_group *region_groups[] = { &cxl_base_attribute_group, &cxl_region_group, &cxl_region_target_group, + &cxl_region_access0_coordinate_group, + &cxl_region_access1_coordinate_group, NULL, }; From 04158652b37e2c8cd6207acb5cd7cd272e9c6d5a Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Fri, 8 Mar 2024 14:59:30 -0700 Subject: [PATCH 087/138] cxl/region: Add memory hotplug notifier for cxl region commit 067353a46d8ccdac279ebab97c038c3658e97541 upstream. When the CXL region is formed, the driver computes the performance data for the region. However this data is not available at the node data collection that has been populated by the HMAT during kernel initialization. Add a memory hotplug notifier to update the access coordinates to the 'struct memory_target' context kept by the HMAT_REPORTING code. Add CXL_CALLBACK_PRI for a memory hotplug callback priority. Set the priority number to be called before HMAT_CALLBACK_PRI. The CXL update must happen before hmat_callback(). A new HMAT_REPORTING helper hmat_update_target_coordinates() is added in order to allow CXL to update the memory_target access coordinates. A new ext_updated member is added to the memory_target to indicate that the access coordinates within the memory_target has been updated by an external agent such as CXL. This prevents data being overwritten by the hmat_update_target_attrs() triggered by hmat_callback(). Cc: Andrew Morton Cc: Rafael J. Wysocki Reviewed-by: Huang, Ying Reviewed-by: Jonathan Cameron Tested-by: Jonathan Cameron Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/20240308220055.2172956-12-dave.jiang@intel.com Signed-off-by: Dan Williams Signed-off-by: Yi Liu Signed-off-by: Bo Li --- drivers/acpi/numa/hmat.c | 34 ++++++++++++++++++++ drivers/cxl/core/cdat.c | 6 ++++ drivers/cxl/core/core.h | 3 ++ drivers/cxl/core/region.c | 68 +++++++++++++++++++++++++++++++++++++++ drivers/cxl/cxl.h | 3 ++ include/linux/acpi.h | 12 +++++++ include/linux/memory.h | 1 + 7 files changed, 127 insertions(+) diff --git a/drivers/acpi/numa/hmat.c b/drivers/acpi/numa/hmat.c index c170691b38f2a..56b8358245af7 100644 --- a/drivers/acpi/numa/hmat.c +++ b/drivers/acpi/numa/hmat.c @@ -74,6 +74,7 @@ struct memory_target { struct node_cache_attrs cache_attrs; u8 gen_port_device_handle[ACPI_SRAT_DEVICE_HANDLE_SIZE]; bool registered; + bool ext_updated; /* externally updated */ }; struct memory_initiator { @@ -328,6 +329,35 @@ static void hmat_update_target_access(struct memory_target *target, } } +int hmat_update_target_coordinates(int nid, struct access_coordinate *coord, + enum access_coordinate_class access) +{ + struct memory_target *target; + int pxm; + + if (nid == NUMA_NO_NODE) + return -EINVAL; + + pxm = node_to_pxm(nid); + guard(mutex)(&target_lock); + target = find_mem_target(pxm); + if (!target) + return -ENODEV; + + hmat_update_target_access(target, ACPI_HMAT_READ_LATENCY, + coord->read_latency, access); + hmat_update_target_access(target, ACPI_HMAT_WRITE_LATENCY, + coord->write_latency, access); + hmat_update_target_access(target, ACPI_HMAT_READ_BANDWIDTH, + coord->read_bandwidth, access); + hmat_update_target_access(target, ACPI_HMAT_WRITE_BANDWIDTH, + coord->write_bandwidth, access); + target->ext_updated = true; + + return 0; +} +EXPORT_SYMBOL_GPL(hmat_update_target_coordinates); + static __init void hmat_add_locality(struct acpi_hmat_locality *hmat_loc) { struct memory_locality *loc; @@ -699,6 +729,10 @@ static void hmat_update_target_attrs(struct memory_target *target, u32 best = 0; int i; + /* Don't update if an external agent has changed the data. */ + if (target->ext_updated) + return; + /* Don't update for generic port if there's no device handle */ if ((access == NODE_ACCESS_CLASS_GENPORT_SINK_LOCAL || access == NODE_ACCESS_CLASS_GENPORT_SINK_CPU) && diff --git a/drivers/cxl/core/cdat.c b/drivers/cxl/core/cdat.c index ab4f21a69df48..3143617a1886f 100644 --- a/drivers/cxl/core/cdat.c +++ b/drivers/cxl/core/cdat.c @@ -579,3 +579,9 @@ void cxl_region_perf_data_calculate(struct cxl_region *cxlr, DIV_ROUND_UP(cxlr->coord[i].write_latency, 1000); } } + +int cxl_update_hmat_access_coordinates(int nid, struct cxl_region *cxlr, + enum access_coordinate_class access) +{ + return hmat_update_target_coordinates(nid, &cxlr->coord[access], access); +} diff --git a/drivers/cxl/core/core.h b/drivers/cxl/core/core.h index 76c2861e20027..01ce740573aab 100644 --- a/drivers/cxl/core/core.h +++ b/drivers/cxl/core/core.h @@ -97,4 +97,7 @@ enum cxl_poison_trace_type { long cxl_pci_get_latency(struct pci_dev *pdev); +int cxl_update_hmat_access_coordinates(int nid, struct cxl_region *cxlr, + enum access_coordinate_class access); + #endif /* __CXL_CORE_H__ */ diff --git a/drivers/cxl/core/region.c b/drivers/cxl/core/region.c index 20d5e0da5dc3e..ef1716dd31197 100644 --- a/drivers/cxl/core/region.c +++ b/drivers/cxl/core/region.c @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -116,12 +117,22 @@ static const struct attribute_group cxl_region_access0_coordinate_group = { .is_visible = cxl_region_access0_coordinate_visible, }; +static const struct attribute_group *get_cxl_region_access0_group(void) +{ + return &cxl_region_access0_coordinate_group; +} + static const struct attribute_group cxl_region_access1_coordinate_group = { .name = "access1", .attrs = access1_coordinate_attrs, .is_visible = cxl_region_access1_coordinate_visible, }; +static const struct attribute_group *get_cxl_region_access1_group(void) +{ + return &cxl_region_access1_coordinate_group; +} + static ssize_t uuid_show(struct device *dev, struct device_attribute *attr, char *buf) { @@ -2297,6 +2308,7 @@ static void unregister_region(void *_cxlr) struct cxl_region_params *p = &cxlr->params; int i; + unregister_memory_notifier(&cxlr->memory_notifier); device_del(&cxlr->dev); /* @@ -2341,6 +2353,58 @@ static struct cxl_region *cxl_region_alloc(struct cxl_root_decoder *cxlrd, int i return cxlr; } +static bool cxl_region_update_coordinates(struct cxl_region *cxlr, int nid) +{ + int cset = 0; + int rc; + + for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) { + if (cxlr->coord[i].read_bandwidth) { + rc = cxl_update_hmat_access_coordinates(nid, cxlr, i); + if (rc == 0) + cset++; + } + } + + if (!cset) + return false; + + rc = sysfs_update_group(&cxlr->dev.kobj, get_cxl_region_access0_group()); + if (rc) + dev_dbg(&cxlr->dev, "Failed to update access0 group\n"); + + rc = sysfs_update_group(&cxlr->dev.kobj, get_cxl_region_access1_group()); + if (rc) + dev_dbg(&cxlr->dev, "Failed to update access1 group\n"); + + return true; +} + +static int cxl_region_perf_attrs_callback(struct notifier_block *nb, + unsigned long action, void *arg) +{ + struct cxl_region *cxlr = container_of(nb, struct cxl_region, + memory_notifier); + struct cxl_region_params *p = &cxlr->params; + struct cxl_endpoint_decoder *cxled = p->targets[0]; + struct cxl_decoder *cxld = &cxled->cxld; + struct memory_notify *mnb = arg; + int nid = mnb->status_change_nid; + int region_nid; + + if (nid == NUMA_NO_NODE || action != MEM_ONLINE) + return NOTIFY_DONE; + + region_nid = phys_to_target_node(cxld->hpa_range.start); + if (nid != region_nid) + return NOTIFY_DONE; + + if (!cxl_region_update_coordinates(cxlr, nid)) + return NOTIFY_DONE; + + return NOTIFY_OK; +} + /** * devm_cxl_add_region - Adds a region to a decoder * @cxlrd: root decoder @@ -2379,6 +2443,10 @@ static struct cxl_region *devm_cxl_add_region(struct cxl_root_decoder *cxlrd, if (rc) goto err; + cxlr->memory_notifier.notifier_call = cxl_region_perf_attrs_callback; + cxlr->memory_notifier.priority = CXL_CALLBACK_PRI; + register_memory_notifier(&cxlr->memory_notifier); + rc = devm_add_action_or_reset(port->uport_dev, unregister_region, cxlr); if (rc) return ERR_PTR(rc); diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h index 6748740e512a7..31f5e2c5688f2 100644 --- a/drivers/cxl/cxl.h +++ b/drivers/cxl/cxl.h @@ -6,6 +6,7 @@ #include #include +#include #include #include #include @@ -520,6 +521,7 @@ struct cxl_region_params { * @flags: Region state flags * @params: active + config params for the region * @coord: QoS access coordinates for the region + * @memory_notifier: notifier for setting the access coordinates to node */ struct cxl_region { struct device dev; @@ -531,6 +533,7 @@ struct cxl_region { unsigned long flags; struct cxl_region_params params; struct access_coordinate coord[ACCESS_COORDINATE_MAX]; + struct notifier_block memory_notifier; }; struct cxl_nvdimm_bridge { diff --git a/include/linux/acpi.h b/include/linux/acpi.h index 3973069877123..7412d1379c211 100644 --- a/include/linux/acpi.h +++ b/include/linux/acpi.h @@ -1551,4 +1551,16 @@ static inline void acpi_use_parent_companion(struct device *dev) ACPI_COMPANION_SET(dev, ACPI_COMPANION(dev->parent)); } +#ifdef CONFIG_ACPI_HMAT +int hmat_update_target_coordinates(int nid, struct access_coordinate *coord, + enum access_coordinate_class access); +#else +static inline int hmat_update_target_coordinates(int nid, + struct access_coordinate *coord, + enum access_coordinate_class access) +{ + return -EOPNOTSUPP; +} +#endif + #endif /*_LINUX_ACPI_H*/ diff --git a/include/linux/memory.h b/include/linux/memory.h index f53cfdaaaa416..d8588256578ae 100644 --- a/include/linux/memory.h +++ b/include/linux/memory.h @@ -114,6 +114,7 @@ struct mem_section; #define DEFAULT_CALLBACK_PRI 0 #define SLAB_CALLBACK_PRI 1 #define HMAT_CALLBACK_PRI 2 +#define CXL_CALLBACK_PRI 5 #define MM_COMPUTE_BATCH_PRI 10 #define CPUSET_CALLBACK_PRI 10 #define MEMTIER_HOTPLUG_PRI 100 From ebe186d4f5125f0f8c82e7ce84cac9d280acee7c Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Fri, 8 Mar 2024 14:59:31 -0700 Subject: [PATCH 088/138] cxl/region: Deal with numa nodes not enumerated by SRAT commit debdce20c4f28b7e5aa48512e7abf270a00e9051 upstream. For the numa nodes that are not created by SRAT, no memory_target is allocated and is not managed by the HMAT_REPORTING code. Therefore hmat_callback() memory hotplug notifier will exit early on those NUMA nodes. The CXL memory hotplug notifier will need to call node_set_perf_attrs() directly in order to setup the access sysfs attributes. In acpi_numa_init(), the last proximity domain (pxm) id created by SRAT is stored. Add a helper function acpi_node_backed_by_real_pxm() in order to check if a NUMA node id is defined by SRAT or created by CFMWS. node_set_perf_attrs() symbol is exported to allow update of perf attribs for a node. The sysfs path of /sys/devices/system/node/nodeX/access0/initiators/* is created by node_set_perf_attrs() for the various attributes where nodeX is matched to the NUMA node of the CXL region. Cc: Rafael J. Wysocki Reviewed-by: Alison Schofield Reviewed-by: Jonathan Cameron Tested-by: Jonathan Cameron Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/20240308220055.2172956-13-dave.jiang@intel.com Signed-off-by: Dan Williams Signed-off-by: Yi Liu Signed-off-by: Bo Li --- drivers/acpi/numa/srat.c | 11 +++++++++++ drivers/base/node.c | 1 + drivers/cxl/core/cdat.c | 5 +++++ drivers/cxl/core/core.h | 1 + drivers/cxl/core/region.c | 7 ++++++- include/linux/acpi.h | 9 +++++++++ 6 files changed, 33 insertions(+), 1 deletion(-) diff --git a/drivers/acpi/numa/srat.c b/drivers/acpi/numa/srat.c index 848942bf883cb..f830d7c499e11 100644 --- a/drivers/acpi/numa/srat.c +++ b/drivers/acpi/numa/srat.c @@ -29,6 +29,8 @@ static int node_to_pxm_map[MAX_NUMNODES] unsigned char acpi_srat_revision __initdata; static int acpi_numa __initdata; +static int last_real_pxm; + void __init disable_srat(void) { acpi_numa = -1; @@ -541,6 +543,7 @@ int __init acpi_numa_init(void) if (node_to_pxm_map[i] > fake_pxm) fake_pxm = node_to_pxm_map[i]; } + last_real_pxm = fake_pxm; fake_pxm++; acpi_table_parse_cedt(ACPI_CEDT_TYPE_CFMWS, acpi_parse_cfmws, &fake_pxm); @@ -552,6 +555,14 @@ int __init acpi_numa_init(void) return 0; } +bool acpi_node_backed_by_real_pxm(int nid) +{ + int pxm = node_to_pxm(nid); + + return pxm <= last_real_pxm; +} +EXPORT_SYMBOL_GPL(acpi_node_backed_by_real_pxm); + static int acpi_get_pxm(acpi_handle h) { unsigned long long pxm; diff --git a/drivers/base/node.c b/drivers/base/node.c index 2b398c8a0f06c..415a99c0adaa7 100644 --- a/drivers/base/node.c +++ b/drivers/base/node.c @@ -215,6 +215,7 @@ void node_set_perf_attrs(unsigned int nid, struct access_coordinate *coord, } } } +EXPORT_SYMBOL_GPL(node_set_perf_attrs); /** * struct node_cache_info - Internal tracking for memory node caches diff --git a/drivers/cxl/core/cdat.c b/drivers/cxl/core/cdat.c index 3143617a1886f..6965d8bea4e06 100644 --- a/drivers/cxl/core/cdat.c +++ b/drivers/cxl/core/cdat.c @@ -585,3 +585,8 @@ int cxl_update_hmat_access_coordinates(int nid, struct cxl_region *cxlr, { return hmat_update_target_coordinates(nid, &cxlr->coord[access], access); } + +bool cxl_need_node_perf_attrs_update(int nid) +{ + return !acpi_node_backed_by_real_pxm(nid); +} diff --git a/drivers/cxl/core/core.h b/drivers/cxl/core/core.h index 01ce740573aab..87008505f8a9e 100644 --- a/drivers/cxl/core/core.h +++ b/drivers/cxl/core/core.h @@ -99,5 +99,6 @@ long cxl_pci_get_latency(struct pci_dev *pdev); int cxl_update_hmat_access_coordinates(int nid, struct cxl_region *cxlr, enum access_coordinate_class access); +bool cxl_need_node_perf_attrs_update(int nid); #endif /* __CXL_CORE_H__ */ diff --git a/drivers/cxl/core/region.c b/drivers/cxl/core/region.c index ef1716dd31197..5dbcc3352eb01 100644 --- a/drivers/cxl/core/region.c +++ b/drivers/cxl/core/region.c @@ -2360,7 +2360,12 @@ static bool cxl_region_update_coordinates(struct cxl_region *cxlr, int nid) for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) { if (cxlr->coord[i].read_bandwidth) { - rc = cxl_update_hmat_access_coordinates(nid, cxlr, i); + rc = 0; + if (cxl_need_node_perf_attrs_update(nid)) + node_set_perf_attrs(nid, &cxlr->coord[i], i); + else + rc = cxl_update_hmat_access_coordinates(nid, cxlr, i); + if (rc == 0) cset++; } diff --git a/include/linux/acpi.h b/include/linux/acpi.h index 7412d1379c211..a057841c7f676 100644 --- a/include/linux/acpi.h +++ b/include/linux/acpi.h @@ -1563,4 +1563,13 @@ static inline int hmat_update_target_coordinates(int nid, } #endif +#ifdef CONFIG_ACPI_NUMA +bool acpi_node_backed_by_real_pxm(int nid); +#else +static inline bool acpi_node_backed_by_real_pxm(int nid) +{ + return false; +} +#endif + #endif /*_LINUX_ACPI_H*/ From 5fd85d07410555c72899fb43266d7bc24207a20e Mon Sep 17 00:00:00 2001 From: Ben Cheatham Date: Mon, 11 Mar 2024 09:25:05 -0500 Subject: [PATCH 089/138] EINJ: Migrate to a platform driver commit 5621fafaac0031e8813be4873f8efa097591b026 upstream. Change the EINJ module to install a platform device/driver on module init and move the module init() and exit() functions to driver probe and remove. This change allows the EINJ module to load regardless of whether setting up EINJ succeeds, which allows dependent modules to still load (i.e. the CXL core). Since EINJ may no longer be initialized when the module loads, any functions that are called from dependent/external modules should safegaurd against the case EINJ didn't load. Reviewed-by: Jonathan Cameron Reviewed-by: Dan Williams Signed-off-by: Ben Cheatham Link: https://lore.kernel.org/r/20240311142508.31717-2-Benjamin.Cheatham@amd.com Signed-off-by: Dan Williams Signed-off-by: Yi Liu Signed-off-by: Bo Li --- drivers/acpi/apei/einj.c | 48 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 44 insertions(+), 4 deletions(-) diff --git a/drivers/acpi/apei/einj.c b/drivers/acpi/apei/einj.c index 89fb9331c611e..937c69844dac0 100644 --- a/drivers/acpi/apei/einj.c +++ b/drivers/acpi/apei/einj.c @@ -21,6 +21,7 @@ #include #include #include +#include #include #include "apei-internal.h" @@ -137,6 +138,11 @@ static struct apei_exec_ins_type einj_ins_type[] = { */ static DEFINE_MUTEX(einj_mutex); +/* + * Exported APIs use this flag to exit early if einj_probe() failed. + */ +static bool einj_initialized __ro_after_init; + static void *einj_param; static void einj_exec_ctx_init(struct apei_exec_context *ctx) @@ -703,21 +709,21 @@ static int einj_check_table(struct acpi_table_einj *einj_tab) return 0; } -static int __init einj_init(void) +static int __init einj_probe(struct platform_device *pdev) { int rc; acpi_status status; struct apei_exec_context ctx; if (acpi_disabled) { - pr_info("ACPI disabled.\n"); + pr_debug("ACPI disabled.\n"); return -ENODEV; } status = acpi_get_table(ACPI_SIG_EINJ, 0, (struct acpi_table_header **)&einj_tab); if (status == AE_NOT_FOUND) { - pr_warn("EINJ table not found.\n"); + pr_debug("EINJ table not found.\n"); return -ENODEV; } else if (ACPI_FAILURE(status)) { pr_err("Failed to get EINJ table: %s\n", @@ -805,7 +811,7 @@ static int __init einj_init(void) return rc; } -static void __exit einj_exit(void) +static void __exit einj_remove(struct platform_device *pdev) { struct apei_exec_context ctx; @@ -826,6 +832,40 @@ static void __exit einj_exit(void) acpi_put_table((struct acpi_table_header *)einj_tab); } +static struct platform_device *einj_dev; +static struct platform_driver einj_driver = { + .remove_new = einj_remove, + .driver = { + .name = "acpi-einj", + }, +}; + +static int __init einj_init(void) +{ + struct platform_device_info einj_dev_info = { + .name = "acpi-einj", + .id = -1, + }; + int rc; + + einj_dev = platform_device_register_full(&einj_dev_info); + if (IS_ERR(einj_dev)) + return PTR_ERR(einj_dev); + + rc = platform_driver_probe(&einj_driver, einj_probe); + einj_initialized = rc == 0; + + return 0; +} + +static void __exit einj_exit(void) +{ + if (einj_initialized) + platform_driver_unregister(&einj_driver); + + platform_device_del(einj_dev); +} + module_init(einj_init); module_exit(einj_exit); From aadd8699cc7552dfe5f45f8953e4bf86280281f6 Mon Sep 17 00:00:00 2001 From: Ben Cheatham Date: Mon, 11 Mar 2024 09:25:06 -0500 Subject: [PATCH 090/138] EINJ: Add CXL error type support commit 12fb28ea6b1cf24bde27c406eb38ee3c108d73f9 upstream. Move CXL protocol error types from einj.c (now einj-core.c) to einj-cxl.c. einj-cxl.c implements the necessary handling for CXL protocol error injection and exposes an API for the CXL core to use said functionality, while also allowing the EINJ module to be built without CXL support. Because CXL error types targeting CXL 1.0/1.1 ports require special handling, only allow them to be injected through the new cxl debugfs interface (next commit) and return an error when attempting to inject through the legacy interface. Reviewed-by: Jonathan Cameron Signed-off-by: Ben Cheatham Link: https://lore.kernel.org/r/20240311142508.31717-3-Benjamin.Cheatham@amd.com Signed-off-by: Dan Williams Signed-off-by: Yi Liu Signed-off-by: Bo Li --- MAINTAINERS | 1 + drivers/acpi/apei/Kconfig | 13 +++ drivers/acpi/apei/Makefile | 2 + drivers/acpi/apei/apei-internal.h | 18 ++++ drivers/acpi/apei/{einj.c => einj-core.c} | 76 +++++++++++---- drivers/acpi/apei/einj-cxl.c | 113 ++++++++++++++++++++++ include/linux/einj-cxl.h | 44 +++++++++ 7 files changed, 249 insertions(+), 18 deletions(-) rename drivers/acpi/apei/{einj.c => einj-core.c} (93%) create mode 100644 drivers/acpi/apei/einj-cxl.c create mode 100644 include/linux/einj-cxl.h diff --git a/MAINTAINERS b/MAINTAINERS index 5871b31669415..12d1ef1a37503 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -5224,6 +5224,7 @@ M: Dan Williams L: linux-cxl@vger.kernel.org S: Maintained F: drivers/cxl/ +F: include/linux/cxl-einj.h F: include/linux/cxl-event.h F: include/uapi/linux/cxl_mem.h F: tools/testing/cxl/ diff --git a/drivers/acpi/apei/Kconfig b/drivers/acpi/apei/Kconfig index 71e0d64a7792e..070c07d68dfb2 100644 --- a/drivers/acpi/apei/Kconfig +++ b/drivers/acpi/apei/Kconfig @@ -61,6 +61,19 @@ config ACPI_APEI_EINJ mainly used for debugging and testing the other parts of APEI and some other RAS features. +config ACPI_APEI_EINJ_CXL + bool "CXL Error INJection Support" + default ACPI_APEI_EINJ + depends on ACPI_APEI_EINJ + depends on CXL_BUS && CXL_BUS <= ACPI_APEI_EINJ + help + Support for CXL protocol Error INJection through debugfs/cxl. + Availability and which errors are supported is dependent on + the host platform. Look to ACPI v6.5 section 18.6.4 and kernel + EINJ documentation for more information. + + If unsure say 'n' + config ACPI_APEI_ERST_DEBUG tristate "APEI Error Record Serialization Table (ERST) Debug Support" depends on ACPI_APEI diff --git a/drivers/acpi/apei/Makefile b/drivers/acpi/apei/Makefile index 4dfac2128737c..2c474e6477e12 100644 --- a/drivers/acpi/apei/Makefile +++ b/drivers/acpi/apei/Makefile @@ -2,6 +2,8 @@ obj-$(CONFIG_ACPI_APEI) += apei.o obj-$(CONFIG_ACPI_APEI_GHES) += ghes.o obj-$(CONFIG_ACPI_APEI_EINJ) += einj.o +einj-y := einj-core.o +einj-$(CONFIG_ACPI_APEI_EINJ_CXL) += einj-cxl.o obj-$(CONFIG_ACPI_APEI_ERST_DEBUG) += erst-dbg.o apei-y := apei-base.o hest.o erst.o bert.o diff --git a/drivers/acpi/apei/apei-internal.h b/drivers/acpi/apei/apei-internal.h index 67c2c3b959e15..cd2766c69d78d 100644 --- a/drivers/acpi/apei/apei-internal.h +++ b/drivers/acpi/apei/apei-internal.h @@ -130,4 +130,22 @@ static inline u32 cper_estatus_len(struct acpi_hest_generic_status *estatus) } int apei_osc_setup(void); + +int einj_get_available_error_type(u32 *type); +int einj_error_inject(u32 type, u32 flags, u64 param1, u64 param2, u64 param3, + u64 param4); +int einj_cxl_rch_error_inject(u32 type, u32 flags, u64 param1, u64 param2, + u64 param3, u64 param4); +bool einj_is_cxl_error_type(u64 type); +int einj_validate_error_type(u64 type); + +#ifndef ACPI_EINJ_CXL_CACHE_CORRECTABLE +#define ACPI_EINJ_CXL_CACHE_CORRECTABLE BIT(12) +#define ACPI_EINJ_CXL_CACHE_UNCORRECTABLE BIT(13) +#define ACPI_EINJ_CXL_CACHE_FATAL BIT(14) +#define ACPI_EINJ_CXL_MEM_CORRECTABLE BIT(15) +#define ACPI_EINJ_CXL_MEM_UNCORRECTABLE BIT(16) +#define ACPI_EINJ_CXL_MEM_FATAL BIT(17) +#endif + #endif diff --git a/drivers/acpi/apei/einj.c b/drivers/acpi/apei/einj-core.c similarity index 93% rename from drivers/acpi/apei/einj.c rename to drivers/acpi/apei/einj-core.c index 937c69844dac0..66e7f529e92fc 100644 --- a/drivers/acpi/apei/einj.c +++ b/drivers/acpi/apei/einj-core.c @@ -37,6 +37,12 @@ #define MEM_ERROR_MASK (ACPI_EINJ_MEMORY_CORRECTABLE | \ ACPI_EINJ_MEMORY_UNCORRECTABLE | \ ACPI_EINJ_MEMORY_FATAL) +#define CXL_ERROR_MASK (ACPI_EINJ_CXL_CACHE_CORRECTABLE | \ + ACPI_EINJ_CXL_CACHE_UNCORRECTABLE | \ + ACPI_EINJ_CXL_CACHE_FATAL | \ + ACPI_EINJ_CXL_MEM_CORRECTABLE | \ + ACPI_EINJ_CXL_MEM_UNCORRECTABLE | \ + ACPI_EINJ_CXL_MEM_FATAL) /* * ACPI version 5 provides a SET_ERROR_TYPE_WITH_ADDRESS action. @@ -141,7 +147,7 @@ static DEFINE_MUTEX(einj_mutex); /* * Exported APIs use this flag to exit early if einj_probe() failed. */ -static bool einj_initialized __ro_after_init; +bool einj_initialized __ro_after_init; static void *einj_param; @@ -166,7 +172,7 @@ static int __einj_get_available_error_type(u32 *type) } /* Get error injection capabilities of the platform */ -static int einj_get_available_error_type(u32 *type) +int einj_get_available_error_type(u32 *type) { int rc; @@ -536,8 +542,8 @@ static int __einj_error_inject(u32 type, u32 flags, u64 param1, u64 param2, } /* Inject the specified hardware error */ -static int einj_error_inject(u32 type, u32 flags, u64 param1, u64 param2, - u64 param3, u64 param4) +int einj_error_inject(u32 type, u32 flags, u64 param1, u64 param2, u64 param3, + u64 param4) { int rc; u64 base_addr, size; @@ -560,8 +566,17 @@ static int einj_error_inject(u32 type, u32 flags, u64 param1, u64 param2, if (type & ACPI5_VENDOR_BIT) { if (vendor_flags != SETWA_FLAGS_MEM) goto inject; - } else if (!(type & MEM_ERROR_MASK) && !(flags & SETWA_FLAGS_MEM)) + } else if (!(type & MEM_ERROR_MASK) && !(flags & SETWA_FLAGS_MEM)) { goto inject; + } + + /* + * Injections targeting a CXL 1.0/1.1 port have to be injected + * via the einj_cxl_rch_error_inject() path as that does the proper + * validation of the given RCRB base (MMIO) address. + */ + if (einj_is_cxl_error_type(type) && (flags & SETWA_FLAGS_MEM)) + return -EINVAL; /* * Disallow crazy address masks that give BIOS leeway to pick @@ -593,6 +608,21 @@ static int einj_error_inject(u32 type, u32 flags, u64 param1, u64 param2, return rc; } +int einj_cxl_rch_error_inject(u32 type, u32 flags, u64 param1, u64 param2, + u64 param3, u64 param4) +{ + int rc; + + if (!(einj_is_cxl_error_type(type) && (flags & SETWA_FLAGS_MEM))) + return -EINVAL; + + mutex_lock(&einj_mutex); + rc = __einj_error_inject(type, flags, param1, param2, param3, param4); + mutex_unlock(&einj_mutex); + + return rc; +} + static u32 error_type; static u32 error_flags; static u64 error_param1; @@ -613,12 +643,6 @@ static struct { u32 mask; const char *str; } const einj_error_type_string[] = { { BIT(9), "Platform Correctable" }, { BIT(10), "Platform Uncorrectable non-fatal" }, { BIT(11), "Platform Uncorrectable fatal"}, - { BIT(12), "CXL.cache Protocol Correctable" }, - { BIT(13), "CXL.cache Protocol Uncorrectable non-fatal" }, - { BIT(14), "CXL.cache Protocol Uncorrectable fatal" }, - { BIT(15), "CXL.mem Protocol Correctable" }, - { BIT(16), "CXL.mem Protocol Uncorrectable non-fatal" }, - { BIT(17), "CXL.mem Protocol Uncorrectable fatal" }, { BIT(31), "Vendor Defined Error Types" }, }; @@ -647,22 +671,26 @@ static int error_type_get(void *data, u64 *val) return 0; } -static int error_type_set(void *data, u64 val) +bool einj_is_cxl_error_type(u64 type) +{ + return (type & CXL_ERROR_MASK) && (!(type & ACPI5_VENDOR_BIT)); +} + +int einj_validate_error_type(u64 type) { + u32 tval, vendor, available_error_type = 0; int rc; - u32 available_error_type = 0; - u32 tval, vendor; /* Only low 32 bits for error type are valid */ - if (val & GENMASK_ULL(63, 32)) + if (type & GENMASK_ULL(63, 32)) return -EINVAL; /* * Vendor defined types have 0x80000000 bit set, and * are not enumerated by ACPI_EINJ_GET_ERROR_TYPE */ - vendor = val & ACPI5_VENDOR_BIT; - tval = val & 0x7fffffff; + vendor = type & ACPI5_VENDOR_BIT; + tval = type & GENMASK(30, 0); /* Only one error type can be specified */ if (tval & (tval - 1)) @@ -671,9 +699,21 @@ static int error_type_set(void *data, u64 val) rc = einj_get_available_error_type(&available_error_type); if (rc) return rc; - if (!(val & available_error_type)) + if (!(type & available_error_type)) return -EINVAL; } + + return 0; +} + +static int error_type_set(void *data, u64 val) +{ + int rc; + + rc = einj_validate_error_type(val); + if (rc) + return rc; + error_type = val; return 0; diff --git a/drivers/acpi/apei/einj-cxl.c b/drivers/acpi/apei/einj-cxl.c new file mode 100644 index 0000000000000..8b8be0c90709f --- /dev/null +++ b/drivers/acpi/apei/einj-cxl.c @@ -0,0 +1,113 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * CXL Error INJection support. Used by CXL core to inject + * protocol errors into CXL ports. + * + * Copyright (C) 2023 Advanced Micro Devices, Inc. + * + * Author: Ben Cheatham + */ +#include +#include +#include + +#include "apei-internal.h" + +/* Defined in einj-core.c */ +extern bool einj_initialized; + +static struct { u32 mask; const char *str; } const einj_cxl_error_type_string[] = { + { ACPI_EINJ_CXL_CACHE_CORRECTABLE, "CXL.cache Protocol Correctable" }, + { ACPI_EINJ_CXL_CACHE_UNCORRECTABLE, "CXL.cache Protocol Uncorrectable non-fatal" }, + { ACPI_EINJ_CXL_CACHE_FATAL, "CXL.cache Protocol Uncorrectable fatal" }, + { ACPI_EINJ_CXL_MEM_CORRECTABLE, "CXL.mem Protocol Correctable" }, + { ACPI_EINJ_CXL_MEM_UNCORRECTABLE, "CXL.mem Protocol Uncorrectable non-fatal" }, + { ACPI_EINJ_CXL_MEM_FATAL, "CXL.mem Protocol Uncorrectable fatal" }, +}; + +int einj_cxl_available_error_type_show(struct seq_file *m, void *v) +{ + int cxl_err, rc; + u32 available_error_type = 0; + + rc = einj_get_available_error_type(&available_error_type); + if (rc) + return rc; + + for (int pos = 0; pos < ARRAY_SIZE(einj_cxl_error_type_string); pos++) { + cxl_err = ACPI_EINJ_CXL_CACHE_CORRECTABLE << pos; + + if (available_error_type & cxl_err) + seq_printf(m, "0x%08x\t%s\n", + einj_cxl_error_type_string[pos].mask, + einj_cxl_error_type_string[pos].str); + } + + return 0; +} +EXPORT_SYMBOL_NS_GPL(einj_cxl_available_error_type_show, CXL); + +static int cxl_dport_get_sbdf(struct pci_dev *dport_dev, u64 *sbdf) +{ + struct pci_bus *pbus; + struct pci_host_bridge *bridge; + u64 seg = 0, bus; + + pbus = dport_dev->bus; + bridge = pci_find_host_bridge(pbus); + + if (!bridge) + return -ENODEV; + + if (bridge->domain_nr != PCI_DOMAIN_NR_NOT_SET) + seg = bridge->domain_nr; + + bus = pbus->number; + *sbdf = (seg << 24) | (bus << 16) | dport_dev->devfn; + + return 0; +} + +int einj_cxl_inject_rch_error(u64 rcrb, u64 type) +{ + int rc; + + /* Only CXL error types can be specified */ + if (!einj_is_cxl_error_type(type)) + return -EINVAL; + + rc = einj_validate_error_type(type); + if (rc) + return rc; + + return einj_cxl_rch_error_inject(type, 0x2, rcrb, GENMASK_ULL(63, 0), + 0, 0); +} +EXPORT_SYMBOL_NS_GPL(einj_cxl_inject_rch_error, CXL); + +int einj_cxl_inject_error(struct pci_dev *dport, u64 type) +{ + u64 param4 = 0; + int rc; + + /* Only CXL error types can be specified */ + if (!einj_is_cxl_error_type(type)) + return -EINVAL; + + rc = einj_validate_error_type(type); + if (rc) + return rc; + + rc = cxl_dport_get_sbdf(dport, ¶m4); + if (rc) + return rc; + + return einj_error_inject(type, 0x4, 0, 0, 0, param4); +} +EXPORT_SYMBOL_NS_GPL(einj_cxl_inject_error, CXL); + +bool einj_cxl_is_initialized(void) +{ + return einj_initialized; +} +EXPORT_SYMBOL_NS_GPL(einj_cxl_is_initialized, CXL); diff --git a/include/linux/einj-cxl.h b/include/linux/einj-cxl.h new file mode 100644 index 0000000000000..624ff6ff41f96 --- /dev/null +++ b/include/linux/einj-cxl.h @@ -0,0 +1,44 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * CXL protocol Error INJection support. + * + * Copyright (c) 2023 Advanced Micro Devices, Inc. + * All Rights Reserved. + * + * Author: Ben Cheatham + */ +#ifndef EINJ_CXL_H +#define EINJ_CXL_H + +#include +#include + +struct pci_dev; +struct seq_file; + +#if IS_ENABLED(CONFIG_ACPI_APEI_EINJ_CXL) +int einj_cxl_available_error_type_show(struct seq_file *m, void *v); +int einj_cxl_inject_error(struct pci_dev *dport_dev, u64 type); +int einj_cxl_inject_rch_error(u64 rcrb, u64 type); +bool einj_cxl_is_initialized(void); +#else /* !IS_ENABLED(CONFIG_ACPI_APEI_EINJ_CXL) */ +static inline int einj_cxl_available_error_type_show(struct seq_file *m, + void *v) +{ + return -ENXIO; +} + +static inline int einj_cxl_inject_error(struct pci_dev *dport_dev, u64 type) +{ + return -ENXIO; +} + +static inline int einj_cxl_inject_rch_error(u64 rcrb, u64 type) +{ + return -ENXIO; +} + +static inline bool einj_cxl_is_initialized(void) { return false; } +#endif /* CONFIG_ACPI_APEI_EINJ_CXL */ + +#endif /* EINJ_CXL_H */ From 2c9a72cc0f60f68f11ec34afcfd61a6356a8f774 Mon Sep 17 00:00:00 2001 From: Ben Cheatham Date: Mon, 11 Mar 2024 09:25:08 -0500 Subject: [PATCH 091/138] EINJ, Documentation: Update EINJ kernel doc commit a0563f58300360ef2a00b8fcfea91711594d70be upstream. Update EINJ kernel document to include how to inject CXL protocol error types, build the kernel to include CXL error types, and give an example injection. Reviewed-by: Jonathan Cameron Signed-off-by: Ben Cheatham Link: https://lore.kernel.org/r/20240311142508.31717-5-Benjamin.Cheatham@amd.com Signed-off-by: Dan Williams Signed-off-by: Yi Liu Signed-off-by: Bo Li --- .../firmware-guide/acpi/apei/einj.rst | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/Documentation/firmware-guide/acpi/apei/einj.rst b/Documentation/firmware-guide/acpi/apei/einj.rst index d6b61d22f5258..c52b9da08fa9a 100644 --- a/Documentation/firmware-guide/acpi/apei/einj.rst +++ b/Documentation/firmware-guide/acpi/apei/einj.rst @@ -32,6 +32,10 @@ configuration:: CONFIG_ACPI_APEI CONFIG_ACPI_APEI_EINJ +...and to (optionally) enable CXL protocol error injection set:: + + CONFIG_ACPI_APEI_EINJ_CXL + The EINJ user interface is in /apei/einj. The following files belong to it: @@ -118,6 +122,24 @@ The following files belong to it: this actually works depends on what operations the BIOS actually includes in the trigger phase. +CXL error types are supported from ACPI 6.5 onwards (given a CXL port +is present). The EINJ user interface for CXL error types is at +/cxl. The following files belong to it: + +- einj_types: + + Provides the same functionality as available_error_types above, but + for CXL error types + +- $dport_dev/einj_inject: + + Injects a CXL error type into the CXL port represented by $dport_dev, + where $dport_dev is the name of the CXL port (usually a PCIe device name). + Error injections targeting a CXL 2.0+ port can use the legacy interface + under /apei/einj, while CXL 1.1/1.0 port injections + must use this file. + + BIOS versions based on the ACPI 4.0 specification have limited options in controlling where the errors are injected. Your BIOS may support an extension (enabled with the param_extension=1 module parameter, or boot @@ -181,6 +203,18 @@ You should see something like this in dmesg:: [22715.834759] EDAC sbridge MC3: PROCESSOR 0:306e7 TIME 1422553404 SOCKET 0 APIC 0 [22716.616173] EDAC MC3: 1 CE memory read error on CPU_SrcID#0_Channel#0_DIMM#0 (channel:0 slot:0 page:0x12345 offset:0x0 grain:32 syndrome:0x0 - area:DRAM err_code:0001:0090 socket:0 channel_mask:1 rank:0) +A CXL error injection example with $dport_dev=0000:e0:01.1:: + + # cd /sys/kernel/debug/cxl/ + # ls + 0000:e0:01.1 0000:0c:00.0 + # cat einj_types # See which errors can be injected + 0x00008000 CXL.mem Protocol Correctable + 0x00010000 CXL.mem Protocol Uncorrectable non-fatal + 0x00020000 CXL.mem Protocol Uncorrectable fatal + # cd 0000:e0:01.1 # Navigate to dport to inject into + # echo 0x8000 > einj_inject # Inject error + Special notes for injection into SGX enclaves: There may be a separate BIOS setup option to enable SGX injection. From 8db32af2c5888cf53b6ff776eee37c7608ad9228 Mon Sep 17 00:00:00 2001 From: Ben Cheatham Date: Mon, 11 Mar 2024 09:25:07 -0500 Subject: [PATCH 092/138] cxl/core: Add CXL EINJ debugfs files commit 8039804cfa7314ad50085a779923aa5469889f88 upstream. Export CXL helper functions in einj-cxl.c for getting/injecting available CXL protocol error types to sysfs under kernel/debug/cxl. The kernel/debug/cxl/einj_types file will print the available CXL protocol errors in the same format as the available_error_types file provided by the einj module. The kernel/debug/cxl/$dport_dev/einj_inject file is functionally the same as the error_type and error_inject files provided by the EINJ module, i.e.: writing an error type into $dport_dev/einj_inject will inject said error type into the CXL dport represented by $dport_dev. Reviewed-by: Jonathan Cameron Signed-off-by: Ben Cheatham Link: https://lore.kernel.org/r/20240311142508.31717-4-Benjamin.Cheatham@amd.com Signed-off-by: Dan Williams Signed-off-by: Yi Liu Signed-off-by: Bo Li --- Documentation/ABI/testing/debugfs-cxl | 30 ++++++++++++++++++++ drivers/cxl/core/port.c | 41 +++++++++++++++++++++++++++ 2 files changed, 71 insertions(+) diff --git a/Documentation/ABI/testing/debugfs-cxl b/Documentation/ABI/testing/debugfs-cxl index fe61d372e3fac..4c0f62f881ca4 100644 --- a/Documentation/ABI/testing/debugfs-cxl +++ b/Documentation/ABI/testing/debugfs-cxl @@ -33,3 +33,33 @@ Description: device cannot clear poison from the address, -ENXIO is returned. The clear_poison attribute is only visible for devices supporting the capability. + +What: /sys/kernel/debug/cxl/einj_types +Date: January, 2024 +KernelVersion: v6.9 +Contact: linux-cxl@vger.kernel.org +Description: + (RO) Prints the CXL protocol error types made available by + the platform in the format "0x ". + The possible error types are (as of ACPI v6.5): + 0x1000 CXL.cache Protocol Correctable + 0x2000 CXL.cache Protocol Uncorrectable non-fatal + 0x4000 CXL.cache Protocol Uncorrectable fatal + 0x8000 CXL.mem Protocol Correctable + 0x10000 CXL.mem Protocol Uncorrectable non-fatal + 0x20000 CXL.mem Protocol Uncorrectable fatal + + The can be written to einj_inject to inject + into a chosen dport. + +What: /sys/kernel/debug/cxl/$dport_dev/einj_inject +Date: January, 2024 +KernelVersion: v6.9 +Contact: linux-cxl@vger.kernel.org +Description: + (WO) Writing an integer to this file injects the corresponding + CXL protocol error into $dport_dev ($dport_dev will be a device + name from /sys/bus/pci/devices). The integer to type mapping for + injection can be found by reading from einj_types. If the dport + was enumerated in RCH mode, a CXL 1.1 error is injected, otherwise + a CXL 2.0 error is injected. diff --git a/drivers/cxl/core/port.c b/drivers/cxl/core/port.c index de61b526954df..0894a093df101 100644 --- a/drivers/cxl/core/port.c +++ b/drivers/cxl/core/port.c @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -793,6 +794,40 @@ static int cxl_dport_setup_regs(struct device *host, struct cxl_dport *dport, return rc; } +DEFINE_SHOW_ATTRIBUTE(einj_cxl_available_error_type); + +static int cxl_einj_inject(void *data, u64 type) +{ + struct cxl_dport *dport = data; + + if (dport->rch) + return einj_cxl_inject_rch_error(dport->rcrb.base, type); + + return einj_cxl_inject_error(to_pci_dev(dport->dport_dev), type); +} +DEFINE_DEBUGFS_ATTRIBUTE(cxl_einj_inject_fops, NULL, cxl_einj_inject, + "0x%llx\n"); + +static void cxl_debugfs_create_dport_dir(struct cxl_dport *dport) +{ + struct dentry *dir; + + if (!einj_cxl_is_initialized()) + return; + + /* + * dport_dev needs to be a PCIe port for CXL 2.0+ ports because + * EINJ expects a dport SBDF to be specified for 2.0 error injection. + */ + if (!dport->rch && !dev_is_pci(dport->dport_dev)) + return; + + dir = cxl_debugfs_create_dir(dev_name(dport->dport_dev)); + + debugfs_create_file("einj_inject", 0200, dir, dport, + &cxl_einj_inject_fops); +} + static struct cxl_port *__devm_cxl_add_port(struct device *host, struct device *uport_dev, resource_size_t component_reg_phys, @@ -1150,6 +1185,8 @@ __devm_cxl_add_dport(struct cxl_port *port, struct device *dport_dev, if (dev_is_pci(dport_dev)) dport->link_latency = cxl_pci_get_latency(to_pci_dev(dport_dev)); + cxl_debugfs_create_dport_dir(dport); + return dport; } @@ -2243,6 +2280,10 @@ static __init int cxl_core_init(void) cxl_debugfs = debugfs_create_dir("cxl", NULL); + if (einj_cxl_is_initialized()) + debugfs_create_file("einj_types", 0400, cxl_debugfs, NULL, + &einj_cxl_available_error_type_fops); + cxl_mbox_init(); rc = cxl_memdev_init(); From ed9f4b4f8c7d9347a32553d16dd56a7a128edc2e Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Fri, 1 Mar 2024 14:09:48 -0700 Subject: [PATCH 093/138] cxl: Fix the incorrect assignment of SSLBIS entry pointer initial location commit 99b52aac2d40203d0f6468325018f68e2c494c24 upstream. The 'entry' pointer in cdat_sslbis_handler() is set to header + sizeof(common header). However, the math missed the addition of the SSLBIS main header. It should be header + sizeof(common header) + sizeof(*sslbis). Use a defined struct for all the SSLBIS parts in order to avoid pointer math errors. The bug causes incorrect parsing of the SSLBIS table and introduces incorrect performance values to the access_coordinates during the CXL access_coordinate calculation path if there are CXL switches present in the topology. The issue was found during testing of new code being added to add additional checks for invalid CDAT values during CXL access_coordinate calculation. The testing was done on qemu with a CXL topology including a CXL switch. Fixes: 80aa780dda20 ("cxl: Add callback to parse the SSLBIS subtable from CDAT") Signed-off-by: Dave Jiang Reviewed-by: Jonathan Cameron Reviewed-by: Fan Ni Link: https://lore.kernel.org/r/20240301210948.1298075-1-dave.jiang@intel.com Signed-off-by: Dan Williams Signed-off-by: Yi Liu Signed-off-by: Bo Li --- drivers/cxl/core/cdat.c | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/drivers/cxl/core/cdat.c b/drivers/cxl/core/cdat.c index 6965d8bea4e06..3d35c06a73a33 100644 --- a/drivers/cxl/core/cdat.c +++ b/drivers/cxl/core/cdat.c @@ -397,36 +397,38 @@ EXPORT_SYMBOL_NS_GPL(cxl_endpoint_parse_cdat, CXL); static int cdat_sslbis_handler(union acpi_subtable_headers *header, void *arg, const unsigned long end) { + struct acpi_cdat_sslbis_table { + struct acpi_cdat_header header; + struct acpi_cdat_sslbis sslbis_header; + struct acpi_cdat_sslbe entries[]; + } *tbl = (struct acpi_cdat_sslbis_table *)header; + int size = sizeof(header->cdat) + sizeof(tbl->sslbis_header); struct acpi_cdat_sslbis *sslbis; - int size = sizeof(header->cdat) + sizeof(*sslbis); struct cxl_port *port = arg; struct device *dev = &port->dev; - struct acpi_cdat_sslbe *entry; int remain, entries, i; u16 len; len = le16_to_cpu((__force __le16)header->cdat.length); remain = len - size; - if (!remain || remain % sizeof(*entry) || + if (!remain || remain % sizeof(tbl->entries[0]) || (unsigned long)header + len > end) { dev_warn(dev, "Malformed SSLBIS table length: (%u)\n", len); return -EINVAL; } - /* Skip common header */ - sslbis = (struct acpi_cdat_sslbis *)((unsigned long)header + - sizeof(header->cdat)); - + sslbis = &tbl->sslbis_header; /* Unrecognized data type, we can skip */ if (sslbis->data_type > ACPI_HMAT_WRITE_BANDWIDTH) return 0; - entries = remain / sizeof(*entry); - entry = (struct acpi_cdat_sslbe *)((unsigned long)header + sizeof(*sslbis)); + entries = remain / sizeof(tbl->entries[0]); + if (struct_size(tbl, entries, entries) != len) + return -EINVAL; for (i = 0; i < entries; i++) { - u16 x = le16_to_cpu((__force __le16)entry->portx_id); - u16 y = le16_to_cpu((__force __le16)entry->porty_id); + u16 x = le16_to_cpu((__force __le16)tbl->entries[i].portx_id); + u16 y = le16_to_cpu((__force __le16)tbl->entries[i].porty_id); __le64 le_base; __le16 le_val; struct cxl_dport *dport; @@ -456,8 +458,8 @@ static int cdat_sslbis_handler(union acpi_subtable_headers *header, void *arg, break; } - le_base = (__force __le64)sslbis->entry_base_unit; - le_val = (__force __le16)entry->latency_or_bandwidth; + le_base = (__force __le64)tbl->sslbis_header.entry_base_unit; + le_val = (__force __le16)tbl->entries[i].latency_or_bandwidth; if (check_mul_overflow(le64_to_cpu(le_base), le16_to_cpu(le_val), &val)) @@ -470,8 +472,6 @@ static int cdat_sslbis_handler(union acpi_subtable_headers *header, void *arg, sslbis->data_type, val); } - - entry++; } return 0; From 47a135336d5287c83f0d4771a5fa4c48c47c97b6 Mon Sep 17 00:00:00 2001 From: Robert Richter Date: Fri, 16 Feb 2024 16:58:42 +0100 Subject: [PATCH 094/138] cxl/pci: Rename DOE mailbox handle to doe_mb commit ec8ffff3a9f86435996f71299ce729126a7ac8a2 upstream. Trivial variable rename for the DOE mailbox handle from cdat_doe to doe_mb. The variable name cdat_doe is too ambiguous, use doe_mb that is commonly used for the mailbox. Signed-off-by: Robert Richter Reviewed-by: Dave Jiang Reviewed-by: Jonathan Cameron Link: https://lore.kernel.org/r/20240216155844.406996-2-rrichter@amd.com Signed-off-by: Dan Williams Signed-off-by: Yi Liu Signed-off-by: Bo Li --- drivers/cxl/core/pci.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/drivers/cxl/core/pci.c b/drivers/cxl/core/pci.c index 47e980e611c50..89005e2511c74 100644 --- a/drivers/cxl/core/pci.c +++ b/drivers/cxl/core/pci.c @@ -512,14 +512,14 @@ EXPORT_SYMBOL_NS_GPL(cxl_hdm_decode_init, CXL); FIELD_PREP(CXL_DOE_TABLE_ACCESS_ENTRY_HANDLE, (entry_handle))) static int cxl_cdat_get_length(struct device *dev, - struct pci_doe_mb *cdat_doe, + struct pci_doe_mb *doe_mb, size_t *length) { __le32 request = CDAT_DOE_REQ(0); __le32 response[2]; int rc; - rc = pci_doe(cdat_doe, PCI_DVSEC_VENDOR_ID_CXL, + rc = pci_doe(doe_mb, PCI_DVSEC_VENDOR_ID_CXL, CXL_DOE_PROTOCOL_TABLE_ACCESS, &request, sizeof(request), &response, sizeof(response)); @@ -537,7 +537,7 @@ static int cxl_cdat_get_length(struct device *dev, } static int cxl_cdat_read_table(struct device *dev, - struct pci_doe_mb *cdat_doe, + struct pci_doe_mb *doe_mb, void *cdat_table, size_t *cdat_length) { size_t length = *cdat_length + sizeof(__le32); @@ -551,7 +551,7 @@ static int cxl_cdat_read_table(struct device *dev, size_t entry_dw; int rc; - rc = pci_doe(cdat_doe, PCI_DVSEC_VENDOR_ID_CXL, + rc = pci_doe(doe_mb, PCI_DVSEC_VENDOR_ID_CXL, CXL_DOE_PROTOCOL_TABLE_ACCESS, &request, sizeof(request), data, length); @@ -611,7 +611,7 @@ void read_cdat_data(struct cxl_port *port) { struct device *uport = port->uport_dev; struct device *dev = &port->dev; - struct pci_doe_mb *cdat_doe; + struct pci_doe_mb *doe_mb; struct pci_dev *pdev = NULL; struct cxl_memdev *cxlmd; size_t cdat_length; @@ -632,16 +632,16 @@ void read_cdat_data(struct cxl_port *port) if (!pdev) return; - cdat_doe = pci_find_doe_mailbox(pdev, PCI_DVSEC_VENDOR_ID_CXL, - CXL_DOE_PROTOCOL_TABLE_ACCESS); - if (!cdat_doe) { + doe_mb = pci_find_doe_mailbox(pdev, PCI_DVSEC_VENDOR_ID_CXL, + CXL_DOE_PROTOCOL_TABLE_ACCESS); + if (!doe_mb) { dev_dbg(dev, "No CDAT mailbox\n"); return; } port->cdat_available = true; - if (cxl_cdat_get_length(dev, cdat_doe, &cdat_length)) { + if (cxl_cdat_get_length(dev, doe_mb, &cdat_length)) { dev_dbg(dev, "No CDAT length\n"); return; } @@ -650,7 +650,7 @@ void read_cdat_data(struct cxl_port *port) if (!cdat_buf) return; - rc = cxl_cdat_read_table(dev, cdat_doe, cdat_buf, &cdat_length); + rc = cxl_cdat_read_table(dev, doe_mb, cdat_buf, &cdat_length); if (rc) goto err; From 61e22fc621eed312e4a64e63703f55f9c8627084 Mon Sep 17 00:00:00 2001 From: Robert Richter Date: Fri, 16 Feb 2024 16:58:43 +0100 Subject: [PATCH 095/138] cxl/pci: Get rid of pointer arithmetic reading CDAT table commit e0c818e00443ce4a704519d85a21e3c14179e3a6 upstream. Reading the CDAT table using DOE requires a Table Access Response Header in addition to the CDAT entry. In current implementation this has caused offsets with sizeof(__le32) to the actual buffers. This led to hardly readable code and even bugs. E.g., see fix of devm_kfree() in read_cdat_data(): commit c65efe3685f5 ("cxl/cdat: Free correct buffer on checksum error") Rework code to avoid calculations with sizeof(__le32). Introduce struct cdat_doe_rsp for this which contains the Table Access Response Header and a variable payload size for various data structures afterwards to access the CDAT table and its CDAT Data Structures without recalculating buffer offsets. Cc: Lukas Wunner Cc: Fan Ni Reviewed-by: Dave Jiang Signed-off-by: Robert Richter Reviewed-by: Jonathan Cameron Link: https://lore.kernel.org/r/20240216155844.406996-3-rrichter@amd.com Signed-off-by: Dan Williams Signed-off-by: Yi Liu Signed-off-by: Bo Li --- drivers/cxl/core/pci.c | 77 ++++++++++++++++++++++-------------------- drivers/cxl/cxlpci.h | 24 +++++++++++++ 2 files changed, 65 insertions(+), 36 deletions(-) diff --git a/drivers/cxl/core/pci.c b/drivers/cxl/core/pci.c index 89005e2511c74..53294080b87bb 100644 --- a/drivers/cxl/core/pci.c +++ b/drivers/cxl/core/pci.c @@ -538,55 +538,57 @@ static int cxl_cdat_get_length(struct device *dev, static int cxl_cdat_read_table(struct device *dev, struct pci_doe_mb *doe_mb, - void *cdat_table, size_t *cdat_length) + struct cdat_doe_rsp *rsp, size_t *length) { - size_t length = *cdat_length + sizeof(__le32); - __le32 *data = cdat_table; - int entry_handle = 0; + size_t received, remaining = *length; + unsigned int entry_handle = 0; + union cdat_data *data; __le32 saved_dw = 0; do { __le32 request = CDAT_DOE_REQ(entry_handle); - struct cdat_entry_header *entry; - size_t entry_dw; int rc; rc = pci_doe(doe_mb, PCI_DVSEC_VENDOR_ID_CXL, CXL_DOE_PROTOCOL_TABLE_ACCESS, &request, sizeof(request), - data, length); + rsp, sizeof(*rsp) + remaining); if (rc < 0) { dev_err(dev, "DOE failed: %d", rc); return rc; } - /* 1 DW Table Access Response Header + CDAT entry */ - entry = (struct cdat_entry_header *)(data + 1); - if ((entry_handle == 0 && - rc != sizeof(__le32) + sizeof(struct cdat_header)) || - (entry_handle > 0 && - (rc < sizeof(__le32) + sizeof(*entry) || - rc != sizeof(__le32) + le16_to_cpu(entry->length)))) + if (rc < sizeof(*rsp)) return -EIO; + data = (union cdat_data *)rsp->data; + received = rc - sizeof(*rsp); + + if (entry_handle == 0) { + if (received != sizeof(data->header)) + return -EIO; + } else { + if (received < sizeof(data->entry) || + received != le16_to_cpu(data->entry.length)) + return -EIO; + } + /* Get the CXL table access header entry handle */ entry_handle = FIELD_GET(CXL_DOE_TABLE_ACCESS_ENTRY_HANDLE, - le32_to_cpu(data[0])); - entry_dw = rc / sizeof(__le32); - /* Skip Header */ - entry_dw -= 1; + le32_to_cpu(rsp->doe_header)); + /* * Table Access Response Header overwrote the last DW of * previous entry, so restore that DW */ - *data = saved_dw; - length -= entry_dw * sizeof(__le32); - data += entry_dw; - saved_dw = *data; + rsp->doe_header = saved_dw; + remaining -= received; + rsp = (void *)rsp + received; + saved_dw = rsp->doe_header; } while (entry_handle != CXL_DOE_TABLE_ACCESS_LAST_ENTRY); /* Length in CDAT header may exceed concatenation of CDAT entries */ - *cdat_length -= length - sizeof(__le32); + *length -= remaining; return 0; } @@ -614,8 +616,8 @@ void read_cdat_data(struct cxl_port *port) struct pci_doe_mb *doe_mb; struct pci_dev *pdev = NULL; struct cxl_memdev *cxlmd; - size_t cdat_length; - void *cdat_table, *cdat_buf; + struct cdat_doe_rsp *buf; + size_t length; int rc; if (is_cxl_memdev(uport)) { @@ -641,30 +643,33 @@ void read_cdat_data(struct cxl_port *port) port->cdat_available = true; - if (cxl_cdat_get_length(dev, doe_mb, &cdat_length)) { + if (cxl_cdat_get_length(dev, doe_mb, &length)) { dev_dbg(dev, "No CDAT length\n"); return; } - cdat_buf = devm_kzalloc(dev, cdat_length + sizeof(__le32), GFP_KERNEL); - if (!cdat_buf) - return; + /* + * The begin of the CDAT buffer needs space for additional 4 + * bytes for the DOE header. Table data starts afterwards. + */ + buf = devm_kzalloc(dev, sizeof(*buf) + length, GFP_KERNEL); + if (!buf) + goto err; - rc = cxl_cdat_read_table(dev, doe_mb, cdat_buf, &cdat_length); + rc = cxl_cdat_read_table(dev, doe_mb, buf, &length); if (rc) goto err; - cdat_table = cdat_buf + sizeof(__le32); - if (cdat_checksum(cdat_table, cdat_length)) + if (cdat_checksum(buf->data, length)) goto err; - port->cdat.table = cdat_table; - port->cdat.length = cdat_length; - return; + port->cdat.table = buf->data; + port->cdat.length = length; + return; err: /* Don't leave table data allocated on error */ - devm_kfree(dev, cdat_buf); + devm_kfree(dev, buf); dev_err(dev, "Failed to read/validate CDAT.\n"); } EXPORT_SYMBOL_NS_GPL(read_cdat_data, CXL); diff --git a/drivers/cxl/cxlpci.h b/drivers/cxl/cxlpci.h index 711b05d9a370e..93992a1c8eecf 100644 --- a/drivers/cxl/cxlpci.h +++ b/drivers/cxl/cxlpci.h @@ -71,6 +71,15 @@ enum cxl_regloc_type { CXL_REGLOC_RBI_TYPES }; +/* + * Table Access DOE, CDAT Read Entry Response + * + * Spec refs: + * + * CXL 3.1 8.1.11, Table 8-14: Read Entry Response + * CDAT Specification 1.03: 2 CDAT Data Structures + */ + struct cdat_header { __le32 length; u8 revision; @@ -85,6 +94,21 @@ struct cdat_entry_header { __le16 length; } __packed; +/* + * The DOE CDAT read response contains a CDAT read entry (either the + * CDAT header or a structure). + */ +union cdat_data { + struct cdat_header header; + struct cdat_entry_header entry; +} __packed; + +/* There is an additional CDAT response header of 4 bytes. */ +struct cdat_doe_rsp { + __le32 doe_header; + u8 data[]; +} __packed; + /* * CXL v3.0 6.2.3 Table 6-4 * The table indicates that if PCIe Flit Mode is set, then CXL is in 256B flits From 9cc1a4565caa47e3c7011a5ab21d90429fd2319b Mon Sep 17 00:00:00 2001 From: Robert Richter Date: Sat, 17 Feb 2024 22:39:46 +0100 Subject: [PATCH 096/138] lib/firmware_table: Provide buffer length argument to cdat_table_parse() commit c6c3187d66bc4e87086036266def4170742d7214 upstream. There exist card implementations with a CDAT table using a fixed size buffer, but with entries filled in that do not fill the whole table length size. Then, the last entry in the CDAT table may not mark the end of the CDAT table buffer specified by the length field in the CDAT header. It can be shorter with trailing unused (zero'ed) data. The actual table length is determined while reading all CDAT entries of the table with DOE. If the table is greater than expected (containing zero'ed trailing data), the CDAT parser fails with: [ 48.691717] Malformed DSMAS table length: (24:0) [ 48.702084] [CDAT:0x00] Invalid zero length [ 48.711460] cxl_port endpoint1: Failed to parse CDAT: -22 In addition, a check of the table buffer length is missing to prevent an out-of-bound access then parsing the CDAT table. Hardening code against device returning borked table. Fix that by providing an optional buffer length argument to acpi_parse_entries_array() that can be used by cdat_table_parse() to propagate the buffer size down to its users to check the buffer length. This also prevents a possible out-of-bound access mentioned. Add a check to warn about a malformed CDAT table length. Cc: Rafael J. Wysocki Cc: Len Brown Reviewed-by: Dave Jiang Signed-off-by: Robert Richter Reviewed-by: Jonathan Cameron Link: https://lore.kernel.org/r/ZdEnopFO0Tl3t2O1@rric.localdomain Signed-off-by: Dan Williams Signed-off-by: Yi Liu Signed-off-by: Bo Li --- drivers/acpi/tables.c | 2 +- drivers/cxl/core/cdat.c | 6 +++--- drivers/cxl/core/pci.c | 8 +++++++- include/linux/fw_table.h | 4 +++- lib/fw_table.c | 15 ++++++++++----- 5 files changed, 24 insertions(+), 11 deletions(-) diff --git a/drivers/acpi/tables.c b/drivers/acpi/tables.c index c59a3617bca7f..9e1b01c350702 100644 --- a/drivers/acpi/tables.c +++ b/drivers/acpi/tables.c @@ -267,7 +267,7 @@ int __init_or_acpilib acpi_table_parse_entries_array( count = acpi_parse_entries_array(id, table_size, (union fw_table_header *)table_header, - proc, proc_num, max_entries); + 0, proc, proc_num, max_entries); acpi_put_table(table_header); return count; diff --git a/drivers/cxl/core/cdat.c b/drivers/cxl/core/cdat.c index 3d35c06a73a33..b4a5c41b071dd 100644 --- a/drivers/cxl/core/cdat.c +++ b/drivers/cxl/core/cdat.c @@ -149,13 +149,13 @@ static int cxl_cdat_endpoint_process(struct cxl_port *port, int rc; rc = cdat_table_parse(ACPI_CDAT_TYPE_DSMAS, cdat_dsmas_handler, - dsmas_xa, port->cdat.table); + dsmas_xa, port->cdat.table, port->cdat.length); rc = cdat_table_parse_output(rc); if (rc) return rc; rc = cdat_table_parse(ACPI_CDAT_TYPE_DSLBIS, cdat_dslbis_handler, - dsmas_xa, port->cdat.table); + dsmas_xa, port->cdat.table, port->cdat.length); return cdat_table_parse_output(rc); } @@ -485,7 +485,7 @@ void cxl_switch_parse_cdat(struct cxl_port *port) return; rc = cdat_table_parse(ACPI_CDAT_TYPE_SSLBIS, cdat_sslbis_handler, - port, port->cdat.table); + port, port->cdat.table, port->cdat.length); rc = cdat_table_parse_output(rc); if (rc) dev_dbg(&port->dev, "Failed to parse SSLBIS: %d\n", rc); diff --git a/drivers/cxl/core/pci.c b/drivers/cxl/core/pci.c index 53294080b87bb..005a49e942b69 100644 --- a/drivers/cxl/core/pci.c +++ b/drivers/cxl/core/pci.c @@ -617,7 +617,7 @@ void read_cdat_data(struct cxl_port *port) struct pci_dev *pdev = NULL; struct cxl_memdev *cxlmd; struct cdat_doe_rsp *buf; - size_t length; + size_t table_length, length; int rc; if (is_cxl_memdev(uport)) { @@ -656,10 +656,16 @@ void read_cdat_data(struct cxl_port *port) if (!buf) goto err; + table_length = length; + rc = cxl_cdat_read_table(dev, doe_mb, buf, &length); if (rc) goto err; + if (table_length != length) + dev_warn(dev, "Malformed CDAT table length (%zu:%zu), discarding trailing data\n", + table_length, length); + if (cdat_checksum(buf->data, length)) goto err; diff --git a/include/linux/fw_table.h b/include/linux/fw_table.h index 95421860397a2..3ff4c277296fb 100644 --- a/include/linux/fw_table.h +++ b/include/linux/fw_table.h @@ -40,12 +40,14 @@ union acpi_subtable_headers { int acpi_parse_entries_array(char *id, unsigned long table_size, union fw_table_header *table_header, + unsigned long max_length, struct acpi_subtable_proc *proc, int proc_num, unsigned int max_entries); int cdat_table_parse(enum acpi_cdat_type type, acpi_tbl_entry_handler_arg handler_arg, void *arg, - struct acpi_table_cdat *table_header); + struct acpi_table_cdat *table_header, + unsigned long length); /* CXL is the only non-ACPI consumer of the FIRMWARE_TABLE library */ #if IS_ENABLED(CONFIG_ACPI) && !IS_ENABLED(CONFIG_CXL_BUS) diff --git a/lib/fw_table.c b/lib/fw_table.c index 1e5e0b2f70120..ddb67853b7ac1 100644 --- a/lib/fw_table.c +++ b/lib/fw_table.c @@ -132,6 +132,7 @@ static __init_or_fwtbl_lib int call_handler(struct acpi_subtable_proc *proc, * * @id: table id (for debugging purposes) * @table_size: size of the root table + * @max_length: maximum size of the table (ignore if 0) * @table_header: where does the table start? * @proc: array of acpi_subtable_proc struct containing entry id * and associated handler with it @@ -153,10 +154,11 @@ static __init_or_fwtbl_lib int call_handler(struct acpi_subtable_proc *proc, int __init_or_fwtbl_lib acpi_parse_entries_array(char *id, unsigned long table_size, union fw_table_header *table_header, + unsigned long max_length, struct acpi_subtable_proc *proc, int proc_num, unsigned int max_entries) { - unsigned long table_end, subtable_len, entry_len; + unsigned long table_len, table_end, subtable_len, entry_len; struct acpi_subtable_entry entry; enum acpi_subtable_type type; int count = 0; @@ -164,8 +166,10 @@ acpi_parse_entries_array(char *id, unsigned long table_size, int i; type = acpi_get_subtable_type(id); - table_end = (unsigned long)table_header + - acpi_table_get_length(type, table_header); + table_len = acpi_table_get_length(type, table_header); + if (max_length && max_length < table_len) + table_len = max_length; + table_end = (unsigned long)table_header + table_len; /* Parse all entries looking for a match. */ @@ -220,7 +224,8 @@ int __init_or_fwtbl_lib cdat_table_parse(enum acpi_cdat_type type, acpi_tbl_entry_handler_arg handler_arg, void *arg, - struct acpi_table_cdat *table_header) + struct acpi_table_cdat *table_header, + unsigned long length) { struct acpi_subtable_proc proc = { .id = type, @@ -234,6 +239,6 @@ cdat_table_parse(enum acpi_cdat_type type, return acpi_parse_entries_array(ACPI_SIG_CDAT, sizeof(struct acpi_table_cdat), (union fw_table_header *)table_header, - &proc, 1, 0); + length, &proc, 1, 0); } EXPORT_SYMBOL_FWTBL_LIB(cdat_table_parse); From 10d217dad77e5f383407370ee71f8f5d4a28a1ad Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Thu, 14 Mar 2024 17:34:26 -0700 Subject: [PATCH 097/138] Documentation/ABI/testing/debugfs-cxl: Fix "Unexpected indentation" commit edc1243437e75ea019ba264d38b2cd793ae83ed0 upstream. Stephen reported that an htmldocs build hit: Documentation/ABI/testing/debugfs-cxl:38: ERROR: Unexpected indentation. It turns out that line was fine but the tool was unhappy about some line breaks in the table of values to error types. It turns out that: make V=1 SPHINXDIRS="admin-guide" htmldocs ...can not be used to get more info about what is behind a documentation build error. It was only pure luck that reflowing the text resulted in an error message that seemed a imply a problem later on with line breaks around the table. Fixes: 8039804cfa73 ("cxl/core: Add CXL EINJ debugfs files") Cc: Jonathan Corbet Reported-by: Stephen Rothwell Closes: http://lore.kernel.org/r/20240314141313.7ba04aff@canb.auug.org.au Cc: Ben Cheatham Signed-off-by: Dan Williams Signed-off-by: Yi Liu Signed-off-by: Bo Li --- Documentation/ABI/testing/debugfs-cxl | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Documentation/ABI/testing/debugfs-cxl b/Documentation/ABI/testing/debugfs-cxl index 4c0f62f881ca4..c61f9b8139730 100644 --- a/Documentation/ABI/testing/debugfs-cxl +++ b/Documentation/ABI/testing/debugfs-cxl @@ -40,8 +40,12 @@ KernelVersion: v6.9 Contact: linux-cxl@vger.kernel.org Description: (RO) Prints the CXL protocol error types made available by - the platform in the format "0x ". + the platform in the format: + + 0x + The possible error types are (as of ACPI v6.5): + 0x1000 CXL.cache Protocol Correctable 0x2000 CXL.cache Protocol Uncorrectable non-fatal 0x4000 CXL.cache Protocol Uncorrectable fatal From 4727ef496795d54d2fd237b94b95c7fd943c3e59 Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Wed, 20 Mar 2024 19:02:50 +0100 Subject: [PATCH 098/138] ACPI: APEI: EINJ: mark remove callback as non-__exit commit a24118a8a6870fe06dbbd59f915b9ca3662b2ddb upstream. The remove callback of a device is called whenever it is unbound, which may happen during runtime e.g. through sysfs, so this is not allowed to be dropped from the binary: WARNING: modpost: vmlinux: section mismatch in reference: einj_driver+0x8 (section: .data) -> einj_remove (section: .exit.text) ERROR: modpost: Section mismatches detected. Remove that annotation. Fixes: 12fb28ea6b1c ("EINJ: Add CXL error type support") Signed-off-by: Arnd Bergmann Reviewed-by: Dan Williams Signed-off-by: Rafael J. Wysocki Signed-off-by: Yi Liu Signed-off-by: Bo Li --- drivers/acpi/apei/einj-core.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/acpi/apei/einj-core.c b/drivers/acpi/apei/einj-core.c index 66e7f529e92fc..01faca3a238a3 100644 --- a/drivers/acpi/apei/einj-core.c +++ b/drivers/acpi/apei/einj-core.c @@ -851,7 +851,7 @@ static int __init einj_probe(struct platform_device *pdev) return rc; } -static void __exit einj_remove(struct platform_device *pdev) +static void einj_remove(struct platform_device *pdev) { struct apei_exec_context ctx; From 8cbed48b27edb66cd3d1fb741de385a1136374fe Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Wed, 3 Apr 2024 08:47:12 -0700 Subject: [PATCH 099/138] cxl: Remove checking of iter in cxl_endpoint_get_perf_coordinates() commit 648dae58a830ecceea3b1bebf68432435980f137 upstream. The while() loop in cxl_endpoint_get_perf_coordinates() checks to see if 'iter' is valid as part of the condition breaking out of the loop. is_cxl_root() will stop the loop before the next iteration could go NULL. Remove the iter check. The presence of the iter or removing the iter does not impact the behavior of the code. This is a code clean up and not a bug fix. Reviewed-by: Jonathan Cameron Reviewed-by: Davidlohr Bueso Reviewed-by: Dan Williams Link: https://lore.kernel.org/r/20240403154844.3403859-2-dave.jiang@intel.com Signed-off-by: Dave Jiang Signed-off-by: Yi Liu Signed-off-by: Bo Li --- drivers/cxl/core/port.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/cxl/core/port.c b/drivers/cxl/core/port.c index 0894a093df101..d0d0d0a3e02bb 100644 --- a/drivers/cxl/core/port.c +++ b/drivers/cxl/core/port.c @@ -2204,7 +2204,7 @@ int cxl_endpoint_get_perf_coordinates(struct cxl_port *port, * port each iteration. If the parent is cxl root then there is * nothing to gather. */ - while (iter && !is_cxl_root(to_cxl_port(iter->dev.parent))) { + while (!is_cxl_root(to_cxl_port(iter->dev.parent))) { cxl_coordinates_combine(&c, &c, &dport->sw_coord); c.write_latency += dport->link_latency; c.read_latency += dport->link_latency; From 69da9a37be977e8b0fe8d54740c80c57655a658e Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Wed, 3 Apr 2024 08:47:13 -0700 Subject: [PATCH 100/138] cxl: Fix retrieving of access_coordinates in PCIe path commit 592780b8391fe31f129ef4823c1513528f4dcb76 upstream. Current loop in cxl_endpoint_get_perf_coordinates() incorrectly assumes the Root Port (RP) dport is the one with generic port access_coordinate. However those coordinates are one level up in the Host Bridge (HB). Current code causes the computation code to pick up 0s as the coordinates and cause minimal bandwidth to result in 0. Add check to skip RP when combining coordinates. Fixes: 14a6960b3e92 ("cxl: Add helper function that calculate performance data for downstream ports") Reported-by: Jonathan Cameron Reviewed-by: Jonathan Cameron Reviewed-by: Dan Williams Link: https://lore.kernel.org/r/20240403154844.3403859-3-dave.jiang@intel.com Signed-off-by: Dave Jiang Signed-off-by: Yi Liu Signed-off-by: Bo Li --- drivers/cxl/core/port.c | 35 ++++++++++++++++++++++------------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/drivers/cxl/core/port.c b/drivers/cxl/core/port.c index d0d0d0a3e02bb..9c1a8f76f0df0 100644 --- a/drivers/cxl/core/port.c +++ b/drivers/cxl/core/port.c @@ -2172,6 +2172,11 @@ int cxl_hb_get_perf_coordinates(struct cxl_port *port, return 0; } +static bool parent_port_is_cxl_root(struct cxl_port *port) +{ + return is_cxl_root(to_cxl_port(port->dev.parent)); +} + /** * cxl_endpoint_get_perf_coordinates - Retrieve performance numbers stored in dports * of CXL path @@ -2191,27 +2196,31 @@ int cxl_endpoint_get_perf_coordinates(struct cxl_port *port, struct cxl_dport *dport; struct pci_dev *pdev; unsigned int bw; + bool is_cxl_root; if (!is_cxl_endpoint(port)) return -EINVAL; - dport = iter->parent_dport; - /* - * Exit the loop when the parent port of the current port is cxl root. - * The iterative loop starts at the endpoint and gathers the - * latency of the CXL link from the current iter to the next downstream - * port each iteration. If the parent is cxl root then there is - * nothing to gather. + * Exit the loop when the parent port of the current iter port is cxl + * root. The iterative loop starts at the endpoint and gathers the + * latency of the CXL link from the current device/port to the connected + * downstream port each iteration. */ - while (!is_cxl_root(to_cxl_port(iter->dev.parent))) { - cxl_coordinates_combine(&c, &c, &dport->sw_coord); + do { + dport = iter->parent_dport; + iter = to_cxl_port(iter->dev.parent); + is_cxl_root = parent_port_is_cxl_root(iter); + + /* + * There's no valid access_coordinate for a root port since RPs do not + * have CDAT and therefore needs to be skipped. + */ + if (!is_cxl_root) + cxl_coordinates_combine(&c, &c, &dport->sw_coord); c.write_latency += dport->link_latency; c.read_latency += dport->link_latency; - - iter = to_cxl_port(iter->dev.parent); - dport = iter->parent_dport; - } + } while (!is_cxl_root); /* Get the calculated PCI paths bandwidth */ pdev = to_pci_dev(port->uport_dev->parent); From 40871a8119a7f4a801d44db5414f04884ade668c Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Wed, 3 Apr 2024 08:47:14 -0700 Subject: [PATCH 101/138] cxl: Fix incorrect region perf data calculation commit 51293c565cf4b8d57c154efadb57b17866c74bcb upstream. Current math in cxl_region_perf_data_calculate divides the latency by 1000 every time the function gets called. This causes the region latency to be divided by 1000 per memory device and the math is incorrect. This is user visible as the latency access_coordinate exposed via sysfs will show incorrect latency data. Normalize values from CDAT to nanoseconds. Adjust sub-nanoseconds latency to at least 1. Remove adjustment of perf numbers from the generic target since hmat handling code has already normalized those numbers. Now all computation and stored numbers should be in nanoseconds. cxl_hb_get_perf_coordinates() is removed and HB coords are calculated in the port access_coordinate calculation path since it no longer need to be treated special. Fixes: 3d9f4a197230 ("cxl/region: Calculate performance data for a region") Reviewed-by: Jonathan Cameron Reviewed-by: Dan Williams Link: https://lore.kernel.org/r/20240403154844.3403859-4-dave.jiang@intel.com Signed-off-by: Dave Jiang Signed-off-by: Yi Liu Signed-off-by: Bo Li --- drivers/cxl/acpi.c | 13 +----- drivers/cxl/core/cdat.c | 94 ++++++++++++++++++----------------------- drivers/cxl/core/port.c | 36 ++-------------- drivers/cxl/cxl.h | 2 - 4 files changed, 45 insertions(+), 100 deletions(-) diff --git a/drivers/cxl/acpi.c b/drivers/cxl/acpi.c index 122974a3fe029..56ed24a5019ee 100644 --- a/drivers/cxl/acpi.c +++ b/drivers/cxl/acpi.c @@ -525,22 +525,11 @@ static int get_genport_coordinates(struct device *dev, struct cxl_dport *dport) { struct acpi_device *hb = to_cxl_host_bridge(NULL, dev); u32 uid; - int rc; if (kstrtou32(acpi_device_uid(hb), 0, &uid)) return -EINVAL; - rc = acpi_get_genport_coordinates(uid, dport->hb_coord); - if (rc < 0) - return rc; - - /* Adjust back to picoseconds from nanoseconds */ - for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) { - dport->hb_coord[i].read_latency *= 1000; - dport->hb_coord[i].write_latency *= 1000; - } - - return 0; + return acpi_get_genport_coordinates(uid, dport->hb_coord); } static int add_host_bridge_dport(struct device *match, void *arg) diff --git a/drivers/cxl/core/cdat.c b/drivers/cxl/core/cdat.c index b4a5c41b071dd..4729ee341a9d4 100644 --- a/drivers/cxl/core/cdat.c +++ b/drivers/cxl/core/cdat.c @@ -19,6 +19,36 @@ struct dsmas_entry { int qos_class; }; +static u32 cdat_normalize(u16 entry, u64 base, u8 type) +{ + u32 value; + + /* + * Check for invalid and overflow values + */ + if (entry == 0xffff || !entry) + return 0; + else if (base > (UINT_MAX / (entry))) + return 0; + + /* + * CDAT fields follow the format of HMAT fields. See table 5 Device + * Scoped Latency and Bandwidth Information Structure in Coherent Device + * Attribute Table (CDAT) Specification v1.01. + */ + value = entry * base; + switch (type) { + case ACPI_HMAT_ACCESS_LATENCY: + case ACPI_HMAT_READ_LATENCY: + case ACPI_HMAT_WRITE_LATENCY: + value = DIV_ROUND_UP(value, 1000); + break; + default: + break; + } + return value; +} + static int cdat_dsmas_handler(union acpi_subtable_headers *header, void *arg, const unsigned long end) { @@ -96,7 +126,6 @@ static int cdat_dslbis_handler(union acpi_subtable_headers *header, void *arg, __le16 le_val; u64 val; u16 len; - int rc; len = le16_to_cpu((__force __le16)hdr->length); if (len != size || (unsigned long)hdr + len > end) { @@ -123,10 +152,8 @@ static int cdat_dslbis_handler(union acpi_subtable_headers *header, void *arg, le_base = (__force __le64)dslbis->entry_base_unit; le_val = (__force __le16)dslbis->entry[0]; - rc = check_mul_overflow(le64_to_cpu(le_base), - le16_to_cpu(le_val), &val); - if (rc) - pr_warn("DSLBIS value overflowed.\n"); + val = cdat_normalize(le16_to_cpu(le_val), le64_to_cpu(le_base), + dslbis->data_type); cxl_access_coordinate_set(&dent->coord, dslbis->data_type, val); @@ -163,7 +190,6 @@ static int cxl_port_perf_data_calculate(struct cxl_port *port, struct xarray *dsmas_xa) { struct access_coordinate ep_c; - struct access_coordinate coord[ACCESS_COORDINATE_MAX]; struct dsmas_entry *dent; int valid_entries = 0; unsigned long index; @@ -175,12 +201,6 @@ static int cxl_port_perf_data_calculate(struct cxl_port *port, return rc; } - rc = cxl_hb_get_perf_coordinates(port, coord); - if (rc) { - dev_dbg(&port->dev, "Failed to retrieve hb perf coordinates.\n"); - return rc; - } - struct cxl_root *cxl_root __free(put_cxl_root) = find_cxl_root(port); if (!cxl_root) @@ -193,18 +213,9 @@ static int cxl_port_perf_data_calculate(struct cxl_port *port, int qos_class; cxl_coordinates_combine(&dent->coord, &dent->coord, &ep_c); - /* - * Keeping the host bridge coordinates separate from the dsmas - * coordinates in order to allow calculation of access class - * 0 and 1 for region later. - */ - cxl_coordinates_combine(&coord[ACCESS_COORDINATE_CPU], - &coord[ACCESS_COORDINATE_CPU], - &dent->coord); dent->entries = 1; - rc = cxl_root->ops->qos_class(cxl_root, - &coord[ACCESS_COORDINATE_CPU], - 1, &qos_class); + rc = cxl_root->ops->qos_class(cxl_root, &dent->coord, 1, + &qos_class); if (rc != 1) continue; @@ -460,10 +471,8 @@ static int cdat_sslbis_handler(union acpi_subtable_headers *header, void *arg, le_base = (__force __le64)tbl->sslbis_header.entry_base_unit; le_val = (__force __le16)tbl->entries[i].latency_or_bandwidth; - - if (check_mul_overflow(le64_to_cpu(le_base), - le16_to_cpu(le_val), &val)) - dev_warn(dev, "SSLBIS value overflowed!\n"); + val = cdat_normalize(le16_to_cpu(le_val), le64_to_cpu(le_base), + sslbis->data_type); xa_for_each(&port->dports, index, dport) { if (dsp_id == ACPI_CDAT_SSLBIS_ANY_PORT || @@ -520,17 +529,13 @@ void cxl_region_perf_data_calculate(struct cxl_region *cxlr, struct cxl_endpoint_decoder *cxled) { struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); - struct cxl_port *port = cxlmd->endpoint; struct cxl_dev_state *cxlds = cxlmd->cxlds; struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds); - struct access_coordinate hb_coord[ACCESS_COORDINATE_MAX]; - struct access_coordinate coord; struct range dpa = { .start = cxled->dpa_res->start, .end = cxled->dpa_res->end, }; struct cxl_dpa_perf *perf; - int rc; switch (cxlr->mode) { case CXL_DECODER_RAM: @@ -548,35 +553,16 @@ void cxl_region_perf_data_calculate(struct cxl_region *cxlr, if (!range_contains(&perf->dpa_range, &dpa)) return; - rc = cxl_hb_get_perf_coordinates(port, hb_coord); - if (rc) { - dev_dbg(&port->dev, "Failed to retrieve hb perf coordinates.\n"); - return; - } - for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) { - /* Pickup the host bridge coords */ - cxl_coordinates_combine(&coord, &hb_coord[i], &perf->coord); - /* Get total bandwidth and the worst latency for the cxl region */ cxlr->coord[i].read_latency = max_t(unsigned int, cxlr->coord[i].read_latency, - coord.read_latency); + perf->coord.read_latency); cxlr->coord[i].write_latency = max_t(unsigned int, cxlr->coord[i].write_latency, - coord.write_latency); - cxlr->coord[i].read_bandwidth += coord.read_bandwidth; - cxlr->coord[i].write_bandwidth += coord.write_bandwidth; - - /* - * Convert latency to nanosec from picosec to be consistent - * with the resulting latency coordinates computed by the - * HMAT_REPORTING code. - */ - cxlr->coord[i].read_latency = - DIV_ROUND_UP(cxlr->coord[i].read_latency, 1000); - cxlr->coord[i].write_latency = - DIV_ROUND_UP(cxlr->coord[i].write_latency, 1000); + perf->coord.write_latency); + cxlr->coord[i].read_bandwidth += perf->coord.read_bandwidth; + cxlr->coord[i].write_bandwidth += perf->coord.write_bandwidth; } } diff --git a/drivers/cxl/core/port.c b/drivers/cxl/core/port.c index 9c1a8f76f0df0..8c9af2e05f590 100644 --- a/drivers/cxl/core/port.c +++ b/drivers/cxl/core/port.c @@ -2140,38 +2140,6 @@ bool schedule_cxl_memdev_detach(struct cxl_memdev *cxlmd) } EXPORT_SYMBOL_NS_GPL(schedule_cxl_memdev_detach, CXL); -/** - * cxl_hb_get_perf_coordinates - Retrieve performance numbers between initiator - * and host bridge - * - * @port: endpoint cxl_port - * @coord: output access coordinates - * - * Return: errno on failure, 0 on success. - */ -int cxl_hb_get_perf_coordinates(struct cxl_port *port, - struct access_coordinate *coord) -{ - struct cxl_port *iter = port; - struct cxl_dport *dport; - - if (!is_cxl_endpoint(port)) - return -EINVAL; - - dport = iter->parent_dport; - while (iter && !is_cxl_root(to_cxl_port(iter->dev.parent))) { - iter = to_cxl_port(iter->dev.parent); - dport = iter->parent_dport; - } - - coord[ACCESS_COORDINATE_LOCAL] = - dport->hb_coord[ACCESS_COORDINATE_LOCAL]; - coord[ACCESS_COORDINATE_CPU] = - dport->hb_coord[ACCESS_COORDINATE_CPU]; - - return 0; -} - static bool parent_port_is_cxl_root(struct cxl_port *port) { return is_cxl_root(to_cxl_port(port->dev.parent)); @@ -2222,6 +2190,10 @@ int cxl_endpoint_get_perf_coordinates(struct cxl_port *port, c.read_latency += dport->link_latency; } while (!is_cxl_root); + dport = iter->parent_dport; + /* Retrieve HB coords */ + cxl_coordinates_combine(&c, &c, dport->hb_coord); + /* Get the calculated PCI paths bandwidth */ pdev = to_pci_dev(port->uport_dev->parent); bw = pcie_bandwidth_available(pdev, NULL, NULL, NULL); diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h index 31f5e2c5688f2..2ee7680c41dd9 100644 --- a/drivers/cxl/cxl.h +++ b/drivers/cxl/cxl.h @@ -887,8 +887,6 @@ void cxl_switch_parse_cdat(struct cxl_port *port); int cxl_endpoint_get_perf_coordinates(struct cxl_port *port, struct access_coordinate *coord); -int cxl_hb_get_perf_coordinates(struct cxl_port *port, - struct access_coordinate *coord); void cxl_region_perf_data_calculate(struct cxl_region *cxlr, struct cxl_endpoint_decoder *cxled); From 7503407ece0eb37e1a2783d150a737b3554b63ca Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Wed, 3 Apr 2024 08:47:15 -0700 Subject: [PATCH 102/138] cxl: Consolidate dport access_coordinate ->hb_coord and ->sw_coord into ->coord commit 001c5d19341a39cb683ab0a18ce4b662a09d96a0 upstream. The driver stores access_coordinate for host bridge in ->hb_coord and switch CDAT access_coordinate in ->sw_coord. Since neither of these access_coordinate clobber each other, the variable name can be consolidated into ->coord to simplify the code. Reviewed-by: Jonathan Cameron Reviewed-by: Davidlohr Bueso Reviewed-by: Dan Williams Link: https://lore.kernel.org/r/20240403154844.3403859-5-dave.jiang@intel.com Signed-off-by: Dave Jiang Signed-off-by: Yi Liu Signed-off-by: Bo Li --- drivers/cxl/acpi.c | 2 +- drivers/cxl/core/cdat.c | 74 +++++++++++++++++++++++------------- drivers/cxl/core/port.c | 48 +++++++++++++++++------ drivers/cxl/cxl.h | 6 +-- drivers/cxl/cxlmem.h | 2 +- tools/testing/cxl/test/cxl.c | 10 +++-- 6 files changed, 94 insertions(+), 48 deletions(-) diff --git a/drivers/cxl/acpi.c b/drivers/cxl/acpi.c index 56ed24a5019ee..3f9040b2b27fe 100644 --- a/drivers/cxl/acpi.c +++ b/drivers/cxl/acpi.c @@ -529,7 +529,7 @@ static int get_genport_coordinates(struct device *dev, struct cxl_dport *dport) if (kstrtou32(acpi_device_uid(hb), 0, &uid)) return -EINVAL; - return acpi_get_genport_coordinates(uid, dport->hb_coord); + return acpi_get_genport_coordinates(uid, dport->coord); } static int add_host_bridge_dport(struct device *match, void *arg) diff --git a/drivers/cxl/core/cdat.c b/drivers/cxl/core/cdat.c index 4729ee341a9d4..1d505a49a2753 100644 --- a/drivers/cxl/core/cdat.c +++ b/drivers/cxl/core/cdat.c @@ -13,7 +13,7 @@ struct dsmas_entry { struct range dpa_range; u8 handle; - struct access_coordinate coord; + struct access_coordinate coord[ACCESS_COORDINATE_MAX]; int entries; int qos_class; @@ -87,8 +87,8 @@ static int cdat_dsmas_handler(union acpi_subtable_headers *header, void *arg, return 0; } -static void cxl_access_coordinate_set(struct access_coordinate *coord, - int access, unsigned int val) +static void __cxl_access_coordinate_set(struct access_coordinate *coord, + int access, unsigned int val) { switch (access) { case ACPI_HMAT_ACCESS_LATENCY: @@ -114,6 +114,13 @@ static void cxl_access_coordinate_set(struct access_coordinate *coord, } } +static void cxl_access_coordinate_set(struct access_coordinate *coord, + int access, unsigned int val) +{ + for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) + __cxl_access_coordinate_set(&coord[i], access, val); +} + static int cdat_dslbis_handler(union acpi_subtable_headers *header, void *arg, const unsigned long end) { @@ -155,7 +162,7 @@ static int cdat_dslbis_handler(union acpi_subtable_headers *header, void *arg, val = cdat_normalize(le16_to_cpu(le_val), le64_to_cpu(le_base), dslbis->data_type); - cxl_access_coordinate_set(&dent->coord, dslbis->data_type, val); + cxl_access_coordinate_set(dent->coord, dslbis->data_type, val); return 0; } @@ -189,13 +196,13 @@ static int cxl_cdat_endpoint_process(struct cxl_port *port, static int cxl_port_perf_data_calculate(struct cxl_port *port, struct xarray *dsmas_xa) { - struct access_coordinate ep_c; + struct access_coordinate ep_c[ACCESS_COORDINATE_MAX]; struct dsmas_entry *dent; int valid_entries = 0; unsigned long index; int rc; - rc = cxl_endpoint_get_perf_coordinates(port, &ep_c); + rc = cxl_endpoint_get_perf_coordinates(port, ep_c); if (rc) { dev_dbg(&port->dev, "Failed to retrieve ep perf coordinates.\n"); return rc; @@ -212,10 +219,11 @@ static int cxl_port_perf_data_calculate(struct cxl_port *port, xa_for_each(dsmas_xa, index, dent) { int qos_class; - cxl_coordinates_combine(&dent->coord, &dent->coord, &ep_c); + cxl_coordinates_combine(dent->coord, dent->coord, ep_c); dent->entries = 1; - rc = cxl_root->ops->qos_class(cxl_root, &dent->coord, 1, - &qos_class); + rc = cxl_root->ops->qos_class(cxl_root, + &dent->coord[ACCESS_COORDINATE_CPU], + 1, &qos_class); if (rc != 1) continue; @@ -232,14 +240,17 @@ static int cxl_port_perf_data_calculate(struct cxl_port *port, static void update_perf_entry(struct device *dev, struct dsmas_entry *dent, struct cxl_dpa_perf *dpa_perf) { + for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) + dpa_perf->coord[i] = dent->coord[i]; dpa_perf->dpa_range = dent->dpa_range; - dpa_perf->coord = dent->coord; dpa_perf->qos_class = dent->qos_class; dev_dbg(dev, "DSMAS: dpa: %#llx qos: %d read_bw: %d write_bw %d read_lat: %d write_lat: %d\n", dent->dpa_range.start, dpa_perf->qos_class, - dent->coord.read_bandwidth, dent->coord.write_bandwidth, - dent->coord.read_latency, dent->coord.write_latency); + dent->coord[ACCESS_COORDINATE_CPU].read_bandwidth, + dent->coord[ACCESS_COORDINATE_CPU].write_bandwidth, + dent->coord[ACCESS_COORDINATE_CPU].read_latency, + dent->coord[ACCESS_COORDINATE_CPU].write_latency); } static void cxl_memdev_set_qos_class(struct cxl_dev_state *cxlds, @@ -476,10 +487,11 @@ static int cdat_sslbis_handler(union acpi_subtable_headers *header, void *arg, xa_for_each(&port->dports, index, dport) { if (dsp_id == ACPI_CDAT_SSLBIS_ANY_PORT || - dsp_id == dport->port_id) - cxl_access_coordinate_set(&dport->sw_coord, + dsp_id == dport->port_id) { + cxl_access_coordinate_set(dport->coord, sslbis->data_type, val); + } } } @@ -501,6 +513,21 @@ void cxl_switch_parse_cdat(struct cxl_port *port) } EXPORT_SYMBOL_NS_GPL(cxl_switch_parse_cdat, CXL); +static void __cxl_coordinates_combine(struct access_coordinate *out, + struct access_coordinate *c1, + struct access_coordinate *c2) +{ + if (c1->write_bandwidth && c2->write_bandwidth) + out->write_bandwidth = min(c1->write_bandwidth, + c2->write_bandwidth); + out->write_latency = c1->write_latency + c2->write_latency; + + if (c1->read_bandwidth && c2->read_bandwidth) + out->read_bandwidth = min(c1->read_bandwidth, + c2->read_bandwidth); + out->read_latency = c1->read_latency + c2->read_latency; +} + /** * cxl_coordinates_combine - Combine the two input coordinates * @@ -512,15 +539,8 @@ void cxl_coordinates_combine(struct access_coordinate *out, struct access_coordinate *c1, struct access_coordinate *c2) { - if (c1->write_bandwidth && c2->write_bandwidth) - out->write_bandwidth = min(c1->write_bandwidth, - c2->write_bandwidth); - out->write_latency = c1->write_latency + c2->write_latency; - - if (c1->read_bandwidth && c2->read_bandwidth) - out->read_bandwidth = min(c1->read_bandwidth, - c2->read_bandwidth); - out->read_latency = c1->read_latency + c2->read_latency; + for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) + __cxl_coordinates_combine(&out[i], &c1[i], &c2[i]); } MODULE_IMPORT_NS(CXL); @@ -557,12 +577,12 @@ void cxl_region_perf_data_calculate(struct cxl_region *cxlr, /* Get total bandwidth and the worst latency for the cxl region */ cxlr->coord[i].read_latency = max_t(unsigned int, cxlr->coord[i].read_latency, - perf->coord.read_latency); + perf->coord[i].read_latency); cxlr->coord[i].write_latency = max_t(unsigned int, cxlr->coord[i].write_latency, - perf->coord.write_latency); - cxlr->coord[i].read_bandwidth += perf->coord.read_bandwidth; - cxlr->coord[i].write_bandwidth += perf->coord.write_bandwidth; + perf->coord[i].write_latency); + cxlr->coord[i].read_bandwidth += perf->coord[i].read_bandwidth; + cxlr->coord[i].write_bandwidth += perf->coord[i].write_bandwidth; } } diff --git a/drivers/cxl/core/port.c b/drivers/cxl/core/port.c index 8c9af2e05f590..958d3c37d3255 100644 --- a/drivers/cxl/core/port.c +++ b/drivers/cxl/core/port.c @@ -2140,6 +2140,29 @@ bool schedule_cxl_memdev_detach(struct cxl_memdev *cxlmd) } EXPORT_SYMBOL_NS_GPL(schedule_cxl_memdev_detach, CXL); +static void add_latency(struct access_coordinate *c, long latency) +{ + for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) { + c[i].write_latency += latency; + c[i].read_latency += latency; + } +} + +static void set_min_bandwidth(struct access_coordinate *c, unsigned int bw) +{ + for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) { + c[i].write_bandwidth = min(c[i].write_bandwidth, bw); + c[i].read_bandwidth = min(c[i].read_bandwidth, bw); + } +} + +static void set_access_coordinates(struct access_coordinate *out, + struct access_coordinate *in) +{ + for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) + out[i] = in[i]; +} + static bool parent_port_is_cxl_root(struct cxl_port *port) { return is_cxl_root(to_cxl_port(port->dev.parent)); @@ -2156,9 +2179,15 @@ static bool parent_port_is_cxl_root(struct cxl_port *port) int cxl_endpoint_get_perf_coordinates(struct cxl_port *port, struct access_coordinate *coord) { - struct access_coordinate c = { - .read_bandwidth = UINT_MAX, - .write_bandwidth = UINT_MAX, + struct access_coordinate c[] = { + { + .read_bandwidth = UINT_MAX, + .write_bandwidth = UINT_MAX, + }, + { + .read_bandwidth = UINT_MAX, + .write_bandwidth = UINT_MAX, + }, }; struct cxl_port *iter = port; struct cxl_dport *dport; @@ -2185,14 +2214,13 @@ int cxl_endpoint_get_perf_coordinates(struct cxl_port *port, * have CDAT and therefore needs to be skipped. */ if (!is_cxl_root) - cxl_coordinates_combine(&c, &c, &dport->sw_coord); - c.write_latency += dport->link_latency; - c.read_latency += dport->link_latency; + cxl_coordinates_combine(c, c, dport->coord); + add_latency(c, dport->link_latency); } while (!is_cxl_root); dport = iter->parent_dport; /* Retrieve HB coords */ - cxl_coordinates_combine(&c, &c, dport->hb_coord); + cxl_coordinates_combine(c, c, dport->coord); /* Get the calculated PCI paths bandwidth */ pdev = to_pci_dev(port->uport_dev->parent); @@ -2201,10 +2229,8 @@ int cxl_endpoint_get_perf_coordinates(struct cxl_port *port, return -ENXIO; bw /= BITS_PER_BYTE; - c.write_bandwidth = min(c.write_bandwidth, bw); - c.read_bandwidth = min(c.read_bandwidth, bw); - - *coord = c; + set_min_bandwidth(c, bw); + set_access_coordinates(coord, c); return 0; } diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h index 2ee7680c41dd9..371945d95b9ab 100644 --- a/drivers/cxl/cxl.h +++ b/drivers/cxl/cxl.h @@ -665,8 +665,7 @@ struct cxl_rcrb_info { * @rch: Indicate whether this dport was enumerated in RCH or VH mode * @port: reference to cxl_port that contains this downstream port * @regs: Dport parsed register blocks - * @sw_coord: access coordinates (performance) for switch from CDAT - * @hb_coord: access coordinates (performance) from ACPI generic port (host bridge) + * @coord: access coordinates (bandwidth and latency performance attributes) * @link_latency: calculated PCIe downstream latency */ struct cxl_dport { @@ -677,8 +676,7 @@ struct cxl_dport { bool rch; struct cxl_port *port; struct cxl_regs regs; - struct access_coordinate sw_coord; - struct access_coordinate hb_coord[ACCESS_COORDINATE_MAX]; + struct access_coordinate coord[ACCESS_COORDINATE_MAX]; long link_latency; }; diff --git a/drivers/cxl/cxlmem.h b/drivers/cxl/cxlmem.h index 675fe674ff3e7..b901b6ad45de5 100644 --- a/drivers/cxl/cxlmem.h +++ b/drivers/cxl/cxlmem.h @@ -401,7 +401,7 @@ enum cxl_devtype { */ struct cxl_dpa_perf { struct range dpa_range; - struct access_coordinate coord; + struct access_coordinate coord[ACCESS_COORDINATE_MAX]; int qos_class; }; diff --git a/tools/testing/cxl/test/cxl.c b/tools/testing/cxl/test/cxl.c index 6da9df7c878ed..4f9ee3fc906cc 100644 --- a/tools/testing/cxl/test/cxl.c +++ b/tools/testing/cxl/test/cxl.c @@ -986,10 +986,12 @@ static void dpa_perf_setup(struct cxl_port *endpoint, struct range *range, { dpa_perf->qos_class = FAKE_QTG_ID; dpa_perf->dpa_range = *range; - dpa_perf->coord.read_latency = 500; - dpa_perf->coord.write_latency = 500; - dpa_perf->coord.read_bandwidth = 1000; - dpa_perf->coord.write_bandwidth = 1000; + for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) { + dpa_perf->coord[i].read_latency = 500; + dpa_perf->coord[i].write_latency = 500; + dpa_perf->coord[i].read_bandwidth = 1000; + dpa_perf->coord[i].write_bandwidth = 1000; + } } static void mock_cxl_endpoint_parse_cdat(struct cxl_port *port) From 3e5f046dad737f1575fe6cd43d2cfd5c9038806c Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Wed, 3 Apr 2024 08:47:16 -0700 Subject: [PATCH 103/138] cxl: Add checks to access_coordinate calculation to fail missing data commit 7bcf809b1e7889ab7e75fe1fcf8f1a98332f36d2 upstream. Jonathan noted that when the coordinates for host bridge and switches can be 0s if no actual data are retrieved and the calculation continues. The resulting number would be inaccurate. Add checks to ensure that the calculation would complete only if the numbers are valid. While not seen in the wild, issue may show up with a BIOS that reported CXL root ports via Generic Ports (via a PCI handle in the SRAT entry). Fixes: 14a6960b3e92 ("cxl: Add helper function that calculate performance data for downstream ports") Reported-by: Jonathan Cameron Reviewed-by: Jonathan Cameron Reviewed-by: Davidlohr Bueso Reviewed-by: Dan Williams Link: https://lore.kernel.org/r/20240403154844.3403859-6-dave.jiang@intel.com Signed-off-by: Dave Jiang Signed-off-by: Yi Liu Signed-off-by: Bo Li --- drivers/cxl/core/port.c | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/drivers/cxl/core/port.c b/drivers/cxl/core/port.c index 958d3c37d3255..49463db5b68be 100644 --- a/drivers/cxl/core/port.c +++ b/drivers/cxl/core/port.c @@ -2148,6 +2148,18 @@ static void add_latency(struct access_coordinate *c, long latency) } } +static bool coordinates_valid(struct access_coordinate *c) +{ + for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) { + if (c[i].read_bandwidth && c[i].write_bandwidth && + c[i].read_latency && c[i].write_latency) + continue; + return false; + } + + return true; +} + static void set_min_bandwidth(struct access_coordinate *c, unsigned int bw) { for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) { @@ -2213,13 +2225,18 @@ int cxl_endpoint_get_perf_coordinates(struct cxl_port *port, * There's no valid access_coordinate for a root port since RPs do not * have CDAT and therefore needs to be skipped. */ - if (!is_cxl_root) + if (!is_cxl_root) { + if (!coordinates_valid(dport->coord)) + return -EINVAL; cxl_coordinates_combine(c, c, dport->coord); + } add_latency(c, dport->link_latency); } while (!is_cxl_root); dport = iter->parent_dport; /* Retrieve HB coords */ + if (!coordinates_valid(dport->coord)) + return -EINVAL; cxl_coordinates_combine(c, c, dport->coord); /* Get the calculated PCI paths bandwidth */ From 77bb49a71f6bb71ee4213a35ffa475a5fcbbe14c Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Fri, 26 Apr 2024 15:47:56 -0700 Subject: [PATCH 104/138] cxl: Fix cxl_endpoint_get_perf_coordinate() support for RCH commit 5d211c7090590033581175d6405ae40917ca3a06 upstream. Robert reported the following when booting a CXL host with Restricted CXL Host (RCH) topology: [ 39.815379] cxl_acpi ACPI0017:00: not a cxl_port device [ 39.827123] WARNING: CPU: 46 PID: 1754 at drivers/cxl/core/port.c:592 to_cxl_port+0x56/0x70 [cxl_core] ... plus some related subsequent NULL pointer dereference: [ 40.718708] BUG: kernel NULL pointer dereference, address: 00000000000002d8 The iterator to walk the PCIe path did not account for RCH topology. However RCH does not support hotplug and the memory exported by the Restricted CXL Device (RCD) should be covered by HMAT and therefore no access_coordinate is needed. Add check to see if the endpoint device is RCD and skip calculation. Also add a call to cxl_endpoint_get_perf_coordinates() in cxl_test in order to exercise the topology iterator. The dev_is_pci() check added is to help with this test and should be harmless for normal operation. Reported-by: Robert Richter Closes: https://lore.kernel.org/all/Ziv8GfSMSbvlBB0h@rric.localdomain/ Fixes: 592780b8391f ("cxl: Fix retrieving of access_coordinates in PCIe path") Reviewed-by: Dan Williams Tested-by: Robert Richter Reviewed-by: Robert Richter Link: https://lore.kernel.org/r/20240426224913.1027420-1-dave.jiang@intel.com Signed-off-by: Dave Jiang Signed-off-by: Yi Liu Signed-off-by: Bo Li --- drivers/cxl/core/port.c | 15 ++++++++++++++- tools/testing/cxl/test/cxl.c | 7 +++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/drivers/cxl/core/port.c b/drivers/cxl/core/port.c index 49463db5b68be..eed45377aa03a 100644 --- a/drivers/cxl/core/port.c +++ b/drivers/cxl/core/port.c @@ -2191,6 +2191,7 @@ static bool parent_port_is_cxl_root(struct cxl_port *port) int cxl_endpoint_get_perf_coordinates(struct cxl_port *port, struct access_coordinate *coord) { + struct cxl_memdev *cxlmd = to_cxl_memdev(port->uport_dev); struct access_coordinate c[] = { { .read_bandwidth = UINT_MAX, @@ -2204,12 +2205,20 @@ int cxl_endpoint_get_perf_coordinates(struct cxl_port *port, struct cxl_port *iter = port; struct cxl_dport *dport; struct pci_dev *pdev; + struct device *dev; unsigned int bw; bool is_cxl_root; if (!is_cxl_endpoint(port)) return -EINVAL; + /* + * Skip calculation for RCD. Expectation is HMAT already covers RCD case + * since RCH does not support hotplug. + */ + if (cxlmd->cxlds->rcd) + return 0; + /* * Exit the loop when the parent port of the current iter port is cxl * root. The iterative loop starts at the endpoint and gathers the @@ -2239,8 +2248,12 @@ int cxl_endpoint_get_perf_coordinates(struct cxl_port *port, return -EINVAL; cxl_coordinates_combine(c, c, dport->coord); + dev = port->uport_dev->parent; + if (!dev_is_pci(dev)) + return -ENODEV; + /* Get the calculated PCI paths bandwidth */ - pdev = to_pci_dev(port->uport_dev->parent); + pdev = to_pci_dev(dev); bw = pcie_bandwidth_available(pdev, NULL, NULL, NULL); if (bw == 0) return -ENXIO; diff --git a/tools/testing/cxl/test/cxl.c b/tools/testing/cxl/test/cxl.c index 4f9ee3fc906cc..c5bbd89b31920 100644 --- a/tools/testing/cxl/test/cxl.c +++ b/tools/testing/cxl/test/cxl.c @@ -1001,6 +1001,7 @@ static void mock_cxl_endpoint_parse_cdat(struct cxl_port *port) struct cxl_memdev *cxlmd = to_cxl_memdev(port->uport_dev); struct cxl_dev_state *cxlds = cxlmd->cxlds; struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds); + struct access_coordinate ep_c[ACCESS_COORDINATE_MAX]; struct range pmem_range = { .start = cxlds->pmem_res.start, .end = cxlds->pmem_res.end, @@ -1020,6 +1021,12 @@ static void mock_cxl_endpoint_parse_cdat(struct cxl_port *port) dpa_perf_setup(port, &pmem_range, &mds->pmem_perf); cxl_memdev_update_perf(cxlmd); + + /* + * This function is here to only test the topology iterator. It serves + * no other purpose. + */ + cxl_endpoint_get_perf_coordinates(port, ep_c); } static struct cxl_mock_ops cxl_mock_ops = { From 41c359d2fdf062c93b8bbca85a4ae8117bfdc91f Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Tue, 21 May 2024 15:46:32 -0700 Subject: [PATCH 105/138] ACPI: APEI: EINJ: Fix einj_dev release leak commit 7ff6c798eca05e4a9dcb80163cb454d7787a4bc3 upstream. The platform driver conversion of EINJ mistakenly used platform_device_del() to unwind platform_device_register_full() at module exit. This leads to a small leak of one 'struct platform_device' instance per module load/unload cycle. Switch to platform_device_unregister() which performs both device_del() and final put_device(). Fixes: 5621fafaac00 ("EINJ: Migrate to a platform driver") Cc: 6.9+ # 6.9+ Signed-off-by: Dan Williams Reviewed-by: Ben Cheatham Signed-off-by: Rafael J. Wysocki Signed-off-by: Yi Liu Signed-off-by: Bo Li --- drivers/acpi/apei/einj-core.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/acpi/apei/einj-core.c b/drivers/acpi/apei/einj-core.c index 01faca3a238a3..bb9f8475ce594 100644 --- a/drivers/acpi/apei/einj-core.c +++ b/drivers/acpi/apei/einj-core.c @@ -903,7 +903,7 @@ static void __exit einj_exit(void) if (einj_initialized) platform_driver_unregister(&einj_driver); - platform_device_del(einj_dev); + platform_device_unregister(einj_dev); } module_init(einj_init); From 293015df7f41dbae57281c48d5bd8f7a41a031ff Mon Sep 17 00:00:00 2001 From: Huang Ying Date: Tue, 18 Jun 2024 16:46:37 +0800 Subject: [PATCH 106/138] cxl/region: Fix a race condition in memory hotplug notifier commit a3483ee7e6a7f2d12b5950246f4e0ef94f4a5df0 upstream. In the memory hotplug notifier function of the CXL region, cxl_region_perf_attrs_callback(), the node ID is obtained by checking the host address range of the region. However, the address range information is not available when the region is registered in devm_cxl_add_region(). Additionally, this information may be removed or added under the protection of cxl_region_rwsem during runtime. If the memory notifier is called for nodes other than that backed by the region, a race condition may occur, potentially leading to a NULL dereference or an invalid address range. The race condition is addressed by checking the availability of the address range information under the protection of cxl_region_rwsem. To enhance code readability and use guard(), the relevant code has been moved into a newly added function: cxl_region_nid(). Fixes: 067353a46d8c ("cxl/region: Add memory hotplug notifier for cxl region") Signed-off-by: Huang, Ying Cc: Dan Williams Cc: Alison Schofield Cc: Andrew Morton Cc: Jonathan Cameron Cc: Dave Jiang Cc: Bharata B Rao Cc: Alistair Popple Cc: Aneesh Kumar K.V Cc: Davidlohr Bueso Cc: Vishal Verma Cc: Ira Weiny Reviewed-by: Jonathan Cameron Link: https://patch.msgid.link/20240618084639.1419629-2-ying.huang@intel.com Signed-off-by: Dave Jiang Signed-off-by: Yi Liu Signed-off-by: Bo Li --- drivers/cxl/core/region.c | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/drivers/cxl/core/region.c b/drivers/cxl/core/region.c index 5dbcc3352eb01..91885df06386a 100644 --- a/drivers/cxl/core/region.c +++ b/drivers/cxl/core/region.c @@ -2385,14 +2385,25 @@ static bool cxl_region_update_coordinates(struct cxl_region *cxlr, int nid) return true; } +static int cxl_region_nid(struct cxl_region *cxlr) +{ + struct cxl_region_params *p = &cxlr->params; + struct cxl_endpoint_decoder *cxled; + struct cxl_decoder *cxld; + + guard(rwsem_read)(&cxl_region_rwsem); + cxled = p->targets[0]; + if (!cxled) + return NUMA_NO_NODE; + cxld = &cxled->cxld; + return phys_to_target_node(cxld->hpa_range.start); +} + static int cxl_region_perf_attrs_callback(struct notifier_block *nb, unsigned long action, void *arg) { struct cxl_region *cxlr = container_of(nb, struct cxl_region, memory_notifier); - struct cxl_region_params *p = &cxlr->params; - struct cxl_endpoint_decoder *cxled = p->targets[0]; - struct cxl_decoder *cxld = &cxled->cxld; struct memory_notify *mnb = arg; int nid = mnb->status_change_nid; int region_nid; @@ -2400,7 +2411,7 @@ static int cxl_region_perf_attrs_callback(struct notifier_block *nb, if (nid == NUMA_NO_NODE || action != MEM_ONLINE) return NOTIFY_DONE; - region_nid = phys_to_target_node(cxld->hpa_range.start); + region_nid = cxl_region_nid(cxlr); if (nid != region_nid) return NOTIFY_DONE; From d305888d0389b05b3361ccaa8519fcf1953c1833 Mon Sep 17 00:00:00 2001 From: Li Ming Date: Fri, 9 Aug 2024 08:27:49 +0000 Subject: [PATCH 107/138] cxl/pci: Get AER capability address from RCRB only for RCH dport commit 8c251c5ab1b7cd204231e4ee936bfe078a33f234 upstream. cxl_setup_parent_dport() needs to get RCH dport AER capability address from RCRB to disable AER interrupt. The function does not check if dport is RCH dport, it will get a wrong pci_host_bridge structure by dport_dev in VH case because dport_dev points to a pci device(RP or switch DSP) rather than a pci host bridge device. Fixes: f05fd10d138d ("cxl/pci: Add RCH downstream port AER register discovery") Signed-off-by: Li Ming Reviewed-by: Dan Williams Reviewed-by: Ira Weiny Tested-by: Ira Weiny Tested-by: Alison Schofield Link: https://patch.msgid.link/20240809082750.3015641-2-ming4.li@intel.com Signed-off-by: Dave Jiang Signed-off-by: Yi Liu Signed-off-by: Bo Li --- drivers/cxl/core/pci.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/drivers/cxl/core/pci.c b/drivers/cxl/core/pci.c index 005a49e942b69..b76ad047c2a3d 100644 --- a/drivers/cxl/core/pci.c +++ b/drivers/cxl/core/pci.c @@ -828,11 +828,13 @@ static void cxl_disable_rch_root_ints(struct cxl_dport *dport) void cxl_setup_parent_dport(struct device *host, struct cxl_dport *dport) { struct device *dport_dev = dport->dport_dev; - struct pci_host_bridge *host_bridge; - host_bridge = to_pci_host_bridge(dport_dev); - if (host_bridge->native_aer) - dport->rcrb.aer_cap = cxl_rcrb_to_aer(dport_dev, dport->rcrb.base); + if (dport->rch) { + struct pci_host_bridge *host_bridge = to_pci_host_bridge(dport_dev); + + if (host_bridge->native_aer) + dport->rcrb.aer_cap = cxl_rcrb_to_aer(dport_dev, dport->rcrb.base); + } dport->reg_map.host = host; cxl_dport_map_regs(dport); From 580b7a7e277f1827662ed4783e7ef6087bf4f65f Mon Sep 17 00:00:00 2001 From: Li Ming Date: Fri, 9 Aug 2024 08:27:50 +0000 Subject: [PATCH 108/138] cxl/test: Skip cxl_setup_parent_dport() for emulated dports commit 2c402bd2e85b44dc00ef85b5c0e217de684b5372 upstream. The cxl_test unit test environment on qemu always hits below call trace with KASAN enabled: BUG: KASAN: slab-out-of-bounds in cxl_setup_parent_dport+0x480/0x530 [cxl_core] Read of size 1 at addr ff110000676014f8 by task (udev-worker)/676[ 24.424403] CPU: 2 PID: 676 Comm: (udev-worker) Tainted: G O N 6.10.0-qemucxl #1 Hardware name: QEMU Standard PC (Q35 + ICH9, 2009), BIOS edk2-20240214-2.el9 02/14/2024 Call Trace: dump_stack_lvl+0xea/0x150 print_report+0xce/0x610 ? kasan_complete_mode_report_info+0x40/0x200 kasan_report+0xcc/0x110 __asan_report_load1_noabort+0x18/0x20 cxl_setup_parent_dport+0x480/0x530 [cxl_core] cxl_mem_probe+0x49b/0xaa0 [cxl_mem] cxl_test module models a CXL topology for testing, it creates some emulated dports with platform devices in the CXL topology, so the dport_dev of an emulated dport points to a platform device rather than a pci device or a pci host bridge in the case. Currently, cxl_setup_parent_dport() is used to set up RAS and AER capability on the dport connected to the CXL memory device, but cxl_test does not support RAS or AER functionality yet, so the fix is implementing a __wrap_cxl_setup_parent_dport() to filter out all emulated dports, guarantees only real dports can be handled by cxl_setup_parent_dport(). Fixes: f05fd10d138d ("cxl/pci: Add RCH downstream port AER register discovery") Reported-by: Pengfei Xu Closes: https://lore.kernel.org/linux-cxl/ZrHTBp2O+HtUe6kt@xpf.sh.intel.com/T/#t Signed-off-by: Li Ming Reviewed-by: Dan Williams Reviewed-by: Ira Weiny Reviewed-by: Alison Schofield Tested-by: Ira Weiny Tested-by: Alison Schofield Link: https://patch.msgid.link/20240809082750.3015641-3-ming4.li@intel.com Signed-off-by: Dave Jiang Signed-off-by: Yi Liu Signed-off-by: Bo Li --- tools/testing/cxl/Kbuild | 1 + tools/testing/cxl/test/mock.c | 12 ++++++++++++ 2 files changed, 13 insertions(+) diff --git a/tools/testing/cxl/Kbuild b/tools/testing/cxl/Kbuild index 030b388800f05..3d1ca9e38b1fa 100644 --- a/tools/testing/cxl/Kbuild +++ b/tools/testing/cxl/Kbuild @@ -14,6 +14,7 @@ ldflags-y += --wrap=cxl_dvsec_rr_decode ldflags-y += --wrap=devm_cxl_add_rch_dport ldflags-y += --wrap=cxl_rcd_component_reg_phys ldflags-y += --wrap=cxl_endpoint_parse_cdat +ldflags-y += --wrap=cxl_setup_parent_dport DRIVERS := ../../../drivers CXL_SRC := $(DRIVERS)/cxl diff --git a/tools/testing/cxl/test/mock.c b/tools/testing/cxl/test/mock.c index 6f737941dc0e1..d619672faa497 100644 --- a/tools/testing/cxl/test/mock.c +++ b/tools/testing/cxl/test/mock.c @@ -299,6 +299,18 @@ void __wrap_cxl_endpoint_parse_cdat(struct cxl_port *port) } EXPORT_SYMBOL_NS_GPL(__wrap_cxl_endpoint_parse_cdat, CXL); +void __wrap_cxl_setup_parent_dport(struct device *host, struct cxl_dport *dport) +{ + int index; + struct cxl_mock_ops *ops = get_cxl_mock_ops(&index); + + if (!ops || !ops->is_mock_port(dport->dport_dev)) + cxl_setup_parent_dport(host, dport); + + put_cxl_mock_ops(index); +} +EXPORT_SYMBOL_NS_GPL(__wrap_cxl_setup_parent_dport, CXL); + MODULE_LICENSE("GPL v2"); MODULE_IMPORT_NS(ACPI); MODULE_IMPORT_NS(CXL); From e24093e52bbf9cdcbb3bf154d9c2374a105eb354 Mon Sep 17 00:00:00 2001 From: Ben Cheatham Date: Fri, 27 Sep 2024 11:34:28 -0500 Subject: [PATCH 109/138] EINJ, CXL: Fix CXL device SBDF calculation commit ee1e3c46ed19c096be22472c728fa7f68b1352c4 upstream. The SBDF of the target CXL 2.0 compliant root port is required to inject a CXL protocol error as per ACPI 6.5. The SBDF given has to be in the following format: 31 24 23 16 15 11 10 8 7 0 +-------------------------------------------------+ | segment | bus | device | function | reserved | +-------------------------------------------------+ The SBDF calculated in cxl_dport_get_sbdf() doesn't account for the reserved bits currently, causing the wrong SBDF to be used. Fix said calculation to properly shift the SBDF. Without this fix, error injection into CXL 2.0 root ports through the CXL debugfs interface (/cxl) is broken. Injection through the legacy interface (/apei/einj/) will still work because the SBDF is manually provided by the user. Fixes: 12fb28ea6b1cf ("EINJ: Add CXL error type support") Signed-off-by: Ben Cheatham Reviewed-by: Dan Williams Tested-by: Srinivasulu Thanneeru Reviewed-by: Srinivasulu Thanneeru Link: https://patch.msgid.link/20240927163428.366557-1-Benjamin.Cheatham@amd.com Signed-off-by: Ira Weiny Signed-off-by: Yi Liu Signed-off-by: Bo Li --- drivers/acpi/apei/einj-cxl.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/acpi/apei/einj-cxl.c b/drivers/acpi/apei/einj-cxl.c index 8b8be0c90709f..d64e2713aae4b 100644 --- a/drivers/acpi/apei/einj-cxl.c +++ b/drivers/acpi/apei/einj-cxl.c @@ -63,7 +63,7 @@ static int cxl_dport_get_sbdf(struct pci_dev *dport_dev, u64 *sbdf) seg = bridge->domain_nr; bus = pbus->number; - *sbdf = (seg << 24) | (bus << 16) | dport_dev->devfn; + *sbdf = (seg << 24) | (bus << 16) | (dport_dev->devfn << 8); return 0; } From fc77bee4a68d6cd83c5c0be2fb3cd5f4b396253f Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Fri, 29 Aug 2025 15:29:05 -0700 Subject: [PATCH 110/138] drivers/base/node: Add a helper function node_update_perf_attrs() commit b57fc652ca24ada3b0c888327f9944ed21559286 upstream. Add helper function node_update_perf_attrs() to allow update of node access coordinates computed by an external agent such as CXL. The helper allows updating of coordinates after the attribute being created by HMAT. Acked-by: David Hildenbrand Reviewed-by: Jonathan Cameron Link: https://patch.msgid.link/20250829222907.1290912-3-dave.jiang@intel.com Signed-off-by: Dave Jiang Signed-off-by: Yi Liu Signed-off-by: Bo Li --- drivers/base/node.c | 31 +++++++++++++++++++++++++++++++ include/linux/node.h | 8 ++++++++ 2 files changed, 39 insertions(+) diff --git a/drivers/base/node.c b/drivers/base/node.c index 415a99c0adaa7..2d2c6dc421d97 100644 --- a/drivers/base/node.c +++ b/drivers/base/node.c @@ -217,6 +217,37 @@ void node_set_perf_attrs(unsigned int nid, struct access_coordinate *coord, } EXPORT_SYMBOL_GPL(node_set_perf_attrs); +/** + * node_update_perf_attrs - Update the performance values for given access class + * @nid: Node identifier to be updated + * @coord: Heterogeneous memory performance coordinates + * @access: The access class for the given attributes + */ +void node_update_perf_attrs(unsigned int nid, struct access_coordinate *coord, + enum access_coordinate_class access) +{ + struct node_access_nodes *access_node; + struct node *node; + int i; + + if (WARN_ON_ONCE(!node_online(nid))) + return; + + node = node_devices[nid]; + list_for_each_entry(access_node, &node->access_list, list_node) { + if (access_node->access != access) + continue; + + access_node->coord = *coord; + for (i = 0; access_attrs[i]; i++) { + sysfs_notify(&access_node->dev.kobj, + NULL, access_attrs[i]->name); + } + break; + } +} +EXPORT_SYMBOL_GPL(node_update_perf_attrs); + /** * struct node_cache_info - Internal tracking for memory node caches * @dev: Device represeting the cache level diff --git a/include/linux/node.h b/include/linux/node.h index dfc004e4bee74..c1d60295cb61c 100644 --- a/include/linux/node.h +++ b/include/linux/node.h @@ -79,6 +79,8 @@ struct node_cache_attrs { void node_add_cache(unsigned int nid, struct node_cache_attrs *cache_attrs); void node_set_perf_attrs(unsigned int nid, struct access_coordinate *coord, enum access_coordinate_class access); +void node_update_perf_attrs(unsigned int nid, struct access_coordinate *coord, + enum access_coordinate_class access); #else static inline void node_add_cache(unsigned int nid, struct node_cache_attrs *cache_attrs) @@ -90,6 +92,12 @@ static inline void node_set_perf_attrs(unsigned int nid, enum access_coordinate_class access) { } + +static inline void node_update_perf_attrs(unsigned int nid, + struct access_coordinate *coord, + enum access_coordinate_class access) +{ +} #endif struct node { From b22ba695d6f5b90c1d0723d204e7f5cc00ea5815 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Fri, 29 Aug 2025 15:29:06 -0700 Subject: [PATCH 111/138] cxl, acpi/hmat: Update CXL access coordinates directly instead of through HMAT commit 2e454fb8056df6da4bba7d89a57bf60e217463c0 upstream. The current implementation of CXL memory hotplug notifier gets called before the HMAT memory hotplug notifier. The CXL driver calculates the access coordinates (bandwidth and latency values) for the CXL end to end path (i.e. CPU to endpoint). When the CXL region is onlined, the CXL memory hotplug notifier writes the access coordinates to the HMAT target structs. Then the HMAT memory hotplug notifier is called and it creates the access coordinates for the node sysfs attributes. During testing on an Intel platform, it was found that although the newly calculated coordinates were pushed to sysfs, the sysfs attributes for the access coordinates showed up with the wrong initiator. The system has 4 nodes (0, 1, 2, 3) where node 0 and 1 are CPU nodes and node 2 and 3 are CXL nodes. The expectation is that node 2 would show up as a target to node 0: /sys/devices/system/node/node2/access0/initiators/node0 However it was observed that node 2 showed up as a target under node 1: /sys/devices/system/node/node2/access0/initiators/node1 The original intent of the 'ext_updated' flag in HMAT handling code was to stop HMAT memory hotplug callback from clobbering the access coordinates after CXL has injected its calculated coordinates and replaced the generic target access coordinates provided by the HMAT table in the HMAT target structs. However the flag is hacky at best and blocks the updates from other CXL regions that are onlined in the same node later on. Remove the 'ext_updated' flag usage and just update the access coordinates for the nodes directly without touching HMAT target data. The hotplug memory callback ordering is changed. Instead of changing CXL, move HMAT back so there's room for the levels rather than have CXL share the same level as SLAB_CALLBACK_PRI. The change will resulting in the CXL callback to be executed after the HMAT callback. With the change, the CXL hotplug memory notifier runs after the HMAT callback. The HMAT callback will create the node sysfs attributes for access coordinates. The CXL callback will write the access coordinates to the now created node sysfs attributes directly and will not pollute the HMAT target values. A nodemask is introduced to keep track if a node has been updated and prevents further updates. Fixes: 067353a46d8c ("cxl/region: Add memory hotplug notifier for cxl region") Cc: stable@vger.kernel.org Tested-by: Marc Herbert Reviewed-by: Dan Williams Reviewed-by: Jonathan Cameron Link: https://patch.msgid.link/20250829222907.1290912-4-dave.jiang@intel.com Signed-off-by: Dave Jiang Signed-off-by: Yi Liu Signed-off-by: Bo Li --- drivers/acpi/numa/hmat.c | 6 ------ drivers/cxl/core/cdat.c | 5 ----- drivers/cxl/core/core.h | 1 - drivers/cxl/core/region.c | 20 ++++++++++++-------- include/linux/memory.h | 2 +- 5 files changed, 13 insertions(+), 21 deletions(-) diff --git a/drivers/acpi/numa/hmat.c b/drivers/acpi/numa/hmat.c index 56b8358245af7..83be8938462e0 100644 --- a/drivers/acpi/numa/hmat.c +++ b/drivers/acpi/numa/hmat.c @@ -74,7 +74,6 @@ struct memory_target { struct node_cache_attrs cache_attrs; u8 gen_port_device_handle[ACPI_SRAT_DEVICE_HANDLE_SIZE]; bool registered; - bool ext_updated; /* externally updated */ }; struct memory_initiator { @@ -352,7 +351,6 @@ int hmat_update_target_coordinates(int nid, struct access_coordinate *coord, coord->read_bandwidth, access); hmat_update_target_access(target, ACPI_HMAT_WRITE_BANDWIDTH, coord->write_bandwidth, access); - target->ext_updated = true; return 0; } @@ -729,10 +727,6 @@ static void hmat_update_target_attrs(struct memory_target *target, u32 best = 0; int i; - /* Don't update if an external agent has changed the data. */ - if (target->ext_updated) - return; - /* Don't update for generic port if there's no device handle */ if ((access == NODE_ACCESS_CLASS_GENPORT_SINK_LOCAL || access == NODE_ACCESS_CLASS_GENPORT_SINK_CPU) && diff --git a/drivers/cxl/core/cdat.c b/drivers/cxl/core/cdat.c index 1d505a49a2753..295d4487b2693 100644 --- a/drivers/cxl/core/cdat.c +++ b/drivers/cxl/core/cdat.c @@ -591,8 +591,3 @@ int cxl_update_hmat_access_coordinates(int nid, struct cxl_region *cxlr, { return hmat_update_target_coordinates(nid, &cxlr->coord[access], access); } - -bool cxl_need_node_perf_attrs_update(int nid) -{ - return !acpi_node_backed_by_real_pxm(nid); -} diff --git a/drivers/cxl/core/core.h b/drivers/cxl/core/core.h index 87008505f8a9e..01ce740573aab 100644 --- a/drivers/cxl/core/core.h +++ b/drivers/cxl/core/core.h @@ -99,6 +99,5 @@ long cxl_pci_get_latency(struct pci_dev *pdev); int cxl_update_hmat_access_coordinates(int nid, struct cxl_region *cxlr, enum access_coordinate_class access); -bool cxl_need_node_perf_attrs_update(int nid); #endif /* __CXL_CORE_H__ */ diff --git a/drivers/cxl/core/region.c b/drivers/cxl/core/region.c index 91885df06386a..e821136a3900d 100644 --- a/drivers/cxl/core/region.c +++ b/drivers/cxl/core/region.c @@ -29,6 +29,12 @@ * 3. Decoder targets */ +/* + * nodemask that sets per node when the access_coordinates for the node has + * been updated by the CXL memory hotplug notifier. + */ +static nodemask_t nodemask_region_seen = NODE_MASK_NONE; + static struct cxl_region *to_cxl_region(struct device *dev); #define __ACCESS_ATTR_RO(_level, _name) { \ @@ -2360,14 +2366,8 @@ static bool cxl_region_update_coordinates(struct cxl_region *cxlr, int nid) for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) { if (cxlr->coord[i].read_bandwidth) { - rc = 0; - if (cxl_need_node_perf_attrs_update(nid)) - node_set_perf_attrs(nid, &cxlr->coord[i], i); - else - rc = cxl_update_hmat_access_coordinates(nid, cxlr, i); - - if (rc == 0) - cset++; + node_update_perf_attrs(nid, &cxlr->coord[i], i); + cset++; } } @@ -2415,6 +2415,10 @@ static int cxl_region_perf_attrs_callback(struct notifier_block *nb, if (nid != region_nid) return NOTIFY_DONE; + /* No action needed if node bit already set */ + if (node_test_and_set(nid, nodemask_region_seen)) + return NOTIFY_DONE; + if (!cxl_region_update_coordinates(cxlr, nid)) return NOTIFY_DONE; diff --git a/include/linux/memory.h b/include/linux/memory.h index d8588256578ae..d5385592a03b8 100644 --- a/include/linux/memory.h +++ b/include/linux/memory.h @@ -113,8 +113,8 @@ struct mem_section; */ #define DEFAULT_CALLBACK_PRI 0 #define SLAB_CALLBACK_PRI 1 -#define HMAT_CALLBACK_PRI 2 #define CXL_CALLBACK_PRI 5 +#define HMAT_CALLBACK_PRI 6 #define MM_COMPUTE_BATCH_PRI 10 #define CPUSET_CALLBACK_PRI 10 #define MEMTIER_HOTPLUG_PRI 100 From fe616b104b5be9ca5fff890dc9a7006c518a434b Mon Sep 17 00:00:00 2001 From: "Cheatham, Benjamin" Date: Fri, 9 Jan 2026 07:57:38 -0600 Subject: [PATCH 112/138] cxl/core: Fix cxl_dport debugfs EINJ entries commit 4ed7952b9e87cf731ebc8251874416e60eb15230 upstream. Protocol error injection is only valid for CXL 2.0+ root ports and CXL 1.1 memory-mapped downstream ports as per the ACPI v6.5 spec (Table 8-31). The core code currently creates an 'einj_inject' file in CXL debugfs for all CXL 1.1 downstream ports and all PCI CXL 2.0+ downstream ports. This results in debugfs EINJ files that won't work due to platform/spec restrictions. Fix by limiting 'einj_inject' file creation to only CXL 1.1 dports and CXL 2.0+ root ports. Update the comment above the check to more accurately represent the requirements expected by the EINJ module and ACPI spec. Fixes: 8039804cfa73 ("cxl/core: Add CXL EINJ debugfs files") Signed-off-by: Ben Cheatham Reviewed-by: Jonathan Cameron Reviewed-by: Alison Schofield Reviewed-by: Dave Jiang Link: https://patch.msgid.link/6e9fb657-8264-4028-92e2-5428e2695bf1@amd.com Signed-off-by: Dave Jiang [Backport-fix: open code parent_port_of() to avoid backport helper introduction commit] Signed-off-by: Yi Liu Signed-off-by: Bo Li --- drivers/cxl/core/port.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/drivers/cxl/core/port.c b/drivers/cxl/core/port.c index eed45377aa03a..a90dff590154f 100644 --- a/drivers/cxl/core/port.c +++ b/drivers/cxl/core/port.c @@ -810,16 +810,20 @@ DEFINE_DEBUGFS_ATTRIBUTE(cxl_einj_inject_fops, NULL, cxl_einj_inject, static void cxl_debugfs_create_dport_dir(struct cxl_dport *dport) { + struct cxl_port *port = dport->port; + struct cxl_port *parent = (port && port->parent_dport) ? + port->parent_dport->port : NULL; struct dentry *dir; if (!einj_cxl_is_initialized()) return; /* - * dport_dev needs to be a PCIe port for CXL 2.0+ ports because - * EINJ expects a dport SBDF to be specified for 2.0 error injection. + * Protocol error injection is only available for CXL 2.0+ root ports + * and CXL 1.1 downstream ports */ - if (!dport->rch && !dev_is_pci(dport->dport_dev)) + if (!dport->rch && + !(dev_is_pci(dport->dport_dev) && parent && is_cxl_root(parent))) return; dir = cxl_debugfs_create_dir(dev_name(dport->dport_dev)); From 9f0bdde26e527047a76987ef2855f3299712b0b6 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Mon, 11 Dec 2023 13:53:34 -0500 Subject: [PATCH 113/138] timers: Split out timer_types.h commit dff0fd233a5104337069603d201f8cad74bc0e5a upstream. Cutting down on sched.h dependencies: this is going to be used in workqueue_types.h in the next patch, so we can kill the sched.h dependency on workqueue.h. Signed-off-by: Kent Overstreet Signed-off-by: Bo Li --- include/linux/timer.h | 16 +--------------- include/linux/timer_types.h | 23 +++++++++++++++++++++++ 2 files changed, 24 insertions(+), 15 deletions(-) create mode 100644 include/linux/timer_types.h diff --git a/include/linux/timer.h b/include/linux/timer.h index 9162f275819a7..992e7538f5939 100644 --- a/include/linux/timer.h +++ b/include/linux/timer.h @@ -7,21 +7,7 @@ #include #include #include - -struct timer_list { - /* - * All fields that change during normal runtime grouped to the - * same cacheline - */ - struct hlist_node entry; - unsigned long expires; - void (*function)(struct timer_list *); - u32 flags; - -#ifdef CONFIG_LOCKDEP - struct lockdep_map lockdep_map; -#endif -}; +#include #ifdef CONFIG_LOCKDEP /* diff --git a/include/linux/timer_types.h b/include/linux/timer_types.h new file mode 100644 index 0000000000000..fae5a388f9149 --- /dev/null +++ b/include/linux/timer_types.h @@ -0,0 +1,23 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LINUX_TIMER_TYPES_H +#define _LINUX_TIMER_TYPES_H + +#include +#include + +struct timer_list { + /* + * All fields that change during normal runtime grouped to the + * same cacheline + */ + struct hlist_node entry; + unsigned long expires; + void (*function)(struct timer_list *); + u32 flags; + +#ifdef CONFIG_LOCKDEP + struct lockdep_map lockdep_map; +#endif +}; + +#endif /* _LINUX_TIMER_TYPES_H */ From 7085befe5531c19a556861452cf3cff1c200244f Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Mon, 11 Dec 2023 13:55:01 -0500 Subject: [PATCH 114/138] workqueue: Split out workqueue_types.h commit b2fa8443db320c4873feca2588b957439e350890 upstream. More sched.h dependency culling - this lets us kill a rhashtable-types.h dependency on workqueue.h. Signed-off-by: Kent Overstreet Signed-off-by: Bo Li --- include/linux/dma-fence.h | 1 + include/linux/rhashtable-types.h | 2 +- include/linux/timekeeping.h | 1 + include/linux/workqueue.h | 16 +--------------- include/linux/workqueue_types.h | 25 +++++++++++++++++++++++++ 5 files changed, 29 insertions(+), 16 deletions(-) create mode 100644 include/linux/workqueue_types.h diff --git a/include/linux/dma-fence.h b/include/linux/dma-fence.h index b3772edca2e6e..e06bad467f55e 100644 --- a/include/linux/dma-fence.h +++ b/include/linux/dma-fence.h @@ -21,6 +21,7 @@ #include #include #include +#include struct dma_fence; struct dma_fence_ops; diff --git a/include/linux/rhashtable-types.h b/include/linux/rhashtable-types.h index 57467cbf4c5b1..b6f3797277ff8 100644 --- a/include/linux/rhashtable-types.h +++ b/include/linux/rhashtable-types.h @@ -12,7 +12,7 @@ #include #include #include -#include +#include struct rhash_head { struct rhash_head __rcu *next; diff --git a/include/linux/timekeeping.h b/include/linux/timekeeping.h index fe1e467ba046f..7c43e98cf2115 100644 --- a/include/linux/timekeeping.h +++ b/include/linux/timekeeping.h @@ -4,6 +4,7 @@ #include #include +#include /* Included from linux/ktime.h */ diff --git a/include/linux/workqueue.h b/include/linux/workqueue.h index 52c6dd6d80ac0..4aab397145ba2 100644 --- a/include/linux/workqueue.h +++ b/include/linux/workqueue.h @@ -14,12 +14,7 @@ #include #include #include - -struct workqueue_struct; - -struct work_struct; -typedef void (*work_func_t)(struct work_struct *work); -void delayed_work_timer_fn(struct timer_list *t); +#include /* * The first word is the work queue pointer and the flags rolled into @@ -95,15 +90,6 @@ enum { #define WORK_STRUCT_FLAG_MASK ((1ul << WORK_STRUCT_FLAG_BITS) - 1) #define WORK_STRUCT_WQ_DATA_MASK (~WORK_STRUCT_FLAG_MASK) -struct work_struct { - atomic_long_t data; - struct list_head entry; - work_func_t func; -#ifdef CONFIG_LOCKDEP - struct lockdep_map lockdep_map; -#endif -}; - #define WORK_DATA_INIT() ATOMIC_LONG_INIT((unsigned long)WORK_STRUCT_NO_POOL) #define WORK_DATA_STATIC_INIT() \ ATOMIC_LONG_INIT((unsigned long)(WORK_STRUCT_NO_POOL | WORK_STRUCT_STATIC)) diff --git a/include/linux/workqueue_types.h b/include/linux/workqueue_types.h new file mode 100644 index 0000000000000..4c38824f3ab4f --- /dev/null +++ b/include/linux/workqueue_types.h @@ -0,0 +1,25 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LINUX_WORKQUEUE_TYPES_H +#define _LINUX_WORKQUEUE_TYPES_H + +#include +#include +#include +#include + +struct workqueue_struct; + +struct work_struct; +typedef void (*work_func_t)(struct work_struct *work); +void delayed_work_timer_fn(struct timer_list *t); + +struct work_struct { + atomic_long_t data; + struct list_head entry; + work_func_t func; +#ifdef CONFIG_LOCKDEP + struct lockdep_map lockdep_map; +#endif +}; + +#endif /* _LINUX_WORKQUEUE_TYPES_H */ From 2dd4edc8ea1a568bc9939b2ca7463357611f7686 Mon Sep 17 00:00:00 2001 From: Lukas Bulwahn Date: Thu, 11 Apr 2024 08:24:43 +0200 Subject: [PATCH 115/138] MAINTAINERS: repair file entry in COMPUTE EXPRESS LINK commit c26a55e513f7f1a2e77d6a76fc8b76d1fde9e8e1 upstream. Commit 12fb28ea6b1c ("EINJ: Add CXL error type support") adds the header file include/linux/einj-cxl.h, but then adds a file entry with cxl-einj.h (note the swapping of words) to the COMPUTE EXPRESS LINK (CXL) section. Hence, ./scripts/get_maintainer.pl --self-test=patterns complains about a broken reference. get_maintainer.pl can only return commit_signer history for file include/linux/einj-cxl.h because the entry in MAINTAINERS is wrong. Correct the entry so that the full MAINTAINER list is returned. Repair the file entry in COMPUTE EXPRESS LINK (CXL). Signed-off-by: Lukas Bulwahn Reviewed-by: Alison Schofield Link: https://lore.kernel.org/r/20240411062443.47372-1-lukas.bulwahn@redhat.com Signed-off-by: Dave Jiang Signed-off-by: Bo Li --- MAINTAINERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MAINTAINERS b/MAINTAINERS index 12d1ef1a37503..cefe1aaa35951 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -5224,7 +5224,7 @@ M: Dan Williams L: linux-cxl@vger.kernel.org S: Maintained F: drivers/cxl/ -F: include/linux/cxl-einj.h +F: include/linux/einj-cxl.h F: include/linux/cxl-event.h F: include/uapi/linux/cxl_mem.h F: tools/testing/cxl/ From 986848ca94d9d9b081ed5307f7cea045465b3aba Mon Sep 17 00:00:00 2001 From: Alison Schofield Date: Tue, 30 Apr 2024 10:28:05 -0700 Subject: [PATCH 116/138] cxl/region: Move cxl_trace_hpa() work to the region driver commit 86954ff5032d9d60a5458334001ab3ae3b2c45e8 upstream. This work belongs in the region driver as it is only useful with CONFIG_CXL_REGION. Add a stub in core.h for when the region driver is not built. Signed-off-by: Alison Schofield Reviewed-by: Jonathan Cameron Reviewed-by: Ira Weiny Link: https://lore.kernel.org/r/183222631f11a43c5e6debc42ec22fe1bd4b818a.1714496730.git.alison.schofield@intel.com Signed-off-by: Dave Jiang Signed-off-by: Bo Li --- drivers/cxl/core/core.h | 7 +++ drivers/cxl/core/region.c | 91 +++++++++++++++++++++++++++++++++++++++ drivers/cxl/core/trace.c | 91 --------------------------------------- drivers/cxl/core/trace.h | 2 - 4 files changed, 98 insertions(+), 93 deletions(-) diff --git a/drivers/cxl/core/core.h b/drivers/cxl/core/core.h index 01ce740573aab..17ac9cb965622 100644 --- a/drivers/cxl/core/core.h +++ b/drivers/cxl/core/core.h @@ -28,8 +28,15 @@ int cxl_region_init(void); void cxl_region_exit(void); int cxl_get_poison_by_endpoint(struct cxl_port *port); struct cxl_region *cxl_dpa_to_region(const struct cxl_memdev *cxlmd, u64 dpa); +u64 cxl_trace_hpa(struct cxl_region *cxlr, const struct cxl_memdev *cxlmd, + u64 dpa); #else +static inline u64 +cxl_trace_hpa(struct cxl_region *cxlr, const struct cxl_memdev *cxlmd, u64 dpa) +{ + return ULLONG_MAX; +} static inline struct cxl_region *cxl_dpa_to_region(const struct cxl_memdev *cxlmd, u64 dpa) { diff --git a/drivers/cxl/core/region.c b/drivers/cxl/core/region.c index e821136a3900d..ecca8594ece6c 100644 --- a/drivers/cxl/core/region.c +++ b/drivers/cxl/core/region.c @@ -2830,6 +2830,97 @@ struct cxl_region *cxl_dpa_to_region(const struct cxl_memdev *cxlmd, u64 dpa) return ctx.cxlr; } +static bool cxl_is_hpa_in_range(u64 hpa, struct cxl_region *cxlr, int pos) +{ + struct cxl_region_params *p = &cxlr->params; + int gran = p->interleave_granularity; + int ways = p->interleave_ways; + u64 offset; + + /* Is the hpa within this region at all */ + if (hpa < p->res->start || hpa > p->res->end) { + dev_dbg(&cxlr->dev, + "Addr trans fail: hpa 0x%llx not in region\n", hpa); + return false; + } + + /* Is the hpa in an expected chunk for its pos(-ition) */ + offset = hpa - p->res->start; + offset = do_div(offset, gran * ways); + if ((offset >= pos * gran) && (offset < (pos + 1) * gran)) + return true; + + dev_dbg(&cxlr->dev, + "Addr trans fail: hpa 0x%llx not in expected chunk\n", hpa); + + return false; +} + +static u64 cxl_dpa_to_hpa(u64 dpa, struct cxl_region *cxlr, + struct cxl_endpoint_decoder *cxled) +{ + u64 dpa_offset, hpa_offset, bits_upper, mask_upper, hpa; + struct cxl_region_params *p = &cxlr->params; + int pos = cxled->pos; + u16 eig = 0; + u8 eiw = 0; + + ways_to_eiw(p->interleave_ways, &eiw); + granularity_to_eig(p->interleave_granularity, &eig); + + /* + * The device position in the region interleave set was removed + * from the offset at HPA->DPA translation. To reconstruct the + * HPA, place the 'pos' in the offset. + * + * The placement of 'pos' in the HPA is determined by interleave + * ways and granularity and is defined in the CXL Spec 3.0 Section + * 8.2.4.19.13 Implementation Note: Device Decode Logic + */ + + /* Remove the dpa base */ + dpa_offset = dpa - cxl_dpa_resource_start(cxled); + + mask_upper = GENMASK_ULL(51, eig + 8); + + if (eiw < 8) { + hpa_offset = (dpa_offset & mask_upper) << eiw; + hpa_offset |= pos << (eig + 8); + } else { + bits_upper = (dpa_offset & mask_upper) >> (eig + 8); + bits_upper = bits_upper * 3; + hpa_offset = ((bits_upper << (eiw - 8)) + pos) << (eig + 8); + } + + /* The lower bits remain unchanged */ + hpa_offset |= dpa_offset & GENMASK_ULL(eig + 7, 0); + + /* Apply the hpa_offset to the region base address */ + hpa = hpa_offset + p->res->start; + + if (!cxl_is_hpa_in_range(hpa, cxlr, cxled->pos)) + return ULLONG_MAX; + + return hpa; +} + +u64 cxl_trace_hpa(struct cxl_region *cxlr, const struct cxl_memdev *cxlmd, + u64 dpa) +{ + struct cxl_region_params *p = &cxlr->params; + struct cxl_endpoint_decoder *cxled = NULL; + + for (int i = 0; i < p->nr_targets; i++) { + cxled = p->targets[i]; + if (cxlmd == cxled_to_memdev(cxled)) + break; + } + if (!cxled || cxlmd != cxled_to_memdev(cxled)) + return ULLONG_MAX; + + return cxl_dpa_to_hpa(dpa, cxlr, cxled); +} + static struct lock_class_key cxl_pmem_region_key; static struct cxl_pmem_region *cxl_pmem_region_alloc(struct cxl_region *cxlr) diff --git a/drivers/cxl/core/trace.c b/drivers/cxl/core/trace.c index d0403dc3c8ab6..7f2a9dd0d0e3f 100644 --- a/drivers/cxl/core/trace.c +++ b/drivers/cxl/core/trace.c @@ -6,94 +6,3 @@ #define CREATE_TRACE_POINTS #include "trace.h" - -static bool cxl_is_hpa_in_range(u64 hpa, struct cxl_region *cxlr, int pos) -{ - struct cxl_region_params *p = &cxlr->params; - int gran = p->interleave_granularity; - int ways = p->interleave_ways; - u64 offset; - - /* Is the hpa within this region at all */ - if (hpa < p->res->start || hpa > p->res->end) { - dev_dbg(&cxlr->dev, - "Addr trans fail: hpa 0x%llx not in region\n", hpa); - return false; - } - - /* Is the hpa in an expected chunk for its pos(-ition) */ - offset = hpa - p->res->start; - offset = do_div(offset, gran * ways); - if ((offset >= pos * gran) && (offset < (pos + 1) * gran)) - return true; - - dev_dbg(&cxlr->dev, - "Addr trans fail: hpa 0x%llx not in expected chunk\n", hpa); - - return false; -} - -static u64 cxl_dpa_to_hpa(u64 dpa, struct cxl_region *cxlr, - struct cxl_endpoint_decoder *cxled) -{ - u64 dpa_offset, hpa_offset, bits_upper, mask_upper, hpa; - struct cxl_region_params *p = &cxlr->params; - int pos = cxled->pos; - u16 eig = 0; - u8 eiw = 0; - - ways_to_eiw(p->interleave_ways, &eiw); - granularity_to_eig(p->interleave_granularity, &eig); - - /* - * The device position in the region interleave set was removed - * from the offset at HPA->DPA translation. To reconstruct the - * HPA, place the 'pos' in the offset. - * - * The placement of 'pos' in the HPA is determined by interleave - * ways and granularity and is defined in the CXL Spec 3.0 Section - * 8.2.4.19.13 Implementation Note: Device Decode Logic - */ - - /* Remove the dpa base */ - dpa_offset = dpa - cxl_dpa_resource_start(cxled); - - mask_upper = GENMASK_ULL(51, eig + 8); - - if (eiw < 8) { - hpa_offset = (dpa_offset & mask_upper) << eiw; - hpa_offset |= pos << (eig + 8); - } else { - bits_upper = (dpa_offset & mask_upper) >> (eig + 8); - bits_upper = bits_upper * 3; - hpa_offset = ((bits_upper << (eiw - 8)) + pos) << (eig + 8); - } - - /* The lower bits remain unchanged */ - hpa_offset |= dpa_offset & GENMASK_ULL(eig + 7, 0); - - /* Apply the hpa_offset to the region base address */ - hpa = hpa_offset + p->res->start; - - if (!cxl_is_hpa_in_range(hpa, cxlr, cxled->pos)) - return ULLONG_MAX; - - return hpa; -} - -u64 cxl_trace_hpa(struct cxl_region *cxlr, struct cxl_memdev *cxlmd, - u64 dpa) -{ - struct cxl_region_params *p = &cxlr->params; - struct cxl_endpoint_decoder *cxled = NULL; - - for (int i = 0; i < p->nr_targets; i++) { - cxled = p->targets[i]; - if (cxlmd == cxled_to_memdev(cxled)) - break; - } - if (!cxled || cxlmd != cxled_to_memdev(cxled)) - return ULLONG_MAX; - - return cxl_dpa_to_hpa(dpa, cxlr, cxled); -} diff --git a/drivers/cxl/core/trace.h b/drivers/cxl/core/trace.h index e110125bc75c2..aee5d190e6ac8 100644 --- a/drivers/cxl/core/trace.h +++ b/drivers/cxl/core/trace.h @@ -653,8 +653,6 @@ TRACE_EVENT(cxl_memory_module, #define cxl_poison_overflow(flags, time) \ (flags & CXL_POISON_FLAG_OVERFLOW ? le64_to_cpu(time) : 0) -u64 cxl_trace_hpa(struct cxl_region *cxlr, struct cxl_memdev *memdev, u64 dpa); - TRACE_EVENT(cxl_poison, TP_PROTO(struct cxl_memdev *cxlmd, struct cxl_region *cxlr, From afa4eafec29066c5a14fe5031f3483bcbdc38543 Mon Sep 17 00:00:00 2001 From: Alison Schofield Date: Tue, 30 Apr 2024 10:28:06 -0700 Subject: [PATCH 117/138] cxl/core: Add region info to cxl_general_media and cxl_dram events commit 6aec00139d3a83e2394d4bcb0084e872b4036e8f upstream. User space may need to know which region, if any, maps the DPAs (device physical addresses) reported in a cxl_general_media or cxl_dram event. Since the mapping can change, the kernel provides this information at the time the event occurs. This informs user space that at event this mapped this to this . Add the same region info that is included in the cxl_poison trace event: the DPA->HPA translation, region name, and region uuid. The new fields are inserted in the trace event and no existing fields are modified. If the DPA is not mapped, user will see: hpa=ULLONG_MAX, region="", and uuid=0 This work must be protected by dpa_rwsem & region_rwsem since it is looking up region mappings. Signed-off-by: Alison Schofield Reviewed-by: Dan Williams Reviewed-by: Ira Weiny Reviewed-by: Jonathan Cameron Link: https://lore.kernel.org/r/dd8d708b7a7ebfb64a27020a5eb338091336b34d.1714496730.git.alison.schofield@intel.com Signed-off-by: Dave Jiang Signed-off-by: Bo Li --- drivers/cxl/core/mbox.c | 36 ++++++++++++++++++++++++++------ drivers/cxl/core/trace.h | 44 +++++++++++++++++++++++++++++++-------- include/linux/cxl-event.h | 10 +++++++++ 3 files changed, 75 insertions(+), 15 deletions(-) diff --git a/drivers/cxl/core/mbox.c b/drivers/cxl/core/mbox.c index 65185c9fa0013..97e938657773b 100644 --- a/drivers/cxl/core/mbox.c +++ b/drivers/cxl/core/mbox.c @@ -842,14 +842,38 @@ void cxl_event_trace_record(const struct cxl_memdev *cxlmd, enum cxl_event_type event_type, const uuid_t *uuid, union cxl_event *evt) { - if (event_type == CXL_CPER_EVENT_GEN_MEDIA) - trace_cxl_general_media(cxlmd, type, &evt->gen_media); - else if (event_type == CXL_CPER_EVENT_DRAM) - trace_cxl_dram(cxlmd, type, &evt->dram); - else if (event_type == CXL_CPER_EVENT_MEM_MODULE) + if (event_type == CXL_CPER_EVENT_MEM_MODULE) { trace_cxl_memory_module(cxlmd, type, &evt->mem_module); - else + return; + } + if (event_type == CXL_CPER_EVENT_GENERIC) { trace_cxl_generic_event(cxlmd, type, uuid, &evt->generic); + return; + } + + if (trace_cxl_general_media_enabled() || trace_cxl_dram_enabled()) { + u64 dpa, hpa = ULLONG_MAX; + struct cxl_region *cxlr; + + /* + * These trace points are annotated with HPA and region + * translations. Take topology mutation locks and lookup + * { HPA, REGION } from { DPA, MEMDEV } in the event record. + */ + guard(rwsem_read)(&cxl_region_rwsem); + guard(rwsem_read)(&cxl_dpa_rwsem); + + dpa = le64_to_cpu(evt->common.phys_addr) & CXL_DPA_MASK; + cxlr = cxl_dpa_to_region(cxlmd, dpa); + if (cxlr) + hpa = cxl_trace_hpa(cxlr, cxlmd, dpa); + + if (event_type == CXL_CPER_EVENT_GEN_MEDIA) + trace_cxl_general_media(cxlmd, type, cxlr, hpa, + &evt->gen_media); + else if (event_type == CXL_CPER_EVENT_DRAM) + trace_cxl_dram(cxlmd, type, cxlr, hpa, &evt->dram); + } } EXPORT_SYMBOL_NS_GPL(cxl_event_trace_record, CXL); diff --git a/drivers/cxl/core/trace.h b/drivers/cxl/core/trace.h index aee5d190e6ac8..0c8f63e893dca 100644 --- a/drivers/cxl/core/trace.h +++ b/drivers/cxl/core/trace.h @@ -316,9 +316,9 @@ TRACE_EVENT(cxl_generic_event, TRACE_EVENT(cxl_general_media, TP_PROTO(const struct cxl_memdev *cxlmd, enum cxl_event_log_type log, - struct cxl_event_gen_media *rec), + struct cxl_region *cxlr, u64 hpa, struct cxl_event_gen_media *rec), - TP_ARGS(cxlmd, log, rec), + TP_ARGS(cxlmd, log, cxlr, hpa, rec), TP_STRUCT__entry( CXL_EVT_TP_entry @@ -330,10 +330,13 @@ TRACE_EVENT(cxl_general_media, __field(u8, channel) __field(u32, device) __array(u8, comp_id, CXL_EVENT_GEN_MED_COMP_ID_SIZE) - __field(u16, validity_flags) /* Following are out of order to pack trace record */ + __field(u64, hpa) + __field_struct(uuid_t, region_uuid) + __field(u16, validity_flags) __field(u8, rank) __field(u8, dpa_flags) + __string(region_name, cxlr ? dev_name(&cxlr->dev) : "") ), TP_fast_assign( @@ -354,18 +357,28 @@ TRACE_EVENT(cxl_general_media, memcpy(__entry->comp_id, &rec->component_id, CXL_EVENT_GEN_MED_COMP_ID_SIZE); __entry->validity_flags = get_unaligned_le16(&rec->validity_flags); + __entry->hpa = hpa; + if (cxlr) { + __assign_str(region_name, dev_name(&cxlr->dev)); + uuid_copy(&__entry->region_uuid, &cxlr->params.uuid); + } else { + __assign_str(region_name, ""); + uuid_copy(&__entry->region_uuid, &uuid_null); + } ), CXL_EVT_TP_printk("dpa=%llx dpa_flags='%s' " \ "descriptor='%s' type='%s' transaction_type='%s' channel=%u rank=%u " \ - "device=%x comp_id=%s validity_flags='%s'", + "device=%x comp_id=%s validity_flags='%s' " \ + "hpa=%llx region=%s region_uuid=%pUb", __entry->dpa, show_dpa_flags(__entry->dpa_flags), show_event_desc_flags(__entry->descriptor), show_gmer_mem_event_type(__entry->type), show_trans_type(__entry->transaction_type), __entry->channel, __entry->rank, __entry->device, __print_hex(__entry->comp_id, CXL_EVENT_GEN_MED_COMP_ID_SIZE), - show_valid_flags(__entry->validity_flags) + show_valid_flags(__entry->validity_flags), + __entry->hpa, __get_str(region_name), &__entry->region_uuid ) ); @@ -411,9 +424,9 @@ TRACE_EVENT(cxl_general_media, TRACE_EVENT(cxl_dram, TP_PROTO(const struct cxl_memdev *cxlmd, enum cxl_event_log_type log, - struct cxl_event_dram *rec), + struct cxl_region *cxlr, u64 hpa, struct cxl_event_dram *rec), - TP_ARGS(cxlmd, log, rec), + TP_ARGS(cxlmd, log, cxlr, hpa, rec), TP_STRUCT__entry( CXL_EVT_TP_entry @@ -428,10 +441,13 @@ TRACE_EVENT(cxl_dram, __field(u32, nibble_mask) __field(u32, row) __array(u8, cor_mask, CXL_EVENT_DER_CORRECTION_MASK_SIZE) + __field(u64, hpa) + __field_struct(uuid_t, region_uuid) __field(u8, rank) /* Out of order to pack trace record */ __field(u8, bank_group) /* Out of order to pack trace record */ __field(u8, bank) /* Out of order to pack trace record */ __field(u8, dpa_flags) /* Out of order to pack trace record */ + __string(region_name, cxlr ? dev_name(&cxlr->dev) : "") ), TP_fast_assign( @@ -455,12 +471,21 @@ TRACE_EVENT(cxl_dram, __entry->column = get_unaligned_le16(rec->column); memcpy(__entry->cor_mask, &rec->correction_mask, CXL_EVENT_DER_CORRECTION_MASK_SIZE); + __entry->hpa = hpa; + if (cxlr) { + __assign_str(region_name, dev_name(&cxlr->dev)); + uuid_copy(&__entry->region_uuid, &cxlr->params.uuid); + } else { + __assign_str(region_name, ""); + uuid_copy(&__entry->region_uuid, &uuid_null); + } ), CXL_EVT_TP_printk("dpa=%llx dpa_flags='%s' descriptor='%s' type='%s' " \ "transaction_type='%s' channel=%u rank=%u nibble_mask=%x " \ "bank_group=%u bank=%u row=%u column=%u cor_mask=%s " \ - "validity_flags='%s'", + "validity_flags='%s' " \ + "hpa=%llx region=%s region_uuid=%pUb", __entry->dpa, show_dpa_flags(__entry->dpa_flags), show_event_desc_flags(__entry->descriptor), show_dram_mem_event_type(__entry->type), @@ -469,7 +494,8 @@ TRACE_EVENT(cxl_dram, __entry->bank_group, __entry->bank, __entry->row, __entry->column, __print_hex(__entry->cor_mask, CXL_EVENT_DER_CORRECTION_MASK_SIZE), - show_dram_valid_flags(__entry->validity_flags) + show_dram_valid_flags(__entry->validity_flags), + __entry->hpa, __get_str(region_name), &__entry->region_uuid ) ); diff --git a/include/linux/cxl-event.h b/include/linux/cxl-event.h index 03fa6d50d46fe..5342755777ccd 100644 --- a/include/linux/cxl-event.h +++ b/include/linux/cxl-event.h @@ -91,11 +91,21 @@ struct cxl_event_mem_module { u8 reserved[0x3d]; } __packed; +/* + * General Media or DRAM Event Common Fields + * - provides common access to phys_addr + */ +struct cxl_event_common { + struct cxl_event_record_hdr hdr; + __le64 phys_addr; +} __packed; + union cxl_event { struct cxl_event_generic generic; struct cxl_event_gen_media gen_media; struct cxl_event_dram dram; struct cxl_event_mem_module mem_module; + struct cxl_event_common common; } __packed; /* From fc4e9e6736865446a451de265f3d876320c37ac4 Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Fri, 5 Apr 2024 15:05:50 -0700 Subject: [PATCH 118/138] cxl/acpi: Cleanup __cxl_parse_cfmws() commit e4ff70a8e3352f71b5db52c752a9417402a098c8 upstream. As a follow on to the recent rework of __cxl_parse_cfmws() to always return errors [1], use cleanup.h helpers to remove goto and other cleanups now that logging is moved to the cxl_parse_cfmws() wrapper. This ends up adding more code than it deletes, but __cxl_parse_cfmws() itself does get smaller. The takeaway from the cond_no_free_ptr() discussion [2] was to not add new macros to handle the cases where no_free_ptr() is awkward, instead rework the code to have helpers and clearer delineation of responsibility. Now one might say that __free(del_cxl_resource) is excessive given it is immediately registered with add_or_reset_cxl_resource(). The rationale for keeping it is that it forces use of "no_free_ptr()" on the argument passed to add_or_reset_cxl_resource(). That in turn makes it clear that @res is NULL for the rest of the function which is part of the point of the cleanup helpers, to turn subtle use after free errors [3] into loud NULL pointer de-references. Link: http://lore.kernel.org/r/170820177238.631006.1012639681618409284.stgit@dwillia2-xfh.jf.intel.com [1] Link: http://lore.kernel.org/r/CAHk-=whBVhnh=KSeBBRet=E7qJAwnPR_aj5em187Q3FiD+LXnA@mail.gmail.com [2] Link: http://lore.kernel.org/r/20230714093146.2253438-1-leitao@debian.org [3] Reported-by: Jonathan Cameron Closes: http://lore.kernel.org/r/20240219124041.00002bda@Huawei.com Reviewed-by: Alison Schofield Reviewed-by: Vishal Verma Signed-off-by: Dan Williams Reviewed-by: Jonathan Cameron Link: https://lore.kernel.org/r/171235474028.2718248.14109646123143505522.stgit@dwillia2-xfh.jf.intel.com Signed-off-by: Dave Jiang Signed-off-by: Bo Li --- drivers/cxl/acpi.c | 93 +++++++++++++++++++++++++--------------------- drivers/cxl/cxl.h | 5 +++ 2 files changed, 56 insertions(+), 42 deletions(-) diff --git a/drivers/cxl/acpi.c b/drivers/cxl/acpi.c index 3f9040b2b27fe..2e4c070ef5f8c 100644 --- a/drivers/cxl/acpi.c +++ b/drivers/cxl/acpi.c @@ -316,28 +316,59 @@ static const struct cxl_root_ops acpi_root_ops = { .qos_class = cxl_acpi_qos_class, }; +static void del_cxl_resource(struct resource *res) +{ + if (!res) + return; + kfree(res->name); + kfree(res); +} + +static struct resource *alloc_cxl_resource(resource_size_t base, + resource_size_t n, int id) +{ + struct resource *res __free(kfree) = kzalloc(sizeof(*res), GFP_KERNEL); + + if (!res) + return NULL; + + res->start = base; + res->end = base + n - 1; + res->flags = IORESOURCE_MEM; + res->name = kasprintf(GFP_KERNEL, "CXL Window %d", id); + if (!res->name) + return NULL; + + return no_free_ptr(res); +} + +static int add_or_reset_cxl_resource(struct resource *parent, struct resource *res) +{ + int rc = insert_resource(parent, res); + + if (rc) + del_cxl_resource(res); + return rc; +} + +DEFINE_FREE(put_cxlrd, struct cxl_root_decoder *, + if (!IS_ERR_OR_NULL(_T)) put_device(&_T->cxlsd.cxld.dev)) +DEFINE_FREE(del_cxl_resource, struct resource *, if (_T) del_cxl_resource(_T)) static int __cxl_parse_cfmws(struct acpi_cedt_cfmws *cfmws, struct cxl_cfmws_context *ctx) { int target_map[CXL_DECODER_MAX_INTERLEAVE]; struct cxl_port *root_port = ctx->root_port; - struct resource *cxl_res = ctx->cxl_res; struct cxl_cxims_context cxims_ctx; - struct cxl_root_decoder *cxlrd; struct device *dev = ctx->dev; cxl_calc_hb_fn cxl_calc_hb; struct cxl_decoder *cxld; unsigned int ways, i, ig; - struct resource *res; int rc; rc = cxl_acpi_cfmws_verify(dev, cfmws); - if (rc) { - dev_err(dev, "CFMWS range %#llx-%#llx not registered\n", - cfmws->base_hpa, - cfmws->base_hpa + cfmws->window_size - 1); + if (rc) return rc; - } rc = eiw_to_ways(cfmws->interleave_ways, &ways); if (rc) @@ -348,29 +379,23 @@ static int __cxl_parse_cfmws(struct acpi_cedt_cfmws *cfmws, for (i = 0; i < ways; i++) target_map[i] = cfmws->interleave_targets[i]; - res = kzalloc(sizeof(*res), GFP_KERNEL); + struct resource *res __free(del_cxl_resource) = alloc_cxl_resource( + cfmws->base_hpa, cfmws->window_size, ctx->id++); if (!res) return -ENOMEM; - res->name = kasprintf(GFP_KERNEL, "CXL Window %d", ctx->id++); - if (!res->name) - goto err_name; - - res->start = cfmws->base_hpa; - res->end = cfmws->base_hpa + cfmws->window_size - 1; - res->flags = IORESOURCE_MEM; - /* add to the local resource tracking to establish a sort order */ - rc = insert_resource(cxl_res, res); + rc = add_or_reset_cxl_resource(ctx->cxl_res, no_free_ptr(res)); if (rc) - goto err_insert; + return rc; if (cfmws->interleave_arithmetic == ACPI_CEDT_CFMWS_ARITHMETIC_MODULO) cxl_calc_hb = cxl_hb_modulo; else cxl_calc_hb = cxl_hb_xor; - cxlrd = cxl_root_decoder_alloc(root_port, ways, cxl_calc_hb); + struct cxl_root_decoder *cxlrd __free(put_cxlrd) = + cxl_root_decoder_alloc(root_port, ways, cxl_calc_hb); if (IS_ERR(cxlrd)) return PTR_ERR(cxlrd); @@ -378,8 +403,8 @@ static int __cxl_parse_cfmws(struct acpi_cedt_cfmws *cfmws, cxld->flags = cfmws_to_decoder_flags(cfmws->restrictions); cxld->target_type = CXL_DECODER_HOSTONLYMEM; cxld->hpa_range = (struct range) { - .start = res->start, - .end = res->end, + .start = cfmws->base_hpa, + .end = cfmws->base_hpa + cfmws->window_size - 1, }; cxld->interleave_ways = ways; /* @@ -399,11 +424,10 @@ static int __cxl_parse_cfmws(struct acpi_cedt_cfmws *cfmws, rc = acpi_table_parse_cedt(ACPI_CEDT_TYPE_CXIMS, cxl_parse_cxims, &cxims_ctx); if (rc < 0) - goto err_xormap; + return rc; if (!cxlrd->platform_data) { dev_err(dev, "No CXIMS for HBIG %u\n", ig); - rc = -EINVAL; - goto err_xormap; + return -EINVAL; } } } @@ -411,18 +435,9 @@ static int __cxl_parse_cfmws(struct acpi_cedt_cfmws *cfmws, cxlrd->qos_class = cfmws->qtg_id; rc = cxl_decoder_add(cxld, target_map); -err_xormap: if (rc) - put_device(&cxld->dev); - else - rc = cxl_decoder_autoremove(dev, cxld); - return rc; - -err_insert: - kfree(res->name); -err_name: - kfree(res); - return -ENOMEM; + return rc; + return cxl_root_decoder_autoremove(dev, no_free_ptr(cxlrd)); } static int cxl_parse_cfmws(union acpi_subtable_headers *header, void *arg, @@ -683,12 +698,6 @@ static void cxl_acpi_lock_reset_class(void *dev) device_lock_reset_class(dev); } -static void del_cxl_resource(struct resource *res) -{ - kfree(res->name); - kfree(res); -} - static void cxl_set_public_resource(struct resource *priv, struct resource *pub) { priv->desc = (unsigned long) pub; diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h index 371945d95b9ab..7b9fd201030fc 100644 --- a/drivers/cxl/cxl.h +++ b/drivers/cxl/cxl.h @@ -782,6 +782,11 @@ int cxl_decoder_add(struct cxl_decoder *cxld, int *target_map); struct cxl_endpoint_decoder *cxl_endpoint_decoder_alloc(struct cxl_port *port); int cxl_decoder_add_locked(struct cxl_decoder *cxld, int *target_map); int cxl_decoder_autoremove(struct device *host, struct cxl_decoder *cxld); +static inline int cxl_root_decoder_autoremove(struct device *host, + struct cxl_root_decoder *cxlrd) +{ + return cxl_decoder_autoremove(host, &cxlrd->cxlsd.cxld); +} int cxl_endpoint_autoremove(struct cxl_memdev *cxlmd, struct cxl_port *endpoint); /** From 4df66b490bfda8d2a44637a1e0276160de7bf2c9 Mon Sep 17 00:00:00 2001 From: Ira Weiny Date: Fri, 26 Apr 2024 20:34:00 -0700 Subject: [PATCH 119/138] acpi/ghes: Process CXL Component Events commit 5e4a264bf8b55d5a1b4c8cd96ac10b99f98d7487 upstream. BIOS can configure memory devices as firmware first. This will send CXL events to the firmware instead of the OS. The firmware can then inform the OS of these events via UEFI. UEFI v2.10 section N.2.14 defines a Common Platform Error Record (CPER) format for CXL Component Events. The format is mostly the same as the CXL Common Event Record Format. The difference lies in the use of a GUID as the CPER Section Type which matches the UUID defined in CXL 3.1 Table 8-43. Currently a configuration such as this will trace a non standard event in the log omitting useful details of the event. In addition the CXL sub-system contains additional region and HPA information useful to the user.[0] The CXL code is required to be called from process context as it needs to take a device lock. The GHES code may be in interrupt context. This complicated the use of a callback. Dan Williams suggested the use of work items as an atomic way of switching between the callback execution and a default handler.[1] The use of a kfifo simplifies queue processing by providing lock free fifo operations. cxl_cper_kfifo_get() allows easier management of the kfifo between the ghes and cxl modules. CXL 3.1 Table 8-127 requires a device to have a queue depth of 1 for each of the four event logs. A combined queue depth of 32 is chosen to provide room for 8 entries of each log type. Add GHES support to detect CXL CPER records. Add the ability for the CXL sub-system to register a work queue to process the events. This patch adds back the functionality which was removed to fix the report by Dan Carpenter[2]. Cc: Ard Biesheuvel Cc: Rafael J. Wysocki Cc: Tony Luck Cc: Borislav Petkov Suggested-by: Dan Carpenter Suggested-by: Dan Williams Link: http://lore.kernel.org/r/cover.1711598777.git.alison.schofield@intel.com [0] Link: http://lore.kernel.org/r/65d111eb87115_6c745294ac@dwillia2-xfh.jf.intel.com.notmuch [1] Link: http://lore.kernel.org/r/b963c490-2c13-4b79-bbe7-34c6568423c7@moroto.mountain [2] Reviewed-by: Dan Williams Signed-off-by: Ira Weiny Reviewed-by: Jonathan Cameron Reviewed-by: Tony Luck Tested-by: Smita Koralahalli Link: https://lore.kernel.org/r/20240426-cxl-cper3-v4-1-58076cce1624@intel.com Signed-off-by: Dave Jiang Signed-off-by: Bo Li --- drivers/acpi/apei/ghes.c | 110 ++++++++++++++++++++++++++++++++++++++ include/linux/cxl-event.h | 27 ++++++++++ 2 files changed, 137 insertions(+) diff --git a/drivers/acpi/apei/ghes.c b/drivers/acpi/apei/ghes.c index e768dfd345fb2..f60e324afc42d 100644 --- a/drivers/acpi/apei/ghes.c +++ b/drivers/acpi/apei/ghes.c @@ -27,6 +27,8 @@ #include #include #include +#include +#include #include #include #include @@ -34,6 +36,7 @@ #include #include #include +#include #include #include #include @@ -675,6 +678,101 @@ static void ghes_defer_non_standard_event(struct acpi_hest_generic_data *gdata, schedule_work(&entry->work); } +/* CXL Event record UUIDs are formated as GUIDs and reported in section type */ + +/* + * General Media Event Record + * CXL rev 3.0 Section 8.2.9.2.1.1; Table 8-43 + */ +#define CPER_SEC_CXL_GEN_MEDIA_GUID \ + GUID_INIT(0xfbcd0a77, 0xc260, 0x417f, \ + 0x85, 0xa9, 0x08, 0x8b, 0x16, 0x21, 0xeb, 0xa6) + +/* + * DRAM Event Record + * CXL rev 3.0 section 8.2.9.2.1.2; Table 8-44 + */ +#define CPER_SEC_CXL_DRAM_GUID \ + GUID_INIT(0x601dcbb3, 0x9c06, 0x4eab, \ + 0xb8, 0xaf, 0x4e, 0x9b, 0xfb, 0x5c, 0x96, 0x24) + +/* + * Memory Module Event Record + * CXL rev 3.0 section 8.2.9.2.1.3; Table 8-45 + */ +#define CPER_SEC_CXL_MEM_MODULE_GUID \ + GUID_INIT(0xfe927475, 0xdd59, 0x4339, \ + 0xa5, 0x86, 0x79, 0xba, 0xb1, 0x13, 0xb7, 0x74) + +/* Room for 8 entries for each of the 4 event log queues */ +#define CXL_CPER_FIFO_DEPTH 32 +DEFINE_KFIFO(cxl_cper_fifo, struct cxl_cper_work_data, CXL_CPER_FIFO_DEPTH); + +/* Synchronize schedule_work() with cxl_cper_work changes */ +static DEFINE_SPINLOCK(cxl_cper_work_lock); +struct work_struct *cxl_cper_work; + +static void cxl_cper_post_event(enum cxl_event_type event_type, + struct cxl_cper_event_rec *rec) +{ + struct cxl_cper_work_data wd; + + if (rec->hdr.length <= sizeof(rec->hdr) || + rec->hdr.length > sizeof(*rec)) { + pr_err(FW_WARN "CXL CPER Invalid section length (%u)\n", + rec->hdr.length); + return; + } + + if (!(rec->hdr.validation_bits & CPER_CXL_COMP_EVENT_LOG_VALID)) { + pr_err(FW_WARN "CXL CPER invalid event\n"); + return; + } + + guard(spinlock_irqsave)(&cxl_cper_work_lock); + + if (!cxl_cper_work) + return; + + wd.event_type = event_type; + memcpy(&wd.rec, rec, sizeof(wd.rec)); + + if (!kfifo_put(&cxl_cper_fifo, wd)) { + pr_err_ratelimited("CXL CPER kfifo overflow\n"); + return; + } + + schedule_work(cxl_cper_work); +} + +int cxl_cper_register_work(struct work_struct *work) +{ + if (cxl_cper_work) + return -EINVAL; + + guard(spinlock)(&cxl_cper_work_lock); + cxl_cper_work = work; + return 0; +} +EXPORT_SYMBOL_NS_GPL(cxl_cper_register_work, CXL); + +int cxl_cper_unregister_work(struct work_struct *work) +{ + if (cxl_cper_work != work) + return -EINVAL; + + guard(spinlock)(&cxl_cper_work_lock); + cxl_cper_work = NULL; + return 0; +} +EXPORT_SYMBOL_NS_GPL(cxl_cper_unregister_work, CXL); + +int cxl_cper_kfifo_get(struct cxl_cper_work_data *wd) +{ + return kfifo_get(&cxl_cper_fifo, wd); +} +EXPORT_SYMBOL_NS_GPL(cxl_cper_kfifo_get, CXL); + static bool ghes_do_proc(struct ghes *ghes, const struct acpi_hest_generic_status *estatus) { @@ -709,6 +807,18 @@ static bool ghes_do_proc(struct ghes *ghes, } else if (guid_equal(sec_type, &CPER_SEC_PROC_ARM)) { queued = ghes_handle_arm_hw_error(gdata, sev, sync); + } else if (guid_equal(sec_type, &CPER_SEC_CXL_GEN_MEDIA_GUID)) { + struct cxl_cper_event_rec *rec = acpi_hest_get_payload(gdata); + + cxl_cper_post_event(CXL_CPER_EVENT_GEN_MEDIA, rec); + } else if (guid_equal(sec_type, &CPER_SEC_CXL_DRAM_GUID)) { + struct cxl_cper_event_rec *rec = acpi_hest_get_payload(gdata); + + cxl_cper_post_event(CXL_CPER_EVENT_DRAM, rec); + } else if (guid_equal(sec_type, &CPER_SEC_CXL_MEM_MODULE_GUID)) { + struct cxl_cper_event_rec *rec = acpi_hest_get_payload(gdata); + + cxl_cper_post_event(CXL_CPER_EVENT_MEM_MODULE, rec); } else { void *err = acpi_hest_get_payload(gdata); diff --git a/include/linux/cxl-event.h b/include/linux/cxl-event.h index 5342755777ccd..d874336225985 100644 --- a/include/linux/cxl-event.h +++ b/include/linux/cxl-event.h @@ -3,6 +3,8 @@ #ifndef _LINUX_CXL_EVENT_H #define _LINUX_CXL_EVENT_H +#include + /* * Common Event Record Format * CXL rev 3.0 section 8.2.9.2.1; Table 8-42 @@ -150,4 +152,29 @@ struct cxl_cper_event_rec { union cxl_event event; } __packed; +struct cxl_cper_work_data { + enum cxl_event_type event_type; + struct cxl_cper_event_rec rec; +}; + +#ifdef CONFIG_ACPI_APEI_GHES +int cxl_cper_register_work(struct work_struct *work); +int cxl_cper_unregister_work(struct work_struct *work); +int cxl_cper_kfifo_get(struct cxl_cper_work_data *wd); +#else +static inline int cxl_cper_register_work(struct work_struct *work); +{ + return 0; +} + +static inline int cxl_cper_unregister_work(struct work_struct *work); +{ + return 0; +} +static inline int cxl_cper_kfifo_get(struct cxl_cper_work_data *wd) +{ + return 0; +} +#endif + #endif /* _LINUX_CXL_EVENT_H */ From 0529b9e6983a6b5b13b438da7877ce81afb160dc Mon Sep 17 00:00:00 2001 From: Ira Weiny Date: Fri, 26 Apr 2024 20:34:01 -0700 Subject: [PATCH 120/138] cxl/pci: Process CPER events commit c19ac30eda3a1d14d4883de0ea214b6c5c96a9b4 upstream. If the firmware has configured CXL event support to be firmware first the OS will receive those events through CPER records. The CXL layer has unique DPA to HPA knowledge and existing event trace parsing in place.[0] Add a CXL CPER work item and register it with the GHES code to process CPER events. Link: http://lore.kernel.org/r/cover.1711598777.git.alison.schofield@intel.com [0] Reviewed-by: Dan Williams Signed-off-by: Ira Weiny Reviewed-by: Jonathan Cameron Tested-by: Smita Koralahalli Reviewed-by: Smita Koralahalli Link: https://lore.kernel.org/r/20240426-cxl-cper3-v4-2-58076cce1624@intel.com Signed-off-by: Dave Jiang Signed-off-by: Bo Li --- drivers/cxl/pci.c | 71 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 70 insertions(+), 1 deletion(-) diff --git a/drivers/cxl/pci.c b/drivers/cxl/pci.c index 0b37f15f3a600..bf900eba504c9 100644 --- a/drivers/cxl/pci.c +++ b/drivers/cxl/pci.c @@ -973,6 +973,75 @@ static struct pci_driver cxl_pci_driver = { }, }; -module_pci_driver(cxl_pci_driver); +#define CXL_EVENT_HDR_FLAGS_REC_SEVERITY GENMASK(1, 0) +static void cxl_handle_cper_event(enum cxl_event_type ev_type, + struct cxl_cper_event_rec *rec) +{ + struct cper_cxl_event_devid *device_id = &rec->hdr.device_id; + struct pci_dev *pdev __free(pci_dev_put) = NULL; + enum cxl_event_log_type log_type; + struct cxl_dev_state *cxlds; + unsigned int devfn; + u32 hdr_flags; + + pr_debug("CPER event %d for device %u:%u:%u.%u\n", ev_type, + device_id->segment_num, device_id->bus_num, + device_id->device_num, device_id->func_num); + + devfn = PCI_DEVFN(device_id->device_num, device_id->func_num); + pdev = pci_get_domain_bus_and_slot(device_id->segment_num, + device_id->bus_num, devfn); + if (!pdev) + return; + + guard(device)(&pdev->dev); + if (pdev->driver != &cxl_pci_driver) + return; + + cxlds = pci_get_drvdata(pdev); + if (!cxlds) + return; + + /* Fabricate a log type */ + hdr_flags = get_unaligned_le24(rec->event.generic.hdr.flags); + log_type = FIELD_GET(CXL_EVENT_HDR_FLAGS_REC_SEVERITY, hdr_flags); + + cxl_event_trace_record(cxlds->cxlmd, log_type, ev_type, + &uuid_null, &rec->event); +} + +static void cxl_cper_work_fn(struct work_struct *work) +{ + struct cxl_cper_work_data wd; + + while (cxl_cper_kfifo_get(&wd)) + cxl_handle_cper_event(wd.event_type, &wd.rec); +} +static DECLARE_WORK(cxl_cper_work, cxl_cper_work_fn); + +static int __init cxl_pci_driver_init(void) +{ + int rc; + + rc = pci_register_driver(&cxl_pci_driver); + if (rc) + return rc; + + rc = cxl_cper_register_work(&cxl_cper_work); + if (rc) + pci_unregister_driver(&cxl_pci_driver); + + return rc; +} + +static void __exit cxl_pci_driver_exit(void) +{ + cxl_cper_unregister_work(&cxl_cper_work); + cancel_work_sync(&cxl_cper_work); + pci_unregister_driver(&cxl_pci_driver); +} + +module_init(cxl_pci_driver_init); +module_exit(cxl_pci_driver_exit); MODULE_LICENSE("GPL v2"); MODULE_IMPORT_NS(CXL); From 5745ba31e3f6bf3d07eee5bffaca3505bc57e880 Mon Sep 17 00:00:00 2001 From: Ira Weiny Date: Wed, 1 May 2024 11:40:23 -0700 Subject: [PATCH 121/138] cxl/cper: Fix non-ACPI-APEI-GHES build commit 55111470b4416a6225abfdb4e1f0b8a8c8c52fdc upstream. If ACPI_APEI_GHES is not configured the [un]register work functions are not properly declared. 0day notices that the cxl_cper_register_work() declaration in the CONFIG_ACPI_APEI_GHES=n is broken, fix it to be typical nop stub. Reported-by: kernel test robot Closes: http://lore.kernel.org/r/202405012230.6kXItWen-lkp@intel.com Reviewed-by: Dan Williams Signed-off-by: Ira Weiny Link: https://lore.kernel.org/r/20240501-cper-fix-0day-v1-1-c0b0056eafbc@intel.com Signed-off-by: Dave Jiang Signed-off-by: Bo Li --- include/linux/cxl-event.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/linux/cxl-event.h b/include/linux/cxl-event.h index d874336225985..67a35d44cdf21 100644 --- a/include/linux/cxl-event.h +++ b/include/linux/cxl-event.h @@ -162,12 +162,12 @@ int cxl_cper_register_work(struct work_struct *work); int cxl_cper_unregister_work(struct work_struct *work); int cxl_cper_kfifo_get(struct cxl_cper_work_data *wd); #else -static inline int cxl_cper_register_work(struct work_struct *work); +static inline int cxl_cper_register_work(struct work_struct *work) { return 0; } -static inline int cxl_cper_unregister_work(struct work_struct *work); +static inline int cxl_cper_unregister_work(struct work_struct *work) { return 0; } From ea98f9b095b70f24a3814fad27a4725c31e5193e Mon Sep 17 00:00:00 2001 From: Ira Weiny Date: Thu, 2 May 2024 11:42:34 -0700 Subject: [PATCH 122/138] cxl/cper: Remove duplicated GUID defines commit d99f13843237cf9dbdc1bd873a901662b4aee16f upstream. Commit 54ce1927eb78 ("cxl/cper: Fix errant CPER prints for CXL events") moved the CXL CPER section defines to include/linux/cper.h from ghes.c When the latest cxl/cper series was reworked those defines were kept in ghes.c by accident. Thus they were duplicated. Delete the duplicate defines keeping them in the header to be shared between efi and apei. Suggested-by: Fabio M. De Francesco Signed-off-by: Ira Weiny Link: https://lore.kernel.org/r/20240502-cper-fix-dup-guid-v1-1-283cc447c7bf@intel.com Signed-off-by: Dave Jiang Signed-off-by: Bo Li --- drivers/acpi/apei/ghes.c | 26 -------------------------- 1 file changed, 26 deletions(-) diff --git a/drivers/acpi/apei/ghes.c b/drivers/acpi/apei/ghes.c index f60e324afc42d..98c5e309445bb 100644 --- a/drivers/acpi/apei/ghes.c +++ b/drivers/acpi/apei/ghes.c @@ -678,32 +678,6 @@ static void ghes_defer_non_standard_event(struct acpi_hest_generic_data *gdata, schedule_work(&entry->work); } -/* CXL Event record UUIDs are formated as GUIDs and reported in section type */ - -/* - * General Media Event Record - * CXL rev 3.0 Section 8.2.9.2.1.1; Table 8-43 - */ -#define CPER_SEC_CXL_GEN_MEDIA_GUID \ - GUID_INIT(0xfbcd0a77, 0xc260, 0x417f, \ - 0x85, 0xa9, 0x08, 0x8b, 0x16, 0x21, 0xeb, 0xa6) - -/* - * DRAM Event Record - * CXL rev 3.0 section 8.2.9.2.1.2; Table 8-44 - */ -#define CPER_SEC_CXL_DRAM_GUID \ - GUID_INIT(0x601dcbb3, 0x9c06, 0x4eab, \ - 0xb8, 0xaf, 0x4e, 0x9b, 0xfb, 0x5c, 0x96, 0x24) - -/* - * Memory Module Event Record - * CXL rev 3.0 section 8.2.9.2.1.3; Table 8-45 - */ -#define CPER_SEC_CXL_MEM_MODULE_GUID \ - GUID_INIT(0xfe927475, 0xdd59, 0x4339, \ - 0xa5, 0x86, 0x79, 0xba, 0xb1, 0x13, 0xb7, 0x74) - /* Room for 8 entries for each of the 4 event log queues */ #define CXL_CPER_FIFO_DEPTH 32 DEFINE_KFIFO(cxl_cper_fifo, struct cxl_cper_work_data, CXL_CPER_FIFO_DEPTH); From 11b46b99b342cfc170315b14cc1232ef27555f4e Mon Sep 17 00:00:00 2001 From: Alison Schofield Date: Tue, 2 Jul 2024 22:29:49 -0700 Subject: [PATCH 123/138] cxl/core: Fold cxl_trace_hpa() into cxl_dpa_to_hpa() commit 9aa5f6235e16ac6fceed789a30e72addf1abd7d8 upstream. Although cxl_trace_hpa() is used to populate TRACE EVENTs with HPA addresses the work it performs is a DPA to HPA translation not a trace. Tidy up this naming by moving the minimal work done in cxl_trace_hpa() into cxl_dpa_to_hpa() and use cxl_dpa_to_hpa() for trace event callbacks. Suggested-by: Dan Williams Signed-off-by: Alison Schofield Reviewed-by: Dan Williams Reviewed-by: Robert Richter Link: https://patch.msgid.link/452a9b0c525b774c72d9d5851515ffa928750132.1719980933.git.alison.schofield@intel.com Signed-off-by: Dave Jiang Signed-off-by: Bo Li --- drivers/cxl/core/core.h | 8 ++++---- drivers/cxl/core/mbox.c | 2 +- drivers/cxl/core/region.c | 33 +++++++++++++-------------------- drivers/cxl/core/trace.h | 4 ++-- 4 files changed, 20 insertions(+), 27 deletions(-) diff --git a/drivers/cxl/core/core.h b/drivers/cxl/core/core.h index 17ac9cb965622..933259c76b3f9 100644 --- a/drivers/cxl/core/core.h +++ b/drivers/cxl/core/core.h @@ -28,12 +28,12 @@ int cxl_region_init(void); void cxl_region_exit(void); int cxl_get_poison_by_endpoint(struct cxl_port *port); struct cxl_region *cxl_dpa_to_region(const struct cxl_memdev *cxlmd, u64 dpa); -u64 cxl_trace_hpa(struct cxl_region *cxlr, const struct cxl_memdev *cxlmd, - u64 dpa); +u64 cxl_dpa_to_hpa(struct cxl_region *cxlr, const struct cxl_memdev *cxlmd, + u64 dpa); #else -static inline u64 -cxl_trace_hpa(struct cxl_region *cxlr, const struct cxl_memdev *cxlmd, u64 dpa) +static inline u64 cxl_dpa_to_hpa(struct cxl_region *cxlr, + const struct cxl_memdev *cxlmd, u64 dpa) { return ULLONG_MAX; } diff --git a/drivers/cxl/core/mbox.c b/drivers/cxl/core/mbox.c index 97e938657773b..16f802c0ef9b9 100644 --- a/drivers/cxl/core/mbox.c +++ b/drivers/cxl/core/mbox.c @@ -866,7 +866,7 @@ void cxl_event_trace_record(const struct cxl_memdev *cxlmd, dpa = le64_to_cpu(evt->common.phys_addr) & CXL_DPA_MASK; cxlr = cxl_dpa_to_region(cxlmd, dpa); if (cxlr) - hpa = cxl_trace_hpa(cxlr, cxlmd, dpa); + hpa = cxl_dpa_to_hpa(cxlr, cxlmd, dpa); if (event_type == CXL_CPER_EVENT_GEN_MEDIA) trace_cxl_general_media(cxlmd, type, cxlr, hpa, diff --git a/drivers/cxl/core/region.c b/drivers/cxl/core/region.c index ecca8594ece6c..ec452be1aa62d 100644 --- a/drivers/cxl/core/region.c +++ b/drivers/cxl/core/region.c @@ -2856,15 +2856,25 @@ static bool cxl_is_hpa_in_range(u64 hpa, struct cxl_region *cxlr, int pos) return false; } -static u64 cxl_dpa_to_hpa(u64 dpa, struct cxl_region *cxlr, - struct cxl_endpoint_decoder *cxled) +u64 cxl_dpa_to_hpa(struct cxl_region *cxlr, const struct cxl_memdev *cxlmd, + u64 dpa) { u64 dpa_offset, hpa_offset, bits_upper, mask_upper, hpa; struct cxl_region_params *p = &cxlr->params; - int pos = cxled->pos; + struct cxl_endpoint_decoder *cxled = NULL; u16 eig = 0; u8 eiw = 0; + int pos; + + for (int i = 0; i < p->nr_targets; i++) { + cxled = p->targets[i]; + if (cxlmd == cxled_to_memdev(cxled)) + break; + } + if (!cxled || cxlmd != cxled_to_memdev(cxled)) + return ULLONG_MAX; + pos = cxled->pos; ways_to_eiw(p->interleave_ways, &eiw); granularity_to_eig(p->interleave_granularity, &eig); @@ -2904,23 +2914,6 @@ static u64 cxl_dpa_to_hpa(u64 dpa, struct cxl_region *cxlr, return hpa; } -u64 cxl_trace_hpa(struct cxl_region *cxlr, const struct cxl_memdev *cxlmd, - u64 dpa) -{ - struct cxl_region_params *p = &cxlr->params; - struct cxl_endpoint_decoder *cxled = NULL; - - for (int i = 0; i < p->nr_targets; i++) { - cxled = p->targets[i]; - if (cxlmd == cxled_to_memdev(cxled)) - break; - } - if (!cxled || cxlmd != cxled_to_memdev(cxled)) - return ULLONG_MAX; - - return cxl_dpa_to_hpa(dpa, cxlr, cxled); -} - static struct lock_class_key cxl_pmem_region_key; static struct cxl_pmem_region *cxl_pmem_region_alloc(struct cxl_region *cxlr) diff --git a/drivers/cxl/core/trace.h b/drivers/cxl/core/trace.h index 0c8f63e893dca..642b1ee4542ed 100644 --- a/drivers/cxl/core/trace.h +++ b/drivers/cxl/core/trace.h @@ -715,8 +715,8 @@ TRACE_EVENT(cxl_poison, if (cxlr) { __assign_str(region, dev_name(&cxlr->dev)); memcpy(__entry->uuid, &cxlr->params.uuid, 16); - __entry->hpa = cxl_trace_hpa(cxlr, cxlmd, - __entry->dpa); + __entry->hpa = cxl_dpa_to_hpa(cxlr, cxlmd, + __entry->dpa); } else { __assign_str(region, ""); memset(__entry->uuid, 0, 16); From d17bc7b42ddc56445d946c6e41f722e23baf09c2 Mon Sep 17 00:00:00 2001 From: Alison Schofield Date: Tue, 2 Jul 2024 22:29:50 -0700 Subject: [PATCH 124/138] cxl: Restore XOR'd position bits during address translation commit 3b2fedcd75e3991e77c2a8c3ebcab0ea68b2d69d upstream. When a device reports a DPA in events like poison, general_media, and dram, the driver translates that DPA back to an HPA. Presently, the CXL driver translation only considers the Modulo position and will report the wrong HPA for XOR configured root decoders. Add a helper function that restores the XOR'd bits during DPA->HPA address translation. Plumb a root decoder callback to the new helper when XOR interleave arithmetic is in use. For Modulo arithmetic, just let the callback be NULL - as in no extra work required. Upon completion of a DPA->HPA translation a couple of checks are performed on the result. One simply confirms that the calculated HPA is within the address range of the region. That test is useful for both Modulo and XOR interleave arithmetic decodes. A second check confirms that the HPA is within an expected chunk based on the endpoints position in the region and the region granularity. An XOR decode disrupts the Modulo pattern making the chunk check useless. To align the checks with the proper decode, pull the region range check inline and use the helper to do the chunk check for Modulo decodes only. A cxl-test unit test is posted for upstream review here: https://lore.kernel.org/20240624210644.495563-1-alison.schofield@intel.com/ Fixes: 28a3ae4ff66c ("cxl/trace: Add an HPA to cxl_poison trace events") Signed-off-by: Alison Schofield Tested-by: Diego Garcia Rodriguez Reviewed-by: Dan Williams Link: https://patch.msgid.link/1a1ac880d9f889bd6384e657e810431b9a0a72e5.1719980933.git.alison.schofield@intel.com Signed-off-by: Dave Jiang Signed-off-by: Bo Li --- drivers/cxl/acpi.c | 40 +++++++++++++++++++++++++++++++++++++++ drivers/cxl/core/region.c | 23 +++++++++++++--------- drivers/cxl/cxl.h | 3 +++ 3 files changed, 57 insertions(+), 9 deletions(-) diff --git a/drivers/cxl/acpi.c b/drivers/cxl/acpi.c index 2e4c070ef5f8c..e8c2121ad0b11 100644 --- a/drivers/cxl/acpi.c +++ b/drivers/cxl/acpi.c @@ -74,6 +74,43 @@ static struct cxl_dport *cxl_hb_xor(struct cxl_root_decoder *cxlrd, int pos) return cxlrd->cxlsd.target[n]; } +static u64 cxl_xor_hpa_to_spa(struct cxl_root_decoder *cxlrd, u64 hpa) +{ + struct cxl_cxims_data *cximsd = cxlrd->platform_data; + int hbiw = cxlrd->cxlsd.nr_targets; + u64 val; + int pos; + + /* No xormaps for host bridge interleave ways of 1 or 3 */ + if (hbiw == 1 || hbiw == 3) + return hpa; + + /* + * For root decoders using xormaps (hbiw: 2,4,6,8,12,16) restore + * the position bit to its value before the xormap was applied at + * HPA->DPA translation. + * + * pos is the lowest set bit in an XORMAP + * val is the XORALLBITS(HPA & XORMAP) + * + * XORALLBITS: The CXL spec (3.1 Table 9-22) defines XORALLBITS + * as an operation that outputs a single bit by XORing all the + * bits in the input (hpa & xormap). Implement XORALLBITS using + * hweight64(). If the hamming weight is even the XOR of those + * bits results in val==0, if odd the XOR result is val==1. + */ + + for (int i = 0; i < cximsd->nr_maps; i++) { + if (!cximsd->xormaps[i]) + continue; + pos = __ffs(cximsd->xormaps[i]); + val = (hweight64(hpa & cximsd->xormaps[i]) & 1); + hpa = (hpa & ~(1ULL << pos)) | (val << pos); + } + + return hpa; +} + struct cxl_cxims_context { struct device *dev; struct cxl_root_decoder *cxlrd; @@ -434,6 +471,9 @@ static int __cxl_parse_cfmws(struct acpi_cedt_cfmws *cfmws, cxlrd->qos_class = cfmws->qtg_id; + if (cfmws->interleave_arithmetic == ACPI_CEDT_CFMWS_ARITHMETIC_XOR) + cxlrd->hpa_to_spa = cxl_xor_hpa_to_spa; + rc = cxl_decoder_add(cxld, target_map); if (rc) return rc; diff --git a/drivers/cxl/core/region.c b/drivers/cxl/core/region.c index ec452be1aa62d..ebf7ddb53a6a5 100644 --- a/drivers/cxl/core/region.c +++ b/drivers/cxl/core/region.c @@ -2830,20 +2830,13 @@ struct cxl_region *cxl_dpa_to_region(const struct cxl_memdev *cxlmd, u64 dpa) return ctx.cxlr; } -static bool cxl_is_hpa_in_range(u64 hpa, struct cxl_region *cxlr, int pos) +static bool cxl_is_hpa_in_chunk(u64 hpa, struct cxl_region *cxlr, int pos) { struct cxl_region_params *p = &cxlr->params; int gran = p->interleave_granularity; int ways = p->interleave_ways; u64 offset; - /* Is the hpa within this region at all */ - if (hpa < p->res->start || hpa > p->res->end) { - dev_dbg(&cxlr->dev, - "Addr trans fail: hpa 0x%llx not in region\n", hpa); - return false; - } - /* Is the hpa in an expected chunk for its pos(-ition) */ offset = hpa - p->res->start; offset = do_div(offset, gran * ways); @@ -2859,6 +2852,7 @@ static bool cxl_is_hpa_in_range(u64 hpa, struct cxl_region *cxlr, int pos) u64 cxl_dpa_to_hpa(struct cxl_region *cxlr, const struct cxl_memdev *cxlmd, u64 dpa) { + struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent); u64 dpa_offset, hpa_offset, bits_upper, mask_upper, hpa; struct cxl_region_params *p = &cxlr->params; struct cxl_endpoint_decoder *cxled = NULL; @@ -2908,7 +2902,18 @@ u64 cxl_dpa_to_hpa(struct cxl_region *cxlr, const struct cxl_memdev *cxlmd, /* Apply the hpa_offset to the region base address */ hpa = hpa_offset + p->res->start; - if (!cxl_is_hpa_in_range(hpa, cxlr, cxled->pos)) + /* Root decoder translation overrides typical modulo decode */ + if (cxlrd->hpa_to_spa) + hpa = cxlrd->hpa_to_spa(cxlrd, hpa); + + if (hpa < p->res->start || hpa > p->res->end) { + dev_dbg(&cxlr->dev, + "Addr trans fail: hpa 0x%llx not in region\n", hpa); + return ULLONG_MAX; + } + + /* Simple chunk check, by pos & gran, only applies to modulo decodes */ + if (!cxlrd->hpa_to_spa && (!cxl_is_hpa_in_chunk(hpa, cxlr, pos))) return ULLONG_MAX; return hpa; diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h index 7b9fd201030fc..c59f36d6e0b4e 100644 --- a/drivers/cxl/cxl.h +++ b/drivers/cxl/cxl.h @@ -434,12 +434,14 @@ struct cxl_switch_decoder { struct cxl_root_decoder; typedef struct cxl_dport *(*cxl_calc_hb_fn)(struct cxl_root_decoder *cxlrd, int pos); +typedef u64 (*cxl_hpa_to_spa_fn)(struct cxl_root_decoder *cxlrd, u64 hpa); /** * struct cxl_root_decoder - Static platform CXL address decoder * @res: host / parent resource for region allocations * @region_id: region id for next region provisioning event * @calc_hb: which host bridge covers the n'th position by granularity + * @hpa_to_spa: translate CXL host-physical-address to Platform system-physical-address * @platform_data: platform specific configuration data * @range_lock: sync region autodiscovery by address range * @qos_class: QoS performance class cookie @@ -449,6 +451,7 @@ struct cxl_root_decoder { struct resource *res; atomic_t region_id; cxl_calc_hb_fn calc_hb; + cxl_hpa_to_spa_fn hpa_to_spa; void *platform_data; struct mutex range_lock; int qos_class; From 6091f330e0b84a9b24b6a786751c084d891f4441 Mon Sep 17 00:00:00 2001 From: Alison Schofield Date: Tue, 2 Jul 2024 22:29:52 -0700 Subject: [PATCH 125/138] cxl: Remove defunct code calculating host bridge target positions commit 8f55ada796565ce801418bf579f31a6a522d0337 upstream. The CXL Spec 3.1 Table 9-22 requires that the BIOS populate the CFMWS target list in interleave target order. This means the calculations the CXL driver added to determine positions when XOR math is in use, along with the entire XOR vs Modulo call back setup is not needed. A prior patch added a common method to verify positions. Remove the now unused code related to the cxl_calc_hb_fn. Signed-off-by: Alison Schofield Reviewed-by: Dan Williams Reviewed-by: Jonathan Cameron Link: https://patch.msgid.link/2e2c32a2d0f1007e920b58712d15edad2e48d857.1719980933.git.alison.schofield@intel.com Signed-off-by: Dave Jiang Signed-off-by: Bo Li --- drivers/cxl/acpi.c | 60 ++--------------------------------------- drivers/cxl/core/port.c | 20 +------------- drivers/cxl/cxl.h | 8 +----- 3 files changed, 4 insertions(+), 84 deletions(-) diff --git a/drivers/cxl/acpi.c b/drivers/cxl/acpi.c index e8c2121ad0b11..2b19b7bedece1 100644 --- a/drivers/cxl/acpi.c +++ b/drivers/cxl/acpi.c @@ -22,57 +22,6 @@ static const guid_t acpi_cxl_qtg_id_guid = GUID_INIT(0xF365F9A6, 0xA7DE, 0x4071, 0xA6, 0x6A, 0xB4, 0x0C, 0x0B, 0x4F, 0x8E, 0x52); -/* - * Find a targets entry (n) in the host bridge interleave list. - * CXL Specification 3.0 Table 9-22 - */ -static int cxl_xor_calc_n(u64 hpa, struct cxl_cxims_data *cximsd, int iw, - int ig) -{ - int i = 0, n = 0; - u8 eiw; - - /* IW: 2,4,6,8,12,16 begin building 'n' using xormaps */ - if (iw != 3) { - for (i = 0; i < cximsd->nr_maps; i++) - n |= (hweight64(hpa & cximsd->xormaps[i]) & 1) << i; - } - /* IW: 3,6,12 add a modulo calculation to 'n' */ - if (!is_power_of_2(iw)) { - if (ways_to_eiw(iw, &eiw)) - return -1; - hpa &= GENMASK_ULL(51, eiw + ig); - n |= do_div(hpa, 3) << i; - } - return n; -} - -static struct cxl_dport *cxl_hb_xor(struct cxl_root_decoder *cxlrd, int pos) -{ - struct cxl_cxims_data *cximsd = cxlrd->platform_data; - struct cxl_switch_decoder *cxlsd = &cxlrd->cxlsd; - struct cxl_decoder *cxld = &cxlsd->cxld; - int ig = cxld->interleave_granularity; - int iw = cxld->interleave_ways; - int n = 0; - u64 hpa; - - if (dev_WARN_ONCE(&cxld->dev, - cxld->interleave_ways != cxlsd->nr_targets, - "misconfigured root decoder\n")) - return NULL; - - hpa = cxlrd->res->start + pos * ig; - - /* Entry (n) is 0 for no interleave (iw == 1) */ - if (iw != 1) - n = cxl_xor_calc_n(hpa, cximsd, iw, ig); - - if (n < 0) - return NULL; - - return cxlrd->cxlsd.target[n]; -} static u64 cxl_xor_hpa_to_spa(struct cxl_root_decoder *cxlrd, u64 hpa) { @@ -398,7 +347,6 @@ static int __cxl_parse_cfmws(struct acpi_cedt_cfmws *cfmws, struct cxl_port *root_port = ctx->root_port; struct cxl_cxims_context cxims_ctx; struct device *dev = ctx->dev; - cxl_calc_hb_fn cxl_calc_hb; struct cxl_decoder *cxld; unsigned int ways, i, ig; int rc; @@ -426,13 +374,9 @@ static int __cxl_parse_cfmws(struct acpi_cedt_cfmws *cfmws, if (rc) return rc; - if (cfmws->interleave_arithmetic == ACPI_CEDT_CFMWS_ARITHMETIC_MODULO) - cxl_calc_hb = cxl_hb_modulo; - else - cxl_calc_hb = cxl_hb_xor; - struct cxl_root_decoder *cxlrd __free(put_cxlrd) = - cxl_root_decoder_alloc(root_port, ways, cxl_calc_hb); + cxl_root_decoder_alloc(root_port, ways); + if (IS_ERR(cxlrd)) return PTR_ERR(cxlrd); diff --git a/drivers/cxl/core/port.c b/drivers/cxl/core/port.c index a90dff590154f..3218e9d3614ba 100644 --- a/drivers/cxl/core/port.c +++ b/drivers/cxl/core/port.c @@ -1737,21 +1737,6 @@ static int decoder_populate_targets(struct cxl_switch_decoder *cxlsd, return 0; } -struct cxl_dport *cxl_hb_modulo(struct cxl_root_decoder *cxlrd, int pos) -{ - struct cxl_switch_decoder *cxlsd = &cxlrd->cxlsd; - struct cxl_decoder *cxld = &cxlsd->cxld; - int iw; - - iw = cxld->interleave_ways; - if (dev_WARN_ONCE(&cxld->dev, iw != cxlsd->nr_targets, - "misconfigured root decoder\n")) - return NULL; - - return cxlrd->cxlsd.target[pos % iw]; -} -EXPORT_SYMBOL_NS_GPL(cxl_hb_modulo, CXL); - static struct lock_class_key cxl_decoder_key; /** @@ -1811,7 +1796,6 @@ static int cxl_switch_decoder_init(struct cxl_port *port, * cxl_root_decoder_alloc - Allocate a root level decoder * @port: owning CXL root of this decoder * @nr_targets: static number of downstream targets - * @calc_hb: which host bridge covers the n'th position by granularity * * Return: A new cxl decoder to be registered by cxl_decoder_add(). A * 'CXL root' decoder is one that decodes from a top-level / static platform @@ -1819,8 +1803,7 @@ static int cxl_switch_decoder_init(struct cxl_port *port, * topology. */ struct cxl_root_decoder *cxl_root_decoder_alloc(struct cxl_port *port, - unsigned int nr_targets, - cxl_calc_hb_fn calc_hb) + unsigned int nr_targets) { struct cxl_root_decoder *cxlrd; struct cxl_switch_decoder *cxlsd; @@ -1842,7 +1825,6 @@ struct cxl_root_decoder *cxl_root_decoder_alloc(struct cxl_port *port, return ERR_PTR(rc); } - cxlrd->calc_hb = calc_hb; mutex_init(&cxlrd->range_lock); cxld = &cxlsd->cxld; diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h index c59f36d6e0b4e..0f8f3aa4d378b 100644 --- a/drivers/cxl/cxl.h +++ b/drivers/cxl/cxl.h @@ -432,15 +432,12 @@ struct cxl_switch_decoder { }; struct cxl_root_decoder; -typedef struct cxl_dport *(*cxl_calc_hb_fn)(struct cxl_root_decoder *cxlrd, - int pos); typedef u64 (*cxl_hpa_to_spa_fn)(struct cxl_root_decoder *cxlrd, u64 hpa); /** * struct cxl_root_decoder - Static platform CXL address decoder * @res: host / parent resource for region allocations * @region_id: region id for next region provisioning event - * @calc_hb: which host bridge covers the n'th position by granularity * @hpa_to_spa: translate CXL host-physical-address to Platform system-physical-address * @platform_data: platform specific configuration data * @range_lock: sync region autodiscovery by address range @@ -450,7 +447,6 @@ typedef u64 (*cxl_hpa_to_spa_fn)(struct cxl_root_decoder *cxlrd, u64 hpa); struct cxl_root_decoder { struct resource *res; atomic_t region_id; - cxl_calc_hb_fn calc_hb; cxl_hpa_to_spa_fn hpa_to_spa; void *platform_data; struct mutex range_lock; @@ -776,9 +772,7 @@ bool is_root_decoder(struct device *dev); bool is_switch_decoder(struct device *dev); bool is_endpoint_decoder(struct device *dev); struct cxl_root_decoder *cxl_root_decoder_alloc(struct cxl_port *port, - unsigned int nr_targets, - cxl_calc_hb_fn calc_hb); -struct cxl_dport *cxl_hb_modulo(struct cxl_root_decoder *cxlrd, int pos); + unsigned int nr_targets); struct cxl_switch_decoder *cxl_switch_decoder_alloc(struct cxl_port *port, unsigned int nr_targets); int cxl_decoder_add(struct cxl_decoder *cxld, int *target_map); From e5aa5627f23a55705244984c9d45e31e36e9dc97 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Thu, 5 Sep 2024 15:35:45 -0700 Subject: [PATCH 126/138] cxl: move cxl headers to new include/cxl/ directory commit 40a895fd9a358eea16901026360bd2cd3ca691a7 upstream. Group all cxl related kernel headers into include/cxl/ directory. Reviewed-by: Alison Schofield Reviewed-by: Ira Weiny Link: https://patch.msgid.link/20240905223711.1990186-2-dave.jiang@intel.com Signed-off-by: Dave Jiang --- MAINTAINERS | 3 +-- drivers/acpi/apei/einj-cxl.c | 2 +- drivers/acpi/apei/ghes.c | 2 +- drivers/cxl/core/port.c | 2 +- drivers/cxl/cxlmem.h | 2 +- include/{linux/einj-cxl.h => cxl/einj.h} | 0 include/{linux/cxl-event.h => cxl/event.h} | 0 7 files changed, 5 insertions(+), 6 deletions(-) rename include/{linux/einj-cxl.h => cxl/einj.h} (100%) rename include/{linux/cxl-event.h => cxl/event.h} (100%) diff --git a/MAINTAINERS b/MAINTAINERS index cefe1aaa35951..f31d9a3471bfa 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -5224,8 +5224,7 @@ M: Dan Williams L: linux-cxl@vger.kernel.org S: Maintained F: drivers/cxl/ -F: include/linux/einj-cxl.h -F: include/linux/cxl-event.h +F: include/cxl/ F: include/uapi/linux/cxl_mem.h F: tools/testing/cxl/ diff --git a/drivers/acpi/apei/einj-cxl.c b/drivers/acpi/apei/einj-cxl.c index d64e2713aae4b..a4e709937236f 100644 --- a/drivers/acpi/apei/einj-cxl.c +++ b/drivers/acpi/apei/einj-cxl.c @@ -7,9 +7,9 @@ * * Author: Ben Cheatham */ -#include #include #include +#include #include "apei-internal.h" diff --git a/drivers/acpi/apei/ghes.c b/drivers/acpi/apei/ghes.c index 98c5e309445bb..c769000fec4d3 100644 --- a/drivers/acpi/apei/ghes.c +++ b/drivers/acpi/apei/ghes.c @@ -28,7 +28,6 @@ #include #include #include -#include #include #include #include @@ -51,6 +50,7 @@ #include #include #include +#include #include #include "apei-internal.h" diff --git a/drivers/cxl/core/port.c b/drivers/cxl/core/port.c index 3218e9d3614ba..23818fcc924aa 100644 --- a/drivers/cxl/core/port.c +++ b/drivers/cxl/core/port.c @@ -3,7 +3,6 @@ #include #include #include -#include #include #include #include @@ -11,6 +10,7 @@ #include #include #include +#include #include #include #include diff --git a/drivers/cxl/cxlmem.h b/drivers/cxl/cxlmem.h index b901b6ad45de5..e0349e16b424a 100644 --- a/drivers/cxl/cxlmem.h +++ b/drivers/cxl/cxlmem.h @@ -7,7 +7,7 @@ #include #include #include -#include +#include #include "cxl.h" /* CXL 2.0 8.2.8.5.1.1 Memory Device Status Register */ diff --git a/include/linux/einj-cxl.h b/include/cxl/einj.h similarity index 100% rename from include/linux/einj-cxl.h rename to include/cxl/einj.h diff --git a/include/linux/cxl-event.h b/include/cxl/event.h similarity index 100% rename from include/linux/cxl-event.h rename to include/cxl/event.h From d517abb76ed5a4e42bee24126b5f50f17bac5ec6 Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Fri, 29 Mar 2024 16:48:48 -0700 Subject: [PATCH 127/138] cleanup: Add usage and style documentation commit d5934e76316e84eced836b6b2bafae1837d1cd58 upstream. When proposing that PCI grow some new cleanup helpers for pci_dev_put() and pci_dev_{lock,unlock} [1], Bjorn had some fundamental questions about expectations and best practices. Upon reviewing an updated changelog with those details he recommended adding them to documentation in the header file itself. Add that documentation and link it into the rendering for Documentation/core-api/. Signed-off-by: Dan Williams Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Jonathan Cameron Reviewed-by: Kevin Tian Link: https://lore.kernel.org/r/171175585714.2192972.12661675876300167762.stgit@dwillia2-xfh.jf.intel.com Signed-off-by: Bo Li --- Documentation/core-api/cleanup.rst | 8 ++ Documentation/core-api/index.rst | 1 + include/linux/cleanup.h | 136 +++++++++++++++++++++++++++++ 3 files changed, 145 insertions(+) create mode 100644 Documentation/core-api/cleanup.rst diff --git a/Documentation/core-api/cleanup.rst b/Documentation/core-api/cleanup.rst new file mode 100644 index 0000000000000..527eb2f8ec6ef --- /dev/null +++ b/Documentation/core-api/cleanup.rst @@ -0,0 +1,8 @@ +.. SPDX-License-Identifier: GPL-2.0 + +=========================== +Scope-based Cleanup Helpers +=========================== + +.. kernel-doc:: include/linux/cleanup.h + :doc: scope-based cleanup helpers diff --git a/Documentation/core-api/index.rst b/Documentation/core-api/index.rst index 7a3a08d81f111..2d2b3719567e6 100644 --- a/Documentation/core-api/index.rst +++ b/Documentation/core-api/index.rst @@ -35,6 +35,7 @@ Library functionality that is used throughout the kernel. kobject kref + cleanup assoc_array xarray maple_tree diff --git a/include/linux/cleanup.h b/include/linux/cleanup.h index 64b8600eb8c0e..16ce687d1fe1d 100644 --- a/include/linux/cleanup.h +++ b/include/linux/cleanup.h @@ -4,6 +4,142 @@ #include +/** + * DOC: scope-based cleanup helpers + * + * The "goto error" pattern is notorious for introducing subtle resource + * leaks. It is tedious and error prone to add new resource acquisition + * constraints into code paths that already have several unwind + * conditions. The "cleanup" helpers enable the compiler to help with + * this tedium and can aid in maintaining LIFO (last in first out) + * unwind ordering to avoid unintentional leaks. + * + * As drivers make up the majority of the kernel code base, here is an + * example of using these helpers to clean up PCI drivers. The target of + * the cleanups are occasions where a goto is used to unwind a device + * reference (pci_dev_put()), or unlock the device (pci_dev_unlock()) + * before returning. + * + * The DEFINE_FREE() macro can arrange for PCI device references to be + * dropped when the associated variable goes out of scope:: + * + * DEFINE_FREE(pci_dev_put, struct pci_dev *, if (_T) pci_dev_put(_T)) + * ... + * struct pci_dev *dev __free(pci_dev_put) = + * pci_get_slot(parent, PCI_DEVFN(0, 0)); + * + * The above will automatically call pci_dev_put() if @dev is non-NULL + * when @dev goes out of scope (automatic variable scope). If a function + * wants to invoke pci_dev_put() on error, but return @dev (i.e. without + * freeing it) on success, it can do:: + * + * return no_free_ptr(dev); + * + * ...or:: + * + * return_ptr(dev); + * + * The DEFINE_GUARD() macro can arrange for the PCI device lock to be + * dropped when the scope where guard() is invoked ends:: + * + * DEFINE_GUARD(pci_dev, struct pci_dev *, pci_dev_lock(_T), pci_dev_unlock(_T)) + * ... + * guard(pci_dev)(dev); + * + * The lifetime of the lock obtained by the guard() helper follows the + * scope of automatic variable declaration. Take the following example:: + * + * func(...) + * { + * if (...) { + * ... + * guard(pci_dev)(dev); // pci_dev_lock() invoked here + * ... + * } // <- implied pci_dev_unlock() triggered here + * } + * + * Observe the lock is held for the remainder of the "if ()" block not + * the remainder of "func()". + * + * Now, when a function uses both __free() and guard(), or multiple + * instances of __free(), the LIFO order of variable definition order + * matters. GCC documentation says: + * + * "When multiple variables in the same scope have cleanup attributes, + * at exit from the scope their associated cleanup functions are run in + * reverse order of definition (last defined, first cleanup)." + * + * When the unwind order matters it requires that variables be defined + * mid-function scope rather than at the top of the file. Take the + * following example and notice the bug highlighted by "!!":: + * + * LIST_HEAD(list); + * DEFINE_MUTEX(lock); + * + * struct object { + * struct list_head node; + * }; + * + * static struct object *alloc_add(void) + * { + * struct object *obj; + * + * lockdep_assert_held(&lock); + * obj = kzalloc(sizeof(*obj), GFP_KERNEL); + * if (obj) { + * LIST_HEAD_INIT(&obj->node); + * list_add(obj->node, &list): + * } + * return obj; + * } + * + * static void remove_free(struct object *obj) + * { + * lockdep_assert_held(&lock); + * list_del(&obj->node); + * kfree(obj); + * } + * + * DEFINE_FREE(remove_free, struct object *, if (_T) remove_free(_T)) + * static int init(void) + * { + * struct object *obj __free(remove_free) = NULL; + * int err; + * + * guard(mutex)(&lock); + * obj = alloc_add(); + * + * if (!obj) + * return -ENOMEM; + * + * err = other_init(obj); + * if (err) + * return err; // remove_free() called without the lock!! + * + * no_free_ptr(obj); + * return 0; + * } + * + * That bug is fixed by changing init() to call guard() and define + + * initialize @obj in this order:: + * + * guard(mutex)(&lock); + * struct object *obj __free(remove_free) = alloc_add(); + * + * Given that the "__free(...) = NULL" pattern for variables defined at + * the top of the function poses this potential interdependency problem + * the recommendation is to always define and assign variables in one + * statement and not group variable definitions at the top of the + * function when __free() is used. + * + * Lastly, given that the benefit of cleanup helpers is removal of + * "goto", and that the "goto" statement can jump between scopes, the + * expectation is that usage of "goto" and cleanup helpers is never + * mixed in the same function. I.e. for a given routine, convert all + * resources that need a "goto" cleanup to scope-based cleanup, or + * convert none of them. + */ + /* * DEFINE_FREE(name, type, free): * simple helper macro that defines the required wrapper for a __free() From 9143d72a34b6ca07310963a672294aa515a47679 Mon Sep 17 00:00:00 2001 From: Smita Koralahalli Date: Thu, 23 Jan 2025 08:44:16 +0000 Subject: [PATCH 128/138] efi/cper, cxl: Prefix protocol error struct and function names with cxl_ commit 84973331442a57bac31b40eaef6f264496c6f3fd upstream. Rename the protocol error struct from struct cper_sec_prot_err to struct cxl_cper_sec_prot_err and cper_print_prot_err() to cxl_cper_print_prot_err() to maintain naming consistency. No functional changes. Signed-off-by: Smita Koralahalli Reviewed-by: Jonathan Cameron Reviewed-by: Ira Weiny Reviewed-by: Dave Jiang Reviewed-by: Fan Ni Reviewed-by: Gregory Price Reviewed-by: Dan Williams Link: https://patch.msgid.link/20250123084421.127697-2-Smita.KoralahalliChannabasappa@amd.com Signed-off-by: Dave Jiang Signed-off-by: Bo Li --- drivers/firmware/efi/cper.c | 4 ++-- drivers/firmware/efi/cper_cxl.c | 3 ++- drivers/firmware/efi/cper_cxl.h | 5 +++-- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/drivers/firmware/efi/cper.c b/drivers/firmware/efi/cper.c index 481af0b6f22c4..d3d9b9a217dc1 100644 --- a/drivers/firmware/efi/cper.c +++ b/drivers/firmware/efi/cper.c @@ -679,11 +679,11 @@ cper_estatus_print_section(const char *pfx, struct acpi_hest_generic_data *gdata else goto err_section_too_small; } else if (guid_equal(sec_type, &CPER_SEC_CXL_PROT_ERR)) { - struct cper_sec_prot_err *prot_err = acpi_hest_get_payload(gdata); + struct cxl_cper_sec_prot_err *prot_err = acpi_hest_get_payload(gdata); printk("%ssection_type: CXL Protocol Error\n", newpfx); if (gdata->error_data_length >= sizeof(*prot_err)) - cper_print_prot_err(newpfx, prot_err); + cxl_cper_print_prot_err(newpfx, prot_err); else goto err_section_too_small; } else { diff --git a/drivers/firmware/efi/cper_cxl.c b/drivers/firmware/efi/cper_cxl.c index a55771b99a97a..cbaabcb7382d9 100644 --- a/drivers/firmware/efi/cper_cxl.c +++ b/drivers/firmware/efi/cper_cxl.c @@ -55,7 +55,8 @@ enum { USP, /* CXL Upstream Switch Port */ }; -void cper_print_prot_err(const char *pfx, const struct cper_sec_prot_err *prot_err) +void cxl_cper_print_prot_err(const char *pfx, + const struct cxl_cper_sec_prot_err *prot_err) { if (prot_err->valid_bits & PROT_ERR_VALID_AGENT_TYPE) pr_info("%s agent_type: %d, %s\n", pfx, prot_err->agent_type, diff --git a/drivers/firmware/efi/cper_cxl.h b/drivers/firmware/efi/cper_cxl.h index 86bfcf7909eca..0e3ab0ba17c36 100644 --- a/drivers/firmware/efi/cper_cxl.h +++ b/drivers/firmware/efi/cper_cxl.h @@ -18,7 +18,7 @@ #pragma pack(1) /* Compute Express Link Protocol Error Section, UEFI v2.10 sec N.2.13 */ -struct cper_sec_prot_err { +struct cxl_cper_sec_prot_err { u64 valid_bits; u8 agent_type; u8 reserved[7]; @@ -61,6 +61,7 @@ struct cper_sec_prot_err { #pragma pack() -void cper_print_prot_err(const char *pfx, const struct cper_sec_prot_err *prot_err); +void cxl_cper_print_prot_err(const char *pfx, + const struct cxl_cper_sec_prot_err *prot_err); #endif //__CPER_CXL_ From 3ec896148b654b9983aeea9f77d9df0bfbdcac61 Mon Sep 17 00:00:00 2001 From: Smita Koralahalli Date: Thu, 23 Jan 2025 08:44:17 +0000 Subject: [PATCH 129/138] efi/cper, cxl: Make definitions and structures global commit 958c3a6706863f9f77b21c90d2428473441cd8a1 upstream. In preparation to add tracepoint support, move protocol error UUID definition to a common location, Also, make struct CXL RAS capability, cxl_cper_sec_prot_err and CPER validation flags global for use across different modules. Signed-off-by: Smita Koralahalli Reviewed-by: Jonathan Cameron Reviewed-by: Ira Weiny Reviewed-by: Dave Jiang Reviewed-by: Fan Ni Reviewed-by: Gregory Price Reviewed-by: Dan Williams Link: https://patch.msgid.link/20250123084421.127697-3-Smita.KoralahalliChannabasappa@amd.com Signed-off-by: Dave Jiang Signed-off-by: Bo Li --- drivers/firmware/efi/cper.c | 1 + drivers/firmware/efi/cper_cxl.c | 35 +-------------- drivers/firmware/efi/cper_cxl.h | 51 --------------------- include/cxl/event.h | 80 +++++++++++++++++++++++++++++++++ include/linux/cper.h | 4 ++ 5 files changed, 86 insertions(+), 85 deletions(-) diff --git a/drivers/firmware/efi/cper.c b/drivers/firmware/efi/cper.c index d3d9b9a217dc1..a27a51c53cbf8 100644 --- a/drivers/firmware/efi/cper.c +++ b/drivers/firmware/efi/cper.c @@ -25,6 +25,7 @@ #include #include #include +#include #include "cper_cxl.h" /* diff --git a/drivers/firmware/efi/cper_cxl.c b/drivers/firmware/efi/cper_cxl.c index cbaabcb7382d9..64c0dd27be6ed 100644 --- a/drivers/firmware/efi/cper_cxl.c +++ b/drivers/firmware/efi/cper_cxl.c @@ -8,27 +8,9 @@ */ #include +#include #include "cper_cxl.h" -#define PROT_ERR_VALID_AGENT_TYPE BIT_ULL(0) -#define PROT_ERR_VALID_AGENT_ADDRESS BIT_ULL(1) -#define PROT_ERR_VALID_DEVICE_ID BIT_ULL(2) -#define PROT_ERR_VALID_SERIAL_NUMBER BIT_ULL(3) -#define PROT_ERR_VALID_CAPABILITY BIT_ULL(4) -#define PROT_ERR_VALID_DVSEC BIT_ULL(5) -#define PROT_ERR_VALID_ERROR_LOG BIT_ULL(6) - -/* CXL RAS Capability Structure, CXL v3.0 sec 8.2.4.16 */ -struct cxl_ras_capability_regs { - u32 uncor_status; - u32 uncor_mask; - u32 uncor_severity; - u32 cor_status; - u32 cor_mask; - u32 cap_control; - u32 header_log[16]; -}; - static const char * const prot_err_agent_type_strs[] = { "Restricted CXL Device", "Restricted CXL Host Downstream Port", @@ -40,21 +22,6 @@ static const char * const prot_err_agent_type_strs[] = { "CXL Upstream Switch Port", }; -/* - * The layout of the enumeration and the values matches CXL Agent Type - * field in the UEFI 2.10 Section N.2.13, - */ -enum { - RCD, /* Restricted CXL Device */ - RCH_DP, /* Restricted CXL Host Downstream Port */ - DEVICE, /* CXL Device */ - LD, /* CXL Logical Device */ - FMLD, /* CXL Fabric Manager managed Logical Device */ - RP, /* CXL Root Port */ - DSP, /* CXL Downstream Switch Port */ - USP, /* CXL Upstream Switch Port */ -}; - void cxl_cper_print_prot_err(const char *pfx, const struct cxl_cper_sec_prot_err *prot_err) { diff --git a/drivers/firmware/efi/cper_cxl.h b/drivers/firmware/efi/cper_cxl.h index 0e3ab0ba17c36..5ce1401ee17af 100644 --- a/drivers/firmware/efi/cper_cxl.h +++ b/drivers/firmware/efi/cper_cxl.h @@ -10,57 +10,6 @@ #ifndef LINUX_CPER_CXL_H #define LINUX_CPER_CXL_H -/* CXL Protocol Error Section */ -#define CPER_SEC_CXL_PROT_ERR \ - GUID_INIT(0x80B9EFB4, 0x52B5, 0x4DE3, 0xA7, 0x77, 0x68, 0x78, \ - 0x4B, 0x77, 0x10, 0x48) - -#pragma pack(1) - -/* Compute Express Link Protocol Error Section, UEFI v2.10 sec N.2.13 */ -struct cxl_cper_sec_prot_err { - u64 valid_bits; - u8 agent_type; - u8 reserved[7]; - - /* - * Except for RCH Downstream Port, all the remaining CXL Agent - * types are uniquely identified by the PCIe compatible SBDF number. - */ - union { - u64 rcrb_base_addr; - struct { - u8 function; - u8 device; - u8 bus; - u16 segment; - u8 reserved_1[3]; - }; - } agent_addr; - - struct { - u16 vendor_id; - u16 device_id; - u16 subsystem_vendor_id; - u16 subsystem_id; - u8 class_code[2]; - u16 slot; - u8 reserved_1[4]; - } device_id; - - struct { - u32 lower_dw; - u32 upper_dw; - } dev_serial_num; - - u8 capability[60]; - u16 dvsec_len; - u16 err_len; - u8 reserved_2[4]; -}; - -#pragma pack() - void cxl_cper_print_prot_err(const char *pfx, const struct cxl_cper_sec_prot_err *prot_err); diff --git a/include/cxl/event.h b/include/cxl/event.h index 67a35d44cdf21..dca6fcce5a07c 100644 --- a/include/cxl/event.h +++ b/include/cxl/event.h @@ -157,6 +157,86 @@ struct cxl_cper_work_data { struct cxl_cper_event_rec rec; }; +#define PROT_ERR_VALID_AGENT_TYPE BIT_ULL(0) +#define PROT_ERR_VALID_AGENT_ADDRESS BIT_ULL(1) +#define PROT_ERR_VALID_DEVICE_ID BIT_ULL(2) +#define PROT_ERR_VALID_SERIAL_NUMBER BIT_ULL(3) +#define PROT_ERR_VALID_CAPABILITY BIT_ULL(4) +#define PROT_ERR_VALID_DVSEC BIT_ULL(5) +#define PROT_ERR_VALID_ERROR_LOG BIT_ULL(6) + +/* + * The layout of the enumeration and the values matches CXL Agent Type + * field in the UEFI 2.10 Section N.2.13, + */ +enum { + RCD, /* Restricted CXL Device */ + RCH_DP, /* Restricted CXL Host Downstream Port */ + DEVICE, /* CXL Device */ + LD, /* CXL Logical Device */ + FMLD, /* CXL Fabric Manager managed Logical Device */ + RP, /* CXL Root Port */ + DSP, /* CXL Downstream Switch Port */ + USP, /* CXL Upstream Switch Port */ +}; + +#pragma pack(1) + +/* Compute Express Link Protocol Error Section, UEFI v2.10 sec N.2.13 */ +struct cxl_cper_sec_prot_err { + u64 valid_bits; + u8 agent_type; + u8 reserved[7]; + + /* + * Except for RCH Downstream Port, all the remaining CXL Agent + * types are uniquely identified by the PCIe compatible SBDF number. + */ + union { + u64 rcrb_base_addr; + struct { + u8 function; + u8 device; + u8 bus; + u16 segment; + u8 reserved_1[3]; + }; + } agent_addr; + + struct { + u16 vendor_id; + u16 device_id; + u16 subsystem_vendor_id; + u16 subsystem_id; + u8 class_code[2]; + u16 slot; + u8 reserved_1[4]; + } device_id; + + struct { + u32 lower_dw; + u32 upper_dw; + } dev_serial_num; + + u8 capability[60]; + u16 dvsec_len; + u16 err_len; + u8 reserved_2[4]; +}; + +#pragma pack() + +/* CXL RAS Capability Structure, CXL v3.0 sec 8.2.4.16 */ +struct cxl_ras_capability_regs { + u32 uncor_status; + u32 uncor_mask; + u32 uncor_severity; + u32 cor_status; + u32 cor_mask; + u32 cap_control; + u32 header_log[16]; +}; + #ifdef CONFIG_ACPI_APEI_GHES int cxl_cper_register_work(struct work_struct *work); int cxl_cper_unregister_work(struct work_struct *work); diff --git a/include/linux/cper.h b/include/linux/cper.h index 3670b866ac119..0465f47ea16a5 100644 --- a/include/linux/cper.h +++ b/include/linux/cper.h @@ -89,6 +89,10 @@ enum { #define CPER_NOTIFY_DMAR \ GUID_INIT(0x667DD791, 0xC6B3, 0x4c27, 0x8A, 0x6B, 0x0F, 0x8E, \ 0x72, 0x2D, 0xEB, 0x41) +/* CXL Protocol Error Section */ +#define CPER_SEC_CXL_PROT_ERR \ + GUID_INIT(0x80B9EFB4, 0x52B5, 0x4DE3, 0xA7, 0x77, 0x68, 0x78, \ + 0x4B, 0x77, 0x10, 0x48) /* CXL Event record UUIDs are formatted as GUIDs and reported in section type */ /* From d5d37a3603da36e3249d651c810ff3c3ded776ce Mon Sep 17 00:00:00 2001 From: Smita Koralahalli Date: Thu, 23 Jan 2025 08:44:18 +0000 Subject: [PATCH 130/138] efi/cper, cxl: Remove cper_cxl.h commit 61eac5f7f6439e8fe99b5fb29406acb0fd7b83c6 upstream. Move the declaration of cxl_cper_print_prot_err() to include/linux/cper.h to avoid maintaining a separate header file just for this function declaration. Remove drivers/firmware/efi/cper_cxl.h as its contents have been reorganized. No functional changes. Signed-off-by: Smita Koralahalli Reviewed-by: Ira Weiny Reviewed-by: Jonathan Cameron Reviewed-by: Dave Jiang Reviewed-by: Fan Ni Reviewed-by: Gregory Price Reviewed-by: Dan Williams Link: https://patch.msgid.link/20250123084421.127697-4-Smita.KoralahalliChannabasappa@amd.com Signed-off-by: Dave Jiang Signed-off-by: Bo Li --- drivers/firmware/efi/cper.c | 1 - drivers/firmware/efi/cper_cxl.c | 1 - drivers/firmware/efi/cper_cxl.h | 16 ---------------- include/linux/cper.h | 4 ++++ 4 files changed, 4 insertions(+), 18 deletions(-) delete mode 100644 drivers/firmware/efi/cper_cxl.h diff --git a/drivers/firmware/efi/cper.c b/drivers/firmware/efi/cper.c index a27a51c53cbf8..52bf54af50e1f 100644 --- a/drivers/firmware/efi/cper.c +++ b/drivers/firmware/efi/cper.c @@ -26,7 +26,6 @@ #include #include #include -#include "cper_cxl.h" /* * CPER record ID need to be unique even after reboot, because record diff --git a/drivers/firmware/efi/cper_cxl.c b/drivers/firmware/efi/cper_cxl.c index 64c0dd27be6ed..8a7667faf9534 100644 --- a/drivers/firmware/efi/cper_cxl.c +++ b/drivers/firmware/efi/cper_cxl.c @@ -9,7 +9,6 @@ #include #include -#include "cper_cxl.h" static const char * const prot_err_agent_type_strs[] = { "Restricted CXL Device", diff --git a/drivers/firmware/efi/cper_cxl.h b/drivers/firmware/efi/cper_cxl.h deleted file mode 100644 index 5ce1401ee17af..0000000000000 --- a/drivers/firmware/efi/cper_cxl.h +++ /dev/null @@ -1,16 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * UEFI Common Platform Error Record (CPER) support for CXL Section. - * - * Copyright (C) 2022 Advanced Micro Devices, Inc. - * - * Author: Smita Koralahalli - */ - -#ifndef LINUX_CPER_CXL_H -#define LINUX_CPER_CXL_H - -void cxl_cper_print_prot_err(const char *pfx, - const struct cxl_cper_sec_prot_err *prot_err); - -#endif //__CPER_CXL_ diff --git a/include/linux/cper.h b/include/linux/cper.h index 0465f47ea16a5..5b1236d8c65bb 100644 --- a/include/linux/cper.h +++ b/include/linux/cper.h @@ -607,4 +607,8 @@ void cper_estatus_print(const char *pfx, int cper_estatus_check_header(const struct acpi_hest_generic_status *estatus); int cper_estatus_check(const struct acpi_hest_generic_status *estatus); +struct cxl_cper_sec_prot_err; +void cxl_cper_print_prot_err(const char *pfx, + const struct cxl_cper_sec_prot_err *prot_err); + #endif From e1ce09550f1603f1bba8155774ed759c5265d847 Mon Sep 17 00:00:00 2001 From: Smita Koralahalli Date: Thu, 23 Jan 2025 08:44:19 +0000 Subject: [PATCH 131/138] acpi/ghes, cper: Recognize and cache CXL Protocol errors commit 315c2f0b53ba2645062627443a12cea73f3dad9c upstream. Add support in GHES to detect and process CXL CPER Protocol errors, as defined in UEFI v2.10, section N.2.13. Define struct cxl_cper_prot_err_work_data to cache CXL protocol error information, including RAS capabilities and severity, for further handling. These cached CXL CPER records will later be processed by workqueues within the CXL subsystem. Signed-off-by: Smita Koralahalli Reviewed-by: Jonathan Cameron Reviewed-by: Dave Jiang Reviewed-by: Ira Weiny Reviewed-by: Tony Luck Reviewed-by: Gregory Price Reviewed-by: Dan Williams Link: https://patch.msgid.link/20250123084421.127697-5-Smita.KoralahalliChannabasappa@amd.com Signed-off-by: Dave Jiang Signed-off-by: Bo Li --- drivers/acpi/apei/ghes.c | 54 ++++++++++++++++++++++++++++++++++++++++ include/cxl/event.h | 6 +++++ 2 files changed, 60 insertions(+) diff --git a/drivers/acpi/apei/ghes.c b/drivers/acpi/apei/ghes.c index c769000fec4d3..9fc581b8ce3fa 100644 --- a/drivers/acpi/apei/ghes.c +++ b/drivers/acpi/apei/ghes.c @@ -678,6 +678,56 @@ static void ghes_defer_non_standard_event(struct acpi_hest_generic_data *gdata, schedule_work(&entry->work); } +static void cxl_cper_post_prot_err(struct cxl_cper_sec_prot_err *prot_err, + int severity) +{ +#ifdef CONFIG_ACPI_APEI_PCIEAER + struct cxl_cper_prot_err_work_data wd; + u8 *dvsec_start, *cap_start; + + if (!(prot_err->valid_bits & PROT_ERR_VALID_AGENT_ADDRESS)) { + pr_err_ratelimited("CXL CPER invalid agent type\n"); + return; + } + + if (!(prot_err->valid_bits & PROT_ERR_VALID_ERROR_LOG)) { + pr_err_ratelimited("CXL CPER invalid protocol error log\n"); + return; + } + + if (prot_err->err_len != sizeof(struct cxl_ras_capability_regs)) { + pr_err_ratelimited("CXL CPER invalid RAS Cap size (%u)\n", + prot_err->err_len); + return; + } + + if (!(prot_err->valid_bits & PROT_ERR_VALID_SERIAL_NUMBER)) + pr_warn(FW_WARN "CXL CPER no device serial number\n"); + + switch (prot_err->agent_type) { + case RCD: + case DEVICE: + case LD: + case FMLD: + case RP: + case DSP: + case USP: + memcpy(&wd.prot_err, prot_err, sizeof(wd.prot_err)); + + dvsec_start = (u8 *)(prot_err + 1); + cap_start = dvsec_start + prot_err->dvsec_len; + + memcpy(&wd.ras_cap, cap_start, sizeof(wd.ras_cap)); + wd.severity = cper_severity_to_aer(severity); + break; + default: + pr_err_ratelimited("CXL CPER invalid agent type: %d\n", + prot_err->agent_type); + return; + } +#endif +} + /* Room for 8 entries for each of the 4 event log queues */ #define CXL_CPER_FIFO_DEPTH 32 DEFINE_KFIFO(cxl_cper_fifo, struct cxl_cper_work_data, CXL_CPER_FIFO_DEPTH); @@ -781,6 +831,10 @@ static bool ghes_do_proc(struct ghes *ghes, } else if (guid_equal(sec_type, &CPER_SEC_PROC_ARM)) { queued = ghes_handle_arm_hw_error(gdata, sev, sync); + } else if (guid_equal(sec_type, &CPER_SEC_CXL_PROT_ERR)) { + struct cxl_cper_sec_prot_err *prot_err = acpi_hest_get_payload(gdata); + + cxl_cper_post_prot_err(prot_err, gdata->error_severity); } else if (guid_equal(sec_type, &CPER_SEC_CXL_GEN_MEDIA_GUID)) { struct cxl_cper_event_rec *rec = acpi_hest_get_payload(gdata); diff --git a/include/cxl/event.h b/include/cxl/event.h index dca6fcce5a07c..3ab9253033b32 100644 --- a/include/cxl/event.h +++ b/include/cxl/event.h @@ -237,6 +237,12 @@ struct cxl_ras_capability_regs { u32 header_log[16]; }; +struct cxl_cper_prot_err_work_data { + struct cxl_cper_sec_prot_err prot_err; + struct cxl_ras_capability_regs ras_cap; + int severity; +}; + #ifdef CONFIG_ACPI_APEI_GHES int cxl_cper_register_work(struct work_struct *work); int cxl_cper_unregister_work(struct work_struct *work); From 53f4da7940e0359bb7c284f6f23e1550efeae093 Mon Sep 17 00:00:00 2001 From: Smita Koralahalli Date: Mon, 10 Mar 2025 22:38:38 +0000 Subject: [PATCH 132/138] acpi/ghes, cxl/pci: Process CXL CPER Protocol Errors commit 36f257e3b0ba904f5a4e7fa8dafaa60e88cdd28c upstream. When PCIe AER is in FW-First, OS should process CXL Protocol errors from CPER records. Introduce support for handling and logging CXL Protocol errors. The defined trace events cxl_aer_uncorrectable_error and cxl_aer_correctable_error trace native CXL AER endpoint errors. Reuse them to trace FW-First Protocol errors. Since the CXL code is required to be called from process context and GHES is in interrupt context, use workqueues for processing. Similar to CXL CPER event handling, use kfifo to handle errors as it simplifies queue processing by providing lock free fifo operations. Add the ability for the CXL sub-system to register a workqueue to process CXL CPER protocol errors. [DJ: return cxl_cper_register_prot_err_work() directly in cxl_ras_init()] Signed-off-by: Smita Koralahalli Reviewed-by: Li Ming Reviewed-by: Alison Schofield Reviewed-by: Ira Weiny Reviewed-by: Tony Luck Link: https://patch.msgid.link/20250310223839.31342-2-Smita.KoralahalliChannabasappa@amd.com Signed-off-by: Dave Jiang Signed-off-by: Bo Li --- drivers/acpi/apei/ghes.c | 49 +++++++++++++++++++++++ drivers/cxl/core/Makefile | 1 + drivers/cxl/core/core.h | 3 ++ drivers/cxl/core/port.c | 7 ++++ drivers/cxl/core/ras.c | 82 +++++++++++++++++++++++++++++++++++++++ include/cxl/event.h | 15 +++++++ tools/testing/cxl/Kbuild | 1 + 7 files changed, 158 insertions(+) create mode 100644 drivers/cxl/core/ras.c diff --git a/drivers/acpi/apei/ghes.c b/drivers/acpi/apei/ghes.c index 9fc581b8ce3fa..f924bc1be16f9 100644 --- a/drivers/acpi/apei/ghes.c +++ b/drivers/acpi/apei/ghes.c @@ -678,6 +678,15 @@ static void ghes_defer_non_standard_event(struct acpi_hest_generic_data *gdata, schedule_work(&entry->work); } +/* Room for 8 entries */ +#define CXL_CPER_PROT_ERR_FIFO_DEPTH 8 +static DEFINE_KFIFO(cxl_cper_prot_err_fifo, struct cxl_cper_prot_err_work_data, + CXL_CPER_PROT_ERR_FIFO_DEPTH); + +/* Synchronize schedule_work() with cxl_cper_prot_err_work changes */ +static DEFINE_SPINLOCK(cxl_cper_prot_err_work_lock); +struct work_struct *cxl_cper_prot_err_work; + static void cxl_cper_post_prot_err(struct cxl_cper_sec_prot_err *prot_err, int severity) { @@ -704,6 +713,11 @@ static void cxl_cper_post_prot_err(struct cxl_cper_sec_prot_err *prot_err, if (!(prot_err->valid_bits & PROT_ERR_VALID_SERIAL_NUMBER)) pr_warn(FW_WARN "CXL CPER no device serial number\n"); + guard(spinlock_irqsave)(&cxl_cper_prot_err_work_lock); + + if (!cxl_cper_prot_err_work) + return; + switch (prot_err->agent_type) { case RCD: case DEVICE: @@ -725,9 +739,44 @@ static void cxl_cper_post_prot_err(struct cxl_cper_sec_prot_err *prot_err, prot_err->agent_type); return; } + + if (!kfifo_put(&cxl_cper_prot_err_fifo, wd)) { + pr_err_ratelimited("CXL CPER kfifo overflow\n"); + return; + } + + schedule_work(cxl_cper_prot_err_work); #endif } +int cxl_cper_register_prot_err_work(struct work_struct *work) +{ + if (cxl_cper_prot_err_work) + return -EINVAL; + + guard(spinlock)(&cxl_cper_prot_err_work_lock); + cxl_cper_prot_err_work = work; + return 0; +} +EXPORT_SYMBOL_NS_GPL(cxl_cper_register_prot_err_work, CXL); + +int cxl_cper_unregister_prot_err_work(struct work_struct *work) +{ + if (cxl_cper_prot_err_work != work) + return -EINVAL; + + guard(spinlock)(&cxl_cper_prot_err_work_lock); + cxl_cper_prot_err_work = NULL; + return 0; +} +EXPORT_SYMBOL_NS_GPL(cxl_cper_unregister_prot_err_work, CXL); + +int cxl_cper_prot_err_kfifo_get(struct cxl_cper_prot_err_work_data *wd) +{ + return kfifo_get(&cxl_cper_prot_err_fifo, wd); +} +EXPORT_SYMBOL_NS_GPL(cxl_cper_prot_err_kfifo_get, CXL); + /* Room for 8 entries for each of the 4 event log queues */ #define CXL_CPER_FIFO_DEPTH 32 DEFINE_KFIFO(cxl_cper_fifo, struct cxl_cper_work_data, CXL_CPER_FIFO_DEPTH); diff --git a/drivers/cxl/core/Makefile b/drivers/cxl/core/Makefile index 9259bcc6773c8..ba5f0916d3796 100644 --- a/drivers/cxl/core/Makefile +++ b/drivers/cxl/core/Makefile @@ -14,5 +14,6 @@ cxl_core-y += pci.o cxl_core-y += hdm.o cxl_core-y += pmu.o cxl_core-y += cdat.o +cxl_core-y += ras.o cxl_core-$(CONFIG_TRACING) += trace.o cxl_core-$(CONFIG_CXL_REGION) += region.o diff --git a/drivers/cxl/core/core.h b/drivers/cxl/core/core.h index 933259c76b3f9..200606a6afc84 100644 --- a/drivers/cxl/core/core.h +++ b/drivers/cxl/core/core.h @@ -107,4 +107,7 @@ long cxl_pci_get_latency(struct pci_dev *pdev); int cxl_update_hmat_access_coordinates(int nid, struct cxl_region *cxlr, enum access_coordinate_class access); +int cxl_ras_init(void); +void cxl_ras_exit(void); + #endif /* __CXL_CORE_H__ */ diff --git a/drivers/cxl/core/port.c b/drivers/cxl/core/port.c index 23818fcc924aa..4d58d529b876d 100644 --- a/drivers/cxl/core/port.c +++ b/drivers/cxl/core/port.c @@ -2327,8 +2327,14 @@ static __init int cxl_core_init(void) if (rc) goto err_region; + rc = cxl_ras_init(); + if (rc) + goto err_ras; + return 0; +err_ras: + cxl_region_exit(); err_region: bus_unregister(&cxl_bus_type); err_bus: @@ -2340,6 +2346,7 @@ static __init int cxl_core_init(void) static void cxl_core_exit(void) { + cxl_ras_exit(); cxl_region_exit(); bus_unregister(&cxl_bus_type); destroy_workqueue(cxl_bus_wq); diff --git a/drivers/cxl/core/ras.c b/drivers/cxl/core/ras.c new file mode 100644 index 0000000000000..91273af6ccbc7 --- /dev/null +++ b/drivers/cxl/core/ras.c @@ -0,0 +1,82 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright(c) 2025 AMD Corporation. All rights reserved. */ + +#include +#include +#include +#include +#include "trace.h" + +static void cxl_cper_trace_corr_prot_err(struct pci_dev *pdev, + struct cxl_ras_capability_regs ras_cap) +{ + u32 status = ras_cap.cor_status & ~ras_cap.cor_mask; + struct cxl_dev_state *cxlds; + + cxlds = pci_get_drvdata(pdev); + if (!cxlds) + return; + + trace_cxl_aer_correctable_error(cxlds->cxlmd, status); +} + +static void cxl_cper_trace_uncorr_prot_err(struct pci_dev *pdev, + struct cxl_ras_capability_regs ras_cap) +{ + u32 status = ras_cap.uncor_status & ~ras_cap.uncor_mask; + struct cxl_dev_state *cxlds; + u32 fe; + + cxlds = pci_get_drvdata(pdev); + if (!cxlds) + return; + + if (hweight32(status) > 1) + fe = BIT(FIELD_GET(CXL_RAS_CAP_CONTROL_FE_MASK, + ras_cap.cap_control)); + else + fe = status; + + trace_cxl_aer_uncorrectable_error(cxlds->cxlmd, status, fe, + ras_cap.header_log); +} + +static void cxl_cper_handle_prot_err(struct cxl_cper_prot_err_work_data *data) +{ + unsigned int devfn = PCI_DEVFN(data->prot_err.agent_addr.device, + data->prot_err.agent_addr.function); + struct pci_dev *pdev __free(pci_dev_put) = + pci_get_domain_bus_and_slot(data->prot_err.agent_addr.segment, + data->prot_err.agent_addr.bus, + devfn); + + if (!pdev) + return; + + guard(device)(&pdev->dev); + + if (data->severity == AER_CORRECTABLE) + cxl_cper_trace_corr_prot_err(pdev, data->ras_cap); + else + cxl_cper_trace_uncorr_prot_err(pdev, data->ras_cap); +} + +static void cxl_cper_prot_err_work_fn(struct work_struct *work) +{ + struct cxl_cper_prot_err_work_data wd; + + while (cxl_cper_prot_err_kfifo_get(&wd)) + cxl_cper_handle_prot_err(&wd); +} +static DECLARE_WORK(cxl_cper_prot_err_work, cxl_cper_prot_err_work_fn); + +int cxl_ras_init(void) +{ + return cxl_cper_register_prot_err_work(&cxl_cper_prot_err_work); +} + +void cxl_ras_exit(void) +{ + cxl_cper_unregister_prot_err_work(&cxl_cper_prot_err_work); + cancel_work_sync(&cxl_cper_prot_err_work); +} diff --git a/include/cxl/event.h b/include/cxl/event.h index 3ab9253033b32..efaa7a1aa2693 100644 --- a/include/cxl/event.h +++ b/include/cxl/event.h @@ -247,6 +247,9 @@ struct cxl_cper_prot_err_work_data { int cxl_cper_register_work(struct work_struct *work); int cxl_cper_unregister_work(struct work_struct *work); int cxl_cper_kfifo_get(struct cxl_cper_work_data *wd); +int cxl_cper_register_prot_err_work(struct work_struct *work); +int cxl_cper_unregister_prot_err_work(struct work_struct *work); +int cxl_cper_prot_err_kfifo_get(struct cxl_cper_prot_err_work_data *wd); #else static inline int cxl_cper_register_work(struct work_struct *work) { @@ -261,6 +264,18 @@ static inline int cxl_cper_kfifo_get(struct cxl_cper_work_data *wd) { return 0; } +static inline int cxl_cper_register_prot_err_work(struct work_struct *work) +{ + return 0; +} +static inline int cxl_cper_unregister_prot_err_work(struct work_struct *work) +{ + return 0; +} +static inline int cxl_cper_prot_err_kfifo_get(struct cxl_cper_prot_err_work_data *wd) +{ + return 0; +} #endif #endif /* _LINUX_CXL_EVENT_H */ diff --git a/tools/testing/cxl/Kbuild b/tools/testing/cxl/Kbuild index 3d1ca9e38b1fa..fa75f49606e7b 100644 --- a/tools/testing/cxl/Kbuild +++ b/tools/testing/cxl/Kbuild @@ -61,6 +61,7 @@ cxl_core-y += $(CXL_CORE_SRC)/pci.o cxl_core-y += $(CXL_CORE_SRC)/hdm.o cxl_core-y += $(CXL_CORE_SRC)/pmu.o cxl_core-y += $(CXL_CORE_SRC)/cdat.o +cxl_core-y += $(CXL_CORE_SRC)/ras.o cxl_core-$(CONFIG_TRACING) += $(CXL_CORE_SRC)/trace.o cxl_core-$(CONFIG_CXL_REGION) += $(CXL_CORE_SRC)/region.o cxl_core-y += config_check.o From e20e6cf81b8cb7557c871dcf81e05fd1c531cdbe Mon Sep 17 00:00:00 2001 From: Smita Koralahalli Date: Mon, 10 Mar 2025 22:38:39 +0000 Subject: [PATCH 133/138] cxl/pci: Add trace logging for CXL PCIe Port RAS errors commit 02f4f0177d8e7647016fc29f11c1a7bb75bc2182 upstream. The CXL drivers use kernel trace functions for logging endpoint and Restricted CXL host (RCH) Downstream Port RAS errors. Similar functionality is required for CXL Root Ports, CXL Downstream Switch Ports, and CXL Upstream Switch Ports. Introduce trace logging functions for both RAS correctable and uncorrectable errors specific to CXL PCIe Ports. Use them to trace FW-First Protocol errors. Co-developed-by: Terry Bowman Signed-off-by: Terry Bowman Signed-off-by: Smita Koralahalli Reviewed-by: Ira Weiny Reviewed-by: Li Ming Reviewed-by: Alison Schofield Reviewed-by: Tony Luck Link: https://patch.msgid.link/20250310223839.31342-3-Smita.KoralahalliChannabasappa@amd.com Signed-off-by: Dave Jiang Signed-off-by: Bo Li --- drivers/cxl/core/ras.c | 37 +++++++++++++++++++++++++++++++ drivers/cxl/core/trace.h | 47 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+) diff --git a/drivers/cxl/core/ras.c b/drivers/cxl/core/ras.c index 91273af6ccbc7..485a831695c70 100644 --- a/drivers/cxl/core/ras.c +++ b/drivers/cxl/core/ras.c @@ -7,6 +7,30 @@ #include #include "trace.h" +static void cxl_cper_trace_corr_port_prot_err(struct pci_dev *pdev, + struct cxl_ras_capability_regs ras_cap) +{ + u32 status = ras_cap.cor_status & ~ras_cap.cor_mask; + + trace_cxl_port_aer_correctable_error(&pdev->dev, status); +} + +static void cxl_cper_trace_uncorr_port_prot_err(struct pci_dev *pdev, + struct cxl_ras_capability_regs ras_cap) +{ + u32 status = ras_cap.uncor_status & ~ras_cap.uncor_mask; + u32 fe; + + if (hweight32(status) > 1) + fe = BIT(FIELD_GET(CXL_RAS_CAP_CONTROL_FE_MASK, + ras_cap.cap_control)); + else + fe = status; + + trace_cxl_port_aer_uncorrectable_error(&pdev->dev, status, fe, + ras_cap.header_log); +} + static void cxl_cper_trace_corr_prot_err(struct pci_dev *pdev, struct cxl_ras_capability_regs ras_cap) { @@ -49,12 +73,25 @@ static void cxl_cper_handle_prot_err(struct cxl_cper_prot_err_work_data *data) pci_get_domain_bus_and_slot(data->prot_err.agent_addr.segment, data->prot_err.agent_addr.bus, devfn); + int port_type; if (!pdev) return; guard(device)(&pdev->dev); + port_type = pci_pcie_type(pdev); + if (port_type == PCI_EXP_TYPE_ROOT_PORT || + port_type == PCI_EXP_TYPE_DOWNSTREAM || + port_type == PCI_EXP_TYPE_UPSTREAM) { + if (data->severity == AER_CORRECTABLE) + cxl_cper_trace_corr_port_prot_err(pdev, data->ras_cap); + else + cxl_cper_trace_uncorr_port_prot_err(pdev, data->ras_cap); + + return; + } + if (data->severity == AER_CORRECTABLE) cxl_cper_trace_corr_prot_err(pdev, data->ras_cap); else diff --git a/drivers/cxl/core/trace.h b/drivers/cxl/core/trace.h index 642b1ee4542ed..51d949de4e6b3 100644 --- a/drivers/cxl/core/trace.h +++ b/drivers/cxl/core/trace.h @@ -48,6 +48,34 @@ { CXL_RAS_UC_IDE_RX_ERR, "IDE Rx Error" } \ ) +TRACE_EVENT(cxl_port_aer_uncorrectable_error, + TP_PROTO(struct device *dev, u32 status, u32 fe, u32 *hl), + TP_ARGS(dev, status, fe, hl), + TP_STRUCT__entry( + __string(device, dev_name(dev)) + __string(host, dev_name(dev->parent)) + __field(u32, status) + __field(u32, first_error) + __array(u32, header_log, CXL_HEADERLOG_SIZE_U32) + ), + TP_fast_assign( + __assign_str(device, dev_name(dev)); + __assign_str(host, dev_name(dev->parent)); + __entry->status = status; + __entry->first_error = fe; + /* + * Embed the 512B headerlog data for user app retrieval and + * parsing, but no need to print this in the trace buffer. + */ + memcpy(__entry->header_log, hl, CXL_HEADERLOG_SIZE); + ), + TP_printk("device=%s host=%s status: '%s' first_error: '%s'", + __get_str(device), __get_str(host), + show_uc_errs(__entry->status), + show_uc_errs(__entry->first_error) + ) +); + TRACE_EVENT(cxl_aer_uncorrectable_error, TP_PROTO(const struct cxl_memdev *cxlmd, u32 status, u32 fe, u32 *hl), TP_ARGS(cxlmd, status, fe, hl), @@ -96,6 +124,25 @@ TRACE_EVENT(cxl_aer_uncorrectable_error, { CXL_RAS_CE_PHYS_LAYER_ERR, "Received Error From Physical Layer" } \ ) +TRACE_EVENT(cxl_port_aer_correctable_error, + TP_PROTO(struct device *dev, u32 status), + TP_ARGS(dev, status), + TP_STRUCT__entry( + __string(device, dev_name(dev)) + __string(host, dev_name(dev->parent)) + __field(u32, status) + ), + TP_fast_assign( + __assign_str(device, dev_name(dev)); + __assign_str(host, dev_name(dev->parent)); + __entry->status = status; + ), + TP_printk("device=%s host=%s status='%s'", + __get_str(device), __get_str(host), + show_ce_errs(__entry->status) + ) +); + TRACE_EVENT(cxl_aer_correctable_error, TP_PROTO(const struct cxl_memdev *cxlmd, u32 status), TP_ARGS(cxlmd, status), From b630c7577820393aa6105e2ed823c3fbf89d2e39 Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Thu, 12 Jun 2025 12:20:43 -0700 Subject: [PATCH 134/138] cxl/ras: Fix CPER handler device confusion commit 3c70ec71abdaf4e4fa48cd8fdfbbd864d78235a8 upstream. By inspection, cxl_cper_handle_prot_err() is making a series of fragile assumptions that can lead to crashes: 1/ It assumes that endpoints identified in the record are a CXL-type-3 device, nothing guarantees that. 2/ It assumes that the device is bound to the cxl_pci driver, nothing guarantees that. 3/ Minor, it holds the device lock over the switch-port tracing for no reason as the trace is 100% generated from data in the record. Correct those by checking that the PCIe endpoint parents a cxl_memdev before assuming the format of the driver data, and move the lock to where it is required. Consequently this also makes the implementation ready for CXL accelerators that are not bound to cxl_pci. Fixes: 36f257e3b0ba ("acpi/ghes, cxl/pci: Process CXL CPER Protocol Errors") Cc: Terry Bowman Cc: Li Ming Cc: Alison Schofield Cc: Ira Weiny Cc: Tony Luck Reviewed-by: Smita Koralahalli Reviewed-by: Dave Jiang Signed-off-by: Dan Williams Reviewed-by: Jonathan Cameron Reviewed-by: Li Ming Link: https://patch.msgid.link/20250612192043.2254617-1-dan.j.williams@intel.com Signed-off-by: Dave Jiang Signed-off-by: Bo Li --- drivers/cxl/core/ras.c | 47 ++++++++++++++++++++++++------------------ 1 file changed, 27 insertions(+), 20 deletions(-) diff --git a/drivers/cxl/core/ras.c b/drivers/cxl/core/ras.c index 485a831695c70..2731ba3a07993 100644 --- a/drivers/cxl/core/ras.c +++ b/drivers/cxl/core/ras.c @@ -31,40 +31,38 @@ static void cxl_cper_trace_uncorr_port_prot_err(struct pci_dev *pdev, ras_cap.header_log); } -static void cxl_cper_trace_corr_prot_err(struct pci_dev *pdev, - struct cxl_ras_capability_regs ras_cap) +static void cxl_cper_trace_corr_prot_err(struct cxl_memdev *cxlmd, + struct cxl_ras_capability_regs ras_cap) { u32 status = ras_cap.cor_status & ~ras_cap.cor_mask; - struct cxl_dev_state *cxlds; - cxlds = pci_get_drvdata(pdev); - if (!cxlds) - return; - - trace_cxl_aer_correctable_error(cxlds->cxlmd, status); + trace_cxl_aer_correctable_error(cxlmd, status); } -static void cxl_cper_trace_uncorr_prot_err(struct pci_dev *pdev, - struct cxl_ras_capability_regs ras_cap) +static void +cxl_cper_trace_uncorr_prot_err(struct cxl_memdev *cxlmd, + struct cxl_ras_capability_regs ras_cap) { u32 status = ras_cap.uncor_status & ~ras_cap.uncor_mask; - struct cxl_dev_state *cxlds; u32 fe; - cxlds = pci_get_drvdata(pdev); - if (!cxlds) - return; - if (hweight32(status) > 1) fe = BIT(FIELD_GET(CXL_RAS_CAP_CONTROL_FE_MASK, ras_cap.cap_control)); else fe = status; - trace_cxl_aer_uncorrectable_error(cxlds->cxlmd, status, fe, + trace_cxl_aer_uncorrectable_error(cxlmd, status, fe, ras_cap.header_log); } +static int match_memdev_by_parent(struct device *dev, const void *uport) +{ + if (is_cxl_memdev(dev) && dev->parent == uport) + return 1; + return 0; +} + static void cxl_cper_handle_prot_err(struct cxl_cper_prot_err_work_data *data) { unsigned int devfn = PCI_DEVFN(data->prot_err.agent_addr.device, @@ -73,13 +71,12 @@ static void cxl_cper_handle_prot_err(struct cxl_cper_prot_err_work_data *data) pci_get_domain_bus_and_slot(data->prot_err.agent_addr.segment, data->prot_err.agent_addr.bus, devfn); + struct cxl_memdev *cxlmd; int port_type; if (!pdev) return; - guard(device)(&pdev->dev); - port_type = pci_pcie_type(pdev); if (port_type == PCI_EXP_TYPE_ROOT_PORT || port_type == PCI_EXP_TYPE_DOWNSTREAM || @@ -92,10 +89,20 @@ static void cxl_cper_handle_prot_err(struct cxl_cper_prot_err_work_data *data) return; } + guard(device)(&pdev->dev); + if (!pdev->dev.driver) + return; + + struct device *mem_dev __free(put_device) = bus_find_device( + &cxl_bus_type, NULL, pdev, match_memdev_by_parent); + if (!mem_dev) + return; + + cxlmd = to_cxl_memdev(mem_dev); if (data->severity == AER_CORRECTABLE) - cxl_cper_trace_corr_prot_err(pdev, data->ras_cap); + cxl_cper_trace_corr_prot_err(cxlmd, data->ras_cap); else - cxl_cper_trace_uncorr_prot_err(pdev, data->ras_cap); + cxl_cper_trace_uncorr_prot_err(cxlmd, data->ras_cap); } static void cxl_cper_prot_err_work_fn(struct work_struct *work) From a7f502a5730dce81888c5922d8349d84a5be3108 Mon Sep 17 00:00:00 2001 From: Alison Schofield Date: Mon, 4 Aug 2025 01:00:09 -0700 Subject: [PATCH 135/138] cxl: Move hpa_to_spa callback to a new root decoder ops structure commit 524b2b76f365fb90a7f894ac17261ea760464e2c upstream. The root decoder's HPA to SPA translation logic was implemented using a single function pointer. In preparation for additional per-decoder callbacks, convert this into a struct cxl_rd_ops and move the hpa_to_spa pointer into it. To avoid maintaining a static ops instance populated with mostly NULL pointers, allocate the ops structure dynamically only when a platform requires overrides (e.g. XOR interleave decoding). The setup can be extended as additional callbacks are added. Co-developed-by: Dave Jiang Signed-off-by: Alison Schofield Reviewed-by: Jonathan Cameron Link: https://patch.msgid.link/818530c82c351a9c0d3a204f593068dd2126a5a9.1754290144.git.alison.schofield@intel.com Signed-off-by: Dave Jiang Signed-off-by: Bo Li --- drivers/cxl/acpi.c | 10 +++++++--- drivers/cxl/core/port.c | 1 + drivers/cxl/core/region.c | 11 ++++++++--- drivers/cxl/cxl.h | 12 +++++++++--- 4 files changed, 25 insertions(+), 9 deletions(-) diff --git a/drivers/cxl/acpi.c b/drivers/cxl/acpi.c index 2b19b7bedece1..8065d5b26d7fa 100644 --- a/drivers/cxl/acpi.c +++ b/drivers/cxl/acpi.c @@ -22,7 +22,6 @@ static const guid_t acpi_cxl_qtg_id_guid = GUID_INIT(0xF365F9A6, 0xA7DE, 0x4071, 0xA6, 0x6A, 0xB4, 0x0C, 0x0B, 0x4F, 0x8E, 0x52); - static u64 cxl_xor_hpa_to_spa(struct cxl_root_decoder *cxlrd, u64 hpa) { struct cxl_cxims_data *cximsd = cxlrd->platform_data; @@ -415,8 +414,13 @@ static int __cxl_parse_cfmws(struct acpi_cedt_cfmws *cfmws, cxlrd->qos_class = cfmws->qtg_id; - if (cfmws->interleave_arithmetic == ACPI_CEDT_CFMWS_ARITHMETIC_XOR) - cxlrd->hpa_to_spa = cxl_xor_hpa_to_spa; + if (cfmws->interleave_arithmetic == ACPI_CEDT_CFMWS_ARITHMETIC_XOR) { + cxlrd->ops = kzalloc(sizeof(*cxlrd->ops), GFP_KERNEL); + if (!cxlrd->ops) + return -ENOMEM; + + cxlrd->ops->hpa_to_spa = cxl_xor_hpa_to_spa; + } rc = cxl_decoder_add(cxld, target_map); if (rc) diff --git a/drivers/cxl/core/port.c b/drivers/cxl/core/port.c index 4d58d529b876d..7d57d4a52d17d 100644 --- a/drivers/cxl/core/port.c +++ b/drivers/cxl/core/port.c @@ -446,6 +446,7 @@ static void cxl_root_decoder_release(struct device *dev) if (atomic_read(&cxlrd->region_id) >= 0) memregion_free(atomic_read(&cxlrd->region_id)); __cxl_decoder_release(&cxlrd->cxlsd.cxld); + kfree(cxlrd->ops); kfree(cxlrd); } diff --git a/drivers/cxl/core/region.c b/drivers/cxl/core/region.c index ebf7ddb53a6a5..f4ecf5d2ea6ed 100644 --- a/drivers/cxl/core/region.c +++ b/drivers/cxl/core/region.c @@ -2849,6 +2849,11 @@ static bool cxl_is_hpa_in_chunk(u64 hpa, struct cxl_region *cxlr, int pos) return false; } +static bool has_hpa_to_spa(struct cxl_root_decoder *cxlrd) +{ + return cxlrd->ops && cxlrd->ops->hpa_to_spa; +} + u64 cxl_dpa_to_hpa(struct cxl_region *cxlr, const struct cxl_memdev *cxlmd, u64 dpa) { @@ -2903,8 +2908,8 @@ u64 cxl_dpa_to_hpa(struct cxl_region *cxlr, const struct cxl_memdev *cxlmd, hpa = hpa_offset + p->res->start; /* Root decoder translation overrides typical modulo decode */ - if (cxlrd->hpa_to_spa) - hpa = cxlrd->hpa_to_spa(cxlrd, hpa); + if (has_hpa_to_spa(cxlrd)) + hpa = cxlrd->ops->hpa_to_spa(cxlrd, hpa); if (hpa < p->res->start || hpa > p->res->end) { dev_dbg(&cxlr->dev, @@ -2913,7 +2918,7 @@ u64 cxl_dpa_to_hpa(struct cxl_region *cxlr, const struct cxl_memdev *cxlmd, } /* Simple chunk check, by pos & gran, only applies to modulo decodes */ - if (!cxlrd->hpa_to_spa && (!cxl_is_hpa_in_chunk(hpa, cxlr, pos))) + if (!has_hpa_to_spa(cxlrd) && (!cxl_is_hpa_in_chunk(hpa, cxlr, pos))) return ULLONG_MAX; return hpa; diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h index 0f8f3aa4d378b..9f3d4aedb3330 100644 --- a/drivers/cxl/cxl.h +++ b/drivers/cxl/cxl.h @@ -432,25 +432,31 @@ struct cxl_switch_decoder { }; struct cxl_root_decoder; -typedef u64 (*cxl_hpa_to_spa_fn)(struct cxl_root_decoder *cxlrd, u64 hpa); +/** + * struct cxl_rd_ops - CXL root decoder callback operations + * @hpa_to_spa: Convert host physical address to system physical address + */ +struct cxl_rd_ops { + u64 (*hpa_to_spa)(struct cxl_root_decoder *cxlrd, u64 hpa); +}; /** * struct cxl_root_decoder - Static platform CXL address decoder * @res: host / parent resource for region allocations * @region_id: region id for next region provisioning event - * @hpa_to_spa: translate CXL host-physical-address to Platform system-physical-address * @platform_data: platform specific configuration data * @range_lock: sync region autodiscovery by address range * @qos_class: QoS performance class cookie + * @ops: CXL root decoder operations * @cxlsd: base cxl switch decoder */ struct cxl_root_decoder { struct resource *res; atomic_t region_id; - cxl_hpa_to_spa_fn hpa_to_spa; void *platform_data; struct mutex range_lock; int qos_class; + struct cxl_rd_ops *ops; struct cxl_switch_decoder cxlsd; }; From 1895405d7f929f41e1b40d42b137859878fef156 Mon Sep 17 00:00:00 2001 From: Alison Schofield Date: Mon, 4 Aug 2025 01:00:10 -0700 Subject: [PATCH 136/138] cxl: Define a SPA->CXL HPA root decoder callback for XOR Math commit b83ee9614a3ec196111f0ae54335b99700f78b45 upstream. When DPA->SPA translation was introduced, it included a helper that applied the XOR maps to do the CXL HPA -> SPA translation for XOR region interleaves. In preparation for adding SPA->DPA address translation, introduce the reverse callback. The root decoder callback is defined generically and not all usages may be self inverting like this XOR function. Add another root decoder callback that is the spa_to_hpa function. Update the existing cxl_xor_hpa_to_spa() with a name that reflects what it does without directionality: cxl_apply_xor_maps(), a generic parameter: addr replaces hpa, and code comments stating that the function supports the translation in either direction. Signed-off-by: Alison Schofield Reviewed-by: Jonathan Cameron Link: https://patch.msgid.link/79d9d72230c599cae94d7221781ead6392ae6d3f.1754290144.git.alison.schofield@intel.com Signed-off-by: Dave Jiang Signed-off-by: Bo Li --- drivers/cxl/acpi.c | 27 ++++++++++++++++----------- drivers/cxl/cxl.h | 2 ++ 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/drivers/cxl/acpi.c b/drivers/cxl/acpi.c index 8065d5b26d7fa..572282d2d6717 100644 --- a/drivers/cxl/acpi.c +++ b/drivers/cxl/acpi.c @@ -22,7 +22,7 @@ static const guid_t acpi_cxl_qtg_id_guid = GUID_INIT(0xF365F9A6, 0xA7DE, 0x4071, 0xA6, 0x6A, 0xB4, 0x0C, 0x0B, 0x4F, 0x8E, 0x52); -static u64 cxl_xor_hpa_to_spa(struct cxl_root_decoder *cxlrd, u64 hpa) +static u64 cxl_apply_xor_maps(struct cxl_root_decoder *cxlrd, u64 addr) { struct cxl_cxims_data *cximsd = cxlrd->platform_data; int hbiw = cxlrd->cxlsd.nr_targets; @@ -31,19 +31,23 @@ static u64 cxl_xor_hpa_to_spa(struct cxl_root_decoder *cxlrd, u64 hpa) /* No xormaps for host bridge interleave ways of 1 or 3 */ if (hbiw == 1 || hbiw == 3) - return hpa; + return addr; /* - * For root decoders using xormaps (hbiw: 2,4,6,8,12,16) restore - * the position bit to its value before the xormap was applied at - * HPA->DPA translation. + * In regions using XOR interleave arithmetic the CXL HPA may not + * be the same as the SPA. This helper performs the SPA->CXL HPA + * or the CXL HPA->SPA translation. Since XOR is self-inverting, + * so is this function. + * + * For root decoders using xormaps (hbiw: 2,4,6,8,12,16) applying the + * xormaps will toggle a position bit. * * pos is the lowest set bit in an XORMAP - * val is the XORALLBITS(HPA & XORMAP) + * val is the XORALLBITS(addr & XORMAP) * * XORALLBITS: The CXL spec (3.1 Table 9-22) defines XORALLBITS * as an operation that outputs a single bit by XORing all the - * bits in the input (hpa & xormap). Implement XORALLBITS using + * bits in the input (addr & xormap). Implement XORALLBITS using * hweight64(). If the hamming weight is even the XOR of those * bits results in val==0, if odd the XOR result is val==1. */ @@ -52,11 +56,11 @@ static u64 cxl_xor_hpa_to_spa(struct cxl_root_decoder *cxlrd, u64 hpa) if (!cximsd->xormaps[i]) continue; pos = __ffs(cximsd->xormaps[i]); - val = (hweight64(hpa & cximsd->xormaps[i]) & 1); - hpa = (hpa & ~(1ULL << pos)) | (val << pos); + val = (hweight64(addr & cximsd->xormaps[i]) & 1); + addr = (addr & ~(1ULL << pos)) | (val << pos); } - return hpa; + return addr; } struct cxl_cxims_context { @@ -419,7 +423,8 @@ static int __cxl_parse_cfmws(struct acpi_cedt_cfmws *cfmws, if (!cxlrd->ops) return -ENOMEM; - cxlrd->ops->hpa_to_spa = cxl_xor_hpa_to_spa; + cxlrd->ops->hpa_to_spa = cxl_apply_xor_maps; + cxlrd->ops->spa_to_hpa = cxl_apply_xor_maps; } rc = cxl_decoder_add(cxld, target_map); diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h index 9f3d4aedb3330..0c3ab0952dd72 100644 --- a/drivers/cxl/cxl.h +++ b/drivers/cxl/cxl.h @@ -435,9 +435,11 @@ struct cxl_root_decoder; /** * struct cxl_rd_ops - CXL root decoder callback operations * @hpa_to_spa: Convert host physical address to system physical address + * @spa_to_hpa: Convert system physical address to host physical address */ struct cxl_rd_ops { u64 (*hpa_to_spa)(struct cxl_root_decoder *cxlrd, u64 hpa); + u64 (*spa_to_hpa)(struct cxl_root_decoder *cxlrd, u64 spa); }; /** From b1fcb8b6ffb01fa6688ef76bfe4f59cb81b36817 Mon Sep 17 00:00:00 2001 From: Alison Schofield Date: Mon, 4 Aug 2025 01:00:11 -0700 Subject: [PATCH 137/138] cxl/region: Introduce SPA to DPA address translation commit dc181170491bda9944f95ca39017667fe7fd767d upstream. Add infrastructure to translate System Physical Addresses (SPA) to Device Physical Addresses (DPA) within CXL regions. This capability will be used by follow-on patches that add poison inject and clear operations at the region level. The SPA-to-DPA translation process follows these steps: 1. Apply root decoder transformations (SPA to HPA) if configured. 2. Extract the position in region interleave from the HPA offset. 3. Extract the DPA offset from the HPA offset. 4. Use position to find endpoint decoder. 5. Use endpoint decoder to find memdev and calculate DPA from offset. 6. Return the result - a memdev and a DPA. It is Step 1 above that makes this a driver level operation and not work we can push to user space. Rather than exporting the XOR maps for root decoders configured with XOR interleave, the driver performs this complex calculation for the user. Steps 2 and 3 follow the CXL Spec 3.2 Section 8.2.4.20.13 Implementation Note: Device Decode Logic. These calculations mirror much of the logic introduced earlier in DPA to SPA translation, see cxl_dpa_to_hpa(), where the driver needed to reverse the spec defined 'Device Decode Logic'. Signed-off-by: Alison Schofield Reviewed-by: Jonathan Cameron Link: https://patch.msgid.link/422f0e27742c6ca9a11f7cd83e6ba9fa1a8d0c74.1754290144.git.alison.schofield@intel.com Signed-off-by: Dave Jiang Signed-off-by: Bo Li --- drivers/cxl/core/region.c | 101 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) diff --git a/drivers/cxl/core/region.c b/drivers/cxl/core/region.c index f4ecf5d2ea6ed..1f69b414b9cba 100644 --- a/drivers/cxl/core/region.c +++ b/drivers/cxl/core/region.c @@ -2854,6 +2854,11 @@ static bool has_hpa_to_spa(struct cxl_root_decoder *cxlrd) return cxlrd->ops && cxlrd->ops->hpa_to_spa; } +static bool has_spa_to_hpa(struct cxl_root_decoder *cxlrd) +{ + return cxlrd->ops && cxlrd->ops->spa_to_hpa; +} + u64 cxl_dpa_to_hpa(struct cxl_region *cxlr, const struct cxl_memdev *cxlmd, u64 dpa) { @@ -2924,6 +2929,102 @@ u64 cxl_dpa_to_hpa(struct cxl_region *cxlr, const struct cxl_memdev *cxlmd, return hpa; } +struct dpa_result { + struct cxl_memdev *cxlmd; + u64 dpa; +}; + +static int __maybe_unused region_offset_to_dpa_result(struct cxl_region *cxlr, + u64 offset, + struct dpa_result *result) +{ + struct cxl_region_params *p = &cxlr->params; + struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent); + struct cxl_endpoint_decoder *cxled; + u64 hpa, hpa_offset, dpa_offset; + u64 bits_upper, bits_lower; + u64 shifted, rem, temp; + u16 eig = 0; + u8 eiw = 0; + int pos; + + lockdep_assert_held(&cxl_region_rwsem); + lockdep_assert_held(&cxl_dpa_rwsem); + + /* Input validation ensures valid ways and gran */ + granularity_to_eig(p->interleave_granularity, &eig); + ways_to_eiw(p->interleave_ways, &eiw); + + /* + * If the root decoder has SPA to CXL HPA callback, use it. Otherwise + * CXL HPA is assumed to equal SPA. + */ + if (has_spa_to_hpa(cxlrd)) { + hpa = cxlrd->ops->spa_to_hpa(cxlrd, p->res->start + offset); + hpa_offset = hpa - p->res->start; + } else { + hpa_offset = offset; + } + /* + * Interleave position: CXL Spec 3.2 Section 8.2.4.20.13 + * eiw < 8 + * Position is in the IW bits at HPA_OFFSET[IG+8+IW-1:IG+8]. + * Per spec "remove IW bits starting with bit position IG+8" + * eiw >= 8 + * Position is not explicitly stored in HPA_OFFSET bits. It is + * derived from the modulo operation of the upper bits using + * the total number of interleave ways. + */ + if (eiw < 8) { + pos = (hpa_offset >> (eig + 8)) & GENMASK(eiw - 1, 0); + } else { + shifted = hpa_offset >> (eig + 8); + div64_u64_rem(shifted, p->interleave_ways, &rem); + pos = rem; + } + if (pos < 0 || pos >= p->nr_targets) { + dev_dbg(&cxlr->dev, "Invalid position %d for %d targets\n", + pos, p->nr_targets); + return -ENXIO; + } + + /* + * DPA offset: CXL Spec 3.2 Section 8.2.4.20.13 + * Lower bits [IG+7:0] pass through unchanged + * (eiw < 8) + * Per spec: DPAOffset[51:IG+8] = (HPAOffset[51:IG+IW+8] >> IW) + * Clear the position bits to isolate upper section, then + * reverse the left shift by eiw that occurred during DPA->HPA + * (eiw >= 8) + * Per spec: DPAOffset[51:IG+8] = HPAOffset[51:IG+IW] / 3 + * Extract upper bits from the correct bit range and divide by 3 + * to recover the original DPA upper bits + */ + bits_lower = hpa_offset & GENMASK_ULL(eig + 7, 0); + if (eiw < 8) { + temp = hpa_offset &= ~((u64)GENMASK(eig + eiw + 8 - 1, 0)); + dpa_offset = temp >> eiw; + } else { + bits_upper = div64_u64(hpa_offset >> (eig + eiw), 3); + dpa_offset = bits_upper << (eig + 8); + } + dpa_offset |= bits_lower; + + /* Look-up and return the result: a memdev and a DPA */ + for (int i = 0; i < p->nr_targets; i++) { + cxled = p->targets[i]; + if (cxled->pos != pos) + continue; + result->cxlmd = cxled_to_memdev(cxled); + result->dpa = cxl_dpa_resource_start(cxled) + dpa_offset; + + return 0; + } + dev_err(&cxlr->dev, "No device found for position %d\n", pos); + + return -ENXIO; +} + static struct lock_class_key cxl_pmem_region_key; static struct cxl_pmem_region *cxl_pmem_region_alloc(struct cxl_region *cxlr) From 7c1b5a35750af68451a59f330ee22e7080433511 Mon Sep 17 00:00:00 2001 From: Terry Bowman Date: Fri, 5 Jun 2026 13:06:10 -0500 Subject: [PATCH 138/138] cxl: Fix CXL_HEADERLOG_SIZE to match RAS Capability size commit c268f949e219f9e179558e836f457f6c5fbec416 upstream. The CXL r4.0 8.2.4.17.7 RAS Capability Structure has total length 0x58 bytes (CXL_RAS_CAPABILITY_LENGTH); the Header Log occupies the trailing 64 bytes at offset 0x18. CXL_HEADERLOG_SIZE was defined as SZ_512, eight times the actual on-device size. header_log_copy() reads CXL_HEADERLOG_SIZE_U32 (128) dwords from the RAS capability iomap, overrunning the 88-byte mapping by 448 bytes. The cxl_aer_uncorrectable_error trace event memcpy()s CXL_HEADERLOG_SIZE (512) bytes from its source. For the CPER caller the source is struct cxl_ras_capability_regs::header_log[16] (64 bytes) embedded in a stack-local cxl_cper_prot_err_work_data, so the memcpy reads 448 bytes of kernel stack into the trace event ring buffer where userspace can read it via tracefs. Set CXL_HEADERLOG_SIZE to 64 and derive CXL_HEADERLOG_SIZE_U32 from it, bringing all iomap readers into agreement on 16 dwords. Userspace tools such as rasdaemon have grown a dependency on the buggy 512-byte (128 u32) header_log layout in the cxl_aer_uncorrectable_error trace event. Add CXL_HEADERLOG_TRACE_SIZE_U32 = 128 and use it for the trace event __array and its memcpy to preserve that ABI. Both callers now pass a zero-filled u32[CXL_HEADERLOG_TRACE_SIZE_U32] staging buffer with only the first CXL_HEADERLOG_SIZE_U32 (16) entries populated from hardware; the remaining 112 u32s are zero-padded, keeping the 512-byte trace ring buffer layout intact. [ dj: Replaced 64 with SZ_64 per RichardC ] Fixes: 36f257e3b0ba ("acpi/ghes, cxl/pci: Process CXL CPER Protocol Errors") Fixes: 2905cb5236cb ("cxl/pci: Add (hopeful) error handling support") Cc: stable@vger.kernel.org Reported-by: Sashiko Signed-off-by: Terry Bowman Reviewed-by: Alison Schofield Reviewed-by: Dave Jiang Reviewed-by: Ben Cheatham Reviewed-by: Richard Cheng Link: https://patch.msgid.link/20260605180610.2249458-1-terry.bowman@amd.com Signed-off-by: Dave Jiang Signed-off-by: Bo Li --- drivers/cxl/core/ras.c | 21 +++++++++++++++++---- drivers/cxl/core/trace.h | 24 ++++++++++++++++-------- drivers/cxl/cxl.h | 14 ++++++++++++-- 3 files changed, 45 insertions(+), 14 deletions(-) diff --git a/drivers/cxl/core/ras.c b/drivers/cxl/core/ras.c index 2731ba3a07993..7d9006ec94491 100644 --- a/drivers/cxl/core/ras.c +++ b/drivers/cxl/core/ras.c @@ -7,6 +7,10 @@ #include #include "trace.h" +/* Check that UCE header definition is maintained to keep ABI intact */ +static_assert(CXL_HEADERLOG_TRACE_SIZE_U32 == 128, + "rasdaemon ABI requires exactly 128 u32s"); + static void cxl_cper_trace_corr_port_prot_err(struct pci_dev *pdev, struct cxl_ras_capability_regs ras_cap) { @@ -18,6 +22,7 @@ static void cxl_cper_trace_corr_port_prot_err(struct pci_dev *pdev, static void cxl_cper_trace_uncorr_port_prot_err(struct pci_dev *pdev, struct cxl_ras_capability_regs ras_cap) { + u32 hl[CXL_HEADERLOG_TRACE_SIZE_U32] = {}; u32 status = ras_cap.uncor_status & ~ras_cap.uncor_mask; u32 fe; @@ -27,8 +32,8 @@ static void cxl_cper_trace_uncorr_port_prot_err(struct pci_dev *pdev, else fe = status; - trace_cxl_port_aer_uncorrectable_error(&pdev->dev, status, fe, - ras_cap.header_log); + memcpy(hl, ras_cap.header_log, CXL_HEADERLOG_SIZE); + trace_cxl_port_aer_uncorrectable_error(&pdev->dev, status, fe, hl); } static void cxl_cper_trace_corr_prot_err(struct cxl_memdev *cxlmd, @@ -43,6 +48,7 @@ static void cxl_cper_trace_uncorr_prot_err(struct cxl_memdev *cxlmd, struct cxl_ras_capability_regs ras_cap) { + u32 hl[CXL_HEADERLOG_TRACE_SIZE_U32] = {}; u32 status = ras_cap.uncor_status & ~ras_cap.uncor_mask; u32 fe; @@ -52,8 +58,15 @@ cxl_cper_trace_uncorr_prot_err(struct cxl_memdev *cxlmd, else fe = status; - trace_cxl_aer_uncorrectable_error(cxlmd, status, fe, - ras_cap.header_log); + /* + * ras_cap.header_log[] holds CXL_HEADERLOG_SIZE_U32 (16) hardware + * dwords. Copy them into the front of a zero-filled + * CXL_HEADERLOG_TRACE_SIZE_U32 (128) u32 staging buffer so the trace + * event memcpy sees a full 512-byte source and the userspace ABI + * (rasdaemon) is preserved. + */ + memcpy(hl, ras_cap.header_log, CXL_HEADERLOG_SIZE); + trace_cxl_aer_uncorrectable_error(cxlmd, status, fe, hl); } static int match_memdev_by_parent(struct device *dev, const void *uport) diff --git a/drivers/cxl/core/trace.h b/drivers/cxl/core/trace.h index 51d949de4e6b3..08595e6869b40 100644 --- a/drivers/cxl/core/trace.h +++ b/drivers/cxl/core/trace.h @@ -56,7 +56,7 @@ TRACE_EVENT(cxl_port_aer_uncorrectable_error, __string(host, dev_name(dev->parent)) __field(u32, status) __field(u32, first_error) - __array(u32, header_log, CXL_HEADERLOG_SIZE_U32) + __array(u32, header_log, CXL_HEADERLOG_TRACE_SIZE_U32) ), TP_fast_assign( __assign_str(device, dev_name(dev)); @@ -64,10 +64,14 @@ TRACE_EVENT(cxl_port_aer_uncorrectable_error, __entry->status = status; __entry->first_error = fe; /* - * Embed the 512B headerlog data for user app retrieval and - * parsing, but no need to print this in the trace buffer. + * Embed headerlog data for user app retrieval and parsing, + * but no need to print in the trace buffer. Only + * CXL_HEADERLOG_SIZE_U32 (16) dwords are hardware data; + * the remaining entries preserve the 512-byte ABI layout + * rasdaemon depends on and are zero-filled by the caller. */ - memcpy(__entry->header_log, hl, CXL_HEADERLOG_SIZE); + memcpy(__entry->header_log, hl, + CXL_HEADERLOG_TRACE_SIZE_U32 * sizeof(u32)); ), TP_printk("device=%s host=%s status: '%s' first_error: '%s'", __get_str(device), __get_str(host), @@ -85,7 +89,7 @@ TRACE_EVENT(cxl_aer_uncorrectable_error, __field(u64, serial) __field(u32, status) __field(u32, first_error) - __array(u32, header_log, CXL_HEADERLOG_SIZE_U32) + __array(u32, header_log, CXL_HEADERLOG_TRACE_SIZE_U32) ), TP_fast_assign( __assign_str(memdev, dev_name(&cxlmd->dev)); @@ -94,10 +98,14 @@ TRACE_EVENT(cxl_aer_uncorrectable_error, __entry->status = status; __entry->first_error = fe; /* - * Embed the 512B headerlog data for user app retrieval and - * parsing, but no need to print this in the trace buffer. + * Embed headerlog data for user app retrieval and parsing, + * but no need to print in the trace buffer. Only + * CXL_HEADERLOG_SIZE_U32 (16) dwords are hardware data; + * the remaining entries preserve the 512-byte ABI layout + * rasdaemon depends on and are zero-filled by the caller. */ - memcpy(__entry->header_log, hl, CXL_HEADERLOG_SIZE); + memcpy(__entry->header_log, hl, + CXL_HEADERLOG_TRACE_SIZE_U32 * sizeof(u32)); ), TP_printk("memdev=%s host=%s serial=%lld: status: '%s' first_error: '%s'", __get_str(memdev), __get_str(host), __entry->serial, diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h index 0c3ab0952dd72..0c08f2ac0cbe2 100644 --- a/drivers/cxl/cxl.h +++ b/drivers/cxl/cxl.h @@ -145,8 +145,18 @@ static inline int ways_to_eiw(unsigned int ways, u8 *eiw) #define CXL_RAS_CAP_CONTROL_FE_MASK GENMASK(5, 0) #define CXL_RAS_HEADER_LOG_OFFSET 0x18 #define CXL_RAS_CAPABILITY_LENGTH 0x58 -#define CXL_HEADERLOG_SIZE SZ_512 -#define CXL_HEADERLOG_SIZE_U32 SZ_512 / sizeof(u32) +#define CXL_HEADERLOG_SIZE SZ_64 +#define CXL_HEADERLOG_SIZE_U32 (CXL_HEADERLOG_SIZE / sizeof(u32)) + +/* + * The RAS UCE trace event header array was originally sized at SZ_512/sizeof(u32) + * = 128 u32s due to a bug. Userspace tools (rasdaemon) have grown a dependency + * on that 512-byte layout. Keep the trace array at 128 u32s to preserve the + * ABI; only CXL_HEADERLOG_SIZE_U32 (16) dwords are valid hardware data, the + * remainder are zero-filled. + */ +#define CXL_HEADERLOG_TRACE_SIZE SZ_512 +#define CXL_HEADERLOG_TRACE_SIZE_U32 (CXL_HEADERLOG_TRACE_SIZE / sizeof(u32)) /* CXL 2.0 8.2.8.1 Device Capabilities Array Register */ #define CXLDEV_CAP_ARRAY_OFFSET 0x0