From 5f50d6b20ca31d91de14cc09be3e5ce67bc99e04 Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Thu, 13 May 2021 22:21:49 -0700 Subject: [PATCH 01/34] cxl/mem: Move some definitions to mem.h In preparation for sharing cxl.h with other generic CXL consumers, move / consolidate some of the memory device specifics to mem.h. The motivation for moving out of cxl.h is to maintain least privilege access to memory-device details since cxl.h is used in multiple files. The motivation for moving definitions into a new mem.h header is for code readability and organization. I.e. minimize implementation details when reading data structures and other definitions. Reviewed-by: Ben Widawsky Reviewed-by: Jonathan Cameron Link: https://lore.kernel.org/r/162096970932.1865304.14510894426562947262.stgit@dwillia2-desk3.amr.corp.intel.com Signed-off-by: Dan Williams --- drivers/cxl/cxl.h | 57 --------------------------------- drivers/cxl/mem.c | 21 +----------- drivers/cxl/mem.h | 81 +++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 82 insertions(+), 77 deletions(-) create mode 100644 drivers/cxl/mem.h diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h index 6f14838c2d25..2e3bdacb32e7 100644 --- a/drivers/cxl/cxl.h +++ b/drivers/cxl/cxl.h @@ -34,62 +34,5 @@ #define CXLDEV_MBOX_BG_CMD_STATUS_OFFSET 0x18 #define CXLDEV_MBOX_PAYLOAD_OFFSET 0x20 -/* CXL 2.0 8.2.8.5.1.1 Memory Device Status Register */ -#define CXLMDEV_STATUS_OFFSET 0x0 -#define CXLMDEV_DEV_FATAL BIT(0) -#define CXLMDEV_FW_HALT BIT(1) -#define CXLMDEV_STATUS_MEDIA_STATUS_MASK GENMASK(3, 2) -#define CXLMDEV_MS_NOT_READY 0 -#define CXLMDEV_MS_READY 1 -#define CXLMDEV_MS_ERROR 2 -#define CXLMDEV_MS_DISABLED 3 -#define CXLMDEV_READY(status) \ - (FIELD_GET(CXLMDEV_STATUS_MEDIA_STATUS_MASK, status) == \ - CXLMDEV_MS_READY) -#define CXLMDEV_MBOX_IF_READY BIT(4) -#define CXLMDEV_RESET_NEEDED_MASK GENMASK(7, 5) -#define CXLMDEV_RESET_NEEDED_NOT 0 -#define CXLMDEV_RESET_NEEDED_COLD 1 -#define CXLMDEV_RESET_NEEDED_WARM 2 -#define CXLMDEV_RESET_NEEDED_HOT 3 -#define CXLMDEV_RESET_NEEDED_CXL 4 -#define CXLMDEV_RESET_NEEDED(status) \ - (FIELD_GET(CXLMDEV_RESET_NEEDED_MASK, status) != \ - CXLMDEV_RESET_NEEDED_NOT) - -struct cxl_memdev; -/** - * struct cxl_mem - A CXL memory device - * @pdev: The PCI device associated with this CXL device. - * @regs: IO mappings to the device's MMIO - * @status_regs: CXL 2.0 8.2.8.3 Device Status Registers - * @mbox_regs: CXL 2.0 8.2.8.4 Mailbox Registers - * @memdev_regs: CXL 2.0 8.2.8.5 Memory Device Registers - * @payload_size: Size of space for payload - * (CXL 2.0 8.2.8.4.3 Mailbox Capabilities Register) - * @mbox_mutex: Mutex to synchronize mailbox access. - * @firmware_version: Firmware version for the memory device. - * @enabled_commands: Hardware commands found enabled in CEL. - * @pmem_range: Persistent memory capacity information. - * @ram_range: Volatile memory capacity information. - */ -struct cxl_mem { - struct pci_dev *pdev; - void __iomem *regs; - struct cxl_memdev *cxlmd; - - void __iomem *status_regs; - void __iomem *mbox_regs; - void __iomem *memdev_regs; - - size_t payload_size; - struct mutex mbox_mutex; /* Protects device mailbox and firmware */ - char firmware_version[0x10]; - unsigned long *enabled_cmds; - - struct range pmem_range; - struct range ram_range; -}; - extern struct bus_type cxl_bus_type; #endif /* __CXL_H__ */ diff --git a/drivers/cxl/mem.c b/drivers/cxl/mem.c index 2acc6173da36..53933d7d8d12 100644 --- a/drivers/cxl/mem.c +++ b/drivers/cxl/mem.c @@ -13,6 +13,7 @@ #include #include "pci.h" #include "cxl.h" +#include "mem.h" /** * DOC: cxl mem @@ -30,12 +31,6 @@ * - Handle and manage error conditions. */ -/* - * An entire PCI topology full of devices should be enough for any - * config - */ -#define CXL_MEM_MAX_DEVS 65536 - #define cxl_doorbell_busy(cxlm) \ (readl((cxlm)->mbox_regs + CXLDEV_MBOX_CTRL_OFFSET) & \ CXLDEV_MBOX_CTRL_DOORBELL) @@ -92,20 +87,6 @@ struct mbox_cmd { #define CXL_MBOX_SUCCESS 0 }; -/** - * struct cxl_memdev - CXL bus object representing a Type-3 Memory Device - * @dev: driver core device object - * @cdev: char dev core object for ioctl operations - * @cxlm: pointer to the parent device driver data - * @id: id number of this memdev instance. - */ -struct cxl_memdev { - struct device dev; - struct cdev cdev; - struct cxl_mem *cxlm; - int id; -}; - static int cxl_mem_major; static DEFINE_IDA(cxl_memdev_ida); static DECLARE_RWSEM(cxl_memdev_rwsem); diff --git a/drivers/cxl/mem.h b/drivers/cxl/mem.h new file mode 100644 index 000000000000..451db0984b92 --- /dev/null +++ b/drivers/cxl/mem.h @@ -0,0 +1,81 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright(c) 2020-2021 Intel Corporation. */ +#ifndef __CXL_MEM_H__ +#define __CXL_MEM_H__ + +/* CXL 2.0 8.2.8.5.1.1 Memory Device Status Register */ +#define CXLMDEV_STATUS_OFFSET 0x0 +#define CXLMDEV_DEV_FATAL BIT(0) +#define CXLMDEV_FW_HALT BIT(1) +#define CXLMDEV_STATUS_MEDIA_STATUS_MASK GENMASK(3, 2) +#define CXLMDEV_MS_NOT_READY 0 +#define CXLMDEV_MS_READY 1 +#define CXLMDEV_MS_ERROR 2 +#define CXLMDEV_MS_DISABLED 3 +#define CXLMDEV_READY(status) \ + (FIELD_GET(CXLMDEV_STATUS_MEDIA_STATUS_MASK, status) == \ + CXLMDEV_MS_READY) +#define CXLMDEV_MBOX_IF_READY BIT(4) +#define CXLMDEV_RESET_NEEDED_MASK GENMASK(7, 5) +#define CXLMDEV_RESET_NEEDED_NOT 0 +#define CXLMDEV_RESET_NEEDED_COLD 1 +#define CXLMDEV_RESET_NEEDED_WARM 2 +#define CXLMDEV_RESET_NEEDED_HOT 3 +#define CXLMDEV_RESET_NEEDED_CXL 4 +#define CXLMDEV_RESET_NEEDED(status) \ + (FIELD_GET(CXLMDEV_RESET_NEEDED_MASK, status) != \ + CXLMDEV_RESET_NEEDED_NOT) + +/* + * An entire PCI topology full of devices should be enough for any + * config + */ +#define CXL_MEM_MAX_DEVS 65536 + +/** + * struct cxl_memdev - CXL bus object representing a Type-3 Memory Device + * @dev: driver core device object + * @cdev: char dev core object for ioctl operations + * @cxlm: pointer to the parent device driver data + * @id: id number of this memdev instance. + */ +struct cxl_memdev { + struct device dev; + struct cdev cdev; + struct cxl_mem *cxlm; + int id; +}; + +/** + * struct cxl_mem - A CXL memory device + * @pdev: The PCI device associated with this CXL device. + * @regs: IO mappings to the device's MMIO + * @status_regs: CXL 2.0 8.2.8.3 Device Status Registers + * @mbox_regs: CXL 2.0 8.2.8.4 Mailbox Registers + * @memdev_regs: CXL 2.0 8.2.8.5 Memory Device Registers + * @payload_size: Size of space for payload + * (CXL 2.0 8.2.8.4.3 Mailbox Capabilities Register) + * @mbox_mutex: Mutex to synchronize mailbox access. + * @firmware_version: Firmware version for the memory device. + * @enabled_cmds: Hardware commands found enabled in CEL. + * @pmem_range: Persistent memory capacity information. + * @ram_range: Volatile memory capacity information. + */ +struct cxl_mem { + struct pci_dev *pdev; + void __iomem *regs; + struct cxl_memdev *cxlmd; + + void __iomem *status_regs; + void __iomem *mbox_regs; + void __iomem *memdev_regs; + + size_t payload_size; + struct mutex mbox_mutex; /* Protects device mailbox and firmware */ + char firmware_version[0x10]; + unsigned long *enabled_cmds; + + struct range pmem_range; + struct range ram_range; +}; +#endif /* __CXL_MEM_H__ */ From 8ac75dd6ab3039ef0656d777a564ea1b65071971 Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Thu, 13 May 2021 22:21:54 -0700 Subject: [PATCH 02/34] cxl/mem: Introduce 'struct cxl_regs' for "composable" CXL devices CXL MMIO register blocks are organized by device type and capabilities. There are Component registers, Device registers (yes, an ambiguous name), and Memory Device registers (a specific extension of Device registers). It is possible for a given device instance (endpoint or port) to implement register sets from multiple of the above categories. The driver code that enumerates and maps the registers is type specific so it is useful to have a dedicated type and helpers for each block type. At the same time, once the registers are mapped the origin type does not matter. It is overly pedantic to reference the register block type in code that is using the registers. In preparation for the endpoint driver to incorporate Component registers into its MMIO operations reorganize the registers to allow typed enumeration + mapping, but anonymous usage. With the end state of 'struct cxl_regs' to be: struct cxl_regs { union { struct { CXL_DEVICE_REGS(); }; struct cxl_device_regs device_regs; }; union { struct { CXL_COMPONENT_REGS(); }; struct cxl_component_regs component_regs; }; }; With this arrangement the driver can share component init code with ports, but when using the registers it can directly reference the component register block type by name without the 'component_regs' prefix. So, map + enumerate can be shared across drivers of different CXL classes e.g.: void cxl_setup_device_regs(struct device *dev, void __iomem *base, struct cxl_device_regs *regs); void cxl_setup_component_regs(struct device *dev, void __iomem *base, struct cxl_component_regs *regs); ...while inline usage in the driver need not indicate where the registers came from: readl(cxlm->regs.mbox + MBOX_OFFSET); readl(cxlm->regs.hdm + HDM_OFFSET); ...instead of: readl(cxlm->regs.device_regs.mbox + MBOX_OFFSET); readl(cxlm->regs.component_regs.hdm + HDM_OFFSET); This complexity of the definition in .h yields improvement in code readability in .c while maintaining type-safety for organization of setup code. It prepares the implementation to maintain organization in the face of CXL devices that compose register interfaces consisting of multiple types. Given that this new container is named 'regs' rename the common register base pointer @base, and fixup the kernel-doc for the missing @cxlmd description. Reviewed-by: Ben Widawsky Reviewed-by: Jonathan Cameron Cc: Christoph Hellwig Link: https://lore.kernel.org/r/162096971451.1865304.13540251513463515153.stgit@dwillia2-desk3.amr.corp.intel.com Signed-off-by: Dan Williams --- drivers/cxl/cxl.h | 32 ++++++++++++++++++++++++++++++++ drivers/cxl/mem.c | 44 ++++++++++++++++++++++++-------------------- drivers/cxl/mem.h | 13 +++++-------- 3 files changed, 61 insertions(+), 28 deletions(-) diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h index 2e3bdacb32e7..1f3434f89ef2 100644 --- a/drivers/cxl/cxl.h +++ b/drivers/cxl/cxl.h @@ -34,5 +34,37 @@ #define CXLDEV_MBOX_BG_CMD_STATUS_OFFSET 0x18 #define CXLDEV_MBOX_PAYLOAD_OFFSET 0x20 +/* + * CXL_DEVICE_REGS - Common set of CXL Device register block base pointers + * @status: CXL 2.0 8.2.8.3 Device Status Registers + * @mbox: CXL 2.0 8.2.8.4 Mailbox Registers + * @memdev: CXL 2.0 8.2.8.5 Memory Device Registers + */ +#define CXL_DEVICE_REGS() \ + void __iomem *status; \ + void __iomem *mbox; \ + void __iomem *memdev + +/* See note for 'struct cxl_regs' for the rationale of this organization */ +struct cxl_device_regs { + CXL_DEVICE_REGS(); +}; + +/* + * Note, the anonymous union organization allows for per + * register-block-type helper routines, without requiring block-type + * agnostic code to include the prefix. I.e. + * cxl_setup_device_regs(&cxlm->regs.dev) vs readl(cxlm->regs.mbox). + * The specificity reads naturally from left-to-right. + */ +struct cxl_regs { + union { + struct { + CXL_DEVICE_REGS(); + }; + struct cxl_device_regs device_regs; + }; +}; + extern struct bus_type cxl_bus_type; #endif /* __CXL_H__ */ diff --git a/drivers/cxl/mem.c b/drivers/cxl/mem.c index 53933d7d8d12..ddc94c7bd422 100644 --- a/drivers/cxl/mem.c +++ b/drivers/cxl/mem.c @@ -32,7 +32,7 @@ */ #define cxl_doorbell_busy(cxlm) \ - (readl((cxlm)->mbox_regs + CXLDEV_MBOX_CTRL_OFFSET) & \ + (readl((cxlm)->regs.mbox + CXLDEV_MBOX_CTRL_OFFSET) & \ CXLDEV_MBOX_CTRL_DOORBELL) /* CXL 2.0 - 8.2.8.4 */ @@ -273,7 +273,7 @@ static void cxl_mem_mbox_timeout(struct cxl_mem *cxlm, static int __cxl_mem_mbox_send_cmd(struct cxl_mem *cxlm, struct mbox_cmd *mbox_cmd) { - void __iomem *payload = cxlm->mbox_regs + CXLDEV_MBOX_PAYLOAD_OFFSET; + void __iomem *payload = cxlm->regs.mbox + CXLDEV_MBOX_PAYLOAD_OFFSET; u64 cmd_reg, status_reg; size_t out_len; int rc; @@ -316,12 +316,12 @@ static int __cxl_mem_mbox_send_cmd(struct cxl_mem *cxlm, } /* #2, #3 */ - writeq(cmd_reg, cxlm->mbox_regs + CXLDEV_MBOX_CMD_OFFSET); + writeq(cmd_reg, cxlm->regs.mbox + CXLDEV_MBOX_CMD_OFFSET); /* #4 */ dev_dbg(&cxlm->pdev->dev, "Sending command\n"); writel(CXLDEV_MBOX_CTRL_DOORBELL, - cxlm->mbox_regs + CXLDEV_MBOX_CTRL_OFFSET); + cxlm->regs.mbox + CXLDEV_MBOX_CTRL_OFFSET); /* #5 */ rc = cxl_mem_wait_for_doorbell(cxlm); @@ -331,7 +331,7 @@ static int __cxl_mem_mbox_send_cmd(struct cxl_mem *cxlm, } /* #6 */ - status_reg = readq(cxlm->mbox_regs + CXLDEV_MBOX_STATUS_OFFSET); + status_reg = readq(cxlm->regs.mbox + CXLDEV_MBOX_STATUS_OFFSET); mbox_cmd->return_code = FIELD_GET(CXLDEV_MBOX_STATUS_RET_CODE_MASK, status_reg); @@ -341,7 +341,7 @@ static int __cxl_mem_mbox_send_cmd(struct cxl_mem *cxlm, } /* #7 */ - cmd_reg = readq(cxlm->mbox_regs + CXLDEV_MBOX_CMD_OFFSET); + cmd_reg = readq(cxlm->regs.mbox + CXLDEV_MBOX_CMD_OFFSET); out_len = FIELD_GET(CXLDEV_MBOX_CMD_PAYLOAD_LENGTH_MASK, cmd_reg); /* #8 */ @@ -402,7 +402,7 @@ static int cxl_mem_mbox_get(struct cxl_mem *cxlm) goto out; } - md_status = readq(cxlm->memdev_regs + CXLMDEV_STATUS_OFFSET); + md_status = readq(cxlm->regs.memdev + CXLMDEV_STATUS_OFFSET); if (!(md_status & CXLMDEV_MBOX_IF_READY && CXLMDEV_READY(md_status))) { dev_err(dev, "mbox: reported doorbell ready, but not mbox ready\n"); rc = -EBUSY; @@ -887,7 +887,7 @@ static int cxl_mem_setup_regs(struct cxl_mem *cxlm) int cap, cap_count; u64 cap_array; - cap_array = readq(cxlm->regs + CXLDEV_CAP_ARRAY_OFFSET); + cap_array = readq(cxlm->base + CXLDEV_CAP_ARRAY_OFFSET); if (FIELD_GET(CXLDEV_CAP_ARRAY_ID_MASK, cap_array) != CXLDEV_CAP_ARRAY_CAP_ID) return -ENODEV; @@ -900,25 +900,25 @@ static int cxl_mem_setup_regs(struct cxl_mem *cxlm) u16 cap_id; cap_id = FIELD_GET(CXLDEV_CAP_HDR_CAP_ID_MASK, - readl(cxlm->regs + cap * 0x10)); - offset = readl(cxlm->regs + cap * 0x10 + 0x4); - register_block = cxlm->regs + offset; + readl(cxlm->base + cap * 0x10)); + offset = readl(cxlm->base + cap * 0x10 + 0x4); + register_block = cxlm->base + offset; switch (cap_id) { case CXLDEV_CAP_CAP_ID_DEVICE_STATUS: dev_dbg(dev, "found Status capability (0x%x)\n", offset); - cxlm->status_regs = register_block; + cxlm->regs.status = register_block; break; case CXLDEV_CAP_CAP_ID_PRIMARY_MAILBOX: dev_dbg(dev, "found Mailbox capability (0x%x)\n", offset); - cxlm->mbox_regs = register_block; + cxlm->regs.mbox = register_block; break; case CXLDEV_CAP_CAP_ID_SECONDARY_MAILBOX: dev_dbg(dev, "found Secondary Mailbox capability (0x%x)\n", offset); break; case CXLDEV_CAP_CAP_ID_MEMDEV: dev_dbg(dev, "found Memory Device capability (0x%x)\n", offset); - cxlm->memdev_regs = register_block; + cxlm->regs.memdev = register_block; break; default: dev_dbg(dev, "Unknown cap ID: %d (0x%x)\n", cap_id, offset); @@ -926,11 +926,11 @@ static int cxl_mem_setup_regs(struct cxl_mem *cxlm) } } - if (!cxlm->status_regs || !cxlm->mbox_regs || !cxlm->memdev_regs) { + if (!cxlm->regs.status || !cxlm->regs.mbox || !cxlm->regs.memdev) { dev_err(dev, "registers not found: %s%s%s\n", - !cxlm->status_regs ? "status " : "", - !cxlm->mbox_regs ? "mbox " : "", - !cxlm->memdev_regs ? "memdev" : ""); + !cxlm->regs.status ? "status " : "", + !cxlm->regs.mbox ? "mbox " : "", + !cxlm->regs.memdev ? "memdev" : ""); return -ENXIO; } @@ -939,7 +939,7 @@ static int cxl_mem_setup_regs(struct cxl_mem *cxlm) static int cxl_mem_setup_mailbox(struct cxl_mem *cxlm) { - const int cap = readl(cxlm->mbox_regs + CXLDEV_MBOX_CAPS_OFFSET); + const int cap = readl(cxlm->regs.mbox + CXLDEV_MBOX_CAPS_OFFSET); cxlm->payload_size = 1 << FIELD_GET(CXLDEV_MBOX_CAP_PAYLOAD_SIZE_MASK, cap); @@ -999,7 +999,7 @@ static struct cxl_mem *cxl_mem_create(struct pci_dev *pdev, u32 reg_lo, mutex_init(&cxlm->mbox_mutex); cxlm->pdev = pdev; - cxlm->regs = regs + offset; + cxlm->base = regs + offset; cxlm->enabled_cmds = devm_kmalloc_array(dev, BITS_TO_LONGS(cxl_cmd_count), sizeof(unsigned long), @@ -1525,6 +1525,10 @@ static __init int cxl_mem_init(void) dev_t devt; int rc; + /* Double check the anonymous union trickery in struct cxl_regs */ + BUILD_BUG_ON(offsetof(struct cxl_regs, memdev) != + offsetof(struct cxl_regs, device_regs.memdev)); + rc = alloc_chrdev_region(&devt, 0, CXL_MEM_MAX_DEVS, "cxl"); if (rc) return rc; diff --git a/drivers/cxl/mem.h b/drivers/cxl/mem.h index 451db0984b92..0a3f70316872 100644 --- a/drivers/cxl/mem.h +++ b/drivers/cxl/mem.h @@ -49,10 +49,9 @@ struct cxl_memdev { /** * struct cxl_mem - A CXL memory device * @pdev: The PCI device associated with this CXL device. - * @regs: IO mappings to the device's MMIO - * @status_regs: CXL 2.0 8.2.8.3 Device Status Registers - * @mbox_regs: CXL 2.0 8.2.8.4 Mailbox Registers - * @memdev_regs: CXL 2.0 8.2.8.5 Memory Device Registers + * @base: IO mappings to the device's MMIO + * @cxlmd: Logical memory device chardev / interface + * @regs: Parsed register blocks * @payload_size: Size of space for payload * (CXL 2.0 8.2.8.4.3 Mailbox Capabilities Register) * @mbox_mutex: Mutex to synchronize mailbox access. @@ -63,12 +62,10 @@ struct cxl_memdev { */ struct cxl_mem { struct pci_dev *pdev; - void __iomem *regs; + void __iomem *base; struct cxl_memdev *cxlmd; - void __iomem *status_regs; - void __iomem *mbox_regs; - void __iomem *memdev_regs; + struct cxl_regs regs; size_t payload_size; struct mutex mbox_mutex; /* Protects device mailbox and firmware */ From 5f653f7590ab7db7379f668b2975744585206b0d Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Thu, 13 May 2021 22:22:00 -0700 Subject: [PATCH 03/34] cxl/core: Rename bus.c to core.c In preparation for more generic shared functionality across endpoint consumers of core cxl resources, and platform-firmware producers of those resources, rename bus.c to core.c. In addition to the central rendezvous for interleave coordination, the core will also define common routines like CXL register block mapping. Acked-by: Ben Widawsky Reviewed-by: Jonathan Cameron Link: https://lore.kernel.org/r/162096972018.1865304.11079951161445408423.stgit@dwillia2-desk3.amr.corp.intel.com Signed-off-by: Dan Williams --- Documentation/driver-api/cxl/memory-devices.rst | 6 +++--- drivers/cxl/Makefile | 4 ++-- drivers/cxl/{bus.c => core.c} | 15 ++++++++------- 3 files changed, 13 insertions(+), 12 deletions(-) rename drivers/cxl/{bus.c => core.c} (55%) diff --git a/Documentation/driver-api/cxl/memory-devices.rst b/Documentation/driver-api/cxl/memory-devices.rst index 1bad466f9167..71495ed77069 100644 --- a/Documentation/driver-api/cxl/memory-devices.rst +++ b/Documentation/driver-api/cxl/memory-devices.rst @@ -28,10 +28,10 @@ CXL Memory Device .. kernel-doc:: drivers/cxl/mem.c :internal: -CXL Bus +CXL Core ------- -.. kernel-doc:: drivers/cxl/bus.c - :doc: cxl bus +.. kernel-doc:: drivers/cxl/core.c + :doc: cxl core External Interfaces =================== diff --git a/drivers/cxl/Makefile b/drivers/cxl/Makefile index a314a1891f4d..3808e39dd31f 100644 --- a/drivers/cxl/Makefile +++ b/drivers/cxl/Makefile @@ -1,7 +1,7 @@ # SPDX-License-Identifier: GPL-2.0 -obj-$(CONFIG_CXL_BUS) += cxl_bus.o +obj-$(CONFIG_CXL_BUS) += cxl_core.o obj-$(CONFIG_CXL_MEM) += cxl_mem.o ccflags-y += -DDEFAULT_SYMBOL_NAMESPACE=CXL -cxl_bus-y := bus.o +cxl_core-y := core.o cxl_mem-y := mem.o diff --git a/drivers/cxl/bus.c b/drivers/cxl/core.c similarity index 55% rename from drivers/cxl/bus.c rename to drivers/cxl/core.c index 58f74796d525..7f8d2034038a 100644 --- a/drivers/cxl/bus.c +++ b/drivers/cxl/core.c @@ -4,26 +4,27 @@ #include /** - * DOC: cxl bus + * DOC: cxl core * - * The CXL bus provides namespace for control devices and a rendezvous - * point for cross-device interleave coordination. + * The CXL core provides a sysfs hierarchy for control devices and a rendezvous + * point for cross-device interleave coordination through cxl ports. */ + struct bus_type cxl_bus_type = { .name = "cxl", }; EXPORT_SYMBOL_GPL(cxl_bus_type); -static __init int cxl_bus_init(void) +static __init int cxl_core_init(void) { return bus_register(&cxl_bus_type); } -static void cxl_bus_exit(void) +static void cxl_core_exit(void) { bus_unregister(&cxl_bus_type); } -module_init(cxl_bus_init); -module_exit(cxl_bus_exit); +module_init(cxl_core_init); +module_exit(cxl_core_exit); MODULE_LICENSE("GPL v2"); From 399d34ebc2483c6091a587e5905c6ed34116fb05 Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Thu, 13 May 2021 22:22:05 -0700 Subject: [PATCH 04/34] cxl/core: Refactor CXL register lookup for bridge reuse While CXL Memory Device endpoints locate the CXL MMIO registers in a PCI BAR, CXL root bridges have their MMIO base address described by platform firmware. Refactor the existing register lookup into a generic facility for endpoints and bridges to share. Reviewed-by: Ben Widawsky Reviewed-by: Jonathan Cameron Link: https://lore.kernel.org/r/162096972534.1865304.3218686216153688039.stgit@dwillia2-desk3.amr.corp.intel.com Signed-off-by: Dan Williams --- drivers/cxl/core.c | 57 ++++++++++++++++++++++++++++++++++++++++++++++ drivers/cxl/cxl.h | 3 +++ drivers/cxl/mem.c | 50 +++++----------------------------------- 3 files changed, 66 insertions(+), 44 deletions(-) diff --git a/drivers/cxl/core.c b/drivers/cxl/core.c index 7f8d2034038a..84b90db57420 100644 --- a/drivers/cxl/core.c +++ b/drivers/cxl/core.c @@ -1,7 +1,9 @@ // SPDX-License-Identifier: GPL-2.0-only /* Copyright(c) 2020 Intel Corporation. All rights reserved. */ +#include #include #include +#include "cxl.h" /** * DOC: cxl core @@ -10,6 +12,61 @@ * point for cross-device interleave coordination through cxl ports. */ +/** + * cxl_setup_device_regs() - Detect CXL Device register blocks + * @dev: Host device of the @base mapping + * @base: Mapping of CXL 2.0 8.2.8 CXL Device Register Interface + * @regs: Base pointers for device register blocks (see CXL_DEVICE_REGS()) + */ +void cxl_setup_device_regs(struct device *dev, void __iomem *base, + struct cxl_device_regs *regs) +{ + int cap, cap_count; + u64 cap_array; + + *regs = (struct cxl_device_regs) { 0 }; + + cap_array = readq(base + CXLDEV_CAP_ARRAY_OFFSET); + if (FIELD_GET(CXLDEV_CAP_ARRAY_ID_MASK, cap_array) != + CXLDEV_CAP_ARRAY_CAP_ID) + return; + + cap_count = FIELD_GET(CXLDEV_CAP_ARRAY_COUNT_MASK, cap_array); + + for (cap = 1; cap <= cap_count; cap++) { + void __iomem *register_block; + u32 offset; + u16 cap_id; + + cap_id = FIELD_GET(CXLDEV_CAP_HDR_CAP_ID_MASK, + readl(base + cap * 0x10)); + offset = readl(base + cap * 0x10 + 0x4); + register_block = base + offset; + + switch (cap_id) { + case CXLDEV_CAP_CAP_ID_DEVICE_STATUS: + dev_dbg(dev, "found Status capability (0x%x)\n", offset); + regs->status = register_block; + break; + case CXLDEV_CAP_CAP_ID_PRIMARY_MAILBOX: + dev_dbg(dev, "found Mailbox capability (0x%x)\n", offset); + regs->mbox = register_block; + break; + case CXLDEV_CAP_CAP_ID_SECONDARY_MAILBOX: + dev_dbg(dev, "found Secondary Mailbox capability (0x%x)\n", offset); + break; + case CXLDEV_CAP_CAP_ID_MEMDEV: + dev_dbg(dev, "found Memory Device capability (0x%x)\n", offset); + regs->memdev = register_block; + break; + default: + dev_dbg(dev, "Unknown cap ID: %d (0x%x)\n", cap_id, offset); + break; + } + } +} +EXPORT_SYMBOL_GPL(cxl_setup_device_regs); + struct bus_type cxl_bus_type = { .name = "cxl", }; diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h index 1f3434f89ef2..d49e0cb679fa 100644 --- a/drivers/cxl/cxl.h +++ b/drivers/cxl/cxl.h @@ -66,5 +66,8 @@ struct cxl_regs { }; }; +void cxl_setup_device_regs(struct device *dev, void __iomem *base, + struct cxl_device_regs *regs); + extern struct bus_type cxl_bus_type; #endif /* __CXL_H__ */ diff --git a/drivers/cxl/mem.c b/drivers/cxl/mem.c index ddc94c7bd422..c5fdf2c57181 100644 --- a/drivers/cxl/mem.c +++ b/drivers/cxl/mem.c @@ -884,53 +884,15 @@ static int cxl_mem_mbox_send_cmd(struct cxl_mem *cxlm, u16 opcode, static int cxl_mem_setup_regs(struct cxl_mem *cxlm) { struct device *dev = &cxlm->pdev->dev; - int cap, cap_count; - u64 cap_array; + struct cxl_regs *regs = &cxlm->regs; - cap_array = readq(cxlm->base + CXLDEV_CAP_ARRAY_OFFSET); - if (FIELD_GET(CXLDEV_CAP_ARRAY_ID_MASK, cap_array) != - CXLDEV_CAP_ARRAY_CAP_ID) - return -ENODEV; + cxl_setup_device_regs(dev, cxlm->base, ®s->device_regs); - cap_count = FIELD_GET(CXLDEV_CAP_ARRAY_COUNT_MASK, cap_array); - - for (cap = 1; cap <= cap_count; cap++) { - void __iomem *register_block; - u32 offset; - u16 cap_id; - - cap_id = FIELD_GET(CXLDEV_CAP_HDR_CAP_ID_MASK, - readl(cxlm->base + cap * 0x10)); - offset = readl(cxlm->base + cap * 0x10 + 0x4); - register_block = cxlm->base + offset; - - switch (cap_id) { - case CXLDEV_CAP_CAP_ID_DEVICE_STATUS: - dev_dbg(dev, "found Status capability (0x%x)\n", offset); - cxlm->regs.status = register_block; - break; - case CXLDEV_CAP_CAP_ID_PRIMARY_MAILBOX: - dev_dbg(dev, "found Mailbox capability (0x%x)\n", offset); - cxlm->regs.mbox = register_block; - break; - case CXLDEV_CAP_CAP_ID_SECONDARY_MAILBOX: - dev_dbg(dev, "found Secondary Mailbox capability (0x%x)\n", offset); - break; - case CXLDEV_CAP_CAP_ID_MEMDEV: - dev_dbg(dev, "found Memory Device capability (0x%x)\n", offset); - cxlm->regs.memdev = register_block; - break; - default: - dev_dbg(dev, "Unknown cap ID: %d (0x%x)\n", cap_id, offset); - break; - } - } - - if (!cxlm->regs.status || !cxlm->regs.mbox || !cxlm->regs.memdev) { + if (!regs->status || !regs->mbox || !regs->memdev) { dev_err(dev, "registers not found: %s%s%s\n", - !cxlm->regs.status ? "status " : "", - !cxlm->regs.mbox ? "mbox " : "", - !cxlm->regs.memdev ? "memdev" : ""); + !regs->status ? "status " : "", + !regs->mbox ? "mbox " : "", + !regs->memdev ? "memdev" : ""); return -ENXIO; } From 35c32e3095d396c750f5cdfdaa94cba83d9b23c6 Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Thu, 20 May 2021 12:52:20 -0700 Subject: [PATCH 05/34] cxl/docs: Fix "Title underline too short" warning When "Bus" was renamed to "Core" the header underline update was missed. Reported-by: Stephen Rothwell Fixes: 5f653f7590ab ("cxl/core: Rename bus.c to core.c") Reviewed-by: Vishal Verma Link: https://lore.kernel.org/r/162154034053.1995075.17047445540000243300.stgit@dwillia2-desk3.amr.corp.intel.com Signed-off-by: Dan Williams --- Documentation/driver-api/cxl/memory-devices.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/driver-api/cxl/memory-devices.rst b/Documentation/driver-api/cxl/memory-devices.rst index 71495ed77069..73f6b246c583 100644 --- a/Documentation/driver-api/cxl/memory-devices.rst +++ b/Documentation/driver-api/cxl/memory-devices.rst @@ -29,7 +29,7 @@ CXL Memory Device :internal: CXL Core -------- +-------- .. kernel-doc:: drivers/cxl/core.c :doc: cxl core From 21e9f76733a8c152b794cba5463ff9bf2db919d4 Mon Sep 17 00:00:00 2001 From: Ben Widawsky Date: Wed, 26 May 2021 10:44:13 -0700 Subject: [PATCH 06/34] cxl: Rename mem to pci As the driver has undergone development, it's become clear that the majority [entirety?] of the current functionality in mem.c is actually a layer encapsulating functionality exposed through PCI based interactions. This layer can be used either in isolation or to provide functionality for higher level functionality. CXL capabilities exist in a parallel domain to PCIe. CXL devices are enumerable and controllable via "legacy" PCIe mechanisms; however, their CXL capabilities are a superset of PCIe. For example, a CXL device may be connected to a non-CXL capable PCIe root port, and therefore will not be able to participate in CXL.mem or CXL.cache operations, but can still be accessed through PCIe mechanisms for CXL.io operations. To properly represent the PCI nature of this driver, and in preparation for introducing a new driver for the CXL.mem / HDM decoder (Host-managed Device Memory) capabilities of a CXL memory expander, rename mem.c to pci.c so that mem.c is available for this new driver. The result of the change is that there is a clear layering distinction in the driver, and a systems administrator may load only the cxl_pci module and gain access to such operations as, firmware update, offline provisioning of devices, and error collection. In addition to freeing up the file name for another purpose, there are two primary reasons this is useful, 1. Acting upon devices which don't have full CXL capabilities. This may happen for instance if the CXL device is connected in a CXL unaware part of the platform topology. 2. Userspace-first provisioning for devices without kernel driver interference. This may be useful when provisioning a new device in a specific manner that might otherwise be blocked or prevented by the real CXL mem driver. Reviewed-by: Dan Williams Signed-off-by: Ben Widawsky Link: https://lore.kernel.org/r/20210526174413.802913-1-ben.widawsky@intel.com Signed-off-by: Dan Williams --- Documentation/driver-api/cxl/memory-devices.rst | 6 +++--- drivers/cxl/Kconfig | 13 ++++--------- drivers/cxl/Makefile | 4 ++-- drivers/cxl/{mem.c => pci.c} | 9 ++++----- 4 files changed, 13 insertions(+), 19 deletions(-) rename drivers/cxl/{mem.c => pci.c} (99%) diff --git a/Documentation/driver-api/cxl/memory-devices.rst b/Documentation/driver-api/cxl/memory-devices.rst index 73f6b246c583..44c8ddbc8415 100644 --- a/Documentation/driver-api/cxl/memory-devices.rst +++ b/Documentation/driver-api/cxl/memory-devices.rst @@ -22,10 +22,10 @@ This section covers the driver infrastructure for a CXL memory device. CXL Memory Device ----------------- -.. kernel-doc:: drivers/cxl/mem.c - :doc: cxl mem +.. kernel-doc:: drivers/cxl/pci.c + :doc: cxl pci -.. kernel-doc:: drivers/cxl/mem.c +.. kernel-doc:: drivers/cxl/pci.c :internal: CXL Core diff --git a/drivers/cxl/Kconfig b/drivers/cxl/Kconfig index 97dc4d751651..5483ba92b6da 100644 --- a/drivers/cxl/Kconfig +++ b/drivers/cxl/Kconfig @@ -21,15 +21,10 @@ config CXL_MEM as if the memory was attached to the typical CPU memory controller. - Say 'y/m' to enable a driver (named "cxl_mem.ko" when built as - a module) that will attach to CXL.mem devices for - configuration, provisioning, and health monitoring. This - driver is required for dynamic provisioning of CXL.mem - attached memory which is a prerequisite for persistent memory - support. Typically volatile memory is mapped by platform - firmware and included in the platform memory map, but in some - cases the OS is responsible for mapping that memory. See - Chapter 2.3 Type 3 CXL Device in the CXL 2.0 specification. + Say 'y/m' to enable a driver that will attach to CXL.mem devices for + configuration and management primarily via the mailbox interface. See + Chapter 2.3 Type 3 CXL Device in the CXL 2.0 specification for more + details. If unsure say 'm'. diff --git a/drivers/cxl/Makefile b/drivers/cxl/Makefile index 3808e39dd31f..d9d282dc15be 100644 --- a/drivers/cxl/Makefile +++ b/drivers/cxl/Makefile @@ -1,7 +1,7 @@ # SPDX-License-Identifier: GPL-2.0 obj-$(CONFIG_CXL_BUS) += cxl_core.o -obj-$(CONFIG_CXL_MEM) += cxl_mem.o +obj-$(CONFIG_CXL_MEM) += cxl_pci.o ccflags-y += -DDEFAULT_SYMBOL_NAMESPACE=CXL cxl_core-y := core.o -cxl_mem-y := mem.o +cxl_pci-y := pci.o diff --git a/drivers/cxl/mem.c b/drivers/cxl/pci.c similarity index 99% rename from drivers/cxl/mem.c rename to drivers/cxl/pci.c index c5fdf2c57181..c7996c2a2054 100644 --- a/drivers/cxl/mem.c +++ b/drivers/cxl/pci.c @@ -16,10 +16,11 @@ #include "mem.h" /** - * DOC: cxl mem + * DOC: cxl pci * - * This implements a CXL memory device ("type-3") as it is defined by the - * Compute Express Link specification. + * This implements the PCI exclusive functionality for a CXL device as it is + * defined by the Compute Express Link specification. CXL devices may surface + * certain functionality even if it isn't CXL enabled. * * The driver has several responsibilities, mainly: * - Create the memX device and register on the CXL bus. @@ -27,8 +28,6 @@ * - Probe the device attributes to establish sysfs interface. * - Provide an IOCTL interface to userspace to communicate with the device for * things like firmware update. - * - Support management of interleave sets. - * - Handle and manage error conditions. */ #define cxl_doorbell_busy(cxlm) \ From 199cf8c3feec2947015da84643312790b21531cb Mon Sep 17 00:00:00 2001 From: Vishal Verma Date: Thu, 20 May 2021 13:47:45 -0600 Subject: [PATCH 07/34] cxl/pci.c: Add a 'label_storage_size' attribute to the memdev The 'Identify Device' mailbox command returns an 'lsa_size', which is the size of the label storage area on the device. Export it as a sysfs attribute so that userspace tooling to read/write the LSA can determine the size without having to run an 'Identify Device' on their own. Cc: Ben Widawsky Cc: Dan Williams Signed-off-by: Vishal Verma Reviewed-by: Dan Williams Link: https://lore.kernel.org/r/20210520194745.1095517-1-vishal.l.verma@intel.com Signed-off-by: Dan Williams --- drivers/cxl/mem.h | 3 +++ drivers/cxl/pci.c | 12 ++++++++++++ 2 files changed, 15 insertions(+) diff --git a/drivers/cxl/mem.h b/drivers/cxl/mem.h index 0a3f70316872..23fc40dde27e 100644 --- a/drivers/cxl/mem.h +++ b/drivers/cxl/mem.h @@ -54,6 +54,8 @@ struct cxl_memdev { * @regs: Parsed register blocks * @payload_size: Size of space for payload * (CXL 2.0 8.2.8.4.3 Mailbox Capabilities Register) + * @lsa_size: Size of Label Storage Area + * (CXL 2.0 8.2.9.5.1.1 Identify Memory Device) * @mbox_mutex: Mutex to synchronize mailbox access. * @firmware_version: Firmware version for the memory device. * @enabled_cmds: Hardware commands found enabled in CEL. @@ -68,6 +70,7 @@ struct cxl_mem { struct cxl_regs regs; size_t payload_size; + size_t lsa_size; struct mutex mbox_mutex; /* Protects device mailbox and firmware */ char firmware_version[0x10]; unsigned long *enabled_cmds; diff --git a/drivers/cxl/pci.c b/drivers/cxl/pci.c index c7996c2a2054..606c88ca902b 100644 --- a/drivers/cxl/pci.c +++ b/drivers/cxl/pci.c @@ -1036,6 +1036,16 @@ static ssize_t payload_max_show(struct device *dev, } static DEVICE_ATTR_RO(payload_max); +static ssize_t label_storage_size_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct cxl_memdev *cxlmd = to_cxl_memdev(dev); + struct cxl_mem *cxlm = cxlmd->cxlm; + + return sysfs_emit(buf, "%zu\n", cxlm->lsa_size); +} +static DEVICE_ATTR_RO(label_storage_size); + static ssize_t ram_size_show(struct device *dev, struct device_attribute *attr, char *buf) { @@ -1065,6 +1075,7 @@ static struct device_attribute dev_attr_pmem_size = static struct attribute *cxl_memdev_attributes[] = { &dev_attr_firmware_version.attr, &dev_attr_payload_max.attr, + &dev_attr_label_storage_size.attr, NULL, }; @@ -1397,6 +1408,7 @@ static int cxl_mem_identify(struct cxl_mem *cxlm) cxlm->pmem_range.end = le64_to_cpu(id.persistent_capacity) * SZ_256M - 1; + cxlm->lsa_size = le32_to_cpu(id.lsa_size); memcpy(cxlm->firmware_version, id.fw_revision, sizeof(id.fw_revision)); return 0; From dd2a93a814e7ffbd4d2c3e361f4609c9bd384a96 Mon Sep 17 00:00:00 2001 From: Ben Widawsky Date: Thu, 20 May 2021 13:48:52 -0700 Subject: [PATCH 08/34] cxl/mem: Demarcate vendor specific capability IDs Vendor capabilities occupy 0x8000 to 0xFFFF according to CXL 2.0 spec 8.2.8.2.1 CXL Device Capabilities. While they are not defined by the spec, they are allowed and not "unknown". Call this detail out in the logs to let users easily distinguish the difference. This patch is a squash of two earlier patches and take in some minor suggestions from both Vishal and Dan. Cc: Vishal Verma Cc: Dan Williams Signed-off-by: Ben Widawsky Reviewed-by: Vishal Verma Link: https://lore.kernel.org/r/20210520204852.1070780-1-ben.widawsky@intel.com Signed-off-by: Dan Williams --- drivers/cxl/core.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/drivers/cxl/core.c b/drivers/cxl/core.c index 84b90db57420..38979c97158d 100644 --- a/drivers/cxl/core.c +++ b/drivers/cxl/core.c @@ -60,7 +60,10 @@ void cxl_setup_device_regs(struct device *dev, void __iomem *base, regs->memdev = register_block; break; default: - dev_dbg(dev, "Unknown cap ID: %d (0x%x)\n", cap_id, offset); + if (cap_id >= 0x8000) + dev_dbg(dev, "Vendor cap ID: %#x offset: %#x\n", cap_id, offset); + else + dev_dbg(dev, "Unknown cap ID: %#x offset: %#x\n", cap_id, offset); break; } } From 5d0c6f02595310a17762755bb6f015786b8900db Mon Sep 17 00:00:00 2001 From: Ben Widawsky Date: Wed, 7 Apr 2021 15:26:19 -0700 Subject: [PATCH 09/34] cxl/mem: Use dev instead of pdev->dev Trivial cleanup. Signed-off-by: Ben Widawsky Acked-by: Jonathan Cameron Link: https://lore.kernel.org/r/20210407222625.320177-2-ben.widawsky@intel.com Signed-off-by: Dan Williams --- drivers/cxl/pci.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/cxl/pci.c b/drivers/cxl/pci.c index 606c88ca902b..e478fbfac3be 100644 --- a/drivers/cxl/pci.c +++ b/drivers/cxl/pci.c @@ -935,7 +935,7 @@ static struct cxl_mem *cxl_mem_create(struct pci_dev *pdev, u32 reg_lo, u8 bar; int rc; - cxlm = devm_kzalloc(&pdev->dev, sizeof(*cxlm), GFP_KERNEL); + cxlm = devm_kzalloc(dev, sizeof(*cxlm), GFP_KERNEL); if (!cxlm) { dev_err(dev, "No memory available\n"); return NULL; From 1b0a1a2a193400d305931a40ac68e89bcfdad9c5 Mon Sep 17 00:00:00 2001 From: Ben Widawsky Date: Wed, 7 Apr 2021 15:26:20 -0700 Subject: [PATCH 10/34] cxl/mem: Split creation from mapping in probe Add a new function specifically for mapping the register blocks and offsets within. The new function can be used more generically for other register block identifiers. No functional change is meant to be introduced in this patch with the exception of a dev_err printed when the device register block isn't found. Signed-off-by: Ben Widawsky Reviewed-by: Jonathan Cameron Link: https://lore.kernel.org/r/20210407222625.320177-3-ben.widawsky@intel.com Signed-off-by: Dan Williams --- drivers/cxl/pci.c | 64 +++++++++++++++++++++++++++++------------------ 1 file changed, 40 insertions(+), 24 deletions(-) diff --git a/drivers/cxl/pci.c b/drivers/cxl/pci.c index e478fbfac3be..be6936c88148 100644 --- a/drivers/cxl/pci.c +++ b/drivers/cxl/pci.c @@ -925,22 +925,40 @@ static int cxl_mem_setup_mailbox(struct cxl_mem *cxlm) return 0; } -static struct cxl_mem *cxl_mem_create(struct pci_dev *pdev, u32 reg_lo, - u32 reg_hi) +static struct cxl_mem *cxl_mem_create(struct pci_dev *pdev) { struct device *dev = &pdev->dev; struct cxl_mem *cxlm; - void __iomem *regs; - u64 offset; - u8 bar; - int rc; cxlm = devm_kzalloc(dev, sizeof(*cxlm), GFP_KERNEL); if (!cxlm) { dev_err(dev, "No memory available\n"); - return NULL; + return ERR_PTR(-ENOMEM); } + mutex_init(&cxlm->mbox_mutex); + cxlm->pdev = pdev; + cxlm->enabled_cmds = + devm_kmalloc_array(dev, BITS_TO_LONGS(cxl_cmd_count), + sizeof(unsigned long), + GFP_KERNEL | __GFP_ZERO); + if (!cxlm->enabled_cmds) { + dev_err(dev, "No memory available for bitmap\n"); + return ERR_PTR(-ENOMEM); + } + + return cxlm; +} + +static int cxl_mem_map_regblock(struct cxl_mem *cxlm, u32 reg_lo, u32 reg_hi) +{ + struct pci_dev *pdev = cxlm->pdev; + struct device *dev = &pdev->dev; + void __iomem *regs; + u64 offset; + u8 bar; + int rc; + offset = ((u64)reg_hi << 32) | (reg_lo & CXL_REGLOC_ADDR_MASK); bar = FIELD_GET(CXL_REGLOC_BIR_MASK, reg_lo); @@ -948,30 +966,20 @@ static struct cxl_mem *cxl_mem_create(struct pci_dev *pdev, u32 reg_lo, if (pci_resource_len(pdev, bar) < offset) { dev_err(dev, "BAR%d: %pr: too small (offset: %#llx)\n", bar, &pdev->resource[bar], (unsigned long long)offset); - return NULL; + return -ENXIO; } rc = pcim_iomap_regions(pdev, BIT(bar), pci_name(pdev)); if (rc) { dev_err(dev, "failed to map registers\n"); - return NULL; + return rc; } regs = pcim_iomap_table(pdev)[bar]; - mutex_init(&cxlm->mbox_mutex); - cxlm->pdev = pdev; cxlm->base = regs + offset; - cxlm->enabled_cmds = - devm_kmalloc_array(dev, BITS_TO_LONGS(cxl_cmd_count), - sizeof(unsigned long), - GFP_KERNEL | __GFP_ZERO); - if (!cxlm->enabled_cmds) { - dev_err(dev, "No memory available for bitmap\n"); - return NULL; - } dev_dbg(dev, "Mapped CXL Memory Device resource\n"); - return cxlm; + return 0; } static int cxl_mem_dvsec(struct pci_dev *pdev, int dvsec) @@ -1417,14 +1425,18 @@ static int cxl_mem_identify(struct cxl_mem *cxlm) static int cxl_mem_probe(struct pci_dev *pdev, const struct pci_device_id *id) { struct device *dev = &pdev->dev; - struct cxl_mem *cxlm = NULL; u32 regloc_size, regblocks; + struct cxl_mem *cxlm; int rc, regloc, i; rc = pcim_enable_device(pdev); if (rc) return rc; + cxlm = cxl_mem_create(pdev); + if (IS_ERR(cxlm)) + return PTR_ERR(cxlm); + regloc = cxl_mem_dvsec(pdev, PCI_DVSEC_ID_CXL_REGLOC_OFFSET); if (!regloc) { dev_err(dev, "register location dvsec not found\n"); @@ -1449,13 +1461,17 @@ static int cxl_mem_probe(struct pci_dev *pdev, const struct pci_device_id *id) reg_type = FIELD_GET(CXL_REGLOC_RBI_MASK, reg_lo); if (reg_type == CXL_REGLOC_RBI_MEMDEV) { - cxlm = cxl_mem_create(pdev, reg_lo, reg_hi); + rc = cxl_mem_map_regblock(cxlm, reg_lo, reg_hi); + if (rc) + return rc; break; } } - if (!cxlm) - return -ENODEV; + if (i == regblocks) { + dev_err(dev, "Missing register locator for device registers\n"); + return -ENXIO; + } rc = cxl_mem_setup_regs(cxlm); if (rc) From 1d5a4159074bde1b2d5e4a6f5ed34de70a83a39f Mon Sep 17 00:00:00 2001 From: Ben Widawsky Date: Wed, 7 Apr 2021 15:26:21 -0700 Subject: [PATCH 11/34] cxl/mem: Move register locator logic into reg setup Start moving code around to ultimately get rid of @cxlm.base. The @cxlm.base member serves no purpose other than intermediate storage of the offset found in cxl_mem_map_regblock() later used by cxl_mem_setup_regs(). Aside from wanting to get rid of this useless member, it will help later when adding new register block identifiers. While @cxlm.base still exists, it will become trivial to remove it in a future patch. No functional change is meant to be introduced in this patch. Signed-off-by: Ben Widawsky Reviewed-by: Jonathan Cameron Link: https://lore.kernel.org/r/20210407222625.320177-4-ben.widawsky@intel.com Signed-off-by: Dan Williams --- drivers/cxl/pci.c | 135 +++++++++++++++++++++++----------------------- 1 file changed, 68 insertions(+), 67 deletions(-) diff --git a/drivers/cxl/pci.c b/drivers/cxl/pci.c index be6936c88148..df6ccdcceea5 100644 --- a/drivers/cxl/pci.c +++ b/drivers/cxl/pci.c @@ -870,34 +870,6 @@ static int cxl_mem_mbox_send_cmd(struct cxl_mem *cxlm, u16 opcode, return 0; } -/** - * cxl_mem_setup_regs() - Setup necessary MMIO. - * @cxlm: The CXL memory device to communicate with. - * - * Return: 0 if all necessary registers mapped. - * - * A memory device is required by spec to implement a certain set of MMIO - * regions. The purpose of this function is to enumerate and map those - * registers. - */ -static int cxl_mem_setup_regs(struct cxl_mem *cxlm) -{ - struct device *dev = &cxlm->pdev->dev; - struct cxl_regs *regs = &cxlm->regs; - - cxl_setup_device_regs(dev, cxlm->base, ®s->device_regs); - - if (!regs->status || !regs->mbox || !regs->memdev) { - dev_err(dev, "registers not found: %s%s%s\n", - !regs->status ? "status " : "", - !regs->mbox ? "mbox " : "", - !regs->memdev ? "memdev" : ""); - return -ENXIO; - } - - return 0; -} - static int cxl_mem_setup_mailbox(struct cxl_mem *cxlm) { const int cap = readl(cxlm->regs.mbox + CXLDEV_MBOX_CAPS_OFFSET); @@ -1005,6 +977,73 @@ static int cxl_mem_dvsec(struct pci_dev *pdev, int dvsec) return 0; } +/** + * cxl_mem_setup_regs() - Setup necessary MMIO. + * @cxlm: The CXL memory device to communicate with. + * + * Return: 0 if all necessary registers mapped. + * + * A memory device is required by spec to implement a certain set of MMIO + * regions. The purpose of this function is to enumerate and map those + * registers. + */ +static int cxl_mem_setup_regs(struct cxl_mem *cxlm) +{ + struct cxl_regs *regs = &cxlm->regs; + struct pci_dev *pdev = cxlm->pdev; + struct device *dev = &pdev->dev; + u32 regloc_size, regblocks; + int rc, regloc, i; + + regloc = cxl_mem_dvsec(pdev, PCI_DVSEC_ID_CXL_REGLOC_OFFSET); + if (!regloc) { + dev_err(dev, "register location dvsec not found\n"); + return -ENXIO; + } + + /* Get the size of the Register Locator DVSEC */ + pci_read_config_dword(pdev, regloc + PCI_DVSEC_HEADER1, ®loc_size); + regloc_size = FIELD_GET(PCI_DVSEC_HEADER1_LENGTH_MASK, regloc_size); + + regloc += PCI_DVSEC_ID_CXL_REGLOC_BLOCK1_OFFSET; + regblocks = (regloc_size - PCI_DVSEC_ID_CXL_REGLOC_BLOCK1_OFFSET) / 8; + + for (i = 0; i < regblocks; i++, regloc += 8) { + u32 reg_lo, reg_hi; + u8 reg_type; + + /* "register low and high" contain other bits */ + pci_read_config_dword(pdev, regloc, ®_lo); + pci_read_config_dword(pdev, regloc + 4, ®_hi); + + reg_type = FIELD_GET(CXL_REGLOC_RBI_MASK, reg_lo); + + if (reg_type == CXL_REGLOC_RBI_MEMDEV) { + rc = cxl_mem_map_regblock(cxlm, reg_lo, reg_hi); + if (rc) + return rc; + break; + } + } + + if (i == regblocks) { + dev_err(dev, "Missing register locator for device registers\n"); + return -ENXIO; + } + + cxl_setup_device_regs(dev, cxlm->base, ®s->device_regs); + + if (!regs->status || !regs->mbox || !regs->memdev) { + dev_err(dev, "registers not found: %s%s%s\n", + !regs->status ? "status " : "", + !regs->mbox ? "mbox " : "", + !regs->memdev ? "memdev" : ""); + return -ENXIO; + } + + return 0; +} + static struct cxl_memdev *to_cxl_memdev(struct device *dev) { return container_of(dev, struct cxl_memdev, dev); @@ -1424,10 +1463,8 @@ static int cxl_mem_identify(struct cxl_mem *cxlm) static int cxl_mem_probe(struct pci_dev *pdev, const struct pci_device_id *id) { - struct device *dev = &pdev->dev; - u32 regloc_size, regblocks; struct cxl_mem *cxlm; - int rc, regloc, i; + int rc; rc = pcim_enable_device(pdev); if (rc) @@ -1437,42 +1474,6 @@ static int cxl_mem_probe(struct pci_dev *pdev, const struct pci_device_id *id) if (IS_ERR(cxlm)) return PTR_ERR(cxlm); - regloc = cxl_mem_dvsec(pdev, PCI_DVSEC_ID_CXL_REGLOC_OFFSET); - if (!regloc) { - dev_err(dev, "register location dvsec not found\n"); - return -ENXIO; - } - - /* Get the size of the Register Locator DVSEC */ - pci_read_config_dword(pdev, regloc + PCI_DVSEC_HEADER1, ®loc_size); - regloc_size = FIELD_GET(PCI_DVSEC_HEADER1_LENGTH_MASK, regloc_size); - - regloc += PCI_DVSEC_ID_CXL_REGLOC_BLOCK1_OFFSET; - regblocks = (regloc_size - PCI_DVSEC_ID_CXL_REGLOC_BLOCK1_OFFSET) / 8; - - for (i = 0; i < regblocks; i++, regloc += 8) { - u32 reg_lo, reg_hi; - u8 reg_type; - - /* "register low and high" contain other bits */ - pci_read_config_dword(pdev, regloc, ®_lo); - pci_read_config_dword(pdev, regloc + 4, ®_hi); - - reg_type = FIELD_GET(CXL_REGLOC_RBI_MASK, reg_lo); - - if (reg_type == CXL_REGLOC_RBI_MEMDEV) { - rc = cxl_mem_map_regblock(cxlm, reg_lo, reg_hi); - if (rc) - return rc; - break; - } - } - - if (i == regblocks) { - dev_err(dev, "Missing register locator for device registers\n"); - return -ENXIO; - } - rc = cxl_mem_setup_regs(cxlm); if (rc) return rc; From 6630d31c912ed2dfbc035caf0f54709b50ce779e Mon Sep 17 00:00:00 2001 From: Ben Widawsky Date: Thu, 20 May 2021 14:29:53 -0700 Subject: [PATCH 12/34] cxl/mem: Get rid of @cxlm.base @cxlm.base only existed to support holding the base found in the register block mapping code, and pass it along to the register setup code. Now that the register setup function has all logic around managing the registers, from DVSEC to iomapping up to populating our CXL specific information, it is easy to turn the @base values into local variables and remove them from our device driver state. Acked-by: Jonathan Cameron Signed-off-by: Ben Widawsky Link: https://lore.kernel.org/r/20210520212953.1181695-1-ben.widawsky@intel.com Signed-off-by: Dan Williams --- drivers/cxl/mem.h | 2 -- drivers/cxl/pci.c | 24 +++++++++++------------- 2 files changed, 11 insertions(+), 15 deletions(-) diff --git a/drivers/cxl/mem.h b/drivers/cxl/mem.h index 23fc40dde27e..13868ff7cadf 100644 --- a/drivers/cxl/mem.h +++ b/drivers/cxl/mem.h @@ -49,7 +49,6 @@ struct cxl_memdev { /** * struct cxl_mem - A CXL memory device * @pdev: The PCI device associated with this CXL device. - * @base: IO mappings to the device's MMIO * @cxlmd: Logical memory device chardev / interface * @regs: Parsed register blocks * @payload_size: Size of space for payload @@ -64,7 +63,6 @@ struct cxl_memdev { */ struct cxl_mem { struct pci_dev *pdev; - void __iomem *base; struct cxl_memdev *cxlmd; struct cxl_regs regs; diff --git a/drivers/cxl/pci.c b/drivers/cxl/pci.c index df6ccdcceea5..8bdae74d7d78 100644 --- a/drivers/cxl/pci.c +++ b/drivers/cxl/pci.c @@ -922,11 +922,10 @@ static struct cxl_mem *cxl_mem_create(struct pci_dev *pdev) return cxlm; } -static int cxl_mem_map_regblock(struct cxl_mem *cxlm, u32 reg_lo, u32 reg_hi) +static void __iomem *cxl_mem_map_regblock(struct cxl_mem *cxlm, u32 reg_lo, u32 reg_hi) { struct pci_dev *pdev = cxlm->pdev; struct device *dev = &pdev->dev; - void __iomem *regs; u64 offset; u8 bar; int rc; @@ -938,20 +937,18 @@ static int cxl_mem_map_regblock(struct cxl_mem *cxlm, u32 reg_lo, u32 reg_hi) if (pci_resource_len(pdev, bar) < offset) { dev_err(dev, "BAR%d: %pr: too small (offset: %#llx)\n", bar, &pdev->resource[bar], (unsigned long long)offset); - return -ENXIO; + return IOMEM_ERR_PTR(-ENXIO); } rc = pcim_iomap_regions(pdev, BIT(bar), pci_name(pdev)); if (rc) { dev_err(dev, "failed to map registers\n"); - return rc; + return IOMEM_ERR_PTR(rc); } - regs = pcim_iomap_table(pdev)[bar]; - - cxlm->base = regs + offset; dev_dbg(dev, "Mapped CXL Memory Device resource\n"); - return 0; + + return pcim_iomap_table(pdev)[bar] + offset; } static int cxl_mem_dvsec(struct pci_dev *pdev, int dvsec) @@ -993,7 +990,8 @@ static int cxl_mem_setup_regs(struct cxl_mem *cxlm) struct pci_dev *pdev = cxlm->pdev; struct device *dev = &pdev->dev; u32 regloc_size, regblocks; - int rc, regloc, i; + void __iomem *base; + int regloc, i; regloc = cxl_mem_dvsec(pdev, PCI_DVSEC_ID_CXL_REGLOC_OFFSET); if (!regloc) { @@ -1019,9 +1017,9 @@ static int cxl_mem_setup_regs(struct cxl_mem *cxlm) reg_type = FIELD_GET(CXL_REGLOC_RBI_MASK, reg_lo); if (reg_type == CXL_REGLOC_RBI_MEMDEV) { - rc = cxl_mem_map_regblock(cxlm, reg_lo, reg_hi); - if (rc) - return rc; + base = cxl_mem_map_regblock(cxlm, reg_lo, reg_hi); + if (IS_ERR(base)) + return PTR_ERR(base); break; } } @@ -1031,7 +1029,7 @@ static int cxl_mem_setup_regs(struct cxl_mem *cxlm) return -ENXIO; } - cxl_setup_device_regs(dev, cxlm->base, ®s->device_regs); + cxl_setup_device_regs(dev, base, ®s->device_regs); if (!regs->status || !regs->mbox || !regs->memdev) { dev_err(dev, "registers not found: %s%s%s\n", From 07d62eac422c5c8aec6ec1dacdc27423334b2d17 Mon Sep 17 00:00:00 2001 From: Ira Weiny Date: Thu, 27 May 2021 17:49:18 -0700 Subject: [PATCH 13/34] cxl/pci: Introduce cxl_decode_register_block() Each register block located in the DVSEC needs to be decoded from 2 words, 'register offset high' and 'register offset low'. Create a function, cxl_decode_register_block() to perform this decode and return the bar, offset, and register type of the register block. Then use the values decoded in cxl_mem_map_regblock() instead of passing the raw registers. Signed-off-by: Ira Weiny Link: https://lore.kernel.org/r/20210528004922.3980613-2-ira.weiny@intel.com Signed-off-by: Dan Williams --- drivers/cxl/pci.c | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/drivers/cxl/pci.c b/drivers/cxl/pci.c index 8bdae74d7d78..b2f978954daa 100644 --- a/drivers/cxl/pci.c +++ b/drivers/cxl/pci.c @@ -922,17 +922,13 @@ static struct cxl_mem *cxl_mem_create(struct pci_dev *pdev) return cxlm; } -static void __iomem *cxl_mem_map_regblock(struct cxl_mem *cxlm, u32 reg_lo, u32 reg_hi) +static void __iomem *cxl_mem_map_regblock(struct cxl_mem *cxlm, + u8 bar, u64 offset) { struct pci_dev *pdev = cxlm->pdev; struct device *dev = &pdev->dev; - u64 offset; - u8 bar; int rc; - offset = ((u64)reg_hi << 32) | (reg_lo & CXL_REGLOC_ADDR_MASK); - bar = FIELD_GET(CXL_REGLOC_BIR_MASK, reg_lo); - /* Basic sanity check that BAR is big enough */ if (pci_resource_len(pdev, bar) < offset) { dev_err(dev, "BAR%d: %pr: too small (offset: %#llx)\n", bar, @@ -974,6 +970,14 @@ static int cxl_mem_dvsec(struct pci_dev *pdev, int dvsec) return 0; } +static void cxl_decode_register_block(u32 reg_lo, u32 reg_hi, + u8 *bar, u64 *offset, u8 *reg_type) +{ + *offset = ((u64)reg_hi << 32) | (reg_lo & CXL_REGLOC_ADDR_MASK); + *bar = FIELD_GET(CXL_REGLOC_BIR_MASK, reg_lo); + *reg_type = FIELD_GET(CXL_REGLOC_RBI_MASK, reg_lo); +} + /** * cxl_mem_setup_regs() - Setup necessary MMIO. * @cxlm: The CXL memory device to communicate with. @@ -1009,15 +1013,21 @@ static int cxl_mem_setup_regs(struct cxl_mem *cxlm) for (i = 0; i < regblocks; i++, regloc += 8) { u32 reg_lo, reg_hi; u8 reg_type; + u64 offset; + u8 bar; /* "register low and high" contain other bits */ pci_read_config_dword(pdev, regloc, ®_lo); pci_read_config_dword(pdev, regloc + 4, ®_hi); - reg_type = FIELD_GET(CXL_REGLOC_RBI_MASK, reg_lo); + cxl_decode_register_block(reg_lo, reg_hi, &bar, &offset, + ®_type); + + dev_dbg(dev, "Found register block in bar %u @ 0x%llx of type %u\n", + bar, offset, reg_type); if (reg_type == CXL_REGLOC_RBI_MEMDEV) { - base = cxl_mem_map_regblock(cxlm, reg_lo, reg_hi); + base = cxl_mem_map_regblock(cxlm, bar, offset); if (IS_ERR(base)) return PTR_ERR(base); break; From f8a7e8c29be873b90fcc426e93bdb6184df5970e Mon Sep 17 00:00:00 2001 From: Ira Weiny Date: Thu, 27 May 2021 17:49:19 -0700 Subject: [PATCH 14/34] cxl/pci: Reserve all device regions at once In order to remap individual register sets each bar region must be reserved prior to mapping. Because the details of individual register sets are contained within the BARs themselves, the bar must be mapped 2 times, once to extract this information and a second time for each register set. Rather than attempt to reserve each BAR individually and track if that bar has been reserved. Open code pcim_iomap_regions() by first reserving all memory regions on the device and then mapping the bars individually as needed. NOTE pci_request_mem_regions() does not need a corresponding pci_release_mem_regions() because the pci device is managed via pcim_enable_device(). Reviewed-by: Jonathan Cameron Signed-off-by: Ira Weiny Link: https://lore.kernel.org/r/20210528004922.3980613-3-ira.weiny@intel.com Signed-off-by: Dan Williams --- drivers/cxl/pci.c | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/drivers/cxl/pci.c b/drivers/cxl/pci.c index b2f978954daa..33fc6e1634e3 100644 --- a/drivers/cxl/pci.c +++ b/drivers/cxl/pci.c @@ -927,7 +927,7 @@ static void __iomem *cxl_mem_map_regblock(struct cxl_mem *cxlm, { struct pci_dev *pdev = cxlm->pdev; struct device *dev = &pdev->dev; - int rc; + void __iomem *addr; /* Basic sanity check that BAR is big enough */ if (pci_resource_len(pdev, bar) < offset) { @@ -936,13 +936,14 @@ static void __iomem *cxl_mem_map_regblock(struct cxl_mem *cxlm, return IOMEM_ERR_PTR(-ENXIO); } - rc = pcim_iomap_regions(pdev, BIT(bar), pci_name(pdev)); - if (rc) { + addr = pcim_iomap(pdev, bar, 0); + if (!addr) { dev_err(dev, "failed to map registers\n"); - return IOMEM_ERR_PTR(rc); + return addr; } - dev_dbg(dev, "Mapped CXL Memory Device resource\n"); + dev_dbg(dev, "Mapped CXL Memory Device resource bar %u @ %#llx\n", + bar, offset); return pcim_iomap_table(pdev)[bar] + offset; } @@ -1003,6 +1004,9 @@ static int cxl_mem_setup_regs(struct cxl_mem *cxlm) return -ENXIO; } + if (pci_request_mem_regions(pdev, pci_name(pdev))) + return -ENODEV; + /* Get the size of the Register Locator DVSEC */ pci_read_config_dword(pdev, regloc + PCI_DVSEC_HEADER1, ®loc_size); regloc_size = FIELD_GET(PCI_DVSEC_HEADER1_LENGTH_MASK, regloc_size); @@ -1028,8 +1032,8 @@ static int cxl_mem_setup_regs(struct cxl_mem *cxlm) if (reg_type == CXL_REGLOC_RBI_MEMDEV) { base = cxl_mem_map_regblock(cxlm, bar, offset); - if (IS_ERR(base)) - return PTR_ERR(base); + if (!base) + return -ENOMEM; break; } } From 30af97296f48d84bc4a6abbaabb92c796a84ca57 Mon Sep 17 00:00:00 2001 From: Ira Weiny Date: Thu, 3 Jun 2021 17:50:36 -0700 Subject: [PATCH 15/34] cxl/pci: Map registers based on capabilities The information required to map registers based on capabilities is contained within the bars themselves. This means the bar must be mapped to read the information needed and then unmapped to map the individual parts of the BAR based on capabilities. Change cxl_setup_device_regs() to return a new cxl_register_map, change the name to cxl_probe_device_regs(). Allocate and place cxl_register_maps on a list while processing all of the specified register blocks. After probing all the register blocks go back and map smaller registers blocks based on their capabilities and dispose of the cxl_register_maps. NOTE: pci_iomap() is not managed automatically via pcim_enable_device() so be careful to call pci_iounmap() correctly. Reviewed-by: Jonathan Cameron Signed-off-by: Ira Weiny Link: https://lore.kernel.org/r/20210604005036.4187184-1-ira.weiny@intel.com Signed-off-by: Dan Williams --- drivers/cxl/core.c | 74 +++++++++++++++++++++++++----- drivers/cxl/cxl.h | 33 ++++++++++++-- drivers/cxl/pci.c | 111 ++++++++++++++++++++++++++++++++++++--------- 3 files changed, 180 insertions(+), 38 deletions(-) diff --git a/drivers/cxl/core.c b/drivers/cxl/core.c index 38979c97158d..f836aaab03e0 100644 --- a/drivers/cxl/core.c +++ b/drivers/cxl/core.c @@ -3,6 +3,7 @@ #include #include #include +#include #include "cxl.h" /** @@ -13,18 +14,20 @@ */ /** - * cxl_setup_device_regs() - Detect CXL Device register blocks + * cxl_probe_device_regs() - Detect CXL Device register blocks * @dev: Host device of the @base mapping * @base: Mapping of CXL 2.0 8.2.8 CXL Device Register Interface - * @regs: Base pointers for device register blocks (see CXL_DEVICE_REGS()) + * @map: Map object describing the register block information found + * + * Probe for device register information and return it in map object. */ -void cxl_setup_device_regs(struct device *dev, void __iomem *base, - struct cxl_device_regs *regs) +void cxl_probe_device_regs(struct device *dev, void __iomem *base, + struct cxl_device_reg_map *map) { int cap, cap_count; u64 cap_array; - *regs = (struct cxl_device_regs) { 0 }; + *map = (struct cxl_device_reg_map){ 0 }; cap_array = readq(base + CXLDEV_CAP_ARRAY_OFFSET); if (FIELD_GET(CXLDEV_CAP_ARRAY_ID_MASK, cap_array) != @@ -34,30 +37,36 @@ void cxl_setup_device_regs(struct device *dev, void __iomem *base, cap_count = FIELD_GET(CXLDEV_CAP_ARRAY_COUNT_MASK, cap_array); for (cap = 1; cap <= cap_count; cap++) { - void __iomem *register_block; - u32 offset; + u32 offset, length; u16 cap_id; cap_id = FIELD_GET(CXLDEV_CAP_HDR_CAP_ID_MASK, readl(base + cap * 0x10)); offset = readl(base + cap * 0x10 + 0x4); - register_block = base + offset; + length = readl(base + cap * 0x10 + 0x8); switch (cap_id) { case CXLDEV_CAP_CAP_ID_DEVICE_STATUS: dev_dbg(dev, "found Status capability (0x%x)\n", offset); - regs->status = register_block; + + map->status.valid = true; + map->status.offset = offset; + map->status.size = length; break; case CXLDEV_CAP_CAP_ID_PRIMARY_MAILBOX: dev_dbg(dev, "found Mailbox capability (0x%x)\n", offset); - regs->mbox = register_block; + map->mbox.valid = true; + map->mbox.offset = offset; + map->mbox.size = length; break; case CXLDEV_CAP_CAP_ID_SECONDARY_MAILBOX: dev_dbg(dev, "found Secondary Mailbox capability (0x%x)\n", offset); break; case CXLDEV_CAP_CAP_ID_MEMDEV: dev_dbg(dev, "found Memory Device capability (0x%x)\n", offset); - regs->memdev = register_block; + map->memdev.valid = true; + map->memdev.offset = offset; + map->memdev.size = length; break; default: if (cap_id >= 0x8000) @@ -68,7 +77,48 @@ void cxl_setup_device_regs(struct device *dev, void __iomem *base, } } } -EXPORT_SYMBOL_GPL(cxl_setup_device_regs); +EXPORT_SYMBOL_GPL(cxl_probe_device_regs); + +int cxl_map_device_regs(struct pci_dev *pdev, + struct cxl_device_regs *regs, + struct cxl_register_map *map) +{ + struct device *dev = &pdev->dev; + resource_size_t phys_addr; + + phys_addr = pci_resource_start(pdev, map->barno); + phys_addr += map->block_offset; + + if (map->device_map.status.valid) { + resource_size_t addr; + resource_size_t length; + + addr = phys_addr + map->device_map.status.offset; + length = map->device_map.status.size; + regs->status = devm_ioremap(dev, addr, length); + } + + if (map->device_map.mbox.valid) { + resource_size_t addr; + resource_size_t length; + + addr = phys_addr + map->device_map.mbox.offset; + length = map->device_map.mbox.size; + regs->mbox = devm_ioremap(dev, addr, length); + } + + if (map->device_map.memdev.valid) { + resource_size_t addr; + resource_size_t length; + + addr = phys_addr + map->device_map.memdev.offset; + length = map->device_map.memdev.size; + regs->memdev = devm_ioremap(dev, addr, length); + } + + return 0; +} +EXPORT_SYMBOL_GPL(cxl_map_device_regs); struct bus_type cxl_bus_type = { .name = "cxl", diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h index d49e0cb679fa..ae4b4c96c6b5 100644 --- a/drivers/cxl/cxl.h +++ b/drivers/cxl/cxl.h @@ -53,9 +53,7 @@ struct cxl_device_regs { /* * Note, the anonymous union organization allows for per * register-block-type helper routines, without requiring block-type - * agnostic code to include the prefix. I.e. - * cxl_setup_device_regs(&cxlm->regs.dev) vs readl(cxlm->regs.mbox). - * The specificity reads naturally from left-to-right. + * agnostic code to include the prefix. */ struct cxl_regs { union { @@ -66,8 +64,33 @@ struct cxl_regs { }; }; -void cxl_setup_device_regs(struct device *dev, void __iomem *base, - struct cxl_device_regs *regs); +struct cxl_reg_map { + bool valid; + unsigned long offset; + unsigned long size; +}; + +struct cxl_device_reg_map { + struct cxl_reg_map status; + struct cxl_reg_map mbox; + struct cxl_reg_map memdev; +}; + +struct cxl_register_map { + struct list_head list; + u64 block_offset; + u8 reg_type; + u8 barno; + union { + struct cxl_device_reg_map device_map; + }; +}; + +void cxl_probe_device_regs(struct device *dev, void __iomem *base, + struct cxl_device_reg_map *map); +int cxl_map_device_regs(struct pci_dev *pdev, + struct cxl_device_regs *regs, + struct cxl_register_map *map); extern struct bus_type cxl_bus_type; #endif /* __CXL_H__ */ diff --git a/drivers/cxl/pci.c b/drivers/cxl/pci.c index 33fc6e1634e3..3ffd5fad74b4 100644 --- a/drivers/cxl/pci.c +++ b/drivers/cxl/pci.c @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -936,7 +937,7 @@ static void __iomem *cxl_mem_map_regblock(struct cxl_mem *cxlm, return IOMEM_ERR_PTR(-ENXIO); } - addr = pcim_iomap(pdev, bar, 0); + addr = pci_iomap(pdev, bar, 0); if (!addr) { dev_err(dev, "failed to map registers\n"); return addr; @@ -945,7 +946,12 @@ static void __iomem *cxl_mem_map_regblock(struct cxl_mem *cxlm, dev_dbg(dev, "Mapped CXL Memory Device resource bar %u @ %#llx\n", bar, offset); - return pcim_iomap_table(pdev)[bar] + offset; + return addr; +} + +static void cxl_mem_unmap_regblock(struct cxl_mem *cxlm, void __iomem *base) +{ + pci_iounmap(cxlm->pdev, base); } static int cxl_mem_dvsec(struct pci_dev *pdev, int dvsec) @@ -971,6 +977,52 @@ static int cxl_mem_dvsec(struct pci_dev *pdev, int dvsec) return 0; } +static int cxl_probe_regs(struct cxl_mem *cxlm, void __iomem *base, + struct cxl_register_map *map) +{ + struct pci_dev *pdev = cxlm->pdev; + struct device *dev = &pdev->dev; + struct cxl_device_reg_map *dev_map; + + switch (map->reg_type) { + case CXL_REGLOC_RBI_MEMDEV: + dev_map = &map->device_map; + cxl_probe_device_regs(dev, base, dev_map); + if (!dev_map->status.valid || !dev_map->mbox.valid || + !dev_map->memdev.valid) { + dev_err(dev, "registers not found: %s%s%s\n", + !dev_map->status.valid ? "status " : "", + !dev_map->mbox.valid ? "status " : "", + !dev_map->memdev.valid ? "status " : ""); + return -ENXIO; + } + + dev_dbg(dev, "Probing device registers...\n"); + break; + default: + break; + } + + return 0; +} + +static int cxl_map_regs(struct cxl_mem *cxlm, struct cxl_register_map *map) +{ + struct pci_dev *pdev = cxlm->pdev; + struct device *dev = &pdev->dev; + + switch (map->reg_type) { + case CXL_REGLOC_RBI_MEMDEV: + cxl_map_device_regs(pdev, &cxlm->regs.device_regs, map); + dev_dbg(dev, "Probing device registers...\n"); + break; + default: + break; + } + + return 0; +} + static void cxl_decode_register_block(u32 reg_lo, u32 reg_hi, u8 *bar, u64 *offset, u8 *reg_type) { @@ -991,12 +1043,14 @@ static void cxl_decode_register_block(u32 reg_lo, u32 reg_hi, */ static int cxl_mem_setup_regs(struct cxl_mem *cxlm) { - struct cxl_regs *regs = &cxlm->regs; struct pci_dev *pdev = cxlm->pdev; struct device *dev = &pdev->dev; u32 regloc_size, regblocks; void __iomem *base; int regloc, i; + struct cxl_register_map *map, *n; + LIST_HEAD(register_maps); + int ret = 0; regloc = cxl_mem_dvsec(pdev, PCI_DVSEC_ID_CXL_REGLOC_OFFSET); if (!regloc) { @@ -1020,7 +1074,14 @@ static int cxl_mem_setup_regs(struct cxl_mem *cxlm) u64 offset; u8 bar; - /* "register low and high" contain other bits */ + map = kzalloc(sizeof(*map), GFP_KERNEL); + if (!map) { + ret = -ENOMEM; + goto free_maps; + } + + list_add(&map->list, ®ister_maps); + pci_read_config_dword(pdev, regloc, ®_lo); pci_read_config_dword(pdev, regloc + 4, ®_hi); @@ -1030,30 +1091,38 @@ static int cxl_mem_setup_regs(struct cxl_mem *cxlm) dev_dbg(dev, "Found register block in bar %u @ 0x%llx of type %u\n", bar, offset, reg_type); - if (reg_type == CXL_REGLOC_RBI_MEMDEV) { - base = cxl_mem_map_regblock(cxlm, bar, offset); - if (!base) - return -ENOMEM; - break; + base = cxl_mem_map_regblock(cxlm, bar, offset); + if (!base) { + ret = -ENOMEM; + goto free_maps; } + + map->barno = bar; + map->block_offset = offset; + map->reg_type = reg_type; + + ret = cxl_probe_regs(cxlm, base + offset, map); + + /* Always unmap the regblock regardless of probe success */ + cxl_mem_unmap_regblock(cxlm, base); + + if (ret) + goto free_maps; } - if (i == regblocks) { - dev_err(dev, "Missing register locator for device registers\n"); - return -ENXIO; + list_for_each_entry(map, ®ister_maps, list) { + ret = cxl_map_regs(cxlm, map); + if (ret) + goto free_maps; } - cxl_setup_device_regs(dev, base, ®s->device_regs); - - if (!regs->status || !regs->mbox || !regs->memdev) { - dev_err(dev, "registers not found: %s%s%s\n", - !regs->status ? "status " : "", - !regs->mbox ? "mbox " : "", - !regs->memdev ? "memdev" : ""); - return -ENXIO; +free_maps: + list_for_each_entry_safe(map, n, ®ister_maps, list) { + list_del(&map->list); + kfree(map); } - return 0; + return ret; } static struct cxl_memdev *to_cxl_memdev(struct device *dev) From 9a016527dcb71e2ecadfeacf52122a79b428790c Mon Sep 17 00:00:00 2001 From: Ira Weiny Date: Thu, 3 Jun 2021 17:53:16 -0700 Subject: [PATCH 16/34] cxl/pci: Reserve individual register block regions Some hardware implementations mix component and device registers into the same BAR and the driver stack is going to need independent mapping implementations for those 2 cases. Furthermore, it will be nice to have finer grained mappings should user space want to map some register blocks. Now that individual register blocks are mapped; those blocks regions should be reserved individually to fully separate the register blocks. Release the 'global' memory reservation and create individual register block region reservations through devm. NOTE: pci_release_mem_regions() is still compatible with pcim_enable_device() because it removes the automatic region release when called. So preserve the pcim_enable_device() so that the pcim interface can be called if needed. Reviewed-by: Jonathan Cameron Signed-off-by: Ira Weiny Link: https://lore.kernel.org/r/20210604005316.4187340-1-ira.weiny@intel.com Signed-off-by: Dan Williams --- drivers/cxl/core.c | 36 ++++++++++++++++++++++++++++++++---- drivers/cxl/pci.c | 2 ++ 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/drivers/cxl/core.c b/drivers/cxl/core.c index f836aaab03e0..c1efa11207b5 100644 --- a/drivers/cxl/core.c +++ b/drivers/cxl/core.c @@ -79,11 +79,33 @@ void cxl_probe_device_regs(struct device *dev, void __iomem *base, } EXPORT_SYMBOL_GPL(cxl_probe_device_regs); +static void __iomem *devm_cxl_iomap_block(struct pci_dev *pdev, + resource_size_t addr, + resource_size_t length) +{ + struct device *dev = &pdev->dev; + void __iomem *ret_val; + struct resource *res; + + res = devm_request_mem_region(dev, addr, length, pci_name(pdev)); + if (!res) { + resource_size_t end = addr + length - 1; + + dev_err(dev, "Failed to request region %pa-%pa\n", &addr, &end); + return NULL; + } + + ret_val = devm_ioremap(dev, addr, length); + if (!ret_val) + dev_err(dev, "Failed to map region %pr\n", res); + + return ret_val; +} + int cxl_map_device_regs(struct pci_dev *pdev, struct cxl_device_regs *regs, struct cxl_register_map *map) { - struct device *dev = &pdev->dev; resource_size_t phys_addr; phys_addr = pci_resource_start(pdev, map->barno); @@ -95,7 +117,9 @@ int cxl_map_device_regs(struct pci_dev *pdev, addr = phys_addr + map->device_map.status.offset; length = map->device_map.status.size; - regs->status = devm_ioremap(dev, addr, length); + regs->status = devm_cxl_iomap_block(pdev, addr, length); + if (!regs->status) + return -ENOMEM; } if (map->device_map.mbox.valid) { @@ -104,7 +128,9 @@ int cxl_map_device_regs(struct pci_dev *pdev, addr = phys_addr + map->device_map.mbox.offset; length = map->device_map.mbox.size; - regs->mbox = devm_ioremap(dev, addr, length); + regs->mbox = devm_cxl_iomap_block(pdev, addr, length); + if (!regs->mbox) + return -ENOMEM; } if (map->device_map.memdev.valid) { @@ -113,7 +139,9 @@ int cxl_map_device_regs(struct pci_dev *pdev, addr = phys_addr + map->device_map.memdev.offset; length = map->device_map.memdev.size; - regs->memdev = devm_ioremap(dev, addr, length); + regs->memdev = devm_cxl_iomap_block(pdev, addr, length); + if (!regs->memdev) + return -ENOMEM; } return 0; diff --git a/drivers/cxl/pci.c b/drivers/cxl/pci.c index 3ffd5fad74b4..e1a2dbc2886b 100644 --- a/drivers/cxl/pci.c +++ b/drivers/cxl/pci.c @@ -1110,6 +1110,8 @@ static int cxl_mem_setup_regs(struct cxl_mem *cxlm) goto free_maps; } + pci_release_mem_regions(pdev); + list_for_each_entry(map, ®ister_maps, list) { ret = cxl_map_regs(cxlm, map); if (ret) From 08422378c4adacf528d573bb1631d4818f8f9a01 Mon Sep 17 00:00:00 2001 From: Ben Widawsky Date: Thu, 27 May 2021 17:49:22 -0700 Subject: [PATCH 17/34] cxl/pci: Add HDM decoder capabilities An HDM decoder is defined in the CXL 2.0 specification as a mechanism that allow devices and upstream ports to claim memory address ranges and participate in interleave sets. HDM decoder registers are within the component register block defined in CXL 2.0 8.2.3 CXL 2.0 Component Registers as part of the CXL.cache and CXL.mem subregion. The Component Register Block is found via the Register Locator DVSEC in a similar fashion to how the CXL Device Register Block is found. The primary difference is the capability id size of the Component Register Block is a single DWORD instead of 4 DWORDS. It's now possible to configure a CXL type 3 device's HDM decoder. Such programming is expected for CXL devices with persistent memory, and hot plugged CXL devices that participate in CXL.mem with volatile memory. Add probe and mapping functions for the component register blocks. Reviewed-by: Jonathan Cameron Co-developed-by: Ira Weiny Signed-off-by: Ira Weiny Co-developed-by: Vishal Verma Signed-off-by: Vishal Verma Signed-off-by: Ben Widawsky Link: https://lore.kernel.org/r/20210528004922.3980613-6-ira.weiny@intel.com Signed-off-by: Dan Williams --- drivers/cxl/core.c | 92 ++++++++++++++++++++++++++++++++++++++++++++++ drivers/cxl/cxl.h | 65 +++++++++++++++++++++++++++++--- drivers/cxl/pci.c | 15 ++++++++ 3 files changed, 166 insertions(+), 6 deletions(-) diff --git a/drivers/cxl/core.c b/drivers/cxl/core.c index c1efa11207b5..d0f6d93d5e49 100644 --- a/drivers/cxl/core.c +++ b/drivers/cxl/core.c @@ -13,6 +13,78 @@ * point for cross-device interleave coordination through cxl ports. */ +/** + * cxl_probe_component_regs() - Detect CXL Component register blocks + * @dev: Host device of the @base mapping + * @base: Mapping containing the HDM Decoder Capability Header + * @map: Map object describing the register block information found + * + * See CXL 2.0 8.2.4 Component Register Layout and Definition + * See CXL 2.0 8.2.5.5 CXL Device Register Interface + * + * Probe for component register information and return it in map object. + */ +void cxl_probe_component_regs(struct device *dev, void __iomem *base, + struct cxl_component_reg_map *map) +{ + int cap, cap_count; + u64 cap_array; + + *map = (struct cxl_component_reg_map) { 0 }; + + /* + * CXL.cache and CXL.mem registers are at offset 0x1000 as defined in + * CXL 2.0 8.2.4 Table 141. + */ + base += CXL_CM_OFFSET; + + cap_array = readq(base + CXL_CM_CAP_HDR_OFFSET); + + if (FIELD_GET(CXL_CM_CAP_HDR_ID_MASK, cap_array) != CM_CAP_HDR_CAP_ID) { + dev_err(dev, + "Couldn't locate the CXL.cache and CXL.mem capability array header./n"); + return; + } + + /* It's assumed that future versions will be backward compatible */ + cap_count = FIELD_GET(CXL_CM_CAP_HDR_ARRAY_SIZE_MASK, cap_array); + + for (cap = 1; cap <= cap_count; cap++) { + void __iomem *register_block; + u32 hdr; + int decoder_cnt; + u16 cap_id, offset; + u32 length; + + hdr = readl(base + cap * 0x4); + + cap_id = FIELD_GET(CXL_CM_CAP_HDR_ID_MASK, hdr); + offset = FIELD_GET(CXL_CM_CAP_PTR_MASK, hdr); + register_block = base + offset; + + switch (cap_id) { + case CXL_CM_CAP_CAP_ID_HDM: + dev_dbg(dev, "found HDM decoder capability (0x%x)\n", + offset); + + hdr = readl(register_block); + + decoder_cnt = FIELD_GET(CXL_HDM_DECODER_COUNT_MASK, hdr); + length = 0x20 * decoder_cnt + 0x10; + + map->hdm_decoder.valid = true; + map->hdm_decoder.offset = offset; + map->hdm_decoder.size = length; + break; + default: + dev_dbg(dev, "Unknown CM cap ID: %d (0x%x)\n", cap_id, + offset); + break; + } + } +} +EXPORT_SYMBOL_GPL(cxl_probe_component_regs); + /** * cxl_probe_device_regs() - Detect CXL Device register blocks * @dev: Host device of the @base mapping @@ -102,6 +174,26 @@ static void __iomem *devm_cxl_iomap_block(struct pci_dev *pdev, return ret_val; } +int cxl_map_component_regs(struct pci_dev *pdev, + struct cxl_component_regs *regs, + struct cxl_register_map *map) +{ + resource_size_t phys_addr; + resource_size_t length; + + phys_addr = pci_resource_start(pdev, map->barno); + phys_addr += map->block_offset; + + phys_addr += map->component_map.hdm_decoder.offset; + length = map->component_map.hdm_decoder.size; + regs->hdm_decoder = devm_cxl_iomap_block(pdev, phys_addr, length); + if (!regs->hdm_decoder) + return -ENOMEM; + + return 0; +} +EXPORT_SYMBOL_GPL(cxl_map_component_regs); + int cxl_map_device_regs(struct pci_dev *pdev, struct cxl_device_regs *regs, struct cxl_register_map *map) diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h index ae4b4c96c6b5..2c47e9cffd44 100644 --- a/drivers/cxl/cxl.h +++ b/drivers/cxl/cxl.h @@ -8,6 +8,31 @@ #include #include +/* CXL 2.0 8.2.5 CXL.cache and CXL.mem Registers*/ +#define CXL_CM_OFFSET 0x1000 +#define CXL_CM_CAP_HDR_OFFSET 0x0 +#define CXL_CM_CAP_HDR_ID_MASK GENMASK(15, 0) +#define CM_CAP_HDR_CAP_ID 1 +#define CXL_CM_CAP_HDR_VERSION_MASK GENMASK(19, 16) +#define CM_CAP_HDR_CAP_VERSION 1 +#define CXL_CM_CAP_HDR_CACHE_MEM_VERSION_MASK GENMASK(23, 20) +#define CM_CAP_HDR_CACHE_MEM_VERSION 1 +#define CXL_CM_CAP_HDR_ARRAY_SIZE_MASK GENMASK(31, 24) +#define CXL_CM_CAP_PTR_MASK GENMASK(31, 20) + +#define CXL_CM_CAP_CAP_ID_HDM 0x5 +#define CXL_CM_CAP_CAP_HDM_VERSION 1 + +/* HDM decoders CXL 2.0 8.2.5.12 CXL HDM Decoder Capability Structure */ +#define CXL_HDM_DECODER_CAP_OFFSET 0x0 +#define CXL_HDM_DECODER_COUNT_MASK GENMASK(3, 0) +#define CXL_HDM_DECODER_TARGET_COUNT_MASK GENMASK(7, 4) +#define CXL_HDM_DECODER0_BASE_LOW_OFFSET 0x10 +#define CXL_HDM_DECODER0_BASE_HIGH_OFFSET 0x14 +#define CXL_HDM_DECODER0_SIZE_LOW_OFFSET 0x18 +#define CXL_HDM_DECODER0_SIZE_HIGH_OFFSET 0x1c +#define CXL_HDM_DECODER0_CTRL_OFFSET 0x20 + /* CXL 2.0 8.2.8.1 Device Capabilities Array Register */ #define CXLDEV_CAP_ARRAY_OFFSET 0x0 #define CXLDEV_CAP_ARRAY_CAP_ID 0 @@ -34,18 +59,30 @@ #define CXLDEV_MBOX_BG_CMD_STATUS_OFFSET 0x18 #define CXLDEV_MBOX_PAYLOAD_OFFSET 0x20 -/* - * CXL_DEVICE_REGS - Common set of CXL Device register block base pointers - * @status: CXL 2.0 8.2.8.3 Device Status Registers - * @mbox: CXL 2.0 8.2.8.4 Mailbox Registers - * @memdev: CXL 2.0 8.2.8.5 Memory Device Registers - */ +#define CXL_COMPONENT_REGS() \ + void __iomem *hdm_decoder + #define CXL_DEVICE_REGS() \ void __iomem *status; \ void __iomem *mbox; \ void __iomem *memdev /* See note for 'struct cxl_regs' for the rationale of this organization */ +/* + * CXL_COMPONENT_REGS - Common set of CXL Component register block base pointers + * @hdm_decoder: CXL 2.0 8.2.5.12 CXL HDM Decoder Capability Structure + */ +struct cxl_component_regs { + CXL_COMPONENT_REGS(); +}; + +/* See note for 'struct cxl_regs' for the rationale of this organization */ +/* + * CXL_DEVICE_REGS - Common set of CXL Device register block base pointers + * @status: CXL 2.0 8.2.8.3 Device Status Registers + * @mbox: CXL 2.0 8.2.8.4 Mailbox Registers + * @memdev: CXL 2.0 8.2.8.5 Memory Device Registers + */ struct cxl_device_regs { CXL_DEVICE_REGS(); }; @@ -56,6 +93,12 @@ struct cxl_device_regs { * agnostic code to include the prefix. */ struct cxl_regs { + union { + struct { + CXL_COMPONENT_REGS(); + }; + struct cxl_component_regs component; + }; union { struct { CXL_DEVICE_REGS(); @@ -70,6 +113,10 @@ struct cxl_reg_map { unsigned long size; }; +struct cxl_component_reg_map { + struct cxl_reg_map hdm_decoder; +}; + struct cxl_device_reg_map { struct cxl_reg_map status; struct cxl_reg_map mbox; @@ -82,12 +129,18 @@ struct cxl_register_map { u8 reg_type; u8 barno; union { + struct cxl_component_reg_map component_map; struct cxl_device_reg_map device_map; }; }; +void cxl_probe_component_regs(struct device *dev, void __iomem *base, + struct cxl_component_reg_map *map); void cxl_probe_device_regs(struct device *dev, void __iomem *base, struct cxl_device_reg_map *map); +int cxl_map_component_regs(struct pci_dev *pdev, + struct cxl_component_regs *regs, + struct cxl_register_map *map); int cxl_map_device_regs(struct pci_dev *pdev, struct cxl_device_regs *regs, struct cxl_register_map *map); diff --git a/drivers/cxl/pci.c b/drivers/cxl/pci.c index e1a2dbc2886b..5a1705b52278 100644 --- a/drivers/cxl/pci.c +++ b/drivers/cxl/pci.c @@ -982,9 +982,20 @@ static int cxl_probe_regs(struct cxl_mem *cxlm, void __iomem *base, { struct pci_dev *pdev = cxlm->pdev; struct device *dev = &pdev->dev; + struct cxl_component_reg_map *comp_map; struct cxl_device_reg_map *dev_map; switch (map->reg_type) { + case CXL_REGLOC_RBI_COMPONENT: + comp_map = &map->component_map; + cxl_probe_component_regs(dev, base, comp_map); + if (!comp_map->hdm_decoder.valid) { + dev_err(dev, "HDM decoder registers not found\n"); + return -ENXIO; + } + + dev_dbg(dev, "Set up component registers\n"); + break; case CXL_REGLOC_RBI_MEMDEV: dev_map = &map->device_map; cxl_probe_device_regs(dev, base, dev_map); @@ -1012,6 +1023,10 @@ static int cxl_map_regs(struct cxl_mem *cxlm, struct cxl_register_map *map) struct device *dev = &pdev->dev; switch (map->reg_type) { + case CXL_REGLOC_RBI_COMPONENT: + cxl_map_component_regs(pdev, &cxlm->regs.component, map); + dev_dbg(dev, "Mapping component registers...\n"); + break; case CXL_REGLOC_RBI_MEMDEV: cxl_map_device_regs(pdev, &cxlm->regs.device_regs, map); dev_dbg(dev, "Probing device registers...\n"); From 605a5e41db7d8c930fb80115686991c4c1d08ee4 Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Thu, 27 May 2021 18:39:11 -0700 Subject: [PATCH 18/34] cxl/pci: Fixup devm_cxl_iomap_block() to take a 'struct device *' The expectation is that devm functions take 'struct device *' and pci functions take 'struct pci_dev *'. Swap out the @pdev argument for @dev and fixup related helpers. Cc: Ira Weiny Reviewed-by: Ira Weiny Acked-by: Jonathan Cameron Link: https://lore.kernel.org/r/162216592374.3833641.13281743585064451514.stgit@dwillia2-desk3.amr.corp.intel.com Signed-off-by: Dan Williams --- drivers/cxl/core.c | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/drivers/cxl/core.c b/drivers/cxl/core.c index d0f6d93d5e49..853666d8a9f5 100644 --- a/drivers/cxl/core.c +++ b/drivers/cxl/core.c @@ -151,15 +151,14 @@ void cxl_probe_device_regs(struct device *dev, void __iomem *base, } EXPORT_SYMBOL_GPL(cxl_probe_device_regs); -static void __iomem *devm_cxl_iomap_block(struct pci_dev *pdev, +static void __iomem *devm_cxl_iomap_block(struct device *dev, resource_size_t addr, resource_size_t length) { - struct device *dev = &pdev->dev; void __iomem *ret_val; struct resource *res; - res = devm_request_mem_region(dev, addr, length, pci_name(pdev)); + res = devm_request_mem_region(dev, addr, length, dev_name(dev)); if (!res) { resource_size_t end = addr + length - 1; @@ -178,6 +177,7 @@ int cxl_map_component_regs(struct pci_dev *pdev, struct cxl_component_regs *regs, struct cxl_register_map *map) { + struct device *dev = &pdev->dev; resource_size_t phys_addr; resource_size_t length; @@ -186,7 +186,7 @@ int cxl_map_component_regs(struct pci_dev *pdev, phys_addr += map->component_map.hdm_decoder.offset; length = map->component_map.hdm_decoder.size; - regs->hdm_decoder = devm_cxl_iomap_block(pdev, phys_addr, length); + regs->hdm_decoder = devm_cxl_iomap_block(dev, phys_addr, length); if (!regs->hdm_decoder) return -ENOMEM; @@ -198,6 +198,7 @@ int cxl_map_device_regs(struct pci_dev *pdev, struct cxl_device_regs *regs, struct cxl_register_map *map) { + struct device *dev = &pdev->dev; resource_size_t phys_addr; phys_addr = pci_resource_start(pdev, map->barno); @@ -209,7 +210,7 @@ int cxl_map_device_regs(struct pci_dev *pdev, addr = phys_addr + map->device_map.status.offset; length = map->device_map.status.size; - regs->status = devm_cxl_iomap_block(pdev, addr, length); + regs->status = devm_cxl_iomap_block(dev, addr, length); if (!regs->status) return -ENOMEM; } @@ -220,7 +221,7 @@ int cxl_map_device_regs(struct pci_dev *pdev, addr = phys_addr + map->device_map.mbox.offset; length = map->device_map.mbox.size; - regs->mbox = devm_cxl_iomap_block(pdev, addr, length); + regs->mbox = devm_cxl_iomap_block(dev, addr, length); if (!regs->mbox) return -ENOMEM; } @@ -231,7 +232,7 @@ int cxl_map_device_regs(struct pci_dev *pdev, addr = phys_addr + map->device_map.memdev.offset; length = map->device_map.memdev.size; - regs->memdev = devm_cxl_iomap_block(pdev, addr, length); + regs->memdev = devm_cxl_iomap_block(dev, addr, length); if (!regs->memdev) return -ENOMEM; } From 4812be97c015bddf12c70155858df43acc35a4eb Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Wed, 9 Jun 2021 09:01:35 -0700 Subject: [PATCH 19/34] cxl/acpi: Introduce the root of a cxl_port topology While CXL builds upon the PCI software model for enumeration and endpoint control, a static platform component is required to bootstrap the CXL memory layout. Similar to how ACPI identifies root-level PCI memory resources, ACPI data enumerates the address space and interleave configuration for CXL Memory. In addition to identifying host bridges, ACPI is responsible for enumerating the CXL memory space that can be addressed by downstream decoders. This is similar to the requirement for ACPI to publish resources via the _CRS method for PCI host bridges. Specifically, ACPI publishes a table, CXL Early Discovery Table (CEDT), which includes a list of CXL Memory resources, CXL Fixed Memory Window Structures (CFMWS). For now, introduce the core infrastructure for a cxl_port hierarchy starting with a root level anchor represented by the ACPI0017 device. Follow on changes model support for the configurable decode capabilities of cxl_port instances, i.e. CXL switch support. Co-developed-by: Alison Schofield Signed-off-by: Alison Schofield Acked-by: Rafael J. Wysocki Reviewed-by: Jonathan Cameron Link: https://lore.kernel.org/r/162325449515.2293126.15303270193010154608.stgit@dwillia2-desk3.amr.corp.intel.com Signed-off-by: Dan Williams --- Documentation/ABI/testing/sysfs-bus-cxl | 20 +++ .../driver-api/cxl/memory-devices.rst | 6 + drivers/cxl/Kconfig | 15 ++ drivers/cxl/Makefile | 2 + drivers/cxl/acpi.c | 39 +++++ drivers/cxl/core.c | 160 ++++++++++++++++++ drivers/cxl/cxl.h | 31 ++++ 7 files changed, 273 insertions(+) create mode 100644 drivers/cxl/acpi.c diff --git a/Documentation/ABI/testing/sysfs-bus-cxl b/Documentation/ABI/testing/sysfs-bus-cxl index 2fe7490ad6a8..bda2cc55cc38 100644 --- a/Documentation/ABI/testing/sysfs-bus-cxl +++ b/Documentation/ABI/testing/sysfs-bus-cxl @@ -24,3 +24,23 @@ Description: (RO) "Persistent Only Capacity" as bytes. Represents the identically named field in the Identify Memory Device Output Payload in the CXL-2.0 specification. + +What: /sys/bus/cxl/devices/*/devtype +Date: June, 2021 +KernelVersion: v5.14 +Contact: linux-cxl@vger.kernel.org +Description: + CXL device objects export the devtype attribute which mirrors + the same value communicated in the DEVTYPE environment variable + for uevents for devices on the "cxl" bus. + +What: /sys/bus/cxl/devices/portX/uport +Date: June, 2021 +KernelVersion: v5.14 +Contact: linux-cxl@vger.kernel.org +Description: + CXL port objects are enumerated from either a platform firmware + device (ACPI0017 and ACPI0016) or PCIe switch upstream port with + CXL component registers. The 'uport' symlink connects the CXL + portX object to the device that published the CXL port + capability. diff --git a/Documentation/driver-api/cxl/memory-devices.rst b/Documentation/driver-api/cxl/memory-devices.rst index 44c8ddbc8415..487ce4f41d77 100644 --- a/Documentation/driver-api/cxl/memory-devices.rst +++ b/Documentation/driver-api/cxl/memory-devices.rst @@ -30,6 +30,12 @@ CXL Memory Device CXL Core -------- +.. kernel-doc:: drivers/cxl/cxl.h + :doc: cxl objects + +.. kernel-doc:: drivers/cxl/cxl.h + :internal: + .. kernel-doc:: drivers/cxl/core.c :doc: cxl core diff --git a/drivers/cxl/Kconfig b/drivers/cxl/Kconfig index 5483ba92b6da..d2573f6aef91 100644 --- a/drivers/cxl/Kconfig +++ b/drivers/cxl/Kconfig @@ -45,4 +45,19 @@ config CXL_MEM_RAW_COMMANDS potential impact to memory currently in use by the kernel. If developing CXL hardware or the driver say Y, otherwise say N. + +config CXL_ACPI + tristate "CXL ACPI: Platform Support" + depends on ACPI + help + Enable support for host managed device memory (HDM) resources + published by a platform's ACPI CXL memory layout description. See + Chapter 9.14.1 CXL Early Discovery Table (CEDT) in the CXL 2.0 + specification, and CXL Fixed Memory Window Structures (CEDT.CFMWS) + (https://www.computeexpresslink.org/spec-landing). The CXL core + consumes these resource to publish the root of a cxl_port decode + hierarchy to map regions that represent System RAM, or Persistent + Memory regions to be managed by LIBNVDIMM. + + If unsure say 'm'. endif diff --git a/drivers/cxl/Makefile b/drivers/cxl/Makefile index d9d282dc15be..a29efb3e8ad2 100644 --- a/drivers/cxl/Makefile +++ b/drivers/cxl/Makefile @@ -1,7 +1,9 @@ # SPDX-License-Identifier: GPL-2.0 obj-$(CONFIG_CXL_BUS) += cxl_core.o obj-$(CONFIG_CXL_MEM) += cxl_pci.o +obj-$(CONFIG_CXL_ACPI) += cxl_acpi.o ccflags-y += -DDEFAULT_SYMBOL_NAMESPACE=CXL cxl_core-y := core.o cxl_pci-y := pci.o +cxl_acpi-y := acpi.o diff --git a/drivers/cxl/acpi.c b/drivers/cxl/acpi.c new file mode 100644 index 000000000000..556d25ab6966 --- /dev/null +++ b/drivers/cxl/acpi.c @@ -0,0 +1,39 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright(c) 2021 Intel Corporation. All rights reserved. */ +#include +#include +#include +#include +#include +#include "cxl.h" + +static int cxl_acpi_probe(struct platform_device *pdev) +{ + struct cxl_port *root_port; + struct device *host = &pdev->dev; + + root_port = devm_cxl_add_port(host, host, CXL_RESOURCE_NONE, NULL); + if (IS_ERR(root_port)) + return PTR_ERR(root_port); + dev_dbg(host, "add: %s\n", dev_name(&root_port->dev)); + + return 0; +} + +static const struct acpi_device_id cxl_acpi_ids[] = { + { "ACPI0017", 0 }, + { "", 0 }, +}; +MODULE_DEVICE_TABLE(acpi, cxl_acpi_ids); + +static struct platform_driver cxl_acpi_driver = { + .probe = cxl_acpi_probe, + .driver = { + .name = KBUILD_MODNAME, + .acpi_match_table = cxl_acpi_ids, + }, +}; + +module_platform_driver(cxl_acpi_driver); +MODULE_LICENSE("GPL v2"); +MODULE_IMPORT_NS(CXL); diff --git a/drivers/cxl/core.c b/drivers/cxl/core.c index 853666d8a9f5..dbbb34618d7d 100644 --- a/drivers/cxl/core.c +++ b/drivers/cxl/core.c @@ -4,6 +4,8 @@ #include #include #include +#include +#include #include "cxl.h" /** @@ -13,6 +15,164 @@ * point for cross-device interleave coordination through cxl ports. */ +static DEFINE_IDA(cxl_port_ida); + +static ssize_t devtype_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + return sysfs_emit(buf, "%s\n", dev->type->name); +} +static DEVICE_ATTR_RO(devtype); + +static struct attribute *cxl_base_attributes[] = { + &dev_attr_devtype.attr, + NULL, +}; + +static struct attribute_group cxl_base_attribute_group = { + .attrs = cxl_base_attributes, +}; + +static void cxl_port_release(struct device *dev) +{ + struct cxl_port *port = to_cxl_port(dev); + + ida_free(&cxl_port_ida, port->id); + kfree(port); +} + +static const struct attribute_group *cxl_port_attribute_groups[] = { + &cxl_base_attribute_group, + NULL, +}; + +static const struct device_type cxl_port_type = { + .name = "cxl_port", + .release = cxl_port_release, + .groups = cxl_port_attribute_groups, +}; + +struct cxl_port *to_cxl_port(struct device *dev) +{ + if (dev_WARN_ONCE(dev, dev->type != &cxl_port_type, + "not a cxl_port device\n")) + return NULL; + return container_of(dev, struct cxl_port, dev); +} + +static void unregister_dev(void *dev) +{ + device_unregister(dev); +} + +static void cxl_unlink_uport(void *_port) +{ + struct cxl_port *port = _port; + + sysfs_remove_link(&port->dev.kobj, "uport"); +} + +static int devm_cxl_link_uport(struct device *host, struct cxl_port *port) +{ + int rc; + + rc = sysfs_create_link(&port->dev.kobj, &port->uport->kobj, "uport"); + if (rc) + return rc; + return devm_add_action_or_reset(host, cxl_unlink_uport, port); +} + +static struct cxl_port *cxl_port_alloc(struct device *uport, + resource_size_t component_reg_phys, + struct cxl_port *parent_port) +{ + struct cxl_port *port; + struct device *dev; + int rc; + + port = kzalloc(sizeof(*port), GFP_KERNEL); + if (!port) + return ERR_PTR(-ENOMEM); + + rc = ida_alloc(&cxl_port_ida, GFP_KERNEL); + if (rc < 0) + goto err; + port->id = rc; + + /* + * The top-level cxl_port "cxl_root" does not have a cxl_port as + * its parent and it does not have any corresponding component + * registers as its decode is described by a fixed platform + * description. + */ + dev = &port->dev; + if (parent_port) + dev->parent = &parent_port->dev; + else + dev->parent = uport; + + port->uport = uport; + port->component_reg_phys = component_reg_phys; + + device_initialize(dev); + device_set_pm_not_required(dev); + dev->bus = &cxl_bus_type; + dev->type = &cxl_port_type; + + return port; + +err: + kfree(port); + return ERR_PTR(rc); +} + +/** + * devm_cxl_add_port - register a cxl_port in CXL memory decode hierarchy + * @host: host device for devm operations + * @uport: "physical" device implementing this upstream port + * @component_reg_phys: (optional) for configurable cxl_port instances + * @parent_port: next hop up in the CXL memory decode hierarchy + */ +struct cxl_port *devm_cxl_add_port(struct device *host, struct device *uport, + resource_size_t component_reg_phys, + struct cxl_port *parent_port) +{ + struct cxl_port *port; + struct device *dev; + int rc; + + port = cxl_port_alloc(uport, component_reg_phys, parent_port); + if (IS_ERR(port)) + return port; + + dev = &port->dev; + if (parent_port) + rc = dev_set_name(dev, "port%d", port->id); + else + rc = dev_set_name(dev, "root%d", port->id); + if (rc) + goto err; + + rc = device_add(dev); + if (rc) + goto err; + + rc = devm_add_action_or_reset(host, unregister_dev, dev); + if (rc) + return ERR_PTR(rc); + + rc = devm_cxl_link_uport(host, port); + if (rc) + return ERR_PTR(rc); + + return port; + +err: + put_device(dev); + return ERR_PTR(rc); +} +EXPORT_SYMBOL_GPL(devm_cxl_add_port); + /** * cxl_probe_component_regs() - Detect CXL Component register blocks * @dev: Host device of the @base mapping diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h index 2c47e9cffd44..5651e5bb8274 100644 --- a/drivers/cxl/cxl.h +++ b/drivers/cxl/cxl.h @@ -8,6 +8,14 @@ #include #include +/** + * DOC: cxl objects + * + * The CXL core objects like ports, decoders, and regions are shared + * between the subsystem drivers cxl_acpi, cxl_pci, and core drivers + * (port-driver, region-driver, nvdimm object-drivers... etc). + */ + /* CXL 2.0 8.2.5 CXL.cache and CXL.mem Registers*/ #define CXL_CM_OFFSET 0x1000 #define CXL_CM_CAP_HDR_OFFSET 0x0 @@ -145,5 +153,28 @@ int cxl_map_device_regs(struct pci_dev *pdev, struct cxl_device_regs *regs, struct cxl_register_map *map); +#define CXL_RESOURCE_NONE ((resource_size_t) -1) + +/** + * struct cxl_port - logical collection of upstream port devices and + * downstream port devices to construct a CXL memory + * decode hierarchy. + * @dev: this port's device + * @uport: PCI or platform device implementing the upstream port capability + * @id: id for port device-name + * @component_reg_phys: component register capability base address (optional) + */ +struct cxl_port { + struct device dev; + struct device *uport; + int id; + resource_size_t component_reg_phys; +}; + +struct cxl_port *to_cxl_port(struct device *dev); +struct cxl_port *devm_cxl_add_port(struct device *host, struct device *uport, + resource_size_t component_reg_phys, + struct cxl_port *parent_port); + extern struct bus_type cxl_bus_type; #endif /* __CXL_H__ */ From 3feaa2d35880de935fc0d02acf808f355564f4e6 Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Wed, 9 Jun 2021 09:01:41 -0700 Subject: [PATCH 20/34] cxl/Kconfig: Default drivers to CONFIG_CXL_BUS CONFIG_CXL_BUS is default 'n' as expected for new functionality. When that is enabled do not make the end user hunt for all the expected sub-options to enable. For example CONFIG_CXL_BUS without CONFIG_CXL_MEM is an odd/expert configuration, so is CONFIG_CXL_MEM without CONFIG_CXL_ACPI (on ACPI capable platforms). Default CONFIG_CXL_MEM and CONFIG_CXL_ACPI to CONFIG_CXL_BUS. Acked-by: Ben Widawsky Acked-by: Jonathan Cameron Link: https://lore.kernel.org/r/162325450105.2293126.17046356425194082921.stgit@dwillia2-desk3.amr.corp.intel.com Signed-off-by: Dan Williams --- drivers/cxl/Kconfig | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/cxl/Kconfig b/drivers/cxl/Kconfig index d2573f6aef91..1a44b173dcbc 100644 --- a/drivers/cxl/Kconfig +++ b/drivers/cxl/Kconfig @@ -15,6 +15,7 @@ if CXL_BUS config CXL_MEM tristate "CXL.mem: Memory Devices" + default CXL_BUS help The CXL.mem protocol allows a device to act as a provider of "System RAM" and/or "Persistent Memory" that is fully coherent @@ -49,6 +50,7 @@ config CXL_MEM_RAW_COMMANDS config CXL_ACPI tristate "CXL ACPI: Platform Support" depends on ACPI + default CXL_BUS help Enable support for host managed device memory (HDM) resources published by a platform's ACPI CXL memory layout description. See From 7d4b5ca2e2cb5d28db628ec79c706bcfa832feea Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Wed, 9 Jun 2021 09:01:46 -0700 Subject: [PATCH 21/34] cxl/acpi: Add downstream port data to cxl_port instances In preparation for infrastructure that enumerates and configures the CXL decode mechanism of an upstream port to its downstream ports, add a representation of a CXL downstream port. On ACPI systems the top-most logical downstream ports in the hierarchy are the host bridges (ACPI0016 devices) that decode the memory windows described by the CXL Early Discovery Table Fixed Memory Window Structures (CEDT.CFMWS). Reviewed-by: Alison Schofield Link: https://lore.kernel.org/r/162325450624.2293126.3533006409920271718.stgit@dwillia2-desk3.amr.corp.intel.com Signed-off-by: Dan Williams --- Documentation/ABI/testing/sysfs-bus-cxl | 13 +++ drivers/cxl/acpi.c | 43 +++++++++- drivers/cxl/core.c | 107 +++++++++++++++++++++++- drivers/cxl/cxl.h | 21 +++++ 4 files changed, 180 insertions(+), 4 deletions(-) diff --git a/Documentation/ABI/testing/sysfs-bus-cxl b/Documentation/ABI/testing/sysfs-bus-cxl index bda2cc55cc38..f680da85fd44 100644 --- a/Documentation/ABI/testing/sysfs-bus-cxl +++ b/Documentation/ABI/testing/sysfs-bus-cxl @@ -44,3 +44,16 @@ Description: CXL component registers. The 'uport' symlink connects the CXL portX object to the device that published the CXL port capability. + +What: /sys/bus/cxl/devices/portX/dportY +Date: June, 2021 +KernelVersion: v5.14 +Contact: linux-cxl@vger.kernel.org +Description: + CXL port objects are enumerated from either a platform firmware + device (ACPI0017 and ACPI0016) or PCIe switch upstream port with + CXL component registers. The 'dportY' symlink identifies one or + more downstream ports that the upstream port may target in its + decode of CXL memory resources. The 'Y' integer reflects the + hardware port unique-id used in the hardware decoder target + list. diff --git a/drivers/cxl/acpi.c b/drivers/cxl/acpi.c index 556d25ab6966..5eb9543c587a 100644 --- a/drivers/cxl/acpi.c +++ b/drivers/cxl/acpi.c @@ -7,17 +7,58 @@ #include #include "cxl.h" +static struct acpi_device *to_cxl_host_bridge(struct device *dev) +{ + struct acpi_device *adev = to_acpi_device(dev); + + if (strcmp(acpi_device_hid(adev), "ACPI0016") == 0) + return adev; + return NULL; +} + +static int add_host_bridge_dport(struct device *match, void *arg) +{ + int rc; + acpi_status status; + unsigned long long uid; + struct cxl_port *root_port = arg; + struct device *host = root_port->dev.parent; + struct acpi_device *bridge = to_cxl_host_bridge(match); + + if (!bridge) + return 0; + + status = acpi_evaluate_integer(bridge->handle, METHOD_NAME__UID, NULL, + &uid); + if (status != AE_OK) { + dev_err(host, "unable to retrieve _UID of %s\n", + dev_name(match)); + return -ENODEV; + } + + rc = cxl_add_dport(root_port, match, uid, CXL_RESOURCE_NONE); + if (rc) { + dev_err(host, "failed to add downstream port: %s\n", + dev_name(match)); + return rc; + } + dev_dbg(host, "add dport%llu: %s\n", uid, dev_name(match)); + return 0; +} + static int cxl_acpi_probe(struct platform_device *pdev) { struct cxl_port *root_port; struct device *host = &pdev->dev; + struct acpi_device *adev = ACPI_COMPANION(host); root_port = devm_cxl_add_port(host, host, CXL_RESOURCE_NONE, NULL); if (IS_ERR(root_port)) return PTR_ERR(root_port); dev_dbg(host, "add: %s\n", dev_name(&root_port->dev)); - return 0; + return bus_for_each_dev(adev->dev.bus, NULL, root_port, + add_host_bridge_dport); } static const struct acpi_device_id cxl_acpi_ids[] = { diff --git a/drivers/cxl/core.c b/drivers/cxl/core.c index dbbb34618d7d..8a3f3804f252 100644 --- a/drivers/cxl/core.c +++ b/drivers/cxl/core.c @@ -33,10 +33,22 @@ static struct attribute_group cxl_base_attribute_group = { .attrs = cxl_base_attributes, }; +static void cxl_dport_release(struct cxl_dport *dport) +{ + list_del(&dport->list); + put_device(dport->dport); + kfree(dport); +} + static void cxl_port_release(struct device *dev) { struct cxl_port *port = to_cxl_port(dev); + struct cxl_dport *dport, *_d; + device_lock(dev); + list_for_each_entry_safe(dport, _d, &port->dports, list) + cxl_dport_release(dport); + device_unlock(dev); ida_free(&cxl_port_ida, port->id); kfree(port); } @@ -60,9 +72,22 @@ struct cxl_port *to_cxl_port(struct device *dev) return container_of(dev, struct cxl_port, dev); } -static void unregister_dev(void *dev) +static void unregister_port(void *_port) { - device_unregister(dev); + struct cxl_port *port = _port; + struct cxl_dport *dport; + + device_lock(&port->dev); + list_for_each_entry(dport, &port->dports, list) { + char link_name[CXL_TARGET_STRLEN]; + + if (snprintf(link_name, CXL_TARGET_STRLEN, "dport%d", + dport->port_id) >= CXL_TARGET_STRLEN) + continue; + sysfs_remove_link(&port->dev.kobj, link_name); + } + device_unlock(&port->dev); + device_unregister(&port->dev); } static void cxl_unlink_uport(void *_port) @@ -113,6 +138,7 @@ static struct cxl_port *cxl_port_alloc(struct device *uport, port->uport = uport; port->component_reg_phys = component_reg_phys; + INIT_LIST_HEAD(&port->dports); device_initialize(dev); device_set_pm_not_required(dev); @@ -157,7 +183,7 @@ struct cxl_port *devm_cxl_add_port(struct device *host, struct device *uport, if (rc) goto err; - rc = devm_add_action_or_reset(host, unregister_dev, dev); + rc = devm_add_action_or_reset(host, unregister_port, port); if (rc) return ERR_PTR(rc); @@ -173,6 +199,81 @@ err: } EXPORT_SYMBOL_GPL(devm_cxl_add_port); +static struct cxl_dport *find_dport(struct cxl_port *port, int id) +{ + struct cxl_dport *dport; + + device_lock_assert(&port->dev); + list_for_each_entry (dport, &port->dports, list) + if (dport->port_id == id) + return dport; + return NULL; +} + +static int add_dport(struct cxl_port *port, struct cxl_dport *new) +{ + struct cxl_dport *dup; + + device_lock(&port->dev); + dup = find_dport(port, new->port_id); + if (dup) + dev_err(&port->dev, + "unable to add dport%d-%s non-unique port id (%s)\n", + new->port_id, dev_name(new->dport), + dev_name(dup->dport)); + else + list_add_tail(&new->list, &port->dports); + device_unlock(&port->dev); + + return dup ? -EEXIST : 0; +} + +/** + * cxl_add_dport - append downstream port data to a cxl_port + * @port: the cxl_port that references this dport + * @dport_dev: firmware or PCI device representing the dport + * @port_id: identifier for this dport in a decoder's target list + * @component_reg_phys: optional location of CXL component registers + * + * Note that all allocations and links are undone by cxl_port deletion + * and release. + */ +int cxl_add_dport(struct cxl_port *port, struct device *dport_dev, int port_id, + resource_size_t component_reg_phys) +{ + char link_name[CXL_TARGET_STRLEN]; + struct cxl_dport *dport; + int rc; + + if (snprintf(link_name, CXL_TARGET_STRLEN, "dport%d", port_id) >= + CXL_TARGET_STRLEN) + return -EINVAL; + + dport = kzalloc(sizeof(*dport), GFP_KERNEL); + if (!dport) + return -ENOMEM; + + INIT_LIST_HEAD(&dport->list); + dport->dport = get_device(dport_dev); + dport->port_id = port_id; + dport->component_reg_phys = component_reg_phys; + dport->port = port; + + rc = add_dport(port, dport); + if (rc) + goto err; + + rc = sysfs_create_link(&port->dev.kobj, &dport_dev->kobj, link_name); + if (rc) + goto err; + + return 0; +err: + cxl_dport_release(dport); + return rc; +} +EXPORT_SYMBOL_GPL(cxl_add_dport); + /** * cxl_probe_component_regs() - Detect CXL Component register blocks * @dev: Host device of the @base mapping diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h index 5651e5bb8274..dd159fd6d692 100644 --- a/drivers/cxl/cxl.h +++ b/drivers/cxl/cxl.h @@ -154,6 +154,7 @@ int cxl_map_device_regs(struct pci_dev *pdev, struct cxl_register_map *map); #define CXL_RESOURCE_NONE ((resource_size_t) -1) +#define CXL_TARGET_STRLEN 20 /** * struct cxl_port - logical collection of upstream port devices and @@ -162,19 +163,39 @@ int cxl_map_device_regs(struct pci_dev *pdev, * @dev: this port's device * @uport: PCI or platform device implementing the upstream port capability * @id: id for port device-name + * @dports: cxl_dport instances referenced by decoders * @component_reg_phys: component register capability base address (optional) */ struct cxl_port { struct device dev; struct device *uport; int id; + struct list_head dports; resource_size_t component_reg_phys; }; +/** + * struct cxl_dport - CXL downstream port + * @dport: PCI bridge or firmware device representing the downstream link + * @port_id: unique hardware identifier for dport in decoder target list + * @component_reg_phys: downstream port component registers + * @port: reference to cxl_port that contains this downstream port + * @list: node for a cxl_port's list of cxl_dport instances + */ +struct cxl_dport { + struct device *dport; + int port_id; + resource_size_t component_reg_phys; + struct cxl_port *port; + struct list_head list; +}; + struct cxl_port *to_cxl_port(struct device *dev); struct cxl_port *devm_cxl_add_port(struct device *host, struct device *uport, resource_size_t component_reg_phys, struct cxl_port *parent_port); +int cxl_add_dport(struct cxl_port *port, struct device *dport, int port_id, + resource_size_t component_reg_phys); extern struct bus_type cxl_bus_type; #endif /* __CXL_H__ */ From 3b94ce7b7bc1b436465a93f19a50e0b495b429a1 Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Wed, 9 Jun 2021 09:01:51 -0700 Subject: [PATCH 22/34] cxl/acpi: Enumerate host bridge root ports While the resources enumerated by the CEDT.CFMWS identify a cxl_port with host bridges as downstream ports, host bridges themselves are upstream ports that decode to downstream ports represented by PCIe Root Ports. Walk the PCIe Root Ports connected to a CXL Host Bridge, identified by the ACPI0016 _HID, and add each one as a cxl_dport of the host bridge cxl_port. For now, component registers are not enumerated, only the first order uport / dport relationships. Reviewed-by: Jonathan Cameron Link: https://lore.kernel.org/r/162325451145.2293126.10149150938788969381.stgit@dwillia2-desk3.amr.corp.intel.com Signed-off-by: Dan Williams --- drivers/cxl/acpi.c | 93 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 92 insertions(+), 1 deletion(-) diff --git a/drivers/cxl/acpi.c b/drivers/cxl/acpi.c index 5eb9543c587a..1f075dffc042 100644 --- a/drivers/cxl/acpi.c +++ b/drivers/cxl/acpi.c @@ -5,8 +5,51 @@ #include #include #include +#include #include "cxl.h" +struct cxl_walk_context { + struct device *dev; + struct pci_bus *root; + struct cxl_port *port; + int error; + int count; +}; + +static int match_add_root_ports(struct pci_dev *pdev, void *data) +{ + struct cxl_walk_context *ctx = data; + struct pci_bus *root_bus = ctx->root; + struct cxl_port *port = ctx->port; + int type = pci_pcie_type(pdev); + struct device *dev = ctx->dev; + u32 lnkcap, port_num; + int rc; + + if (pdev->bus != root_bus) + return 0; + if (!pci_is_pcie(pdev)) + return 0; + if (type != PCI_EXP_TYPE_ROOT_PORT) + return 0; + if (pci_read_config_dword(pdev, pci_pcie_cap(pdev) + PCI_EXP_LNKCAP, + &lnkcap) != PCIBIOS_SUCCESSFUL) + return 0; + + /* TODO walk DVSEC to find component register base */ + port_num = FIELD_GET(PCI_EXP_LNKCAP_PN, lnkcap); + rc = cxl_add_dport(port, &pdev->dev, port_num, CXL_RESOURCE_NONE); + if (rc) { + ctx->error = rc; + return rc; + } + ctx->count++; + + dev_dbg(dev, "add dport%d: %s\n", port_num, dev_name(&pdev->dev)); + + return 0; +} + static struct acpi_device *to_cxl_host_bridge(struct device *dev) { struct acpi_device *adev = to_acpi_device(dev); @@ -16,6 +59,44 @@ static struct acpi_device *to_cxl_host_bridge(struct device *dev) return NULL; } +/* + * A host bridge is a dport to a CFMWS decode and it is a uport to the + * dport (PCIe Root Ports) in the host bridge. + */ +static int add_host_bridge_uport(struct device *match, void *arg) +{ + struct acpi_device *bridge = to_cxl_host_bridge(match); + struct cxl_port *root_port = arg; + struct device *host = root_port->dev.parent; + struct acpi_pci_root *pci_root; + struct cxl_walk_context ctx; + struct cxl_port *port; + + if (!bridge) + return 0; + + pci_root = acpi_pci_find_root(bridge->handle); + if (!pci_root) + return -ENXIO; + + /* TODO: fold in CEDT.CHBS retrieval */ + port = devm_cxl_add_port(host, match, CXL_RESOURCE_NONE, root_port); + if (IS_ERR(port)) + return PTR_ERR(port); + dev_dbg(host, "%s: add: %s\n", dev_name(match), dev_name(&port->dev)); + + ctx = (struct cxl_walk_context){ + .dev = host, + .root = pci_root->bus, + .port = port, + }; + pci_walk_bus(pci_root->bus, match_add_root_ports, &ctx); + + if (ctx.count == 0) + return -ENODEV; + return ctx.error; +} + static int add_host_bridge_dport(struct device *match, void *arg) { int rc; @@ -48,6 +129,7 @@ static int add_host_bridge_dport(struct device *match, void *arg) static int cxl_acpi_probe(struct platform_device *pdev) { + int rc; struct cxl_port *root_port; struct device *host = &pdev->dev; struct acpi_device *adev = ACPI_COMPANION(host); @@ -57,8 +139,17 @@ static int cxl_acpi_probe(struct platform_device *pdev) return PTR_ERR(root_port); dev_dbg(host, "add: %s\n", dev_name(&root_port->dev)); + rc = bus_for_each_dev(adev->dev.bus, NULL, root_port, + add_host_bridge_dport); + if (rc) + return rc; + + /* + * Root level scanned with host-bridge as dports, now scan host-bridges + * for their role as CXL uports to their CXL-capable PCIe Root Ports. + */ return bus_for_each_dev(adev->dev.bus, NULL, root_port, - add_host_bridge_dport); + add_host_bridge_uport); } static const struct acpi_device_id cxl_acpi_ids[] = { From 40ba17afdfabb01688c61565dbe02a916241bc05 Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Wed, 9 Jun 2021 09:43:29 -0700 Subject: [PATCH 23/34] cxl/acpi: Introduce cxl_decoder objects MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A cxl_decoder is a child of a cxl_port. It represents a hardware decoder configuration of an upstream port to one or more of its downstream ports. The decoder is either represented in CXL standard HDM decoder registers (see CXL 2.0 section 8.2.5.12 CXL HDM Decoder Capability Structure), or it is a static decode configuration communicated by platform firmware (see the CXL Early Discovery Table: Fixed Memory Window Structure). The firmware described and hardware described decoders differ slightly leading to 2 different sub-types of decoders, cxl_decoder_root and cxl_decoder_switch. At the root level the decode capabilities restrict what can be mapped beneath them. Mid-level switch decoders are configured for either acclerator (type-2) or memory-expander (type-3) operation, but they are otherwise agnostic to the type of memory (volatile vs persistent) being mapped. Here is an example topology from a single-ported host-bridge environment without CFMWS decodes enumerated. /sys/bus/cxl/devices/root0 ├── devtype ├── dport0 -> ../../../LNXSYSTM:00/LNXSYBUS:00/ACPI0016:00 ├── port1 │   ├── decoder1.0 │   │   ├── devtype │   │   ├── locked │   │   ├── size │   │   ├── start │   │   ├── subsystem -> ../../../../../../bus/cxl │   │   ├── target_list │   │   ├── target_type │   │   └── uevent │   ├── devtype │   ├── dport0 -> ../../../../pci0000:34/0000:34:00.0 │   ├── subsystem -> ../../../../../bus/cxl │   ├── uevent │   └── uport -> ../../../../LNXSYSTM:00/LNXSYBUS:00/ACPI0016:00 ├── subsystem -> ../../../../bus/cxl ├── uevent └── uport -> ../../ACPI0017:00 Reviewed-by: Jonathan Cameron Link: https://lore.kernel.org/r/162325695128.2293823.17519927266014762694.stgit@dwillia2-desk3.amr.corp.intel.com Signed-off-by: Dan Williams --- Documentation/ABI/testing/sysfs-bus-cxl | 70 +++++++ drivers/cxl/acpi.c | 20 +- drivers/cxl/core.c | 265 ++++++++++++++++++++++++ drivers/cxl/cxl.h | 63 ++++++ 4 files changed, 417 insertions(+), 1 deletion(-) diff --git a/Documentation/ABI/testing/sysfs-bus-cxl b/Documentation/ABI/testing/sysfs-bus-cxl index f680da85fd44..0b6a2e6e8fbb 100644 --- a/Documentation/ABI/testing/sysfs-bus-cxl +++ b/Documentation/ABI/testing/sysfs-bus-cxl @@ -57,3 +57,73 @@ Description: decode of CXL memory resources. The 'Y' integer reflects the hardware port unique-id used in the hardware decoder target list. + +What: /sys/bus/cxl/devices/decoderX.Y +Date: June, 2021 +KernelVersion: v5.14 +Contact: linux-cxl@vger.kernel.org +Description: + CXL decoder objects are enumerated from either a platform + firmware description, or a CXL HDM decoder register set in a + PCIe device (see CXL 2.0 section 8.2.5.12 CXL HDM Decoder + Capability Structure). The 'X' in decoderX.Y represents the + cxl_port container of this decoder, and 'Y' represents the + instance id of a given decoder resource. + +What: /sys/bus/cxl/devices/decoderX.Y/{start,size} +Date: June, 2021 +KernelVersion: v5.14 +Contact: linux-cxl@vger.kernel.org +Description: + The 'start' and 'size' attributes together convey the physical + address base and number of bytes mapped in the decoder's decode + window. For decoders of devtype "cxl_decoder_root" the address + range is fixed. For decoders of devtype "cxl_decoder_switch" the + address is bounded by the decode range of the cxl_port ancestor + of the decoder's cxl_port, and dynamically updates based on the + active memory regions in that address space. + +What: /sys/bus/cxl/devices/decoderX.Y/locked +Date: June, 2021 +KernelVersion: v5.14 +Contact: linux-cxl@vger.kernel.org +Description: + CXL HDM decoders have the capability to lock the configuration + until the next device reset. For decoders of devtype + "cxl_decoder_root" there is no standard facility to unlock them. + For decoders of devtype "cxl_decoder_switch" a secondary bus + reset, of the PCIe bridge that provides the bus for this + decoders uport, unlocks / resets the decoder. + +What: /sys/bus/cxl/devices/decoderX.Y/target_list +Date: June, 2021 +KernelVersion: v5.14 +Contact: linux-cxl@vger.kernel.org +Description: + Display a comma separated list of the current decoder target + configuration. The list is ordered by the current configured + interleave order of the decoder's dport instances. Each entry in + the list is a dport id. + +What: /sys/bus/cxl/devices/decoderX.Y/cap_{pmem,ram,type2,type3} +Date: June, 2021 +KernelVersion: v5.14 +Contact: linux-cxl@vger.kernel.org +Description: + When a CXL decoder is of devtype "cxl_decoder_root", it + represents a fixed memory window identified by platform + firmware. A fixed window may only support a subset of memory + types. The 'cap_*' attributes indicate whether persistent + memory, volatile memory, accelerator memory, and / or expander + memory may be mapped behind this decoder's memory window. + +What: /sys/bus/cxl/devices/decoderX.Y/target_type +Date: June, 2021 +KernelVersion: v5.14 +Contact: linux-cxl@vger.kernel.org +Description: + When a CXL decoder is of devtype "cxl_decoder_switch", it can + optionally decode either accelerator memory (type-2) or expander + memory (type-3). The 'target_type' attribute indicates the + current setting which may dynamically change based on what + memory regions are activated in this decode hierarchy. diff --git a/drivers/cxl/acpi.c b/drivers/cxl/acpi.c index 1f075dffc042..be357eea552c 100644 --- a/drivers/cxl/acpi.c +++ b/drivers/cxl/acpi.c @@ -70,6 +70,7 @@ static int add_host_bridge_uport(struct device *match, void *arg) struct device *host = root_port->dev.parent; struct acpi_pci_root *pci_root; struct cxl_walk_context ctx; + struct cxl_decoder *cxld; struct cxl_port *port; if (!bridge) @@ -94,7 +95,24 @@ static int add_host_bridge_uport(struct device *match, void *arg) if (ctx.count == 0) return -ENODEV; - return ctx.error; + if (ctx.error) + return ctx.error; + + /* TODO: Scan CHBCR for HDM Decoder resources */ + + /* + * In the single-port host-bridge case there are no HDM decoders + * in the CHBCR and a 1:1 passthrough decode is implied. + */ + if (ctx.count == 1) { + cxld = devm_cxl_add_passthrough_decoder(host, port); + if (IS_ERR(cxld)) + return PTR_ERR(cxld); + + dev_dbg(host, "add: %s\n", dev_name(&cxld->dev)); + } + + return 0; } static int add_host_bridge_dport(struct device *match, void *arg) diff --git a/drivers/cxl/core.c b/drivers/cxl/core.c index 8a3f3804f252..1b9ee0b08384 100644 --- a/drivers/cxl/core.c +++ b/drivers/cxl/core.c @@ -33,6 +33,168 @@ static struct attribute_group cxl_base_attribute_group = { .attrs = cxl_base_attributes, }; +static ssize_t start_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + struct cxl_decoder *cxld = to_cxl_decoder(dev); + + return sysfs_emit(buf, "%#llx\n", cxld->range.start); +} +static DEVICE_ATTR_RO(start); + +static ssize_t size_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + struct cxl_decoder *cxld = to_cxl_decoder(dev); + + return sysfs_emit(buf, "%#llx\n", range_len(&cxld->range)); +} +static DEVICE_ATTR_RO(size); + +#define CXL_DECODER_FLAG_ATTR(name, flag) \ +static ssize_t name##_show(struct device *dev, \ + struct device_attribute *attr, char *buf) \ +{ \ + struct cxl_decoder *cxld = to_cxl_decoder(dev); \ + \ + return sysfs_emit(buf, "%s\n", \ + (cxld->flags & (flag)) ? "1" : "0"); \ +} \ +static DEVICE_ATTR_RO(name) + +CXL_DECODER_FLAG_ATTR(cap_pmem, CXL_DECODER_F_PMEM); +CXL_DECODER_FLAG_ATTR(cap_ram, CXL_DECODER_F_RAM); +CXL_DECODER_FLAG_ATTR(cap_type2, CXL_DECODER_F_TYPE2); +CXL_DECODER_FLAG_ATTR(cap_type3, CXL_DECODER_F_TYPE3); +CXL_DECODER_FLAG_ATTR(locked, CXL_DECODER_F_LOCK); + +static ssize_t target_type_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct cxl_decoder *cxld = to_cxl_decoder(dev); + + switch (cxld->target_type) { + case CXL_DECODER_ACCELERATOR: + return sysfs_emit(buf, "accelerator\n"); + case CXL_DECODER_EXPANDER: + return sysfs_emit(buf, "expander\n"); + } + return -ENXIO; +} +static DEVICE_ATTR_RO(target_type); + +static ssize_t target_list_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct cxl_decoder *cxld = to_cxl_decoder(dev); + ssize_t offset = 0; + int i, rc = 0; + + device_lock(dev); + for (i = 0; i < cxld->interleave_ways; i++) { + struct cxl_dport *dport = cxld->target[i]; + struct cxl_dport *next = NULL; + + if (!dport) + break; + + if (i + 1 < cxld->interleave_ways) + next = cxld->target[i + 1]; + rc = sysfs_emit_at(buf, offset, "%d%s", dport->port_id, + next ? "," : ""); + if (rc < 0) + break; + offset += rc; + } + device_unlock(dev); + + if (rc < 0) + return rc; + + rc = sysfs_emit_at(buf, offset, "\n"); + if (rc < 0) + return rc; + + return offset + rc; +} +static DEVICE_ATTR_RO(target_list); + +static struct attribute *cxl_decoder_base_attrs[] = { + &dev_attr_start.attr, + &dev_attr_size.attr, + &dev_attr_locked.attr, + &dev_attr_target_list.attr, + NULL, +}; + +static struct attribute_group cxl_decoder_base_attribute_group = { + .attrs = cxl_decoder_base_attrs, +}; + +static struct attribute *cxl_decoder_root_attrs[] = { + &dev_attr_cap_pmem.attr, + &dev_attr_cap_ram.attr, + &dev_attr_cap_type2.attr, + &dev_attr_cap_type3.attr, + NULL, +}; + +static struct attribute_group cxl_decoder_root_attribute_group = { + .attrs = cxl_decoder_root_attrs, +}; + +static const struct attribute_group *cxl_decoder_root_attribute_groups[] = { + &cxl_decoder_root_attribute_group, + &cxl_decoder_base_attribute_group, + &cxl_base_attribute_group, + NULL, +}; + +static struct attribute *cxl_decoder_switch_attrs[] = { + &dev_attr_target_type.attr, + NULL, +}; + +static struct attribute_group cxl_decoder_switch_attribute_group = { + .attrs = cxl_decoder_switch_attrs, +}; + +static const struct attribute_group *cxl_decoder_switch_attribute_groups[] = { + &cxl_decoder_switch_attribute_group, + &cxl_decoder_base_attribute_group, + &cxl_base_attribute_group, + NULL, +}; + +static void cxl_decoder_release(struct device *dev) +{ + struct cxl_decoder *cxld = to_cxl_decoder(dev); + struct cxl_port *port = to_cxl_port(dev->parent); + + ida_free(&port->decoder_ida, cxld->id); + kfree(cxld); +} + +static const struct device_type cxl_decoder_switch_type = { + .name = "cxl_decoder_switch", + .release = cxl_decoder_release, + .groups = cxl_decoder_switch_attribute_groups, +}; + +static const struct device_type cxl_decoder_root_type = { + .name = "cxl_decoder_root", + .release = cxl_decoder_release, + .groups = cxl_decoder_root_attribute_groups, +}; + +struct cxl_decoder *to_cxl_decoder(struct device *dev) +{ + if (dev_WARN_ONCE(dev, dev->type->release != cxl_decoder_release, + "not a cxl_decoder device\n")) + return NULL; + return container_of(dev, struct cxl_decoder, dev); +} + static void cxl_dport_release(struct cxl_dport *dport) { list_del(&dport->list); @@ -138,6 +300,7 @@ static struct cxl_port *cxl_port_alloc(struct device *uport, port->uport = uport; port->component_reg_phys = component_reg_phys; + ida_init(&port->decoder_ida); INIT_LIST_HEAD(&port->dports); device_initialize(dev); @@ -274,6 +437,108 @@ err: } EXPORT_SYMBOL_GPL(cxl_add_dport); +static struct cxl_decoder * +cxl_decoder_alloc(struct cxl_port *port, int nr_targets, resource_size_t base, + resource_size_t len, int interleave_ways, + int interleave_granularity, enum cxl_decoder_type type, + unsigned long flags) +{ + struct cxl_decoder *cxld; + struct device *dev; + int rc = 0; + + if (interleave_ways < 1) + return ERR_PTR(-EINVAL); + + device_lock(&port->dev); + if (list_empty(&port->dports)) + rc = -EINVAL; + device_unlock(&port->dev); + if (rc) + return ERR_PTR(rc); + + cxld = kzalloc(struct_size(cxld, target, nr_targets), GFP_KERNEL); + if (!cxld) + return ERR_PTR(-ENOMEM); + + rc = ida_alloc(&port->decoder_ida, GFP_KERNEL); + if (rc < 0) + goto err; + + *cxld = (struct cxl_decoder) { + .id = rc, + .range = { + .start = base, + .end = base + len - 1, + }, + .flags = flags, + .interleave_ways = interleave_ways, + .interleave_granularity = interleave_granularity, + .target_type = type, + }; + + /* handle implied target_list */ + if (interleave_ways == 1) + cxld->target[0] = + list_first_entry(&port->dports, struct cxl_dport, list); + dev = &cxld->dev; + device_initialize(dev); + device_set_pm_not_required(dev); + dev->parent = &port->dev; + dev->bus = &cxl_bus_type; + + /* root ports do not have a cxl_port_type parent */ + if (port->dev.parent->type == &cxl_port_type) + dev->type = &cxl_decoder_switch_type; + else + dev->type = &cxl_decoder_root_type; + + return cxld; +err: + kfree(cxld); + return ERR_PTR(rc); +} + +static void unregister_dev(void *dev) +{ + device_unregister(dev); +} + +struct cxl_decoder * +devm_cxl_add_decoder(struct device *host, struct cxl_port *port, int nr_targets, + resource_size_t base, resource_size_t len, + int interleave_ways, int interleave_granularity, + enum cxl_decoder_type type, unsigned long flags) +{ + struct cxl_decoder *cxld; + struct device *dev; + int rc; + + cxld = cxl_decoder_alloc(port, nr_targets, base, len, interleave_ways, + interleave_granularity, type, flags); + if (IS_ERR(cxld)) + return cxld; + + dev = &cxld->dev; + rc = dev_set_name(dev, "decoder%d.%d", port->id, cxld->id); + if (rc) + goto err; + + rc = device_add(dev); + if (rc) + goto err; + + rc = devm_add_action_or_reset(host, unregister_dev, dev); + if (rc) + return ERR_PTR(rc); + return cxld; + +err: + put_device(dev); + return ERR_PTR(rc); +} +EXPORT_SYMBOL_GPL(devm_cxl_add_decoder); + /** * cxl_probe_component_regs() - Detect CXL Component register blocks * @dev: Host device of the @base mapping diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h index dd159fd6d692..b988ea288f53 100644 --- a/drivers/cxl/cxl.h +++ b/drivers/cxl/cxl.h @@ -156,6 +156,45 @@ int cxl_map_device_regs(struct pci_dev *pdev, #define CXL_RESOURCE_NONE ((resource_size_t) -1) #define CXL_TARGET_STRLEN 20 +/* + * cxl_decoder flags that define the type of memory / devices this + * decoder supports as well as configuration lock status See "CXL 2.0 + * 8.2.5.12.7 CXL HDM Decoder 0 Control Register" for details. + */ +#define CXL_DECODER_F_RAM BIT(0) +#define CXL_DECODER_F_PMEM BIT(1) +#define CXL_DECODER_F_TYPE2 BIT(2) +#define CXL_DECODER_F_TYPE3 BIT(3) +#define CXL_DECODER_F_LOCK BIT(4) +#define CXL_DECODER_F_MASK GENMASK(4, 0) + +enum cxl_decoder_type { + CXL_DECODER_ACCELERATOR = 2, + CXL_DECODER_EXPANDER = 3, +}; + +/** + * struct cxl_decoder - CXL address range decode configuration + * @dev: this decoder's device + * @id: kernel device name id + * @range: address range considered by this decoder + * @interleave_ways: number of cxl_dports in this decode + * @interleave_granularity: data stride per dport + * @target_type: accelerator vs expander (type2 vs type3) selector + * @flags: memory type capabilities and locking + * @target: active ordered target list in current decoder configuration + */ +struct cxl_decoder { + struct device dev; + int id; + struct range range; + int interleave_ways; + int interleave_granularity; + enum cxl_decoder_type target_type; + unsigned long flags; + struct cxl_dport *target[]; +}; + /** * struct cxl_port - logical collection of upstream port devices and * downstream port devices to construct a CXL memory @@ -164,6 +203,7 @@ int cxl_map_device_regs(struct pci_dev *pdev, * @uport: PCI or platform device implementing the upstream port capability * @id: id for port device-name * @dports: cxl_dport instances referenced by decoders + * @decoder_ida: allocator for decoder ids * @component_reg_phys: component register capability base address (optional) */ struct cxl_port { @@ -171,6 +211,7 @@ struct cxl_port { struct device *uport; int id; struct list_head dports; + struct ida decoder_ida; resource_size_t component_reg_phys; }; @@ -197,5 +238,27 @@ struct cxl_port *devm_cxl_add_port(struct device *host, struct device *uport, int cxl_add_dport(struct cxl_port *port, struct device *dport, int port_id, resource_size_t component_reg_phys); + +struct cxl_decoder *to_cxl_decoder(struct device *dev); +struct cxl_decoder * +devm_cxl_add_decoder(struct device *host, struct cxl_port *port, int nr_targets, + resource_size_t base, resource_size_t len, + int interleave_ways, int interleave_granularity, + enum cxl_decoder_type type, unsigned long flags); + +/* + * Per the CXL specification (8.2.5.12 CXL HDM Decoder Capability Structure) + * single ported host-bridges need not publish a decoder capability when a + * passthrough decode can be assumed, i.e. all transactions that the uport sees + * are claimed and passed to the single dport. Default the range a 0-base + * 0-length until the first CXL region is activated. + */ +static inline struct cxl_decoder * +devm_cxl_add_passthrough_decoder(struct device *host, struct cxl_port *port) +{ + return devm_cxl_add_decoder(host, port, 1, 0, 0, 1, PAGE_SIZE, + CXL_DECODER_EXPANDER, 0); +} + extern struct bus_type cxl_bus_type; #endif /* __CXL_H__ */ From 6423035fd26c1ecb72f90ecab909e9afa36942b8 Mon Sep 17 00:00:00 2001 From: Ben Widawsky Date: Fri, 11 Jun 2021 12:01:11 -0700 Subject: [PATCH 24/34] cxl/hdm: Fix decoder count calculation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The decoder count in the HDM decoder capability structure is an encoded field. As defined in the spec: Decoder Count: Reports the number of memory address decoders implemented by the component. 0 – 1 Decoder 1 – 2 Decoders 2 – 4 Decoders 3 – 6 Decoders 4 – 8 Decoders 5 – 10 Decoders All other values are reserved Nothing is actually fixed by this as nothing actually used this mapping yet. Cc: Ira Weiny Fixes: 08422378c4ad ("cxl/pci: Add HDM decoder capabilities") Acked-by: Jonathan Cameron Signed-off-by: Ben Widawsky Link: https://lore.kernel.org/r/20210611190111.121295-1-ben.widawsky@intel.com Signed-off-by: Dan Williams --- drivers/cxl/core.c | 2 +- drivers/cxl/cxl.h | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/drivers/cxl/core.c b/drivers/cxl/core.c index 1b9ee0b08384..b134b29923ca 100644 --- a/drivers/cxl/core.c +++ b/drivers/cxl/core.c @@ -595,7 +595,7 @@ void cxl_probe_component_regs(struct device *dev, void __iomem *base, hdr = readl(register_block); - decoder_cnt = FIELD_GET(CXL_HDM_DECODER_COUNT_MASK, hdr); + decoder_cnt = cxl_hdm_decoder_count(hdr); length = 0x20 * decoder_cnt + 0x10; map->hdm_decoder.valid = true; diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h index b988ea288f53..97a273ae3947 100644 --- a/drivers/cxl/cxl.h +++ b/drivers/cxl/cxl.h @@ -41,6 +41,13 @@ #define CXL_HDM_DECODER0_SIZE_HIGH_OFFSET 0x1c #define CXL_HDM_DECODER0_CTRL_OFFSET 0x20 +static inline int cxl_hdm_decoder_count(u32 cap_hdr) +{ + int val = FIELD_GET(CXL_HDM_DECODER_COUNT_MASK, cap_hdr); + + return val ? val * 2 : 1; +} + /* CXL 2.0 8.2.8.1 Device Capabilities Array Register */ #define CXLDEV_CAP_ARRAY_OFFSET 0x0 #define CXLDEV_CAP_ARRAY_CAP_ID 0 From ba268647368844ed290e2f7b4da7a28cd12ee049 Mon Sep 17 00:00:00 2001 From: Ben Widawsky Date: Thu, 10 Jun 2021 22:11:13 -0700 Subject: [PATCH 25/34] cxl/component_regs: Fix offset The CXL.cache and CXL.mem registers begin after the CXL.io registers which occupy the first 0x1000 bytes. The current code wasn't setting this up properly for future users of the component registers. It was correct for the probing code however. Cc: Jonathan Cameron Cc: Ira Weiny Fixes: 08422378c4ad ("cxl/pci: Add HDM decoder capabilities") Signed-off-by: Ben Widawsky Acked-by: Jonathan Cameron Link: https://lore.kernel.org/r/20210611051113.224328-1-ben.widawsky@intel.com Signed-off-by: Dan Williams --- drivers/cxl/core.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/cxl/core.c b/drivers/cxl/core.c index b134b29923ca..c613dc795498 100644 --- a/drivers/cxl/core.c +++ b/drivers/cxl/core.c @@ -599,7 +599,7 @@ void cxl_probe_component_regs(struct device *dev, void __iomem *base, length = 0x20 * decoder_cnt + 0x10; map->hdm_decoder.valid = true; - map->hdm_decoder.offset = offset; + map->hdm_decoder.offset = CXL_CM_OFFSET + offset; map->hdm_decoder.size = length; break; default: From 87815ee9d0060a91bdf18266e42837a9adb5972e Mon Sep 17 00:00:00 2001 From: Ben Widawsky Date: Tue, 13 Apr 2021 07:09:07 -0700 Subject: [PATCH 26/34] cxl/pci: Add media provisioning required commands Some of the commands have already been defined for the support of RAW commands (to be blocked). Unlike their usage in the RAW interface, when used through the supported interface, they will be coordinated and marshalled along with other commands being issued by userspace and the driver itself. That coordination will be added later. The list of commands was determined based on the learnings from libnvdimm and this list is provided directly from Dan. Recommended-by: Dan Williams Signed-off-by: Ben Widawsky Reviewed-by: Jonathan Cameron Link: https://lore.kernel.org/r/20210413140907.534404-1-ben.widawsky@intel.com Signed-off-by: Dan Williams --- drivers/cxl/pci.c | 19 +++++++++++++++++++ include/uapi/linux/cxl_mem.h | 12 ++++++++++++ 2 files changed, 31 insertions(+) diff --git a/drivers/cxl/pci.c b/drivers/cxl/pci.c index 5a1705b52278..a8f29ce35c2a 100644 --- a/drivers/cxl/pci.c +++ b/drivers/cxl/pci.c @@ -51,7 +51,14 @@ enum opcode { CXL_MBOX_OP_GET_LSA = 0x4102, CXL_MBOX_OP_SET_LSA = 0x4103, CXL_MBOX_OP_GET_HEALTH_INFO = 0x4200, + CXL_MBOX_OP_GET_ALERT_CONFIG = 0x4201, + CXL_MBOX_OP_SET_ALERT_CONFIG = 0x4202, + CXL_MBOX_OP_GET_SHUTDOWN_STATE = 0x4203, CXL_MBOX_OP_SET_SHUTDOWN_STATE = 0x4204, + CXL_MBOX_OP_GET_POISON = 0x4300, + CXL_MBOX_OP_INJECT_POISON = 0x4301, + CXL_MBOX_OP_CLEAR_POISON = 0x4302, + CXL_MBOX_OP_GET_SCAN_MEDIA_CAPS = 0x4303, CXL_MBOX_OP_SCAN_MEDIA = 0x4304, CXL_MBOX_OP_GET_SCAN_MEDIA = 0x4305, CXL_MBOX_OP_MAX = 0x10000 @@ -159,6 +166,18 @@ static struct cxl_mem_command mem_commands[CXL_MEM_COMMAND_ID_MAX] = { CXL_CMD(GET_LSA, 0x8, ~0, 0), CXL_CMD(GET_HEALTH_INFO, 0, 0x12, 0), CXL_CMD(GET_LOG, 0x18, ~0, CXL_CMD_FLAG_FORCE_ENABLE), + CXL_CMD(SET_PARTITION_INFO, 0x0a, 0, 0), + CXL_CMD(SET_LSA, ~0, 0, 0), + CXL_CMD(GET_ALERT_CONFIG, 0, 0x10, 0), + CXL_CMD(SET_ALERT_CONFIG, 0xc, 0, 0), + CXL_CMD(GET_SHUTDOWN_STATE, 0, 0x1, 0), + CXL_CMD(SET_SHUTDOWN_STATE, 0x1, 0, 0), + CXL_CMD(GET_POISON, 0x10, ~0, 0), + CXL_CMD(INJECT_POISON, 0x8, 0, 0), + CXL_CMD(CLEAR_POISON, 0x48, 0, 0), + CXL_CMD(GET_SCAN_MEDIA_CAPS, 0x10, 0x4, 0), + CXL_CMD(SCAN_MEDIA, 0x11, 0, 0), + CXL_CMD(GET_SCAN_MEDIA, 0, ~0, 0), }; /* diff --git a/include/uapi/linux/cxl_mem.h b/include/uapi/linux/cxl_mem.h index 3155382dfc9b..f6e8a005b113 100644 --- a/include/uapi/linux/cxl_mem.h +++ b/include/uapi/linux/cxl_mem.h @@ -29,6 +29,18 @@ ___C(GET_LSA, "Get Label Storage Area"), \ ___C(GET_HEALTH_INFO, "Get Health Info"), \ ___C(GET_LOG, "Get Log"), \ + ___C(SET_PARTITION_INFO, "Set Partition Information"), \ + ___C(SET_LSA, "Set Label Storage Area"), \ + ___C(GET_ALERT_CONFIG, "Get Alert Configuration"), \ + ___C(SET_ALERT_CONFIG, "Set Alert Configuration"), \ + ___C(GET_SHUTDOWN_STATE, "Get Shutdown State"), \ + ___C(SET_SHUTDOWN_STATE, "Set Shutdown State"), \ + ___C(GET_POISON, "Get Poison List"), \ + ___C(INJECT_POISON, "Inject Poison"), \ + ___C(CLEAR_POISON, "Clear Poison"), \ + ___C(GET_SCAN_MEDIA_CAPS, "Get Scan Media Capabilities"), \ + ___C(SCAN_MEDIA, "Scan Media"), \ + ___C(GET_SCAN_MEDIA, "Get Scan Media Results"), \ ___C(MAX, "invalid / last command") #define ___C(a, b) CXL_MEM_COMMAND_ID_##a From 6af7139c979474a29a6ad642c9bf32d92e24c5bc Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Tue, 15 Jun 2021 16:18:11 -0700 Subject: [PATCH 27/34] cxl/core: Add cxl-bus driver infrastructure Enable devices on the 'cxl' bus to be attached to drivers. The initial user of this functionality is a driver for an 'nvdimm-bridge' device that anchors a libnvdimm hierarchy attached to CXL persistent memory resources. Other device types that will leverage this include: cxl_port: map and use component register functionality (HDM Decoders) cxl_nvdimm: translate CXL memory expander endpoints to libnvdimm 'nvdimm' objects cxl_region: translate CXL interleave sets to libnvdimm 'region' objects The pairing of devices to drivers is handled through the cxl_device_id() matching to cxl_driver.id values. A cxl_device_id() of '0' indicates no driver support. In addition to ->match(), ->probe(), and ->remove() support for the 'cxl' bus introduce MODULE_ALIAS_CXL() to autoload modules containing cxl-drivers. Drivers are added in follow-on changes. Reviewed-by: Jonathan Cameron Link: https://lore.kernel.org/r/162379909190.2993820.6134168109678004186.stgit@dwillia2-desk3.amr.corp.intel.com Signed-off-by: Dan Williams --- drivers/cxl/core.c | 73 ++++++++++++++++++++++++++++++++++++++++++++++ drivers/cxl/cxl.h | 22 ++++++++++++++ 2 files changed, 95 insertions(+) diff --git a/drivers/cxl/core.c b/drivers/cxl/core.c index c613dc795498..68d6afea363b 100644 --- a/drivers/cxl/core.c +++ b/drivers/cxl/core.c @@ -767,8 +767,81 @@ int cxl_map_device_regs(struct pci_dev *pdev, } EXPORT_SYMBOL_GPL(cxl_map_device_regs); +/** + * __cxl_driver_register - register a driver for the cxl bus + * @cxl_drv: cxl driver structure to attach + * @owner: owning module/driver + * @modname: KBUILD_MODNAME for parent driver + */ +int __cxl_driver_register(struct cxl_driver *cxl_drv, struct module *owner, + const char *modname) +{ + if (!cxl_drv->probe) { + pr_debug("%s ->probe() must be specified\n", modname); + return -EINVAL; + } + + if (!cxl_drv->name) { + pr_debug("%s ->name must be specified\n", modname); + return -EINVAL; + } + + if (!cxl_drv->id) { + pr_debug("%s ->id must be specified\n", modname); + return -EINVAL; + } + + cxl_drv->drv.bus = &cxl_bus_type; + cxl_drv->drv.owner = owner; + cxl_drv->drv.mod_name = modname; + cxl_drv->drv.name = cxl_drv->name; + + return driver_register(&cxl_drv->drv); +} +EXPORT_SYMBOL_GPL(__cxl_driver_register); + +void cxl_driver_unregister(struct cxl_driver *cxl_drv) +{ + driver_unregister(&cxl_drv->drv); +} +EXPORT_SYMBOL_GPL(cxl_driver_unregister); + +static int cxl_device_id(struct device *dev) +{ + return 0; +} + +static int cxl_bus_uevent(struct device *dev, struct kobj_uevent_env *env) +{ + return add_uevent_var(env, "MODALIAS=" CXL_MODALIAS_FMT, + cxl_device_id(dev)); +} + +static int cxl_bus_match(struct device *dev, struct device_driver *drv) +{ + return cxl_device_id(dev) == to_cxl_drv(drv)->id; +} + +static int cxl_bus_probe(struct device *dev) +{ + return to_cxl_drv(dev->driver)->probe(dev); +} + +static int cxl_bus_remove(struct device *dev) +{ + struct cxl_driver *cxl_drv = to_cxl_drv(dev->driver); + + if (cxl_drv->remove) + cxl_drv->remove(dev); + return 0; +} + struct bus_type cxl_bus_type = { .name = "cxl", + .uevent = cxl_bus_uevent, + .match = cxl_bus_match, + .probe = cxl_bus_probe, + .remove = cxl_bus_remove, }; EXPORT_SYMBOL_GPL(cxl_bus_type); diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h index 97a273ae3947..a6f8e7097ea2 100644 --- a/drivers/cxl/cxl.h +++ b/drivers/cxl/cxl.h @@ -268,4 +268,26 @@ devm_cxl_add_passthrough_decoder(struct device *host, struct cxl_port *port) } extern struct bus_type cxl_bus_type; + +struct cxl_driver { + const char *name; + int (*probe)(struct device *dev); + void (*remove)(struct device *dev); + struct device_driver drv; + int id; +}; + +static inline struct cxl_driver *to_cxl_drv(struct device_driver *drv) +{ + return container_of(drv, struct cxl_driver, drv); +} + +int __cxl_driver_register(struct cxl_driver *cxl_drv, struct module *owner, + const char *modname); +#define cxl_driver_register(x) __cxl_driver_register(x, THIS_MODULE, KBUILD_MODNAME) +void cxl_driver_unregister(struct cxl_driver *cxl_drv); + +#define MODULE_ALIAS_CXL(type) MODULE_ALIAS("cxl:t" __stringify(type) "*") +#define CXL_MODALIAS_FMT "cxl:t%d" + #endif /* __CXL_H__ */ From 8fdcb1704f61a8fd9be0f3849a174d084def0666 Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Tue, 15 Jun 2021 16:18:17 -0700 Subject: [PATCH 28/34] cxl/pmem: Add initial infrastructure for pmem support Register an 'nvdimm-bridge' device to act as an anchor for a libnvdimm bus hierarchy. Also, flesh out the cxl_bus definition to allow a cxl_nvdimm_bridge_driver to attach to the bridge and trigger the nvdimm-bus registration. The creation of the bridge is gated on the detection of a PMEM capable address space registered to the root. The bridge indirection allows the libnvdimm module to remain unloaded on platforms without PMEM support. Given that the probing of ACPI0017 is asynchronous to CXL endpoint devices, and the expectation that CXL endpoint devices register other PMEM resources on the 'CXL' nvdimm bus, a workqueue is added. The workqueue is needed to run bus_rescan_devices() outside of the device_lock() of the nvdimm-bridge device to rendezvous nvdimm resources as they arrive. For now only the bus is taken online/offline in the workqueue. Reviewed-by: Jonathan Cameron Link: https://lore.kernel.org/r/162379909706.2993820.14051258608641140169.stgit@dwillia2-desk3.amr.corp.intel.com Signed-off-by: Dan Williams --- drivers/cxl/Kconfig | 13 ++++ drivers/cxl/Makefile | 2 + drivers/cxl/acpi.c | 37 +++++++++++- drivers/cxl/core.c | 121 +++++++++++++++++++++++++++++++++++++ drivers/cxl/cxl.h | 24 ++++++++ drivers/cxl/pmem.c | 140 +++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 335 insertions(+), 2 deletions(-) create mode 100644 drivers/cxl/pmem.c diff --git a/drivers/cxl/Kconfig b/drivers/cxl/Kconfig index 1a44b173dcbc..e6de221cc568 100644 --- a/drivers/cxl/Kconfig +++ b/drivers/cxl/Kconfig @@ -61,5 +61,18 @@ config CXL_ACPI hierarchy to map regions that represent System RAM, or Persistent Memory regions to be managed by LIBNVDIMM. + If unsure say 'm'. + +config CXL_PMEM + tristate "CXL PMEM: Persistent Memory Support" + depends on LIBNVDIMM + default CXL_BUS + help + In addition to typical memory resources a platform may also advertise + support for persistent memory attached via CXL. This support is + managed via a bridge driver from CXL to the LIBNVDIMM system + subsystem. Say 'y/m' to enable support for enumerating and + provisioning the persistent memory capacity of CXL memory expanders. + If unsure say 'm'. endif diff --git a/drivers/cxl/Makefile b/drivers/cxl/Makefile index a29efb3e8ad2..32954059b37b 100644 --- a/drivers/cxl/Makefile +++ b/drivers/cxl/Makefile @@ -2,8 +2,10 @@ obj-$(CONFIG_CXL_BUS) += cxl_core.o obj-$(CONFIG_CXL_MEM) += cxl_pci.o obj-$(CONFIG_CXL_ACPI) += cxl_acpi.o +obj-$(CONFIG_CXL_PMEM) += cxl_pmem.o ccflags-y += -DDEFAULT_SYMBOL_NAMESPACE=CXL cxl_core-y := core.o cxl_pci-y := pci.o cxl_acpi-y := acpi.o +cxl_pmem-y := pmem.o diff --git a/drivers/cxl/acpi.c b/drivers/cxl/acpi.c index be357eea552c..8a723f7f3f73 100644 --- a/drivers/cxl/acpi.c +++ b/drivers/cxl/acpi.c @@ -145,6 +145,30 @@ static int add_host_bridge_dport(struct device *match, void *arg) return 0; } +static int add_root_nvdimm_bridge(struct device *match, void *data) +{ + struct cxl_decoder *cxld; + struct cxl_port *root_port = data; + struct cxl_nvdimm_bridge *cxl_nvb; + struct device *host = root_port->dev.parent; + + if (!is_root_decoder(match)) + return 0; + + cxld = to_cxl_decoder(match); + if (!(cxld->flags & CXL_DECODER_F_PMEM)) + return 0; + + cxl_nvb = devm_cxl_add_nvdimm_bridge(host, root_port); + if (IS_ERR(cxl_nvb)) { + dev_dbg(host, "failed to register pmem\n"); + return PTR_ERR(cxl_nvb); + } + dev_dbg(host, "%s: add: %s\n", dev_name(&root_port->dev), + dev_name(&cxl_nvb->dev)); + return 1; +} + static int cxl_acpi_probe(struct platform_device *pdev) { int rc; @@ -166,8 +190,17 @@ static int cxl_acpi_probe(struct platform_device *pdev) * Root level scanned with host-bridge as dports, now scan host-bridges * for their role as CXL uports to their CXL-capable PCIe Root Ports. */ - return bus_for_each_dev(adev->dev.bus, NULL, root_port, - add_host_bridge_uport); + rc = bus_for_each_dev(adev->dev.bus, NULL, root_port, + add_host_bridge_uport); + if (rc) + return rc; + + if (IS_ENABLED(CONFIG_CXL_PMEM)) + rc = device_for_each_child(&root_port->dev, root_port, + add_root_nvdimm_bridge); + if (rc < 0) + return rc; + return 0; } static const struct acpi_device_id cxl_acpi_ids[] = { diff --git a/drivers/cxl/core.c b/drivers/cxl/core.c index 68d6afea363b..aaffb390d168 100644 --- a/drivers/cxl/core.c +++ b/drivers/cxl/core.c @@ -187,6 +187,12 @@ static const struct device_type cxl_decoder_root_type = { .groups = cxl_decoder_root_attribute_groups, }; +bool is_root_decoder(struct device *dev) +{ + return dev->type == &cxl_decoder_root_type; +} +EXPORT_SYMBOL_GPL(is_root_decoder); + struct cxl_decoder *to_cxl_decoder(struct device *dev) { if (dev_WARN_ONCE(dev, dev->type->release != cxl_decoder_release, @@ -194,6 +200,7 @@ struct cxl_decoder *to_cxl_decoder(struct device *dev) return NULL; return container_of(dev, struct cxl_decoder, dev); } +EXPORT_SYMBOL_GPL(to_cxl_decoder); static void cxl_dport_release(struct cxl_dport *dport) { @@ -611,6 +618,118 @@ void cxl_probe_component_regs(struct device *dev, void __iomem *base, } EXPORT_SYMBOL_GPL(cxl_probe_component_regs); +static void cxl_nvdimm_bridge_release(struct device *dev) +{ + struct cxl_nvdimm_bridge *cxl_nvb = to_cxl_nvdimm_bridge(dev); + + kfree(cxl_nvb); +} + +static const struct attribute_group *cxl_nvdimm_bridge_attribute_groups[] = { + &cxl_base_attribute_group, + NULL, +}; + +static const struct device_type cxl_nvdimm_bridge_type = { + .name = "cxl_nvdimm_bridge", + .release = cxl_nvdimm_bridge_release, + .groups = cxl_nvdimm_bridge_attribute_groups, +}; + +struct cxl_nvdimm_bridge *to_cxl_nvdimm_bridge(struct device *dev) +{ + if (dev_WARN_ONCE(dev, dev->type != &cxl_nvdimm_bridge_type, + "not a cxl_nvdimm_bridge device\n")) + return NULL; + return container_of(dev, struct cxl_nvdimm_bridge, dev); +} +EXPORT_SYMBOL_GPL(to_cxl_nvdimm_bridge); + +static struct cxl_nvdimm_bridge * +cxl_nvdimm_bridge_alloc(struct cxl_port *port) +{ + struct cxl_nvdimm_bridge *cxl_nvb; + struct device *dev; + + cxl_nvb = kzalloc(sizeof(*cxl_nvb), GFP_KERNEL); + if (!cxl_nvb) + return ERR_PTR(-ENOMEM); + + dev = &cxl_nvb->dev; + cxl_nvb->port = port; + cxl_nvb->state = CXL_NVB_NEW; + device_initialize(dev); + device_set_pm_not_required(dev); + dev->parent = &port->dev; + dev->bus = &cxl_bus_type; + dev->type = &cxl_nvdimm_bridge_type; + + return cxl_nvb; +} + +static void unregister_nvb(void *_cxl_nvb) +{ + struct cxl_nvdimm_bridge *cxl_nvb = _cxl_nvb; + bool flush; + + /* + * If the bridge was ever activated then there might be in-flight state + * work to flush. Once the state has been changed to 'dead' then no new + * work can be queued by user-triggered bind. + */ + device_lock(&cxl_nvb->dev); + flush = cxl_nvb->state != CXL_NVB_NEW; + cxl_nvb->state = CXL_NVB_DEAD; + device_unlock(&cxl_nvb->dev); + + /* + * Even though the device core will trigger device_release_driver() + * before the unregister, it does not know about the fact that + * cxl_nvdimm_bridge_driver defers ->remove() work. So, do the driver + * release not and flush it before tearing down the nvdimm device + * hierarchy. + */ + device_release_driver(&cxl_nvb->dev); + if (flush) + flush_work(&cxl_nvb->state_work); + device_unregister(&cxl_nvb->dev); +} + +struct cxl_nvdimm_bridge *devm_cxl_add_nvdimm_bridge(struct device *host, + struct cxl_port *port) +{ + struct cxl_nvdimm_bridge *cxl_nvb; + struct device *dev; + int rc; + + if (!IS_ENABLED(CONFIG_CXL_PMEM)) + return ERR_PTR(-ENXIO); + + cxl_nvb = cxl_nvdimm_bridge_alloc(port); + if (IS_ERR(cxl_nvb)) + return cxl_nvb; + + dev = &cxl_nvb->dev; + rc = dev_set_name(dev, "nvdimm-bridge"); + if (rc) + goto err; + + rc = device_add(dev); + if (rc) + goto err; + + rc = devm_add_action_or_reset(host, unregister_nvb, cxl_nvb); + if (rc) + return ERR_PTR(rc); + + return cxl_nvb; + +err: + put_device(dev); + return ERR_PTR(rc); +} +EXPORT_SYMBOL_GPL(devm_cxl_add_nvdimm_bridge); + /** * cxl_probe_device_regs() - Detect CXL Device register blocks * @dev: Host device of the @base mapping @@ -808,6 +927,8 @@ EXPORT_SYMBOL_GPL(cxl_driver_unregister); static int cxl_device_id(struct device *dev) { + if (dev->type == &cxl_nvdimm_bridge_type) + return CXL_DEVICE_NVDIMM_BRIDGE; return 0; } diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h index a6f8e7097ea2..5da2163ca45f 100644 --- a/drivers/cxl/cxl.h +++ b/drivers/cxl/cxl.h @@ -4,6 +4,7 @@ #ifndef __CXL_H__ #define __CXL_H__ +#include #include #include #include @@ -202,6 +203,23 @@ struct cxl_decoder { struct cxl_dport *target[]; }; + +enum cxl_nvdimm_brige_state { + CXL_NVB_NEW, + CXL_NVB_DEAD, + CXL_NVB_ONLINE, + CXL_NVB_OFFLINE, +}; + +struct cxl_nvdimm_bridge { + struct device dev; + struct cxl_port *port; + struct nvdimm_bus *nvdimm_bus; + struct nvdimm_bus_descriptor nd_desc; + struct work_struct state_work; + enum cxl_nvdimm_brige_state state; +}; + /** * struct cxl_port - logical collection of upstream port devices and * downstream port devices to construct a CXL memory @@ -247,6 +265,7 @@ int cxl_add_dport(struct cxl_port *port, struct device *dport, int port_id, resource_size_t component_reg_phys); struct cxl_decoder *to_cxl_decoder(struct device *dev); +bool is_root_decoder(struct device *dev); struct cxl_decoder * devm_cxl_add_decoder(struct device *host, struct cxl_port *port, int nr_targets, resource_size_t base, resource_size_t len, @@ -287,7 +306,12 @@ int __cxl_driver_register(struct cxl_driver *cxl_drv, struct module *owner, #define cxl_driver_register(x) __cxl_driver_register(x, THIS_MODULE, KBUILD_MODNAME) void cxl_driver_unregister(struct cxl_driver *cxl_drv); +#define CXL_DEVICE_NVDIMM_BRIDGE 1 + #define MODULE_ALIAS_CXL(type) MODULE_ALIAS("cxl:t" __stringify(type) "*") #define CXL_MODALIAS_FMT "cxl:t%d" +struct cxl_nvdimm_bridge *to_cxl_nvdimm_bridge(struct device *dev); +struct cxl_nvdimm_bridge *devm_cxl_add_nvdimm_bridge(struct device *host, + struct cxl_port *port); #endif /* __CXL_H__ */ diff --git a/drivers/cxl/pmem.c b/drivers/cxl/pmem.c new file mode 100644 index 000000000000..8365839856dc --- /dev/null +++ b/drivers/cxl/pmem.c @@ -0,0 +1,140 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright(c) 2021 Intel Corporation. All rights reserved. */ +#include +#include +#include +#include +#include "cxl.h" + +/* + * Ordered workqueue for cxl nvdimm device arrival and departure + * to coordinate bus rescans when a bridge arrives and trigger remove + * operations when the bridge is removed. + */ +static struct workqueue_struct *cxl_pmem_wq; + +static int cxl_pmem_ctl(struct nvdimm_bus_descriptor *nd_desc, + struct nvdimm *nvdimm, unsigned int cmd, void *buf, + unsigned int buf_len, int *cmd_rc) +{ + return -ENOTTY; +} + +static bool online_nvdimm_bus(struct cxl_nvdimm_bridge *cxl_nvb) +{ + if (cxl_nvb->nvdimm_bus) + return true; + cxl_nvb->nvdimm_bus = + nvdimm_bus_register(&cxl_nvb->dev, &cxl_nvb->nd_desc); + return cxl_nvb->nvdimm_bus != NULL; +} + +static void offline_nvdimm_bus(struct cxl_nvdimm_bridge *cxl_nvb) +{ + if (!cxl_nvb->nvdimm_bus) + return; + nvdimm_bus_unregister(cxl_nvb->nvdimm_bus); + cxl_nvb->nvdimm_bus = NULL; +} + +static void cxl_nvb_update_state(struct work_struct *work) +{ + struct cxl_nvdimm_bridge *cxl_nvb = + container_of(work, typeof(*cxl_nvb), state_work); + bool release = false; + + device_lock(&cxl_nvb->dev); + switch (cxl_nvb->state) { + case CXL_NVB_ONLINE: + if (!online_nvdimm_bus(cxl_nvb)) { + dev_err(&cxl_nvb->dev, + "failed to establish nvdimm bus\n"); + release = true; + } + break; + case CXL_NVB_OFFLINE: + case CXL_NVB_DEAD: + offline_nvdimm_bus(cxl_nvb); + break; + default: + break; + } + device_unlock(&cxl_nvb->dev); + + if (release) + device_release_driver(&cxl_nvb->dev); + + put_device(&cxl_nvb->dev); +} + +static void cxl_nvdimm_bridge_remove(struct device *dev) +{ + struct cxl_nvdimm_bridge *cxl_nvb = to_cxl_nvdimm_bridge(dev); + + if (cxl_nvb->state == CXL_NVB_ONLINE) + cxl_nvb->state = CXL_NVB_OFFLINE; + if (queue_work(cxl_pmem_wq, &cxl_nvb->state_work)) + get_device(&cxl_nvb->dev); +} + +static int cxl_nvdimm_bridge_probe(struct device *dev) +{ + struct cxl_nvdimm_bridge *cxl_nvb = to_cxl_nvdimm_bridge(dev); + + if (cxl_nvb->state == CXL_NVB_DEAD) + return -ENXIO; + + if (cxl_nvb->state == CXL_NVB_NEW) { + cxl_nvb->nd_desc = (struct nvdimm_bus_descriptor) { + .provider_name = "CXL", + .module = THIS_MODULE, + .ndctl = cxl_pmem_ctl, + }; + + INIT_WORK(&cxl_nvb->state_work, cxl_nvb_update_state); + } + + cxl_nvb->state = CXL_NVB_ONLINE; + if (queue_work(cxl_pmem_wq, &cxl_nvb->state_work)) + get_device(&cxl_nvb->dev); + + return 0; +} + +static struct cxl_driver cxl_nvdimm_bridge_driver = { + .name = "cxl_nvdimm_bridge", + .probe = cxl_nvdimm_bridge_probe, + .remove = cxl_nvdimm_bridge_remove, + .id = CXL_DEVICE_NVDIMM_BRIDGE, +}; + +static __init int cxl_pmem_init(void) +{ + int rc; + + cxl_pmem_wq = alloc_ordered_workqueue("cxl_pmem", 0); + if (!cxl_pmem_wq) + return -ENXIO; + + rc = cxl_driver_register(&cxl_nvdimm_bridge_driver); + if (rc) + goto err; + + return 0; + +err: + destroy_workqueue(cxl_pmem_wq); + return rc; +} + +static __exit void cxl_pmem_exit(void) +{ + cxl_driver_unregister(&cxl_nvdimm_bridge_driver); + destroy_workqueue(cxl_pmem_wq); +} + +MODULE_LICENSE("GPL v2"); +module_init(cxl_pmem_init); +module_exit(cxl_pmem_exit); +MODULE_IMPORT_NS(CXL); +MODULE_ALIAS_CXL(CXL_DEVICE_NVDIMM_BRIDGE); From fd14602d05229671be81018fa226f9afdafdba88 Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Tue, 15 Jun 2021 16:18:22 -0700 Subject: [PATCH 29/34] libnvdimm: Export nvdimm shutdown helper, nvdimm_delete() CXL is a hotplug bus and arranges for nvdimm devices to be dynamically discovered and removed. The libnvdimm core manages shutdown of nvdimm security operations when the device is unregistered. That functionality is moved to nvdimm_delete() and invoked by the CXL-to-nvdimm glue code. Reviewed-by: Jonathan Cameron Link: https://lore.kernel.org/r/162379910271.2993820.2955889139842401250.stgit@dwillia2-desk3.amr.corp.intel.com Signed-off-by: Dan Williams --- drivers/nvdimm/bus.c | 19 ++++--------------- drivers/nvdimm/dimm_devs.c | 18 ++++++++++++++++++ include/linux/libnvdimm.h | 1 + 3 files changed, 23 insertions(+), 15 deletions(-) diff --git a/drivers/nvdimm/bus.c b/drivers/nvdimm/bus.c index 3a777d0073b7..a11821df83b5 100644 --- a/drivers/nvdimm/bus.c +++ b/drivers/nvdimm/bus.c @@ -396,21 +396,10 @@ static int child_unregister(struct device *dev, void *data) if (dev->class) return 0; - if (is_nvdimm(dev)) { - struct nvdimm *nvdimm = to_nvdimm(dev); - bool dev_put = false; - - /* We are shutting down. Make state frozen artificially. */ - nvdimm_bus_lock(dev); - set_bit(NVDIMM_SECURITY_FROZEN, &nvdimm->sec.flags); - if (test_and_clear_bit(NDD_WORK_PENDING, &nvdimm->flags)) - dev_put = true; - nvdimm_bus_unlock(dev); - cancel_delayed_work_sync(&nvdimm->dwork); - if (dev_put) - put_device(dev); - } - nd_device_unregister(dev, ND_SYNC); + if (is_nvdimm(dev)) + nvdimm_delete(to_nvdimm(dev)); + else + nd_device_unregister(dev, ND_SYNC); return 0; } diff --git a/drivers/nvdimm/dimm_devs.c b/drivers/nvdimm/dimm_devs.c index 9d208570d059..dc7449a40003 100644 --- a/drivers/nvdimm/dimm_devs.c +++ b/drivers/nvdimm/dimm_devs.c @@ -642,6 +642,24 @@ struct nvdimm *__nvdimm_create(struct nvdimm_bus *nvdimm_bus, } EXPORT_SYMBOL_GPL(__nvdimm_create); +void nvdimm_delete(struct nvdimm *nvdimm) +{ + struct device *dev = &nvdimm->dev; + bool dev_put = false; + + /* We are shutting down. Make state frozen artificially. */ + nvdimm_bus_lock(dev); + set_bit(NVDIMM_SECURITY_FROZEN, &nvdimm->sec.flags); + if (test_and_clear_bit(NDD_WORK_PENDING, &nvdimm->flags)) + dev_put = true; + nvdimm_bus_unlock(dev); + cancel_delayed_work_sync(&nvdimm->dwork); + if (dev_put) + put_device(dev); + nd_device_unregister(dev, ND_SYNC); +} +EXPORT_SYMBOL_GPL(nvdimm_delete); + static void shutdown_security_notify(void *data) { struct nvdimm *nvdimm = data; diff --git a/include/linux/libnvdimm.h b/include/linux/libnvdimm.h index 89b69e645ac7..7074aa9af525 100644 --- a/include/linux/libnvdimm.h +++ b/include/linux/libnvdimm.h @@ -278,6 +278,7 @@ static inline struct nvdimm *nvdimm_create(struct nvdimm_bus *nvdimm_bus, return __nvdimm_create(nvdimm_bus, provider_data, groups, flags, cmd_mask, num_flush, flush_wpq, NULL, NULL, NULL); } +void nvdimm_delete(struct nvdimm *nvdimm); const struct nd_cmd_desc *nd_cmd_dimm_desc(int cmd); const struct nd_cmd_desc *nd_cmd_bus_desc(int cmd); From 2bbafda405c04cfed1b57b761d13ada3154c0f89 Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Tue, 15 Jun 2021 16:18:28 -0700 Subject: [PATCH 30/34] libnvdimm: Drop unused device power management support LIBNVDIMM device objects register sysfs power attributes despite nothing requiring that support. Clean up sysfs remove the power/ attribute group. This requires a device_create() and a device_register() usage to be converted to the device_initialize() + device_add() pattern. Reviewed-by: Jonathan Cameron Link: https://lore.kernel.org/r/162379910795.2993820.10130417680551632288.stgit@dwillia2-desk3.amr.corp.intel.com Signed-off-by: Dan Williams --- drivers/nvdimm/bus.c | 45 ++++++++++++++++++++++++++++++++++++-------- 1 file changed, 37 insertions(+), 8 deletions(-) diff --git a/drivers/nvdimm/bus.c b/drivers/nvdimm/bus.c index a11821df83b5..e6aa87043a95 100644 --- a/drivers/nvdimm/bus.c +++ b/drivers/nvdimm/bus.c @@ -363,8 +363,13 @@ struct nvdimm_bus *nvdimm_bus_register(struct device *parent, nvdimm_bus->dev.groups = nd_desc->attr_groups; nvdimm_bus->dev.bus = &nvdimm_bus_type; nvdimm_bus->dev.of_node = nd_desc->of_node; - dev_set_name(&nvdimm_bus->dev, "ndbus%d", nvdimm_bus->id); - rc = device_register(&nvdimm_bus->dev); + device_initialize(&nvdimm_bus->dev); + device_set_pm_not_required(&nvdimm_bus->dev); + rc = dev_set_name(&nvdimm_bus->dev, "ndbus%d", nvdimm_bus->id); + if (rc) + goto err; + + rc = device_add(&nvdimm_bus->dev); if (rc) { dev_dbg(&nvdimm_bus->dev, "registration failed: %d\n", rc); goto err; @@ -525,6 +530,7 @@ void __nd_device_register(struct device *dev) set_dev_node(dev, to_nd_region(dev)->numa_node); dev->bus = &nvdimm_bus_type; + device_set_pm_not_required(dev); if (dev->parent) { get_device(dev->parent); if (dev_to_node(dev) == NUMA_NO_NODE) @@ -717,18 +723,41 @@ const struct attribute_group nd_numa_attribute_group = { .is_visible = nd_numa_attr_visible, }; +static void ndctl_release(struct device *dev) +{ + kfree(dev); +} + int nvdimm_bus_create_ndctl(struct nvdimm_bus *nvdimm_bus) { dev_t devt = MKDEV(nvdimm_bus_major, nvdimm_bus->id); struct device *dev; + int rc; - dev = device_create(nd_class, &nvdimm_bus->dev, devt, nvdimm_bus, - "ndctl%d", nvdimm_bus->id); + dev = kzalloc(sizeof(*dev), GFP_KERNEL); + if (!dev) + return -ENOMEM; + device_initialize(dev); + device_set_pm_not_required(dev); + dev->class = nd_class; + dev->parent = &nvdimm_bus->dev; + dev->devt = devt; + dev->release = ndctl_release; + rc = dev_set_name(dev, "ndctl%d", nvdimm_bus->id); + if (rc) + goto err; - if (IS_ERR(dev)) - dev_dbg(&nvdimm_bus->dev, "failed to register ndctl%d: %ld\n", - nvdimm_bus->id, PTR_ERR(dev)); - return PTR_ERR_OR_ZERO(dev); + rc = device_add(dev); + if (rc) { + dev_dbg(&nvdimm_bus->dev, "failed to register ndctl%d: %d\n", + nvdimm_bus->id, rc); + goto err; + } + return 0; + +err: + put_device(dev); + return rc; } void nvdimm_bus_destroy_ndctl(struct nvdimm_bus *nvdimm_bus) From 21083f51521fb0f60dbac591f175c3ed48435af4 Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Tue, 15 Jun 2021 16:36:31 -0700 Subject: [PATCH 31/34] cxl/pmem: Register 'pmem' / cxl_nvdimm devices While a memX device on /sys/bus/cxl represents a CXL memory expander control interface, a pmemX device represents the persistent memory sub-functionality. It bridges the CXL subystem to the libnvdimm nmemX control interface. With this skeleton ndctl can now see persistent memory devices on a "CXL" bus. Later patches add support for translating libnvdimm native commands to CXL commands. # ndctl list -BDiu -b CXL { "provider":"CXL", "dev":"ndbus1", "dimms":[ { "dev":"nmem1", "state":"disabled" }, { "dev":"nmem0", "state":"disabled" } ] } Given nvdimm_bus_unregister() removes all devices on an ndbus0 the cxl_pmem infrastructure needs to arrange ->remove() to be triggered on cxl_nvdimm devices to keep their enabled state synchronized with the registration state of their corresponding device on the nvdimm_bus. In other words, always arrange for cxl_nvdimm_driver.remove() to unregister nvdimms from an nvdimm_bus ahead of the bus being unregistered. Reviewed-by: Jonathan Cameron Link: https://lore.kernel.org/r/162380012696.3039556.4293801691038740850.stgit@dwillia2-desk3.amr.corp.intel.com Signed-off-by: Dan Williams --- drivers/cxl/core.c | 86 ++++++++++++++++++++++++++++++++++++ drivers/cxl/cxl.h | 12 ++++- drivers/cxl/mem.h | 2 + drivers/cxl/pci.c | 23 +++++++--- drivers/cxl/pmem.c | 108 +++++++++++++++++++++++++++++++++++++++++---- 5 files changed, 215 insertions(+), 16 deletions(-) diff --git a/drivers/cxl/core.c b/drivers/cxl/core.c index aaffb390d168..a2e4d54fc7bc 100644 --- a/drivers/cxl/core.c +++ b/drivers/cxl/core.c @@ -7,6 +7,7 @@ #include #include #include "cxl.h" +#include "mem.h" /** * DOC: cxl core @@ -730,6 +731,89 @@ err: } EXPORT_SYMBOL_GPL(devm_cxl_add_nvdimm_bridge); +static void cxl_nvdimm_release(struct device *dev) +{ + struct cxl_nvdimm *cxl_nvd = to_cxl_nvdimm(dev); + + kfree(cxl_nvd); +} + +static const struct attribute_group *cxl_nvdimm_attribute_groups[] = { + &cxl_base_attribute_group, + NULL, +}; + +static const struct device_type cxl_nvdimm_type = { + .name = "cxl_nvdimm", + .release = cxl_nvdimm_release, + .groups = cxl_nvdimm_attribute_groups, +}; + +bool is_cxl_nvdimm(struct device *dev) +{ + return dev->type == &cxl_nvdimm_type; +} +EXPORT_SYMBOL_GPL(is_cxl_nvdimm); + +struct cxl_nvdimm *to_cxl_nvdimm(struct device *dev) +{ + if (dev_WARN_ONCE(dev, !is_cxl_nvdimm(dev), + "not a cxl_nvdimm device\n")) + return NULL; + return container_of(dev, struct cxl_nvdimm, dev); +} +EXPORT_SYMBOL_GPL(to_cxl_nvdimm); + +static struct cxl_nvdimm *cxl_nvdimm_alloc(struct cxl_memdev *cxlmd) +{ + struct cxl_nvdimm *cxl_nvd; + struct device *dev; + + cxl_nvd = kzalloc(sizeof(*cxl_nvd), GFP_KERNEL); + if (!cxl_nvd) + return ERR_PTR(-ENOMEM); + + dev = &cxl_nvd->dev; + cxl_nvd->cxlmd = cxlmd; + device_initialize(dev); + device_set_pm_not_required(dev); + dev->parent = &cxlmd->dev; + dev->bus = &cxl_bus_type; + dev->type = &cxl_nvdimm_type; + + return cxl_nvd; +} + +int devm_cxl_add_nvdimm(struct device *host, struct cxl_memdev *cxlmd) +{ + struct cxl_nvdimm *cxl_nvd; + struct device *dev; + int rc; + + cxl_nvd = cxl_nvdimm_alloc(cxlmd); + if (IS_ERR(cxl_nvd)) + return PTR_ERR(cxl_nvd); + + dev = &cxl_nvd->dev; + rc = dev_set_name(dev, "pmem%d", cxlmd->id); + if (rc) + goto err; + + rc = device_add(dev); + if (rc) + goto err; + + dev_dbg(host, "%s: register %s\n", dev_name(dev->parent), + dev_name(dev)); + + return devm_add_action_or_reset(host, unregister_dev, dev); + +err: + put_device(dev); + return rc; +} +EXPORT_SYMBOL_GPL(devm_cxl_add_nvdimm); + /** * cxl_probe_device_regs() - Detect CXL Device register blocks * @dev: Host device of the @base mapping @@ -929,6 +1013,8 @@ static int cxl_device_id(struct device *dev) { if (dev->type == &cxl_nvdimm_bridge_type) return CXL_DEVICE_NVDIMM_BRIDGE; + if (dev->type == &cxl_nvdimm_type) + return CXL_DEVICE_NVDIMM; return 0; } diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h index 5da2163ca45f..b6bda39a59e3 100644 --- a/drivers/cxl/cxl.h +++ b/drivers/cxl/cxl.h @@ -220,6 +220,12 @@ struct cxl_nvdimm_bridge { enum cxl_nvdimm_brige_state state; }; +struct cxl_nvdimm { + struct device dev; + struct cxl_memdev *cxlmd; + struct nvdimm *nvdimm; +}; + /** * struct cxl_port - logical collection of upstream port devices and * downstream port devices to construct a CXL memory @@ -306,7 +312,8 @@ int __cxl_driver_register(struct cxl_driver *cxl_drv, struct module *owner, #define cxl_driver_register(x) __cxl_driver_register(x, THIS_MODULE, KBUILD_MODNAME) void cxl_driver_unregister(struct cxl_driver *cxl_drv); -#define CXL_DEVICE_NVDIMM_BRIDGE 1 +#define CXL_DEVICE_NVDIMM_BRIDGE 1 +#define CXL_DEVICE_NVDIMM 2 #define MODULE_ALIAS_CXL(type) MODULE_ALIAS("cxl:t" __stringify(type) "*") #define CXL_MODALIAS_FMT "cxl:t%d" @@ -314,4 +321,7 @@ void cxl_driver_unregister(struct cxl_driver *cxl_drv); struct cxl_nvdimm_bridge *to_cxl_nvdimm_bridge(struct device *dev); struct cxl_nvdimm_bridge *devm_cxl_add_nvdimm_bridge(struct device *host, struct cxl_port *port); +struct cxl_nvdimm *to_cxl_nvdimm(struct device *dev); +bool is_cxl_nvdimm(struct device *dev); +int devm_cxl_add_nvdimm(struct device *host, struct cxl_memdev *cxlmd); #endif /* __CXL_H__ */ diff --git a/drivers/cxl/mem.h b/drivers/cxl/mem.h index 13868ff7cadf..8f02d02b26b4 100644 --- a/drivers/cxl/mem.h +++ b/drivers/cxl/mem.h @@ -2,6 +2,8 @@ /* Copyright(c) 2020-2021 Intel Corporation. */ #ifndef __CXL_MEM_H__ #define __CXL_MEM_H__ +#include +#include "cxl.h" /* CXL 2.0 8.2.8.5.1.1 Memory Device Status Register */ #define CXLMDEV_STATUS_OFFSET 0x0 diff --git a/drivers/cxl/pci.c b/drivers/cxl/pci.c index a8f29ce35c2a..f8408e5f0754 100644 --- a/drivers/cxl/pci.c +++ b/drivers/cxl/pci.c @@ -1332,7 +1332,8 @@ err: return ERR_PTR(rc); } -static int cxl_mem_add_memdev(struct cxl_mem *cxlm) +static struct cxl_memdev *devm_cxl_add_memdev(struct device *host, + struct cxl_mem *cxlm) { struct cxl_memdev *cxlmd; struct device *dev; @@ -1341,7 +1342,7 @@ static int cxl_mem_add_memdev(struct cxl_mem *cxlm) cxlmd = cxl_memdev_alloc(cxlm); if (IS_ERR(cxlmd)) - return PTR_ERR(cxlmd); + return cxlmd; dev = &cxlmd->dev; rc = dev_set_name(dev, "mem%d", cxlmd->id); @@ -1359,8 +1360,10 @@ static int cxl_mem_add_memdev(struct cxl_mem *cxlm) if (rc) goto err; - return devm_add_action_or_reset(dev->parent, cxl_memdev_unregister, - cxlmd); + rc = devm_add_action_or_reset(host, cxl_memdev_unregister, cxlmd); + if (rc) + return ERR_PTR(rc); + return cxlmd; err: /* @@ -1369,7 +1372,7 @@ err: */ cxl_memdev_shutdown(cxlmd); put_device(dev); - return rc; + return ERR_PTR(rc); } static int cxl_xfer_log(struct cxl_mem *cxlm, uuid_t *uuid, u32 size, u8 *out) @@ -1580,6 +1583,7 @@ static int cxl_mem_identify(struct cxl_mem *cxlm) static int cxl_mem_probe(struct pci_dev *pdev, const struct pci_device_id *id) { + struct cxl_memdev *cxlmd; struct cxl_mem *cxlm; int rc; @@ -1607,7 +1611,14 @@ static int cxl_mem_probe(struct pci_dev *pdev, const struct pci_device_id *id) if (rc) return rc; - return cxl_mem_add_memdev(cxlm); + cxlmd = devm_cxl_add_memdev(&pdev->dev, cxlm); + if (IS_ERR(cxlmd)) + return PTR_ERR(cxlmd); + + if (range_len(&cxlm->pmem_range) && IS_ENABLED(CONFIG_CXL_PMEM)) + rc = devm_cxl_add_nvdimm(&pdev->dev, cxlmd); + + return rc; } static const struct pci_device_id cxl_mem_pci_tbl[] = { diff --git a/drivers/cxl/pmem.c b/drivers/cxl/pmem.c index 8365839856dc..0088e41dd2f3 100644 --- a/drivers/cxl/pmem.c +++ b/drivers/cxl/pmem.c @@ -3,7 +3,10 @@ #include #include #include +#include +#include #include +#include "mem.h" #include "cxl.h" /* @@ -13,6 +16,62 @@ */ static struct workqueue_struct *cxl_pmem_wq; +static void unregister_nvdimm(void *nvdimm) +{ + nvdimm_delete(nvdimm); +} + +static int match_nvdimm_bridge(struct device *dev, const void *data) +{ + return strcmp(dev_name(dev), "nvdimm-bridge") == 0; +} + +static struct cxl_nvdimm_bridge *cxl_find_nvdimm_bridge(void) +{ + struct device *dev; + + dev = bus_find_device(&cxl_bus_type, NULL, NULL, match_nvdimm_bridge); + if (!dev) + return NULL; + return to_cxl_nvdimm_bridge(dev); +} + +static int cxl_nvdimm_probe(struct device *dev) +{ + struct cxl_nvdimm *cxl_nvd = to_cxl_nvdimm(dev); + struct cxl_nvdimm_bridge *cxl_nvb; + unsigned long flags = 0; + struct nvdimm *nvdimm; + int rc = -ENXIO; + + cxl_nvb = cxl_find_nvdimm_bridge(); + if (!cxl_nvb) + return -ENXIO; + + device_lock(&cxl_nvb->dev); + if (!cxl_nvb->nvdimm_bus) + goto out; + + set_bit(NDD_LABELING, &flags); + nvdimm = nvdimm_create(cxl_nvb->nvdimm_bus, cxl_nvd, NULL, flags, 0, 0, + NULL); + if (!nvdimm) + goto out; + + rc = devm_add_action_or_reset(dev, unregister_nvdimm, nvdimm); +out: + device_unlock(&cxl_nvb->dev); + put_device(&cxl_nvb->dev); + + return rc; +} + +static struct cxl_driver cxl_nvdimm_driver = { + .name = "cxl_nvdimm", + .probe = cxl_nvdimm_probe, + .id = CXL_DEVICE_NVDIMM, +}; + static int cxl_pmem_ctl(struct nvdimm_bus_descriptor *nd_desc, struct nvdimm *nvdimm, unsigned int cmd, void *buf, unsigned int buf_len, int *cmd_rc) @@ -29,19 +88,34 @@ static bool online_nvdimm_bus(struct cxl_nvdimm_bridge *cxl_nvb) return cxl_nvb->nvdimm_bus != NULL; } -static void offline_nvdimm_bus(struct cxl_nvdimm_bridge *cxl_nvb) +static int cxl_nvdimm_release_driver(struct device *dev, void *data) { - if (!cxl_nvb->nvdimm_bus) + if (!is_cxl_nvdimm(dev)) + return 0; + device_release_driver(dev); + return 0; +} + +static void offline_nvdimm_bus(struct nvdimm_bus *nvdimm_bus) +{ + if (!nvdimm_bus) return; - nvdimm_bus_unregister(cxl_nvb->nvdimm_bus); - cxl_nvb->nvdimm_bus = NULL; + + /* + * Set the state of cxl_nvdimm devices to unbound / idle before + * nvdimm_bus_unregister() rips the nvdimm objects out from + * underneath them. + */ + bus_for_each_dev(&cxl_bus_type, NULL, NULL, cxl_nvdimm_release_driver); + nvdimm_bus_unregister(nvdimm_bus); } static void cxl_nvb_update_state(struct work_struct *work) { struct cxl_nvdimm_bridge *cxl_nvb = container_of(work, typeof(*cxl_nvb), state_work); - bool release = false; + struct nvdimm_bus *victim_bus = NULL; + bool release = false, rescan = false; device_lock(&cxl_nvb->dev); switch (cxl_nvb->state) { @@ -50,11 +124,13 @@ static void cxl_nvb_update_state(struct work_struct *work) dev_err(&cxl_nvb->dev, "failed to establish nvdimm bus\n"); release = true; - } + } else + rescan = true; break; case CXL_NVB_OFFLINE: case CXL_NVB_DEAD: - offline_nvdimm_bus(cxl_nvb); + victim_bus = cxl_nvb->nvdimm_bus; + cxl_nvb->nvdimm_bus = NULL; break; default: break; @@ -63,6 +139,12 @@ static void cxl_nvb_update_state(struct work_struct *work) if (release) device_release_driver(&cxl_nvb->dev); + if (rescan) { + int rc = bus_rescan_devices(&cxl_bus_type); + + dev_dbg(&cxl_nvb->dev, "rescan: %d\n", rc); + } + offline_nvdimm_bus(victim_bus); put_device(&cxl_nvb->dev); } @@ -118,17 +200,24 @@ static __init int cxl_pmem_init(void) rc = cxl_driver_register(&cxl_nvdimm_bridge_driver); if (rc) - goto err; + goto err_bridge; + + rc = cxl_driver_register(&cxl_nvdimm_driver); + if (rc) + goto err_nvdimm; return 0; -err: +err_nvdimm: + cxl_driver_unregister(&cxl_nvdimm_bridge_driver); +err_bridge: destroy_workqueue(cxl_pmem_wq); return rc; } static __exit void cxl_pmem_exit(void) { + cxl_driver_unregister(&cxl_nvdimm_driver); cxl_driver_unregister(&cxl_nvdimm_bridge_driver); destroy_workqueue(cxl_pmem_wq); } @@ -138,3 +227,4 @@ module_init(cxl_pmem_init); module_exit(cxl_pmem_exit); MODULE_IMPORT_NS(CXL); MODULE_ALIAS_CXL(CXL_DEVICE_NVDIMM_BRIDGE); +MODULE_ALIAS_CXL(CXL_DEVICE_NVDIMM); From da6aafec3dca6132dd80a74a4d918ffd86c7ae35 Mon Sep 17 00:00:00 2001 From: Alison Schofield Date: Thu, 17 Jun 2021 16:12:15 -0700 Subject: [PATCH 32/34] cxl/acpi: Add the Host Bridge base address to CXL port objects The base address for the Host Bridge port component registers is located in the CXL Host Bridge Structure (CHBS) of the ACPI CXL Early Discovery Table (CEDT). Retrieve the CHBS for each Host Bridge (ACPI0016 device) and include that base address in the port object. Co-developed-by: Vishal Verma Signed-off-by: Vishal Verma Signed-off-by: Alison Schofield Reviewed-by: Jonathan Cameron Reviewed-by: Ben Widawsky Link: https://lore.kernel.org/r/a475ce137b899bc7ae5ba9550b5f198cb29ccbfd.1623968958.git.alison.schofield@intel.com Signed-off-by: Dan Williams --- drivers/cxl/acpi.c | 100 ++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 95 insertions(+), 5 deletions(-) diff --git a/drivers/cxl/acpi.c b/drivers/cxl/acpi.c index 8a723f7f3f73..3afdee8f334a 100644 --- a/drivers/cxl/acpi.c +++ b/drivers/cxl/acpi.c @@ -8,6 +8,60 @@ #include #include "cxl.h" +static struct acpi_table_header *acpi_cedt; + +static struct acpi_cedt_chbs *cxl_acpi_match_chbs(struct device *dev, u32 uid) +{ + struct acpi_cedt_chbs *chbs, *chbs_match = NULL; + acpi_size len, cur = 0; + void *cedt_subtable; + + len = acpi_cedt->length - sizeof(*acpi_cedt); + cedt_subtable = acpi_cedt + 1; + + while (cur < len) { + struct acpi_cedt_header *c = cedt_subtable + cur; + + if (c->type != ACPI_CEDT_TYPE_CHBS) { + cur += c->length; + continue; + } + + chbs = cedt_subtable + cur; + + if (chbs->header.length < sizeof(*chbs)) { + dev_warn_once(dev, + "CHBS entry skipped: invalid length:%u\n", + chbs->header.length); + cur += c->length; + continue; + } + + if (chbs->uid != uid) { + cur += c->length; + continue; + } + + if (chbs_match) { + dev_warn_once(dev, + "CHBS entry skipped: duplicate UID:%u\n", + uid); + cur += c->length; + continue; + } + + chbs_match = chbs; + cur += c->length; + } + + return chbs_match ? chbs_match : ERR_PTR(-ENODEV); +} + +static resource_size_t get_chbcr(struct acpi_cedt_chbs *chbs) +{ + return IS_ERR(chbs) ? CXL_RESOURCE_NONE : chbs->base; +} + struct cxl_walk_context { struct device *dev; struct pci_bus *root; @@ -50,6 +104,21 @@ static int match_add_root_ports(struct pci_dev *pdev, void *data) return 0; } +static struct cxl_dport *find_dport_by_dev(struct cxl_port *port, struct device *dev) +{ + struct cxl_dport *dport; + + device_lock(&port->dev); + list_for_each_entry(dport, &port->dports, list) + if (dport->dport == dev) { + device_unlock(&port->dev); + return dport; + } + + device_unlock(&port->dev); + return NULL; +} + static struct acpi_device *to_cxl_host_bridge(struct device *dev) { struct acpi_device *adev = to_acpi_device(dev); @@ -71,6 +140,7 @@ static int add_host_bridge_uport(struct device *match, void *arg) struct acpi_pci_root *pci_root; struct cxl_walk_context ctx; struct cxl_decoder *cxld; + struct cxl_dport *dport; struct cxl_port *port; if (!bridge) @@ -80,8 +150,14 @@ static int add_host_bridge_uport(struct device *match, void *arg) if (!pci_root) return -ENXIO; - /* TODO: fold in CEDT.CHBS retrieval */ - port = devm_cxl_add_port(host, match, CXL_RESOURCE_NONE, root_port); + dport = find_dport_by_dev(root_port, match); + if (!dport) { + dev_dbg(host, "host bridge expected and not found\n"); + return -ENODEV; + } + + port = devm_cxl_add_port(host, match, dport->component_reg_phys, + root_port); if (IS_ERR(port)) return PTR_ERR(port); dev_dbg(host, "%s: add: %s\n", dev_name(match), dev_name(&port->dev)); @@ -120,6 +196,7 @@ static int add_host_bridge_dport(struct device *match, void *arg) int rc; acpi_status status; unsigned long long uid; + struct acpi_cedt_chbs *chbs; struct cxl_port *root_port = arg; struct device *host = root_port->dev.parent; struct acpi_device *bridge = to_cxl_host_bridge(match); @@ -135,7 +212,12 @@ static int add_host_bridge_dport(struct device *match, void *arg) return -ENODEV; } - rc = cxl_add_dport(root_port, match, uid, CXL_RESOURCE_NONE); + chbs = cxl_acpi_match_chbs(host, uid); + if (IS_ERR(chbs)) + dev_dbg(host, "No CHBS found for Host Bridge: %s\n", + dev_name(match)); + + rc = cxl_add_dport(root_port, match, uid, get_chbcr(chbs)); if (rc) { dev_err(host, "failed to add downstream port: %s\n", dev_name(match)); @@ -172,6 +254,7 @@ static int add_root_nvdimm_bridge(struct device *match, void *data) static int cxl_acpi_probe(struct platform_device *pdev) { int rc; + acpi_status status; struct cxl_port *root_port; struct device *host = &pdev->dev; struct acpi_device *adev = ACPI_COMPANION(host); @@ -181,10 +264,14 @@ static int cxl_acpi_probe(struct platform_device *pdev) return PTR_ERR(root_port); dev_dbg(host, "add: %s\n", dev_name(&root_port->dev)); + status = acpi_get_table(ACPI_SIG_CEDT, 0, &acpi_cedt); + if (ACPI_FAILURE(status)) + return -ENXIO; + rc = bus_for_each_dev(adev->dev.bus, NULL, root_port, add_host_bridge_dport); if (rc) - return rc; + goto out; /* * Root level scanned with host-bridge as dports, now scan host-bridges @@ -193,11 +280,14 @@ static int cxl_acpi_probe(struct platform_device *pdev) rc = bus_for_each_dev(adev->dev.bus, NULL, root_port, add_host_bridge_uport); if (rc) - return rc; + goto out; if (IS_ENABLED(CONFIG_CXL_PMEM)) rc = device_for_each_child(&root_port->dev, root_port, add_root_nvdimm_bridge); + +out: + acpi_put_table(acpi_cedt); if (rc < 0) return rc; return 0; From 3e23d17ce1980c7cbd9426a3764eef7d7bcd443f Mon Sep 17 00:00:00 2001 From: Alison Schofield Date: Thu, 17 Jun 2021 16:12:16 -0700 Subject: [PATCH 33/34] cxl/acpi: Use the ACPI CFMWS to create static decoder objects The ACPI CXL Early Discovery Table (CEDT) includes a list of CXL memory resources in CXL Fixed Memory Window Structures (CFMWS). Retrieve each CFMWS in the CEDT and add a cxl_decoder object to the root port (root0) for each memory resource. Signed-off-by: Alison Schofield Reviewed-by: Jonathan Cameron Reviewed-by: Ben Widawsky Link: https://lore.kernel.org/r/d2b73eecfb7ea22e1103f1894b271a89958b4c41.1623968958.git.alison.schofield@intel.com Signed-off-by: Dan Williams --- drivers/cxl/acpi.c | 122 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 122 insertions(+) diff --git a/drivers/cxl/acpi.c b/drivers/cxl/acpi.c index 3afdee8f334a..8ae89273f58e 100644 --- a/drivers/cxl/acpi.c +++ b/drivers/cxl/acpi.c @@ -10,6 +10,126 @@ static struct acpi_table_header *acpi_cedt; +/* Encode defined in CXL 2.0 8.2.5.12.7 HDM Decoder Control Register */ +#define CFMWS_INTERLEAVE_WAYS(x) (1 << (x)->interleave_ways) +#define CFMWS_INTERLEAVE_GRANULARITY(x) ((x)->granularity + 8) + +static unsigned long cfmws_to_decoder_flags(int restrictions) +{ + unsigned long flags = 0; + + if (restrictions & ACPI_CEDT_CFMWS_RESTRICT_TYPE2) + flags |= CXL_DECODER_F_TYPE2; + if (restrictions & ACPI_CEDT_CFMWS_RESTRICT_TYPE3) + flags |= CXL_DECODER_F_TYPE3; + if (restrictions & ACPI_CEDT_CFMWS_RESTRICT_VOLATILE) + flags |= CXL_DECODER_F_RAM; + if (restrictions & ACPI_CEDT_CFMWS_RESTRICT_PMEM) + flags |= CXL_DECODER_F_PMEM; + if (restrictions & ACPI_CEDT_CFMWS_RESTRICT_FIXED) + flags |= CXL_DECODER_F_LOCK; + + return flags; +} + +static int cxl_acpi_cfmws_verify(struct device *dev, + struct acpi_cedt_cfmws *cfmws) +{ + int expected_len; + + if (cfmws->interleave_arithmetic != ACPI_CEDT_CFMWS_ARITHMETIC_MODULO) { + dev_err(dev, "CFMWS Unsupported Interleave Arithmetic\n"); + return -EINVAL; + } + + if (!IS_ALIGNED(cfmws->base_hpa, SZ_256M)) { + dev_err(dev, "CFMWS Base HPA not 256MB aligned\n"); + return -EINVAL; + } + + if (!IS_ALIGNED(cfmws->window_size, SZ_256M)) { + dev_err(dev, "CFMWS Window Size not 256MB aligned\n"); + return -EINVAL; + } + + expected_len = struct_size((cfmws), interleave_targets, + CFMWS_INTERLEAVE_WAYS(cfmws)); + + if (cfmws->header.length < expected_len) { + dev_err(dev, "CFMWS length %d less than expected %d\n", + cfmws->header.length, expected_len); + return -EINVAL; + } + + if (cfmws->header.length > expected_len) + dev_dbg(dev, "CFMWS length %d greater than expected %d\n", + cfmws->header.length, expected_len); + + return 0; +} + +static void cxl_add_cfmws_decoders(struct device *dev, + struct cxl_port *root_port) +{ + struct acpi_cedt_cfmws *cfmws; + struct cxl_decoder *cxld; + acpi_size len, cur = 0; + void *cedt_subtable; + unsigned long flags; + int rc; + + len = acpi_cedt->length - sizeof(*acpi_cedt); + cedt_subtable = acpi_cedt + 1; + + while (cur < len) { + struct acpi_cedt_header *c = cedt_subtable + cur; + + if (c->type != ACPI_CEDT_TYPE_CFMWS) { + cur += c->length; + continue; + } + + cfmws = cedt_subtable + cur; + + if (cfmws->header.length < sizeof(*cfmws)) { + dev_warn_once(dev, + "CFMWS entry skipped:invalid length:%u\n", + cfmws->header.length); + cur += c->length; + continue; + } + + rc = cxl_acpi_cfmws_verify(dev, cfmws); + if (rc) { + dev_err(dev, "CFMWS range %#llx-%#llx not registered\n", + cfmws->base_hpa, cfmws->base_hpa + + cfmws->window_size - 1); + cur += c->length; + continue; + } + + flags = cfmws_to_decoder_flags(cfmws->restrictions); + cxld = devm_cxl_add_decoder(dev, root_port, + CFMWS_INTERLEAVE_WAYS(cfmws), + cfmws->base_hpa, cfmws->window_size, + CFMWS_INTERLEAVE_WAYS(cfmws), + CFMWS_INTERLEAVE_GRANULARITY(cfmws), + CXL_DECODER_EXPANDER, + flags); + + if (IS_ERR(cxld)) { + dev_err(dev, "Failed to add decoder for %#llx-%#llx\n", + cfmws->base_hpa, cfmws->base_hpa + + cfmws->window_size - 1); + } else { + dev_dbg(dev, "add: %s range %#llx-%#llx\n", + dev_name(&cxld->dev), cfmws->base_hpa, + cfmws->base_hpa + cfmws->window_size - 1); + } + cur += c->length; + } +} + static struct acpi_cedt_chbs *cxl_acpi_match_chbs(struct device *dev, u32 uid) { struct acpi_cedt_chbs *chbs, *chbs_match = NULL; @@ -273,6 +393,8 @@ static int cxl_acpi_probe(struct platform_device *pdev) if (rc) goto out; + cxl_add_cfmws_decoders(host, root_port); + /* * Root level scanned with host-bridge as dports, now scan host-bridges * for their role as CXL uports to their CXL-capable PCIe Root Ports. From 4ad6181e4b216ed0cb52f45d3c6d2c70c8ae9243 Mon Sep 17 00:00:00 2001 From: Ben Widawsky Date: Thu, 17 Jun 2021 17:30:09 -0700 Subject: [PATCH 34/34] cxl/pci: Rename CXL REGLOC ID The current naming is confusing and wrong. The Register Locator is identified by the DSVSEC identifier, not an offset. Cc: Dan Williams Signed-off-by: Ben Widawsky Link: https://lore.kernel.org/r/20210618003009.956929-1-ben.widawsky@intel.com Signed-off-by: Dan Williams --- drivers/cxl/pci.c | 2 +- drivers/cxl/pci.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/cxl/pci.c b/drivers/cxl/pci.c index f8408e5f0754..4cf351a3cf99 100644 --- a/drivers/cxl/pci.c +++ b/drivers/cxl/pci.c @@ -1086,7 +1086,7 @@ static int cxl_mem_setup_regs(struct cxl_mem *cxlm) LIST_HEAD(register_maps); int ret = 0; - regloc = cxl_mem_dvsec(pdev, PCI_DVSEC_ID_CXL_REGLOC_OFFSET); + regloc = cxl_mem_dvsec(pdev, PCI_DVSEC_ID_CXL_REGLOC_DVSEC_ID); if (!regloc) { dev_err(dev, "register location dvsec not found\n"); return -ENXIO; diff --git a/drivers/cxl/pci.h b/drivers/cxl/pci.h index af3ec078cf6c..dad7a831f65f 100644 --- a/drivers/cxl/pci.h +++ b/drivers/cxl/pci.h @@ -13,7 +13,7 @@ #define PCI_DVSEC_VENDOR_ID_CXL 0x1E98 #define PCI_DVSEC_ID_CXL 0x0 -#define PCI_DVSEC_ID_CXL_REGLOC_OFFSET 0x8 +#define PCI_DVSEC_ID_CXL_REGLOC_DVSEC_ID 0x8 #define PCI_DVSEC_ID_CXL_REGLOC_BLOCK1_OFFSET 0xC /* BAR Indicator Register (BIR) */