From bd1e02982aec7b6d30c44210d90fd96a7d78a03f Mon Sep 17 00:00:00 2001 From: Bibek Kumar Patro Date: Sat, 8 Aug 2026 14:49:13 +0530 Subject: [PATCH] iommu: qcom: Add iommu-debug framework Add the QCOM IOMMU debug/testing framework (CONFIG_QCOM_IOMMU_DEBUG) with debugfs-based profiling and ATOS testing support. Add the iommu_test_device node to nord-rrd and nord-ride-sx using apps_smmu_0 SID 0x0001 which is reserved for IOMMU debug use. Change-Id: Iaba7cc8c7ecbbaa045089a6af9ba955b8cc96286 Signed-off-by: Bibek Kumar Patro --- arch/arm64/boot/dts/qcom/nord-ride-sx.dts | 35 + arch/arm64/boot/dts/qcom/nord-rrd.dts | 35 + arch/arm64/configs/defconfig | 1 + arch/arm64/configs/qcom.config | 3 + drivers/iommu/Kconfig | 9 + drivers/iommu/Makefile | 2 + .../iommu/arm/arm-smmu/arm-smmu-qcom-debug.c | 14 + drivers/iommu/arm/arm-smmu/arm-smmu.c | 2 +- drivers/iommu/qcom-iommu-debug-user.c | 1457 +++++++++++++++++ drivers/iommu/qcom-iommu-debug.c | 400 +++++ drivers/iommu/qcom-iommu-debug.h | 76 + drivers/iommu/qcom-iommu-util.c | 826 ++++++++++ include/linux/qcom-dma-mapping.h | 39 + include/linux/qcom-iommu-util.h | 144 ++ 14 files changed, 3042 insertions(+), 1 deletion(-) create mode 100644 drivers/iommu/qcom-iommu-debug-user.c create mode 100644 drivers/iommu/qcom-iommu-debug.c create mode 100644 drivers/iommu/qcom-iommu-debug.h create mode 100644 drivers/iommu/qcom-iommu-util.c create mode 100644 include/linux/qcom-dma-mapping.h create mode 100644 include/linux/qcom-iommu-util.h diff --git a/arch/arm64/boot/dts/qcom/nord-ride-sx.dts b/arch/arm64/boot/dts/qcom/nord-ride-sx.dts index b8839e1b86421..64fe75c158fd7 100644 --- a/arch/arm64/boot/dts/qcom/nord-ride-sx.dts +++ b/arch/arm64/boot/dts/qcom/nord-ride-sx.dts @@ -707,3 +707,38 @@ status = "okay"; }; + +&soc { + iommu_test_device { + compatible = "qcom,iommu-debug-test"; + + usecase0_apps { + compatible = "qcom,iommu-debug-usecase"; + iommus = <&apps_smmu_0 0x0001 0x0>; + }; + + usecase1_apps_fastmap { + compatible = "qcom,iommu-debug-usecase"; + iommus = <&apps_smmu_0 0x0001 0x0>; + qcom,iommu-dma = "fastmap"; + }; + + usecase2_apps_atomic { + compatible = "qcom,iommu-debug-usecase"; + iommus = <&apps_smmu_0 0x0001 0x0>; + qcom,iommu-dma = "atomic"; + }; + + usecase3_apps_dma { + compatible = "qcom,iommu-debug-usecase"; + iommus = <&apps_smmu_0 0x0001 0x0>; + dma-coherent; + }; + + sec_pix: secure-pixel { + compatible = "qcom,iommu-debug-usecase"; + iommus = <&apps_smmu_0 0x0001 0x0>; + qcom,iommu-vmid = <0xa>; /* VMID_CP_PIXEL */ + }; + }; +}; diff --git a/arch/arm64/boot/dts/qcom/nord-rrd.dts b/arch/arm64/boot/dts/qcom/nord-rrd.dts index ddc534e2214b4..4c9ae13772e47 100644 --- a/arch/arm64/boot/dts/qcom/nord-rrd.dts +++ b/arch/arm64/boot/dts/qcom/nord-rrd.dts @@ -626,3 +626,38 @@ status = "okay"; }; + +&soc { + iommu_test_device { + compatible = "qcom,iommu-debug-test"; + + usecase0_apps { + compatible = "qcom,iommu-debug-usecase"; + iommus = <&apps_smmu_0 0x0001 0x0>; + }; + + usecase1_apps_fastmap { + compatible = "qcom,iommu-debug-usecase"; + iommus = <&apps_smmu_0 0x0001 0x0>; + qcom,iommu-dma = "fastmap"; + }; + + usecase2_apps_atomic { + compatible = "qcom,iommu-debug-usecase"; + iommus = <&apps_smmu_0 0x0001 0x0>; + qcom,iommu-dma = "atomic"; + }; + + usecase3_apps_dma { + compatible = "qcom,iommu-debug-usecase"; + iommus = <&apps_smmu_0 0x0001 0x0>; + dma-coherent; + }; + + sec_pix: secure-pixel { + compatible = "qcom,iommu-debug-usecase"; + iommus = <&apps_smmu_0 0x0001 0x0>; + qcom,iommu-vmid = <0xa>; /* VMID_CP_PIXEL */ + }; + }; +}; diff --git a/arch/arm64/configs/defconfig b/arch/arm64/configs/defconfig index 60178ffbcb2e3..dca966103b72b 100644 --- a/arch/arm64/configs/defconfig +++ b/arch/arm64/configs/defconfig @@ -1974,3 +1974,4 @@ CONFIG_CORESIGHT_STM=m CONFIG_CORESIGHT_CPU_DEBUG=m CONFIG_CORESIGHT_CTI=m CONFIG_MEMTEST=y +CONFIG_QCOM_IOMMU_DEBUG=y diff --git a/arch/arm64/configs/qcom.config b/arch/arm64/configs/qcom.config index b286309f0c251..1f9a4240c9548 100644 --- a/arch/arm64/configs/qcom.config +++ b/arch/arm64/configs/qcom.config @@ -140,3 +140,6 @@ CONFIG_SCMI_QCOM_MEMLAT_DEVFREQ=m # GPU Virtualization support CONFIG_DRM_VIRTIO_GPU=m CONFIG_DRM_VIRTIO_GPU_KMS=m + +# QCOM SMMU debug support (required for qcom_smmu_itop_hard) +CONFIG_ARM_SMMU_QCOM_DEBUG=y diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index 6e07bd69467a3..eb4d24aa87916 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -315,6 +315,15 @@ config APPLE_DART Say Y here if you are using an Apple SoC. +config QCOM_IOMMU_DEBUG + tristate "IOMMU debugging and testing" + depends on DEBUG_FS + help + This option is used to enable profiling and debugging in + the IOMMU framework code. IOMMU profiling and debugging + can be done through the debugfs nodes which this option + makes available. + config S390_IOMMU def_bool y if S390 && PCI depends on S390 && PCI diff --git a/drivers/iommu/Makefile b/drivers/iommu/Makefile index 2f05725eaab18..81f4d29e7f3fa 100644 --- a/drivers/iommu/Makefile +++ b/drivers/iommu/Makefile @@ -38,3 +38,5 @@ obj-$(CONFIG_SPRD_IOMMU) += sprd-iommu.o obj-$(CONFIG_APPLE_DART) += apple-dart.o obj-$(CONFIG_VSI_IOMMU) += vsi-iommu.o obj-$(CONFIG_IOMMU_DEBUG_PAGEALLOC) += iommu-debug-pagealloc.o +obj-$(CONFIG_QCOM_IOMMU_DEBUG) += qcom_iommu_debug.o +qcom_iommu_debug-y += qcom-iommu-debug.o qcom-iommu-debug-user.o diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu-qcom-debug.c b/drivers/iommu/arm/arm-smmu/arm-smmu-qcom-debug.c index 531b29fbf4924..e4fdd534a9095 100644 --- a/drivers/iommu/arm/arm-smmu/arm-smmu-qcom-debug.c +++ b/drivers/iommu/arm/arm-smmu/arm-smmu-qcom-debug.c @@ -354,9 +354,23 @@ static phys_addr_t qcom_smmu_iova_to_phys_hard(struct arm_smmu_domain *smmu_doma frsynra = arm_smmu_gr1_read(smmu, ARM_SMMU_GR1_CBFRSYNRA(idx)); sid = FIELD_GET(ARM_SMMU_CBFRSYNRA_SID, frsynra); + /* + * This is hardcoded because there is junk in ARM_SMMU_CBFRSYNRA_SID. + * This value is specific to target you're checking on. + **/ + sid = 0x00E0; + pr_err("Trying for SID: %x\n", sid); return qcom_iova_to_phys(smmu_domain, iova, sid); } +phys_addr_t qcom_smmu_itop_hard(struct iommu_domain *dom, dma_addr_t iova) +{ + struct arm_smmu_domain *smmu_domain = container_of(dom, struct arm_smmu_domain, domain); + + return qcom_smmu_iova_to_phys_hard(smmu_domain, iova); +} +EXPORT_SYMBOL(qcom_smmu_itop_hard); + static phys_addr_t qcom_smmu_verify_fault(struct arm_smmu_domain *smmu_domain, dma_addr_t iova, u32 fsr) { struct io_pgtable *iop = io_pgtable_ops_to_pgtable(smmu_domain->pgtbl_ops); diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu.c b/drivers/iommu/arm/arm-smmu/arm-smmu.c index 744ff8b4d5733..af22bfe146d5b 100644 --- a/drivers/iommu/arm/arm-smmu/arm-smmu.c +++ b/drivers/iommu/arm/arm-smmu/arm-smmu.c @@ -1391,7 +1391,7 @@ static void arm_smmu_iotlb_sync(struct iommu_domain *domain, arm_smmu_rpm_put(smmu); } -static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain, +phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain, dma_addr_t iova) { struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); diff --git a/drivers/iommu/qcom-iommu-debug-user.c b/drivers/iommu/qcom-iommu-debug-user.c new file mode 100644 index 0000000000000..b2e6507a0a632 --- /dev/null +++ b/drivers/iommu/qcom-iommu-debug-user.c @@ -0,0 +1,1457 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2015-2021, The Linux Foundation. All rights reserved. + * Copyright (c) 2022-2023, Qualcomm Innovation Center, Inc. All rights reserved. + * + */ + +#include +#include +#include +#include +#include +#include +#include "qcom-iommu-debug.h" + +#ifdef CONFIG_64BIT +#define kstrtoux kstrtou64 +#define kstrtox_from_user kstrtoull_from_user +#define kstrtosize_t kstrtoul +#else +#define kstrtoux kstrtou32 +#define kstrtox_from_user kstrtouint_from_user +#define kstrtosize_t kstrtouint +#endif + +static void *test_virt_addr; +static DEFINE_MUTEX(test_virt_addr_lock); +static DEFINE_XARRAY(xa_qcom_iommu_ops); + +static struct qcom_iommu_ops *to_qcom_iommu_ops(const struct iommu_domain_ops *ops) +{ + return xa_load(&xa_qcom_iommu_ops, (unsigned long)ops); +} + +extern phys_addr_t qcom_smmu_itop_hard(struct iommu_domain *dom, dma_addr_t iova); +phys_addr_t qcom_iommu_iova_to_phys_hard(struct iommu_domain *domain, + struct qcom_iommu_atos_txn *txn) +{ + + return qcom_smmu_itop_hard(domain, txn->addr); +} +EXPORT_SYMBOL(qcom_iommu_iova_to_phys_hard); + +static ssize_t iommu_debug_dma_atos_read(struct file *file, char __user *ubuf, + size_t count, loff_t *offset) +{ + struct iommu_debug_device *ddev = file->private_data; + struct iommu_fwspec *fwspec; + phys_addr_t phys; + char buf[100] = {0}; + struct qcom_iommu_atos_txn txn; + int len; + + if (*offset) + return 0; + + mutex_lock(&ddev->state_lock); + if (!ddev->domain) { + pr_err("%s: No domain. Have you selected a usecase?\n", __func__); + mutex_unlock(&ddev->state_lock); + return -EINVAL; + } + + fwspec = dev_iommu_fwspec_get(ddev->test_dev); + if (!fwspec) { + pr_err("%s: No fwspec.\n", __func__); + mutex_unlock(&ddev->state_lock); + return 0; + } + + txn.addr = ddev->iova; + txn.flags = IOMMU_TRANS_DEFAULT; + txn.id = FIELD_GET(ARM_SMMU_SMR_ID, fwspec->ids[0]); + phys = qcom_iommu_iova_to_phys_hard(ddev->domain, &txn); + if (!phys) + len = strscpy(buf, "FAIL\n", sizeof(buf)); + else + len = scnprintf(buf, sizeof(buf), "%pa\n", &phys); + mutex_unlock(&ddev->state_lock); + return simple_read_from_buffer(ubuf, count, offset, buf, len); +} + +static ssize_t iommu_debug_atos_write(struct file *file, + const char __user *ubuf, + size_t count, loff_t *offset) +{ + struct iommu_debug_device *ddev = file->private_data; + dma_addr_t iova; + phys_addr_t phys; + unsigned long pfn; + + mutex_lock(&ddev->state_lock); + if (!ddev->domain) { + pr_err("%s: No domain. Have you selected a usecase?\n", __func__); + mutex_unlock(&ddev->state_lock); + return -EINVAL; + } + + if (kstrtox_from_user(ubuf, count, 0, &iova)) { + dev_err(ddev->test_dev, "Invalid format for iova\n"); + ddev->iova = 0; + mutex_unlock(&ddev->state_lock); + return -EINVAL; + } + + phys = iommu_iova_to_phys(ddev->domain, iova); + pfn = __phys_to_pfn(phys); + if (!pfn_valid(pfn)) { + dev_err(ddev->test_dev, "Invalid ATOS operation page %pa\n", &phys); + mutex_unlock(&ddev->state_lock); + return -EINVAL; + } + ddev->iova = iova; + + mutex_unlock(&ddev->state_lock); + pr_info("Saved iova=%pa for future ATOS commands\n", &iova); + return count; +} + +const struct file_operations iommu_debug_atos_fops = { + .open = simple_open, + .write = iommu_debug_atos_write, + .read = iommu_debug_dma_atos_read, +}; + +static ssize_t iommu_debug_map_write(struct file *file, const char __user *ubuf, + size_t count, loff_t *offset) +{ + ssize_t retval = -EINVAL; + int ret; + char *comma1, *comma2, *comma3; + char buf[100] = {0}; + dma_addr_t iova; + phys_addr_t phys; + size_t size; + int prot; + struct iommu_debug_device *ddev = file->private_data; + + if (count >= 100) { + pr_err_ratelimited("Value too large\n"); + return -EINVAL; + } + + if (copy_from_user(buf, ubuf, count)) { + pr_err_ratelimited("Couldn't copy from user\n"); + retval = -EFAULT; + } + + comma1 = strnchr(buf, count, ','); + if (!comma1) + goto invalid_format; + + comma2 = strnchr(comma1 + 1, count, ','); + if (!comma2) + goto invalid_format; + + comma3 = strnchr(comma2 + 1, count, ','); + if (!comma3) + goto invalid_format; + + /* split up the words */ + *comma1 = *comma2 = *comma3 = '\0'; + + if (kstrtoux(buf, 0, &iova)) + goto invalid_format; + + if (kstrtoux(comma1 + 1, 0, &phys)) + goto invalid_format; + + if (kstrtosize_t(comma2 + 1, 0, &size)) + goto invalid_format; + + if (kstrtoint(comma3 + 1, 0, &prot)) + goto invalid_format; + + mutex_lock(&ddev->state_lock); + if (!ddev->domain) { + pr_err_ratelimited("%s: No domain. Have you selected a usecase?\n", __func__); + mutex_unlock(&ddev->state_lock); + return -EINVAL; + } + + ret = iommu_map(ddev->domain, iova, phys, size, prot, GFP_KERNEL); + if (ret) { + pr_err_ratelimited("iommu_map failed with %d\n", ret); + retval = -EIO; + goto out; + } + + retval = count; + pr_info("Mapped %pa to %pa (len=0x%zx, prot=0x%x)\n", &iova, &phys, size, prot); +out: + mutex_unlock(&ddev->state_lock); + return retval; + +invalid_format: + pr_err_ratelimited("Invalid format. Expected: iova,phys,len,prot where `prot' is the bitwise OR of IOMMU_READ, IOMMU_WRITE, etc.\n"); + return -EINVAL; +} + +const struct file_operations iommu_debug_map_fops = { + .open = simple_open, + .write = iommu_debug_map_write, +}; + +static ssize_t iommu_debug_unmap_write(struct file *file, + const char __user *ubuf, + size_t count, loff_t *offset) +{ + ssize_t retval = 0; + char *comma1; + char buf[100] = {0}; + dma_addr_t iova; + size_t size; + size_t unmapped; + struct iommu_debug_device *ddev = file->private_data; + + if (count >= 100) { + pr_err_ratelimited("Value too large\n"); + return -EINVAL; + } + + if (!ddev->domain) { + pr_err_ratelimited("%s: No domain. Have you selected a usecase?\n", __func__); + return -EINVAL; + } + + if (copy_from_user(buf, ubuf, count)) { + pr_err_ratelimited("Couldn't copy from user\n"); + retval = -EFAULT; + goto out; + } + + comma1 = strnchr(buf, count, ','); + if (!comma1) + goto invalid_format; + + /* split up the words */ + *comma1 = '\0'; + + if (kstrtoux(buf, 0, &iova)) + goto invalid_format; + + if (kstrtosize_t(comma1 + 1, 0, &size)) + goto invalid_format; + + mutex_lock(&ddev->state_lock); + if (!ddev->domain) { + pr_err_ratelimited("No domain. Did you already attach?\n"); + mutex_unlock(&ddev->state_lock); + return -EINVAL; + } + + unmapped = iommu_unmap(ddev->domain, iova, size); + if (unmapped != size) { + pr_err_ratelimited("iommu_unmap failed. Expected to unmap: 0x%zx, unmapped: 0x%zx", + size, unmapped); + retval = -EIO; + goto out; + } + + retval = count; + pr_info("Unmapped %pa (len=0x%zx)\n", &iova, size); +out: + mutex_unlock(&ddev->state_lock); + return retval; + +invalid_format: + pr_err_ratelimited("Invalid format. Expected: iova,len\n"); + return -EINVAL; +} + +const struct file_operations iommu_debug_unmap_fops = { + .open = simple_open, + .write = iommu_debug_unmap_write, +}; + +/* + * Performs DMA mapping of a given virtual address and size to an iova address. + * User input format: (addr,len,dma attr) where dma attr is: + * 0: normal mapping + * 1: force coherent mapping + * 2: force non-cohernet mapping + * 3: use system cache + */ +static ssize_t iommu_debug_dma_map_write(struct file *file, + const char __user *ubuf, size_t count, loff_t *offset) +{ + ssize_t retval = -EINVAL; + int ret; + char *comma1, *comma2; + char buf[100] = {0}; + unsigned long addr; + void *v_addr; + dma_addr_t iova; + size_t size; + unsigned int attr; + unsigned long dma_attrs; + struct iommu_debug_device *ddev = file->private_data; + struct device *dev = ddev->test_dev; + + if (count >= sizeof(buf)) { + pr_err_ratelimited("Value too large\n"); + return -EINVAL; + } + + if (copy_from_user(buf, ubuf, count)) { + pr_err_ratelimited("Couldn't copy from user\n"); + return -EFAULT; + } + + comma1 = strnchr(buf, count, ','); + if (!comma1) + goto invalid_format; + + comma2 = strnchr(comma1 + 1, count, ','); + if (!comma2) + goto invalid_format; + + *comma1 = *comma2 = '\0'; + + if (kstrtoul(buf, 0, &addr)) + goto invalid_format; + + v_addr = (void *)addr; + + if (kstrtosize_t(comma1 + 1, 0, &size)) + goto invalid_format; + + if (kstrtouint(comma2 + 1, 0, &attr)) + goto invalid_format; + + mutex_lock(&test_virt_addr_lock); + if (IS_ERR(test_virt_addr)) { + mutex_unlock(&test_virt_addr_lock); + goto allocation_failure; + } + + if (!test_virt_addr) { + mutex_unlock(&test_virt_addr_lock); + goto missing_allocation; + } + mutex_unlock(&test_virt_addr_lock); + + if (v_addr < test_virt_addr || v_addr + size > test_virt_addr + SZ_1M) + goto invalid_addr; + + if (attr == 0) + dma_attrs = 0; + else if (attr == 1) + dma_attrs = DMA_ATTR_FORCE_COHERENT; + else if (attr == 2) + dma_attrs = DMA_ATTR_FORCE_NON_COHERENT; + else if (attr == 3) + dma_attrs = DMA_ATTR_SYS_CACHE; + else + goto invalid_format; + + mutex_lock(&ddev->state_lock); + if (!ddev->domain) { + pr_err_ratelimited("%s: No domain. Have you selected a usecase?\n", __func__); + mutex_unlock(&ddev->state_lock); + return -EINVAL; + } + + iova = dma_map_single_attrs(dev, v_addr, size, DMA_TO_DEVICE, dma_attrs); + + if (dma_mapping_error(dev, iova)) { + pr_err_ratelimited("Failed to perform dma_map_single\n"); + ret = -EINVAL; + goto out; + } + + retval = count; + pr_err_ratelimited("Mapped 0x%p to %pa (len=0x%zx)\n", v_addr, &iova, size); + ddev->iova = iova; + pr_err_ratelimited("Saved iova=%pa for future PTE commands\n", &iova); + +out: + mutex_unlock(&ddev->state_lock); + return retval; + +invalid_format: + pr_err_ratelimited("Invalid format. Expected: addr,len,dma attr where 'dma attr' is\n0: normal mapping\n1: force coherent\n2: force non-cohernet\n3: use system cache\n"); + return retval; + +invalid_addr: + pr_err_ratelimited("Invalid addr given (0x%p)! Address should be within 1MB size from start addr returned by doing 'cat test_virt_addr'.\n", + v_addr); + return retval; + +allocation_failure: + pr_err_ratelimited("Allocation of test_virt_addr failed.\n"); + return -ENOMEM; + +missing_allocation: + pr_err_ratelimited("Please attempt to do 'cat test_virt_addr'.\n"); + return retval; +} + +static ssize_t iommu_debug_dma_map_read(struct file *file, char __user *ubuf, + size_t count, loff_t *offset) +{ + struct iommu_debug_device *ddev = file->private_data; + char buf[100] = {}; + dma_addr_t iova; + int len; + + if (*offset) + return 0; + + iova = ddev->iova; + len = scnprintf(buf, sizeof(buf), "%pa\n", &iova); + return simple_read_from_buffer(ubuf, count, offset, buf, len); +} + +const struct file_operations iommu_debug_dma_map_fops = { + .open = simple_open, + .read = iommu_debug_dma_map_read, + .write = iommu_debug_dma_map_write, +}; + +static ssize_t iommu_debug_dma_unmap_write(struct file *file, const char __user *ubuf, + size_t count, loff_t *offset) +{ + ssize_t retval = 0; + char *comma1, *comma2; + char buf[100] = {}; + size_t size; + unsigned int attr; + dma_addr_t iova; + unsigned long dma_attrs; + struct iommu_debug_device *ddev = file->private_data; + struct device *dev = ddev->test_dev; + + if (count >= sizeof(buf)) { + pr_err_ratelimited("Value too large\n"); + return -EINVAL; + } + + if (copy_from_user(buf, ubuf, count)) { + pr_err_ratelimited("Couldn't copy from user\n"); + retval = -EFAULT; + goto out; + } + + comma1 = strnchr(buf, count, ','); + if (!comma1) + goto invalid_format; + + comma2 = strnchr(comma1 + 1, count, ','); + if (!comma2) + goto invalid_format; + + *comma1 = *comma2 = '\0'; + + if (kstrtoux(buf, 0, &iova)) + goto invalid_format; + + if (kstrtosize_t(comma1 + 1, 0, &size)) + goto invalid_format; + + if (kstrtouint(comma2 + 1, 0, &attr)) + goto invalid_format; + + if (attr == 0) + dma_attrs = 0; + else if (attr == 1) + dma_attrs = DMA_ATTR_FORCE_COHERENT; + else if (attr == 2) + dma_attrs = DMA_ATTR_FORCE_NON_COHERENT; + else if (attr == 3) + dma_attrs = DMA_ATTR_SYS_CACHE; + else + goto invalid_format; + + mutex_lock(&ddev->state_lock); + if (!ddev->domain) { + pr_err_ratelimited("%s: No domain. Have you selected a usecase?\n", __func__); + mutex_unlock(&ddev->state_lock); + return -EINVAL; + } + dma_unmap_single_attrs(dev, iova, size, DMA_TO_DEVICE, dma_attrs); + + retval = count; + pr_err_ratelimited("Unmapped %pa (len=0x%zx)\n", &iova, size); +out: + mutex_unlock(&ddev->state_lock); + return retval; + +invalid_format: + pr_err_ratelimited("Invalid format. Expected: iova,len, dma attr\n"); + return -EINVAL; +} + +const struct file_operations iommu_debug_dma_unmap_fops = { + .open = simple_open, + .write = iommu_debug_dma_unmap_write, +}; + +static int iommu_debug_build_phoney_sg_table(struct device *dev, + struct sg_table *table, + unsigned long total_size, + unsigned long chunk_size) +{ + unsigned long nents = total_size / chunk_size; + struct scatterlist *sg; + int i, j; + struct page *page; + + if (!IS_ALIGNED(total_size, PAGE_SIZE)) + return -EINVAL; + if (!IS_ALIGNED(total_size, chunk_size)) + return -EINVAL; + if (sg_alloc_table(table, nents, GFP_KERNEL)) + return -EINVAL; + + for_each_sg(table->sgl, sg, table->nents, i) { + page = alloc_pages(GFP_KERNEL, get_order(chunk_size)); + if (!page) + goto free_pages; + sg_set_page(sg, page, chunk_size, 0); + } + + return 0; +free_pages: + for_each_sg(table->sgl, sg, i--, j) + __free_pages(sg_page(sg), get_order(chunk_size)); + sg_free_table(table); + return -ENOMEM; +} + +static void iommu_debug_destroy_phoney_sg_table(struct device *dev, + struct sg_table *table, + unsigned long chunk_size) +{ + struct scatterlist *sg; + int i; + + for_each_sg(table->sgl, sg, table->nents, i) + __free_pages(sg_page(sg), get_order(chunk_size)); + sg_free_table(table); +} + +#define ps_printf(name, s, fmt, ...) ({ \ + pr_err("%s: " fmt, name, ##__VA_ARGS__); \ + seq_printf(s, fmt, ##__VA_ARGS__); \ + }) + +static int __functional_dma_api_alloc_test(struct device *dev, + struct seq_file *s, + struct iommu_domain *domain, + void *ignored) +{ + size_t size = SZ_1K * 742; + int ret = 0; + u8 *data; + dma_addr_t iova; + + /* Make sure we can allocate and use a buffer */ + ps_printf(dev_name(dev), s, "Allocating coherent buffer"); + data = dma_alloc_coherent(dev, size, &iova, GFP_KERNEL); + if (!data) { + ret = -ENOMEM; + } else { + int i; + + ps_printf(dev_name(dev), s, " -> SUCCEEDED\n"); + ps_printf(dev_name(dev), s, "Using coherent buffer"); + for (i = 0; i < 742; ++i) { + int ind = SZ_1K * i; + u8 *p = data + ind; + u8 val = i % 255; + + memset(data, 0xa5, size); + *p = val; + (*p)++; + if ((*p) != val + 1) { + ps_printf(dev_name(dev), s, + " -> FAILED on iter %d since %d != %d\n", + i, *p, val + 1); + ret = -EINVAL; + } + } + if (!ret) + ps_printf(dev_name(dev), s, " -> SUCCEEDED\n"); + dma_free_coherent(dev, size, data, iova); + } + + return ret; +} + +static int __functional_dma_api_basic_test(struct device *dev, + struct seq_file *s, + struct iommu_domain *domain, + void *ignored) +{ + size_t size = 1518; + int i, j, ret = 0; + u8 *data; + dma_addr_t iova; + + ps_printf(dev_name(dev), s, "Basic DMA API test"); + /* Make sure we can allocate and use a buffer */ + for (i = 0; i < 1000; ++i) { + data = kmalloc(size, GFP_KERNEL); + if (!data) { + ret = -ENOMEM; + goto out; + } + memset(data, 0xa5, size); + iova = dma_map_single(dev, data, size, DMA_TO_DEVICE); + ret = iommu_debug_check_mapping_fast(dev, iova, size, virt_to_phys(data)); + if (ret) + goto out; + + dma_unmap_single(dev, iova, size, DMA_TO_DEVICE); + for (j = 0; j < size; ++j) { + if (data[j] != 0xa5) { + dev_err_ratelimited(dev, "data[%d] != 0xa5\n", data[j]); + ret = -EINVAL; + goto out; + } + } + kfree(data); + } + +out: + if (ret) + ps_printf(dev_name(dev), s, " -> FAILED\n"); + else + ps_printf(dev_name(dev), s, " -> SUCCEEDED\n"); + + return ret; +} + +static int __functional_dma_api_map_sg_test(struct device *dev, struct seq_file *s, + struct iommu_domain *domain, size_t sizes[]) +{ + const size_t *sz; + int ret = 0, count = 0; + + ps_printf(dev_name(dev), s, "Map SG DMA API test"); + + for (sz = sizes; *sz; ++sz) { + size_t size = *sz; + struct sg_table table; + unsigned long chunk_size = SZ_4K; + + /* Build us a table */ + ret = iommu_debug_build_phoney_sg_table(dev, &table, size, chunk_size); + if (ret) { + seq_puts(s, "couldn't build phoney sg table! bailing...\n"); + goto out; + } + + count = dma_map_sg(dev, table.sgl, table.nents, DMA_BIDIRECTIONAL); + if (!count) { + ret = -EINVAL; + goto destroy_table; + } + + /* Check mappings... */ + ret = iommu_debug_check_mapping_sg_fast(dev, table.sgl, 0, table.nents, count); + + dma_unmap_sg(dev, table.sgl, table.nents, DMA_BIDIRECTIONAL); +destroy_table: + iommu_debug_destroy_phoney_sg_table(dev, &table, chunk_size); + } +out: + if (ret) + ps_printf(dev_name(dev), s, " -> FAILED\n"); + else + ps_printf(dev_name(dev), s, " -> SUCCEEDED\n"); + + return ret; +} + +static int iommu_debug_functional_arm_dma_api_show(struct seq_file *s, + void *ignored) +{ + struct iommu_debug_device *ddev = s->private; + struct device *dev; + size_t sizes[] = {SZ_4K, SZ_64K, SZ_2M, SZ_1M * 12, 0}; + int ret = -EINVAL; + + mutex_lock(&ddev->state_lock); + if (!iommu_debug_usecase_reset(ddev)) + goto out; + dev = ddev->test_dev; + + ret = __functional_dma_api_alloc_test(dev, s, ddev->domain, sizes); + ret |= __functional_dma_api_basic_test(dev, s, ddev->domain, sizes); + ret |= __functional_dma_api_map_sg_test(dev, s, ddev->domain, sizes); + +out: + mutex_unlock(&ddev->state_lock); + if (ret) + seq_printf(s, "FAIL %d\n", ret); + else + seq_puts(s, "SUCCESS\n"); + + return 0; +} + +static int iommu_debug_functional_arm_dma_api_open(struct inode *inode, + struct file *file) +{ + return single_open(file, iommu_debug_functional_arm_dma_api_show, + inode->i_private); +} + +const struct file_operations iommu_debug_functional_arm_dma_api_fops = { + .open = iommu_debug_functional_arm_dma_api_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +/* Creates a fresh fast mapping and applies @fn to it */ +static int __apply_to_new_mapping(struct seq_file *s, + int (*fn)(struct device *dev, + struct seq_file *s, + struct iommu_domain *domain, + void *priv), + void *priv) +{ + struct iommu_domain *domain; + struct iommu_debug_device *ddev = s->private; + struct device *dev; + int ret = -EINVAL; + + mutex_lock(&ddev->state_lock); + if (!iommu_debug_usecase_reset(ddev)) + goto out; + + domain = ddev->domain; + dev = ddev->test_dev; + + ret = fn(dev, s, domain, priv); + +out: + mutex_unlock(&ddev->state_lock); + seq_printf(s, "%s\n", ret ? "FAIL" : "SUCCESS"); + return ret; +} + +static const char * const _size_to_string(unsigned long size) +{ + switch (size) { + case SZ_4K: + return "4K"; + case SZ_8K: + return "8K"; + case SZ_16K: + return "16K"; + case SZ_64K: + return "64K"; + case SZ_1M: + return "1M"; + case SZ_2M: + return "2M"; + case SZ_1M * 12: + return "12M"; + case SZ_1M * 20: + return "20M"; + case SZ_1M * 24: + return "24M"; + case SZ_1M * 32: + return "32M"; + } + + pr_err("unknown size, please add to %s\n", __func__); + return "unknown size"; +} + +static int __check_mapping(struct device *dev, struct iommu_domain *domain, + dma_addr_t iova, phys_addr_t expected) +{ + struct iommu_fwspec *fwspec; + phys_addr_t res, res2; + struct qcom_iommu_atos_txn txn; + + fwspec = dev_iommu_fwspec_get(dev); + if (!fwspec) { + dev_err_ratelimited(dev, "No fwspec.\n"); + return -EINVAL; + } + + txn.addr = iova; + txn.flags = IOMMU_TRANS_DEFAULT; + txn.id = FIELD_GET(ARM_SMMU_SMR_ID, fwspec->ids[0]); + + res = qcom_iommu_iova_to_phys_hard(domain, &txn); + res2 = iommu_iova_to_phys(domain, iova); + + WARN(res != res2, "hard/soft iova_to_phys fns don't agree..."); + + if (res != expected) { + dev_err_ratelimited(dev, "Bad translation for %pa! Expected: %pa Got: %pa\n", + &iova, &expected, &res); + return -EINVAL; + } + + return 0; +} + +static int __full_va_sweep(struct device *dev, struct seq_file *s, + struct iommu_domain *domain, void *priv) +{ + u64 iova; + int nr_maps = 0; + dma_addr_t dma_addr; + void *virt; + phys_addr_t phys; + const u64 max = SZ_1G * 4ULL - 1; + int ret = 0, i; + const size_t size = (size_t)priv; + unsigned long theiova; + + virt = (void *)__get_free_pages(GFP_KERNEL, get_order(size)); + if (!virt) { + if (size > SZ_8K) { + dev_err_ratelimited(dev, "Failed to allocate %s of memory, which is a lot. Skipping test for this size\n", + _size_to_string(size)); + return 0; + } + return -ENOMEM; + } + phys = virt_to_phys(virt); + + for (iova = 0, i = 0; iova < max; iova += size, ++i) { + unsigned long expected = iova; + + if (iova == MSI_IOVA_BASE) { + iova = MSI_IOVA_BASE + MSI_IOVA_LENGTH - size; + continue; + } + + dma_addr = dma_map_single(dev, virt, size, DMA_TO_DEVICE); + if (dma_addr != expected) { + dev_err_ratelimited(dev, "Unexpected iova on iter %d (expected: 0x%lx got: 0x%lx)\n", + i, expected, (unsigned long)dma_addr); + ret = -EINVAL; + if (!dma_mapping_error(dev, dma_addr)) + dma_unmap_single(dev, dma_addr, size, + DMA_TO_DEVICE); + goto out; + } + nr_maps++; + } + + if (domain) { + /* check every mapping from 0..6M */ + for (iova = 0, i = 0; iova < SZ_2M * 3; iova += size, ++i) { + phys_addr_t expected = phys; + + if (__check_mapping(dev, domain, iova, expected)) { + dev_err_ratelimited(dev, "iter: %d\n", i); + ret = -EINVAL; + goto out; + } + } + /* and from 4G..4G-6M */ + for (iova = 0, i = 0; iova < SZ_2M * 3; iova += size, ++i) { + phys_addr_t expected = phys; + + if (iova == MSI_IOVA_BASE) { + iova = MSI_IOVA_BASE + MSI_IOVA_LENGTH - size; + continue; + } + theiova = ((SZ_1G * 4ULL) - size) - iova; + + if (__check_mapping(dev, domain, theiova, expected)) { + dev_err_ratelimited(dev, "iter: %d\n", i); + ret = -EINVAL; + goto out; + } + } + } + + /* at this point, our VA space should be full */ + dma_addr = dma_map_single(dev, virt, size, DMA_TO_DEVICE); + if (dma_addr != DMA_MAPPING_ERROR) { + dev_err_ratelimited(dev, "dma_map_single succeeded when it should have failed. Got iova: 0x%lx\n", + (unsigned long)dma_addr); + ret = -EINVAL; + } + +out: + for (iova = 0; iova < max && nr_maps--; iova += size) { + if (iova == MSI_IOVA_BASE) { + iova = MSI_IOVA_BASE + MSI_IOVA_LENGTH - size; + continue; + } + dma_unmap_single(dev, (dma_addr_t)iova, size, DMA_TO_DEVICE); + } + + free_pages((unsigned long)virt, get_order(size)); + return ret; +} + +static int __tlb_stress_sweep(struct device *dev, struct seq_file *s, + struct iommu_domain *domain, void *unused) +{ + int i, ret = 0; + int nr_maps = 0; + u64 iova; + u64 first_iova = 0; + const u64 max = SZ_1G * 4ULL - 1; + void *virt; + phys_addr_t phys; + dma_addr_t dma_addr; + + /* + * we'll be doing 4K and 8K mappings. Need to own an entire 8K + * chunk that we can work with. + */ + virt = (void *)__get_free_pages(GFP_KERNEL, get_order(SZ_8K)); + phys = virt_to_phys(virt); + + /* fill the whole 4GB space */ + for (iova = 0, i = 0; iova < max; iova += SZ_8K, ++i) { + if (iova == MSI_IOVA_BASE) { + iova = MSI_IOVA_BASE + MSI_IOVA_LENGTH - SZ_8K; + continue; + } + + dma_addr = dma_map_single(dev, virt, SZ_8K, DMA_TO_DEVICE); + if (dma_addr == DMA_MAPPING_ERROR) { + dev_err_ratelimited(dev, "Failed map on iter %d\n", i); + ret = -EINVAL; + goto out; + } else if (dma_addr != iova) { + dma_unmap_single(dev, dma_addr, SZ_8K, DMA_TO_DEVICE); + dev_err_ratelimited(dev, "Failed map on iter %d\n", i); + ret = -EINVAL; + goto out; + } + nr_maps++; + } + + if (dma_map_single(dev, virt, SZ_4K, DMA_TO_DEVICE) != DMA_MAPPING_ERROR) { + dev_err_ratelimited(dev, "dma_map_single unexpectedly (VA should have been exhausted)\n"); + ret = -EINVAL; + goto out; + } + + /* + * free up 4K at the very beginning, then leave one 4K mapping, + * then free up 8K. This will result in the next 8K map to skip + * over the 4K hole and take the 8K one. + * i.e + * 0K..4K Hole + * 4K..8K Map R1 + * 8K..12K Hole + * 12K..4G Map R2 + */ + dma_unmap_single(dev, 0, SZ_4K, DMA_TO_DEVICE); + dma_unmap_single(dev, SZ_8K, SZ_4K, DMA_TO_DEVICE); + dma_unmap_single(dev, SZ_8K + SZ_4K, SZ_4K, DMA_TO_DEVICE); + + /* remap 8K */ + dma_addr = dma_map_single(dev, virt, SZ_8K, DMA_TO_DEVICE); + if (dma_addr != SZ_8K) { + dma_addr_t expected = SZ_8K; + + dev_err_ratelimited(dev, "Unexpected dma_addr. got: %pa expected: %pa\n", + &dma_addr, &expected); + + /* To simplify error handling, unmap the 4K regions (4K..8K + * and 12K..16K) here and the rest (16K..4G) in 8K increments + * in the for loop. + */ + dma_unmap_single(dev, SZ_4K, SZ_4K, DMA_TO_DEVICE); + dma_unmap_single(dev, SZ_8K+SZ_4K, SZ_4K, DMA_TO_DEVICE); + nr_maps -= 2; + first_iova = SZ_8K + SZ_8K; + + ret = -EINVAL; + goto out; + } + + /* + * Now we have 0..4K hole and 4K..4G mapped. + * Remap 4K. We should get the first 4K chunk that was skipped + * over during the previous 8K map. If we missed a TLB invalidate + * at that point this should explode. + */ + dma_addr = dma_map_single(dev, virt, SZ_4K, DMA_TO_DEVICE); + if (dma_addr != 0) { + dma_addr_t expected = 0; + + dev_err_ratelimited(dev, "Unexpected dma_addr. got: %pa expected: %pa\n", + &dma_addr, &expected); + /* To simplify error handling, unmap the 4K region (4K..8K) + * here and rest (8K..4G) in 8K increments in the for loop. + */ + dma_unmap_single(dev, SZ_4K, SZ_4K, DMA_TO_DEVICE); + first_iova = SZ_8K; + nr_maps -= 1; + ret = -EINVAL; + goto out; + } + + first_iova = 0; + + if (dma_map_single(dev, virt, SZ_4K, DMA_TO_DEVICE) != DMA_MAPPING_ERROR) { + dev_err_ratelimited(dev, "dma_map_single unexpectedly after remaps (VA should have been exhausted)\n"); + ret = -EINVAL; + goto out; + } + +out: + /* we're all full again. unmap everything. */ + for (iova = first_iova; iova < max && nr_maps--; iova += SZ_8K) { + if (iova == MSI_IOVA_BASE) { + iova = MSI_IOVA_BASE + MSI_IOVA_LENGTH - SZ_8K; + continue; + } + dma_unmap_single(dev, (dma_addr_t)iova, SZ_8K, DMA_TO_DEVICE); + } + + free_pages((unsigned long)virt, get_order(SZ_8K)); + return ret; +} + +struct fib_state { + unsigned long cur; + unsigned long prev; +}; + +static void fib_init(struct fib_state *f) +{ + f->cur = f->prev = 1; +} + +static unsigned long get_next_fib(struct fib_state *f) +{ + int next = f->cur + f->prev; + + f->prev = f->cur; + f->cur = next; + return next; +} + +/* + * Not actually random. Just testing the fibs (and max - the fibs). + */ +static int __rand_va_sweep(struct device *dev, struct seq_file *s, + struct iommu_domain *domain, void *priv) +{ + u64 iova; + const u64 max = SZ_1G * 4ULL - 1; + int i, remapped, unmapped, ret = 0; + int nr_maps = 0; + void *virt; + dma_addr_t dma_addr, dma_addr2; + struct fib_state fib; + const size_t size = (size_t)priv; + + virt = (void *)__get_free_pages(GFP_KERNEL, get_order(size)); + if (!virt) { + if (size > SZ_8K) { + dev_err_ratelimited(dev, "Failed to allocate %s of memory, which is a lot. Skipping test for this size\n", + _size_to_string(size)); + return 0; + } + return -ENOMEM; + } + + /* fill the whole 4GB space */ + for (iova = 0, i = 0; iova < max; iova += size, ++i) { + if (iova == MSI_IOVA_BASE) { + iova = MSI_IOVA_BASE + MSI_IOVA_LENGTH - size; + continue; + } + + dma_addr = dma_map_single(dev, virt, size, DMA_TO_DEVICE); + if (dma_addr == DMA_MAPPING_ERROR) { + dev_err_ratelimited(dev, "Failed map on iter %d\n", i); + ret = -EINVAL; + goto out; + } else if (dma_addr != iova) { + dma_unmap_single(dev, dma_addr, size, DMA_TO_DEVICE); + dev_err_ratelimited(dev, "Unexpected dma_addr. got: %lx, expected: %lx\n", + (unsigned long)dma_addr, (unsigned long)iova); + ret = -EINVAL; + goto out; + } + nr_maps++; + } + + /* now unmap "random" iovas */ + unmapped = 0; + fib_init(&fib); + for (iova = get_next_fib(&fib) * size; + iova < max - size; + iova = (u64)get_next_fib(&fib) * size) { + dma_addr = (dma_addr_t)(iova); + dma_addr2 = (dma_addr_t)((max + 1) - size - iova); + if (dma_addr == dma_addr2) { + WARN(1, "%s test needs update! The random number sequence is folding in on itself and should be changed.\n", + __func__); + return -EINVAL; + } + + if (!(MSI_IOVA_BASE <= dma_addr && MSI_IOVA_BASE + MSI_IOVA_LENGTH > dma_addr)) + dma_unmap_single(dev, dma_addr, size, DMA_TO_DEVICE); + + if (!(MSI_IOVA_BASE <= dma_addr2 && MSI_IOVA_BASE + MSI_IOVA_LENGTH > dma_addr2)) + dma_unmap_single(dev, dma_addr2, size, DMA_TO_DEVICE); + + unmapped += 2; + } + + /* and map until everything fills back up */ + for (remapped = 0; ; ++remapped) { + dma_addr = dma_map_single(dev, virt, size, DMA_TO_DEVICE); + if (dma_addr == DMA_MAPPING_ERROR) + break; + } + + if (unmapped != remapped) { + dev_err_ratelimited(dev, "Unexpected random remap count! Unmapped %d but remapped %d\n", + unmapped, remapped); + ret = -EINVAL; + } + +out: + for (iova = 0; iova < max && nr_maps--; iova += size) { + if (iova == MSI_IOVA_BASE) { + iova = MSI_IOVA_BASE + MSI_IOVA_LENGTH - size; + continue; + } + dma_unmap_single(dev, (dma_addr_t)iova, size, DMA_TO_DEVICE); + } + + free_pages((unsigned long)virt, get_order(size)); + return ret; +} + +static int __functional_dma_api_va_test(struct seq_file *s) +{ + int ret = 0; + size_t *sz; + size_t sizes[] = {SZ_4K, SZ_8K, SZ_16K, SZ_64K, 0}; + struct iommu_debug_device *ddev = s->private; + char *usecase_name; + + /* + * dev_name() cannot be used to get the usecase name as ddev->test_dev + * will be NULL in case __apply_to_new_mapping() fails. Since + * ddev->test_dev changes across calls to __apply_to_new_mapping(), we + * also can't hold a reference to its name by caching the result of + * dev_name() initially. + */ + mutex_lock(&ddev->state_lock); + if (!ddev->test_dev) { + mutex_unlock(&ddev->state_lock); + return -ENODEV; + } + + usecase_name = kstrdup(dev_name(ddev->test_dev), GFP_KERNEL); + mutex_unlock(&ddev->state_lock); + if (!usecase_name) + return -ENOMEM; + + for (sz = sizes; *sz; ++sz) { + ps_printf(usecase_name, s, "Full VA sweep @%s:", _size_to_string(*sz)); + if (__apply_to_new_mapping(s, __full_va_sweep, (void *)*sz)) { + ps_printf(usecase_name, s, " -> FAILED\n"); + ret = -EINVAL; + } else { + ps_printf(usecase_name, s, " -> SUCCEEDED\n"); + } + } + + ps_printf(usecase_name, s, "bonus map:"); + if (__apply_to_new_mapping(s, __full_va_sweep, (void *)SZ_4K)) { + ps_printf(usecase_name, s, " -> FAILED\n"); + ret = -EINVAL; + } else { + ps_printf(usecase_name, s, " -> SUCCEEDED\n"); + } + + for (sz = sizes; *sz; ++sz) { + ps_printf(usecase_name, s, "Rand VA sweep @%s:", _size_to_string(*sz)); + if (__apply_to_new_mapping(s, __rand_va_sweep, (void *)*sz)) { + ps_printf(usecase_name, s, " -> FAILED\n"); + ret = -EINVAL; + } else { + ps_printf(usecase_name, s, " -> SUCCEEDED\n"); + } + } + + ps_printf(usecase_name, s, "TLB stress sweep:"); + if (__apply_to_new_mapping(s, __tlb_stress_sweep, NULL)) { + ps_printf(usecase_name, s, " -> FAILED\n"); + ret = -EINVAL; + } else { + ps_printf(usecase_name, s, " -> SUCCEEDED\n"); + } + + ps_printf(usecase_name, s, "second bonus map:"); + if (__apply_to_new_mapping(s, __full_va_sweep, (void *)SZ_4K)) { + ps_printf(usecase_name, s, " -> FAILED\n"); + ret = -EINVAL; + } else { + ps_printf(usecase_name, s, " -> SUCCEEDED\n"); + } + + kfree(usecase_name); + return ret; +} + +static int iommu_debug_functional_fast_dma_api_show(struct seq_file *s, + void *ignored) +{ + int ret = 0; + struct iommu_debug_device *ddev = s->private; + + if (!ddev->test_dev) { + pr_err("%s:Have you selected a uscase?\n", __func__); + return -EINVAL; + } + + if (!ddev->fastmap_usecase) { + ps_printf(dev_name(ddev->test_dev), s, + "Not a fastmap usecase\n"); + return 0; + } else if (!IS_ENABLED(CONFIG_IOMMU_IO_PGTABLE_FAST)) { + ps_printf(dev_name(ddev->test_dev), s, + "CONFIG_IOMMU_IO_PGTABLE_FAST not enabled\n"); + return 0; + } + + ret |= __apply_to_new_mapping(s, __functional_dma_api_alloc_test, NULL); + ret |= __apply_to_new_mapping(s, __functional_dma_api_basic_test, NULL); + ret |= __functional_dma_api_va_test(s); + return ret; +} + +static int iommu_debug_functional_fast_dma_api_open(struct inode *inode, + struct file *file) +{ + return single_open(file, iommu_debug_functional_fast_dma_api_show, + inode->i_private); +} + +const struct file_operations iommu_debug_functional_fast_dma_api_fops = { + .open = iommu_debug_functional_fast_dma_api_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +static ssize_t iommu_debug_test_virt_addr_read(struct file *file, + char __user *ubuf, + size_t count, loff_t *offset) +{ + char buf[100]; + int len; + + if (*offset) + return 0; + + mutex_lock(&test_virt_addr_lock); + if (IS_ERR_OR_NULL(test_virt_addr)) + test_virt_addr = kzalloc(SZ_1M, GFP_KERNEL); + + if (!test_virt_addr) { + test_virt_addr = ERR_PTR(-ENOMEM); + len = strscpy(buf, "FAIL\n", sizeof(buf)); + } else { + len = scnprintf(buf, sizeof(buf), "0x%p\n", test_virt_addr); + } + mutex_unlock(&test_virt_addr_lock); + + return simple_read_from_buffer(ubuf, count, offset, buf, len); +} + +const struct file_operations iommu_debug_test_virt_addr_fops = { + .open = simple_open, + .read = iommu_debug_test_virt_addr_read, +}; + +#ifdef CONFIG_IOMMU_IOVA_ALIGNMENT +static unsigned long iommu_debug_get_align_mask(size_t size) +{ + unsigned long align_mask = ~0UL; + + align_mask <<= min_t(unsigned long, CONFIG_IOMMU_IOVA_ALIGNMENT + PAGE_SHIFT, + fls_long(size - 1)); + return ~align_mask; +} +#else +static unsigned long iommu_debug_get_align_mask(size_t size) +{ + unsigned long align_mask = ~0UL; + + align_mask <<= fls_long(size - 1); + return ~align_mask; +} +#endif + +static void iommu_debug_device_profiling(struct seq_file *s, struct iommu_debug_device *ddev, + const size_t sizes[]) +{ + const size_t *sz; + struct iommu_domain *domain; + struct device *dev; + unsigned long iova = 0x10000; + phys_addr_t paddr = 0x80000000; + + mutex_lock(&ddev->state_lock); + if (!iommu_debug_usecase_reset(ddev)) + goto out; + + domain = ddev->domain; + dev = ddev->test_dev; + + seq_printf(s, "(average over %d iterations)\n", ddev->nr_iters); + seq_printf(s, "%8s %19s %16s\n", "size", "iommu_map", "iommu_unmap"); + for (sz = sizes; *sz; ++sz) { + size_t size = *sz; + size_t unmapped; + u64 map_elapsed_ns = 0, unmap_elapsed_ns = 0; + u64 map_elapsed_us = 0, unmap_elapsed_us = 0; + u32 map_elapsed_rem = 0, unmap_elapsed_rem = 0; + ktime_t tbefore, tafter, diff; + int i; + unsigned long align_mask = iommu_debug_get_align_mask(size); + + for (i = 0; i < ddev->nr_iters; ++i) { + tbefore = ktime_get(); + if (iommu_map(domain, __ALIGN_MASK(iova, align_mask), + ALIGN(paddr, size), size, + IOMMU_READ | IOMMU_WRITE, GFP_KERNEL)) { + seq_puts(s, "Failed to map\n"); + continue; + } + tafter = ktime_get(); + diff = ktime_sub(tafter, tbefore); + map_elapsed_ns += ktime_to_ns(diff); + + tbefore = ktime_get(); + unmapped = iommu_unmap(domain, + __ALIGN_MASK(iova, align_mask), + size); + if (unmapped != size) { + seq_printf(s, + "Only unmapped %zx instead of %zx\n", + unmapped, size); + continue; + } + tafter = ktime_get(); + diff = ktime_sub(tafter, tbefore); + unmap_elapsed_ns += ktime_to_ns(diff); + } + + map_elapsed_ns = div_u64_rem(map_elapsed_ns, ddev->nr_iters, &map_elapsed_rem); + unmap_elapsed_ns = div_u64_rem(unmap_elapsed_ns, ddev->nr_iters, + &unmap_elapsed_rem); + + map_elapsed_us = div_u64_rem(map_elapsed_ns, 1000, &map_elapsed_rem); + unmap_elapsed_us = div_u64_rem(unmap_elapsed_ns, 1000, &unmap_elapsed_rem); + + seq_printf(s, "%8s %12lld.%03d us %9lld.%03d us\n", + _size_to_string(size), map_elapsed_us, map_elapsed_rem, + unmap_elapsed_us, unmap_elapsed_rem); + } + + seq_putc(s, '\n'); + seq_printf(s, "%8s %19s %16s\n", "size", "iommu_map_sg", "iommu_unmap"); + for (sz = sizes; *sz; ++sz) { + size_t size = *sz; + size_t unmapped; + u64 map_elapsed_ns = 0, unmap_elapsed_ns = 0; + u64 map_elapsed_us = 0, unmap_elapsed_us = 0; + u32 map_elapsed_rem = 0, unmap_elapsed_rem = 0; + ktime_t tbefore, tafter, diff; + struct sg_table table; + unsigned long chunk_size = SZ_4K; + int i; + unsigned long align_mask = iommu_debug_get_align_mask(size); + + if (iommu_debug_build_phoney_sg_table(dev, &table, size, + chunk_size)) { + seq_puts(s, "couldn't build phoney sg table! bailing...\n"); + goto out; + } + + for (i = 0; i < ddev->nr_iters; ++i) { + tbefore = ktime_get(); + if (iommu_map_sgtable(domain, __ALIGN_MASK(iova, align_mask), + &table, IOMMU_READ | IOMMU_WRITE) != size) { + seq_puts(s, "Failed to map_sg\n"); + goto next; + } + tafter = ktime_get(); + diff = ktime_sub(tafter, tbefore); + map_elapsed_ns += ktime_to_ns(diff); + + tbefore = ktime_get(); + unmapped = iommu_unmap(domain, + __ALIGN_MASK(iova, align_mask), + size); + if (unmapped != size) { + seq_printf(s, "Only unmapped %zx instead of %zx\n", + unmapped, size); + goto next; + } + tafter = ktime_get(); + diff = ktime_sub(tafter, tbefore); + unmap_elapsed_ns += ktime_to_ns(diff); + } + + map_elapsed_ns = div_u64_rem(map_elapsed_ns, ddev->nr_iters, &map_elapsed_rem); + unmap_elapsed_ns = div_u64_rem(unmap_elapsed_ns, ddev->nr_iters, + &unmap_elapsed_rem); + + map_elapsed_us = div_u64_rem(map_elapsed_ns, 1000, &map_elapsed_rem); + unmap_elapsed_us = div_u64_rem(unmap_elapsed_ns, 1000, &unmap_elapsed_rem); + + seq_printf(s, "%8s %12lld.%03d us %9lld.%03d us\n", _size_to_string(size), + map_elapsed_us, map_elapsed_rem, unmap_elapsed_us, unmap_elapsed_rem); + +next: + iommu_debug_destroy_phoney_sg_table(dev, &table, chunk_size); + } + +out: + mutex_unlock(&ddev->state_lock); +} + +static int iommu_debug_profiling_show(struct seq_file *s, void *ignored) +{ + struct iommu_debug_device *ddev = s->private; + const size_t sizes[] = { SZ_4K, SZ_64K, SZ_1M, SZ_2M, SZ_1M * 12, + SZ_1M * 24, SZ_1M * 32, 0 }; + + iommu_debug_device_profiling(s, ddev, sizes); + return 0; +} + +static int iommu_debug_profiling_open(struct inode *inode, struct file *file) +{ + return single_open(file, iommu_debug_profiling_show, inode->i_private); +} + +const struct file_operations iommu_debug_profiling_fops = { + .open = iommu_debug_profiling_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; diff --git a/drivers/iommu/qcom-iommu-debug.c b/drivers/iommu/qcom-iommu-debug.c new file mode 100644 index 0000000000000..1838405561999 --- /dev/null +++ b/drivers/iommu/qcom-iommu-debug.c @@ -0,0 +1,400 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2015-2021, The Linux Foundation. All rights reserved. + * Copyright (c) 2022-2023 Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#define pr_fmt(fmt) "iommu-debug: %s: " fmt, __func__ + +#include +#include +#include +#include +#include +#include +#include "qcom-iommu-debug.h" +#include + +#define USECASE_SWITCH_TIMEOUT_MSECS (500) + +static int iommu_debug_nr_iters_set(void *data, u64 val) +{ + struct iommu_debug_device *ddev = data; + + if (!val) + val = 1; + + if (val > 10000) + val = 10000; + + ddev->nr_iters = (u32)val; + + return 0; +} + +static int iommu_debug_nr_iters_get(void *data, u64 *val) +{ + struct iommu_debug_device *ddev = data; + + *val = ddev->nr_iters; + + return 0; +} + +DEFINE_DEBUGFS_ATTRIBUTE(iommu_debug_nr_iters_fops, + iommu_debug_nr_iters_get, + iommu_debug_nr_iters_set, + "%llu\n"); + +int iommu_debug_check_mapping_flags(struct device *dev, dma_addr_t iova, size_t size, + phys_addr_t expected_pa, u32 flags) +{ + struct qcom_iommu_atos_txn txn; + struct iommu_fwspec *fwspec; + struct iommu_domain *domain; + + domain = iommu_get_domain_for_dev(dev); + if (!domain) { + dev_err(dev, "iommu_get_domain_for_dev() failed\n"); + return -EINVAL; + } + + fwspec = dev_iommu_fwspec_get(dev); + if (!fwspec) { + dev_err(dev, "dev_iommu_fwspec_get() failed\n"); + return -EINVAL; + } + + txn.addr = iova; + txn.id = FIELD_GET(ARM_SMMU_SMR_ID, (fwspec->ids[0])); + txn.flags = flags; + + size = PAGE_ALIGN(size); + while (size) { + phys_addr_t walk_pa, atos_pa; + + atos_pa = qcom_iommu_iova_to_phys_hard(domain, &txn); + walk_pa = iommu_iova_to_phys(domain, iova); + + if (expected_pa != atos_pa || expected_pa != walk_pa) { + dev_err_ratelimited(dev, + "Bad translation for %pad! Expected: %pa Got: %pa (ATOS) %pa (Table Walk) sid=%08x\n", + &iova, &expected_pa, &atos_pa, &walk_pa, txn.id); + return -EINVAL; + } + + size -= PAGE_SIZE; + iova += PAGE_SIZE; + expected_pa += PAGE_SIZE; + } + + return 0; +} + +int iommu_debug_check_mapping_sg_flags(struct device *dev, struct scatterlist *sgl, + unsigned int pgoffset, unsigned int dma_nents, + unsigned int nents, u32 flags) +{ + int ret; + struct sg_page_iter piter; + struct sg_dma_page_iter diter; + + for (__sg_page_iter_start(&piter, sgl, nents, pgoffset), + __sg_page_iter_start(&diter.base, sgl, dma_nents, pgoffset); + __sg_page_iter_next(&piter) && __sg_page_iter_dma_next(&diter);) { + + struct page *page = sg_page_iter_page(&piter); + dma_addr_t dma_addr = sg_page_iter_dma_address(&diter); + + ret = iommu_debug_check_mapping_flags(dev, dma_addr, PAGE_SIZE, + page_to_phys(page), flags); + if (ret) + return ret; + } + + return 0; +} + +static void iommu_debug_destroy_test_dev(struct iommu_debug_device *ddev) +{ + if (ddev->test_dev) { + of_platform_device_destroy(ddev->test_dev, NULL); + ddev->test_dev = NULL; + ddev->domain = NULL; + } +} + +/* + * Returns struct device corresponding to the new usecase. + * ddev->test_dev will change - caller must not use old value! + * Caller must hold ddev->state_lock + */ +struct device * +iommu_debug_switch_usecase(struct iommu_debug_device *ddev, u32 usecase_nr) +{ + struct platform_device *pdev; + struct device_node *child; + const char *str; + int child_nr = 0; + int ret; + + if (ddev->test_dev) + iommu_debug_destroy_test_dev(ddev); + + if (usecase_nr >= of_get_child_count(ddev->self->of_node)) { + dev_err(ddev->self, "Invalid usecase nr requested: %u\n", + usecase_nr); + return NULL; + } + + reinit_completion(&ddev->probe_wait); + for_each_child_of_node(ddev->self->of_node, child) { + if (child_nr == usecase_nr) + break; + child_nr++; + } + + pdev = of_device_alloc(child, NULL, ddev->self); + if (!pdev) + goto out; + + pdev->dev.coherent_dma_mask = DMA_BIT_MASK(32); + if (!pdev->dev.dma_mask) + pdev->dev.dma_mask = &pdev->dev.coherent_dma_mask; + pdev->dev.bus = &platform_bus_type; + pdev->dev.platform_data = NULL; + //of_msi_configure(&pdev->dev, pdev->dev.of_node); + + dev_set_name(&pdev->dev, "secure-pixel"); + if (of_device_add(pdev) != 0) { + platform_device_put(pdev); + goto out; + } + + /* + * Wait for child device's probe function to be called. + * Its very unlikely to be asynchonrous... + */ + ret = wait_for_completion_interruptible_timeout(&ddev->probe_wait, + msecs_to_jiffies(USECASE_SWITCH_TIMEOUT_MSECS)); + if (ret <= 0) { + dev_err(ddev->self, "Timed out waiting for usecase to register\n"); + goto out; + } + + if (of_property_read_string(child, "qcom,iommu-dma", &str)) + str = "default"; + + ddev->fastmap_usecase = !strcmp(str, "fastmap"); + ddev->usecase_nr = usecase_nr; + ddev->test_dev = &pdev->dev; + ddev->domain = iommu_get_domain_for_dev(ddev->test_dev); + if (!ddev->domain) { + dev_err(ddev->self, "Oops, usecase not associated with iommu\n"); + goto out; + } + + return ddev->test_dev; +out: + iommu_debug_destroy_test_dev(ddev); + return NULL; +} + +/* + * Caller must hold ddev->state_lock + */ +struct device *iommu_debug_usecase_reset(struct iommu_debug_device *ddev) +{ + return iommu_debug_switch_usecase(ddev, ddev->usecase_nr); +} + +static int iommu_debug_usecase_register(struct device *dev) +{ + struct iommu_debug_device *ddev = dev_get_drvdata(dev->parent); + + complete(&ddev->probe_wait); + return 0; +} + +static ssize_t iommu_debug_usecase_read(struct file *file, char __user *ubuf, + size_t count, loff_t *offset) +{ + struct iommu_debug_device *ddev = file->private_data; + + return simple_read_from_buffer(ubuf, count, offset, ddev->buffer, + strnlen(ddev->buffer, PAGE_SIZE)); +} + +static ssize_t iommu_debug_usecase_write(struct file *file, const char __user *ubuf, + size_t count, loff_t *offset) +{ + struct iommu_debug_device *ddev = file->private_data; + unsigned int usecase_nr; + int ret; + + ret = kstrtouint_from_user(ubuf, count, 0, &usecase_nr); + if (ret || usecase_nr >= ddev->nr_children) + return -EINVAL; + + mutex_lock(&ddev->state_lock); + if (!iommu_debug_switch_usecase(ddev, usecase_nr)) { + mutex_unlock(&ddev->state_lock); + return -EINVAL; + } + mutex_unlock(&ddev->state_lock); + + return count; +} + +static const struct file_operations iommu_debug_usecase_fops = { + .open = simple_open, + .read = iommu_debug_usecase_read, + .write = iommu_debug_usecase_write, + //.llseek = no_llseek, +}; + +static int iommu_debug_debugfs_setup(struct iommu_debug_device *ddev) +{ + struct dentry *dir; + + dir = debugfs_create_dir("iommu-test", NULL); + if (IS_ERR(dir)) + return -EINVAL; + + ddev->root_dir = dir; + + debugfs_create_file("usecase", 0600, dir, ddev, &iommu_debug_usecase_fops); + debugfs_create_file("functional_arm_dma_api", 0400, dir, ddev, + &iommu_debug_functional_arm_dma_api_fops); + debugfs_create_file("functional_fast_dma_api", 0400, dir, ddev, + &iommu_debug_functional_fast_dma_api_fops); + debugfs_create_file("atos", 0600, dir, ddev, &iommu_debug_atos_fops); + debugfs_create_file("map", 0200, dir, ddev, &iommu_debug_map_fops); + debugfs_create_file("unmap", 0200, dir, ddev, &iommu_debug_unmap_fops); + debugfs_create_file("dma_map", 0200, dir, ddev, &iommu_debug_dma_map_fops); + debugfs_create_file("dma_unmap", 0200, dir, ddev, &iommu_debug_dma_unmap_fops); + debugfs_create_file("nr_iters", 0600, dir, ddev, &iommu_debug_nr_iters_fops); + debugfs_create_file("test_virt_addr", 0400, dir, ddev, &iommu_debug_test_virt_addr_fops); + debugfs_create_file("profiling", 0400, dir, ddev, &iommu_debug_profiling_fops); + + return 0; +} + +static int iommu_debug_probe(struct platform_device *pdev) +{ + struct iommu_debug_device *ddev; + struct device *dev = &pdev->dev; + struct device_node *child; + int ret; + int offset = 0; + + ddev = devm_kzalloc(dev, sizeof(*ddev), GFP_KERNEL); + if (!ddev) + return -ENOMEM; + + ddev->self = dev; + ddev->usecase_nr = U32_MAX; + ddev->nr_iters = 1; + mutex_init(&ddev->state_lock); + init_completion(&ddev->probe_wait); + + ddev->buffer = devm_kzalloc(dev, PAGE_SIZE, GFP_KERNEL); + if (!ddev->buffer) { + ret = -ENOMEM; + goto out; + } + + ddev->nr_children = 0; + for_each_child_of_node(dev->of_node, child) { + offset += scnprintf(ddev->buffer + offset, PAGE_SIZE - offset, + "%d: %s\n", ddev->nr_children, child->name); + if (offset + 1 == PAGE_SIZE) { + dev_err(dev, "Too many testcases?\n"); + break; + } + ddev->nr_children++; + } + dev_set_drvdata(dev, ddev); + + ret = iommu_debug_debugfs_setup(ddev); + if (ret) + goto out; + + return 0; + +out: + mutex_destroy(&ddev->state_lock); + return ret; +} + +static void iommu_debug_remove(struct platform_device *pdev) +{ + struct iommu_debug_device *ddev = platform_get_drvdata(pdev); + + debugfs_remove_recursive(ddev->root_dir); + if (ddev->test_dev) + of_platform_device_destroy(ddev->test_dev, NULL); + + mutex_destroy(&ddev->state_lock); +} + +static const struct of_device_id iommu_debug_of_match[] = { + { .compatible = "qcom,iommu-debug-test" }, + { }, +}; + +static struct platform_driver iommu_debug_driver = { + .probe = iommu_debug_probe, + .remove = iommu_debug_remove, + .driver = { + .name = "qcom-iommu-debug", + .of_match_table = iommu_debug_of_match, + }, +}; + +/* + * This isn't really a "driver", we just need something in the device tree + * to hook up to the `iommus' property. + */ +static int iommu_debug_usecase_probe(struct platform_device *pdev) +{ + return iommu_debug_usecase_register(&pdev->dev); +} + +static const struct of_device_id iommu_debug_usecase_of_match[] = { + { .compatible = "qcom,iommu-debug-usecase" }, + { }, +}; + +static struct platform_driver iommu_debug_usecase_driver = { + .probe = iommu_debug_usecase_probe, + .driver = { + .name = "qcom-iommu-debug-usecase", + .of_match_table = iommu_debug_usecase_of_match, + }, +}; + +static int iommu_debug_init(void) +{ + int ret; + + ret = platform_driver_register(&iommu_debug_driver); + if (ret) + return ret; + + ret = platform_driver_register(&iommu_debug_usecase_driver); + if (ret) + platform_driver_unregister(&iommu_debug_driver); + return ret; +} + +static void iommu_debug_exit(void) +{ + platform_driver_unregister(&iommu_debug_usecase_driver); + platform_driver_unregister(&iommu_debug_driver); +} + +module_init(iommu_debug_init); +module_exit(iommu_debug_exit); + +MODULE_LICENSE("GPL"); diff --git a/drivers/iommu/qcom-iommu-debug.h b/drivers/iommu/qcom-iommu-debug.h new file mode 100644 index 0000000000000..3c47de41fd9a6 --- /dev/null +++ b/drivers/iommu/qcom-iommu-debug.h @@ -0,0 +1,76 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2015-2021, The Linux Foundation. All rights reserved. + * Copyright (c) 2022-2023, Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#ifndef __DRIVERS_IOMMU_QCOM_IOMMU_DEBUG_H__ +#define __DRIVERS_IOMMU_QCOM_IOMMU_DEBUG_H__ + +#include +#include +#include +#include +#include + +#define MSI_IOVA_BASE 0x8000000 +#define MSI_IOVA_LENGTH 0x100000 +#define ARM_SMMU_SMR_ID GENMASK(15, 0) + +struct iommu_debug_device { + struct device *self; + u32 nr_children; + char *buffer; + struct dentry *root_dir; + /* for usecase under test */ + struct device *test_dev; + struct iommu_domain *domain; + u32 usecase_nr; + bool fastmap_usecase; + /* Protects test_dev */ + struct mutex state_lock; + /* For waiting for child probe to complete */ + struct completion probe_wait; + /* Used for atos */ + u64 iova; + /* number of iterations */ + u32 nr_iters; +}; + +struct device *iommu_debug_usecase_reset(struct iommu_debug_device *ddev); +struct device *iommu_debug_switch_usecase(struct iommu_debug_device *ddev, u32 usecase_nr); + +int iommu_debug_check_mapping_flags(struct device *dev, dma_addr_t iova, size_t size, + phys_addr_t expected_pa, u32 flags); +#define iommu_debug_check_mapping(d, i, s, p) \ + iommu_debug_check_mapping_flags(d, i, s, p, 0) +/* Only checks a single page */ +#define iommu_debug_check_mapping_fast(d, i, s, p) \ + iommu_debug_check_mapping_flags(d, i, PAGE_SIZE, p, 0) + +int iommu_debug_check_mapping_sg_flags(struct device *dev, struct scatterlist *sgl, + unsigned int pgoffset, unsigned int dma_nents, + unsigned int nents, u32 flags); +#define iommu_debug_check_mapping_sg(d, s, o, e1, e2) \ + iommu_debug_check_mapping_sg_flags(d, s, o, e1, e2, 0) + +/* Only checks the last page of first sgl */ +static inline int iommu_debug_check_mapping_sg_fast(struct device *dev, struct scatterlist *sgl, + unsigned int pgoffset, unsigned int dma_nents, + unsigned int nents) +{ + pgoffset = PAGE_ALIGN(sgl->offset + sgl->length) >> PAGE_SHIFT; + return iommu_debug_check_mapping_sg_flags(dev, sgl, pgoffset - 1, dma_nents, 1, 0); +} + +extern const struct file_operations iommu_debug_functional_arm_dma_api_fops; +extern const struct file_operations iommu_debug_functional_fast_dma_api_fops; +extern const struct file_operations iommu_debug_atos_fops; +extern const struct file_operations iommu_debug_map_fops; +extern const struct file_operations iommu_debug_unmap_fops; +extern const struct file_operations iommu_debug_dma_map_fops; +extern const struct file_operations iommu_debug_dma_unmap_fops; +extern const struct file_operations iommu_debug_test_virt_addr_fops; +extern const struct file_operations iommu_debug_profiling_fops; + +#endif diff --git a/drivers/iommu/qcom-iommu-util.c b/drivers/iommu/qcom-iommu-util.c new file mode 100644 index 0000000000000..645856e32fbb9 --- /dev/null +++ b/drivers/iommu/qcom-iommu-util.c @@ -0,0 +1,826 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Portions based off of __alloc_and_insert_iova_range() implementation + * in drivers/iommu/iova.c: + * Author: Anil S Keshavamurthy + * Copyright © 2006-2009, Intel Corporation. + * + * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved. + * Copyright (c) 2022-2024 Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include "drivers/iommu/iommu-priv.h" +#include +#include +#include "qcom-dma-iommu-generic.h" +#include "qcom-io-pgtable-alloc.h" + +static DEFINE_XARRAY(xa_qcom_iommu_ops); + +struct qcom_iommu_range_prop_cb_data { + int (*range_prop_entry_cb_fn)(const __be32 *p, int naddr, int nsize, void *arg); + void *arg; +}; + +struct iova_range { + u64 base; + u64 end; +}; + +struct device_node *qcom_iommu_group_parse_phandle(struct device *dev) +{ + struct device_node *np; + + if (!dev->of_node) + return NULL; + + np = of_parse_phandle(dev->of_node, "qcom,iommu-group", 0); + return np ? np : dev->of_node; +} + +static int of_property_walk_each_entry(struct device *dev, const char *propname, + struct qcom_iommu_range_prop_cb_data *cb_data) +{ + struct device_node *np; + const __be32 *p, *property_end; + int ret, len, naddr, nsize; + + np = qcom_iommu_group_parse_phandle(dev); + if (!np) + return -EINVAL; + + p = of_get_property(np, propname, &len); + if (!p) + return -ENODEV; + + len /= sizeof(u32); + naddr = of_n_addr_cells(np); + nsize = of_n_size_cells(np); + if (!naddr || !nsize || len % (naddr + nsize)) { + dev_err(dev, "%s Invalid length %d. Address cells %d. Size cells %d\n", + propname, len, naddr, nsize); + return -EINVAL; + } + property_end = p + len; + + while (p < property_end) { + ret = cb_data->range_prop_entry_cb_fn(p, naddr, nsize, cb_data->arg); + if (ret) + return ret; + + p += naddr + nsize; + } + + return 0; +} + +void qcom_iommu_get_resv_regions(struct device *dev, struct list_head *list) +{ + const struct iommu_ops *ops = dev_iommu_ops(dev); + + if (ops && ops->get_resv_regions) + ops->get_resv_regions(dev, list); +} +EXPORT_SYMBOL(qcom_iommu_get_resv_regions); + +static int get_addr_range(const __be32 *p, int naddr, int nsize, void *arg) +{ + + u64 start = of_read_number(p, naddr); + u64 end = start + of_read_number(p + naddr, nsize) - 1; + struct iova_range *range = arg; + + if (start >= SZ_4G || end >= SZ_4G) { + pr_err("fastmap does not support IOVAs >= 4 GB\n"); + return -EINVAL; + } + + range->base = min_not_zero(range->base, start); + range->end = max(range->end, end); + + return 0; +} + +/* + * similar to list_for_each_entry_safe_reverse from include/linux/list.h, + * except starting position is prior to pos, instead of the list tail. + */ +#define list_for_each_entry_safe_continue_reverse(pos, n, head, member) \ + for (pos = list_prev_entry(pos, member), \ + n = list_prev_entry(pos, member); \ + !list_entry_is_head(pos, head, member); \ + pos = n, n = list_prev_entry(n, member)) + +/* + * "new" has been added to list in sorted order, but may overlap with preceding + * or following entries. Merge as required, deleting old nodes. + * + * A valid region has following properties: + * A.length > 0 + * A.end == A.length + A.start - 1 does not overflow or underflow. + * A.end is in range [0, U64_MAX] + * A.start is in range [0, U64_MAX] + * + * This causes an issue when checking if A.start is adjacent to B.end: + * B.end + 1 can overflow, and A.start - 1 can underflow. + * + * This is resolved by short ciruiting the comparison if A.start == 0. + */ +static void merge_resv_region(struct list_head *list, struct iommu_resv_region *new) +{ + struct iommu_resv_region *cur, *tmp; + u64 new_end, cur_end, end, start; + + /* Merge against entries with smaller start address */ + cur = new; + list_for_each_entry_safe_continue_reverse(cur, tmp, list, list) { + new_end = new->start + new->length - 1; + cur_end = cur->start + cur->length - 1; + if (new->start && new->start - 1 > cur_end) + break; + + start = min(new->start, cur->start); + end = max(new_end, cur_end); + pr_debug("%s: Merging %llx-%llx into %llx-%llx\n", + __func__, new->start, new_end, cur->start, cur_end); + + list_del(&cur->list); + kfree(cur); + new->start = start; + new->length = end + 1 - new->start; + } + + /* Merge against entries with greater start address */ + cur = new; + list_for_each_entry_safe_continue(cur, tmp, list, list) { + new_end = new->start + new->length - 1; + cur_end = cur->start + cur->length - 1; + if (cur->start && new_end < cur->start - 1) + break; + + start = min(new->start, cur->start); + end = max(new_end, cur_end); + pr_debug("%s: Merging %llx-%llx into %llx-%llx\n", + __func__, new->start, new_end, cur->start, cur_end); + + list_del(&cur->list); + kfree(cur); + new->start = start; + new->length = end + 1 - new->start; + } +} + +/* + * On success, dma_range contains a single range which spans all usable addresses. + * Returns -ENODEV if property not present, or another negative value on error + */ +static int get_iova_range_from_iommu_addresses(struct device *dev, struct iova_range *dma_range, + u64 fastmap_max_iova) +{ + LIST_HEAD(list); + LIST_HEAD(mergelist); + struct iommu_resv_region *new, *region, *x; + int ret = -EINVAL; + bool found = false; + struct of_phandle_iterator it; + + of_for_each_phandle(&it, ret, dev->of_node, "memory-region", NULL, 0) { + if (of_find_property(it.node, "iommu-addresses", NULL)) { + found = true; + break; + } + } + if (!found) + return -ENODEV; + + qcom_iommu_get_resv_regions(dev, &list); + if (list_empty(&list)) { + ret = -EINVAL; + goto out; + } + + list_for_each_entry(region, &list, list) { + new = kmemdup(region, sizeof(*region), GFP_KERNEL); + if (!new) { + ret = -ENOMEM; + goto out; + } + + list_for_each_entry(x, &mergelist, list) + if (x->start > new->start) + break; + list_add_tail(&new->list, &x->list); + merge_resv_region(&mergelist, new); + } + + pr_debug("%s: Sorted & Merged iommu-address regions\n", __func__); + list_for_each_entry(x, &mergelist, list) + pr_debug("%s: %llx-%llx\n", __func__, x->start, x->start + x->length - 1); + + region = list_first_entry(&mergelist, struct iommu_resv_region, list); + dma_range->base = 0; + if (region->start == 0) + dma_range->base = region->start + region->length; + + region = list_last_entry(&mergelist, struct iommu_resv_region, list); + dma_range->end = fastmap_max_iova; + if (region->start <= fastmap_max_iova && + region->start + region->length - 1 >= fastmap_max_iova) + dma_range->end = region->start - 1; + + pr_debug("%s: Result: %llx-%llx\n", __func__, dma_range->base, dma_range->end); + ret = 0; + +out: + list_for_each_entry_safe(region, x, &mergelist, list) { + list_del(®ion->list); + kfree(region); + } + iommu_put_resv_regions(dev, &list); + return ret; +} + +int qcom_iommu_get_fast_iova_range(struct device *dev, dma_addr_t *ret_iova_base, + dma_addr_t *ret_iova_end) +{ + struct iova_range dma_range = {}; + struct iova_range geometry_range = {}; + struct qcom_iommu_range_prop_cb_data get_addr_range_cb_data = { + .range_prop_entry_cb_fn = get_addr_range, + }; + int ret; + u64 fastmap_max_iova = SZ_4G - 1; + struct device_node *np; + + np = qcom_iommu_group_parse_phandle(dev); + if (!np) + return -EINVAL; + + if (!dev || !ret_iova_base || !ret_iova_end) + return -EINVAL; + + get_addr_range_cb_data.arg = &dma_range; + + /* + * Legacy property - should be removed post kernel version 6.6. + */ + if (of_property_present(np, "qcom,iommu-dma-addr-pool")) { + WARN(1, "qcom,iommu-dma-addr-pool is deprecated. Switch to using iommu-addresses."); + return -EINVAL; + } + + ret = get_iova_range_from_iommu_addresses(dev, &dma_range, + fastmap_max_iova); + if (ret && ret != -ENODEV) { + dev_err(dev, "Parsing iommu-addresses into set failed with %d\n", ret); + return ret; + } else if (ret) { + dma_range.base = 0; + dma_range.end = fastmap_max_iova; + } + + get_addr_range_cb_data.arg = &geometry_range; + ret = of_property_walk_each_entry(dev, "qcom,iommu-geometry", + &get_addr_range_cb_data); + if (ret == -ENODEV) { + geometry_range.base = 0; + geometry_range.end = fastmap_max_iova; + } else if (ret) { + return ret; + } + + *ret_iova_base = min(geometry_range.base, dma_range.base); + *ret_iova_end = max(geometry_range.end, dma_range.end); + return 0; +} +EXPORT_SYMBOL(qcom_iommu_get_fast_iova_range); + +/* + * Caller should check for NULL. Will return NULL for identity domains on + * kernel 6.8+ + */ +static struct qcom_iommu_ops *to_qcom_iommu_ops(const struct iommu_domain_ops *ops) +{ + return xa_load(&xa_qcom_iommu_ops, (unsigned long)ops); +} + +int register_qcom_iommu_ops(struct qcom_iommu_ops *ops) +{ + void *ret; + + ret = xa_store(&xa_qcom_iommu_ops, (unsigned long)&ops->domain_ops, + ops, GFP_KERNEL); + if (xa_is_err(ret)) { + pr_err("Failed to register qcom_iommu_ops\n"); + return xa_err(ret); + } + return 0; +} +EXPORT_SYMBOL_GPL(register_qcom_iommu_ops); + +phys_addr_t qcom_iommu_iova_to_phys_hard(struct iommu_domain *domain, + struct qcom_iommu_atos_txn *txn) +{ + return arm_smmu_iova_to_phys_hard(domain, txn->addr); +} +EXPORT_SYMBOL(qcom_iommu_iova_to_phys_hard); + +int qcom_iommu_sid_switch(struct device *dev, enum sid_switch_direction dir) +{ + struct qcom_iommu_ops *ops; + struct iommu_domain *domain; + + domain = iommu_get_domain_for_dev(dev); + if (!domain) + return -EINVAL; + + ops = to_qcom_iommu_ops(domain->ops); + if (!ops || unlikely(ops->sid_switch == NULL)) + return -EINVAL; + + return ops->sid_switch(dev, dir); +} +EXPORT_SYMBOL(qcom_iommu_sid_switch); + +int qcom_iommu_get_fault_ids(struct iommu_domain *domain, + struct qcom_iommu_fault_ids *f_ids) +{ + struct qcom_iommu_ops *ops = to_qcom_iommu_ops(domain->ops); + + if (!ops || unlikely(ops->get_fault_ids == NULL)) + return -EINVAL; + + return ops->get_fault_ids(domain, f_ids); +} +EXPORT_SYMBOL(qcom_iommu_get_fault_ids); + +int qcom_skip_tlb_management(struct device *dev, bool skip) +{ + struct qcom_iommu_ops *ops; + struct iommu_domain *domain; + + domain = iommu_get_domain_for_dev(dev); + if (!domain) + return -EINVAL; + + ops = to_qcom_iommu_ops(domain->ops); + if (!ops || unlikely(ops->skip_tlb_management == NULL)) + return -EINVAL; + + ops->skip_tlb_management(domain, skip); + return 0; +} +EXPORT_SYMBOL_GPL(qcom_skip_tlb_management); + +int qcom_iommu_get_msi_size(struct device *dev, u32 *msi_size) +{ + struct device_node *np = qcom_iommu_group_parse_phandle(dev); + + if (!np) + return -EINVAL; + + return of_property_read_u32(np, "qcom,iommu-msi-size", msi_size); +} + +int qcom_iommu_get_context_bank_nr(struct iommu_domain *domain) +{ + struct qcom_iommu_ops *ops = to_qcom_iommu_ops(domain->ops); + + if (!ops || unlikely(ops->get_context_bank_nr == NULL)) + return -EINVAL; + + return ops->get_context_bank_nr(domain); +} +EXPORT_SYMBOL(qcom_iommu_get_context_bank_nr); + +int qcom_iommu_get_asid_nr(struct iommu_domain *domain) +{ + struct qcom_iommu_ops *ops = to_qcom_iommu_ops(domain->ops); + + if (!ops || unlikely(ops->get_asid_nr == NULL)) + return -EINVAL; + + return ops->get_asid_nr(domain); +} +EXPORT_SYMBOL(qcom_iommu_get_asid_nr); + +int qcom_iommu_set_secure_vmid(struct iommu_domain *domain, enum vmid vmid) +{ + struct qcom_iommu_ops *ops = to_qcom_iommu_ops(domain->ops); + + if (!ops || unlikely(ops->set_secure_vmid == NULL)) + return -EINVAL; + + return ops->set_secure_vmid(domain, vmid); +} +EXPORT_SYMBOL(qcom_iommu_set_secure_vmid); + +int qcom_iommu_set_fault_model(struct iommu_domain *domain, int fault_model) +{ + struct qcom_iommu_ops *ops = to_qcom_iommu_ops(domain->ops); + + if (!ops || unlikely(ops->set_fault_model == NULL)) + return -EINVAL; + else if (fault_model & ~(QCOM_IOMMU_FAULT_MODEL_NON_FATAL | + QCOM_IOMMU_FAULT_MODEL_NO_CFRE | + QCOM_IOMMU_FAULT_MODEL_NO_STALL | + QCOM_IOMMU_FAULT_MODEL_HUPCF)) + return -EINVAL; + + return ops->set_fault_model(domain, fault_model); +} +EXPORT_SYMBOL(qcom_iommu_set_fault_model); + +int qcom_iommu_enable_s1_translation(struct iommu_domain *domain) +{ + struct qcom_iommu_ops *ops = to_qcom_iommu_ops(domain->ops); + + if (!ops || unlikely(ops->enable_s1_translation == NULL)) + return -EINVAL; + + return ops->enable_s1_translation(domain); +} +EXPORT_SYMBOL(qcom_iommu_enable_s1_translation); + +int qcom_iommu_get_mappings_configuration(struct iommu_domain *domain) +{ + struct qcom_iommu_ops *ops = to_qcom_iommu_ops(domain->ops); + + if (!ops || unlikely(ops->get_mappings_configuration == NULL)) + return -EINVAL; + + return ops->get_mappings_configuration(domain); +} +EXPORT_SYMBOL(qcom_iommu_get_mappings_configuration); + +struct io_pgtable_ops *qcom_alloc_io_pgtable_ops(enum io_pgtable_fmt fmt, + struct qcom_io_pgtable_info *pgtbl_info, + void *cookie) +{ + struct io_pgtable *iop; + const struct io_pgtable_init_fns *fns; + struct io_pgtable_cfg *cfg = &pgtbl_info->cfg; + + if (fmt < IO_PGTABLE_NUM_FMTS) + return alloc_io_pgtable_ops(fmt, cfg, cookie); +#ifdef CONFIG_IOMMU_IO_PGTABLE_FAST + else if (fmt == ARM_V8L_FAST) + fns = &io_pgtable_av8l_fast_init_fns; +#endif +#ifdef CONFIG_IOMMU_IO_PGTABLE_LPAE + else if (fmt == QCOM_ARM_64_LPAE_S1) + fns = &qcom_io_pgtable_arm_64_lpae_s1_init_fns; +#endif + else { + pr_err("Invalid io-pgtable fmt %u\n", fmt); + return NULL; + } + + iop = fns->alloc(cfg, cookie); + if (!iop) + return NULL; + + iop->fmt = fmt; + iop->cookie = cookie; + iop->cfg = *cfg; + + return &iop->ops; +} +EXPORT_SYMBOL(qcom_alloc_io_pgtable_ops); + +void qcom_free_io_pgtable_ops(struct io_pgtable_ops *ops) +{ + struct io_pgtable *iop; + enum io_pgtable_fmt fmt; + const struct io_pgtable_init_fns *fns; + + if (!ops) + return; + + iop = io_pgtable_ops_to_pgtable(ops); + fmt = iop->fmt; + if (fmt < IO_PGTABLE_NUM_FMTS) + return free_io_pgtable_ops(ops); +#ifdef CONFIG_IOMMU_IO_PGTABLE_FAST + else if (fmt == ARM_V8L_FAST) + fns = &io_pgtable_av8l_fast_init_fns; +#endif +#ifdef CONFIG_IOMMU_IO_PGTABLE_LPAE + else if (fmt == QCOM_ARM_64_LPAE_S1) + fns = &qcom_io_pgtable_arm_64_lpae_s1_init_fns; +#endif + else { + pr_err("Invalid io-pgtable fmt %u\n", fmt); + return; + } + + io_pgtable_tlb_flush_all(iop); + fns->free(iop); +} +EXPORT_SYMBOL(qcom_free_io_pgtable_ops); + +#if defined(CONFIG_TRACEPOINTS) && defined(CONFIG_ANDROID_VENDOR_HOOKS) && \ + defined(CONFIG_ANDROID_VENDOR_OEM_DATA) +/* + * iovad->vendor_data1 i.e, ANDROID_VENDOR_DATA(1), field is a 64-bit field. + * + * Use Bits 7:0 to encode the max_alignment_shift. + * Use Bit 16 for selecting best_fit algorithm. + * Reserve remaining bits for future use. + */ +#define QCOM_IOVAD_VENDOR_BEST_FIT_MASK BIT_MASK(16) +#define QCOM_IOVAD_VENDOR_MAX_ALIGN_SHIFT_MASK GENMASK(7, 0) + +static inline void iovad_set_best_fit_iova(struct iova_domain *iovad) +{ + iovad->android_vendor_data1 |= QCOM_IOVAD_VENDOR_BEST_FIT_MASK; +} + +static inline bool iovad_use_best_fit_iova(struct iova_domain *iovad) +{ + return !!(iovad->android_vendor_data1 & QCOM_IOVAD_VENDOR_BEST_FIT_MASK); +} + +static inline void iovad_set_max_align_shift(struct iova_domain *iovad, + unsigned long max_shift) +{ + if (max_shift > QCOM_IOVAD_VENDOR_MAX_ALIGN_SHIFT_MASK) { + /* Use the default value of 9, or 2M alignment for 4K pages */ + WARN_ON_ONCE("Invalid value of max_align_shift!\n"); + max_shift = 9; + } + + /* + * When extracting/computing max_align_shift, we assume that it + * is encoded in the LSB of ->android_vendor_data. Ensure this + * with BUILD_BUG_ON. + */ + BUILD_BUG_ON(QCOM_IOVAD_VENDOR_MAX_ALIGN_SHIFT_MASK > 255); + iovad->android_vendor_data1 |= max_shift; +} + +static inline unsigned long iovad_get_max_align_shift(struct iova_domain *iovad) +{ + u64 max_shift = iovad->android_vendor_data1; + + /* + * When extracting/computing max_align_shift, we assume that it + * is encoded in the LSB of ->android_vendor_data. Ensure this + * with BUILD_BUG_ON. + */ + BUILD_BUG_ON(QCOM_IOVAD_VENDOR_MAX_ALIGN_SHIFT_MASK > 255); + + max_shift &= QCOM_IOVAD_VENDOR_MAX_ALIGN_SHIFT_MASK; + + return (unsigned long)max_shift; +} + +static void init_iovad_attr(void *unused, struct device *dev, + struct iova_domain *iovad) +{ + struct device_node *node; + u32 shift; + + node = dev->of_node; + if (of_property_read_bool(node, "qcom,iova-best-fit")) + iovad_set_best_fit_iova(iovad); + + if (!of_property_read_u32(node, "qcom,iova-max-align-shift", &shift)) + iovad_set_max_align_shift(iovad, (unsigned long)shift); +} + +static void register_iommu_iovad_init_alloc_algo_vh(void) +{ + if (register_trace_android_rvh_iommu_iovad_init_alloc_algo( + init_iovad_attr, NULL)) + pr_err("Failed to register init_iovad_attr vendor hook\n"); +} + +static struct iova *to_iova(struct rb_node *node) +{ + return rb_entry(node, struct iova, node); +} + +/* Insert the iova into domain rbtree by holding writer lock */ +static void iova_insert_rbtree(struct rb_root *root, struct iova *iova, + struct rb_node *start) +{ + struct rb_node **new, *parent = NULL; + + new = (start) ? &start : &(root->rb_node); + /* Figure out where to put new node */ + while (*new) { + struct iova *this = to_iova(*new); + + parent = *new; + + if (iova->pfn_lo < this->pfn_lo) + new = &((*new)->rb_left); + else if (iova->pfn_lo > this->pfn_lo) + new = &((*new)->rb_right); + else { + WARN_ON(1); /* this should not happen */ + return; + } + } + /* Add new node and rebalance tree. */ + rb_link_node(&iova->node, parent, new); + rb_insert_color(&iova->node, root); +} + +static unsigned long limit_align_shift(struct iova_domain *iovad, + unsigned long shift) +{ + unsigned long max_align_shift; + unsigned long new_shift; + + new_shift = iovad_get_max_align_shift(iovad); + + /* If device doesn't override reuse current value */ + if (!new_shift) + return shift; + + max_align_shift = new_shift + PAGE_SHIFT - iova_shift(iovad); + + return min_t(unsigned long, max_align_shift, shift); +} + +static int __alloc_and_insert_iova_best_fit(struct iova_domain *iovad, + unsigned long size, + unsigned long limit_pfn, + struct iova *new, + bool size_aligned) +{ + struct rb_node *curr, *prev; + struct iova *curr_iova, *prev_iova; + unsigned long flags; + unsigned long align_mask = ~0UL; + struct rb_node *candidate_rb_parent; + unsigned long new_pfn, candidate_pfn = ~0UL; + unsigned long gap, candidate_gap = ~0UL; + + if (!iovad_use_best_fit_iova(iovad)) + return -EINVAL; + + if (size_aligned) + align_mask <<= limit_align_shift(iovad, fls_long(size - 1)); + + /* Walk the tree backwards */ + spin_lock_irqsave(&iovad->iova_rbtree_lock, flags); + curr = &iovad->anchor.node; + prev = rb_prev(curr); + for (; prev; curr = prev, prev = rb_prev(curr)) { + curr_iova = rb_entry(curr, struct iova, node); + prev_iova = rb_entry(prev, struct iova, node); + + limit_pfn = min(limit_pfn, curr_iova->pfn_lo); + new_pfn = (limit_pfn - size) & align_mask; + gap = curr_iova->pfn_lo - prev_iova->pfn_hi - 1; + if ((limit_pfn >= size) && (new_pfn > prev_iova->pfn_hi) + && (gap < candidate_gap)) { + candidate_gap = gap; + candidate_pfn = new_pfn; + candidate_rb_parent = curr; + if (gap == size) + goto insert; + } + } + + curr_iova = rb_entry(curr, struct iova, node); + limit_pfn = min(limit_pfn, curr_iova->pfn_lo); + new_pfn = (limit_pfn - size) & align_mask; + gap = curr_iova->pfn_lo - iovad->start_pfn; + if (limit_pfn >= size && new_pfn >= iovad->start_pfn && + gap < candidate_gap) { + candidate_gap = gap; + candidate_pfn = new_pfn; + candidate_rb_parent = curr; + } + +insert: + if (candidate_pfn == ~0UL) { + spin_unlock_irqrestore(&iovad->iova_rbtree_lock, flags); + return -ENOMEM; + } + + /* pfn_lo will point to size aligned address if size_aligned is set */ + new->pfn_lo = candidate_pfn; + new->pfn_hi = new->pfn_lo + size - 1; + + /* If we have 'prev', it's a valid place to start the insertion. */ + iova_insert_rbtree(&iovad->rbroot, new, candidate_rb_parent); + spin_unlock_irqrestore(&iovad->iova_rbtree_lock, flags); + return 0; +} + +static void __qcom_alloc_insert_iova(void *data, struct iova_domain *iovad, + unsigned long size, + unsigned long limit_pfn, struct iova *new, + bool size_aligned, int *ret) +{ + *ret = __alloc_and_insert_iova_best_fit(iovad, size, limit_pfn, new, + size_aligned); +} + +static void register_iommu_alloc_insert_iova_vh(void) +{ + if (register_trace_android_rvh_iommu_alloc_insert_iova( + __qcom_alloc_insert_iova, NULL)) { + pr_err("Failed to register alloc_inser_iova vendor hook\n"); + } +} + +static void __qcom_limit_align_shift(void *data, struct iova_domain *iovad, + unsigned long size, unsigned long *shift) +{ + *shift = limit_align_shift(iovad, *shift); +} + +static void register_iommu_limit_align_shift(void) +{ + if (register_trace_android_rvh_iommu_limit_align_shift( + __qcom_limit_align_shift, NULL)) { + pr_err("Failed to register limit_align_shift vendor hook\n"); + } +} + +#else +static void register_iommu_iovad_init_alloc_algo_vh(void) +{ +} + +static void register_iommu_alloc_insert_iova_vh(void) +{ +} + +static void register_iommu_limit_align_shift(void) +{ +} +#endif + +/* + * These tables must have the same length. + * It is allowed to have a NULL exitcall corresponding to a non-NULL initcall. + */ +static initcall_t init_table[] __initdata = { + dma_mapping_fast_init, + qcom_dma_iommu_generic_driver_init, + qcom_arm_lpae_do_selftests, + qcom_io_pgtable_alloc_init, + NULL +}; + +static exitcall_t exit_table[] = { + NULL, /* dma_mapping_fast_exit */ + qcom_dma_iommu_generic_driver_exit, + NULL, /*qcom_arm_lpae_do_selftests */ + qcom_io_pgtable_alloc_exit, + NULL, +}; + +static int __init qcom_iommu_util_init(void) +{ + initcall_t *init_fn; + exitcall_t *exit_fn; + int ret; + + if (ARRAY_SIZE(init_table) != ARRAY_SIZE(exit_table)) { + pr_err("qcom-iommu-util: Invalid initcall/exitcall table\n"); + return -EINVAL; + } + + for (init_fn = init_table; *init_fn; init_fn++) { + ret = (**init_fn)(); + if (ret) { + pr_err("%ps returned %d\n", *init_fn, ret); + goto out_undo; + } + } + + register_iommu_iovad_init_alloc_algo_vh(); + register_iommu_alloc_insert_iova_vh(); + register_iommu_limit_align_shift(); + + return 0; + +out_undo: + exit_fn = exit_table + (init_fn - init_table); + for (exit_fn--; exit_fn >= exit_table; exit_fn--) { + if (!*exit_fn) + continue; + (**exit_fn)(); + } + return ret; +} +#if IS_MODULE(CONFIG_QCOM_IOMMU_UTIL) +module_init(qcom_iommu_util_init); +#else +arch_initcall_sync(qcom_iommu_util_init); +#endif + +MODULE_LICENSE("GPL"); diff --git a/include/linux/qcom-dma-mapping.h b/include/linux/qcom-dma-mapping.h new file mode 100644 index 0000000000000..09be7d411eaee --- /dev/null +++ b/include/linux/qcom-dma-mapping.h @@ -0,0 +1,39 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2021, The Linux Foundation. All rights reserved. + & Copyright (c) 2023 Qualcomm Innovation Center, Inc. All rights reserved. + */ + +/* + * DMA_ATTR_NO_DELAYED_UNMAP: Used by msm specific lazy mapping to indicate + * that the mapping can be freed on unmap, rather than when the ion_buffer + * is freed. + */ +#define DMA_ATTR_NO_DELAYED_UNMAP (1UL << 13) +/* + * When passed to a DMA map call the DMA_ATTR_FORCE_COHERENT DMA + * attribute can be used to force a buffer to be mapped as IO coherent. + */ +#define DMA_ATTR_FORCE_COHERENT (1UL << 15) +/* + * When passed to a DMA map call the DMA_ATTR_FORCE_NON_COHERENT DMA + * attribute can be used to force a buffer to not be mapped as IO + * coherent. + */ +#define DMA_ATTR_FORCE_NON_COHERENT (1UL << 16) +/* + * DMA_ATTR_DELAYED_UNMAP: Used by ION, it will ensure that mappings are not + * removed on unmap but instead are removed when the ion_buffer is freed. + */ +#define DMA_ATTR_DELAYED_UNMAP (1UL << 17) +/* + * DMA_ATTR_QTI_SMMU_PROXY_MAP : Map this buffer in the TVM SMMU if supported + * on the target. + */ +#define DMA_ATTR_QTI_SMMU_PROXY_MAP (1UL << 18) + +#ifndef DMA_ATTR_SYS_CACHE +/* Attributes are not supported, so render them ineffective. */ +#define DMA_ATTR_SYS_CACHE (0UL) +#define DMA_ATTR_SYS_CACHE_NWA (0UL) +#endif diff --git a/include/linux/qcom-iommu-util.h b/include/linux/qcom-iommu-util.h new file mode 100644 index 0000000000000..e28d160ee5a47 --- /dev/null +++ b/include/linux/qcom-iommu-util.h @@ -0,0 +1,144 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved. + * Copyright (c) 2022-2024 Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#ifndef __QCOM_IOMMU_UTIL_H +#define __QCOM_IOMMU_UTIL_H + +#include +#include +#include + +//#include + +/* IOMMU fault behaviors */ +#define QCOM_IOMMU_FAULT_MODEL_NON_FATAL BIT(0) +#define QCOM_IOMMU_FAULT_MODEL_NO_CFRE BIT(1) +#define QCOM_IOMMU_FAULT_MODEL_NO_STALL BIT(2) +#define QCOM_IOMMU_FAULT_MODEL_HUPCF BIT(3) + +/* IOMMU mapping configurations */ +#define QCOM_IOMMU_MAPPING_CONF_S1_BYPASS BIT(0) +#define QCOM_IOMMU_MAPPING_CONF_ATOMIC BIT(1) +#define QCOM_IOMMU_MAPPING_CONF_FAST BIT(2) + +/* iommu transaction flags */ +/* 1 Write, 0 Read */ +#define QCOM_IOMMU_ATOS_TRANS_WRITE BIT(0) +/* 1 Privileged, 0 Unprivileged */ +#define QCOM_IOMMU_ATOS_TRANS_PRIV BIT(1) +/* 1 Instruction fetch, 0 Data access */ +#define QCOM_IOMMU_ATOS_TRANS_INST BIT(2) +/* Non secure unprivileged Data read operation */ +#define QCOM_IOMMU_ATOS_TRANS_DEFAULT (0U) + +#ifndef IOMMU_SYS_CACHE +/* Attributes are not supported, so render them ineffective. */ +#define IOMMU_SYS_CACHE (0) +#define IOMMU_SYS_CACHE_NWA (0) +#endif + +/* vendor iommu fault flags */ +#define IOMMU_FAULT_TRANSLATION (1 << 2) +#define IOMMU_FAULT_PERMISSION (1 << 3) +#define IOMMU_FAULT_EXTERNAL (1 << 4) +#define IOMMU_FAULT_TRANSACTION_STALLED (1 << 5) + +/* iommu transaction flags */ +#define IOMMU_TRANS_WRITE BIT(0) /* 1 Write, 0 Read */ +#define IOMMU_TRANS_PRIV BIT(1) /* 1 Privileged, 0 Unprivileged */ +#define IOMMU_TRANS_INST BIT(2) /* 1 Instruction fetch, 0 Data access */ +#define IOMMU_TRANS_SEC BIT(3) /* 1 Secure, 0 Non-secure access*/ + +/* Non secure unprivileged Data read operation */ +#define IOMMU_TRANS_DEFAULT (0U) + +typedef void (*fault_handler_irq_t)(struct iommu_domain *, void *); + +struct iommu_pgtbl_info { + void *ops; +}; + +struct qcom_iommu_atos_txn { + u64 addr; + u32 flags; + u32 id; +}; + +enum sid_switch_direction { + SID_ACQUIRE, + SID_RELEASE, +}; + +struct qcom_iommu_fault_ids { + u32 bid; + u32 pid; + u32 mid; +}; + +/* + * @sid_switch: add/remove all SIDS in the iommu domain containing dev from + * iommu registers. + */ +struct qcom_iommu_ops { + phys_addr_t (*iova_to_phys_hard)(struct iommu_domain *domain, + struct qcom_iommu_atos_txn *txn); + int (*sid_switch)(struct device *dev, enum sid_switch_direction dir); + int (*get_fault_ids)(struct iommu_domain *domain, + struct qcom_iommu_fault_ids *ids); + int (*get_context_bank_nr)(struct iommu_domain *domain); + int (*get_asid_nr)(struct iommu_domain *domain); + int (*set_secure_vmid)(struct iommu_domain *domain, enum vmid vmid); + int (*set_fault_model)(struct iommu_domain *domain, int fault_model); + void (*set_fault_handler_irq)(struct iommu_domain *domain, + fault_handler_irq_t handler_irq, void *token); + int (*enable_s1_translation)(struct iommu_domain *domain); + int (*get_mappings_configuration)(struct iommu_domain *domain); + void (*skip_tlb_management)(struct iommu_domain *domain, bool skip); + struct iommu_ops iommu_ops; + struct iommu_domain_ops domain_ops; +}; +int register_qcom_iommu_ops(struct qcom_iommu_ops *ops); + +struct device_node *qcom_iommu_group_parse_phandle(struct device *dev); +int qcom_iommu_get_fast_iova_range(struct device *dev, + dma_addr_t *ret_iova_base, + dma_addr_t *ret_iova_end); + +/* Remove once this function is exported by upstream kernel */ +void qcom_iommu_get_resv_regions(struct device *dev, struct list_head *list); + +phys_addr_t qcom_iommu_iova_to_phys_hard(struct iommu_domain *domain, + struct qcom_iommu_atos_txn *txn); + +int qcom_iommu_sid_switch(struct device *dev, enum sid_switch_direction dir); + +int qcom_skip_tlb_management(struct device *dev, bool skip); + +extern int qcom_iommu_get_fault_ids(struct iommu_domain *domain, + struct qcom_iommu_fault_ids *f_ids); +extern int qcom_iommu_get_msi_size(struct device *dev, u32 *msi_size); + +int qcom_iommu_get_context_bank_nr(struct iommu_domain *domain); + +int qcom_iommu_get_asid_nr(struct iommu_domain *domain); + +int qcom_iommu_set_secure_vmid(struct iommu_domain *domain, enum vmid vmid); + +int qcom_iommu_set_fault_model(struct iommu_domain *domain, int fault_model); + +int qcom_iommu_enable_s1_translation(struct iommu_domain *domain); + +int qcom_iommu_get_mappings_configuration(struct iommu_domain *domain); + +#ifdef CONFIG_IOMMU_IO_PGTABLE_LPAE +int __init qcom_arm_lpae_do_selftests(void); +#else +static inline int __init qcom_arm_lpae_do_selftests(void) +{ + return 0; +} +#endif +#endif /* __QCOM_IOMMU_UTIL_H */