diff --git a/Documentation/core-api/swiotlb.rst b/Documentation/core-api/swiotlb.rst index 71b4e4c27eb5c0..06d9f88fdf7b88 100644 --- a/Documentation/core-api/swiotlb.rst +++ b/Documentation/core-api/swiotlb.rst @@ -148,10 +148,10 @@ may also be adjusted due to other conditions, such as running in a CoCo VM, as described above. If CONFIG_SWIOTLB_DYNAMIC is enabled, additional pools may be allocated later in the life of the system. Each pool must be a contiguous range of physical -memory. The default pool is allocated below the 4 GiB physical address line so -it works for devices that can only address 32-bits of physical memory (unless -architecture-specific code provides the SWIOTLB_ANY flag). In a CoCo VM, the -pool memory must be decrypted before swiotlb is used. +memory. The default pool is allocated below the architecture's low address +limit when it is needed for devices with limited DMA addressing. Otherwise, +it may be allocated anywhere in directly mapped memory. In a CoCo VM, the pool +memory must be decrypted before swiotlb is used. Each pool is divided into "slots" of size IO_TLB_SIZE, which is 2 KiB with current definitions. IO_TLB_SEGSIZE contiguous slots (128 slots) constitute diff --git a/arch/arm/mm/init.c b/arch/arm/mm/init.c index 0cc1bf04686d83..aca97a4e5dcdf1 100644 --- a/arch/arm/mm/init.c +++ b/arch/arm/mm/init.c @@ -223,7 +223,11 @@ static inline void poison_init_mem(void *s, size_t count) void __init arch_mm_preinit(void) { #ifdef CONFIG_ARM_LPAE - swiotlb_init(max_pfn > arm_dma_pfn_limit, SWIOTLB_VERBOSE); + unsigned int flags = SWIOTLB_VERBOSE; + + if (max_pfn > arm_dma_pfn_limit) + flags |= SWIOTLB_INIT_ADDRESSING_LIMIT; + swiotlb_init(flags); #endif #ifdef CONFIG_SA1111 diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c index fbf215ecc7d061..95075b5c82079a 100644 --- a/arch/arm64/mm/init.c +++ b/arch/arm64/mm/init.c @@ -340,18 +340,10 @@ void __init arch_mm_preinit(void) { unsigned int flags = SWIOTLB_VERBOSE; - if (max_pfn <= PFN_DOWN(arm64_dma_phys_limit)) { - /* - * If no bouncing needed for ZONE_DMA, reduce the swiotlb - * buffer for kmalloc() bouncing to 1MB per 1GB of RAM. - */ - unsigned long size = - DIV_ROUND_UP(memblock_phys_mem_size(), 1024); - - swiotlb_adjust_size(min(swiotlb_size_or_default(), size)); - } + if (max_pfn > PFN_DOWN(arm64_dma_phys_limit)) + flags |= SWIOTLB_INIT_ADDRESSING_LIMIT; - swiotlb_init(true, flags); + swiotlb_init(flags); /* * Check boundaries twice: Some fundamental inconsistencies can be diff --git a/arch/loongarch/kernel/setup.c b/arch/loongarch/kernel/setup.c index 6fa4a22a58fd6a..02f52f6b274721 100644 --- a/arch/loongarch/kernel/setup.c +++ b/arch/loongarch/kernel/setup.c @@ -404,7 +404,7 @@ static void __init arch_mem_init(char **cmdline_p) memblock_set_bottom_up(true); - swiotlb_init(true, SWIOTLB_VERBOSE); + swiotlb_init(SWIOTLB_VERBOSE | SWIOTLB_INIT_ADDRESSING_LIMIT); dma_contiguous_reserve(PFN_PHYS(max_low_pfn)); diff --git a/arch/mips/cavium-octeon/dma-octeon.c b/arch/mips/cavium-octeon/dma-octeon.c index 9fbba6a8fa4c5c..ca7c37bc070f8e 100644 --- a/arch/mips/cavium-octeon/dma-octeon.c +++ b/arch/mips/cavium-octeon/dma-octeon.c @@ -235,5 +235,5 @@ void __init plat_swiotlb_setup(void) #endif swiotlb_adjust_size(swiotlbsize); - swiotlb_init(true, SWIOTLB_VERBOSE); + swiotlb_init(SWIOTLB_VERBOSE | SWIOTLB_INIT_ADDRESSING_LIMIT); } diff --git a/arch/mips/loongson64/dma.c b/arch/mips/loongson64/dma.c index 52801442ea86ce..5b8056e6c5a87f 100644 --- a/arch/mips/loongson64/dma.c +++ b/arch/mips/loongson64/dma.c @@ -25,5 +25,5 @@ phys_addr_t dma_to_phys(struct device *dev, dma_addr_t daddr) void __init plat_swiotlb_setup(void) { - swiotlb_init(true, SWIOTLB_VERBOSE); + swiotlb_init(SWIOTLB_VERBOSE | SWIOTLB_INIT_ADDRESSING_LIMIT); } diff --git a/arch/mips/sibyte/common/dma.c b/arch/mips/sibyte/common/dma.c index c5c2c782aff688..3835fee21489a3 100644 --- a/arch/mips/sibyte/common/dma.c +++ b/arch/mips/sibyte/common/dma.c @@ -10,5 +10,5 @@ void __init plat_swiotlb_setup(void) { - swiotlb_init(true, SWIOTLB_VERBOSE); + swiotlb_init(SWIOTLB_VERBOSE | SWIOTLB_INIT_ADDRESSING_LIMIT); } diff --git a/arch/powerpc/kernel/dma-swiotlb.c b/arch/powerpc/kernel/dma-swiotlb.c index ba256c37bcc0f7..97fffa46f05ac6 100644 --- a/arch/powerpc/kernel/dma-swiotlb.c +++ b/arch/powerpc/kernel/dma-swiotlb.c @@ -14,8 +14,10 @@ unsigned int ppc_swiotlb_flags; void __init swiotlb_detect_4g(void) { - if ((memblock_end_of_DRAM() - 1) > 0xffffffff) + if ((memblock_end_of_DRAM() - 1) > 0xffffffff) { ppc_swiotlb_enable = 1; + ppc_swiotlb_flags |= SWIOTLB_INIT_ADDRESSING_LIMIT; + } } static int __init check_swiotlb_enabled(void) diff --git a/arch/powerpc/mm/mem.c b/arch/powerpc/mm/mem.c index b617b69452cd6b..1bdea6c03848b5 100644 --- a/arch/powerpc/mm/mem.c +++ b/arch/powerpc/mm/mem.c @@ -287,6 +287,21 @@ void __init arch_mm_preinit(void) BUILD_BUG_ON(MMU_PAGE_COUNT > 16); #ifdef CONFIG_SWIOTLB + if (is_secure_guest()) { + + /* Don't release the SWIOTLB buffer. */ + ppc_swiotlb_enable = 1; + + /* + * Since the guest memory is inaccessible to the host, + * devices always need to use the SWIOTLB buffer for DMA + * even if dma_capable() says otherwise. The hypervisor has + * no addressing limitation, so the buffer may be allocated + * anywhere. + */ + ppc_swiotlb_flags &= ~SWIOTLB_INIT_ADDRESSING_LIMIT; + } + /* * Some platforms (e.g. 85xx) limit DMA-able memory way below * 4G. We force memblock to bottom-up mode to ensure that the @@ -295,7 +310,7 @@ void __init arch_mm_preinit(void) * back to to-down. */ memblock_set_bottom_up(true); - swiotlb_init(ppc_swiotlb_enable, ppc_swiotlb_flags); + swiotlb_init(ppc_swiotlb_flags); #endif kasan_late_init(); diff --git a/arch/powerpc/platforms/pseries/svm.c b/arch/powerpc/platforms/pseries/svm.c index 7a403dbd35ee55..4a0be631dc6da2 100644 --- a/arch/powerpc/platforms/pseries/svm.c +++ b/arch/powerpc/platforms/pseries/svm.c @@ -21,16 +21,6 @@ static int __init init_svm(void) if (!is_secure_guest()) return 0; - /* Don't release the SWIOTLB buffer. */ - ppc_swiotlb_enable = 1; - - /* - * Since the guest memory is inaccessible to the host, devices always - * need to use the SWIOTLB buffer for DMA even if dma_capable() says - * otherwise. - */ - ppc_swiotlb_flags |= SWIOTLB_ANY; - /* Share the SWIOTLB buffer with the host. */ swiotlb_update_mem_attributes(); diff --git a/arch/powerpc/sysdev/fsl_pci.c b/arch/powerpc/sysdev/fsl_pci.c index 600f83cea1cd34..49264e25108b46 100644 --- a/arch/powerpc/sysdev/fsl_pci.c +++ b/arch/powerpc/sysdev/fsl_pci.c @@ -444,6 +444,7 @@ static void setup_pci_atmu(struct pci_controller *hose) if (hose->dma_window_size < mem) { #ifdef CONFIG_SWIOTLB ppc_swiotlb_enable = 1; + ppc_swiotlb_flags |= SWIOTLB_INIT_ADDRESSING_LIMIT; #else pr_err("%pOF: ERROR: Memory size exceeds PCI ATMU ability to " "map - enable CONFIG_SWIOTLB to avoid dma errors.\n", diff --git a/arch/riscv/mm/init.c b/arch/riscv/mm/init.c index fb37b0b67efec6..734712230f3549 100644 --- a/arch/riscv/mm/init.c +++ b/arch/riscv/mm/init.c @@ -165,32 +165,16 @@ static void print_vm_layout(void) { } void __init arch_mm_preinit(void) { - bool swiotlb = max_pfn > PFN_DOWN(dma32_phys_limit) && - memblock_start_of_DRAM() < dma32_phys_limit; unsigned int swiotlb_flags = SWIOTLB_VERBOSE; #ifdef CONFIG_FLATMEM BUG_ON(!mem_map); #endif /* CONFIG_FLATMEM */ - if (IS_ENABLED(CONFIG_DMA_BOUNCE_UNALIGNED_KMALLOC) && !swiotlb && - dma_cache_alignment != 1) { - /* - * No 32-bit DMA bouncing needed (either all DRAM is within - * the 32-bit limit, or it all starts above it), but - * kmalloc() buffers whose sizes are not cache-line-aligned - * still require bouncing for non-coherent DMA. Use - * SWIOTLB_ANY so that the buffer can be allocated from high - * memory when DRAM starts above dma32_phys_limit. Allocate - * ~1 MB per 1 GB of RAM. - */ - unsigned long size = - DIV_ROUND_UP(memblock_phys_mem_size(), 1024); - swiotlb_adjust_size(min(swiotlb_size_or_default(), size)); - swiotlb = true; - swiotlb_flags |= SWIOTLB_ANY; - } + if ((max_pfn > PFN_DOWN(dma32_phys_limit)) && + (memblock_start_of_DRAM() < dma32_phys_limit)) + swiotlb_flags |= SWIOTLB_INIT_ADDRESSING_LIMIT; - swiotlb_init(swiotlb, swiotlb_flags); + swiotlb_init(swiotlb_flags); print_vm_layout(); } diff --git a/arch/s390/mm/init.c b/arch/s390/mm/init.c index be7e009e7b59ed..ce10292447f1c0 100644 --- a/arch/s390/mm/init.c +++ b/arch/s390/mm/init.c @@ -166,7 +166,7 @@ static void __init pv_init(void) virtio_set_mem_acc_cb(virtio_require_restricted_mem_acc); /* make sure bounce buffers are shared */ - swiotlb_init(true, SWIOTLB_VERBOSE | SWIOTLB_ANY); + swiotlb_init(SWIOTLB_VERBOSE); swiotlb_update_mem_attributes(); } diff --git a/arch/x86/include/asm/crash_reserve.h b/arch/x86/include/asm/crash_reserve.h index 7835b2cdff04a7..9f6b06e2f9bdbe 100644 --- a/arch/x86/include/asm/crash_reserve.h +++ b/arch/x86/include/asm/crash_reserve.h @@ -18,7 +18,7 @@ * no good way to detect the paging mode of the target kernel which will be * loaded for dumping. */ -extern unsigned long swiotlb_size_or_default(void); +unsigned long __init swiotlb_adjusted_size(void); #ifdef CONFIG_X86_32 # define CRASH_ADDR_LOW_MAX SZ_512M @@ -33,7 +33,7 @@ extern unsigned long swiotlb_size_or_default(void); static inline unsigned long crash_low_size_default(void) { #ifdef CONFIG_X86_64 - return max(swiotlb_size_or_default() + (8UL << 20), 256UL << 20); + return max(swiotlb_adjusted_size() + (8UL << 20), 256UL << 20); #else return 0; #endif diff --git a/arch/x86/include/asm/iommu.h b/arch/x86/include/asm/iommu.h index 3be2451e7bc857..22c8190fe34d09 100644 --- a/arch/x86/include/asm/iommu.h +++ b/arch/x86/include/asm/iommu.h @@ -14,8 +14,10 @@ extern bool amd_iommu_snp_en; #ifdef CONFIG_SWIOTLB extern bool x86_swiotlb_enable; +extern unsigned int x86_swiotlb_flags; #else #define x86_swiotlb_enable false +#define x86_swiotlb_flags 0 #endif /* 10 seconds */ diff --git a/arch/x86/kernel/amd_gart_64.c b/arch/x86/kernel/amd_gart_64.c index b5f1f031d45b8d..d0fbc0271e4323 100644 --- a/arch/x86/kernel/amd_gart_64.c +++ b/arch/x86/kernel/amd_gart_64.c @@ -814,6 +814,7 @@ int __init gart_iommu_init(void) dma_ops = &gart_dma_ops; x86_platform.iommu_shutdown = gart_iommu_shutdown; x86_swiotlb_enable = false; + x86_swiotlb_flags = 0; return 0; } diff --git a/arch/x86/kernel/pci-dma.c b/arch/x86/kernel/pci-dma.c index 75cf8f6ae8cd09..e472631a5bd730 100644 --- a/arch/x86/kernel/pci-dma.c +++ b/arch/x86/kernel/pci-dma.c @@ -39,13 +39,15 @@ int iommu_detected __read_mostly = 0; #ifdef CONFIG_SWIOTLB bool x86_swiotlb_enable; -static unsigned int x86_swiotlb_flags; +unsigned int x86_swiotlb_flags; static void __init pci_swiotlb_detect(void) { /* don't initialize swiotlb if iommu=off (no_iommu=1) */ - if (!no_iommu && max_possible_pfn > MAX_DMA32_PFN) + if (!no_iommu && max_possible_pfn > MAX_DMA32_PFN) { x86_swiotlb_enable = true; + x86_swiotlb_flags |= SWIOTLB_INIT_ADDRESSING_LIMIT; + } /* * Set swiotlb to 1 so that bounce buffers are allocated and used for @@ -66,7 +68,6 @@ static void __init pci_swiotlb_detect(void) static inline void __init pci_swiotlb_detect(void) { } -#define x86_swiotlb_flags 0 #endif /* CONFIG_SWIOTLB */ #ifdef CONFIG_SWIOTLB_XEN @@ -81,8 +82,10 @@ static void __init pci_xen_swiotlb_init(void) if (!xen_swiotlb_enabled()) return; x86_swiotlb_enable = true; - x86_swiotlb_flags |= SWIOTLB_ANY; - swiotlb_init_remap(true, x86_swiotlb_flags, xen_swiotlb_fixup); + /* Xen can use a SWIOTLB pool anywhere in directly mapped memory. */ + x86_swiotlb_flags &= ~SWIOTLB_INIT_ADDRESSING_LIMIT; + x86_swiotlb_flags |= SWIOTLB_INIT_REMAP; + swiotlb_init_remap(x86_swiotlb_flags, xen_swiotlb_fixup); dma_ops = &xen_swiotlb_dma_ops; if (IS_ENABLED(CONFIG_PCI)) pci_request_acs(); @@ -103,7 +106,7 @@ void __init pci_iommu_alloc(void) gart_iommu_hole_init(); amd_iommu_detect(); detect_intel_iommu(); - swiotlb_init(x86_swiotlb_enable, x86_swiotlb_flags); + swiotlb_init(x86_swiotlb_flags); } static __init int iommu_setup(char *p) @@ -149,8 +152,10 @@ static __init int iommu_setup(char *p) return 1; } #ifdef CONFIG_SWIOTLB - if (!strncmp(p, "soft", 4)) + if (!strncmp(p, "soft", 4)) { x86_swiotlb_enable = true; + x86_swiotlb_flags |= SWIOTLB_INIT_ADDRESSING_LIMIT; + } #endif if (!strncmp(p, "pt", 2)) iommu_set_default_passthrough(true); diff --git a/arch/x86/mm/mem_encrypt.c b/arch/x86/mm/mem_encrypt.c index 95bae74fdab233..912f22ca838f87 100644 --- a/arch/x86/mm/mem_encrypt.c +++ b/arch/x86/mm/mem_encrypt.c @@ -101,9 +101,6 @@ void __init mem_encrypt_init(void) void __init mem_encrypt_setup_arch(void) { - phys_addr_t total_mem = memblock_phys_mem_size(); - unsigned long size; - /* * Do RMP table fixups after the e820 tables have been setup by * e820__memory_setup(). @@ -114,27 +111,6 @@ void __init mem_encrypt_setup_arch(void) if (!cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT)) return; - /* - * For SEV and TDX, all DMA has to occur via shared/unencrypted pages. - * Kernel uses SWIOTLB to make this happen without changing device - * drivers. However, depending on the workload being run, the - * default 64MB of SWIOTLB may not be enough and SWIOTLB may - * run out of buffers for DMA, resulting in I/O errors and/or - * performance degradation especially with high I/O workloads. - * - * Adjust the default size of SWIOTLB using a percentage of guest - * memory for SWIOTLB buffers. Also, as the SWIOTLB bounce buffer - * memory is allocated from low memory, ensure that the adjusted size - * is within the limits of low available memory. - * - * The percentage of guest memory used here for SWIOTLB buffers - * is more of an approximation of the static adjustment which - * 64MB for <1G, and ~128M to 256M for 1G-to-4G, i.e., the 6% - */ - size = total_mem * 6 / 100; - size = clamp_val(size, IO_TLB_DEFAULT_SIZE, SZ_1G); - swiotlb_adjust_size(size); - /* Set restricted memory access for virtio. */ virtio_set_mem_acc_cb(virtio_require_restricted_mem_acc); } diff --git a/include/linux/swiotlb.h b/include/linux/swiotlb.h index 2a421241b980ca..b3da751a81dd4c 100644 --- a/include/linux/swiotlb.h +++ b/include/linux/swiotlb.h @@ -15,7 +15,10 @@ struct page; struct scatterlist; #define SWIOTLB_VERBOSE (1 << 0) /* verbose initialization */ -#define SWIOTLB_ANY (1 << 1) /* allow any memory for the buffer */ +/* Initialize a default-sized pool for devices with limited DMA addressing. */ +#define SWIOTLB_INIT_ADDRESSING_LIMIT (1 << 1) +/* Initialize a default-sized pool that requires architecture remapping. */ +#define SWIOTLB_INIT_REMAP (1 << 2) /* * Maximum allowable number of contiguous slabs to map, @@ -39,8 +42,8 @@ struct scatterlist; #endif unsigned long swiotlb_size_or_default(void); -void __init swiotlb_init_remap(bool addressing_limit, unsigned int flags, - int (*remap)(void *tlb, unsigned long nslabs)); +void __init swiotlb_init_remap(unsigned int flags, + int (*remap)(void *tlb, unsigned long nslabs)); int swiotlb_init_late(size_t size, gfp_t gfp_mask, int (*remap)(void *tlb, unsigned long nslabs)); extern void __init swiotlb_update_mem_attributes(void); @@ -183,17 +186,18 @@ static inline bool is_swiotlb_force_bounce(struct device *dev) return mem && mem->force_bounce; } -void swiotlb_init(bool addressing_limited, unsigned int flags); +void swiotlb_init(unsigned int flags); void __init swiotlb_exit(void); void swiotlb_dev_init(struct device *dev); size_t swiotlb_max_mapping_size(struct device *dev); bool is_swiotlb_allocated(void); bool is_swiotlb_active(struct device *dev); +unsigned long __init swiotlb_adjusted_size(void); void __init swiotlb_adjust_size(unsigned long size); phys_addr_t default_swiotlb_base(void); phys_addr_t default_swiotlb_limit(void); #else -static inline void swiotlb_init(bool addressing_limited, unsigned int flags) +static inline void swiotlb_init(unsigned int flags) { } @@ -228,6 +232,11 @@ static inline bool is_swiotlb_active(struct device *dev) return false; } +static inline unsigned long __init swiotlb_adjusted_size(void) +{ + return 0; +} + static inline void swiotlb_adjust_size(unsigned long size) { } diff --git a/kernel/dma/swiotlb.c b/kernel/dma/swiotlb.c index ded7016a46a71f..2715833a21f8a2 100644 --- a/kernel/dma/swiotlb.c +++ b/kernel/dma/swiotlb.c @@ -80,6 +80,14 @@ struct io_tlb_slot { static bool swiotlb_force_bounce; static bool swiotlb_force_disable; +static bool restricted_dma_pool_present __initdata; + +enum swiotlb_pool_policy { + SWIOTLB_POOL_NONE, + SWIOTLB_POOL_MINIMAL, + SWIOTLB_POOL_DEFAULT, + SWIOTLB_POOL_CC_GUEST, +}; #ifdef CONFIG_SWIOTLB_DYNAMIC @@ -117,26 +125,33 @@ struct io_tlb_area { spinlock_t lock; }; -/* - * Round up number of slabs to the next power of 2. The last area is going - * be smaller than the rest if default_nslabs is not power of two. - * The number of slot in an area should be a multiple of IO_TLB_SEGSIZE, - * otherwise a segment may span two or more areas. It conflicts with free - * contiguous slots tracking: free slots are treated contiguous no matter - * whether they cross an area boundary. - * - * Return true if default_nslabs is rounded up. - */ -static bool round_up_default_nslabs(void) +/* Return a segment-aligned size that can be split evenly between areas. */ +static unsigned long swiotlb_aligned_nslabs(unsigned long size) { + unsigned long nslabs; + + nslabs = ALIGN(DIV_ROUND_UP(size, IO_TLB_SIZE), IO_TLB_SEGSIZE); if (!default_nareas) - return false; + return nslabs; + + if (nslabs < IO_TLB_SEGSIZE * default_nareas) + nslabs = IO_TLB_SEGSIZE * default_nareas; + else if (!is_power_of_2(nslabs)) + nslabs = roundup_pow_of_two(nslabs); - if (default_nslabs < IO_TLB_SEGSIZE * default_nareas) - default_nslabs = IO_TLB_SEGSIZE * default_nareas; - else if (is_power_of_2(default_nslabs)) + return nslabs; +} + +/* Return true if default_nslabs is rounded up for the configured areas. */ +static bool round_up_default_nslabs(void) +{ + unsigned long nslabs; + + nslabs = swiotlb_aligned_nslabs(default_nslabs << IO_TLB_SHIFT); + if (nslabs == default_nslabs) return false; - default_nslabs = roundup_pow_of_two(default_nslabs); + + default_nslabs = nslabs; return true; } @@ -290,6 +305,11 @@ unsigned long swiotlb_size_or_default(void) return default_nslabs << IO_TLB_SHIFT; } +static bool __init swiotlb_default_size_changed(void) +{ + return default_nslabs != IO_TLB_DEFAULT_SIZE >> IO_TLB_SHIFT; +} + void __init swiotlb_adjust_size(unsigned long size) { /* @@ -297,13 +317,11 @@ void __init swiotlb_adjust_size(unsigned long size) * architectures such as those supporting memory encryption to * adjust/expand SWIOTLB size for their use. */ - if (default_nslabs != IO_TLB_DEFAULT_SIZE >> IO_TLB_SHIFT) + if (swiotlb_default_size_changed()) return; - size = ALIGN(size, IO_TLB_SIZE); - default_nslabs = ALIGN(size >> IO_TLB_SHIFT, IO_TLB_SEGSIZE); - if (round_up_default_nslabs()) - size = default_nslabs << IO_TLB_SHIFT; + default_nslabs = swiotlb_aligned_nslabs(size); + size = default_nslabs << IO_TLB_SHIFT; pr_info("SWIOTLB bounce buffer size adjusted to %luMB", size >> 20); } @@ -356,24 +374,15 @@ static void swiotlb_mark_pool_used(struct io_tlb_pool *pool) void __init swiotlb_update_mem_attributes(void) { struct io_tlb_pool *mem = &io_tlb_default_mem.defpool; - unsigned long bytes; - - /* - * if platform support memory encryption, swiotlb buffers are - * shared by default. - */ - if (cc_platform_has(CC_ATTR_MEM_ENCRYPT)) - io_tlb_default_mem.cc_shared = true; - else - io_tlb_default_mem.cc_shared = false; if (!mem->nslabs || mem->late_alloc) return; - bytes = PAGE_ALIGN(mem->nslabs << IO_TLB_SHIFT); if (io_tlb_default_mem.cc_shared) { int ret; + unsigned long bytes; + bytes = PAGE_ALIGN(mem->nslabs << IO_TLB_SHIFT); ret = set_memory_decrypted((unsigned long)mem->vaddr, bytes >> PAGE_SHIFT); if (ret) { @@ -439,15 +448,10 @@ static void __init *swiotlb_memblock_alloc(unsigned long nslabs, size_t bytes = PAGE_ALIGN(nslabs << IO_TLB_SHIFT); void *tlb; - /* - * By default allocate the bounce buffer memory from low memory, but - * allow to pick a location everywhere for hypervisors with guest - * memory encryption. - */ - if (flags & SWIOTLB_ANY) - tlb = memblock_alloc(bytes, PAGE_SIZE); - else + if (flags & SWIOTLB_INIT_ADDRESSING_LIMIT) tlb = memblock_alloc_low(bytes, PAGE_SIZE); + else + tlb = memblock_alloc(bytes, PAGE_SIZE); if (!tlb) { pr_warn("%s: Failed to allocate %zu bytes tlb structure\n", @@ -464,22 +468,124 @@ static void __init *swiotlb_memblock_alloc(unsigned long nslabs, return tlb; } +static bool __init swiotlb_kmalloc_needs_bounce(void) +{ + return IS_ENABLED(CONFIG_DMA_BOUNCE_UNALIGNED_KMALLOC) && + (dma_get_cache_alignment() > 1); +} + +/** + * swiotlb_adjusted_size() - get the prospective adjusted SWIOTLB size + * + * Return the size that confidential-computing guest sizing would select for + * the default pool, without changing the configured SWIOTLB size. An + * explicit swiotlb= size is always preserved. An explicit area count is + * included in the size calculation. Automatic area sizing is initialized + * later from the running kernel's possible CPU map and any resulting size + * adjustment is therefore not reflected in the returned size. + */ +unsigned long __init swiotlb_adjusted_size(void) +{ + unsigned long nslabs, size = swiotlb_size_or_default(); + + if (swiotlb_default_size_changed() || + !cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT)) + return size; + /* + * For SEV and TDX and CCA, all DMA has to occur via + * shared/unencrypted pages. Kernel uses SWIOTLB to make this + * happen without changing device drivers. However, depending on + * the workload being run, the default 64MB of SWIOTLB may not be + * enough and SWIOTLB may run out of buffers for DMA, resulting in + * I/O errors and/or performance degradation especially with high + * I/O workloads. + * + * Adjust the default size of SWIOTLB using a percentage of guest + * memory for SWIOTLB buffers. Also, as the SWIOTLB bounce buffer + * memory is allocated from low memory, ensure that the adjusted + * size is within the limits of low available memory. + * + * The percentage of guest memory used here for SWIOTLB buffers is + * more of an approximation of the static adjustment which 64MB for + * <1G, and ~128M to 256M for 1G-to-4G, i.e., the 6% + */ + size = memblock_phys_mem_size() * 6 / 100; + size = clamp_val(size, IO_TLB_DEFAULT_SIZE, SZ_1G); + nslabs = swiotlb_aligned_nslabs(size); + + return nslabs << IO_TLB_SHIFT; +} + +static void __init +swiotlb_adjust_pool_size(enum swiotlb_pool_policy policy) +{ + if (swiotlb_default_size_changed()) + return; + + switch (policy) { + case SWIOTLB_POOL_MINIMAL: { + unsigned long size; + + /* Use 1MB per 1GB of RAM for kmalloc() bouncing. */ + size = DIV_ROUND_UP(memblock_phys_mem_size(), 1024); + swiotlb_adjust_size(min(swiotlb_size_or_default(), size)); + break; + } + case SWIOTLB_POOL_CC_GUEST: + swiotlb_adjust_size(swiotlb_adjusted_size()); + break; + case SWIOTLB_POOL_NONE: + WARN(true, "Cannot adjust SWIOTLB size without a pool\n"); + break; + case SWIOTLB_POOL_DEFAULT: + break; + } +} + +static enum swiotlb_pool_policy __init +swiotlb_select_pool_policy(unsigned int flags) +{ + if (swiotlb_force_disable) + return SWIOTLB_POOL_NONE; + + if (cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT) && + !restricted_dma_pool_present) + return SWIOTLB_POOL_CC_GUEST; + + if (cc_platform_has(CC_ATTR_HOST_MEM_ENCRYPT)) + return SWIOTLB_POOL_DEFAULT; + + if (flags & SWIOTLB_INIT_REMAP) + return SWIOTLB_POOL_DEFAULT; + + if (flags & SWIOTLB_INIT_ADDRESSING_LIMIT) + return SWIOTLB_POOL_DEFAULT; + + if (swiotlb_force_bounce) + return SWIOTLB_POOL_DEFAULT; + + if (swiotlb_kmalloc_needs_bounce()) + return SWIOTLB_POOL_MINIMAL; + + return SWIOTLB_POOL_NONE; +} + /* * Statically reserve bounce buffer space and initialize bounce buffer data * structures for the software IO TLB used to implement the DMA API. */ -void __init swiotlb_init_remap(bool addressing_limit, unsigned int flags, - int (*remap)(void *tlb, unsigned long nslabs)) +void __init swiotlb_init_remap(unsigned int flags, + int (*remap)(void *tlb, unsigned long nslabs)) { struct io_tlb_pool *mem = &io_tlb_default_mem.defpool; + enum swiotlb_pool_policy policy; unsigned long nslabs; unsigned int nareas; size_t alloc_size; void *tlb; - if (!addressing_limit && !swiotlb_force_bounce) - return; - if (swiotlb_force_disable) + policy = swiotlb_select_pool_policy(flags); + if (policy == SWIOTLB_POOL_NONE) return; io_tlb_default_mem.force_bounce = swiotlb_force_bounce; @@ -487,12 +593,18 @@ void __init swiotlb_init_remap(bool addressing_limit, unsigned int flags, #ifdef CONFIG_SWIOTLB_DYNAMIC if (!remap) io_tlb_default_mem.can_grow = true; - if (flags & SWIOTLB_ANY) - io_tlb_default_mem.phys_limit = virt_to_phys(high_memory - 1); - else + if (flags & SWIOTLB_INIT_ADDRESSING_LIMIT) io_tlb_default_mem.phys_limit = ARCH_LOW_ADDRESS_LIMIT; + else + io_tlb_default_mem.phys_limit = virt_to_phys(high_memory - 1); #endif + /* if we have host or guest memory encryption */ + if (cc_platform_has(CC_ATTR_MEM_ENCRYPT)) + io_tlb_default_mem.cc_shared = true; + + swiotlb_adjust_pool_size(policy); + if (!default_nareas) swiotlb_adjust_nareas(num_possible_cpus()); @@ -533,9 +645,9 @@ void __init swiotlb_init_remap(bool addressing_limit, unsigned int flags, swiotlb_print_info(); } -void __init swiotlb_init(bool addressing_limit, unsigned int flags) +void __init swiotlb_init(unsigned int flags) { - swiotlb_init_remap(addressing_limit, flags, NULL); + swiotlb_init_remap(flags, NULL); } /* @@ -2083,6 +2195,8 @@ static int __init rmem_swiotlb_setup(unsigned long node, of_get_flat_dt_prop(node, "no-map", NULL)) return -EINVAL; + restricted_dma_pool_present = true; + pr_info("Reserved memory: created restricted DMA pool at %pa, size %ld MiB\n", &rmem->base, (unsigned long)rmem->size / SZ_1M); return 0;