[SRU][jammy:linux-azure][PATCH v2 1/4] Revert "UBUNTU: SAUCE: swiotlb: Split up single swiotlb lock"
John Cabaj
john.cabaj at canonical.com
Tue Jan 28 04:18:03 UTC 2025
BugLink: https://bugs.launchpad.net/bugs/2096813
This reverts commit 8bd834015a5eb685d7fa883a05b03f38117d1a85.
Signed-off-by: John Cabaj <john.cabaj at canonical.com>
Acked-by: Marcelo Henrique Cerri <marcelo.cerri at canonical.com>
Acked-by: Aaron Jauregui <aaron.jauregui at canonical.com>
Signed-off-by: John Cabaj <john.cabaj at canonical.com>
---
.../admin-guide/kernel-parameters.txt | 4 +-
arch/x86/kernel/acpi/boot.c | 4 -
include/linux/swiotlb.h | 27 +--
kernel/dma/swiotlb.c | 179 +++---------------
4 files changed, 30 insertions(+), 184 deletions(-)
diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt
index 2830eab62138..dc9b7b9db377 100644
--- a/Documentation/admin-guide/kernel-parameters.txt
+++ b/Documentation/admin-guide/kernel-parameters.txt
@@ -5783,10 +5783,8 @@
it if 0 is given (See Documentation/admin-guide/cgroup-v1/memory.rst)
swiotlb= [ARM,IA-64,PPC,MIPS,X86]
- Format: { <int> [,<int>] | force | noforce }
+ Format: { <int> | force | noforce }
<int> -- Number of I/O TLB slabs
- <int> -- Second integer after comma. Number of swiotlb
- areas with their own lock. Must be power of 2.
force -- force using of bounce buffers even if they
wouldn't be automatically used by the kernel
noforce -- Never use bounce buffers (for debugging)
diff --git a/arch/x86/kernel/acpi/boot.c b/arch/x86/kernel/acpi/boot.c
index a0fb49ebcb73..44646540061c 100644
--- a/arch/x86/kernel/acpi/boot.c
+++ b/arch/x86/kernel/acpi/boot.c
@@ -22,7 +22,6 @@
#include <linux/efi-bgrt.h>
#include <linux/serial_core.h>
#include <linux/pgtable.h>
-#include <linux/swiotlb.h>
#include <asm/e820/api.h>
#include <asm/irqdomain.h>
@@ -1067,9 +1066,6 @@ static int __init acpi_parse_madt_lapic_entries(void)
return count;
}
- /* This does not take overrides into consideration */
- swiotlb_hint_cpus(max(count, x2count));
-
x2count = acpi_table_parse_madt(ACPI_MADT_TYPE_LOCAL_X2APIC_NMI,
acpi_parse_x2apic_nmi, 0);
count = acpi_table_parse_madt(ACPI_MADT_TYPE_LOCAL_APIC_NMI,
diff --git a/include/linux/swiotlb.h b/include/linux/swiotlb.h
index 0eedb44649a2..b4bdd3ffb65d 100644
--- a/include/linux/swiotlb.h
+++ b/include/linux/swiotlb.h
@@ -38,7 +38,6 @@ enum swiotlb_force {
extern void swiotlb_init(int verbose);
int swiotlb_init_with_tbl(char *tlb, unsigned long nslabs, int verbose);
-void swiotlb_hint_cpus(int cpus);
unsigned long swiotlb_size_or_default(void);
void swiotlb_set_alloc_from_low_pages(bool low);
extern int swiotlb_late_init_with_tbl(char *tlb, unsigned long nslabs);
@@ -73,25 +72,7 @@ struct io_tlb_slot {
};
/**
- * struct io_tlb_area - IO TLB memory area descriptor
- *
- * This is a single area with a single lock.
- *
- * @used: The number of used IO TLB block.
- * @list: The free list describing the number of free entries available
- * from each index.
- * @lock: The lock to protect the above data structures in the map and
- * unmap calls.
- */
-
-struct io_tlb_area {
- unsigned long used;
- struct list_head free_slots;
- spinlock_t lock;
-};
-
-/**
- * struct io_tlb_mem - io tlb memory pool descriptor
+ * struct io_tlb_mem - IO TLB Memory Pool Descriptor
*
* @start: The start address of the swiotlb memory pool. Used to do a quick
* range check to see if the memory was in fact allocated by this
@@ -109,6 +90,8 @@ struct io_tlb_area {
* @index: The index to start searching in the next round.
* @orig_addr: The original address corresponding to a mapped entry.
* @alloc_size: Size of the allocated buffer.
+ * @lock: The lock to protect the above data structures in the map and
+ * unmap calls.
* @debugfs: The dentry to debugfs.
* @late_alloc: %true if allocated using the page allocator
* @force_bounce: %true if swiotlb bouncing is forced
@@ -121,11 +104,13 @@ struct io_tlb_mem {
phys_addr_t end;
void *vaddr;
unsigned long nslabs;
+ unsigned long used;
+ struct list_head free_slots;
+ spinlock_t lock;
struct dentry *debugfs;
bool late_alloc;
bool force_bounce;
bool for_alloc;
- struct io_tlb_area *areas;
struct io_tlb_slot *slots;
unsigned long *bitmap;
};
diff --git a/kernel/dma/swiotlb.c b/kernel/dma/swiotlb.c
index 68348444decf..c28eed64d905 100644
--- a/kernel/dma/swiotlb.c
+++ b/kernel/dma/swiotlb.c
@@ -69,8 +69,6 @@
#define INVALID_PHYS_ADDR (~(phys_addr_t)0)
-#define NUM_AREAS_DEFAULT 1
-
enum swiotlb_force swiotlb_force;
struct io_tlb_mem io_tlb_default_mem;
@@ -87,60 +85,14 @@ static unsigned int max_segment;
static unsigned long default_nslabs = IO_TLB_DEFAULT_SIZE >> IO_TLB_SHIFT;
-static __read_mostly unsigned int area_index_shift;
-static __read_mostly unsigned int area_bits;
-static __read_mostly int num_areas = NUM_AREAS_DEFAULT;
-
-static __init int setup_areas(int n)
-{
- unsigned long nslabs;
-
- if (n < 1 || !is_power_of_2(n)) {
- pr_err("swiotlb: Invalid areas parameter %d\n", n);
- return -EINVAL;
- }
-
- /* Round up number of slabs to the next power of 2.
- * The last area is going be smaller than the rest if default_nslabs is
- * not power of two.
- */
- nslabs = roundup_pow_of_two(default_nslabs);
-
- pr_info("swiotlb: Using %d areas\n", n);
- num_areas = n;
- area_index_shift = __fls(nslabs) - __fls(num_areas);
- area_bits = __fls(n);
- return 0;
-}
-
-/*
- * Can be called from architecture specific code when swiotlb is set up before
- * possible cpus are setup.
- */
-
-void __init swiotlb_hint_cpus(int cpus)
-{
- if (num_areas == NUM_AREAS_DEFAULT && cpus > 1) {
- if (!is_power_of_2(cpus))
- cpus = 1U << (__fls(cpus) + 1);
- setup_areas(cpus);
- }
-}
-
static int __init
setup_io_tlb_npages(char *str)
{
- int ret = 0;
-
if (isdigit(*str)) {
/* avoid tail segment of size < IO_TLB_SEGSIZE */
default_nslabs =
ALIGN(simple_strtoul(str, &str, 0), IO_TLB_SEGSIZE);
}
- if (*str == ',')
- ++str;
- if (isdigit(*str))
- ret = setup_areas(simple_strtoul(str, &str, 0));
if (*str == ',')
++str;
if (!strcmp(str, "force"))
@@ -148,7 +100,7 @@ setup_io_tlb_npages(char *str)
else if (!strcmp(str, "noforce"))
swiotlb_force = SWIOTLB_NO_FORCE;
- return ret;
+ return 0;
}
early_param("swiotlb", setup_io_tlb_npages);
@@ -187,7 +139,6 @@ void __init swiotlb_adjust_size(unsigned long size)
return;
size = ALIGN(size, IO_TLB_SIZE);
default_nslabs = ALIGN(size >> IO_TLB_SHIFT, IO_TLB_SEGSIZE);
- setup_areas(num_areas);
pr_info("SWIOTLB bounce buffer size adjusted to %luMB", size >> 20);
}
@@ -276,22 +227,18 @@ static void swiotlb_init_io_tlb_mem(struct io_tlb_mem *mem, phys_addr_t start,
mem->nslabs = nslabs;
mem->start = start;
mem->end = mem->start + bytes;
+ INIT_LIST_HEAD(&mem->free_slots);
mem->late_alloc = late_alloc;
if (swiotlb_force == SWIOTLB_FORCE)
mem->force_bounce = true;
- for (i = 0; i < num_areas; i++) {
- INIT_LIST_HEAD(&mem->areas[i].free_slots);
- spin_lock_init(&mem->areas[i].lock);
- }
+ spin_lock_init(&mem->lock);
for (i = 0; i < mem->nslabs; i++) {
- int aindex = area_index_shift ? i >> area_index_shift : 0;
__set_bit(i, mem->bitmap);
mem->slots[i].orig_addr = INVALID_PHYS_ADDR;
mem->slots[i].alloc_size = 0;
- list_add_tail(&mem->slots[i].node,
- &mem->areas[aindex].free_slots);
+ list_add_tail(&mem->slots[i].node, &mem->free_slots);
}
/*
@@ -323,10 +270,6 @@ int __init swiotlb_init_with_tbl(char *tlb, unsigned long nslabs, int verbose)
if (!mem->slots)
panic("%s: Failed to allocate %zu bytes align=0x%lx\n",
__func__, alloc_size, PAGE_SIZE);
- mem->areas = memblock_alloc(sizeof(struct io_tlb_area) * num_areas,
- SMP_CACHE_BYTES);
- if (!mem->areas)
- panic("Cannot allocate io_tlb_areas");
mem->bitmap = memblock_alloc(DIV_ROUND_UP(nslabs, BITS_PER_BYTE), SMP_CACHE_BYTES);
if (!mem->bitmap)
@@ -428,7 +371,6 @@ swiotlb_late_init_with_tbl(char *tlb, unsigned long nslabs)
{
struct io_tlb_mem *mem = &io_tlb_default_mem;
unsigned long bytes = nslabs << IO_TLB_SHIFT;
- int order;
if (swiotlb_force == SWIOTLB_NO_FORCE)
return 0;
@@ -438,22 +380,14 @@ swiotlb_late_init_with_tbl(char *tlb, unsigned long nslabs)
return -ENOMEM;
mem->bitmap = kzalloc(DIV_ROUND_UP(nslabs, BITS_PER_BYTE), GFP_KERNEL);
- order = get_order(array_size(sizeof(*mem->slots), nslabs));
- mem->slots = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, order);
+ mem->slots = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
+ get_order(array_size(sizeof(*mem->slots), nslabs)));
if (!mem->slots || !mem->bitmap) {
kfree(mem->bitmap);
kfree(mem->slots);
return -ENOMEM;
}
- mem->areas = (struct io_tlb_area *)kcalloc(num_areas,
- sizeof(struct io_tlb_area),
- GFP_KERNEL);
- if (!mem->areas) {
- free_pages((unsigned long)mem->slots, order);
- return -ENOMEM;
- }
-
set_memory_decrypted((unsigned long)tlb, bytes >> PAGE_SHIFT);
swiotlb_init_io_tlb_mem(mem, virt_to_phys(tlb), nslabs, true);
@@ -478,12 +412,9 @@ void __init swiotlb_exit(void)
set_memory_encrypted(tbl_vaddr, tbl_size >> PAGE_SHIFT);
if (mem->late_alloc) {
- kfree(mem->areas);
free_pages(tbl_vaddr, get_order(tbl_size));
free_pages((unsigned long)mem->slots, get_order(slots_size));
} else {
- memblock_free_late(__pa(mem->areas),
- num_areas * sizeof(struct io_tlb_area));
memblock_free_late(mem->start, tbl_size);
memblock_free_late(__pa(mem->slots), slots_size);
}
@@ -589,32 +520,14 @@ static inline unsigned long get_max_slots(unsigned long boundary_mask)
return nr_slots(boundary_mask + 1);
}
-static inline unsigned long area_nslabs(struct io_tlb_mem *mem)
-{
- return area_index_shift ? (1UL << area_index_shift) + 1 :
- mem->nslabs;
-}
-
-static inline unsigned int area_start(int aindex)
-{
- return aindex << area_index_shift;
-}
-
-static inline unsigned int area_end(struct io_tlb_mem *mem, int aindex)
-{
- return area_start(aindex) + area_nslabs(mem);
-}
-
/*
* Find a suitable number of IO TLB entries size that will fit this request and
* allocate a buffer from that IO TLB pool.
*/
-static int swiotlb_do_find_slots(struct io_tlb_mem *mem,
- struct io_tlb_area *area,
- int area_index,
- struct device *dev, phys_addr_t orig_addr,
- size_t alloc_size)
+static int swiotlb_find_slots(struct device *dev, phys_addr_t orig_addr,
+ size_t alloc_size, unsigned int alloc_align_mask)
{
+ struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
struct io_tlb_slot *slot, *tmp;
unsigned long boundary_mask = dma_get_seg_boundary(dev);
dma_addr_t tbl_dma_addr =
@@ -628,13 +541,12 @@ static int swiotlb_do_find_slots(struct io_tlb_mem *mem,
unsigned long flags;
BUG_ON(!nslots);
- BUG_ON(area_index >= num_areas);
- spin_lock_irqsave(&area->lock, flags);
- if (unlikely(nslots > area_nslabs(mem) - area->used))
+ spin_lock_irqsave(&mem->lock, flags);
+ if (unlikely(nslots > mem->nslabs - mem->used))
goto not_found;
- list_for_each_entry_safe(slot, tmp, &area->free_slots, node) {
+ list_for_each_entry_safe(slot, tmp, &mem->free_slots, node) {
index = slot - mem->slots;
if (orig_addr &&
(slot_addr(tbl_dma_addr, index) & iotlb_align_mask) !=
@@ -667,7 +579,7 @@ static int swiotlb_do_find_slots(struct io_tlb_mem *mem,
}
not_found:
- spin_unlock_irqrestore(&area->lock, flags);
+ spin_unlock_irqrestore(&mem->lock, flags);
return -1;
found:
@@ -678,42 +590,12 @@ static int swiotlb_do_find_slots(struct io_tlb_mem *mem,
list_del(&mem->slots[i].node);
}
- area->used += nslots;
+ mem->used += nslots;
- spin_unlock_irqrestore(&area->lock, flags);
+ spin_unlock_irqrestore(&mem->lock, flags);
return index;
}
-static int swiotlb_find_slots(struct device *dev, phys_addr_t orig_addr,
- size_t alloc_size)
-{
- struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
- int start = raw_smp_processor_id() & ((1U << area_bits) - 1);
- int i, index;
-
- i = start;
- do {
- index = swiotlb_do_find_slots(mem, mem->areas + i, i,
- dev, orig_addr, alloc_size);
- if (index >= 0)
- return index;
- if (++i >= num_areas)
- i = 0;
- } while (i != start);
- return -1;
-}
-
-/* Somewhat racy estimate */
-static unsigned long mem_used(struct io_tlb_mem *mem)
-{
- int i;
- unsigned long used = 0;
-
- for (i = 0; i < num_areas; i++)
- used += mem->areas[i].used;
- return used;
-}
-
phys_addr_t swiotlb_tbl_map_single(struct device *dev, phys_addr_t orig_addr,
size_t mapping_size, size_t alloc_size,
unsigned int alloc_align_mask, enum dma_data_direction dir,
@@ -738,12 +620,12 @@ phys_addr_t swiotlb_tbl_map_single(struct device *dev, phys_addr_t orig_addr,
}
index = swiotlb_find_slots(dev, orig_addr,
- alloc_size + offset);
+ alloc_size + offset, alloc_align_mask);
if (index == -1) {
if (!(attrs & DMA_ATTR_NO_WARN))
dev_warn_ratelimited(dev,
"swiotlb buffer is full (sz: %zd bytes), total %lu (slots), used %lu (slots)\n",
- alloc_size, mem->nslabs, mem_used(mem));
+ alloc_size, mem->nslabs, mem->used);
return (phys_addr_t)DMA_MAPPING_ERROR;
}
@@ -773,14 +655,9 @@ static void swiotlb_release_slots(struct device *dev, phys_addr_t tlb_addr)
unsigned int offset = swiotlb_align_offset(dev, tlb_addr);
int index = (tlb_addr - offset - mem->start) >> IO_TLB_SHIFT;
int nslots = nr_slots(mem->slots[index].alloc_size + offset);
- int aindex = area_index_shift ? index >> area_index_shift : 0;
- struct io_tlb_area *area = mem->areas + aindex;
int i;
- BUG_ON(aindex >= num_areas);
-
- spin_lock_irqsave(&area->lock, flags);
-
+ spin_lock_irqsave(&mem->lock, flags);
/*
* Return the slots to swiotlb, updating bitmap to indicate
* corresponding entries are free.
@@ -789,11 +666,11 @@ static void swiotlb_release_slots(struct device *dev, phys_addr_t tlb_addr)
__set_bit(i, mem->bitmap);
mem->slots[i].orig_addr = INVALID_PHYS_ADDR;
mem->slots[i].alloc_size = 0;
- list_add(&mem->slots[i].node, &area->free_slots);
+ list_add(&mem->slots[i].node, &mem->free_slots);
}
- area->used -= nslots;
- spin_unlock_irqrestore(&area->lock, flags);
+ mem->used -= nslots;
+ spin_unlock_irqrestore(&mem->lock, flags);
}
/*
@@ -885,27 +762,17 @@ bool is_swiotlb_active(struct device *dev)
{
struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
- return mem && mem->start != mem->end;
+ return mem && mem->nslabs;
}
EXPORT_SYMBOL_GPL(is_swiotlb_active);
#ifdef CONFIG_DEBUG_FS
-
-static int used_get(void *data, u64 *val)
-{
- struct io_tlb_mem *mem = (struct io_tlb_mem *)data;
- *val = mem_used(mem);
- return 0;
-}
-
-DEFINE_DEBUGFS_ATTRIBUTE(used_fops, used_get, NULL, "%llu\n");
-
static struct dentry *debugfs_dir;
static void swiotlb_create_debugfs_files(struct io_tlb_mem *mem)
{
debugfs_create_ulong("io_tlb_nslabs", 0400, mem->debugfs, &mem->nslabs);
- debugfs_create_file("io_tlb_used", 0400, mem->debugfs, mem, &used_fops);
+ debugfs_create_ulong("io_tlb_used", 0400, mem->debugfs, &mem->used);
}
static int __init swiotlb_create_default_debugfs(void)
--
2.43.0
More information about the kernel-team
mailing list