irqbalance overheads...

Daniel J Blueman daniel.blueman at gmail.com
Mon Mar 29 23:12:44 BST 2010


I'm finding on my 8-core (with Hyperthreading and no way to disable it
correctly) Core-i7 system that irqbalance does more bad than good.

It has clocked up 25 seconds in 2.5 hrs of uptime, and I see it doing
a fair amount of system work each wakeup [1], which is unfortunately
O(n) cores - it is unlikely to have saved significantly more than 25
seconds, particularly since it hasn't spread device (ie non-local)
interrupts so much [2].

Since the kernel hands out MSI vectors spread across cores, and has
choice to enable reception on the lowest-priority core, is irqbalance
really justified being installed by default, rather than say someone
who knows a particular non-MSI interrupt-heavy workload favours it?

Daniel

--- [1]

Process 4909 attached - interrupt to quit
restart_syscall(<... resuming interrupted call ...>) = 0
open("/proc/stat", O_RDONLY)            = 3
fstat(3, {st_mode=S_IFREG|0444, st_size=0, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1,
0) = 0x7fa757d08000
read(3, "cpu  380611 1010827 184959 46959"..., 1024) = 1024
close(3)                                = 0
munmap(0x7fa757d08000, 4096)            = 0
open("/proc/interrupts", O_RDONLY)      = 3
fstat(3, {st_mode=S_IFREG|0444, st_size=0, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1,
0) = 0x7fa757d08000
read(3, "            CPU0       CPU1     "..., 1024) = 1024
read(3, "       0          0          0  "..., 1024) = 1024
read(3, " 39:          0          0      "..., 1024) = 1024
read(3, "949      32656      32039      3"..., 1024) = 568
read(3, "", 1024)                       = 0
close(3)                                = 0
munmap(0x7fa757d08000, 4096)            = 0
open("/sys/devices/system/cpu", O_RDONLY|O_NONBLOCK|O_DIRECTORY|O_CLOEXEC) = 3
getdents(3, /* 19 entries */, 32768)    = 544
openat(3, "cpu0/online", O_RDONLY)      = -1 ENOENT (No such file or directory)
openat(3, "cpu0/topology/core_siblings", O_RDONLY) = 4
fcntl(4, F_GETFL)                       = 0x8000 (flags O_RDONLY|O_LARGEFILE)
fstat(4, {st_mode=S_IFREG|0444, st_size=4096, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1,
0) = 0x7fa757d08000
lseek(4, 0, SEEK_CUR)                   = 0
read(4, "ff\n", 4096)                   = 3
close(4)                                = 0
munmap(0x7fa757d08000, 4096)            = 0
openat(3, "cpu0/cache/index1/shared_cpu_map", O_RDONLY) = 4
fcntl(4, F_GETFL)                       = 0x8000 (flags O_RDONLY|O_LARGEFILE)
fstat(4, {st_mode=S_IFREG|0444, st_size=4096, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1,
0) = 0x7fa757d08000
lseek(4, 0, SEEK_CUR)                   = 0
read(4, "11\n", 4096)                   = 3
close(4)                                = 0
munmap(0x7fa757d08000, 4096)            = 0
openat(3, "cpu0/cache/index2/shared_cpu_map", O_RDONLY) = 4
fcntl(4, F_GETFL)                       = 0x8000 (flags O_RDONLY|O_LARGEFILE)
fstat(4, {st_mode=S_IFREG|0444, st_size=4096, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1,
0) = 0x7fa757d08000
lseek(4, 0, SEEK_CUR)                   = 0
read(4, "11\n", 4096)                   = 3
close(4)                                = 0
munmap(0x7fa757d08000, 4096)            = 0
openat(3, "cpu1/online", O_RDONLY)      = 4
read(4, "1", 1)                         = 1
close(4)                                = 0
openat(3, "cpu1/topology/core_siblings", O_RDONLY) = 4
fcntl(4, F_GETFL)                       = 0x8000 (flags O_RDONLY|O_LARGEFILE)
fstat(4, {st_mode=S_IFREG|0444, st_size=4096, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1,
0) = 0x7fa757d08000
lseek(4, 0, SEEK_CUR)                   = 0
read(4, "ff\n", 4096)                   = 3
close(4)                                = 0
munmap(0x7fa757d08000, 4096)            = 0
openat(3, "cpu1/cache/index1/shared_cpu_map", O_RDONLY) = 4
fcntl(4, F_GETFL)                       = 0x8000 (flags O_RDONLY|O_LARGEFILE)
fstat(4, {st_mode=S_IFREG|0444, st_size=4096, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1,
0) = 0x7fa757d08000
lseek(4, 0, SEEK_CUR)                   = 0
read(4, "22\n", 4096)                   = 3
close(4)                                = 0
munmap(0x7fa757d08000, 4096)            = 0
openat(3, "cpu1/cache/index2/shared_cpu_map", O_RDONLY) = 4
fcntl(4, F_GETFL)                       = 0x8000 (flags O_RDONLY|O_LARGEFILE)
fstat(4, {st_mode=S_IFREG|0444, st_size=4096, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1,
0) = 0x7fa757d08000
lseek(4, 0, SEEK_CUR)                   = 0
read(4, "22\n", 4096)                   = 3
close(4)                                = 0
munmap(0x7fa757d08000, 4096)            = 0
openat(3, "cpu2/online", O_RDONLY)      = 4
read(4, "1", 1)                         = 1
close(4)                                = 0
openat(3, "cpu2/topology/core_siblings", O_RDONLY) = 4
fcntl(4, F_GETFL)                       = 0x8000 (flags O_RDONLY|O_LARGEFILE)
fstat(4, {st_mode=S_IFREG|0444, st_size=4096, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1,
0) = 0x7fa757d08000
lseek(4, 0, SEEK_CUR)                   = 0
read(4, "ff\n", 4096)                   = 3
close(4)                                = 0
munmap(0x7fa757d08000, 4096)            = 0
openat(3, "cpu2/cache/index1/shared_cpu_map", O_RDONLY) = 4
fcntl(4, F_GETFL)                       = 0x8000 (flags O_RDONLY|O_LARGEFILE)
fstat(4, {st_mode=S_IFREG|0444, st_size=4096, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1,
0) = 0x7fa757d08000
lseek(4, 0, SEEK_CUR)                   = 0
read(4, "44\n", 4096)                   = 3
close(4)                                = 0
munmap(0x7fa757d08000, 4096)            = 0
openat(3, "cpu2/cache/index2/shared_cpu_map", O_RDONLY) = 4
fcntl(4, F_GETFL)                       = 0x8000 (flags O_RDONLY|O_LARGEFILE)
fstat(4, {st_mode=S_IFREG|0444, st_size=4096, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1,
0) = 0x7fa757d08000
lseek(4, 0, SEEK_CUR)                   = 0
read(4, "44\n", 4096)                   = 3
close(4)                                = 0
munmap(0x7fa757d08000, 4096)            = 0
openat(3, "cpu3/online", O_RDONLY)      = 4
read(4, "1", 1)                         = 1
close(4)                                = 0
openat(3, "cpu3/topology/core_siblings", O_RDONLY) = 4
fcntl(4, F_GETFL)                       = 0x8000 (flags O_RDONLY|O_LARGEFILE)
fstat(4, {st_mode=S_IFREG|0444, st_size=4096, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1,
0) = 0x7fa757d08000
lseek(4, 0, SEEK_CUR)                   = 0
read(4, "ff\n", 4096)                   = 3
close(4)                                = 0
munmap(0x7fa757d08000, 4096)            = 0
openat(3, "cpu3/cache/index1/shared_cpu_map", O_RDONLY) = 4
fcntl(4, F_GETFL)                       = 0x8000 (flags O_RDONLY|O_LARGEFILE)
fstat(4, {st_mode=S_IFREG|0444, st_size=4096, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1,
0) = 0x7fa757d08000
lseek(4, 0, SEEK_CUR)                   = 0
read(4, "88\n", 4096)                   = 3
close(4)                                = 0
munmap(0x7fa757d08000, 4096)            = 0
openat(3, "cpu3/cache/index2/shared_cpu_map", O_RDONLY) = 4
fcntl(4, F_GETFL)                       = 0x8000 (flags O_RDONLY|O_LARGEFILE)
fstat(4, {st_mode=S_IFREG|0444, st_size=4096, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1,
0) = 0x7fa757d08000
lseek(4, 0, SEEK_CUR)                   = 0
read(4, "88\n", 4096)                   = 3
close(4)                                = 0
munmap(0x7fa757d08000, 4096)            = 0
openat(3, "cpu4/online", O_RDONLY)      = 4
read(4, "1", 1)                         = 1
close(4)                                = 0
openat(3, "cpu4/topology/core_siblings", O_RDONLY) = 4
fcntl(4, F_GETFL)                       = 0x8000 (flags O_RDONLY|O_LARGEFILE)
fstat(4, {st_mode=S_IFREG|0444, st_size=4096, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1,
0) = 0x7fa757d08000
lseek(4, 0, SEEK_CUR)                   = 0
read(4, "ff\n", 4096)                   = 3
close(4)                                = 0
munmap(0x7fa757d08000, 4096)            = 0
openat(3, "cpu4/cache/index1/shared_cpu_map", O_RDONLY) = 4
fcntl(4, F_GETFL)                       = 0x8000 (flags O_RDONLY|O_LARGEFILE)
fstat(4, {st_mode=S_IFREG|0444, st_size=4096, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1,
0) = 0x7fa757d08000
lseek(4, 0, SEEK_CUR)                   = 0
read(4, "11\n", 4096)                   = 3
close(4)                                = 0
munmap(0x7fa757d08000, 4096)            = 0
openat(3, "cpu4/cache/index2/shared_cpu_map", O_RDONLY) = 4
fcntl(4, F_GETFL)                       = 0x8000 (flags O_RDONLY|O_LARGEFILE)
fstat(4, {st_mode=S_IFREG|0444, st_size=4096, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1,
0) = 0x7fa757d08000
lseek(4, 0, SEEK_CUR)                   = 0
read(4, "11\n", 4096)                   = 3
close(4)                                = 0
munmap(0x7fa757d08000, 4096)            = 0
openat(3, "cpu5/online", O_RDONLY)      = 4
read(4, "1", 1)                         = 1
close(4)                                = 0
openat(3, "cpu5/topology/core_siblings", O_RDONLY) = 4
fcntl(4, F_GETFL)                       = 0x8000 (flags O_RDONLY|O_LARGEFILE)
fstat(4, {st_mode=S_IFREG|0444, st_size=4096, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1,
0) = 0x7fa757d08000
lseek(4, 0, SEEK_CUR)                   = 0
read(4, "ff\n", 4096)                   = 3
close(4)                                = 0
munmap(0x7fa757d08000, 4096)            = 0
openat(3, "cpu5/cache/index1/shared_cpu_map", O_RDONLY) = 4
fcntl(4, F_GETFL)                       = 0x8000 (flags O_RDONLY|O_LARGEFILE)
fstat(4, {st_mode=S_IFREG|0444, st_size=4096, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1,
0) = 0x7fa757d08000
lseek(4, 0, SEEK_CUR)                   = 0
read(4, "22\n", 4096)                   = 3
close(4)                                = 0
munmap(0x7fa757d08000, 4096)            = 0
openat(3, "cpu5/cache/index2/shared_cpu_map", O_RDONLY) = 4
fcntl(4, F_GETFL)                       = 0x8000 (flags O_RDONLY|O_LARGEFILE)
fstat(4, {st_mode=S_IFREG|0444, st_size=4096, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1,
0) = 0x7fa757d08000
lseek(4, 0, SEEK_CUR)                   = 0
read(4, "22\n", 4096)                   = 3
close(4)                                = 0
munmap(0x7fa757d08000, 4096)            = 0
openat(3, "cpu6/online", O_RDONLY)      = 4
read(4, "1", 1)                         = 1
close(4)                                = 0
openat(3, "cpu6/topology/core_siblings", O_RDONLY) = 4
fcntl(4, F_GETFL)                       = 0x8000 (flags O_RDONLY|O_LARGEFILE)
fstat(4, {st_mode=S_IFREG|0444, st_size=4096, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1,
0) = 0x7fa757d08000
lseek(4, 0, SEEK_CUR)                   = 0
read(4, "ff\n", 4096)                   = 3
close(4)                                = 0
munmap(0x7fa757d08000, 4096)            = 0
openat(3, "cpu6/cache/index1/shared_cpu_map", O_RDONLY) = 4
fcntl(4, F_GETFL)                       = 0x8000 (flags O_RDONLY|O_LARGEFILE)
fstat(4, {st_mode=S_IFREG|0444, st_size=4096, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1,
0) = 0x7fa757d08000
lseek(4, 0, SEEK_CUR)                   = 0
read(4, "44\n", 4096)                   = 3
close(4)                                = 0
munmap(0x7fa757d08000, 4096)            = 0
openat(3, "cpu6/cache/index2/shared_cpu_map", O_RDONLY) = 4
fcntl(4, F_GETFL)                       = 0x8000 (flags O_RDONLY|O_LARGEFILE)
fstat(4, {st_mode=S_IFREG|0444, st_size=4096, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1,
0) = 0x7fa757d08000
lseek(4, 0, SEEK_CUR)                   = 0
read(4, "44\n", 4096)                   = 3
close(4)                                = 0
munmap(0x7fa757d08000, 4096)            = 0
openat(3, "cpu7/online", O_RDONLY)      = 4
read(4, "1", 1)                         = 1
close(4)                                = 0
openat(3, "cpu7/topology/core_siblings", O_RDONLY) = 4
fcntl(4, F_GETFL)                       = 0x8000 (flags O_RDONLY|O_LARGEFILE)
fstat(4, {st_mode=S_IFREG|0444, st_size=4096, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1,
0) = 0x7fa757d08000
lseek(4, 0, SEEK_CUR)                   = 0
read(4, "ff\n", 4096)                   = 3
close(4)                                = 0
munmap(0x7fa757d08000, 4096)            = 0
openat(3, "cpu7/cache/index1/shared_cpu_map", O_RDONLY) = 4
fcntl(4, F_GETFL)                       = 0x8000 (flags O_RDONLY|O_LARGEFILE)
fstat(4, {st_mode=S_IFREG|0444, st_size=4096, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1,
0) = 0x7fa757d08000
lseek(4, 0, SEEK_CUR)                   = 0
read(4, "88\n", 4096)                   = 3
close(4)                                = 0
munmap(0x7fa757d08000, 4096)            = 0
openat(3, "cpu7/cache/index2/shared_cpu_map", O_RDONLY) = 4
fcntl(4, F_GETFL)                       = 0x8000 (flags O_RDONLY|O_LARGEFILE)
fstat(4, {st_mode=S_IFREG|0444, st_size=4096, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1,
0) = 0x7fa757d08000
lseek(4, 0, SEEK_CUR)                   = 0
read(4, "88\n", 4096)                   = 3
close(4)                                = 0
munmap(0x7fa757d08000, 4096)            = 0
getdents(3, /* 0 entries */, 32768)     = 0
close(3)                                = 0
open("/proc/net/dev", O_RDONLY)         = 3
fstat(3, {st_mode=S_IFREG|0444, st_size=0, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1,
0) = 0x7fa757d08000
read(3, "Inter-|   Receive               "..., 1024) = 578
socket(PF_INET, SOCK_DGRAM, IPPROTO_IP) = 4
ioctl(4, SIOCETHTOOL, 0x7fff05bc7e30)   = -1 EOPNOTSUPP (Operation not
supported)
close(4)                                = 0
socket(PF_INET, SOCK_DGRAM, IPPROTO_IP) = 4
ioctl(4, SIOCETHTOOL, 0x7fff05bc7e30)   = 0
close(4)                                = 0
open("/sys/bus/pci/devices/0000:0b:00.0/irq", O_RDONLY) = 4
fstat(4, {st_mode=S_IFREG|0444, st_size=4096, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1,
0) = 0x7fa757d07000
read(4, "36\n", 4096)                   = 3
close(4)                                = 0
munmap(0x7fa757d07000, 4096)            = 0
socket(PF_INET, SOCK_DGRAM, IPPROTO_IP) = 4
ioctl(4, SIOCETHTOOL, 0x7fff05bc7e30)   = 0
close(4)                                = 0
open("/sys/bus/pci/devices/0000:05:00.0/irq", O_RDONLY) = 4
fstat(4, {st_mode=S_IFREG|0444, st_size=4096, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1,
0) = 0x7fa757d07000
read(4, "39\n", 4096)                   = 3
close(4)                                = 0
munmap(0x7fa757d07000, 4096)            = 0
read(3, "", 1024)                       = 0
close(3)                                = 0
munmap(0x7fa757d08000, 4096)            = 0
nanosleep({9, 657576000}, ^C <unfinished ...>
Process 4909 detached

--- [2]

# cat /proc/interrupts
            CPU0       CPU1       CPU2       CPU3       CPU4
CPU5       CPU6       CPU7
   0:         31          0          0     492865          0
0          0          0   IO-APIC-edge      timer
   1:          1          0          0      12079          0
0          0          0   IO-APIC-edge      i8042
   8:          0          0          0          1          0
0          0          0   IO-APIC-edge      rtc0
   9:          0          0          0         61          0
4        319          0   IO-APIC-fasteoi   acpi
  12:          1          0          0     154174          0
808315          0          0   IO-APIC-edge      i8042
  16:          0          0          0          0          0
0         62          0   IO-APIC-fasteoi   ehci_hcd:usb1, mmc0
  23:          0          0          0          0         45
0          0          0   IO-APIC-fasteoi   ehci_hcd:usb2
  24:     773788          0          0          0          0
0          0          0  HPET_MSI-edge      hpet2
  25:          0     515679          0          0          0
0          0          0  HPET_MSI-edge      hpet3
  26:          0          0     465773          0          0
0          0          0  HPET_MSI-edge      hpet4
  27:          0          0          0     425523          0
0          0          0  HPET_MSI-edge      hpet5
  28:          0          0          0          0     578571
0          0          0  HPET_MSI-edge      hpet6
  35:          1          0          0       4550          0
63602          0          0   PCI-MSI-edge      ahci
  36:          0          0          0          0    9940164
0          0          0   PCI-MSI-edge      eth0
  37:          0          0          0          0          0
16045         75          0   PCI-MSI-edge      hda_intel
  38:          0          0          0          0          0
0         76          0   PCI-MSI-edge      hda_intel
  39:          0          0          0          0          0
0          0          0   PCI-MSI-edge      iwlagn
 NMI:      91935     222671      57154      97747      58357
199249      50607      56186   Non-maskable interrupts
 LOC:        190        175        155        135        115
378385     351065     370748   Local timer interrupts
 SPU:          0          0          0          0          0
0          0          0   Spurious interrupts
 PMI:          0          0          0          0          0
0          0          0   Performance monitoring interrupts
 PND:          0          0          0          0          0
0          0          0   Performance pending work
 RES:      56646      32751      28175      26262      43184
22535      23754      22636   Rescheduling interrupts
 CAL:       3013       3024       2555       3114       2668
2651       2440       2670   Function call interrupts
 TLB:     122387      32478      36100      28363      47961
32659      32043      35694   TLB shootdowns
 TRM:       3018       3004       3006       3009       3018
3004       3006       3009   Thermal event interrupts
 THR:          0          0          0          0          0
0          0          0   Threshold APIC interrupts
 MCE:          0          0          0          0          0
0          0          0   Machine check exceptions
 MCP:         36         33         33         34         34
33         33         33   Machine check polls
 ERR:          7
 MIS:          0
-- 
Daniel J Blueman



More information about the kernel-team mailing list