| lists.openwall.net | lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC | |
|
Open Source and information security mailing list archives
| ||
|
Message-ID: <CABWYdi1=CwMH1McYkVy+HOQcVHWqZerhjqyn8irQq10wee08Zg@mail.gmail.com>
Date: Wed, 30 Jan 2019 14:26:32 -0800
From: Ivan Babrou <ivan@...udflare.com>
To: Linux Kernel Network Developers <netdev@...r.kernel.org>
Cc: mkubecek@...e.cz, "David S. Miller" <davem@...emloft.net>,
Eric Dumazet <edumazet@...gle.com>,
Ignat Korchagin <ignat@...udflare.com>,
Shawn Bohrer <sbohrer@...udflare.com>,
Jakub Sitnicki <jakub@...udflare.com>
Subject: BUG: KASAN: double-free or invalid-free in ip_defrag after upgrade
from 4.19.13
Hey,
Continuing from this thread earlier today:
* https://marc.info/?t=154886729100001&r=1&w=2
We fired up KASAN enabled kernel one one of those machine and this is
what we saw:
$ /tmp/decode_stacktrace.sh
/usr/lib/debug/lib/modules/4.19.18-cloudflare-2019.1.8-1-gcabf55c/vmlinux
linux-4.19.18 < kasan.txt
[ 2300.250278] ==================================================================
[ 2300.266575] BUG: KASAN: double-free or invalid-free in ip_defrag
(net/ipv4/ip_fragment.c:507 net/ipv4/ip_fragment.c:699)
[ 2300.282860]
[ 2300.293415] CPU: 28 PID: 0 Comm: swapper/28 Tainted: G B O
4.19.18-cloudflare-2019.1.8-1-gcabf55c #gcabf55c
[ 2300.313767] Hardware name: Quanta Computer Inc. QuantaPlex
T41S-2U/S2S-MB, BIOS S2S_3B10.03 06/21/2018
[ 2300.332707] Call Trace:
[ 2300.344701] <IRQ>
[ 2300.356188] dump_stack (lib/dump_stack.c:115)
[ 2300.368967] print_address_description (mm/kasan/report.c:257)
[ 2300.383192] ? ip_defrag (net/ipv4/ip_fragment.c:507
net/ipv4/ip_fragment.c:699)
[ 2300.396330] kasan_report_invalid_free (mm/kasan/report.c:337)
[ 2300.410448] ? ip_defrag (net/ipv4/ip_fragment.c:507
net/ipv4/ip_fragment.c:699)
[ 2300.423599] __kasan_slab_free (mm/kasan/kasan.c:502)
[ 2300.437165] ? ip_defrag (net/ipv4/ip_fragment.c:507
net/ipv4/ip_fragment.c:699)
[ 2300.450251] kmem_cache_free (mm/slub.c:1398 mm/slub.c:2953 mm/slub.c:2969)
[ 2300.463497] ip_defrag (net/ipv4/ip_fragment.c:507 net/ipv4/ip_fragment.c:699)
[ 2300.476352] ? ip4_obj_hashfn (net/ipv4/ip_fragment.c:684)
[ 2300.489711] ? ip_route_input_rcu (net/ipv4/route.c:2122)
[ 2300.503416] ip_local_deliver (net/ipv4/ip_input.c:252)
[ 2300.516739] ? ip_call_ra_chain (net/ipv4/ip_input.c:245)
[ 2300.530174] ? ip_rcv_finish_core.isra.19 (net/ipv4/ip_input.c:366)
[ 2300.544535] ? ip_local_deliver (net/ipv4/ip_input.c:518)
[ 2300.557862] ip_rcv (net/ipv4/ip_input.c:518)
[ 2300.569972] ? ip_local_deliver (net/ipv4/ip_input.c:518)
[ 2300.583216] ? ip_rcv_core.isra.20 (net/ipv4/ip_input.c:403)
[ 2300.596683] __netif_receive_skb_one_core (net/core/dev.c:4911)
[ 2300.610732] ? __netif_receive_skb_core (net/core/dev.c:4911)
[ 2300.624666] ? eth_gro_receive (net/ethernet/eth.c:157)
[ 2300.637374] ? recalibrate_cpu_khz (arch/x86/kernel/tsc.c:1066
arch/x86/kernel/tsc.c:1066)
[ 2300.650015] ? ktime_get_with_offset (kernel/time/timekeeping.c:267
kernel/time/timekeeping.c:371 kernel/time/timekeeping.c:799)
[ 2300.662708] ? __build_skb (include/linux/compiler.h:214
arch/x86/include/asm/atomic.h:43
include/asm-generic/atomic-instrumented.h:34 net/core/skbuff.c:300)
[ 2300.674529] netif_receive_skb_internal (net/core/dev.c:5097)
[ 2300.687430] ? dev_cpu_dead (net/core/dev.c:5097)
[ 2300.699351] ? efx_rx_mk_skb+0x5d0/0x1210 sfc]
[ 2300.711999] ? efx_time_sync_event+0x1b0/0x1b0 sfc]
[ 2300.725126] efx_rx_deliver+0x447/0x640 sfc]
[ 2300.737697] ? efx_free_rx_buffers+0x180/0x180 sfc]
[ 2300.750803] ? __efx_rx_packet+0x76e/0x23b0 sfc]
[ 2300.763572] ? efx_ssr+0x19c0/0x19c0 sfc]
[ 2300.775502] ? efx_ef10_ptp_set_ts_config+0x120/0x120 sfc]
[ 2300.788713] ? reweight_entity (kernel/sched/fair.c:2762
kernel/sched/fair.c:2830)
[ 2300.800224] ? efx_poll+0x991/0x12b0 sfc]
[ 2300.811467] ? net_rx_action (arch/x86/include/asm/jump_label.h:36
include/linux/jump_label.h:142 include/trace/events/napi.h:14
net/core/dev.c:6263 net/core/dev.c:6328)
[ 2300.822343] ? napi_complete_done (net/core/dev.c:6306)
[ 2300.833468] ? hrtimer_init (kernel/time/hrtimer.c:1430)
[ 2300.843830] ? recalibrate_cpu_khz (arch/x86/kernel/tsc.c:1066
arch/x86/kernel/tsc.c:1066)
[ 2300.854377] ? _raw_spin_lock (arch/x86/include/asm/atomic.h:194
include/asm-generic/atomic-instrumented.h:58
include/asm-generic/qspinlock.h:85 include/linux/spinlock.h:180
include/linux/spinlock_api_smp.h:143 kernel/locking/spinlock.c:144)
[ 2300.864214] ? handle_irq_event (kernel/irq/handle.c:209)
[ 2300.874106] ? __do_softirq (arch/x86/include/asm/jump_label.h:36
include/linux/jump_label.h:142 include/trace/events/irq.h:142
kernel/softirq.c:293)
[ 2300.883609] ? handle_irq (arch/x86/kernel/irq_64.c:79)
[ 2300.892849] ? irq_exit (kernel/softirq.c:372 kernel/softirq.c:412)
[ 2300.901709] ? do_IRQ (arch/x86/include/asm/irq_regs.h:19
arch/x86/include/asm/irq_regs.h:26 arch/x86/kernel/irq.c:260)
[ 2300.910059] ? common_interrupt (arch/x86/entry/entry_64.S:646)
[ 2300.918862] </IRQ>
[ 2300.925956] ? cpuidle_enter_state (drivers/cpuidle/cpuidle.c:251)
[ 2300.935470] ? do_idle (kernel/sched/idle.c:204 kernel/sched/idle.c:262)
[ 2300.943904] ? arch_cpu_idle_exit (??:?)
[ 2300.953108] ? cpu_startup_entry (kernel/sched/idle.c:368 (discriminator 1))
[ 2300.962229] ? cpu_in_idle (kernel/sched/idle.c:349)
[ 2300.970788] ? clockevents_config.part.12 (kernel/time/clockevents.c:503)
[ 2300.980788] ? start_secondary (arch/x86/kernel/smpboot.c:213)
[ 2300.989915] ? set_cpu_sibling_map (arch/x86/kernel/smpboot.c:213)
[ 2300.999569] ? secondary_startup_64 (arch/x86/kernel/head_64.S:243)
[ 2301.008969]
[ 2301.015480] Allocated by task 0:
[ 2301.023718] kasan_kmalloc (mm/kasan/kasan.c:460 mm/kasan/kasan.c:553)
[ 2301.032340] kmem_cache_alloc (arch/x86/include/asm/jump_label.h:36
include/linux/memcontrol.h:1292 mm/slab.h:447 mm/slub.c:2706
mm/slub.c:2714 mm/slub.c:2719)
[ 2301.041269] __build_skb (net/core/skbuff.c:282 (discriminator 4))
[ 2301.049724] __netdev_alloc_skb (net/core/skbuff.c:423)
[ 2301.058898] efx_rx_mk_skb+0x10e/0x1210 sfc]
[ 2301.068239]
[ 2301.074615] Freed by task 0:
[ 2301.082411] __kasan_slab_free (mm/kasan/kasan.c:522)
[ 2301.091429] kmem_cache_free (mm/slub.c:1398 mm/slub.c:2953 mm/slub.c:2969)
[ 2301.100160] ip_defrag (net/ipv4/ip_fragment.c:507 net/ipv4/ip_fragment.c:699)
[ 2301.108518] ipv4_conntrack_defrag+0x323/0x490 nf_defrag_ipv4]
[ 2301.119408] nf_hook_slow (net/netfilter/core.c:512)
[ 2301.127942] ip_rcv (include/linux/netfilter.h:288 net/ipv4/ip_input.c:524)
[ 2301.135977] __netif_receive_skb_one_core (net/core/dev.c:4911)
[ 2301.145905] netif_receive_skb_internal (net/core/dev.c:5097)
[ 2301.155687] efx_rx_deliver+0x447/0x640 sfc]
[ 2301.164986]
[ 2301.171326] The buggy address belongs to the object at ffff888bd8f543c0
[ 2301.171326] which belongs to the cache skbuff_head_cache of size 232
[ 2301.194483] The buggy address is located 0 bytes inside of
[ 2301.194483] 232-byte region [ffff888bd8f543c0, ffff888bd8f544a8)
[ 2301.216346] The buggy address belongs to the page:
[ 2301.226355] page:ffffea002f63d500 count:1 mapcount:0
mapping:ffff88a03c294540 index:0xffff888bd8f561c0 compound_mapcount: 0
[ 2301.243024] flags: 0x2ffff800008100(slab|head)
[ 2301.253041] raw: 002ffff800008100 ffffea002341d300 0000002d00000002
ffff88a03c294540
[ 2301.266600] raw: ffff888bd8f561c0 0000000080330030 00000001ffffffff
0000000000000000
[ 2301.280190] page dumped because: kasan: bad access detected
[ 2301.291627]
[ 2301.298900] Memory state around the buggy address:
[ 2301.309617] ffff888bd8f54280: fb fb fb fb fb fb fb fb fb fb fb fb
fb fb fb fb
[ 2301.322930] ffff888bd8f54300: fb fb fb fb fb fb fb fb fb fb fb fb
fb fc fc fc
[ 2301.336183] >ffff888bd8f54380: fc fc fc fc fc fc fc fc fb fb fb fb
fb fb fb fb
[ 2301.349449] ^
[ 2301.360817] ffff888bd8f54400: fb fb fb fb fb fb fb fb fb fb fb fb
fb fb fb fb
[ 2301.374248] ffff888bd8f54480: fb fb fb fb fb fc fc fc fc fc fc fc
fc fc fc fc
[ 2301.387663] ==================================================================
[ 2301.401334] ==================================================================
[ 2301.414780] BUG: KASAN: double-free or invalid-free in tcp_v4_rcv
(net/ipv4/tcp_ipv4.c:1693)
[ 2301.428222]
[ 2301.435965] CPU: 28 PID: 0 Comm: swapper/28 Tainted: G B O
4.19.18-cloudflare-2019.1.8-1-gcabf55c #gcabf55c
[ 2301.453552] Hardware name: Quanta Computer Inc. QuantaPlex
T41S-2U/S2S-MB, BIOS S2S_3B10.03 06/21/2018
[ 2301.469737] Call Trace:
[ 2301.478962] <IRQ>
[ 2301.487699] dump_stack (lib/dump_stack.c:115)
[ 2301.497768] print_address_description (mm/kasan/report.c:257)
[ 2301.509256] ? tcp_v4_rcv (net/ipv4/tcp_ipv4.c:1693)
[ 2301.519681] kasan_report_invalid_free (mm/kasan/report.c:337)
[ 2301.531138] ? tcp_v4_rcv (net/ipv4/tcp_ipv4.c:1693)
[ 2301.541628] __kasan_slab_free (mm/kasan/kasan.c:502)
[ 2301.552571] ? tcp_v4_rcv (net/ipv4/tcp_ipv4.c:1693)
[ 2301.563087] kmem_cache_free (mm/slub.c:1398 mm/slub.c:2953 mm/slub.c:2969)
[ 2301.573831] tcp_v4_rcv (net/ipv4/tcp_ipv4.c:1693)
[ 2301.584110] ? icmp_checkentry+0x70/0x70 ip_tables]
[ 2301.595966] ? tcp_v4_early_demux (net/ipv4/tcp_ipv4.c:1693)
[ 2301.607224] ip_local_deliver_finish (net/ipv4/ip_input.c:216)
[ 2301.618764] ip_local_deliver (net/ipv4/ip_input.c:245)
[ 2301.629636] ? ip_call_ra_chain (net/ipv4/ip_input.c:245)
[ 2301.640683] ? ip_sublist_rcv (net/ipv4/ip_input.c:192)
[ 2301.651493] ? ip_local_deliver (net/ipv4/ip_input.c:518)
[ 2301.662419] ip_rcv (net/ipv4/ip_input.c:518)
[ 2301.672198] ? ip_local_deliver (net/ipv4/ip_input.c:518)
[ 2301.683164] ? ip_rcv_core.isra.20 (net/ipv4/ip_input.c:403)
[ 2301.694340] __netif_receive_skb_one_core (net/core/dev.c:4911)
[ 2301.694344] ? __netif_receive_skb_core (net/core/dev.c:4911)
[ 2301.694361] ? eth_gro_receive (net/ethernet/eth.c:157)
[ 2301.694369] ? recalibrate_cpu_khz (arch/x86/kernel/tsc.c:1066
arch/x86/kernel/tsc.c:1066)
[ 2301.694375] ? ktime_get_with_offset (kernel/time/timekeeping.c:267
kernel/time/timekeeping.c:371 kernel/time/timekeeping.c:799)
[ 2301.694385] ? __build_skb (include/linux/compiler.h:214
arch/x86/include/asm/atomic.h:43
include/asm-generic/atomic-instrumented.h:34 net/core/skbuff.c:300)
[ 2301.760745] netif_receive_skb_internal (net/core/dev.c:5097)
[ 2301.760750] ? dev_cpu_dead (net/core/dev.c:5097)
[ 2301.760786] ? efx_rx_mk_skb+0x5d0/0x1210 sfc]
[ 2301.760808] ? efx_time_sync_event+0x1b0/0x1b0 sfc]
[ 2301.760831] efx_rx_deliver+0x447/0x640 sfc]
[ 2301.760851] ? efx_free_rx_buffers+0x180/0x180 sfc]
[ 2301.760872] ? __efx_rx_packet+0x76e/0x23b0 sfc]
[ 2301.835110] ? efx_ssr+0x19c0/0x19c0 sfc]
[ 2301.835142] ? efx_ef10_ptp_set_ts_config+0x120/0x120 sfc]
[ 2301.835152] ? reweight_entity (kernel/sched/fair.c:2762
kernel/sched/fair.c:2830)
[ 2301.835186] ? efx_poll+0x991/0x12b0 sfc]
[ 2301.876013] ? net_rx_action (arch/x86/include/asm/jump_label.h:36
include/linux/jump_label.h:142 include/trace/events/napi.h:14
net/core/dev.c:6263 net/core/dev.c:6328)
[ 2301.876019] ? napi_complete_done (net/core/dev.c:6306)
[ 2301.895619] ? hrtimer_init (kernel/time/hrtimer.c:1430)
[ 2301.895630] ? recalibrate_cpu_khz (arch/x86/kernel/tsc.c:1066
arch/x86/kernel/tsc.c:1066)
[ 2301.914880] ? _raw_spin_lock (arch/x86/include/asm/atomic.h:194
include/asm-generic/atomic-instrumented.h:58
include/asm-generic/qspinlock.h:85 include/linux/spinlock.h:180
include/linux/spinlock_api_smp.h:143 kernel/locking/spinlock.c:144)
[ 2301.914887] ? handle_irq_event (kernel/irq/handle.c:209)
[ 2301.914895] ? __do_softirq (arch/x86/include/asm/jump_label.h:36
include/linux/jump_label.h:142 include/trace/events/irq.h:142
kernel/softirq.c:293)
[ 2301.943072] ? handle_irq (arch/x86/kernel/irq_64.c:79)
[ 2301.943085] ? irq_exit (kernel/softirq.c:372 kernel/softirq.c:412)
[ 2301.960340] ? do_IRQ (arch/x86/include/asm/irq_regs.h:19
arch/x86/include/asm/irq_regs.h:26 arch/x86/kernel/irq.c:260)
[ 2301.960346] ? common_interrupt (arch/x86/entry/entry_64.S:646)
[ 2301.960348] </IRQ>
[ 2301.960359] ? cpuidle_enter_state (drivers/cpuidle/cpuidle.c:251)
[ 2301.960380] ? do_idle (kernel/sched/idle.c:204 kernel/sched/idle.c:262)
[ 2301.960383] ? arch_cpu_idle_exit (??:?)
[ 2301.960389] ? cpu_startup_entry (kernel/sched/idle.c:368 (discriminator 1))
[ 2301.960392] ? cpu_in_idle (kernel/sched/idle.c:349)
[ 2301.960413] ? clockevents_config.part.12 (kernel/time/clockevents.c:503)
[ 2301.960420] ? start_secondary (arch/x86/kernel/smpboot.c:213)
[ 2301.960423] ? set_cpu_sibling_map (arch/x86/kernel/smpboot.c:213)
[ 2301.960430] ? secondary_startup_64 (arch/x86/kernel/head_64.S:243)
[ 2301.960435]
[ 2302.070728] Allocated by task 0:
[ 2302.070739] kasan_kmalloc (mm/kasan/kasan.c:460 mm/kasan/kasan.c:553)
[ 2302.070764] kmem_cache_alloc (arch/x86/include/asm/jump_label.h:36
include/linux/memcontrol.h:1292 mm/slab.h:447 mm/slub.c:2706
mm/slub.c:2714 mm/slub.c:2719)
[ 2302.095562] __build_skb (net/core/skbuff.c:282 (discriminator 4))
[ 2302.095565] __netdev_alloc_skb (net/core/skbuff.c:423)
[ 2302.095604] efx_rx_mk_skb+0x10e/0x1210 sfc]
[ 2302.095611]
[ 2302.127968] Freed by task 0:
[ 2302.127983] __kasan_slab_free (mm/kasan/kasan.c:522)
[ 2302.127993] kmem_cache_free (mm/slub.c:1398 mm/slub.c:2953 mm/slub.c:2969)
[ 2302.152762] ip_defrag (net/ipv4/ip_fragment.c:507 net/ipv4/ip_fragment.c:699)
[ 2302.152768] ipv4_conntrack_defrag+0x323/0x490 nf_defrag_ipv4]
[ 2302.152771] nf_hook_slow (net/netfilter/core.c:512)
[ 2302.152775] ip_rcv (include/linux/netfilter.h:288 net/ipv4/ip_input.c:524)
[ 2302.152779] __netif_receive_skb_one_core (net/core/dev.c:4911)
[ 2302.152782] netif_receive_skb_internal (net/core/dev.c:5097)
[ 2302.152808] efx_rx_deliver+0x447/0x640 sfc]
[ 2302.152810]
[ 2302.152813] The buggy address belongs to the object at ffff888bd8f543c0
[ 2302.152813] which belongs to the cache skbuff_head_cache of size 232
[ 2302.152815] The buggy address is located 0 bytes inside of
[ 2302.152815] 232-byte region [ffff888bd8f543c0, ffff888bd8f544a8)
[ 2302.152816] The buggy address belongs to the page:
[ 2302.152819] page:ffffea002f63d500 count:1 mapcount:0
mapping:ffff88a03c294540 index:0xffff888bd8f561c0 compound_mapcount: 0
[ 2302.152822] flags: 0x2ffff800008100(slab|head)
[ 2302.152827] raw: 002ffff800008100 ffffea002341d300 0000002d00000002
ffff88a03c294540
[ 2302.152829] raw: ffff888bd8f561c0 0000000080330030 00000001ffffffff
0000000000000000
[ 2302.152830] page dumped because: kasan: bad access detected
[ 2302.152830]
[ 2302.152831] Memory state around the buggy address:
[ 2302.152833] ffff888bd8f54280: fb fb fb fb fb fb fb fb fb fb fb fb
fb fb fb fb
[ 2302.152835] ffff888bd8f54300: fb fb fb fb fb fb fb fb fb fb fb fb
fb fc fc fc
[ 2302.152836] >ffff888bd8f54380: fc fc fc fc fc fc fc fc fb fb fb fb
fb fb fb fb
[ 2302.152837] ^
[ 2302.152839] ffff888bd8f54400: fb fb fb fb fb fb fb fb fb fb fb fb
fb fb fb fb
[ 2302.152840] ffff888bd8f54480: fb fb fb fb fb fc fc fc fc fc fc fc
fc fc fc fc
[ 2302.152841] ==================================================================
[ 2302.187379] BUG: Bad page state in process nginx-origin pfn:28b7f8
[ 2302.462537] page:ffffea000a2dfe00 count:-1 mapcount:0
mapping:0000000000000000 index:0x0
[ 2302.462542] flags: 0x2ffff800000000()
[ 2302.462549] raw: 002ffff800000000 dead000000000100 dead000000000200
0000000000000000
[ 2302.462553] raw: 0000000000000000 0000000000000000 ffffffffffffffff
0000000000000000
[ 2302.462554] page dumped because: nonzero _count
[ 2302.462555] Modules linked in: tun xt_connlimit nf_conncount xt_bpf
xt_hashlimit iptable_security cls_flow cls_u32 sch_htb sch_fq md_mod
dm_crypt algif_skcipher af_alg dm_mod dax ip6table_nat nf_nat_ipv6
ip6table_mangle ip6table_security ip6table_raw ip6table_filter
ip6_tables xt_nat iptable_nat nf_nat_ipv4 nf_nat xt_TPROXY
nf_tproxy_ipv6 nf_tproxy_ipv4 xt_connmark iptable_mangle xt_owner
xt_CT xt_socket nf_socket_ipv4 nf_socket_ipv6 iptable_raw
nfnetlink_log xt_NFLOG xt_tcpudp xt_comment xt_conntrack nf_conntrack
nf_defrag_ipv6 nf_defrag_ipv4 xt_mark xt_multiport xt_set
iptable_filter bpfilter ip_set_hash_netport ip_set_hash_net
ip_set_hash_ip ip_set nfnetlink 8021q garp mrp stp llc sb_edac
x86_pkg_temp_thermal kvm_intel kvm ipmi_ssif irqbypass crc32_pclmul
crc32c_intel sfc(O) pcbc aesni_intel aes_x86_64
[ 2302.650012] crypto_simd igb cryptd i2c_algo_bit glue_helper mdio
dca ipmi_si ipmi_devintf ipmi_msghandler efivarfs ip_tables x_tables
[ 2302.650031] CPU: 1 PID: 74997 Comm: nginx-origin Tainted: G B
O 4.19.18-cloudflare-2019.1.8-1-gcabf55c #gcabf55c
[ 2302.650033] Hardware name: Quanta Computer Inc. QuantaPlex
T41S-2U/S2S-MB, BIOS S2S_3B10.03 06/21/2018
[ 2302.650035] Call Trace:
[ 2302.650049] dump_stack (lib/dump_stack.c:115)
[ 2302.650062] bad_page.cold.116 (mm/page_alloc.c:542)
[ 2302.755115] ? si_mem_available (mm/page_alloc.c:507)
[ 2302.755119] ? ksys_write (fs/read_write.c:599)
[ 2302.755126] ? entry_SYSCALL_64_after_hwframe (arch/x86/entry/entry_64.S:247)
[ 2302.755130] ? __switch_to_asm (arch/x86/entry/entry_64.S:373)
[ 2302.755135] get_page_from_freelist (mm/page_alloc.c:2997
mm/page_alloc.c:3342)
[ 2302.755140] ? __switch_to_asm (arch/x86/entry/entry_64.S:373)
[ 2302.755144] ? __switch_to_asm (arch/x86/entry/entry_64.S:373)
[ 2302.755153] ? kasan_unpoison_shadow (mm/kasan/kasan.c:71)
[ 2302.861765] ? __isolate_free_page (mm/page_alloc.c:3252)
[ 2302.861769] ? __kmalloc_node_track_caller (mm/slab.h:448
mm/slub.c:2706 mm/slub.c:4320)
[ 2302.861775] ? __alloc_skb (net/core/skbuff.c:206)
[ 2302.861783] __alloc_pages_nodemask (mm/page_alloc.c:4369)
[ 2302.915129] ? __alloc_pages_slowpath (mm/page_alloc.c:4345)
[ 2302.915135] skb_page_frag_refill (net/core/sock.c:2213)
[ 2302.915139] sk_page_frag_refill (net/core/sock.c:2234)
[ 2302.915144] tcp_sendmsg_locked (net/ipv4/tcp.c:1321)
[ 2302.915149] ? interrupt_entry (arch/x86/entry/entry_64.S:607)
[ 2302.915153] ? kasan_unpoison_shadow (mm/kasan/kasan.c:68)
[ 2302.915160] ? tcp_sendpage (net/ipv4/tcp.c:1175)
[ 2303.003254] ? selinux_secmark_relabel_packet (security/selinux/hooks.c:4532)
[ 2303.003260] ? release_pages (mm/swap.c:716)
[ 2303.028592] ? inet_sk_set_state (net/ipv4/af_inet.c:794)
[ 2303.028596] tcp_sendmsg (net/ipv4/tcp.c:1444)
[ 2303.028603] sock_sendmsg (net/socket.c:622 net/socket.c:631)
[ 2303.028609] sock_write_iter (net/socket.c:901)
[ 2303.075968] ? sock_sendmsg (net/socket.c:884)
[ 2303.075978] __vfs_write (fs/read_write.c:475 fs/read_write.c:487)
[ 2303.075986] ? __handle_mm_fault (mm/memory.c:3211 mm/memory.c:4030
mm/memory.c:4156)
[ 2303.111370] ? kernel_read (fs/read_write.c:483)
[ 2303.111375] ? file_has_perm (security/selinux/hooks.c:1919)
[ 2303.111379] ? bpf_fd_pass (security/selinux/hooks.c:1890)
[ 2303.111386] vfs_write (fs/read_write.c:550)
[ 2303.111389] ksys_write (fs/read_write.c:599)
[ 2303.111394] ? __ia32_sys_read (fs/read_write.c:592)
[ 2303.111401] do_syscall_64 (arch/x86/entry/common.c:290)
[ 2303.188508] ? page_fault (arch/x86/entry/entry_64.S:1161)
[ 2303.188513] entry_SYSCALL_64_after_hwframe (arch/x86/entry/entry_64.S:247)
[ 2303.188517] RIP: 0033:0x7f53e469f190
[ 2303.188521] Code: 2e 0f 1f 84 00 00 00 00 00 90 48 8b 05 39 7e 20
00 c3 0f 1f 84 00 00 00 00 00 83 3d 39 c2 20 00 00 75 10 b8 01 00 00
00 0f 05 <48> 3d 01 f0 ff ff 73 31 c3 48 83 ec 08 e8 ae fc ff ff 48 89
04 24
All code
========
0: 2e 0f 1f 84 00 00 00 nopl %cs:0x0(%rax,%rax,1)
7: 00 00
9: 90 nop
a: 48 8b 05 39 7e 20 00 mov 0x207e39(%rip),%rax # 0x207e4a
11: c3 retq
12: 0f 1f 84 00 00 00 00 nopl 0x0(%rax,%rax,1)
19: 00
1a: 83 3d 39 c2 20 00 00 cmpl $0x0,0x20c239(%rip) # 0x20c25a
21: 75 10 jne 0x33
23: b8 01 00 00 00 mov $0x1,%eax
28: 0f 05 syscall
2a:* 48 3d 01 f0 ff ff cmp $0xfffffffffffff001,%rax <--
trapping instruction
30: 73 31 jae 0x63
32: c3 retq
33: 48 83 ec 08 sub $0x8,%rsp
37: e8 ae fc ff ff callq 0xfffffffffffffcea
3c: 48 89 04 24 mov %rax,(%rsp)
Code starting with the faulting instruction
===========================================
0: 48 3d 01 f0 ff ff cmp $0xfffffffffffff001,%rax
6: 73 31 jae 0x39
8: c3 retq
9: 48 83 ec 08 sub $0x8,%rsp
d: e8 ae fc ff ff callq 0xfffffffffffffcc0
12: 48 89 04 24 mov %rax,(%rsp)
[ 2303.188523] RSP: 002b:00007ffcc6a0c118 EFLAGS: 00000246 ORIG_RAX:
0000000000000001
[ 2303.188528] RAX: ffffffffffffffda RBX: 00005562df6160b3 RCX: 00007f53e469f190
[ 2303.188531] RDX: 000000000000401d RSI: 00005562df6160b3 RDI: 0000000000000d4f
[ 2303.188533] RBP: 00007ffcc6a0c150 R08: 0000000000000005 R09: 0000000060640d3e
[ 2303.188535] R10: 00005562d20f7b10 R11: 0000000000000246 R12: 000000000000401d
[ 2303.188541] R13: 000000000000401d R14: 00007ffcc6a0c3a8 R15: 00005562dc0e6ec8
[ 2303.407074] WARNING: CPU: 21 PID: 74997 at lib/iov_iter.c:825
copy_page_to_iter (lib/iov_iter.c:825 lib/iov_iter.c:832)
[ 2303.420983] Modules linked in: tun xt_connlimit nf_conncount xt_bpf
xt_hashlimit iptable_security cls_flow cls_u32 sch_htb sch_fq md_mod
dm_crypt algif_skcipher af_alg dm_mod dax ip6table_nat nf_nat_ipv6
ip6table_mangle ip6table_security ip6table_raw ip6table_filter
ip6_tables xt_nat iptable_nat nf_nat_ipv4 nf_nat xt_TPROXY
nf_tproxy_ipv6 nf_tproxy_ipv4 xt_connmark iptable_mangle xt_owner
xt_CT xt_socket nf_socket_ipv4 nf_socket_ipv6 iptable_raw
nfnetlink_log xt_NFLOG xt_tcpudp xt_comment xt_conntrack nf_conntrack
nf_defrag_ipv6 nf_defrag_ipv4 xt_mark xt_multiport xt_set
iptable_filter bpfilter ip_set_hash_netport ip_set_hash_net
ip_set_hash_ip ip_set nfnetlink 8021q garp mrp stp llc sb_edac
x86_pkg_temp_thermal kvm_intel kvm ipmi_ssif irqbypass crc32_pclmul
crc32c_intel sfc(O) pcbc aesni_intel aes_x86_64
[ 2303.538009] crypto_simd igb cryptd i2c_algo_bit glue_helper mdio
dca ipmi_si ipmi_devintf ipmi_msghandler efivarfs ip_tables x_tables
[ 2303.538034] CPU: 21 PID: 74997 Comm: nginx-origin Tainted: G B
O 4.19.18-cloudflare-2019.1.8-1-gcabf55c #gcabf55c
[ 2303.538037] Hardware name: Quanta Computer Inc. QuantaPlex
T41S-2U/S2S-MB, BIOS S2S_3B10.03 06/21/2018
[ 2303.538050] RIP: 0010:copy_page_to_iter (??:?)
[ 2303.538055] Code: 07 00 00 4d 85 f6 4c 89 54 24 10 4d 8b 6f 18 4c
89 44 24 08 74 0c 4c 89 ff e8 65 43 ff ff 84 c0 75 12 45 31 f6 e9 d9
fe ff ff <0f> 0b 45 31 f6 e9 cf fe ff ff 49 8d 6f 08 4c 8b 44 24 08 48
b8 00
All code
========
0: 07 (bad)
1: 00 00 add %al,(%rax)
3: 4d 85 f6 test %r14,%r14
6: 4c 89 54 24 10 mov %r10,0x10(%rsp)
b: 4d 8b 6f 18 mov 0x18(%r15),%r13
f: 4c 89 44 24 08 mov %r8,0x8(%rsp)
14: 74 0c je 0x22
16: 4c 89 ff mov %r15,%rdi
19: e8 65 43 ff ff callq 0xffffffffffff4383
1e: 84 c0 test %al,%al
20: 75 12 jne 0x34
22: 45 31 f6 xor %r14d,%r14d
25: e9 d9 fe ff ff jmpq 0xffffffffffffff03
2a:* 0f 0b ud2 <-- trapping instruction
2c: 45 31 f6 xor %r14d,%r14d
2f: e9 cf fe ff ff jmpq 0xffffffffffffff03
34: 49 8d 6f 08 lea 0x8(%r15),%rbp
38: 4c 8b 44 24 08 mov 0x8(%rsp),%r8
3d: 48 rex.W
3e: b8 .byte 0xb8
...
Code starting with the faulting instruction
===========================================
0: 0f 0b ud2
2: 45 31 f6 xor %r14d,%r14d
5: e9 cf fe ff ff jmpq 0xfffffffffffffed9
a: 49 8d 6f 08 lea 0x8(%r15),%rbp
e: 4c 8b 44 24 08 mov 0x8(%rsp),%r8
13: 48 rex.W
14: b8 .byte 0xb8
...
[ 2303.538057] RSP: 0018:ffff88a005e0f7c0 EFLAGS: 00010293
[ 2303.538061] RAX: 0000000000001000 RBX: 000000000000168d RCX: 002ffff800000000
[ 2303.538064] RDX: ffffffffa66bdcb0 RSI: ffffffffa66bdca0 RDI: ffffea000a2dfe00
[ 2303.538066] RBP: 0000000000000005 R08: ffffea000a2dfe00 R09: dffffc0000000000
[ 2303.538069] R10: 0000000000001688 R11: 0000000000000004 R12: ffffea000a2dfe08
[ 2303.538071] R13: ffffea000a2dfe00 R14: ffffea0000000000 R15: ffff88a005e0fc40
[ 2303.538075] FS: 00007f53e4ac0740(0000) GS:ffff888c3f4c0000(0000)
knlGS:0000000000000000
[ 2303.538077] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 2303.538079] CR2: 00005562d36cc000 CR3: 0000002015486001 CR4: 00000000003606e0
[ 2303.538081] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[ 2303.538083] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
[ 2303.538085] Call Trace:
[ 2303.538099] skb_copy_datagram_iter (net/core/datagram.c:453)
[ 2303.538108] tcp_recvmsg (net/ipv4/tcp.c:2104)
[ 2303.538115] ? tcp_get_md5sig_pool (net/ipv4/tcp.c:1917)
[ 2303.538119] ? tcp_poll (include/net/sock.h:1204
include/net/sock.h:1210 net/ipv4/tcp.c:569)
[ 2303.538123] ? tcp_splice_read (net/ipv4/tcp.c:504)
[ 2303.538131] ? bad_area_access_error (arch/x86/mm/fault.c:1213)
[ 2303.538134] ? tcp_splice_read (net/ipv4/tcp.c:504)
[ 2303.538144] ? ep_item_poll.isra.20 (fs/eventpoll.c:892)
[ 2303.538151] ? selinux_secmark_relabel_packet (security/selinux/hooks.c:4532)
[ 2303.538159] inet_recvmsg (net/ipv4/af_inet.c:838)
[ 2303.538164] ? inet_sendpage (net/ipv4/af_inet.c:828)
[ 2303.538172] sock_read_iter (net/socket.c:879)
[ 2303.538177] ? sock_recvmsg (net/socket.c:862)
[ 2303.538187] __vfs_read (fs/read_write.c:407 fs/read_write.c:418)
[ 2303.538193] ? __switch_to_asm (arch/x86/entry/entry_64.S:373)
[ 2303.538197] ? __switch_to_asm (arch/x86/entry/entry_64.S:373)
[ 2303.538202] ? __x64_sys_copy_file_range (fs/read_write.c:414)
[ 2303.538208] ? file_has_perm (security/selinux/hooks.c:1919)
[ 2303.538216] vfs_read (fs/read_write.c:453)
[ 2303.538221] ksys_read (fs/read_write.c:579)
[ 2303.538225] ? kernel_write (fs/read_write.c:572)
[ 2303.538232] do_syscall_64 (arch/x86/entry/common.c:290)
[ 2303.538236] ? prepare_exit_to_usermode (arch/x86/entry/common.c:197)
[ 2303.538240] entry_SYSCALL_64_after_hwframe (arch/x86/entry/entry_64.S:247)
[ 2303.538245] RIP: 0033:0x7f53e469f1f0
[ 2303.538249] Code: 73 01 c3 48 8b 0d b8 7d 20 00 f7 d8 64 89 01 48
83 c8 ff c3 66 0f 1f 44 00 00 83 3d d9 c1 20 00 00 75 10 b8 00 00 00
00 0f 05 <48> 3d 01 f0 ff ff 73 31 c3 48 83 ec 08 e8 4e fc ff ff 48 89
04 24
All code
========
0: 73 01 jae 0x3
2: c3 retq
3: 48 8b 0d b8 7d 20 00 mov 0x207db8(%rip),%rcx # 0x207dc2
a: f7 d8 neg %eax
c: 64 89 01 mov %eax,%fs:(%rcx)
f: 48 83 c8 ff or $0xffffffffffffffff,%rax
13: c3 retq
14: 66 0f 1f 44 00 00 nopw 0x0(%rax,%rax,1)
1a: 83 3d d9 c1 20 00 00 cmpl $0x0,0x20c1d9(%rip) # 0x20c1fa
21: 75 10 jne 0x33
23: b8 00 00 00 00 mov $0x0,%eax
28: 0f 05 syscall
2a:* 48 3d 01 f0 ff ff cmp $0xfffffffffffff001,%rax <--
trapping instruction
30: 73 31 jae 0x63
32: c3 retq
33: 48 83 ec 08 sub $0x8,%rsp
37: e8 4e fc ff ff callq 0xfffffffffffffc8a
3c: 48 89 04 24 mov %rax,(%rsp)
Code starting with the faulting instruction
===========================================
0: 48 3d 01 f0 ff ff cmp $0xfffffffffffff001,%rax
6: 73 31 jae 0x39
8: c3 retq
9: 48 83 ec 08 sub $0x8,%rsp
d: e8 4e fc ff ff callq 0xfffffffffffffc60
12: 48 89 04 24 mov %rax,(%rsp)
[ 2303.538251] RSP: 002b:00007ffcc6a0c188 EFLAGS: 00000246 ORIG_RAX:
0000000000000000
[ 2303.538254] RAX: ffffffffffffffda RBX: 00005562d5f89883 RCX: 00007f53e469f1f0
[ 2303.538256] RDX: 0000000000000005 RSI: 00005562d5f89883 RDI: 0000000000000dfb
[ 2303.538258] RBP: 00007ffcc6a0c1c0 R08: 0000000000000032 R09: 0000000000000020
[ 2303.538260] R10: 00005562d20944de R11: 0000000000000246 R12: 0000000000000005
[ 2303.538262] R13: 00005562dbb17f60 R14: 00005562d2570e80 R15: 00007f53c5866d98
[ 2303.538268] ---[ end trace d791391e77eef582 ]---
[ 2330.200708] kasan: CONFIG_KASAN_INLINE enabled
[ 2330.211020] kasan: GPF could be caused by NULL-ptr deref or user
memory access
[ 2330.224169] general protection fault: 0000 [#1] SMP KASAN PTI
[ 2330.235791] CPU: 28 PID: 69371 Comm: nginx-fl Tainted: G B W
O 4.19.18-cloudflare-2019.1.8-1-gcabf55c #gcabf55c
[ 2330.253036] Hardware name: Quanta Computer Inc. QuantaPlex
T41S-2U/S2S-MB, BIOS S2S_3B10.03 06/21/2018
[ 2330.268679] RIP: 0010:rb_replace_node (??:?)
[ 2330.279645] Code: 55 48 89 f5 53 48 89 fb 48 83 ec 08 80 3c 01 00
0f 85 64 02 00 00 48 b9 00 00 00 00 00 fc ff df 48 89 e8 4c 8b 23 48
c1 e8 03 <0f> b6 34 08 48 8d 45 17 48 89 c7 83 e0 07 48 c1 ef 03 49 83
e4 fc
All code
========
0: 55 push %rbp
1: 48 89 f5 mov %rsi,%rbp
4: 53 push %rbx
5: 48 89 fb mov %rdi,%rbx
8: 48 83 ec 08 sub $0x8,%rsp
c: 80 3c 01 00 cmpb $0x0,(%rcx,%rax,1)
10: 0f 85 64 02 00 00 jne 0x27a
16: 48 b9 00 00 00 00 00 movabs $0xdffffc0000000000,%rcx
1d: fc ff df
20: 48 89 e8 mov %rbp,%rax
23: 4c 8b 23 mov (%rbx),%r12
26: 48 c1 e8 03 shr $0x3,%rax
2a:* 0f b6 34 08 movzbl (%rax,%rcx,1),%esi <-- trapping instruction
2e: 48 8d 45 17 lea 0x17(%rbp),%rax
32: 48 89 c7 mov %rax,%rdi
35: 83 e0 07 and $0x7,%eax
38: 48 c1 ef 03 shr $0x3,%rdi
3c: 49 83 e4 fc and $0xfffffffffffffffc,%r12
Code starting with the faulting instruction
===========================================
0: 0f b6 34 08 movzbl (%rax,%rcx,1),%esi
4: 48 8d 45 17 lea 0x17(%rbp),%rax
8: 48 89 c7 mov %rax,%rdi
b: 83 e0 07 and $0x7,%eax
e: 48 c1 ef 03 shr $0x3,%rdi
12: 49 83 e4 fc and $0xfffffffffffffffc,%r12
[ 2330.311757] RSP: 0018:ffff888c3f687d88 EFLAGS: 00010206
[ 2330.323631] RAX: 0000000000000003 RBX: ffff888c081fc000 RCX: dffffc0000000000
[ 2330.323634] RDX: ffff888c0a5c38e0 RSI: 000000000000001a RDI: ffff888c081fc000
[ 2330.323636] RBP: 000000000000001a R08: fffffbfff4d88d09 R09: fffffbfff4d88d08
[ 2330.323639] R10: fffffbfff4d88d08 R11: ffffffffa6c46847 R12: 0000000030747865
[ 2330.323641] R13: ffff888c0a5c3910 R14: ffff888c0a5c3870 R15: ffff888c0a5c38e0
[ 2330.323644] FS: 00007f3375a30780(0000) GS:ffff888c3f680000(0000)
knlGS:0000000000000000
[ 2330.323647] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 2330.323649] CR2: 00007f19d3da5000 CR3: 0000000bee77a001 CR4: 00000000003606e0
[ 2330.323651] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[ 2330.323653] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
[ 2330.323655] Call Trace:
[ 2330.323658] <IRQ>
[ 2330.323673] ip_expire (net/ipv4/ip_fragment.c:223)
[ 2330.323680] ? ip_check_defrag (net/ipv4/ip_fragment.c:187)
[ 2330.323686] call_timer_fn (arch/x86/include/asm/jump_label.h:36
include/linux/jump_label.h:142 include/trace/events/timer.h:121
kernel/time/timer.c:1327)
[ 2330.323691] run_timer_softirq (kernel/time/timer.c:1364
kernel/time/timer.c:1682 kernel/time/timer.c:1695)
[ 2330.323695] ? add_timer (kernel/time/timer.c:1692)
[ 2330.323699] ? hrtimer_init (kernel/time/hrtimer.c:1430)
[ 2330.323705] ? recalibrate_cpu_khz (arch/x86/kernel/tsc.c:1066
arch/x86/kernel/tsc.c:1066)
[ 2330.323709] ? recalibrate_cpu_khz (arch/x86/kernel/tsc.c:1066
arch/x86/kernel/tsc.c:1066)
[ 2330.323713] ? ktime_get (kernel/time/timekeeping.c:267
kernel/time/timekeeping.c:371 kernel/time/timekeeping.c:756)
[ 2330.323720] ? lapic_timer_set_oneshot (arch/x86/kernel/apic/apic.c:467)
[ 2330.323727] ? clockevents_program_event (kernel/time/clockevents.c:346)
[ 2330.323733] __do_softirq (arch/x86/include/asm/jump_label.h:36
include/linux/jump_label.h:142 include/trace/events/irq.h:142
kernel/softirq.c:293)
[ 2330.323741] irq_exit (kernel/softirq.c:372 kernel/softirq.c:412)
[ 2330.323744] smp_apic_timer_interrupt
(arch/x86/include/asm/irq_regs.h:19 arch/x86/include/asm/irq_regs.h:26
arch/x86/kernel/apic/apic.c:1058)
[ 2330.323751] apic_timer_interrupt (arch/x86/entry/entry_64.S:864)
[ 2330.323753] </IRQ>
[ 2330.323760] RIP: 0010:check_memory_region (??:?)
[ 2330.323765] Code: ff 41 54 49 b9 00 00 00 00 00 fc ff df 4d 89 da
55 49 c1 ea 03 53 48 89 fb 4d 01 ca 48 c1 eb 03 49 8d 6a 01 49 01 d9
49 89 e8 <4c> 89 c8 4d 29 c8 49 83 f8 10 0f 8e 98 00 00 00 44 89 cb 83
e3 07
All code
========
0: ff 41 54 incl 0x54(%rcx)
3: 49 b9 00 00 00 00 00 movabs $0xdffffc0000000000,%r9
a: fc ff df
d: 4d 89 da mov %r11,%r10
10: 55 push %rbp
11: 49 c1 ea 03 shr $0x3,%r10
15: 53 push %rbx
16: 48 89 fb mov %rdi,%rbx
19: 4d 01 ca add %r9,%r10
1c: 48 c1 eb 03 shr $0x3,%rbx
20: 49 8d 6a 01 lea 0x1(%r10),%rbp
24: 49 01 d9 add %rbx,%r9
27: 49 89 e8 mov %rbp,%r8
2a:* 4c 89 c8 mov %r9,%rax <-- trapping instruction
2d: 4d 29 c8 sub %r9,%r8
30: 49 83 f8 10 cmp $0x10,%r8
34: 0f 8e 98 00 00 00 jle 0xd2
3a: 44 89 cb mov %r9d,%ebx
3d: 83 e3 07 and $0x7,%ebx
Code starting with the faulting instruction
===========================================
0: 4c 89 c8 mov %r9,%rax
3: 4d 29 c8 sub %r9,%r8
6: 49 83 f8 10 cmp $0x10,%r8
a: 0f 8e 98 00 00 00 jle 0xa8
10: 44 89 cb mov %r9d,%ebx
13: 83 e3 07 and $0x7,%ebx
[ 2330.323767] RSP: 0018:ffff888bcb66f830 EFLAGS: 00000286 ORIG_RAX:
ffffffffffffff13
[ 2330.323771] RAX: ffff7fffffffffff RBX: 1ffffd400601a58e RCX: ffffffffa5591192
[ 2330.323772] RDX: 0000000000000001 RSI: 0000000000000004 RDI: ffffea00300d2c74
[ 2330.323775] RBP: fffff9400601a58f R08: fffff9400601a58f R09: fffff9400601a58e
[ 2330.323777] R10: fffff9400601a58e R11: ffffea00300d2c77 R12: dffffc0000000000
[ 2330.323779] R13: ffff888bf01d0500 R14: ffff88826902a7c0 R15: ffffea00300d2c40
[ 2330.323787] ? skb_release_data (arch/x86/include/asm/atomic.h:125
(discriminator 3) include/asm-generic/atomic-instrumented.h:260
(discriminator 3) include/linux/page_ref.h:139 (discriminator 3)
include/linux/mm.h:520 (discriminator 3) include/linux/mm.h:942
(discriminator 3) include/linux/skbuff.h:2795 (discriminator 3)
net/core/skbuff.c:564 (discriminator 3))
[ 2330.323793] skb_release_data (arch/x86/include/asm/atomic.h:125
(discriminator 3) include/asm-generic/atomic-instrumented.h:260
(discriminator 3) include/linux/page_ref.h:139 (discriminator 3)
include/linux/mm.h:520 (discriminator 3) include/linux/mm.h:942
(discriminator 3) include/linux/skbuff.h:2795 (discriminator 3)
net/core/skbuff.c:564 (discriminator 3))
[ 2330.323798] __kfree_skb (net/core/skbuff.c:642)
[ 2330.323804] tcp_recvmsg (include/net/sock.h:2405 net/ipv4/tcp.c:2134)
[ 2330.323808] ? sock_def_readable (arch/x86/include/asm/bitops.h:328
include/net/sock.h:828 include/net/sock.h:2181 net/core/sock.c:2698)
[ 2330.323814] ? tcp_get_md5sig_pool (net/ipv4/tcp.c:1917)
[ 2330.323817] ? tcp_poll (include/net/sock.h:1204
include/net/sock.h:1210 net/ipv4/tcp.c:569)
[ 2330.323825] ? unix_stream_sendpage (net/unix/af_unix.c:1829)
[ 2330.323831] ? sock_sendmsg (net/socket.c:622 net/socket.c:631)
[ 2330.323834] ? sock_write_iter (net/socket.c:901)
[ 2330.323838] ? sock_sendmsg (net/socket.c:884)
[ 2330.323846] inet_recvmsg (net/ipv4/af_inet.c:838)
[ 2330.323851] ? inet_sendpage (net/ipv4/af_inet.c:828)
[ 2330.323856] sock_read_iter (net/socket.c:879)
[ 2330.323860] ? sock_recvmsg (net/socket.c:862)
[ 2330.323870] __vfs_read (fs/read_write.c:407 fs/read_write.c:418)
[ 2330.323874] ? __switch_to_asm (arch/x86/entry/entry_64.S:373)
[ 2330.323878] ? __switch_to_asm (arch/x86/entry/entry_64.S:373)
[ 2330.323883] ? __x64_sys_copy_file_range (fs/read_write.c:414)
[ 2330.323890] ? file_has_perm (security/selinux/hooks.c:1919)
[ 2330.323898] vfs_read (fs/read_write.c:453)
[ 2330.323903] ksys_read (fs/read_write.c:579)
[ 2330.323908] ? kernel_write (fs/read_write.c:572)
[ 2330.323911] ? fput (arch/x86/include/asm/atomic64_64.h:118
include/asm-generic/atomic-instrumented.h:269
include/asm-generic/atomic-long.h:218 fs/file_table.c:331)
[ 2330.323918] do_syscall_64 (arch/x86/entry/common.c:290)
[ 2330.323921] ? prepare_exit_to_usermode (arch/x86/entry/common.c:197)
[ 2330.323926] entry_SYSCALL_64_after_hwframe (arch/x86/entry/entry_64.S:247)
[ 2330.323930] RIP: 0033:0x7f337540b20d
[ 2330.323934] Code: c1 20 00 00 75 10 b8 00 00 00 00 0f 05 48 3d 01
f0 ff ff 73 31 c3 48 83 ec 08 e8 4e fc ff ff 48 89 04 24 b8 00 00 00
00 0f 05 <48> 8b 3c 24 48 89 c2 e8 97 fc ff ff 48 89 d0 48 83 c4 08 48
3d 01
All code
========
0: c1 20 00 shll $0x0,(%rax)
3: 00 75 10 add %dh,0x10(%rbp)
6: b8 00 00 00 00 mov $0x0,%eax
b: 0f 05 syscall
d: 48 3d 01 f0 ff ff cmp $0xfffffffffffff001,%rax
13: 73 31 jae 0x46
15: c3 retq
16: 48 83 ec 08 sub $0x8,%rsp
1a: e8 4e fc ff ff callq 0xfffffffffffffc6d
1f: 48 89 04 24 mov %rax,(%rsp)
23: b8 00 00 00 00 mov $0x0,%eax
28: 0f 05 syscall
2a:* 48 8b 3c 24 mov (%rsp),%rdi <-- trapping instruction
2e: 48 89 c2 mov %rax,%rdx
31: e8 97 fc ff ff callq 0xfffffffffffffccd
36: 48 89 d0 mov %rdx,%rax
39: 48 83 c4 08 add $0x8,%rsp
3d: 48 rex.W
3e: 3d .byte 0x3d
3f: 01 .byte 0x1
Code starting with the faulting instruction
===========================================
0: 48 8b 3c 24 mov (%rsp),%rdi
4: 48 89 c2 mov %rax,%rdx
7: e8 97 fc ff ff callq 0xfffffffffffffca3
c: 48 89 d0 mov %rdx,%rax
f: 48 83 c4 08 add $0x8,%rsp
13: 48 rex.W
14: 3d .byte 0x3d
15: 01 .byte 0x1
[ 2330.323936] RSP: 002b:00007ffe077a9510 EFLAGS: 00000293 ORIG_RAX:
0000000000000000
[ 2330.323940] RAX: ffffffffffffffda RBX: 00005640dee9dcb8 RCX: 00007f337540b20d
[ 2330.323942] RDX: 0000000000004018 RSI: 00005640dee9dcb8 RDI: 0000000000000185
[ 2330.323945] RBP: 00007ffe077a9550 R08: 00005640dd627720 R09: 0000000000004000
[ 2330.323947] R10: 0000000000000300 R11: 0000000000000293 R12: 0000000000004018
[ 2330.323949] R13: 00005640dddcb4c0 R14: 0000000000004000 R15: 00007f32435090e0
[ 2330.323954] Modules linked in: tun xt_connlimit nf_conncount xt_bpf
xt_hashlimit iptable_security cls_flow cls_u32 sch_htb sch_fq md_mod
dm_crypt algif_skcipher af_alg dm_mod dax ip6table_nat nf_nat_ipv6
ip6table_mangle ip6table_security ip6table_raw ip6table_filter
ip6_tables xt_nat iptable_nat nf_nat_ipv4 nf_nat xt_TPROXY
nf_tproxy_ipv6 nf_tproxy_ipv4 xt_connmark iptable_mangle xt_owner
xt_CT xt_socket nf_socket_ipv4 nf_socket_ipv6 iptable_raw
nfnetlink_log xt_NFLOG xt_tcpudp xt_comment xt_conntrack nf_conntrack
nf_defrag_ipv6 nf_defrag_ipv4 xt_mark xt_multiport xt_set
iptable_filter bpfilter ip_set_hash_netport ip_set_hash_net
ip_set_hash_ip ip_set nfnetlink 8021q garp mrp stp llc sb_edac
x86_pkg_temp_thermal kvm_intel kvm ipmi_ssif irqbypass crc32_pclmul
crc32c_intel sfc(O) pcbc aesni_intel aes_x86_64
[ 2330.324038] crypto_simd igb cryptd i2c_algo_bit glue_helper mdio
dca ipmi_si ipmi_devintf ipmi_msghandler efivarfs ip_tables x_tables
[ 2330.324111] ---[ end trace d791391e77eef583 ]---
[ 2330.324118] RIP: 0010:rb_replace_node (??:?)
[ 2330.324122] Code: 55 48 89 f5 53 48 89 fb 48 83 ec 08 80 3c 01 00
0f 85 64 02 00 00 48 b9 00 00 00 00 00 fc ff df 48 89 e8 4c 8b 23 48
c1 e8 03 <0f> b6 34 08 48 8d 45 17 48 89 c7 83 e0 07 48 c1 ef 03 49 83
e4 fc
All code
========
0: 55 push %rbp
1: 48 89 f5 mov %rsi,%rbp
4: 53 push %rbx
5: 48 89 fb mov %rdi,%rbx
8: 48 83 ec 08 sub $0x8,%rsp
c: 80 3c 01 00 cmpb $0x0,(%rcx,%rax,1)
10: 0f 85 64 02 00 00 jne 0x27a
16: 48 b9 00 00 00 00 00 movabs $0xdffffc0000000000,%rcx
1d: fc ff df
20: 48 89 e8 mov %rbp,%rax
23: 4c 8b 23 mov (%rbx),%r12
26: 48 c1 e8 03 shr $0x3,%rax
2a:* 0f b6 34 08 movzbl (%rax,%rcx,1),%esi <-- trapping instruction
2e: 48 8d 45 17 lea 0x17(%rbp),%rax
32: 48 89 c7 mov %rax,%rdi
35: 83 e0 07 and $0x7,%eax
38: 48 c1 ef 03 shr $0x3,%rdi
3c: 49 83 e4 fc and $0xfffffffffffffffc,%r12
Code starting with the faulting instruction
===========================================
0: 0f b6 34 08 movzbl (%rax,%rcx,1),%esi
4: 48 8d 45 17 lea 0x17(%rbp),%rax
8: 48 89 c7 mov %rax,%rdi
b: 83 e0 07 and $0x7,%eax
e: 48 c1 ef 03 shr $0x3,%rdi
12: 49 83 e4 fc and $0xfffffffffffffffc,%r12
[ 2330.324129] RSP: 0018:ffff888c3f687d88 EFLAGS: 00010206
[ 2330.324133] RAX: 0000000000000003 RBX: ffff888c081fc000 RCX: dffffc0000000000
[ 2330.324135] RDX: ffff888c0a5c38e0 RSI: 000000000000001a RDI: ffff888c081fc000
[ 2330.324137] RBP: 000000000000001a R08: fffffbfff4d88d09 R09: fffffbfff4d88d08
[ 2330.324140] R10: fffffbfff4d88d08 R11: ffffffffa6c46847 R12: 0000000030747865
[ 2330.324142] R13: ffff888c0a5c3910 R14: ffff888c0a5c3870 R15: ffff888c0a5c38e0
[ 2330.324151] FS: 00007f3375a30780(0000) GS:ffff888c3f680000(0000)
knlGS:0000000000000000
[ 2330.324154] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 2330.324156] CR2: 00007f19d3da5000 CR3: 0000000bee77a001 CR4: 00000000003606e0
[ 2330.324158] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[ 2330.324161] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
[ 2330.324163] Kernel panic - not syncing: Fatal exception in interrupt
[ 2330.324214] Kernel Offset: 0x23000000 from 0xffffffff81000000
(relocation range: 0xffffffff80000000-0xffffffffbfffffff)
This commit from 4.19.14 seems relevant:
* https://github.com/torvalds/linux/commit/d5f9565c8d5ad3cf94982223cfcef1169b0bb60f
As a reminder, we upgraded from 4.19.13 and started seeing crashes.
Powered by blists - more mailing lists