| lists.openwall.net | lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC | |
|
Open Source and information security mailing list archives
| ||
|
Message-ID: <CANn89i+4OQihFo8+ONU2tqEexpvq+mLAYPD9kfxQ9U2zzuRuJQ@mail.gmail.com>
Date: Wed, 30 Jan 2019 14:50:59 -0800
From: Eric Dumazet <edumazet@...gle.com>
To: Ivan Babrou <ivan@...udflare.com>
Cc: Linux Kernel Network Developers <netdev@...r.kernel.org>,
mkubecek@...e.cz, "David S. Miller" <davem@...emloft.net>,
Ignat Korchagin <ignat@...udflare.com>,
Shawn Bohrer <sbohrer@...udflare.com>,
Jakub Sitnicki <jakub@...udflare.com>
Subject: Re: BUG: KASAN: double-free or invalid-free in ip_defrag after
upgrade from 4.19.13
On Wed, Jan 30, 2019 at 2:26 PM Ivan Babrou <ivan@...udflare.com> wrote:
>
> Hey,
>
> Continuing from this thread earlier today:
>
> * https://marc.info/?t=154886729100001&r=1&w=2
>
> We fired up KASAN enabled kernel one one of those machine and this is
> what we saw:
>
> $ /tmp/decode_stacktrace.sh
> /usr/lib/debug/lib/modules/4.19.18-cloudflare-2019.1.8-1-gcabf55c/vmlinux
> linux-4.19.18 < kasan.txt
> [ 2300.250278] ==================================================================
> [ 2300.266575] BUG: KASAN: double-free or invalid-free in ip_defrag
> (net/ipv4/ip_fragment.c:507 net/ipv4/ip_fragment.c:699)
> [ 2300.282860]
> [ 2300.293415] CPU: 28 PID: 0 Comm: swapper/28 Tainted: G B O
> 4.19.18-cloudflare-2019.1.8-1-gcabf55c #gcabf55c
> [ 2300.313767] Hardware name: Quanta Computer Inc. QuantaPlex
> T41S-2U/S2S-MB, BIOS S2S_3B10.03 06/21/2018
> [ 2300.332707] Call Trace:
> [ 2300.344701] <IRQ>
> [ 2300.356188] dump_stack (lib/dump_stack.c:115)
> [ 2300.368967] print_address_description (mm/kasan/report.c:257)
> [ 2300.383192] ? ip_defrag (net/ipv4/ip_fragment.c:507
> net/ipv4/ip_fragment.c:699)
> [ 2300.396330] kasan_report_invalid_free (mm/kasan/report.c:337)
> [ 2300.410448] ? ip_defrag (net/ipv4/ip_fragment.c:507
> net/ipv4/ip_fragment.c:699)
> [ 2300.423599] __kasan_slab_free (mm/kasan/kasan.c:502)
> [ 2300.437165] ? ip_defrag (net/ipv4/ip_fragment.c:507
> net/ipv4/ip_fragment.c:699)
> [ 2300.450251] kmem_cache_free (mm/slub.c:1398 mm/slub.c:2953 mm/slub.c:2969)
> [ 2300.463497] ip_defrag (net/ipv4/ip_fragment.c:507 net/ipv4/ip_fragment.c:699)
> [ 2300.476352] ? ip4_obj_hashfn (net/ipv4/ip_fragment.c:684)
> [ 2300.489711] ? ip_route_input_rcu (net/ipv4/route.c:2122)
> [ 2300.503416] ip_local_deliver (net/ipv4/ip_input.c:252)
> [ 2300.516739] ? ip_call_ra_chain (net/ipv4/ip_input.c:245)
> [ 2300.530174] ? ip_rcv_finish_core.isra.19 (net/ipv4/ip_input.c:366)
> [ 2300.544535] ? ip_local_deliver (net/ipv4/ip_input.c:518)
> [ 2300.557862] ip_rcv (net/ipv4/ip_input.c:518)
> [ 2300.569972] ? ip_local_deliver (net/ipv4/ip_input.c:518)
> [ 2300.583216] ? ip_rcv_core.isra.20 (net/ipv4/ip_input.c:403)
> [ 2300.596683] __netif_receive_skb_one_core (net/core/dev.c:4911)
> [ 2300.610732] ? __netif_receive_skb_core (net/core/dev.c:4911)
> [ 2300.624666] ? eth_gro_receive (net/ethernet/eth.c:157)
> [ 2300.637374] ? recalibrate_cpu_khz (arch/x86/kernel/tsc.c:1066
> arch/x86/kernel/tsc.c:1066)
> [ 2300.650015] ? ktime_get_with_offset (kernel/time/timekeeping.c:267
> kernel/time/timekeeping.c:371 kernel/time/timekeeping.c:799)
> [ 2300.662708] ? __build_skb (include/linux/compiler.h:214
> arch/x86/include/asm/atomic.h:43
> include/asm-generic/atomic-instrumented.h:34 net/core/skbuff.c:300)
> [ 2300.674529] netif_receive_skb_internal (net/core/dev.c:5097)
> [ 2300.687430] ? dev_cpu_dead (net/core/dev.c:5097)
> [ 2300.699351] ? efx_rx_mk_skb+0x5d0/0x1210 sfc]
> [ 2300.711999] ? efx_time_sync_event+0x1b0/0x1b0 sfc]
> [ 2300.725126] efx_rx_deliver+0x447/0x640 sfc]
> [ 2300.737697] ? efx_free_rx_buffers+0x180/0x180 sfc]
> [ 2300.750803] ? __efx_rx_packet+0x76e/0x23b0 sfc]
> [ 2300.763572] ? efx_ssr+0x19c0/0x19c0 sfc]
> [ 2300.775502] ? efx_ef10_ptp_set_ts_config+0x120/0x120 sfc]
> [ 2300.788713] ? reweight_entity (kernel/sched/fair.c:2762
> kernel/sched/fair.c:2830)
> [ 2300.800224] ? efx_poll+0x991/0x12b0 sfc]
> [ 2300.811467] ? net_rx_action (arch/x86/include/asm/jump_label.h:36
> include/linux/jump_label.h:142 include/trace/events/napi.h:14
> net/core/dev.c:6263 net/core/dev.c:6328)
> [ 2300.822343] ? napi_complete_done (net/core/dev.c:6306)
> [ 2300.833468] ? hrtimer_init (kernel/time/hrtimer.c:1430)
> [ 2300.843830] ? recalibrate_cpu_khz (arch/x86/kernel/tsc.c:1066
> arch/x86/kernel/tsc.c:1066)
> [ 2300.854377] ? _raw_spin_lock (arch/x86/include/asm/atomic.h:194
> include/asm-generic/atomic-instrumented.h:58
> include/asm-generic/qspinlock.h:85 include/linux/spinlock.h:180
> include/linux/spinlock_api_smp.h:143 kernel/locking/spinlock.c:144)
> [ 2300.864214] ? handle_irq_event (kernel/irq/handle.c:209)
> [ 2300.874106] ? __do_softirq (arch/x86/include/asm/jump_label.h:36
> include/linux/jump_label.h:142 include/trace/events/irq.h:142
> kernel/softirq.c:293)
> [ 2300.883609] ? handle_irq (arch/x86/kernel/irq_64.c:79)
> [ 2300.892849] ? irq_exit (kernel/softirq.c:372 kernel/softirq.c:412)
> [ 2300.901709] ? do_IRQ (arch/x86/include/asm/irq_regs.h:19
> arch/x86/include/asm/irq_regs.h:26 arch/x86/kernel/irq.c:260)
> [ 2300.910059] ? common_interrupt (arch/x86/entry/entry_64.S:646)
> [ 2300.918862] </IRQ>
> [ 2300.925956] ? cpuidle_enter_state (drivers/cpuidle/cpuidle.c:251)
> [ 2300.935470] ? do_idle (kernel/sched/idle.c:204 kernel/sched/idle.c:262)
> [ 2300.943904] ? arch_cpu_idle_exit (??:?)
> [ 2300.953108] ? cpu_startup_entry (kernel/sched/idle.c:368 (discriminator 1))
> [ 2300.962229] ? cpu_in_idle (kernel/sched/idle.c:349)
> [ 2300.970788] ? clockevents_config.part.12 (kernel/time/clockevents.c:503)
> [ 2300.980788] ? start_secondary (arch/x86/kernel/smpboot.c:213)
> [ 2300.989915] ? set_cpu_sibling_map (arch/x86/kernel/smpboot.c:213)
> [ 2300.999569] ? secondary_startup_64 (arch/x86/kernel/head_64.S:243)
> [ 2301.008969]
> [ 2301.015480] Allocated by task 0:
> [ 2301.023718] kasan_kmalloc (mm/kasan/kasan.c:460 mm/kasan/kasan.c:553)
> [ 2301.032340] kmem_cache_alloc (arch/x86/include/asm/jump_label.h:36
> include/linux/memcontrol.h:1292 mm/slab.h:447 mm/slub.c:2706
> mm/slub.c:2714 mm/slub.c:2719)
> [ 2301.041269] __build_skb (net/core/skbuff.c:282 (discriminator 4))
> [ 2301.049724] __netdev_alloc_skb (net/core/skbuff.c:423)
> [ 2301.058898] efx_rx_mk_skb+0x10e/0x1210 sfc]
> [ 2301.068239]
> [ 2301.074615] Freed by task 0:
> [ 2301.082411] __kasan_slab_free (mm/kasan/kasan.c:522)
> [ 2301.091429] kmem_cache_free (mm/slub.c:1398 mm/slub.c:2953 mm/slub.c:2969)
> [ 2301.100160] ip_defrag (net/ipv4/ip_fragment.c:507 net/ipv4/ip_fragment.c:699)
> [ 2301.108518] ipv4_conntrack_defrag+0x323/0x490 nf_defrag_ipv4]
> [ 2301.119408] nf_hook_slow (net/netfilter/core.c:512)
> [ 2301.127942] ip_rcv (include/linux/netfilter.h:288 net/ipv4/ip_input.c:524)
> [ 2301.135977] __netif_receive_skb_one_core (net/core/dev.c:4911)
> [ 2301.145905] netif_receive_skb_internal (net/core/dev.c:5097)
> [ 2301.155687] efx_rx_deliver+0x447/0x640 sfc]
> [ 2301.164986]
> [ 2301.171326] The buggy address belongs to the object at ffff888bd8f543c0
> [ 2301.171326] which belongs to the cache skbuff_head_cache of size 232
> [ 2301.194483] The buggy address is located 0 bytes inside of
> [ 2301.194483] 232-byte region [ffff888bd8f543c0, ffff888bd8f544a8)
> [ 2301.216346] The buggy address belongs to the page:
> [ 2301.226355] page:ffffea002f63d500 count:1 mapcount:0
> mapping:ffff88a03c294540 index:0xffff888bd8f561c0 compound_mapcount: 0
> [ 2301.243024] flags: 0x2ffff800008100(slab|head)
> [ 2301.253041] raw: 002ffff800008100 ffffea002341d300 0000002d00000002
> ffff88a03c294540
> [ 2301.266600] raw: ffff888bd8f561c0 0000000080330030 00000001ffffffff
> 0000000000000000
> [ 2301.280190] page dumped because: kasan: bad access detected
> [ 2301.291627]
> [ 2301.298900] Memory state around the buggy address:
> [ 2301.309617] ffff888bd8f54280: fb fb fb fb fb fb fb fb fb fb fb fb
> fb fb fb fb
> [ 2301.322930] ffff888bd8f54300: fb fb fb fb fb fb fb fb fb fb fb fb
> fb fc fc fc
> [ 2301.336183] >ffff888bd8f54380: fc fc fc fc fc fc fc fc fb fb fb fb
> fb fb fb fb
> [ 2301.349449] ^
> [ 2301.360817] ffff888bd8f54400: fb fb fb fb fb fb fb fb fb fb fb fb
> fb fb fb fb
> [ 2301.374248] ffff888bd8f54480: fb fb fb fb fb fc fc fc fc fc fc fc
> fc fc fc fc
> [ 2301.387663] ==================================================================
> [ 2301.401334] ==================================================================
> [ 2301.414780] BUG: KASAN: double-free or invalid-free in tcp_v4_rcv
> (net/ipv4/tcp_ipv4.c:1693)
> [ 2301.428222]
> [ 2301.435965] CPU: 28 PID: 0 Comm: swapper/28 Tainted: G B O
> 4.19.18-cloudflare-2019.1.8-1-gcabf55c #gcabf55c
> [ 2301.453552] Hardware name: Quanta Computer Inc. QuantaPlex
> T41S-2U/S2S-MB, BIOS S2S_3B10.03 06/21/2018
> [ 2301.469737] Call Trace:
> [ 2301.478962] <IRQ>
> [ 2301.487699] dump_stack (lib/dump_stack.c:115)
> [ 2301.497768] print_address_description (mm/kasan/report.c:257)
> [ 2301.509256] ? tcp_v4_rcv (net/ipv4/tcp_ipv4.c:1693)
> [ 2301.519681] kasan_report_invalid_free (mm/kasan/report.c:337)
> [ 2301.531138] ? tcp_v4_rcv (net/ipv4/tcp_ipv4.c:1693)
> [ 2301.541628] __kasan_slab_free (mm/kasan/kasan.c:502)
> [ 2301.552571] ? tcp_v4_rcv (net/ipv4/tcp_ipv4.c:1693)
> [ 2301.563087] kmem_cache_free (mm/slub.c:1398 mm/slub.c:2953 mm/slub.c:2969)
> [ 2301.573831] tcp_v4_rcv (net/ipv4/tcp_ipv4.c:1693)
> [ 2301.584110] ? icmp_checkentry+0x70/0x70 ip_tables]
> [ 2301.595966] ? tcp_v4_early_demux (net/ipv4/tcp_ipv4.c:1693)
> [ 2301.607224] ip_local_deliver_finish (net/ipv4/ip_input.c:216)
> [ 2301.618764] ip_local_deliver (net/ipv4/ip_input.c:245)
> [ 2301.629636] ? ip_call_ra_chain (net/ipv4/ip_input.c:245)
> [ 2301.640683] ? ip_sublist_rcv (net/ipv4/ip_input.c:192)
> [ 2301.651493] ? ip_local_deliver (net/ipv4/ip_input.c:518)
> [ 2301.662419] ip_rcv (net/ipv4/ip_input.c:518)
> [ 2301.672198] ? ip_local_deliver (net/ipv4/ip_input.c:518)
> [ 2301.683164] ? ip_rcv_core.isra.20 (net/ipv4/ip_input.c:403)
> [ 2301.694340] __netif_receive_skb_one_core (net/core/dev.c:4911)
> [ 2301.694344] ? __netif_receive_skb_core (net/core/dev.c:4911)
> [ 2301.694361] ? eth_gro_receive (net/ethernet/eth.c:157)
> [ 2301.694369] ? recalibrate_cpu_khz (arch/x86/kernel/tsc.c:1066
> arch/x86/kernel/tsc.c:1066)
> [ 2301.694375] ? ktime_get_with_offset (kernel/time/timekeeping.c:267
> kernel/time/timekeeping.c:371 kernel/time/timekeeping.c:799)
> [ 2301.694385] ? __build_skb (include/linux/compiler.h:214
> arch/x86/include/asm/atomic.h:43
> include/asm-generic/atomic-instrumented.h:34 net/core/skbuff.c:300)
> [ 2301.760745] netif_receive_skb_internal (net/core/dev.c:5097)
> [ 2301.760750] ? dev_cpu_dead (net/core/dev.c:5097)
> [ 2301.760786] ? efx_rx_mk_skb+0x5d0/0x1210 sfc]
> [ 2301.760808] ? efx_time_sync_event+0x1b0/0x1b0 sfc]
> [ 2301.760831] efx_rx_deliver+0x447/0x640 sfc]
> [ 2301.760851] ? efx_free_rx_buffers+0x180/0x180 sfc]
> [ 2301.760872] ? __efx_rx_packet+0x76e/0x23b0 sfc]
> [ 2301.835110] ? efx_ssr+0x19c0/0x19c0 sfc]
> [ 2301.835142] ? efx_ef10_ptp_set_ts_config+0x120/0x120 sfc]
> [ 2301.835152] ? reweight_entity (kernel/sched/fair.c:2762
> kernel/sched/fair.c:2830)
> [ 2301.835186] ? efx_poll+0x991/0x12b0 sfc]
> [ 2301.876013] ? net_rx_action (arch/x86/include/asm/jump_label.h:36
> include/linux/jump_label.h:142 include/trace/events/napi.h:14
> net/core/dev.c:6263 net/core/dev.c:6328)
> [ 2301.876019] ? napi_complete_done (net/core/dev.c:6306)
> [ 2301.895619] ? hrtimer_init (kernel/time/hrtimer.c:1430)
> [ 2301.895630] ? recalibrate_cpu_khz (arch/x86/kernel/tsc.c:1066
> arch/x86/kernel/tsc.c:1066)
> [ 2301.914880] ? _raw_spin_lock (arch/x86/include/asm/atomic.h:194
> include/asm-generic/atomic-instrumented.h:58
> include/asm-generic/qspinlock.h:85 include/linux/spinlock.h:180
> include/linux/spinlock_api_smp.h:143 kernel/locking/spinlock.c:144)
> [ 2301.914887] ? handle_irq_event (kernel/irq/handle.c:209)
> [ 2301.914895] ? __do_softirq (arch/x86/include/asm/jump_label.h:36
> include/linux/jump_label.h:142 include/trace/events/irq.h:142
> kernel/softirq.c:293)
> [ 2301.943072] ? handle_irq (arch/x86/kernel/irq_64.c:79)
> [ 2301.943085] ? irq_exit (kernel/softirq.c:372 kernel/softirq.c:412)
> [ 2301.960340] ? do_IRQ (arch/x86/include/asm/irq_regs.h:19
> arch/x86/include/asm/irq_regs.h:26 arch/x86/kernel/irq.c:260)
> [ 2301.960346] ? common_interrupt (arch/x86/entry/entry_64.S:646)
> [ 2301.960348] </IRQ>
> [ 2301.960359] ? cpuidle_enter_state (drivers/cpuidle/cpuidle.c:251)
> [ 2301.960380] ? do_idle (kernel/sched/idle.c:204 kernel/sched/idle.c:262)
> [ 2301.960383] ? arch_cpu_idle_exit (??:?)
> [ 2301.960389] ? cpu_startup_entry (kernel/sched/idle.c:368 (discriminator 1))
> [ 2301.960392] ? cpu_in_idle (kernel/sched/idle.c:349)
> [ 2301.960413] ? clockevents_config.part.12 (kernel/time/clockevents.c:503)
> [ 2301.960420] ? start_secondary (arch/x86/kernel/smpboot.c:213)
> [ 2301.960423] ? set_cpu_sibling_map (arch/x86/kernel/smpboot.c:213)
> [ 2301.960430] ? secondary_startup_64 (arch/x86/kernel/head_64.S:243)
> [ 2301.960435]
> [ 2302.070728] Allocated by task 0:
> [ 2302.070739] kasan_kmalloc (mm/kasan/kasan.c:460 mm/kasan/kasan.c:553)
> [ 2302.070764] kmem_cache_alloc (arch/x86/include/asm/jump_label.h:36
> include/linux/memcontrol.h:1292 mm/slab.h:447 mm/slub.c:2706
> mm/slub.c:2714 mm/slub.c:2719)
> [ 2302.095562] __build_skb (net/core/skbuff.c:282 (discriminator 4))
> [ 2302.095565] __netdev_alloc_skb (net/core/skbuff.c:423)
> [ 2302.095604] efx_rx_mk_skb+0x10e/0x1210 sfc]
> [ 2302.095611]
> [ 2302.127968] Freed by task 0:
> [ 2302.127983] __kasan_slab_free (mm/kasan/kasan.c:522)
> [ 2302.127993] kmem_cache_free (mm/slub.c:1398 mm/slub.c:2953 mm/slub.c:2969)
> [ 2302.152762] ip_defrag (net/ipv4/ip_fragment.c:507 net/ipv4/ip_fragment.c:699)
> [ 2302.152768] ipv4_conntrack_defrag+0x323/0x490 nf_defrag_ipv4]
> [ 2302.152771] nf_hook_slow (net/netfilter/core.c:512)
> [ 2302.152775] ip_rcv (include/linux/netfilter.h:288 net/ipv4/ip_input.c:524)
> [ 2302.152779] __netif_receive_skb_one_core (net/core/dev.c:4911)
> [ 2302.152782] netif_receive_skb_internal (net/core/dev.c:5097)
> [ 2302.152808] efx_rx_deliver+0x447/0x640 sfc]
> [ 2302.152810]
> [ 2302.152813] The buggy address belongs to the object at ffff888bd8f543c0
> [ 2302.152813] which belongs to the cache skbuff_head_cache of size 232
> [ 2302.152815] The buggy address is located 0 bytes inside of
> [ 2302.152815] 232-byte region [ffff888bd8f543c0, ffff888bd8f544a8)
> [ 2302.152816] The buggy address belongs to the page:
> [ 2302.152819] page:ffffea002f63d500 count:1 mapcount:0
> mapping:ffff88a03c294540 index:0xffff888bd8f561c0 compound_mapcount: 0
> [ 2302.152822] flags: 0x2ffff800008100(slab|head)
> [ 2302.152827] raw: 002ffff800008100 ffffea002341d300 0000002d00000002
> ffff88a03c294540
> [ 2302.152829] raw: ffff888bd8f561c0 0000000080330030 00000001ffffffff
> 0000000000000000
> [ 2302.152830] page dumped because: kasan: bad access detected
> [ 2302.152830]
> [ 2302.152831] Memory state around the buggy address:
> [ 2302.152833] ffff888bd8f54280: fb fb fb fb fb fb fb fb fb fb fb fb
> fb fb fb fb
> [ 2302.152835] ffff888bd8f54300: fb fb fb fb fb fb fb fb fb fb fb fb
> fb fc fc fc
> [ 2302.152836] >ffff888bd8f54380: fc fc fc fc fc fc fc fc fb fb fb fb
> fb fb fb fb
> [ 2302.152837] ^
> [ 2302.152839] ffff888bd8f54400: fb fb fb fb fb fb fb fb fb fb fb fb
> fb fb fb fb
> [ 2302.152840] ffff888bd8f54480: fb fb fb fb fb fc fc fc fc fc fc fc
> fc fc fc fc
> [ 2302.152841] ==================================================================
> [ 2302.187379] BUG: Bad page state in process nginx-origin pfn:28b7f8
> [ 2302.462537] page:ffffea000a2dfe00 count:-1 mapcount:0
> mapping:0000000000000000 index:0x0
> [ 2302.462542] flags: 0x2ffff800000000()
> [ 2302.462549] raw: 002ffff800000000 dead000000000100 dead000000000200
> 0000000000000000
> [ 2302.462553] raw: 0000000000000000 0000000000000000 ffffffffffffffff
> 0000000000000000
> [ 2302.462554] page dumped because: nonzero _count
> [ 2302.462555] Modules linked in: tun xt_connlimit nf_conncount xt_bpf
> xt_hashlimit iptable_security cls_flow cls_u32 sch_htb sch_fq md_mod
> dm_crypt algif_skcipher af_alg dm_mod dax ip6table_nat nf_nat_ipv6
> ip6table_mangle ip6table_security ip6table_raw ip6table_filter
> ip6_tables xt_nat iptable_nat nf_nat_ipv4 nf_nat xt_TPROXY
> nf_tproxy_ipv6 nf_tproxy_ipv4 xt_connmark iptable_mangle xt_owner
> xt_CT xt_socket nf_socket_ipv4 nf_socket_ipv6 iptable_raw
> nfnetlink_log xt_NFLOG xt_tcpudp xt_comment xt_conntrack nf_conntrack
> nf_defrag_ipv6 nf_defrag_ipv4 xt_mark xt_multiport xt_set
> iptable_filter bpfilter ip_set_hash_netport ip_set_hash_net
> ip_set_hash_ip ip_set nfnetlink 8021q garp mrp stp llc sb_edac
> x86_pkg_temp_thermal kvm_intel kvm ipmi_ssif irqbypass crc32_pclmul
> crc32c_intel sfc(O) pcbc aesni_intel aes_x86_64
> [ 2302.650012] crypto_simd igb cryptd i2c_algo_bit glue_helper mdio
> dca ipmi_si ipmi_devintf ipmi_msghandler efivarfs ip_tables x_tables
> [ 2302.650031] CPU: 1 PID: 74997 Comm: nginx-origin Tainted: G B
> O 4.19.18-cloudflare-2019.1.8-1-gcabf55c #gcabf55c
> [ 2302.650033] Hardware name: Quanta Computer Inc. QuantaPlex
> T41S-2U/S2S-MB, BIOS S2S_3B10.03 06/21/2018
> [ 2302.650035] Call Trace:
> [ 2302.650049] dump_stack (lib/dump_stack.c:115)
> [ 2302.650062] bad_page.cold.116 (mm/page_alloc.c:542)
> [ 2302.755115] ? si_mem_available (mm/page_alloc.c:507)
> [ 2302.755119] ? ksys_write (fs/read_write.c:599)
> [ 2302.755126] ? entry_SYSCALL_64_after_hwframe (arch/x86/entry/entry_64.S:247)
> [ 2302.755130] ? __switch_to_asm (arch/x86/entry/entry_64.S:373)
> [ 2302.755135] get_page_from_freelist (mm/page_alloc.c:2997
> mm/page_alloc.c:3342)
> [ 2302.755140] ? __switch_to_asm (arch/x86/entry/entry_64.S:373)
> [ 2302.755144] ? __switch_to_asm (arch/x86/entry/entry_64.S:373)
> [ 2302.755153] ? kasan_unpoison_shadow (mm/kasan/kasan.c:71)
> [ 2302.861765] ? __isolate_free_page (mm/page_alloc.c:3252)
> [ 2302.861769] ? __kmalloc_node_track_caller (mm/slab.h:448
> mm/slub.c:2706 mm/slub.c:4320)
> [ 2302.861775] ? __alloc_skb (net/core/skbuff.c:206)
> [ 2302.861783] __alloc_pages_nodemask (mm/page_alloc.c:4369)
> [ 2302.915129] ? __alloc_pages_slowpath (mm/page_alloc.c:4345)
> [ 2302.915135] skb_page_frag_refill (net/core/sock.c:2213)
> [ 2302.915139] sk_page_frag_refill (net/core/sock.c:2234)
> [ 2302.915144] tcp_sendmsg_locked (net/ipv4/tcp.c:1321)
> [ 2302.915149] ? interrupt_entry (arch/x86/entry/entry_64.S:607)
> [ 2302.915153] ? kasan_unpoison_shadow (mm/kasan/kasan.c:68)
> [ 2302.915160] ? tcp_sendpage (net/ipv4/tcp.c:1175)
> [ 2303.003254] ? selinux_secmark_relabel_packet (security/selinux/hooks.c:4532)
> [ 2303.003260] ? release_pages (mm/swap.c:716)
> [ 2303.028592] ? inet_sk_set_state (net/ipv4/af_inet.c:794)
> [ 2303.028596] tcp_sendmsg (net/ipv4/tcp.c:1444)
> [ 2303.028603] sock_sendmsg (net/socket.c:622 net/socket.c:631)
> [ 2303.028609] sock_write_iter (net/socket.c:901)
> [ 2303.075968] ? sock_sendmsg (net/socket.c:884)
> [ 2303.075978] __vfs_write (fs/read_write.c:475 fs/read_write.c:487)
> [ 2303.075986] ? __handle_mm_fault (mm/memory.c:3211 mm/memory.c:4030
> mm/memory.c:4156)
> [ 2303.111370] ? kernel_read (fs/read_write.c:483)
> [ 2303.111375] ? file_has_perm (security/selinux/hooks.c:1919)
> [ 2303.111379] ? bpf_fd_pass (security/selinux/hooks.c:1890)
> [ 2303.111386] vfs_write (fs/read_write.c:550)
> [ 2303.111389] ksys_write (fs/read_write.c:599)
> [ 2303.111394] ? __ia32_sys_read (fs/read_write.c:592)
> [ 2303.111401] do_syscall_64 (arch/x86/entry/common.c:290)
> [ 2303.188508] ? page_fault (arch/x86/entry/entry_64.S:1161)
> [ 2303.188513] entry_SYSCALL_64_after_hwframe (arch/x86/entry/entry_64.S:247)
> [ 2303.188517] RIP: 0033:0x7f53e469f190
> [ 2303.188521] Code: 2e 0f 1f 84 00 00 00 00 00 90 48 8b 05 39 7e 20
> 00 c3 0f 1f 84 00 00 00 00 00 83 3d 39 c2 20 00 00 75 10 b8 01 00 00
> 00 0f 05 <48> 3d 01 f0 ff ff 73 31 c3 48 83 ec 08 e8 ae fc ff ff 48 89
> 04 24
> All code
> ========
> 0: 2e 0f 1f 84 00 00 00 nopl %cs:0x0(%rax,%rax,1)
> 7: 00 00
> 9: 90 nop
> a: 48 8b 05 39 7e 20 00 mov 0x207e39(%rip),%rax # 0x207e4a
> 11: c3 retq
> 12: 0f 1f 84 00 00 00 00 nopl 0x0(%rax,%rax,1)
> 19: 00
> 1a: 83 3d 39 c2 20 00 00 cmpl $0x0,0x20c239(%rip) # 0x20c25a
> 21: 75 10 jne 0x33
> 23: b8 01 00 00 00 mov $0x1,%eax
> 28: 0f 05 syscall
> 2a:* 48 3d 01 f0 ff ff cmp $0xfffffffffffff001,%rax <--
> trapping instruction
> 30: 73 31 jae 0x63
> 32: c3 retq
> 33: 48 83 ec 08 sub $0x8,%rsp
> 37: e8 ae fc ff ff callq 0xfffffffffffffcea
> 3c: 48 89 04 24 mov %rax,(%rsp)
>
> Code starting with the faulting instruction
> ===========================================
> 0: 48 3d 01 f0 ff ff cmp $0xfffffffffffff001,%rax
> 6: 73 31 jae 0x39
> 8: c3 retq
> 9: 48 83 ec 08 sub $0x8,%rsp
> d: e8 ae fc ff ff callq 0xfffffffffffffcc0
> 12: 48 89 04 24 mov %rax,(%rsp)
> [ 2303.188523] RSP: 002b:00007ffcc6a0c118 EFLAGS: 00000246 ORIG_RAX:
> 0000000000000001
> [ 2303.188528] RAX: ffffffffffffffda RBX: 00005562df6160b3 RCX: 00007f53e469f190
> [ 2303.188531] RDX: 000000000000401d RSI: 00005562df6160b3 RDI: 0000000000000d4f
> [ 2303.188533] RBP: 00007ffcc6a0c150 R08: 0000000000000005 R09: 0000000060640d3e
> [ 2303.188535] R10: 00005562d20f7b10 R11: 0000000000000246 R12: 000000000000401d
> [ 2303.188541] R13: 000000000000401d R14: 00007ffcc6a0c3a8 R15: 00005562dc0e6ec8
> [ 2303.407074] WARNING: CPU: 21 PID: 74997 at lib/iov_iter.c:825
> copy_page_to_iter (lib/iov_iter.c:825 lib/iov_iter.c:832)
> [ 2303.420983] Modules linked in: tun xt_connlimit nf_conncount xt_bpf
> xt_hashlimit iptable_security cls_flow cls_u32 sch_htb sch_fq md_mod
> dm_crypt algif_skcipher af_alg dm_mod dax ip6table_nat nf_nat_ipv6
> ip6table_mangle ip6table_security ip6table_raw ip6table_filter
> ip6_tables xt_nat iptable_nat nf_nat_ipv4 nf_nat xt_TPROXY
> nf_tproxy_ipv6 nf_tproxy_ipv4 xt_connmark iptable_mangle xt_owner
> xt_CT xt_socket nf_socket_ipv4 nf_socket_ipv6 iptable_raw
> nfnetlink_log xt_NFLOG xt_tcpudp xt_comment xt_conntrack nf_conntrack
> nf_defrag_ipv6 nf_defrag_ipv4 xt_mark xt_multiport xt_set
> iptable_filter bpfilter ip_set_hash_netport ip_set_hash_net
> ip_set_hash_ip ip_set nfnetlink 8021q garp mrp stp llc sb_edac
> x86_pkg_temp_thermal kvm_intel kvm ipmi_ssif irqbypass crc32_pclmul
> crc32c_intel sfc(O) pcbc aesni_intel aes_x86_64
> [ 2303.538009] crypto_simd igb cryptd i2c_algo_bit glue_helper mdio
> dca ipmi_si ipmi_devintf ipmi_msghandler efivarfs ip_tables x_tables
> [ 2303.538034] CPU: 21 PID: 74997 Comm: nginx-origin Tainted: G B
> O 4.19.18-cloudflare-2019.1.8-1-gcabf55c #gcabf55c
> [ 2303.538037] Hardware name: Quanta Computer Inc. QuantaPlex
> T41S-2U/S2S-MB, BIOS S2S_3B10.03 06/21/2018
> [ 2303.538050] RIP: 0010:copy_page_to_iter (??:?)
> [ 2303.538055] Code: 07 00 00 4d 85 f6 4c 89 54 24 10 4d 8b 6f 18 4c
> 89 44 24 08 74 0c 4c 89 ff e8 65 43 ff ff 84 c0 75 12 45 31 f6 e9 d9
> fe ff ff <0f> 0b 45 31 f6 e9 cf fe ff ff 49 8d 6f 08 4c 8b 44 24 08 48
> b8 00
> All code
> ========
> 0: 07 (bad)
> 1: 00 00 add %al,(%rax)
> 3: 4d 85 f6 test %r14,%r14
> 6: 4c 89 54 24 10 mov %r10,0x10(%rsp)
> b: 4d 8b 6f 18 mov 0x18(%r15),%r13
> f: 4c 89 44 24 08 mov %r8,0x8(%rsp)
> 14: 74 0c je 0x22
> 16: 4c 89 ff mov %r15,%rdi
> 19: e8 65 43 ff ff callq 0xffffffffffff4383
> 1e: 84 c0 test %al,%al
> 20: 75 12 jne 0x34
> 22: 45 31 f6 xor %r14d,%r14d
> 25: e9 d9 fe ff ff jmpq 0xffffffffffffff03
> 2a:* 0f 0b ud2 <-- trapping instruction
> 2c: 45 31 f6 xor %r14d,%r14d
> 2f: e9 cf fe ff ff jmpq 0xffffffffffffff03
> 34: 49 8d 6f 08 lea 0x8(%r15),%rbp
> 38: 4c 8b 44 24 08 mov 0x8(%rsp),%r8
> 3d: 48 rex.W
> 3e: b8 .byte 0xb8
> ...
>
> Code starting with the faulting instruction
> ===========================================
> 0: 0f 0b ud2
> 2: 45 31 f6 xor %r14d,%r14d
> 5: e9 cf fe ff ff jmpq 0xfffffffffffffed9
> a: 49 8d 6f 08 lea 0x8(%r15),%rbp
> e: 4c 8b 44 24 08 mov 0x8(%rsp),%r8
> 13: 48 rex.W
> 14: b8 .byte 0xb8
> ...
> [ 2303.538057] RSP: 0018:ffff88a005e0f7c0 EFLAGS: 00010293
> [ 2303.538061] RAX: 0000000000001000 RBX: 000000000000168d RCX: 002ffff800000000
> [ 2303.538064] RDX: ffffffffa66bdcb0 RSI: ffffffffa66bdca0 RDI: ffffea000a2dfe00
> [ 2303.538066] RBP: 0000000000000005 R08: ffffea000a2dfe00 R09: dffffc0000000000
> [ 2303.538069] R10: 0000000000001688 R11: 0000000000000004 R12: ffffea000a2dfe08
> [ 2303.538071] R13: ffffea000a2dfe00 R14: ffffea0000000000 R15: ffff88a005e0fc40
> [ 2303.538075] FS: 00007f53e4ac0740(0000) GS:ffff888c3f4c0000(0000)
> knlGS:0000000000000000
> [ 2303.538077] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
> [ 2303.538079] CR2: 00005562d36cc000 CR3: 0000002015486001 CR4: 00000000003606e0
> [ 2303.538081] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
> [ 2303.538083] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
> [ 2303.538085] Call Trace:
> [ 2303.538099] skb_copy_datagram_iter (net/core/datagram.c:453)
> [ 2303.538108] tcp_recvmsg (net/ipv4/tcp.c:2104)
> [ 2303.538115] ? tcp_get_md5sig_pool (net/ipv4/tcp.c:1917)
> [ 2303.538119] ? tcp_poll (include/net/sock.h:1204
> include/net/sock.h:1210 net/ipv4/tcp.c:569)
> [ 2303.538123] ? tcp_splice_read (net/ipv4/tcp.c:504)
> [ 2303.538131] ? bad_area_access_error (arch/x86/mm/fault.c:1213)
> [ 2303.538134] ? tcp_splice_read (net/ipv4/tcp.c:504)
> [ 2303.538144] ? ep_item_poll.isra.20 (fs/eventpoll.c:892)
> [ 2303.538151] ? selinux_secmark_relabel_packet (security/selinux/hooks.c:4532)
> [ 2303.538159] inet_recvmsg (net/ipv4/af_inet.c:838)
> [ 2303.538164] ? inet_sendpage (net/ipv4/af_inet.c:828)
> [ 2303.538172] sock_read_iter (net/socket.c:879)
> [ 2303.538177] ? sock_recvmsg (net/socket.c:862)
> [ 2303.538187] __vfs_read (fs/read_write.c:407 fs/read_write.c:418)
> [ 2303.538193] ? __switch_to_asm (arch/x86/entry/entry_64.S:373)
> [ 2303.538197] ? __switch_to_asm (arch/x86/entry/entry_64.S:373)
> [ 2303.538202] ? __x64_sys_copy_file_range (fs/read_write.c:414)
> [ 2303.538208] ? file_has_perm (security/selinux/hooks.c:1919)
> [ 2303.538216] vfs_read (fs/read_write.c:453)
> [ 2303.538221] ksys_read (fs/read_write.c:579)
> [ 2303.538225] ? kernel_write (fs/read_write.c:572)
> [ 2303.538232] do_syscall_64 (arch/x86/entry/common.c:290)
> [ 2303.538236] ? prepare_exit_to_usermode (arch/x86/entry/common.c:197)
> [ 2303.538240] entry_SYSCALL_64_after_hwframe (arch/x86/entry/entry_64.S:247)
> [ 2303.538245] RIP: 0033:0x7f53e469f1f0
> [ 2303.538249] Code: 73 01 c3 48 8b 0d b8 7d 20 00 f7 d8 64 89 01 48
> 83 c8 ff c3 66 0f 1f 44 00 00 83 3d d9 c1 20 00 00 75 10 b8 00 00 00
> 00 0f 05 <48> 3d 01 f0 ff ff 73 31 c3 48 83 ec 08 e8 4e fc ff ff 48 89
> 04 24
> All code
> ========
> 0: 73 01 jae 0x3
> 2: c3 retq
> 3: 48 8b 0d b8 7d 20 00 mov 0x207db8(%rip),%rcx # 0x207dc2
> a: f7 d8 neg %eax
> c: 64 89 01 mov %eax,%fs:(%rcx)
> f: 48 83 c8 ff or $0xffffffffffffffff,%rax
> 13: c3 retq
> 14: 66 0f 1f 44 00 00 nopw 0x0(%rax,%rax,1)
> 1a: 83 3d d9 c1 20 00 00 cmpl $0x0,0x20c1d9(%rip) # 0x20c1fa
> 21: 75 10 jne 0x33
> 23: b8 00 00 00 00 mov $0x0,%eax
> 28: 0f 05 syscall
> 2a:* 48 3d 01 f0 ff ff cmp $0xfffffffffffff001,%rax <--
> trapping instruction
> 30: 73 31 jae 0x63
> 32: c3 retq
> 33: 48 83 ec 08 sub $0x8,%rsp
> 37: e8 4e fc ff ff callq 0xfffffffffffffc8a
> 3c: 48 89 04 24 mov %rax,(%rsp)
>
> Code starting with the faulting instruction
> ===========================================
> 0: 48 3d 01 f0 ff ff cmp $0xfffffffffffff001,%rax
> 6: 73 31 jae 0x39
> 8: c3 retq
> 9: 48 83 ec 08 sub $0x8,%rsp
> d: e8 4e fc ff ff callq 0xfffffffffffffc60
> 12: 48 89 04 24 mov %rax,(%rsp)
> [ 2303.538251] RSP: 002b:00007ffcc6a0c188 EFLAGS: 00000246 ORIG_RAX:
> 0000000000000000
> [ 2303.538254] RAX: ffffffffffffffda RBX: 00005562d5f89883 RCX: 00007f53e469f1f0
> [ 2303.538256] RDX: 0000000000000005 RSI: 00005562d5f89883 RDI: 0000000000000dfb
> [ 2303.538258] RBP: 00007ffcc6a0c1c0 R08: 0000000000000032 R09: 0000000000000020
> [ 2303.538260] R10: 00005562d20944de R11: 0000000000000246 R12: 0000000000000005
> [ 2303.538262] R13: 00005562dbb17f60 R14: 00005562d2570e80 R15: 00007f53c5866d98
> [ 2303.538268] ---[ end trace d791391e77eef582 ]---
> [ 2330.200708] kasan: CONFIG_KASAN_INLINE enabled
> [ 2330.211020] kasan: GPF could be caused by NULL-ptr deref or user
> memory access
> [ 2330.224169] general protection fault: 0000 [#1] SMP KASAN PTI
> [ 2330.235791] CPU: 28 PID: 69371 Comm: nginx-fl Tainted: G B W
> O 4.19.18-cloudflare-2019.1.8-1-gcabf55c #gcabf55c
> [ 2330.253036] Hardware name: Quanta Computer Inc. QuantaPlex
> T41S-2U/S2S-MB, BIOS S2S_3B10.03 06/21/2018
> [ 2330.268679] RIP: 0010:rb_replace_node (??:?)
> [ 2330.279645] Code: 55 48 89 f5 53 48 89 fb 48 83 ec 08 80 3c 01 00
> 0f 85 64 02 00 00 48 b9 00 00 00 00 00 fc ff df 48 89 e8 4c 8b 23 48
> c1 e8 03 <0f> b6 34 08 48 8d 45 17 48 89 c7 83 e0 07 48 c1 ef 03 49 83
> e4 fc
> All code
> ========
> 0: 55 push %rbp
> 1: 48 89 f5 mov %rsi,%rbp
> 4: 53 push %rbx
> 5: 48 89 fb mov %rdi,%rbx
> 8: 48 83 ec 08 sub $0x8,%rsp
> c: 80 3c 01 00 cmpb $0x0,(%rcx,%rax,1)
> 10: 0f 85 64 02 00 00 jne 0x27a
> 16: 48 b9 00 00 00 00 00 movabs $0xdffffc0000000000,%rcx
> 1d: fc ff df
> 20: 48 89 e8 mov %rbp,%rax
> 23: 4c 8b 23 mov (%rbx),%r12
> 26: 48 c1 e8 03 shr $0x3,%rax
> 2a:* 0f b6 34 08 movzbl (%rax,%rcx,1),%esi <-- trapping instruction
> 2e: 48 8d 45 17 lea 0x17(%rbp),%rax
> 32: 48 89 c7 mov %rax,%rdi
> 35: 83 e0 07 and $0x7,%eax
> 38: 48 c1 ef 03 shr $0x3,%rdi
> 3c: 49 83 e4 fc and $0xfffffffffffffffc,%r12
>
> Code starting with the faulting instruction
> ===========================================
> 0: 0f b6 34 08 movzbl (%rax,%rcx,1),%esi
> 4: 48 8d 45 17 lea 0x17(%rbp),%rax
> 8: 48 89 c7 mov %rax,%rdi
> b: 83 e0 07 and $0x7,%eax
> e: 48 c1 ef 03 shr $0x3,%rdi
> 12: 49 83 e4 fc and $0xfffffffffffffffc,%r12
> [ 2330.311757] RSP: 0018:ffff888c3f687d88 EFLAGS: 00010206
> [ 2330.323631] RAX: 0000000000000003 RBX: ffff888c081fc000 RCX: dffffc0000000000
> [ 2330.323634] RDX: ffff888c0a5c38e0 RSI: 000000000000001a RDI: ffff888c081fc000
> [ 2330.323636] RBP: 000000000000001a R08: fffffbfff4d88d09 R09: fffffbfff4d88d08
> [ 2330.323639] R10: fffffbfff4d88d08 R11: ffffffffa6c46847 R12: 0000000030747865
> [ 2330.323641] R13: ffff888c0a5c3910 R14: ffff888c0a5c3870 R15: ffff888c0a5c38e0
> [ 2330.323644] FS: 00007f3375a30780(0000) GS:ffff888c3f680000(0000)
> knlGS:0000000000000000
> [ 2330.323647] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
> [ 2330.323649] CR2: 00007f19d3da5000 CR3: 0000000bee77a001 CR4: 00000000003606e0
> [ 2330.323651] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
> [ 2330.323653] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
> [ 2330.323655] Call Trace:
> [ 2330.323658] <IRQ>
> [ 2330.323673] ip_expire (net/ipv4/ip_fragment.c:223)
> [ 2330.323680] ? ip_check_defrag (net/ipv4/ip_fragment.c:187)
> [ 2330.323686] call_timer_fn (arch/x86/include/asm/jump_label.h:36
> include/linux/jump_label.h:142 include/trace/events/timer.h:121
> kernel/time/timer.c:1327)
> [ 2330.323691] run_timer_softirq (kernel/time/timer.c:1364
> kernel/time/timer.c:1682 kernel/time/timer.c:1695)
> [ 2330.323695] ? add_timer (kernel/time/timer.c:1692)
> [ 2330.323699] ? hrtimer_init (kernel/time/hrtimer.c:1430)
> [ 2330.323705] ? recalibrate_cpu_khz (arch/x86/kernel/tsc.c:1066
> arch/x86/kernel/tsc.c:1066)
> [ 2330.323709] ? recalibrate_cpu_khz (arch/x86/kernel/tsc.c:1066
> arch/x86/kernel/tsc.c:1066)
> [ 2330.323713] ? ktime_get (kernel/time/timekeeping.c:267
> kernel/time/timekeeping.c:371 kernel/time/timekeeping.c:756)
> [ 2330.323720] ? lapic_timer_set_oneshot (arch/x86/kernel/apic/apic.c:467)
> [ 2330.323727] ? clockevents_program_event (kernel/time/clockevents.c:346)
> [ 2330.323733] __do_softirq (arch/x86/include/asm/jump_label.h:36
> include/linux/jump_label.h:142 include/trace/events/irq.h:142
> kernel/softirq.c:293)
> [ 2330.323741] irq_exit (kernel/softirq.c:372 kernel/softirq.c:412)
> [ 2330.323744] smp_apic_timer_interrupt
> (arch/x86/include/asm/irq_regs.h:19 arch/x86/include/asm/irq_regs.h:26
> arch/x86/kernel/apic/apic.c:1058)
> [ 2330.323751] apic_timer_interrupt (arch/x86/entry/entry_64.S:864)
> [ 2330.323753] </IRQ>
> [ 2330.323760] RIP: 0010:check_memory_region (??:?)
> [ 2330.323765] Code: ff 41 54 49 b9 00 00 00 00 00 fc ff df 4d 89 da
> 55 49 c1 ea 03 53 48 89 fb 4d 01 ca 48 c1 eb 03 49 8d 6a 01 49 01 d9
> 49 89 e8 <4c> 89 c8 4d 29 c8 49 83 f8 10 0f 8e 98 00 00 00 44 89 cb 83
> e3 07
> All code
> ========
> 0: ff 41 54 incl 0x54(%rcx)
> 3: 49 b9 00 00 00 00 00 movabs $0xdffffc0000000000,%r9
> a: fc ff df
> d: 4d 89 da mov %r11,%r10
> 10: 55 push %rbp
> 11: 49 c1 ea 03 shr $0x3,%r10
> 15: 53 push %rbx
> 16: 48 89 fb mov %rdi,%rbx
> 19: 4d 01 ca add %r9,%r10
> 1c: 48 c1 eb 03 shr $0x3,%rbx
> 20: 49 8d 6a 01 lea 0x1(%r10),%rbp
> 24: 49 01 d9 add %rbx,%r9
> 27: 49 89 e8 mov %rbp,%r8
> 2a:* 4c 89 c8 mov %r9,%rax <-- trapping instruction
> 2d: 4d 29 c8 sub %r9,%r8
> 30: 49 83 f8 10 cmp $0x10,%r8
> 34: 0f 8e 98 00 00 00 jle 0xd2
> 3a: 44 89 cb mov %r9d,%ebx
> 3d: 83 e3 07 and $0x7,%ebx
>
> Code starting with the faulting instruction
> ===========================================
> 0: 4c 89 c8 mov %r9,%rax
> 3: 4d 29 c8 sub %r9,%r8
> 6: 49 83 f8 10 cmp $0x10,%r8
> a: 0f 8e 98 00 00 00 jle 0xa8
> 10: 44 89 cb mov %r9d,%ebx
> 13: 83 e3 07 and $0x7,%ebx
> [ 2330.323767] RSP: 0018:ffff888bcb66f830 EFLAGS: 00000286 ORIG_RAX:
> ffffffffffffff13
> [ 2330.323771] RAX: ffff7fffffffffff RBX: 1ffffd400601a58e RCX: ffffffffa5591192
> [ 2330.323772] RDX: 0000000000000001 RSI: 0000000000000004 RDI: ffffea00300d2c74
> [ 2330.323775] RBP: fffff9400601a58f R08: fffff9400601a58f R09: fffff9400601a58e
> [ 2330.323777] R10: fffff9400601a58e R11: ffffea00300d2c77 R12: dffffc0000000000
> [ 2330.323779] R13: ffff888bf01d0500 R14: ffff88826902a7c0 R15: ffffea00300d2c40
> [ 2330.323787] ? skb_release_data (arch/x86/include/asm/atomic.h:125
> (discriminator 3) include/asm-generic/atomic-instrumented.h:260
> (discriminator 3) include/linux/page_ref.h:139 (discriminator 3)
> include/linux/mm.h:520 (discriminator 3) include/linux/mm.h:942
> (discriminator 3) include/linux/skbuff.h:2795 (discriminator 3)
> net/core/skbuff.c:564 (discriminator 3))
> [ 2330.323793] skb_release_data (arch/x86/include/asm/atomic.h:125
> (discriminator 3) include/asm-generic/atomic-instrumented.h:260
> (discriminator 3) include/linux/page_ref.h:139 (discriminator 3)
> include/linux/mm.h:520 (discriminator 3) include/linux/mm.h:942
> (discriminator 3) include/linux/skbuff.h:2795 (discriminator 3)
> net/core/skbuff.c:564 (discriminator 3))
> [ 2330.323798] __kfree_skb (net/core/skbuff.c:642)
> [ 2330.323804] tcp_recvmsg (include/net/sock.h:2405 net/ipv4/tcp.c:2134)
> [ 2330.323808] ? sock_def_readable (arch/x86/include/asm/bitops.h:328
> include/net/sock.h:828 include/net/sock.h:2181 net/core/sock.c:2698)
> [ 2330.323814] ? tcp_get_md5sig_pool (net/ipv4/tcp.c:1917)
> [ 2330.323817] ? tcp_poll (include/net/sock.h:1204
> include/net/sock.h:1210 net/ipv4/tcp.c:569)
> [ 2330.323825] ? unix_stream_sendpage (net/unix/af_unix.c:1829)
> [ 2330.323831] ? sock_sendmsg (net/socket.c:622 net/socket.c:631)
> [ 2330.323834] ? sock_write_iter (net/socket.c:901)
> [ 2330.323838] ? sock_sendmsg (net/socket.c:884)
> [ 2330.323846] inet_recvmsg (net/ipv4/af_inet.c:838)
> [ 2330.323851] ? inet_sendpage (net/ipv4/af_inet.c:828)
> [ 2330.323856] sock_read_iter (net/socket.c:879)
> [ 2330.323860] ? sock_recvmsg (net/socket.c:862)
> [ 2330.323870] __vfs_read (fs/read_write.c:407 fs/read_write.c:418)
> [ 2330.323874] ? __switch_to_asm (arch/x86/entry/entry_64.S:373)
> [ 2330.323878] ? __switch_to_asm (arch/x86/entry/entry_64.S:373)
> [ 2330.323883] ? __x64_sys_copy_file_range (fs/read_write.c:414)
> [ 2330.323890] ? file_has_perm (security/selinux/hooks.c:1919)
> [ 2330.323898] vfs_read (fs/read_write.c:453)
> [ 2330.323903] ksys_read (fs/read_write.c:579)
> [ 2330.323908] ? kernel_write (fs/read_write.c:572)
> [ 2330.323911] ? fput (arch/x86/include/asm/atomic64_64.h:118
> include/asm-generic/atomic-instrumented.h:269
> include/asm-generic/atomic-long.h:218 fs/file_table.c:331)
> [ 2330.323918] do_syscall_64 (arch/x86/entry/common.c:290)
> [ 2330.323921] ? prepare_exit_to_usermode (arch/x86/entry/common.c:197)
> [ 2330.323926] entry_SYSCALL_64_after_hwframe (arch/x86/entry/entry_64.S:247)
> [ 2330.323930] RIP: 0033:0x7f337540b20d
> [ 2330.323934] Code: c1 20 00 00 75 10 b8 00 00 00 00 0f 05 48 3d 01
> f0 ff ff 73 31 c3 48 83 ec 08 e8 4e fc ff ff 48 89 04 24 b8 00 00 00
> 00 0f 05 <48> 8b 3c 24 48 89 c2 e8 97 fc ff ff 48 89 d0 48 83 c4 08 48
> 3d 01
> All code
> ========
> 0: c1 20 00 shll $0x0,(%rax)
> 3: 00 75 10 add %dh,0x10(%rbp)
> 6: b8 00 00 00 00 mov $0x0,%eax
> b: 0f 05 syscall
> d: 48 3d 01 f0 ff ff cmp $0xfffffffffffff001,%rax
> 13: 73 31 jae 0x46
> 15: c3 retq
> 16: 48 83 ec 08 sub $0x8,%rsp
> 1a: e8 4e fc ff ff callq 0xfffffffffffffc6d
> 1f: 48 89 04 24 mov %rax,(%rsp)
> 23: b8 00 00 00 00 mov $0x0,%eax
> 28: 0f 05 syscall
> 2a:* 48 8b 3c 24 mov (%rsp),%rdi <-- trapping instruction
> 2e: 48 89 c2 mov %rax,%rdx
> 31: e8 97 fc ff ff callq 0xfffffffffffffccd
> 36: 48 89 d0 mov %rdx,%rax
> 39: 48 83 c4 08 add $0x8,%rsp
> 3d: 48 rex.W
> 3e: 3d .byte 0x3d
> 3f: 01 .byte 0x1
>
> Code starting with the faulting instruction
> ===========================================
> 0: 48 8b 3c 24 mov (%rsp),%rdi
> 4: 48 89 c2 mov %rax,%rdx
> 7: e8 97 fc ff ff callq 0xfffffffffffffca3
> c: 48 89 d0 mov %rdx,%rax
> f: 48 83 c4 08 add $0x8,%rsp
> 13: 48 rex.W
> 14: 3d .byte 0x3d
> 15: 01 .byte 0x1
> [ 2330.323936] RSP: 002b:00007ffe077a9510 EFLAGS: 00000293 ORIG_RAX:
> 0000000000000000
> [ 2330.323940] RAX: ffffffffffffffda RBX: 00005640dee9dcb8 RCX: 00007f337540b20d
> [ 2330.323942] RDX: 0000000000004018 RSI: 00005640dee9dcb8 RDI: 0000000000000185
> [ 2330.323945] RBP: 00007ffe077a9550 R08: 00005640dd627720 R09: 0000000000004000
> [ 2330.323947] R10: 0000000000000300 R11: 0000000000000293 R12: 0000000000004018
> [ 2330.323949] R13: 00005640dddcb4c0 R14: 0000000000004000 R15: 00007f32435090e0
> [ 2330.323954] Modules linked in: tun xt_connlimit nf_conncount xt_bpf
> xt_hashlimit iptable_security cls_flow cls_u32 sch_htb sch_fq md_mod
> dm_crypt algif_skcipher af_alg dm_mod dax ip6table_nat nf_nat_ipv6
> ip6table_mangle ip6table_security ip6table_raw ip6table_filter
> ip6_tables xt_nat iptable_nat nf_nat_ipv4 nf_nat xt_TPROXY
> nf_tproxy_ipv6 nf_tproxy_ipv4 xt_connmark iptable_mangle xt_owner
> xt_CT xt_socket nf_socket_ipv4 nf_socket_ipv6 iptable_raw
> nfnetlink_log xt_NFLOG xt_tcpudp xt_comment xt_conntrack nf_conntrack
> nf_defrag_ipv6 nf_defrag_ipv4 xt_mark xt_multiport xt_set
> iptable_filter bpfilter ip_set_hash_netport ip_set_hash_net
> ip_set_hash_ip ip_set nfnetlink 8021q garp mrp stp llc sb_edac
> x86_pkg_temp_thermal kvm_intel kvm ipmi_ssif irqbypass crc32_pclmul
> crc32c_intel sfc(O) pcbc aesni_intel aes_x86_64
> [ 2330.324038] crypto_simd igb cryptd i2c_algo_bit glue_helper mdio
> dca ipmi_si ipmi_devintf ipmi_msghandler efivarfs ip_tables x_tables
> [ 2330.324111] ---[ end trace d791391e77eef583 ]---
> [ 2330.324118] RIP: 0010:rb_replace_node (??:?)
> [ 2330.324122] Code: 55 48 89 f5 53 48 89 fb 48 83 ec 08 80 3c 01 00
> 0f 85 64 02 00 00 48 b9 00 00 00 00 00 fc ff df 48 89 e8 4c 8b 23 48
> c1 e8 03 <0f> b6 34 08 48 8d 45 17 48 89 c7 83 e0 07 48 c1 ef 03 49 83
> e4 fc
> All code
> ========
> 0: 55 push %rbp
> 1: 48 89 f5 mov %rsi,%rbp
> 4: 53 push %rbx
> 5: 48 89 fb mov %rdi,%rbx
> 8: 48 83 ec 08 sub $0x8,%rsp
> c: 80 3c 01 00 cmpb $0x0,(%rcx,%rax,1)
> 10: 0f 85 64 02 00 00 jne 0x27a
> 16: 48 b9 00 00 00 00 00 movabs $0xdffffc0000000000,%rcx
> 1d: fc ff df
> 20: 48 89 e8 mov %rbp,%rax
> 23: 4c 8b 23 mov (%rbx),%r12
> 26: 48 c1 e8 03 shr $0x3,%rax
> 2a:* 0f b6 34 08 movzbl (%rax,%rcx,1),%esi <-- trapping instruction
> 2e: 48 8d 45 17 lea 0x17(%rbp),%rax
> 32: 48 89 c7 mov %rax,%rdi
> 35: 83 e0 07 and $0x7,%eax
> 38: 48 c1 ef 03 shr $0x3,%rdi
> 3c: 49 83 e4 fc and $0xfffffffffffffffc,%r12
>
> Code starting with the faulting instruction
> ===========================================
> 0: 0f b6 34 08 movzbl (%rax,%rcx,1),%esi
> 4: 48 8d 45 17 lea 0x17(%rbp),%rax
> 8: 48 89 c7 mov %rax,%rdi
> b: 83 e0 07 and $0x7,%eax
> e: 48 c1 ef 03 shr $0x3,%rdi
> 12: 49 83 e4 fc and $0xfffffffffffffffc,%r12
> [ 2330.324129] RSP: 0018:ffff888c3f687d88 EFLAGS: 00010206
> [ 2330.324133] RAX: 0000000000000003 RBX: ffff888c081fc000 RCX: dffffc0000000000
> [ 2330.324135] RDX: ffff888c0a5c38e0 RSI: 000000000000001a RDI: ffff888c081fc000
> [ 2330.324137] RBP: 000000000000001a R08: fffffbfff4d88d09 R09: fffffbfff4d88d08
> [ 2330.324140] R10: fffffbfff4d88d08 R11: ffffffffa6c46847 R12: 0000000030747865
> [ 2330.324142] R13: ffff888c0a5c3910 R14: ffff888c0a5c3870 R15: ffff888c0a5c38e0
> [ 2330.324151] FS: 00007f3375a30780(0000) GS:ffff888c3f680000(0000)
> knlGS:0000000000000000
> [ 2330.324154] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
> [ 2330.324156] CR2: 00007f19d3da5000 CR3: 0000000bee77a001 CR4: 00000000003606e0
> [ 2330.324158] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
> [ 2330.324161] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
> [ 2330.324163] Kernel panic - not syncing: Fatal exception in interrupt
> [ 2330.324214] Kernel Offset: 0x23000000 from 0xffffffff81000000
> (relocation range: 0xffffffff80000000-0xffffffffbfffffff)
>
> This commit from 4.19.14 seems relevant:
>
> * https://github.com/torvalds/linux/commit/d5f9565c8d5ad3cf94982223cfcef1169b0bb60f
>
> As a reminder, we upgraded from 4.19.13 and started seeing crashes.
Right, @err needs to be set properly.
Probably something like :
diff --git a/net/ipv4/ip_fragment.c b/net/ipv4/ip_fragment.c
index f8bbd693c19c247e41839c2d0b5318ca51b23ee8..dbd14530510a934230096b293c4042dd65c672c5
100644
--- a/net/ipv4/ip_fragment.c
+++ b/net/ipv4/ip_fragment.c
@@ -443,6 +443,7 @@ static int ip_frag_queue(struct ipq *qp, struct
sk_buff *skb)
* but not the last (covered above).
*/
rbn = &qp->q.rb_fragments.rb_node;
+ err = -EINVAL;
do {
parent = *rbn;
skb1 = rb_to_skb(parent);
@@ -501,7 +502,6 @@ static int ip_frag_queue(struct ipq *qp, struct
sk_buff *skb)
discard_qp:
inet_frag_kill(&qp->q);
- err = -EINVAL;
__IP_INC_STATS(net, IPSTATS_MIB_REASM_OVERLAPS);
err:
kfree_skb(skb);
Powered by blists - more mailing lists