[3142674.625842][T283860] BUG: unable to handle page fault for address: ffff893ebf410362 [3142757.450512][ C4] Modules linked in: binfmt_misc(E) rpcsec_gss_krb5(E) nfsv4(E) dns_resolver(E) nfs(E) fscache(E) nfsd(E) nfs_acl(E) lockd(E) auth_rpcgss(E) grace(E) nf_conntrack(E) nf_defrag_ipv6(E) nf_defrag_ipv4(E) nls_utf8(E) isofs(E) sunrpc(E) ip_tables(E) ufs(E) hfsplus(E) hfs(E) vfat(E) msdos(E) fat(E) xfs(E) libcrc32c(E) dm_mod(E) ext4(E) mbcache(E) jbd2(E) st(E) sd_mod(E) fuse(E) squashfs(E) zstd_decompress(E) loop(E) brd(E) isst_if_common(E) nfit(E) kvm_intel(E) sr_mod(E) cdrom(E) sg(E) kvm(E) ata_generic(E) irqbypass(E) crct10dif_pclmul(E) crc32_pclmul(E) crc32c_intel(E) ghash_clmulni_intel(E) cirrus(E) drm_kms_helper(E) syscopyarea(E) sysfillrect(E) sysimgblt(E) fb_sys_fops(E) drm(E) ata_piix(E) libata(E) virtio_net(E) joydev(E) serio_raw(E) net_failover(E) virtio_blk(E) virtio_balloon(E) pcspkr(E) i2c_piix4(E) failover(E) qemu_fw_cfg(E) sch_fq_codel(E) [last unloaded: dm_mod] [3142757.451750][ C5] Modules linked in: binfmt_misc(E) rpcsec_gss_krb5(E) nfsv4(E) dns_resolver(E) nfs(E) fscache(E) nfsd(E) nfs_acl(E) lockd(E) auth_rpcgss(E) grace(E) nf_conntrack(E) nf_defrag_ipv6(E) nf_defrag_ipv4(E) nls_utf8(E) isofs(E) sunrpc(E) ip_tables(E) ufs(E) hfsplus(E) hfs(E) vfat(E) msdos(E) fat(E) xfs(E) libcrc32c(E) dm_mod(E) ext4(E) mbcache(E) jbd2(E) st(E) sd_mod(E) fuse(E) squashfs(E) zstd_decompress(E) loop(E) brd(E) isst_if_common(E) nfit(E) kvm_intel(E) sr_mod(E) cdrom(E) sg(E) kvm(E) ata_generic(E) irqbypass(E) crct10dif_pclmul(E) crc32_pclmul(E) crc32c_intel(E) ghash_clmulni_intel(E) cirrus(E) drm_kms_helper(E) syscopyarea(E) sysfillrect(E) sysimgblt(E) fb_sys_fops(E) drm(E) ata_piix(E) libata(E) virtio_net(E) joydev(E) serio_raw(E) net_failover(E) virtio_blk(E) virtio_balloon(E) pcspkr(E) i2c_piix4(E) failover(E) qemu_fw_cfg(E) sch_fq_codel(E) [last unloaded: dm_mod] [3142757.454613][ C4] CPU: 4 PID: 283852 Comm: vertica Tainted: G EL 5.4.34-1.el7.gdc.x86_64 #1 [3142757.455863][ C5] CPU: 5 PID: 284163 Comm: vertica Tainted: G EL 5.4.34-1.el7.gdc.x86_64 #1 [3142757.456482][ C7] watchdog: BUG: soft lockup - CPU#7 stuck for 22s! [sshd:284646] [3142757.456483][ C7] Modules linked in: binfmt_misc(E) rpcsec_gss_krb5(E) nfsv4(E) dns_resolver(E) nfs(E) fscache(E) nfsd(E) nfs_acl(E) lockd(E) auth_rpcgss(E) grace(E) nf_conntrack(E) nf_defrag_ipv6(E) nf_defrag_ipv4(E) nls_utf8(E) isofs(E) sunrpc(E) ip_tables(E) ufs(E) hfsplus(E) hfs(E) vfat(E) msdos(E) fat(E) xfs(E) libcrc32c(E) dm_mod(E) ext4(E) mbcache(E) jbd2(E) st(E) sd_mod(E) fuse(E) squashfs(E) zstd_decompress(E) loop(E) brd(E) isst_if_common(E) nfit(E) kvm_intel(E) sr_mod(E) cdrom(E) sg(E) kvm(E) ata_generic(E) irqbypass(E) crct10dif_pclmul(E) crc32_pclmul(E) crc32c_intel(E) ghash_clmulni_intel(E) cirrus(E) drm_kms_helper(E) syscopyarea(E) sysfillrect(E) sysimgblt(E) fb_sys_fops(E) drm(E) ata_piix(E) libata(E) virtio_net(E) joydev(E) serio_raw(E) net_failover(E) virtio_blk(E) virtio_balloon(E) pcspkr(E) i2c_piix4(E) failover(E) qemu_fw_cfg(E) sch_fq_codel(E) [last unloaded: dm_mod] [3142757.456505][ C7] CPU: 7 PID: 284646 Comm: sshd Tainted: G EL 5.4.34-1.el7.gdc.x86_64 #1 [3142757.456506][ C7] Hardware name: RDO OpenStack Compute, BIOS 1.11.0-2.el7 04/01/2014 [3142757.456511][ C7] RIP: 0010:smp_call_function_many+0x21e/0x280 [3142757.456512][ C7] Code: c7 e8 56 25 77 00 3b 05 04 9f 48 01 0f 83 7b fe ff ff 48 63 d0 49 8b 0f 48 03 0c d5 80 99 19 82 8b 51 18 83 e2 01 74 0a f3 90 <8b> 51 18 83 e2 01 75 f6 eb c7 0f b6 4c 24 04 48 83 c4 10 48 89 de [3142757.456513][ C7] RSP: 0018:ffffc90000c53ba0 EFLAGS: 00000202 ORIG_RAX: ffffffffffffff13 [3142757.456514][ C7] RAX: 0000000000000003 RBX: ffffffff8107ba40 RCX: ffff88d4e74f1620 [3142757.456514][ C7] RDX: 0000000000000001 RSI: 000000000000001e RDI: ffff88d4db029710 [3142757.456514][ C7] RBP: 0000000000000000 R08: 0000000000000000 R09: 0000000000000000 [3142757.456515][ C7] R10: 0000000000000000 R11: 0000000000000000 R12: 000000000002ba40 [3142757.456515][ C7] R13: 0000000000000001 R14: 000000000000001e R15: ffff88d4e75eba80 [3142757.456516][ C7] FS: 00007f7f683548c0(0000) GS:ffff88d4e75c0000(0000) knlGS:0000000000000000 [3142757.456517][ C7] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [3142757.456517][ C7] CR2: 00007ffc4d461f44 CR3: 0000001c3a278004 CR4: 0000000000760ee0 [3142757.456520][ C7] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [3142757.456520][ C7] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [3142757.456520][ C7] PKRU: 55555554 [3142757.456521][ C7] Call Trace: [3142757.456526][ C7] ? do_kernel_range_flush+0x50/0x50 [3142757.456527][ C7] on_each_cpu+0x28/0x50 [3142757.456530][ C7] __purge_vmap_area_lazy+0x86/0x680 [3142757.456531][ C7] ? purge_fragmented_blocks_allcpus+0x42/0x1f0 [3142757.456533][ C7] _vm_unmap_aliases+0xf2/0x130 [3142757.456535][ C7] change_page_attr_set_clr+0xb6/0x1e0 [3142757.456537][ C7] set_memory_ro+0x2d/0x40 [3142757.456539][ C7] bpf_int_jit_compile+0x2ee/0x3e0 [3142757.456543][ C7] bpf_prog_select_runtime+0xdf/0x140 [3142757.456546][ C7] bpf_prepare_filter+0x597/0x610 [3142757.456547][ C7] bpf_prog_create_from_user+0xd5/0x120 [3142757.456550][ C7] ? hardlockup_detector_perf_cleanup+0xa0/0xa0 [3142757.456551][ C7] do_seccomp+0x294/0x8c0 [3142757.456554][ C7] __x64_sys_prctl+0x280/0x58d [3142757.456557][ C7] do_syscall_64+0x5b/0x1b0 [3142757.456560][ C7] entry_SYSCALL_64_after_hwframe+0x44/0xa9 [3142757.456562][ C7] RIP: 0033:0x7f7f655490da [3142757.456564][ C7] Code: 48 8b 0d 99 7d 2c 00 f7 d8 64 89 01 48 83 c8 ff c3 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 49 89 ca b8 9d 00 00 00 0f 05 <48> 3d 01 f0 ff ff 73 01 c3 48 8b 0d 66 7d 2c 00 f7 d8 64 89 01 48 [3142757.456564][ C7] RSP: 002b:00007ffc4d45fc68 EFLAGS: 00000246 ORIG_RAX: 000000000000009d [3142757.456565][ C7] RAX: ffffffffffffffda RBX: 000055fc76435a10 RCX: 00007f7f655490da [3142757.456565][ C7] RDX: 000055fc75198900 RSI: 0000000000000002 RDI: 0000000000000016 [3142757.456566][ C7] RBP: 000055fc76435570 R08: 0000000000000000 R09: 00007f7f6549727d [3142757.456566][ C7] R10: 00007f7f655490da R11: 0000000000000246 R12: 0000000000000000 [3142757.456567][ C7] R13: 000055fc76433660 R14: 000055fc751a4068 R15: 0000000000000018 [3142757.458830][ C4] Hardware name: RDO OpenStack Compute, BIOS 1.11.0-2.el7 04/01/2014 [3142757.458835][ C4] RIP: 0010:smp_call_function_many+0x21e/0x280 [3142757.459481][ C8] watchdog: BUG: soft lockup - CPU#8 stuck for 22s! [vertica:283847] [3142757.459482][ C8] Modules linked in: binfmt_misc(E) rpcsec_gss_krb5(E) nfsv4(E) dns_resolver(E) nfs(E) fscache(E) nfsd(E) nfs_acl(E) lockd(E) auth_rpcgss(E) grace(E) nf_conntrack(E) nf_defrag_ipv6(E) nf_defrag_ipv4(E) nls_utf8(E) isofs(E) sunrpc(E) ip_tables(E) ufs(E) hfsplus(E) hfs(E) vfat(E) msdos(E) fat(E) xfs(E) libcrc32c(E) dm_mod(E) ext4(E) mbcache(E) jbd2(E) st(E) sd_mod(E) fuse(E) squashfs(E) zstd_decompress(E) loop(E) brd(E) isst_if_common(E) nfit(E) kvm_intel(E) sr_mod(E) cdrom(E) sg(E) kvm(E) ata_generic(E) irqbypass(E) crct10dif_pclmul(E) crc32_pclmul(E) crc32c_intel(E) ghash_clmulni_intel(E) cirrus(E) drm_kms_helper(E) syscopyarea(E) sysfillrect(E) sysimgblt(E) fb_sys_fops(E) drm(E) ata_piix(E) libata(E) virtio_net(E) joydev(E) serio_raw(E) net_failover(E) virtio_blk(E) virtio_balloon(E) pcspkr(E) i2c_piix4(E) failover(E) qemu_fw_cfg(E) sch_fq_codel(E) [last unloaded: dm_mod] [3142757.459503][ C8] CPU: 8 PID: 283847 Comm: vertica Tainted: G EL 5.4.34-1.el7.gdc.x86_64 #1 [3142757.459504][ C8] Hardware name: RDO OpenStack Compute, BIOS 1.11.0-2.el7 04/01/2014 [3142757.459506][ C8] RIP: 0010:smp_call_function_many+0x21e/0x280 [3142757.459508][ C8] Code: c7 e8 56 25 77 00 3b 05 04 9f 48 01 0f 83 7b fe ff ff 48 63 d0 49 8b 0f 48 03 0c d5 80 99 19 82 8b 51 18 83 e2 01 74 0a f3 90 <8b> 51 18 83 e2 01 75 f6 eb c7 0f b6 4c 24 04 48 83 c4 10 48 89 de [3142757.459508][ C8] RSP: 0000:ffffc90001413c58 EFLAGS: 00000202 ORIG_RAX: ffffffffffffff13 [3142757.459509][ C8] RAX: 0000000000000003 RBX: ffffffff8107c1e0 RCX: ffff88d4e74f1640 [3142757.459510][ C8] RDX: 0000000000000001 RSI: 000000000000001e RDI: ffff88d4db029940 [3142757.459510][ C8] RBP: ffff88d4e762a300 R08: 0000000000000000 R09: 0000000000000000 [3142757.459511][ C8] R10: 0000000000000000 R11: 0000000080fe6cc0 R12: 000000000002ba40 [3142757.459512][ C8] R13: 0000000000000001 R14: 000000000000001e R15: ffff88d4e762ba80 [3142757.459513][ C8] FS: 00007f2b567fc700(0000) GS:ffff88d4e7600000(0000) knlGS:0000000000000000 [3142757.459513][ C8] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [3142757.459514][ C8] CR2: 00007f315f1f9ca0 CR3: 000000518913e006 CR4: 0000000000760ee0 [3142757.459516][ C8] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [3142757.459516][ C8] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [3142757.459517][ C8] PKRU: 55555554 [3142757.459517][ C8] Call Trace: [3142757.459521][ C8] ? flush_tlb_func_common.isra.8+0x230/0x230 [3142757.459522][ C8] ? flush_tlb_func_common.isra.8+0x230/0x230 [3142757.459523][ C8] on_each_cpu_mask+0x24/0x60 [3142757.459524][ C8] ? x86_configure_nx+0x50/0x50 [3142757.459525][ C8] on_each_cpu_cond_mask+0xa5/0x140 [3142757.459526][ C8] ? flush_tlb_func_common.isra.8+0x147/0x230 [3142757.459527][ C8] flush_tlb_mm_range+0xab/0xe0 [3142757.459530][ C8] ptep_clear_flush+0x56/0x60 [3142757.459532][ C8] wp_page_copy+0x34a/0x7e0 [3142757.459534][ C8] do_wp_page+0x8b/0x660 [3142757.459536][ C8] __handle_mm_fault+0x777/0xef0 [3142757.459538][ C8] handle_mm_fault+0xe2/0x1f0 [3142757.459541][ C8] __do_page_fault+0x226/0x490 [3142757.459543][ C8] do_page_fault+0x31/0x120 [3142757.459545][ C8] async_page_fault+0x3e/0x50 [3142757.459546][ C8] RIP: 0033:0xe148c5 [3142757.459547][ C8] Code: fa 08 0f 84 44 01 00 00 48 83 ff 07 4c 89 c2 76 33 48 83 ef 08 31 d2 48 89 f9 48 c1 e9 03 48 8d 34 cd 08 00 00 00 4d 8b 0c 10 <4c> 89 0c 10 48 83 c2 08 48 39 f2 75 ef 48 f7 d9 48 01 d0 4c 01 c2 [3142757.459548][ C8] RSP: 002b:00007f302cd2ad30 EFLAGS: 00010202 [3142757.459549][ C8] RAX: 00007f315f1f9ca0 RBX: 000000000000001f RCX: 0000000000000002 [3142757.459549][ C8] RDX: 0000000000000000 RSI: 0000000000000018 RDI: 0000000000000010 [3142757.459550][ C8] RBP: 00007f302cd2aea0 R08: 00007f25fa54b258 R09: 400008d0d1a2dc18 [3142757.459550][ C8] R10: 0000000000000018 R11: 0000000000005419 R12: 0000000000200000 [3142757.459551][ C8] R13: 00007f315e06f000 R14: 00007f2b5de19950 R15: 00007f302cd2af10 [3142757.462481][ C9] watchdog: BUG: soft lockup - CPU#9 stuck for 22s! [vertica:283858] [3142757.462481][ C9] Modules linked in: binfmt_misc(E) rpcsec_gss_krb5(E) nfsv4(E) dns_resolver(E) nfs(E) fscache(E) nfsd(E) nfs_acl(E) lockd(E) auth_rpcgss(E) grace(E) nf_conntrack(E) nf_defrag_ipv6(E) nf_defrag_ipv4(E) nls_utf8(E) isofs(E) sunrpc(E) ip_tables(E) ufs(E) hfsplus(E) hfs(E) vfat(E) msdos(E) fat(E) xfs(E) libcrc32c(E) dm_mod(E) ext4(E) mbcache(E) jbd2(E) st(E) sd_mod(E) fuse(E) squashfs(E) zstd_decompress(E) loop(E) brd(E) isst_if_common(E) nfit(E) kvm_intel(E) sr_mod(E) cdrom(E) sg(E) kvm(E) ata_generic(E) irqbypass(E) crct10dif_pclmul(E) crc32_pclmul(E) crc32c_intel(E) ghash_clmulni_intel(E) cirrus(E) drm_kms_helper(E) syscopyarea(E) sysfillrect(E) sysimgblt(E) fb_sys_fops(E) drm(E) ata_piix(E) libata(E) virtio_net(E) joydev(E) serio_raw(E) net_failover(E) virtio_blk(E) virtio_balloon(E) pcspkr(E) i2c_piix4(E) failover(E) qemu_fw_cfg(E) sch_fq_codel(E) [last unloaded: dm_mod] [3142757.462497][ C9] CPU: 9 PID: 283858 Comm: vertica Tainted: G EL 5.4.34-1.el7.gdc.x86_64 #1 [3142757.462498][ C9] Hardware name: RDO OpenStack Compute, BIOS 1.11.0-2.el7 04/01/2014 [3142757.462499][ C9] RIP: 0010:smp_call_function_many+0x21e/0x280 [3142757.462500][ C9] Code: c7 e8 56 25 77 00 3b 05 04 9f 48 01 0f 83 7b fe ff ff 48 63 d0 49 8b 0f 48 03 0c d5 80 99 19 82 8b 51 18 83 e2 01 74 0a f3 90 <8b> 51 18 83 e2 01 75 f6 eb c7 0f b6 4c 24 04 48 83 c4 10 48 89 de [3142757.462501][ C9] RSP: 0000:ffffc900015cbc58 EFLAGS: 00000202 ORIG_RAX: ffffffffffffff13 [3142757.462502][ C9] RAX: 0000000000000003 RBX: ffffffff8107c1e0 RCX: ffff88d4e74f1660 [3142757.462502][ C9] RDX: 0000000000000001 RSI: 000000000000001e RDI: ffff88d4db029a68 [3142757.462503][ C9] RBP: ffff88d4e766a300 R08: 0000000000000000 R09: 0000000000000000 [3142757.462503][ C9] R10: 0000000000000000 R11: 00000001096b3e80 R12: 000000000002ba40 [3142757.462504][ C9] R13: 0000000000000001 R14: 000000000000001e R15: ffff88d4e766ba80 [3142757.462505][ C9] FS: 00007f2f080de700(0000) GS:ffff88d4e7640000(0000) knlGS:0000000000000000 [3142757.462505][ C9] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [3142757.462506][ C9] CR2: 00007f3025c6d000 CR3: 000000518913e004 CR4: 0000000000760ee0 [3142757.462508][ C9] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [3142757.462508][ C9] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [3142757.462508][ C9] PKRU: 55555554 [3142757.462509][ C9] Call Trace: [3142757.462510][ C9] ? flush_tlb_func_common.isra.8+0x230/0x230 [3142757.462511][ C9] ? flush_tlb_func_common.isra.8+0x230/0x230 [3142757.462512][ C9] on_each_cpu_mask+0x24/0x60 [3142757.462513][ C9] ? x86_configure_nx+0x50/0x50 [3142757.462514][ C9] on_each_cpu_cond_mask+0xa5/0x140 [3142757.462515][ C9] ? flush_tlb_func_common.isra.8+0x92/0x230 [3142757.462516][ C9] flush_tlb_mm_range+0xab/0xe0 [3142757.462518][ C9] ptep_clear_flush+0x56/0x60 [3142757.462519][ C9] wp_page_copy+0x34a/0x7e0 [3142757.462521][ C9] do_wp_page+0x8b/0x660 [3142757.462522][ C9] __handle_mm_fault+0x777/0xef0 [3142757.462524][ C9] handle_mm_fault+0xe2/0x1f0 [3142757.462525][ C9] __do_page_fault+0x226/0x490 [3142757.462527][ C9] do_page_fault+0x31/0x120 [3142757.462528][ C9] async_page_fault+0x3e/0x50 [3142757.462528][ C9] RIP: 0033:0xe148c5 [3142757.462529][ C9] Code: fa 08 0f 84 44 01 00 00 48 83 ff 07 4c 89 c2 76 33 48 83 ef 08 31 d2 48 89 f9 48 c1 e9 03 48 8d 34 cd 08 00 00 00 4d 8b 0c 10 <4c> 89 0c 10 48 83 c2 08 48 39 f2 75 ef 48 f7 d9 48 01 d0 4c 01 c2 [3142757.462530][ C9] RSP: 002b:00007f312c7f3560 EFLAGS: 00010202 [3142757.462531][ C9] RAX: 00007f3025c6d000 RBX: 000000000000001f RCX: 0000000000000002 [3142757.462531][ C9] RDX: 0000000000000000 RSI: 0000000000000018 RDI: 0000000000000010 [3142757.462532][ C9] RBP: 00007f312c7f36d0 R08: 00007f257a0670e0 R09: 400008d0ce0b746f [3142757.462532][ C9] R10: 0000000000000018 R11: 00000000000044b4 R12: 0000000000200000 [3142757.462533][ C9] R13: 00007f30257ff000 R14: 00007f302caa46e0 R15: 00007f312c7f3740 [3142757.462626][ C5] Hardware name: RDO OpenStack Compute, BIOS 1.11.0-2.el7 04/01/2014 [3142757.465687][ C4] Code: c7 e8 56 25 77 00 3b 05 04 9f 48 01 0f 83 7b fe ff ff 48 63 d0 49 8b 0f 48 03 0c d5 80 99 19 82 8b 51 18 83 e2 01 74 0a f3 90 <8b> 51 18 83 e2 01 75 f6 eb c7 0f b6 4c 24 04 48 83 c4 10 48 89 de [3142757.468774][ C5] RIP: 0010:smp_call_function_many+0x21e/0x280 [3142757.469481][ C11] watchdog: BUG: soft lockup - CPU#11 stuck for 22s! [vertica:284316] [3142757.469482][ C11] Modules linked in: binfmt_misc(E) rpcsec_gss_krb5(E) nfsv4(E) dns_resolver(E) nfs(E) fscache(E) nfsd(E) nfs_acl(E) lockd(E) auth_rpcgss(E) grace(E) nf_conntrack(E) nf_defrag_ipv6(E) nf_defrag_ipv4(E) nls_utf8(E) isofs(E) sunrpc(E) ip_tables(E) ufs(E) hfsplus(E) hfs(E) vfat(E) msdos(E) fat(E) xfs(E) libcrc32c(E) dm_mod(E) ext4(E) mbcache(E) jbd2(E) st(E) sd_mod(E) fuse(E) squashfs(E) zstd_decompress(E) loop(E) brd(E) isst_if_common(E) nfit(E) kvm_intel(E) sr_mod(E) cdrom(E) sg(E) kvm(E) ata_generic(E) irqbypass(E) crct10dif_pclmul(E) crc32_pclmul(E) crc32c_intel(E) ghash_clmulni_intel(E) cirrus(E) drm_kms_helper(E) syscopyarea(E) sysfillrect(E) sysimgblt(E) fb_sys_fops(E) drm(E) ata_piix(E) libata(E) virtio_net(E) joydev(E) serio_raw(E) net_failover(E) virtio_blk(E) virtio_balloon(E) pcspkr(E) i2c_piix4(E) failover(E) qemu_fw_cfg(E) sch_fq_codel(E) [last unloaded: dm_mod] [3142757.469499][ C11] CPU: 11 PID: 284316 Comm: vertica Tainted: G EL 5.4.34-1.el7.gdc.x86_64 #1 [3142757.469500][ C11] Hardware name: RDO OpenStack Compute, BIOS 1.11.0-2.el7 04/01/2014 [3142757.469502][ C11] RIP: 0010:smp_call_function_many+0x21e/0x280 [3142757.469503][ C11] Code: c7 e8 56 25 77 00 3b 05 04 9f 48 01 0f 83 7b fe ff ff 48 63 d0 49 8b 0f 48 03 0c d5 80 99 19 82 8b 51 18 83 e2 01 74 0a f3 90 <8b> 51 18 83 e2 01 75 f6 eb c7 0f b6 4c 24 04 48 83 c4 10 48 89 de [3142757.469504][ C11] RSP: 0018:ffffc9000187fce0 EFLAGS: 00000202 ORIG_RAX: ffffffffffffff13 [3142757.469505][ C11] RAX: 000000000000001a RBX: ffffffff8107c1e0 RCX: ffff88d4e7ab16a0 [3142757.469505][ C11] RDX: 0000000000000001 RSI: 000000000000001e RDI: ffff88d4db0291f0 [3142757.469506][ C11] RBP: ffff88d4e76ea300 R08: 0000000000000000 R09: 0000000000000000 [3142757.469506][ C11] R10: 0000000000000000 R11: 0000000000000000 R12: 000000000002ba40 [3142757.469507][ C11] R13: 0000000000000001 R14: 000000000000001e R15: ffff88d4e76eba80 [3142757.469508][ C11] FS: 00007f301effe700(0000) GS:ffff88d4e76c0000(0000) knlGS:0000000000000000 [3142757.469508][ C11] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [3142757.469509][ C11] CR2: 00007fa6b445f000 CR3: 000000518913e006 CR4: 0000000000760ee0 [3142757.469511][ C11] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [3142757.469511][ C11] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [3142757.469512][ C11] PKRU: 55555554 [3142757.469512][ C11] Call Trace: [3142757.469515][ C11] flush_tlb_mm_range+0xab/0xe0 [3142757.469516][ C11] tlb_flush_mmu+0x89/0x150 [3142757.469518][ C11] tlb_finish_mmu+0x3d/0x70 [3142757.469519][ C11] zap_page_range+0x115/0x160 [3142757.469521][ C11] ? find_vma+0x59/0x60 [3142757.469523][ C11] __do_sys_madvise+0x738/0xc00 [3142757.469525][ C11] ? finish_task_switch+0x75/0x2a0 [3142757.469528][ C11] ? exit_to_usermode_loop+0xc3/0xe7 [3142757.469529][ C11] do_syscall_64+0x5b/0x1b0 [3142757.469531][ C11] entry_SYSCALL_64_after_hwframe+0x44/0xa9 [3142757.469532][ C11] RIP: 0033:0x7f35e1571e97 [3142757.469533][ C11] Code: 83 c4 08 48 3d 01 f0 ff ff 73 01 c3 48 8b 0d d0 df 2c 00 f7 d8 64 89 01 48 83 c8 ff c3 66 0f 1f 44 00 00 b8 1c 00 00 00 0f 05 <48> 3d 01 f0 ff ff 73 01 c3 48 8b 0d a9 df 2c 00 f7 d8 64 89 01 48 [3142757.469534][ C11] RSP: 002b:00007f301effb3d8 EFLAGS: 00000206 ORIG_RAX: 000000000000001c [3142757.469535][ C11] RAX: ffffffffffffffda RBX: 00007f301effe700 RCX: 00007f35e1571e97 [3142757.469535][ C11] RDX: 0000000000000004 RSI: 00000000007f8000 RDI: 00007f301e7ff000 [3142757.469535][ C11] RBP: 0000000000000000 R08: 0000000000000001 R09: 00007f2ac8228d50 [3142757.469536][ C11] R10: 0000000000000001 R11: 0000000000000206 R12: 0000000000000000 [3142757.469536][ C11] R13: 00007f34c37f5620 R14: 00007f34c37f56e0 R15: 00007f301effe700 [3142757.472481][ C12] watchdog: BUG: soft lockup - CPU#12 stuck for 22s! [vertica:283862] [3142757.472482][ C12] Modules linked in: binfmt_misc(E) rpcsec_gss_krb5(E) nfsv4(E) dns_resolver(E) nfs(E) fscache(E) nfsd(E) nfs_acl(E) lockd(E) auth_rpcgss(E) grace(E) nf_conntrack(E) nf_defrag_ipv6(E) nf_defrag_ipv4(E) nls_utf8(E) isofs(E) sunrpc(E) ip_tables(E) ufs(E) hfsplus(E) hfs(E) vfat(E) msdos(E) fat(E) xfs(E) libcrc32c(E) dm_mod(E) ext4(E) mbcache(E) jbd2(E) st(E) sd_mod(E) fuse(E) squashfs(E) zstd_decompress(E) loop(E) brd(E) isst_if_common(E) nfit(E) kvm_intel(E) sr_mod(E) cdrom(E) sg(E) kvm(E) ata_generic(E) irqbypass(E) crct10dif_pclmul(E) crc32_pclmul(E) crc32c_intel(E) ghash_clmulni_intel(E) cirrus(E) drm_kms_helper(E) syscopyarea(E) sysfillrect(E) sysimgblt(E) fb_sys_fops(E) drm(E) ata_piix(E) libata(E) virtio_net(E) joydev(E) serio_raw(E) net_failover(E) virtio_blk(E) virtio_balloon(E) pcspkr(E) i2c_piix4(E) failover(E) qemu_fw_cfg(E) sch_fq_codel(E) [last unloaded: dm_mod] [3142757.472501][ C12] CPU: 12 PID: 283862 Comm: vertica Tainted: G EL 5.4.34-1.el7.gdc.x86_64 #1 [3142757.472502][ C12] Hardware name: RDO OpenStack Compute, BIOS 1.11.0-2.el7 04/01/2014 [3142757.472504][ C12] RIP: 0010:smp_call_function_many+0x21e/0x280 [3142757.472505][ C12] Code: c7 e8 56 25 77 00 3b 05 04 9f 48 01 0f 83 7b fe ff ff 48 63 d0 49 8b 0f 48 03 0c d5 80 99 19 82 8b 51 18 83 e2 01 74 0a f3 90 <8b> 51 18 83 e2 01 75 f6 eb c7 0f b6 4c 24 04 48 83 c4 10 48 89 de [3142757.472506][ C12] RSP: 0000:ffffc900015fbc58 EFLAGS: 00000202 ORIG_RAX: ffffffffffffff13 [3142757.472507][ C12] RAX: 0000000000000003 RBX: ffffffff8107c1e0 RCX: ffff88d4e74f16c0 [3142757.472508][ C12] RDX: 0000000000000001 RSI: 000000000000001e RDI: ffff88d4db029c28 [3142757.472508][ C12] RBP: ffff88d4e772a300 R08: 0000000000000000 R09: 0000000000000000 [3142757.472509][ C12] R10: 0000000000000000 R11: 00000000b5130b40 R12: 000000000002ba40 [3142757.472509][ C12] R13: 0000000000000001 R14: 000000000000001e R15: ffff88d4e772ba80 [3142757.472510][ C12] FS: 00007f2f3af94700(0000) GS:ffff88d4e7700000(0000) knlGS:0000000000000000 [3142757.472511][ C12] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [3142757.472512][ C12] CR2: 00007f237a5073b8 CR3: 000000518913e004 CR4: 0000000000760ee0 [3142757.472514][ C12] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [3142757.472514][ C12] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [3142757.472515][ C12] PKRU: 55555554 [3142757.472515][ C12] Call Trace: [3142757.472518][ C12] ? flush_tlb_func_common.isra.8+0x230/0x230 [3142757.472519][ C12] ? flush_tlb_func_common.isra.8+0x230/0x230 [3142757.472520][ C12] on_each_cpu_mask+0x24/0x60 [3142757.472521][ C12] ? x86_configure_nx+0x50/0x50 [3142757.472522][ C12] on_each_cpu_cond_mask+0xa5/0x140 [3142757.472523][ C12] ? flush_tlb_func_common.isra.8+0x92/0x230 [3142757.472524][ C12] flush_tlb_mm_range+0xab/0xe0 [3142757.472526][ C12] ptep_clear_flush+0x56/0x60 [3142757.472528][ C12] wp_page_copy+0x34a/0x7e0 [3142757.472530][ C12] do_wp_page+0x8b/0x660 [3142757.472531][ C12] __handle_mm_fault+0x777/0xef0 [3142757.472533][ C12] handle_mm_fault+0xe2/0x1f0 [3142757.472535][ C12] __do_page_fault+0x226/0x490 [3142757.472537][ C12] do_page_fault+0x31/0x120 [3142757.472538][ C12] async_page_fault+0x3e/0x50 [3142757.472539][ C12] RIP: 0033:0xe148c5 [3142757.472540][ C12] Code: fa 08 0f 84 44 01 00 00 48 83 ff 07 4c 89 c2 76 33 48 83 ef 08 31 d2 48 89 f9 48 c1 e9 03 48 8d 34 cd 08 00 00 00 4d 8b 0c 10 <4c> 89 0c 10 48 83 c2 08 48 39 f2 75 ef 48 f7 d9 48 01 d0 4c 01 c2 [3142757.472541][ C12] RSP: 002b:00007f312c9f60e0 EFLAGS: 00010202 [3142757.472542][ C12] RAX: 00007f237a5073b8 RBX: 000000000000001f RCX: 0000000000000002 [3142757.472542][ C12] RDX: 0000000000000000 RSI: 0000000000000018 RDI: 0000000000000010 [3142757.472543][ C12] RBP: 00007f312c9f6250 R08: 00007f25f853e568 R09: 400008d0cf532ee6 [3142757.472544][ C12] R10: 0000000000000018 R11: 0000000000004b8f R12: 0000000000200000 [3142757.472544][ C12] R13: 00007f2378000000 R14: 00007f302caaa8c0 R15: 00007f312c9f62c0 [3142757.472624][ C4] RSP: 0000:ffffc9000158bc58 EFLAGS: 00000202 ORIG_RAX: ffffffffffffff13 [3142757.475480][ C13] watchdog: BUG: soft lockup - CPU#13 stuck for 22s! [vertica:283848] [3142757.475481][ C13] Modules linked in: binfmt_misc(E) rpcsec_gss_krb5(E) nfsv4(E) dns_resolver(E) nfs(E) fscache(E) nfsd(E) nfs_acl(E) lockd(E) auth_rpcgss(E) grace(E) nf_conntrack(E) nf_defrag_ipv6(E) nf_defrag_ipv4(E) nls_utf8(E) isofs(E) sunrpc(E) ip_tables(E) ufs(E) hfsplus(E) hfs(E) vfat(E) msdos(E) fat(E) xfs(E) libcrc32c(E) dm_mod(E) ext4(E) mbcache(E) jbd2(E) st(E) sd_mod(E) fuse(E) squashfs(E) zstd_decompress(E) loop(E) brd(E) isst_if_common(E) nfit(E) kvm_intel(E) sr_mod(E) cdrom(E) sg(E) kvm(E) ata_generic(E) irqbypass(E) crct10dif_pclmul(E) crc32_pclmul(E) crc32c_intel(E) ghash_clmulni_intel(E) cirrus(E) drm_kms_helper(E) syscopyarea(E) sysfillrect(E) sysimgblt(E) fb_sys_fops(E) drm(E) ata_piix(E) libata(E) virtio_net(E) joydev(E) serio_raw(E) net_failover(E) virtio_blk(E) virtio_balloon(E) pcspkr(E) i2c_piix4(E) failover(E) qemu_fw_cfg(E) sch_fq_codel(E) [last unloaded: dm_mod] [3142757.475497][ C13] CPU: 13 PID: 283848 Comm: vertica Tainted: G EL 5.4.34-1.el7.gdc.x86_64 #1 [3142757.475498][ C13] Hardware name: RDO OpenStack Compute, BIOS 1.11.0-2.el7 04/01/2014 [3142757.475499][ C13] RIP: 0010:smp_call_function_many+0x21e/0x280 [3142757.475500][ C13] Code: c7 e8 56 25 77 00 3b 05 04 9f 48 01 0f 83 7b fe ff ff 48 63 d0 49 8b 0f 48 03 0c d5 80 99 19 82 8b 51 18 83 e2 01 74 0a f3 90 <8b> 51 18 83 e2 01 75 f6 eb c7 0f b6 4c 24 04 48 83 c4 10 48 89 de [3142757.475501][ C13] RSP: 0000:ffffc90001423c58 EFLAGS: 00000202 ORIG_RAX: ffffffffffffff13 [3142757.475502][ C13] RAX: 0000000000000003 RBX: ffffffff8107c1e0 RCX: ffff88d4e74f16e0 [3142757.475502][ C13] RDX: 0000000000000001 RSI: 000000000000001e RDI: ffff88d4db0298e8 [3142757.475503][ C13] RBP: ffff88d4e776a300 R08: 0000000000000000 R09: 0000000000000000 [3142757.475504][ C13] R10: 0000000000000000 R11: 00000000ad2f2600 R12: 000000000002ba40 [3142757.475504][ C13] R13: 0000000000000001 R14: 000000000000001e R15: ffff88d4e776ba80 [3142757.475505][ C13] FS: 00007f2b5a5fb700(0000) GS:ffff88d4e7740000(0000) knlGS:0000000000000000 [3142757.475506][ C13] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [3142757.475506][ C13] CR2: 00007f3202e22760 CR3: 000000518913e003 CR4: 0000000000760ee0 [3142757.475508][ C13] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [3142757.475509][ C13] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [3142757.475509][ C13] PKRU: 55555554 [3142757.475509][ C13] Call Trace: [3142757.475511][ C13] ? flush_tlb_func_common.isra.8+0x230/0x230 [3142757.475512][ C13] ? flush_tlb_func_common.isra.8+0x230/0x230 [3142757.475513][ C13] on_each_cpu_mask+0x24/0x60 [3142757.475514][ C13] ? x86_configure_nx+0x50/0x50 [3142757.475515][ C13] on_each_cpu_cond_mask+0xa5/0x140 [3142757.475516][ C13] ? flush_tlb_func_common.isra.8+0x92/0x230 [3142757.475517][ C13] flush_tlb_mm_range+0xab/0xe0 [3142757.475518][ C13] ptep_clear_flush+0x56/0x60 [3142757.475520][ C13] wp_page_copy+0x34a/0x7e0 [3142757.475521][ C13] do_wp_page+0x8b/0x660 [3142757.475523][ C13] __handle_mm_fault+0x777/0xef0 [3142757.475525][ C13] handle_mm_fault+0xe2/0x1f0 [3142757.475526][ C13] __do_page_fault+0x226/0x490 [3142757.475528][ C13] do_page_fault+0x31/0x120 [3142757.475529][ C13] async_page_fault+0x3e/0x50 [3142757.475529][ C13] RIP: 0033:0xe148c5 [3142757.475530][ C13] Code: fa 08 0f 84 44 01 00 00 48 83 ff 07 4c 89 c2 76 33 48 83 ef 08 31 d2 48 89 f9 48 c1 e9 03 48 8d 34 cd 08 00 00 00 4d 8b 0c 10 <4c> 89 0c 10 48 83 c2 08 48 39 f2 75 ef 48 f7 d9 48 01 d0 4c 01 c2 [3142757.475531][ C13] RSP: 002b:00007f264cbfd2b0 EFLAGS: 00010202 [3142757.475531][ C13] RAX: 00007f3202e22760 RBX: 000000000000001f RCX: 0000000000000002 [3142757.475532][ C13] RDX: 0000000000000000 RSI: 0000000000000018 RDI: 0000000000000010 [3142757.475533][ C13] RBP: 00007f264cbfd420 R08: 00007f275c17c280 R09: 400008d0d18e3052 [3142757.475535][ C5] Code: c7 e8 56 25 77 00 3b 05 04 9f 48 01 0f 83 7b fe ff ff 48 63 d0 49 8b 0f 48 03 0c d5 80 99 19 82 8b 51 18 83 e2 01 74 0a f3 90 <8b> 51 18 83 e2 01 75 f6 eb c7 0f b6 4c 24 04 48 83 c4 10 48 89 de [3142757.475536][ C13] R10: 0000000000000018 R11: 0000000000005570 R12: 0000000000200000 [3142757.475537][ C13] R13: 00007f3200000000 R14: 00007f302ca8bf60 R15: 00007f264cbfd490 [3142757.476772][ C4] RAX: 0000000000000003 RBX: ffffffff8107c1e0 RCX: ffff88d4e74f15c0 [3142757.479689][ C5] RSP: 0000:ffffc90002ebfc58 EFLAGS: 00000202 ORIG_RAX: ffffffffffffff13 [3142757.480964][ C4] RDX: 0000000000000001 RSI: 000000000000001e RDI: ffff88d4db029b38 [3142757.481481][ C15] watchdog: BUG: soft lockup - CPU#15 stuck for 22s! [vertica:283864] [3142757.481481][ C15] Modules linked in: binfmt_misc(E) rpcsec_gss_krb5(E) nfsv4(E) dns_resolver(E) nfs(E) fscache(E) nfsd(E) nfs_acl(E) lockd(E) auth_rpcgss(E) grace(E) nf_conntrack(E) nf_defrag_ipv6(E) nf_defrag_ipv4(E) nls_utf8(E) isofs(E) sunrpc(E) ip_tables(E) ufs(E) hfsplus(E) hfs(E) vfat(E) msdos(E) fat(E) xfs(E) libcrc32c(E) dm_mod(E) ext4(E) mbcache(E) jbd2(E) st(E) sd_mod(E) fuse(E) squashfs(E) zstd_decompress(E) loop(E) brd(E) isst_if_common(E) nfit(E) kvm_intel(E) sr_mod(E) cdrom(E) sg(E) kvm(E) ata_generic(E) irqbypass(E) crct10dif_pclmul(E) crc32_pclmul(E) crc32c_intel(E) ghash_clmulni_intel(E) cirrus(E) drm_kms_helper(E) syscopyarea(E) sysfillrect(E) sysimgblt(E) fb_sys_fops(E) drm(E) ata_piix(E) libata(E) virtio_net(E) joydev(E) serio_raw(E) net_failover(E) virtio_blk(E) virtio_balloon(E) pcspkr(E) i2c_piix4(E) failover(E) qemu_fw_cfg(E) sch_fq_codel(E) [last unloaded: dm_mod] [3142757.481500][ C15] CPU: 15 PID: 283864 Comm: vertica Tainted: G EL 5.4.34-1.el7.gdc.x86_64 #1 [3142757.481500][ C15] Hardware name: RDO OpenStack Compute, BIOS 1.11.0-2.el7 04/01/2014 [3142757.481502][ C15] RIP: 0010:smp_call_function_many+0x21e/0x280 [3142757.481504][ C15] Code: c7 e8 56 25 77 00 3b 05 04 9f 48 01 0f 83 7b fe ff ff 48 63 d0 49 8b 0f 48 03 0c d5 80 99 19 82 8b 51 18 83 e2 01 74 0a f3 90 <8b> 51 18 83 e2 01 75 f6 eb c7 0f b6 4c 24 04 48 83 c4 10 48 89 de [3142757.481505][ C15] RSP: 0000:ffffc9000161bc58 EFLAGS: 00000202 ORIG_RAX: ffffffffffffff13 [3142757.481506][ C15] RAX: 0000000000000003 RBX: ffffffff8107c1e0 RCX: ffff88d4e74f1720 [3142757.481506][ C15] RDX: 0000000000000001 RSI: 000000000000001e RDI: ffff88d4db029f28 [3142757.481507][ C15] RBP: ffff88d4e77ea300 R08: 0000000000000000 R09: 0000000000000000 [3142757.481507][ C15] R10: 0000000000000000 R11: 000000001e5bdb80 R12: 000000000002ba40 [3142757.481508][ C15] R13: 0000000000000001 R14: 000000000000001e R15: ffff88d4e77eba80 [3142757.481509][ C15] FS: 00007f2f4aff6700(0000) GS:ffff88d4e77c0000(0000) knlGS:0000000000000000 [3142757.481509][ C15] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [3142757.481510][ C15] CR2: 00007f308ed0ec18 CR3: 000000518913e003 CR4: 0000000000760ee0 [3142757.481512][ C15] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [3142757.481512][ C15] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [3142757.481513][ C15] PKRU: 55555554 [3142757.481513][ C15] Call Trace: [3142757.481516][ C15] ? flush_tlb_func_common.isra.8+0x230/0x230 [3142757.481517][ C15] ? flush_tlb_func_common.isra.8+0x230/0x230 [3142757.481518][ C15] on_each_cpu_mask+0x24/0x60 [3142757.481519][ C15] ? x86_configure_nx+0x50/0x50 [3142757.481520][ C15] on_each_cpu_cond_mask+0xa5/0x140 [3142757.481521][ C15] ? flush_tlb_func_common.isra.8+0x92/0x230 [3142757.481522][ C15] flush_tlb_mm_range+0xab/0xe0 [3142757.481524][ C15] ptep_clear_flush+0x56/0x60 [3142757.481525][ C15] wp_page_copy+0x34a/0x7e0 [3142757.481527][ C15] do_wp_page+0x8b/0x660 [3142757.481529][ C15] __handle_mm_fault+0x777/0xef0 [3142757.481531][ C15] handle_mm_fault+0xe2/0x1f0 [3142757.481532][ C15] __do_page_fault+0x226/0x490 [3142757.481534][ C15] do_page_fault+0x31/0x120 [3142757.481535][ C15] async_page_fault+0x3e/0x50 [3142757.481536][ C15] RIP: 0033:0xe148c5 [3142757.481537][ C15] Code: fa 08 0f 84 44 01 00 00 48 83 ff 07 4c 89 c2 76 33 48 83 ef 08 31 d2 48 89 f9 48 c1 e9 03 48 8d 34 cd 08 00 00 00 4d 8b 0c 10 <4c> 89 0c 10 48 83 c2 08 48 39 f2 75 ef 48 f7 d9 48 01 d0 4c 01 c2 [3142757.481538][ C15] RSP: 002b:00007f312cbf8c60 EFLAGS: 00010202 [3142757.481539][ C15] RAX: 00007f308ed0ec18 RBX: 000000000000001f RCX: 0000000000000002 [3142757.481539][ C15] RDX: 0000000000000000 RSI: 0000000000000018 RDI: 0000000000000010 [3142757.481540][ C15] RBP: 00007f312cbf8dd0 R08: 00007f26a2073860 R09: 400008d0c927fae4 [3142757.481541][ C15] R10: 0000000000000018 R11: 0000000000004d04 R12: 0000000000200000 [3142757.481541][ C15] R13: 00007f308d5f5000 R14: 00007f302cab0aa0 R15: 00007f312cbf8e40 [3142757.483909][ C5] RAX: 000000000000001a RBX: ffffffff8107c1e0 RCX: ffff88d4e7ab15e0 [3142757.483910][ C5] RDX: 0000000000000001 RSI: 000000000000001e RDI: ffff88d4db0291c8 [3142757.484482][ C16] watchdog: BUG: soft lockup - CPU#16 stuck for 22s! [vertica:284270] [3142757.484482][ C16] Modules linked in: binfmt_misc(E) rpcsec_gss_krb5(E) nfsv4(E) dns_resolver(E) nfs(E) fscache(E) nfsd(E) nfs_acl(E) lockd(E) auth_rpcgss(E) grace(E) nf_conntrack(E) nf_defrag_ipv6(E) nf_defrag_ipv4(E) nls_utf8(E) isofs(E) sunrpc(E) ip_tables(E) ufs(E) hfsplus(E) hfs(E) vfat(E) msdos(E) fat(E) xfs(E) libcrc32c(E) dm_mod(E) ext4(E) mbcache(E) jbd2(E) st(E) sd_mod(E) fuse(E) squashfs(E) zstd_decompress(E) loop(E) brd(E) isst_if_common(E) nfit(E) kvm_intel(E) sr_mod(E) cdrom(E) sg(E) kvm(E) ata_generic(E) irqbypass(E) crct10dif_pclmul(E) crc32_pclmul(E) crc32c_intel(E) ghash_clmulni_intel(E) cirrus(E) drm_kms_helper(E) syscopyarea(E) sysfillrect(E) sysimgblt(E) fb_sys_fops(E) drm(E) ata_piix(E) libata(E) virtio_net(E) joydev(E) serio_raw(E) net_failover(E) virtio_blk(E) virtio_balloon(E) pcspkr(E) i2c_piix4(E) failover(E) qemu_fw_cfg(E) sch_fq_codel(E) [last unloaded: dm_mod] [3142757.484500][ C16] CPU: 16 PID: 284270 Comm: vertica Tainted: G EL 5.4.34-1.el7.gdc.x86_64 #1 [3142757.484501][ C16] Hardware name: RDO OpenStack Compute, BIOS 1.11.0-2.el7 04/01/2014 [3142757.484503][ C16] RIP: 0010:smp_call_function_many+0x21e/0x280 [3142757.484504][ C16] Code: c7 e8 56 25 77 00 3b 05 04 9f 48 01 0f 83 7b fe ff ff 48 63 d0 49 8b 0f 48 03 0c d5 80 99 19 82 8b 51 18 83 e2 01 74 0a f3 90 <8b> 51 18 83 e2 01 75 f6 eb c7 0f b6 4c 24 04 48 83 c4 10 48 89 de [3142757.484505][ C16] RSP: 0018:ffffc90000463948 EFLAGS: 00000202 ORIG_RAX: ffffffffffffff13 [3142757.484506][ C16] RAX: 0000000000000003 RBX: ffffffff8107c1e0 RCX: ffff88d4e74f1740 [3142757.484507][ C16] RDX: 0000000000000001 RSI: 000000000000001e RDI: ffff88d4db029008 [3142757.484507][ C16] RBP: ffff88d4e782a300 R08: 0000000000000000 R09: 0000000000000000 [3142757.484508][ C16] R10: 0000000000000000 R11: 000000011f521840 R12: 000000000002ba40 [3142757.484508][ C16] R13: 0000000000000001 R14: 000000000000001e R15: ffff88d4e782ba80 [3142757.484509][ C16] FS: 00007f307f5ea700(0000) GS:ffff88d4e7800000(0000) knlGS:0000000000000000 [3142757.484510][ C16] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [3142757.484511][ C16] CR2: 00007f302df32740 CR3: 000000518913e004 CR4: 0000000000760ee0 [3142757.484513][ C16] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [3142757.484513][ C16] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [3142757.484514][ C16] PKRU: 55555554 [3142757.484514][ C16] Call Trace: [3142757.484517][ C16] ? flush_tlb_func_common.isra.8+0x230/0x230 [3142757.484518][ C16] ? flush_tlb_func_common.isra.8+0x230/0x230 [3142757.484519][ C16] on_each_cpu_mask+0x24/0x60 [3142757.484520][ C16] ? x86_configure_nx+0x50/0x50 [3142757.484521][ C16] on_each_cpu_cond_mask+0xa5/0x140 [3142757.484522][ C16] ? flush_tlb_func_common.isra.8+0x147/0x230 [3142757.484523][ C16] flush_tlb_mm_range+0xab/0xe0 [3142757.484525][ C16] ptep_clear_flush+0x56/0x60 [3142757.484526][ C16] wp_page_copy+0x34a/0x7e0 [3142757.484528][ C16] do_wp_page+0x8b/0x660 [3142757.484530][ C16] __handle_mm_fault+0x777/0xef0 [3142757.484532][ C16] handle_mm_fault+0xe2/0x1f0 [3142757.484533][ C16] __do_page_fault+0x226/0x490 [3142757.484535][ C16] do_page_fault+0x31/0x120 [3142757.484536][ C16] async_page_fault+0x3e/0x50 [3142757.484538][ C16] RIP: 0010:copy_user_enhanced_fast_string+0xe/0x30 [3142757.484539][ C16] Code: 89 d1 c1 e9 03 83 e2 07 f3 48 a5 89 d1 f3 a4 31 c0 0f 01 ca c3 0f 1f 80 00 00 00 00 0f 01 cb 83 fa 40 0f 82 70 ff ff ff 89 d1 a4 31 c0 0f 01 ca c3 66 2e 0f 1f 84 00 00 00 00 00 89 d1 f3 a4 [3142757.484540][ C16] RSP: 0018:ffffc90000463cf8 EFLAGS: 00050202 [3142757.484541][ C16] RAX: 00007f302df32930 RBX: ffffc90000463e58 RCX: 00000000000001f0 [3142757.484541][ C16] RDX: 00000000000001f0 RSI: ffff88bc2be34e10 RDI: 00007f302df32740 [3142757.484542][ C16] RBP: 00000000000001f0 R08: 0000000000004009 R09: 0000000000000000 [3142757.484542][ C16] R10: 0000000000004000 R11: ffff88d1005ec538 R12: ffffc90000463e48 [3142757.484543][ C16] R13: 00000000000001f0 R14: ffff88bc2be34e10 R15: 00000000000001f0 [3142757.484548][ C16] copyout+0x22/0x30 [3142757.484549][ C16] copy_page_to_iter+0xfd/0x350 [3142757.484552][ C16] generic_file_buffered_read+0x4d6/0xbd0 [3142757.484553][ C16] ? do_futex+0x44c/0x940 [3142757.484555][ C16] new_sync_read+0x106/0x1a0 [3142757.484558][ C16] vfs_read+0x89/0x140 [3142757.484559][ C16] ksys_read+0x55/0xd0 [3142757.484561][ C16] do_syscall_64+0x5b/0x1b0 [3142757.484563][ C16] entry_SYSCALL_64_after_hwframe+0x44/0xa9 [3142757.484564][ C16] RIP: 0033:0x7f35e1c6171d [3142757.484565][ C16] Code: cc 20 00 00 75 10 b8 00 00 00 00 0f 05 48 3d 01 f0 ff ff 73 31 c3 48 83 ec 08 e8 ee fc ff ff 48 89 04 24 b8 00 00 00 00 0f 05 <48> 8b 3c 24 48 89 c2 e8 37 fd ff ff 48 89 d0 48 83 c4 08 48 3d 01 [3142757.484565][ C16] RSP: 002b:00007f307f5e6d80 EFLAGS: 00000293 ORIG_RAX: 0000000000000000 [3142757.484566][ C16] RAX: ffffffffffffffda RBX: 00007f302df32740 RCX: 00007f35e1c6171d [3142757.484567][ C16] RDX: 0000000000004009 RSI: 00007f302df32740 RDI: 00000000000000e5 [3142757.484567][ C16] RBP: 00007f307f5e6ef0 R08: 0000000000000000 R09: 0000000000000028 [3142757.484568][ C16] R10: 0000000000000000 R11: 0000000000000293 R12: 0000000000004009 [3142757.484568][ C16] R13: 00007f35d0099b30 R14: 0000000000000000 R15: 0000000000000000 [3142757.485141][ C4] RBP: ffff88d4e752a300 R08: 0000000000000000 R09: 0000000000000000 [3142757.487480][ C17] watchdog: BUG: soft lockup - CPU#17 stuck for 22s! [vertica:284226] [3142757.487481][ C17] Modules linked in: binfmt_misc(E) rpcsec_gss_krb5(E) nfsv4(E) dns_resolver(E) nfs(E) fscache(E) nfsd(E) nfs_acl(E) lockd(E) auth_rpcgss(E) grace(E) nf_conntrack(E) nf_defrag_ipv6(E) nf_defrag_ipv4(E) nls_utf8(E) isofs(E) sunrpc(E) ip_tables(E) ufs(E) hfsplus(E) hfs(E) vfat(E) msdos(E) fat(E) xfs(E) libcrc32c(E) dm_mod(E) ext4(E) mbcache(E) jbd2(E) st(E) sd_mod(E) fuse(E) squashfs(E) zstd_decompress(E) loop(E) brd(E) isst_if_common(E) nfit(E) kvm_intel(E) sr_mod(E) cdrom(E) sg(E) kvm(E) ata_generic(E) irqbypass(E) crct10dif_pclmul(E) crc32_pclmul(E) crc32c_intel(E) ghash_clmulni_intel(E) cirrus(E) drm_kms_helper(E) syscopyarea(E) sysfillrect(E) sysimgblt(E) fb_sys_fops(E) drm(E) ata_piix(E) libata(E) virtio_net(E) joydev(E) serio_raw(E) net_failover(E) virtio_blk(E) virtio_balloon(E) pcspkr(E) i2c_piix4(E) failover(E) qemu_fw_cfg(E) sch_fq_codel(E) [last unloaded: dm_mod] [3142757.487497][ C17] CPU: 17 PID: 284226 Comm: vertica Tainted: G EL 5.4.34-1.el7.gdc.x86_64 #1 [3142757.487497][ C17] Hardware name: RDO OpenStack Compute, BIOS 1.11.0-2.el7 04/01/2014 [3142757.487499][ C17] RIP: 0010:smp_call_function_many+0x21e/0x280 [3142757.487500][ C17] Code: c7 e8 56 25 77 00 3b 05 04 9f 48 01 0f 83 7b fe ff ff 48 63 d0 49 8b 0f 48 03 0c d5 80 99 19 82 8b 51 18 83 e2 01 74 0a f3 90 <8b> 51 18 83 e2 01 75 f6 eb c7 0f b6 4c 24 04 48 83 c4 10 48 89 de [3142757.487500][ C17] RSP: 0000:ffffc9000343fc58 EFLAGS: 00000202 ORIG_RAX: ffffffffffffff13 [3142757.487501][ C17] RAX: 000000000000001a RBX: ffffffff8107c1e0 RCX: ffff88d4e7ab1760 [3142757.487502][ C17] RDX: 0000000000000001 RSI: 000000000000001e RDI: ffff88d4db0298a0 [3142757.487502][ C17] RBP: ffff88d4e786a300 R08: 0000000000000000 R09: 0000000000000000 [3142757.487503][ C17] R10: 0000000000000000 R11: 00000000adeeb600 R12: 000000000002ba40 [3142757.487503][ C17] R13: 0000000000000001 R14: 000000000000001e R15: ffff88d4e786ba80 [3142757.487504][ C17] FS: 00007f3103fff700(0000) GS:ffff88d4e7840000(0000) knlGS:0000000000000000 [3142757.487505][ C17] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [3142757.487506][ C17] CR2: 00007f3509f02368 CR3: 000000518913e004 CR4: 0000000000760ee0 [3142757.487508][ C17] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [3142757.487508][ C17] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [3142757.487508][ C17] PKRU: 55555554 [3142757.487509][ C17] Call Trace: [3142757.487510][ C17] ? flush_tlb_func_common.isra.8+0x230/0x230 [3142757.487511][ C17] ? flush_tlb_func_common.isra.8+0x230/0x230 [3142757.487512][ C17] on_each_cpu_mask+0x24/0x60 [3142757.487513][ C17] ? x86_configure_nx+0x50/0x50 [3142757.487514][ C17] on_each_cpu_cond_mask+0xa5/0x140 [3142757.487515][ C17] ? flush_tlb_func_common.isra.8+0x147/0x230 [3142757.487517][ C17] flush_tlb_mm_range+0xab/0xe0 [3142757.487518][ C17] ptep_clear_flush+0x56/0x60 [3142757.487519][ C17] wp_page_copy+0x34a/0x7e0 [3142757.487521][ C17] do_wp_page+0x8b/0x660 [3142757.487522][ C17] __handle_mm_fault+0x777/0xef0 [3142757.487524][ C17] handle_mm_fault+0xe2/0x1f0 [3142757.487526][ C17] __do_page_fault+0x226/0x490 [3142757.487527][ C17] do_page_fault+0x31/0x120 [3142757.487528][ C17] async_page_fault+0x3e/0x50 [3142757.487529][ C17] RIP: 0033:0x101d045 [3142757.487530][ C17] Code: 48 31 c1 4c 21 c9 49 89 0c fe 8b 46 24 48 0f af c1 48 03 46 28 48 8b 10 48 85 d2 75 80 48 ba 00 00 00 00 00 00 00 40 49 09 d0 <4c> 89 00 48 8b 56 40 48 3b 56 60 73 0c 4c 8b 46 68 49 89 0c d0 48 [3142757.487530][ C17] RSP: 002b:00007f28e81fc5d0 EFLAGS: 00010206 [3142757.487531][ C17] RAX: 00007f3509f02368 RBX: 0000000000000019 RCX: 0000000000000b49 [3142757.487532][ C17] RDX: 4000000000000000 RSI: 00007f28e81fdab0 RDI: 0000000000000019 [3142757.487532][ C17] RBP: 00007f28e81fc6a0 R08: 40758102271da741 R09: 0000000000007fff [3142757.487533][ C17] R10: 3fffffffffffffff R11: 00007f2ecdd7eaf0 R12: 0000000000000001 [3142757.487533][ C17] R13: 00007f28e8204090 R14: 00007f28e8244090 R15: 0000000000000034 [3142757.488228][ C5] RBP: ffff88d4e756a300 R08: 0000000000000000 R09: 0000000000000000 [3142757.490482][ C18] watchdog: BUG: soft lockup - CPU#18 stuck for 22s! [vertica:284166] [3142757.490482][ C18] Modules linked in: binfmt_misc(E) rpcsec_gss_krb5(E) nfsv4(E) dns_resolver(E) nfs(E) fscache(E) nfsd(E) nfs_acl(E) lockd(E) auth_rpcgss(E) grace(E) nf_conntrack(E) nf_defrag_ipv6(E) nf_defrag_ipv4(E) nls_utf8(E) isofs(E) sunrpc(E) ip_tables(E) ufs(E) hfsplus(E) hfs(E) vfat(E) msdos(E) fat(E) xfs(E) libcrc32c(E) dm_mod(E) ext4(E) mbcache(E) jbd2(E) st(E) sd_mod(E) fuse(E) squashfs(E) zstd_decompress(E) loop(E) brd(E) isst_if_common(E) nfit(E) kvm_intel(E) sr_mod(E) cdrom(E) sg(E) kvm(E) ata_generic(E) irqbypass(E) crct10dif_pclmul(E) crc32_pclmul(E) crc32c_intel(E) ghash_clmulni_intel(E) cirrus(E) drm_kms_helper(E) syscopyarea(E) sysfillrect(E) sysimgblt(E) fb_sys_fops(E) drm(E) ata_piix(E) libata(E) virtio_net(E) joydev(E) serio_raw(E) net_failover(E) virtio_blk(E) virtio_balloon(E) pcspkr(E) i2c_piix4(E) failover(E) qemu_fw_cfg(E) sch_fq_codel(E) [last unloaded: dm_mod] [3142757.490501][ C18] CPU: 18 PID: 284166 Comm: vertica Tainted: G EL 5.4.34-1.el7.gdc.x86_64 #1 [3142757.490502][ C18] Hardware name: RDO OpenStack Compute, BIOS 1.11.0-2.el7 04/01/2014 [3142757.490504][ C18] RIP: 0010:smp_call_function_many+0x21e/0x280 [3142757.490506][ C18] Code: c7 e8 56 25 77 00 3b 05 04 9f 48 01 0f 83 7b fe ff ff 48 63 d0 49 8b 0f 48 03 0c d5 80 99 19 82 8b 51 18 83 e2 01 74 0a f3 90 <8b> 51 18 83 e2 01 75 f6 eb c7 0f b6 4c 24 04 48 83 c4 10 48 89 de [3142757.490506][ C18] RSP: 0000:ffffc90002ed7c58 EFLAGS: 00000202 ORIG_RAX: ffffffffffffff13 [3142757.490507][ C18] RAX: 000000000000001a RBX: ffffffff8107c1e0 RCX: ffff88d4e7ab1780 [3142757.490508][ C18] RDX: 0000000000000001 RSI: 000000000000001e RDI: ffff88d4db0291b0 [3142757.490509][ C18] RBP: ffff88d4e78aa300 R08: 0000000000000000 R09: 0000000000000000 [3142757.490509][ C18] R10: 0000000000000000 R11: 0000000069564e00 R12: 000000000002ba40 [3142757.490510][ C18] R13: 0000000000000001 R14: 000000000000001e R15: ffff88d4e78aba80 [3142757.490511][ C18] FS: 00007f31e15f1700(0000) GS:ffff88d4e7880000(0000) knlGS:0000000000000000 [3142757.490511][ C18] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [3142757.490512][ C18] CR2: 00007f28da2a3520 CR3: 000000518913e003 CR4: 0000000000760ee0 [3142757.490514][ C18] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [3142757.490515][ C18] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [3142757.490515][ C18] PKRU: 55555554 [3142757.490516][ C18] Call Trace: [3142757.490518][ C18] ? flush_tlb_func_common.isra.8+0x230/0x230 [3142757.490519][ C18] ? flush_tlb_func_common.isra.8+0x230/0x230 [3142757.490520][ C18] on_each_cpu_mask+0x24/0x60 [3142757.490521][ C18] ? x86_configure_nx+0x50/0x50 [3142757.490522][ C18] on_each_cpu_cond_mask+0xa5/0x140 [3142757.490524][ C18] ? flush_tlb_func_common.isra.8+0x147/0x230 [3142757.490525][ C18] flush_tlb_mm_range+0xab/0xe0 [3142757.490526][ C18] ptep_clear_flush+0x56/0x60 [3142757.490528][ C18] wp_page_copy+0x34a/0x7e0 [3142757.490530][ C18] do_wp_page+0x8b/0x660 [3142757.490531][ C18] __handle_mm_fault+0x777/0xef0 [3142757.490533][ C18] handle_mm_fault+0xe2/0x1f0 [3142757.490535][ C18] __do_page_fault+0x226/0x490 [3142757.490536][ C18] do_page_fault+0x31/0x120 [3142757.490538][ C18] async_page_fault+0x3e/0x50 [3142757.490539][ C18] RIP: 0033:0x101d045 [3142757.490540][ C18] Code: 48 31 c1 4c 21 c9 49 89 0c fe 8b 46 24 48 0f af c1 48 03 46 28 48 8b 10 48 85 d2 75 80 48 ba 00 00 00 00 00 00 00 40 49 09 d0 <4c> 89 00 48 8b 56 40 48 3b 56 60 73 0c 4c 8b 46 68 49 89 0c d0 48 [3142757.490540][ C18] RSP: 002b:00007f2fb21e36d0 EFLAGS: 00010202 [3142757.490541][ C18] RAX: 00007f28da2a3520 RBX: 00000000000000cd RCX: 000000000000718c [3142757.490542][ C18] RDX: 4000000000000000 RSI: 00007f2fb21ec9c0 RDI: 00000000000000cd [3142757.490542][ C18] RBP: 00007f2fb21e37a0 R08: 400008d0bf697101 R09: 000000000000ffff [3142757.490543][ C18] R10: 3fffffffffffffff R11: 00007f302c5f95e0 R12: 0000000000000001 [3142757.490544][ C18] R13: 00007f2fb21eedf0 R14: 00007f2fb222edf0 R15: 0000000000000034 [3142757.491170][ C4] R10: 0000000000000000 R11: 00000000cd929540 R12: 000000000002ba40 [3142757.491172][ C4] R13: 0000000000000001 R14: 000000000000001e R15: ffff88d4e752ba80 [3142757.492413][ C5] R10: 0000000000000000 R11: 00000000e2b3f0c0 R12: 000000000002ba40 [3142757.495361][ C4] FS: 00007f2b5e9ff700(0000) GS:ffff88d4e7500000(0000) knlGS:0000000000000000 [3142757.496580][ C5] R13: 0000000000000001 R14: 000000000000001e R15: ffff88d4e756ba80 [3142757.499718][ C4] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [3142757.500480][ C21] watchdog: BUG: soft lockup - CPU#21 stuck for 23s! [vertica:284315] [3142757.500481][ C21] Modules linked in: binfmt_misc(E) rpcsec_gss_krb5(E) nfsv4(E) dns_resolver(E) nfs(E) fscache(E) nfsd(E) nfs_acl(E) lockd(E) auth_rpcgss(E) grace(E) nf_conntrack(E) nf_defrag_ipv6(E) nf_defrag_ipv4(E) nls_utf8(E) isofs(E) sunrpc(E) ip_tables(E) ufs(E) hfsplus(E) hfs(E) vfat(E) msdos(E) fat(E) xfs(E) libcrc32c(E) dm_mod(E) ext4(E) mbcache(E) jbd2(E) st(E) sd_mod(E) fuse(E) squashfs(E) zstd_decompress(E) loop(E) brd(E) isst_if_common(E) nfit(E) kvm_intel(E) sr_mod(E) cdrom(E) sg(E) kvm(E) ata_generic(E) irqbypass(E) crct10dif_pclmul(E) crc32_pclmul(E) crc32c_intel(E) ghash_clmulni_intel(E) cirrus(E) drm_kms_helper(E) syscopyarea(E) sysfillrect(E) sysimgblt(E) fb_sys_fops(E) drm(E) ata_piix(E) libata(E) virtio_net(E) joydev(E) serio_raw(E) net_failover(E) virtio_blk(E) virtio_balloon(E) pcspkr(E) i2c_piix4(E) failover(E) qemu_fw_cfg(E) sch_fq_codel(E) [last unloaded: dm_mod] [3142757.500499][ C21] CPU: 21 PID: 284315 Comm: vertica Tainted: G EL 5.4.34-1.el7.gdc.x86_64 #1 [3142757.500499][ C21] Hardware name: RDO OpenStack Compute, BIOS 1.11.0-2.el7 04/01/2014 [3142757.500501][ C21] RIP: 0010:smp_call_function_many+0x21e/0x280 [3142757.500503][ C21] Code: c7 e8 56 25 77 00 3b 05 04 9f 48 01 0f 83 7b fe ff ff 48 63 d0 49 8b 0f 48 03 0c d5 80 99 19 82 8b 51 18 83 e2 01 74 0a f3 90 <8b> 51 18 83 e2 01 75 f6 eb c7 0f b6 4c 24 04 48 83 c4 10 48 89 de [3142757.500503][ C21] RSP: 0018:ffffc90001877ce0 EFLAGS: 00000202 ORIG_RAX: ffffffffffffff13 [3142757.500504][ C21] RAX: 000000000000001a RBX: ffffffff8107c1e0 RCX: ffff88d4e7ab17e0 [3142757.500505][ C21] RDX: 0000000000000001 RSI: 000000000000001e RDI: ffff88d4db029c30 [3142757.500505][ C21] RBP: ffff88d4e796a300 R08: 0000000000000000 R09: 0000000000000000 [3142757.500506][ C21] R10: 0000000000000000 R11: 0000000000000000 R12: 000000000002ba40 [3142757.500506][ C21] R13: 0000000000000001 R14: 000000000000001e R15: ffff88d4e796ba80 [3142757.500507][ C21] FS: 00007f305c9c1700(0000) GS:ffff88d4e7940000(0000) knlGS:0000000000000000 [3142757.500508][ C21] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [3142757.500508][ C21] CR2: 00007fa6c84ecffc CR3: 000000518913e004 CR4: 0000000000760ee0 [3142757.500510][ C21] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [3142757.500511][ C21] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [3142757.500511][ C21] PKRU: 55555554 [3142757.500511][ C21] Call Trace: [3142757.500514][ C21] flush_tlb_mm_range+0xab/0xe0 [3142757.500515][ C21] tlb_flush_mmu+0x89/0x150 [3142757.500516][ C21] tlb_finish_mmu+0x3d/0x70 [3142757.500518][ C21] zap_page_range+0x115/0x160 [3142757.500519][ C21] ? do_futex+0x44c/0x940 [3142757.500521][ C21] ? enqueue_entity+0x231/0x5d0 [3142757.500522][ C21] ? find_vma+0x59/0x60 [3142757.500524][ C21] __do_sys_madvise+0x738/0xc00 [3142757.500526][ C21] do_syscall_64+0x5b/0x1b0 [3142757.500528][ C21] entry_SYSCALL_64_after_hwframe+0x44/0xa9 [3142757.500529][ C21] RIP: 0033:0x7f35e1571e97 [3142757.500530][ C21] Code: 83 c4 08 48 3d 01 f0 ff ff 73 01 c3 48 8b 0d d0 df 2c 00 f7 d8 64 89 01 48 83 c8 ff c3 66 0f 1f 44 00 00 b8 1c 00 00 00 0f 05 <48> 3d 01 f0 ff ff 73 01 c3 48 8b 0d a9 df 2c 00 f7 d8 64 89 01 48 [3142757.500530][ C21] RSP: 002b:00007f305c9be3d8 EFLAGS: 00000206 ORIG_RAX: 000000000000001c [3142757.500531][ C21] RAX: ffffffffffffffda RBX: 00007f305c9c1700 RCX: 00007f35e1571e97 [3142757.500532][ C21] RDX: 0000000000000004 RSI: 00000000007f8000 RDI: 00007f305c1c2000 [3142757.500532][ C21] RBP: 0000000000000000 R08: 0000000000000001 R09: 00007f2bc45d6f90 [3142757.500532][ C21] R10: 0000000000000001 R11: 0000000000000206 R12: 0000000000000000 [3142757.500533][ C21] R13: 00007f34c37f5620 R14: 00007f34c37f56e0 R15: 00007f305c9c1700 [3142757.502748][ C5] FS: 00007f31e85f1700(0000) GS:ffff88d4e7540000(0000) knlGS:0000000000000000 [3142757.506480][ C23] watchdog: BUG: soft lockup - CPU#23 stuck for 23s! [vertica:283851] [3142757.506481][ C23] Modules linked in: binfmt_misc(E) rpcsec_gss_krb5(E) nfsv4(E) dns_resolver(E) nfs(E) fscache(E) nfsd(E) nfs_acl(E) lockd(E) auth_rpcgss(E) grace(E) nf_conntrack(E) nf_defrag_ipv6(E) nf_defrag_ipv4(E) nls_utf8(E) isofs(E) sunrpc(E) ip_tables(E) ufs(E) hfsplus(E) hfs(E) vfat(E) msdos(E) fat(E) xfs(E) libcrc32c(E) dm_mod(E) ext4(E) mbcache(E) jbd2(E) st(E) sd_mod(E) fuse(E) squashfs(E) zstd_decompress(E) loop(E) brd(E) isst_if_common(E) nfit(E) kvm_intel(E) sr_mod(E) cdrom(E) sg(E) kvm(E) ata_generic(E) irqbypass(E) crct10dif_pclmul(E) crc32_pclmul(E) crc32c_intel(E) ghash_clmulni_intel(E) cirrus(E) drm_kms_helper(E) syscopyarea(E) sysfillrect(E) sysimgblt(E) fb_sys_fops(E) drm(E) ata_piix(E) libata(E) virtio_net(E) joydev(E) serio_raw(E) net_failover(E) virtio_blk(E) virtio_balloon(E) pcspkr(E) i2c_piix4(E) failover(E) qemu_fw_cfg(E) sch_fq_codel(E) [last unloaded: dm_mod] [3142757.506498][ C23] CPU: 23 PID: 283851 Comm: vertica Tainted: G EL 5.4.34-1.el7.gdc.x86_64 #1 [3142757.506498][ C23] Hardware name: RDO OpenStack Compute, BIOS 1.11.0-2.el7 04/01/2014 [3142757.506500][ C23] RIP: 0010:smp_call_function_many+0x21e/0x280 [3142757.506502][ C23] Code: c7 e8 56 25 77 00 3b 05 04 9f 48 01 0f 83 7b fe ff ff 48 63 d0 49 8b 0f 48 03 0c d5 80 99 19 82 8b 51 18 83 e2 01 74 0a f3 90 <8b> 51 18 83 e2 01 75 f6 eb c7 0f b6 4c 24 04 48 83 c4 10 48 89 de [3142757.506502][ C23] RSP: 0000:ffffc9000157bc58 EFLAGS: 00000202 ORIG_RAX: ffffffffffffff13 [3142757.506503][ C23] RAX: 0000000000000003 RBX: ffffffff8107c1e0 RCX: ffff88d4e74f1820 [3142757.506504][ C23] RDX: 0000000000000001 RSI: 000000000000001e RDI: ffff88d4db0290b0 [3142757.506504][ C23] RBP: ffff88d4e79ea300 R08: 0000000000000000 R09: 0000000000000000 [3142757.506505][ C23] R10: 0000000000000000 R11: 00000000aa0ab680 R12: 000000000002ba40 [3142757.506505][ C23] R13: 0000000000000001 R14: 000000000000001e R15: ffff88d4e79eba80 [3142757.506506][ C23] FS: 00007f2b5d9fe700(0000) GS:ffff88d4e79c0000(0000) knlGS:0000000000000000 [3142757.506507][ C23] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [3142757.506507][ C23] CR2: 00007f30232a4e20 CR3: 000000518913e003 CR4: 0000000000760ee0 [3142757.506509][ C23] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [3142757.506510][ C23] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [3142757.506510][ C23] PKRU: 55555554 [3142757.506510][ C23] Call Trace: [3142757.506513][ C23] ? flush_tlb_func_common.isra.8+0x230/0x230 [3142757.506514][ C23] ? flush_tlb_func_common.isra.8+0x230/0x230 [3142757.506515][ C23] on_each_cpu_mask+0x24/0x60 [3142757.506516][ C23] ? x86_configure_nx+0x50/0x50 [3142757.506517][ C23] on_each_cpu_cond_mask+0xa5/0x140 [3142757.506518][ C23] ? flush_tlb_func_common.isra.8+0x92/0x230 [3142757.506519][ C23] flush_tlb_mm_range+0xab/0xe0 [3142757.506520][ C23] ptep_clear_flush+0x56/0x60 [3142757.506522][ C23] wp_page_copy+0x34a/0x7e0 [3142757.506523][ C23] do_wp_page+0x8b/0x660 [3142757.506525][ C23] __handle_mm_fault+0x777/0xef0 [3142757.506527][ C23] handle_mm_fault+0xe2/0x1f0 [3142757.506529][ C23] __do_page_fault+0x226/0x490 [3142757.506530][ C23] do_page_fault+0x31/0x120 [3142757.506531][ C23] async_page_fault+0x3e/0x50 [3142757.506532][ C23] RIP: 0033:0xe148c5 [3142757.506533][ C23] Code: fa 08 0f 84 44 01 00 00 48 83 ff 07 4c 89 c2 76 33 48 83 ef 08 31 d2 48 89 f9 48 c1 e9 03 48 8d 34 cd 08 00 00 00 4d 8b 0c 10 <4c> 89 0c 10 48 83 c2 08 48 39 f2 75 ef 48 f7 d9 48 01 d0 4c 01 c2 [3142757.506533][ C23] RSP: 002b:00007f264cf013f0 EFLAGS: 00010202 [3142757.506534][ C23] RAX: 00007f30232a4e20 RBX: 000000000000001f RCX: 0000000000000002 [3142757.506535][ C23] RDX: 0000000000000000 RSI: 0000000000000018 RDI: 0000000000000010 [3142757.506535][ C23] RBP: 00007f264cf01560 R08: 00007f261b8baee8 R09: 400008d0c94b528d [3142757.506536][ C23] R10: 0000000000000018 R11: 000000000000869f R12: 0000000000200000 [3142757.506536][ C23] R13: 00007f30217ff000 R14: 00007f302ca95230 R15: 00007f264cf015d0 [3142757.506765][ C4] CR2: 00007f271a871a58 CR3: 000000518913e001 CR4: 0000000000760ee0 [3142757.509829][ C5] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [3142757.512480][ C25] watchdog: BUG: soft lockup - CPU#25 stuck for 23s! [vertica:283850] [3142757.512481][ C25] Modules linked in: binfmt_misc(E) rpcsec_gss_krb5(E) nfsv4(E) dns_resolver(E) nfs(E) fscache(E) nfsd(E) nfs_acl(E) lockd(E) auth_rpcgss(E) grace(E) nf_conntrack(E) nf_defrag_ipv6(E) nf_defrag_ipv4(E) nls_utf8(E) isofs(E) sunrpc(E) ip_tables(E) ufs(E) hfsplus(E) hfs(E) vfat(E) msdos(E) fat(E) xfs(E) libcrc32c(E) dm_mod(E) ext4(E) mbcache(E) jbd2(E) st(E) sd_mod(E) fuse(E) squashfs(E) zstd_decompress(E) loop(E) brd(E) isst_if_common(E) nfit(E) kvm_intel(E) sr_mod(E) cdrom(E) sg(E) kvm(E) ata_generic(E) irqbypass(E) crct10dif_pclmul(E) crc32_pclmul(E) crc32c_intel(E) ghash_clmulni_intel(E) cirrus(E) drm_kms_helper(E) syscopyarea(E) sysfillrect(E) sysimgblt(E) fb_sys_fops(E) drm(E) ata_piix(E) libata(E) virtio_net(E) joydev(E) serio_raw(E) net_failover(E) virtio_blk(E) virtio_balloon(E) pcspkr(E) i2c_piix4(E) failover(E) qemu_fw_cfg(E) sch_fq_codel(E) [last unloaded: dm_mod] [3142757.512499][ C25] CPU: 25 PID: 283850 Comm: vertica Tainted: G EL 5.4.34-1.el7.gdc.x86_64 #1 [3142757.512500][ C25] Hardware name: RDO OpenStack Compute, BIOS 1.11.0-2.el7 04/01/2014 [3142757.512502][ C25] RIP: 0010:smp_call_function_many+0x21e/0x280 [3142757.512504][ C25] Code: c7 e8 56 25 77 00 3b 05 04 9f 48 01 0f 83 7b fe ff ff 48 63 d0 49 8b 0f 48 03 0c d5 80 99 19 82 8b 51 18 83 e2 01 74 0a f3 90 <8b> 51 18 83 e2 01 75 f6 eb c7 0f b6 4c 24 04 48 83 c4 10 48 89 de [3142757.512504][ C25] RSP: 0000:ffffc90001543c58 EFLAGS: 00000202 ORIG_RAX: ffffffffffffff13 [3142757.512505][ C25] RAX: 0000000000000003 RBX: ffffffff8107c1e0 RCX: ffff88d4e74f1860 [3142757.512506][ C25] RDX: 0000000000000001 RSI: 000000000000001e RDI: ffff88d4db029b18 [3142757.512507][ C25] RBP: ffff88d4e7a6a300 R08: 0000000000000000 R09: 0000000000000000 [3142757.512507][ C25] R10: 0000000000000000 R11: 00000000c1c720c0 R12: 000000000002ba40 [3142757.512508][ C25] R13: 0000000000000001 R14: 000000000000001e R15: ffff88d4e7a6ba80 [3142757.512509][ C25] FS: 00007f2b5c9fd700(0000) GS:ffff88d4e7a40000(0000) knlGS:0000000000000000 [3142757.512510][ C25] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [3142757.512510][ C25] CR2: 00007f30a2c32758 CR3: 000000518913e002 CR4: 0000000000760ee0 [3142757.512512][ C25] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [3142757.512513][ C25] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [3142757.512513][ C25] PKRU: 55555554 [3142757.512514][ C25] Call Trace: [3142757.512516][ C25] ? flush_tlb_func_common.isra.8+0x230/0x230 [3142757.512518][ C25] ? flush_tlb_func_common.isra.8+0x230/0x230 [3142757.512519][ C25] on_each_cpu_mask+0x24/0x60 [3142757.512520][ C25] ? x86_configure_nx+0x50/0x50 [3142757.512521][ C25] on_each_cpu_cond_mask+0xa5/0x140 [3142757.512522][ C25] ? flush_tlb_func_common.isra.8+0x92/0x230 [3142757.512523][ C25] flush_tlb_mm_range+0xab/0xe0 [3142757.512525][ C25] ptep_clear_flush+0x56/0x60 [3142757.512526][ C25] wp_page_copy+0x34a/0x7e0 [3142757.512528][ C25] do_wp_page+0x8b/0x660 [3142757.512530][ C25] __handle_mm_fault+0x777/0xef0 [3142757.512532][ C25] handle_mm_fault+0xe2/0x1f0 [3142757.512533][ C25] __do_page_fault+0x226/0x490 [3142757.512535][ C25] do_page_fault+0x31/0x120 [3142757.512536][ C25] async_page_fault+0x3e/0x50 [3142757.512537][ C25] RIP: 0033:0xe148c5 [3142757.512538][ C25] Code: fa 08 0f 84 44 01 00 00 48 83 ff 07 4c 89 c2 76 33 48 83 ef 08 31 d2 48 89 f9 48 c1 e9 03 48 8d 34 cd 08 00 00 00 4d 8b 0c 10 <4c> 89 0c 10 48 83 c2 08 48 39 f2 75 ef 48 f7 d9 48 01 d0 4c 01 c2 [3142757.512539][ C25] RSP: 002b:00007f264cdffe30 EFLAGS: 00010202 [3142757.512540][ C25] RAX: 00007f30a2c32758 RBX: 000000000000001f RCX: 0000000000000002 [3142757.512540][ C25] RDX: 0000000000000000 RSI: 0000000000000018 RDI: 0000000000000010 [3142757.512541][ C25] RBP: 00007f264cdfffa0 R08: 00007f26450adf10 R09: 40003c9dff53ec68 [3142757.512542][ C25] R10: 0000000000000018 R11: 00000000000073f6 R12: 0000000000200000 [3142757.512542][ C25] R13: 00007f30a0000000 R14: 00007f302ca92140 R15: 00007f264ce00010 [3142757.513843][ C4] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [3142757.515480][ C26] watchdog: BUG: soft lockup - CPU#26 stuck for 23s! [vertica:283854] [3142757.515481][ C26] Modules linked in: binfmt_misc(E) rpcsec_gss_krb5(E) nfsv4(E) dns_resolver(E) nfs(E) fscache(E) nfsd(E) nfs_acl(E) lockd(E) auth_rpcgss(E) grace(E) nf_conntrack(E) nf_defrag_ipv6(E) nf_defrag_ipv4(E) nls_utf8(E) isofs(E) sunrpc(E) ip_tables(E) ufs(E) hfsplus(E) hfs(E) vfat(E) msdos(E) fat(E) xfs(E) libcrc32c(E) dm_mod(E) ext4(E) mbcache(E) jbd2(E) st(E) sd_mod(E) fuse(E) squashfs(E) zstd_decompress(E) loop(E) brd(E) isst_if_common(E) nfit(E) kvm_intel(E) sr_mod(E) cdrom(E) sg(E) kvm(E) ata_generic(E) irqbypass(E) crct10dif_pclmul(E) crc32_pclmul(E) crc32c_intel(E) ghash_clmulni_intel(E) cirrus(E) drm_kms_helper(E) syscopyarea(E) sysfillrect(E) sysimgblt(E) fb_sys_fops(E) drm(E) ata_piix(E) libata(E) virtio_net(E) joydev(E) serio_raw(E) net_failover(E) virtio_blk(E) virtio_balloon(E) pcspkr(E) i2c_piix4(E) failover(E) qemu_fw_cfg(E) sch_fq_codel(E) [last unloaded: dm_mod] [3142757.515497][ C26] CPU: 26 PID: 283854 Comm: vertica Tainted: G EL 5.4.34-1.el7.gdc.x86_64 #1 [3142757.515497][ C26] Hardware name: RDO OpenStack Compute, BIOS 1.11.0-2.el7 04/01/2014 [3142757.515500][ C26] RIP: 0010:smp_call_function_many+0x221/0x280 [3142757.515501][ C26] Code: 25 77 00 3b 05 04 9f 48 01 0f 83 7b fe ff ff 48 63 d0 49 8b 0f 48 03 0c d5 80 99 19 82 8b 51 18 83 e2 01 74 0a f3 90 8b 51 18 <83> e2 01 75 f6 eb c7 0f b6 4c 24 04 48 83 c4 10 48 89 de 5b 48 89 [3142757.515502][ C26] RSP: 0000:ffffc900015a3c58 EFLAGS: 00000202 ORIG_RAX: ffffffffffffff13 [3142757.515503][ C26] RAX: 0000000000000003 RBX: ffffffff8107c1e0 RCX: ffff88d4e74f1880 [3142757.515503][ C26] RDX: 0000000000000003 RSI: 000000000000001e RDI: ffff88d4db029f38 [3142757.515504][ C26] RBP: ffff88d4e7aaa300 R08: 0000000000000000 R09: 0000000000000000 [3142757.515504][ C26] R10: 0000000000000000 R11: 0000000115aafd80 R12: 000000000002ba40 [3142757.515505][ C26] R13: 0000000000000001 R14: 000000000000001e R15: ffff88d4e7aaba80 [3142757.515506][ C26] FS: 00007f2ede7fe700(0000) GS:ffff88d4e7a80000(0000) knlGS:0000000000000000 [3142757.515506][ C26] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [3142757.515507][ C26] CR2: 00007f2f82861ed8 CR3: 000000518913e004 CR4: 0000000000760ee0 [3142757.515509][ C26] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [3142757.515509][ C26] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [3142757.515509][ C26] PKRU: 55555554 [3142757.515510][ C26] Call Trace: [3142757.515512][ C26] ? flush_tlb_func_common.isra.8+0x230/0x230 [3142757.515513][ C26] ? flush_tlb_func_common.isra.8+0x230/0x230 [3142757.515514][ C26] on_each_cpu_mask+0x24/0x60 [3142757.515515][ C26] ? x86_configure_nx+0x50/0x50 [3142757.515516][ C26] on_each_cpu_cond_mask+0xa5/0x140 [3142757.515517][ C26] ? flush_tlb_func_common.isra.8+0x92/0x230 [3142757.515518][ C26] flush_tlb_mm_range+0xab/0xe0 [3142757.515520][ C26] ptep_clear_flush+0x56/0x60 [3142757.515521][ C26] wp_page_copy+0x34a/0x7e0 [3142757.515523][ C26] do_wp_page+0x8b/0x660 [3142757.515524][ C26] __handle_mm_fault+0x777/0xef0 [3142757.515526][ C26] handle_mm_fault+0xe2/0x1f0 [3142757.515528][ C26] __do_page_fault+0x226/0x490 [3142757.515529][ C26] do_page_fault+0x31/0x120 [3142757.515530][ C26] async_page_fault+0x3e/0x50 [3142757.515531][ C26] RIP: 0033:0xe148c5 [3142757.515532][ C26] Code: fa 08 0f 84 44 01 00 00 48 83 ff 07 4c 89 c2 76 33 48 83 ef 08 31 d2 48 89 f9 48 c1 e9 03 48 8d 34 cd 08 00 00 00 4d 8b 0c 10 <4c> 89 0c 10 48 83 c2 08 48 39 f2 75 ef 48 f7 d9 48 01 d0 4c 01 c2 [3142757.515533][ C26] RSP: 002b:00007f312c5f09e0 EFLAGS: 00010202 [3142757.515534][ C26] RAX: 00007f2f82861ed8 RBX: 000000000000001f RCX: 0000000000000002 [3142757.515534][ C26] RDX: 0000000000000000 RSI: 0000000000000018 RDI: 0000000000000010 [3142757.515535][ C26] RBP: 00007f312c5f0b50 R08: 00007f2633066540 R09: 400008d0d1a481a5 [3142757.515535][ C26] R10: 0000000000000018 R11: 0000000000004438 R12: 0000000000200000 [3142757.515536][ C26] R13: 00007f2f827d6000 R14: 00007f302ca9e500 R15: 00007f312c5f0bc0 [3142757.516828][ C5] CR2: 00007f29887620b0 CR3: 000000518913e005 CR4: 0000000000760ee0 [3142757.518116][ C4] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [3142757.521223][ C5] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [3142757.521480][ C28] watchdog: BUG: soft lockup - CPU#28 stuck for 23s! [vertica:283853] [3142757.521481][ C28] Modules linked in: binfmt_misc(E) rpcsec_gss_krb5(E) nfsv4(E) dns_resolver(E) nfs(E) fscache(E) nfsd(E) nfs_acl(E) lockd(E) auth_rpcgss(E) grace(E) nf_conntrack(E) nf_defrag_ipv6(E) nf_defrag_ipv4(E) nls_utf8(E) isofs(E) sunrpc(E) ip_tables(E) ufs(E) hfsplus(E) hfs(E) vfat(E) msdos(E) fat(E) xfs(E) libcrc32c(E) dm_mod(E) ext4(E) mbcache(E) jbd2(E) st(E) sd_mod(E) fuse(E) squashfs(E) zstd_decompress(E) loop(E) brd(E) isst_if_common(E) nfit(E) kvm_intel(E) sr_mod(E) cdrom(E) sg(E) kvm(E) ata_generic(E) irqbypass(E) crct10dif_pclmul(E) crc32_pclmul(E) crc32c_intel(E) ghash_clmulni_intel(E) cirrus(E) drm_kms_helper(E) syscopyarea(E) sysfillrect(E) sysimgblt(E) fb_sys_fops(E) drm(E) ata_piix(E) libata(E) virtio_net(E) joydev(E) serio_raw(E) net_failover(E) virtio_blk(E) virtio_balloon(E) pcspkr(E) i2c_piix4(E) failover(E) qemu_fw_cfg(E) sch_fq_codel(E) [last unloaded: dm_mod] [3142757.521498][ C28] CPU: 28 PID: 283853 Comm: vertica Tainted: G EL 5.4.34-1.el7.gdc.x86_64 #1 [3142757.521499][ C28] Hardware name: RDO OpenStack Compute, BIOS 1.11.0-2.el7 04/01/2014 [3142757.521501][ C28] RIP: 0010:smp_call_function_many+0x21e/0x280 [3142757.521502][ C28] Code: c7 e8 56 25 77 00 3b 05 04 9f 48 01 0f 83 7b fe ff ff 48 63 d0 49 8b 0f 48 03 0c d5 80 99 19 82 8b 51 18 83 e2 01 74 0a f3 90 <8b> 51 18 83 e2 01 75 f6 eb c7 0f b6 4c 24 04 48 83 c4 10 48 89 de [3142757.521503][ C28] RSP: 0000:ffffc90001593c58 EFLAGS: 00000202 ORIG_RAX: ffffffffffffff13 [3142757.521504][ C28] RAX: 0000000000000003 RBX: ffffffff8107c1e0 RCX: ffff88d4e74f18c0 [3142757.521505][ C28] RDX: 0000000000000001 RSI: 000000000000001e RDI: ffff88d4db029fa8 [3142757.521505][ C28] RBP: ffff88d4e7b2a300 R08: 0000000000000000 R09: 0000000000000000 [3142757.521506][ C28] R10: 0000000000000000 R11: 0000000109627440 R12: 000000000002ba40 [3142757.521507][ C28] R13: 0000000000000001 R14: 000000000000001e R15: ffff88d4e7b2ba80 [3142757.521508][ C28] FS: 00007f2eddffd700(0000) GS:ffff88d4e7b00000(0000) knlGS:0000000000000000 [3142757.521509][ C28] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [3142757.521509][ C28] CR2: 00007f2892b59860 CR3: 000000518913e005 CR4: 0000000000760ee0 [3142757.521511][ C28] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [3142757.521512][ C28] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [3142757.521512][ C28] PKRU: 55555554 [3142757.521513][ C28] Call Trace: [3142757.521515][ C28] ? flush_tlb_func_common.isra.8+0x230/0x230 [3142757.521516][ C28] ? flush_tlb_func_common.isra.8+0x230/0x230 [3142757.521517][ C28] on_each_cpu_mask+0x24/0x60 [3142757.521518][ C28] ? x86_configure_nx+0x50/0x50 [3142757.521519][ C28] on_each_cpu_cond_mask+0xa5/0x140 [3142757.521520][ C28] ? flush_tlb_func_common.isra.8+0x92/0x230 [3142757.521521][ C28] flush_tlb_mm_range+0xab/0xe0 [3142757.521523][ C28] ptep_clear_flush+0x56/0x60 [3142757.521525][ C28] wp_page_copy+0x34a/0x7e0 [3142757.521526][ C28] do_wp_page+0x8b/0x660 [3142757.521528][ C28] __handle_mm_fault+0x777/0xef0 [3142757.521530][ C28] handle_mm_fault+0xe2/0x1f0 [3142757.521532][ C28] __do_page_fault+0x226/0x490 [3142757.521533][ C28] do_page_fault+0x31/0x120 [3142757.521534][ C28] async_page_fault+0x3e/0x50 [3142757.521535][ C28] RIP: 0033:0xe148c5 [3142757.521536][ C28] Code: fa 08 0f 84 44 01 00 00 48 83 ff 07 4c 89 c2 76 33 48 83 ef 08 31 d2 48 89 f9 48 c1 e9 03 48 8d 34 cd 08 00 00 00 4d 8b 0c 10 <4c> 89 0c 10 48 83 c2 08 48 39 f2 75 ef 48 f7 d9 48 01 d0 4c 01 c2 [3142757.521537][ C28] RSP: 002b:00007f312c4ef420 EFLAGS: 00010202 [3142757.521538][ C28] RAX: 00007f2892b59860 RBX: 000000000000001f RCX: 0000000000000002 [3142757.521538][ C28] RDX: 0000000000000000 RSI: 0000000000000018 RDI: 0000000000000010 [3142757.521539][ C28] RBP: 00007f312c4ef590 R08: 00007f2dff0501a8 R09: 400008d0ca40bbb7 [3142757.521539][ C28] R10: 0000000000000018 R11: 0000000000003967 R12: 0000000000200000 [3142757.521540][ C28] R13: 00007f28927fb000 R14: 00007f302ca9b410 R15: 00007f312c4ef600 [3142757.524400][ C4] PKRU: 55555554 [3142757.524402][ C4] Call Trace: [3142757.524480][ C29] watchdog: BUG: soft lockup - CPU#29 stuck for 23s! [vertica:283983] [3142757.524481][ C29] Modules linked in: binfmt_misc(E) rpcsec_gss_krb5(E) nfsv4(E) dns_resolver(E) nfs(E) fscache(E) nfsd(E) nfs_acl(E) lockd(E) auth_rpcgss(E) grace(E) nf_conntrack(E) nf_defrag_ipv6(E) nf_defrag_ipv4(E) nls_utf8(E) isofs(E) sunrpc(E) ip_tables(E) ufs(E) hfsplus(E) hfs(E) vfat(E) msdos(E) fat(E) xfs(E) libcrc32c(E) dm_mod(E) ext4(E) mbcache(E) jbd2(E) st(E) sd_mod(E) fuse(E) squashfs(E) zstd_decompress(E) loop(E) brd(E) isst_if_common(E) nfit(E) kvm_intel(E) sr_mod(E) cdrom(E) sg(E) kvm(E) ata_generic(E) irqbypass(E) crct10dif_pclmul(E) crc32_pclmul(E) crc32c_intel(E) ghash_clmulni_intel(E) cirrus(E) drm_kms_helper(E) syscopyarea(E) sysfillrect(E) sysimgblt(E) fb_sys_fops(E) drm(E) ata_piix(E) libata(E) virtio_net(E) joydev(E) serio_raw(E) net_failover(E) virtio_blk(E) virtio_balloon(E) pcspkr(E) i2c_piix4(E) failover(E) qemu_fw_cfg(E) sch_fq_codel(E) [last unloaded: dm_mod] [3142757.524497][ C29] CPU: 29 PID: 283983 Comm: vertica Tainted: G EL 5.4.34-1.el7.gdc.x86_64 #1 [3142757.524497][ C29] Hardware name: RDO OpenStack Compute, BIOS 1.11.0-2.el7 04/01/2014 [3142757.524499][ C29] RIP: 0010:smp_call_function_many+0x21e/0x280 [3142757.524500][ C29] Code: c7 e8 56 25 77 00 3b 05 04 9f 48 01 0f 83 7b fe ff ff 48 63 d0 49 8b 0f 48 03 0c d5 80 99 19 82 8b 51 18 83 e2 01 74 0a f3 90 <8b> 51 18 83 e2 01 75 f6 eb c7 0f b6 4c 24 04 48 83 c4 10 48 89 de [3142757.524501][ C29] RSP: 0000:ffffc90001153c58 EFLAGS: 00000202 ORIG_RAX: ffffffffffffff13 [3142757.524501][ C29] RAX: 0000000000000003 RBX: ffffffff8107c1e0 RCX: ffff88d4e74f18e0 [3142757.524502][ C29] RDX: 0000000000000001 RSI: 000000000000001e RDI: ffff88d4da104d80 [3142757.524503][ C29] RBP: ffff88d4e7b6a300 R08: 0000000000000000 R09: 0000000000000000 [3142757.524503][ C29] R10: 0000000000000000 R11: 000000004f421b80 R12: 000000000002ba40 [3142757.524504][ C29] R13: 0000000000000001 R14: 000000000000001e R15: ffff88d4e7b6ba80 [3142757.524505][ C29] FS: 00007f2d677fe700(0000) GS:ffff88d4e7b40000(0000) knlGS:0000000000000000 [3142757.524505][ C29] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [3142757.524506][ C29] CR2: 00007f3509ec32a0 CR3: 000000518913e005 CR4: 0000000000760ee0 [3142757.524508][ C29] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [3142757.524508][ C29] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [3142757.524508][ C29] PKRU: 55555554 [3142757.524509][ C29] Call Trace: [3142757.524510][ C29] ? flush_tlb_func_common.isra.8+0x230/0x230 [3142757.524511][ C29] ? flush_tlb_func_common.isra.8+0x230/0x230 [3142757.524512][ C29] on_each_cpu_mask+0x24/0x60 [3142757.524513][ C29] ? x86_configure_nx+0x50/0x50 [3142757.524514][ C29] on_each_cpu_cond_mask+0xa5/0x140 [3142757.524515][ C29] ? flush_tlb_func_common.isra.8+0x147/0x230 [3142757.524516][ C29] flush_tlb_mm_range+0xab/0xe0 [3142757.524518][ C29] ptep_clear_flush+0x56/0x60 [3142757.524519][ C29] wp_page_copy+0x34a/0x7e0 [3142757.524521][ C29] do_wp_page+0x8b/0x660 [3142757.524522][ C29] __handle_mm_fault+0x777/0xef0 [3142757.524524][ C29] handle_mm_fault+0xe2/0x1f0 [3142757.524525][ C29] __do_page_fault+0x226/0x490 [3142757.524527][ C29] do_page_fault+0x31/0x120 [3142757.524528][ C29] async_page_fault+0x3e/0x50 [3142757.524529][ C29] RIP: 0033:0x16a563d [3142757.524530][ C29] Code: fa 08 0f 84 65 09 00 00 48 83 fe 07 49 89 f8 76 34 48 83 ee 08 31 c9 49 89 f2 49 c1 ea 03 4e 8d 04 d5 08 00 00 00 4c 8b 34 0f <4c> 89 34 08 48 83 c1 08 4c 39 c1 75 ef 4c 8d 04 0f 49 f7 da 48 01 [3142757.524530][ C29] RSP: 002b:00007f34ea5c8380 EFLAGS: 00010202 [3142757.524531][ C29] RAX: 00007f2f878398e8 RBX: 00007f354b9b3fa0 RCX: 0000000000000000 [3142757.524532][ C29] RDX: 0000000000068e39 RSI: 0000000000000020 RDI: 00007f275ea32890 [3142757.524532][ C29] RBP: 00007f34ea5c8790 R08: 0000000000000028 R09: 00000000000fffff [3142757.524533][ C29] R10: 0000000000000004 R11: 0000000000000000 R12: 00000000000025cd [3142757.524533][ C29] R13: 00007f2f867d6000 R14: 400000003b5baf6a R15: 00007f34ea5f90e0 [3142757.527377][ C5] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [3142757.527378][ C5] PKRU: 55555554 [3142757.528666][ C4] ? flush_tlb_func_common.isra.8+0x230/0x230 [3142757.531627][ C5] Call Trace: [3142757.532865][ C4] ? flush_tlb_func_common.isra.8+0x230/0x230 [3142757.535956][ C5] ? flush_tlb_func_common.isra.8+0x230/0x230 [3142757.539982][ C4] on_each_cpu_mask+0x24/0x60 [3142757.542857][ C5] ? flush_tlb_func_common.isra.8+0x230/0x230 [3142757.544095][ C4] ? x86_configure_nx+0x50/0x50 [3142757.547236][ C5] on_each_cpu_mask+0x24/0x60 [3142757.550250][ C4] on_each_cpu_cond_mask+0xa5/0x140 [3142757.551479][ C5] ? x86_configure_nx+0x50/0x50 [3142757.554622][ C4] ? flush_tlb_func_common.isra.8+0x147/0x230 [3142757.557589][ C5] on_each_cpu_cond_mask+0xa5/0x140 [3142757.558867][ C4] flush_tlb_mm_range+0xab/0xe0 [3142757.561789][ C5] ? flush_tlb_func_common.isra.8+0x147/0x230 [3142757.563013][ C4] ptep_clear_flush+0x56/0x60 [3142757.565933][ C5] flush_tlb_mm_range+0xab/0xe0 [3142757.567219][ C4] wp_page_copy+0x34a/0x7e0 [3142757.570091][ C5] ptep_clear_flush+0x56/0x60 [3142757.571350][ C4] do_wp_page+0x8b/0x660 [3142757.574405][ C5] wp_page_copy+0x34a/0x7e0 [3142757.578402][ C4] __handle_mm_fault+0x777/0xef0 [3142757.581498][ C5] do_wp_page+0x8b/0x660 [3142757.584617][ C4] handle_mm_fault+0xe2/0x1f0 [3142757.588475][ C5] __handle_mm_fault+0x777/0xef0 [3142757.589948][ C4] __do_page_fault+0x226/0x490 [3142757.593031][ C5] handle_mm_fault+0xe2/0x1f0 [3142757.597090][ C4] do_page_fault+0x31/0x120 [3142757.600147][ C5] __do_page_fault+0x226/0x490 [3142757.604142][ C4] async_page_fault+0x3e/0x50 [3142757.607069][ C5] do_page_fault+0x31/0x120 [3142757.608415][ C4] RIP: 0033:0xe148c5 [3142757.611533][ C5] async_page_fault+0x3e/0x50 [3142757.614544][ C4] Code: fa 08 0f 84 44 01 00 00 48 83 ff 07 4c 89 c2 76 33 48 83 ef 08 31 d2 48 89 f9 48 c1 e9 03 48 8d 34 cd 08 00 00 00 4d 8b 0c 10 <4c> 89 0c 10 48 83 c2 08 48 39 f2 75 ef 48 f7 d9 48 01 d0 4c 01 c2 [3142757.615770][ C5] RIP: 0033:0x101d045 [3142757.618836][ C4] RSP: 002b:00007f312c3ede60 EFLAGS: 00010202 [3142757.622738][ C5] Code: 48 31 c1 4c 21 c9 49 89 0c fe 8b 46 24 48 0f af c1 48 03 46 28 48 8b 10 48 85 d2 75 80 48 ba 00 00 00 00 00 00 00 40 49 09 d0 <4c> 89 00 48 8b 56 40 48 3b 56 60 73 0c 4c 8b 46 68 49 89 0c d0 48 [3142757.625666][ C4] RAX: 00007f271a871a58 RBX: 000000000000001f RCX: 0000000000000002 [3142757.626912][ C5] RSP: 002b:00007f2fb103e0f0 EFLAGS: 00010202 [3142757.629853][ C4] RDX: 0000000000000000 RSI: 0000000000000018 RDI: 0000000000000010 [3142757.629855][ C4] RBP: 00007f312c3edfd0 R08: 00007f265b09cb10 R09: 400008d0cf19ded3 [3142757.631088][ C5] RAX: 00007f29887620b0 RBX: 000000000000000f RCX: 0000000000007859 [3142757.634187][ C4] R10: 0000000000000018 R11: 0000000000006876 R12: 0000000000200000 [3142757.634188][ C4] R13: 00007f27188f6000 R14: 00007f302ca98320 R15: 00007f312c3ee040 [3142757.638161][ C5] RDX: 4000000000000000 RSI: 00007f2fb1040750 RDI: 000000000000000f [3142757.638163][ C5] RBP: 00007f2fb103e1c0 R08: 400008d07e06f81f R09: 0000000000007fff [3142759.395918][ C5] R10: 3fffffffffffffff R11: 00007f302c59ff40 R12: 0000000000000001 [3142759.398975][ C5] R13: 00007f2fb1049810 R14: 00007f2fb1089810 R15: 0000000000000034 [3142761.436412][ C0] watchdog: BUG: soft lockup - CPU#0 stuck for 22s! [vertica:283856] [3142761.438412][ C1] watchdog: BUG: soft lockup - CPU#1 stuck for 22s! [vertica:283863] [3142761.439353][ C0] Modules linked in: binfmt_misc(E) rpcsec_gss_krb5(E) nfsv4(E) dns_resolver(E) nfs(E) fscache(E) nfsd(E) nfs_acl(E) lockd(E) auth_rpcgss(E) grace(E) nf_conntrack(E) nf_defrag_ipv6(E) nf_defrag_ipv4(E) nls_utf8(E) isofs(E) sunrpc(E) ip_tables(E) ufs(E) hfsplus(E) hfs(E) vfat(E) msdos(E) fat(E) xfs(E) libcrc32c(E) dm_mod(E) ext4(E) mbcache(E) jbd2(E) st(E) sd_mod(E) fuse(E) squashfs(E) zstd_decompress(E) loop(E) brd(E) isst_if_common(E) nfit(E) kvm_intel(E) sr_mod(E) cdrom(E) sg(E) kvm(E) ata_generic(E) irqbypass(E) crct10dif_pclmul(E) crc32_pclmul(E) crc32c_intel(E) ghash_clmulni_intel(E) cirrus(E) drm_kms_helper(E) syscopyarea(E) sysfillrect(E) sysimgblt(E) fb_sys_fops(E) drm(E) ata_piix(E) libata(E) virtio_net(E) joydev(E) serio_raw(E) net_failover(E) virtio_blk(E) virtio_balloon(E) pcspkr(E) i2c_piix4(E) failover(E) qemu_fw_cfg(E) sch_fq_codel(E) [last unloaded: dm_mod] [3142761.440597][ C1] Modules linked in: binfmt_misc(E) rpcsec_gss_krb5(E) nfsv4(E) dns_resolver(E) nfs(E) fscache(E) nfsd(E) nfs_acl(E) lockd(E) auth_rpcgss(E) grace(E) nf_conntrack(E) nf_defrag_ipv6(E) nf_defrag_ipv4(E) nls_utf8(E) isofs(E) sunrpc(E) ip_tables(E) ufs(E) hfsplus(E) hfs(E) vfat(E) msdos(E) fat(E) xfs(E) libcrc32c(E) dm_mod(E) ext4(E) mbcache(E) jbd2(E) st(E) sd_mod(E) fuse(E) squashfs(E) zstd_decompress(E) loop(E) brd(E) isst_if_common(E) nfit(E) kvm_intel(E) sr_mod(E) cdrom(E) sg(E) kvm(E) ata_generic(E) irqbypass(E) crct10dif_pclmul(E) crc32_pclmul(E) crc32c_intel(E) ghash_clmulni_intel(E) cirrus(E) drm_kms_helper(E) syscopyarea(E) sysfillrect(E) sysimgblt(E) fb_sys_fops(E) drm(E) ata_piix(E) libata(E) virtio_net(E) joydev(E) serio_raw(E) net_failover(E) virtio_blk(E) virtio_balloon(E) pcspkr(E) i2c_piix4(E) failover(E) qemu_fw_cfg(E) sch_fq_codel(E) [last unloaded: dm_mod] [3142761.443681][ C0] CPU: 0 PID: 283856 Comm: vertica Tainted: G EL 5.4.34-1.el7.gdc.x86_64 #1 [3142761.446779][ C1] CPU: 1 PID: 283863 Comm: vertica Tainted: G EL 5.4.34-1.el7.gdc.x86_64 #1 [3142761.450667][ C0] Hardware name: RDO OpenStack Compute, BIOS 1.11.0-2.el7 04/01/2014 [3142761.450673][ C0] RIP: 0010:smp_call_function_many+0x21e/0x280 [3142761.453580][ C1] Hardware name: RDO OpenStack Compute, BIOS 1.11.0-2.el7 04/01/2014 [3142761.454787][ C0] Code: c7 e8 56 25 77 00 3b 05 04 9f 48 01 0f 83 7b fe ff ff 48 63 d0 49 8b 0f 48 03 0c d5 80 99 19 82 8b 51 18 83 e2 01 74 0a f3 90 <8b> 51 18 83 e2 01 75 f6 eb c7 0f b6 4c 24 04 48 83 c4 10 48 89 de [3142761.457812][ C1] RIP: 0010:smp_call_function_many+0x21e/0x280 [3142761.460860][ C0] RSP: 0000:ffffc900015c3c58 EFLAGS: 00000202 ORIG_RAX: ffffffffffffff13 [3142761.464738][ C1] Code: c7 e8 56 25 77 00 3b 05 04 9f 48 01 0f 83 7b fe ff ff 48 63 d0 49 8b 0f 48 03 0c d5 80 99 19 82 8b 51 18 83 e2 01 74 0a f3 90 <8b> 51 18 83 e2 01 75 f6 eb c7 0f b6 4c 24 04 48 83 c4 10 48 89 de [3142761.466169][ C0] RAX: 0000000000000003 RBX: ffffffff8107c1e0 RCX: ffff88d4e74f0280 [3142761.466171][ C0] RDX: 0000000000000001 RSI: 000000000000001e RDI: ffff88d4db029268 [3142761.469106][ C1] RSP: 0000:ffffc90001603c58 EFLAGS: 00000202 ORIG_RAX: ffffffffffffff13 [3142761.470350][ C0] RBP: ffff88d4e742a300 R08: 0000000000000000 R09: 0000000000000000 [3142761.470351][ C0] R10: 0000000000000000 R11: 000000004a700180 R12: 000000000002ba40 [3142761.473421][ C1] RAX: 0000000000000003 RBX: ffffffff8107c1e0 RCX: ffff88d4e74f12c0 [3142761.476564][ C0] R13: 0000000000000001 R14: 000000000000001e R15: ffff88d4e742ba80 [3142761.476566][ C0] FS: 00007f2efe4d2700(0000) GS:ffff88d4e7400000(0000) knlGS:0000000000000000 [3142761.479539][ C1] RDX: 0000000000000001 RSI: 000000000000001e RDI: ffff88d4db029d28 [3142761.479540][ C1] RBP: ffff88d4e746a300 R08: 0000000000000000 R09: 0000000000000000 [3142761.480767][ C0] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [3142761.483712][ C1] R10: 0000000000000000 R11: 00000000e0fcb440 R12: 000000000002ba40 [3142761.483713][ C1] R13: 0000000000000001 R14: 000000000000001e R15: ffff88d4e746ba80 [3142761.484956][ C0] CR2: 00007f270f577e80 CR3: 000000518913e002 CR4: 0000000000760ef0 [3142761.487907][ C1] FS: 00007f2f3cf97700(0000) GS:ffff88d4e7440000(0000) knlGS:0000000000000000 [3142761.489169][ C0] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [3142761.492216][ C1] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [3142761.492218][ C1] CR2: 00007f2e20e28708 CR3: 000000518913e005 CR4: 0000000000760ee0 [3142761.496074][ C0] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [3142761.499019][ C1] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [3142761.500272][ C0] PKRU: 55555554 [3142761.500273][ C0] Call Trace: [3142761.503223][ C1] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [3142761.503224][ C1] PKRU: 55555554 [3142761.504477][ C0] ? flush_tlb_func_common.isra.8+0x230/0x230 [3142761.507357][ C1] Call Trace: [3142761.507361][ C1] ? flush_tlb_func_common.isra.8+0x230/0x230 [3142761.508613][ C0] ? flush_tlb_func_common.isra.8+0x230/0x230 [3142761.509411][ C24] watchdog: BUG: soft lockup - CPU#24 stuck for 22s! [vertica:283849] [3142761.509412][ C24] Modules linked in: binfmt_misc(E) rpcsec_gss_krb5(E) nfsv4(E) dns_resolver(E) nfs(E) fscache(E) nfsd(E) nfs_acl(E) lockd(E) auth_rpcgss(E) grace(E) nf_conntrack(E) nf_defrag_ipv6(E) nf_defrag_ipv4(E) nls_utf8(E) isofs(E) sunrpc(E) ip_tables(E) ufs(E) hfsplus(E) hfs(E) vfat(E) msdos(E) fat(E) xfs(E) libcrc32c(E) dm_mod(E) ext4(E) mbcache(E) jbd2(E) st(E) sd_mod(E) fuse(E) squashfs(E) zstd_decompress(E) loop(E) brd(E) isst_if_common(E) nfit(E) kvm_intel(E) sr_mod(E) cdrom(E) sg(E) kvm(E) ata_generic(E) irqbypass(E) crct10dif_pclmul(E) crc32_pclmul(E) crc32c_intel(E) ghash_clmulni_intel(E) cirrus(E) drm_kms_helper(E) syscopyarea(E) sysfillrect(E) sysimgblt(E) fb_sys_fops(E) drm(E) ata_piix(E) libata(E) virtio_net(E) joydev(E) serio_raw(E) net_failover(E) virtio_blk(E) virtio_balloon(E) pcspkr(E) i2c_piix4(E) failover(E) qemu_fw_cfg(E) sch_fq_codel(E) [last unloaded: dm_mod] [3142761.509435][ C24] CPU: 24 PID: 283849 Comm: vertica Tainted: G EL 5.4.34-1.el7.gdc.x86_64 #1 [3142761.509436][ C24] Hardware name: RDO OpenStack Compute, BIOS 1.11.0-2.el7 04/01/2014 [3142761.509439][ C24] RIP: 0010:smp_call_function_many+0x21e/0x280 [3142761.509441][ C24] Code: c7 e8 56 25 77 00 3b 05 04 9f 48 01 0f 83 7b fe ff ff 48 63 d0 49 8b 0f 48 03 0c d5 80 99 19 82 8b 51 18 83 e2 01 74 0a f3 90 <8b> 51 18 83 e2 01 75 f6 eb c7 0f b6 4c 24 04 48 83 c4 10 48 89 de [3142761.509442][ C24] RSP: 0000:ffffc90001453c58 EFLAGS: 00000202 ORIG_RAX: ffffffffffffff13 [3142761.509443][ C24] RAX: 0000000000000003 RBX: ffffffff8107c1e0 RCX: ffff88d4e74f1840 [3142761.509443][ C24] RDX: 0000000000000001 RSI: 000000000000001e RDI: ffff88d4db029588 [3142761.509444][ C24] RBP: ffff88d4e7a2a300 R08: 0000000000000000 R09: 0000000000000000 [3142761.509444][ C24] R10: 0000000000000000 R11: 0000000125b3b900 R12: 000000000002ba40 [3142761.509445][ C24] R13: 0000000000000001 R14: 000000000000001e R15: ffff88d4e7a2ba80 [3142761.509446][ C24] FS: 00007f2b5b1fc700(0000) GS:ffff88d4e7a00000(0000) knlGS:0000000000000000 [3142761.509447][ C24] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [3142761.509447][ C24] CR2: 00007f31079372e0 CR3: 000000518913e003 CR4: 0000000000760ee0 [3142761.509449][ C24] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [3142761.509450][ C24] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [3142761.509450][ C24] PKRU: 55555554 [3142761.509451][ C24] Call Trace: [3142761.509453][ C24] ? flush_tlb_func_common.isra.8+0x230/0x230 [3142761.509455][ C24] ? flush_tlb_func_common.isra.8+0x230/0x230 [3142761.509455][ C24] on_each_cpu_mask+0x24/0x60 [3142761.509457][ C24] ? x86_configure_nx+0x50/0x50 [3142761.509458][ C24] on_each_cpu_cond_mask+0xa5/0x140 [3142761.509459][ C24] ? flush_tlb_func_common.isra.8+0x92/0x230 [3142761.509460][ C24] flush_tlb_mm_range+0xab/0xe0 [3142761.509463][ C24] ptep_clear_flush+0x56/0x60 [3142761.509466][ C24] wp_page_copy+0x34a/0x7e0 [3142761.509468][ C24] do_wp_page+0x8b/0x660 [3142761.509470][ C24] __handle_mm_fault+0x777/0xef0 [3142761.509472][ C24] handle_mm_fault+0xe2/0x1f0 [3142761.509474][ C24] __do_page_fault+0x226/0x490 [3142761.509476][ C24] do_page_fault+0x31/0x120 [3142761.509479][ C24] async_page_fault+0x3e/0x50 [3142761.509480][ C24] RIP: 0033:0xe148c5 [3142761.509481][ C24] Code: fa 08 0f 84 44 01 00 00 48 83 ff 07 4c 89 c2 76 33 48 83 ef 08 31 d2 48 89 f9 48 c1 e9 03 48 8d 34 cd 08 00 00 00 4d 8b 0c 10 <4c> 89 0c 10 48 83 c2 08 48 39 f2 75 ef 48 f7 d9 48 01 d0 4c 01 c2 [3142761.509482][ C24] RSP: 002b:00007f264ccfe870 EFLAGS: 00010202 [3142761.509483][ C24] RAX: 00007f31079372e0 RBX: 000000000000001f RCX: 0000000000000002 [3142761.509483][ C24] RDX: 0000000000000000 RSI: 0000000000000018 RDI: 0000000000000010 [3142761.509484][ C24] RBP: 00007f264ccfe9e0 R08: 00007f26a00d3e48 R09: 40003e12f19a81c6 [3142761.509485][ C24] R10: 0000000000000018 R11: 0000000000008d43 R12: 0000000000200000 [3142761.509485][ C24] R13: 00007f3106000000 R14: 00007f302ca8f050 R15: 00007f264ccfea50 [3142761.509963][ C1] ? flush_tlb_func_common.isra.8+0x230/0x230 [3142761.513040][ C0] on_each_cpu_mask+0x24/0x60 [3142761.515932][ C1] on_each_cpu_mask+0x24/0x60 [3142761.517186][ C0] ? x86_configure_nx+0x50/0x50 [3142761.520262][ C1] ? x86_configure_nx+0x50/0x50 [3142761.523138][ C0] on_each_cpu_cond_mask+0xa5/0x140 [3142761.524336][ C1] on_each_cpu_cond_mask+0xa5/0x140 [3142761.527229][ C0] ? flush_tlb_func_common.isra.8+0x92/0x230 [3142761.528435][ C1] ? flush_tlb_func_common.isra.8+0x92/0x230 [3142761.531558][ C0] flush_tlb_mm_range+0xab/0xe0 [3142761.534375][ C1] flush_tlb_mm_range+0xab/0xe0 [3142761.535616][ C0] ptep_clear_flush+0x56/0x60 [3142761.538440][ C1] ptep_clear_flush+0x56/0x60 [3142761.539675][ C0] wp_page_copy+0x34a/0x7e0 [3142761.542734][ C1] wp_page_copy+0x34a/0x7e0 [3142761.545841][ C0] do_wp_page+0x8b/0x660 [3142761.548922][ C1] do_wp_page+0x8b/0x660 [3142761.552922][ C0] __handle_mm_fault+0x777/0xef0 [3142761.555952][ C1] __handle_mm_fault+0x777/0xef0 [3142761.559942][ C0] handle_mm_fault+0xe2/0x1f0 [3142761.563022][ C1] handle_mm_fault+0xe2/0x1f0 [3142761.566194][ C0] __do_page_fault+0x226/0x490 [3142761.569083][ C1] __do_page_fault+0x226/0x490 [3142761.570373][ C0] do_page_fault+0x31/0x120 [3142761.571812][ C1] do_page_fault+0x31/0x120 [3142761.574913][ C0] async_page_fault+0x3e/0x50 [3142761.578774][ C1] async_page_fault+0x3e/0x50 [3142761.581769][ C0] RIP: 0033:0xe148c5 [3142761.583032][ C1] RIP: 0033:0xe148c5 [3142761.586248][ C0] Code: fa 08 0f 84 44 01 00 00 48 83 ff 07 4c 89 c2 76 33 48 83 ef 08 31 d2 48 89 f9 48 c1 e9 03 48 8d 34 cd 08 00 00 00 4d 8b 0c 10 <4c> 89 0c 10 48 83 c2 08 48 39 f2 75 ef 48 f7 d9 48 01 d0 4c 01 c2 [3142761.589393][ C1] Code: fa 08 0f 84 44 01 00 00 48 83 ff 07 4c 89 c2 76 33 48 83 ef 08 31 d2 48 89 f9 48 c1 e9 03 48 8d 34 cd 08 00 00 00 4d 8b 0c 10 <4c> 89 0c 10 48 83 c2 08 48 39 f2 75 ef 48 f7 d9 48 01 d0 4c 01 c2 [3142761.592365][ C0] RSP: 002b:00007f312c6f1fa0 EFLAGS: 00010202 [3142761.593634][ C1] RSP: 002b:00007f312caf76a0 EFLAGS: 00010202 [3142761.596544][ C0] RAX: 00007f270f577e80 RBX: 000000000000001f RCX: 0000000000000002 [3142761.596545][ C0] RDX: 0000000000000000 RSI: 0000000000000018 RDI: 0000000000000010 [3142761.597826][ C1] RAX: 00007f2e20e28708 RBX: 000000000000001f RCX: 0000000000000002 [3142761.597827][ C1] RDX: 0000000000000000 RSI: 0000000000000018 RDI: 0000000000000010 [3142761.600979][ C0] RBP: 00007f312c6f2110 R08: 00007f2eca486bc0 R09: 400008d0cdbdeacd [3142761.600980][ C0] R10: 0000000000000018 R11: 0000000000005d28 R12: 0000000000200000 [3142761.604214][ C1] RBP: 00007f312caf7810 R08: 00007f25ee527120 R09: 400008d0c8ccbcb9 [3142761.608161][ C0] R13: 00007f270c8f6000 R14: 00007f302caa15f0 R15: 00007f312c6f2180 [3142761.749050][ C1] R10: 0000000000000018 R11: 0000000000003c0c R12: 0000000000200000 [3142761.750520][ C1] R13: 00007f2e20000000 R14: 00007f302caad9b0 R15: 00007f312caf7880 [3142781.478059][ C14] watchdog: BUG: soft lockup - CPU#14 stuck for 23s! [vertica:191361] [3142781.481127][ C14] Modules linked in: binfmt_misc(E) rpcsec_gss_krb5(E) nfsv4(E) dns_resolver(E) nfs(E) fscache(E) nfsd(E) nfs_acl(E) lockd(E) auth_rpcgss(E) grace(E) nf_conntrack(E) nf_defrag_ipv6(E) nf_defrag_ipv4(E) nls_utf8(E) isofs(E) sunrpc(E) ip_tables(E) ufs(E) hfsplus(E) hfs(E) vfat(E) msdos(E) fat(E) xfs(E) libcrc32c(E) dm_mod(E) ext4(E) mbcache(E) jbd2(E) st(E) sd_mod(E) fuse(E) squashfs(E) zstd_decompress(E) loop(E) brd(E) isst_if_common(E) nfit(E) kvm_intel(E) sr_mod(E) cdrom(E) sg(E) kvm(E) ata_generic(E) irqbypass(E) crct10dif_pclmul(E) crc32_pclmul(E) crc32c_intel(E) ghash_clmulni_intel(E) cirrus(E) drm_kms_helper(E) syscopyarea(E) sysfillrect(E) sysimgblt(E) fb_sys_fops(E) drm(E) ata_piix(E) libata(E) virtio_net(E) joydev(E) serio_raw(E) net_failover(E) virtio_blk(E) virtio_balloon(E) pcspkr(E) i2c_piix4(E) failover(E) qemu_fw_cfg(E) sch_fq_codel(E) [last unloaded: dm_mod] [3142781.482404][ C14] CPU: 14 PID: 191361 Comm: vertica Tainted: G EL 5.4.34-1.el7.gdc.x86_64 #1 [3142781.485557][ C14] Hardware name: RDO OpenStack Compute, BIOS 1.11.0-2.el7 04/01/2014 [3142781.486809][ C14] RIP: 0010:smp_call_function_many+0x21e/0x280 [3142781.489934][ C14] Code: c7 e8 56 25 77 00 3b 05 04 9f 48 01 0f 83 7b fe ff ff 48 63 d0 49 8b 0f 48 03 0c d5 80 99 19 82 8b 51 18 83 e2 01 74 0a f3 90 <8b> 51 18 83 e2 01 75 f6 eb c7 0f b6 4c 24 04 48 83 c4 10 48 89 de [3142781.491175][ C14] RSP: 0018:ffffc9000267fc68 EFLAGS: 00000202 ORIG_RAX: ffffffffffffff13 [3142781.493058][ C19] watchdog: BUG: soft lockup - CPU#19 stuck for 22s! [vertica:283865] [3142781.494119][ C14] RAX: 0000000000000003 RBX: ffffffff8107c1e0 RCX: ffff88d4e74f1700 [3142781.494120][ C14] RDX: 0000000000000001 RSI: 000000000000001e RDI: ffff88d4db029890 [3142781.495461][ C19] Modules linked in: binfmt_misc(E) rpcsec_gss_krb5(E) nfsv4(E) dns_resolver(E) nfs(E) fscache(E) nfsd(E) nfs_acl(E) lockd(E) auth_rpcgss(E) grace(E) nf_conntrack(E) nf_defrag_ipv6(E) nf_defrag_ipv4(E) nls_utf8(E) isofs(E) sunrpc(E) ip_tables(E) ufs(E) hfsplus(E) hfs(E) vfat(E) msdos(E) fat(E) xfs(E) libcrc32c(E) dm_mod(E) ext4(E) mbcache(E) jbd2(E) st(E) sd_mod(E) fuse(E) squashfs(E) zstd_decompress(E) loop(E) brd(E) isst_if_common(E) nfit(E) kvm_intel(E) sr_mod(E) cdrom(E) sg(E) kvm(E) ata_generic(E) irqbypass(E) crct10dif_pclmul(E) crc32_pclmul(E) crc32c_intel(E) ghash_clmulni_intel(E) cirrus(E) drm_kms_helper(E) syscopyarea(E) sysfillrect(E) sysimgblt(E) fb_sys_fops(E) drm(E) ata_piix(E) libata(E) virtio_net(E) joydev(E) serio_raw(E) net_failover(E) virtio_blk(E) virtio_balloon(E) pcspkr(E) i2c_piix4(E) failover(E) qemu_fw_cfg(E) sch_fq_codel(E) [last unloaded: dm_mod] [3142781.498425][ C14] RBP: ffff88d4e77aa300 R08: 0000000000000000 R09: 0000000000000000 [3142781.498427][ C14] R10: 0000000000000000 R11: 0000000000000000 R12: 000000000002ba40 [3142781.499724][ C19] CPU: 19 PID: 283865 Comm: vertica Tainted: G EL 5.4.34-1.el7.gdc.x86_64 #1 [3142781.502900][ C14] R13: 0000000000000001 R14: 000000000000001e R15: ffff88d4e77aba80 [3142781.502901][ C14] FS: 00007f3277de9700(0000) GS:ffff88d4e7780000(0000) knlGS:0000000000000000 [3142781.506830][ C19] Hardware name: RDO OpenStack Compute, BIOS 1.11.0-2.el7 04/01/2014 [3142781.506833][ C19] RIP: 0010:smp_call_function_many+0x21e/0x280 [3142781.509939][ C14] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [3142781.509940][ C14] CR2: 00007fa6c81f4ffc CR3: 000000518913e002 CR4: 0000000000760ee0 [3142781.513874][ C19] Code: c7 e8 56 25 77 00 3b 05 04 9f 48 01 0f 83 7b fe ff ff 48 63 d0 49 8b 0f 48 03 0c d5 80 99 19 82 8b 51 18 83 e2 01 74 0a f3 90 <8b> 51 18 83 e2 01 75 f6 eb c7 0f b6 4c 24 04 48 83 c4 10 48 89 de [3142781.517028][ C14] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [3142781.520405][ C19] RSP: 0000:ffffc90001623c58 EFLAGS: 00000202 ORIG_RAX: ffffffffffffff13 [3142781.524702][ C14] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [3142781.524704][ C14] PKRU: 55555554 [3142781.526227][ C19] RAX: 000000000000001a RBX: ffffffff8107c1e0 RCX: ffff88d4e7ab17a0 [3142781.529434][ C14] Call Trace: [3142781.529439][ C14] ? flush_tlb_func_common.isra.8+0x230/0x230 [3142781.532813][ C19] RDX: 0000000000000001 RSI: 000000000000001e RDI: ffff88d4db029b70 [3142781.532814][ C19] RBP: ffff88d4e78ea300 R08: 0000000000000000 R09: 0000000000000000 [3142781.534077][ C14] ? flush_tlb_func_common.isra.8+0x230/0x230 [3142781.537408][ C19] R10: 0000000000000000 R11: 00000000f32d36c0 R12: 000000000002ba40 [3142781.537410][ C19] R13: 0000000000000001 R14: 000000000000001e R15: ffff88d4e78eba80 [3142781.538653][ C14] on_each_cpu_mask+0x24/0x60 [3142781.541646][ C19] FS: 00007f2f57cbc700(0000) GS:ffff88d4e78c0000(0000) knlGS:0000000000000000 [3142781.542880][ C14] ? x86_configure_nx+0x50/0x50 [3142781.545968][ C19] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [3142781.545969][ C19] CR2: 00007f31511f02d0 CR3: 000000518913e004 CR4: 0000000000760ee0 [3142781.549099][ C14] on_each_cpu_cond_mask+0xa5/0x140 [3142781.552247][ C19] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [3142781.555418][ C14] ? flush_tlb_func_common.isra.8+0x147/0x230 [3142781.558582][ C19] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [3142781.558583][ C19] PKRU: 55555554 [3142781.562568][ C14] flush_tlb_mm_range+0xab/0xe0 [3142781.565686][ C19] Call Trace: [3142781.565690][ C19] ? flush_tlb_func_common.isra.8+0x230/0x230 [3142781.567009][ C14] tlb_flush_mmu+0x89/0x150 [3142781.568472][ C19] ? flush_tlb_func_common.isra.8+0x230/0x230 [3142781.571447][ C14] tlb_finish_mmu+0x3d/0x70 [3142781.572701][ C19] on_each_cpu_mask+0x24/0x60 [3142781.575840][ C14] zap_page_range+0x115/0x160 [3142781.577140][ C19] ? x86_configure_nx+0x50/0x50 [3142781.580283][ C14] ? find_vma+0x59/0x60 [3142781.583551][ C19] on_each_cpu_cond_mask+0xa5/0x140 [3142781.583553][ C19] ? flush_tlb_func_common.isra.8+0x147/0x230 [3142781.586698][ C14] __do_sys_madvise+0x738/0xc00 [3142781.590671][ C19] flush_tlb_mm_range+0xab/0xe0 [3142781.593847][ C14] ? finish_task_switch+0x201/0x2a0 [3142781.597009][ C19] ptep_clear_flush+0x56/0x60 [3142781.600015][ C14] do_syscall_64+0x5b/0x1b0 [3142781.601319][ C19] wp_page_copy+0x34a/0x7e0 [3142781.604322][ C14] entry_SYSCALL_64_after_hwframe+0x44/0xa9 [3142781.605587][ C19] do_wp_page+0x8b/0x660 [3142781.608565][ C14] RIP: 0033:0x7f35e1571e97 [3142781.609855][ C19] __handle_mm_fault+0x777/0xef0 [3142781.611310][ C14] Code: 83 c4 08 48 3d 01 f0 ff ff 73 01 c3 48 8b 0d d0 df 2c 00 f7 d8 64 89 01 48 83 c8 ff c3 66 0f 1f 44 00 00 b8 1c 00 00 00 0f 05 <48> 3d 01 f0 ff ff 73 01 c3 48 8b 0d a9 df 2c 00 f7 d8 64 89 01 48 [3142781.614339][ C19] handle_mm_fault+0xe2/0x1f0 [3142781.615609][ C14] RSP: 002b:00007f3277dd6a68 EFLAGS: 00000206 ORIG_RAX: 000000000000001c [3142781.618843][ C19] __do_page_fault+0x226/0x490 [3142781.621955][ C14] RAX: ffffffffffffffda RBX: 00000000079defc8 RCX: 00007f35e1571e97 [3142781.621957][ C14] RDX: 0000000000000004 RSI: 0000000000200000 RDI: 00007f35b097f000 [3142781.626021][ C19] do_page_fault+0x31/0x120 [3142781.629037][ C14] RBP: 00007f3277dd6b80 R08: 0000000000000000 R09: 0000000000006f68 [3142781.629039][ C14] R10: 00007f3124344060 R11: 0000000000000206 R12: 0000000000200000 [3142781.630296][ C19] async_page_fault+0x3e/0x50 [3142781.633389][ C14] R13: 00007f35b097f000 R14: 0000000000200000 R15: 00007f35b097f000 [3142781.678242][ C19] RIP: 0033:0xe148c5 [3142781.682181][ C19] Code: fa 08 0f 84 44 01 00 00 48 83 ff 07 4c 89 c2 76 33 48 83 ef 08 31 d2 48 89 f9 48 c1 e9 03 48 8d 34 cd 08 00 00 00 4d 8b 0c 10 <4c> 89 0c 10 48 83 c2 08 48 39 f2 75 ef 48 f7 d9 48 01 d0 4c 01 c2 [3142781.685237][ C19] RSP: 002b:00007f312ccfa220 EFLAGS: 00010202 [3142781.686520][ C19] RAX: 00007f31511f02d0 RBX: 000000000000001f RCX: 0000000000000002 [3142781.689702][ C19] RDX: 0000000000000000 RSI: 0000000000000018 RDI: 0000000000000010 [3142781.692979][ C19] RBP: 00007f312ccfa390 R08: 00007f26a8055350 R09: 4000537d981cc377 [3142781.694466][ C19] R10: 0000000000000018 R11: 00000000000038ce R12: 0000000000200000 [3142781.697472][ C19] R13: 00007f3150ffe000 R14: 00007f302cab3b90 R15: 00007f312ccfa400 [3142785.446990][ C4] watchdog: BUG: soft lockup - CPU#4 stuck for 23s! [vertica:283852] [3142785.449990][ C5] watchdog: BUG: soft lockup - CPU#5 stuck for 23s! [vertica:284163] [3142785.450063][ C4] Modules linked in: binfmt_misc(E) rpcsec_gss_krb5(E) nfsv4(E) dns_resolver(E) nfs(E) fscache(E) nfsd(E) nfs_acl(E) lockd(E) auth_rpcgss(E) grace(E) nf_conntrack(E) nf_defrag_ipv6(E) nf_defrag_ipv4(E) nls_utf8(E) isofs(E) sunrpc(E) ip_tables(E) ufs(E) hfsplus(E) hfs(E) vfat(E) msdos(E) fat(E) xfs(E) libcrc32c(E) dm_mod(E) ext4(E) mbcache(E) jbd2(E) st(E) sd_mod(E) fuse(E) squashfs(E) zstd_decompress(E) loop(E) brd(E) isst_if_common(E) nfit(E) kvm_intel(E) sr_mod(E) cdrom(E) sg(E) kvm(E) ata_generic(E) irqbypass(E) crct10dif_pclmul(E) crc32_pclmul(E) crc32c_intel(E) ghash_clmulni_intel(E) cirrus(E) drm_kms_helper(E) syscopyarea(E) sysfillrect(E) sysimgblt(E) fb_sys_fops(E) drm(E) ata_piix(E) libata(E) virtio_net(E) joydev(E) serio_raw(E) net_failover(E) virtio_blk(E) virtio_balloon(E) pcspkr(E) i2c_piix4(E) failover(E) qemu_fw_cfg(E) sch_fq_codel(E) [last unloaded: dm_mod] [3142785.451309][ C5] Modules linked in: binfmt_misc(E) rpcsec_gss_krb5(E) nfsv4(E) dns_resolver(E) nfs(E) fscache(E) nfsd(E) nfs_acl(E) lockd(E) auth_rpcgss(E) grace(E) nf_conntrack(E) nf_defrag_ipv6(E) nf_defrag_ipv4(E) nls_utf8(E) isofs(E) sunrpc(E) ip_tables(E) ufs(E) hfsplus(E) hfs(E) vfat(E) msdos(E) fat(E) xfs(E) libcrc32c(E) dm_mod(E) ext4(E) mbcache(E) jbd2(E) st(E) sd_mod(E) fuse(E) squashfs(E) zstd_decompress(E) loop(E) brd(E) isst_if_common(E) nfit(E) kvm_intel(E) sr_mod(E) cdrom(E) sg(E) kvm(E) ata_generic(E) irqbypass(E) crct10dif_pclmul(E) crc32_pclmul(E) crc32c_intel(E) ghash_clmulni_intel(E) cirrus(E) drm_kms_helper(E) syscopyarea(E) sysfillrect(E) sysimgblt(E) fb_sys_fops(E) drm(E) ata_piix(E) libata(E) virtio_net(E) joydev(E) serio_raw(E) net_failover(E) virtio_blk(E) virtio_balloon(E) pcspkr(E) i2c_piix4(E) failover(E) qemu_fw_cfg(E) sch_fq_codel(E) [last unloaded: dm_mod] [3142785.454464][ C4] CPU: 4 PID: 283852 Comm: vertica Tainted: G EL 5.4.34-1.el7.gdc.x86_64 #1 [3142785.455990][ C7] watchdog: BUG: soft lockup - CPU#7 stuck for 23s! [sshd:284646] [3142785.455991][ C7] Modules linked in: binfmt_misc(E) rpcsec_gss_krb5(E) nfsv4(E) dns_resolver(E) nfs(E) fscache(E) nfsd(E) nfs_acl(E) lockd(E) auth_rpcgss(E) grace(E) nf_conntrack(E) nf_defrag_ipv6(E) nf_defrag_ipv4(E) nls_utf8(E) isofs(E) sunrpc(E) ip_tables(E) ufs(E) hfsplus(E) hfs(E) vfat(E) msdos(E) fat(E) xfs(E) libcrc32c(E) dm_mod(E) ext4(E) mbcache(E) jbd2(E) st(E) sd_mod(E) fuse(E) squashfs(E) zstd_decompress(E) loop(E) brd(E) isst_if_common(E) nfit(E) kvm_intel(E) sr_mod(E) cdrom(E) sg(E) kvm(E) ata_generic(E) irqbypass(E) crct10dif_pclmul(E) crc32_pclmul(E) crc32c_intel(E) ghash_clmulni_intel(E) cirrus(E) drm_kms_helper(E) syscopyarea(E) sysfillrect(E) sysimgblt(E) fb_sys_fops(E) drm(E) ata_piix(E) libata(E) virtio_net(E) joydev(E) serio_raw(E) net_failover(E) virtio_blk(E) virtio_balloon(E) pcspkr(E) i2c_piix4(E) failover(E) qemu_fw_cfg(E) sch_fq_codel(E) [last unloaded: dm_mod] [3142785.456013][ C7] CPU: 7 PID: 284646 Comm: sshd Tainted: G EL 5.4.34-1.el7.gdc.x86_64 #1 [3142785.456014][ C7] Hardware name: RDO OpenStack Compute, BIOS 1.11.0-2.el7 04/01/2014 [3142785.456018][ C7] RIP: 0010:smp_call_function_many+0x21e/0x280 [3142785.456020][ C7] Code: c7 e8 56 25 77 00 3b 05 04 9f 48 01 0f 83 7b fe ff ff 48 63 d0 49 8b 0f 48 03 0c d5 80 99 19 82 8b 51 18 83 e2 01 74 0a f3 90 <8b> 51 18 83 e2 01 75 f6 eb c7 0f b6 4c 24 04 48 83 c4 10 48 89 de [3142785.456020][ C7] RSP: 0018:ffffc90000c53ba0 EFLAGS: 00000202 ORIG_RAX: ffffffffffffff13 [3142785.456021][ C7] RAX: 0000000000000003 RBX: ffffffff8107ba40 RCX: ffff88d4e74f1620 [3142785.456022][ C7] RDX: 0000000000000001 RSI: 000000000000001e RDI: ffff88d4db029710 [3142785.456022][ C7] RBP: 0000000000000000 R08: 0000000000000000 R09: 0000000000000000 [3142785.456023][ C7] R10: 0000000000000000 R11: 0000000000000000 R12: 000000000002ba40 [3142785.456023][ C7] R13: 0000000000000001 R14: 000000000000001e R15: ffff88d4e75eba80 [3142785.456024][ C7] FS: 00007f7f683548c0(0000) GS:ffff88d4e75c0000(0000) knlGS:0000000000000000 [3142785.456025][ C7] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [3142785.456025][ C7] CR2: 00007ffc4d461f44 CR3: 0000001c3a278004 CR4: 0000000000760ee0 [3142785.456028][ C7] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [3142785.456028][ C7] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [3142785.456029][ C7] PKRU: 55555554 [3142785.456029][ C7] Call Trace: [3142785.456034][ C7] ? do_kernel_range_flush+0x50/0x50 [3142785.456035][ C7] on_each_cpu+0x28/0x50 [3142785.456038][ C7] __purge_vmap_area_lazy+0x86/0x680 [3142785.456040][ C7] ? purge_fragmented_blocks_allcpus+0x42/0x1f0 [3142785.456041][ C7] _vm_unmap_aliases+0xf2/0x130 [3142785.456043][ C7] change_page_attr_set_clr+0xb6/0x1e0 [3142785.456045][ C7] set_memory_ro+0x2d/0x40 [3142785.456047][ C7] bpf_int_jit_compile+0x2ee/0x3e0 [3142785.456051][ C7] bpf_prog_select_runtime+0xdf/0x140 [3142785.456054][ C7] bpf_prepare_filter+0x597/0x610 [3142785.456056][ C7] bpf_prog_create_from_user+0xd5/0x120 [3142785.456058][ C7] ? hardlockup_detector_perf_cleanup+0xa0/0xa0 [3142785.456059][ C7] do_seccomp+0x294/0x8c0 [3142785.456062][ C7] __x64_sys_prctl+0x280/0x58d [3142785.456065][ C7] do_syscall_64+0x5b/0x1b0 [3142785.456068][ C7] entry_SYSCALL_64_after_hwframe+0x44/0xa9 [3142785.456069][ C7] RIP: 0033:0x7f7f655490da [3142785.456070][ C7] Code: 48 8b 0d 99 7d 2c 00 f7 d8 64 89 01 48 83 c8 ff c3 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 49 89 ca b8 9d 00 00 00 0f 05 <48> 3d 01 f0 ff ff 73 01 c3 48 8b 0d 66 7d 2c 00 f7 d8 64 89 01 48 [3142785.456071][ C7] RSP: 002b:00007ffc4d45fc68 EFLAGS: 00000246 ORIG_RAX: 000000000000009d [3142785.456072][ C7] RAX: ffffffffffffffda RBX: 000055fc76435a10 RCX: 00007f7f655490da [3142785.456072][ C7] RDX: 000055fc75198900 RSI: 0000000000000002 RDI: 0000000000000016 [3142785.456073][ C7] RBP: 000055fc76435570 R08: 0000000000000000 R09: 00007f7f6549727d [3142785.456073][ C7] R10: 00007f7f655490da R11: 0000000000000246 R12: 0000000000000000 [3142785.456074][ C7] R13: 000055fc76433660 R14: 000055fc751a4068 R15: 0000000000000018 [3142785.457591][ C5] CPU: 5 PID: 284163 Comm: vertica Tainted: G EL 5.4.34-1.el7.gdc.x86_64 #1 [3142785.458988][ C8] watchdog: BUG: soft lockup - CPU#8 stuck for 23s! [vertica:283847] [3142785.458989][ C8] Modules linked in: binfmt_misc(E) rpcsec_gss_krb5(E) nfsv4(E) dns_resolver(E) nfs(E) fscache(E) nfsd(E) nfs_acl(E) lockd(E) auth_rpcgss(E) grace(E) nf_conntrack(E) nf_defrag_ipv6(E) nf_defrag_ipv4(E) nls_utf8(E) isofs(E) sunrpc(E) ip_tables(E) ufs(E) hfsplus(E) hfs(E) vfat(E) msdos(E) fat(E) xfs(E) libcrc32c(E) dm_mod(E) ext4(E) mbcache(E) jbd2(E) st(E) sd_mod(E) fuse(E) squashfs(E) zstd_decompress(E) loop(E) brd(E) isst_if_common(E) nfit(E) kvm_intel(E) sr_mod(E) cdrom(E) sg(E) kvm(E) ata_generic(E) irqbypass(E) crct10dif_pclmul(E) crc32_pclmul(E) crc32c_intel(E) ghash_clmulni_intel(E) cirrus(E) drm_kms_helper(E) syscopyarea(E) sysfillrect(E) sysimgblt(E) fb_sys_fops(E) drm(E) ata_piix(E) libata(E) virtio_net(E) joydev(E) serio_raw(E) net_failover(E) virtio_blk(E) virtio_balloon(E) pcspkr(E) i2c_piix4(E) failover(E) qemu_fw_cfg(E) sch_fq_codel(E) [last unloaded: dm_mod] [3142785.459008][ C8] CPU: 8 PID: 283847 Comm: vertica Tainted: G EL 5.4.34-1.el7.gdc.x86_64 #1 [3142785.459008][ C8] Hardware name: RDO OpenStack Compute, BIOS 1.11.0-2.el7 04/01/2014 [3142785.459012][ C8] RIP: 0010:smp_call_function_many+0x21e/0x280 [3142785.459013][ C8] Code: c7 e8 56 25 77 00 3b 05 04 9f 48 01 0f 83 7b fe ff ff 48 63 d0 49 8b 0f 48 03 0c d5 80 99 19 82 8b 51 18 83 e2 01 74 0a f3 90 <8b> 51 18 83 e2 01 75 f6 eb c7 0f b6 4c 24 04 48 83 c4 10 48 89 de [3142785.459014][ C8] RSP: 0000:ffffc90001413c58 EFLAGS: 00000202 ORIG_RAX: ffffffffffffff13 [3142785.459015][ C8] RAX: 0000000000000003 RBX: ffffffff8107c1e0 RCX: ffff88d4e74f1640 [3142785.459015][ C8] RDX: 0000000000000001 RSI: 000000000000001e RDI: ffff88d4db029940 [3142785.459016][ C8] RBP: ffff88d4e762a300 R08: 0000000000000000 R09: 0000000000000000 [3142785.459016][ C8] R10: 0000000000000000 R11: 0000000080fe6cc0 R12: 000000000002ba40 [3142785.459017][ C8] R13: 0000000000000001 R14: 000000000000001e R15: ffff88d4e762ba80 [3142785.459018][ C8] FS: 00007f2b567fc700(0000) GS:ffff88d4e7600000(0000) knlGS:0000000000000000 [3142785.459018][ C8] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [3142785.459019][ C8] CR2: 00007f315f1f9ca0 CR3: 000000518913e006 CR4: 0000000000760ee0 [3142785.459021][ C8] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [3142785.459022][ C8] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [3142785.459022][ C8] PKRU: 55555554 [3142785.459022][ C8] Call Trace: [3142785.459026][ C8] ? flush_tlb_func_common.isra.8+0x230/0x230 [3142785.459027][ C8] ? flush_tlb_func_common.isra.8+0x230/0x230 [3142785.459028][ C8] on_each_cpu_mask+0x24/0x60 [3142785.459029][ C8] ? x86_configure_nx+0x50/0x50 [3142785.459030][ C8] on_each_cpu_cond_mask+0xa5/0x140 [3142785.459031][ C8] ? flush_tlb_func_common.isra.8+0x147/0x230 [3142785.459032][ C8] flush_tlb_mm_range+0xab/0xe0 [3142785.459035][ C8] ptep_clear_flush+0x56/0x60 [3142785.459037][ C8] wp_page_copy+0x34a/0x7e0 [3142785.459039][ C8] do_wp_page+0x8b/0x660 [3142785.459041][ C8] __handle_mm_fault+0x777/0xef0 [3142785.459042][ C8] handle_mm_fault+0xe2/0x1f0 [3142785.459045][ C8] __do_page_fault+0x226/0x490 [3142785.459047][ C8] do_page_fault+0x31/0x120 [3142785.459048][ C8] async_page_fault+0x3e/0x50 [3142785.459049][ C8] RIP: 0033:0xe148c5 [3142785.459050][ C8] Code: fa 08 0f 84 44 01 00 00 48 83 ff 07 4c 89 c2 76 33 48 83 ef 08 31 d2 48 89 f9 48 c1 e9 03 48 8d 34 cd 08 00 00 00 4d 8b 0c 10 <4c> 89 0c 10 48 83 c2 08 48 39 f2 75 ef 48 f7 d9 48 01 d0 4c 01 c2 [3142785.459051][ C8] RSP: 002b:00007f302cd2ad30 EFLAGS: 00010202 [3142785.459052][ C8] RAX: 00007f315f1f9ca0 RBX: 000000000000001f RCX: 0000000000000002 [3142785.459052][ C8] RDX: 0000000000000000 RSI: 0000000000000018 RDI: 0000000000000010 [3142785.459053][ C8] RBP: 00007f302cd2aea0 R08: 00007f25fa54b258 R09: 400008d0d1a2dc18 [3142785.459053][ C8] R10: 0000000000000018 R11: 0000000000005419 R12: 0000000000200000 [3142785.459054][ C8] R13: 00007f315e06f000 R14: 00007f2b5de19950 R15: 00007f302cd2af10 [3142785.460538][ C4] Hardware name: RDO OpenStack Compute, BIOS 1.11.0-2.el7 04/01/2014 [3142785.461841][ C5] Hardware name: RDO OpenStack Compute, BIOS 1.11.0-2.el7 04/01/2014 [3142785.461988][ C9] watchdog: BUG: soft lockup - CPU#9 stuck for 23s! [vertica:283858] [3142785.461988][ C9] Modules linked in: binfmt_misc(E) rpcsec_gss_krb5(E) nfsv4(E) dns_resolver(E) nfs(E) fscache(E) nfsd(E) nfs_acl(E) lockd(E) auth_rpcgss(E) grace(E) nf_conntrack(E) nf_defrag_ipv6(E) nf_defrag_ipv4(E) nls_utf8(E) isofs(E) sunrpc(E) ip_tables(E) ufs(E) hfsplus(E) hfs(E) vfat(E) msdos(E) fat(E) xfs(E) libcrc32c(E) dm_mod(E) ext4(E) mbcache(E) jbd2(E) st(E) sd_mod(E) fuse(E) squashfs(E) zstd_decompress(E) loop(E) brd(E) isst_if_common(E) nfit(E) kvm_intel(E) sr_mod(E) cdrom(E) sg(E) kvm(E) ata_generic(E) irqbypass(E) crct10dif_pclmul(E) crc32_pclmul(E) crc32c_intel(E) ghash_clmulni_intel(E) cirrus(E) drm_kms_helper(E) syscopyarea(E) sysfillrect(E) sysimgblt(E) fb_sys_fops(E) drm(E) ata_piix(E) libata(E) virtio_net(E) joydev(E) serio_raw(E) net_failover(E) virtio_blk(E) virtio_balloon(E) pcspkr(E) i2c_piix4(E) failover(E) qemu_fw_cfg(E) sch_fq_codel(E) [last unloaded: dm_mod] [3142785.462004][ C9] CPU: 9 PID: 283858 Comm: vertica Tainted: G EL 5.4.34-1.el7.gdc.x86_64 #1 [3142785.462004][ C9] Hardware name: RDO OpenStack Compute, BIOS 1.11.0-2.el7 04/01/2014 [3142785.462006][ C9] RIP: 0010:smp_call_function_many+0x21e/0x280 [3142785.462007][ C9] Code: c7 e8 56 25 77 00 3b 05 04 9f 48 01 0f 83 7b fe ff ff 48 63 d0 49 8b 0f 48 03 0c d5 80 99 19 82 8b 51 18 83 e2 01 74 0a f3 90 <8b> 51 18 83 e2 01 75 f6 eb c7 0f b6 4c 24 04 48 83 c4 10 48 89 de [3142785.462007][ C9] RSP: 0000:ffffc900015cbc58 EFLAGS: 00000202 ORIG_RAX: ffffffffffffff13 [3142785.462008][ C9] RAX: 0000000000000003 RBX: ffffffff8107c1e0 RCX: ffff88d4e74f1660 [3142785.462009][ C9] RDX: 0000000000000001 RSI: 000000000000001e RDI: ffff88d4db029a68 [3142785.462009][ C9] RBP: ffff88d4e766a300 R08: 0000000000000000 R09: 0000000000000000 [3142785.462010][ C9] R10: 0000000000000000 R11: 00000001096b3e80 R12: 000000000002ba40 [3142785.462010][ C9] R13: 0000000000000001 R14: 000000000000001e R15: ffff88d4e766ba80 [3142785.462011][ C9] FS: 00007f2f080de700(0000) GS:ffff88d4e7640000(0000) knlGS:0000000000000000 [3142785.462012][ C9] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [3142785.462012][ C9] CR2: 00007f3025c6d000 CR3: 000000518913e004 CR4: 0000000000760ee0 [3142785.462014][ C9] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [3142785.462014][ C9] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [3142785.462015][ C9] PKRU: 55555554 [3142785.462015][ C9] Call Trace: [3142785.462016][ C9] ? flush_tlb_func_common.isra.8+0x230/0x230 [3142785.462017][ C9] ? flush_tlb_func_common.isra.8+0x230/0x230 [3142785.462018][ C9] on_each_cpu_mask+0x24/0x60 [3142785.462019][ C9] ? x86_configure_nx+0x50/0x50 [3142785.462020][ C9] on_each_cpu_cond_mask+0xa5/0x140 [3142785.462021][ C9] ? flush_tlb_func_common.isra.8+0x92/0x230 [3142785.462023][ C9] flush_tlb_mm_range+0xab/0xe0 [3142785.462024][ C9] ptep_clear_flush+0x56/0x60 [3142785.462025][ C9] wp_page_copy+0x34a/0x7e0 [3142785.462027][ C9] do_wp_page+0x8b/0x660 [3142785.462028][ C9] __handle_mm_fault+0x777/0xef0 [3142785.462030][ C9] handle_mm_fault+0xe2/0x1f0 [3142785.462031][ C9] __do_page_fault+0x226/0x490 [3142785.462033][ C9] do_page_fault+0x31/0x120 [3142785.462033][ C9] async_page_fault+0x3e/0x50 [3142785.462034][ C9] RIP: 0033:0xe148c5 [3142785.462035][ C9] Code: fa 08 0f 84 44 01 00 00 48 83 ff 07 4c 89 c2 76 33 48 83 ef 08 31 d2 48 89 f9 48 c1 e9 03 48 8d 34 cd 08 00 00 00 4d 8b 0c 10 <4c> 89 0c 10 48 83 c2 08 48 39 f2 75 ef 48 f7 d9 48 01 d0 4c 01 c2 [3142785.462035][ C9] RSP: 002b:00007f312c7f3560 EFLAGS: 00010202 [3142785.462036][ C9] RAX: 00007f3025c6d000 RBX: 000000000000001f RCX: 0000000000000002 [3142785.462037][ C9] RDX: 0000000000000000 RSI: 0000000000000018 RDI: 0000000000000010 [3142785.462037][ C9] RBP: 00007f312c7f36d0 R08: 00007f257a0670e0 R09: 400008d0ce0b746f [3142785.462038][ C9] R10: 0000000000000018 R11: 00000000000044b4 R12: 0000000000200000 [3142785.462038][ C9] R13: 00007f30257ff000 R14: 00007f302caa46e0 R15: 00007f312c7f3740 [3142785.464958][ C4] RIP: 0010:smp_call_function_many+0x221/0x280 [3142785.468076][ C5] RIP: 0010:smp_call_function_many+0x21e/0x280 [3142785.468988][ C11] watchdog: BUG: soft lockup - CPU#11 stuck for 23s! [vertica:284316] [3142785.468989][ C11] Modules linked in: binfmt_misc(E) rpcsec_gss_krb5(E) nfsv4(E) dns_resolver(E) nfs(E) fscache(E) nfsd(E) nfs_acl(E) lockd(E) auth_rpcgss(E) grace(E) nf_conntrack(E) nf_defrag_ipv6(E) nf_defrag_ipv4(E) nls_utf8(E) isofs(E) sunrpc(E) ip_tables(E) ufs(E) hfsplus(E) hfs(E) vfat(E) msdos(E) fat(E) xfs(E) libcrc32c(E) dm_mod(E) ext4(E) mbcache(E) jbd2(E) st(E) sd_mod(E) fuse(E) squashfs(E) zstd_decompress(E) loop(E) brd(E) isst_if_common(E) nfit(E) kvm_intel(E) sr_mod(E) cdrom(E) sg(E) kvm(E) ata_generic(E) irqbypass(E) crct10dif_pclmul(E) crc32_pclmul(E) crc32c_intel(E) ghash_clmulni_intel(E) cirrus(E) drm_kms_helper(E) syscopyarea(E) sysfillrect(E) sysimgblt(E) fb_sys_fops(E) drm(E) ata_piix(E) libata(E) virtio_net(E) joydev(E) serio_raw(E) net_failover(E) virtio_blk(E) virtio_balloon(E) pcspkr(E) i2c_piix4(E) failover(E) qemu_fw_cfg(E) sch_fq_codel(E) [last unloaded: dm_mod] [3142785.469010][ C11] CPU: 11 PID: 284316 Comm: vertica Tainted: G EL 5.4.34-1.el7.gdc.x86_64 #1 [3142785.469010][ C11] Hardware name: RDO OpenStack Compute, BIOS 1.11.0-2.el7 04/01/2014 [3142785.469012][ C11] RIP: 0010:smp_call_function_many+0x21e/0x280 [3142785.469014][ C11] Code: c7 e8 56 25 77 00 3b 05 04 9f 48 01 0f 83 7b fe ff ff 48 63 d0 49 8b 0f 48 03 0c d5 80 99 19 82 8b 51 18 83 e2 01 74 0a f3 90 <8b> 51 18 83 e2 01 75 f6 eb c7 0f b6 4c 24 04 48 83 c4 10 48 89 de [3142785.469015][ C11] RSP: 0018:ffffc9000187fce0 EFLAGS: 00000202 ORIG_RAX: ffffffffffffff13 [3142785.469016][ C11] RAX: 000000000000001a RBX: ffffffff8107c1e0 RCX: ffff88d4e7ab16a0 [3142785.469016][ C11] RDX: 0000000000000001 RSI: 000000000000001e RDI: ffff88d4db0291f0 [3142785.469017][ C11] RBP: ffff88d4e76ea300 R08: 0000000000000000 R09: 0000000000000000 [3142785.469017][ C11] R10: 0000000000000000 R11: 0000000000000000 R12: 000000000002ba40 [3142785.469018][ C11] R13: 0000000000000001 R14: 000000000000001e R15: ffff88d4e76eba80 [3142785.469019][ C11] FS: 00007f301effe700(0000) GS:ffff88d4e76c0000(0000) knlGS:0000000000000000 [3142785.469020][ C11] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [3142785.469020][ C11] CR2: 00007fa6b445f000 CR3: 000000518913e006 CR4: 0000000000760ee0 [3142785.469022][ C11] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [3142785.469023][ C11] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [3142785.469023][ C11] PKRU: 55555554 [3142785.469024][ C11] Call Trace: [3142785.469027][ C11] flush_tlb_mm_range+0xab/0xe0 [3142785.469028][ C11] tlb_flush_mmu+0x89/0x150 [3142785.469030][ C11] tlb_finish_mmu+0x3d/0x70 [3142785.469032][ C11] zap_page_range+0x115/0x160 [3142785.469033][ C11] ? find_vma+0x59/0x60 [3142785.469035][ C11] __do_sys_madvise+0x738/0xc00 [3142785.469038][ C11] ? finish_task_switch+0x75/0x2a0 [3142785.469041][ C11] ? exit_to_usermode_loop+0xc3/0xe7 [3142785.469042][ C11] do_syscall_64+0x5b/0x1b0 [3142785.469044][ C11] entry_SYSCALL_64_after_hwframe+0x44/0xa9 [3142785.469046][ C11] RIP: 0033:0x7f35e1571e97 [3142785.469047][ C11] Code: 83 c4 08 48 3d 01 f0 ff ff 73 01 c3 48 8b 0d d0 df 2c 00 f7 d8 64 89 01 48 83 c8 ff c3 66 0f 1f 44 00 00 b8 1c 00 00 00 0f 05 <48> 3d 01 f0 ff ff 73 01 c3 48 8b 0d a9 df 2c 00 f7 d8 64 89 01 48 [3142785.469047][ C11] RSP: 002b:00007f301effb3d8 EFLAGS: 00000206 ORIG_RAX: 000000000000001c [3142785.469048][ C11] RAX: ffffffffffffffda RBX: 00007f301effe700 RCX: 00007f35e1571e97 [3142785.469049][ C11] RDX: 0000000000000004 RSI: 00000000007f8000 RDI: 00007f301e7ff000 [3142785.469050][ C11] RBP: 0000000000000000 R08: 0000000000000001 R09: 00007f2ac8228d50 [3142785.469050][ C11] R10: 0000000000000001 R11: 0000000000000206 R12: 0000000000000000 [3142785.469051][ C11] R13: 00007f34c37f5620 R14: 00007f34c37f56e0 R15: 00007f301effe700 [3142785.471152][ C4] Code: 25 77 00 3b 05 04 9f 48 01 0f 83 7b fe ff ff 48 63 d0 49 8b 0f 48 03 0c d5 80 99 19 82 8b 51 18 83 e2 01 74 0a f3 90 8b 51 18 <83> e2 01 75 f6 eb c7 0f b6 4c 24 04 48 83 c4 10 48 89 de 5b 48 89 [3142785.471988][ C12] watchdog: BUG: soft lockup - CPU#12 stuck for 23s! [vertica:283862] [3142785.471989][ C12] Modules linked in: binfmt_misc(E) rpcsec_gss_krb5(E) nfsv4(E) dns_resolver(E) nfs(E) fscache(E) nfsd(E) nfs_acl(E) lockd(E) auth_rpcgss(E) grace(E) nf_conntrack(E) nf_defrag_ipv6(E) nf_defrag_ipv4(E) nls_utf8(E) isofs(E) sunrpc(E) ip_tables(E) ufs(E) hfsplus(E) hfs(E) vfat(E) msdos(E) fat(E) xfs(E) libcrc32c(E) dm_mod(E) ext4(E) mbcache(E) jbd2(E) st(E) sd_mod(E) fuse(E) squashfs(E) zstd_decompress(E) loop(E) brd(E) isst_if_common(E) nfit(E) kvm_intel(E) sr_mod(E) cdrom(E) sg(E) kvm(E) ata_generic(E) irqbypass(E) crct10dif_pclmul(E) crc32_pclmul(E) crc32c_intel(E) ghash_clmulni_intel(E) cirrus(E) drm_kms_helper(E) syscopyarea(E) sysfillrect(E) sysimgblt(E) fb_sys_fops(E) drm(E) ata_piix(E) libata(E) virtio_net(E) joydev(E) serio_raw(E) net_failover(E) virtio_blk(E) virtio_balloon(E) pcspkr(E) i2c_piix4(E) failover(E) qemu_fw_cfg(E) sch_fq_codel(E) [last unloaded: dm_mod] [3142785.472007][ C12] CPU: 12 PID: 283862 Comm: vertica Tainted: G EL 5.4.34-1.el7.gdc.x86_64 #1 [3142785.472007][ C12] Hardware name: RDO OpenStack Compute, BIOS 1.11.0-2.el7 04/01/2014 [3142785.472009][ C12] RIP: 0010:smp_call_function_many+0x21e/0x280 [3142785.472011][ C12] Code: c7 e8 56 25 77 00 3b 05 04 9f 48 01 0f 83 7b fe ff ff 48 63 d0 49 8b 0f 48 03 0c d5 80 99 19 82 8b 51 18 83 e2 01 74 0a f3 90 <8b> 51 18 83 e2 01 75 f6 eb c7 0f b6 4c 24 04 48 83 c4 10 48 89 de [3142785.472011][ C12] RSP: 0000:ffffc900015fbc58 EFLAGS: 00000202 ORIG_RAX: ffffffffffffff13 [3142785.472012][ C12] RAX: 0000000000000003 RBX: ffffffff8107c1e0 RCX: ffff88d4e74f16c0 [3142785.472013][ C12] RDX: 0000000000000001 RSI: 000000000000001e RDI: ffff88d4db029c28 [3142785.472014][ C12] RBP: ffff88d4e772a300 R08: 0000000000000000 R09: 0000000000000000 [3142785.472014][ C12] R10: 0000000000000000 R11: 00000000b5130b40 R12: 000000000002ba40 [3142785.472015][ C12] R13: 0000000000000001 R14: 000000000000001e R15: ffff88d4e772ba80 [3142785.472016][ C12] FS: 00007f2f3af94700(0000) GS:ffff88d4e7700000(0000) knlGS:0000000000000000 [3142785.472016][ C12] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [3142785.472017][ C12] CR2: 00007f237a5073b8 CR3: 000000518913e004 CR4: 0000000000760ee0 [3142785.472019][ C12] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [3142785.472019][ C12] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [3142785.472020][ C12] PKRU: 55555554 [3142785.472020][ C12] Call Trace: [3142785.472023][ C12] ? flush_tlb_func_common.isra.8+0x230/0x230 [3142785.472024][ C12] ? flush_tlb_func_common.isra.8+0x230/0x230 [3142785.472025][ C12] on_each_cpu_mask+0x24/0x60 [3142785.472026][ C12] ? x86_configure_nx+0x50/0x50 [3142785.472027][ C12] on_each_cpu_cond_mask+0xa5/0x140 [3142785.472028][ C12] ? flush_tlb_func_common.isra.8+0x92/0x230 [3142785.472030][ C12] flush_tlb_mm_range+0xab/0xe0 [3142785.472031][ C12] ptep_clear_flush+0x56/0x60 [3142785.472033][ C12] wp_page_copy+0x34a/0x7e0 [3142785.472035][ C12] do_wp_page+0x8b/0x660 [3142785.472036][ C12] __handle_mm_fault+0x777/0xef0 [3142785.472038][ C12] handle_mm_fault+0xe2/0x1f0 [3142785.472040][ C12] __do_page_fault+0x226/0x490 [3142785.472042][ C12] do_page_fault+0x31/0x120 [3142785.472043][ C12] async_page_fault+0x3e/0x50 [3142785.472044][ C12] RIP: 0033:0xe148c5 [3142785.472045][ C12] Code: fa 08 0f 84 44 01 00 00 48 83 ff 07 4c 89 c2 76 33 48 83 ef 08 31 d2 48 89 f9 48 c1 e9 03 48 8d 34 cd 08 00 00 00 4d 8b 0c 10 <4c> 89 0c 10 48 83 c2 08 48 39 f2 75 ef 48 f7 d9 48 01 d0 4c 01 c2 [3142785.472046][ C12] RSP: 002b:00007f312c9f60e0 EFLAGS: 00010202 [3142785.472047][ C12] RAX: 00007f237a5073b8 RBX: 000000000000001f RCX: 0000000000000002 [3142785.472047][ C12] RDX: 0000000000000000 RSI: 0000000000000018 RDI: 0000000000000010 [3142785.472048][ C12] RBP: 00007f312c9f6250 R08: 00007f25f853e568 R09: 400008d0cf532ee6 [3142785.472048][ C12] R10: 0000000000000018 R11: 0000000000004b8f R12: 0000000000200000 [3142785.472049][ C12] R13: 00007f2378000000 R14: 00007f302caaa8c0 R15: 00007f312c9f62c0 [3142785.474277][ C5] Code: c7 e8 56 25 77 00 3b 05 04 9f 48 01 0f 83 7b fe ff ff 48 63 d0 49 8b 0f 48 03 0c d5 80 99 19 82 8b 51 18 83 e2 01 74 0a f3 90 <8b> 51 18 83 e2 01 75 f6 eb c7 0f b6 4c 24 04 48 83 c4 10 48 89 de [3142785.474988][ C13] watchdog: BUG: soft lockup - CPU#13 stuck for 23s! [vertica:283848] [3142785.474988][ C13] Modules linked in: binfmt_misc(E) rpcsec_gss_krb5(E) nfsv4(E) dns_resolver(E) nfs(E) fscache(E) nfsd(E) nfs_acl(E) lockd(E) auth_rpcgss(E) grace(E) nf_conntrack(E) nf_defrag_ipv6(E) nf_defrag_ipv4(E) nls_utf8(E) isofs(E) sunrpc(E) ip_tables(E) ufs(E) hfsplus(E) hfs(E) vfat(E) msdos(E) fat(E) xfs(E) libcrc32c(E) dm_mod(E) ext4(E) mbcache(E) jbd2(E) st(E) sd_mod(E) fuse(E) squashfs(E) zstd_decompress(E) loop(E) brd(E) isst_if_common(E) nfit(E) kvm_intel(E) sr_mod(E) cdrom(E) sg(E) kvm(E) ata_generic(E) irqbypass(E) crct10dif_pclmul(E) crc32_pclmul(E) crc32c_intel(E) ghash_clmulni_intel(E) cirrus(E) drm_kms_helper(E) syscopyarea(E) sysfillrect(E) sysimgblt(E) fb_sys_fops(E) drm(E) ata_piix(E) libata(E) virtio_net(E) joydev(E) serio_raw(E) net_failover(E) virtio_blk(E) virtio_balloon(E) pcspkr(E) i2c_piix4(E) failover(E) qemu_fw_cfg(E) sch_fq_codel(E) [last unloaded: dm_mod] [3142785.475004][ C13] CPU: 13 PID: 283848 Comm: vertica Tainted: G EL 5.4.34-1.el7.gdc.x86_64 #1 [3142785.475005][ C13] Hardware name: RDO OpenStack Compute, BIOS 1.11.0-2.el7 04/01/2014 [3142785.475006][ C13] RIP: 0010:smp_call_function_many+0x21e/0x280 [3142785.475008][ C13] Code: c7 e8 56 25 77 00 3b 05 04 9f 48 01 0f 83 7b fe ff ff 48 63 d0 49 8b 0f 48 03 0c d5 80 99 19 82 8b 51 18 83 e2 01 74 0a f3 90 <8b> 51 18 83 e2 01 75 f6 eb c7 0f b6 4c 24 04 48 83 c4 10 48 89 de [3142785.475008][ C13] RSP: 0000:ffffc90001423c58 EFLAGS: 00000202 ORIG_RAX: ffffffffffffff13 [3142785.475009][ C13] RAX: 0000000000000003 RBX: ffffffff8107c1e0 RCX: ffff88d4e74f16e0 [3142785.475010][ C13] RDX: 0000000000000001 RSI: 000000000000001e RDI: ffff88d4db0298e8 [3142785.475010][ C13] RBP: ffff88d4e776a300 R08: 0000000000000000 R09: 0000000000000000 [3142785.475011][ C13] R10: 0000000000000000 R11: 00000000ad2f2600 R12: 000000000002ba40 [3142785.475011][ C13] R13: 0000000000000001 R14: 000000000000001e R15: ffff88d4e776ba80 [3142785.475012][ C13] FS: 00007f2b5a5fb700(0000) GS:ffff88d4e7740000(0000) knlGS:0000000000000000 [3142785.475013][ C13] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [3142785.475013][ C13] CR2: 00007f3202e22760 CR3: 000000518913e003 CR4: 0000000000760ee0 [3142785.475015][ C13] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [3142785.475016][ C13] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [3142785.475016][ C13] PKRU: 55555554 [3142785.475017][ C13] Call Trace: [3142785.475018][ C13] ? flush_tlb_func_common.isra.8+0x230/0x230 [3142785.475019][ C13] ? flush_tlb_func_common.isra.8+0x230/0x230 [3142785.475020][ C13] on_each_cpu_mask+0x24/0x60 [3142785.475021][ C13] ? x86_configure_nx+0x50/0x50 [3142785.475022][ C13] on_each_cpu_cond_mask+0xa5/0x140 [3142785.475023][ C13] ? flush_tlb_func_common.isra.8+0x92/0x230 [3142785.475024][ C13] flush_tlb_mm_range+0xab/0xe0 [3142785.475025][ C13] ptep_clear_flush+0x56/0x60 [3142785.475027][ C13] wp_page_copy+0x34a/0x7e0 [3142785.475028][ C13] do_wp_page+0x8b/0x660 [3142785.475030][ C13] __handle_mm_fault+0x777/0xef0 [3142785.475032][ C13] handle_mm_fault+0xe2/0x1f0 [3142785.475033][ C13] __do_page_fault+0x226/0x490 [3142785.475034][ C13] do_page_fault+0x31/0x120 [3142785.475035][ C13] async_page_fault+0x3e/0x50 [3142785.475036][ C13] RIP: 0033:0xe148c5 [3142785.475037][ C13] Code: fa 08 0f 84 44 01 00 00 48 83 ff 07 4c 89 c2 76 33 48 83 ef 08 31 d2 48 89 f9 48 c1 e9 03 48 8d 34 cd 08 00 00 00 4d 8b 0c 10 <4c> 89 0c 10 48 83 c2 08 48 39 f2 75 ef 48 f7 d9 48 01 d0 4c 01 c2 [3142785.475038][ C13] RSP: 002b:00007f264cbfd2b0 EFLAGS: 00010202 [3142785.475038][ C13] RAX: 00007f3202e22760 RBX: 000000000000001f RCX: 0000000000000002 [3142785.475039][ C13] RDX: 0000000000000000 RSI: 0000000000000018 RDI: 0000000000000010 [3142785.475040][ C13] RBP: 00007f264cbfd420 R08: 00007f275c17c280 R09: 400008d0d18e3052 [3142785.475040][ C13] R10: 0000000000000018 R11: 0000000000005570 R12: 0000000000200000 [3142785.475041][ C13] R13: 00007f3200000000 R14: 00007f302ca8bf60 R15: 00007f264cbfd490 [3142785.477399][ C4] RSP: 0000:ffffc9000158bc58 EFLAGS: 00000202 ORIG_RAX: ffffffffffffff13 [3142785.478856][ C5] RSP: 0000:ffffc90002ebfc58 EFLAGS: 00000202 ORIG_RAX: ffffffffffffff13 [3142785.480988][ C15] watchdog: BUG: soft lockup - CPU#15 stuck for 23s! [vertica:283864] [3142785.480989][ C15] Modules linked in: binfmt_misc(E) rpcsec_gss_krb5(E) nfsv4(E) dns_resolver(E) nfs(E) fscache(E) nfsd(E) nfs_acl(E) lockd(E) auth_rpcgss(E) grace(E) nf_conntrack(E) nf_defrag_ipv6(E) nf_defrag_ipv4(E) nls_utf8(E) isofs(E) sunrpc(E) ip_tables(E) ufs(E) hfsplus(E) hfs(E) vfat(E) msdos(E) fat(E) xfs(E) libcrc32c(E) dm_mod(E) ext4(E) mbcache(E) jbd2(E) st(E) sd_mod(E) fuse(E) squashfs(E) zstd_decompress(E) loop(E) brd(E) isst_if_common(E) nfit(E) kvm_intel(E) sr_mod(E) cdrom(E) sg(E) kvm(E) ata_generic(E) irqbypass(E) crct10dif_pclmul(E) crc32_pclmul(E) crc32c_intel(E) ghash_clmulni_intel(E) cirrus(E) drm_kms_helper(E) syscopyarea(E) sysfillrect(E) sysimgblt(E) fb_sys_fops(E) drm(E) ata_piix(E) libata(E) virtio_net(E) joydev(E) serio_raw(E) net_failover(E) virtio_blk(E) virtio_balloon(E) pcspkr(E) i2c_piix4(E) failover(E) qemu_fw_cfg(E) sch_fq_codel(E) [last unloaded: dm_mod] [3142785.481006][ C15] CPU: 15 PID: 283864 Comm: vertica Tainted: G EL 5.4.34-1.el7.gdc.x86_64 #1 [3142785.481007][ C15] Hardware name: RDO OpenStack Compute, BIOS 1.11.0-2.el7 04/01/2014 [3142785.481009][ C15] RIP: 0010:smp_call_function_many+0x21e/0x280 [3142785.481010][ C15] Code: c7 e8 56 25 77 00 3b 05 04 9f 48 01 0f 83 7b fe ff ff 48 63 d0 49 8b 0f 48 03 0c d5 80 99 19 82 8b 51 18 83 e2 01 74 0a f3 90 <8b> 51 18 83 e2 01 75 f6 eb c7 0f b6 4c 24 04 48 83 c4 10 48 89 de [3142785.481011][ C15] RSP: 0000:ffffc9000161bc58 EFLAGS: 00000202 ORIG_RAX: ffffffffffffff13 [3142785.481012][ C15] RAX: 0000000000000003 RBX: ffffffff8107c1e0 RCX: ffff88d4e74f1720 [3142785.481012][ C15] RDX: 0000000000000001 RSI: 000000000000001e RDI: ffff88d4db029f28 [3142785.481013][ C15] RBP: ffff88d4e77ea300 R08: 0000000000000000 R09: 0000000000000000 [3142785.481014][ C15] R10: 0000000000000000 R11: 000000001e5bdb80 R12: 000000000002ba40 [3142785.481014][ C15] R13: 0000000000000001 R14: 000000000000001e R15: ffff88d4e77eba80 [3142785.481015][ C15] FS: 00007f2f4aff6700(0000) GS:ffff88d4e77c0000(0000) knlGS:0000000000000000 [3142785.481016][ C15] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [3142785.481016][ C15] CR2: 00007f308ed0ec18 CR3: 000000518913e003 CR4: 0000000000760ee0 [3142785.481018][ C15] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [3142785.481019][ C15] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [3142785.481019][ C15] PKRU: 55555554 [3142785.481020][ C15] Call Trace: [3142785.481022][ C15] ? flush_tlb_func_common.isra.8+0x230/0x230 [3142785.481023][ C15] ? flush_tlb_func_common.isra.8+0x230/0x230 [3142785.481024][ C15] on_each_cpu_mask+0x24/0x60 [3142785.481025][ C15] ? x86_configure_nx+0x50/0x50 [3142785.481026][ C15] on_each_cpu_cond_mask+0xa5/0x140 [3142785.481027][ C15] ? flush_tlb_func_common.isra.8+0x92/0x230 [3142785.481029][ C15] flush_tlb_mm_range+0xab/0xe0 [3142785.481030][ C15] ptep_clear_flush+0x56/0x60 [3142785.481032][ C15] wp_page_copy+0x34a/0x7e0 [3142785.481033][ C15] do_wp_page+0x8b/0x660 [3142785.481035][ C15] __handle_mm_fault+0x777/0xef0 [3142785.481037][ C15] handle_mm_fault+0xe2/0x1f0 [3142785.481038][ C15] __do_page_fault+0x226/0x490 [3142785.481040][ C15] do_page_fault+0x31/0x120 [3142785.481041][ C15] async_page_fault+0x3e/0x50 [3142785.481042][ C15] RIP: 0033:0xe148c5 [3142785.481043][ C15] Code: fa 08 0f 84 44 01 00 00 48 83 ff 07 4c 89 c2 76 33 48 83 ef 08 31 d2 48 89 f9 48 c1 e9 03 48 8d 34 cd 08 00 00 00 4d 8b 0c 10 <4c> 89 0c 10 48 83 c2 08 48 39 f2 75 ef 48 f7 d9 48 01 d0 4c 01 c2 [3142785.481044][ C15] RSP: 002b:00007f312cbf8c60 EFLAGS: 00010202 [3142785.481044][ C15] RAX: 00007f308ed0ec18 RBX: 000000000000001f RCX: 0000000000000002 [3142785.481045][ C15] RDX: 0000000000000000 RSI: 0000000000000018 RDI: 0000000000000010 [3142785.481046][ C15] RBP: 00007f312cbf8dd0 R08: 00007f26a2073860 R09: 400008d0c927fae4 [3142785.481046][ C15] R10: 0000000000000018 R11: 0000000000004d04 R12: 0000000000200000 [3142785.481047][ C15] R13: 00007f308d5f5000 R14: 00007f302cab0aa0 R15: 00007f312cbf8e40 [3142785.481839][ C4] RAX: 0000000000000003 RBX: ffffffff8107c1e0 RCX: ffff88d4e74f15c0 [3142785.483062][ C5] RAX: 000000000000001a RBX: ffffffff8107c1e0 RCX: ffff88d4e7ab15e0 [3142785.483989][ C16] watchdog: BUG: soft lockup - CPU#16 stuck for 23s! [vertica:284270] [3142785.483989][ C16] Modules linked in: binfmt_misc(E) rpcsec_gss_krb5(E) nfsv4(E) dns_resolver(E) nfs(E) fscache(E) nfsd(E) nfs_acl(E) lockd(E) auth_rpcgss(E) grace(E) nf_conntrack(E) nf_defrag_ipv6(E) nf_defrag_ipv4(E) nls_utf8(E) isofs(E) sunrpc(E) ip_tables(E) ufs(E) hfsplus(E) hfs(E) vfat(E) msdos(E) fat(E) xfs(E) libcrc32c(E) dm_mod(E) ext4(E) mbcache(E) jbd2(E) st(E) sd_mod(E) fuse(E) squashfs(E) zstd_decompress(E) loop(E) brd(E) isst_if_common(E) nfit(E) kvm_intel(E) sr_mod(E) cdrom(E) sg(E) kvm(E) ata_generic(E) irqbypass(E) crct10dif_pclmul(E) crc32_pclmul(E) crc32c_intel(E) ghash_clmulni_intel(E) cirrus(E) drm_kms_helper(E) syscopyarea(E) sysfillrect(E) sysimgblt(E) fb_sys_fops(E) drm(E) ata_piix(E) libata(E) virtio_net(E) joydev(E) serio_raw(E) net_failover(E) virtio_blk(E) virtio_balloon(E) pcspkr(E) i2c_piix4(E) failover(E) qemu_fw_cfg(E) sch_fq_codel(E) [last unloaded: dm_mod] [3142785.484007][ C16] CPU: 16 PID: 284270 Comm: vertica Tainted: G EL 5.4.34-1.el7.gdc.x86_64 #1 [3142785.484008][ C16] Hardware name: RDO OpenStack Compute, BIOS 1.11.0-2.el7 04/01/2014 [3142785.484010][ C16] RIP: 0010:smp_call_function_many+0x21e/0x280 [3142785.484012][ C16] Code: c7 e8 56 25 77 00 3b 05 04 9f 48 01 0f 83 7b fe ff ff 48 63 d0 49 8b 0f 48 03 0c d5 80 99 19 82 8b 51 18 83 e2 01 74 0a f3 90 <8b> 51 18 83 e2 01 75 f6 eb c7 0f b6 4c 24 04 48 83 c4 10 48 89 de [3142785.484012][ C16] RSP: 0018:ffffc90000463948 EFLAGS: 00000202 ORIG_RAX: ffffffffffffff13 [3142785.484013][ C16] RAX: 0000000000000003 RBX: ffffffff8107c1e0 RCX: ffff88d4e74f1740 [3142785.484014][ C16] RDX: 0000000000000001 RSI: 000000000000001e RDI: ffff88d4db029008 [3142785.484014][ C16] RBP: ffff88d4e782a300 R08: 0000000000000000 R09: 0000000000000000 [3142785.484015][ C16] R10: 0000000000000000 R11: 000000011f521840 R12: 000000000002ba40 [3142785.484015][ C16] R13: 0000000000000001 R14: 000000000000001e R15: ffff88d4e782ba80 [3142785.484016][ C16] FS: 00007f307f5ea700(0000) GS:ffff88d4e7800000(0000) knlGS:0000000000000000 [3142785.484017][ C16] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [3142785.484018][ C16] CR2: 00007f302df32740 CR3: 000000518913e004 CR4: 0000000000760ee0 [3142785.484020][ C16] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [3142785.484020][ C16] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [3142785.484021][ C16] PKRU: 55555554 [3142785.484021][ C16] Call Trace: [3142785.484024][ C16] ? flush_tlb_func_common.isra.8+0x230/0x230 [3142785.484025][ C16] ? flush_tlb_func_common.isra.8+0x230/0x230 [3142785.484026][ C16] on_each_cpu_mask+0x24/0x60 [3142785.484027][ C16] ? x86_configure_nx+0x50/0x50 [3142785.484028][ C16] on_each_cpu_cond_mask+0xa5/0x140 [3142785.484029][ C16] ? flush_tlb_func_common.isra.8+0x147/0x230 [3142785.484030][ C16] flush_tlb_mm_range+0xab/0xe0 [3142785.484032][ C16] ptep_clear_flush+0x56/0x60 [3142785.484033][ C16] wp_page_copy+0x34a/0x7e0 [3142785.484035][ C16] do_wp_page+0x8b/0x660 [3142785.484037][ C16] __handle_mm_fault+0x777/0xef0 [3142785.484039][ C16] handle_mm_fault+0xe2/0x1f0 [3142785.484040][ C16] __do_page_fault+0x226/0x490 [3142785.484042][ C16] do_page_fault+0x31/0x120 [3142785.484043][ C16] async_page_fault+0x3e/0x50 [3142785.484045][ C16] RIP: 0010:copy_user_enhanced_fast_string+0xe/0x30 [3142785.484046][ C16] Code: 89 d1 c1 e9 03 83 e2 07 f3 48 a5 89 d1 f3 a4 31 c0 0f 01 ca c3 0f 1f 80 00 00 00 00 0f 01 cb 83 fa 40 0f 82 70 ff ff ff 89 d1 a4 31 c0 0f 01 ca c3 66 2e 0f 1f 84 00 00 00 00 00 89 d1 f3 a4 [3142785.484047][ C16] RSP: 0018:ffffc90000463cf8 EFLAGS: 00050202 [3142785.484048][ C16] RAX: 00007f302df32930 RBX: ffffc90000463e58 RCX: 00000000000001f0 [3142785.484049][ C16] RDX: 00000000000001f0 RSI: ffff88bc2be34e10 RDI: 00007f302df32740 [3142785.484049][ C16] RBP: 00000000000001f0 R08: 0000000000004009 R09: 0000000000000000 [3142785.484050][ C16] R10: 0000000000004000 R11: ffff88d1005ec538 R12: ffffc90000463e48 [3142785.484050][ C16] R13: 00000000000001f0 R14: ffff88bc2be34e10 R15: 00000000000001f0 [3142785.484055][ C16] copyout+0x22/0x30 [3142785.484056][ C16] copy_page_to_iter+0xfd/0x350 [3142785.484059][ C16] generic_file_buffered_read+0x4d6/0xbd0 [3142785.484061][ C16] ? do_futex+0x44c/0x940 [3142785.484064][ C16] new_sync_read+0x106/0x1a0 [3142785.484066][ C16] vfs_read+0x89/0x140 [3142785.484067][ C16] ksys_read+0x55/0xd0 [3142785.484069][ C16] do_syscall_64+0x5b/0x1b0 [3142785.484071][ C16] entry_SYSCALL_64_after_hwframe+0x44/0xa9 [3142785.484072][ C16] RIP: 0033:0x7f35e1c6171d [3142785.484073][ C16] Code: cc 20 00 00 75 10 b8 00 00 00 00 0f 05 48 3d 01 f0 ff ff 73 31 c3 48 83 ec 08 e8 ee fc ff ff 48 89 04 24 b8 00 00 00 00 0f 05 <48> 8b 3c 24 48 89 c2 e8 37 fd ff ff 48 89 d0 48 83 c4 08 48 3d 01 [3142785.484074][ C16] RSP: 002b:00007f307f5e6d80 EFLAGS: 00000293 ORIG_RAX: 0000000000000000 [3142785.484074][ C16] RAX: ffffffffffffffda RBX: 00007f302df32740 RCX: 00007f35e1c6171d [3142785.484075][ C16] RDX: 0000000000004009 RSI: 00007f302df32740 RDI: 00000000000000e5 [3142785.484076][ C16] RBP: 00007f307f5e6ef0 R08: 0000000000000000 R09: 0000000000000028 [3142785.484076][ C16] R10: 0000000000000000 R11: 0000000000000293 R12: 0000000000004009 [3142785.484077][ C16] R13: 00007f35d0099b30 R14: 0000000000000000 R15: 0000000000000000 [3142785.486193][ C4] RDX: 0000000000000003 RSI: 000000000000001e RDI: ffff88d4db029b38 [3142785.486987][ C17] watchdog: BUG: soft lockup - CPU#17 stuck for 23s! [vertica:284226] [3142785.486988][ C17] Modules linked in: binfmt_misc(E) rpcsec_gss_krb5(E) nfsv4(E) dns_resolver(E) nfs(E) fscache(E) nfsd(E) nfs_acl(E) lockd(E) auth_rpcgss(E) grace(E) nf_conntrack(E) nf_defrag_ipv6(E) nf_defrag_ipv4(E) nls_utf8(E) isofs(E) sunrpc(E) ip_tables(E) ufs(E) hfsplus(E) hfs(E) vfat(E) msdos(E) fat(E) xfs(E) libcrc32c(E) dm_mod(E) ext4(E) mbcache(E) jbd2(E) st(E) sd_mod(E) fuse(E) squashfs(E) zstd_decompress(E) loop(E) brd(E) isst_if_common(E) nfit(E) kvm_intel(E) sr_mod(E) cdrom(E) sg(E) kvm(E) ata_generic(E) irqbypass(E) crct10dif_pclmul(E) crc32_pclmul(E) crc32c_intel(E) ghash_clmulni_intel(E) cirrus(E) drm_kms_helper(E) syscopyarea(E) sysfillrect(E) sysimgblt(E) fb_sys_fops(E) drm(E) ata_piix(E) libata(E) virtio_net(E) joydev(E) serio_raw(E) net_failover(E) virtio_blk(E) virtio_balloon(E) pcspkr(E) i2c_piix4(E) failover(E) qemu_fw_cfg(E) sch_fq_codel(E) [last unloaded: dm_mod] [3142785.487004][ C17] CPU: 17 PID: 284226 Comm: vertica Tainted: G EL 5.4.34-1.el7.gdc.x86_64 #1 [3142785.487005][ C17] Hardware name: RDO OpenStack Compute, BIOS 1.11.0-2.el7 04/01/2014 [3142785.487006][ C17] RIP: 0010:smp_call_function_many+0x21e/0x280 [3142785.487007][ C17] Code: c7 e8 56 25 77 00 3b 05 04 9f 48 01 0f 83 7b fe ff ff 48 63 d0 49 8b 0f 48 03 0c d5 80 99 19 82 8b 51 18 83 e2 01 74 0a f3 90 <8b> 51 18 83 e2 01 75 f6 eb c7 0f b6 4c 24 04 48 83 c4 10 48 89 de [3142785.487007][ C17] RSP: 0000:ffffc9000343fc58 EFLAGS: 00000202 ORIG_RAX: ffffffffffffff13 [3142785.487008][ C17] RAX: 000000000000001a RBX: ffffffff8107c1e0 RCX: ffff88d4e7ab1760 [3142785.487009][ C17] RDX: 0000000000000001 RSI: 000000000000001e RDI: ffff88d4db0298a0 [3142785.487009][ C17] RBP: ffff88d4e786a300 R08: 0000000000000000 R09: 0000000000000000 [3142785.487010][ C17] R10: 0000000000000000 R11: 00000000adeeb600 R12: 000000000002ba40 [3142785.487010][ C17] R13: 0000000000000001 R14: 000000000000001e R15: ffff88d4e786ba80 [3142785.487011][ C17] FS: 00007f3103fff700(0000) GS:ffff88d4e7840000(0000) knlGS:0000000000000000 [3142785.487012][ C17] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [3142785.487013][ C17] CR2: 00007f3509f02368 CR3: 000000518913e004 CR4: 0000000000760ee0 [3142785.487015][ C17] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [3142785.487015][ C17] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [3142785.487015][ C17] PKRU: 55555554 [3142785.487016][ C17] Call Trace: [3142785.487017][ C17] ? flush_tlb_func_common.isra.8+0x230/0x230 [3142785.487018][ C17] ? flush_tlb_func_common.isra.8+0x230/0x230 [3142785.487019][ C17] on_each_cpu_mask+0x24/0x60 [3142785.487020][ C17] ? x86_configure_nx+0x50/0x50 [3142785.487021][ C17] on_each_cpu_cond_mask+0xa5/0x140 [3142785.487023][ C17] ? flush_tlb_func_common.isra.8+0x147/0x230 [3142785.487024][ C17] flush_tlb_mm_range+0xab/0xe0 [3142785.487025][ C17] ptep_clear_flush+0x56/0x60 [3142785.487026][ C17] wp_page_copy+0x34a/0x7e0 [3142785.487028][ C17] do_wp_page+0x8b/0x660 [3142785.487030][ C17] __handle_mm_fault+0x777/0xef0 [3142785.487031][ C17] handle_mm_fault+0xe2/0x1f0 [3142785.487033][ C17] __do_page_fault+0x226/0x490 [3142785.487034][ C17] do_page_fault+0x31/0x120 [3142785.487035][ C17] async_page_fault+0x3e/0x50 [3142785.487036][ C17] RIP: 0033:0x101d045 [3142785.487037][ C17] Code: 48 31 c1 4c 21 c9 49 89 0c fe 8b 46 24 48 0f af c1 48 03 46 28 48 8b 10 48 85 d2 75 80 48 ba 00 00 00 00 00 00 00 40 49 09 d0 <4c> 89 00 48 8b 56 40 48 3b 56 60 73 0c 4c 8b 46 68 49 89 0c d0 48 [3142785.487038][ C17] RSP: 002b:00007f28e81fc5d0 EFLAGS: 00010206 [3142785.487038][ C17] RAX: 00007f3509f02368 RBX: 0000000000000019 RCX: 0000000000000b49 [3142785.487039][ C17] RDX: 4000000000000000 RSI: 00007f28e81fdab0 RDI: 0000000000000019 [3142785.487040][ C17] RBP: 00007f28e81fc6a0 R08: 40758102271da741 R09: 0000000000007fff [3142785.487040][ C17] R10: 3fffffffffffffff R11: 00007f2ecdd7eaf0 R12: 0000000000000001 [3142785.487041][ C17] R13: 00007f28e8204090 R14: 00007f28e8244090 R15: 0000000000000034 [3142785.489989][ C18] watchdog: BUG: soft lockup - CPU#18 stuck for 23s! [vertica:284166] [3142785.489990][ C18] Modules linked in: binfmt_misc(E) rpcsec_gss_krb5(E) nfsv4(E) dns_resolver(E) nfs(E) fscache(E) nfsd(E) nfs_acl(E) lockd(E) auth_rpcgss(E) grace(E) nf_conntrack(E) nf_defrag_ipv6(E) nf_defrag_ipv4(E) nls_utf8(E) isofs(E) sunrpc(E) ip_tables(E) ufs(E) hfsplus(E) hfs(E) vfat(E) msdos(E) fat(E) xfs(E) libcrc32c(E) dm_mod(E) ext4(E) mbcache(E) jbd2(E) st(E) sd_mod(E) fuse(E) squashfs(E) zstd_decompress(E) loop(E) brd(E) isst_if_common(E) nfit(E) kvm_intel(E) sr_mod(E) cdrom(E) sg(E) kvm(E) ata_generic(E) irqbypass(E) crct10dif_pclmul(E) crc32_pclmul(E) crc32c_intel(E) ghash_clmulni_intel(E) cirrus(E) drm_kms_helper(E) syscopyarea(E) sysfillrect(E) sysimgblt(E) fb_sys_fops(E) drm(E) ata_piix(E) libata(E) virtio_net(E) joydev(E) serio_raw(E) net_failover(E) virtio_blk(E) virtio_balloon(E) pcspkr(E) i2c_piix4(E) failover(E) qemu_fw_cfg(E) sch_fq_codel(E) [last unloaded: dm_mod] [3142785.490008][ C18] CPU: 18 PID: 284166 Comm: vertica Tainted: G EL 5.4.34-1.el7.gdc.x86_64 #1 [3142785.490008][ C18] Hardware name: RDO OpenStack Compute, BIOS 1.11.0-2.el7 04/01/2014 [3142785.490011][ C18] RIP: 0010:smp_call_function_many+0x21e/0x280 [3142785.490012][ C18] Code: c7 e8 56 25 77 00 3b 05 04 9f 48 01 0f 83 7b fe ff ff 48 63 d0 49 8b 0f 48 03 0c d5 80 99 19 82 8b 51 18 83 e2 01 74 0a f3 90 <8b> 51 18 83 e2 01 75 f6 eb c7 0f b6 4c 24 04 48 83 c4 10 48 89 de [3142785.490012][ C18] RSP: 0000:ffffc90002ed7c58 EFLAGS: 00000202 ORIG_RAX: ffffffffffffff13 [3142785.490013][ C18] RAX: 000000000000001a RBX: ffffffff8107c1e0 RCX: ffff88d4e7ab1780 [3142785.490014][ C18] RDX: 0000000000000001 RSI: 000000000000001e RDI: ffff88d4db0291b0 [3142785.490015][ C18] RBP: ffff88d4e78aa300 R08: 0000000000000000 R09: 0000000000000000 [3142785.490015][ C18] R10: 0000000000000000 R11: 0000000069564e00 R12: 000000000002ba40 [3142785.490016][ C18] R13: 0000000000000001 R14: 000000000000001e R15: ffff88d4e78aba80 [3142785.490017][ C18] FS: 00007f31e15f1700(0000) GS:ffff88d4e7880000(0000) knlGS:0000000000000000 [3142785.490018][ C18] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [3142785.490018][ C18] CR2: 00007f28da2a3520 CR3: 000000518913e003 CR4: 0000000000760ee0 [3142785.490020][ C18] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [3142785.490021][ C18] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [3142785.490021][ C18] PKRU: 55555554 [3142785.490021][ C18] Call Trace: [3142785.490024][ C18] ? flush_tlb_func_common.isra.8+0x230/0x230 [3142785.490025][ C18] ? flush_tlb_func_common.isra.8+0x230/0x230 [3142785.490026][ C18] on_each_cpu_mask+0x24/0x60 [3142785.490027][ C18] ? x86_configure_nx+0x50/0x50 [3142785.490028][ C18] on_each_cpu_cond_mask+0xa5/0x140 [3142785.490029][ C18] ? flush_tlb_func_common.isra.8+0x147/0x230 [3142785.490031][ C18] flush_tlb_mm_range+0xab/0xe0 [3142785.490032][ C18] ptep_clear_flush+0x56/0x60 [3142785.490034][ C18] wp_page_copy+0x34a/0x7e0 [3142785.490036][ C18] do_wp_page+0x8b/0x660 [3142785.490037][ C18] __handle_mm_fault+0x777/0xef0 [3142785.490039][ C18] handle_mm_fault+0xe2/0x1f0 [3142785.490041][ C18] __do_page_fault+0x226/0x490 [3142785.490042][ C18] do_page_fault+0x31/0x120 [3142785.490043][ C18] async_page_fault+0x3e/0x50 [3142785.490045][ C18] RIP: 0033:0x101d045 [3142785.490046][ C18] Code: 48 31 c1 4c 21 c9 49 89 0c fe 8b 46 24 48 0f af c1 48 03 46 28 48 8b 10 48 85 d2 75 80 48 ba 00 00 00 00 00 00 00 40 49 09 d0 <4c> 89 00 48 8b 56 40 48 3b 56 60 73 0c 4c 8b 46 68 49 89 0c d0 48 [3142785.490046][ C18] RSP: 002b:00007f2fb21e36d0 EFLAGS: 00010202 [3142785.490047][ C18] RAX: 00007f28da2a3520 RBX: 00000000000000cd RCX: 000000000000718c [3142785.490048][ C18] RDX: 4000000000000000 RSI: 00007f2fb21ec9c0 RDI: 00000000000000cd [3142785.490049][ C18] RBP: 00007f2fb21e37a0 R08: 400008d0bf697101 R09: 000000000000ffff [3142785.490049][ C18] R10: 3fffffffffffffff R11: 00007f302c5f95e0 R12: 0000000000000001 [3142785.490050][ C18] R13: 00007f2fb21eedf0 R14: 00007f2fb222edf0 R15: 0000000000000034 [3142785.490087][ C5] RDX: 0000000000000001 RSI: 000000000000001e RDI: ffff88d4db0291c8 [3142785.490088][ C5] RBP: ffff88d4e756a300 R08: 0000000000000000 R09: 0000000000000000 [3142785.493070][ C4] RBP: ffff88d4e752a300 R08: 0000000000000000 R09: 0000000000000000 [3142785.494324][ C5] R10: 0000000000000000 R11: 00000000e2b3f0c0 R12: 000000000002ba40 [3142785.494325][ C5] R13: 0000000000000001 R14: 000000000000001e R15: ffff88d4e756ba80 [3142785.497421][ C4] R10: 0000000000000000 R11: 00000000cd929540 R12: 000000000002ba40 [3142785.497423][ C4] R13: 0000000000000001 R14: 000000000000001e R15: ffff88d4e752ba80 [3142785.499988][ C21] watchdog: BUG: soft lockup - CPU#21 stuck for 23s! [vertica:284315] [3142785.499989][ C21] Modules linked in: binfmt_misc(E) rpcsec_gss_krb5(E) nfsv4(E) dns_resolver(E) nfs(E) fscache(E) nfsd(E) nfs_acl(E) lockd(E) auth_rpcgss(E) grace(E) nf_conntrack(E) nf_defrag_ipv6(E) nf_defrag_ipv4(E) nls_utf8(E) isofs(E) sunrpc(E) ip_tables(E) ufs(E) hfsplus(E) hfs(E) vfat(E) msdos(E) fat(E) xfs(E) libcrc32c(E) dm_mod(E) ext4(E) mbcache(E) jbd2(E) st(E) sd_mod(E) fuse(E) squashfs(E) zstd_decompress(E) loop(E) brd(E) isst_if_common(E) nfit(E) kvm_intel(E) sr_mod(E) cdrom(E) sg(E) kvm(E) ata_generic(E) irqbypass(E) crct10dif_pclmul(E) crc32_pclmul(E) crc32c_intel(E) ghash_clmulni_intel(E) cirrus(E) drm_kms_helper(E) syscopyarea(E) sysfillrect(E) sysimgblt(E) fb_sys_fops(E) drm(E) ata_piix(E) libata(E) virtio_net(E) joydev(E) serio_raw(E) net_failover(E) virtio_blk(E) virtio_balloon(E) pcspkr(E) i2c_piix4(E) failover(E) qemu_fw_cfg(E) sch_fq_codel(E) [last unloaded: dm_mod] [3142785.500006][ C21] CPU: 21 PID: 284315 Comm: vertica Tainted: G EL 5.4.34-1.el7.gdc.x86_64 #1 [3142785.500007][ C21] Hardware name: RDO OpenStack Compute, BIOS 1.11.0-2.el7 04/01/2014 [3142785.500009][ C21] RIP: 0010:smp_call_function_many+0x21e/0x280 [3142785.500010][ C21] Code: c7 e8 56 25 77 00 3b 05 04 9f 48 01 0f 83 7b fe ff ff 48 63 d0 49 8b 0f 48 03 0c d5 80 99 19 82 8b 51 18 83 e2 01 74 0a f3 90 <8b> 51 18 83 e2 01 75 f6 eb c7 0f b6 4c 24 04 48 83 c4 10 48 89 de [3142785.500010][ C21] RSP: 0018:ffffc90001877ce0 EFLAGS: 00000202 ORIG_RAX: ffffffffffffff13 [3142785.500011][ C21] RAX: 000000000000001a RBX: ffffffff8107c1e0 RCX: ffff88d4e7ab17e0 [3142785.500012][ C21] RDX: 0000000000000001 RSI: 000000000000001e RDI: ffff88d4db029c30 [3142785.500012][ C21] RBP: ffff88d4e796a300 R08: 0000000000000000 R09: 0000000000000000 [3142785.500013][ C21] R10: 0000000000000000 R11: 0000000000000000 R12: 000000000002ba40 [3142785.500013][ C21] R13: 0000000000000001 R14: 000000000000001e R15: ffff88d4e796ba80 [3142785.500014][ C21] FS: 00007f305c9c1700(0000) GS:ffff88d4e7940000(0000) knlGS:0000000000000000 [3142785.500015][ C21] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [3142785.500015][ C21] CR2: 00007fa6c84ecffc CR3: 000000518913e004 CR4: 0000000000760ee0 [3142785.500017][ C21] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [3142785.500018][ C21] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [3142785.500018][ C21] PKRU: 55555554 [3142785.500018][ C21] Call Trace: [3142785.500021][ C21] flush_tlb_mm_range+0xab/0xe0 [3142785.500022][ C21] tlb_flush_mmu+0x89/0x150 [3142785.500023][ C21] tlb_finish_mmu+0x3d/0x70 [3142785.500025][ C21] zap_page_range+0x115/0x160 [3142785.500026][ C21] ? do_futex+0x44c/0x940 [3142785.500028][ C21] ? enqueue_entity+0x231/0x5d0 [3142785.500029][ C21] ? find_vma+0x59/0x60 [3142785.500031][ C21] __do_sys_madvise+0x738/0xc00 [3142785.500033][ C21] do_syscall_64+0x5b/0x1b0 [3142785.500035][ C21] entry_SYSCALL_64_after_hwframe+0x44/0xa9 [3142785.500036][ C21] RIP: 0033:0x7f35e1571e97 [3142785.500037][ C21] Code: 83 c4 08 48 3d 01 f0 ff ff 73 01 c3 48 8b 0d d0 df 2c 00 f7 d8 64 89 01 48 83 c8 ff c3 66 0f 1f 44 00 00 b8 1c 00 00 00 0f 05 <48> 3d 01 f0 ff ff 73 01 c3 48 8b 0d a9 df 2c 00 f7 d8 64 89 01 48 [3142785.500038][ C21] RSP: 002b:00007f305c9be3d8 EFLAGS: 00000206 ORIG_RAX: 000000000000001c [3142785.500038][ C21] RAX: ffffffffffffffda RBX: 00007f305c9c1700 RCX: 00007f35e1571e97 [3142785.500039][ C21] RDX: 0000000000000004 RSI: 00000000007f8000 RDI: 00007f305c1c2000 [3142785.500039][ C21] RBP: 0000000000000000 R08: 0000000000000001 R09: 00007f2bc45d6f90 [3142785.500040][ C21] R10: 0000000000000001 R11: 0000000000000206 R12: 0000000000000000 [3142785.500040][ C21] R13: 00007f34c37f5620 R14: 00007f34c37f56e0 R15: 00007f305c9c1700 [3142785.500335][ C5] FS: 00007f31e85f1700(0000) GS:ffff88d4e7540000(0000) knlGS:0000000000000000 [3142785.501544][ C4] FS: 00007f2b5e9ff700(0000) GS:ffff88d4e7500000(0000) knlGS:0000000000000000 [3142785.504499][ C5] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [3142785.505750][ C4] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [3142785.505988][ C23] watchdog: BUG: soft lockup - CPU#23 stuck for 23s! [vertica:283851] [3142785.505988][ C23] Modules linked in: binfmt_misc(E) rpcsec_gss_krb5(E) nfsv4(E) dns_resolver(E) nfs(E) fscache(E) nfsd(E) nfs_acl(E) lockd(E) auth_rpcgss(E) grace(E) nf_conntrack(E) nf_defrag_ipv6(E) nf_defrag_ipv4(E) nls_utf8(E) isofs(E) sunrpc(E) ip_tables(E) ufs(E) hfsplus(E) hfs(E) vfat(E) msdos(E) fat(E) xfs(E) libcrc32c(E) dm_mod(E) ext4(E) mbcache(E) jbd2(E) st(E) sd_mod(E) fuse(E) squashfs(E) zstd_decompress(E) loop(E) brd(E) isst_if_common(E) nfit(E) kvm_intel(E) sr_mod(E) cdrom(E) sg(E) kvm(E) ata_generic(E) irqbypass(E) crct10dif_pclmul(E) crc32_pclmul(E) crc32c_intel(E) ghash_clmulni_intel(E) cirrus(E) drm_kms_helper(E) syscopyarea(E) sysfillrect(E) sysimgblt(E) fb_sys_fops(E) drm(E) ata_piix(E) libata(E) virtio_net(E) joydev(E) serio_raw(E) net_failover(E) virtio_blk(E) virtio_balloon(E) pcspkr(E) i2c_piix4(E) failover(E) qemu_fw_cfg(E) sch_fq_codel(E) [last unloaded: dm_mod] [3142785.506005][ C23] CPU: 23 PID: 283851 Comm: vertica Tainted: G EL 5.4.34-1.el7.gdc.x86_64 #1 [3142785.506005][ C23] Hardware name: RDO OpenStack Compute, BIOS 1.11.0-2.el7 04/01/2014 [3142785.506007][ C23] RIP: 0010:smp_call_function_many+0x21e/0x280 [3142785.506008][ C23] Code: c7 e8 56 25 77 00 3b 05 04 9f 48 01 0f 83 7b fe ff ff 48 63 d0 49 8b 0f 48 03 0c d5 80 99 19 82 8b 51 18 83 e2 01 74 0a f3 90 <8b> 51 18 83 e2 01 75 f6 eb c7 0f b6 4c 24 04 48 83 c4 10 48 89 de [3142785.506009][ C23] RSP: 0000:ffffc9000157bc58 EFLAGS: 00000202 ORIG_RAX: ffffffffffffff13 [3142785.506010][ C23] RAX: 0000000000000003 RBX: ffffffff8107c1e0 RCX: ffff88d4e74f1820 [3142785.506010][ C23] RDX: 0000000000000001 RSI: 000000000000001e RDI: ffff88d4db0290b0 [3142785.506011][ C23] RBP: ffff88d4e79ea300 R08: 0000000000000000 R09: 0000000000000000 [3142785.506011][ C23] R10: 0000000000000000 R11: 00000000aa0ab680 R12: 000000000002ba40 [3142785.506012][ C23] R13: 0000000000000001 R14: 000000000000001e R15: ffff88d4e79eba80 [3142785.506013][ C23] FS: 00007f2b5d9fe700(0000) GS:ffff88d4e79c0000(0000) knlGS:0000000000000000 [3142785.506014][ C23] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [3142785.506014][ C23] CR2: 00007f30232a4e20 CR3: 000000518913e003 CR4: 0000000000760ee0 [3142785.506016][ C23] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [3142785.506017][ C23] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [3142785.506017][ C23] PKRU: 55555554 [3142785.506017][ C23] Call Trace: [3142785.506020][ C23] ? flush_tlb_func_common.isra.8+0x230/0x230 [3142785.506021][ C23] ? flush_tlb_func_common.isra.8+0x230/0x230 [3142785.506022][ C23] on_each_cpu_mask+0x24/0x60 [3142785.506023][ C23] ? x86_configure_nx+0x50/0x50 [3142785.506024][ C23] on_each_cpu_cond_mask+0xa5/0x140 [3142785.506025][ C23] ? flush_tlb_func_common.isra.8+0x92/0x230 [3142785.506026][ C23] flush_tlb_mm_range+0xab/0xe0 [3142785.506027][ C23] ptep_clear_flush+0x56/0x60 [3142785.506029][ C23] wp_page_copy+0x34a/0x7e0 [3142785.506030][ C23] do_wp_page+0x8b/0x660 [3142785.506032][ C23] __handle_mm_fault+0x777/0xef0 [3142785.506034][ C23] handle_mm_fault+0xe2/0x1f0 [3142785.506035][ C23] __do_page_fault+0x226/0x490 [3142785.506036][ C23] do_page_fault+0x31/0x120 [3142785.506038][ C23] async_page_fault+0x3e/0x50 [3142785.506039][ C23] RIP: 0033:0xe148c5 [3142785.506040][ C23] Code: fa 08 0f 84 44 01 00 00 48 83 ff 07 4c 89 c2 76 33 48 83 ef 08 31 d2 48 89 f9 48 c1 e9 03 48 8d 34 cd 08 00 00 00 4d 8b 0c 10 <4c> 89 0c 10 48 83 c2 08 48 39 f2 75 ef 48 f7 d9 48 01 d0 4c 01 c2 [3142785.506040][ C23] RSP: 002b:00007f264cf013f0 EFLAGS: 00010202 [3142785.506041][ C23] RAX: 00007f30232a4e20 RBX: 000000000000001f RCX: 0000000000000002 [3142785.506041][ C23] RDX: 0000000000000000 RSI: 0000000000000018 RDI: 0000000000000010 [3142785.506042][ C23] RBP: 00007f264cf01560 R08: 00007f261b8baee8 R09: 400008d0c94b528d [3142785.506042][ C23] R10: 0000000000000018 R11: 000000000000869f R12: 0000000000200000 [3142785.506043][ C23] R13: 00007f30217ff000 R14: 00007f302ca95230 R15: 00007f264cf015d0 [3142785.508801][ C5] CR2: 00007f29887620b0 CR3: 000000518913e005 CR4: 0000000000760ee0 [3142785.511987][ C25] watchdog: BUG: soft lockup - CPU#25 stuck for 23s! [vertica:283850] [3142785.511988][ C25] Modules linked in: binfmt_misc(E) rpcsec_gss_krb5(E) nfsv4(E) dns_resolver(E) nfs(E) fscache(E) nfsd(E) nfs_acl(E) lockd(E) auth_rpcgss(E) grace(E) nf_conntrack(E) nf_defrag_ipv6(E) nf_defrag_ipv4(E) nls_utf8(E) isofs(E) sunrpc(E) ip_tables(E) ufs(E) hfsplus(E) hfs(E) vfat(E) msdos(E) fat(E) xfs(E) libcrc32c(E) dm_mod(E) ext4(E) mbcache(E) jbd2(E) st(E) sd_mod(E) fuse(E) squashfs(E) zstd_decompress(E) loop(E) brd(E) isst_if_common(E) nfit(E) kvm_intel(E) sr_mod(E) cdrom(E) sg(E) kvm(E) ata_generic(E) irqbypass(E) crct10dif_pclmul(E) crc32_pclmul(E) crc32c_intel(E) ghash_clmulni_intel(E) cirrus(E) drm_kms_helper(E) syscopyarea(E) sysfillrect(E) sysimgblt(E) fb_sys_fops(E) drm(E) ata_piix(E) libata(E) virtio_net(E) joydev(E) serio_raw(E) net_failover(E) virtio_blk(E) virtio_balloon(E) pcspkr(E) i2c_piix4(E) failover(E) qemu_fw_cfg(E) sch_fq_codel(E) [last unloaded: dm_mod] [3142785.512006][ C25] CPU: 25 PID: 283850 Comm: vertica Tainted: G EL 5.4.34-1.el7.gdc.x86_64 #1 [3142785.512006][ C25] Hardware name: RDO OpenStack Compute, BIOS 1.11.0-2.el7 04/01/2014 [3142785.512009][ C25] RIP: 0010:smp_call_function_many+0x21e/0x280 [3142785.512010][ C25] Code: c7 e8 56 25 77 00 3b 05 04 9f 48 01 0f 83 7b fe ff ff 48 63 d0 49 8b 0f 48 03 0c d5 80 99 19 82 8b 51 18 83 e2 01 74 0a f3 90 <8b> 51 18 83 e2 01 75 f6 eb c7 0f b6 4c 24 04 48 83 c4 10 48 89 de [3142785.512011][ C25] RSP: 0000:ffffc90001543c58 EFLAGS: 00000202 ORIG_RAX: ffffffffffffff13 [3142785.512012][ C25] RAX: 0000000000000003 RBX: ffffffff8107c1e0 RCX: ffff88d4e74f1860 [3142785.512012][ C25] RDX: 0000000000000001 RSI: 000000000000001e RDI: ffff88d4db029b18 [3142785.512013][ C25] RBP: ffff88d4e7a6a300 R08: 0000000000000000 R09: 0000000000000000 [3142785.512014][ C25] R10: 0000000000000000 R11: 00000000c1c720c0 R12: 000000000002ba40 [3142785.512014][ C25] R13: 0000000000000001 R14: 000000000000001e R15: ffff88d4e7a6ba80 [3142785.512015][ C25] FS: 00007f2b5c9fd700(0000) GS:ffff88d4e7a40000(0000) knlGS:0000000000000000 [3142785.512016][ C25] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [3142785.512017][ C25] CR2: 00007f30a2c32758 CR3: 000000518913e002 CR4: 0000000000760ee0 [3142785.512019][ C25] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [3142785.512019][ C25] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [3142785.512020][ C25] PKRU: 55555554 [3142785.512020][ C25] Call Trace: [3142785.512023][ C25] ? flush_tlb_func_common.isra.8+0x230/0x230 [3142785.512024][ C25] ? flush_tlb_func_common.isra.8+0x230/0x230 [3142785.512025][ C25] on_each_cpu_mask+0x24/0x60 [3142785.512026][ C25] ? x86_configure_nx+0x50/0x50 [3142785.512027][ C25] on_each_cpu_cond_mask+0xa5/0x140 [3142785.512028][ C25] ? flush_tlb_func_common.isra.8+0x92/0x230 [3142785.512029][ C25] flush_tlb_mm_range+0xab/0xe0 [3142785.512031][ C25] ptep_clear_flush+0x56/0x60 [3142785.512033][ C25] wp_page_copy+0x34a/0x7e0 [3142785.512034][ C25] do_wp_page+0x8b/0x660 [3142785.512036][ C25] __handle_mm_fault+0x777/0xef0 [3142785.512037][ C25] handle_mm_fault+0xe2/0x1f0 [3142785.512039][ C25] __do_page_fault+0x226/0x490 [3142785.512041][ C25] do_page_fault+0x31/0x120 [3142785.512042][ C25] async_page_fault+0x3e/0x50 [3142785.512043][ C25] RIP: 0033:0xe148c5 [3142785.512044][ C25] Code: fa 08 0f 84 44 01 00 00 48 83 ff 07 4c 89 c2 76 33 48 83 ef 08 31 d2 48 89 f9 48 c1 e9 03 48 8d 34 cd 08 00 00 00 4d 8b 0c 10 <4c> 89 0c 10 48 83 c2 08 48 39 f2 75 ef 48 f7 d9 48 01 d0 4c 01 c2 [3142785.512044][ C25] RSP: 002b:00007f264cdffe30 EFLAGS: 00010202 [3142785.512045][ C25] RAX: 00007f30a2c32758 RBX: 000000000000001f RCX: 0000000000000002 [3142785.512046][ C25] RDX: 0000000000000000 RSI: 0000000000000018 RDI: 0000000000000010 [3142785.512047][ C25] RBP: 00007f264cdfffa0 R08: 00007f26450adf10 R09: 40003c9dff53ec68 [3142785.512047][ C25] R10: 0000000000000018 R11: 00000000000073f6 R12: 0000000000200000 [3142785.512048][ C25] R13: 00007f30a0000000 R14: 00007f302ca92140 R15: 00007f264ce00010 [3142785.512691][ C4] CR2: 00007f271a871a58 CR3: 000000518913e001 CR4: 0000000000760ee0 [3142785.512694][ C4] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [3142785.514988][ C26] watchdog: BUG: soft lockup - CPU#26 stuck for 23s! [vertica:283854] [3142785.514989][ C26] Modules linked in: binfmt_misc(E) rpcsec_gss_krb5(E) nfsv4(E) dns_resolver(E) nfs(E) fscache(E) nfsd(E) nfs_acl(E) lockd(E) auth_rpcgss(E) grace(E) nf_conntrack(E) nf_defrag_ipv6(E) nf_defrag_ipv4(E) nls_utf8(E) isofs(E) sunrpc(E) ip_tables(E) ufs(E) hfsplus(E) hfs(E) vfat(E) msdos(E) fat(E) xfs(E) libcrc32c(E) dm_mod(E) ext4(E) mbcache(E) jbd2(E) st(E) sd_mod(E) fuse(E) squashfs(E) zstd_decompress(E) loop(E) brd(E) isst_if_common(E) nfit(E) kvm_intel(E) sr_mod(E) cdrom(E) sg(E) kvm(E) ata_generic(E) irqbypass(E) crct10dif_pclmul(E) crc32_pclmul(E) crc32c_intel(E) ghash_clmulni_intel(E) cirrus(E) drm_kms_helper(E) syscopyarea(E) sysfillrect(E) sysimgblt(E) fb_sys_fops(E) drm(E) ata_piix(E) libata(E) virtio_net(E) joydev(E) serio_raw(E) net_failover(E) virtio_blk(E) virtio_balloon(E) pcspkr(E) i2c_piix4(E) failover(E) qemu_fw_cfg(E) sch_fq_codel(E) [last unloaded: dm_mod] [3142785.515006][ C26] CPU: 26 PID: 283854 Comm: vertica Tainted: G EL 5.4.34-1.el7.gdc.x86_64 #1 [3142785.515006][ C26] Hardware name: RDO OpenStack Compute, BIOS 1.11.0-2.el7 04/01/2014 [3142785.515009][ C26] RIP: 0010:smp_call_function_many+0x21e/0x280 [3142785.515010][ C26] Code: c7 e8 56 25 77 00 3b 05 04 9f 48 01 0f 83 7b fe ff ff 48 63 d0 49 8b 0f 48 03 0c d5 80 99 19 82 8b 51 18 83 e2 01 74 0a f3 90 <8b> 51 18 83 e2 01 75 f6 eb c7 0f b6 4c 24 04 48 83 c4 10 48 89 de [3142785.515011][ C26] RSP: 0000:ffffc900015a3c58 EFLAGS: 00000202 ORIG_RAX: ffffffffffffff13 [3142785.515011][ C26] RAX: 0000000000000003 RBX: ffffffff8107c1e0 RCX: ffff88d4e74f1880 [3142785.515012][ C26] RDX: 0000000000000001 RSI: 000000000000001e RDI: ffff88d4db029f38 [3142785.515012][ C26] RBP: ffff88d4e7aaa300 R08: 0000000000000000 R09: 0000000000000000 [3142785.515013][ C26] R10: 0000000000000000 R11: 0000000115aafd80 R12: 000000000002ba40 [3142785.515013][ C26] R13: 0000000000000001 R14: 000000000000001e R15: ffff88d4e7aaba80 [3142785.515014][ C26] FS: 00007f2ede7fe700(0000) GS:ffff88d4e7a80000(0000) knlGS:0000000000000000 [3142785.515015][ C26] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [3142785.515015][ C26] CR2: 00007f2f82861ed8 CR3: 000000518913e004 CR4: 0000000000760ee0 [3142785.515017][ C26] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [3142785.515018][ C26] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [3142785.515018][ C26] PKRU: 55555554 [3142785.515018][ C26] Call Trace: [3142785.515021][ C26] ? flush_tlb_func_common.isra.8+0x230/0x230 [3142785.515022][ C26] ? flush_tlb_func_common.isra.8+0x230/0x230 [3142785.515023][ C26] on_each_cpu_mask+0x24/0x60 [3142785.515024][ C26] ? x86_configure_nx+0x50/0x50 [3142785.515025][ C26] on_each_cpu_cond_mask+0xa5/0x140 [3142785.515026][ C26] ? flush_tlb_func_common.isra.8+0x92/0x230 [3142785.515027][ C26] flush_tlb_mm_range+0xab/0xe0 [3142785.515028][ C26] ptep_clear_flush+0x56/0x60 [3142785.515030][ C26] wp_page_copy+0x34a/0x7e0 [3142785.515031][ C26] do_wp_page+0x8b/0x660 [3142785.515033][ C26] __handle_mm_fault+0x777/0xef0 [3142785.515034][ C26] handle_mm_fault+0xe2/0x1f0 [3142785.515036][ C26] __do_page_fault+0x226/0x490 [3142785.515038][ C26] do_page_fault+0x31/0x120 [3142785.515039][ C26] async_page_fault+0x3e/0x50 [3142785.515040][ C26] RIP: 0033:0xe148c5 [3142785.515041][ C26] Code: fa 08 0f 84 44 01 00 00 48 83 ff 07 4c 89 c2 76 33 48 83 ef 08 31 d2 48 89 f9 48 c1 e9 03 48 8d 34 cd 08 00 00 00 4d 8b 0c 10 <4c> 89 0c 10 48 83 c2 08 48 39 f2 75 ef 48 f7 d9 48 01 d0 4c 01 c2 [3142785.515041][ C26] RSP: 002b:00007f312c5f09e0 EFLAGS: 00010202 [3142785.515042][ C26] RAX: 00007f2f82861ed8 RBX: 000000000000001f RCX: 0000000000000002 [3142785.515042][ C26] RDX: 0000000000000000 RSI: 0000000000000018 RDI: 0000000000000010 [3142785.515043][ C26] RBP: 00007f312c5f0b50 R08: 00007f2633066540 R09: 400008d0d1a481a5 [3142785.515043][ C26] R10: 0000000000000018 R11: 0000000000004438 R12: 0000000000200000 [3142785.515044][ C26] R13: 00007f2f827d6000 R14: 00007f302ca9e500 R15: 00007f312c5f0bc0 [3142785.515834][ C5] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [3142785.518803][ C4] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [3142785.520045][ C5] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [3142785.520047][ C5] PKRU: 55555554 [3142785.520987][ C28] watchdog: BUG: soft lockup - CPU#28 stuck for 23s! [vertica:283853] [3142785.520988][ C28] Modules linked in: binfmt_misc(E) rpcsec_gss_krb5(E) nfsv4(E) dns_resolver(E) nfs(E) fscache(E) nfsd(E) nfs_acl(E) lockd(E) auth_rpcgss(E) grace(E) nf_conntrack(E) nf_defrag_ipv6(E) nf_defrag_ipv4(E) nls_utf8(E) isofs(E) sunrpc(E) ip_tables(E) ufs(E) hfsplus(E) hfs(E) vfat(E) msdos(E) fat(E) xfs(E) libcrc32c(E) dm_mod(E) ext4(E) mbcache(E) jbd2(E) st(E) sd_mod(E) fuse(E) squashfs(E) zstd_decompress(E) loop(E) brd(E) isst_if_common(E) nfit(E) kvm_intel(E) sr_mod(E) cdrom(E) sg(E) kvm(E) ata_generic(E) irqbypass(E) crct10dif_pclmul(E) crc32_pclmul(E) crc32c_intel(E) ghash_clmulni_intel(E) cirrus(E) drm_kms_helper(E) syscopyarea(E) sysfillrect(E) sysimgblt(E) fb_sys_fops(E) drm(E) ata_piix(E) libata(E) virtio_net(E) joydev(E) serio_raw(E) net_failover(E) virtio_blk(E) virtio_balloon(E) pcspkr(E) i2c_piix4(E) failover(E) qemu_fw_cfg(E) sch_fq_codel(E) [last unloaded: dm_mod] [3142785.521005][ C28] CPU: 28 PID: 283853 Comm: vertica Tainted: G EL 5.4.34-1.el7.gdc.x86_64 #1 [3142785.521005][ C28] Hardware name: RDO OpenStack Compute, BIOS 1.11.0-2.el7 04/01/2014 [3142785.521007][ C28] RIP: 0010:smp_call_function_many+0x221/0x280 [3142785.521009][ C28] Code: 25 77 00 3b 05 04 9f 48 01 0f 83 7b fe ff ff 48 63 d0 49 8b 0f 48 03 0c d5 80 99 19 82 8b 51 18 83 e2 01 74 0a f3 90 8b 51 18 <83> e2 01 75 f6 eb c7 0f b6 4c 24 04 48 83 c4 10 48 89 de 5b 48 89 [3142785.521009][ C28] RSP: 0000:ffffc90001593c58 EFLAGS: 00000202 ORIG_RAX: ffffffffffffff13 [3142785.521010][ C28] RAX: 0000000000000003 RBX: ffffffff8107c1e0 RCX: ffff88d4e74f18c0 [3142785.521011][ C28] RDX: 0000000000000003 RSI: 000000000000001e RDI: ffff88d4db029fa8 [3142785.521012][ C28] RBP: ffff88d4e7b2a300 R08: 0000000000000000 R09: 0000000000000000 [3142785.521012][ C28] R10: 0000000000000000 R11: 0000000109627440 R12: 000000000002ba40 [3142785.521013][ C28] R13: 0000000000000001 R14: 000000000000001e R15: ffff88d4e7b2ba80 [3142785.521014][ C28] FS: 00007f2eddffd700(0000) GS:ffff88d4e7b00000(0000) knlGS:0000000000000000 [3142785.521014][ C28] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [3142785.521015][ C28] CR2: 00007f2892b59860 CR3: 000000518913e005 CR4: 0000000000760ee0 [3142785.521017][ C28] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [3142785.521017][ C28] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [3142785.521018][ C28] PKRU: 55555554 [3142785.521018][ C28] Call Trace: [3142785.521021][ C28] ? flush_tlb_func_common.isra.8+0x230/0x230 [3142785.521022][ C28] ? flush_tlb_func_common.isra.8+0x230/0x230 [3142785.521023][ C28] on_each_cpu_mask+0x24/0x60 [3142785.521024][ C28] ? x86_configure_nx+0x50/0x50 [3142785.521025][ C28] on_each_cpu_cond_mask+0xa5/0x140 [3142785.521026][ C28] ? flush_tlb_func_common.isra.8+0x92/0x230 [3142785.521027][ C28] flush_tlb_mm_range+0xab/0xe0 [3142785.521029][ C28] ptep_clear_flush+0x56/0x60 [3142785.521030][ C28] wp_page_copy+0x34a/0x7e0 [3142785.521032][ C28] do_wp_page+0x8b/0x660 [3142785.521033][ C28] __handle_mm_fault+0x777/0xef0 [3142785.521035][ C28] handle_mm_fault+0xe2/0x1f0 [3142785.521037][ C28] __do_page_fault+0x226/0x490 [3142785.521038][ C28] do_page_fault+0x31/0x120 [3142785.521039][ C28] async_page_fault+0x3e/0x50 [3142785.521040][ C28] RIP: 0033:0xe148c5 [3142785.521041][ C28] Code: fa 08 0f 84 44 01 00 00 48 83 ff 07 4c 89 c2 76 33 48 83 ef 08 31 d2 48 89 f9 48 c1 e9 03 48 8d 34 cd 08 00 00 00 4d 8b 0c 10 <4c> 89 0c 10 48 83 c2 08 48 39 f2 75 ef 48 f7 d9 48 01 d0 4c 01 c2 [3142785.521042][ C28] RSP: 002b:00007f312c4ef420 EFLAGS: 00010202 [3142785.521043][ C28] RAX: 00007f2892b59860 RBX: 000000000000001f RCX: 0000000000000002 [3142785.521043][ C28] RDX: 0000000000000000 RSI: 0000000000000018 RDI: 0000000000000010 [3142785.521044][ C28] RBP: 00007f312c4ef590 R08: 00007f2dff0501a8 R09: 400008d0ca40bbb7 [3142785.521044][ C28] R10: 0000000000000018 R11: 0000000000003967 R12: 0000000000200000 [3142785.521045][ C28] R13: 00007f28927fb000 R14: 00007f302ca9b410 R15: 00007f312c4ef600 [3142785.523030][ C4] PKRU: 55555554 [3142785.523031][ C4] Call Trace: [3142785.523988][ C29] watchdog: BUG: soft lockup - CPU#29 stuck for 23s! [vertica:283983] [3142785.523988][ C29] Modules linked in: binfmt_misc(E) rpcsec_gss_krb5(E) nfsv4(E) dns_resolver(E) nfs(E) fscache(E) nfsd(E) nfs_acl(E) lockd(E) auth_rpcgss(E) grace(E) nf_conntrack(E) nf_defrag_ipv6(E) nf_defrag_ipv4(E) nls_utf8(E) isofs(E) sunrpc(E) ip_tables(E) ufs(E) hfsplus(E) hfs(E) vfat(E) msdos(E) fat(E) xfs(E) libcrc32c(E) dm_mod(E) ext4(E) mbcache(E) jbd2(E) st(E) sd_mod(E) fuse(E) squashfs(E) zstd_decompress(E) loop(E) brd(E) isst_if_common(E) nfit(E) kvm_intel(E) sr_mod(E) cdrom(E) sg(E) kvm(E) ata_generic(E) irqbypass(E) crct10dif_pclmul(E) crc32_pclmul(E) crc32c_intel(E) ghash_clmulni_intel(E) cirrus(E) drm_kms_helper(E) syscopyarea(E) sysfillrect(E) sysimgblt(E) fb_sys_fops(E) drm(E) ata_piix(E) libata(E) virtio_net(E) joydev(E) serio_raw(E) net_failover(E) virtio_blk(E) virtio_balloon(E) pcspkr(E) i2c_piix4(E) failover(E) qemu_fw_cfg(E) sch_fq_codel(E) [last unloaded: dm_mod] [3142785.524003][ C29] CPU: 29 PID: 283983 Comm: vertica Tainted: G EL 5.4.34-1.el7.gdc.x86_64 #1 [3142785.524004][ C29] Hardware name: RDO OpenStack Compute, BIOS 1.11.0-2.el7 04/01/2014 [3142785.524005][ C29] RIP: 0010:smp_call_function_many+0x221/0x280 [3142785.524006][ C29] Code: 25 77 00 3b 05 04 9f 48 01 0f 83 7b fe ff ff 48 63 d0 49 8b 0f 48 03 0c d5 80 99 19 82 8b 51 18 83 e2 01 74 0a f3 90 8b 51 18 <83> e2 01 75 f6 eb c7 0f b6 4c 24 04 48 83 c4 10 48 89 de 5b 48 89 [3142785.524006][ C29] RSP: 0000:ffffc90001153c58 EFLAGS: 00000202 ORIG_RAX: ffffffffffffff13 [3142785.524007][ C29] RAX: 0000000000000003 RBX: ffffffff8107c1e0 RCX: ffff88d4e74f18e0 [3142785.524008][ C29] RDX: 0000000000000003 RSI: 000000000000001e RDI: ffff88d4da104d80 [3142785.524008][ C29] RBP: ffff88d4e7b6a300 R08: 0000000000000000 R09: 0000000000000000 [3142785.524009][ C29] R10: 0000000000000000 R11: 000000004f421b80 R12: 000000000002ba40 [3142785.524010][ C29] R13: 0000000000000001 R14: 000000000000001e R15: ffff88d4e7b6ba80 [3142785.524010][ C29] FS: 00007f2d677fe700(0000) GS:ffff88d4e7b40000(0000) knlGS:0000000000000000 [3142785.524011][ C29] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [3142785.524012][ C29] CR2: 00007f3509ec32a0 CR3: 000000518913e005 CR4: 0000000000760ee0 [3142785.524013][ C29] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [3142785.524014][ C29] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [3142785.524014][ C29] PKRU: 55555554 [3142785.524015][ C29] Call Trace: [3142785.524016][ C29] ? flush_tlb_func_common.isra.8+0x230/0x230 [3142785.524017][ C29] ? flush_tlb_func_common.isra.8+0x230/0x230 [3142785.524018][ C29] on_each_cpu_mask+0x24/0x60 [3142785.524019][ C29] ? x86_configure_nx+0x50/0x50 [3142785.524020][ C29] on_each_cpu_cond_mask+0xa5/0x140 [3142785.524021][ C29] ? flush_tlb_func_common.isra.8+0x147/0x230 [3142785.524022][ C29] flush_tlb_mm_range+0xab/0xe0 [3142785.524023][ C29] ptep_clear_flush+0x56/0x60 [3142785.524025][ C29] wp_page_copy+0x34a/0x7e0 [3142785.524026][ C29] do_wp_page+0x8b/0x660 [3142785.524028][ C29] __handle_mm_fault+0x777/0xef0 [3142785.524030][ C29] handle_mm_fault+0xe2/0x1f0 [3142785.524031][ C29]