for >1hr as load continued to climb (was unable to login or connect to disks MEH). reboot ok * Checking root filesystem .../dev/sda3 has gone 264 days without being checked, check forced. /dev/sda3: 317637/2003120 files (0.1% non-contiguous), 1859567/8000000 blocks on console {{{ fff810b6cc0>] ? isolate_pages_global+0x0/0x23f [1974761.848939] [] ? autoremove_wake_function+0x0/0x38 [1974761.855916] [] ? kswapd+0x0/0x8bb [1974761.861320] [] ? kthread+0x7d/0x85 [1974761.866855] [] ? kernel_thread_helper+0x4/0x10 [1974761.873471] [] ? kthread+0x0/0x85 [1974761.878929] [] ? kernel_thread_helper+0 [1974761.885546] Code: d0 48 23 02 48 85 c0 49 89 87 40 6d c1 81 74 20 48 8b 05 60 b8 af 00 41 8d b6 f0 00 00 00 48 89 df ff 90 e0 00 00 00 eb 02 f3 90 <83> 3b 00 75 f9 48 8b 7d d0 49 c7 87 00 6d c1 81 00 00 00 00 49 [1974761.906895] Call Trace: [1974761.909772] [] ? flush_tlb_page+0x6e/0x76 [1974761.915939] [] ? page_check_address+0xb0/0x106 [1974761.922545] [] ? ptep_clear_flush_young+0x20/0x26 [1974761.929426] [] ? page_referenced_one+0x65/0xe5 [1974761.935993] [] ? prio_tree_next+0x208/0x22b [1974761.942322] [] ? page_referenced+0x2e0/0x335 [1974761.948736] [] ? mem_cgroup_del_lru_list+0x7b/0x8e [1974761.955654] [] ? isolate_pages_global+0x1c6/0x23f [1974761.962500] [] ? shrink_active_list+0x1 [1974761.969183] [] ? shrink_zone+0x484/0xb86 [1974761.975258] [] ? xfs_perag_get+0x29/0x99 [1974761.981352] [] ? up_read+0x1e/0x36 [1974761.986916] [] ? kswapd+0x61a/0x8bb [1974761.992519] [] ? isolate_pages_global+0x0/0x23f [1974761.999186] [] ? autoremove_wake_function+0x0/0x38 [1974762.006153] [] ? kswapd+0x0/0x8bb [1974762.011603] [] ? kthread+0x7d/0x85 [1974762.017085] [] ? kernel_thread_helper+0x4/0x10 [1974762.023702] [] ? kthread+0x0/0x85 [1974762.029135] [] ? kernel_thread_helper+0x0/0x10 [1974803.417084] BUG: soft lockup - CPU#1 stuck for 61s! [sshd:9057] [1974803.423524] Modules linked in: w83793 hwmon_vid autofs4 e1000e i2c_i801 i2c_core iTCO_wdt ioatdma dca rtc_cmos tg3 libphy e1000 fuse dm_snapshot dm_mirror dm_region_hash dm_log scsi_wait_scan qla2xxx megaraid_sas megaraid_mbox megaraid_mm aacraid 3w_9xxx 3w_xxxx arcmsr aic7xxx aic79xx sr_mod sg ata_piix ahci sata_sis sata_nv sata_via pata_sl82c105 pata_sis [1974803.457702] CPU 1 [1974803.459772] Modules linked in: w83793 hwmon_vid autofs4 e1000e i2c_i801 i2c_core iTCO_wdt ioatdma dca rtc_cmos tg3 libphy e1000 fuse dm_snapshot dm_mirror dm_region_hash dm_log scsi_wait_scan qla2xxx megaraid_sas megaraid_mbox megaraid_mm aacraid 3w_9xxx 3w_xxxx arcmsr aic7xxx aic79xx sr_mod sg ata_piix ahci sata_sis sata_nv sata_via pata_sl82c105 pata_sis [1974803.498881] [1974803.500790] Pid: 9057, comm: sshd Tainted: G D W 2.6.34.4 #3 S7010/empty [1974803.508702] RIP: 0010:[] [] smp_call_function_many+0x18c/0x1a5 [1974803.518532] RSP: 0018:ffff880c18747b20 EFLAGS: 00000202 [1974803.524316] RAX: 0000000000000020 RBX: ffff880c18747b60 RCX: 0000000000000020 [1974803.532100] RDX: 0000000000000000 RSI: 0000000000000020 RDI: 0000000000000000 [1974803.539908] RBP: ffffffff8100334e R08: 0000000000000020 R09: ffffffff81b24210 [1974803.547783] R10: ffffffff812734b7 R11: 0000000000000286 R12: ffff880c18747ae0 [1974803.555589] R13: 0000000000000046 R14: ffff880c18747ae0 R15: 0000000000000000 [1974803.563437] FS: 00007f23edbf76f0(0000) GS:ffff880002620000(0000) knlGS:0000000000000000 [1974803.572291] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b [1974803.578509] CR2: 00007fffec51b310 CR3: 0000000c239bf000 CR4: 00000000000006e0 [1974803.586327] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [1974803.594123] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [1974803.601959] Process sshd (pid: 9057, threadinfo ffff880c18746000, task ffff880c187a0000) [1974803.610778] Stack: [1974803.613202] 01ff880c18747ba0 0000000000000000 ffffffff810b086c 0000000000000040 [1974803.621033] <0> 0000000000000000 ffffffff810b086c 0000000000000003 0000000000000002 [1974803.629469] <0> ffff880c18747b70 ffffffff8106a519 ffff880c18747ba0 ffffffff81040d28 [1974803.638290] Call Trace: [1974803.641166] [] ? drain_local_pages+0x0/0x13 [1974803.647493] [] ? drain_local_pages+0x0/0x13 [1974803.653840] [] ? smp_call_function+0x20/0x24 [1974803.660279] [] ? on_each_cpu+0x18/0x2c [1974803.666173] [] ? isolate_pages_global+0x0/0x23f [1974803.672848] [] ? __alloc_pages_nodemask+0x44d/0x671 [1974803.679886] [] ? up_read+0x1e/0x36 [1974803.685407] [] ? alloc_pages_current+0x96/0x9f [1974803.692013] [] ? new_slab+0x57/0x26b [1974803.697765] [] ? __slab_alloc+0x21c/0x45f [1974803.703941] [] ? copy_process+0xce/0xfc1 [1974803.709999] [] ? copy_process+0xce/0xfc [1974803.716093] [] ? kmem_cache_alloc+0x6a/0xd2 [1974803.722414] [] ? copy_process+0xce/0xfc1 [1974803.728498] [] ? lockdep_init_map+0x9f/0x52e [1974803.734967] [] ? __raw_spin_lock_init+0x31/0x54 [1974803.741721] [] ? do_fork+0x160/0x336 [1974803.747470] [] ? _raw_spin_unlock+0x26/0x2b [1974803.753812] [] ? fd_install+0x30/0x5f [1974803.759592] [] ? fd_install+0x56/0x5f [1974803.765426] [] ? sys_clone+0x23/0x25 [1974803.771122] [] ? stub_clone+0x13/0x20 [1974803.776897] [] ? system_call_fastpath+0 [1974803.783571] Code: 48 c7 c7 d0 21 a0 81 4c 89 60 08 4c 89 25 f9 7c 99 00 e8 59 51 4b 00 0f ae f0 48 89 df ff 15 f0 d1 a2 00 80 7d c7 00 75 04 eb 0a 90 41 f6 44 24 20 01 75 f6 48 83 c4 18 5b 41 5c 41 5d 41 5e [1974803.804920] Call Trace: [1974803.807774] [] ? drain_local_pages+0x0/0x13 [1974803.814131] [] ? drain_local_pages+0x0/0x13 [1974803.820502] [] ? smp_call_function+0x20/0x24 [1974803.826934] [] ? on_each_cpu+0x18/0x2c [1974803.832853] [] ? isolate_pages_global+0x0/0x23f [1974803.839512] [] ? __alloc_pages_nodemask+0x44d/0x671 [1974803.846515] [] ? up_read+0x1e/0x36 [1974803.852016] [] ? alloc_pages_current+0x96/0x9f [1974803.858634] [] ? new_slab+0x57/0x26b [1974803.864361] [] ? __slab_alloc+0x21c/0x45f [1974803.870509] [] ? copy_process+0xce/0xfc1 [1974803.876593] [] ? copy_process+0xce/0xfc1 [1974803.882625] [] ? kmem_cache_alloc+0x6a/0xd2 [1974803.888949] [] ? copy_process+0xce/0xfc1 [1974803.895033] [] ? lockdep_init_map+0x9f/0x52e [1974803.901450] [] ? __raw_spin_lock_init+0x31/0x54 [1974803.908155] [] ? do_fork+0x160/0x336 [1974803.913845] [] ? _raw_spin_unlock+0x26/0x2b [1974803.920173] [] ? fd_install+0x30/0x5f [1974803.925986] [] ? fd_install+0x56/0x5f [1974803.931789] [] ? sys_clone+0x23/0x25 [1974803.937494] [] ? stub_clone+0x13/0x20 [1974803.943260] [] ? system_call_fastpath+0x16/0x1b [1974816.301255] BUG: soft lockup - CPU#3 stuck for 61s! [tcsh:3708] [1974816.307680] Modules linked in: w83793 hwmon_vid autofs4 e1000e i2c_i801 i2c_core iTCO_wdt ioatdma dca rtc_cmos tg3 libphy e1000 fuse dm_snapshot dm_mirror dm_region_hash dm_log scsi_wait_scan qla2xxx megaraid_sas megaraid_mbox megaraid_mm aacraid 3w_9xxx 3w_xxxx arcmsr aic7xxx aic79xx sr_mod sg ata_piix ahci sata_sis sata_nv sata_via pata_sl82c105 pata_sis [1974816.341822] CPU 3 [1974816.343902] Modules linked in: w83793 hwmon_vid autofs4 e1000e i2c_i801 i2c_core iTCO_wdt ioatdma dca rtc_cmos tg3 libphy e1000 fuse dm_snapshot dm_mirror dm_region_hash dm_log scsi_wait_scan qla2xxx megaraid_sas megaraid_mbox megaraid_mm aacraid 3w_9xxx 3w_xxxx arcmsr aic7xxx aic79xx sr_mod sg ata_piix ahci sata_sis sata_nv sata_via pata_sl82c105 pata_sis [1974816.378389] [1974816.380252] Pid: 3708, comm: tcsh Tainted: G D W 2.6.34.4 #3 S7010/empty [1974816.388191] RIP: 0010:[] [] smp_call_function_many+0x194/0x1a5 [1974816.398044] RSP: 0018:ffff880347d45b20 EFLAGS: 00000202 [1974816.403856] RAX: 0000000000000020 RBX: ffff880347d45b60 RCX: 0000000000000020 [1974816.411676] RDX: 0000000000000000 RSI: 0000000000000020 RDI: 0000000000000000 [1974816.419544] RBP: ffffffff8100334e R08: 0000000000000020 R09: ffffffff81b24210 [1974816.427384] R10: ffffffff812734b7 R11: 0000000000000286 R12: ffff880347d45ae0 [1974816.435224] R13: 0000000000000046 R14: ffff880347d45ae0 R15: 0000000000000000 [1974816.443145] FS: 00007ff01d3fc6f0(0000) GS:ffff880002660000(0000) knlGS:0000000000000000 [1974816.452031] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b [1974816.458274] CR2: 000000000065bbf8 CR3: 00000005ea477000 CR4: 00000000000006e0 [1974816.466094] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [1974816.473949] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [1974816.481793] Process tcsh (pid: 3708, threadinfo ffff880347d44000, task ffff8800b3493090) [1974816.490617] Stack: [1974816.493029] 01ff880347d45ba0 0000000000000000 ffffffff810b086c 0000000000000040 [1974816.500856] <0> 0000000000000000 ffffffff810b086c 0000000000000003 0000000000000002 [1974816.509328] <0> ffff880347d45b70 ffffffff8106a519 ffff880347d45ba0 ffffffff81040d28 [1974816.518150] Call Trace: [1974816.521024] [] ? drain_local_pages+0x0/0x13 [1974816.527342] [] ? drain_local_pages+0x0/0x13 [1974816.533698] [] ? smp_call_function+0x20 [1974816.540113] [] ? on_each_cpu+0x18/0x2c [1974816.545979] [] ? isolate_pages_global+0x0/0x23f [1974816.552656] [] ? __alloc_pages_nodemask+0x44d/0x671 [1974816.559678] [] ? alloc_pages_current+0x96/0x9f [1974816.566262] [] ? new_slab+0x57/0x26b [1974816.571963] [] ? __slab_alloc+0x21c/0x45f [1974816.578149] [] ? _raw_spin_unlock+0x26/0x2b [1974816.584470] [] ? copy_process+0xce/0xfc1 [1974816.590516] [] ? copy_process+0xce/0xfc1 [1974816.596568] [] ? kmem_cache_alloc+0x6a/0xd2 [1974816.602896] [] ? copy_process+0xce/0xfc1 [1974816.608938] [] ? handle_mm_fault+0x6e3/0x73e [1974816.615322] [] ? up_read+0x1e/0x36 [1974816.620890] [] ? do_fork+0x160/0x336 [1974816.626597] [] ? _raw_spin_unlock+0x26/0x2b [1974816.632952] [] ? set_close_on_exec+0x30 [1974816.639391] [] ? lockdep_sys_exit_thunk+0x35/0x67 [1974816.646267] [] ? sys_clone+0x23/0x25 [1974816.651960] [] ? stub_clone+0x13/0x20 [1974816.657735] [] ? system_call_fastpath+0x16/0x1b [1974816.664417] Code: 89 60 08 4c 89 25 f9 7c 99 00 e8 59 51 4b 00 0f ae f0 48 89 df ff 15 f0 d1 a2 00 80 7d c7 00 75 04 eb 0a f3 90 41 f6 44 24 20 01 <75> f6 48 83 c4 18 5b 41 5c 41 5d 41 5e 41 5f c9 c3 48 89 f8 55 [1974816.685731] Call Trace: [1974816.688604] [] ? drain_local_pages+0x0/0x13 [1974816.694915] [] ? drain_local_pages+0x0/0x13 [1974816.701253] [] ? smp_call_function+0x20 [1974816.707677] [] ? on_each_cpu+0x18/0x2c [1974816.713595] [] ? isolate_pages_global+0x0/0x23f [1974816.720276] [] ? __alloc_pages_nodemask+0x44d/0x671 [1974816.727326] [] ? alloc_pages_current+0x96/0x9f [1974816.733911] [] ? new_slab+0x57/0x26b [1974816.739629] [] ? __slab_alloc+0x21c/0x45f [1974816.745789] [] ? _raw_spin_unlock+0x26/0x2b [1974816.752126] [] ? copy_process+0xce/0xfc1 [1974816.758210] [] ? copy_process+0xce/0xfc1 [1974816.764268] [] ? kmem_cache_alloc+0x6a/0xd2 [1974816.770588] [] ? copy_process+0xce/0xfc1 [1974816.776683] [] ? handle_mm_fault+0x6e3/0x73e [1974816.783090] [] ? up_read+0x1e/0x36 [1974816.793139] [] ? do_fork+0x160/0x336 [1974816.798879] [] ? _raw_spin_unlock+0x26/0x2b [1974816.805193] [] ? set_close_on_exec+0x30/0x85 [1974816.811596] [] ? lockdep_sys_exit_thunk+0x35/0x67 [1974816.818484] [] ? sys_clone+0x23/0x25 [1974816.824156] [] ? stub_clone+0x13/0x20 [1974816.829965] [] ? system_call_fastpath+0x16/0x1b [1974817.375340] BUG: soft lockup - CPU#5 stuck for 61s! [sshd:25093] [1974817.381844] Modules linked in: w83793 hwmon_vid autofs4 e1000e i2c_i801 i2c_core iTCO_wdt ioatdma dca rtc_cmos tg3 libphy e1000 fuse dm_snapshot dm_mirror dm_region_hash dm_log scsi_wait_scan qla2xxx megaraid_sas megaraid_mbox megaraid_mm aacraid 3w_9xxx 3w_xxxx arcmsr aic7xxx aic79xx sr_mod sg ata_piix ahci sata_sis sata_nv sata_via pata_sl82c105 pata_sis [1974817.415907] CPU 5 [1974817.417977] Modules linked in: w83793 hwmon_vid autofs4 e1000e i2c_i801 i2c_core iTCO_wdt ioatdma dca rtc_cmos tg3 libphy e1000 fuse dm_snapshot dm_mirror dm_region_hash dm_log scsi_wait_scan qla2xxx megaraid_sas megaraid_mbox megaraid_mm aacraid 3w_9xxx 3w_xxxx arcmsr aic7xxx aic79xx sr_mod sg ata_piix ahci sata_sis sata_nv sata_via pata_sl82c105 pata_sis [1974817.452316] [1974817.454198] Pid: 25093, comm: sshd Tainted: G D W 2.6.34.4 #3 S7010/empty [1974817.462186] RIP: 0010:[] [] smp_call_function_many+0x18e/0x1a5 [1974817.471982] RSP: 0018:ffff88035da9d978 EFLAGS: 00000202 [1974817.477757] RAX: 0000000000000020 RBX: ffff88035da9d9b8 RCX: 0000000000000020 [1974817.485569] RDX: 0000000000000000 RSI: 0000000000000020 RDI: 0000000000000000 [1974817.493348] RBP: ffffffff8100334e R08: 0000000000000020 R09: ffffffff81b24210 [1974817.501154] R10: ffffffff812734b7 R11: 0000000000000286 R12: 0000000000000046 [1974817.508959] R13: ffff88035da9d938 R14: 0000000000000000 R15: 0000000000000046 [1974817.516784] FS: 00007f71cc38c6f0(0000) GS:ffff8800026a0000(0000) knlGS:0000000000000000 [1974817.525612] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b [1974817.531848] CR2: 0000000000cc8058 CR3: 00000002d5944000 CR4: 00000000000006e0 [1974817.539654] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [1974817.547473] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [1974817.555314] Process sshd (pid: 25093, threadinfo ffff88035da9c000, task ffff880335739030) [1974817.564229] Stack: [1974817.566636] 01ff88035da9d9f8 0000000000000000 ffffffff810b086c 0000000000000040 [1974817.574440] <0> 0000000000000000 ffffffff810b086c 0000000000000003 0000000000000002 [1974817.582894] <0> ffff88035da9d9c8 ffffffff8106a519 ffff88035da9d9f8 ffffffff81040d28 [1974817.591759] Call Trace: [1974817.594607] [] ? drain_local_pages+0x0/0x13 [1974817.600926] [] ? drain_local_pages+0x0/0x13 [1974817.607246] [] ? smp_call_function+0x20/0x24 [1974817.613635] [] ? on_each_cpu+0x18/0x2c [1974817.619521] [] ? isolate_pages_global+0x0/0x23f [1974817.626185] [] ? __alloc_pages_nodemask+0x44d/0x671 [1974817.633210] [] ? alloc_pages_current+0x96/0x9f [1974817.639810] [] ? new_slab+0x57/0x26b [1974817.645477] [] ? __slab_alloc+0x21c/0x45f [1974817.651625] [] ? sk_stream_alloc_skb+0x38/0xed [1974817.658213] [] ? __kmalloc_node_track_caller+0x9f/0x114 [1974817.665599] [] ? sk_stream_alloc_skb+0x38/0xed [1974817.672206] [] ? __alloc_skb+0x6f/0x168 [1974817.678180] [] ? tcp_current_mss+0x46/0x65 [1974817.684405] [] ? sk_stream_alloc_skb+0x38/0xed [1974817.690994] [] ? tcp_sendmsg+0x1b2/0xa39 [1974817.697027] [] ? sock_aio_write+0xdf/0xef [1974817.703139] [] ? do_sync_write+0xc7/0x1 [1974817.709312] [] ? sigprocmask+0x30/0xc8 [1974817.715160] [] ? security_file_permission+0x11/0x13 [1974817.722193] [] ? vfs_write+0xc1/0x14a [1974817.728007] [] ? sys_write+0x47/0x6e [1974817.733688] [] ? system_call_fastpath+0x16/0x1b [1974817.740344] Code: c7 d0 21 a0 81 4c 89 60 08 4c 89 25 f9 7c 99 00 e8 59 51 4b 00 0f ae f0 48 89 df ff 15 f0 d1 a2 00 80 7d c7 00 75 04 eb 0a f3 90 <41> f6 44 24 20 01 75 f6 48 83 c4 18 5b 41 5c 41 5d 41 5e 41 5f [1974817.761639] Call Trace: [1974817.764520] [] ? drain_local_pages+0x0/0x13 [1974817.770857] [] ? drain_local_pages+0x0/0x13 [1974817.777178] [] ? smp_call_function+0x20/0x24 [1974817.783575] [] ? on_each_cpu+0x18/0x2c [1974817.789460] [] ? isolate_pages_global+0x0/0x23f [1974817.796111] [] ? __alloc_pages_nodemask+0x44d/0x671 [1974817.803156] [] ? alloc_pages_current+0x96/0x9f [1974817.809768] [] ? new_slab+0x57/0x26b [1974817.815491] [] ? __slab_alloc+0x21c/0x45f [1974817.821604] [] ? sk_stream_alloc_skb+0x38/0xed [1974817.828198] [] ? __kmalloc_node_track_caller+0x9f/0x114 [1974817.835591] [] ? sk_stream_alloc_skb+0x38/0xed [1974817.842186] [] ? __alloc_skb+0x6f/0x168 [1974817.848138] [] ? tcp_current_mss+0x46/0x65 [1974817.854363] [] ? sk_stream_alloc_skb+0x38/0xed [1974817.860969] [] ? tcp_sendmsg+0x1b2/0xa39 [1974817.867070] [] ? sock_aio_write+0xdf/0xef [1974817.873192] [] ? do_sync_write+0xc7/0x10d [1974817.879338] [] ? sigprocmask+0x30/0xc8 [1974817.885207] [] ? security_file_permissi [1974817.892246] [] ? vfs_write+0xc1/0x14a [1974817.898043] [] ? sys_write+0x47/0x6e [1974817.903730] [] ? system_call_fastpath+0x16/0x1b [1974827.037187] BUG: soft lockup - CPU#0 stuck for 61s! [kswapd0:608] [1974827.043764] Modules linked in: w83793 hwmon_vid autofs4 e1000e i2c_i801 i2c_core iTCO_wdt ioatdma dca rtc_cmos tg3 libphy e1000 fuse dm_snapshot dm_mirror dm_region_hash dm_log scsi_wait_scan qla2xxx megaraid_sas megaraid_mbox megaraid_mm aacraid 3w_9xxx 3w_xxxx arcmsr aic7xxx aic79xx sr_mod sg ata_piix ahci sata_sis sata_nv sata_via pata_sl82c105 pata_sis [1974827.077972] CPU 0 [1974827.080035] Modules linked in: w83793 hwmon_vid autofs4 e1000e i2c_i801 i2c_core iTCO_wdt ioatdma dca rtc_cmos tg3 libphy e1000 fuse dm_snapshot dm_mirror dm_region_hash dm_log scsi_wait_scan qla2xxx megaraid_sas megaraid_mbox megaraid_mm aacraid 3w_9xxx 3w_xxxx arcmsr aic7xxx aic79xx sr_mod sg ata_piix ahci sata_sis sata_nv sata_via pata_sl82c105 pata_sis [1974827.114575] [1974827.116438] Pid: 608, comm: kswapd0 Tainted: G D W 2.6.34.4 #3 S7010/empty [1974827.124565] RIP: 0010:[] [] flush_tlb_others_ipi+0xb3/0xe6 [1974827.134005] RSP: 0018:ffff880623fe1890 EFLAGS: 00000206 [1974827.139781] RAX: 0000000000000018 RBX: ffff880623fe18d0 RCX: 0000000000000020 [1974827.147620] RDX: ffffffff81c16d40 RSI: 0000000000000020 RDI: 0000000000000000 [1974827.155478] RBP: ffffffff8100334e R08: 0000000000000020 R09: ffffffff81c16d40 [1974827.163318] R10: ffffffff81027fd3 R11: ffffffff812bba1d R12: ffff880623fe1860 [1974827.171168] R13: 0000000000000000 R14: 0000000000000246 R15: 0000000000000001 [1974827.178997] FS: 0000000000000000(0000) GS:ffff880002600000(0000) knlGS:0000000000000000 [1974827.187889] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b [1974827.194105] CR2: 0000000000c7f108 CR3: 0000000001a8a000 CR4: 00000000000006f0 [1974827.201907] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [1974827.209773] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [1974827.217597] Process kswapd0 (pid: 608, threadinfo ffff880623fe0000, task ffff880623f5b090) [1974827.226581] Stack: [1974827.228998] ffff880500000000 ffff880c2363dc68 ffffffff81c16d10 ffff880c2363d940 [1974827.236845] <0> ffff880c2363dc68 00007f02a2136000 ffff880662be36b0 ffff880623fe1af0 [1974827.245299] <0> ffff880623fe1900 ffffffff810281b2 ffffffff810cc352 00000000ffffffff [1974827.254130] Call Trace: [1974827.257011] [] ? flush_tlb_page+0x6e/0x76 [1974827.263168] [] ? page_check_address+0xb0/0x106 [1974827.269789] [] ? ptep_clear_flush_young+0x20/0x26 [1974827.276636] [] ? page_referenced_one+0x65/0xe5 [1974827.283240] [] ? prio_tree_next+0x208/0x22b [1974827.289583] [] ? page_referenced+0x2e0/0x335 [1974827.295981] [] ? mem_cgroup_del_lru_list+0x7b/0x8e [1974827.302942] [] ? isolate_pages_global+0x1c6/0x23f [1974827.309770] [] ? shrink_active_list+0x1d3/0x2a0 [1974827.316434] [] ? shrink_zone+0x484/0xb86 [1974827.322495] [] ? xfs_perag_get+0x29/0x99 [1974827.328588] [] ? up_read+0x1e/0x36 [1974827.334140] [] ? kswapd+0x61a/0x8bb [1974827.339744] [] ? isolate_pages_global+0x0/0x23f [1974827.346449] [] ? autoremove_wake_function+0x0/0x38 [1974827.353390] [] ? kswapd+0x0/0x8bb [1974827.358831] [] ? kthread+0x7d/0x85 [1974827.364346] [] ? kernel_thread_helper+0x4/0x10 [1974827.370972] [] ? kthread+0x0/0x85 [1974827.376394] [] ? kernel_thread_helper+0x0/0x10 [1974827.383004] Code: d0 48 23 02 48 85 c0 49 89 87 40 6d c1 81 74 20 48 8b 05 60 b8 af 00 41 8d b6 f0 00 00 00 48 89 df ff 90 e0 00 00 00 eb 02 f3 90 <83> 3b 00 75 f9 48 8b 7d d0 49 c7 87 00 6d c1 81 00 00 00 00 49 [1974827.404369] Call Trace: [1974827.407248] [] ? flush_tlb_page+0x6e/0x76 [1974827.413406] [] ? page_check_address+0xb0/0x106 [1974827.420029] [] ? ptep_clear_flush_young+0x20/0x26 [1974827.426876] [] ? page_referenced_one+0x65/0xe5 [1974827.433470] [] ? prio_tree_next+0x208/0x22b [1974827.444311] [] ? page_referenced+0x2e0/0x335 [1974827.450741] [] ? mem_cgroup_del_lru_list+0x7b/0x8e [1974827.457685] [] ? isolate_pages_global+0x1c6/0x23f [1974827.464538] [] ? shrink_active_list+0x1d3/0x2a0 [1974827.471187] [] ? shrink_zone+0x484/0xb86 [1974827.477262] [] ? xfs_perag_get+0x29/0x99 [1974827.483312] [] ? up_read+0x1e/0x36 [1974827.488852] [] ? kswapd+0x61a/0x8bb [1974827.494461] [] ? isolate_pages_global+0x0/0x23f [1974827.501165] [] ? autoremove_wake_function+0x0/0x38 [1974827.508090] [] ? kswapd+0x0/0x8bb [1974827.513530] [] ? kthread+0x7d/0x85 [1974827.519081] [] ? kernel_thread_helper+0x4/0x10 [1974827.525620] [] ? kthread+0x0/0x85 [1974827.531079] [] ? kernel_thread_helper+0x0/0x10 [1974868.914516] BUG: soft lockup - CPU#1 stuck for 61s! [sshd:9057] [1974868.920929] Modules linked in: w83793 hwmon_vid autofs4 e1000e i2c_i801 i2c_core iTCO_wdt ioatdma dca rtc_cmos tg3 libphy e1000 fuse dm_snapshot dm_mirror dm_region_hash dm_log scsi_wait_scan qla2xxx megaraid_sas megaraid_mbox megaraid_mm aacraid 3w_9xxx 3w_xxxx arcmsr aic7xxx aic79xx sr_mod sg ata_piix ahci sata_sis sata_nv sata_via pata_sl82c105 pata_sis [1974868.955124] CPU 1 [1974868.957202] Modules linked in: w83793 hwmon_vid autofs4 e1000e i2c_i801 i2c_core iTCO_wdt ioatdma dca rtc_cmos tg3 libphy e1000 fuse dm_snapshot dm_mirror dm_region_hash dm_log scsi_wait_scan qla2xxx megaraid_sas megaraid_mbox megaraid_mm aacraid 3w_9xxx 3w_xxxx arcmsr aic7xxx aic79xx sr_mod sg ata_piix ahci sata_sis sata_nv sata_via pata_sl82c105 pata_sis [1974868.991698] [1974868.993571] Pid: 9057, comm: sshd Tainted: G D W 2.6.34.4 #3 S7010/empty [1974869.001517] RIP: 0010:[] [] smp_call_function_many+0x18e/0x1a5 [1974869.011328] RSP: 0018:ffff880c18747b20 EFLAGS: 00000202 [1974869.017104] RAX: 0000000000000020 RBX: ffff880c18747b60 RCX: 0000000000000020 [1974869.024952] RDX: 0000000000000000 RSI: 0000000000000020 RDI: 0000000000000000 [1974869.032783] RBP: ffffffff8100334e R08: 0000000000000020 R09: ffffffff81b24210 [1974869.040598] R10: ffffffff812734b7 R11: 0000000000000286 R12: ffff880c18747ae0 [1974869.048473] R13: 0000000000000046 R14: ffff880c18747ae0 R15: 0000000000000000 [1974869.056279] FS: 00007f23edbf76f0(0000) GS:ffff880002620000(0000) knlGS:0000000000000000 [1974869.065085] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b [1974869.071345] CR2: 00007fffec51b310 CR3: 0000000c239bf000 CR4: 00000000000006e0 [1974869.079195] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [1974869.087026] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [1974869.094852] Process sshd (pid: 9057, threadinfo ffff880c18746000, task ffff880c187a0000) [1974869.103704] Stack: [1974869.106114] 01ff880c18747ba0 0000000000000000 ffffffff810b086c 0000000000000040 [1974869.113986] <0> 0000000000000000 ffffffff810b086c 0000000000000003 0000000000000002 [1974869.122422] <0> ffff880c18747b70 ffffffff8106a519 ffff880c18747ba0 ffffffff81040d28 [1974869.131261] Call Trace: [1974869.134127] [] ? drain_local_pages+0x0/0x13 [1974869.140456] [] ? drain_local_pages+0x0/0x13 [1974869.146776] [] ? smp_call_function+0x20/0x24 [1974869.153242] [] ? on_each_cpu+0x18/0x2c [1974869.159124] [] ? isolate_pages_global+0x0/0x23f [1974869.165803] [] ? __alloc_pages_nodemask+0x44d/0x671 [1974869.172832] [] ? up_read+0x1e/0x36 [1974869.178360] [] ? alloc_pages_current+0x96/0x9f [1974869.184966] [] ? new_slab+0x57/0x26b [1974869.190667] [] ? __slab_alloc+0x21c/0x45f [1974869.196826] [] ? copy_process+0xce/0xfc1 [1974869.202886] [] ? copy_process+0xce/0xfc1 [1974869.208947] [] ? kmem_cache_alloc+0x6a/0xd2 [1974869.215300] [] ? copy_process+0xce/0xfc1 [1974869.221388] [] ? lockdep_init_map+0x9f/0x52e [1974869.227835] [] ? __raw_spin_lock_init+0x31/0x54 [1974869.234576] [] ? do_fork+0x160/0x336 [1974869.240290] [] ? _raw_spin_unlock+0x26/0x2b [1974869.246617] [] ? fd_install+0x30/0x5f [1974869.252440] [] ? fd_install+0x56/0x5f [1974869.258269] [] ? sys_clone+0x23/0x25 [1974869.263973] [] ? stub_clone+0x13/0x20 [1974869.269781] [] ? system_call_fastpath+0x16/0x1b [1974869.276482] Code: c7 d0 21 a0 81 4c 89 60 08 4c 89 25 f9 7c 99 00 e8 59 51 4b 00 0f ae f0 48 89 df ff 15 f0 d1 a2 00 80 7d c7 00 75 04 eb 0a f3 90 <41> f6 44 24 20 01 75 f6 48 83 c4 18 5b 41 5c 41 5d 41 5e 41 5f [1974869.297728] Call Trace: [1974869.300581] [] ? drain_local_pages+0x0/0x13 [1974869.306928] [] ? drain_local_pages+0x0/0x13 [1974869.313275] [] ? smp_call_function+0x20/0x24 [1974869.319707] [] ? on_each_cpu+0x18/0x2c [1974869.325625] [] ? isolate_pages_global+0x0/0x23f [1974869.332336] [] ? __alloc_pages_nodemask [1974869.339388] [] ? up_read+0x1e/0x36 [1974869.344912] [] ? alloc_pages_current+0x96/0x9f [1974869.351500] [] ? new_slab+0x57/0x26b [1974869.357211] [] ? __slab_alloc+0x21c/0x45f [1974869.363377] [] ? copy_process+0xce/0xfc1 [1974869.369419] [] ? copy_process+0xce/0xfc1 [1974869.375487] [] ? kmem_cache_alloc+0x6a/0xd2 [1974869.381841] [] ? copy_process+0xce/0xfc1 [1974869.387917] [] ? lockdep_init_map+0x9f/0x52e [1974869.394350] [] ? __raw_spin_lock_init+0x31/0x54 [1974869.401024] [] ? do_fork+0x160/0x336 [1974869.406754] [] ? _raw_spin_unlock+0x26/0x2b [1974869.413102] [] ? fd_install+0x30/0x5f [1974869.418879] [] ? fd_install+0x56/0x5f [1974869.424666] [] ? sys_clone+0x23/0x25 [1974869.430404] [] ? stub_clone+0x13/0x20 [1974869.436203] [] ? system_call_fastpath+0x16/0x1b [1974881.798686] BUG: soft lockup - CPU#3 stuck for 61s! [tcsh:3708] [1974881.805085] Modules linked in: w83793 hwmon_vid autofs4 e1000e i2c_i801 i2c_core iTCO_wdt ioatdma dca rtc_cmos tg3 libphy e1000 fuse dm_snapshot dm_mirror dm_region_hash dm_log scsi_wait_scan qla2xxx megaraid_sas megaraid_mbox megaraid_mm aacraid 3w_9xxx 3w_xxxx arcmsr aic7xxx aic79xx sr_mod sg ata_piix ahci sata_sis sata_nv sata_via pata_sl82c105 pata_sis [1974881.839350] CPU 3 [1974881.841420] Modules linked in: w83793 hwmon_vid autofs4 e1000e i2c_i801 i2c_core iTCO_wdt ioatdma dca rtc_cmos tg3 libphy e1000 fuse dm_snapshot dm_mirror dm_region_hash dm_log scsi_wait_scan qla2xxx megaraid_sas megaraid_mbox megaraid_mm aacraid 3w_9xxx 3w_xxxx arcmsr aic7xxx aic79xx sr_mod sg ata_piix ahci sata_sis sata_nv sata_via pata_sl82c105 pata_sis [1974881.876018] [1974881.877892] Pid: 3708, comm: tcsh Tainted: G D W 2.6.34.4 #3 S7010/empty [1974881.885837] RIP: 0010:[] [] smp_call_function_many+0x18e/0x1a5 [1974881.895675] RSP: 0018:ffff880347d45b20 EFLAGS: 00000202 [1974881.901477] RAX: 0000000000000020 RBX: ffff880347d45b60 RCX: 0000000000000020 [1974881.909350] RDX: 0000000000000000 RSI: 0000000000000020 RDI: 0000000000000000 [1974881.917208] RBP: ffffffff8100334e R08: 0000000000000020 R09: ffffffff81b24210 [1974881.925092] R10: ffffffff812734b7 R11: 0000000000000286 R12: ffff880347d45ae0 [1974881.932957] R13: 0000000000000046 R14: ffff880347d45ae0 R15: 0000000000000000 [1974881.940815] FS: 00007ff01d3fc6f0(0000) GS:ffff880002660000(0000) knlGS:0000000000000000 [1974881.949636] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b [1974881.955832] CR2: 000000000065bbf8 CR3: 00000005ea477000 CR4: 00000000000006e0 [1974881.963708] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [1974881.971512] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [1974881.979360] Process tcsh (pid: 3708, threadinfo ffff880347d44000, task ffff8800b3493090) [1974881.988173] Stack: [1974881.990607] 01ff880347d45ba0 0000000000000000 ffffffff810b086c 0000000000000040 [1974881.998462] <0> 0000000000000000 ffffffff810b086c 0000000000000003 0000000000000002 [1974882.006959] <0> ffff880347d45b70 ffffffff8106a519 ffff880347d45ba0 ffffffff81040d28 [1974882.015789] Call Trace: [1974882.018655] [] ? drain_local_pages+0x0/0x13 [1974882.024991] [] ? drain_local_pages+0x0/0x13 [1974882.031321] [] ? smp_call_function+0x20/0x24 [1974882.037701] [] ? on_each_cpu+0x18/0x2c [1974882.043583] [] ? isolate_pages_global+0x0/0x23f [1974882.050303] [] ? __alloc_pages_nodemask+0x44d/0x671 [1974882.057306] [] ? alloc_pages_current+0x96/0x9f [1974882.063919] [] ? new_slab+0x57/0x26b [1974882.069611] [] ? __slab_alloc+0x21c/0x45f [1974882.075780] [] ? _raw_spin_unlock+0x26/0x2b [1974882.082150] [] ? copy_process+0xce/0xfc1 [1974882.088191] [] ? copy_process+0xce/0xfc1 [1974882.094239] [] ? kmem_cache_alloc+0x6a/0xd2 [1974882.100604] [] ? copy_process+0xce/0xfc1 [1974882.106656] [] ? handle_mm_fault+0x6e3/0x73e [1974882.113061] [] ? up_read+0x1e/0x36 [1974882.118564] [] ? do_fork+0x160/0x336 [1974882.124296] [] ? _raw_spin_unlock+0x26/0x2b [1974882.130637] [] ? set_close_on_exec+0x30/0x85 [1974882.137008] [] ? lockdep_sys_exit_thunk+0x35/0x67 [1974882.143842] [] ? sys_clone+0x23/0x25 [1974882.154095] [] ? stub_clone+0x13/0x20 [1974882.159893] [] ? system_call_fastpath+0x16/0x1b [1974882.166593] Code: c7 d0 21 a0 81 4c 89 60 08 4c 89 25 f9 7c 99 00 e8 59 51 4b 00 0f ae f0 48 89 df ff 15 f0 d1 a2 00 80 7d c7 00 75 04 eb 0a f3 90 <41> f6 44 24 20 01 75 f6 48 83 c4 18 5b 41 5c 41 5d 41 5e 41 5f [1974882.187916] Call Trace: [1974882.190806] [] ? drain_local_pages+0x0/0x13 [1974882.197154] [] ? drain_local_pages+0x0/0x13 [1974882.203464] [] ? smp_call_function+0x20/0x24 [1974882.209913] [] ? on_each_cpu+0x18/0x2c [1974882.215772] [] ? isolate_pages_global+0x0/0x23f [1974882.222456] [] ? __alloc_pages_nodemask+0x44d/0x671 [1974882.229521] [] ? alloc_pages_current+0x96/0x9f [1974882.236098] [] ? new_slab+0x57/0x26b [1974882.241790] [] ? __slab_alloc+0x21c/0x45f [1974882.247966] [] ? _raw_spin_unlock+0x26/0x2b [1974882.254302] [] ? copy_process+0xce/0xfc1 [1974882.260359] [] ? copy_process+0xce/0xfc1 [1974882.266409] [] ? kmem_cache_alloc+0x6a/0xd2 [1974882.272776] [] ? copy_process+0xce/0xfc1 [1974882.278845] [] ? handle_mm_fault+0x6e3/0x73e [1974882.285258] [] ? up_read+0x1e/0x36 [1974882.290814] [] ? do_fork+0x160/0x336 [1974882.296486] [] ? _raw_spin_unlock+0x26/0x2b [1974882.302832] [] ? set_close_on_exec+0x30/0x85 [1974882.309280] [] ? lockdep_sys_exit_thunk+0x35/0x67 [1974882.316154] [] ? sys_clone+0x23/0x25 [1974882.321873] [] ? stub_clone+0x13/0x20 [1974882.327671] [] ? system_call_fastpath+0x16/0x1b [1974882.872772] BUG: soft lockup - CPU#5 stuck for 61s! [sshd:25093] [1974882.879273] Modules linked in: w83793 hwmon_vid autofs4 e1000e i2c_i801 i2c_core iTCO_wdt ioatdma dca rtc_cmos tg3 libphy e1000 fuse dm_snapshot dm_mirror dm_region_hash dm_log scsi_wait_scan qla2xxx megaraid_sas megaraid_mbox megaraid_mm aacraid 3w_9xxx 3w_xxxx arcmsr aic7xxx aic79xx sr_mod sg ata_piix ahci sata_sis sata_nv sata_via pata_sl82c105 pata_sis [1974882.913336] CPU 5 [1974882.915401] Modules linked in: w83793 hwmon_vid autofs4 e1000e i2c_i801 i2c_core iTCO_wdt ioatdma dca rtc_cmos tg3 libphy e1000 fuse dm_snapshot dm_mirror dm_region_hash dm_log scsi_wait_scan qla2xxx megaraid_sas megaraid_mbox megaraid_mm aacraid 3w_9xxx 3w_xxxx arcmsr aic7xxx aic79xx sr_mod sg ata_piix ahci sata_sis sata_nv sata_via pata_sl82c105 pata_sis [1974882.949869] [1974882.951743] Pid: 25093, comm: sshd Tainted: G D W 2.6.34.4 #3 S7010/empty [1974882.959745] RIP: 0010:[] [] smp_call_function_many+0x194/0x1a5 [1974882.969518] RSP: 0018:ffff88035da9d978 EFLAGS: 00000202 [1974882.975293] RAX: 0000000000000020 RBX: ffff88035da9d9b8 RCX: 0000000000000020 [1974882.983087] RDX: 0000000000000000 RSI: 0000000000000020 RDI: 0000000000000000 [1974882.990920] RBP: ffffffff8100334e R08: 0000000000000020 R09: ffffffff81b24210 [1974882.998743] R10: ffffffff812734b7 R11: 0000000000000286 R12: 0000000000000046 [1974883.006548] R13: ffff88035da9d938 R14: 0000000000000000 R15: 0000000000000046 [1974883.014355] FS: 00007f71cc38c6f0(0000) GS:ffff8800026a0000(0000) knlGS:0000000000000000 [1974883.023142] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b [1974883.029373] CR2: 0000000000cc8058 CR3: 00000002d5944000 CR4: 00000000000006e0 [1974883.037201] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [1974883.045017] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [1974883.052823] Process sshd (pid: 25093, threadinfo ffff88035da9c000, task ffff880335739030) [1974883.061739] Stack: [1974883.064136] 01ff88035da9d9f8 0000000000000000 ffffffff810b086c 0000000000000040 [1974883.071935] <0> 0000000000000000 ffffffff810b086c 0000000000000002 [1974883.080370] <0> ffff88035da9d9c8 ffffffff8106a519 ffff88035da9d9f8 ffffffff81040d28 [1974883.089261] Call Trace: [1974883.092116] [] ? drain_local_pages+0x0/0x13 [1974883.098385] [] ? drain_local_pages+0x0/0x13 [1974883.104705] [] ? smp_call_function+0x20/0x24 [1974883.111111] [] ? on_each_cpu+0x18/0x2c [1974883.116986] [] ? isolate_pages_global+0x0/0x23f [1974883.123680] [] ? __alloc_pages_nodemask+0x44d/0x671 [1974883.130716] [] ? alloc_pages_current+0x96/0x9f [1974883.137320] [] ? new_slab+0x57/0x26b [1974883.143020] [] ? __slab_alloc+0x21c/0x45f [1974883.149178] [] ? sk_stream_alloc_skb+0x38/0xed [1974883.155784] [] ? __kmalloc_node_track_caller+0x9f/0x114 [1974883.163170] [] ? sk_stream_alloc_skb+0x38/0xed [1974883.169771] [] ? __alloc_skb+0x6f/0x168 [1974883.175750] [] ? tcp_current_mss+0x46/0x65 [1974883.182010] [] ? sk_stream_alloc_skb+0x38/0xed [1974883.188598] [] ? tcp_sendmsg+0x1b2/0xa39 [1974883.194631] [] ? sock_aio_write+0xdf/0xef [1974883.200768] [] ? do_sync_write+0xc7/0x10d [1974883.206916] [] ? sigprocmask+0x30/0xc8 [1974883.212801] [] ? security_file_permission+0x11/0x13 [1974883.219842] [] ? vfs_write+0xc1/0x14a [1974883.225611] [] ? sys_write+0x47/0x6e [1974883.231292] [] ? system_call_fastpath+0x16/0x1b [1974883.237975] Code: 89 60 08 4c 89 25 f9 7c 99 00 e8 59 51 4b 00 0f ae f0 48 89 df ff 15 f0 d1 a2 00 80 7d c7 00 75 04 eb 0a f3 90 41 f6 44 24 20 01 <75> f6 48 83 c4 18 5b 41 5c 41 5d 41 5e 41 5f c9 c3 48 89 f8 55 [1974883.259253] Call Trace: [1974883.262106] [] ? drain_local_pages+0x0/0x13 [1974883.268427] [] ? drain_local_pages+0x0/0x13 [1974883.274760] [] ? smp_call_function+0x20/0x24 [1974883.281171] [] ? on_each_cpu+0x18/0x2c [1974883.287021] [] ? isolate_pages_global+0x0/0x23f [1974883.293671] [] ? __alloc_pages_nodemask+0x44d/0x671 [1974883.300709] [] ? alloc_pages_current+0x96/0x9f [1974883.307295] [] ? new_slab+0x57/0x26b [1974883.312971] [] ? __slab_alloc+0x21c/0x45f [1974883.319101] [] ? sk_stream_alloc_skb+0x38/0xed [1974883.325690] [] ? __kmalloc_node_track_caller+0x9f/0x114 [1974883.333083] [] ? sk_stream_alloc_skb+0x38/0xed [1974883.339642] [] ? __alloc_skb+0x6f/0x168 [1974883.345612] [] ? tcp_current_mss+0x46/0x65 [1974883.351820] [] ? sk_stream_alloc_skb+0x38/0xed [1974883.358402] [] ? tcp_sendmsg+0x1b2/0xa39 [1974883.364440] [] ? sock_aio_write+0xdf/0xef [1974883.370572] [] ? do_sync_write+0xc7/0x1 [1974883.376703] [] ? sigprocmask+0x30/0xc8 [1974883.382598] [] ? security_file_permission+0x11/0x13 [1974883.389627] [] ? vfs_write+0xc1/0x14a [1974883.395387] [] ? sys_write+0x47/0x6e [1974883.401067] [] ? system_call_fastpath+0x16/0x1b [1974892.537943] BUG: soft lockup - CPU#0 stuck for 61s! [kswapd0:608] [1974892.544481] Modules linked in: w83793 hwmon_vid autofs4 e1000e i2c_i801 i2c_core iTCO_wdt ioatdma dca rtc_cmos tg3 libphy e1000 fuse dm_snapshot dm_mirror dm_region_hash dm_log scsi_wait_scan qla2xxx megaraid_sas megaraid_mbox megaraid_mm aacraid 3w_9xxx 3w_xxxx arcmsr aic7xxx aic79xx sr_mod sg ata_piix ahci sata_sis sata_nv sata_via pata_sl82c105 pata_sis [1974892.578638] CPU 0 [1974892.580682] Modules linked in: w83793 hwmon_vid autofs4 e1000e i2c_i801 i2c_core iTCO_wdt ioatdma dca rtc_cmos tg3 libphy e1000 fuse dm_snapshot dm_mirror dm_region_hash dm_log scsi_wait_scan qla2xxx megaraid_sas megaraid_mbox megaraid_mm aacraid 3w_9xxx 3w_xxxx arcmsr aic7xxx aic79xx sr_mod sg ata_piix ahci sata_sis sata_nv sata_via pata_sl82c105 pata_sis [1974892.615187] [1974892.617069] Pid: 608, comm: kswapd0 Tainted: G D W 2.6.34.4 #3 S7010/empty [1974892.625188] RIP: 0010:[] [] flush_tlb_others_ipi+0xb3/0xe6 [1974892.634661] RSP: 0018:ffff880623fe1890 EFLAGS: 00000206 [1974892.640447] RAX: 0000000000000018 RBX: ffff880623fe18d0 RCX: 0000000000000020 [1974892.648319] RDX: ffffffff81c16d40 RSI: 0000000000000020 RDI: 0000000000000000 [1974892.656134] RBP: ffffffff8100334e R08: 0000000000000020 R09: ffffffff81c16d40 [1974892.663991] R10: ffffffff81027fd3 R11: ffffffff812bba1d R12: ffff880623fe1860 [1974892.671813] R13: 0000000000000000 R14: 0000000000000246 R15: 0000000000000001 [1974892.679681] FS: 0000000000000000(0000) GS:ffff880002600000(0000) knlGS:0000000000000000 [1974892.688483] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b [1974892.694725] CR2: 0000000000c7f108 CR3: 0000000001a8a000 CR4: 00000000000006f0 [1974892.702586] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [1974892.710462] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [1974892.718277] Process kswapd0 (pid: 608, threadinfo ffff880623fe0000, task ffff880623f5b090) [1974892.727263] Stack: [1974892.729697] ffff880500000000 ffff880c2363dc68 ffffffff81c16d10 ffff880c2363d940 [1974892.737501] <0> ffff880c2363dc68 00007f02a2136000 ffff880662be36b0 ffff880623fe1af0 [1974892.746007] <0> ffff880623fe1900 ffffffff810281b2 ffffffff810cc352 00000000ffffffff [1974892.754828] Call Trace: [1974892.757709] [] ? flush_tlb_page+0x6e/0x76 [1974892.763874] [] ? page_check_address+0xb0/0x106 [1974892.770452] [] ? ptep_clear_flush_young+0x20/0x26 [1974892.777327] [] ? page_referenced_one+0x65/0xe5 [1974892.783912] [] ? prio_tree_next+0x208/0x22b [1974892.790212] [] ? page_referenced+0x2e0/0x335 [1974892.796636] [] ? mem_cgroup_del_lru_list+0x7b/0x8e [1974892.808101] [] ? isolate_pages_global+0x1c6/0x23f [1974892.814930] [] ? shrink_active_list+0x1d3/0x2a0 [1974892.821602] [] ? shrink_zone+0x484/0xb86 [1974892.827688] [] ? xfs_perag_get+0x29/0x99 [1974892.833774] [] ? up_read+0x1e/0x36 [1974892.839311] [] ? kswapd+0x61a/0x8bb [1974892.844939] [] ? isolate_pages_global+0x0/0x23f [1974892.851634] [] ? autoremove_wake_function+0x0/0x38 [1974892.858608] [] ? kswapd+0x0/0x8bb [1974892.864043] [] ? kthread+0x7d/0x85 [1974892.869627] [] ? kernel_thread_helper+0x4/0x10 [1974892.876201] [] ? kthread+0x0/0x85 [1974892.881632] [] ? kernel_thread_helper+0x0/0x10 [1974892.888222] Code: d0 48 23 02 48 85 c0 49 89 87 40 6d c1 81 74 20 48 8b 05 60 b8 af 00 41 8d b6 f0 00 00 00 48 89 df ff 90 e0 00 00 00 eb 02 f3 90 <83> 3b 00 75 f9 48 8b 7d d0 49 c7 87 00 6d c1 81 00 00 00 00 49 [1974892.909580] Call Trace: [1974892.912467] [] ? flush_tlb_page+0x6e/0x76 [1974892.918643] [] ? page_check_address+0xb0/0x106 [1974892.925275] [] ? ptep_clear_flush_young+0x20/0x26 [1974892.932157] [] ? page_referenced_one+0x65/0xe5 [1974892.938726] [] ? prio_tree_next+0x208/0x22b [1974892.945079] [] ? page_referenced+0x2e0/0x335 [1974892.951486] [] ? mem_cgroup_del_lru_list+0x7b/0x8e [1974892.958479] [] ? isolate_pages_global+0x1c6/0x23f [1974892.965394] [] ? shrink_active_list+0x1d3/0x2a0 [1974892.972119] [] ? shrink_zone+0x484/0xb86 [1974892.978177] [] ? xfs_perag_get+0x29/0x99 [1974892.984253] [] ? up_read+0x1e/0x36 [1974892.989783] [] ? kswapd+0x61a/0x8bb [1974892.995428] [] ? isolate_pages_global+0x0/0x23f [1974893.002158] [] ? autoremove_wake_function+0x0/0x38 [1974893.009091] [] ? kswapd+0x0/0x8bb [1974893.014540] [] ? kthread+0x7d/0x85 [1974893.020056] [] ? kernel_thread_helper+0x4/0x10 [1974893.026664] [] ? kthread+0x0/0x85 [1974893.032115] [] ? kernel_thread_helper+0x0/0x10 [1974934.411948] BUG: soft lockup - CPU#1 stuck for 61s! [sshd:9057] [1974934.418363] Modules linked in: w83793 hwmon_vid autofs4 e1000e i2c_i801 i2c_core iTCO_wdt ioatdma dca rtc_cmos tg3 libphy e1000 fuse dm_snapshot dm_mirror dm_region_hash dm_log scsi_wait_scan qla2xxx megaraid_sas megaraid_mbox megaraid_mm aacraid 3w_9xxx 3w_xxxx arcmsr aic7xxx aic79xx sr_mod sg ata_piix ahci sata_sis sata_nv sata_via pata_sl82c105 pata_sis [1974934.452495] CPU 1 [1974934.454558] Modules linked in: w83793 hwmon_vid autofs4 e1000e i2c_i801 i2c_core iTCO_wdt ioatdma dca rtc_cmos tg3 libphy e1000 fuse dm_snapshot dm_mirror dm_region_hash dm_log scsi_wait_scan qla2xxx megaraid_sas megaraid_mbox megaraid_mm aacraid 3w_9xxx 3w_xxxx arcmsr aic7xxx aic79xx sr_mod sg ata_piix ahci sata_sis sata_nv sata_via pata_sl82c105 pata_sis [1974934.489025] [1974934.490918] Pid: 9057, comm: sshd Tainted: G D W 2.6.34.4 #3 S7010/empty [1974934.498853] RIP: 0010:[] [] smp_call_function_many+0x18e/0x1a5 [1974934.508638] RSP: 0018:ffff880c18747b20 EFLAGS: 00000202 [1974934.514425] RAX: 0000000000000020 RBX: ffff880c18747b60 RCX: 0000000000000020 [1974934.522287] RDX: 0000000000000000 RSI: 0000000000000020 RDI: 0000000000000000 [1974934.530121] RBP: ffffffff8100334e R08: 0000000000000020 R09: ffffffff81b24210 [1974934.537970] R10: ffffffff812734b7 R11: 0000000000000286 R12: ffff880c18747ae0 [1974934.545776] R13: 0000000000000046 R14: ffff880c18747ae0 R15: 0000000000000000 [1974934.553645] FS: 00007f23edbf76f0(0000) GS:ffff880002620000(0000) knlGS:0000000000000000 [1974934.562455] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b [1974934.568670] CR2: 00007fffec51b310 CR3: 0000000c239bf000 CR4: 00000000000006e0 [1974934.576507] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [1974934.584373] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [1974934.592223] Process sshd (pid: 9057, threadinfo ffff880c18746000, task ffff880c187a0000) [1974934.601068] Stack: [1974934.603486] 01ff880c18747ba0 0000000000000000 ffffffff810b086c 0000000000000040 [1974934.611321] <0> 0000000000000000 ffffffff810b086c 0000000000000003 0000000000000002 [1974934.619802] <0> ffff880c18747b70 ffffffff8106a519 ffff880c18747ba0 ffffffff81040d28 [1974934.628624] Call Trace: [1974934.631507] [] ? drain_local_pages+0x0/0x13 [1974934.637887] [] ? drain_local_pages+0x0/0x13 [1974934.644210] [] ? smp_call_function+0x20/0x24 [1974934.650658] [] ? on_each_cpu+0x18/0x2c [1974934.656542] [] ? isolate_pages_global+0 [1974934.663260] [] ? __alloc_pages_nodemask+0x44d/0x671 [1974934.670281] [] ? up_read+0x1e/0x36 [1974934.675833] [] ? alloc_pages_current+0x96/0x9f [1974934.682449] [] ? new_slab+0x57/0x26b [1974934.688124] [] ? __slab_alloc+0x21c/0x45f [1974934.694309] [] ? copy_process+0xce/0xfc1 [1974934.700384] [] ? copy_process+0xce/0xfc1 [1974934.706443] [] ? kmem_cache_alloc+0x6a/0xd2 [1974934.712781] [] ? copy_process+0xce/0xfc1 [1974934.718831] [] ? lockdep_init_map+0x9f/0x52e [1974934.725250] [] ? __raw_spin_lock_init+0x31/0x54 [1974934.731939] [] ? do_fork+0x160/0x336 [1974934.737652] [] ? _raw_spin_unlock+0x26/0x2b [1974934.744006] [] ? fd_install+0x30/0x5f [1974934.749846] [] ? fd_install+0x56/0x5f [1974934.755616] [] ? sys_clone+0x23/0x25 [1974934.761328] [] ? stub_clone+0x13/0x20 [1974934.767139] [] ? system_call_fastpath+0x16/0x1b [1974934.773811] Code: c7 d0 21 a0 81 4c 89 60 08 4c 89 25 f9 7c 99 00 e8 59 51 4b 00 0f ae f0 48 89 df ff 15 f0 d1 a2 00 80 7d c7 00 75 04 eb 0a f3 90 <41> f6 44 24 20 01 75 f6 48 83 c4 18 5b 41 5c 41 5d 41 5e 41 5f [1974934.795184] Call Trace: [1974934.798051] [] ? drain_local_pages+0x0/0x13 [1974934.804403] [] ? drain_local_pages+0x0/0x13 [1974934.810716] [] ? smp_call_function+0x20/0x24 [1974934.817113] [] ? on_each_cpu+0x18/0x2c [1974934.823005] [] ? isolate_pages_global+0x0/0x23f [1974934.829691] [] ? __alloc_pages_nodemask+0x44d/0x671 [1974934.836737] [] ? up_read+0x1e/0x36 [1974934.842231] [] ? alloc_pages_current+0x96/0x9f [1974934.848863] [] ? new_slab+0x57/0x26b [1974934.854572] [] ? __slab_alloc+0x21c/0x45f [1974934.860714] [] ? copy_process+0xce/0xfc1 [1974934.866739] [] ? copy_process+0xce/0xfc1 [1974934.872831] [] ? kmem_cache_alloc+0x6a/0xd2 [1974934.879187] [] ? copy_process+0xce/0xfc1 [1974934.885280] [] ? lockdep_init_map+0x9f/0x52e [1974934.891680] [] ? __raw_spin_lock_init+0x31/0x54 [1974934.898377] [] ? do_fork+0x160/0x336 [1974934.904051] [] ? _raw_spin_unlock+0x26/0x2b [1974934.910401] [] ? fd_install+0x30/0x5f [1974934.916179] [] ? fd_install+0x56/0x5f [1974934.922010] [] ? sys_clone+0x23/0x25 [1974934.927724] [] ? stub_clone+0x13/0x20 [1974934.933558] [] ? system_call_fastpath+0x16/0x1b [1974947.296119] BUG: soft lockup - CPU#3 stuck for 61s! [tcsh:3708] [1974947.302544] Modules linked in: w83793 hwmon_vid autofs4 e1000e i2c_i801 i2c_core iTCO_wdt ioatdma dca rtc_cmos tg3 libphy e1000 fuse dm_snapshot dm_mirror dm_region_hash dm_log scsi_wait_scan qla2xxx megaraid_sas megaraid_mbox megaraid_mm aacraid 3w_9xxx 3w_xxxx arcmsr aic7xxx aic79xx sr_mod sg ata_piix ahci sata_sis sata_nv sata_via pata_sl82c105 pata_sis [1974947.336710] CPU 3 [1974947.338756] Modules linked in: w83793 hwmon_vid autofs4 e1000e i2c_i801 i2c_core iTCO_wdt ioatdma dca rtc_cmos tg3 libphy e1000 fuse dm_snapshot dm_mirror dm_region_hash dm_log scsi_wait_scan qla2xxx megaraid_sas megaraid_mbox megaraid_mm aacraid 3w_9xxx 3w_xxxx arcmsr aic7xxx aic79xx sr_mod sg ata_piix ahci sata_sis sata_nv sata_via pata_sl82c105 pata_sis [1974947.373425] [1974947.375315] Pid: 3708, comm: tcsh Tainted: G D W 2.6.34.4 #3 S7010/empty [1974947.383192] RIP: 0010:[] [] smp_call_function_many+0x194/0x1a5 [1974947.392994] RSP: 0018:ffff880347d45b20 EFLAGS: 00000202 [1974947.398815] RAX: 0000000000000020 RBX: ffff880347d45b60 RCX: 0000000000000020 [1974947.406711] RDX: 0000000000000000 RSI: 0000000000000020 RDI: 0000000000000000 [1974947.414527] RBP: ffffffff8100334e R08: 0000000000000020 R09: ffffffff81b24210 [1974947.422412] R10: ffffffff812734b7 R11: 0000000000000286 R12: ffff880347d45ae0 [1974947.430268] R13: 0000000000000046 R14: ffff880347d45ae0 R15: 0000000000000000 [1974947.438127] FS: 00007ff01d3fc6f0(0000) GS:ffff880002660000(0000) knlGS:0000000000000000 [1974947.446938] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b [1974947.453162] CR2: 000000000065bbf8 CR3: 00000005ea477000 CR4: 00000000000006e0 [1974947.461026] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [1974947.468874] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [1974947.476741] Process tcsh (pid: 3708, threadinfo ffff880347d44000, task ffff8800b3493090) [1974947.485593] Stack: [1974947.488013] 01ff880347d45ba0 0000000000000000 ffffffff810b086c 0000000000000040 [1974947.495858] <0> 0000000000000000 ffffffff810b086c 0000000000000003 0000000000000002 [1974947.504297] <0> ffff880347d45b70 ffffffff8106a519 ffff880347d45ba0 ffffffff81040d28 [1974947.517683] Call Trace: [1974947.520555] [] ? drain_local_pages+0x0/0x13 [1974947.526884] [] ? drain_local_pages+0x0/0x13 [1974947.533230] [] ? smp_call_function+0x20/0x24 [1974947.539636] [] ? on_each_cpu+0x18/0x2c [1974947.545541] [] ? isolate_pages_global+0x0/0x23f [1974947.552199] [] ? __alloc_pages_nodemask+0x44d/0x671 [1974947.559210] [] ? alloc_pages_current+0x96/0x9f [1974947.565811] [] ? new_slab+0x57/0x26b [1974947.571511] [] ? __slab_alloc+0x21c/0x45f [1974947.577722] [] ? _raw_spin_unlock+0x26/0x2b [1974947.584083] [] ? copy_process+0xce/0xfc1 [1974947.590178] [] ? copy_process+0xce/0xfc1 [1974947.596218] [] ? kmem_cache_alloc+0x6a/0xd2 [1974947.602591] [] ? copy_process+0xce/0xfc1 [1974947.608659] [] ? handle_mm_fault+0x6e3/0x73e [1974947.615075] [] ? up_read+0x1e/0x36 [1974947.620594] [] ? do_fork+0x160/0x336 [1974947.626275] [] ? _raw_spin_unlock+0x26/0x2b [1974947.632640] [] ? set_close_on_exec+0x30/0x85 [1974947.639095] [] ? lockdep_sys_exit_thunk+0x35/0x67 [1974947.645954] [] ? sys_clone+0x23/0x25 [1974947.651674] [] ? stub_clone+0x13/0x20 [1974947.657455] [] ? system_call_fastpath+0x16/0x1b [1974947.664122] Code: 89 60 08 4c 89 25 f9 7c 99 00 e8 59 51 4b 00 0f ae f0 48 89 df ff 15 f0 d1 a2 00 80 7d c7 00 75 04 eb 0a f3 90 41 f6 44 24 20 01 <75> f6 48 83 c4 18 5b 41 5c 41 5d 41 5e 41 5f c9 c3 48 89 f8 55 [1974947.685564] Call Trace: [1974947.688444] [] ? drain_local_pages+0x0/0x13 [1974947.694826] [] ? drain_local_pages+0x0/0x13 [1974947.701171] [] ? smp_call_function+0x20/0x24 [1974947.707585] [] ? on_each_cpu+0x18/0x2c [1974947.713446] [] ? isolate_pages_global+0x0/0x23f [1974947.720173] [] ? __alloc_pages_nodemask [1974947.727210] [] ? alloc_pages_current+0x96/0x9f [1974947.733796] [] ? new_slab+0x57/0x26b [1974947.739533] [] ? __slab_alloc+0x21c/0x45f [1974947.745733] [] ? _raw_spin_unlock+0x26/0x2b [1974947.752087] [] ? copy_process+0xce/0xfc1 [1974947.758130] [] ? copy_process+0xce/0xfc1 [1974947.764188] [] ? kmem_cache_alloc+0x6a/0xd2 [1974947.770509] [] ? copy_process+0xce/0xfc1 [1974947.776576] [] ? handle_mm_fault+0x6e3/0x73e [1974947.783035] [] ? up_read+0x1e/0x36 [1974947.788599] [] ? do_fork+0x160/0x336 [1974947.794348] [] ? _raw_spin_unlock+0x26/0x2b [1974947.800679] [] ? set_close_on_exec+0x30/0x85 [1974947.807083] [] ? lockdep_sys_exit_thunk+0x35/0x67 [1974947.813959] [] ? sys_clone+0x23/0x25 [1974947.819676] [] ? stub_clone+0x13/0x20 [1974947.825485] [] ? system_call_fastpath+0x16/0x1b [1974948.370203] BUG: soft lockup - CPU#5 stuck for 61s! [sshd:25093] [1974948.376688] Modules linked in: w83793 hwmon_vid autofs4 e1000e i2c_i801 i2c_core iTCO_wdt ioatdma dca rtc_cmos tg3 libphy e1000 fuse dm_snapshot dm_mirror dm_region_hash dm_log scsi_wait_scan qla2xxx megaraid_sas megaraid_mbox megaraid_mm aacraid 3w_9xxx 3w_xxxx arcmsr aic7xxx aic79xx sr_mod sg ata_piix ahci sata_sis sata_nv sata_via pata_sl82c105 pata_sis [1974948.410735] CPU 5 [1974948.412797] Modules linked in: w83793 hwmon_vid autofs4 e1000e i2c_i801 i2c_core iTCO_wdt ioatdma dca rtc_cmos tg3 libphy e1000 fuse dm_snapshot dm_mirror dm_region_hash dm_log scsi_wait_scan qla2xxx megaraid_sas megaraid_mbox megaraid_mm aacraid 3w_9xxx 3w_xxxx arcmsr aic7xxx aic79xx sr_mod sg ata_piix ahci sata_sis sata_nv sata_via pata_sl82c105 pata_sis [1974948.447310] [1974948.449183] Pid: 25093, comm: sshd Tainted: G D W 2.6.34.4 #3 S7010/empty [1974948.457180] RIP: 0010:[] [] smp_call_function_many+0x194/0x1a5 [1974948.466931] RSP: 0018:ffff88035da9d978 EFLAGS: 00000202 [1974948.472709] RAX: 0000000000000020 RBX: ffff88035da9d9b8 RCX: 0000000000000020 [1974948.480556] RDX: 0000000000000000 RSI: 0000000000000020 RDI: 0000000000000000 [1974948.488376] RBP: ffffffff8100334e R08: 0000000000000020 R09: ffffffff81b24210 [1974948.496193] R10: ffffffff812734b7 R11: 0000000000000286 R12: 0000000000000046 [1974948.504007] R13: ffff88035da9d938 R14: 0000000000000000 R15: 0000000000000046 [1974948.511812] FS: 00007f71cc38c6f0(0000) GS:ffff8800026a0000(0000) knlGS:0000000000000000 [1974948.520640] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b [1974948.526873] CR2: 0000000000cc8058 CR3: 00000002d5944000 CR4: 00000000000006e0 [1974948.534736] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [1974948.542587] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [1974948.550471] Process sshd (pid: 25093, threadinfo ffff88035da9c000, task ffff880335739030) [1974948.559361] Stack: [1974948.561769] 01ff88035da9d9f8 0000000000000000 ffffffff810b086c 0000000000000040 [1974948.569571] <0> 0000000000000000 ffffffff810b086c 0000000000000003 0000000000000002 [1974948.577991] <0> ffff88035da9d9c8 ffffffff8106a519 ffff88035da9d9f8 ffffffff81040d28 [1974948.586813] Call Trace: [1974948.589679] [] ? drain_local_pages+0x0/0x13 [1974948.595987] [] ? drain_local_pages+0x0/0x13 [1974948.602310] [] ? smp_call_function+0x20/0x24 [1974948.608698] [] ? on_each_cpu+0x18/0x2c [1974948.614556] [] ? isolate_pages_global+0x0/0x23f [1974948.621230] [] ? __alloc_pages_nodemask+0x44d/0x671 [1974948.628261] [] ? alloc_pages_current+0x96/0x9f [1974948.634844] [] ? new_slab+0x57/0x26b [1974948.640523] [] ? __slab_alloc+0x21c/0x45f [1974948.646662] [] ? sk_stream_alloc_skb+0x38/0xed [1974948.653259] [] ? __kmalloc_node_track_caller+0x9f/0x114 [1974948.660599] [] ? sk_stream_alloc_skb+0x38/0xed [1974948.667169] [] ? __alloc_skb+0x6f/0x168 [1974948.673195] [] ? tcp_current_mss+0x46/0x65 [1974948.679407] [] ? sk_stream_alloc_skb+0x38/0xed [1974948.685977] [] ? tcp_sendmsg+0x1b2/0xa39 [1974948.691994] [] ? sock_aio_write+0xdf/0xef [1974948.698158] [] ? do_sync_write+0xc7/0x10d [1974948.704268] [] ? sigprocmask+0x30/0xc8 [1974948.710130] [] ? security_file_permission+0x11/0x13 [1974948.717134] [] ? vfs_write+0xc1/0x14a [1974948.722929] [] ? sys_write+0x47/0x6e [1974948.728627] [] ? system_call_fastpath+0x16/0x1b [1974948.735302] Code: 89 60 08 4c 89 25 f9 7c 99 00 e8 59 51 4b 00 0f ae f0 48 89 df ff 15 f0 d1 a2 00 80 7d c7 00 75 04 eb 0a f3 90 41 f6 44 48 83 c4 18 5b 41 5c 41 5d 41 5e 41 5f c9 c3 48 89 f8 55 [1974948.756649] Call Trace: [1974948.759504] [] ? drain_local_pages+0x0/0x13 [1974948.765808] [] ? drain_local_pages+0x0/ [1974948.772127] [] ? smp_call_function+0x20/0x24 [1974948.778527] [] ? on_each_cpu+0x18/0x2c [1974948.784408] [] ? isolate_pages_global+0x0/0x23f [1974948.791068] [] ? __alloc_pages_nodemask+0x44d/0x671 [1974948.798059] [] ? alloc_pages_current+0x96/0x9f [1974948.804665] [] ? new_slab+0x57/0x26b [1974948.810374] [] ? __slab_alloc+0x21c/0x4 [1974948.816492] [] ? sk_stream_alloc_skb+0x38/0xed [1974948.823060] [] ? __kmalloc_node_track_caller+0x9f/0x114 [1974948.830446] [] ? sk_stream_alloc_skb+0x38/0xed [1974948.837049] [] ? __alloc_skb+0x6f/0x168 [1974948.843011] [] ? tcp_current_mss+0x46/0x65 [1974948.849260] [] ? sk_stream_alloc_skb+0x38/0xed [1974948.855815] [] ? tcp_sendmsg+0x1b2/0xa39 [1974948.861855] [] ? sock_aio_write+0xdf/0xef [1974948.868021] [] ? do_sync_write+0xc7/0x10d [1974948.874174] [] ? sigprocmask+0x30/0xc8 [1974948.880079] [] ? security_file_permission+0x11/0x13 [1974948.887118] [] ? vfs_write+0xc1/0x14a [1974948.892895] [] ? sys_write+0x47/0x6e [1974948.898602] [] ? system_call_fastpath+0x16/0x1b [1974958.035375] BUG: soft lockup - CPU#0 stuck for 61s! [kswapd0:608] [1974958.041964] Modules linked in: w83793 hwmon_vid autofs4 e1000e i2c_i801 i2c_core iTCO_wdt ioatdma dca rtc_cmos tg3 libphy e1000 fuse dm_snapshot dm_mirror dm_region_hash dm_log scsi_wait_scan qla2xxx megaraid_sas megaraid_mbox megaraid_mm aacraid 3w_9xxx 3w_xxxx arcmsr aic7xxx aic79xx sr_mod sg ata_piix ahci sata_sis sata_nv sata_via pata_sl82c105 pata_sis [1974958.076122] CPU 0 [1974958.078176] Modules linked in: w83793 hwmon_vid autofs4 e1000e i2c_i801 i2c_core iTCO_wdt ioatdma dca rtc_cmos tg3 libphy e1000 fuse dm_snapshot dm_mirror dm_region_hash dm_log scsi_wait_scan qla2xxx megaraid_sas megaraid_mbox megaraid_mm aacraid 3w_9xxx 3w_xxxx arcmsr aic7xxx aic79xx sr_mod sg ata_piix ahci sata_sis sata_nv sata_via pata_sl82c105 pata_sis [1974958.112676] [1974958.114568] Pid: 608, comm: kswapd0 Tainted: G D W 2.6.34.4 #3 S7010/empty [1974958.122670] RIP: 0010:[] [] flush_tlb_others_ipi+0xb1/0xe6 [1974958.132162] RSP: 0018:ffff880623fe1890 EFLAGS: 00000206 [1974958.137974] RAX: 0000000000000018 RBX: ffff880623fe18d0 RCX: 0000000000000020 [1974958.145820] RDX: ffffffff81c16d40 RSI: 0000000000000020 RDI: 0000000000000000 [1974958.153650] RBP: ffffffff8100334e R08: 0000000000000020 R09: ffffffff81c16d40 [1974958.161492] R10: ffffffff81027fd3 R11: ffffffff812bba1d R12: ffff880623fe1860 [1974958.169367] R13: 0000000000000000 R14: 0000000000000246 R15: 0000000000000001 [1974958.177190] FS: 0000000000000000(0000) GS:ffff880002600000(0000) knlGS:0000000000000000 [1974958.186013] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b [1974958.192280] CR2: 0000000000c7f108 CR3: 0000000001a8a000 CR4: 00000000000006f0 [1974958.204701] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [1974958.212564] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [1974958.220403] Process kswapd0 (pid: 608, threadinfo ffff880623fe0000, task ffff880623f5b090) [1974958.229405] Stack: [1974958.231805] ffff880500000000 ffff880c2363dc68 ffffffff81c16d10 ffff880c2363d940 [1974958.239600] <0> ffff880c2363dc68 00007f02a2136000 ffff880662be36b0 ffff880623fe1af0 [1974958.248088] <0> ffff880623fe1900 ffffffff810281b2 ffffffff810cc352 00000000ffffffff [1974958.256919] Call Trace: [1974958.259782] [] ? flush_tlb_page+0x6e/0x76 [1974958.265966] [] ? page_check_address+0xb0/0x106 [1974958.272552] [] ? ptep_clear_flush_young+0x20/0x26 [1974958.279417] [] ? page_referenced_one+0x65/0xe5 [1974958.286021] [] ? prio_tree_next+0x208/0x22b [1974958.292364] [] ? page_referenced+0x2e0/0x335 [1974958.298772] [] ? mem_cgroup_del_lru_list+0x7b/0x8e [1974958.305730] [] ? isolate_pages_global+0x1c6/0x23f [1974958.312558] [] ? shrink_active_list+0x1d3/0x2a0 [1974958.319266] [] ? shrink_zone+0x484/0xb86 [1974958.325308] [] ? xfs_perag_get+0x29/0x99 [1974958.331352] [] ? up_read+0x1e/0x36 [1974958.336862] [] ? kswapd+0x61a/0x8bb [1974958.342484] [] ? isolate_pages_global+0x0/0x23f [1974958.349221] [] ? autoremove_wake_function+0x0/0x38 [1974958.356162] [] ? kswapd+0x0/0x8bb [1974958.361621] [] ? kthread+0x7d/0x85 [1974958.367162] [] ? kernel_thread_helper+0x4/0x10 [1974958.373735] [] ? kthread+0x0/0x85 [1974958.379167] [] ? kernel_thread_helper+0x0/0x10 [1974958.385793] Code: 48 f7 d0 48 23 02 48 85 c0 49 89 87 40 6d c1 81 74 20 48 8b 05 60 b8 af 00 41 8d b6 f0 00 00 00 48 89 df ff 90 e0 00 00 00 eb 02 90 83 3b 00 75 f9 48 8b 7d d0 49 c7 87 00 6d c1 81 00 00 00 [1974958.407132] Call Trace: [1974958.409977] [] ? flush_tlb_page+0x6e/0x76 [1974958.416117] [] ? page_check_address+0xb0/0x106 [1974958.422750] [] ? ptep_clear_flush_young+0x20/0x26 [1974958.429632] [] ? page_referenced_one+0x65/0xe5 [1974958.436198] [] ? prio_tree_next+0x208/0x22b [1974958.442517] [] ? page_referenced+0x2e0/0x335 [1974958.448915] [] ? mem_cgroup_del_lru_lis [1974958.455884] [] ? isolate_pages_global+0x1c6/0x23f [1974958.462757] [] ? shrink_active_list+0x1d3/0x2a0 [1974958.469456] [] ? shrink_zone+0x484/0xb86 [1974958.475513] [] ? xfs_perag_get+0x29/0x99 [1974958.481573] [] ? up_read+0x1e/0x36 [1974958.487111] [] ? kswapd+0x61a/0x8bb [1974958.492731] [] ? isolate_pages_global+0x0/0x23f [1974958.499455] [] ? autoremove_wake_function+0x0/0x38 [1974958.506455] [] ? kswapd+0x0/0x8bb [1974958.511938] [] ? kthread+0x7d/0x85 [1974958.517471] [] ? kernel_thread_helper+0x4/0x10 [1974958.524060] [] ? kthread+0x0/0x85 [1974958.529528] [] ? kernel_thread_helper+0x0/0x10 [1974999.909379] BUG: soft lockup - CPU#1 stuck for 61s! [sshd:9057] [1974999.915785] Modules linked in: w83793 hwmon_vid autofs4 e1000e i2c_i801 i2c_core iTCO_wdt ioatdma dca rtc_cmos tg3 libphy e1000 fuse dm_snapshot dm_mirror dm_region_hash dm_log scsi_wait_scan qla2xxx megaraid_sas megaraid_mbox megaraid_mm aacraid 3w_9xxx 3w_xxxx arcmsr aic7xxx aic79xx sr_mod sg ata_piix ahci sata_sis sata_nv sata_via pata_sl82c105 pata_sis [1974999.950057] CPU 1 [1974999.952128] Modules linked in: w83793 hwmon_vid autofs4 e1000e i2c_i801 i2c_core iTCO_wdt ioatdma dca rtc_cmos tg3 libphy e1000 fuse dm_snapshot dm_mirror dm_region_hash dm_log scsi_wait_scan qla2xxx megaraid_sas megaraid_mbox megaraid_mm aacraid 3w_9xxx 3w_xxxx arcmsr aic7xxx aic79xx sr_mod sg ata_piix ahci sata_sis sata_nv sata_via pata_sl82c105 pata_sis [1974999.986683] [1974999.988600] Pid: 9057, comm: sshd Tainted: G D W 2.6.34.4 #3 S7010/empty [1974999.996511] RIP: 0010:[] [] smp_call_function_many+0x18e/0x1a5 [1975000.006303] RSP: 0018:ffff880c18747b20 EFLAGS: 00000202 [1975000.012116] RAX: 0000000000000020 RBX: ffff880c18747b60 RCX: 0000000000000020 [1975000.019979] RDX: 0000000000000000 RSI: 0000000000000020 RDI: 0000000000000000 [1975000.027845] RBP: ffffffff8100334e R08: 0000000000000020 R09: ffffffff81b24210 [1975000.035668] R10: ffffffff812734b7 R11: 0000000000000286 R12: ffff880c18747ae0 [1975000.043535] R13: 0000000000000046 R14: ffff880c18747ae0 R15: 0000000000000000 [1975000.051352] FS: 00007f23edbf76f0(0000) GS:ffff880002620000(0000) knlGS:0000000000000000 [1975000.060178] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b [1975000.066447] CR2: 00007fffec51b310 CR3: 0000000c239bf000 CR4: 00000000000006e0 [1975000.074292] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [1975000.082175] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [1975000.089999] Process sshd (pid: 9057, threadinfo ffff880c18746000, task ffff880c187a0000) [1975000.098768] Stack: [1975000.101167] 01ff880c18747ba0 0000000000000000 ffffffff810b086c 0000000000000040 [1975000.108978] <0> 0000000000000000 ffffffff810b086c 0000000000000003 0000000000000002 [1975000.117494] <0> ffff880c18747b70 ffffffff8106a519 ffff880c18747ba0 ffffffff81040d28 [1975000.126367] Call Trace: [1975000.129249] [] ? drain_local_pages+0x0/0x13 [1975000.135604] [] ? drain_local_pages+0x0/0x13 [1975000.141923] [] ? smp_call_function+0x20/0x24 [1975000.148382] [] ? on_each_cpu+0x18/0x2c [1975000.154257] [] ? isolate_pages_global+0x0/0x23f [1975000.160949] [] ? __alloc_pages_nodemask+0x44d/0x671 [1975000.168005] [] ? up_read+0x1e/0x36 [1975000.173525] [] ? alloc_pages_current+0x96/0x9f [1975000.180114] [] ? new_slab+0x57/0x26b [1975000.185868] [] ? __slab_alloc+0x21c/0x45f [1975000.192052] [] ? copy_process+0xce/0xfc1 [1975000.198128] [] ? copy_process+0xce/0xfc1 [1975000.204243] [] ? kmem_cache_alloc+0x6a/0xd2 [1975000.210537] [] ? copy_process+0xce/0xfc1 [1975000.216620] [] ? lockdep_init_map+0x9f/0x52e [1975000.223052] [] ? __raw_spin_lock_init+0x31/0x54 [1975000.229751] [] ? do_fork+0x160/0x336 [1975000.235473] [] ? _raw_spin_unlock+0x26/0x2b [1975000.241834] [] ? fd_install+0x30/0x5f [1975000.247651] [] ? fd_install+0x56/0x5f [1975000.253426] [] ? sys_clone+0x23/0x25 [1975000.259097] [] ? stub_clone+0x13/0x20 [1975000.264913] [] ? system_call_fastpath+0x16/0x1b [1975000.271642] Code: c7 d0 21 a0 81 4c 89 60 08 4c 89 25 f9 7c 99 00 e8 59 51 4b 00 0f ae f0 48 89 df ff 15 f0 d1 a2 00 80 7d c7 00 75 04 eb 0a f3 90 <41> f6 44 24 20 01 75 f6 48 83 c4 18 5b 41 5c 41 5d 41 5e 41 5f [1975000.292989] Call Trace: [1975000.295868] [] ? drain_local_pages+0x0/0x13 [1975000.302180] [] ? drain_local_pages+0x0/0x13 [1975000.308543] [] ? smp_call_function+0x20/0x24 [1975000.314952] [] ? on_each_cpu+0x18/0x2c [1975000.320844] [] ? isolate_pages_global+0x0/0x23f [1975000.327528] [] ? __alloc_pages_nodemask+0x44d/0x671 [1975000.334567] [] ? up_read+0x1e/0x36 [1975000.340133] [] ? alloc_pages_current+0x [1975000.346765] [] ? new_slab+0x57/0x26b [1975000.352510] [] ? __slab_alloc+0x21c/0x45f [1975000.358674] [] ? copy_process+0xce/0xfc1 [1975000.364732] [] ? copy_process+0xce/0xfc1 [1975000.370826] [] ? kmem_cache_alloc+0x6a/0xd2 [1975000.377129] [] ? copy_process+0xce/0xfc1 [1975000.383214] [] ? lockdep_init_map+0x9f/0x52e [1975000.389620] [] ? __raw_spin_lock_init+0x31/0x54 [1975000.396295] [] ? do_fork+0x160/0x336 [1975000.402023] [] ? _raw_spin_unlock+0x26/0x2b [1975000.408353] [] ? fd_install+0x30/0x5f [1975000.414142] [] ? fd_install+0x56/0x5f [1975000.419943] [] ? sys_clone+0x23/0x25 [1975000.425665] [] ? stub_clone+0x13/0x20 [1975000.431499] [] ? system_call_fastpath+0x16/0x1b }}}