[ 20.363960] md: bind [ 20.843934] RAID conf printout: [ 20.843943] --- level:5 rd:3 wd:2 [ 20.843946] disk 0, o:1, dev:sdb1 [ 20.843949] disk 1, o:1, dev:sdc1 [ 20.843951] disk 2, o:1, dev:md1 [ 20.844640] md: recovery of RAID array md0 [ 20.844648] md: minimum _guaranteed_ speed: 1000 KB/sec/disk. [ 20.844651] md: using maximum available idle IO bandwidth (but not more than 200000 KB/sec) for recovery. [ 20.844656] md: using 128k window, over a total of 2930133504k. [ 20.844674] md: md0: recovery done. [ 20.975669] RAID conf printout: [ 20.975675] --- level:5 rd:3 wd:3 [ 20.975679] disk 0, o:1, dev:sdb1 [ 20.975682] disk 1, o:1, dev:sdc1 [ 20.975685] disk 2, o:1, dev:md1 [ 21.369591] bio: create slab at 1 [ 21.395028] ip_tables: (C) 2000-2006 Netfilter Core Team [ 21.532067] bio: create slab at 1 [ 21.636836] device fsid 88241708-5668-4c67-b0b5-536a11ec6371 devid 1 transid 10857 /dev/mapper/zero [ 21.638803] btrfs: disk space caching is enabled [ 160.981003] SysRq : Show Blocked State [ 160.981109] task PC stack pid father [ 160.981124] kworker/1:1 D ffff880077c93f40 0 28 2 0x00000000 [ 160.981133] ffff880074749b30 0000000000000046 ffff8800744fdd00 ffff880074749fd8 [ 160.981142] ffff880074749fd8 ffff880074749fd8 ffff880074ba1740 ffff8800744fdd00 [ 160.981150] ffff880036a83f60 ffff880036a83e00 ffff880036a83f70 00000000001b7d40 [ 160.981158] Call Trace: [ 160.981174] [] schedule+0x29/0x70 [ 160.981196] [] get_active_stripe+0x243/0x5e0 [raid456] [ 160.981209] [] ? default_spin_lock_flags+0x9/0x10 [ 160.981217] [] ? finish_wait+0x80/0x80 [ 160.981229] [] make_request+0x173/0xaa0 [raid456] [ 160.981236] [] ? finish_wait+0x80/0x80 [ 160.981245] [] md_make_request+0xc3/0x200 [ 160.981254] [] ? async_encrypt+0x3d/0x40 [ 160.981262] [] generic_make_request+0xc2/0x110 [ 160.981272] [] kcryptd_crypt_write_io_submit+0x4f/0xc0 [dm_crypt] [ 160.981281] [] kcryptd_crypt+0x24d/0x3b0 [dm_crypt] [ 160.981290] [] process_one_work+0x13e/0x4e0 [ 160.981298] [] ? kcryptd_async_done+0x100/0x100 [dm_crypt] [ 160.981306] [] worker_thread+0x16e/0x480 [ 160.981313] [] ? manage_workers+0x2a0/0x2a0 [ 160.981319] [] kthread+0xc0/0xd0 [ 160.981326] [] ? kthread_create_on_node+0x120/0x120 [ 160.981333] [] ret_from_fork+0x7c/0xb0 [ 160.981340] [] ? kthread_create_on_node+0x120/0x120 [ 160.981350] kworker/0:2 D ffff880077c13f40 0 52 2 0x00000000 [ 160.981357] ffff880070af7b30 0000000000000046 ffff8800709f0000 ffff880070af7fd8 [ 160.981365] ffff880070af7fd8 ffff880070af7fd8 ffffffff81c15440 ffff8800709f0000 [ 160.981372] ffff880036a83f60 ffff880036a83e00 ffff880036a83f70 0000000000029f70 [ 160.981379] Call Trace: [ 160.981387] [] schedule+0x29/0x70 [ 160.981397] [] get_active_stripe+0x243/0x5e0 [raid456] [ 160.981406] [] ? default_spin_lock_flags+0x9/0x10 [ 160.981413] [] ? finish_wait+0x80/0x80 [ 160.981423] [] make_request+0x173/0xaa0 [raid456] [ 160.981430] [] ? finish_wait+0x80/0x80 [ 160.981438] [] md_make_request+0xc3/0x200 [ 160.981446] [] ? async_encrypt+0x3d/0x40 [ 160.981453] [] generic_make_request+0xc2/0x110 [ 160.981462] [] kcryptd_crypt_write_io_submit+0x4f/0xc0 [dm_crypt] [ 160.981470] [] kcryptd_crypt+0x24d/0x3b0 [dm_crypt] [ 160.981478] [] process_one_work+0x13e/0x4e0 [ 160.981486] [] ? kcryptd_async_done+0x100/0x100 [dm_crypt] [ 160.981493] [] worker_thread+0x16e/0x480 [ 160.981500] [] ? manage_workers+0x2a0/0x2a0 [ 160.981506] [] kthread+0xc0/0xd0 [ 160.981513] [] ? kthread_create_on_node+0x120/0x120 [ 160.981520] [] ret_from_fork+0x7c/0xb0 [ 160.981527] [] ? kthread_create_on_node+0x120/0x120 [ 160.981536] md0_raid5 D ffff880077c93f40 0 333 2 0x00000000 [ 160.981542] ffff880035b1da20 0000000000000046 ffff880035f345c0 ffff880035b1dfd8 [ 160.981549] ffff880035b1dfd8 ffff880035b1dfd8 ffff880074ba1740 ffff880035f345c0 [ 160.981556] ffff880035f345c0 ffff880077c947f8 ffff880070aa4650 ffff880070aa4680 [ 160.981563] Call Trace: [ 160.981570] [] schedule+0x29/0x70 [ 160.981576] [] io_schedule+0x8f/0xd0 [ 160.981583] [] get_request+0x197/0x6d0 [ 160.981590] [] ? finish_wait+0x80/0x80 [ 160.981598] [] blk_queue_bio+0x7d/0x3e0 [ 160.981605] [] generic_make_request+0xc2/0x110 [ 160.981615] [] ops_run_io+0x29e/0x6b0 [raid456] [ 160.981624] [] ? ttwu_do_wakeup+0x2c/0xf0 [ 160.981633] [] ? try_to_wake_up+0x1e2/0x2a0 [ 160.981643] [] handle_stripe+0x3cc/0x1e50 [raid456] [ 160.981651] [] ? default_wake_function+0x12/0x20 [ 160.981657] [] ? autoremove_wake_function+0x12/0x40 [ 160.981668] [] handle_active_stripes+0x166/0x200 [raid456] [ 160.981679] [] raid5d+0x346/0x550 [raid456] [ 160.981688] [] md_thread+0x10d/0x140 [ 160.981694] [] ? finish_wait+0x80/0x80 [ 160.981701] [] ? md_rdev_init+0x140/0x140 [ 160.981707] [] kthread+0xc0/0xd0 [ 160.981714] [] ? kthread_create_on_node+0x120/0x120 [ 160.981721] [] ret_from_fork+0x7c/0xb0 [ 160.981727] [] ? kthread_create_on_node+0x120/0x120 [ 160.981760] btrfs-transacti D ffff880077c13f40 0 1565 2 0x00000000 [ 160.981767] ffff8800678dbbb0 0000000000000046 ffff8800678d1740 ffff8800678dbfd8 [ 160.981773] ffff8800678dbfd8 ffff8800678dbfd8 ffff8800709f0000 ffff8800678d1740 [ 160.981780] ffff8800678d1740 ffff880077c147f8 0000000000000002 ffffffff8112e8b0 [ 160.981787] Call Trace: [ 160.981795] [] ? __lock_page+0x70/0x70 [ 160.981802] [] schedule+0x29/0x70 [ 160.981808] [] io_schedule+0x8f/0xd0 [ 160.981814] [] sleep_on_page+0xe/0x20 [ 160.981821] [] __wait_on_bit+0x60/0x90 [ 160.981827] [] wait_on_page_bit+0x80/0x90 [ 160.981834] [] ? autoremove_wake_function+0x40/0x40 [ 160.981841] [] filemap_fdatawait_range+0x101/0x190 [ 160.981888] [] ? free_extent_state+0x1e/0x30 [btrfs] [ 160.981926] [] ? clear_extent_bit+0x223/0x3a0 [btrfs] [ 160.981961] [] btrfs_wait_marked_extents+0x87/0xe0 [btrfs] [ 160.981995] [] btrfs_write_and_wait_marked_extents+0x3b/0x60 [btrfs] [ 160.982029] [] btrfs_write_and_wait_transaction+0x2b/0x50 [btrfs] [ 160.982063] [] btrfs_commit_transaction+0x81a/0xab0 [btrfs] [ 160.982070] [] ? finish_wait+0x80/0x80 [ 160.982102] [] transaction_kthread+0x1bd/0x240 [btrfs] [ 160.982134] [] ? write_dev_flush.part.107+0xc0/0xc0 [btrfs] [ 160.982141] [] kthread+0xc0/0xd0 [ 160.982148] [] ? kthread_create_on_node+0x120/0x120 [ 160.982155] [] ret_from_fork+0x7c/0xb0 [ 160.982162] [] ? kthread_create_on_node+0x120/0x120 [ 160.982180] smbd D ffff880077c13f40 0 1831 877 0x00000000 [ 160.982187] ffff880066cab680 0000000000000086 ffff8800679e1740 ffff880066cabfd8 [ 160.982193] ffff880066cabfd8 ffff880066cabfd8 ffff880070e48000 ffff8800679e1740 [ 160.982200] ffff8800679e1740 ffff880077c147f8 0000000000000002 ffffffff8112e8b0 [ 160.982207] Call Trace: [ 160.982214] [] ? __lock_page+0x70/0x70 [ 160.982220] [] schedule+0x29/0x70 [ 160.982227] [] io_schedule+0x8f/0xd0 [ 160.982233] [] sleep_on_page+0xe/0x20 [ 160.982239] [] __wait_on_bit+0x60/0x90 [ 160.982245] [] wait_on_page_bit+0x80/0x90 [ 160.982252] [] ? autoremove_wake_function+0x40/0x40 [ 160.982290] [] read_extent_buffer_pages+0x25e/0x330 [btrfs] [ 160.982299] [] ? radix_tree_insert+0x92/0x250 [ 160.982331] [] ? verify_parent_transid+0x170/0x170 [btrfs] [ 160.982363] [] btree_read_extent_buffer_pages.constprop.119+0xb1/0x110 [btrfs] [ 160.982397] [] read_tree_block+0x3a/0x50 [btrfs] [ 160.982423] [] read_block_for_search.isra.47+0x134/0x3f0 [btrfs] [ 160.982450] [] ? generic_bin_search.constprop.49+0x6b/0x180 [btrfs] [ 160.982477] [] btrfs_search_slot+0x24a/0x7b0 [btrfs] [ 160.982508] [] btrfs_lookup_csum+0x76/0x180 [btrfs] [ 160.982545] [] ? btrfs_find_ordered_sum+0x39/0x100 [btrfs] [ 160.982576] [] __btrfs_lookup_bio_sums.isra.10+0x173/0x3f0 [btrfs] [ 160.982609] [] ? btrfs_bio_wq_end_io+0x3d/0x90 [btrfs] [ 160.982640] [] btrfs_lookup_bio_sums+0x36/0x40 [btrfs] [ 160.982674] [] btrfs_submit_bio_hook+0x14f/0x1e0 [btrfs] [ 160.982709] [] ? can_nocow_odirect+0x280/0x280 [btrfs] [ 160.982745] [] submit_one_bio+0x67/0xa0 [btrfs] [ 160.982779] [] ? can_nocow_odirect+0x280/0x280 [btrfs] [ 160.982816] [] extent_readpages+0x1d0/0x200 [btrfs] [ 160.982825] [] ? alloc_pages_current+0xb8/0x180 [ 160.982859] [] btrfs_readpages+0x1f/0x30 [btrfs] [ 160.982869] [] __do_page_cache_readahead+0x1b9/0x260 [ 160.982878] [] ondemand_readahead+0x234/0x240 [ 160.982886] [] page_cache_sync_readahead+0x33/0x50 [ 160.982893] [] generic_file_aio_read+0x535/0x6b0 [ 160.982901] [] ? sock_aio_read.part.7+0x126/0x140 [ 160.982909] [] do_sync_read+0xa7/0xe0 [ 160.982917] [] vfs_read+0xa9/0x180 [ 160.982924] [] sys_pread64+0x9a/0xb0 [ 160.982931] [] system_call_fastpath+0x1a/0x1f [ 160.982943] Sched Debug Version: v0.10, 3.8.0-7-generic #15-Ubuntu [ 160.982948] ktime : 161057.497144 [ 160.982952] sched_clk : 160982.940426 [ 160.982956] cpu_clk : 160982.940560 [ 160.982960] jiffies : 4294932560 [ 160.982963] sched_clock_stable : 1 [ 160.982966] [ 160.982968] sysctl_sched [ 160.982972] .sysctl_sched_latency : 12.000000 [ 160.982976] .sysctl_sched_min_granularity : 1.500000 [ 160.982980] .sysctl_sched_wakeup_granularity : 2.000000 [ 160.982983] .sysctl_sched_child_runs_first : 0 [ 160.982987] .sysctl_sched_features : 24187 [ 160.982991] .sysctl_sched_tunable_scaling : 1 (logaritmic) [ 160.982996] [ 160.982996] cpu#0, 1497.547 MHz [ 160.983001] .nr_running : 2 [ 160.983005] .load : 2048 [ 160.983008] .nr_switches : 93358 [ 160.983012] .nr_load_updates : 12711 [ 160.983015] .nr_uninterruptible : 116 [ 160.983019] .next_balance : 4294.932560 [ 160.983022] .curr->pid : 2072 [ 160.983026] .clock : 160980.368340 [ 160.983030] .cpu_load[0] : 0 [ 160.983033] .cpu_load[1] : 0 [ 160.983036] .cpu_load[2] : 0 [ 160.983039] .cpu_load[3] : 0 [ 160.983043] .cpu_load[4] : 0 [ 160.983046] .yld_count : 0 [ 160.983050] .sched_count : 110921 [ 160.983054] .sched_goidle : 34671 [ 160.983057] .avg_idle : 884583 [ 160.983060] .ttwu_count : 56277 [ 160.983064] .ttwu_local : 21407 [ 160.983070] [ 160.983070] cfs_rq[0]:/autogroup-121 [ 160.983075] .exec_clock : 171.560170 [ 160.983080] .MIN_vruntime : 0.000001 [ 160.983084] .min_vruntime : 231.993381 [ 160.983087] .max_vruntime : 0.000001 [ 160.983091] .spread : 0.000000 [ 160.983095] .spread0 : -11663.526036 [ 160.983099] .nr_spread_over : 20 [ 160.983102] .nr_running : 1 [ 160.983106] .load : 1024 [ 160.983109] .runnable_load_avg : 0 [ 160.983113] .blocked_load_avg : 0 [ 160.983116] .tg_load_avg : 0 [ 160.983120] .tg_load_contrib : 0 [ 160.983123] .tg_runnable_contrib : 0 [ 160.983126] .tg->runnable_avg : 0 [ 160.983131] .se->exec_start : 160980.368340 [ 160.983135] .se->vruntime : 11892.612507 [ 160.983139] .se->sum_exec_runtime : 171.615027 [ 160.983143] .se->statistics.wait_start : 0.000000 [ 160.983146] .se->statistics.sleep_start : 0.000000 [ 160.983150] .se->statistics.block_start : 0.000000 [ 160.983154] .se->statistics.sleep_max : 0.000000 [ 160.983158] .se->statistics.block_max : 0.000000 [ 160.983161] .se->statistics.exec_max : 3.996511 [ 160.983165] .se->statistics.slice_max : 0.000000 [ 160.983169] .se->statistics.wait_max : 0.233084 [ 160.983173] .se->statistics.wait_sum : 3.662268 [ 160.983176] .se->statistics.wait_count : 193 [ 160.983180] .se->load.weight : 1024 [ 160.983183] .se->avg.runnable_avg_sum : 364 [ 160.983187] .se->avg.runnable_avg_period : 46811 [ 160.983190] .se->avg.load_avg_contrib : 0 [ 160.983193] .se->avg.decay_count : 0 [ 160.983198] [ 160.983198] cfs_rq[0]:/autogroup-116 [ 160.983203] .exec_clock : 603.150796 [ 160.983208] .MIN_vruntime : 1011.435457 [ 160.983211] .min_vruntime : 1011.435457 [ 160.983215] .max_vruntime : 1011.435457 [ 160.983219] .spread : 0.000000 [ 160.983223] .spread0 : -10884.083960 [ 160.983227] .nr_spread_over : 86 [ 160.983230] .nr_running : 1 [ 160.983233] .load : 1024 [ 160.983236] .runnable_load_avg : 3 [ 160.983240] .blocked_load_avg : 0 [ 160.983243] .tg_load_avg : 0 [ 160.983246] .tg_load_contrib : 0 [ 160.983250] .tg_runnable_contrib : 0 [ 160.983253] .tg->runnable_avg : 0 [ 160.983257] .se->exec_start : 160979.858163 [ 160.983261] .se->vruntime : 11890.774390 [ 160.983265] .se->sum_exec_runtime : 603.182598 [ 160.983268] .se->statistics.wait_start : 160980.368340 [ 160.983272] .se->statistics.sleep_start : 0.000000 [ 160.983276] .se->statistics.block_start : 0.000000 [ 160.983279] .se->statistics.sleep_max : 0.000000 [ 160.983283] .se->statistics.block_max : 0.000000 [ 160.983287] .se->statistics.exec_max : 4.009080 [ 160.983291] .se->statistics.slice_max : 0.212313 [ 160.983294] .se->statistics.wait_max : 5.815878 [ 160.983298] .se->statistics.wait_sum : 15.906568 [ 160.983302] .se->statistics.wait_count : 764 [ 160.983305] .se->load.weight : 1024 [ 160.983309] .se->avg.runnable_avg_sum : 158 [ 160.983312] .se->avg.runnable_avg_period : 47874 [ 160.983315] .se->avg.load_avg_contrib : 0 [ 160.983319] .se->avg.decay_count : 153523 [ 160.983322] [ 160.983322] cfs_rq[0]:/ [ 160.983327] .exec_clock : 12434.577109 [ 160.983331] .MIN_vruntime : 11890.774390 [ 160.983335] .min_vruntime : 11895.519417 [ 160.983339] .max_vruntime : 11890.774390 [ 160.983343] .spread : 0.000000 [ 160.983346] .spread0 : 0.000000 [ 160.983350] .nr_spread_over : 104 [ 160.983353] .nr_running : 2 [ 160.983356] .load : 2048 [ 160.983360] .runnable_load_avg : 0 [ 160.983363] .blocked_load_avg : 0 [ 160.983366] .tg_load_avg : 239 [ 160.983370] .tg_load_contrib : 0 [ 160.983373] .tg_runnable_contrib : 14 [ 160.983376] .tg->runnable_avg : 260 [ 160.983380] .avg->runnable_avg_sum : 689 [ 160.983384] .avg->runnable_avg_period : 47754 [ 160.983392] [ 160.983392] rt_rq[0]:/ [ 160.983397] .rt_nr_running : 0 [ 160.983400] .rt_throttled : 0 [ 160.983403] .rt_time : 0.000000 [ 160.983407] .rt_runtime : 950.000000 [ 160.983411] [ 160.983411] runnable tasks: [ 160.983411] task PID tree-key switches prio exec-runtime sum-exec sum-sleep [ 160.983411] ---------------------------------------------------------------------------------------------------------- [ 160.983463] sshd 1971 1011.435457 178 120 1011.435457 60.230667 22313.626716 /autogroup-116 [ 160.983479] R zsh 2072 231.993381 137 120 231.993381 148.631326 8276.563245 /autogroup-121 [ 160.983491] [ 160.983491] cpu#1, 1497.547 MHz [ 160.983496] .nr_running : 1 [ 160.983500] .load : 1024 [ 160.983503] .nr_switches : 167851 [ 160.983507] .nr_load_updates : 15064 [ 160.983510] .nr_uninterruptible : -111 [ 160.983514] .next_balance : 4294.932603 [ 160.983518] .curr->pid : 2067 [ 160.983522] .clock : 160981.449817 [ 160.983525] .cpu_load[0] : 1024 [ 160.983528] .cpu_load[1] : 960 [ 160.983531] .cpu_load[2] : 700 [ 160.983535] .cpu_load[3] : 424 [ 160.983538] .cpu_load[4] : 234 [ 160.983541] .yld_count : 0 [ 160.983545] .sched_count : 172843 [ 160.983548] .sched_goidle : 60906 [ 160.983552] .avg_idle : 1000000 [ 160.983555] .ttwu_count : 86463 [ 160.983558] .ttwu_local : 57431 [ 160.983563] [ 160.983563] cfs_rq[1]:/autogroup-119 [ 160.983568] .exec_clock : 481.298309 [ 160.983572] .MIN_vruntime : 0.000001 [ 160.983575] .min_vruntime : 618.159274 [ 160.983579] .max_vruntime : 0.000001 [ 160.983583] .spread : 0.000000 [ 160.983587] .spread0 : -11277.360143 [ 160.983590] .nr_spread_over : 22 [ 160.983593] .nr_running : 1 [ 160.983597] .load : 1024 [ 160.983600] .runnable_load_avg : 237 [ 160.983603] .blocked_load_avg : 0 [ 160.983607] .tg_load_avg : 237 [ 160.983610] .tg_load_contrib : 237 [ 160.983614] .tg_runnable_contrib : 240 [ 160.983617] .tg->runnable_avg : 240 [ 160.983621] .se->exec_start : 160981.449817 [ 160.983625] .se->vruntime : 27209.272754 [ 160.983629] .se->sum_exec_runtime : 481.298309 [ 160.983632] .se->statistics.wait_start : 0.000000 [ 160.983636] .se->statistics.sleep_start : 0.000000 [ 160.983640] .se->statistics.block_start : 0.000000 [ 160.983643] .se->statistics.sleep_max : 0.000000 [ 160.983647] .se->statistics.block_max : 0.000000 [ 160.983651] .se->statistics.exec_max : 4.003304 [ 160.983655] .se->statistics.slice_max : 0.000000 [ 160.983658] .se->statistics.wait_max : 0.225848 [ 160.983662] .se->statistics.wait_sum : 3.867410 [ 160.983665] .se->statistics.wait_count : 215 [ 160.983669] .se->load.weight : 1024 [ 160.983672] .se->avg.runnable_avg_sum : 10976 [ 160.983676] .se->avg.runnable_avg_period : 46655 [ 160.983679] .se->avg.load_avg_contrib : 238 [ 160.983682] .se->avg.decay_count : 0 [ 160.983686] [ 160.983686] cfs_rq[1]:/autogroup-118 [ 160.983692] .exec_clock : 21.381877 [ 160.983695] .MIN_vruntime : 0.000001 [ 160.983699] .min_vruntime : 23.333301 [ 160.983703] .max_vruntime : 0.000001 [ 160.983706] .spread : 0.000000 [ 160.983710] .spread0 : -11872.186116 [ 160.983714] .nr_spread_over : 0 [ 160.983717] .nr_running : 0 [ 160.983720] .load : 0 [ 160.983723] .runnable_load_avg : 0 [ 160.983727] .blocked_load_avg : 2 [ 160.983730] .tg_load_avg : 0 [ 160.983733] .tg_load_contrib : 0 [ 160.983736] .tg_runnable_contrib : 2 [ 160.983740] .tg->runnable_avg : 2 [ 160.983744] .se->exec_start : 160980.362482 [ 160.983748] .se->vruntime : 27202.257082 [ 160.983751] .se->sum_exec_runtime : 21.425287 [ 160.983755] .se->statistics.wait_start : 0.000000 [ 160.983759] .se->statistics.sleep_start : 0.000000 [ 160.983762] .se->statistics.block_start : 0.000000 [ 160.983766] .se->statistics.sleep_max : 0.000000 [ 160.983770] .se->statistics.block_max : 0.000000 [ 160.983773] .se->statistics.exec_max : 1.571127 [ 160.983777] .se->statistics.slice_max : 0.144806 [ 160.983781] .se->statistics.wait_max : 0.035878 [ 160.983785] .se->statistics.wait_sum : 0.118710 [ 160.983788] .se->statistics.wait_count : 127 [ 160.983791] .se->load.weight : 2 [ 160.983795] .se->avg.runnable_avg_sum : 184 [ 160.983798] .se->avg.runnable_avg_period : 46568 [ 160.983802] .se->avg.load_avg_contrib : 0 [ 160.983805] .se->avg.decay_count : 153523 [ 160.983808] [ 160.983808] cfs_rq[1]:/ [ 160.983813] .exec_clock : 30116.019547 [ 160.983817] .MIN_vruntime : 0.000001 [ 160.983821] .min_vruntime : 27209.272754 [ 160.983825] .max_vruntime : 0.000001 [ 160.983828] .spread : 0.000000 [ 160.983832] .spread0 : 15313.753337 [ 160.983835] .nr_spread_over : 70 [ 160.983839] .nr_running : 1 [ 160.983842] .load : 1024 [ 160.983845] .runnable_load_avg : 238 [ 160.983849] .blocked_load_avg : 1 [ 160.983852] .tg_load_avg : 239 [ 160.983855] .tg_load_contrib : 239 [ 160.983859] .tg_runnable_contrib : 246 [ 160.983862] .tg->runnable_avg : 260 [ 160.983865] .avg->runnable_avg_sum : 11318 [ 160.983869] .avg->runnable_avg_period : 47081 [ 160.983874] [ 160.983874] rt_rq[1]:/ [ 160.983879] .rt_nr_running : 0 [ 160.983882] .rt_throttled : 0 [ 160.983885] .rt_time : 0.000000 [ 160.983889] .rt_runtime : 950.000000 [ 160.983893] [ 160.983893] runnable tasks: [ 160.983893] task PID tree-key switches prio exec-runtime sum-exec sum-sleep [ 160.983893] ---------------------------------------------------------------------------------------------------------- [ 160.983942] R iotop 2067 618.159274 149 120 618.159274 473.349454 9025.463783 /autogroup-119 [ 160.983954] [ 266.125735] SysRq : Show Blocked State [ 266.125880] task PC stack pid father [ 266.125900] kworker/0:2 D ffff880077c13f40 0 52 2 0x00000000 [ 266.125910] ffff880070af7b30 0000000000000046 ffff8800709f0000 ffff880070af7fd8 [ 266.125919] ffff880070af7fd8 ffff880070af7fd8 ffffffff81c15440 ffff8800709f0000 [ 266.125927] ffff880036a83f60 ffff880036a83e00 ffff880036a83f70 0000000000214928 [ 266.125934] Call Trace: [ 266.125951] [] schedule+0x29/0x70 [ 266.125973] [] get_active_stripe+0x243/0x5e0 [raid456] [ 266.125986] [] ? default_spin_lock_flags+0x9/0x10 [ 266.125994] [] ? finish_wait+0x80/0x80 [ 266.126006] [] make_request+0x173/0xaa0 [raid456] [ 266.126013] [] ? finish_wait+0x80/0x80 [ 266.126022] [] md_make_request+0xc3/0x200 [ 266.126031] [] ? async_encrypt+0x3d/0x40 [ 266.126040] [] generic_make_request+0xc2/0x110 [ 266.126050] [] kcryptd_crypt_write_io_submit+0x4f/0xc0 [dm_crypt] [ 266.126059] [] kcryptd_crypt+0x24d/0x3b0 [dm_crypt] [ 266.126067] [] process_one_work+0x13e/0x4e0 [ 266.126076] [] ? kcryptd_async_done+0x100/0x100 [dm_crypt] [ 266.126083] [] worker_thread+0x16e/0x480 [ 266.126090] [] ? manage_workers+0x2a0/0x2a0 [ 266.126097] [] kthread+0xc0/0xd0 [ 266.126104] [] ? kthread_create_on_node+0x120/0x120 [ 266.126111] [] ret_from_fork+0x7c/0xb0 [ 266.126118] [] ? kthread_create_on_node+0x120/0x120 [ 266.126128] md0_raid5 D ffff880077c93f40 0 333 2 0x00000000 [ 266.126136] ffff880035b1dcd0 0000000000000046 ffff880035f345c0 ffff880035b1dfd8 [ 266.126143] ffff880035b1dfd8 ffff880035b1dfd8 ffff880067a4ae80 ffff880035f345c0 [ 266.126150] ffff880036866680 ffff880036866400 0000000000000001 0000000000000000 [ 266.126157] Call Trace: [ 266.126165] [] schedule+0x29/0x70 [ 266.126173] [] md_super_wait+0x55/0x90 [ 266.126179] [] ? finish_wait+0x80/0x80 [ 266.126186] [] bitmap_unplug+0x173/0x180 [ 266.126197] [] raid5d+0xe4/0x550 [raid456] [ 266.126205] [] md_thread+0x10d/0x140 [ 266.126212] [] ? finish_wait+0x80/0x80 [ 266.126219] [] ? md_rdev_init+0x140/0x140 [ 266.126225] [] kthread+0xc0/0xd0 [ 266.126232] [] ? kthread_create_on_node+0x120/0x120 [ 266.126239] [] ret_from_fork+0x7c/0xb0 [ 266.126245] [] ? kthread_create_on_node+0x120/0x120 [ 266.126278] btrfs-transacti D ffff880077c13f40 0 1565 2 0x00000000 [ 266.126285] ffff8800678dbbb0 0000000000000046 ffff8800678d1740 ffff8800678dbfd8 [ 266.126291] ffff8800678dbfd8 ffff8800678dbfd8 ffff8800709f0000 ffff8800678d1740 [ 266.126298] ffff8800678d1740 ffff880077c147f8 0000000000000002 ffffffff8112e8b0 [ 266.126305] Call Trace: [ 266.126314] [] ? __lock_page+0x70/0x70 [ 266.126320] [] schedule+0x29/0x70 [ 266.126327] [] io_schedule+0x8f/0xd0 [ 266.126333] [] sleep_on_page+0xe/0x20 [ 266.126339] [] __wait_on_bit+0x60/0x90 [ 266.126346] [] wait_on_page_bit+0x80/0x90 [ 266.126352] [] ? autoremove_wake_function+0x40/0x40 [ 266.126359] [] filemap_fdatawait_range+0x101/0x190 [ 266.126405] [] ? free_extent_state+0x1e/0x30 [btrfs] [ 266.126443] [] ? clear_extent_bit+0x223/0x3a0 [btrfs] [ 266.126478] [] btrfs_wait_marked_extents+0x87/0xe0 [btrfs] [ 266.126513] [] btrfs_write_and_wait_marked_extents+0x3b/0x60 [btrfs] [ 266.126547] [] btrfs_write_and_wait_transaction+0x2b/0x50 [btrfs] [ 266.126581] [] btrfs_commit_transaction+0x81a/0xab0 [btrfs] [ 266.126588] [] ? finish_wait+0x80/0x80 [ 266.126620] [] transaction_kthread+0x1bd/0x240 [btrfs] [ 266.126652] [] ? write_dev_flush.part.107+0xc0/0xc0 [btrfs] [ 266.126659] [] kthread+0xc0/0xd0 [ 266.126666] [] ? kthread_create_on_node+0x120/0x120 [ 266.126673] [] ret_from_fork+0x7c/0xb0 [ 266.126679] [] ? kthread_create_on_node+0x120/0x120 [ 266.126699] smbd D ffff880077c13f40 0 1831 877 0x00000000 [ 266.126705] ffff880066cabc58 0000000000000086 ffff8800679e1740 ffff880066cabfd8 [ 266.126712] ffff880066cabfd8 ffff880066cabfd8 ffffffff81c15440 ffff8800679e1740 [ 266.126719] ffff8800679e1740 ffff880077c147f8 0000000000000082 ffffffff8112e8d0 [ 266.126726] Call Trace: [ 266.126733] [] ? sleep_on_page+0x20/0x20 [ 266.126739] [] schedule+0x29/0x70 [ 266.126745] [] io_schedule+0x8f/0xd0 [ 266.126752] [] sleep_on_page_killable+0xe/0x40 [ 266.126758] [] __wait_on_bit_lock+0x5b/0xc0 [ 266.126765] [] __lock_page_killable+0x67/0x70 [ 266.126771] [] ? autoremove_wake_function+0x40/0x40 [ 266.126778] [] generic_file_aio_read+0x44e/0x6b0 [ 266.126786] [] ? sock_aio_read.part.7+0x126/0x140 [ 266.126795] [] do_sync_read+0xa7/0xe0 [ 266.126803] [] vfs_read+0xa9/0x180 [ 266.126810] [] sys_pread64+0x9a/0xb0 [ 266.126817] [] system_call_fastpath+0x1a/0x1f [ 266.126828] Sched Debug Version: v0.10, 3.8.0-7-generic #15-Ubuntu [ 266.126833] ktime : 266325.165575 [ 266.126838] sched_clk : 266126.825849 [ 266.126842] cpu_clk : 266126.825985 [ 266.126846] jiffies : 4294958877 [ 266.126849] sched_clock_stable : 1 [ 266.126852] [ 266.126854] sysctl_sched [ 266.126858] .sysctl_sched_latency : 12.000000 [ 266.126862] .sysctl_sched_min_granularity : 1.500000 [ 266.126866] .sysctl_sched_wakeup_granularity : 2.000000 [ 266.126869] .sysctl_sched_child_runs_first : 0 [ 266.126873] .sysctl_sched_features : 24187 [ 266.126877] .sysctl_sched_tunable_scaling : 1 (logaritmic) [ 266.126882] [ 266.126882] cpu#0, 1497.547 MHz [ 266.126887] .nr_running : 2 [ 266.126891] .load : 2048 [ 266.126894] .nr_switches : 124750 [ 266.126897] .nr_load_updates : 16712 [ 266.126901] .nr_uninterruptible : 146 [ 266.126905] .next_balance : 4294.958925 [ 266.126908] .curr->pid : 2072 [ 266.126912] .clock : 266125.695563 [ 266.126916] .cpu_load[0] : 2048 [ 266.126919] .cpu_load[1] : 1030 [ 266.126922] .cpu_load[2] : 571 [ 266.126926] .cpu_load[3] : 349 [ 266.126929] .cpu_load[4] : 205 [ 266.126932] .yld_count : 0 [ 266.126936] .sched_count : 143750 [ 266.126939] .sched_goidle : 41464 [ 266.126943] .avg_idle : 890431 [ 266.126946] .ttwu_count : 72918 [ 266.126949] .ttwu_local : 35222 [ 266.126955] [ 266.126955] cfs_rq[0]:/autogroup-121 [ 266.126960] .exec_clock : 213.172383 [ 266.126965] .MIN_vruntime : 0.000001 [ 266.126969] .min_vruntime : 272.085632 [ 266.126973] .max_vruntime : 0.000001 [ 266.126976] .spread : 0.000000 [ 266.126981] .spread0 : -15857.514076 [ 266.126984] .nr_spread_over : 25 [ 266.126988] .nr_running : 1 [ 266.126991] .load : 1024 [ 266.126995] .runnable_load_avg : 23 [ 266.126998] .blocked_load_avg : 0 [ 266.127002] .tg_load_avg : 23 [ 266.127005] .tg_load_contrib : 23 [ 266.127009] .tg_runnable_contrib : 24 [ 266.127012] .tg->runnable_avg : 24 [ 266.127016] .se->exec_start : 266125.675642 [ 266.127020] .se->vruntime : 16124.768114 [ 266.127024] .se->sum_exec_runtime : 213.172383 [ 266.127028] .se->statistics.wait_start : 0.000000 [ 266.127032] .se->statistics.sleep_start : 0.000000 [ 266.127036] .se->statistics.block_start : 0.000000 [ 266.127039] .se->statistics.sleep_max : 0.000000 [ 266.127043] .se->statistics.block_max : 0.000000 [ 266.127047] .se->statistics.exec_max : 3.996511 [ 266.127051] .se->statistics.slice_max : 3.654345 [ 266.127054] .se->statistics.wait_max : 13.488788 [ 266.127058] .se->statistics.wait_sum : 31.757097 [ 266.127062] .se->statistics.wait_count : 2017 [ 266.127065] .se->load.weight : 1024 [ 266.127068] .se->avg.runnable_avg_sum : 1133 [ 266.127072] .se->avg.runnable_avg_period : 46133 [ 266.127075] .se->avg.load_avg_contrib : 22 [ 266.127079] .se->avg.decay_count : 0 [ 266.127084] [ 266.127084] cfs_rq[0]:/autogroup-116 [ 266.127089] .exec_clock : 663.000942 [ 266.127093] .MIN_vruntime : 0.000001 [ 266.127097] .min_vruntime : 1071.285603 [ 266.127101] .max_vruntime : 0.000001 [ 266.127104] .spread : 0.000000 [ 266.127108] .spread0 : -15058.314105 [ 266.127111] .nr_spread_over : 86 [ 266.127115] .nr_running : 0 [ 266.127118] .load : 0 [ 266.127121] .runnable_load_avg : 0 [ 266.127124] .blocked_load_avg : 1 [ 266.127128] .tg_load_avg : 1 [ 266.127131] .tg_load_contrib : 1 [ 266.127134] .tg_runnable_contrib : 2 [ 266.127137] .tg->runnable_avg : 2 [ 266.127141] .se->exec_start : 266124.360081 [ 266.127145] .se->vruntime : 16123.742716 [ 266.127149] .se->sum_exec_runtime : 663.060578 [ 266.127153] .se->statistics.wait_start : 0.000000 [ 266.127156] .se->statistics.sleep_start : 0.000000 [ 266.127160] .se->statistics.block_start : 0.000000 [ 266.127164] .se->statistics.sleep_max : 0.000000 [ 266.127167] .se->statistics.block_max : 0.000000 [ 266.127171] .se->statistics.exec_max : 4.009080 [ 266.127175] .se->statistics.slice_max : 0.910761 [ 266.127179] .se->statistics.wait_max : 5.815878 [ 266.127182] .se->statistics.wait_sum : 25.369477 [ 266.127186] .se->statistics.wait_count : 1040 [ 266.127189] .se->load.weight : 2 [ 266.127192] .se->avg.runnable_avg_sum : 136 [ 266.127196] .se->avg.runnable_avg_period : 48052 [ 266.127199] .se->avg.load_avg_contrib : 1 [ 266.127203] .se->avg.decay_count : 253796 [ 266.127207] [ 266.127207] cfs_rq[0]:/autogroup-58 [ 266.127212] .exec_clock : 511.090242 [ 266.127216] .MIN_vruntime : 0.000001 [ 266.127220] .min_vruntime : 511.880053 [ 266.127223] .max_vruntime : 0.000001 [ 266.127227] .spread : 0.000000 [ 266.127231] .spread0 : -15617.719655 [ 266.127234] .nr_spread_over : 0 [ 266.127237] .nr_running : 0 [ 266.127241] .load : 0 [ 266.127244] .runnable_load_avg : 0 [ 266.127247] .blocked_load_avg : 29 [ 266.127250] .tg_load_avg : 29 [ 266.127254] .tg_load_contrib : 29 [ 266.127257] .tg_runnable_contrib : 35 [ 266.127260] .tg->runnable_avg : 35 [ 266.127264] .se->exec_start : 266081.881504 [ 266.127268] .se->vruntime : 16124.587964 [ 266.127272] .se->sum_exec_runtime : 511.117068 [ 266.127276] .se->statistics.wait_start : 0.000000 [ 266.127279] .se->statistics.sleep_start : 0.000000 [ 266.127283] .se->statistics.block_start : 0.000000 [ 266.127287] .se->statistics.sleep_max : 0.000000 [ 266.127290] .se->statistics.block_max : 0.000000 [ 266.127294] .se->statistics.exec_max : 3.988331 [ 266.127298] .se->statistics.slice_max : 5.063907 [ 266.127302] .se->statistics.wait_max : 14.364149 [ 266.127305] .se->statistics.wait_sum : 193.505156 [ 266.127309] .se->statistics.wait_count : 1947 [ 266.127312] .se->load.weight : 2 [ 266.127315] .se->avg.runnable_avg_sum : 1614 [ 266.127319] .se->avg.runnable_avg_period : 47148 [ 266.127322] .se->avg.load_avg_contrib : 33 [ 266.127326] .se->avg.decay_count : 253756 [ 266.127329] [ 266.127329] cfs_rq[0]:/ [ 266.127334] .exec_clock : 17840.330452 [ 266.127339] .MIN_vruntime : 16123.669186 [ 266.127342] .min_vruntime : 16129.599708 [ 266.127346] .max_vruntime : 16123.669186 [ 266.127350] .spread : 0.000000 [ 266.127353] .spread0 : 0.000000 [ 266.127357] .nr_spread_over : 106 [ 266.127360] .nr_running : 2 [ 266.127363] .load : 2048 [ 266.127367] .runnable_load_avg : 24 [ 266.127370] .blocked_load_avg : 0 [ 266.127373] .tg_load_avg : 403 [ 266.127377] .tg_load_contrib : 24 [ 266.127380] .tg_runnable_contrib : 116 [ 266.127384] .tg->runnable_avg : 497 [ 266.127387] .avg->runnable_avg_sum : 5284 [ 266.127391] .avg->runnable_avg_period : 46449 [ 266.127401] [ 266.127401] rt_rq[0]:/ [ 266.127405] .rt_nr_running : 0 [ 266.127409] .rt_throttled : 0 [ 266.127412] .rt_time : 0.000000 [ 266.127416] .rt_runtime : 950.000000 [ 266.127420] [ 266.127420] runnable tasks: [ 266.127420] task PID tree-key switches prio exec-runtime sum-exec sum-sleep [ 266.127420] ---------------------------------------------------------------------------------------------------------- [ 266.127434] kworker/0:1 29 16123.669186 11765 120 16123.669186 420.857571 263193.284576 / [ 266.127485] R zsh 2072 268.681040 169 120 268.681040 165.803481 113390.696904 /autogroup-121 [ 266.127499] [ 266.127499] cpu#1, 1497.547 MHz [ 266.127504] .nr_running : 0 [ 266.127508] .load : 0 [ 266.127511] .nr_switches : 199189 [ 266.127515] .nr_load_updates : 18467 [ 266.127518] .nr_uninterruptible : -142 [ 266.127522] .next_balance : 4294.958878 [ 266.127526] .curr->pid : 0 [ 266.127529] .clock : 266125.784591 [ 266.127533] .cpu_load[0] : 0 [ 266.127536] .cpu_load[1] : 128 [ 266.127539] .cpu_load[2] : 418 [ 266.127543] .cpu_load[3] : 549 [ 266.127546] .cpu_load[4] : 457 [ 266.127549] .yld_count : 0 [ 266.127552] .sched_count : 205942 [ 266.127556] .sched_goidle : 70174 [ 266.127559] .avg_idle : 1000000 [ 266.127563] .ttwu_count : 106465 [ 266.127566] .ttwu_local : 73269 [ 266.127570] [ 266.127570] cfs_rq[1]:/autogroup-118 [ 266.127575] .exec_clock : 191.160707 [ 266.127579] .MIN_vruntime : 0.000001 [ 266.127583] .min_vruntime : 193.112131 [ 266.127587] .max_vruntime : 0.000001 [ 266.127590] .spread : 0.000000 [ 266.127594] .spread0 : -15936.487577 [ 266.127597] .nr_spread_over : 0 [ 266.127601] .nr_running : 0 [ 266.127604] .load : 0 [ 266.127607] .runnable_load_avg : 0 [ 266.127610] .blocked_load_avg : 0 [ 266.127614] .tg_load_avg : 0 [ 266.127617] .tg_load_contrib : 0 [ 266.127620] .tg_runnable_contrib : 2 [ 266.127624] .tg->runnable_avg : 2 [ 266.127628] .se->exec_start : 266124.492918 [ 266.127632] .se->vruntime : 28995.004696 [ 266.127635] .se->sum_exec_runtime : 191.349588 [ 266.127639] .se->statistics.wait_start : 0.000000 [ 266.127643] .se->statistics.sleep_start : 0.000000 [ 266.127646] .se->statistics.block_start : 0.000000 [ 266.127650] .se->statistics.sleep_max : 0.000000 [ 266.127654] .se->statistics.block_max : 0.000000 [ 266.127658] .se->statistics.exec_max : 2.778479 [ 266.127661] .se->statistics.slice_max : 0.949502 [ 266.127665] .se->statistics.wait_max : 1.275431 [ 266.127669] .se->statistics.wait_sum : 13.731487 [ 266.127672] .se->statistics.wait_count : 1260 [ 266.127676] .se->load.weight : 2 [ 266.127679] .se->avg.runnable_avg_sum : 113 [ 266.127683] .se->avg.runnable_avg_period : 48036 [ 266.127686] .se->avg.load_avg_contrib : 0 [ 266.127689] .se->avg.decay_count : 253797 [ 266.127693] [ 266.127693] cfs_rq[1]:/ [ 266.127698] .exec_clock : 32767.622700 [ 266.127702] .MIN_vruntime : 0.000001 [ 266.127706] .min_vruntime : 29000.907612 [ 266.127709] .max_vruntime : 0.000001 [ 266.127713] .spread : 0.000000 [ 266.127717] .spread0 : 12871.307904 [ 266.127720] .nr_spread_over : 73 [ 266.127723] .nr_running : 0 [ 266.127727] .load : 0 [ 266.127730] .runnable_load_avg : 0 [ 266.127733] .blocked_load_avg : 379 [ 266.127737] .tg_load_avg : 403 [ 266.127740] .tg_load_contrib : 379 [ 266.127743] .tg_runnable_contrib : 381 [ 266.127747] .tg->runnable_avg : 497 [ 266.127750] .avg->runnable_avg_sum : 17628 [ 266.127754] .avg->runnable_avg_period : 47233 [ 266.127759] [ 266.127759] rt_rq[1]:/ [ 266.127763] .rt_nr_running : 0 [ 266.127767] .rt_throttled : 0 [ 266.127770] .rt_time : 0.000000 [ 266.127774] .rt_runtime : 950.000000 [ 266.127778] [ 266.127778] runnable tasks: [ 266.127778] task PID tree-key switches prio exec-runtime sum-exec sum-sleep [ 266.127778] ---------------------------------------------------------------------------------------------------------- [ 266.127826]